diff --git a/operator/api/loki/v1/lokistack_types.go b/operator/api/loki/v1/lokistack_types.go index e0c55ec5d569..a9ef632d7044 100644 --- a/operator/api/loki/v1/lokistack_types.go +++ b/operator/api/loki/v1/lokistack_types.go @@ -289,6 +289,34 @@ type OpenshiftTenantSpec struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Admin Groups" AdminGroups []string `json:"adminGroups"` + + // OTLP contains settings for ingesting data using OTLP in the OpenShift tenancy mode. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="OpenTelemetry Protocol" + OTLP *OpenshiftOTLPConfig `json:"otlp,omitempty"` +} + +// OpenshiftOTLPConfig defines configuration specific to users using OTLP together with an OpenShift tenancy mode. +type OpenshiftOTLPConfig struct { + // DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + // metadata. + // + // Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + // meta information to not be available as stream labels or structured metadata, potentially making queries more + // expensive and less performant. + // + // Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + // added to the configuration, even if this field is set to true. + // + // This option is supposed to be combined with a custom label configuration customizing the labels for the specific + // usecase. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Disable recommended OTLP attributes" + DisableRecommendedAttributes bool `json:"disableRecommendedAttributes,omitempty"` } // LokiComponentSpec defines the requirements to configure scheduling @@ -791,138 +819,70 @@ type IngestionLimitSpec struct { PerStreamRateLimitBurst int32 `json:"perStreamRateLimitBurst,omitempty"` } -// OTLPAttributeAction defines the action to executed when indexing -// OTLP resource attributes. Resource attributes can be either added -// to the index, the chunk structured metadata or entirely dropped. -type OTLPAttributeAction string - -const ( - // OTLPAttributeActionIndexLabel stores a resource attribute as a label, which is part of the index identifying streams. - OTLPAttributeActionIndexLabel OTLPAttributeAction = "indexLabel" - // OTLPAttributeActionStructuredMetadata stores an attribute as structured metadata with each log entry. - OTLPAttributeActionStructuredMetadata OTLPAttributeAction = "structuredMetadata" - // OTLPAttributeActionDrop removes the matching attributes from the log entry. - OTLPAttributeActionDrop OTLPAttributeAction = "drop" -) - -// OTLPAttributesSpec contains the configuration for a set of attributes -// to store them as index labels or structured metadata or drop them altogether. -type OTLPAttributesSpec struct { - // Action defines the indexing action for the selected attributes. They - // can be either added to structured metadata or drop altogether. - // - // +required - // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=structured_metadata;drop - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Action" - Action OTLPAttributeAction `json:"action"` - - // Attributes allows choosing the attributes by listing their names. +// OTLPSpec defines which resource, scope and log attributes should be used as stream labels or +// stored as structured metadata. +type OTLPSpec struct { + // StreamLabels configures which resource attributes are converted to Loki stream labels. // // +optional // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attribute Names" - Attributes []string `json:"attributes,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Stream Labels" + StreamLabels *OTLPStreamLabelSpec `json:"streamLabels,omitempty"` - // Regex allows choosing the attributes by matching a regular expression. + // StructuredMetadata configures which attributes are saved in structured metadata. // // +optional // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Regular Expression" - Regex string `json:"regex,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Structured Metadata" + StructuredMetadata *OTLPMetadataSpec `json:"structuredMetadata,omitempty"` } -// OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes -// to store them as index labels or structured metadata or drop them altogether. -type OTLPResourceAttributesConfigSpec struct { - // Action defines the indexing action for the selected resoure attributes. They - // can be either indexed as labels, added to structured metadata or drop altogether. - // - // +required - // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=index_label;structured_metadata;drop - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Action" - Action OTLPAttributeAction `json:"action"` - - // Attributes is the list of attributes to configure indexing or drop them - // altogether. - // - // +optional - // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attribute Names" - Attributes []string `json:"attributes,omitempty"` - - // Regex allows choosing the attributes by matching a regular expression. +type OTLPStreamLabelSpec struct { + // ResourceAttributes lists the names of the resource attributes that should be converted into Loki stream labels. // // +optional // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Regular Expression" - Regex string `json:"regex,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resource Attributes" + ResourceAttributes []OTLPAttributeReference `json:"resourceAttributes,omitempty"` } -// OTLPResourceAttributesSpec contains the configuration for resource attributes -// to store them as index labels or structured metadata or drop them altogether. -type OTLPResourceAttributesSpec struct { - // IgnoreDefaults controls whether to ignore the global configuration for resource attributes - // indexed as labels. - // - // If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. +type OTLPMetadataSpec struct { + // ResourceAttributes lists the names of resource attributes that should be included in structured metadata. // // +optional // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch",displayName="Ignore Global Defaults" - IgnoreDefaults bool `json:"ignoreDefaults,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resource Attributes" + ResourceAttributes []OTLPAttributeReference `json:"resourceAttributes,omitempty"` - // Attributes contains the configuration for resource attributes - // to store them as index labels or structured metadata or drop them altogether. + // ScopeAttributes lists the names of scope attributes that should be included in structured metadata. // // +optional // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attributes" - Attributes []OTLPResourceAttributesConfigSpec `json:"attributes,omitempty"` -} + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Scope Attributes" + ScopeAttributes []OTLPAttributeReference `json:"scopeAttributes,omitempty"` -// GlobalOTLPSpec defines which resource, scope and log attributes to -// be stored as index or structured metadata or drop altogether for all -// tenants. -type GlobalOTLPSpec struct { - // IndexedResourceAttributes contains the global configuration for resource attributes - // to store them as index labels. + // LogAttributes lists the names of log attributes that should be included in structured metadata. // // +optional // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Indexed Resource Attributes" - IndexedResourceAttributes []string `json:"indexedResourceAttributes,omitempty"` - - OTLPSpec `json:",omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Log Attributes" + LogAttributes []OTLPAttributeReference `json:"logAttributes,omitempty"` } -// OTLPSpec defines which resource, scope and log attributes to -// be stored as index or structured metadata or drop altogether -type OTLPSpec struct { - // ResourceAttributes contains the configuration for resource attributes - // to store them as index labels or structured metadata or drop them altogether. +type OTLPAttributeReference struct { + // Name contains either a verbatim name of an attribute or a regular expression matching many attributes. // - // +optional - // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resource Attributes" - ResourceAttributes *OTLPResourceAttributesSpec `json:"resourceAttributes,omitempty"` - - // ScopeAttributes contains the configuration for scope attributes - // to store them as structured metadata or drop them altogether. - // - // +optional - // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Scope Attributes" - ScopeAttributes []OTLPAttributesSpec `json:"scopeAttributes,omitempty"` + // +required + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Name" + Name string `json:"name"` - // LogAttributes contains the configuration for log attributes - // to store them as structured metadata or drop them altogether. + // If Regex is true, then Name is treated as a regular expression instead of as a verbatim attribute name. // // +optional // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Log Attributes" - LogAttributes []OTLPAttributesSpec `json:"logAttributes,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Treat name as regular expression" + Regex bool `json:"regex,omitempty"` } // RetentionStreamSpec defines a log stream with separate retention time. @@ -978,13 +938,14 @@ type LimitsTemplateSpec struct { // +kubebuilder:validation:Optional QueryLimits *QueryLimitSpec `json:"queries,omitempty"` - // OTLP to configure which resource, scope and log attributes - // to store as labels or structured metadata or drop them altogether - // for all tenants. + // OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + // + // Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + // enforce the use of some required attributes. // // +optional // +kubebuilder:validation:Optional - OTLP *GlobalOTLPSpec `json:"otlp,omitempty"` + OTLP *OTLPSpec `json:"otlp,omitempty"` // Retention defines how long logs are kept in storage. // @@ -993,7 +954,7 @@ type LimitsTemplateSpec struct { Retention *RetentionLimitSpec `json:"retention,omitempty"` } -// LimitsTemplateSpec defines the limits applied at ingestion or query path. +// PerTenantLimitsTemplateSpec defines the limits applied at ingestion or query path. type PerTenantLimitsTemplateSpec struct { // IngestionLimits defines the limits applied on ingested log streams. // @@ -1007,9 +968,12 @@ type PerTenantLimitsTemplateSpec struct { // +kubebuilder:validation:Optional QueryLimits *PerTenantQueryLimitSpec `json:"queries,omitempty"` - // OTLP to configure which resource, scope and log attributes - // to store as labels or structured metadata or drop them altogether - // for a single tenants. + // OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + // + // Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + // enforce the use of some required attributes. + // + // The per-tenant configuration for OTLP attributes will be merged with the global configuration. // // +optional // +kubebuilder:validation:Optional @@ -1463,16 +1427,3 @@ func (t BlockedQueryTypes) String() string { return strings.Join(res, ",") } - -func (a OTLPAttributeAction) Value() string { - switch a { - case OTLPAttributeActionIndexLabel: - return "index_label" - case OTLPAttributeActionStructuredMetadata: - return "structured_metadata" - case OTLPAttributeActionDrop: - return "drop" - default: - return string(a) - } -} diff --git a/operator/api/loki/v1/v1.go b/operator/api/loki/v1/v1.go index a17e7244dfb5..77ae0fa417ce 100644 --- a/operator/api/loki/v1/v1.go +++ b/operator/api/loki/v1/v1.go @@ -84,12 +84,12 @@ var ( // ErrIPv6InstanceAddrTypeNotAllowed when the default InstanceAddrType is used with enableIPv6. ErrIPv6InstanceAddrTypeNotAllowed = errors.New(`instanceAddrType "default" cannot be used with enableIPv6 at the same time`) - // ErrOTLPResourceAttributesEmptyNotAllowed when the OTLP ResourceAttributes are empty even though ignoreDefaults is enabled. - ErrOTLPResourceAttributesEmptyNotAllowed = errors.New(`resourceAttributes cannot be empty when ignoreDefaults is true`) - // ErrOTLPResourceAttributesIndexLabelActionMissing when OTLP ResourceAttributes does not contain at least one index label when ignoreDefaults is enabled. - ErrOTLPResourceAttributesIndexLabelActionMissing = errors.New(`resourceAttributes does not contain at least one attributed mapped to "index_label"`) - // ErrOTLPAttributesSpecInvalid when the OTLPAttributesSpec attibutes and regex fields are both empty. - ErrOTLPAttributesSpecInvalid = errors.New(`attributes and regex cannot be empty at the same time`) + // ErrOTLPGlobalNoStreamLabel when the global OTLP configuration does not define at least one stream label. + ErrOTLPGlobalNoStreamLabel = errors.New("global OTLP configuration needs to define at least one stream label") + // ErrOTLPTenantMissing when a tenant is missing from the OTLP configuration although it has been defined in the tenancy. + ErrOTLPTenantMissing = errors.New("if no global OTLP configuration is present which defines at least one stream label, every tenant must have an OTLP configuration") + // ErrOTLPTenantNoStreamLabel when a tenant is defined but has no stream labels and there also no global stream labels. + ErrOTLPTenantNoStreamLabel = errors.New("if no global OTLP configuration is present which defines at least one stream label, every tenant must define at least one stream label") // ErrRuleMustMatchNamespace indicates that an expression used in an alerting or recording rule is missing // matchers for a namespace. diff --git a/operator/api/loki/v1/zz_generated.deepcopy.go b/operator/api/loki/v1/zz_generated.deepcopy.go index faab229b2a56..99c8caa49432 100644 --- a/operator/api/loki/v1/zz_generated.deepcopy.go +++ b/operator/api/loki/v1/zz_generated.deepcopy.go @@ -504,27 +504,6 @@ func (in *ClusterProxy) DeepCopy() *ClusterProxy { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GlobalOTLPSpec) DeepCopyInto(out *GlobalOTLPSpec) { - *out = *in - if in.IndexedResourceAttributes != nil { - in, out := &in.IndexedResourceAttributes, &out.IndexedResourceAttributes - *out = make([]string, len(*in)) - copy(*out, *in) - } - in.OTLPSpec.DeepCopyInto(&out.OTLPSpec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalOTLPSpec. -func (in *GlobalOTLPSpec) DeepCopy() *GlobalOTLPSpec { - if in == nil { - return nil - } - out := new(GlobalOTLPSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HashRingSpec) DeepCopyInto(out *HashRingSpec) { *out = *in @@ -602,7 +581,7 @@ func (in *LimitsTemplateSpec) DeepCopyInto(out *LimitsTemplateSpec) { } if in.OTLP != nil { in, out := &in.OTLP, &out.OTLP - *out = new(GlobalOTLPSpec) + *out = new(OTLPSpec) (*in).DeepCopyInto(*out) } if in.Retention != nil { @@ -1084,97 +1063,91 @@ func (in *OPASpec) DeepCopy() *OPASpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTLPAttributesSpec) DeepCopyInto(out *OTLPAttributesSpec) { +func (in *OTLPAttributeReference) DeepCopyInto(out *OTLPAttributeReference) { *out = *in - if in.Attributes != nil { - in, out := &in.Attributes, &out.Attributes - *out = make([]string, len(*in)) - copy(*out, *in) - } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPAttributesSpec. -func (in *OTLPAttributesSpec) DeepCopy() *OTLPAttributesSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPAttributeReference. +func (in *OTLPAttributeReference) DeepCopy() *OTLPAttributeReference { if in == nil { return nil } - out := new(OTLPAttributesSpec) + out := new(OTLPAttributeReference) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTLPResourceAttributesConfigSpec) DeepCopyInto(out *OTLPResourceAttributesConfigSpec) { +func (in *OTLPMetadataSpec) DeepCopyInto(out *OTLPMetadataSpec) { *out = *in - if in.Attributes != nil { - in, out := &in.Attributes, &out.Attributes - *out = make([]string, len(*in)) + if in.ResourceAttributes != nil { + in, out := &in.ResourceAttributes, &out.ResourceAttributes + *out = make([]OTLPAttributeReference, len(*in)) + copy(*out, *in) + } + if in.ScopeAttributes != nil { + in, out := &in.ScopeAttributes, &out.ScopeAttributes + *out = make([]OTLPAttributeReference, len(*in)) + copy(*out, *in) + } + if in.LogAttributes != nil { + in, out := &in.LogAttributes, &out.LogAttributes + *out = make([]OTLPAttributeReference, len(*in)) copy(*out, *in) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPResourceAttributesConfigSpec. -func (in *OTLPResourceAttributesConfigSpec) DeepCopy() *OTLPResourceAttributesConfigSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPMetadataSpec. +func (in *OTLPMetadataSpec) DeepCopy() *OTLPMetadataSpec { if in == nil { return nil } - out := new(OTLPResourceAttributesConfigSpec) + out := new(OTLPMetadataSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTLPResourceAttributesSpec) DeepCopyInto(out *OTLPResourceAttributesSpec) { +func (in *OTLPSpec) DeepCopyInto(out *OTLPSpec) { *out = *in - if in.Attributes != nil { - in, out := &in.Attributes, &out.Attributes - *out = make([]OTLPResourceAttributesConfigSpec, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + if in.StreamLabels != nil { + in, out := &in.StreamLabels, &out.StreamLabels + *out = new(OTLPStreamLabelSpec) + (*in).DeepCopyInto(*out) + } + if in.StructuredMetadata != nil { + in, out := &in.StructuredMetadata, &out.StructuredMetadata + *out = new(OTLPMetadataSpec) + (*in).DeepCopyInto(*out) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPResourceAttributesSpec. -func (in *OTLPResourceAttributesSpec) DeepCopy() *OTLPResourceAttributesSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPSpec. +func (in *OTLPSpec) DeepCopy() *OTLPSpec { if in == nil { return nil } - out := new(OTLPResourceAttributesSpec) + out := new(OTLPSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTLPSpec) DeepCopyInto(out *OTLPSpec) { +func (in *OTLPStreamLabelSpec) DeepCopyInto(out *OTLPStreamLabelSpec) { *out = *in if in.ResourceAttributes != nil { in, out := &in.ResourceAttributes, &out.ResourceAttributes - *out = new(OTLPResourceAttributesSpec) - (*in).DeepCopyInto(*out) - } - if in.ScopeAttributes != nil { - in, out := &in.ScopeAttributes, &out.ScopeAttributes - *out = make([]OTLPAttributesSpec, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.LogAttributes != nil { - in, out := &in.LogAttributes, &out.LogAttributes - *out = make([]OTLPAttributesSpec, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + *out = make([]OTLPAttributeReference, len(*in)) + copy(*out, *in) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPSpec. -func (in *OTLPSpec) DeepCopy() *OTLPSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPStreamLabelSpec. +func (in *OTLPStreamLabelSpec) DeepCopy() *OTLPStreamLabelSpec { if in == nil { return nil } - out := new(OTLPSpec) + out := new(OTLPStreamLabelSpec) in.DeepCopyInto(out) return out } @@ -1251,6 +1224,21 @@ func (in *ObjectStorageTLSSpec) DeepCopy() *ObjectStorageTLSSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenshiftOTLPConfig) DeepCopyInto(out *OpenshiftOTLPConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenshiftOTLPConfig. +func (in *OpenshiftOTLPConfig) DeepCopy() *OpenshiftOTLPConfig { + if in == nil { + return nil + } + out := new(OpenshiftOTLPConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenshiftTenantSpec) DeepCopyInto(out *OpenshiftTenantSpec) { *out = *in @@ -1259,6 +1247,11 @@ func (in *OpenshiftTenantSpec) DeepCopyInto(out *OpenshiftTenantSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.OTLP != nil { + in, out := &in.OTLP, &out.OTLP + *out = new(OpenshiftOTLPConfig) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenshiftTenantSpec. diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index a84820defa35..fce1efd9432a 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.6.2 - createdAt: "2024-10-21T11:24:52Z" + createdAt: "2024-10-21T14:32:40Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. features.operators.openshift.io/disconnected: "true" @@ -373,76 +373,62 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - displayName: Indexed Resource Attributes - path: limits.global.otlp.indexedResourceAttributes - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.global.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.global.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.global.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.global.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.global.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.global.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.global.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.global.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.global.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.scopeAttributes[0].regex + path: limits.global.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -548,71 +534,62 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.tenants.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.tenants.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.tenants.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.tenants.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.tenants.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.tenants.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.tenants.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.tenants.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.scopeAttributes[0].regex + path: limits.tenants.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked @@ -1030,6 +1007,28 @@ spec: - dedicated-admin displayName: Admin Groups path: tenants.openshift.adminGroups + - description: OTLP contains settings for ingesting data using OTLP in the OpenShift + tenancy mode. + displayName: OpenTelemetry Protocol + path: tenants.openshift.otlp + - description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + displayName: Disable recommended OTLP attributes + path: tenants.openshift.otlp.disableRecommendedAttributes statusDescriptors: - description: Distributor is a map to the per pod status of the distributor deployment diff --git a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml index d1c01b5cb082..87b204da9736 100644 --- a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -165,125 +165,101 @@ spec: type: object otlp: description: |- - OTLP to configure which resource, scope and log attributes - to store as labels or structured metadata or drop them altogether - for all tenants. + OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + + Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + enforce the use of some required attributes. properties: - indexedResourceAttributes: - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - items: - type: string - type: array - logAttributes: - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array - resourceAttributes: - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + streamLabels: + description: StreamLabels configures which resource attributes + are converted to Loki stream labels. properties: - attributes: - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + resourceAttributes: + description: ResourceAttributes lists the names of + the resource attributes that should be converted + into Loki stream labels. items: - description: |- - OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes - to store them as index labels or structured metadata or drop them altogether. properties: - action: - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - enum: - - index_label - - structured_metadata - - drop + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string - attributes: - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - items: - type: string - type: array regex: - description: Regex allows choosing the attributes - by matching a regular expression. + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object + structuredMetadata: + description: StructuredMetadata configures which attributes + are saved in structured metadata. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean required: - - action + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names of + resource attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of scope + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name type: object type: array - ignoreDefaults: - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - type: boolean type: object - scopeAttributes: - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array type: object queries: description: QueryLimits defines the limit applied on querying @@ -363,7 +339,7 @@ spec: type: object tenants: additionalProperties: - description: LimitsTemplateSpec defines the limits applied + description: PerTenantLimitsTemplateSpec defines the limits applied at ingestion or query path. properties: ingestion: @@ -430,118 +406,103 @@ spec: type: object otlp: description: |- - OTLP to configure which resource, scope and log attributes - to store as labels or structured metadata or drop them altogether - for a single tenants. + OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + + Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + enforce the use of some required attributes. + + The per-tenant configuration for OTLP attributes will be merged with the global configuration. properties: - logAttributes: - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array - resourceAttributes: - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + streamLabels: + description: StreamLabels configures which resource + attributes are converted to Loki stream labels. properties: - attributes: - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + resourceAttributes: + description: ResourceAttributes lists the names + of the resource attributes that should be converted + into Loki stream labels. items: - description: |- - OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes - to store them as index labels or structured metadata or drop them altogether. properties: - action: - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - enum: - - index_label - - structured_metadata - - drop + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string - attributes: - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - items: - type: string - type: array regex: - description: Regex allows choosing the attributes - by matching a regular expression. + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object + structuredMetadata: + description: StructuredMetadata configures which attributes + are saved in structured metadata. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean required: - - action + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names + of resource attributes that should be included + in structured metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of + scope attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name type: object type: array - ignoreDefaults: - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - type: boolean type: object - scopeAttributes: - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array type: object queries: description: QueryLimits defines the limit applied on querying @@ -3903,6 +3864,26 @@ spec: items: type: string type: array + otlp: + description: OTLP contains settings for ingesting data using + OTLP in the OpenShift tenancy mode. + properties: + disableRecommendedAttributes: + description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + type: boolean + type: object type: object required: - mode diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml index 585bdd34b130..8dc0d34e36f6 100644 --- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.6.2 - createdAt: "2024-10-21T11:24:51Z" + createdAt: "2024-10-21T14:32:37Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -366,76 +366,62 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - displayName: Indexed Resource Attributes - path: limits.global.otlp.indexedResourceAttributes - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.global.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.global.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.global.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.global.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.global.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.global.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.global.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.global.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.global.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.scopeAttributes[0].regex + path: limits.global.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -541,71 +527,62 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.tenants.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.tenants.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.tenants.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.tenants.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.tenants.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.tenants.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.tenants.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.tenants.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.scopeAttributes[0].regex + path: limits.tenants.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked @@ -1023,6 +1000,28 @@ spec: - dedicated-admin displayName: Admin Groups path: tenants.openshift.adminGroups + - description: OTLP contains settings for ingesting data using OTLP in the OpenShift + tenancy mode. + displayName: OpenTelemetry Protocol + path: tenants.openshift.otlp + - description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + displayName: Disable recommended OTLP attributes + path: tenants.openshift.otlp.disableRecommendedAttributes statusDescriptors: - description: Distributor is a map to the per pod status of the distributor deployment diff --git a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml index 73e3ff4bde67..8dd8ad938683 100644 --- a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml @@ -166,125 +166,101 @@ spec: type: object otlp: description: |- - OTLP to configure which resource, scope and log attributes - to store as labels or structured metadata or drop them altogether - for all tenants. + OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + + Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + enforce the use of some required attributes. properties: - indexedResourceAttributes: - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - items: - type: string - type: array - logAttributes: - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array - resourceAttributes: - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + streamLabels: + description: StreamLabels configures which resource attributes + are converted to Loki stream labels. properties: - attributes: - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + resourceAttributes: + description: ResourceAttributes lists the names of + the resource attributes that should be converted + into Loki stream labels. items: - description: |- - OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes - to store them as index labels or structured metadata or drop them altogether. properties: - action: - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - enum: - - index_label - - structured_metadata - - drop + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string - attributes: - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - items: - type: string - type: array regex: - description: Regex allows choosing the attributes - by matching a regular expression. + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object + structuredMetadata: + description: StructuredMetadata configures which attributes + are saved in structured metadata. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean required: - - action + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names of + resource attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of scope + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name type: object type: array - ignoreDefaults: - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - type: boolean type: object - scopeAttributes: - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array type: object queries: description: QueryLimits defines the limit applied on querying @@ -364,7 +340,7 @@ spec: type: object tenants: additionalProperties: - description: LimitsTemplateSpec defines the limits applied + description: PerTenantLimitsTemplateSpec defines the limits applied at ingestion or query path. properties: ingestion: @@ -431,118 +407,103 @@ spec: type: object otlp: description: |- - OTLP to configure which resource, scope and log attributes - to store as labels or structured metadata or drop them altogether - for a single tenants. + OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + + Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + enforce the use of some required attributes. + + The per-tenant configuration for OTLP attributes will be merged with the global configuration. properties: - logAttributes: - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array - resourceAttributes: - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + streamLabels: + description: StreamLabels configures which resource + attributes are converted to Loki stream labels. properties: - attributes: - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + resourceAttributes: + description: ResourceAttributes lists the names + of the resource attributes that should be converted + into Loki stream labels. items: - description: |- - OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes - to store them as index labels or structured metadata or drop them altogether. properties: - action: - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - enum: - - index_label - - structured_metadata - - drop + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string - attributes: - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - items: - type: string - type: array regex: - description: Regex allows choosing the attributes - by matching a regular expression. + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object + structuredMetadata: + description: StructuredMetadata configures which attributes + are saved in structured metadata. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean required: - - action + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names + of resource attributes that should be included + in structured metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of + scope attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name type: object type: array - ignoreDefaults: - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - type: boolean type: object - scopeAttributes: - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array type: object queries: description: QueryLimits defines the limit applied on querying @@ -3904,6 +3865,26 @@ spec: items: type: string type: array + otlp: + description: OTLP contains settings for ingesting data using + OTLP in the OpenShift tenancy mode. + properties: + disableRecommendedAttributes: + description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + type: boolean + type: object type: object required: - mode diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index 8cbf60130ce1..9d42513fc749 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: quay.io/openshift-logging/loki-operator:0.1.0 - createdAt: "2024-10-21T11:24:54Z" + createdAt: "2024-10-21T14:32:43Z" description: | The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging. ## Prerequisites and Requirements @@ -386,76 +386,62 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - displayName: Indexed Resource Attributes - path: limits.global.otlp.indexedResourceAttributes - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.global.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.global.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.global.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.global.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.global.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.global.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.global.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.global.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.global.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.scopeAttributes[0].regex + path: limits.global.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -561,71 +547,62 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.tenants.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.tenants.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.tenants.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.tenants.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.tenants.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.tenants.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.tenants.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.tenants.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.scopeAttributes[0].regex + path: limits.tenants.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked @@ -1043,6 +1020,28 @@ spec: - dedicated-admin displayName: Admin Groups path: tenants.openshift.adminGroups + - description: OTLP contains settings for ingesting data using OTLP in the OpenShift + tenancy mode. + displayName: OpenTelemetry Protocol + path: tenants.openshift.otlp + - description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + displayName: Disable recommended OTLP attributes + path: tenants.openshift.otlp.disableRecommendedAttributes statusDescriptors: - description: Distributor is a map to the per pod status of the distributor deployment diff --git a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml index 6651f96453e2..828a99481535 100644 --- a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -165,125 +165,101 @@ spec: type: object otlp: description: |- - OTLP to configure which resource, scope and log attributes - to store as labels or structured metadata or drop them altogether - for all tenants. + OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + + Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + enforce the use of some required attributes. properties: - indexedResourceAttributes: - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - items: - type: string - type: array - logAttributes: - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array - resourceAttributes: - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + streamLabels: + description: StreamLabels configures which resource attributes + are converted to Loki stream labels. properties: - attributes: - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + resourceAttributes: + description: ResourceAttributes lists the names of + the resource attributes that should be converted + into Loki stream labels. items: - description: |- - OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes - to store them as index labels or structured metadata or drop them altogether. properties: - action: - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - enum: - - index_label - - structured_metadata - - drop + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string - attributes: - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - items: - type: string - type: array regex: - description: Regex allows choosing the attributes - by matching a regular expression. + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object + structuredMetadata: + description: StructuredMetadata configures which attributes + are saved in structured metadata. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean required: - - action + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names of + resource attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of scope + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name type: object type: array - ignoreDefaults: - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - type: boolean type: object - scopeAttributes: - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array type: object queries: description: QueryLimits defines the limit applied on querying @@ -363,7 +339,7 @@ spec: type: object tenants: additionalProperties: - description: LimitsTemplateSpec defines the limits applied + description: PerTenantLimitsTemplateSpec defines the limits applied at ingestion or query path. properties: ingestion: @@ -430,118 +406,103 @@ spec: type: object otlp: description: |- - OTLP to configure which resource, scope and log attributes - to store as labels or structured metadata or drop them altogether - for a single tenants. + OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + + Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + enforce the use of some required attributes. + + The per-tenant configuration for OTLP attributes will be merged with the global configuration. properties: - logAttributes: - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array - resourceAttributes: - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + streamLabels: + description: StreamLabels configures which resource + attributes are converted to Loki stream labels. properties: - attributes: - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + resourceAttributes: + description: ResourceAttributes lists the names + of the resource attributes that should be converted + into Loki stream labels. items: - description: |- - OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes - to store them as index labels or structured metadata or drop them altogether. properties: - action: - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - enum: - - index_label - - structured_metadata - - drop + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string - attributes: - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - items: - type: string - type: array regex: - description: Regex allows choosing the attributes - by matching a regular expression. + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object + structuredMetadata: + description: StructuredMetadata configures which attributes + are saved in structured metadata. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean required: - - action + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names + of resource attributes that should be included + in structured metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of + scope attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name type: object type: array - ignoreDefaults: - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - type: boolean type: object - scopeAttributes: - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array type: object queries: description: QueryLimits defines the limit applied on querying @@ -3903,6 +3864,26 @@ spec: items: type: string type: array + otlp: + description: OTLP contains settings for ingesting data using + OTLP in the OpenShift tenancy mode. + properties: + disableRecommendedAttributes: + description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + type: boolean + type: object type: object required: - mode diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 6df041245f15..a4ce94922ffe 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -147,125 +147,101 @@ spec: type: object otlp: description: |- - OTLP to configure which resource, scope and log attributes - to store as labels or structured metadata or drop them altogether - for all tenants. + OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + + Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + enforce the use of some required attributes. properties: - indexedResourceAttributes: - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - items: - type: string - type: array - logAttributes: - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array - resourceAttributes: - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + streamLabels: + description: StreamLabels configures which resource attributes + are converted to Loki stream labels. properties: - attributes: - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + resourceAttributes: + description: ResourceAttributes lists the names of + the resource attributes that should be converted + into Loki stream labels. items: - description: |- - OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes - to store them as index labels or structured metadata or drop them altogether. properties: - action: - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - enum: - - index_label - - structured_metadata - - drop + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string - attributes: - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - items: - type: string - type: array regex: - description: Regex allows choosing the attributes - by matching a regular expression. + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object + structuredMetadata: + description: StructuredMetadata configures which attributes + are saved in structured metadata. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean required: - - action + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names of + resource attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of scope + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name type: object type: array - ignoreDefaults: - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - type: boolean type: object - scopeAttributes: - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array type: object queries: description: QueryLimits defines the limit applied on querying @@ -345,7 +321,7 @@ spec: type: object tenants: additionalProperties: - description: LimitsTemplateSpec defines the limits applied + description: PerTenantLimitsTemplateSpec defines the limits applied at ingestion or query path. properties: ingestion: @@ -412,118 +388,103 @@ spec: type: object otlp: description: |- - OTLP to configure which resource, scope and log attributes - to store as labels or structured metadata or drop them altogether - for a single tenants. + OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata. + + Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even + enforce the use of some required attributes. + + The per-tenant configuration for OTLP attributes will be merged with the global configuration. properties: - logAttributes: - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array - resourceAttributes: - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + streamLabels: + description: StreamLabels configures which resource + attributes are converted to Loki stream labels. properties: - attributes: - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + resourceAttributes: + description: ResourceAttributes lists the names + of the resource attributes that should be converted + into Loki stream labels. items: - description: |- - OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes - to store them as index labels or structured metadata or drop them altogether. properties: - action: - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - enum: - - index_label - - structured_metadata - - drop + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string - attributes: - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - items: - type: string - type: array regex: - description: Regex allows choosing the attributes - by matching a regular expression. + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object + structuredMetadata: + description: StructuredMetadata configures which attributes + are saved in structured metadata. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean required: - - action + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names + of resource attributes that should be included + in structured metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of + scope attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name type: object type: array - ignoreDefaults: - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - type: boolean type: object - scopeAttributes: - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. - items: - description: |- - OTLPAttributesSpec contains the configuration for a set of attributes - to store them as index labels or structured metadata or drop them altogether. - properties: - action: - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - enum: - - structured_metadata - - drop - type: string - attributes: - description: Attributes allows choosing the attributes - by listing their names. - items: - type: string - type: array - regex: - description: Regex allows choosing the attributes - by matching a regular expression. - type: string - required: - - action - type: object - type: array type: object queries: description: QueryLimits defines the limit applied on querying @@ -3885,6 +3846,26 @@ spec: items: type: string type: array + otlp: + description: OTLP contains settings for ingesting data using + OTLP in the OpenShift tenancy mode. + properties: + disableRecommendedAttributes: + description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + type: boolean + type: object type: object required: - mode diff --git a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml index 0e6798f255c2..78710b48121d 100644 --- a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml @@ -288,76 +288,62 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - displayName: Indexed Resource Attributes - path: limits.global.otlp.indexedResourceAttributes - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.global.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.global.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.global.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.global.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.global.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.global.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.global.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.global.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.global.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.scopeAttributes[0].regex + path: limits.global.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -463,71 +449,62 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.tenants.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.tenants.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.tenants.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.tenants.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.tenants.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.tenants.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.tenants.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.tenants.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.scopeAttributes[0].regex + path: limits.tenants.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked @@ -945,6 +922,28 @@ spec: - dedicated-admin displayName: Admin Groups path: tenants.openshift.adminGroups + - description: OTLP contains settings for ingesting data using OTLP in the OpenShift + tenancy mode. + displayName: OpenTelemetry Protocol + path: tenants.openshift.otlp + - description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + displayName: Disable recommended OTLP attributes + path: tenants.openshift.otlp.disableRecommendedAttributes statusDescriptors: - description: Distributor is a map to the per pod status of the distributor deployment diff --git a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml index 616d5e653184..77760c0c6633 100644 --- a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml @@ -281,76 +281,62 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - displayName: Indexed Resource Attributes - path: limits.global.otlp.indexedResourceAttributes - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.global.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.global.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.global.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.global.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.global.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.global.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.global.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.global.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.global.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.scopeAttributes[0].regex + path: limits.global.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -456,71 +442,62 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.tenants.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.tenants.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.tenants.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.tenants.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.tenants.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.tenants.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.tenants.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.tenants.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.scopeAttributes[0].regex + path: limits.tenants.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked @@ -938,6 +915,28 @@ spec: - dedicated-admin displayName: Admin Groups path: tenants.openshift.adminGroups + - description: OTLP contains settings for ingesting data using OTLP in the OpenShift + tenancy mode. + displayName: OpenTelemetry Protocol + path: tenants.openshift.otlp + - description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + displayName: Disable recommended OTLP attributes + path: tenants.openshift.otlp.disableRecommendedAttributes statusDescriptors: - description: Distributor is a map to the per pod status of the distributor deployment diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml index f5d6b5efd659..66bcc30f2b52 100644 --- a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml @@ -300,76 +300,62 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - IndexedResourceAttributes contains the global configuration for resource attributes - to store them as index labels. - displayName: Indexed Resource Attributes - path: limits.global.otlp.indexedResourceAttributes - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.global.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.global.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.global.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.global.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.global.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.global.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.global.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.global.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.global.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.global.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.global.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.global.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.global.otlp.scopeAttributes[0].regex + path: limits.global.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -475,71 +461,62 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: |- - LogAttributes contains the configuration for log attributes - to store them as structured metadata or drop them altogether. + - description: StreamLabels configures which resource attributes are converted + to Loki stream labels. + displayName: Stream Labels + path: limits.tenants.otlp.streamLabels + - description: ResourceAttributes lists the names of the resource attributes + that should be converted into Loki stream labels. + displayName: Resource Attributes + path: limits.tenants.otlp.streamLabels.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex + - description: StructuredMetadata configures which attributes are saved in structured + metadata. + displayName: Structured Metadata + path: limits.tenants.otlp.structuredMetadata + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. displayName: Log Attributes - path: limits.tenants.otlp.logAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.logAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.logAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.logAttributes[0].regex - - description: |- - ResourceAttributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. displayName: Resource Attributes - path: limits.tenants.otlp.resourceAttributes - - description: |- - Attributes contains the configuration for resource attributes - to store them as index labels or structured metadata or drop them altogether. - displayName: Attributes - path: limits.tenants.otlp.resourceAttributes.attributes - - description: |- - Action defines the indexing action for the selected resoure attributes. They - can be either indexed as labels, added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.resourceAttributes.attributes[0].action - - description: |- - Attributes is the list of attributes to configure indexing or drop them - altogether. - displayName: Attribute Names - path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.resourceAttributes.attributes[0].regex - - description: |- - IgnoreDefaults controls whether to ignore the global configuration for resource attributes - indexed as labels. - - - If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. - displayName: Ignore Global Defaults - path: limits.tenants.otlp.resourceAttributes.ignoreDefaults - x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - - description: |- - ScopeAttributes contains the configuration for scope attributes - to store them as structured metadata or drop them altogether. + path: limits.tenants.otlp.structuredMetadata.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. displayName: Scope Attributes - path: limits.tenants.otlp.scopeAttributes - - description: |- - Action defines the indexing action for the selected attributes. They - can be either added to structured metadata or drop altogether. - displayName: Action - path: limits.tenants.otlp.scopeAttributes[0].action - - description: Attributes allows choosing the attributes by listing their names. - displayName: Attribute Names - path: limits.tenants.otlp.scopeAttributes[0].attributes - - description: Regex allows choosing the attributes by matching a regular expression. - displayName: Regular Expression - path: limits.tenants.otlp.scopeAttributes[0].regex + path: limits.tenants.otlp.structuredMetadata.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked @@ -957,6 +934,28 @@ spec: - dedicated-admin displayName: Admin Groups path: tenants.openshift.adminGroups + - description: OTLP contains settings for ingesting data using OTLP in the OpenShift + tenancy mode. + displayName: OpenTelemetry Protocol + path: tenants.openshift.otlp + - description: |- + DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured + metadata. + + + Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause + meta information to not be available as stream labels or structured metadata, potentially making queries more + expensive and less performant. + + + Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be + added to the configuration, even if this field is set to true. + + + This option is supposed to be combined with a custom label configuration customizing the labels for the specific + usecase. + displayName: Disable recommended OTLP attributes + path: tenants.openshift.otlp.disableRecommendedAttributes statusDescriptors: - description: Distributor is a map to the per pod status of the distributor deployment diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index 47e690b3888c..8ea65485112d 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -1146,51 +1146,6 @@ a secret. This mode is only supported for certain object storage types in certai -## GlobalOTLPSpec { #loki-grafana-com-v1-GlobalOTLPSpec } -

-(Appears on:LimitsTemplateSpec) -

-
-

GlobalOTLPSpec defines which resource, scope and log attributes to -be stored as index or structured metadata or drop altogether for all -tenants.

-
- - - - - - - - - - - - - - - - - -
FieldDescription
-indexedResourceAttributes
- -[]string - -		 -(Optional) -

IndexedResourceAttributes contains the global configuration for resource attributes -to store them as index labels.

-
-OTLPSpec
- - -OTLPSpec - - -		 -
- ## HashRingSpec { #loki-grafana-com-v1-HashRingSpec }

(Appears on:LokiStackSpec) @@ -1520,16 +1475,16 @@ QueryLimitSpec otlp
- -GlobalOTLPSpec + +OTLPSpec (Optional) -

OTLP to configure which resource, scope and log attributes -to store as labels or structured metadata or drop them altogether -for all tenants.

+

OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata.

+

Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even +enforce the use of some required attributes.

@@ -2663,42 +2618,11 @@ string -## OTLPAttributeAction { #loki-grafana-com-v1-OTLPAttributeAction } -(string alias) +## OTLPAttributeReference { #loki-grafana-com-v1-OTLPAttributeReference }

-(Appears on:OTLPAttributesSpec, OTLPResourceAttributesConfigSpec) -

-
-

OTLPAttributeAction defines the action to executed when indexing -OTLP resource attributes. Resource attributes can be either added -to the index, the chunk structured metadata or entirely dropped.

-
- - - - - - - - - - - - - - -
ValueDescription

"drop"

OTLPAttributeActionDrop removes the matching attributes from the log entry.

-

"indexLabel"

OTLPAttributeActionIndexLabel stores a resource attribute as a label, which is part of the index identifying streams.

-

"structuredMetadata"

OTLPAttributeActionStructuredMetadata stores an attribute as structured metadata with each log entry.

-
- -## OTLPAttributesSpec { #loki-grafana-com-v1-OTLPAttributesSpec } -

-(Appears on:OTLPSpec) +(Appears on:OTLPMetadataSpec, OTLPStreamLabelSpec)

-

OTLPAttributesSpec contains the configuration for a set of attributes -to store them as index labels or structured metadata or drop them altogether.

@@ -2710,52 +2634,35 @@ to store them as index labels or structured metadata or drop them altogether.

- - - -
-action
- - -OTLPAttributeAction - - -		 -

Action defines the indexing action for the selected attributes. They -can be either added to structured metadata or drop altogether.

-
-attributes
+name
-[]string +string 		-(Optional) -

Attributes allows choosing the attributes by listing their names.

+

Name contains either a verbatim name of an attribute or a regular expression matching many attributes.
regex
-string +bool 		(Optional) -

Regex allows choosing the attributes by matching a regular expression.

+

If Regex is true, then Name is treated as a regular expression instead of as a verbatim attribute name.
-## OTLPResourceAttributesConfigSpec { #loki-grafana-com-v1-OTLPResourceAttributesConfigSpec } +## OTLPMetadataSpec { #loki-grafana-com-v1-OTLPMetadataSpec }

-(Appears on:OTLPResourceAttributesSpec) +(Appears on:OTLPSpec)

-

OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes -to store them as index labels or structured metadata or drop them altogether.

@@ -2767,53 +2674,56 @@ to store them as index labels or structured metadata or drop them altogether.

-action
+resourceAttributes
- -OTLPAttributeAction + +[]OTLPAttributeReference 		-

Action defines the indexing action for the selected resoure attributes. They -can be either indexed as labels, added to structured metadata or drop altogether.

+(Optional) +

ResourceAttributes lists the names of resource attributes that should be included in structured metadata.
-attributes
+scopeAttributes
-[]string + +[]OTLPAttributeReference + 		(Optional) -

Attributes is the list of attributes to configure indexing or drop them -altogether.

+

ScopeAttributes lists the names of scope attributes that should be included in structured metadata.
-regex
+logAttributes
-string + +[]OTLPAttributeReference + 		(Optional) -

Regex allows choosing the attributes by matching a regular expression.

+

LogAttributes lists the names of log attributes that should be included in structured metadata.
-## OTLPResourceAttributesSpec { #loki-grafana-com-v1-OTLPResourceAttributesSpec } +## OTLPSpec { #loki-grafana-com-v1-OTLPSpec }

-(Appears on:OTLPSpec) +(Appears on:LimitsTemplateSpec, PerTenantLimitsTemplateSpec)

-

OTLPResourceAttributesSpec contains the configuration for resource attributes -to store them as index labels or structured metadata or drop them altogether.

+

OTLPSpec defines which resource, scope and log attributes should be used as stream labels or +stored as structured metadata.

@@ -2825,43 +2735,40 @@ to store them as index labels or structured metadata or drop them altogether.

-ignoreDefaults
+streamLabels
-bool + +OTLPStreamLabelSpec + 		(Optional) -

IgnoreDefaults controls whether to ignore the global configuration for resource attributes -indexed as labels.

-

If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label.

+

StreamLabels configures which resource attributes are converted to Loki stream labels.
-attributes
+structuredMetadata
- -[]OTLPResourceAttributesConfigSpec + +OTLPMetadataSpec 		(Optional) -

Attributes contains the configuration for resource attributes -to store them as index labels or structured metadata or drop them altogether.

+

StructuredMetadata configures which attributes are saved in structured metadata.
-## OTLPSpec { #loki-grafana-com-v1-OTLPSpec } +## OTLPStreamLabelSpec { #loki-grafana-com-v1-OTLPStreamLabelSpec }

-(Appears on:GlobalOTLPSpec, PerTenantLimitsTemplateSpec) +(Appears on:OTLPSpec)

-

OTLPSpec defines which resource, scope and log attributes to -be stored as index or structured metadata or drop altogether

@@ -2875,45 +2782,14 @@ be stored as index or structured metadata or drop altogether

- - - - - - - - @@ -3191,6 +3067,44 @@ It needs to be in the same namespace as the LokiStack custom resource.

resourceAttributes
- -OTLPResourceAttributesSpec - - -		 -(Optional) -

ResourceAttributes contains the configuration for resource attributes -to store them as index labels or structured metadata or drop them altogether.

-
-scopeAttributes
- - -[]OTLPAttributesSpec - - -		 -(Optional) -

ScopeAttributes contains the configuration for scope attributes -to store them as structured metadata or drop them altogether.

-
-logAttributes
- - -[]OTLPAttributesSpec + +[]OTLPAttributeReference 		(Optional) -

LogAttributes contains the configuration for log attributes -to store them as structured metadata or drop them altogether.

+

ResourceAttributes lists the names of the resource attributes that should be converted into Loki stream labels.
+## OpenshiftOTLPConfig { #loki-grafana-com-v1-OpenshiftOTLPConfig } +

+(Appears on:OpenshiftTenantSpec) +

+
+

OpenshiftOTLPConfig defines configuration specific to users using OTLP together with an OpenShift tenancy mode.

+
+ + + + + + + + + + + + + +
FieldDescription
+disableRecommendedAttributes
+ +bool + +		 +(Optional) +

DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured +metadata.

+

Enabling this setting removes the “recommended attributes” from the generated Loki configuration. This will cause +meta information to not be available as stream labels or structured metadata, potentially making queries more +expensive and less performant.

+

Note that there is a set of “required attributes”, needed for OpenShift Logging to work properly. Those will be +added to the configuration, even if this field is set to true.

+

This option is supposed to be combined with a custom label configuration customizing the labels for the specific +usecase.

+
+ ## OpenshiftTenantSpec { #loki-grafana-com-v1-OpenshiftTenantSpec }

(Appears on:TenantsSpec) @@ -3223,6 +3137,20 @@ Setting this to an empty array disables admin groups.

- dedicated-admin

+ + +otlp
+ + +OpenshiftOTLPConfig + + + + +(Optional) +

OTLP contains settings for ingesting data using OTLP in the OpenShift tenancy mode.

+ + @@ -3231,7 +3159,7 @@ Setting this to an empty array disables admin groups.

(Appears on:LimitsSpec)

-

LimitsTemplateSpec defines the limits applied at ingestion or query path.

+

PerTenantLimitsTemplateSpec defines the limits applied at ingestion or query path.

@@ -3280,9 +3208,10 @@ OTLPSpec diff --git a/operator/internal/manifests/config.go b/operator/internal/manifests/config.go index f40a8ad21c54..163d6f6d381d 100644 --- a/operator/internal/manifests/config.go +++ b/operator/internal/manifests/config.go @@ -202,8 +202,9 @@ func ConfigOptions(opt Options) config.Options { AlertManager: amConfig, RemoteWrite: rwConfig, }, - Retention: retentionConfig(&opt.Stack), - Overrides: overrides, + Retention: retentionConfig(&opt.Stack), + OTLPAttributes: otlpAttributeConfig(&opt.Stack), + Overrides: overrides, } } diff --git a/operator/internal/manifests/config_otlp.go b/operator/internal/manifests/config_otlp.go new file mode 100644 index 000000000000..8530b7e63cfb --- /dev/null +++ b/operator/internal/manifests/config_otlp.go @@ -0,0 +1,273 @@ +package manifests + +import ( + "slices" + "strings" + + lokiv1 "github.com/grafana/loki/operator/api/loki/v1" + "github.com/grafana/loki/operator/internal/manifests/internal/config" + "github.com/grafana/loki/operator/internal/manifests/openshift" +) + +func defaultOTLPAttributeConfig(ts *lokiv1.TenantsSpec) config.OTLPAttributeConfig { + if ts == nil || ts.Mode != lokiv1.OpenshiftLogging { + return config.OTLPAttributeConfig{} + } + + disableRecommended := false + if ts.Openshift != nil && ts.Openshift.OTLP != nil { + disableRecommended = ts.Openshift.OTLP.DisableRecommendedAttributes + } + + return openshift.DefaultOTLPAttributes(disableRecommended) +} + +func convertAttributeReferences(refs []lokiv1.OTLPAttributeReference, action config.OTLPAttributeAction) []config.OTLPAttribute { + var ( + names []string + result []config.OTLPAttribute + ) + + for _, attr := range refs { + if attr.Regex { + result = append(result, config.OTLPAttribute{ + Action: action, + Regex: attr.Name, + }) + continue + } + + names = append(names, attr.Name) + } + + if len(names) > 0 { + result = append(result, config.OTLPAttribute{ + Action: action, + Names: names, + }) + } + + return result +} + +func copyOTLPAttributes(in []config.OTLPAttribute) []config.OTLPAttribute { + result := make([]config.OTLPAttribute, 0, len(in)) + for _, attr := range in { + result = append(result, config.OTLPAttribute{ + Action: attr.Action, + Names: slices.Clone(attr.Names), + Regex: attr.Regex, + }) + } + + return result +} + +func copyTenantAttributeConfig(in *config.OTLPTenantAttributeConfig) *config.OTLPTenantAttributeConfig { + result := &config.OTLPTenantAttributeConfig{} + if in == nil { + return result + } + + if len(in.ResourceAttributes) > 0 { + result.ResourceAttributes = copyOTLPAttributes(in.ResourceAttributes) + } + + if len(in.ScopeAttributes) > 0 { + result.ScopeAttributes = copyOTLPAttributes(in.ScopeAttributes) + } + + if len(in.LogAttributes) > 0 { + result.LogAttributes = copyOTLPAttributes(in.LogAttributes) + } + + return result +} + +func convertTenantAttributeReferences(otlpSpec *lokiv1.OTLPSpec, base *config.OTLPTenantAttributeConfig) *config.OTLPTenantAttributeConfig { + result := copyTenantAttributeConfig(base) + + if streamLabels := otlpSpec.StreamLabels; streamLabels != nil { + result.ResourceAttributes = append(result.ResourceAttributes, + convertAttributeReferences(streamLabels.ResourceAttributes, config.OTLPAttributeActionStreamLabel)...) + } + + if structuredMetadata := otlpSpec.StructuredMetadata; structuredMetadata != nil { + if resAttr := structuredMetadata.ResourceAttributes; len(resAttr) > 0 { + result.ResourceAttributes = append(result.ResourceAttributes, + convertAttributeReferences(resAttr, config.OTLPAttributeActionMetadata)...) + } + + if scopeAttr := structuredMetadata.ScopeAttributes; len(scopeAttr) > 0 { + result.ScopeAttributes = append(result.ScopeAttributes, + convertAttributeReferences(scopeAttr, config.OTLPAttributeActionMetadata)...) + } + + if logAttr := structuredMetadata.LogAttributes; len(logAttr) > 0 { + result.LogAttributes = append(result.LogAttributes, + convertAttributeReferences(logAttr, config.OTLPAttributeActionMetadata)...) + } + } + + return result +} + +func sortAndDeduplicateOTLPConfig(cfg config.OTLPAttributeConfig) config.OTLPAttributeConfig { + if cfg.Global != nil { + if len(cfg.Global.ResourceAttributes) > 0 { + cfg.Global.ResourceAttributes = sortAndDeduplicateOTLPAttributes(cfg.Global.ResourceAttributes) + } + + if len(cfg.Global.ScopeAttributes) > 0 { + cfg.Global.ScopeAttributes = sortAndDeduplicateOTLPAttributes(cfg.Global.ScopeAttributes) + } + + if len(cfg.Global.LogAttributes) > 0 { + cfg.Global.LogAttributes = sortAndDeduplicateOTLPAttributes(cfg.Global.LogAttributes) + } + } + + for _, t := range cfg.Tenants { + if len(t.ResourceAttributes) > 0 { + t.ResourceAttributes = sortAndDeduplicateOTLPAttributes(t.ResourceAttributes) + } + + if len(t.ScopeAttributes) > 0 { + t.ScopeAttributes = sortAndDeduplicateOTLPAttributes(t.ScopeAttributes) + } + + if len(t.LogAttributes) > 0 { + t.LogAttributes = sortAndDeduplicateOTLPAttributes(t.LogAttributes) + } + } + + return cfg +} + +func sortAndDeduplicateOTLPAttributes(attrs []config.OTLPAttribute) []config.OTLPAttribute { + if len(attrs) == 0 { + // Skip everything for zero items + return attrs + } + + if len(attrs[0].Names) > 1 { + // If the first OTLPAttribute has names, then sort and de-duplicate them + slices.Sort(attrs[0].Names) + attrs[0].Names = slices.Compact(attrs[0].Names) + } + + if len(attrs) == 1 { + // If there's only one item, then skip the complex sorting + return attrs + } + + slices.SortFunc(attrs, func(a, b config.OTLPAttribute) int { + action := strings.Compare(string(a.Action), string(b.Action)) + if action != 0 { + return action + } + + if a.Regex != "" && b.Regex != "" { + return strings.Compare(a.Regex, b.Regex) + } + + if a.Regex != "" && b.Regex == "" { + return 1 + } + + if a.Regex == "" && b.Regex != "" { + return -1 + } + + return 0 + }) + + for i := 0; i < len(attrs)-1; i++ { + a := attrs[i] + if a.Regex != "" { + continue + } + + slices.Sort(a.Names) + attrs[i] = a + + next := attrs[i+1] + if next.Regex != "" { + continue + } + + if a.Action != next.Action { + continue + } + + // Combine attribute definitions if they have the same action and just contain names + a.Names = append(a.Names, next.Names...) + slices.Sort(a.Names) + a.Names = slices.Compact(a.Names) + + // Remove the "next" attribute definition + attrs[i] = a + attrs = append(attrs[:i+1], attrs[i+2:]...) + i-- + } + + return attrs +} + +func otlpAttributeConfig(ls *lokiv1.LokiStackSpec) config.OTLPAttributeConfig { + result := defaultOTLPAttributeConfig(ls.Tenants) + + if ls.Limits != nil { + if ls.Limits.Global != nil && ls.Limits.Global.OTLP != nil { + result.RemoveDefaultLabels = true + globalOTLP := ls.Limits.Global.OTLP + + if streamLabels := globalOTLP.StreamLabels; streamLabels != nil { + if result.Global == nil { + result.Global = &config.OTLPTenantAttributeConfig{} + } + + if resAttr := streamLabels.ResourceAttributes; len(resAttr) > 0 { + result.Global.ResourceAttributes = append(result.Global.ResourceAttributes, + convertAttributeReferences(resAttr, config.OTLPAttributeActionStreamLabel)...) + } + } + + if structuredMetadata := globalOTLP.StructuredMetadata; structuredMetadata != nil { + if result.Global == nil { + result.Global = &config.OTLPTenantAttributeConfig{} + } + + if resAttr := structuredMetadata.ResourceAttributes; len(resAttr) > 0 { + result.Global.ResourceAttributes = append(result.Global.ResourceAttributes, + convertAttributeReferences(resAttr, config.OTLPAttributeActionMetadata)...) + } + + if scopeAttr := structuredMetadata.ScopeAttributes; len(scopeAttr) > 0 { + result.Global.ScopeAttributes = append(result.Global.ScopeAttributes, + convertAttributeReferences(scopeAttr, config.OTLPAttributeActionMetadata)...) + } + + if logAttr := structuredMetadata.LogAttributes; len(logAttr) > 0 { + result.Global.LogAttributes = append(result.Global.LogAttributes, + convertAttributeReferences(logAttr, config.OTLPAttributeActionMetadata)...) + } + } + } + + for tenant, tenantLimits := range ls.Limits.Tenants { + if tenantLimits.OTLP != nil { + result.RemoveDefaultLabels = true + + if result.Tenants == nil { + result.Tenants = map[string]*config.OTLPTenantAttributeConfig{} + } + + tenantResult := convertTenantAttributeReferences(tenantLimits.OTLP, result.Global) + result.Tenants[tenant] = tenantResult + } + } + } + + return sortAndDeduplicateOTLPConfig(result) +} diff --git a/operator/internal/manifests/config_otlp_test.go b/operator/internal/manifests/config_otlp_test.go new file mode 100644 index 000000000000..73f4970c8dbf --- /dev/null +++ b/operator/internal/manifests/config_otlp_test.go @@ -0,0 +1,799 @@ +package manifests + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + lokiv1 "github.com/grafana/loki/operator/api/loki/v1" + "github.com/grafana/loki/operator/internal/manifests/internal/config" + "github.com/grafana/loki/operator/internal/manifests/openshift" +) + +func TestOtlpAttributeConfig(t *testing.T) { + tt := []struct { + desc string + spec lokiv1.LokiStackSpec + wantConfig config.OTLPAttributeConfig + }{ + { + desc: "empty", + }, + { + desc: "global stream label", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "stream.label", + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{ + "stream.label", + }, + }, + }, + }, + }, + }, + { + desc: "global stream label regex", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "stream\\.label\\.regex\\..+", + Regex: true, + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Regex: "stream\\.label\\.regex\\..+", + }, + }, + }, + }, + }, + { + desc: "global metadata", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "metadata", + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"metadata"}, + }, + }, + }, + }, + }, + { + desc: "global combined", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "stream.label", + }, + { + Name: "stream\\.label\\.regex\\..+", + Regex: true, + }, + }, + }, + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "resource.metadata", + }, + { + Name: "resource.metadata\\.other\\..+", + Regex: true, + }, + }, + ScopeAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "scope.metadata", + }, + { + Name: "scope.metadata\\.other\\..+", + Regex: true, + }, + }, + LogAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "log.metadata", + }, + { + Name: "log.metadata\\.other\\..+", + Regex: true, + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"stream.label"}, + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Regex: "stream\\.label\\.regex\\..+", + }, + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"resource.metadata"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "resource.metadata\\.other\\..+", + }, + }, + ScopeAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"scope.metadata"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "scope.metadata\\.other\\..+", + }, + }, + LogAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"log.metadata"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "log.metadata\\.other\\..+", + }, + }, + }, + }, + }, + { + desc: "tenant stream label", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.stream.label", + }, + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Tenants: map[string]*config.OTLPTenantAttributeConfig{ + "test-tenant": { + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"tenant.stream.label"}, + }, + }, + }, + }, + }, + }, + { + desc: "tenant stream label regex", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant\\.stream\\.label\\.regex\\..+", + Regex: true, + }, + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Tenants: map[string]*config.OTLPTenantAttributeConfig{ + "test-tenant": { + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Regex: "tenant\\.stream\\.label\\.regex\\..+", + }, + }, + }, + }, + }, + }, + { + desc: "tenant metadata", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.metadata", + }, + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Tenants: map[string]*config.OTLPTenantAttributeConfig{ + "test-tenant": { + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"tenant.metadata"}, + }, + }, + }, + }, + }, + }, + { + desc: "tenant combined", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.stream.label", + }, + { + Name: `tenant\.stream\.label\.regex\..+`, + Regex: true, + }, + }, + }, + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.resource.metadata", + }, + { + Name: `tenant\.resource.metadata\.other\..+`, + Regex: true, + }, + }, + ScopeAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.scope.metadata", + }, + { + Name: `tenant\.scope\.metadata\.other\..+`, + Regex: true, + }, + }, + LogAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.log.metadata", + }, + { + Name: `tenant\.log\.metadata\.other\..+`, + Regex: true, + }, + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Tenants: map[string]*config.OTLPTenantAttributeConfig{ + "test-tenant": { + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"tenant.stream.label"}, + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Regex: "tenant\\.stream\\.label\\.regex\\..+", + }, + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"tenant.resource.metadata"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: `tenant\.resource.metadata\.other\..+`, + }, + }, + ScopeAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"tenant.scope.metadata"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: `tenant\.scope\.metadata\.other\..+`, + }, + }, + LogAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"tenant.log.metadata"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: `tenant\.log\.metadata\.other\..+`, + }, + }, + }, + }, + }, + }, + { + desc: "global and tenant configuration with de-duplication", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "global.stream.label", + }, + { + Name: "another.stream.label", + }, + }, + }, + }, + }, + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.stream.label", + }, + { + Name: "another.stream.label", + }, + }, + }, + }, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{ + "another.stream.label", + "global.stream.label", + }, + }, + }, + }, + Tenants: map[string]*config.OTLPTenantAttributeConfig{ + "test-tenant": { + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{ + "another.stream.label", + "global.stream.label", + "tenant.stream.label", + }, + }, + }, + }, + }, + }, + }, + { + desc: "openshift-logging defaults", + spec: lokiv1.LokiStackSpec{ + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + }, + }, + wantConfig: openshift.DefaultOTLPAttributes(false), + }, + { + desc: "openshift-logging defaults without recommended", + spec: lokiv1.LokiStackSpec{ + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + Openshift: &lokiv1.OpenshiftTenantSpec{ + OTLP: &lokiv1.OpenshiftOTLPConfig{ + DisableRecommendedAttributes: true, + }, + }, + }, + }, + wantConfig: openshift.DefaultOTLPAttributes(true), + }, + { + desc: "openshift-logging defaults with additional custom attributes", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "custom.stream.label", + }, + }, + }, + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + LogAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "custom.log.metadata", + }, + }, + }, + }, + }, + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "application": { + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "custom.application.label", + }, + }, + }, + }, + }, + }, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + Openshift: &lokiv1.OpenshiftTenantSpec{ + OTLP: &lokiv1.OpenshiftOTLPConfig{ + DisableRecommendedAttributes: true, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{ + "custom.stream.label", + "k8s.namespace.name", + "kubernetes.namespace_name", + "log_source", + "log_type", + "openshift.cluster.uid", + "openshift.log.source", + "openshift.log.type", + }, + }, + }, + LogAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"custom.log.metadata"}, + }, + }, + }, + Tenants: map[string]*config.OTLPTenantAttributeConfig{ + "application": { + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{ + "custom.application.label", + "custom.stream.label", + "k8s.namespace.name", + "kubernetes.namespace_name", + "log_source", + "log_type", + "openshift.cluster.uid", + "openshift.log.source", + "openshift.log.type", + }, + }, + }, + LogAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"custom.log.metadata"}, + }, + }, + }, + }, + }, + }, + { + desc: "openshift-logging defaults with de-duplication", + spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "custom.stream.label", + }, + { + Name: "k8s.namespace.name", + }, + }, + }, + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + LogAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "custom.log.metadata", + }, + }, + }, + }, + }, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + Openshift: &lokiv1.OpenshiftTenantSpec{ + OTLP: &lokiv1.OpenshiftOTLPConfig{ + DisableRecommendedAttributes: true, + }, + }, + }, + }, + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{ + "custom.stream.label", + "k8s.namespace.name", + "kubernetes.namespace_name", + "log_source", + "log_type", + "openshift.cluster.uid", + "openshift.log.source", + "openshift.log.type", + }, + }, + }, + LogAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"custom.log.metadata"}, + }, + }, + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + cfg := otlpAttributeConfig(&tc.spec) + + assert.Equal(t, tc.wantConfig, cfg) + }) + } +} + +func TestSortOTLPAttributes(t *testing.T) { + tt := []struct { + desc string + attrs []config.OTLPAttribute + wantAttrs []config.OTLPAttribute + }{ + { + desc: "sort", + attrs: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"test.a"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "test.regex.c", + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.b"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "test.regex.a", + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Regex: "test.regex.b", + }, + }, + wantAttrs: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.b"}, + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Regex: "test.regex.b", + }, + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"test.a"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "test.regex.a", + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "test.regex.c", + }, + }, + }, + { + desc: "simple combine", + attrs: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.a"}, + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.b"}, + }, + }, + wantAttrs: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.a", "test.b"}, + }, + }, + }, + { + desc: "complex combine", + attrs: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.a"}, + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.c"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"test.d", "test.e"}, + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.b"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"test.f"}, + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Regex: "test.regex.b", + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "test.regex.a", + }, + }, + wantAttrs: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{"test.a", "test.b", "test.c"}, + }, + { + Action: config.OTLPAttributeActionStreamLabel, + Regex: "test.regex.b", + }, + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{"test.d", "test.e", "test.f"}, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: "test.regex.a", + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + attrs := sortAndDeduplicateOTLPAttributes(tc.attrs) + + assert.Equal(t, tc.wantAttrs, attrs) + }) + } +} diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 0ce06d606517..3d064c623965 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -6061,9 +6061,7 @@ compactor: working_directory: /tmp/loki/compactor distributor: otlp_config: - default_resource_attributes_as_index_labels: - - foo.bar - - bar.baz + default_resource_attributes_as_index_labels: [] frontend: tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 compress_responses: true @@ -6141,37 +6139,24 @@ limits_config: allow_structured_metadata: true otlp_config: resource_attributes: - ignore_defaults: true attributes_config: - action: index_label attributes: - res.foo.bar - res.bar.baz - regex: .* - action: structured_metadata attributes: - res.service.env - regex: .* scope_attributes: - - action: index_label + - action: structured_metadata attributes: - scope.foo.bar - scope.bar.baz - regex: .* - - action: structured_metadata - attributes: - - scope.service.env - regex: .* log_attributes: - - action: index_label + - action: structured_metadata attributes: - log.foo.bar - log.bar.baz - regex: .* - - action: structured_metadata - attributes: - - log.service.env - regex: .* memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -6303,37 +6288,24 @@ overrides: test-a: otlp_config: resource_attributes: - ignore_defaults: true attributes_config: - action: index_label attributes: - res.foo.bar - res.bar.baz - regex: .* - action: structured_metadata attributes: - res.service.env - regex: .* scope_attributes: - - action: index_label + - action: structured_metadata attributes: - scope.foo.bar - scope.bar.baz - regex: .* - - action: structured_metadata - attributes: - - scope.service.env - regex: .* log_attributes: - - action: index_label + - action: structured_metadata attributes: - log.foo.bar - log.bar.baz - regex: .* - - action: structured_metadata - attributes: - - log.service.env - regex: .* ` opts := Options{ Stack: lokiv1.LokiStackSpec{ @@ -6354,68 +6326,6 @@ overrides: PerStreamRateLimitBurst: 15, PerStreamDesiredRate: 3, }, - OTLP: &lokiv1.GlobalOTLPSpec{ - IndexedResourceAttributes: []string{ - "foo.bar", - "bar.baz", - }, - OTLPSpec: lokiv1.OTLPSpec{ - ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ - IgnoreDefaults: true, - Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ - { - Action: lokiv1.OTLPAttributeActionIndexLabel, - Attributes: []string{ - "res.foo.bar", - "res.bar.baz", - }, - Regex: ".*", - }, - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - Attributes: []string{ - "res.service.env", - }, - Regex: ".*", - }, - }, - }, - ScopeAttributes: []lokiv1.OTLPAttributesSpec{ - { - Action: lokiv1.OTLPAttributeActionIndexLabel, - Attributes: []string{ - "scope.foo.bar", - "scope.bar.baz", - }, - Regex: ".*", - }, - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - Attributes: []string{ - "scope.service.env", - }, - Regex: ".*", - }, - }, - LogAttributes: []lokiv1.OTLPAttributesSpec{ - { - Action: lokiv1.OTLPAttributeActionIndexLabel, - Attributes: []string{ - "log.foo.bar", - "log.bar.baz", - }, - Regex: ".*", - }, - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - Attributes: []string{ - "log.service.env", - }, - Regex: ".*", - }, - }, - }, - }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, @@ -6551,58 +6461,81 @@ overrides: "test-a": { Limits: lokiv1.PerTenantLimitsTemplateSpec{ OTLP: &lokiv1.OTLPSpec{ - ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ - IgnoreDefaults: true, - Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ - { - Action: lokiv1.OTLPAttributeActionIndexLabel, - Attributes: []string{ - "res.foo.bar", - "res.bar.baz", - }, - Regex: ".*", - }, - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - Attributes: []string{ - "res.service.env", - }, - Regex: ".*", - }, - }, + // This part of the spec is not actually used in this step. + // It has already been pre-processed into the OTLPAttributes below. + }, + }, + }, + }, + OTLPAttributes: OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &OTLPTenantAttributeConfig{ + ResourceAttributes: []OTLPAttribute{ + { + Action: OTLPAttributeActionStreamLabel, + Names: []string{ + "res.foo.bar", + "res.bar.baz", }, - ScopeAttributes: []lokiv1.OTLPAttributesSpec{ - { - Action: lokiv1.OTLPAttributeActionIndexLabel, - Attributes: []string{ - "scope.foo.bar", - "scope.bar.baz", - }, - Regex: ".*", + }, + { + Action: OTLPAttributeActionMetadata, + Names: []string{ + "res.service.env", + }, + }, + }, + ScopeAttributes: []OTLPAttribute{ + { + Action: OTLPAttributeActionMetadata, + Names: []string{ + "scope.foo.bar", + "scope.bar.baz", + }, + }, + }, + LogAttributes: []OTLPAttribute{ + { + Action: OTLPAttributeActionMetadata, + Names: []string{ + "log.foo.bar", + "log.bar.baz", + }, + }, + }, + }, + Tenants: map[string]*OTLPTenantAttributeConfig{ + "test-a": { + ResourceAttributes: []OTLPAttribute{ + { + Action: OTLPAttributeActionStreamLabel, + Names: []string{ + "res.foo.bar", + "res.bar.baz", }, - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - Attributes: []string{ - "scope.service.env", - }, - Regex: ".*", + }, + { + Action: OTLPAttributeActionMetadata, + Names: []string{ + "res.service.env", }, }, - LogAttributes: []lokiv1.OTLPAttributesSpec{ - { - Action: lokiv1.OTLPAttributeActionIndexLabel, - Attributes: []string{ - "log.foo.bar", - "log.bar.baz", - }, - Regex: ".*", + }, + ScopeAttributes: []OTLPAttribute{ + { + Action: OTLPAttributeActionMetadata, + Names: []string{ + "scope.foo.bar", + "scope.bar.baz", }, - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - Attributes: []string{ - "log.service.env", - }, - Regex: ".*", + }, + }, + LogAttributes: []OTLPAttribute{ + { + Action: OTLPAttributeActionMetadata, + Names: []string{ + "log.foo.bar", + "log.bar.baz", }, }, }, diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index e7204df71ce5..3c015b8fe725 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -107,16 +107,11 @@ compactor: {{- end }} delete_request_store: {{.ObjectStorage.SharedStore}} {{- end }} -{{- if $l := .Stack.Limits.Global }} -{{- with $l.OTLP }}{{- with .IndexedResourceAttributes }} +{{- if .OTLPAttributes.RemoveDefaultLabels }} distributor: otlp_config: - default_resource_attributes_as_index_labels: - {{- range . }} - - {{ . }} - {{- end }} + default_resource_attributes_as_index_labels: [] {{- end }} -{{- end }}{{- end }} frontend: tail_proxy_url: {{ .Querier.Protocol }}://{{ .Querier.FQDN }}:{{ .Querier.Port }} {{- if .Gates.HTTPEncryption }} @@ -234,19 +229,18 @@ limits_config: enabled: true desired_rate: {{ . }}MB {{- end }} -{{- with .Stack.Limits.Global.OTLP }} +{{- with .OTLPAttributes.Global }} otlp_config: - {{- with .ResourceAttributes }} + {{- if .ResourceAttributes }} resource_attributes: - ignore_defaults: {{ .IgnoreDefaults }} - {{- with .Attributes}} + {{- with .ResourceAttributes}} attributes_config: {{- range . }} - - action: {{ .Action.Value }} + - action: {{ .Action }} {{- with .Regex }} regex: {{ . }} {{- end }} - {{- with .Attributes }} + {{- with .Names }} attributes: {{- range . }} - {{ . }} @@ -258,11 +252,11 @@ limits_config: {{- with .ScopeAttributes }} scope_attributes: {{- range . }} - - action: {{ .Action.Value }} + - action: {{ .Action }} {{- with .Regex }} regex: {{ . }} {{- end }} - {{- with .Attributes }} + {{- with .Names }} attributes: {{- range . }} - {{ . }} @@ -273,11 +267,11 @@ limits_config: {{- with .LogAttributes }} log_attributes: {{- range . }} - - action: {{ .Action.Value }} + - action: {{ .Action }} {{- with .Regex }} regex: {{ . }} {{- end }} - {{- with .Attributes }} + {{- with .Names }} attributes: {{- range . }} - {{ . }} diff --git a/operator/internal/manifests/internal/config/loki-runtime-config.yaml b/operator/internal/manifests/internal/config/loki-runtime-config.yaml index 21b75508dc7a..0eb458660a67 100644 --- a/operator/internal/manifests/internal/config/loki-runtime-config.yaml +++ b/operator/internal/manifests/internal/config/loki-runtime-config.yaml @@ -1,3 +1,4 @@ +{{- /*gotype: github.com/grafana/loki/operator/internal/manifests/internal/config.Options*/ -}} --- overrides: {{- $opts := . }} @@ -73,19 +74,18 @@ overrides: {{- end }} {{- end}} {{- end -}} - {{- if $l := $spec.OTLP }} + {{- if $otlp := index $opts.OTLPAttributes.Tenants $tenant }} otlp_config: - {{- with $l.ResourceAttributes }} + {{- if $otlp.ResourceAttributes }} resource_attributes: - ignore_defaults: {{ .IgnoreDefaults }} - {{- with .Attributes}} + {{- with $otlp.ResourceAttributes}} attributes_config: {{- range . }} - - action: {{ .Action.Value }} + - action: {{ .Action }} {{- with .Regex }} regex: {{ . }} {{- end }} - {{- with .Attributes }} + {{- with .Names }} attributes: {{- range . }} - {{ . }} @@ -94,14 +94,14 @@ overrides: {{- end }} {{- end }} {{- end}} - {{- with $l.ScopeAttributes }} + {{- with $otlp.ScopeAttributes }} scope_attributes: {{- range . }} - - action: {{ .Action.Value }} + - action: {{ .Action }} {{- with .Regex }} regex: {{ . }} {{- end }} - {{- with .Attributes }} + {{- with .Names }} attributes: {{- range . }} - {{ . }} @@ -109,14 +109,14 @@ overrides: {{- end }} {{- end }} {{- end }} - {{- with $l.LogAttributes }} + {{- with $otlp.LogAttributes }} log_attributes: {{- range . }} - - action: {{ .Action.Value }} + - action: {{ .Action }} {{- with .Regex }} regex: {{ . }} {{- end }} - {{- with .Attributes }} + {{- with .Names }} attributes: {{- range . }} - {{ . }} diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index c22f0f898fb4..7ac3dcbac04f 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -37,6 +37,8 @@ type Options struct { Retention RetentionOptions + OTLPAttributes OTLPAttributeConfig + Overrides map[string]LokiOverrides } @@ -246,6 +248,34 @@ type RetentionOptions struct { DeleteWorkerCount uint } +// OTLPAttributeConfig contains the rendered OTLP label configuration. +// This is both influenced by the tenancy mode and the custom OTLP configuration on the LokiStack and might +// contain more labels than the user has configured if some labels are deemed "required". +type OTLPAttributeConfig struct { + RemoveDefaultLabels bool + Global *OTLPTenantAttributeConfig + Tenants map[string]*OTLPTenantAttributeConfig +} + +type OTLPTenantAttributeConfig struct { + ResourceAttributes []OTLPAttribute + ScopeAttributes []OTLPAttribute + LogAttributes []OTLPAttribute +} + +type OTLPAttributeAction string + +const ( + OTLPAttributeActionStreamLabel OTLPAttributeAction = "index_label" + OTLPAttributeActionMetadata OTLPAttributeAction = "structured_metadata" +) + +type OTLPAttribute struct { + Action OTLPAttributeAction + Names []string + Regex string +} + type TLSOptions struct { Ciphers []string MinTLSVersion string diff --git a/operator/internal/manifests/openshift/otlp.go b/operator/internal/manifests/openshift/otlp.go new file mode 100644 index 000000000000..911f0393f39e --- /dev/null +++ b/operator/internal/manifests/openshift/otlp.go @@ -0,0 +1,108 @@ +package openshift + +import ( + "slices" + + "github.com/grafana/loki/operator/internal/manifests/internal/config" +) + +// DefaultOTLPAttributes provides the required/recommended set of OTLP attributes for OpenShift Logging. +func DefaultOTLPAttributes(disableRecommended bool) config.OTLPAttributeConfig { + result := config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: []string{ + "k8s.namespace.name", + "kubernetes.namespace_name", + "log_source", + "log_type", + "openshift.cluster.uid", + "openshift.log.source", + "openshift.log.type", + }, + }, + }, + }, + } + + if disableRecommended { + return result + } + + // TODO decide whether we want to split the default configuration by tenant + result.Global.ResourceAttributes[0].Names = append(result.Global.ResourceAttributes[0].Names, + "k8s.container.name", + "k8s.cronjob.name", + "k8s.daemonset.name", + "k8s.deployment.name", + "k8s.job.name", + "k8s.node.name", + "k8s.pod.name", + "kubernetes.container_name", + "kubernetes.host", + "kubernetes.pod_name", + "service.name", + ) + slices.Sort(result.Global.ResourceAttributes[0].Names) + + result.Global.ResourceAttributes = append(result.Global.ResourceAttributes, + config.OTLPAttribute{ + Action: config.OTLPAttributeActionMetadata, + Names: []string{ + "k8s.node.uid", + "k8s.pod.uid", + "k8s.replicaset.name", + "k8s.statefulset.name", + "process.command_line", + "process.executable.name", + "process.executable.path", + "process.pid", + }, + }, + config.OTLPAttribute{ + Action: config.OTLPAttributeActionMetadata, + Regex: `k8s\.pod\.labels\..+`, + }, + config.OTLPAttribute{ + Action: config.OTLPAttributeActionMetadata, + Regex: `openshift\.labels\..+`, + }, + ) + + result.Global.LogAttributes = []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionMetadata, + Names: []string{ + "k8s.event.level", + "k8s.event.object_ref.api.group", + "k8s.event.object_ref.api.version", + "k8s.event.object_ref.name", + "k8s.event.object_ref.resource", + "k8s.event.request.uri", + "k8s.event.response.code", + "k8s.event.stage", + "k8s.event.user_agent", + "k8s.user.groups", + "k8s.user.username", + "log.iostream", + }, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: `k8s\.event\.annotations\..+`, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: `systemd\.t\..+`, + }, + { + Action: config.OTLPAttributeActionMetadata, + Regex: `systemd\.u\..+`, + }, + } + + return result +} diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index 12fe94a36960..c5eacd8d5729 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -79,12 +79,10 @@ func (v *LokiStackValidator) validate(ctx context.Context, obj runtime.Object) ( } if stack.Spec.Limits != nil { - if stack.Spec.Limits.Global != nil && stack.Spec.Limits.Global.OTLP != nil { - allErrs = append(allErrs, v.validateGlobalOTLPSpec(stack.Spec.Limits.Global.OTLP)...) - } - - if stack.Spec.Limits.Tenants != nil { - allErrs = append(allErrs, v.validatePerTenantOTLPSpec(stack.Spec.Limits.Tenants)...) + if (stack.Spec.Limits.Global != nil && stack.Spec.Limits.Global.OTLP != nil) || + len(stack.Spec.Limits.Tenants) > 0 { + // Only need to validate custom OTLP configuration + allErrs = append(allErrs, v.validateOTLPConfiguration(&stack.Spec)...) } } @@ -103,98 +101,86 @@ func (v *LokiStackValidator) validate(ctx context.Context, obj runtime.Object) ( ) } -func (v *LokiStackValidator) validateGlobalOTLPSpec(s *lokiv1.GlobalOTLPSpec) field.ErrorList { - basePath := field.NewPath("spec", "limits", "global") +func (v *LokiStackValidator) validateOTLPConfiguration(spec *lokiv1.LokiStackSpec) field.ErrorList { + if spec.Tenants == nil { + return nil + } - return v.validateOTLPSpec(basePath, &s.OTLPSpec) -} + if spec.Tenants.Mode == lokiv1.OpenshiftLogging { + // This tenancy mode always provides stream labels + return nil + } -func (v *LokiStackValidator) validatePerTenantOTLPSpec(tenants map[string]lokiv1.PerTenantLimitsTemplateSpec) field.ErrorList { - var allErrs field.ErrorList + if spec.Tenants.Mode == lokiv1.OpenshiftNetwork { + // No validation defined for openshift-network tenancy mode + // TODO can we define a validation for this mode? + return nil + } - for key, tenant := range tenants { - basePath := field.NewPath("spec", "limits", "tenants").Key(key) - allErrs = append(allErrs, v.validateOTLPSpec(basePath, tenant.OTLP)...) + if spec.Limits == nil { + return nil } - return allErrs -} + hasGlobalStreamLabels := false + if spec.Limits.Global != nil && spec.Limits.Global.OTLP != nil { + hasGlobalStreamLabels = v.hasOTLPStreamLabel(spec.Limits.Global.OTLP) + } -func (v *LokiStackValidator) validateOTLPSpec(parent *field.Path, s *lokiv1.OTLPSpec) field.ErrorList { - var allErrs field.ErrorList + if hasGlobalStreamLabels { + // When the global configuration has at least one stream label, then the configuration will be valid + return nil + } - if s.ResourceAttributes != nil && s.ResourceAttributes.IgnoreDefaults { - switch { - case len(s.ResourceAttributes.Attributes) == 0: - allErrs = append(allErrs, - field.Invalid( - parent.Child("otlp", "resourceAttributes"), - []lokiv1.OTLPAttributesSpec{}, - lokiv1.ErrOTLPResourceAttributesEmptyNotAllowed.Error(), - ), - ) - default: - var indexLabelActionFound bool - for _, attr := range s.ResourceAttributes.Attributes { - if attr.Action == lokiv1.OTLPAttributeActionIndexLabel { - indexLabelActionFound = true - break - } - } - - if !indexLabelActionFound { - allErrs = append(allErrs, - field.Invalid( - parent.Child("otlp", "resourceAttributes"), - s.ResourceAttributes.Attributes, - lokiv1.ErrOTLPResourceAttributesIndexLabelActionMissing.Error(), - ), - ) - } - - for idx, attr := range s.ResourceAttributes.Attributes { - if len(attr.Attributes) == 0 && attr.Regex == "" { - allErrs = append(allErrs, - field.Invalid( - parent.Child("otlp", "resourceAttributes").Index(idx), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), - ), - ) - } - } + if spec.Limits.Tenants == nil { + // No tenant config and no global stream labels -> error + return field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "streamLabels", "resourceAttributes"), + nil, + lokiv1.ErrOTLPGlobalNoStreamLabel.Error(), + ), } } - if len(s.ScopeAttributes) != 0 { - for idx, attr := range s.ScopeAttributes { - if len(attr.Attributes) == 0 && attr.Regex == "" { - allErrs = append(allErrs, - field.Invalid( - parent.Child("otlp", "scopeAttributes").Index(idx), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), - ), - ) - } + errList := field.ErrorList{} + for _, tenant := range spec.Tenants.Authentication { + tenantName := tenant.TenantName + tenantLimits, ok := spec.Limits.Tenants[tenantName] + if !ok || tenantLimits.OTLP == nil { + // No tenant limits defined and no global stream labels -> error + errList = append(errList, field.Invalid( + field.NewPath("spec", "limits", "tenants", tenantName, "otlp"), + nil, + lokiv1.ErrOTLPTenantMissing.Error(), + )) + + continue } - } - if len(s.LogAttributes) != 0 { - for idx, attr := range s.LogAttributes { - if len(attr.Attributes) == 0 && attr.Regex == "" { - allErrs = append(allErrs, - field.Invalid( - parent.Child("otlp", "logAttributes").Index(idx), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), - ), - ) - } + if v.hasOTLPStreamLabel(tenantLimits.OTLP) { + continue } + + errList = append(errList, field.Invalid( + field.NewPath("spec", "limits", "tenants", tenantName, "otlp", "streamLabels", "resourceAttributes"), + nil, + lokiv1.ErrOTLPTenantNoStreamLabel.Error(), + )) } - return allErrs + return errList +} + +func (v *LokiStackValidator) hasOTLPStreamLabel(otlp *lokiv1.OTLPSpec) bool { + if otlp == nil { + return false + } + + if otlp.StreamLabels == nil { + return false + } + + return len(otlp.StreamLabels.ResourceAttributes) > 0 } func (v *LokiStackValidator) validateHashRingSpec(s lokiv1.LokiStackSpec) field.ErrorList { diff --git a/operator/internal/validation/lokistack_test.go b/operator/internal/validation/lokistack_test.go index f0e9e6d0274a..18434c5fa20b 100644 --- a/operator/internal/validation/lokistack_test.go +++ b/operator/internal/validation/lokistack_test.go @@ -391,64 +391,60 @@ var ltt = []struct { ), }, { - desc: "enabling global limits OTLP IgnoreDefaults without resource attributes", + desc: "lokistack with custom OTLP configuration with a global stream label", spec: lokiv1.LokiStack{ Spec: lokiv1.LokiStackSpec{ - Storage: lokiv1.ObjectStorageSpec{ - Schemas: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2020-10-11", - }, - }, - }, Limits: &lokiv1.LimitsSpec{ Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.GlobalOTLPSpec{ - OTLPSpec: lokiv1.OTLPSpec{ - ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ - IgnoreDefaults: true, + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "global.stream.label", + }, }, }, }, }, }, - }, - }, - err: apierrors.NewInvalid( - schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, - "testing-stack", - field.ErrorList{ - field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes"), - []lokiv1.OTLPAttributesSpec{}, - lokiv1.ErrOTLPResourceAttributesEmptyNotAllowed.Error(), - ), - }, - ), - }, - { - desc: "enabling global limits OTLP IgnoreDefaults without index label action for resource attributes", - spec: lokiv1.LokiStack{ - Spec: lokiv1.LokiStackSpec{ Storage: lokiv1.ObjectStorageSpec{ Schemas: []lokiv1.ObjectStorageSchema{ { Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2020-10-11", + EffectiveDate: "2024-10-22", }, }, }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + }, + }, + }, + err: nil, + }, + { + desc: "lokistack with custom OTLP configuration with a global stream label and a tenant with no stream label", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ Limits: &lokiv1.LimitsSpec{ Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.GlobalOTLPSpec{ - OTLPSpec: lokiv1.OTLPSpec{ - ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ - IgnoreDefaults: true, - Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "global.stream.label", + }, + }, + }, + }, + }, + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - Attributes: []string{"test"}, + Name: "custom.resource.attribute", }, }, }, @@ -456,46 +452,38 @@ var ltt = []struct { }, }, }, - }, - }, - err: apierrors.NewInvalid( - schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, - "testing-stack", - field.ErrorList{ - field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes"), - []lokiv1.OTLPResourceAttributesConfigSpec{ + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - Attributes: []string{"test"}, + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2024-10-22", }, }, - lokiv1.ErrOTLPResourceAttributesIndexLabelActionMissing.Error(), - ), + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + Authentication: []lokiv1.AuthenticationSpec{ + { + TenantName: "test-tenant", + }, + }, + }, }, - ), + }, + err: nil, }, { - desc: "enabling global limits OTLP IgnoreDefaults with invalid resource attributes config", + desc: "lokistack with custom OTLP configuration with no global stream label and a tenant with a stream label", spec: lokiv1.LokiStack{ Spec: lokiv1.LokiStackSpec{ - Storage: lokiv1.ObjectStorageSpec{ - Schemas: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2020-10-11", - }, - }, - }, Limits: &lokiv1.LimitsSpec{ - Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.GlobalOTLPSpec{ - OTLPSpec: lokiv1.OTLPSpec{ - ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ - IgnoreDefaults: true, - Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Name: "tenant.stream.label", }, }, }, @@ -503,99 +491,54 @@ var ltt = []struct { }, }, }, - }, - }, - err: apierrors.NewInvalid( - schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, - "testing-stack", - field.ErrorList{ - field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes"), - []lokiv1.OTLPResourceAttributesConfigSpec{ - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - }, - }, - lokiv1.ErrOTLPResourceAttributesIndexLabelActionMissing.Error(), - ), - field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes").Index(0), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), - ), - }, - ), - }, - { - desc: "enabling global limits OTLP with invalid resource attributes config", - spec: lokiv1.LokiStack{ - Spec: lokiv1.LokiStackSpec{ Storage: lokiv1.ObjectStorageSpec{ Schemas: []lokiv1.ObjectStorageSchema{ { Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2020-10-11", + EffectiveDate: "2024-10-22", }, }, }, - Limits: &lokiv1.LimitsSpec{ - Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.GlobalOTLPSpec{ - OTLPSpec: lokiv1.OTLPSpec{ - ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ - IgnoreDefaults: true, - Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ - { - Action: lokiv1.OTLPAttributeActionIndexLabel, - }, - }, - }, - }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + Authentication: []lokiv1.AuthenticationSpec{ + { + TenantName: "test-tenant", }, }, }, }, }, - err: apierrors.NewInvalid( - schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, - "testing-stack", - field.ErrorList{ - field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes").Index(0), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), - ), - }, - ), + err: nil, }, { - desc: "invalid global OTLP scope attribute specs", + desc: "lokistack with custom OTLP configuration missing a global stream label", spec: lokiv1.LokiStack{ Spec: lokiv1.LokiStackSpec{ - Storage: lokiv1.ObjectStorageSpec{ - Schemas: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2020-10-11", - }, - }, - }, Limits: &lokiv1.LimitsSpec{ Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.GlobalOTLPSpec{ - OTLPSpec: lokiv1.OTLPSpec{ - ScopeAttributes: []lokiv1.OTLPAttributesSpec{ + OTLP: &lokiv1.OTLPSpec{ + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ { - Action: lokiv1.OTLPAttributeActionIndexLabel, - }, - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Name: "custom.resource.attribute", }, }, }, }, }, }, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2024-10-22", + }, + }, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + }, }, }, err: apierrors.NewInvalid( @@ -603,43 +546,48 @@ var ltt = []struct { "testing-stack", field.ErrorList{ field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "scopeAttributes").Index(0), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), - ), - field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "scopeAttributes").Index(1), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + field.NewPath("spec", "limits", "global", "otlp", "streamLabels", "resourceAttributes"), + nil, + lokiv1.ErrOTLPGlobalNoStreamLabel.Error(), ), }, ), }, { - desc: "invalid global OTLP log attribute specs", + desc: "lokistack with custom OTLP configuration missing a tenant", spec: lokiv1.LokiStack{ Spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.stream.label", + }, + }, + }, + }, + }, + }, + }, Storage: lokiv1.ObjectStorageSpec{ Schemas: []lokiv1.ObjectStorageSchema{ { Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2020-10-11", + EffectiveDate: "2024-10-22", }, }, }, - Limits: &lokiv1.LimitsSpec{ - Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.GlobalOTLPSpec{ - OTLPSpec: lokiv1.OTLPSpec{ - LogAttributes: []lokiv1.OTLPAttributesSpec{ - { - Action: lokiv1.OTLPAttributeActionIndexLabel, - }, - { - Action: lokiv1.OTLPAttributeActionStructuredMetadata, - }, - }, - }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + Authentication: []lokiv1.AuthenticationSpec{ + { + TenantName: "test-tenant", + }, + { + TenantName: "second-tenant", }, }, }, @@ -650,38 +598,45 @@ var ltt = []struct { "testing-stack", field.ErrorList{ field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "logAttributes").Index(0), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), - ), - field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "logAttributes").Index(1), - []string{}, - lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + field.NewPath("spec", "limits", "tenants", "second-tenant", "otlp"), + nil, + lokiv1.ErrOTLPTenantMissing.Error(), ), }, ), }, { - desc: "enabling per-tenant limits OTLP IgnoreDefaults without resource attributes", + desc: "lokistack with custom OTLP configuration with a tenant without stream label", spec: lokiv1.LokiStack{ Spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "tenant.resource.attribute", + }, + }, + }, + }, + }, + }, + }, Storage: lokiv1.ObjectStorageSpec{ Schemas: []lokiv1.ObjectStorageSchema{ { Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2020-10-11", + EffectiveDate: "2024-10-22", }, }, }, - Limits: &lokiv1.LimitsSpec{ - Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ - "tenant-a": { - OTLP: &lokiv1.OTLPSpec{ - ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ - IgnoreDefaults: true, - }, - }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + Authentication: []lokiv1.AuthenticationSpec{ + { + TenantName: "test-tenant", }, }, }, @@ -692,9 +647,9 @@ var ltt = []struct { "testing-stack", field.ErrorList{ field.Invalid( - field.NewPath("spec", "limits", "tenants").Key("tenant-a").Child("otlp", "resourceAttributes"), - []lokiv1.OTLPAttributesSpec{}, - lokiv1.ErrOTLPResourceAttributesEmptyNotAllowed.Error(), + field.NewPath("spec", "limits", "tenants", "test-tenant", "otlp", "streamLabels", "resourceAttributes"), + nil, + lokiv1.ErrOTLPTenantNoStreamLabel.Error(), ), }, ),
(Optional) -

OTLP to configure which resource, scope and log attributes -to store as labels or structured metadata or drop them altogether -for a single tenants.

+

OTLP to configure which resource, scope and log attributes are stored as stream labels or structured metadata.

+

Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even +enforce the use of some required attributes.

+

The per-tenant configuration for OTLP attributes will be merged with the global configuration.