Skip to content

Commit

Permalink
chore: cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@srikanthccv
srikanthccv committed Dec 14, 2024
1 parent 4428024 commit c2d6c79
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 44 deletions.
7 changes: 2 additions & 5 deletions pkg/query-service/app/clickhouseReader/filter_suggestions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,14 +139,14 @@ func (r *ClickHouseReader) getValuesForLogAttributes(
select * from (
(
select tag_key, string_value, number_value
from signoz_logs.distributed_tag_attributes
from signoz_logs.distributed_tag_attributes_v2
where tag_key = $1 and (
string_value != '' or number_value is not null
)
limit 2
) UNION DISTINCT (
select tag_key, string_value, number_value
from signoz_logs.distributed_tag_attributes
from signoz_logs.distributed_tag_attributes_v2
where tag_key = $2 and (
string_value != '' or number_value is not null
)
Expand All @@ -156,9 +156,6 @@ func (r *ClickHouseReader) getValuesForLogAttributes(
```
Since tag_attributes table uses ReplacingMergeTree, the values would be distinct and no order by
is being used to ensure the `limit` clause minimizes the amount of data scanned.
This query scanned ~30k rows per attribute on fiscalnote-v2 for attributes like `message` and `time`
that had >~110M values each
*/

if len(attributes) > 10 {
Expand Down
6 changes: 0 additions & 6 deletions pkg/query-service/app/clickhouseReader/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,13 @@ const (
defaultSpansTable string = "distributed_signoz_spans"
defaultDependencyGraphTable string = "distributed_dependency_graph_minutes_v2"
defaultTopLevelOperationsTable string = "distributed_top_level_operations"
defaultSpanAttributeTable string = "distributed_span_attributes"
defaultSpanAttributeTableV2 string = "distributed_tag_attributes_v2"
defaultSpanAttributeKeysTable string = "distributed_span_attributes_keys"
defaultLogsDB string = "signoz_logs"
defaultLogsTable string = "distributed_logs"
defaultLogsLocalTable string = "logs"
defaultLogAttributeKeysTable string = "distributed_logs_attribute_keys"
defaultLogResourceKeysTable string = "distributed_logs_resource_keys"
defaultLogTagAttributeTable string = "distributed_tag_attributes"
defaultLogTagAttributeTableV2 string = "distributed_tag_attributes_v2"
defaultLiveTailRefreshSeconds int = 5
defaultWriteBatchDelay time.Duration = 5 * time.Second
Expand Down Expand Up @@ -71,7 +69,6 @@ type namespaceConfig struct {
UsageExplorerTable string
SpansTable string
ErrorTable string
SpanAttributeTable string
SpanAttributeTableV2 string
SpanAttributeKeysTable string
DependencyGraphTable string
Expand All @@ -81,7 +78,6 @@ type namespaceConfig struct {
LogsLocalTable string
LogsAttributeKeysTable string
LogsResourceKeysTable string
LogsTagAttributeTable string
LogsTagAttributeTableV2 string
LiveTailRefreshSeconds int
WriteBatchDelay time.Duration
Expand Down Expand Up @@ -171,7 +167,6 @@ func NewOptions(
DurationTable: defaultDurationTable,
UsageExplorerTable: defaultUsageExplorerTable,
SpansTable: defaultSpansTable,
SpanAttributeTable: defaultSpanAttributeTable,
SpanAttributeTableV2: defaultSpanAttributeTableV2,
SpanAttributeKeysTable: defaultSpanAttributeKeysTable,
DependencyGraphTable: defaultDependencyGraphTable,
Expand All @@ -181,7 +176,6 @@ func NewOptions(
LogsLocalTable: defaultLogsLocalTable,
LogsAttributeKeysTable: defaultLogAttributeKeysTable,
LogsResourceKeysTable: defaultLogResourceKeysTable,
LogsTagAttributeTable: defaultLogTagAttributeTable,
LogsTagAttributeTableV2: defaultLogTagAttributeTableV2,
LiveTailRefreshSeconds: defaultLiveTailRefreshSeconds,
WriteBatchDelay: defaultWriteBatchDelay,
Expand Down
39 changes: 6 additions & 33 deletions pkg/query-service/app/clickhouseReader/reader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,6 @@ type ClickHouseReader struct {
errorTable string
usageExplorerTable string
SpansTable string
spanAttributeTable string
spanAttributeTableV2 string
spanAttributesKeysTable string
dependencyGraphTable string
Expand All @@ -128,7 +127,6 @@ type ClickHouseReader struct {
logsLocalTable string
logsAttributeKeys string
logsResourceKeys string
logsTagAttributeTable string
logsTagAttributeTableV2 string
queryEngine *promql.Engine
remoteStorage *remote.Storage
Expand Down Expand Up @@ -248,7 +246,6 @@ func NewReaderFromClickhouseConnection(
usageExplorerTable: options.primary.UsageExplorerTable,
durationTable: options.primary.DurationTable,
SpansTable: options.primary.SpansTable,
spanAttributeTable: options.primary.SpanAttributeTable,
spanAttributeTableV2: options.primary.SpanAttributeTableV2,
spanAttributesKeysTable: options.primary.SpanAttributeKeysTable,
dependencyGraphTable: options.primary.DependencyGraphTable,
Expand All @@ -258,7 +255,6 @@ func NewReaderFromClickhouseConnection(
logsLocalTable: options.primary.LogsLocalTable,
logsAttributeKeys: options.primary.LogsAttributeKeysTable,
logsResourceKeys: options.primary.LogsResourceKeysTable,
logsTagAttributeTable: options.primary.LogsTagAttributeTable,
logsTagAttributeTableV2: options.primary.LogsTagAttributeTableV2,
liveTailRefreshSeconds: options.primary.LiveTailRefreshSeconds,
promConfigFile: configFile,
Expand Down Expand Up @@ -1039,29 +1035,6 @@ func addExistsOperator(item model.TagQuery, tagMapType string, not bool) (string
return fmt.Sprintf(" AND %s (%s)", notStr, strings.Join(tagOperatorPair, " OR ")), args
}

func excludeTags(_ context.Context, tags []string) []string {
excludedTagsMap := map[string]bool{
"http.code": true,
"http.route": true,
"http.method": true,
"http.url": true,
"http.status_code": true,
"http.host": true,
"messaging.system": true,
"messaging.operation": true,
"error": true,
"service.name": true,
}
newTags := make([]string, 0)
for _, tag := range tags {
_, ok := excludedTagsMap[tag]
if !ok {
newTags = append(newTags, tag)
}
}
return newTags
}

func (r *ClickHouseReader) GetTopOperationsV2(ctx context.Context, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError) {

namedArgs := []interface{}{
Expand Down Expand Up @@ -3698,10 +3671,10 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi
if req.FilterAttributeKeyDataType != v3.AttributeKeyDataTypeString {
filterValueColumnWhere = fmt.Sprintf("toString(%s)", filterValueColumn)
}
query = fmt.Sprintf("select distinct %s from %s.%s where tag_key=$1 and %s ILIKE $2 and tag_type=$3 limit $4", filterValueColumn, r.logsDB, r.logsTagAttributeTableV2, filterValueColumnWhere)
query = fmt.Sprintf("SELECT DISTINCT %s FROM %s.%s WHERE tag_key=$1 AND %s ILIKE $2 AND tag_type=$3 LIMIT $4", filterValueColumn, r.logsDB, r.logsTagAttributeTableV2, filterValueColumnWhere)
rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, searchText, req.TagType, req.Limit)
} else {
query = fmt.Sprintf("select distinct %s from %s.%s where tag_key=$1 and tag_type=$2 limit $3", filterValueColumn, r.logsDB, r.logsTagAttributeTableV2)
query = fmt.Sprintf("SELECT DISTINCT %s FROM %s.%s WHERE tag_key=$1 AND tag_type=$2 LIMIT $3", filterValueColumn, r.logsDB, r.logsTagAttributeTableV2)
rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, req.TagType, req.Limit)
}

Expand Down Expand Up @@ -4339,10 +4312,10 @@ func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3.
}, nil
}

query = "select distinct"
query = "SELECT DISTINCT"
switch req.FilterAttributeKeyDataType {
case v3.AttributeKeyDataTypeFloat64:
filterValueColumn = "float64_value"
filterValueColumn = "number_value"
case v3.AttributeKeyDataTypeString:
filterValueColumn = "string_value"
}
Expand All @@ -4365,14 +4338,14 @@ func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3.
if r.useTraceNewSchema {
where += " AND ts_bucket_start >= toUInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR))"
}
query = fmt.Sprintf("select distinct %s from %s.%s where %s and %s ILIKE $1 limit $2", selectKey, r.TraceDB, r.traceTableName, where, filterValueColumnWhere)
query = fmt.Sprintf("SELECT DISTINCT %s FROM %s.%s WHERE %s AND %s ILIKE $1 LIMIT $2", selectKey, r.TraceDB, r.traceTableName, where, filterValueColumnWhere)
rows, err = r.db.Query(ctx, query, searchText, req.Limit)
} else {
filterValueColumnWhere := filterValueColumn
if req.FilterAttributeKeyDataType != v3.AttributeKeyDataTypeString {
filterValueColumnWhere = fmt.Sprintf("toString(%s)", filterValueColumn)
}
query = fmt.Sprintf("select distinct %s from %s.%s where tag_key=$1 and %s ILIKE $2 and tag_type=$3 limit $4", filterValueColumn, r.TraceDB, r.spanAttributeTableV2, filterValueColumnWhere)
query = fmt.Sprintf("SELECT DISTINCT %s FROM %s.%s WHERE tag_key=$1 AND %s ILIKE $2 AND tag_type=$3 LIMIT $4", filterValueColumn, r.TraceDB, r.spanAttributeTableV2, filterValueColumnWhere)
rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, searchText, req.TagType, req.Limit)
}

Expand Down

0 comments on commit c2d6c79

Please sign in to comment.