forked from cockroachdb/cockroach
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This commit introduces the logic which will be used in a later commit for logging SQL schema telemetry. In the meantime, this commit applies it in a test which checks that the schema of a bootstrapped cluster matches expectations. Informs cockroachdb#84284. Release note: None
- Loading branch information
Marius Posta
committed
Jul 21, 2022
1 parent
a3f3295
commit ce0fbbe
Showing
15 changed files
with
1,644 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| load("//build/bazelutil/unused_checker:unused.bzl", "get_x_data") | ||
| load("@io_bazel_rules_go//go:def.bzl", "go_library") | ||
|
|
||
| go_library( | ||
| name = "schematelemetry", | ||
| srcs = ["schema_telemetry_event.go"], | ||
| importpath = "github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry", | ||
| visibility = ["//visibility:public"], | ||
| deps = [ | ||
| "//pkg/kv", | ||
| "//pkg/sql", | ||
| "//pkg/sql/catalog", | ||
| "//pkg/sql/catalog/colinfo", | ||
| "//pkg/sql/catalog/descpb", | ||
| "//pkg/sql/catalog/descs", | ||
| "//pkg/sql/descmetadata", | ||
| "//pkg/sql/schemachanger/scdecomp", | ||
| "//pkg/sql/schemachanger/scpb", | ||
| "//pkg/sql/schemachanger/screl", | ||
| "//pkg/util/hlc", | ||
| "//pkg/util/log/eventpb", | ||
| "@com_github_gogo_protobuf//types", | ||
| ], | ||
| ) | ||
|
|
||
| get_x_data(name = "get_x_data") |
137 changes: 137 additions & 0 deletions
137
pkg/sql/catalog/schematelemetry/schema_telemetry_event.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| // Copyright 2022 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package schematelemetry | ||
|
|
||
| import ( | ||
| "context" | ||
| "time" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/kv" | ||
| "github.com/cockroachdb/cockroach/pkg/sql" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/catalog" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/descmetadata" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scdecomp" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scpb" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/screl" | ||
| "github.com/cockroachdb/cockroach/pkg/util/hlc" | ||
| "github.com/cockroachdb/cockroach/pkg/util/log/eventpb" | ||
| gogotypes "github.com/gogo/protobuf/types" | ||
| ) | ||
|
|
||
| // MaxElementsPerPayload is the upper bound on how many elements can feature in | ||
| // the payload of one eventpb.Schema object. | ||
| const MaxElementsPerPayload = 50 | ||
|
|
||
| func buildLogEvents( | ||
| ctx context.Context, cfg *sql.ExecutorConfig, aostOffset time.Duration, | ||
| ) ([]eventpb.EventPayload, error) { | ||
| asOf := cfg.Clock.Now().Add(aostOffset.Nanoseconds(), 0) | ||
| ess, err := CollectClusterSchemaForTelemetry(ctx, cfg, asOf) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // Bundle the element-status pairs into log events. | ||
| events := make([]eventpb.EventPayload, 0, len(ess)/MaxElementsPerPayload+1) | ||
| var pl scpb.TelemetryPayload | ||
| flush := func() error { | ||
| if len(pl.ElementStatuses) == 0 { | ||
| return nil | ||
| } | ||
| event := &eventpb.Schema{} | ||
| event.Timestamp = asOf.WallTime | ||
| event.CurrentPage = uint32(len(events) + 1) | ||
| any, err := gogotypes.MarshalAny(&pl) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| event.Payload = any | ||
| events = append(events, event) | ||
| pl.ElementStatuses = pl.ElementStatuses[:0] | ||
| return nil | ||
| } | ||
|
|
||
| for _, es := range ess { | ||
| pl.ElementStatuses = append(pl.ElementStatuses, es) | ||
| if len(pl.ElementStatuses) >= MaxElementsPerPayload { | ||
| if err := flush(); err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
| } | ||
| if err := flush(); err != nil { | ||
| return nil, err | ||
| } | ||
| for i := range events { | ||
| events[i].(*eventpb.Schema).NumPages = uint32(len(events)) | ||
| } | ||
| return events, nil | ||
| } | ||
|
|
||
| // Keep the linter happy. | ||
| // TODO(postamar): use this function | ||
| var _ = buildLogEvents | ||
|
|
||
| // CollectClusterSchemaForTelemetry returns a projection of the cluster's SQL | ||
| // schema as of the provided system time, suitably filtered for the purposes of | ||
| // schema telemetry. | ||
| // | ||
| // This function is tested in the systemschema package. | ||
| func CollectClusterSchemaForTelemetry( | ||
| ctx context.Context, cfg *sql.ExecutorConfig, asOf hlc.Timestamp, | ||
| ) (ess []scpb.TelemetryPayload_ElementStatus, _ error) { | ||
| visitorFn := func(status scpb.Status, element scpb.Element) { | ||
| if es := filteredForTelemetry(status, element); es != nil { | ||
| ess = append(ess, *es) | ||
| } | ||
| } | ||
| if err := sql.DescsTxn(ctx, cfg, func(ctx context.Context, txn *kv.Txn, col *descs.Collection) error { | ||
| err := txn.SetFixedTimestamp(ctx, asOf) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| c, err := col.GetAllDescriptors(ctx, txn) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| cc := descmetadata.NewCommentCache(txn, cfg.InternalExecutor) | ||
| _ = c.ForEachDescriptorEntry(func(desc catalog.Descriptor) error { | ||
| _ = scdecomp.WalkDescriptor(ctx, desc, c.LookupDescriptorEntry, visitorFn, cc) | ||
| return nil | ||
| }) | ||
| return nil | ||
| }); err != nil { | ||
| return nil, err | ||
| } | ||
| return ess, nil | ||
| } | ||
|
|
||
| // filteredForTelemetry removes unwanted information from the element | ||
| // decomposition for the purposes of schema telemetry. | ||
| func filteredForTelemetry( | ||
| status scpb.Status, element scpb.Element, | ||
| ) *scpb.TelemetryPayload_ElementStatus { | ||
| // Exclude elements related to system columns. | ||
| a, _ := screl.Schema.GetAttribute(screl.ColumnID, element) | ||
| if colID, ok := a.(descpb.ColumnID); ok && colinfo.IsColIDSystemColumn(colID) { | ||
| return nil | ||
| } | ||
| // Return the visited element-status pair as a telemetry payload element. | ||
| es := &scpb.TelemetryPayload_ElementStatus{} | ||
| es.SetValue(element) | ||
| // Only set non-public statuses, public status should be an implicit default. | ||
| if status != scpb.Status_PUBLIC { | ||
| es.Status = status | ||
| } | ||
| return es | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.