Skip to content

Commit 9239bd4

Browse files
authored
[Go] logging for GCP (#146)
Add logging to the google-cloud plugin. Use the GCP logging client for more control over logging that is provided with JSON output, like the log name. Write a slog Handler that constructs a logging.Entry. Init installs a default logger with the handler.
1 parent 05a6069 commit 9239bd4

File tree

6 files changed

+245
-6
lines changed

6 files changed

+245
-6
lines changed

go/go.mod

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ module github.com/google/genkit/go
33
go 1.22.0
44

55
require (
6+
cloud.google.com/go/aiplatform v1.60.0
7+
cloud.google.com/go/logging v1.9.0
68
cloud.google.com/go/vertexai v0.7.1
79
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.46.0
810
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.22.0
@@ -11,6 +13,7 @@ require (
1113
github.com/google/go-cmp v0.6.0
1214
github.com/google/uuid v1.6.0
1315
github.com/invopop/jsonschema v0.12.0
16+
github.com/jba/slog v0.2.0
1417
github.com/wk8/go-ordered-map/v2 v2.1.8
1518
go.opentelemetry.io/otel v1.26.0
1619
go.opentelemetry.io/otel/metric v1.26.0
@@ -19,13 +22,13 @@ require (
1922
go.opentelemetry.io/otel/trace v1.26.0
2023
golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81
2124
google.golang.org/api v0.177.0
25+
google.golang.org/protobuf v1.34.0
2226
gopkg.in/yaml.v3 v3.0.1
2327
)
2428

2529
require (
2630
cloud.google.com/go v0.112.2 // indirect
2731
cloud.google.com/go/ai v0.3.0 // indirect
28-
cloud.google.com/go/aiplatform v1.60.0 // indirect
2932
cloud.google.com/go/auth v0.3.0 // indirect
3033
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
3134
cloud.google.com/go/compute/metadata v0.3.0 // indirect
@@ -60,6 +63,5 @@ require (
6063
google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
6164
google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect
6265
google.golang.org/grpc v1.63.2 // indirect
63-
google.golang.org/protobuf v1.34.0 // indirect
6466
gopkg.in/yaml.v2 v2.4.0 // indirect
6567
)

go/go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/
9393
github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
9494
github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
9595
github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
96+
github.com/jba/slog v0.1.0 h1:m7pbPxGRvFcQy4vONykm/9X+0Fx4FGEDl7A6E/C/z9Q=
97+
github.com/jba/slog v0.1.0/go.mod h1:R9u+1Qbl7LcDnJaFNIPer+AJa3yK9eZ8SQUE4waKFiw=
98+
github.com/jba/slog v0.2.0 h1:jI0U5NRR3EJKGsbeEVpItJNogk0c4RMeCl7vJmogCJI=
99+
github.com/jba/slog v0.2.0/go.mod h1:0Dh7Vyz3Td68Z1OwzadfincHwr7v+PpzadrS2Jua338=
96100
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
97101
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
98102
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=

go/plugins/googlecloud/googlecloud.go

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@ package googlecloud
2020

2121
import (
2222
"context"
23+
"log/slog"
2324
"os"
2425
"time"
2526

27+
"cloud.google.com/go/logging"
2628
mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
2729
texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
2830
"github.com/google/genkit/go/genkit"
@@ -40,6 +42,10 @@ type Options struct {
4042
// The interval for exporting metric data.
4143
// The default is 60 seconds.
4244
MetricInterval time.Duration
45+
46+
// The minimum level at which logs will be written.
47+
// Defaults to [slog.LevelInfo].
48+
LogLevel slog.Leveler
4349
}
4450

4551
// Init initializes all telemetry in this package.
@@ -59,8 +65,10 @@ func Init(ctx context.Context, projectID string, opts *Options) error {
5965
}
6066
aexp := &adjustingTraceExporter{texp}
6167
genkit.RegisterSpanProcessor(sdktrace.NewBatchSpanProcessor(aexp))
62-
63-
return setMeterProvider(projectID, opts.MetricInterval)
68+
if err := setMeterProvider(projectID, opts.MetricInterval); err != nil {
69+
return err
70+
}
71+
return setLogHandler(projectID, opts.LogLevel)
6472
}
6573

6674
func setMeterProvider(projectID string, interval time.Duration) error {
@@ -109,3 +117,13 @@ func (s adjustedSpan) Attributes() []attribute.KeyValue {
109117
}
110118
return ts
111119
}
120+
121+
func setLogHandler(projectID string, level slog.Leveler) error {
122+
c, err := logging.NewClient(context.Background(), "projects/"+projectID)
123+
if err != nil {
124+
return err
125+
}
126+
logger := c.Logger("genkit_log")
127+
slog.SetDefault(slog.New(newHandler(level, logger.Log)))
128+
return nil
129+
}

go/plugins/googlecloud/googlecloud_test.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ package googlecloud
1919
import (
2020
"context"
2121
"flag"
22+
"log/slog"
23+
"os"
24+
"runtime"
2225
"testing"
2326
"time"
2427

@@ -30,8 +33,11 @@ import (
3033
var projectID = flag.String("project", "", "GCP project ID")
3134

3235
// This test is part of verifying that we can export traces to GCP.
33-
// To verify, run the test, then visit the GCP Trace Explorer and look for the "test"
34-
// trace, and visit the Metrics Explorer and look for the "Generic Node - test" metric.
36+
// To verify, run the test, then:
37+
// - visit the GCP Trace Explorer and look for the "test" trace
38+
// - visit the Metrics Explorer and look for the "Generic Node - test" metric.
39+
// - visit the Logging Explorer and look for the genkit_log logName, or run
40+
// gcloud --project PROJECT_ID logging read 'logName:genkit_log'
3541
func TestGCP(t *testing.T) {
3642
if *projectID == "" {
3743
t.Skip("no -project")
@@ -64,4 +70,14 @@ func TestGCP(t *testing.T) {
6470
// Allow time to sample and export.
6571
time.Sleep(2 * time.Second)
6672
})
73+
t.Run("logging", func(t *testing.T) {
74+
if err := setLogHandler(*projectID, slog.LevelInfo); err != nil {
75+
t.Fatal(err)
76+
}
77+
slog.Info("testing GCP logging",
78+
"binaryName", os.Args[0],
79+
"goVersion", runtime.Version())
80+
// Allow time to export.
81+
time.Sleep(2 * time.Second)
82+
})
6783
}
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// The googlecloud package supports telemetry (tracing, metrics and logging) using
16+
// Google Cloud services.
17+
package googlecloud
18+
19+
import (
20+
"context"
21+
"log/slog"
22+
23+
"cloud.google.com/go/logging"
24+
"github.com/jba/slog/withsupport"
25+
)
26+
27+
func newHandler(level slog.Leveler, f func(logging.Entry)) *handler {
28+
if level == nil {
29+
level = slog.LevelInfo
30+
}
31+
return &handler{
32+
level: level,
33+
handleEntry: f,
34+
}
35+
}
36+
37+
type handler struct {
38+
level slog.Leveler
39+
handleEntry func(logging.Entry)
40+
goa *withsupport.GroupOrAttrs
41+
}
42+
43+
func (h *handler) Enabled(ctx context.Context, level slog.Level) bool {
44+
return level >= h.level.Level()
45+
}
46+
47+
func (h *handler) WithAttrs(as []slog.Attr) slog.Handler {
48+
h2 := *h
49+
h2.goa = h2.goa.WithAttrs(as)
50+
return &h2
51+
}
52+
53+
func (h *handler) WithGroup(name string) slog.Handler {
54+
h2 := *h
55+
h2.goa = h2.goa.WithGroup(name)
56+
return &h2
57+
}
58+
59+
func (h *handler) Handle(ctx context.Context, r slog.Record) error {
60+
h.handleEntry(h.recordToEntry(ctx, r))
61+
return nil
62+
}
63+
64+
func (h *handler) recordToEntry(ctx context.Context, r slog.Record) logging.Entry {
65+
return logging.Entry{
66+
Timestamp: r.Time,
67+
Severity: levelToSeverity(r.Level),
68+
Payload: recordToMap(r, h.goa.Collect()),
69+
Labels: map[string]string{"module": "genkit"},
70+
// TODO: add a monitored resource
71+
// Resource: &monitoredres.MonitoredResource{},
72+
// TODO: add trace information from the context.
73+
// Trace: "",
74+
// SpanID: "",
75+
// TraceSampled: false,
76+
}
77+
}
78+
79+
func levelToSeverity(l slog.Level) logging.Severity {
80+
switch {
81+
case l < slog.LevelInfo:
82+
return logging.Debug
83+
case l == slog.LevelInfo:
84+
return logging.Info
85+
case l < slog.LevelWarn:
86+
return logging.Notice
87+
case l < slog.LevelError:
88+
return logging.Warning
89+
case l == slog.LevelError:
90+
return logging.Error
91+
case l <= slog.LevelError+4:
92+
return logging.Critical
93+
case l <= slog.LevelError+8:
94+
return logging.Alert
95+
default:
96+
return logging.Emergency
97+
}
98+
}
99+
func recordToMap(r slog.Record, goras []*withsupport.GroupOrAttrs) map[string]any {
100+
root := map[string]any{}
101+
root[slog.MessageKey] = r.Message
102+
103+
m := root
104+
for i, gora := range goras {
105+
if gora.Group != "" {
106+
if i == len(goras)-1 && r.NumAttrs() == 0 {
107+
continue
108+
}
109+
m2 := map[string]any{}
110+
m[gora.Group] = m2
111+
m = m2
112+
} else {
113+
for _, a := range gora.Attrs {
114+
handleAttr(a, m)
115+
}
116+
}
117+
}
118+
r.Attrs(func(a slog.Attr) bool {
119+
handleAttr(a, m)
120+
return true
121+
})
122+
return root
123+
}
124+
125+
func handleAttr(a slog.Attr, m map[string]any) {
126+
if a.Equal(slog.Attr{}) {
127+
return
128+
}
129+
v := a.Value.Resolve()
130+
if v.Kind() == slog.KindGroup {
131+
gas := v.Group()
132+
if len(gas) == 0 {
133+
return
134+
}
135+
if a.Key == "" {
136+
for _, ga := range gas {
137+
handleAttr(ga, m)
138+
}
139+
} else {
140+
gm := map[string]any{}
141+
for _, ga := range gas {
142+
handleAttr(ga, gm)
143+
}
144+
m[a.Key] = gm
145+
}
146+
} else {
147+
m[a.Key] = v.Any()
148+
}
149+
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// The googlecloud package supports telemetry (tracing, metrics and logging) using
16+
// Google Cloud services.
17+
package googlecloud
18+
19+
import (
20+
"log/slog"
21+
"testing"
22+
"testing/slogtest"
23+
24+
"cloud.google.com/go/logging"
25+
)
26+
27+
func TestHandler(t *testing.T) {
28+
var results []map[string]any
29+
30+
f := func(e logging.Entry) {
31+
results = append(results, entryToMap(e))
32+
}
33+
34+
if err := slogtest.TestHandler(newHandler(slog.LevelInfo, f), func() []map[string]any { return results }); err != nil {
35+
t.Fatal(err)
36+
}
37+
}
38+
39+
func entryToMap(e logging.Entry) map[string]any {
40+
m := map[string]any{}
41+
if !e.Timestamp.IsZero() {
42+
m[slog.TimeKey] = e.Timestamp
43+
}
44+
m[slog.LevelKey] = e.Severity
45+
pm := e.Payload.(map[string]any)
46+
for k, v := range pm {
47+
m[k] = v
48+
}
49+
return m
50+
}

0 commit comments

Comments
 (0)