diff --git a/TODO.md b/TODO.md index 9c4347c1..cd184c85 100644 --- a/TODO.md +++ b/TODO.md @@ -152,11 +152,24 @@ - allow custom error formats - allow go-routine id to be logged - allow int64 to switch to string encoding when >2**50 + - do something useful with Boring + + - write to xop server + + - keep dictionary of static strings + + - Line.Static() + - Enum descriptions and types + + - xopbytes.BytesWriter + + - write to writing rotating files based time or size - xopotel - add wrapper to use otel exporter directly as xopbase - add wrapper to use xopbase as otel exporter + - do something useful with Boring - Performmance diff --git a/xopat/attributes.go b/xopat/attributes.go index e1659acb..aadc95bb 100644 --- a/xopat/attributes.go +++ b/xopat/attributes.go @@ -156,26 +156,31 @@ func (s Make) make(exampleValue interface{}, subType AttributeType) (Attribute, if s.Namespace == "" { s.Namespace = DefaultNamespace } + jsonKey, err := json.Marshal(s.Key) + if err != nil { + return Attribute{}, fmt.Errorf("cannot marshal attribute name '%s': %w", s.Key, err) + } ra := Attribute{ properties: s, exampleValue: exampleValue, reflectType: reflect.TypeOf(exampleValue), typeName: fmt.Sprintf("%T", exampleValue), subType: subType, + jsonKey: string(jsonKey) + ":", } - jsonKey, err := json.Marshal(s.Key) - if err != nil { - return ra, fmt.Errorf("cannot marshal attribute name '%s': %w", s.Key, err) - } - ra.jsonKey = string(jsonKey) registeredNames[s.Key] = &ra allAttributes = append(allAttributes, &ra) return ra, nil } +type JSONKey string + +func (jk JSONKey) String() string { return string(jk) } + // JSONKey returns a JSON-quoted string that can be used as the key to the -// name of the attribute. It is a string because []byte is mutable -func (r Attribute) JSONKey() string { return r.jsonKey } +// name of the attribute. It is a string because []byte is mutable. JSONKey +// includes a colon at the end since it's uses is as a key. +func (r Attribute) JSONKey() JSONKey { return JSONKey(r.jsonKey) } // ReflectType can be nil if the example value was nil func (r Attribute) ReflectType() reflect.Type { return r.reflectType } diff --git a/xopat/attributes.zzzgo b/xopat/attributes.zzzgo index 10b79b60..1c5ed593 100644 --- a/xopat/attributes.zzzgo +++ b/xopat/attributes.zzzgo @@ -45,17 +45,17 @@ var DefaultNamespace = os.Args[0] // Make is used to construct attributes. // Some keys are reserved. See https://github.com/muir/xop-go/blob/main/xopconst/reserved.go -// for the list of reserved keys. Some keys are already registered. +// for the list of reserved keys. Some keys are already registered. type Make struct { - Key string // the attribute name - Description string // the attribute description - Namespace string // the namespace for this attribute (otherwise DefaultNamespace is used) - Indexed bool // hint: this attribute should be indexed - Prominence int // hint: how important is this attribute (lower is more important) - Multiple bool // keep all values if the attribute is given multiple times - Distinct bool // when keeping all values, only keep distinct values (not supported for interface{}) - Ranged bool // hint: comparisons between values are meaningful (eg: time, integers) - Locked bool // only keep the first value + Key string // the attribute name + Description string // the attribute description + Namespace string // the namespace for this attribute (otherwise DefaultNamespace is used) + Indexed bool // hint: this attribute should be indexed + Prominence int // hint: how important is this attribute (lower is more important) + Multiple bool // keep all values if the attribute is given multiple times + Distinct bool // when keeping all values, only keep distinct values (not supported for interface{}) + Ranged bool // hint: comparisons between values are meaningful (eg: time, integers) + Locked bool // only keep the first value } var lock sync.RWMutex @@ -140,30 +140,35 @@ func (s Make) make(exampleValue interface{}, subType AttributeType) (Attribute, if _, ok := reservedKeys[s.Key]; ok { return Attribute{}, fmt.Errorf("key is reserved for internal use '%s'", s.Key) } - + if s.Namespace == "" { s.Namespace = DefaultNamespace } + jsonKey, err := json.Marshal(s.Key) + if err != nil { + return Attribute{}, fmt.Errorf("cannot marshal attribute name '%s': %w", s.Key, err) + } ra := Attribute{ properties: s, exampleValue: exampleValue, reflectType: reflect.TypeOf(exampleValue), typeName: fmt.Sprintf("%T", exampleValue), subType: subType, + jsonKey: string(jsonKey) + ":", } - jsonKey, err := json.Marshal(s.Key) - if err != nil { - return ra, fmt.Errorf("cannot marshal attribute name '%s': %w", s.Key, err) - } - ra.jsonKey = string(jsonKey) registeredNames[s.Key] = &ra allAttributes = append(allAttributes, &ra) return ra, nil } +type JSONKey string + +func (jk JSONKey) String() string { return string(jk) } + // JSONKey returns a JSON-quoted string that can be used as the key to the -// name of the attribute. It is a string because []byte is mutable -func (r Attribute) JSONKey() string { return r.jsonKey } +// name of the attribute. It is a string because []byte is mutable. JSONKey +// includes a colon at the end since it's uses is as a key. +func (r Attribute) JSONKey() JSONKey { return JSONKey(r.jsonKey) } // ReflectType can be nil if the example value was nil func (r Attribute) ReflectType() reflect.Type { return r.reflectType } @@ -210,10 +215,9 @@ const ( // ZZZAttribute is a just an Int64Attribute that with // SubType() == AttributeTypeDuration. A base logger may // look at SubType() to provide specialized behavior. -type ZZZAttribute struct{Int64Attribute} +type ZZZAttribute struct{ Int64Attribute } //MACRO ZZZAttribute SKIP:Enum,Duration,Int,Int8,Int16,Int32 // ZZZAttribute represents an attribute key that can be used // with zzz values. type ZZZAttribute struct{ Attribute } - diff --git a/xopjson/attributes.go b/xopjson/attributes.go index 3692164c..03f9cef3 100644 --- a/xopjson/attributes.go +++ b/xopjson/attributes.go @@ -54,7 +54,6 @@ type multiAttribute struct { } type attribute struct { - Name []byte Changed bool Type xopbase.DataType } @@ -101,20 +100,15 @@ func (a *AttributeBuilder) Append(b *xoputil.JBuilder, onlyChanged bool) { } } -func (m *multiAttribute) init(a *AttributeBuilder, k string) { +func (m *multiAttribute) init(a *AttributeBuilder, jsonKey xopat.JSONKey) { m.Builder.B = m.Buf[:0] m.Builder.reset(a.span) - m.Builder.AddString(k) - if len(m.Builder.B) == len(k)+2 { - m.Name = m.Builder.B[1 : len(m.Builder.B)-1] - } else { - m.Name = []byte(k) - } - m.Builder.AppendBytes([]byte{':', '['}) // ] + m.Builder.AppendString(jsonKey.String()) + m.Builder.AppendByte('[') // ] m.Distinct = nil } -func (a *AttributeBuilder) addMulti(k string) *multiAttribute { +func (a *AttributeBuilder) addMulti(k string, jsonKey xopat.JSONKey) *multiAttribute { var m *multiAttribute var ok bool m, ok = a.multiMap[k] @@ -124,7 +118,7 @@ func (a *AttributeBuilder) addMulti(k string) *multiAttribute { } a.multis = a.multis[:len(a.multis)+1] m = &a.multis[len(a.multis)-1] - m.init(a, k) + m.init(a, jsonKey) a.multiMap[k] = m } m.Changed = true @@ -132,22 +126,16 @@ func (a *AttributeBuilder) addMulti(k string) *multiAttribute { return m } -func (s *singleAttribute) init(k string) { +func (s *singleAttribute) init(jsonKey xopat.JSONKey) { b := xoputil.JBuilder{ B: s.Buf[:0], } - b.AddString(k) - if len(b.B) == len(k)+2 { - s.Name = b.B[1 : len(b.B)-1] - } else { - s.Name = []byte(k) - } - b.AppendByte(':') + b.AppendString(jsonKey.String()) s.Changed = true s.KeyValue = b.B } -func (a *AttributeBuilder) addSingle(k string) (*singleAttribute, bool) { +func (a *AttributeBuilder) addSingle(k string, jsonKey xopat.JSONKey) (*singleAttribute, bool) { s, ok := a.singleMap[k] if !ok { if len(a.singles) == cap(a.singles) { @@ -155,7 +143,7 @@ func (a *AttributeBuilder) addSingle(k string) (*singleAttribute, bool) { } a.singles = a.singles[:len(a.singles)+1] s = &a.singles[len(a.singles)-1] - s.init(k) + s.init(jsonKey) a.singleMap[k] = s } s.Changed = true @@ -171,7 +159,7 @@ func (a *AttributeBuilder) MetadataAny(k *xopat.AnyAttribute, v interface{}) { a.encoder.SetEscapeHTML(false) } if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -194,7 +182,7 @@ func (a *AttributeBuilder) MetadataAny(k *xopat.AnyAttribute, v interface{}) { s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.AnyDataType a.encodeTarget = &m.Builder.B m.Builder.encoder = a.encoder @@ -223,7 +211,7 @@ func (a *AttributeBuilder) MetadataBool(k *xopat.BoolAttribute, v bool) { defer a.lock.Unlock() a.anyChanged = true if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -244,7 +232,7 @@ func (a *AttributeBuilder) MetadataBool(k *xopat.BoolAttribute, v bool) { s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.BoolDataType lenBefore := len(m.Builder.B) m.Builder.AddBool(v) @@ -271,7 +259,7 @@ func (a *AttributeBuilder) MetadataEnum(k *xopat.EnumAttribute, v xopat.Enum) { defer a.lock.Unlock() a.anyChanged = true if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -292,7 +280,7 @@ func (a *AttributeBuilder) MetadataEnum(k *xopat.EnumAttribute, v xopat.Enum) { s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.EnumDataType lenBefore := len(m.Builder.B) m.Builder.AddEnum(v) @@ -319,7 +307,7 @@ func (a *AttributeBuilder) MetadataFloat64(k *xopat.Float64Attribute, v float64) defer a.lock.Unlock() a.anyChanged = true if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -340,7 +328,7 @@ func (a *AttributeBuilder) MetadataFloat64(k *xopat.Float64Attribute, v float64) s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.Float64DataType lenBefore := len(m.Builder.B) m.Builder.AddFloat64(v) @@ -367,7 +355,7 @@ func (a *AttributeBuilder) MetadataInt64(k *xopat.Int64Attribute, v int64) { defer a.lock.Unlock() a.anyChanged = true if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -388,7 +376,7 @@ func (a *AttributeBuilder) MetadataInt64(k *xopat.Int64Attribute, v int64) { s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.Int64DataType lenBefore := len(m.Builder.B) m.Builder.AddInt64(v) @@ -415,7 +403,7 @@ func (a *AttributeBuilder) MetadataLink(k *xopat.LinkAttribute, v trace.Trace) { defer a.lock.Unlock() a.anyChanged = true if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -436,7 +424,7 @@ func (a *AttributeBuilder) MetadataLink(k *xopat.LinkAttribute, v trace.Trace) { s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.LinkDataType lenBefore := len(m.Builder.B) m.Builder.AddLink(v) @@ -463,7 +451,7 @@ func (a *AttributeBuilder) MetadataString(k *xopat.StringAttribute, v string) { defer a.lock.Unlock() a.anyChanged = true if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -484,7 +472,7 @@ func (a *AttributeBuilder) MetadataString(k *xopat.StringAttribute, v string) { s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.StringDataType lenBefore := len(m.Builder.B) m.Builder.AddString(v) @@ -511,7 +499,7 @@ func (a *AttributeBuilder) MetadataTime(k *xopat.TimeAttribute, v time.Time) { defer a.lock.Unlock() a.anyChanged = true if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -532,7 +520,7 @@ func (a *AttributeBuilder) MetadataTime(k *xopat.TimeAttribute, v time.Time) { s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.TimeDataType lenBefore := len(m.Builder.B) m.Builder.AddTime(v) diff --git a/xopjson/attributes.zzzgo b/xopjson/attributes.zzzgo index d9a6dde1..6315ba77 100644 --- a/xopjson/attributes.zzzgo +++ b/xopjson/attributes.zzzgo @@ -48,7 +48,6 @@ type multiAttribute struct { } type attribute struct { - Name []byte Changed bool Type xopbase.DataType } @@ -95,20 +94,15 @@ func (a *AttributeBuilder) Append(b *xoputil.JBuilder, onlyChanged bool) { } } -func (m *multiAttribute) init(a *AttributeBuilder, k string) { +func (m *multiAttribute) init(a *AttributeBuilder, jsonKey xopat.JSONKey) { m.Builder.B = m.Buf[:0] m.Builder.reset(a.span) - m.Builder.AddString(k) - if len(m.Builder.B) == len(k)+2 { - m.Name = m.Builder.B[1 : len(m.Builder.B)-1] - } else { - m.Name = []byte(k) - } - m.Builder.AppendBytes([]byte{':', '['}) // ] + m.Builder.AppendString(jsonKey.String()) + m.Builder.AppendByte('[') // ] m.Distinct = nil } -func (a *AttributeBuilder) addMulti(k string) *multiAttribute { +func (a *AttributeBuilder) addMulti(k string, jsonKey xopat.JSONKey) *multiAttribute { var m *multiAttribute var ok bool m, ok = a.multiMap[k] @@ -118,7 +112,7 @@ func (a *AttributeBuilder) addMulti(k string) *multiAttribute { } a.multis = a.multis[:len(a.multis)+1] m = &a.multis[len(a.multis)-1] - m.init(a, k) + m.init(a, jsonKey) a.multiMap[k] = m } m.Changed = true @@ -126,22 +120,16 @@ func (a *AttributeBuilder) addMulti(k string) *multiAttribute { return m } -func (s *singleAttribute) init(k string) { +func (s *singleAttribute) init(jsonKey xopat.JSONKey) { b := xoputil.JBuilder{ B: s.Buf[:0], } - b.AddString(k) - if len(b.B) == len(k)+2 { - s.Name = b.B[1 : len(b.B)-1] - } else { - s.Name = []byte(k) - } - b.AppendByte(':') + b.AppendString(jsonKey.String()) s.Changed = true s.KeyValue = b.B } -func (a *AttributeBuilder) addSingle(k string) (*singleAttribute, bool) { +func (a *AttributeBuilder) addSingle(k string, jsonKey xopat.JSONKey) (*singleAttribute, bool) { s, ok := a.singleMap[k] if !ok { if len(a.singles) == cap(a.singles) { @@ -149,7 +137,7 @@ func (a *AttributeBuilder) addSingle(k string) (*singleAttribute, bool) { } a.singles = a.singles[:len(a.singles)+1] s = &a.singles[len(a.singles)-1] - s.init(k) + s.init(jsonKey) a.singleMap[k] = s } s.Changed = true @@ -168,7 +156,7 @@ func (a *AttributeBuilder) MetadataZZZ(k *xopat.ZZZAttribute, v zzz) { } //END CONDITIONAL if !k.Multiple() { - s, preExisting := a.addSingle(k.Key()) + s, preExisting := a.addSingle(k.Key(), k.JSONKey()) if preExisting { if k.Locked() { return @@ -195,7 +183,7 @@ func (a *AttributeBuilder) MetadataZZZ(k *xopat.ZZZAttribute, v zzz) { s.KeyValue = b.B return } - m := a.addMulti(k.Key()) + m := a.addMulti(k.Key(), k.JSONKey()) m.Type = xopbase.ZZZDataType //CONDITIONAL ONLY:Any a.encodeTarget = &m.Builder.B diff --git a/xopjson/jsonlogger.go b/xopjson/jsonlogger.go index 7d374b0c..da87e2a1 100644 --- a/xopjson/jsonlogger.go +++ b/xopjson/jsonlogger.go @@ -194,7 +194,6 @@ func (r *request) maintainBuffer() { } func (r *request) flushBuffer() { - // TODO: trigger spans to write their stuff if len(r.writeBuffer) == 0 { return } @@ -335,7 +334,7 @@ func (s *span) Done(t time.Time, _ bool) { s.FlushAttributes() } -func (s *span) Boring(bool) {} // TODO +func (s *span) Boring(bool) {} func (s *span) ID() string { return s.logger.id.String() } func (s *span) NoPrefill() xopbase.Prefilled { @@ -415,7 +414,6 @@ func (p *prefilled) Line(level xopnum.Level, t time.Time, pc []uintptr) xopbase. l.Int64("lvl", int64(level), 0) l.Time("ts", t) if len(pc) > 0 { - // TODO: debug this! frames := runtime.CallersFrames(pc) l.AppendBytes([]byte(`,"stack":[`)) for { @@ -482,7 +480,7 @@ func (l *line) Msg(m string) { } func (l *line) Static(m string) { - l.Msg(m) // TODO + l.Msg(m) } func (b *builder) reclaimMemory() { @@ -545,7 +543,8 @@ func (b *builder) AddAny(v interface{}) { func (b *builder) Enum(k *xopat.EnumAttribute, v xopat.Enum) { b.startAttributes() - b.AddSafeKey(k.Key()) // TODO: check attribute keys at registration time + b.Comma() + b.AppendString(k.JSONKey().String()) b.AddEnum(v) } diff --git a/xopjson/jsonlogger.zzzgo b/xopjson/jsonlogger.zzzgo index 12f8944e..125f51a5 100644 --- a/xopjson/jsonlogger.zzzgo +++ b/xopjson/jsonlogger.zzzgo @@ -192,7 +192,6 @@ func (r *request) maintainBuffer() { } func (r *request) flushBuffer() { - // TODO: trigger spans to write their stuff if len(r.writeBuffer) == 0 { return } @@ -333,7 +332,7 @@ func (s *span) Done(t time.Time, _ bool) { s.FlushAttributes() } -func (s *span) Boring(bool) {} // TODO +func (s *span) Boring(bool) {} func (s *span) ID() string { return s.logger.id.String() } func (s *span) NoPrefill() xopbase.Prefilled { @@ -413,7 +412,6 @@ func (p *prefilled) Line(level xopnum.Level, t time.Time, pc []uintptr) xopbase. l.Int64("lvl", int64(level), 0) l.Time("ts", t) if len(pc) > 0 { - // TODO: debug this! frames := runtime.CallersFrames(pc) l.AppendBytes([]byte(`,"stack":[`)) for { @@ -480,7 +478,7 @@ func (l *line) Msg(m string) { } func (l *line) Static(m string) { - l.Msg(m) // TODO + l.Msg(m) } func (b *builder) reclaimMemory() { @@ -543,7 +541,8 @@ func (b *builder) AddAny(v interface{}) { func (b *builder) Enum(k *xopat.EnumAttribute, v xopat.Enum) { b.startAttributes() - b.AddSafeKey(k.Key()) // TODO: check attribute keys at registration time + b.Comma() + b.AppendString(k.JSONKey().String()) b.AddEnum(v) }