diff --git a/.gitattributes b/.gitattributes index c3833cd8..9429c2fe 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,30 +2,30 @@ doc.go linguist-documentation *.md linguist-documentation *.pb.go linguist-generated -basegroup.go linguist-generated -line.go linguist-generated -seed.go linguist-generated -span.go linguist-generated -sub.go linguist-generated -xopat/attributes.go linguist-generated -xopbase/abbr.go linguist-generated -xopbase/base.go linguist-generated -xopbase/skip.go linguist-generated -xopcon/console.go linguist-generated -xopconsole/attributes.go linguist-generated -xopconsole/replay.go linguist-generated -xopjson/jsonlogger.go linguist-generated -xopjson/replay.go linguist-generated -xopotel/buffered.go linguist-generated -xopotel/export.go linguist-generated -xopotel/otel.go linguist-generated -xoppb/pb.go linguist-generated -xoppb/replay.go linguist-generated -xoprecorder/recorder.go linguist-generated -xoprecorder/replay.go linguist-generated -xoptrace/hexbytes.go linguist-generated -xoputil/enumer_test.go linguist-generated -xopbase/xopbaseutil/metadata.go linguist-generated -xopjson/xopjsonutil/attributes.go linguist-generated -xoptest/xoptestutil/enums.go linguist-generated -xoptest/xoptestutil/verify_replay.go linguist-generated +../xop-go/basegroup.go linguist-generated +../xop-go/line.go linguist-generated +../xop-go/seed.go linguist-generated +../xop-go/span.go linguist-generated +../xop-go/sub.go linguist-generated +../xopotel-go/buffered.go linguist-generated +../xopotel-go/export.go linguist-generated +../xopotel-go/otel.go linguist-generated +../xop-go/xopat/attributes.go linguist-generated +../xop-go/xopbase/abbr.go linguist-generated +../xop-go/xopbase/base.go linguist-generated +../xop-go/xopbase/skip.go linguist-generated +../xop-go/xopcon/console.go linguist-generated +../xop-go/xopconsole/attributes.go linguist-generated +../xop-go/xopconsole/replay.go linguist-generated +../xop-go/xopjson/jsonlogger.go linguist-generated +../xop-go/xopjson/replay.go linguist-generated +../xop-go/xoppb/pb.go linguist-generated +../xop-go/xoppb/replay.go linguist-generated +../xop-go/xoprecorder/recorder.go linguist-generated +../xop-go/xoprecorder/replay.go linguist-generated +../xop-go/xoptrace/hexbytes.go linguist-generated +../xop-go/xoputil/enumer_test.go linguist-generated +../xop-go/xopbase/xopbaseutil/metadata.go linguist-generated +../xop-go/xopjson/xopjsonutil/attributes.go linguist-generated +../xop-go/xoptest/xoptestutil/enums.go linguist-generated +../xop-go/xoptest/xoptestutil/verify_replay.go linguist-generated diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index d713ebca..4c315a40 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -12,6 +12,8 @@ jobs: - uses: actions/setup-go@v2 with: go-version: '1.18' + - name: Checkout peers + run: make ci_checkout_peers CLONE_DEPTH=2 - name: Run coverage run: make calculate_coverage diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index cac17a20..f5351d4d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,5 +14,7 @@ jobs: go-version: ${{ matrix.go-version }} - name: Checkout code uses: actions/checkout@v2 + - name: Checkout peers + run: make ci_checkout_peers - name: Test run: make citest diff --git a/LICENSE b/LICENSE index 3abb6305..fc408082 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021, 2022 David Sharnoff +Copyright (c) 2021-2024 David Sharnoff Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 55f58238..71e3275e 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,21 @@ -GOHOME ?= ${HOME} -GOPATH ?= ${GOHOME} -GOBIN ?= ${GOPATH}/bin +ME = xop-go +RELATED = xopresty-go xopotel-go +EXTRA_TEST_DEPS = testadjuster -ZZZGO = $(wildcard *.zzzgo */*.zzzgo */*/*.zzzgo) -ZZZGENERATED = $(patsubst %.zzzgo, %.go, $(ZZZGO)) -PB = xopproto/ingest.pb.go xopproto/ingest_grpc.pb.go -TOOLS = ${GOBIN}/gofumpt ${GOBIN}/goimports ${GOBIN}/enumer -TEST_ONLY =? +include Makefile.common -all: $(ZZZGENERATED) $(PB) .gitattributes - go generate ./... - go build ./... - -.gitattributes: $(ZZZGENERATED) - echo '*.zzzgo linguist-language=Go' > $@ - echo 'doc.go linguist-documentation' >> $@ - echo '*.md linguist-documentation' >> $@ - echo '*.pb.go linguist-generated' >> $@ - for i in $(ZZZGENERATED); do echo "$$i linguist-generated" >> $@; done +ci_checkout_peers:; + branch="$${GITHUB_REF##*/}"; for i in $(RELATED); do (cd ..; git clone https://github.com/xoplog/$$i --depth $(CLONE_DEPTH) -b $$branch || git clone https://github.com/xoplog/$$i --depth $(CLONE_DEPTH)); done test: $(ZZZGENERATED) testadjuster go generate ./... go test -v ./xopjson/... -run TestASingleLine go test -v ./xopjson/... -tags xoptesting -run TestParameters -failfast $(TEST_ONLY) go test -tags xoptesting ./... -failfast $(TEST_ONLY) + for i in $(RELATED); do (echo $$i...; cd ../$$i && go test -tags xoptesting ./... $(TEST_ONLY) ); done go test -tags xoptesting -race ./... -failfast $(TEST_ONLY) + for i in $(RELATED); do (echo $$i...; cd ../$$i && go test -tags xoptesting -race ./... $(TEST_ONLY) ); done testadjuster: $(ZZZGenerated) @@ -34,27 +24,12 @@ testadjuster: $(ZZZGenerated) citest: go test ./... -failfast + for i in $(RELATED); do (echo $$i...; cd ../$$i && go test -tags xoptesting ./... $(TEST_ONLY) ); done go test -race ./... -failfast + for i in $(RELATED); do (echo $$i...; cd ../$$i && go test -tags xoptesting -race ./... $(TEST_ONLY) ); done XOPLEVEL_xoptestutil=warn XOPLEVEL_foo=debug go test -tags xoptesting ./xoptest/xoptestutil -run TestAdjuster -count 1 XOPLEVEL_xoptestutil=debug XOPLEVEL_foo=warn go test -tags xoptesting ./xoptest/xoptestutil -run TestAdjuster -count 1 -${GOBIN}/gofumpt:; - go install mvdan.cc/gofumpt@latest - -${GOBIN}/goimports:; - go install golang.org/x/tools/cmd/goimports@latest - -${GOBIN}/enumer:; - go install github.com/dmarkham/enumer@latest - -%.go : %.zzzgo tools/xopzzz/xopzzz.go $(TOOLS) Makefile - go run tools/xopzzz/xopzzz.go < $< > $@.tmp - -chmod +w $@ - gofumpt -w $@.tmp - goimports -w $@.tmp - -mv $@.tmp $@ - -chmod -w $@ - calculate_coverage: echo "mode: atomic" > coverage.txt for d in $$(go list ./...); do \ @@ -73,21 +48,6 @@ calculate_coverage: egrep -v 'xoptestutil/|xopoteltest/' > coverage.txt.tmp mv coverage.txt.tmp coverage.txt -coverage: calculate_coverage - go tool cover -html=coverage.txt - -golanglint: - # binary will be $(go env GOPATH)/bin/golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.52.2 - golangci-lint --version - -lint:; - golangci-lint run - -misspell:; - go install github.com/client9/misspell/cmd/misspell@latest - misspell -w `find . -name \*.md` - OTEL_TAG="v1.12.0" ../opentelemetry-specification: diff --git a/Makefile.common b/Makefile.common new file mode 100644 index 00000000..7250059e --- /dev/null +++ b/Makefile.common @@ -0,0 +1,55 @@ + +GOHOME ?= ${HOME} +GOPATH ?= ${GOHOME} +GOBIN ?= ${GOPATH}/bin + +ZZZGO = $(wildcard ../xop*-go/*.zzzgo ../xop*-go/*/*.zzzgo ../xop*-go/*/*/*.zzzgo) +ZZZGENERATED = $(patsubst %.zzzgo, %.go, $(ZZZGO)) +PB = xopproto/ingest.pb.go xopproto/ingest_grpc.pb.go +TOOLS = ${GOBIN}/gofumpt ${GOBIN}/goimports ${GOBIN}/enumer +TEST_ONLY ?= +CLONE_DEPTH ?= 1 + +all: $(ZZZGENERATED) $(PB) .gitattributes + go generate ./... + go build ./... + for i in $(RELATED); do (echo $$i ...; cd ../$$i && go generate ./... && go build ./...); done + +.gitattributes: $(ZZZGENERATED) + echo '*.zzzgo linguist-language=Go' > $@ + echo 'doc.go linguist-documentation' >> $@ + echo '*.md linguist-documentation' >> $@ + echo '*.pb.go linguist-generated' >> $@ + for i in $(ZZZGENERATED); do echo "$$i linguist-generated" >> $@; done + +${GOBIN}/gofumpt:; + go install mvdan.cc/gofumpt@latest + +${GOBIN}/goimports:; + go install golang.org/x/tools/cmd/goimports@latest + +${GOBIN}/enumer:; + go install github.com/dmarkham/enumer@latest + +%.go : %.zzzgo tools/xopzzz/xopzzz.go $(TOOLS) Makefile + go run tools/xopzzz/xopzzz.go < $< > $@.tmp + -chmod +w $@ + gofumpt -w $@.tmp + goimports -w $@.tmp + -mv $@.tmp $@ + -chmod -w $@ + +coverage: calculate_coverage + go tool cover -html=coverage.txt + +golanglint: + # binary will be $(go env GOPATH)/bin/golangci-lint + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.52.2 + golangci-lint --version + +lint:; + golangci-lint run + +misspell:; + go install github.com/client9/misspell/cmd/misspell@latest + misspell -w `find . -name \*.md` diff --git a/Makefile.peer b/Makefile.peer new file mode 100644 index 00000000..25e17239 --- /dev/null +++ b/Makefile.peer @@ -0,0 +1,25 @@ + +include ../xop-go/Makefile.common + +test: $(ZZZGENERATED) + go generate ./... + go test -tags xoptesting ./... -failfast $(TEST_ONLY) + go test -tags xoptesting -race ./... -failfast $(TEST_ONLY) + +citest: + go test ./... -failfast + go test -race ./... -failfast + +calculate_coverage: + echo "mode: atomic" > coverage.txt + for d in $$(go list ./...); do \ + go test -race -covermode=atomic -coverprofile=profile.out -coverpkg=github.com/xoplog/$(ME)/... $$d; \ + if [ -f profile.out ]; then \ + grep -v ^mode profile.out >> coverage.txt; \ + rm profile.out; \ + fi; \ + done + grep -v '\.pb.go:' coverage.txt | \ + egrep -v 'xoptestutil/|xopoteltest/' > coverage.txt.tmp + mv coverage.txt.tmp coverage.txt + diff --git a/README.md b/README.md index a6a9ab51..5c8821f1 100644 --- a/README.md +++ b/README.md @@ -56,20 +56,11 @@ to open pull requests, especially to added base loggers or propagators. Expect the following changes as development continues: -- Log line keys (for key/value attributes) will become typed and - pre-registered (cheap, inline-possible registrations, not complex) - - API changes as additional features are added Currently xop has no metrics support. That will change and adding metrics will probably be the biggest API change -- xopresty and xopotel will split off to their own repos. - -- Additional base loggers are coming - - A full-fidelity console logger is expected soon. - - Additional gateway base loggers will be written To make xop the best logging library for library writers, a full compliment @@ -109,6 +100,28 @@ to strike a blance between safety and usability. Metadata on spans are fully type-safe and keywords must be pre-registered. Data elements on log lines are mostly type-safe but do not need to be pre-registered. +## Base loggers + +Xop is a two-level logger. The top-level logger provides the API for +logging lines and spans, etc. The bottom-level loggers translate the logs +to different formats. + +Some of the bottom-level loggers are "full fidelity" which means that +they are bundled with a function that can consume their own output and +re-log it to a different bottom-level logger thus translating from one +format to another. Xop bottom-level loggers must implement the +[xopbase](https://pkg.go.dev/github.com/xoplog/xop-go/xopbase) Logger interface. + +| name | full fidelity | description | +| -- | -- | -- | +| [xopjson](https://pkg.go.dev/github.com/xoplog/xop-go/xopjson) | yes | JSON output | +| [xopotel](https://pkg.go.dev/github.com/xoplog/xopotel-go) | yes | Output though OpenTelemetry spans (Go logger not available) | +| [xopcon](https://pkg.go.dev/github.com/xoplog/xop-go/xopcon) | no | Console/text logger emphasizing human readability | +| [xopconsole](https://pkg.go.dev/github.com/xoplog/xop-go/xopconsole) | yes | Console/text logger with no information loss | +| [xoppb](https://pkg.go.dev/github.com/xoplog/xop-go/xoppb) | yes | Protobuf output | +| [xoprecorder](https://pkg.go.dev/github.com/xoplog/xop-go/xoprecorder) | yes | Output into a structured in-memory buffer | +| [xoptest](https://pkg.go.dev/github.com/xoplog/xop-go/xoptest) | no | Output to testing.T logger | + ## Using xop To log, you must have a `*Log` object. To create one you must start with a @@ -218,11 +231,12 @@ into various http router frameworks. Outgoing propagation is sharing the current trace id as the parent request to another server when making a request. Xop currently only supports HTTP and that only with [resty](https://github.com/go-resty/resty) in the -xopmiddle package. Adding additional outgoing propagators is an outstanding priority. +[xopresty](https://github.com/xoplog/xopresty-go) package. Adding additional +outgoing propagators is an outstanding priority. ### Version compatibility -xop is currently tested with go1.18 and go1.19. It is probably +xop is currently tested with go1.18 through go1.20. It is probably compatible with go1.17 and perhaps earlier. ## Terminology diff --git a/go.mod b/go.mod index 3285dc28..5a552ad7 100644 --- a/go.mod +++ b/go.mod @@ -8,14 +8,8 @@ require ( github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/muir/gwrap v0.1.0 github.com/muir/list v1.1.1 - github.com/muir/reflectutils v0.7.0 - github.com/muir/resty v0.0.1 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.2 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 - go.opentelemetry.io/otel/sdk v1.14.0 - go.opentelemetry.io/otel/trace v1.14.0 golang.org/x/text v0.3.6 google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.1 @@ -23,9 +17,8 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.0.0-20211029224645-99673261e6eb // indirect golang.org/x/sys v0.5.0 // indirect diff --git a/go.sum b/go.sum index 4e841f5c..fe0175f1 100644 --- a/go.sum +++ b/go.sum @@ -9,13 +9,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= -github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -35,6 +28,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= @@ -43,10 +37,6 @@ github.com/muir/gwrap v0.1.0 h1:o63/q6YgVge1MPwjvH9EJva1EdvOXWKc2mRdNXyo9Zo= github.com/muir/gwrap v0.1.0/go.mod h1:lPm430bQmQkudhpdC8txcEwZgl3azfQAN3Ppg2kmUws= github.com/muir/list v1.1.1 h1:y8wD4u9Jmphcr2480Xndo1N3wVZmrwTnO5NK1eKDStU= github.com/muir/list v1.1.1/go.mod h1:w2K+uRWCjFPlKuQGkl3vusHkUo3GU5xiJXErhvCiU60= -github.com/muir/reflectutils v0.7.0 h1:7ez7OLYTThDQ5kpEpxtOgFvJgtE4E11D6PVTVw+Lwl0= -github.com/muir/reflectutils v0.7.0/go.mod h1:l8W7iTj6zMdmsWcPfsdnaAYLEuipJ7baVROqpfuonIc= -github.com/muir/resty v0.0.1 h1:dSWIF/uLX01/AzniDi+XukP9Q6g5QO10DPVJsZ+49n0= -github.com/muir/resty v0.0.1/go.mod h1:Vjk1Uhi1m4/Och2tNJ2nxuGJnoaI8L0U4a27TbzKMR4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -55,19 +45,10 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -85,11 +66,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/xopotel/NOTES.md b/xopotel/NOTES.md deleted file mode 100644 index df4e9e57..00000000 --- a/xopotel/NOTES.md +++ /dev/null @@ -1,103 +0,0 @@ - -TODO/XXX: test changing OTEL Span name - -## Doc notes - -In this doc, we'll use the names of contants, like `xopVersion` -rather than the values. The values can be found in `models.go`. - -## Where OTEL and XOP differ - -As of the time of writing, OTEL doesn't have a Go logger so logging -is done as span Events. - -Data types. The set of data types is different with OTEL having -slices that XOP doesn't have and XOP having Any/Model, unsigned int, -time.Duration, and time.Time which OTEL doesn't have. - -Both OTEL and XOP require some information at the creation of a span, -but there are differences: - -- XOP span creation only (where OTEL can change it later) - - Name/Description - - "SourceInfo" -- OTEL span creation only (where XOP can change it later) - - Links - - "Instrumentation.Scope" - -In XOP, links can be log data. Links only have attributes (other -than a text description) only when they're log data and not when -they're span-level attributes (metadata). - -In OTEL, links are only span-level attributes and each link can -have arbitary attributes. - -## XOP -> OTEL - -Data that originaged in XOP is identified by having a set of -XOP-specific span-level attributes, at least on the request. - -- xopVersion a version string -- xopOTELVersiona version string -- xopSource encoding the SourceInfo Source & SourceVersion -- xopNamespace encoding the SourceInfo Namespace & Namespace Version - -These four attributes will always be present but the presense of -xopVersion (`"xop.version"`) is enough to judge span as originating -with XOP. - -Translating from XOP to OTEL means encoding log lines as span Events -since (at the time of this writing) OTEL doesn't have a Go logger. - -Since the types don't match up and XOP has more types, almost all XOP -types are encoded as OTEL `StringSlice` with the value as the the first -element in the slice and the type as the second element. Some XOP types, -like, Any, use additional slice elements. Xop bool gets encoded as a -`Bool` since there is only one kind of bool. - -The encoding of XOP links varies a bit depending on which XOP->OTEL -encoder is used. - -- For `BufferedReplayLogger` and `ReadOnlySpan`s that have been - post-processed with `NewUnhacker`, span Metadata links - become OTEL span-level links directly. -- For other encoders links in span Metadata becomes sub-spans that are - marked as existing just to encode links in their parent span. These - extra `Span`s are marked with a spanIsLinkAttributeKey attribute. -- For log lines that are links, if they're encoded both as an `Event` - and also with a sub-Span that is used just to encode the links. These - extra `Span`s are marked with a spanIsLinkEventKey attribute. - WIth `BufferedReplayLogger`, line links are also added to the span - attributes. - -- Baggage - While OTEL includes functions for manipulating Baggage, the Baggage is - not retained in the span directly. XOP baggage is saved in OTEL as - a span-level string attribute keyed by the `xopBaggage` constant. - -## XOP -> OTEL -> XOP - -When translating from OTEL to XOP, data that originated in XOP is detected -so that it can be reconstructed at full fidelity. - -The detection is based upon the presense of specific attributes in the -`Span`s. - -## OTEL -> XOP - -Data imported from OTEL is marked with a MetadataAny: otelReplayStuff. -That marking allows XOP -> OTEL translation to detect where the data came -from origianlly so that it can be reconstructed. This object contains -all of the attributes that aren't easily mapped to XOP. The type of this -object is `otelStuff`. - -Some of the OTEL types translate directly to XOP and they're sent with their -natural representation. - -The remaining OTEL types are mostly encoded using `xopbase.ModelArg` and sent -at the line live with `Model()` and at the span level with `MetadataAny`. - -OTEL links are all span-level, but they all have attribute slices rather than -just name/description so they do not fit well as MetadataLinks. OTEL links are -sent twice, once as MetadataLink with `otelLink` and also as line Links where -their attributes (if any) are included, always described as `xopOTELLinkDetail`. diff --git a/xopotel/README.md b/xopotel/README.md deleted file mode 100644 index b8081796..00000000 --- a/xopotel/README.md +++ /dev/null @@ -1,42 +0,0 @@ - -Package xopotel provides gateways/connections between -[xop](https://github.com/muir/xop-go) -[OpenTelemetry](https://opentelemetry.io/). - -## `SpanLog` - -`SpanLog()` allows xop to be used as a logger to add "Events" to an -existing OTEL span. - -## `BaseLogger` - -`BaseLogger()` creates a `xopbase.Logger` that can be used as a -to gateway xop logs and spans into OTEL. - -## Compatibility - -### Data types - -OTEL supports far fewer data types than xop. Mostly, xop types -can be converted cleanly, but links are a special case: links can -only be added to OTEL spans when the span is created. Since xop -allows links to be made at any time, links will be added as -ephemeral sub-spans. Distinct, Multiple, and Locked attributes will -be ignored for links. - -OTEL does not support unsigned ints. `uint64` will be converted to a -string and smaller unsigned ints will convert to `int64`. - -OTEL provides no way to record links (trace_id/span_id) except as part -of the initial set of "Links" associated with a span. Those links are -values only (no key, no name). Xop supports arbitrary numbers of links -on a per-logline basis. There is no way to record those as links using -the OTEL structures so if there is more than one of them, then they'll -be recorded as strings. Most of the time, a log line won't have more -than one trace reference. - -For both events with links, and links as span metadata, an extra OTEL -span will be created just to hold the link. The extra span will be marked -with span.is-link-attribute:true for span metadata and -span.is-link-event for events. - diff --git a/xopotel/buffered.go b/xopotel/buffered.go deleted file mode 100644 index 5c3a5067..00000000 --- a/xopotel/buffered.go +++ /dev/null @@ -1,623 +0,0 @@ -// This file is generated, DO NOT EDIT. It comes from the corresponding .zzzgo file - -package xopotel - -import ( - "context" - "encoding/json" - "fmt" - "sync/atomic" - "time" - - "github.com/muir/gwrap" - "github.com/xoplog/xop-go/xopat" - "github.com/xoplog/xop-go/xopbase" - "github.com/xoplog/xop-go/xopbase/xopbaseutil" - "github.com/xoplog/xop-go/xoprecorder" - "github.com/xoplog/xop-go/xoptrace" - "github.com/xoplog/xop-go/xoputil/pointer" - - "github.com/google/uuid" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/sdk/instrumentation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" -) - -type BufferedReplayExporterWrapper interface { - // WrapExporter augments the data that the wrapped exporter recevies so that if - // the ReadOnlySpan came from a BufferedReplayLogger that is replaying data that - // originally came from OTEL, then it can fill data that otherwise has no way - // to be propagated through the OTEL APIs. For example, instrumentation.Scope.Name. - WrapExporter(sdktrace.SpanExporter) sdktrace.SpanExporter - - // BufferedReplayLogger creates a Logger that can be used when replaying from other - // xopbase.Logger implementations into xopotel. It buffers all the logged data until - // Done() is called on a per-request basis. Additional logging after Done() is discarded. - // - // A TracerProvider and Tracer are constructed for each Request and discarded afterwards. - // - // VERY IMPORTANT: every exporter wrapped with WrapExporter must be - // passed to BufferedReplayLogger. If not, memory will leak. - // - // Also import: if the exporter's ExportSpans() isn't called with all - // spans, memory will leak. The amount of leaked memory is not large, maybe - // 100 bytes per span, but it isn't zero. - BufferedReplayLogger(...sdktrace.TracerProviderOption) xopbase.Logger -} - -// BufferedReplayLogger creates a Logger that can be used when replaying from other -// xopbase.Logger implementations into xopotel. It buffers all the logged data until -// Done() is called on a per-request basis. Additional logging after Done() is discarded. -// -// A TracerProvider and Tracer are constructed for each Request and discarded afterwards. -// -// For improved fideltity of OTEL -> XOP -> OTEL replay, use -// BufferedReplayExporterWrapper.BufferedReplayLogger instead. -func BufferedReplayLogger(tracerProviderOpts ...sdktrace.TracerProviderOption) xopbase.Logger { - return bufferedReplayLogger(&exporterWrapper{}, tracerProviderOpts) -} - -func bufferedReplayLogger(exporterWrapper *exporterWrapper, tracerProviderOpts []sdktrace.TracerProviderOption) xopbase.Logger { - return &bufferedLogger{ - tracerProviderOpts: tracerProviderOpts, - id: "otelbuf-" + uuid.New().String(), - exporterWrapper: exporterWrapper, - } -} - -type bufferedLogger struct { - id string - tracerProviderOpts []sdktrace.TracerProviderOption - exporterWrapper *exporterWrapper -} - -func (logger *bufferedLogger) ID() string { return logger.id } -func (logger *bufferedLogger) ReferencesKept() bool { return true } -func (logger *bufferedLogger) Buffered() bool { return false } - -// bufferedRequest implements Request for a buferedLogger. The bufferedLogger is a wrapper -// wrapper around request{}. It waits until the request is complete before it invokes -// logger.Request and in the meantime buffers all data into an ephemeral xoprecorder.Logger. -// -// It uses xoprecorder's replay functionality to dump the buffered logs. -// -// It creates a logger{} for each request data can be passed directly from the bufferedRequest -// to the logger{} and from the logger{} to the request{} because there is only one request{} -// per logger{} in this situation. -type bufferedRequest struct { - xopbase.Request - recorder *xoprecorder.Logger - finalized bool - logger *bufferedLogger - ctx context.Context - bundle xoptrace.Bundle - errorReporter func(error) - isXOP bool // request originally came from XOP -} - -func (logger *bufferedLogger) Request(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, sourceInfo xopbase.SourceInfo) xopbase.Request { - recorder := xoprecorder.New() - return &bufferedRequest{ - recorder: recorder, - Request: recorder.Request(ctx, ts, bundle, description, sourceInfo), - logger: logger, - ctx: ctx, - bundle: bundle, - isXOP: sourceInfo.Source != otelDataSource, - } -} - -func (request *bufferedRequest) SetErrorReporter(f func(error)) { request.errorReporter = f } - -func (request *bufferedRequest) Done(endTime time.Time, final bool) { - if request.finalized { - return - } - request.Request.Done(endTime, final) - if !final { - return - } - request.finalized = true - tpOpts := []sdktrace.TracerProviderOption{ - sdktrace.WithSpanLimits(sdktrace.SpanLimits{ - AttributeValueLengthLimit: -1, - AttributeCountLimit: -1, - EventCountLimit: -1, - LinkCountLimit: -1, - AttributePerEventCountLimit: -1, - AttributePerLinkCountLimit: -1, - }), - } - tpOpts = append(tpOpts, request.logger.tracerProviderOpts...) - tpOpts = append(tpOpts, IDGenerator()) - - otelStuff := request.getStuff(request.bundle, false) - // we do not call augment() here because that would result in - // a duplicate call when it is also called from Reqeust() - tpOpts = append(tpOpts, otelStuff.TracerProviderOptions()...) - - tracerProvider := sdktrace.NewTracerProvider(tpOpts...) - defer tracerProvider.Shutdown(request.ctx) - - var tOpts []oteltrace.TracerOption - if request.isXOP { - tOpts = append(tOpts, - oteltrace.WithInstrumentationAttributes( - xopOTELVersion.String(xopotelVersionValue), - xopVersion.String(xopVersionValue), - ), - oteltrace.WithInstrumentationVersion(xopotelVersionValue), - ) - } - tOpts = append(tOpts, otelStuff.TracerOptions()...) - tracer := tracerProvider.Tracer("xopotel", tOpts...) - otel := &logger{ - id: "bufotel-" + uuid.New().String(), - doLogging: true, - tracer: tracer, - bufferedRequest: request, - } - request.recorder.Replay(request.ctx, otel) - err := tracerProvider.ForceFlush(request.ctx) - if err != nil && request.errorReporter != nil { - request.errorReporter(err) - } -} - -func (req *bufferedRequest) getStuff(bundle xoptrace.Bundle, augment bool) (stuff *otelStuff) { - if req == nil || req.recorder == nil { - return - } - spanID := bundle.Trace.SpanID().Array() - _ = req.recorder.WithLock(func(r *xoprecorder.Logger) error { - span, ok := r.SpanIndex[spanID] - if !ok { - return nil - } - var otelStuff otelStuff - stuff = &otelStuff - - addMetadataLink := func(key string, link xoptrace.Trace) { - otelStuff.links = append(otelStuff.links, oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: link.TraceID().Array(), - SpanID: link.SpanID().Array(), - TraceFlags: oteltrace.TraceFlags(link.Flags().Array()[0]), - TraceState: emptyTraceState, - Remote: true, - }), - Attributes: []attribute.KeyValue{ - xopLinkMetadataKey.String(key), - }, - }) - } - span.SpanMetadata.Map.Range(func(key string, mt *xopbaseutil.MetadataTracker) bool { - switch mt.Type { - case xopbase.LinkDataType: - addMetadataLink(key, mt.Value.(xoptrace.Trace)) - case xopbase.LinkArrayDataType: - for _, link := range mt.Value.([]interface{}) { - addMetadataLink(key, link.(xoptrace.Trace)) - } - } - return true - }) - - var missingData missingSpanData - - if len(span.Links) > 0 { - for _, linkLine := range span.Links { - plainBuilder := builder{ - attributes: make([]attribute.KeyValue, 0, len(linkLine.Data)), - } - linkLine = pointer.To(linkLine.Copy()) - - var ts oteltrace.TraceState - - tsRaw, ok := extractValue[string](linkLine, xopOTELLinkTranceState, xopbase.StringDataType, xopLinkTraceStateError) - if ok { - var err error - ts, err = oteltrace.ParseTraceState(tsRaw) - if err != nil { - linkLine.Data[xopLinkTraceStateError] = err.Error() - linkLine.DataType[xopLinkTraceStateError] = xopbase.ErrorDataType - } - } - - isRemote, _ := extractValue[bool](linkLine, xopOTELLinkIsRemote, xopbase.BoolDataType, xopLinkRemoteError) - - linkSpanID := linkLine.AsLink.SpanID().Array() - - if augment { - droppedCount, _ := extractValue[int64](linkLine, xopOTELLinkDroppedAttributeCount, xopbase.IntDataType, xopLinkeDroppedError) - if droppedCount != 0 { - if missingData.droppedLinkAttributeCount == nil { - missingData.droppedLinkAttributeCount = make(map[[8]byte]int) - } - missingData.droppedLinkAttributeCount[linkSpanID] = int(droppedCount) - } - } - - xoprecorder.ReplayLineData(linkLine, &plainBuilder) - - otelStuff.links = append(otelStuff.links, oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: linkLine.AsLink.TraceID().Array(), - SpanID: linkSpanID, - TraceFlags: oteltrace.TraceFlags(linkLine.AsLink.Flags().Array()[0]), - TraceState: ts, - Remote: isRemote, - }), - Attributes: plainBuilder.attributes, - }) - } - } - - if failure := func() string { - md := span.SpanMetadata.Get(otelReplayStuff.Key().String()) - if md == nil { - return "span metdata missing replay key expected for BufferedReplayLogger " + otelReplayStuff.Key().String() - } - ma, ok := md.Value.(xopbase.ModelArg) - if !ok { - return fmt.Sprintf("cast of %s data to ModelArg failed, is %T", otelReplayStuff.Key(), md.Value) - } - err := ma.DecodeTo(&otelStuff) - if err != nil { - return fmt.Sprintf("failed to decode encoded data in %s: %s", otelReplayStuff.Key(), err) - } - return "" - }(); failure != "" { - missingData.error = failure - } else { - missingData.spanCounters = otelStuff.spanCounters - missingData.scopeName = otelStuff.InstrumentationScope.Name - } - var zeroMissing missingSpanDataComparable - if augment && atomic.LoadInt32(&req.logger.exporterWrapper.exporterCount) > 0 && - (missingData.missingSpanDataComparable != zeroMissing || missingData.droppedLinkAttributeCount != nil) { - req.logger.exporterWrapper.augmentMap.Store(spanID, &missingData) - } - return nil - }) - return -} - -func extractValue[T any](linkLine *xoprecorder.Line, name xopat.K, dataType xopbase.DataType, errorKey xopat.K) (T, bool) { - var zero T - if raw, ok := linkLine.Data[name]; ok { - if linkLine.DataType[name] == dataType { - cooked, ok := raw.(T) - if ok { - delete(linkLine.Data, name) - delete(linkLine.DataType, name) - return cooked, true - } else { - linkLine.Data[errorKey] = fmt.Sprintf("%s should be a %T, but is a %T", - name, zero, raw) - linkLine.DataType[errorKey] = xopbase.ErrorDataType - } - } else { - linkLine.Data[errorKey] = fmt.Sprintf("%s should be a %s, but is a %s", - name, dataType, linkLine.DataType[name]) - linkLine.DataType[errorKey] = xopbase.ErrorDataType - } - } - return zero, false -} - -// NewBufferedReplayExportWrapper creates an export wrapper that can be used to -// improve the accuracy of data exported after replay. This comes into play -// when data is run from OTEL to XOP and then back to OTEL. When it comes back -// to OTEL, the export wrapper improves what a SpanExporter exports. -// -// Use the BufferedReplayExporterWrapper to both wrap SpanExporter and to -// create the BufferedReplayLogger that is used to export from XOP to OTEL. -func NewBufferedReplayExporterWrapper() BufferedReplayExporterWrapper { - return &exporterWrapper{} -} - -type exporterWrapper struct { - exporterCount int32 - augmentMap gwrap.SyncMap[[8]byte, *missingSpanData] -} - -func (ew *exporterWrapper) WrapExporter(exporter sdktrace.SpanExporter) sdktrace.SpanExporter { - _ = atomic.AddInt32(&ew.exporterCount, 1) - return &wrappedExporter{ - exporter: exporter, - wrapper: ew, - } -} - -func (ew *exporterWrapper) BufferedReplayLogger(tracerProviderOpts ...sdktrace.TracerProviderOption) xopbase.Logger { - return bufferedReplayLogger(ew, tracerProviderOpts) -} - -type wrappedExporter struct { - wrapper *exporterWrapper - exporter sdktrace.SpanExporter - shutdown int32 -} - -func (w *wrappedExporter) Shutdown(ctx context.Context) error { - if atomic.AddInt32(&w.shutdown, 1) == 1 { - atomic.AddInt32(&w.wrapper.exporterCount, -1) - } - return w.exporter.Shutdown(ctx) -} - -func (w wrappedExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error { - n := make([]sdktrace.ReadOnlySpan, len(spans)) - for i, span := range spans { - missingSpanData, ok := w.wrapper.augmentMap.Load([8]byte(span.SpanContext().SpanID())) - if ok { - if atomic.AddInt32(&missingSpanData.consumptionCount, 1) >= - atomic.LoadInt32(&w.wrapper.exporterCount) { - w.wrapper.augmentMap.Delete([8]byte(span.SpanContext().SpanID())) - } - n[i] = wrappedSpan{ - ReadOnlySpan: span, - missingSpanData: missingSpanData, - } - } else { - n[i] = span - } - } - return w.exporter.ExportSpans(ctx, n) -} - -// We delete the missingSpanData from the augmentMap when consumptionCount -// equals or exceeds the number of wrapped exporters. -type missingSpanData struct { - missingSpanDataComparable - droppedLinkAttributeCount map[[8]byte]int -} - -type missingSpanDataComparable struct { - spanCounters - consumptionCount int32 - scopeName string - error string // for when then round-trip fails -} - -type wrappedSpan struct { - sdktrace.ReadOnlySpan - missingSpanData *missingSpanData -} - -func (s wrappedSpan) InstrumentationScope() instrumentation.Scope { - scope := s.ReadOnlySpan.InstrumentationScope() - scope.Name = s.missingSpanData.scopeName - return scope -} - -func (s wrappedSpan) InstrumentationLibrary() instrumentation.Library { - library := s.ReadOnlySpan.InstrumentationLibrary() - library.Name = s.missingSpanData.scopeName - return library -} - -func (s wrappedSpan) DroppedAttributes() int { return s.missingSpanData.DroppedAttributes } -func (s wrappedSpan) DroppedLinks() int { return s.missingSpanData.DroppedLinks } -func (s wrappedSpan) DroppedEvents() int { return s.missingSpanData.DroppedEvents } -func (s wrappedSpan) ChildSpanCount() int { return s.missingSpanData.ChildSpanCount } - -func (s wrappedSpan) Links() []sdktrace.Link { - links := s.ReadOnlySpan.Links() - if s.missingSpanData != nil { - for i, link := range links { - if dropped, ok := s.missingSpanData.droppedLinkAttributeCount[([8]byte)(link.SpanContext.SpanID())]; ok { - links[i].DroppedAttributeCount = dropped - } - } - } - return links -} - -type bufferedResource struct { - *resource.Resource -} - -var _ json.Unmarshaler = &bufferedResource{} - -func (r *bufferedResource) UnmarshalJSON(b []byte) error { - var bufferedAttributes bufferedAttributes - err := json.Unmarshal(b, &bufferedAttributes) - if err != nil { - return err - } - r.Resource = resource.NewWithAttributes("", bufferedAttributes.attributes...) - return nil -} - -func (o *otelStuff) TracerOptions() []oteltrace.TracerOption { - if o == nil { - return nil - } - return []oteltrace.TracerOption{ - oteltrace.WithSchemaURL(o.InstrumentationScope.SchemaURL), - oteltrace.WithInstrumentationVersion(o.InstrumentationScope.Version), - } -} - -func (o *otelStuff) SpanOptions() []oteltrace.SpanStartOption { - if o == nil { - return nil - } - opts := []oteltrace.SpanStartOption{ - oteltrace.WithSpanKind(oteltrace.SpanKind(o.SpanKind)), - } - if len(o.links) != 0 { - opts = append(opts, oteltrace.WithLinks(o.links...)) - } - return opts -} - -func (o *otelStuff) Set(otelSpan oteltrace.Span) { - if o == nil { - return - } - otelSpan.SetStatus(o.Status.Code, o.Status.Description) -} - -func (o *otelStuff) TracerProviderOptions() []sdktrace.TracerProviderOption { - return []sdktrace.TracerProviderOption{ - sdktrace.WithResource(o.Resource.Resource), - } -} - -type bufferedAttributes struct { - attributes []attribute.KeyValue -} - -var _ json.Unmarshaler = &bufferedAttributes{} - -func (a *bufferedAttributes) UnmarshalJSON(b []byte) error { - var standIn []bufferedKeyValue - err := json.Unmarshal(b, &standIn) - if err != nil { - return err - } - a.attributes = make([]attribute.KeyValue, len(standIn)) - for i, si := range standIn { - a.attributes[i] = si.KeyValue - } - return nil -} - -type bufferedKeyValue struct { - attribute.KeyValue -} - -var _ json.Unmarshaler = &bufferedKeyValue{} - -func (a *bufferedKeyValue) UnmarshalJSON(b []byte) error { - var standIn struct { - Key string - Value struct { - Type string - Value any - } - } - err := json.Unmarshal(b, &standIn) - if err != nil { - return err - } - switch standIn.Value.Type { - case "BOOL": - if c, ok := standIn.Value.Value.(bool); ok { - a.KeyValue = attribute.Bool(standIn.Key, c) - } else { - var si2 struct { - Value struct { - Value bool - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.Bool(standIn.Key, si2.Value.Value) - } - case "BOOLSLICE": - var si2 struct { - Value struct { - Value []bool - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.BoolSlice(standIn.Key, si2.Value.Value) - // blank line required here - case "FLOAT64": - if c, ok := standIn.Value.Value.(float64); ok { - a.KeyValue = attribute.Float64(standIn.Key, c) - } else { - var si2 struct { - Value struct { - Value float64 - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.Float64(standIn.Key, si2.Value.Value) - } - case "FLOAT64SLICE": - var si2 struct { - Value struct { - Value []float64 - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.Float64Slice(standIn.Key, si2.Value.Value) - // blank line required here - case "INT64": - if c, ok := standIn.Value.Value.(int64); ok { - a.KeyValue = attribute.Int64(standIn.Key, c) - } else { - var si2 struct { - Value struct { - Value int64 - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.Int64(standIn.Key, si2.Value.Value) - } - case "INT64SLICE": - var si2 struct { - Value struct { - Value []int64 - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.Int64Slice(standIn.Key, si2.Value.Value) - // blank line required here - case "STRING": - if c, ok := standIn.Value.Value.(string); ok { - a.KeyValue = attribute.String(standIn.Key, c) - } else { - var si2 struct { - Value struct { - Value string - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.String(standIn.Key, si2.Value.Value) - } - case "STRINGSLICE": - var si2 struct { - Value struct { - Value []string - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.StringSlice(standIn.Key, si2.Value.Value) - // blank line required here - - default: - return fmt.Errorf("unknown attribute.KeyValue type '%s'", standIn.Value.Type) - } - return nil -} diff --git a/xopotel/buffered.zzzgo b/xopotel/buffered.zzzgo deleted file mode 100644 index 685e4c05..00000000 --- a/xopotel/buffered.zzzgo +++ /dev/null @@ -1,543 +0,0 @@ -// TEMPLATE-FILE -// TEMPLATE-FILE - -package xopotel - -import ( - "context" - "encoding/json" - "fmt" - "sync/atomic" - "time" - - "github.com/muir/gwrap" - "github.com/xoplog/xop-go/xopat" - "github.com/xoplog/xop-go/xopbase" - "github.com/xoplog/xop-go/xopbase/xopbaseutil" - "github.com/xoplog/xop-go/xoprecorder" - "github.com/xoplog/xop-go/xoptrace" - "github.com/xoplog/xop-go/xoputil/pointer" - - "github.com/google/uuid" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/sdk/instrumentation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" -) - -type BufferedReplayExporterWrapper interface { - // WrapExporter augments the data that the wrapped exporter recevies so that if - // the ReadOnlySpan came from a BufferedReplayLogger that is replaying data that - // originally came from OTEL, then it can fill data that otherwise has no way - // to be propagated through the OTEL APIs. For example, instrumentation.Scope.Name. - WrapExporter(sdktrace.SpanExporter) sdktrace.SpanExporter - - // BufferedReplayLogger creates a Logger that can be used when replaying from other - // xopbase.Logger implementations into xopotel. It buffers all the logged data until - // Done() is called on a per-request basis. Additional logging after Done() is discarded. - // - // A TracerProvider and Tracer are constructed for each Request and discarded afterwards. - // - // VERY IMPORTANT: every exporter wrapped with WrapExporter must be - // passed to BufferedReplayLogger. If not, memory will leak. - // - // Also import: if the exporter's ExportSpans() isn't called with all - // spans, memory will leak. The amount of leaked memory is not large, maybe - // 100 bytes per span, but it isn't zero. - BufferedReplayLogger(...sdktrace.TracerProviderOption) xopbase.Logger -} - -// BufferedReplayLogger creates a Logger that can be used when replaying from other -// xopbase.Logger implementations into xopotel. It buffers all the logged data until -// Done() is called on a per-request basis. Additional logging after Done() is discarded. -// -// A TracerProvider and Tracer are constructed for each Request and discarded afterwards. -// -// For improved fideltity of OTEL -> XOP -> OTEL replay, use -// BufferedReplayExporterWrapper.BufferedReplayLogger instead. -func BufferedReplayLogger(tracerProviderOpts ...sdktrace.TracerProviderOption) xopbase.Logger { - return bufferedReplayLogger(&exporterWrapper{}, tracerProviderOpts) -} - -func bufferedReplayLogger(exporterWrapper *exporterWrapper, tracerProviderOpts []sdktrace.TracerProviderOption) xopbase.Logger { - return &bufferedLogger{ - tracerProviderOpts: tracerProviderOpts, - id: "otelbuf-" + uuid.New().String(), - exporterWrapper: exporterWrapper, - } -} - -type bufferedLogger struct { - id string - tracerProviderOpts []sdktrace.TracerProviderOption - exporterWrapper *exporterWrapper -} - -func (logger *bufferedLogger) ID() string { return logger.id } -func (logger *bufferedLogger) ReferencesKept() bool { return true } -func (logger *bufferedLogger) Buffered() bool { return false } - -// bufferedRequest implements Request for a buferedLogger. The bufferedLogger is a wrapper -// wrapper around request{}. It waits until the request is complete before it invokes -// logger.Request and in the meantime buffers all data into an ephemeral xoprecorder.Logger. -// -// It uses xoprecorder's replay functionality to dump the buffered logs. -// -// It creates a logger{} for each request data can be passed directly from the bufferedRequest -// to the logger{} and from the logger{} to the request{} because there is only one request{} -// per logger{} in this situation. -type bufferedRequest struct { - xopbase.Request - recorder *xoprecorder.Logger - finalized bool - logger *bufferedLogger - ctx context.Context - bundle xoptrace.Bundle - errorReporter func(error) - isXOP bool // request originally came from XOP -} - -func (logger *bufferedLogger) Request(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, sourceInfo xopbase.SourceInfo) xopbase.Request { - recorder := xoprecorder.New() - return &bufferedRequest{ - recorder: recorder, - Request: recorder.Request(ctx, ts, bundle, description, sourceInfo), - logger: logger, - ctx: ctx, - bundle: bundle, - isXOP: sourceInfo.Source != otelDataSource, - } -} - -func (request *bufferedRequest) SetErrorReporter(f func(error)) { request.errorReporter = f } - -func (request *bufferedRequest) Done(endTime time.Time, final bool) { - if request.finalized { - return - } - request.Request.Done(endTime, final) - if !final { - return - } - request.finalized = true - tpOpts := []sdktrace.TracerProviderOption{ - sdktrace.WithSpanLimits(sdktrace.SpanLimits{ - AttributeValueLengthLimit: -1, - AttributeCountLimit: -1, - EventCountLimit: -1, - LinkCountLimit: -1, - AttributePerEventCountLimit: -1, - AttributePerLinkCountLimit: -1, - })} - tpOpts = append(tpOpts, request.logger.tracerProviderOpts...) - tpOpts = append(tpOpts, IDGenerator()) - - otelStuff := request.getStuff(request.bundle, false) - // we do not call augment() here because that would result in - // a duplicate call when it is also called from Reqeust() - tpOpts = append(tpOpts, otelStuff.TracerProviderOptions()...) - - tracerProvider := sdktrace.NewTracerProvider(tpOpts...) - defer tracerProvider.Shutdown(request.ctx) - - var tOpts []oteltrace.TracerOption - if request.isXOP { - tOpts = append(tOpts, - oteltrace.WithInstrumentationAttributes( - xopOTELVersion.String(xopotelVersionValue), - xopVersion.String(xopVersionValue), - ), - oteltrace.WithInstrumentationVersion(xopotelVersionValue), - ) - } - tOpts = append(tOpts, otelStuff.TracerOptions()...) - tracer := tracerProvider.Tracer("xopotel", tOpts...) - otel := &logger{ - id: "bufotel-" + uuid.New().String(), - doLogging: true, - tracer: tracer, - bufferedRequest: request, - } - request.recorder.Replay(request.ctx, otel) - err := tracerProvider.ForceFlush(request.ctx) - if err != nil && request.errorReporter != nil { - request.errorReporter(err) - } -} - -func (req *bufferedRequest) getStuff(bundle xoptrace.Bundle, augment bool) (stuff *otelStuff) { - if req == nil || req.recorder == nil { - return - } - spanID := bundle.Trace.SpanID().Array() - _ = req.recorder.WithLock(func(r *xoprecorder.Logger) error { - span, ok := r.SpanIndex[spanID] - if !ok { - return nil - } - var otelStuff otelStuff - stuff = &otelStuff - - addMetadataLink := func(key string, link xoptrace.Trace) { - otelStuff.links = append(otelStuff.links, oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: link.TraceID().Array(), - SpanID: link.SpanID().Array(), - TraceFlags: oteltrace.TraceFlags(link.Flags().Array()[0]), - TraceState: emptyTraceState, - Remote: true, - }), - Attributes: []attribute.KeyValue{ - xopLinkMetadataKey.String(key), - }, - }) - } - span.SpanMetadata.Map.Range(func(key string, mt *xopbaseutil.MetadataTracker) bool { - switch mt.Type { - case xopbase.LinkDataType: - addMetadataLink(key, mt.Value.(xoptrace.Trace)) - case xopbase.LinkArrayDataType: - for _, link := range mt.Value.([]interface{}) { - addMetadataLink(key, link.(xoptrace.Trace)) - } - } - return true - }) - - var missingData missingSpanData - - if len(span.Links) > 0 { - for _, linkLine := range span.Links { - plainBuilder := builder{ - attributes: make([]attribute.KeyValue, 0, len(linkLine.Data)), - } - linkLine = pointer.To(linkLine.Copy()) - - var ts oteltrace.TraceState - - tsRaw, ok := extractValue[string](linkLine, xopOTELLinkTranceState, xopbase.StringDataType, xopLinkTraceStateError) - if ok { - var err error - ts, err = oteltrace.ParseTraceState(tsRaw) - if err != nil { - linkLine.Data[xopLinkTraceStateError] = err.Error() - linkLine.DataType[xopLinkTraceStateError] = xopbase.ErrorDataType - } - } - - isRemote, _ := extractValue[bool](linkLine, xopOTELLinkIsRemote, xopbase.BoolDataType, xopLinkRemoteError) - - linkSpanID := linkLine.AsLink.SpanID().Array() - - if augment { - droppedCount, _ := extractValue[int64](linkLine, xopOTELLinkDroppedAttributeCount, xopbase.IntDataType, xopLinkeDroppedError) - if droppedCount != 0 { - if missingData.droppedLinkAttributeCount == nil { - missingData.droppedLinkAttributeCount = make(map[[8]byte]int) - } - missingData.droppedLinkAttributeCount[linkSpanID] = int(droppedCount) - } - } - - xoprecorder.ReplayLineData(linkLine, &plainBuilder) - - otelStuff.links = append(otelStuff.links, oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: linkLine.AsLink.TraceID().Array(), - SpanID: linkSpanID, - TraceFlags: oteltrace.TraceFlags(linkLine.AsLink.Flags().Array()[0]), - TraceState: ts, - Remote: isRemote, - }), - Attributes: plainBuilder.attributes, - }) - } - } - - if failure := func() string { - md := span.SpanMetadata.Get(otelReplayStuff.Key().String()) - if md == nil { - return "span metdata missing replay key expected for BufferedReplayLogger " + otelReplayStuff.Key().String() - } - ma, ok := md.Value.(xopbase.ModelArg) - if !ok { - return fmt.Sprintf("cast of %s data to ModelArg failed, is %T", otelReplayStuff.Key(), md.Value) - } - err := ma.DecodeTo(&otelStuff) - if err != nil { - return fmt.Sprintf("failed to decode encoded data in %s: %s", otelReplayStuff.Key(), err) - } - return "" - }(); failure != "" { - missingData.error = failure - } else { - missingData.spanCounters = otelStuff.spanCounters - missingData.scopeName = otelStuff.InstrumentationScope.Name - } - var zeroMissing missingSpanDataComparable - if augment && atomic.LoadInt32(&req.logger.exporterWrapper.exporterCount) > 0 && - (missingData.missingSpanDataComparable != zeroMissing || missingData.droppedLinkAttributeCount != nil) { - req.logger.exporterWrapper.augmentMap.Store(spanID, &missingData) - } - return nil - }) - return -} - -func extractValue[T any](linkLine *xoprecorder.Line, name xopat.K, dataType xopbase.DataType, errorKey xopat.K) (T, bool) { - var zero T - if raw, ok := linkLine.Data[name]; ok { - if linkLine.DataType[name] == dataType { - cooked, ok := raw.(T) - if ok { - delete(linkLine.Data, name) - delete(linkLine.DataType, name) - return cooked, true - } else { - linkLine.Data[errorKey] = fmt.Sprintf("%s should be a %T, but is a %T", - name, zero, raw) - linkLine.DataType[errorKey] = xopbase.ErrorDataType - } - } else { - linkLine.Data[errorKey] = fmt.Sprintf("%s should be a %s, but is a %s", - name, dataType, linkLine.DataType[name]) - linkLine.DataType[errorKey] = xopbase.ErrorDataType - } - } - return zero, false -} - -// NewBufferedReplayExportWrapper creates an export wrapper that can be used to -// improve the accuracy of data exported after replay. This comes into play -// when data is run from OTEL to XOP and then back to OTEL. When it comes back -// to OTEL, the export wrapper improves what a SpanExporter exports. -// -// Use the BufferedReplayExporterWrapper to both wrap SpanExporter and to -// create the BufferedReplayLogger that is used to export from XOP to OTEL. -func NewBufferedReplayExporterWrapper() BufferedReplayExporterWrapper { - return &exporterWrapper{} -} - -type exporterWrapper struct { - exporterCount int32 - augmentMap gwrap.SyncMap[[8]byte, *missingSpanData] -} - -func (ew *exporterWrapper) WrapExporter(exporter sdktrace.SpanExporter) sdktrace.SpanExporter { - _ = atomic.AddInt32(&ew.exporterCount, 1) - return &wrappedExporter{ - exporter: exporter, - wrapper: ew, - } -} - -func (ew *exporterWrapper) BufferedReplayLogger(tracerProviderOpts ...sdktrace.TracerProviderOption) xopbase.Logger { - return bufferedReplayLogger(ew, tracerProviderOpts) -} - -type wrappedExporter struct { - wrapper *exporterWrapper - exporter sdktrace.SpanExporter - shutdown int32 -} - -func (w *wrappedExporter) Shutdown(ctx context.Context) error { - if atomic.AddInt32(&w.shutdown, 1) == 1 { - atomic.AddInt32(&w.wrapper.exporterCount, -1) - } - return w.exporter.Shutdown(ctx) -} - -func (w wrappedExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error { - n := make([]sdktrace.ReadOnlySpan, len(spans)) - for i, span := range spans { - missingSpanData, ok := w.wrapper.augmentMap.Load([8]byte(span.SpanContext().SpanID())) - if ok { - if atomic.AddInt32(&missingSpanData.consumptionCount, 1) >= - atomic.LoadInt32(&w.wrapper.exporterCount) { - w.wrapper.augmentMap.Delete([8]byte(span.SpanContext().SpanID())) - } - n[i] = wrappedSpan{ - ReadOnlySpan: span, - missingSpanData: missingSpanData, - } - } else { - n[i] = span - } - } - return w.exporter.ExportSpans(ctx, n) -} - -// We delete the missingSpanData from the augmentMap when consumptionCount -// equals or exceeds the number of wrapped exporters. -type missingSpanData struct { - missingSpanDataComparable - droppedLinkAttributeCount map[[8]byte]int -} - -type missingSpanDataComparable struct { - spanCounters - consumptionCount int32 - scopeName string - error string // for when then round-trip fails -} - -type wrappedSpan struct { - sdktrace.ReadOnlySpan - missingSpanData *missingSpanData -} - -func (s wrappedSpan) InstrumentationScope() instrumentation.Scope { - scope := s.ReadOnlySpan.InstrumentationScope() - scope.Name = s.missingSpanData.scopeName - return scope -} - -func (s wrappedSpan) InstrumentationLibrary() instrumentation.Library { - library := s.ReadOnlySpan.InstrumentationLibrary() - library.Name = s.missingSpanData.scopeName - return library -} - -func (s wrappedSpan) DroppedAttributes() int { return s.missingSpanData.DroppedAttributes } -func (s wrappedSpan) DroppedLinks() int { return s.missingSpanData.DroppedLinks } -func (s wrappedSpan) DroppedEvents() int { return s.missingSpanData.DroppedEvents } -func (s wrappedSpan) ChildSpanCount() int { return s.missingSpanData.ChildSpanCount } - -func (s wrappedSpan) Links() []sdktrace.Link { - links := s.ReadOnlySpan.Links() - if s.missingSpanData != nil { - for i, link := range links { - if dropped, ok := s.missingSpanData.droppedLinkAttributeCount[([8]byte)(link.SpanContext.SpanID())]; ok { - links[i].DroppedAttributeCount = dropped - } - } - } - return links -} - -type bufferedResource struct { - *resource.Resource -} - -var _ json.Unmarshaler = &bufferedResource{} - -func (r *bufferedResource) UnmarshalJSON(b []byte) error { - var bufferedAttributes bufferedAttributes - err := json.Unmarshal(b, &bufferedAttributes) - if err != nil { - return err - } - r.Resource = resource.NewWithAttributes("", bufferedAttributes.attributes...) - return nil -} - -func (o *otelStuff) TracerOptions() []oteltrace.TracerOption { - if o == nil { - return nil - } - return []oteltrace.TracerOption{ - oteltrace.WithSchemaURL(o.InstrumentationScope.SchemaURL), - oteltrace.WithInstrumentationVersion(o.InstrumentationScope.Version), - } -} - -func (o *otelStuff) SpanOptions() []oteltrace.SpanStartOption { - if o == nil { - return nil - } - opts := []oteltrace.SpanStartOption{ - oteltrace.WithSpanKind(oteltrace.SpanKind(o.SpanKind)), - } - if len(o.links) != 0 { - opts = append(opts, oteltrace.WithLinks(o.links...)) - } - return opts -} - -func (o *otelStuff) Set(otelSpan oteltrace.Span) { - if o == nil { - return - } - otelSpan.SetStatus(o.Status.Code, o.Status.Description) -} - -func (o *otelStuff) TracerProviderOptions() []sdktrace.TracerProviderOption { - return []sdktrace.TracerProviderOption{ - sdktrace.WithResource(o.Resource.Resource), - } -} - -type bufferedAttributes struct { - attributes []attribute.KeyValue -} - -var _ json.Unmarshaler = &bufferedAttributes{} - -func (a *bufferedAttributes) UnmarshalJSON(b []byte) error { - var standIn []bufferedKeyValue - err := json.Unmarshal(b, &standIn) - if err != nil { - return err - } - a.attributes = make([]attribute.KeyValue, len(standIn)) - for i, si := range standIn { - a.attributes[i] = si.KeyValue - } - return nil -} - -type bufferedKeyValue struct { - attribute.KeyValue -} - -var _ json.Unmarshaler = &bufferedKeyValue{} - -func (a *bufferedKeyValue) UnmarshalJSON(b []byte) error { - var standIn struct { - Key string - Value struct { - Type string - Value any - } - } - err := json.Unmarshal(b, &standIn) - if err != nil { - return err - } - switch standIn.Value.Type { - //MACRO OTELTypes - case "ZZZ": - if c, ok := standIn.Value.Value.(zzz); ok { - a.KeyValue = attribute.Zzz(standIn.Key, c) - } else { - var si2 struct { - Value struct { - Value zzz - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.Zzz(standIn.Key, si2.Value.Value) - } - case "ZZZSLICE": - var si2 struct { - Value struct { - Value []zzz - } - } - err := json.Unmarshal(b, &si2) - if err != nil { - return err - } - a.KeyValue = attribute.ZzzSlice(standIn.Key, si2.Value.Value) - // blank line required here - - default: - return fmt.Errorf("unknown attribute.KeyValue type '%s'", standIn.Value.Type) - } - return nil -} diff --git a/xopotel/doc.go b/xopotel/doc.go deleted file mode 100644 index bd9eba56..00000000 --- a/xopotel/doc.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Package xopotel provides a gateways between XOP and Open Telemetry. - -There is a mismatch of features between Open Telemetry and XOP. Open Telemetry -supports only a very limited set of attribute types. When gatewaying from -XOP into Open Telemetry the richer set of types are almost always converted -to string slices. - -There are several integration points. - -# BaseLogger - -The BaseLogger() function returns a xopbase.Logger that can be used like -any other base logger to configure XOP output. In this case, the XOP logs -and traces will be output through the Open Telemtry system using the -primary interfaces of TracerProvider, Tracer, Span, etc. There is a -restriction though: to use this you MUST create the TracerProvider with -the xopotel IDGenerator: - - import ( - "github.com/xoplog/xop-go/xopotel" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - ) - - tracerProvider := NewTraceProvider(xopotel.IDGenerator(), sdktrace.WithBatcher(...)) - -This allows the TraceIDs and SpanIDs created by XOP to be used by -Open Telemetry. - -# SeedModifier - -If for some reason, you do not have control over the creation of your TracerProvider, -you can use SeedModifer() modify your xop.Seed so that it delgates SpanID and TraceID -creation to Open Telemetry. - -# SpanToLog - -If you don't have access to a TracerProvider at all and instead have -a "go.opentelemetry.io/otel/trace".Span, you can use that as the basis for generating -logs with XOP by converting it directly to a *xop.Logger. - -# BufferedReplayLogger - -BufferedReplayLogger creates a fresh TracerProvider and Tracer for each XOP Request. -It offeres the higher quality translation from XOP into OTEL but at a cost: all data -relating to each Request is fully buffered in memory before the TracerProvider and -Tracer are crated. There is no output until the Request is complete. - -BufferedReplayLogger is meant for the situation where another xopbase.Logger is being -replayed into xopotel. It is also the only way to losslessly round trip OTEL logs to -XOP and then back to OTEL. - -# BufferedReplayExporterWrapper - -BufferedReplayExporterWrapper augments BufferedReplayLogger by passing information -around the OTEL TracerProvider, Tracer, and Span. When not using it, Scope.Name, -and all the counters that ReadOnlySpan provides are lost. - -# ExportToXOP - -Integration can go the other direction. You can flow traces from Open Telemetry to -XOP base loggers. Use ExportToXOP() to wrap a xopbase.Logger so that it can be used -as a SpanExporter. - -# Limitations - -Ideally, it should be possible to run data in a round trip from XOP to OTEL back to XOP -and have it unchanged and also run data from OTEL to XOP and back to OTEL and -have it unchanged. - -The former (XOP -> OTEL -> XOP) works. Unfortunately, OTEL -> XOP -> OTEL is -difficult to get working and only fully works when using the BufferedReplayLogger -and the BufferedReplayExporterWrapper. This complexity could be avoided if -it were possible for others to implement ReadOnlySpan. -*/ -package xopotel diff --git a/xopotel/export.go b/xopotel/export.go deleted file mode 100644 index 0d39f842..00000000 --- a/xopotel/export.go +++ /dev/null @@ -1,1454 +0,0 @@ -// This file is generated, DO NOT EDIT. It comes from the corresponding .zzzgo file - -package xopotel - -import ( - "context" - "encoding/json" - "fmt" - "regexp" - "runtime" - "sort" - "strconv" - "strings" - "sync/atomic" - "time" - - "github.com/xoplog/xop-go/xopat" - "github.com/xoplog/xop-go/xopbase" - "github.com/xoplog/xop-go/xopconst" - "github.com/xoplog/xop-go/xopnum" - "github.com/xoplog/xop-go/xopproto" - "github.com/xoplog/xop-go/xoptrace" - "github.com/xoplog/xop-go/xoputil/xopversion" - - "github.com/muir/gwrap" - "github.com/muir/list" - "github.com/pkg/errors" - "go.opentelemetry.io/otel/attribute" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" -) - -var ( - _ sdktrace.SpanExporter = &spanExporter{} - _ sdktrace.SpanExporter = &unhack{} -) - -var ErrShutdown = fmt.Errorf("Shutdown called") - -type spanExporter struct { - base xopbase.Logger - orderedFinish []orderedFinish - sequenceNumber int32 - done int32 -} - -type spanReplay struct { - *spanExporter - id2Index map[oteltrace.SpanID]int - spans []sdktrace.ReadOnlySpan - subSpans [][]int - data []*datum -} - -type datum struct { - baseSpan xopbase.Span - requestIndex int // index of request ancestor - attributeDefinitions map[string]*decodeAttributeDefinition - xopSpan bool - registry *xopat.Registry -} - -func (x *spanExporter) addOrdered(seq int32, f func()) { - x.orderedFinish = append(x.orderedFinish, orderedFinish{ - num: seq, - f: f, - }) -} - -type orderedFinish struct { - num int32 - f func() -} - -type baseSpanReplay struct { - spanReplay - *datum - span sdktrace.ReadOnlySpan -} - -type decodeAttributeDefinition struct { - xopat.Make - AttributeType xopproto.AttributeType `json:"vtype"` -} - -type wrappedReadOnlySpan struct { - sdktrace.ReadOnlySpan - links []sdktrace.Link -} - -// ExportToXOP allows open telementry spans to be exported through -// a xopbase.Logger. If the open telementry spans were generated -// originally using xoputil, then the exported data should almost -// exactly match the original inputs. -func ExportToXOP(base xopbase.Logger) sdktrace.SpanExporter { - return &spanExporter{ - base: base, - } -} - -func (e *spanExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) (err error) { - // TODO: avoid returning error when possible - id2Index := makeIndex(spans) - subSpans, todo := makeSubspans(id2Index, spans) - x := spanReplay{ - spanExporter: e, - id2Index: id2Index, - spans: spans, - subSpans: subSpans, - data: make([]*datum, len(spans)), - } - - var toFinish []func() - - var processSpan func(int) error - processSpan = func(i int) error { - x.data[i] = &datum{} - finisher, err := x.Replay(ctx, spans[i], x.data[i], i) - if err != nil { - return err - } - for _, subSpan := range subSpans[i] { - err := processSpan(subSpan) - if err != nil { - return err - } - } - toFinish = append(toFinish, finisher) - return nil - } - for _, i := range todo { - err := processSpan(i) - if err != nil { - return err - } - - sort.Slice(e.orderedFinish, func(i, j int) bool { - return e.orderedFinish[i].num < e.orderedFinish[j].num - }) - for _, o := range x.orderedFinish { - o.f() - } - x.orderedFinish = x.orderedFinish[:0] - - for _, finisher := range toFinish { - finisher() - } - toFinish = toFinish[:0] - } - return nil -} - -func (x spanReplay) Replay(ctx context.Context, span sdktrace.ReadOnlySpan, data *datum, myIndex int) (func(), error) { - var bundle xoptrace.Bundle - spanContext := span.SpanContext() - if spanContext.HasTraceID() { - bundle.Trace.TraceID().SetArray(spanContext.TraceID()) - } - if spanContext.HasSpanID() { - bundle.Trace.SpanID().SetArray(spanContext.SpanID()) - } - if spanContext.IsSampled() { - bundle.Trace.Flags().SetArray([1]byte{1}) - } - if spanContext.TraceState().Len() != 0 { - bundle.State.SetString(spanContext.TraceState().String()) - } - parentIndex, hasParent := lookupParent(x.id2Index, span) - var xopParent *datum - if hasParent { - parentContext := x.spans[parentIndex].SpanContext() - xopParent = x.data[parentIndex] - if parentContext.HasTraceID() { - bundle.Parent.TraceID().SetArray(parentContext.TraceID()) - if bundle.Trace.TraceID().IsZero() { - bundle.Trace.TraceID().Set(bundle.Parent.GetTraceID()) - } - } - if parentContext.HasSpanID() { - bundle.Parent.SpanID().SetArray(parentContext.SpanID()) - } - if parentContext.IsSampled() { - bundle.Parent.Flags().SetArray([1]byte{1}) - } - } else if span.Parent().HasTraceID() { - bundle.Parent.TraceID().SetArray(span.Parent().TraceID()) - if span.Parent().HasSpanID() { - bundle.Parent.SpanID().SetArray(span.Parent().SpanID()) - } - if span.Parent().IsSampled() { - bundle.Parent.Flags().SetArray([1]byte{1}) - } - } - - var downStreamError gwrap.AtomicValue[error] - errorReporter := func(err error) { - if err != nil { - downStreamError.Store(err) - } - } - bundle.Parent.Flags().SetBytes([]byte{1}) - bundle.Trace.Flags().SetBytes([]byte{1}) - spanKind := span.SpanKind() - attributeMap := mapAttributes(span.Attributes()) - if b := attributeMap.GetString(xopBaggage); b != "" { - bundle.Baggage.SetString(b) - } - if spanKind == oteltrace.SpanKindUnspecified { - spanKind = oteltrace.SpanKind(defaulted(attributeMap.GetInt(otelSpanKind), int64(oteltrace.SpanKindUnspecified))) - } - if attributeMap.GetBool(spanIsLinkEventKey) { - // span is extra just for link - return func() {}, nil - } - switch spanKind { - case oteltrace.SpanKindUnspecified, oteltrace.SpanKindInternal: - if hasParent { - spanSeq := defaulted(attributeMap.GetString(xopSpanSequence), "") - data.xopSpan = xopParent.xopSpan - data.baseSpan = xopParent.baseSpan.Span(ctx, span.StartTime(), bundle, span.Name(), spanSeq) - data.requestIndex = xopParent.requestIndex - data.attributeDefinitions = xopParent.attributeDefinitions - data.registry = xopParent.registry - } else { - // This is a difficult sitatuion. We have an internal/unspecified span - // that does not have a parent present. There is no right answer for what - // to do. In the Xop world, such a span isn't allowed to exist. We'll treat - // this span as a request, but mark it as promoted. - data.xopSpan = attributeMap.GetString(xopVersion) != "" - baseRequest := x.base.Request(ctx, span.StartTime(), bundle, span.Name(), buildSourceInfo(span, attributeMap)) - baseRequest.SetErrorReporter(errorReporter) - data.baseSpan = baseRequest - data.baseSpan.MetadataBool(xopPromotedMetadata, true) - data.requestIndex = myIndex - data.attributeDefinitions = make(map[string]*decodeAttributeDefinition) - data.registry = xopat.NewRegistry(false) - } - default: - baseRequest := x.base.Request(ctx, span.StartTime(), bundle, span.Name(), buildSourceInfo(span, attributeMap)) - baseRequest.SetErrorReporter(errorReporter) - data.baseSpan = baseRequest - data.requestIndex = myIndex - data.attributeDefinitions = make(map[string]*decodeAttributeDefinition) - data.xopSpan = attributeMap.GetString(xopVersion) != "" - data.registry = xopat.NewRegistry(false) - if !data.xopSpan { - data.baseSpan.MetadataAny(otelReplayStuff, xopbase.ModelArg{ - Model: &otelStuff{ - SpanKind: xopconst.SpanKindEnum(span.SpanKind()), - Status: span.Status(), - Resource: bufferedResource{span.Resource()}, - InstrumentationScope: span.InstrumentationScope(), - spanCounters: spanCounters{ - DroppedAttributes: span.DroppedAttributes(), - DroppedLinks: span.DroppedLinks(), - DroppedEvents: span.DroppedEvents(), - ChildSpanCount: span.ChildSpanCount(), - }, - }, - }) - } - } - y := baseSpanReplay{ - spanReplay: x, - span: span, - datum: data, - } - for _, attribute := range span.Attributes() { - err := y.AddSpanAttribute(ctx, attribute) - if err != nil { - return func() {}, err - } - } - var maxNumber int32 - for _, event := range span.Events() { - lastNumber, err := y.AddEvent(ctx, event) - if err != nil { - return func() {}, err - } - if lastNumber > maxNumber { - maxNumber = lastNumber - } - } - for _, link := range span.Links() { - if !data.xopSpan { - z := lineAttributesReplay{ - baseSpanReplay: y, - lineType: lineTypeLink, - lineFormat: lineFormatDefault, - level: xopnum.InfoLevel, - } - line, err := z.AddLineAttributes(ctx, "link", span.StartTime(), link.Attributes) - if err != nil { - return func() {}, err - } - var trace xoptrace.Trace - trace.Flags().SetArray([1]byte{byte(link.SpanContext.TraceFlags())}) - trace.TraceID().SetArray(link.SpanContext.TraceID()) - trace.SpanID().SetArray(link.SpanContext.SpanID()) - data.baseSpan.MetadataLink(otelLink, trace) - z.link = trace - if ts := link.SpanContext.TraceState(); ts.Len() != 0 { - line.String(xopOTELLinkTranceState, ts.String(), xopbase.StringDataType) - } - if link.SpanContext.IsRemote() { - line.Bool(xopOTELLinkIsRemote, true) - } - if link.DroppedAttributeCount != 0 { - line.Int64(xopOTELLinkDroppedAttributeCount, int64(link.DroppedAttributeCount), xopbase.IntDataType) - } - - err = z.finishLine(ctx, "link", xopOTELLinkDetail.String(), line) - if err != nil { - return func() {}, err - } - } - } - if endTime := span.EndTime(); !endTime.IsZero() { - return func() { - data.baseSpan.Done(endTime, true) - }, nil - } - return func() {}, downStreamError.Load() -} - -type lineType int - -const ( - lineTypeLine lineType = iota - lineTypeLink - lineTypeLinkEvent - lineTypeModel -) - -type lineFormat int - -const ( - lineFormatDefault lineFormat = iota - lineFormatTemplate -) - -var lineRE = regexp.MustCompile(`^(.+):(\d+)$`) - -func (x baseSpanReplay) AddEvent(ctx context.Context, event sdktrace.Event) (int32, error) { - z := lineAttributesReplay{ - baseSpanReplay: x, - lineType: lineTypeLine, - lineFormat: lineFormatDefault, - level: xopnum.InfoLevel, - } - line, err := z.AddLineAttributes(ctx, "event", event.Time, event.Attributes) - if err != nil { - return 0, err - } - err = z.finishLine(ctx, "event", event.Name, line) - return z.lineNumber, err -} - -type lineAttributesReplay struct { - baseSpanReplay - lineType lineType - lineFormat lineFormat - template string - link xoptrace.Trace - modelArg xopbase.ModelArg - frames []runtime.Frame - lineNumber int32 - level xopnum.Level -} - -func (x *lineAttributesReplay) AddLineAttributes(ctx context.Context, what string, ts time.Time, attributes []attribute.KeyValue) (xopbase.Line, error) { - x.sequenceNumber++ - x.lineNumber = x.sequenceNumber - nonSpecial := make([]attribute.KeyValue, 0, len(attributes)) - for _, a := range attributes { - switch a.Key { - case xopLineNumber: - if a.Value.Type() == attribute.INT64 { - x.lineNumber = int32(a.Value.AsInt64()) - } else { - return nil, errors.Errorf("invalid line number attribute type %s", a.Value.Type()) - } - case xopLevel: - if a.Value.Type() == attribute.STRING { - var err error - x.level, err = xopnum.LevelString(a.Value.AsString()) - if err != nil { - x.level = xopnum.InfoLevel - } - } else { - return nil, errors.Errorf("invalid line level attribute type %s", a.Value.Type()) - } - case xopType: - if a.Value.Type() == attribute.STRING { - switch a.Value.AsString() { - case "link": - x.lineType = lineTypeLink - case "link-event": - x.lineType = lineTypeLinkEvent - case "model": - x.lineType = lineTypeModel - case "line": - // defaulted - default: - return nil, errors.Errorf("invalid line type attribute value %s", a.Value.AsString()) - } - } else { - return nil, errors.Errorf("invalid line type attribute type %s", a.Value.Type()) - } - case xopModelType: - if a.Value.Type() == attribute.STRING { - x.modelArg.ModelType = a.Value.AsString() - } else { - return nil, errors.Errorf("invalid model type attribute type %s", a.Value.Type()) - } - case xopEncoding: - if a.Value.Type() == attribute.STRING { - e, ok := xopproto.Encoding_value[a.Value.AsString()] - if !ok { - return nil, errors.Errorf("invalid model encoding '%s'", a.Value.AsString()) - } - x.modelArg.Encoding = xopproto.Encoding(e) - } else { - return nil, errors.Errorf("invalid model encoding attribute type %s", a.Value.Type()) - } - case xopModel: - if a.Value.Type() == attribute.STRING { - x.modelArg.Encoded = []byte(a.Value.AsString()) - } else { - return nil, errors.Errorf("invalid model encoding attribute type %s", a.Value.Type()) - } - case xopTemplate: - if a.Value.Type() == attribute.STRING { - x.lineFormat = lineFormatTemplate - x.template = a.Value.AsString() - } else { - return nil, errors.Errorf("invalid line template attribute type %s", a.Value.Type()) - } - case xopLinkData: - if a.Value.Type() == attribute.STRING { - var ok bool - x.link, ok = xoptrace.TraceFromString(a.Value.AsString()) - if !ok { - return nil, errors.Errorf("invalid link data attribute value %s", a.Value.AsString()) - } - } else { - return nil, errors.Errorf("invalid link data attribute type %s", a.Value.Type()) - } - case xopStackTrace: - if a.Value.Type() == attribute.STRINGSLICE { - raw := a.Value.AsStringSlice() - x.frames = make([]runtime.Frame, len(raw)) - for i, s := range raw { - m := lineRE.FindStringSubmatch(s) - if m == nil { - return nil, errors.Errorf("could not match stack line '%s'", s) - } - x.frames[i].File = m[1] - num, _ := strconv.ParseInt(m[2], 10, 64) - x.frames[i].Line = int(num) - } - } else { - return nil, errors.Errorf("invalid stack trace attribute type %s", a.Value.Type()) - } - default: - nonSpecial = append(nonSpecial, a) - } - } - line := x.baseSpan.NoPrefill().Line( - x.level, - ts, - x.frames, - ) - for _, a := range nonSpecial { - if x.xopSpan { - err := x.AddXopEventAttribute(ctx, a, line) - if err != nil { - return nil, errors.Wrapf(err, "add xop %s attribute %s", what, string(a.Key)) - } - } else { - err := x.AddEventAttribute(ctx, a, line) - if err != nil { - return nil, errors.Wrapf(err, "add %s attribute %s with type %s", what, string(a.Key), a.Value.Type()) - } - } - } - return line, nil -} - -func (x lineAttributesReplay) finishLine(ctx context.Context, what string, name string, line xopbase.Line) error { - switch x.lineType { - case lineTypeLine: - switch x.lineFormat { - case lineFormatDefault: - x.addOrdered(x.lineNumber, func() { - line.Msg(name) - }) - case lineFormatTemplate: - x.addOrdered(x.lineNumber, func() { - line.Template(x.template) - }) - default: - return errors.Errorf("unexpected lineType %d", x.lineType) - } - case lineTypeLink: - x.addOrdered(x.lineNumber, func() { - line.Link(name, x.link) - }) - case lineTypeLinkEvent: - return errors.Errorf("unexpected lineType: link event") - case lineTypeModel: - x.addOrdered(x.lineNumber, func() { - line.Model(name, x.modelArg) - }) - default: - return errors.Errorf("unexpected lineType %d", x.lineType) - } - return nil -} - -func (e *spanExporter) Shutdown(ctx context.Context) error { - atomic.StoreInt32(&e.done, 1) - return nil -} - -type unhack struct { - next sdktrace.SpanExporter -} - -// NewUnhacker wraps a SpanExporter and if the input is from BaseLogger or SpanLog, -// then it "fixes" the data hack in the output that puts inter-span links in sub-spans -// rather than in the span that defined them. -func NewUnhacker(exporter sdktrace.SpanExporter) sdktrace.SpanExporter { - return &unhack{next: exporter} -} - -func (u *unhack) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error { - // TODO: fix up SpanKind if spanKind is one of the attributes - id2Index := makeIndex(spans) - subLinks := make([][]sdktrace.Link, len(spans)) - for i, span := range spans { - parentIndex, ok := lookupParent(id2Index, span) - if !ok { - continue - } - var addToParent bool - for _, attribute := range span.Attributes() { - switch attribute.Key { - case spanIsLinkAttributeKey, spanIsLinkEventKey: - spans[i] = nil - addToParent = true - } - } - if !addToParent { - continue - } - subLinks[parentIndex] = append(subLinks[parentIndex], span.Links()...) - } - n := make([]sdktrace.ReadOnlySpan, 0, len(spans)) - for i, span := range spans { - span := span - switch { - case len(subLinks[i]) > 0: - n = append(n, wrappedReadOnlySpan{ - ReadOnlySpan: span, - links: append(list.Copy(span.Links()), subLinks[i]...), - }) - case span == nil: - // skip - default: - n = append(n, span) - } - } - return u.next.ExportSpans(ctx, n) -} - -func (u *unhack) Shutdown(ctx context.Context) error { - return u.next.Shutdown(ctx) -} - -var _ sdktrace.ReadOnlySpan = wrappedReadOnlySpan{} - -func (w wrappedReadOnlySpan) Links() []sdktrace.Link { - return w.links -} - -func makeIndex(spans []sdktrace.ReadOnlySpan) map[oteltrace.SpanID]int { - id2Index := make(map[oteltrace.SpanID]int) - for i, span := range spans { - spanContext := span.SpanContext() - if spanContext.HasSpanID() { - id2Index[spanContext.SpanID()] = i - } - } - return id2Index -} - -func lookupParent(id2Index map[oteltrace.SpanID]int, span sdktrace.ReadOnlySpan) (int, bool) { - parent := span.Parent() - if !parent.HasSpanID() { - return 0, false - } - parentIndex, ok := id2Index[parent.SpanID()] - if !ok { - return 0, false - } - return parentIndex, true -} - -// makeSubspans figures out what subspans each span has and also which spans -// have no parent span (and thus are not a subspan). We are assuming that there -// are no cycles in the graph of spans & subspans. UNSAFE -func makeSubspans(id2Index map[oteltrace.SpanID]int, spans []sdktrace.ReadOnlySpan) ([][]int, []int) { - ss := make([][]int, len(spans)) - noParent := make([]int, 0, len(spans)) - for i, span := range spans { - parentIndex, ok := lookupParent(id2Index, span) - if !ok { - noParent = append(noParent, i) - continue - } - ss[parentIndex] = append(ss[parentIndex], i) - } - return ss, noParent -} - -func buildSourceInfo(span sdktrace.ReadOnlySpan, attributeMap aMap) xopbase.SourceInfo { - var si xopbase.SourceInfo - var source string - var namespace string - if attributeMap.GetString(xopVersion) == "" { - // span did not come from XOP - source = otelDataSource - namespace = span.SpanKind().String() - } else { - if s := attributeMap.GetString(xopSource); s != "" { - source = s - } else if n := span.InstrumentationScope().Name; n != "" { - if v := span.InstrumentationScope().Version; v != "" { - source = n + " " + v - } else { - source = n - } - } else { - source = "OTEL" - } - namespace = defaulted(attributeMap.GetString(xopNamespace), source) - } - si.Source, si.SourceVersion = xopversion.SplitVersion(source) - si.Namespace, si.NamespaceVersion = xopversion.SplitVersion(namespace) - return si -} - -type aMap struct { - strings map[attribute.Key]string - ints map[attribute.Key]int64 - bools map[attribute.Key]bool -} - -func mapAttributes(list []attribute.KeyValue) aMap { - m := aMap{ - strings: make(map[attribute.Key]string), - ints: make(map[attribute.Key]int64), - bools: make(map[attribute.Key]bool), - } - for _, a := range list { - switch a.Value.Type() { - case attribute.STRING: - m.strings[a.Key] = a.Value.AsString() - case attribute.INT64: - m.ints[a.Key] = a.Value.AsInt64() - case attribute.BOOL: - m.bools[a.Key] = a.Value.AsBool() - } - } - return m -} - -func (m aMap) GetString(k attribute.Key) string { return m.strings[k] } -func (m aMap) GetInt(k attribute.Key) int64 { return m.ints[k] } -func (m aMap) GetBool(k attribute.Key) bool { return m.bools[k] } - -func defaulted[T comparable](a, b T) T { - var zero T - if a == zero { - return b - } - return a -} - -func (x baseSpanReplay) AddXopEventAttribute(ctx context.Context, a attribute.KeyValue, line xopbase.Line) error { - switch a.Value.Type() { - case attribute.STRINGSLICE: - slice := a.Value.AsStringSlice() - if len(slice) < 2 { - return errors.Errorf("invalid xop attribute encoding slice is too short") - } - switch slice[1] { - case "any": - if len(slice) != 4 { - return errors.Errorf("key %s invalid any encoding, slice too short", a.Key) - } - var ma xopbase.ModelArg - ma.Encoded = []byte(slice[0]) - e, ok := xopproto.Encoding_value[slice[2]] - if !ok { - return errors.Errorf("invalid model encoding '%s'", a.Value.AsString()) - } - ma.Encoding = xopproto.Encoding(e) - ma.ModelType = slice[3] - line.Any(xopat.K(a.Key), ma) - case "bool": - case "dur": - dur, err := time.ParseDuration(slice[0]) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Duration(xopat.K(a.Key), dur) - case "enum": - if len(slice) != 3 { - return errors.Errorf("key %s invalid enum encoding, slice too short", a.Key) - } - ea, err := x.registry.ConstructEnumAttribute(xopat.Make{ - Key: string(a.Key), - }, xopat.AttributeTypeEnum) - if err != nil { - return errors.Errorf("could not turn key %s into an enum", a.Key) - } - i, err := strconv.ParseInt(slice[2], 10, 64) - if err != nil { - return errors.Wrapf(err, "could not turn key %s into an enum", a.Key) - } - enum := ea.Add64(i, slice[0]) - line.Enum(&ea.EnumAttribute, enum) - case "error": - line.String(xopat.K(a.Key), slice[0], xopbase.StringToDataType["error"]) - case "f32": - f, err := strconv.ParseFloat(slice[0], 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Float64(xopat.K(a.Key), f, xopbase.StringToDataType["f32"]) - case "f64": - f, err := strconv.ParseFloat(slice[0], 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Float64(xopat.K(a.Key), f, xopbase.StringToDataType["f64"]) - case "i": - i, err := strconv.ParseInt(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Int64(xopat.K(a.Key), i, xopbase.StringToDataType["i"]) - case "i16": - i, err := strconv.ParseInt(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Int64(xopat.K(a.Key), i, xopbase.StringToDataType["i16"]) - case "i32": - i, err := strconv.ParseInt(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Int64(xopat.K(a.Key), i, xopbase.StringToDataType["i32"]) - case "i64": - i, err := strconv.ParseInt(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Int64(xopat.K(a.Key), i, xopbase.StringToDataType["i64"]) - case "i8": - i, err := strconv.ParseInt(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Int64(xopat.K(a.Key), i, xopbase.StringToDataType["i8"]) - case "s": - line.String(xopat.K(a.Key), slice[0], xopbase.StringToDataType["s"]) - case "stringer": - line.String(xopat.K(a.Key), slice[0], xopbase.StringToDataType["stringer"]) - case "time": - ts, err := time.Parse(time.RFC3339Nano, slice[0]) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Time(xopat.K(a.Key), ts) - case "u": - i, err := strconv.ParseUint(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Uint64(xopat.K(a.Key), i, xopbase.StringToDataType["u"]) - case "u16": - i, err := strconv.ParseUint(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Uint64(xopat.K(a.Key), i, xopbase.StringToDataType["u16"]) - case "u32": - i, err := strconv.ParseUint(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Uint64(xopat.K(a.Key), i, xopbase.StringToDataType["u32"]) - case "u64": - i, err := strconv.ParseUint(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Uint64(xopat.K(a.Key), i, xopbase.StringToDataType["u64"]) - case "u8": - i, err := strconv.ParseUint(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Uint64(xopat.K(a.Key), i, xopbase.StringToDataType["u8"]) - case "uintptr": - i, err := strconv.ParseUint(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.Uint64(xopat.K(a.Key), i, xopbase.StringToDataType["uintptr"]) - - } - - case attribute.BOOL: - line.Bool(xopat.K(a.Key), a.Value.AsBool()) - default: - return errors.Errorf("unexpected event attribute type %s for xop-encoded line", a.Value.Type()) - } - return nil -} - -func (x baseSpanReplay) AddEventAttribute(ctx context.Context, a attribute.KeyValue, line xopbase.Line) error { - switch a.Value.Type() { - case attribute.BOOL: - line.Bool(xopat.K(a.Key), a.Value.AsBool()) - case attribute.BOOLSLICE: - var ma xopbase.ModelArg - ma.Model = a.Value.AsBoolSlice() - ma.ModelType = toTypeSliceName["BOOL"] - line.Any(xopat.K(a.Key), ma) - case attribute.FLOAT64: - line.Float64(xopat.K(a.Key), a.Value.AsFloat64(), xopbase.Float64DataType) - case attribute.FLOAT64SLICE: - var ma xopbase.ModelArg - ma.Model = a.Value.AsFloat64Slice() - ma.ModelType = toTypeSliceName["FLOAT64"] - line.Any(xopat.K(a.Key), ma) - case attribute.INT64: - line.Int64(xopat.K(a.Key), a.Value.AsInt64(), xopbase.Int64DataType) - case attribute.INT64SLICE: - var ma xopbase.ModelArg - ma.Model = a.Value.AsInt64Slice() - ma.ModelType = toTypeSliceName["INT64"] - line.Any(xopat.K(a.Key), ma) - case attribute.STRING: - line.String(xopat.K(a.Key), a.Value.AsString(), xopbase.StringDataType) - case attribute.STRINGSLICE: - var ma xopbase.ModelArg - ma.Model = a.Value.AsStringSlice() - ma.ModelType = toTypeSliceName["STRING"] - line.Any(xopat.K(a.Key), ma) - - case attribute.INVALID: - fallthrough - default: - return errors.Errorf("invalid type") - } - return nil -} - -var toTypeSliceName = map[string]string{ - "BOOL": "[]bool", - "STRING": "[]string", - "INT64": "[]int64", - "FLOAT64": "[]float64", -} - -func (x baseSpanReplay) AddSpanAttribute(ctx context.Context, a attribute.KeyValue) (err error) { - switch a.Key { - case spanIsLinkAttributeKey, - spanIsLinkEventKey, - xopSource, - xopNamespace, - xopBaggage, - xopSpanSequence, - xopType, - otelSpanKind: - // special cases handled elsewhere - return nil - case xopVersion, - xopOTELVersion: - // dropped - return nil - } - key := string(a.Key) - defer func() { - if err != nil { - err = errors.Wrapf(err, "add span attribute %s with type %s", key, a.Value.Type()) - } - }() - if strings.HasPrefix(key, attributeDefinitionPrefix) { - key := strings.TrimPrefix(key, attributeDefinitionPrefix) - if _, ok := x.data[x.requestIndex].attributeDefinitions[key]; ok { - return nil - } - if a.Value.Type() != attribute.STRING { - return errors.Errorf("expected type to be string") - } - var aDef decodeAttributeDefinition - err := json.Unmarshal([]byte(a.Value.AsString()), &aDef) - if err != nil { - return errors.Wrapf(err, "could not unmarshal attribute defintion") - } - x.data[x.requestIndex].attributeDefinitions[key] = &aDef - return nil - } - - if aDef, ok := x.data[x.requestIndex].attributeDefinitions[key]; ok { - return x.AddXopMetadataAttribute(ctx, a, aDef) - } - if x.xopSpan { - return errors.Errorf("missing attribute defintion for key %s in xop span", key) - } - - mkMake := func(key string, multiple bool) xopat.Make { - return xopat.Make{ - Description: xopSynthesizedForOTEL, - Key: key, - Multiple: multiple, - } - } - switch a.Value.Type() { - case attribute.BOOL: - registeredAttribute, err := x.registry.ConstructBoolAttribute(mkMake(key, false), xopat.AttributeTypeBool) - if err != nil { - return err - } - x.baseSpan.MetadataBool(registeredAttribute, a.Value.AsBool()) - case attribute.BOOLSLICE: - registeredAttribute, err := x.registry.ConstructBoolAttribute(mkMake(key, true), xopat.AttributeTypeBool) - if err != nil { - return err - } - for _, v := range a.Value.AsBoolSlice() { - x.baseSpan.MetadataBool(registeredAttribute, v) - } - case attribute.FLOAT64: - registeredAttribute, err := x.registry.ConstructFloat64Attribute(mkMake(key, false), xopat.AttributeTypeFloat64) - if err != nil { - return err - } - x.baseSpan.MetadataFloat64(registeredAttribute, a.Value.AsFloat64()) - case attribute.FLOAT64SLICE: - registeredAttribute, err := x.registry.ConstructFloat64Attribute(mkMake(key, true), xopat.AttributeTypeFloat64) - if err != nil { - return err - } - for _, v := range a.Value.AsFloat64Slice() { - x.baseSpan.MetadataFloat64(registeredAttribute, v) - } - case attribute.INT64: - registeredAttribute, err := x.registry.ConstructInt64Attribute(mkMake(key, false), xopat.AttributeTypeInt64) - if err != nil { - return err - } - x.baseSpan.MetadataInt64(registeredAttribute, a.Value.AsInt64()) - case attribute.INT64SLICE: - registeredAttribute, err := x.registry.ConstructInt64Attribute(mkMake(key, true), xopat.AttributeTypeInt64) - if err != nil { - return err - } - for _, v := range a.Value.AsInt64Slice() { - x.baseSpan.MetadataInt64(registeredAttribute, v) - } - case attribute.STRING: - registeredAttribute, err := x.registry.ConstructStringAttribute(mkMake(key, false), xopat.AttributeTypeString) - if err != nil { - return err - } - x.baseSpan.MetadataString(registeredAttribute, a.Value.AsString()) - case attribute.STRINGSLICE: - registeredAttribute, err := x.registry.ConstructStringAttribute(mkMake(key, true), xopat.AttributeTypeString) - if err != nil { - return err - } - for _, v := range a.Value.AsStringSlice() { - x.baseSpan.MetadataString(registeredAttribute, v) - } - - case attribute.INVALID: - fallthrough - default: - return errors.Errorf("span attribute key (%s) has value type (%s) that is not expected", key, a.Value.Type()) - } - return nil -} - -func (x baseSpanReplay) AddXopMetadataAttribute(ctx context.Context, a attribute.KeyValue, aDef *decodeAttributeDefinition) error { - switch aDef.AttributeType { - case xopproto.AttributeType_Any: - registeredAttribute, err := x.registry.ConstructAnyAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.STRING, attribute.STRINGSLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v string) error { - var ma xopbase.ModelArg - err := ma.UnmarshalJSON([]byte(v)) - if err != nil { - return err - } - x.baseSpan.MetadataAny(registeredAttribute, ma) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsStringSlice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsString() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Bool: - registeredAttribute, err := x.registry.ConstructBoolAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.BOOL, attribute.BOOLSLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v bool) error { - x.baseSpan.MetadataBool(registeredAttribute, v) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsBoolSlice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsBool() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Duration: - registeredAttribute, err := x.registry.ConstructDurationAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.STRING, attribute.STRINGSLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v string) error { - d, err := time.ParseDuration(v) - if err != nil { - return err - } - x.baseSpan.MetadataInt64(®isteredAttribute.Int64Attribute, int64(d)) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsStringSlice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsString() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Enum: - registeredAttribute, err := x.registry.ConstructEnumAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.STRING, attribute.STRINGSLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v string) error { - i := strings.LastIndexByte(v, '/') - if i == -1 { - return errors.Errorf("invalid enum %s", v) - } - if i == len(v)-1 { - return errors.Errorf("invalid enum %s", v) - } - vi, err := strconv.ParseInt(v[i+1:], 10, 64) - if err != nil { - return errors.Wrap(err, "invalid enum") - } - enum := registeredAttribute.Add64(vi, v[:i]) - x.baseSpan.MetadataEnum(®isteredAttribute.EnumAttribute, enum) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsStringSlice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsString() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Float64: - registeredAttribute, err := x.registry.ConstructFloat64Attribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.FLOAT64, attribute.FLOAT64SLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v float64) error { - x.baseSpan.MetadataFloat64(registeredAttribute, v) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsFloat64Slice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsFloat64() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Int: - registeredAttribute, err := x.registry.ConstructIntAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.INT64, attribute.INT64SLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v int64) error { - x.baseSpan.MetadataInt64(®isteredAttribute.Int64Attribute, int64(v)) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsInt64Slice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsInt64() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Int16: - registeredAttribute, err := x.registry.ConstructInt16Attribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.INT64, attribute.INT64SLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v int64) error { - x.baseSpan.MetadataInt64(®isteredAttribute.Int64Attribute, int64(v)) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsInt64Slice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsInt64() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Int32: - registeredAttribute, err := x.registry.ConstructInt32Attribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.INT64, attribute.INT64SLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v int64) error { - x.baseSpan.MetadataInt64(®isteredAttribute.Int64Attribute, int64(v)) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsInt64Slice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsInt64() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Int64: - registeredAttribute, err := x.registry.ConstructInt64Attribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.INT64, attribute.INT64SLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v int64) error { - x.baseSpan.MetadataInt64(registeredAttribute, v) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsInt64Slice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsInt64() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Int8: - registeredAttribute, err := x.registry.ConstructInt8Attribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.INT64, attribute.INT64SLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v int64) error { - x.baseSpan.MetadataInt64(®isteredAttribute.Int64Attribute, int64(v)) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsInt64Slice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsInt64() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Link: - registeredAttribute, err := x.registry.ConstructLinkAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.STRING, attribute.STRINGSLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v string) error { - t, ok := xoptrace.TraceFromString(v) - if !ok { - return errors.Errorf("invalid trace string %s", v) - } - x.baseSpan.MetadataLink(registeredAttribute, t) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsStringSlice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsString() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_String: - registeredAttribute, err := x.registry.ConstructStringAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.STRING, attribute.STRINGSLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v string) error { - x.baseSpan.MetadataString(registeredAttribute, v) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsStringSlice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsString() - err := setter(value) - if err != nil { - return err - } - } - case xopproto.AttributeType_Time: - registeredAttribute, err := x.registry.ConstructTimeAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - expectedSingleType, expectedMultiType := attribute.STRING, attribute.STRINGSLICE - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - setter := func(v string) error { - t, err := time.Parse(time.RFC3339Nano, v) - if err != nil { - return err - } - x.baseSpan.MetadataTime(registeredAttribute, t) - return nil - } - if registeredAttribute.Multiple() { - values := a.Value.AsStringSlice() - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - value := a.Value.AsString() - err := setter(value) - if err != nil { - return err - } - } - - default: - return errors.Errorf("unexpected attribute type %s", aDef.AttributeType) - } - return nil -} diff --git a/xopotel/export.zzzgo b/xopotel/export.zzzgo deleted file mode 100644 index acf73243..00000000 --- a/xopotel/export.zzzgo +++ /dev/null @@ -1,1004 +0,0 @@ -// TEMPLATE-FILE -// TEMPLATE-FILE -// TEMPLATE-FILE - -package xopotel - -import ( - "context" - "encoding/json" - "fmt" - "regexp" - "runtime" - "sort" - "strconv" - "strings" - "sync/atomic" - "time" - - "github.com/xoplog/xop-go/xopat" - "github.com/xoplog/xop-go/xopbase" - "github.com/xoplog/xop-go/xopconst" - "github.com/xoplog/xop-go/xopnum" - "github.com/xoplog/xop-go/xopproto" - "github.com/xoplog/xop-go/xoptrace" - "github.com/xoplog/xop-go/xoputil/xopversion" - - "github.com/muir/gwrap" - "github.com/muir/list" - "github.com/pkg/errors" - "go.opentelemetry.io/otel/attribute" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" -) - -var _ sdktrace.SpanExporter = &spanExporter{} -var _ sdktrace.SpanExporter = &unhack{} - -var ErrShutdown = fmt.Errorf("Shutdown called") - -type spanExporter struct { - base xopbase.Logger - orderedFinish []orderedFinish - sequenceNumber int32 - done int32 -} - -type spanReplay struct { - *spanExporter - id2Index map[oteltrace.SpanID]int - spans []sdktrace.ReadOnlySpan - subSpans [][]int - data []*datum -} - -type datum struct { - baseSpan xopbase.Span - requestIndex int // index of request ancestor - attributeDefinitions map[string]*decodeAttributeDefinition - xopSpan bool - registry *xopat.Registry -} - -func (x *spanExporter) addOrdered(seq int32, f func()) { - x.orderedFinish = append(x.orderedFinish, orderedFinish{ - num: seq, - f: f, - }) -} - -type orderedFinish struct { - num int32 - f func() -} - -type baseSpanReplay struct { - spanReplay - *datum - span sdktrace.ReadOnlySpan -} - -type decodeAttributeDefinition struct { - xopat.Make - AttributeType xopproto.AttributeType `json:"vtype"` -} - -type wrappedReadOnlySpan struct { - sdktrace.ReadOnlySpan - links []sdktrace.Link -} - -// ExportToXOP allows open telementry spans to be exported through -// a xopbase.Logger. If the open telementry spans were generated -// originally using xoputil, then the exported data should almost -// exactly match the original inputs. -func ExportToXOP(base xopbase.Logger) sdktrace.SpanExporter { - return &spanExporter{ - base: base, - } -} - -func (e *spanExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) (err error) { - // TODO: avoid returning error when possible - id2Index := makeIndex(spans) - subSpans, todo := makeSubspans(id2Index, spans) - x := spanReplay{ - spanExporter: e, - id2Index: id2Index, - spans: spans, - subSpans: subSpans, - data: make([]*datum, len(spans)), - } - - var toFinish []func() - - var processSpan func(int) error - processSpan = func(i int) error { - x.data[i] = &datum{} - finisher, err := x.Replay(ctx, spans[i], x.data[i], i) - if err != nil { - return err - } - for _, subSpan := range subSpans[i] { - err := processSpan(subSpan) - if err != nil { - return err - } - } - toFinish = append(toFinish, finisher) - return nil - } - for _, i := range todo { - err := processSpan(i) - if err != nil { - return err - } - - sort.Slice(e.orderedFinish, func(i, j int) bool { - return e.orderedFinish[i].num < e.orderedFinish[j].num - }) - for _, o := range x.orderedFinish { - o.f() - } - x.orderedFinish = x.orderedFinish[:0] - - for _, finisher := range toFinish { - finisher() - } - toFinish = toFinish[:0] - } - return nil -} - -func (x spanReplay) Replay(ctx context.Context, span sdktrace.ReadOnlySpan, data *datum, myIndex int) (func(), error) { - var bundle xoptrace.Bundle - spanContext := span.SpanContext() - if spanContext.HasTraceID() { - bundle.Trace.TraceID().SetArray(spanContext.TraceID()) - } - if spanContext.HasSpanID() { - bundle.Trace.SpanID().SetArray(spanContext.SpanID()) - } - if spanContext.IsSampled() { - bundle.Trace.Flags().SetArray([1]byte{1}) - } - if spanContext.TraceState().Len() != 0 { - bundle.State.SetString(spanContext.TraceState().String()) - } - parentIndex, hasParent := lookupParent(x.id2Index, span) - var xopParent *datum - if hasParent { - parentContext := x.spans[parentIndex].SpanContext() - xopParent = x.data[parentIndex] - if parentContext.HasTraceID() { - bundle.Parent.TraceID().SetArray(parentContext.TraceID()) - if bundle.Trace.TraceID().IsZero() { - bundle.Trace.TraceID().Set(bundle.Parent.GetTraceID()) - } - } - if parentContext.HasSpanID() { - bundle.Parent.SpanID().SetArray(parentContext.SpanID()) - } - if parentContext.IsSampled() { - bundle.Parent.Flags().SetArray([1]byte{1}) - } - } else if span.Parent().HasTraceID() { - bundle.Parent.TraceID().SetArray(span.Parent().TraceID()) - if span.Parent().HasSpanID() { - bundle.Parent.SpanID().SetArray(span.Parent().SpanID()) - } - if span.Parent().IsSampled() { - bundle.Parent.Flags().SetArray([1]byte{1}) - } - } - - var downStreamError gwrap.AtomicValue[error] - errorReporter := func(err error) { - if err != nil { - downStreamError.Store(err) - } - } - bundle.Parent.Flags().SetBytes([]byte{1}) - bundle.Trace.Flags().SetBytes([]byte{1}) - spanKind := span.SpanKind() - attributeMap := mapAttributes(span.Attributes()) - if b := attributeMap.GetString(xopBaggage); b != "" { - bundle.Baggage.SetString(b) - } - if spanKind == oteltrace.SpanKindUnspecified { - spanKind = oteltrace.SpanKind(defaulted(attributeMap.GetInt(otelSpanKind), int64(oteltrace.SpanKindUnspecified))) - } - if attributeMap.GetBool(spanIsLinkEventKey) { - // span is extra just for link - return func() {}, nil - } - switch spanKind { - case oteltrace.SpanKindUnspecified, oteltrace.SpanKindInternal: - if hasParent { - spanSeq := defaulted(attributeMap.GetString(xopSpanSequence), "") - data.xopSpan = xopParent.xopSpan - data.baseSpan = xopParent.baseSpan.Span(ctx, span.StartTime(), bundle, span.Name(), spanSeq) - data.requestIndex = xopParent.requestIndex - data.attributeDefinitions = xopParent.attributeDefinitions - data.registry = xopParent.registry - } else { - // This is a difficult sitatuion. We have an internal/unspecified span - // that does not have a parent present. There is no right answer for what - // to do. In the Xop world, such a span isn't allowed to exist. We'll treat - // this span as a request, but mark it as promoted. - data.xopSpan = attributeMap.GetString(xopVersion) != "" - baseRequest := x.base.Request(ctx, span.StartTime(), bundle, span.Name(), buildSourceInfo(span, attributeMap)) - baseRequest.SetErrorReporter(errorReporter) - data.baseSpan = baseRequest - data.baseSpan.MetadataBool(xopPromotedMetadata, true) - data.requestIndex = myIndex - data.attributeDefinitions = make(map[string]*decodeAttributeDefinition) - data.registry = xopat.NewRegistry(false) - } - default: - baseRequest := x.base.Request(ctx, span.StartTime(), bundle, span.Name(), buildSourceInfo(span, attributeMap)) - baseRequest.SetErrorReporter(errorReporter) - data.baseSpan = baseRequest - data.requestIndex = myIndex - data.attributeDefinitions = make(map[string]*decodeAttributeDefinition) - data.xopSpan = attributeMap.GetString(xopVersion) != "" - data.registry = xopat.NewRegistry(false) - if !data.xopSpan { - data.baseSpan.MetadataAny(otelReplayStuff, xopbase.ModelArg{ - Model: &otelStuff{ - SpanKind: xopconst.SpanKindEnum(span.SpanKind()), - Status: span.Status(), - Resource: bufferedResource{span.Resource()}, - InstrumentationScope: span.InstrumentationScope(), - spanCounters: spanCounters{ - DroppedAttributes: span.DroppedAttributes(), - DroppedLinks: span.DroppedLinks(), - DroppedEvents: span.DroppedEvents(), - ChildSpanCount: span.ChildSpanCount(), - }, - }, - }) - } - } - y := baseSpanReplay{ - spanReplay: x, - span: span, - datum: data, - } - for _, attribute := range span.Attributes() { - err := y.AddSpanAttribute(ctx, attribute) - if err != nil { - return func() {}, err - } - } - var maxNumber int32 - for _, event := range span.Events() { - lastNumber, err := y.AddEvent(ctx, event) - if err != nil { - return func() {}, err - } - if lastNumber > maxNumber { - maxNumber = lastNumber - } - } - for _, link := range span.Links() { - if !data.xopSpan { - z := lineAttributesReplay{ - baseSpanReplay: y, - lineType: lineTypeLink, - lineFormat: lineFormatDefault, - level: xopnum.InfoLevel, - } - line, err := z.AddLineAttributes(ctx, "link", span.StartTime(), link.Attributes) - if err != nil { - return func() {}, err - } - var trace xoptrace.Trace - trace.Flags().SetArray([1]byte{byte(link.SpanContext.TraceFlags())}) - trace.TraceID().SetArray(link.SpanContext.TraceID()) - trace.SpanID().SetArray(link.SpanContext.SpanID()) - data.baseSpan.MetadataLink(otelLink, trace) - z.link = trace - if ts := link.SpanContext.TraceState(); ts.Len() != 0 { - line.String(xopOTELLinkTranceState, ts.String(), xopbase.StringDataType) - } - if link.SpanContext.IsRemote() { - line.Bool(xopOTELLinkIsRemote, true) - } - if link.DroppedAttributeCount != 0 { - line.Int64(xopOTELLinkDroppedAttributeCount, int64(link.DroppedAttributeCount), xopbase.IntDataType) - } - - err = z.finishLine(ctx, "link", xopOTELLinkDetail.String(), line) - if err != nil { - return func() {}, err - } - } - } - if endTime := span.EndTime(); !endTime.IsZero() { - return func() { - data.baseSpan.Done(endTime, true) - }, nil - } - return func() {}, downStreamError.Load() -} - -type lineType int - -const ( - lineTypeLine lineType = iota - lineTypeLink - lineTypeLinkEvent - lineTypeModel -) - -type lineFormat int - -const ( - lineFormatDefault lineFormat = iota - lineFormatTemplate -) - -var lineRE = regexp.MustCompile(`^(.+):(\d+)$`) - -func (x baseSpanReplay) AddEvent(ctx context.Context, event sdktrace.Event) (int32, error) { - z := lineAttributesReplay{ - baseSpanReplay: x, - lineType: lineTypeLine, - lineFormat: lineFormatDefault, - level: xopnum.InfoLevel, - } - line, err := z.AddLineAttributes(ctx, "event", event.Time, event.Attributes) - if err != nil { - return 0, err - } - err = z.finishLine(ctx, "event", event.Name, line) - return z.lineNumber, err -} - -type lineAttributesReplay struct { - baseSpanReplay - lineType lineType - lineFormat lineFormat - template string - link xoptrace.Trace - modelArg xopbase.ModelArg - frames []runtime.Frame - lineNumber int32 - level xopnum.Level -} - -func (x *lineAttributesReplay) AddLineAttributes(ctx context.Context, what string, ts time.Time, attributes []attribute.KeyValue) (xopbase.Line, error) { - x.sequenceNumber++ - x.lineNumber = x.sequenceNumber - nonSpecial := make([]attribute.KeyValue, 0, len(attributes)) - for _, a := range attributes { - switch a.Key { - case xopLineNumber: - if a.Value.Type() == attribute.INT64 { - x.lineNumber = int32(a.Value.AsInt64()) - } else { - return nil, errors.Errorf("invalid line number attribute type %s", a.Value.Type()) - } - case xopLevel: - if a.Value.Type() == attribute.STRING { - var err error - x.level, err = xopnum.LevelString(a.Value.AsString()) - if err != nil { - x.level = xopnum.InfoLevel - } - } else { - return nil, errors.Errorf("invalid line level attribute type %s", a.Value.Type()) - } - case xopType: - if a.Value.Type() == attribute.STRING { - switch a.Value.AsString() { - case "link": - x.lineType = lineTypeLink - case "link-event": - x.lineType = lineTypeLinkEvent - case "model": - x.lineType = lineTypeModel - case "line": - // defaulted - default: - return nil, errors.Errorf("invalid line type attribute value %s", a.Value.AsString()) - } - } else { - return nil, errors.Errorf("invalid line type attribute type %s", a.Value.Type()) - } - case xopModelType: - if a.Value.Type() == attribute.STRING { - x.modelArg.ModelType = a.Value.AsString() - } else { - return nil, errors.Errorf("invalid model type attribute type %s", a.Value.Type()) - } - case xopEncoding: - if a.Value.Type() == attribute.STRING { - e, ok := xopproto.Encoding_value[a.Value.AsString()] - if !ok { - return nil, errors.Errorf("invalid model encoding '%s'", a.Value.AsString()) - } - x.modelArg.Encoding = xopproto.Encoding(e) - } else { - return nil, errors.Errorf("invalid model encoding attribute type %s", a.Value.Type()) - } - case xopModel: - if a.Value.Type() == attribute.STRING { - x.modelArg.Encoded = []byte(a.Value.AsString()) - } else { - return nil, errors.Errorf("invalid model encoding attribute type %s", a.Value.Type()) - } - case xopTemplate: - if a.Value.Type() == attribute.STRING { - x.lineFormat = lineFormatTemplate - x.template = a.Value.AsString() - } else { - return nil, errors.Errorf("invalid line template attribute type %s", a.Value.Type()) - } - case xopLinkData: - if a.Value.Type() == attribute.STRING { - var ok bool - x.link, ok = xoptrace.TraceFromString(a.Value.AsString()) - if !ok { - return nil, errors.Errorf("invalid link data attribute value %s", a.Value.AsString()) - } - } else { - return nil, errors.Errorf("invalid link data attribute type %s", a.Value.Type()) - } - case xopStackTrace: - if a.Value.Type() == attribute.STRINGSLICE { - raw := a.Value.AsStringSlice() - x.frames = make([]runtime.Frame, len(raw)) - for i, s := range raw { - m := lineRE.FindStringSubmatch(s) - if m == nil { - return nil, errors.Errorf("could not match stack line '%s'", s) - } - x.frames[i].File = m[1] - num, _ := strconv.ParseInt(m[2], 10, 64) - x.frames[i].Line = int(num) - } - } else { - return nil, errors.Errorf("invalid stack trace attribute type %s", a.Value.Type()) - } - default: - nonSpecial = append(nonSpecial, a) - } - } - line := x.baseSpan.NoPrefill().Line( - x.level, - ts, - x.frames, - ) - for _, a := range nonSpecial { - if x.xopSpan { - err := x.AddXopEventAttribute(ctx, a, line) - if err != nil { - return nil, errors.Wrapf(err, "add xop %s attribute %s", what, string(a.Key)) - } - } else { - err := x.AddEventAttribute(ctx, a, line) - if err != nil { - return nil, errors.Wrapf(err, "add %s attribute %s with type %s", what, string(a.Key), a.Value.Type()) - } - } - } - return line, nil -} - -func (x lineAttributesReplay) finishLine(ctx context.Context, what string, name string, line xopbase.Line) error { - switch x.lineType { - case lineTypeLine: - switch x.lineFormat { - case lineFormatDefault: - x.addOrdered(x.lineNumber, func() { - line.Msg(name) - }) - case lineFormatTemplate: - x.addOrdered(x.lineNumber, func() { - line.Template(x.template) - }) - default: - return errors.Errorf("unexpected lineType %d", x.lineType) - } - case lineTypeLink: - x.addOrdered(x.lineNumber, func() { - line.Link(name, x.link) - }) - case lineTypeLinkEvent: - return errors.Errorf("unexpected lineType: link event") - case lineTypeModel: - x.addOrdered(x.lineNumber, func() { - line.Model(name, x.modelArg) - }) - default: - return errors.Errorf("unexpected lineType %d", x.lineType) - } - return nil -} - -func (e *spanExporter) Shutdown(ctx context.Context) error { - atomic.StoreInt32(&e.done, 1) - return nil -} - -type unhack struct { - next sdktrace.SpanExporter -} - -// NewUnhacker wraps a SpanExporter and if the input is from BaseLogger or SpanLog, -// then it "fixes" the data hack in the output that puts inter-span links in sub-spans -// rather than in the span that defined them. -func NewUnhacker(exporter sdktrace.SpanExporter) sdktrace.SpanExporter { - return &unhack{next: exporter} -} - -func (u *unhack) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error { - // TODO: fix up SpanKind if spanKind is one of the attributes - id2Index := makeIndex(spans) - subLinks := make([][]sdktrace.Link, len(spans)) - for i, span := range spans { - parentIndex, ok := lookupParent(id2Index, span) - if !ok { - continue - } - var addToParent bool - for _, attribute := range span.Attributes() { - switch attribute.Key { - case spanIsLinkAttributeKey, spanIsLinkEventKey: - spans[i] = nil - addToParent = true - } - } - if !addToParent { - continue - } - subLinks[parentIndex] = append(subLinks[parentIndex], span.Links()...) - } - n := make([]sdktrace.ReadOnlySpan, 0, len(spans)) - for i, span := range spans { - span := span - switch { - case len(subLinks[i]) > 0: - n = append(n, wrappedReadOnlySpan{ - ReadOnlySpan: span, - links: append(list.Copy(span.Links()), subLinks[i]...), - }) - case span == nil: - // skip - default: - n = append(n, span) - } - } - return u.next.ExportSpans(ctx, n) -} - -func (u *unhack) Shutdown(ctx context.Context) error { - return u.next.Shutdown(ctx) -} - -var _ sdktrace.ReadOnlySpan = wrappedReadOnlySpan{} - -func (w wrappedReadOnlySpan) Links() []sdktrace.Link { - return w.links -} - -func makeIndex(spans []sdktrace.ReadOnlySpan) map[oteltrace.SpanID]int { - id2Index := make(map[oteltrace.SpanID]int) - for i, span := range spans { - spanContext := span.SpanContext() - if spanContext.HasSpanID() { - id2Index[spanContext.SpanID()] = i - } - } - return id2Index -} - -func lookupParent(id2Index map[oteltrace.SpanID]int, span sdktrace.ReadOnlySpan) (int, bool) { - parent := span.Parent() - if !parent.HasSpanID() { - return 0, false - } - parentIndex, ok := id2Index[parent.SpanID()] - if !ok { - return 0, false - } - return parentIndex, true -} - -// makeSubspans figures out what subspans each span has and also which spans -// have no parent span (and thus are not a subspan). We are assuming that there -// are no cycles in the graph of spans & subspans. UNSAFE -func makeSubspans(id2Index map[oteltrace.SpanID]int, spans []sdktrace.ReadOnlySpan) ([][]int, []int) { - ss := make([][]int, len(spans)) - noParent := make([]int, 0, len(spans)) - for i, span := range spans { - parentIndex, ok := lookupParent(id2Index, span) - if !ok { - noParent = append(noParent, i) - continue - } - ss[parentIndex] = append(ss[parentIndex], i) - } - return ss, noParent -} - -func buildSourceInfo(span sdktrace.ReadOnlySpan, attributeMap aMap) xopbase.SourceInfo { - var si xopbase.SourceInfo - var source string - var namespace string - if attributeMap.GetString(xopVersion) == "" { - // span did not come from XOP - source = otelDataSource - namespace = span.SpanKind().String() - } else { - if s := attributeMap.GetString(xopSource); s != "" { - source = s - } else if n := span.InstrumentationScope().Name; n != "" { - if v := span.InstrumentationScope().Version; v != "" { - source = n + " " + v - } else { - source = n - } - } else { - source = "OTEL" - } - namespace = defaulted(attributeMap.GetString(xopNamespace), source) - } - si.Source, si.SourceVersion = xopversion.SplitVersion(source) - si.Namespace, si.NamespaceVersion = xopversion.SplitVersion(namespace) - return si -} - -type aMap struct { - strings map[attribute.Key]string - ints map[attribute.Key]int64 - bools map[attribute.Key]bool -} - -func mapAttributes(list []attribute.KeyValue) aMap { - m := aMap{ - strings: make(map[attribute.Key]string), - ints: make(map[attribute.Key]int64), - bools: make(map[attribute.Key]bool), - } - for _, a := range list { - switch a.Value.Type() { - case attribute.STRING: - m.strings[a.Key] = a.Value.AsString() - case attribute.INT64: - m.ints[a.Key] = a.Value.AsInt64() - case attribute.BOOL: - m.bools[a.Key] = a.Value.AsBool() - } - } - return m -} - -func (m aMap) GetString(k attribute.Key) string { return m.strings[k] } -func (m aMap) GetInt(k attribute.Key) int64 { return m.ints[k] } -func (m aMap) GetBool(k attribute.Key) bool { return m.bools[k] } - -func defaulted[T comparable](a, b T) T { - var zero T - if a == zero { - return b - } - return a -} - -func (x baseSpanReplay) AddXopEventAttribute(ctx context.Context, a attribute.KeyValue, line xopbase.Line) error { - switch a.Value.Type() { - case attribute.STRINGSLICE: - slice := a.Value.AsStringSlice() - if len(slice) < 2 { - return errors.Errorf("invalid xop attribute encoding slice is too short") - } - switch slice[1] { - // MACRO DataTypeAbbreviations - case "ZZZ": - // CONDITIONAL ONLY:i,i8,i16,i32,i64 - i, err := strconv.ParseInt(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.zzz(xopat.K(a.Key), i, xopbase.StringToDataType["ZZZ"]) - // CONDITIONAL ONLY:u,u8,u16,u32,u64,uintptr - i, err := strconv.ParseUint(slice[0], 10, 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.zzz(xopat.K(a.Key), i, xopbase.StringToDataType["ZZZ"]) - // CONDITIONAL ONLY:s,stringer,error - line.zzz(xopat.K(a.Key), slice[0], xopbase.StringToDataType["ZZZ"]) - // CONDITIONAL ONLY:dur - dur, err := time.ParseDuration(slice[0]) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.zzz(xopat.K(a.Key), dur) - // CONDITIONAL ONLY:f32,f64 - f, err := strconv.ParseFloat(slice[0], 64) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.zzz(xopat.K(a.Key), f, xopbase.StringToDataType["ZZZ"]) - // CONDITIONAL ONLY:time - ts, err := time.Parse(time.RFC3339Nano, slice[0]) - if err != nil { - return errors.Wrapf(err, "key %s invalid %s", a.Key, slice[1]) - } - line.zzz(xopat.K(a.Key), ts) - // CONDITIONAL ONLY:enum - if len(slice) != 3 { - return errors.Errorf("key %s invalid enum encoding, slice too short", a.Key) - } - ea, err := x.registry.ConstructEnumAttribute(xopat.Make{ - Key: string(a.Key), - }, xopat.AttributeTypeEnum) - if err != nil { - return errors.Errorf("could not turn key %s into an enum", a.Key) - } - i, err := strconv.ParseInt(slice[2], 10, 64) - if err != nil { - return errors.Wrapf(err, "could not turn key %s into an enum", a.Key) - } - enum := ea.Add64(i, slice[0]) - line.Enum(&ea.EnumAttribute, enum) - // CONDITIONAL ONLY:any - if len(slice) != 4 { - return errors.Errorf("key %s invalid any encoding, slice too short", a.Key) - } - var ma xopbase.ModelArg - ma.Encoded = []byte(slice[0]) - e, ok := xopproto.Encoding_value[slice[2]] - if !ok { - return errors.Errorf("invalid model encoding '%s'", a.Value.AsString()) - } - ma.Encoding = xopproto.Encoding(e) - ma.ModelType = slice[3] - line.zzz(xopat.K(a.Key), ma) - // END CONDITIONAL - - } - - case attribute.BOOL: - line.Bool(xopat.K(a.Key), a.Value.AsBool()) - default: - return errors.Errorf("unexpected event attribute type %s for xop-encoded line", a.Value.Type()) - } - return nil -} - -func (x baseSpanReplay) AddEventAttribute(ctx context.Context, a attribute.KeyValue, line xopbase.Line) error { - switch a.Value.Type() { - // MACRO OTELTypes - case attribute.ZZZ: - // CONDITIONAL SKIP:BOOL - line.Zzz(xopat.K(a.Key), a.Value.AsZzz(), xopbase.ZzzDataType) - // ELSE CONDITIONAL - line.Bool(xopat.K(a.Key), a.Value.AsZzz()) - // END CONDITIONAL - case attribute.ZZZSLICE: - var ma xopbase.ModelArg - ma.Model = a.Value.AsZzzSlice() - ma.ModelType = toTypeSliceName["ZZZ"] - line.Any(xopat.K(a.Key), ma) - - case attribute.INVALID: - fallthrough - default: - return errors.Errorf("invalid type") - } - return nil -} - -var toTypeSliceName = map[string]string{ - "BOOL": "[]bool", - "STRING": "[]string", - "INT64": "[]int64", - "FLOAT64": "[]float64", -} - -func (x baseSpanReplay) AddSpanAttribute(ctx context.Context, a attribute.KeyValue) (err error) { - switch a.Key { - case spanIsLinkAttributeKey, - spanIsLinkEventKey, - xopSource, - xopNamespace, - xopBaggage, - xopSpanSequence, - xopType, - otelSpanKind: - // special cases handled elsewhere - return nil - case xopVersion, - xopOTELVersion: - // dropped - return nil - } - key := string(a.Key) - defer func() { - if err != nil { - err = errors.Wrapf(err, "add span attribute %s with type %s", key, a.Value.Type()) - } - }() - if strings.HasPrefix(key, attributeDefinitionPrefix) { - key := strings.TrimPrefix(key, attributeDefinitionPrefix) - if _, ok := x.data[x.requestIndex].attributeDefinitions[key]; ok { - return nil - } - if a.Value.Type() != attribute.STRING { - return errors.Errorf("expected type to be string") - } - var aDef decodeAttributeDefinition - err := json.Unmarshal([]byte(a.Value.AsString()), &aDef) - if err != nil { - return errors.Wrapf(err, "could not unmarshal attribute defintion") - } - x.data[x.requestIndex].attributeDefinitions[key] = &aDef - return nil - } - - if aDef, ok := x.data[x.requestIndex].attributeDefinitions[key]; ok { - return x.AddXopMetadataAttribute(ctx, a, aDef) - } - if x.xopSpan { - return errors.Errorf("missing attribute defintion for key %s in xop span", key) - } - - mkMake := func(key string, multiple bool) xopat.Make { - return xopat.Make{ - Description: xopSynthesizedForOTEL, - Key: key, - Multiple: multiple, - } - } - switch a.Value.Type() { - // MACRO OTELTypes - case attribute.ZZZ: - registeredAttribute, err := x.registry.ConstructZzzAttribute(mkMake(key, false), xopat.AttributeTypeZzz) - if err != nil { - return err - } - x.baseSpan.MetadataZzz(registeredAttribute, a.Value.AsZzz()) - case attribute.ZZZSLICE: - registeredAttribute, err := x.registry.ConstructZzzAttribute(mkMake(key, true), xopat.AttributeTypeZzz) - if err != nil { - return err - } - for _, v := range a.Value.AsZzzSlice() { - x.baseSpan.MetadataZzz(registeredAttribute, v) - } - - case attribute.INVALID: - fallthrough - default: - return errors.Errorf("span attribute key (%s) has value type (%s) that is not expected", key, a.Value.Type()) - } - return nil -} - -func (x baseSpanReplay) AddXopMetadataAttribute(ctx context.Context, a attribute.KeyValue, aDef *decodeAttributeDefinition) error { - switch aDef.AttributeType { - // MACRO ZZZAttribute - case xopproto.AttributeType_ZZZ: - registeredAttribute, err := x.registry.ConstructZZZAttribute(aDef.Make, xopat.AttributeType(aDef.AttributeType)) - if err != nil { - return err - } - // CONDITIONAL ONLY:Enum,Time,String,Any,Link,Duration - expectedSingleType, expectedMultiType := attribute.STRING, attribute.STRINGSLICE - // CONDITIONAL ONLY:Int64,Int,Int8,Int16,Int32 - expectedSingleType, expectedMultiType := attribute.INT64, attribute.INT64SLICE - // CONDITIONAL ONLY:Bool - expectedSingleType, expectedMultiType := attribute.BOOL, attribute.BOOLSLICE - // CONDITIONAL ONLY:Float64 - expectedSingleType, expectedMultiType := attribute.FLOAT64, attribute.FLOAT64SLICE - // END CONDITIONAL - expectedType := expectedSingleType - if registeredAttribute.Multiple() { - expectedType = expectedMultiType - } - if a.Value.Type() != expectedType { - return errors.Errorf("expected type %s", expectedMultiType) - } - // CONDITIONAL ONLY:String,Int64,Float64,Bool - setter := func(v zzz) error { - x.baseSpan.MetadataZZZ(registeredAttribute, v) - return nil - } - // CONDITIONAL ONLY:Int,Int8,Int16,Int32 - setter := func(v int64) error { - x.baseSpan.MetadataInt64(®isteredAttribute.Int64Attribute, int64(v)) - return nil - } - // CONDITIONAL ONLY:Duration - setter := func(v string) error { - d, err := time.ParseDuration(v) - if err != nil { - return err - } - x.baseSpan.MetadataInt64(®isteredAttribute.Int64Attribute, int64(d)) - return nil - } - // CONDITIONAL ONLY:Time - setter := func(v string) error { - t, err := time.Parse(time.RFC3339Nano, v) - if err != nil { - return err - } - x.baseSpan.MetadataZZZ(registeredAttribute, t) - return nil - } - // CONDITIONAL ONLY:Enum - setter := func(v string) error { - i := strings.LastIndexByte(v, '/') - if i == -1 { - return errors.Errorf("invalid enum %s", v) - } - if i == len(v)-1 { - return errors.Errorf("invalid enum %s", v) - } - vi, err := strconv.ParseInt(v[i+1:], 10, 64) - if err != nil { - return errors.Wrap(err, "invalid enum") - } - enum := registeredAttribute.Add64(vi, v[:i]) - x.baseSpan.MetadataEnum(®isteredAttribute.EnumAttribute, enum) - return nil - } - // CONDITIONAL ONLY:Any - setter := func(v string) error { - var ma xopbase.ModelArg - err := ma.UnmarshalJSON([]byte(v)) - if err != nil { - return err - } - x.baseSpan.MetadataAny(registeredAttribute, ma) - return nil - } - // CONDITIONAL ONLY:Link - setter := func(v string) error { - t, ok := xoptrace.TraceFromString(v) - if !ok { - return errors.Errorf("invalid trace string %s", v) - } - x.baseSpan.MetadataLink(registeredAttribute, t) - return nil - } - // END CONDITIONAL - if registeredAttribute.Multiple() { - // CONDITIONAL ONLY:Enum,Time,Any,Link,Duration - values := a.Value.AsStringSlice() - // CONDITIONAL ONLY:Bool,Int64,String,Float64 - values := a.Value.AsZZZSlice() - // CONDITIONAL ONLY:Int,Int8,Int16,Int32 - values := a.Value.AsInt64Slice() - // END CONDITIONAL - for _, value := range values { - err := setter(value) - if err != nil { - return err - } - } - } else { - // CONDITIONAL ONLY:Enum,Time,Any,Link,Duration - value := a.Value.AsString() - // CONDITIONAL ONLY:Bool,Int64,String,Float64 - value := a.Value.AsZZZ() - // CONDITIONAL ONLY:Int,Int8,Int16,Int32 - value := a.Value.AsInt64() - // END CONDITIONAL - err := setter(value) - if err != nil { - return err - } - } - - default: - return errors.Errorf("unexpected attribute type %s", aDef.AttributeType) - } - return nil -} diff --git a/xopotel/models.go b/xopotel/models.go deleted file mode 100644 index 34f6a722..00000000 --- a/xopotel/models.go +++ /dev/null @@ -1,246 +0,0 @@ -package xopotel - -import ( - "context" - "crypto/rand" - "sync" - "sync/atomic" - "time" - - "github.com/xoplog/xop-go" - "github.com/xoplog/xop-go/xopat" - "github.com/xoplog/xop-go/xopbase" - "github.com/xoplog/xop-go/xopconst" - "github.com/xoplog/xop-go/xopnum" - "github.com/xoplog/xop-go/xoptrace" - - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/sdk/instrumentation" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" -) - -const attributeDefinitionPrefix = "xop.defineKey." -const xopSynthesizedForOTEL = "xopotel-shim type" - -type logger struct { - tracer oteltrace.Tracer - id string - doLogging bool - ignoreDone oteltrace.Span - spanFromContext bool - bufferedRequest *bufferedRequest // only set when BufferedReplayLogger is used -} - -type request struct { - *span - attributesDefined map[string]struct{} - lineCount int32 - errorReporter func(error) -} - -type span struct { - otelSpan oteltrace.Span - logger *logger - request *request - ctx context.Context - lock sync.Mutex - priorBoolSlices map[string][]bool - priorFloat64Slices map[string][]float64 - priorStringSlices map[string][]string - priorInt64Slices map[string][]int64 - hasPrior map[string]struct{} - metadataSeen map[string]interface{} - spanPrefill []attribute.KeyValue // holds spanID & traceID - isXOP bool // true unless data is imported from OTEL -} - -type prefilling struct { - builderWithSpan -} - -type prefilled struct { - builderWithSpan -} - -type line struct { - builderWithSpan - prealloc [15]attribute.KeyValue - level xopnum.Level - timestamp time.Time -} - -type builderWithSpan struct { - span *span - builder -} - -type builder struct { - attributes []attribute.KeyValue - prefillMsg string - linkKey string - linkValue xoptrace.Trace -} - -type otelStuff struct { - spanCounters - Status sdktrace.Status - SpanKind xopconst.SpanKindEnum - Resource bufferedResource - InstrumentationScope instrumentation.Scope - links []oteltrace.Link // filled in by getStuff() -} - -type spanCounters struct { - DroppedAttributes int - DroppedLinks int - DroppedEvents int - ChildSpanCount int -} - -var _ xopbase.Logger = &logger{} -var _ xopbase.Request = &request{} -var _ xopbase.Span = &span{} -var _ xopbase.Line = &line{} -var _ xopbase.Prefilling = &prefilling{} -var _ xopbase.Prefilled = &prefilled{} - -// Span-level -var otelSpanKind = attribute.Key("span.kind") -var spanIsLinkAttributeKey = attribute.Key("xop.span.is-link-attribute") -var spanIsLinkEventKey = attribute.Key("xop.span.is-link-event") -var xopBaggage = attribute.Key("xop.baggage") -var xopNamespace = attribute.Key("xop.namespace") -var xopOTELVersion = attribute.Key("xop.otel-version") -var xopSource = attribute.Key("xop.source") -var xopSpanSequence = attribute.Key("xop.xopSpanSequence") -var xopVersion = attribute.Key("xop.version") - -// Line -var xopLevel = attribute.Key("xop.level") -var xopLineNumber = attribute.Key("xop.lineNumber") -var xopStackTrace = attribute.Key("xop.stackTrace") -var xopTemplate = attribute.Key("xop.template") -var xopType = attribute.Key("xop.type") - -// Model -var xopEncoding = attribute.Key("xop.encoding") -var xopModel = attribute.Key("xop.model") -var xopModelType = attribute.Key("xop.modelType") - -// Link -var xopLinkData = attribute.Key("xop.link") -var otelLink = xopat.Make{Key: "span.otelLinks", Namespace: "XOP", Indexed: false, Prominence: 300, - Multiple: true, Distinct: true, - Description: "Data origin is OTEL, span links w/o attributes; links also sent as Link()"}.LinkAttribute() -var xopLinkMetadataKey = attribute.Key("xop.linkMetadataKey") - -var xopLinkTraceStateError = xop.Key("xop.linkTraceStateError") -var xopOTELLinkTranceState = xop.Key("xop.otelLinkTraceState") -var xopOTELLinkIsRemote = xop.Key("xop.otelLinkIsRemote") -var xopOTELLinkDetail = xop.Key("xop.otelLinkDetail") -var xopLinkRemoteError = xop.Key("xop.otelLinkRemoteError") -var xopOTELLinkDroppedAttributeCount = xop.Key("xop.otelLinkDroppedAttributeCount") -var xopLinkeDroppedError = xop.Key("xop.otelLinkDroppedError") - -var otelReplayStuff = xopat.Make{Key: "span.replayedFromOTEL", Namespace: "XOP", Indexed: false, Prominence: 300, - Description: "Data origin is OTEL, translated through xopotel.ExportToXOP, bundle of span config"}.AnyAttribute(&otelStuff{}) - -// TODO: find a better way to set this version string -const xopVersionValue = "0.3.0" -const xopotelVersionValue = xopVersionValue - -const otelDataSource = "source-is-not-xop" - -var xopPromotedMetadata = xopat.Make{Key: "xop.span-is-promoted", Namespace: "xopotel"}.BoolAttribute() - -var emptyTraceState oteltrace.TraceState - -type overrideContextKeyType struct{} - -var overrideContextKey = overrideContextKeyType{} - -func overrideIntoContext(ctx context.Context, bundle xoptrace.Bundle) context.Context { - override := &idOverride{ - valid: 1, - traceID: bundle.Trace.GetTraceID(), - spanID: bundle.Trace.GetSpanID(), - } - ctx = context.WithValue(ctx, overrideContextKey, override) - if !bundle.Parent.IsZero() || !bundle.State.IsZero() { - spanConfig := oteltrace.SpanContextConfig{ - TraceID: bundle.Parent.TraceID().Array(), - SpanID: bundle.Parent.SpanID().Array(), - TraceFlags: oteltrace.TraceFlags(bundle.Parent.Flags().Array()[0]), - } - if !bundle.State.IsZero() { - state, err := oteltrace.ParseTraceState(bundle.State.String()) - if err == nil { - spanConfig.TraceState = state - } - } - spanContext := oteltrace.NewSpanContext(spanConfig) - ctx = oteltrace.ContextWithSpanContext(ctx, spanContext) - } - return ctx -} - -func overrideFromContext(ctx context.Context) *idOverride { - v := ctx.Value(overrideContextKey) - if v == nil { - return nil - } - return v.(*idOverride) -} - -type idGenerator struct{} - -var _ sdktrace.IDGenerator = idGenerator{} - -type idOverride struct { - valid int32 - traceID xoptrace.HexBytes16 - spanID xoptrace.HexBytes8 -} - -func (o *idOverride) Get() (oteltrace.TraceID, oteltrace.SpanID, bool) { - if atomic.CompareAndSwapInt32(&o.valid, 1, 0) { - return o.traceID.Array(), o.spanID.Array(), true - } - return random16(), random8(), false -} - -var zero8 = oteltrace.SpanID{} -var zero16 = oteltrace.TraceID{} - -func random8() oteltrace.SpanID { - var b oteltrace.SpanID - for { - _, _ = rand.Read(b[:]) - if b != zero8 { - return b - } - } -} - -func random16() oteltrace.TraceID { - var b oteltrace.TraceID - for { - _, _ = rand.Read(b[:]) - if b != zero16 { - return b - } - } -} - -// IDGenerator generates an override that must be used with -// https://pkg.go.dev/go.opentelemetry.io/otel/sdk/trace#NewTracerProvider -// when creating a TracerProvider. This override causes the -// TracerProvider to respect the TraceIDs and SpanIDs generated by -// Xop. For accurate replay via xopotel, this is required. For general -// log generation, if this ID generator is not used, then the Span IDs -// and TraceIDs created by by the TracerProvider will be used for Xop -// logging. -func IDGenerator() sdktrace.TracerProviderOption { - return sdktrace.WithIDGenerator(idGenerator{}) -} diff --git a/xopotel/otel.go b/xopotel/otel.go deleted file mode 100644 index edbe7863..00000000 --- a/xopotel/otel.go +++ /dev/null @@ -1,968 +0,0 @@ -// This file is generated, DO NOT EDIT. It comes from the corresponding .zzzgo file - -package xopotel - -import ( - "context" - "fmt" - "regexp" - "runtime" - "strconv" - "sync/atomic" - "time" - - "github.com/xoplog/xop-go" - "github.com/xoplog/xop-go/xopat" - "github.com/xoplog/xop-go/xopbase" - "github.com/xoplog/xop-go/xopnum" - "github.com/xoplog/xop-go/xoptrace" - - "github.com/google/uuid" - "go.opentelemetry.io/otel/attribute" - oteltrace "go.opentelemetry.io/otel/trace" -) - -func xopTraceFromSpan(span oteltrace.Span) xoptrace.Trace { - var xoptrace xoptrace.Trace - sc := span.SpanContext() - xoptrace.TraceID().SetArray(sc.TraceID()) - xoptrace.SpanID().SetArray(sc.SpanID()) - xoptrace.Flags().SetArray([1]byte{byte(sc.TraceFlags())}) - // xoptrace.Version().SetArray([1]byte{0}) - return xoptrace -} - -// SpanToLog allows xop to add logs to an existing OTEL span. log.Done() will be -// ignored for this span. -func SpanToLog(ctx context.Context, name string, extraModifiers ...xop.SeedModifier) *xop.Logger { - span := oteltrace.SpanFromContext(ctx) - xoptrace := xopTraceFromSpan(span) - tracer := span.TracerProvider().Tracer("xoputil") - log := xop.NewSeed(makeSeedModifier(ctx, tracer, - xop.WithTrace(xoptrace), - xop.WithBase(&logger{ - id: "otel-" + uuid.New().String(), - doLogging: true, - ignoreDone: span, - tracer: tracer, - }), - )).SubSpan(name) - go func() { - <-ctx.Done() - log.Done() - }() - return log -} - -func (_ idGenerator) NewIDs(ctx context.Context) (oteltrace.TraceID, oteltrace.SpanID) { - override := overrideFromContext(ctx) - traceID, spanID, _ := override.Get() - return traceID, spanID -} - -func (_ idGenerator) NewSpanID(ctx context.Context, _ oteltrace.TraceID) oteltrace.SpanID { - override := overrideFromContext(ctx) - _, spanID, _ := override.Get() - return spanID -} - -// SeedModifier provides a xop.SeedModifier to set up an OTEL Tracer as a xopbase.Logger -// so that xop logs are output through the OTEL Tracer. -// -// As of the writing of this comment, the Open Telemetry Go library does not support -// logging so to use it for logging purposes, log lines are sent as span "Events". -// -// The recommended way to create a TracerProvider includes using WithBatcher to -// control the flow of data to SpanExporters. The default configuration for the Batcher -// limits spans to 128 Events each. It imposes other limits too but the default event -// limit is the one that is likely to be hit with even modest usage. -// -// Using SeedModifier, the TraceProvider does not have to have been created using IDGenerator(). -func SeedModifier(ctx context.Context, traceProvider oteltrace.TracerProvider) xop.SeedModifier { - tracer := traceProvider.Tracer("xopotel", - oteltrace.WithInstrumentationAttributes( - xopOTELVersion.String(xopotelVersionValue), - xopVersion.String(xopVersionValue), - ), - oteltrace.WithInstrumentationVersion(xopotelVersionValue), - ) - return makeSeedModifier(ctx, tracer) -} - -func makeSeedModifier(ctx context.Context, tracer oteltrace.Tracer, extraModifiers ...xop.SeedModifier) xop.SeedModifier { - modifiers := []xop.SeedModifier{ - xop.WithBase(&logger{ - id: "otel-" + uuid.New().String(), - doLogging: true, - tracer: tracer, - spanFromContext: true, - }), - xop.WithContext(ctx), - xop.WithReactive(func(ctx context.Context, seed xop.Seed, nameOrDescription string, isChildSpan bool, ts time.Time) []xop.SeedModifier { - if isChildSpan { - ctx, span := buildSpan(ctx, ts, seed.Bundle(), nameOrDescription, tracer, nil) - return []xop.SeedModifier{ - xop.WithContext(ctx), - xop.WithSpan(span.SpanContext().SpanID()), - } - } - parentCtx := ctx - bundle := seed.Bundle() - ctx, _, otelSpan := buildRequestSpan(ctx, ts, bundle, nameOrDescription, seed.SourceInfo(), tracer, nil) - if bundle.Parent.IsZero() { - parentSpan := oteltrace.SpanFromContext(parentCtx) - if parentSpan.SpanContext().HasTraceID() { - bundle.Parent.Flags().SetArray([1]byte{byte(parentSpan.SpanContext().TraceFlags())}) - bundle.Parent.TraceID().SetArray(parentSpan.SpanContext().TraceID()) - bundle.Parent.SpanID().SetArray(parentSpan.SpanContext().SpanID()) - } - bundle.State.SetString(otelSpan.SpanContext().TraceState().String()) - bundle.Trace.Flags().SetArray([1]byte{byte(otelSpan.SpanContext().TraceFlags())}) - bundle.Trace.TraceID().SetArray(otelSpan.SpanContext().TraceID()) - bundle.Trace.SpanID().SetArray(otelSpan.SpanContext().SpanID()) - } - bundle.Trace.SpanID().SetArray(otelSpan.SpanContext().SpanID()) - return []xop.SeedModifier{ - xop.WithContext(ctx), - xop.WithBundle(bundle), - } - }), - } - return xop.CombineSeedModifiers(append(modifiers, extraModifiers...)...) -} - -// BaseLogger provides SeedModifiers to set up an OTEL Tracer as a xopbase.Logger -// so that xop logs are output through the OTEL Tracer. -// -// As of the writing of this comment, the Open Telemetry Go library does not support -// logging so to use it for logging purposes, log lines are sent as span "Events". -// -// The recommended way to create a TracerProvider includes using WithBatcher to -// control the flow of data to SpanExporters. The default configuration for the Batcher -// limits spans to 128 Events each. It imposes other limits too but the default event -// limit is the one that is likely to be hit with even modest usage. -// -// The TracerProvider MUST be created with IDGenerator(). Without that, the SpanID -// created by Xop will be ignored and that will cause problems with propagation. -func BaseLogger(traceProvider oteltrace.TracerProvider) xopbase.Logger { - tracer := traceProvider.Tracer("xopotel", - oteltrace.WithInstrumentationAttributes( - xopOTELVersion.String(xopotelVersionValue), - xopVersion.String(xopVersionValue), - ), - oteltrace.WithInstrumentationVersion(xopotelVersionValue), - ) - return &logger{ - id: "otel-" + uuid.New().String(), - doLogging: true, - tracer: tracer, - spanFromContext: false, - } -} - -func (logger *logger) ID() string { return logger.id } -func (logger *logger) ReferencesKept() bool { return true } -func (logger *logger) Buffered() bool { return false } - -func buildRequestSpan(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, sourceInfo xopbase.SourceInfo, tracer oteltrace.Tracer, bufferedRequest *bufferedRequest) (context.Context, bool, oteltrace.Span) { - spanKind := oteltrace.SpanKindServer - isXOP := sourceInfo.Source != otelDataSource - if !isXOP { - // replaying data originally coming from OTEL. - // The spanKind is encoded as the namespace. - if sk, ok := spanKindFromString[sourceInfo.Namespace]; ok { - spanKind = sk - } - } - opts := []oteltrace.SpanStartOption{ - oteltrace.WithSpanKind(spanKind), - oteltrace.WithTimestamp(ts), - } - - otelStuff := bufferedRequest.getStuff(bundle, true) - opts = append(opts, otelStuff.SpanOptions()...) - - if bundle.Parent.TraceID().IsZero() { - opts = append(opts, oteltrace.WithNewRoot()) - } - ctx, otelSpan := tracer.Start(overrideIntoContext(ctx, bundle), description, opts...) - if isXOP { - if !bundle.Baggage.IsZero() { - otelSpan.SetAttributes(xopBaggage.String(bundle.Baggage.String())) - } - otelSpan.SetAttributes( - xopVersion.String(xopVersionValue), - xopOTELVersion.String(xopotelVersionValue), - xopSource.String(sourceInfo.Source+" "+sourceInfo.SourceVersion.String()), - xopNamespace.String(sourceInfo.Namespace+" "+sourceInfo.NamespaceVersion.String()), - ) - } - otelStuff.Set(otelSpan) - return ctx, isXOP, otelSpan -} - -func (logger *logger) Request(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, sourceInfo xopbase.SourceInfo) xopbase.Request { - var otelSpan oteltrace.Span - var isXOP bool - if logger.spanFromContext { - otelSpan = oteltrace.SpanFromContext(ctx) - isXOP = sourceInfo.Source != otelDataSource - } else { - ctx, isXOP, otelSpan = buildRequestSpan(ctx, ts, bundle, description, sourceInfo, logger.tracer, logger.bufferedRequest) - } - r := &request{ - span: &span{ - logger: logger, - otelSpan: otelSpan, - ctx: ctx, - isXOP: isXOP, - }, - attributesDefined: make(map[string]struct{}), - } - r.span.request = r - return r -} - -func (request *request) SetErrorReporter(f func(error)) { request.errorReporter = f } -func (request *request) Flush() {} -func (request *request) Final() {} - -func (span *span) Boring(_ bool) {} -func (span *span) ID() string { return span.logger.id } -func (span *span) Done(endTime time.Time, final bool) { - if !final { - return - } - if span.logger.ignoreDone == span.otelSpan { - // skip Done for spans passed in to SpanLog() - return - } - span.otelSpan.End(oteltrace.WithTimestamp(endTime)) -} - -func buildSpan(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, tracer oteltrace.Tracer, bufferedRequest *bufferedRequest) (context.Context, oteltrace.Span) { - opts := []oteltrace.SpanStartOption{ - oteltrace.WithTimestamp(ts), - oteltrace.WithSpanKind(oteltrace.SpanKindInternal), - } - otelStuff := bufferedRequest.getStuff(bundle, true) - opts = append(opts, otelStuff.SpanOptions()...) - ctx, otelSpan := tracer.Start(overrideIntoContext(ctx, bundle), description, opts...) - otelStuff.Set(otelSpan) - return ctx, otelSpan -} - -func (parentSpan *span) Span(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, spanSequenceCode string) xopbase.Span { - var otelSpan oteltrace.Span - if parentSpan.logger.spanFromContext { - otelSpan = oteltrace.SpanFromContext(ctx) - } else { - if parentSpan.logger.bufferedRequest != nil { - ctx = parentSpan.ctx - } - ctx, otelSpan = buildSpan(ctx, ts, bundle, description, parentSpan.logger.tracer, parentSpan.logger.bufferedRequest) - } - s := &span{ - logger: parentSpan.logger, - otelSpan: otelSpan, - ctx: ctx, - request: parentSpan.request, - isXOP: parentSpan.isXOP, - } - if parentSpan.isXOP && spanSequenceCode != "" { - otelSpan.SetAttributes(xopSpanSequence.String(spanSequenceCode)) - } - return s -} - -func (span *span) NoPrefill() xopbase.Prefilled { - return &prefilled{ - builderWithSpan: builderWithSpan{ - span: span, - }, - } -} - -func (span *span) StartPrefill() xopbase.Prefilling { - return &prefilling{ - builderWithSpan: builderWithSpan{ - span: span, - }, - } -} - -func (prefill *prefilling) PrefillComplete(msg string) xopbase.Prefilled { - prefill.builder.prefillMsg = msg - return &prefilled{ - builderWithSpan: prefill.builderWithSpan, - } -} - -func (prefilled *prefilled) Line(level xopnum.Level, ts time.Time, frames []runtime.Frame) xopbase.Line { - if !prefilled.span.logger.doLogging || !prefilled.span.otelSpan.IsRecording() { - return xopbase.SkipLine - } - // PERFORMANCE: get line from a pool - line := &line{} - line.level = level - line.span = prefilled.span - line.attributes = line.prealloc[:2] // reserving two spots at the beginnging - line.attributes = append(line.attributes, prefilled.span.spanPrefill...) - line.attributes = append(line.attributes, prefilled.attributes...) - line.prefillMsg = prefilled.prefillMsg - line.linkKey = prefilled.linkKey - line.linkValue = prefilled.linkValue - line.timestamp = ts - if len(frames) > 0 { - fs := make([]string, len(frames)) - for i, frame := range frames { - fs[i] = frame.File + ":" + strconv.Itoa(frame.Line) - } - line.attributes = append(line.attributes, xopStackTrace.StringSlice(fs)) - } - return line -} - -func (line *line) Link(k string, v xoptrace.Trace) { - if k == xopOTELLinkDetail.String() { - // Link will not be called with OTEL->XOP->OTEL so no need to - // suppress anything - return - } - line.attributes = append(line.attributes, - xopType.String("link"), - xopLinkData.String(v.String()), - ) - line.done(line.prefillMsg + k) - if line.span.logger.bufferedRequest != nil { - // add span.Links() is handled in buffered.zzzgo so - // a sub-span is not needed here. - return - } - _, tmpSpan := line.span.logger.tracer.Start(line.span.ctx, k, - oteltrace.WithLinks( - oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: v.TraceID().Array(), - SpanID: v.SpanID().Array(), - TraceFlags: oteltrace.TraceFlags(v.Flags().Array()[0]), - TraceState: emptyTraceState, // TODO: is this right? - Remote: true, // information not available - }), - }), - oteltrace.WithAttributes( - spanIsLinkEventKey.Bool(true), - ), - ) - tmpSpan.AddEvent(line.level.String(), - oteltrace.WithTimestamp(line.timestamp), - oteltrace.WithAttributes(line.attributes...)) - tmpSpan.SetAttributes(xopType.String("link-event")) - tmpSpan.End() -} - -func (line *line) Model(msg string, v xopbase.ModelArg) { - v.Encode() - line.attributes = append(line.attributes, - xopType.String("model"), - xopModelType.String(v.ModelType), - xopEncoding.String(v.Encoding.String()), - xopModel.String(string(v.Encoded)), - ) - line.done(line.prefillMsg + msg) -} - -func (line *line) Msg(msg string) { - if line.span.isXOP { - line.attributes = append(line.attributes, xopType.String("line")) - } - line.done(line.prefillMsg + msg) - // PERFORMANCE: return line to pool -} - -func (line *line) done(msg string) { - if line.span.isXOP { - line.attributes[0] = xopLineNumber.Int64(int64(atomic.AddInt32(&line.span.request.lineCount, 1))) - line.attributes[1] = xopLevel.String(line.level.String()) - } else { - line.attributes = line.attributes[2:] - } - if line.timestamp.IsZero() { - line.span.otelSpan.AddEvent(msg, - oteltrace.WithAttributes(line.attributes...)) - } else { - line.span.otelSpan.AddEvent(msg, - oteltrace.WithTimestamp(line.timestamp), - oteltrace.WithAttributes(line.attributes...)) - } -} - -var templateRE = regexp.MustCompile(`\{.+?\}`) - -func (line *line) Template(template string) { - kv := make(map[string]int) - for i, a := range line.attributes { - kv[string(a.Key)] = i - } - msg := templateRE.ReplaceAllStringFunc(template, func(k string) string { - k = k[1 : len(k)-1] - if i, ok := kv[k]; ok { - a := line.attributes[i] - switch a.Value.Type() { - case attribute.BOOL: - return strconv.FormatBool(a.Value.AsBool()) - case attribute.INT64: - return strconv.FormatInt(a.Value.AsInt64(), 10) - case attribute.FLOAT64: - return strconv.FormatFloat(a.Value.AsFloat64(), 'g', -1, 64) - case attribute.STRING: - return a.Value.AsString() - case attribute.BOOLSLICE: - return fmt.Sprint(a.Value.AsBoolSlice()) - case attribute.INT64SLICE: - return fmt.Sprint(a.Value.AsInt64Slice()) - case attribute.FLOAT64SLICE: - return fmt.Sprint(a.Value.AsFloat64Slice()) - case attribute.STRINGSLICE: - return fmt.Sprint(a.Value.AsStringSlice()) - default: - return "{" + k + "}" - } - } - return "''" - }) - line.attributes = append(line.attributes, - xopType.String("line"), - xopTemplate.String(template), - ) - line.done(line.prefillMsg + msg) -} - -func (builder *builder) Enum(k *xopat.EnumAttribute, v xopat.Enum) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.Key().String(), []string{v.String(), "enum", strconv.FormatInt(v.Int64(), 10)})) -} - -func (builder *builder) Any(k xopat.K, v xopbase.ModelArg) { - v.Encode() - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{string(v.Encoded), "any", v.Encoding.String(), v.ModelType})) -} - -func (builder *builder) Time(k xopat.K, v time.Time) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{v.Format(time.RFC3339Nano), "time"})) -} - -func (builder *builder) Duration(k xopat.K, v time.Duration) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{v.String(), "dur"})) -} - -func (builder *builder) Uint64(k xopat.K, v uint64, dt xopbase.DataType) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{strconv.FormatUint(v, 10), xopbase.DataTypeToString[dt]})) -} - -func (builder *builder) Int64(k xopat.K, v int64, dt xopbase.DataType) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{strconv.FormatInt(v, 10), xopbase.DataTypeToString[dt]})) -} - -func (builder *builder) Float64(k xopat.K, v float64, dt xopbase.DataType) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{strconv.FormatFloat(v, 'g', -1, 64), xopbase.DataTypeToString[dt]})) -} - -func (builder *builder) String(k xopat.K, v string, dt xopbase.DataType) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{v, xopbase.DataTypeToString[dt]})) -} - -func (builder *builder) Bool(k xopat.K, v bool) { - builder.attributes = append(builder.attributes, attribute.Bool(k.String(), v)) -} - -var skipIfOTEL = map[string]struct{}{ - otelReplayStuff.Key().String(): {}, -} - -func (span *span) MetadataAny(k *xopat.AnyAttribute, v xopbase.ModelArg) { - if k.Key().String() == otelReplayStuff.Key().String() { - return - } - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - enc, err := v.MarshalJSON() - var value string - if err != nil { - value = fmt.Sprintf("[zopotel] could not marshal %T value: %s", v, err) - } else { - value = string(enc) - } - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - span.otelSpan.SetAttributes(attribute.String(key.String(), value)) - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - seen := make(map[string]struct{}) - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - seen := seenRaw.(map[string]struct{}) - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - if span.priorStringSlices == nil { - span.priorStringSlices = make(map[string][]string) - } - s := span.priorStringSlices[key.String()] - s = append(s, value) - span.priorStringSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.StringSlice(key.String(), s)) -} - -func (span *span) MetadataBool(k *xopat.BoolAttribute, v bool) { - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - value := v - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - span.otelSpan.SetAttributes(attribute.Bool(key.String(), value)) - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - seen := make(map[bool]struct{}) - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - seen := seenRaw.(map[bool]struct{}) - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - if span.priorBoolSlices == nil { - span.priorBoolSlices = make(map[string][]bool) - } - s := span.priorBoolSlices[key.String()] - s = append(s, value) - span.priorBoolSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.BoolSlice(key.String(), s)) -} - -func (span *span) MetadataEnum(k *xopat.EnumAttribute, v xopat.Enum) { - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - value := v.String() + "/" + strconv.FormatInt(v.Int64(), 10) - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - span.otelSpan.SetAttributes(attribute.String(key.String(), value)) - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - seen := make(map[string]struct{}) - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - seen := seenRaw.(map[string]struct{}) - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - if span.priorStringSlices == nil { - span.priorStringSlices = make(map[string][]string) - } - s := span.priorStringSlices[key.String()] - s = append(s, value) - span.priorStringSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.StringSlice(key.String(), s)) -} - -func (span *span) MetadataFloat64(k *xopat.Float64Attribute, v float64) { - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - value := v - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - span.otelSpan.SetAttributes(attribute.Float64(key.String(), value)) - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - seen := make(map[float64]struct{}) - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - seen := seenRaw.(map[float64]struct{}) - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - if span.priorFloat64Slices == nil { - span.priorFloat64Slices = make(map[string][]float64) - } - s := span.priorFloat64Slices[key.String()] - s = append(s, value) - span.priorFloat64Slices[key.String()] = s - span.otelSpan.SetAttributes(attribute.Float64Slice(key.String(), s)) -} - -func (span *span) MetadataInt64(k *xopat.Int64Attribute, v int64) { - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - value := v - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - if k.SubType() == xopat.AttributeTypeDuration { - span.otelSpan.SetAttributes(attribute.String(key.String(), time.Duration(value).String())) - } else { - span.otelSpan.SetAttributes(attribute.Int64(key.String(), value)) - } - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - seen := make(map[int64]struct{}) - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - seen := seenRaw.(map[int64]struct{}) - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - if k.SubType() == xopat.AttributeTypeDuration { - if span.priorStringSlices == nil { - span.priorStringSlices = make(map[string][]string) - } - s := span.priorStringSlices[key.String()] - s = append(s, time.Duration(value).String()) - span.priorStringSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.StringSlice(key.String(), s)) - } else { - if span.priorInt64Slices == nil { - span.priorInt64Slices = make(map[string][]int64) - } - s := span.priorInt64Slices[key.String()] - s = append(s, value) - span.priorInt64Slices[key.String()] = s - span.otelSpan.SetAttributes(attribute.Int64Slice(key.String(), s)) - } -} - -func (span *span) MetadataLink(k *xopat.LinkAttribute, v xoptrace.Trace) { - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - if k.Key().String() == otelLink.Key().String() { - return - } - value := v.String() - if span.logger.bufferedRequest == nil { - _, tmpSpan := span.logger.tracer.Start(span.ctx, k.Key().String(), - oteltrace.WithLinks( - oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: v.TraceID().Array(), - SpanID: v.SpanID().Array(), - TraceFlags: oteltrace.TraceFlags(v.Flags().Array()[0]), - TraceState: emptyTraceState, // TODO: is this right? - Remote: true, // information not available - }), - Attributes: []attribute.KeyValue{ - xopLinkMetadataKey.String(key.String()), - }, - }), - oteltrace.WithAttributes( - spanIsLinkAttributeKey.Bool(true), - ), - ) - tmpSpan.SetAttributes(spanIsLinkAttributeKey.Bool(true)) - tmpSpan.End() - } - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - span.otelSpan.SetAttributes(attribute.String(key.String(), value)) - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - seen := make(map[string]struct{}) - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - seen := seenRaw.(map[string]struct{}) - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - if span.priorStringSlices == nil { - span.priorStringSlices = make(map[string][]string) - } - s := span.priorStringSlices[key.String()] - s = append(s, value) - span.priorStringSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.StringSlice(key.String(), s)) -} - -func (span *span) MetadataString(k *xopat.StringAttribute, v string) { - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - value := v - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - span.otelSpan.SetAttributes(attribute.String(key.String(), value)) - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - seen := make(map[string]struct{}) - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - seen := seenRaw.(map[string]struct{}) - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - if span.priorStringSlices == nil { - span.priorStringSlices = make(map[string][]string) - } - s := span.priorStringSlices[key.String()] - s = append(s, value) - span.priorStringSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.StringSlice(key.String(), s)) -} - -func (span *span) MetadataTime(k *xopat.TimeAttribute, v time.Time) { - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - value := v.Format(time.RFC3339Nano) - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - span.otelSpan.SetAttributes(attribute.String(key.String(), value)) - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - seen := make(map[string]struct{}) - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - seen := seenRaw.(map[string]struct{}) - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - if span.priorStringSlices == nil { - span.priorStringSlices = make(map[string][]string) - } - s := span.priorStringSlices[key.String()] - s = append(s, value) - span.priorStringSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.StringSlice(key.String(), s)) -} - -var spanKindFromString = map[string]oteltrace.SpanKind{ - oteltrace.SpanKindClient.String(): oteltrace.SpanKindClient, - oteltrace.SpanKindConsumer.String(): oteltrace.SpanKindConsumer, - oteltrace.SpanKindInternal.String(): oteltrace.SpanKindInternal, - oteltrace.SpanKindProducer.String(): oteltrace.SpanKindProducer, - oteltrace.SpanKindServer.String(): oteltrace.SpanKindServer, -} diff --git a/xopotel/otel.zzzgo b/xopotel/otel.zzzgo deleted file mode 100644 index 90ee7939..00000000 --- a/xopotel/otel.zzzgo +++ /dev/null @@ -1,632 +0,0 @@ -// TEMPLATE-FILE -// TEMPLATE-FILE - -package xopotel - -import ( - "context" - "fmt" - "regexp" - "runtime" - "strconv" - "sync/atomic" - "time" - - "github.com/xoplog/xop-go" - "github.com/xoplog/xop-go/xopbase" - "github.com/xoplog/xop-go/xopnum" - "github.com/xoplog/xop-go/xoptrace" - - "github.com/google/uuid" - "go.opentelemetry.io/otel/attribute" - oteltrace "go.opentelemetry.io/otel/trace" -) - -func xopTraceFromSpan(span oteltrace.Span) xoptrace.Trace { - var xoptrace xoptrace.Trace - sc := span.SpanContext() - xoptrace.TraceID().SetArray(sc.TraceID()) - xoptrace.SpanID().SetArray(sc.SpanID()) - xoptrace.Flags().SetArray([1]byte{byte(sc.TraceFlags())}) - // xoptrace.Version().SetArray([1]byte{0}) - return xoptrace -} - -// SpanToLog allows xop to add logs to an existing OTEL span. log.Done() will be -// ignored for this span. -func SpanToLog(ctx context.Context, name string, extraModifiers ...xop.SeedModifier) *xop.Logger { - span := oteltrace.SpanFromContext(ctx) - xoptrace := xopTraceFromSpan(span) - tracer := span.TracerProvider().Tracer("xoputil") - log := xop.NewSeed(makeSeedModifier(ctx, tracer, - xop.WithTrace(xoptrace), - xop.WithBase(&logger{ - id: "otel-" + uuid.New().String(), - doLogging: true, - ignoreDone: span, - tracer: tracer, - }), - )).SubSpan(name) - go func() { - <-ctx.Done() - log.Done() - }() - return log -} - -func (_ idGenerator) NewIDs(ctx context.Context) (oteltrace.TraceID, oteltrace.SpanID) { - override := overrideFromContext(ctx) - traceID, spanID, _ := override.Get() - return traceID, spanID -} - -func (_ idGenerator) NewSpanID(ctx context.Context, _ oteltrace.TraceID) oteltrace.SpanID { - override := overrideFromContext(ctx) - _, spanID, _ := override.Get() - return spanID -} - -// SeedModifier provides a xop.SeedModifier to set up an OTEL Tracer as a xopbase.Logger -// so that xop logs are output through the OTEL Tracer. -// -// As of the writing of this comment, the Open Telemetry Go library does not support -// logging so to use it for logging purposes, log lines are sent as span "Events". -// -// The recommended way to create a TracerProvider includes using WithBatcher to -// control the flow of data to SpanExporters. The default configuration for the Batcher -// limits spans to 128 Events each. It imposes other limits too but the default event -// limit is the one that is likely to be hit with even modest usage. -// -// Using SeedModifier, the TraceProvider does not have to have been created using IDGenerator(). -func SeedModifier(ctx context.Context, traceProvider oteltrace.TracerProvider) xop.SeedModifier { - tracer := traceProvider.Tracer("xopotel", - oteltrace.WithInstrumentationAttributes( - xopOTELVersion.String(xopotelVersionValue), - xopVersion.String(xopVersionValue), - ), - oteltrace.WithInstrumentationVersion(xopotelVersionValue), - ) - return makeSeedModifier(ctx, tracer) -} - -func makeSeedModifier(ctx context.Context, tracer oteltrace.Tracer, extraModifiers ...xop.SeedModifier) xop.SeedModifier { - modifiers := []xop.SeedModifier{ - xop.WithBase(&logger{ - id: "otel-" + uuid.New().String(), - doLogging: true, - tracer: tracer, - spanFromContext: true, - }), - xop.WithContext(ctx), - xop.WithReactive(func(ctx context.Context, seed xop.Seed, nameOrDescription string, isChildSpan bool, ts time.Time) []xop.SeedModifier { - if isChildSpan { - ctx, span := buildSpan(ctx, ts, seed.Bundle(), nameOrDescription, tracer, nil) - return []xop.SeedModifier{ - xop.WithContext(ctx), - xop.WithSpan(span.SpanContext().SpanID()), - } - } - parentCtx := ctx - bundle := seed.Bundle() - ctx, _, otelSpan := buildRequestSpan(ctx, ts, bundle, nameOrDescription, seed.SourceInfo(), tracer, nil) - if bundle.Parent.IsZero() { - parentSpan := oteltrace.SpanFromContext(parentCtx) - if parentSpan.SpanContext().HasTraceID() { - bundle.Parent.Flags().SetArray([1]byte{byte(parentSpan.SpanContext().TraceFlags())}) - bundle.Parent.TraceID().SetArray(parentSpan.SpanContext().TraceID()) - bundle.Parent.SpanID().SetArray(parentSpan.SpanContext().SpanID()) - } - bundle.State.SetString(otelSpan.SpanContext().TraceState().String()) - bundle.Trace.Flags().SetArray([1]byte{byte(otelSpan.SpanContext().TraceFlags())}) - bundle.Trace.TraceID().SetArray(otelSpan.SpanContext().TraceID()) - bundle.Trace.SpanID().SetArray(otelSpan.SpanContext().SpanID()) - } - bundle.Trace.SpanID().SetArray(otelSpan.SpanContext().SpanID()) - return []xop.SeedModifier{ - xop.WithContext(ctx), - xop.WithBundle(bundle), - } - }), - } - return xop.CombineSeedModifiers(append(modifiers, extraModifiers...)...) -} - -// BaseLogger provides SeedModifiers to set up an OTEL Tracer as a xopbase.Logger -// so that xop logs are output through the OTEL Tracer. -// -// As of the writing of this comment, the Open Telemetry Go library does not support -// logging so to use it for logging purposes, log lines are sent as span "Events". -// -// The recommended way to create a TracerProvider includes using WithBatcher to -// control the flow of data to SpanExporters. The default configuration for the Batcher -// limits spans to 128 Events each. It imposes other limits too but the default event -// limit is the one that is likely to be hit with even modest usage. -// -// The TracerProvider MUST be created with IDGenerator(). Without that, the SpanID -// created by Xop will be ignored and that will cause problems with propagation. -func BaseLogger(traceProvider oteltrace.TracerProvider) xopbase.Logger { - tracer := traceProvider.Tracer("xopotel", - oteltrace.WithInstrumentationAttributes( - xopOTELVersion.String(xopotelVersionValue), - xopVersion.String(xopVersionValue), - ), - oteltrace.WithInstrumentationVersion(xopotelVersionValue), - ) - return &logger{ - id: "otel-" + uuid.New().String(), - doLogging: true, - tracer: tracer, - spanFromContext: false, - } -} - -func (logger *logger) ID() string { return logger.id } -func (logger *logger) ReferencesKept() bool { return true } -func (logger *logger) Buffered() bool { return false } - -func buildRequestSpan(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, sourceInfo xopbase.SourceInfo, tracer oteltrace.Tracer, bufferedRequest *bufferedRequest) (context.Context, bool, oteltrace.Span) { - spanKind := oteltrace.SpanKindServer - isXOP := sourceInfo.Source != otelDataSource - if !isXOP { - // replaying data originally coming from OTEL. - // The spanKind is encoded as the namespace. - if sk, ok := spanKindFromString[sourceInfo.Namespace]; ok { - spanKind = sk - } - } - opts := []oteltrace.SpanStartOption{ - oteltrace.WithSpanKind(spanKind), - oteltrace.WithTimestamp(ts), - } - - otelStuff := bufferedRequest.getStuff(bundle, true) - opts = append(opts, otelStuff.SpanOptions()...) - - if bundle.Parent.TraceID().IsZero() { - opts = append(opts, oteltrace.WithNewRoot()) - } - ctx, otelSpan := tracer.Start(overrideIntoContext(ctx, bundle), description, opts...) - if isXOP { - if !bundle.Baggage.IsZero() { - otelSpan.SetAttributes(xopBaggage.String(bundle.Baggage.String())) - } - otelSpan.SetAttributes( - xopVersion.String(xopVersionValue), - xopOTELVersion.String(xopotelVersionValue), - xopSource.String(sourceInfo.Source+" "+sourceInfo.SourceVersion.String()), - xopNamespace.String(sourceInfo.Namespace+" "+sourceInfo.NamespaceVersion.String()), - ) - } - otelStuff.Set(otelSpan) - return ctx, isXOP, otelSpan -} - -func (logger *logger) Request(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, sourceInfo xopbase.SourceInfo) xopbase.Request { - var otelSpan oteltrace.Span - var isXOP bool - if logger.spanFromContext { - otelSpan = oteltrace.SpanFromContext(ctx) - isXOP = sourceInfo.Source != otelDataSource - } else { - ctx, isXOP, otelSpan = buildRequestSpan(ctx, ts, bundle, description, sourceInfo, logger.tracer, logger.bufferedRequest) - } - r := &request{ - span: &span{ - logger: logger, - otelSpan: otelSpan, - ctx: ctx, - isXOP: isXOP, - }, - attributesDefined: make(map[string]struct{}), - } - r.span.request = r - return r -} - -func (request *request) SetErrorReporter(f func(error)) { request.errorReporter = f } -func (request *request) Flush() {} -func (request *request) Final() {} - -func (span *span) Boring(_ bool) {} -func (span *span) ID() string { return span.logger.id } -func (span *span) Done(endTime time.Time, final bool) { - if !final { - return - } - if span.logger.ignoreDone == span.otelSpan { - // skip Done for spans passed in to SpanLog() - return - } - span.otelSpan.End(oteltrace.WithTimestamp(endTime)) -} - -func buildSpan(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, tracer oteltrace.Tracer, bufferedRequest *bufferedRequest) (context.Context, oteltrace.Span) { - opts := []oteltrace.SpanStartOption{ - oteltrace.WithTimestamp(ts), - oteltrace.WithSpanKind(oteltrace.SpanKindInternal), - } - otelStuff := bufferedRequest.getStuff(bundle, true) - opts = append(opts, otelStuff.SpanOptions()...) - ctx, otelSpan := tracer.Start(overrideIntoContext(ctx, bundle), description, opts...) - otelStuff.Set(otelSpan) - return ctx, otelSpan -} - -func (parentSpan *span) Span(ctx context.Context, ts time.Time, bundle xoptrace.Bundle, description string, spanSequenceCode string) xopbase.Span { - var otelSpan oteltrace.Span - if parentSpan.logger.spanFromContext { - otelSpan = oteltrace.SpanFromContext(ctx) - } else { - if parentSpan.logger.bufferedRequest != nil { - ctx = parentSpan.ctx - } - ctx, otelSpan = buildSpan(ctx, ts, bundle, description, parentSpan.logger.tracer, parentSpan.logger.bufferedRequest) - } - s := &span{ - logger: parentSpan.logger, - otelSpan: otelSpan, - ctx: ctx, - request: parentSpan.request, - isXOP: parentSpan.isXOP, - } - if parentSpan.isXOP && spanSequenceCode != "" { - otelSpan.SetAttributes(xopSpanSequence.String(spanSequenceCode)) - } - return s -} - -func (span *span) NoPrefill() xopbase.Prefilled { - return &prefilled{ - builderWithSpan: builderWithSpan{ - span: span, - }, - } -} - -func (span *span) StartPrefill() xopbase.Prefilling { - return &prefilling{ - builderWithSpan: builderWithSpan{ - span: span, - }, - } -} - -func (prefill *prefilling) PrefillComplete(msg string) xopbase.Prefilled { - prefill.builder.prefillMsg = msg - return &prefilled{ - builderWithSpan: prefill.builderWithSpan, - } -} - -func (prefilled *prefilled) Line(level xopnum.Level, ts time.Time, frames []runtime.Frame) xopbase.Line { - if !prefilled.span.logger.doLogging || !prefilled.span.otelSpan.IsRecording() { - return xopbase.SkipLine - } - // PERFORMANCE: get line from a pool - line := &line{} - line.level = level - line.span = prefilled.span - line.attributes = line.prealloc[:2] // reserving two spots at the beginnging - line.attributes = append(line.attributes, prefilled.span.spanPrefill...) - line.attributes = append(line.attributes, prefilled.attributes...) - line.prefillMsg = prefilled.prefillMsg - line.linkKey = prefilled.linkKey - line.linkValue = prefilled.linkValue - line.timestamp = ts - if len(frames) > 0 { - fs := make([]string, len(frames)) - for i, frame := range frames { - fs[i] = frame.File + ":" + strconv.Itoa(frame.Line) - } - line.attributes = append(line.attributes, xopStackTrace.StringSlice(fs)) - } - return line -} - -func (line *line) Link(k string, v xoptrace.Trace) { - if k == xopOTELLinkDetail.String() { - // Link will not be called with OTEL->XOP->OTEL so no need to - // suppress anything - return - } - line.attributes = append(line.attributes, - xopType.String("link"), - xopLinkData.String(v.String()), - ) - line.done(line.prefillMsg + k) - if line.span.logger.bufferedRequest != nil { - // add span.Links() is handled in buffered.zzzgo so - // a sub-span is not needed here. - return - } - _, tmpSpan := line.span.logger.tracer.Start(line.span.ctx, k, - oteltrace.WithLinks( - oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: v.TraceID().Array(), - SpanID: v.SpanID().Array(), - TraceFlags: oteltrace.TraceFlags(v.Flags().Array()[0]), - TraceState: emptyTraceState, // TODO: is this right? - Remote: true, // information not available - }), - }), - oteltrace.WithAttributes( - spanIsLinkEventKey.Bool(true), - ), - ) - tmpSpan.AddEvent(line.level.String(), - oteltrace.WithTimestamp(line.timestamp), - oteltrace.WithAttributes(line.attributes...)) - tmpSpan.SetAttributes(xopType.String("link-event")) - tmpSpan.End() -} - -func (line *line) Model(msg string, v xopbase.ModelArg) { - v.Encode() - line.attributes = append(line.attributes, - xopType.String("model"), - xopModelType.String(v.ModelType), - xopEncoding.String(v.Encoding.String()), - xopModel.String(string(v.Encoded)), - ) - line.done(line.prefillMsg + msg) -} - -func (line *line) Msg(msg string) { - if line.span.isXOP { - line.attributes = append(line.attributes, xopType.String("line")) - } - line.done(line.prefillMsg + msg) - // PERFORMANCE: return line to pool -} - -func (line *line) done(msg string) { - if line.span.isXOP { - line.attributes[0] = xopLineNumber.Int64(int64(atomic.AddInt32(&line.span.request.lineCount, 1))) - line.attributes[1] = xopLevel.String(line.level.String()) - } else { - line.attributes = line.attributes[2:] - } - if line.timestamp.IsZero() { - line.span.otelSpan.AddEvent(msg, - oteltrace.WithAttributes(line.attributes...)) - } else { - line.span.otelSpan.AddEvent(msg, - oteltrace.WithTimestamp(line.timestamp), - oteltrace.WithAttributes(line.attributes...)) - } -} - -var templateRE = regexp.MustCompile(`\{.+?\}`) - -func (line *line) Template(template string) { - kv := make(map[string]int) - for i, a := range line.attributes { - kv[string(a.Key)] = i - } - msg := templateRE.ReplaceAllStringFunc(template, func(k string) string { - k = k[1 : len(k)-1] - if i, ok := kv[k]; ok { - a := line.attributes[i] - switch a.Value.Type() { - case attribute.BOOL: - return strconv.FormatBool(a.Value.AsBool()) - case attribute.INT64: - return strconv.FormatInt(a.Value.AsInt64(), 10) - case attribute.FLOAT64: - return strconv.FormatFloat(a.Value.AsFloat64(), 'g', -1, 64) - case attribute.STRING: - return a.Value.AsString() - case attribute.BOOLSLICE: - return fmt.Sprint(a.Value.AsBoolSlice()) - case attribute.INT64SLICE: - return fmt.Sprint(a.Value.AsInt64Slice()) - case attribute.FLOAT64SLICE: - return fmt.Sprint(a.Value.AsFloat64Slice()) - case attribute.STRINGSLICE: - return fmt.Sprint(a.Value.AsStringSlice()) - default: - return "{" + k + "}" - } - } - return "''" - }) - line.attributes = append(line.attributes, - xopType.String("line"), - xopTemplate.String(template), - ) - line.done(line.prefillMsg + msg) -} - -func (builder *builder) Enum(k *xopat.EnumAttribute, v xopat.Enum) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.Key().String(), []string{v.String(), "enum", strconv.FormatInt(v.Int64(), 10)})) -} - -func (builder *builder) Any(k xopat.K, v xopbase.ModelArg) { - v.Encode() - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{string(v.Encoded), "any", v.Encoding.String(), v.ModelType})) -} - -func (builder *builder) Time(k xopat.K, v time.Time) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{v.Format(time.RFC3339Nano), "time"})) -} - -func (builder *builder) Duration(k xopat.K, v time.Duration) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{v.String(), "dur"})) -} - -func (builder *builder) Uint64(k xopat.K, v uint64, dt xopbase.DataType) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{strconv.FormatUint(v, 10), xopbase.DataTypeToString[dt]})) -} - -func (builder *builder) Int64(k xopat.K, v int64, dt xopbase.DataType) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{strconv.FormatInt(v, 10), xopbase.DataTypeToString[dt]})) -} - -func (builder *builder) Float64(k xopat.K, v float64, dt xopbase.DataType) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{strconv.FormatFloat(v, 'g', -1, 64), xopbase.DataTypeToString[dt]})) -} - -func (builder *builder) String(k xopat.K, v string, dt xopbase.DataType) { - builder.attributes = append(builder.attributes, attribute.StringSlice(k.String(), []string{v, xopbase.DataTypeToString[dt]})) -} - -func (builder *builder) Bool(k xopat.K, v bool) { - builder.attributes = append(builder.attributes, attribute.Bool(k.String(), v)) -} - -var skipIfOTEL = map[string]struct{}{ - otelReplayStuff.Key().String(): {}, -} - -// MACRO BaseAttribute -func (span *span) MetadataZZZ(k *xopat.ZZZAttribute, v zzz) { - //CONDITIONAL ONLY:Any - if k.Key().String() == otelReplayStuff.Key().String() { - return - } - //END CONDITIONAL - key := k.Key() - if span.isXOP { - if _, ok := span.request.attributesDefined[key.String()]; !ok { - if k.Description() != xopSynthesizedForOTEL { - span.request.otelSpan.SetAttributes(attribute.String(attributeDefinitionPrefix+key.String(), k.DefinitionJSONString())) - span.request.attributesDefined[key.String()] = struct{}{} - } - } - } - //CONDITIONAL ONLY:Link - if k.Key().String() == otelLink.Key().String() { - return - } - value := v.String() - if span.logger.bufferedRequest == nil { - _, tmpSpan := span.logger.tracer.Start(span.ctx, k.Key().String(), - oteltrace.WithLinks( - oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: v.TraceID().Array(), - SpanID: v.SpanID().Array(), - TraceFlags: oteltrace.TraceFlags(v.Flags().Array()[0]), - TraceState: emptyTraceState, // TODO: is this right? - Remote: true, // information not available - }), - Attributes: []attribute.KeyValue{ - xopLinkMetadataKey.String(key.String()), - }, - }), - oteltrace.WithAttributes( - spanIsLinkAttributeKey.Bool(true), - ), - ) - tmpSpan.SetAttributes(spanIsLinkAttributeKey.Bool(true)) - tmpSpan.End() - } - //CONDITIONAL ONLY:Enum - value := v.String() + "/" + strconv.FormatInt(v.Int64(), 10) - //CONDITIONAL ONLY:Time - value := v.Format(time.RFC3339Nano) - //CONDITIONAL ONLY:Any - enc, err := v.MarshalJSON() - var value string - if err != nil { - value = fmt.Sprintf("[zopotel] could not marshal %T value: %s", v, err) - } else { - value = string(enc) - } - //CONDITIONAL ONLY:Int64,String,Float64,Bool - value := v - //END CONDITIONAL - if !k.Multiple() { - if k.Locked() { - span.lock.Lock() - defer span.lock.Unlock() - if span.hasPrior == nil { - span.hasPrior = make(map[string]struct{}) - } - if _, ok := span.hasPrior[key.String()]; ok { - return - } - span.hasPrior[key.String()] = struct{}{} - } - //CONDITIONAL ONLY:Enum,Time,String,Any,Link - span.otelSpan.SetAttributes(attribute.String(key.String(), value)) - //CONDITIONAL ONLY:Int64 - if k.SubType() == xopat.AttributeTypeDuration { - span.otelSpan.SetAttributes(attribute.String(key.String(), time.Duration(value).String())) - } else { - span.otelSpan.SetAttributes(attribute.ZZZ(key.String(), value)) - } - //CONDITIONAL SKIP:Enum,Time,String,Any,Link,Int64 - span.otelSpan.SetAttributes(attribute.ZZZ(key.String(), value)) - //END CONDITIONAL - return - } - span.lock.Lock() - defer span.lock.Unlock() - if k.Distinct() { - if span.metadataSeen == nil { - span.metadataSeen = make(map[string]interface{}) - } - seenRaw, ok := span.metadataSeen[key.String()] - if !ok { - //CONDITIONAL ONLY:Enum,Time,String,Any,Link - seen := make(map[string]struct{}) - //ELSE CONDITIONAL - seen := make(map[zzz]struct{}) - //END CONDITIONAL - span.metadataSeen[key.String()] = seen - seen[value] = struct{}{} - } else { - //CONDITIONAL ONLY:Enum,Time,String,Any,Link - seen := seenRaw.(map[string]struct{}) - //ELSE CONDITIONAL - seen := seenRaw.(map[zzz]struct{}) - //END CONDITIONAL - if _, ok := seen[value]; ok { - return - } - seen[value] = struct{}{} - } - } - //CONDITIONAL ONLY:Enum,Time,String,Any,Link - if span.priorStringSlices == nil { - span.priorStringSlices = make(map[string][]string) - } - s := span.priorStringSlices[key.String()] - s = append(s, value) - span.priorStringSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.StringSlice(key.String(), s)) - //CONDITIONAL ONLY:Int64 - if k.SubType() == xopat.AttributeTypeDuration { - if span.priorStringSlices == nil { - span.priorStringSlices = make(map[string][]string) - } - s := span.priorStringSlices[key.String()] - s = append(s, time.Duration(value).String()) - span.priorStringSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.StringSlice(key.String(), s)) - } else { - if span.priorZZZSlices == nil { - span.priorZZZSlices = make(map[string][]zzz) - } - s := span.priorZZZSlices[key.String()] - s = append(s, value) - span.priorZZZSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.ZZZSlice(key.String(), s)) - } - //CONDITIONAL SKIP:Enum,Time,String,Any,Link,Int64 - if span.priorZZZSlices == nil { - span.priorZZZSlices = make(map[string][]zzz) - } - s := span.priorZZZSlices[key.String()] - s = append(s, value) - span.priorZZZSlices[key.String()] = s - span.otelSpan.SetAttributes(attribute.ZZZSlice(key.String(), s)) - //END CONDITIONAL -} - -var spanKindFromString = map[string]oteltrace.SpanKind{ - // MACRO OTELSpanKinds - oteltrace.ZZZ.String(): oteltrace.ZZZ, -} diff --git a/xopotel/otel_test.go b/xopotel/otel_test.go deleted file mode 100644 index 12a3486b..00000000 --- a/xopotel/otel_test.go +++ /dev/null @@ -1,380 +0,0 @@ -package xopotel_test - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "testing" - "time" - - "github.com/xoplog/xop-go" - "github.com/xoplog/xop-go/xopat" - "github.com/xoplog/xop-go/xopbytes" - "github.com/xoplog/xop-go/xopjson" - "github.com/xoplog/xop-go/xopotel" - "github.com/xoplog/xop-go/xopotel/xopoteltest" - "github.com/xoplog/xop-go/xoptest" - "github.com/xoplog/xop-go/xoptest/xoptestutil" - "github.com/xoplog/xop-go/xoptrace" - "github.com/xoplog/xop-go/xoputil" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" -) - -func TestSingleLineOTEL(t *testing.T) { - var buffer xoputil.Buffer - - exporter, err := stdouttrace.New( - stdouttrace.WithWriter(&buffer), - stdouttrace.WithPrettyPrint(), - ) - require.NoError(t, err, "exporter") - - tracerProvider := sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), - ) - ctx := context.Background() - defer func() { - err := tracerProvider.Shutdown(ctx) - assert.NoError(t, err, "shutdown") - }() - - tracer := tracerProvider.Tracer("") - - ctx, span := tracer.Start(ctx, "test-span") - log := xopotel.SpanToLog(ctx, "test-span") - log.Alert().String(xopat.K("foo"), "bar").Int(xopat.K("blast"), 99).Msg("a test line") - log.Done() - span.End() - tracerProvider.ForceFlush(context.Background()) - t.Log("logged:", buffer.String()) - assert.NotEmpty(t, buffer.String()) -} - -const jsonToo = false -const otelToo = true - -func TestOTELBaseLoggerReplay(t *testing.T) { - cases := []struct { - name string - idGen bool - useUnhacker bool - useBaseLogger bool - }{ - { - name: "seedModifier-with-id", - idGen: true, - }, - { - name: "seedModifier-without-id", - idGen: false, - }, - { - name: "seedModifier-with-unhacker-and-id", - idGen: true, - useUnhacker: true, - }, - { - name: "baselogger", - idGen: true, - useUnhacker: false, - useBaseLogger: true, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - for _, mc := range xoptestutil.MessageCases { - mc := mc - if mc.SkipOTEL { - continue - } - t.Run(mc.Name, func(t *testing.T) { - - var tpo []sdktrace.TracerProviderOption - - var jBuffer xoputil.Buffer - if jsonToo { - jLog := xopjson.New( - xopbytes.WriteToIOWriter(&jBuffer), - ) - - jExporter := xopotel.ExportToXOP(jLog) - tpo = append(tpo, sdktrace.WithBatcher(jExporter)) - } - - rLog := xoptest.New(t) - rLog.SetPrefix("REPLAY ") - exporter := xopotel.ExportToXOP(rLog) - if tc.useUnhacker { - unhacker := xopotel.NewUnhacker(exporter) - tpo = append(tpo, sdktrace.WithBatcher(unhacker)) - } else { - tpo = append(tpo, sdktrace.WithBatcher(exporter)) - } - - var buffer xoputil.Buffer - if otelToo { - otelExporter, err := stdouttrace.New( - stdouttrace.WithWriter(&buffer), - stdouttrace.WithPrettyPrint(), - ) - require.NoError(t, err, "exporter") - tpo = append(tpo, sdktrace.WithBatcher(otelExporter)) - } - - if tc.idGen { - tpo = append(tpo, xopotel.IDGenerator()) - } - - tracerProvider := sdktrace.NewTracerProvider(tpo...) - ctx, cancel := context.WithCancel(context.Background()) - defer func() { - err := tracerProvider.Shutdown(context.Background()) - assert.NoError(t, err, "shutdown") - }() - - tLog := xoptest.New(t) - - var seed xop.Seed - if tc.useBaseLogger { - seed = xop.NewSeed( - xop.WithBase(tLog), - xop.WithBase(xopotel.BaseLogger(tracerProvider)), - ) - } else { - seed = xop.NewSeed( - xop.WithBase(tLog), - xopotel.SeedModifier(ctx, tracerProvider), - ) - } - if len(mc.SeedMods) != 0 { - t.Logf("Applying %d extra seed mods", len(mc.SeedMods)) - seed = seed.Copy(mc.SeedMods...) - } - log := seed.Request(t.Name()) - mc.Do(t, log, tLog) - - cancel() - tracerProvider.ForceFlush(context.Background()) - - if otelToo { - t.Log("logged:", buffer.String()) - } - if jsonToo { - t.Log("Jlogged:", jBuffer.String()) - } - - t.Log("verify replay equals original") - xoptestutil.VerifyTestReplay(t, tLog, rLog) - }) - } - }) - } -} - -// TestOTELRoundTrip does a round trip of logging: -// -// test OTEL log actions -// | -// v -// OTEL -> JSON -> unpack "origin" -// | -// v -// ExportToXOP -// | -// v -// combinedBaseLogger -> xoptest.Logger -> xopcon.Logger -> "O" -// | -// v -// xopotel.BufferedBaseLogger -// | -// v -// OTEL -> JSON -> unpack "replay" -// | -// v -// ExportToXOP -// | -// v -// xoptest.Logger -> xopcon.Logger "R" -// -// Do we get the same JSON? -func TestOTELRoundTrip(t *testing.T) { - exampleResource := resource.NewWithAttributes("http://test/foo", - attribute.String("environment", "demo"), - ) - - enc, e := json.Marshal(exampleResource) - require.NoError(t, e) - t.Log("example resource", string(enc)) - - // JSON created directly by OTEL (no XOP involved) lands here - var origin bytes.Buffer - originExporter, err := stdouttrace.New( - stdouttrace.WithWriter(&origin), - ) - require.NoError(t, err) - - // The final resulting JSON lands here after going to XOP and back to OTEL - var replay bytes.Buffer - replayExporter, err := stdouttrace.New( - stdouttrace.WithWriter(&replay), - ) - require.NoError(t, err) - - exporterWrapper := xopotel.NewBufferedReplayExporterWrapper() - - wrappedReplayExporter := exporterWrapper.WrapExporter(replayExporter) - - reExportXoptest := xoptest.New(t) - reExportXoptest.SetPrefix("R:") - - reExportToXop := xopotel.ExportToXOP(reExportXoptest) - - // This is the base Logger that writes to OTEL - replayBaseLogger := exporterWrapper.BufferedReplayLogger( - // Notice: WithResource() is not used here since - // this will come from the replay logger. - sdktrace.WithBatcher(wrappedReplayExporter), - sdktrace.WithBatcher(reExportToXop), - ) - - // This provides some text output - testLogger := xoptest.New(t) - testLogger.SetPrefix("O:") - - // This combines the replayBaseLogger with an xoptest.Logger so we can see the output - combinedReplayBaseLogger := xop.CombineBaseLoggers(replayBaseLogger, testLogger) - - // This is the OTEL exporter that writes to the replay base logger - xopotelExporter := xopotel.ExportToXOP(combinedReplayBaseLogger) - - // This is the TracerProvider that writes to the unmodified JSON exporter - // and also to the exporter that writes to XOP and back to OTEL - tpOrigin := sdktrace.NewTracerProvider( - sdktrace.WithResource(exampleResource), - sdktrace.WithBatcher(originExporter), - sdktrace.WithBatcher(xopotelExporter), - ) - - // This is the OTEL tracer we'll use to generate some OTEL logs. - tracer := tpOrigin.Tracer("round-trip", - oteltrace.WithSchemaURL("http://something"), - oteltrace.WithInstrumentationAttributes(kvExamples("ia")...), - oteltrace.WithInstrumentationVersion("0.3.0-test4"), - ) - ctx := context.Background() - - // Now we will generate some rich OTEl logs. - span1Ctx, span1 := tracer.Start(ctx, "span1 first-name", - oteltrace.WithNewRoot(), - oteltrace.WithSpanKind(oteltrace.SpanKindProducer), - oteltrace.WithAttributes(kvExamples("s1start")...), - ) - span1.AddEvent("span1-event", - oteltrace.WithTimestamp(time.Now()), - oteltrace.WithAttributes(kvExamples("s1event")...), - ) - span1.SetAttributes(kvExamples("s1set")...) - span1.SetStatus(codes.Error, "a-okay here") - span1.SetName("span1 new-name") - span1.RecordError(fmt.Errorf("an error"), - oteltrace.WithTimestamp(time.Now()), - oteltrace.WithAttributes(kvExamples("s1error")...), - ) - var bundle1 xoptrace.Bundle - bundle1.Parent.TraceID().SetRandom() - bundle1.Parent.SpanID().SetRandom() - var traceState oteltrace.TraceState - traceState, err = traceState.Insert("foo", "bar") - require.NoError(t, err) - traceState, err = traceState.Insert("abc", "xyz") - require.NoError(t, err) - spanConfig1 := oteltrace.SpanContextConfig{ - TraceID: bundle1.Parent.TraceID().Array(), - SpanID: bundle1.Parent.SpanID().Array(), - Remote: true, - TraceFlags: oteltrace.TraceFlags(bundle1.Parent.Flags().Array()[0]), - TraceState: traceState, - } - var bundle2 xoptrace.Bundle - bundle2.Parent.TraceID().SetRandom() - bundle2.Parent.SpanID().SetRandom() - spanConfig2 := oteltrace.SpanContextConfig{ - TraceID: bundle2.Parent.TraceID().Array(), - SpanID: bundle2.Parent.SpanID().Array(), - Remote: false, - TraceFlags: oteltrace.TraceFlags(bundle2.Parent.Flags().Array()[0]), - } - _, span2 := tracer.Start(span1Ctx, "span2", - oteltrace.WithLinks( - oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(spanConfig1), - Attributes: kvExamples("la"), - }, - oteltrace.Link{ - SpanContext: oteltrace.NewSpanContext(spanConfig2), - }, - ), - ) - span1.AddEvent("span2-event", - oteltrace.WithTimestamp(time.Now()), - oteltrace.WithAttributes(kvExamples("s2event")...), - ) - span2.End() - span1.End() - - // We've finished generating logs. Flush them. - require.NoError(t, tpOrigin.ForceFlush(ctx), "flush origin") - - // Now we verify the end result, looking for differences - originSpans := unpack(t, "origin", origin.Bytes()) - replaySpans := unpack(t, "replay", replay.Bytes()) - assert.NotEmpty(t, originSpans, "some spans") - assert.Equal(t, len(originSpans), len(replaySpans), "count of spans") - diffs := xopoteltest.CompareSpanStubSlice("", originSpans, replaySpans) - filtered := make([]xopoteltest.Diff, 0, len(diffs)) - for _, diff := range diffs { - t.Log("diff", diff) - filtered = append(filtered, diff) - } - assert.Equal(t, 0, len(filtered), "count of unfiltered diffs") -} - -func unpack(t *testing.T, what string, data []byte) []xopoteltest.SpanStub { - var spans []xopoteltest.SpanStub - for _, chunk := range bytes.Split(data, []byte{'\n'}) { - if len(chunk) == 0 { - continue - } - var span xopoteltest.SpanStub - err := json.Unmarshal(chunk, &span) - require.NoErrorf(t, err, "unmarshal '%s'", string(chunk)) - t.Logf("%s unpacking %s", what, string(chunk)) - t.Logf("%s unpacked %s %s", what, span.SpanContext.TraceID().String(), span.SpanContext.SpanID().String()) - spans = append(spans, span) - } - return spans -} - -func kvExamples(p string) []attribute.KeyValue { - return []attribute.KeyValue{ - attribute.Bool(p+"one-bool", true), - attribute.BoolSlice(p+"bool-slice", []bool{false, true, false}), - attribute.String(p+"one-string", "slilly stuff"), - attribute.StringSlice(p+"string-slice", []string{"one", "two", "three"}), - attribute.Int(p+"one-int", 389), - attribute.IntSlice(p+"int-slice", []int{93, -4}), - attribute.Int64(p+"one-int64", 299943), - attribute.Int64Slice(p+"int64-slice", []int64{-7}), - attribute.Float64(p+"one-float", 299943), - attribute.Float64Slice(p+"float-slice", []float64{-7.3, 19.2}), - } -} diff --git a/xopotel/xopoteltest/compare.go b/xopotel/xopoteltest/compare.go deleted file mode 100644 index fe08b7a1..00000000 --- a/xopotel/xopoteltest/compare.go +++ /dev/null @@ -1,432 +0,0 @@ -package xopoteltest - -import ( - "fmt" - "reflect" - "strconv" - "strings" - "time" - "unicode" - - "github.com/muir/reflectutils" - "go.opentelemetry.io/otel/attribute" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" -) - -type keyConstraint interface { - comparable - fmt.Stringer -} - -func makeListCompare[T any, K keyConstraint](mapper func([]T) map[K]T, compare func(string, T, T) []Diff) func(string, []T, []T) []Diff { - return func(name string, a []T, b []T) []Diff { - aMap := mapper(a) - bMap := mapper(b) - var diffs []Diff - for key, aThing := range aMap { - if bThing, ok := bMap[key]; ok { - diffs = append(diffs, compare(key.String(), aThing, bThing)...) - } else { - diffs = append(diffs, Diff{Path: []string{key.String()}, A: aThing}) - } - } - for key, bThing := range bMap { - if _, ok := aMap[key]; !ok { - diffs = append(diffs, Diff{Path: []string{key.String()}, B: bThing}) - } - } - return diffPrefix(name, diffs) - } -} - -func CompareMap[K comparable, V any](name string, aMap map[K]V, bMap map[K]V, compare func(string, V, V) []Diff) []Diff { - var diffs []Diff - for key, aThing := range aMap { - if bThing, ok := bMap[key]; ok { - diffs = append(diffs, compare(fmt.Sprint(key), aThing, bThing)...) - } else { - diffs = append(diffs, Diff{Path: []string{fmt.Sprint(key)}, A: aThing}) - } - } - for key, bThing := range bMap { - if _, ok := aMap[key]; !ok { - diffs = append(diffs, Diff{Path: []string{fmt.Sprint(key)}, B: bThing}) - } - } - return diffPrefix(name, diffs) -} - -type Diff struct { - Path []string - A any - B any -} - -func (d Diff) String() string { - return fmt.Sprintf("%s: %s vs %s", strings.Join(d.Path, "."), toString(d.A), toString(d.B)) -} - -func (d Diff) MatchTail(tail ...string) bool { - if len(tail) > len(d.Path) { - return false - } - for i, tailPart := range tail { - if d.Path[len(d.Path)-len(tail)+i] != tailPart { - return false - } - } - return true -} - -var stringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem() - -func toString(a any) string { - if a == nil { - return "missing" - } - if stringer, ok := a.(fmt.Stringer); ok { - return stringer.String() - } - v := reflect.ValueOf(a) - if v.IsValid() && - v.Type().Kind() == reflect.Func && - v.Type().NumIn() == 0 && - v.Type().NumOut() == 1 && - v.Type().Out(0).AssignableTo(stringerType) { - out := v.Call([]reflect.Value{}) - ov := out[0].Interface() - return ov.(fmt.Stringer).String() - } - return reflect.TypeOf(a).String() + "/" + fmt.Sprint(a) -} - -func diffPrefix(prefix string, diffs []Diff) []Diff { - if diffs == nil { - return nil - } - if prefix == "" { - return diffs - } - m := make([]Diff, len(diffs)) - for i, diff := range diffs { - m[i] = Diff{ - Path: append([]string{prefix}, diff.Path...), - A: diff.A, - B: diff.B, - } - } - return m -} - -var CompareSpanStubSlice = makeListCompare(makeSpanSubIndex, CompareSpanStub) - -func makeSpanSubIndex(list []SpanStub) map[oteltrace.SpanID]SpanStub { - m := make(map[oteltrace.SpanID]SpanStub) - for _, e := range list { - m[e.SpanContext.SpanID()] = e - } - return m -} - -func CompareSpanStub(name string, a SpanStub, b SpanStub) []Diff { - var diffs []Diff - diffs = append(diffs, Compare("Name", a.Name, b.Name)...) - diffs = append(diffs, CompareSpanContext("SpanContext", a.SpanContext, b.SpanContext)...) - diffs = append(diffs, CompareSpanContext("Parent", a.Parent, b.Parent)...) - diffs = append(diffs, CompareTime("StartTime", a.StartTime, b.StartTime)...) - diffs = append(diffs, CompareTime("EndTime", a.EndTime, b.EndTime)...) - diffs = append(diffs, CompareAttributes("Attributes", a.Attributes, b.Attributes)...) - diffs = append(diffs, CompareEvents("Events", a.Events, b.Events)...) - diffs = append(diffs, CompareLinks("Links", a.Links, b.Links)...) - diffs = append(diffs, CompareStatus("Status", a.Status, b.Status)...) - diffs = append(diffs, Compare("SpanKind", a.SpanKind.String(), b.SpanKind.String())...) - diffs = append(diffs, Compare("DroppedAttributes", a.DroppedAttributes, b.DroppedAttributes)...) - diffs = append(diffs, Compare("DroppedEvents", a.DroppedEvents, b.DroppedEvents)...) - diffs = append(diffs, Compare("DroppedLinks", a.DroppedLinks, b.DroppedLinks)...) - diffs = append(diffs, Compare("ChildSpanCount", a.ChildSpanCount, b.ChildSpanCount)...) - diffs = append(diffs, CompareAny("Resource", reflect.ValueOf(a.Resource), reflect.ValueOf(b.Resource))...) - diffs = append(diffs, CompareAny("Scope", reflect.ValueOf(a.Scope), reflect.ValueOf(b.Scope))...) - return diffPrefix(name, diffs) -} - -func CompareStatus(name string, a sdktrace.Status, b sdktrace.Status) []Diff { - var diffs []Diff - diffs = append(diffs, Compare("Code", a.Code.String(), b.Code.String())...) - diffs = append(diffs, Compare("Description", a.Description, b.Description)...) - return diffPrefix(name, diffs) -} - -var CompareLinks = makeListCompare(makeLinkMap, CompareLink) - -func makeLinkMap(list []Link) map[oteltrace.SpanID]Link { - m := make(map[oteltrace.SpanID]Link) - for _, e := range list { - m[e.SpanContext.SpanID()] = e - } - return m -} - -func CompareLink(name string, a Link, b Link) []Diff { - var diffs []Diff - diffs = append(diffs, CompareSpanContext("SpanContext", a.SpanContext, b.SpanContext)...) - diffs = append(diffs, CompareAttributes("Attributes", a.Attributes, b.Attributes)...) - diffs = append(diffs, Compare("DroppedAttributeCount", a.DroppedAttributeCount, b.DroppedAttributeCount)...) - return diffPrefix(name, diffs) -} - -var CompareEvents = makeListCompare(makeEventMap, CompareEvent) - -type eventKey struct { - name string - ts int64 -} - -func (e eventKey) String() string { - return e.name + "@" + time.Unix(0, e.ts).Format(time.RFC3339Nano) -} - -func makeEventMap(events []sdktrace.Event) map[eventKey]sdktrace.Event { - m := make(map[eventKey]sdktrace.Event) - for _, event := range events { - m[eventKey{ - name: event.Name, - ts: event.Time.UnixNano(), - }] = event - } - return m -} - -func CompareEvent(name string, a sdktrace.Event, b sdktrace.Event) []Diff { - var diffs []Diff - diffs = append(diffs, Compare("Name", a.Name, b.Name)...) - diffs = append(diffs, CompareAttributes("Attributes", a.Attributes, b.Attributes)...) - diffs = append(diffs, Compare("DroppedAttributeCount", a.DroppedAttributeCount, b.DroppedAttributeCount)...) - diffs = append(diffs, CompareTime("Time", a.Time, b.Time)...) - return diffPrefix(name, diffs) -} - -var CompareAttributes = makeListCompare(makeAttributesIndex, CompareAttribute) - -type stringKey string - -func (s stringKey) String() string { return string(s) } - -func makeAttributesIndex(list []attribute.KeyValue) map[stringKey]attribute.KeyValue { - m := make(map[stringKey]attribute.KeyValue) - for _, kv := range list { - m[stringKey(kv.Key)] = kv - } - return m -} - -func CompareAttribute(name string, a attribute.KeyValue, b attribute.KeyValue) []Diff { - if a.Value.Type() != b.Value.Type() { - return []Diff{{Path: []string{name, "Type"}, A: a.Value.Type().String(), B: b.Value.Type().String()}} - } - switch a.Value.Type() { - case attribute.STRING: - return diffPrefix(name, Compare("String", a.Value.AsString(), b.Value.AsString())) - case attribute.BOOL: - return diffPrefix(name, Compare("Bool", a.Value.AsBool(), b.Value.AsBool())) - case attribute.INT64: - return diffPrefix(name, Compare("Int64", a.Value.AsInt64(), b.Value.AsInt64())) - case attribute.FLOAT64: - return diffPrefix(name, Compare("Float64", a.Value.AsFloat64(), b.Value.AsFloat64())) - case attribute.STRINGSLICE: - return diffPrefix(name, CompareSlice("StringSlice", a.Value.AsStringSlice(), b.Value.AsStringSlice())) - case attribute.BOOLSLICE: - return diffPrefix(name, CompareSlice("BoolSlice", a.Value.AsBoolSlice(), b.Value.AsBoolSlice())) - case attribute.INT64SLICE: - return diffPrefix(name, CompareSlice("Int64Slice", a.Value.AsInt64Slice(), b.Value.AsInt64Slice())) - case attribute.FLOAT64SLICE: - return diffPrefix(name, CompareSlice("Float64Slice", a.Value.AsFloat64Slice(), b.Value.AsFloat64Slice())) - default: - return nil - } -} - -func CompareSlice[T comparable](name string, a []T, b []T) []Diff { - var diffs []Diff - if len(a) != len(b) { - return []Diff{{Path: []string{name, "len"}, A: strconv.Itoa(len(a)), B: strconv.Itoa(len(b))}} - } - for i := 0; i < len(a); i++ { - diffs = append(diffs, Compare("["+strconv.Itoa(i)+"]", a[i], b[i])...) - } - return diffPrefix(name, diffs) -} - -func CompareTime(name string, a time.Time, b time.Time) []Diff { - if a.Equal(b) { - return nil - } - return []Diff{{Path: []string{name}, A: a.Format(time.RFC3339Nano), B: b.Format(time.RFC3339Nano)}} -} - -func CompareSpanContext(name string, a SpanContext, b SpanContext) []Diff { - if a.IsValid() != b.IsValid() { - if a.IsValid() { - return []Diff{{Path: []string{name}, A: a}} - } else { - return []Diff{{Path: []string{name}, B: b}} - } - } - if !a.IsValid() { - return nil - } - var diffs []Diff - if a.SpanID() != b.SpanID() { - diffs = append(diffs, Diff{Path: []string{"SpanID"}, A: a.SpanID(), B: b.SpanID()}) - } - if a.TraceID() != b.TraceID() { - diffs = append(diffs, Diff{Path: []string{"TraceID"}, A: a.TraceID(), B: b.TraceID()}) - } - diffs = append(diffs, Compare("IsRemote", a.IsRemote(), b.IsRemote())...) - diffs = append(diffs, Compare("IsSampled", a.IsSampled(), b.IsSampled())...) - diffs = append(diffs, Compare("TraceFlags", int(a.TraceFlags()), int(b.TraceFlags()))...) - diffs = append(diffs, Compare("TraceState", a.TraceState().String(), b.TraceState().String())...) - return diffPrefix(name, diffs) -} - -func Compare[T comparable](name string, a T, b T) []Diff { - if a == b { - return nil - } - return []Diff{{Path: []string{name}, A: a, B: b}} -} - -var zeroValue reflect.Value - -func CompareInterface(name string, a any, b any) []Diff { - return CompareAny(name, reflect.ValueOf(a), reflect.ValueOf(b)) -} - -func CompareAny(name string, a reflect.Value, b reflect.Value) []Diff { - if !a.IsValid() { - if !b.IsValid() { - return nil - } - // calling Interface() w/o checking CanInterface() is a bit - // dangerous. We'll depend on our caller to not mess us up. - return []Diff{{Path: []string{name}, B: b.Interface()}} - } - if !b.IsValid() { - return []Diff{{Path: []string{name}, A: a.Interface()}} - } - if a.Type() != b.Type() { - return []Diff{{Path: []string{name}, A: a.Interface(), B: b.Interface()}} - } - var diffs []Diff - switch a.Type().Kind() { - case reflect.Invalid: - return nil - case reflect.Bool: - return Compare(name, a.Interface().(bool), b.Interface().(bool)) - case reflect.Int: - return Compare(name, a.Interface().(int), b.Interface().(int)) - case reflect.Int8: - return Compare(name, a.Interface().(int8), b.Interface().(int8)) - case reflect.Int16: - return Compare(name, a.Interface().(int16), b.Interface().(int16)) - case reflect.Int32: - return Compare(name, a.Interface().(int32), b.Interface().(int32)) - case reflect.Int64: - return Compare(name, a.Interface().(int64), b.Interface().(int64)) - case reflect.Uint: - return Compare(name, a.Interface().(uint), b.Interface().(uint)) - case reflect.Uint8: - return Compare(name, a.Interface().(uint8), b.Interface().(uint8)) - case reflect.Uint16: - return Compare(name, a.Interface().(uint16), b.Interface().(uint16)) - case reflect.Uint32: - return Compare(name, a.Interface().(uint32), b.Interface().(uint32)) - case reflect.Uint64: - return Compare(name, a.Interface().(uint64), b.Interface().(uint64)) - case reflect.Uintptr: - return Compare(name, a.Interface().(uintptr), b.Interface().(uintptr)) - case reflect.Float32: - return Compare(name, a.Interface().(float32), b.Interface().(float32)) - case reflect.Float64: - return Compare(name, a.Interface().(float64), b.Interface().(float64)) - case reflect.Complex64: - return Compare(name, a.Interface().(complex64), b.Interface().(complex64)) - case reflect.Complex128: - return Compare(name, a.Interface().(complex128), b.Interface().(complex128)) - case reflect.Array: - for i := 0; i < a.Len(); i++ { - diffs = append(diffs, CompareAny("["+strconv.Itoa(i)+"]", a.Index(i), b.Index(i))...) - } - case reflect.Chan: - panic("unexpected, chan") - case reflect.Func: - panic("unexpected, func") - case reflect.Interface: - if iA := reflect.ValueOf(a.Interface()); iA.Type().Kind() != reflect.Interface { - if iB := reflect.ValueOf(b.Interface()); iB.Type().Kind() != reflect.Interface { - return CompareAny(name, iA, iB) - } - } - return []Diff{{Path: []string{name, a.Type().String()}, A: b.Interface(), B: b.Interface()}} - case reflect.Map: - for _, k := range a.MapKeys() { - av := a.MapIndex(k) - bv := b.MapIndex(k) - ks := fmt.Sprint(k.Interface()) - if bv == zeroValue { - diffs = append(diffs, Diff{Path: []string{ks}, A: av.Interface()}) - } else { - diffs = append(diffs, CompareAny(ks, av, bv)...) - } - } - for _, k := range b.MapKeys() { - av := a.MapIndex(k) - if av == zeroValue { - bv := b.MapIndex(k) - ks := fmt.Sprint(k.Interface()) - diffs = append(diffs, Diff{Path: []string{ks}, B: bv.Interface()}) - } - } - case reflect.Pointer: - if a.IsNil() != b.IsNil() { - return []Diff{{Path: []string{name}, A: a.Interface(), B: b.Interface()}} - } - if a.IsNil() { - return nil - } - return CompareAny(name, a.Elem(), b.Elem()) - case reflect.Slice: - maxLen := a.Len() - if b.Len() > maxLen { - maxLen = b.Len() - } - for i := 0; i < maxLen; i++ { - if i > a.Len()-1 { - diffs = append(diffs, Diff{Path: []string{"[" + strconv.Itoa(i) + "]"}, B: b.Index(i).Interface()}) - } else if i > b.Len()-1 { - diffs = append(diffs, Diff{Path: []string{"[" + strconv.Itoa(i) + "]"}, A: a.Index(i).Interface()}) - } else { - diffs = append(diffs, CompareAny("["+strconv.Itoa(i)+"]", a.Index(i), b.Index(i))...) - } - } - case reflect.String: - return Compare(name, a.Interface().(string), b.Interface().(string)) - case reflect.Struct: - reflectutils.WalkStructElements(a.Type(), func(field reflect.StructField) bool { - for _, c := range field.Name { - if unicode.IsUpper(c) { - // ignore un-exported fields - return false - } - break //nolint:staticcheck // on purpose - } - diffs = append(diffs, CompareAny(field.Name, a.FieldByIndex(field.Index), b.FieldByIndex(field.Index))...) - return true - }) - case reflect.UnsafePointer: - panic("unexpected, unsafe ptr") - default: - panic("unexpected " + a.Type().String()) - } - return diffPrefix(name, diffs) -} diff --git a/xopotel/xopoteltest/span.go b/xopotel/xopoteltest/span.go deleted file mode 100644 index c9f5a19b..00000000 --- a/xopotel/xopoteltest/span.go +++ /dev/null @@ -1,132 +0,0 @@ -package xopoteltest - -import ( - "encoding/hex" - "encoding/json" - "fmt" - "time" - - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/sdk/instrumentation" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" -) - -// SpanStub from https://pkg.go.dev/go.opentelemetry.io/otel/sdk@v1.14.0/trace/tracetest#SpanStub because -// it doesn't implement UnmarshalJSON. Why not? -type SpanStub struct { - Name string - SpanContext SpanContext - Parent SpanContext - SpanKind oteltrace.SpanKind - StartTime time.Time - EndTime time.Time - Attributes []attribute.KeyValue - Events []sdktrace.Event - Links []Link - Status sdktrace.Status - DroppedAttributes int - DroppedEvents int - DroppedLinks int - ChildSpanCount int - Resource any - Scope instrumentation.Scope `json:"InstrumentationLibrary"` -} - -func (s SpanStub) String() string { return fmt.Sprintf("span %s - %s", s.Name, s.SpanContext) } - -// SpanContext copied from https://github.com/open-telemetry/opentelemetry-go/blob/2e54fbb3fede5b54f316b3a08eab236febd854e0/trace/trace.go#L290 -// because it doesn't implement UnmarshalJSON. Why not? -type SpanContext struct { - oteltrace.SpanContext -} - -func (sc SpanContext) String() string { - return fmt.Sprintf("00-%s-%s-%s", sc.TraceID(), sc.SpanID(), sc.TraceFlags()) -} - -func (sc *SpanContext) UnmarshalJSON(i []byte) error { - var tmp struct { - TraceID TraceID - SpanID SpanID - TraceFlags TraceFlags - TraceState TraceState - Remote bool - } - err := json.Unmarshal(i, &tmp) - if err != nil { - return err - } - scc := oteltrace.SpanContextConfig{ - TraceID: tmp.TraceID.TraceID, - SpanID: tmp.SpanID.SpanID, - TraceFlags: tmp.TraceFlags.TraceFlags, - TraceState: tmp.TraceState.TraceState, - Remote: tmp.Remote, - } - sc.SpanContext = oteltrace.NewSpanContext(scc) - return nil -} - -// Link copied from https://pkg.go.dev/go.opentelemetry.io/otel/trace#Link because -// it doesn't implement UnmarshalJSON. Why not? -type Link struct { - SpanContext SpanContext - Attributes []attribute.KeyValue - DroppedAttributeCount int -} - -// SpanID exists because oteltrace.SpanID doesn't implement UnmarshalJSON -type SpanID struct { - oteltrace.SpanID -} - -func (s *SpanID) UnmarshalText(h []byte) error { - return decode(s.SpanID[:], h) -} - -// TraceID exists because oteltrace.TraceID doesn't implement UnmarshalJSON -type TraceID struct { - oteltrace.TraceID -} - -func (t *TraceID) UnmarshalText(h []byte) error { return decode(t.TraceID[:], h) } - -func decode(s []byte, h []byte) error { - b, err := hex.DecodeString(string(h)) - if err != nil { - return err - } - if len(b) != len(s) { - return fmt.Errorf("wrong length") - } - copy(s[:], b) - return nil -} - -type TraceFlags struct { - oteltrace.TraceFlags -} - -func (tf *TraceFlags) UnmarshalText(h []byte) error { - var a [1]byte - err := decode(a[:], h) - if err != nil { - return err - } - tf.TraceFlags = oteltrace.TraceFlags(a[0]) - return nil -} - -type TraceState struct { - oteltrace.TraceState -} - -func (ts *TraceState) UnmarshalText(i []byte) error { - s, err := oteltrace.ParseTraceState(string(i)) - if err != nil { - return err - } - ts.TraceState = s - return nil -} diff --git a/xopresty/resty.go b/xopresty/resty.go deleted file mode 100644 index c807f29f..00000000 --- a/xopresty/resty.go +++ /dev/null @@ -1,246 +0,0 @@ -/* -package xopresty adds to the resty package to -propagate xop context to through an HTTP request. - -As of March 29th, 2023, the released resty package does not provide -a way to have a logger that knows which request it is logging about. -The resty package does not provide a way to know when requests are -complete. - -Pull requests to fix these issues have been merged but not -made part of a release. - -In the meantime, this package depends upon https://github.com/muir/resty. - -The agumented resty Client requires that a context that -has the parent log span be provided: - - client.R().SetContext(log.IntoContext(context.Background())) - -If there is no logger in the context, the request will fail. - -If you use resty's Client.SetDebug(true), note that the output -will be logged at Debug level which is below the default -minimum level for xop. -*/ -package xopresty - -import ( - "context" - "fmt" - "strings" - "time" - - "github.com/xoplog/xop-go" - "github.com/xoplog/xop-go/xopconst" - "github.com/xoplog/xop-go/xoptrace" - - "github.com/muir/resty" - "github.com/pkg/errors" -) - -var _ resty.Logger = restyLogger{} - -type restyLogger struct { - log *xop.Logger -} - -func (rl restyLogger) Errorf(format string, v ...interface{}) { rl.log.Error().Msgf(format, v...) } -func (rl restyLogger) Warnf(format string, v ...interface{}) { rl.log.Warn().Msgf(format, v...) } -func (rl restyLogger) Debugf(format string, v ...interface{}) { rl.log.Debug().Msgf(format, v...) } - -type contextKeyType struct{} - -var contextKey = contextKeyType{} - -type contextNameType struct{} - -var contextNameKey = contextNameType{} - -type contextValue struct { - b3Sent bool - b3Trace xoptrace.Trace - response bool - log *xop.Logger - retryCount int - originalStartTime time.Time -} - -type config struct { - requestToName func(r *resty.Request) string - extraLogging ExtraLogging -} - -type ClientOpt func(*config) - -var traceResponseHeaderKey = xop.Key("header") -var requestTimeKey = xop.Key("request_time.total") -var requestTimeServerKey = xop.Key("request_time.server") -var requestTimeDNSKey = xop.Key("request_time.dns") - -// WithNameGenerate provides a function to convert a request into -// a description for the span. -func WithNameGenerate(f func(*resty.Request) string) ClientOpt { - return func(config *config) { - config.requestToName = f - } -} - -// ExtraLogging provides a hook for extra logging to be done. -// It is possible that the response parameter will be null. -// If error is not null, then the request has failed. -// ExtraLogging should only be called once but if another resty -// callback panic's, it is possible ExtraLogging will be called -// twice. -type ExtraLogging func(log *xop.Logger, originalStartTime time.Time, retryCount int, request *resty.Request, response *resty.Response, err error) - -func WithExtraLogging(f ExtraLogging) ClientOpt { - return func(config *config) { - config.extraLogging = f - } -} - -// WithNameInDescription adds a span name to a context. If present, -// a name in context overrides WithNameGenerate. -func WithNameInContext(ctx context.Context, nameOrDescription string) context.Context { - return context.WithValue(ctx, contextNameKey, nameOrDescription) -} - -func Client(client *resty.Client, opts ...ClientOpt) *resty.Client { - config := &config{ - requestToName: func(r *resty.Request) string { - url := r.URL - i := strings.IndexByte(url, '?') - if i != -1 { - url = url[:i] - } - return r.Method + " " + url - }, - extraLogging: func(log *xop.Logger, originalStartTime time.Time, retryCount int, request *resty.Request, response *resty.Response, err error) { - }, - } - for _, f := range opts { - f(config) - } - - // c := *client - // c.Header = client.Header.Clone() - // clinet = &c - return client. - OnBeforeRequest(func(_ *resty.Client, r *resty.Request) error { - // OnBeforeRequest can execute multiple times for the same attempt if there - // are retries. It won't execute at all of the request is invalid. - ctx := r.Context() - cvRaw := ctx.Value(contextKey) - var cv *contextValue - if cvRaw != nil { - cv = cvRaw.(*contextValue) - cv.retryCount++ - return nil - } - log, ok := xop.FromContext(r.Context()) - if !ok { - return errors.Errorf("context is missing logger, use Request.SetContext(Log.IntoContext(request.Context()))") - } - nameRaw := ctx.Value(contextNameKey) - var name string - if nameRaw != nil { - name = nameRaw.(string) - } else { - name = config.requestToName(r) - } - log = log.Sub().Step(name) - cv = &contextValue{ - originalStartTime: time.Now(), - log: log, - } - r.SetContext(context.WithValue(ctx, contextKey, cv)) - r.SetLogger(restyLogger{log: log}) - - if r.Body != nil { - log.Trace().Model(r.Body, "request") - } - - log.Span().EmbeddedEnum(xopconst.SpanTypeHTTPClientRequest) - log.Span().String(xopconst.URL, r.URL) - log.Span().String(xopconst.HTTPMethod, r.Method) - r.Header.Set("traceparent", log.Span().Bundle().Trace.String()) - if !log.Span().TraceBaggage().IsZero() { - r.Header.Set("baggage", log.Span().TraceBaggage().String()) - } - if !log.Span().TraceState().IsZero() { - r.Header.Set("state", log.Span().TraceState().String()) - } - if log.Config().UseB3 { - b3Trace := log.Span().Bundle().Trace - b3Trace.SpanID().SetRandom() - r.Header.Set("b3", - b3Trace.GetTraceID().String()+"-"+ - b3Trace.TraceID().String()+"-"+ - b3Trace.GetFlags().String()[1:2]+"-"+ - log.Span().Trace().GetSpanID().String()) - cv.b3Trace = b3Trace - cv.b3Sent = true - } - return nil - }). - OnAfterResponse(func(_ *resty.Client, resp *resty.Response) error { - // OnAfterRequest is run for each individual request attempt - r := resp.Request - ctx := r.Context() - cvRaw := ctx.Value(contextKey) - var cv *contextValue - if cvRaw == nil { - return fmt.Errorf("xopresty: internal error, context missing in response") - } - cv = cvRaw.(*contextValue) - log := cv.log - - tr := resp.Header().Get("traceresponse") - if tr != "" { - trace, ok := xoptrace.TraceFromString(tr) - if ok { - cv.response = true - log.Info().Link(trace, xopconst.RemoteTrace.Key().String()) - log.Span().Link(xopconst.RemoteTrace, trace) - } else { - log.Warn().String(traceResponseHeaderKey, tr).Msg("invalid traceresponse received") - } - } - if r.Result != nil { - log.Info().Model(resp.Result(), "response") - } - ti := r.TraceInfo() - if ti.TotalTime != 0 { - log.Info(). - Duration(requestTimeKey, ti.TotalTime). - Duration(requestTimeServerKey, ti.ServerTime). - Duration(requestTimeDNSKey, ti.DNSLookup). - Msg("timings") - } - return nil - }). - OnError(func(r *resty.Request, err error) { - ctx := r.Context() - cv := ctx.Value(contextKey).(*contextValue) - log := cv.log - var re *resty.ResponseError - if errors.As(err, &re) { - config.extraLogging(log, cv.originalStartTime, cv.retryCount, r, re.Response, re.Err) - } else { - config.extraLogging(log, cv.originalStartTime, cv.retryCount, r, nil, err) - } - }). - OnPanic(func(r *resty.Request, err error) { - ctx := r.Context() - cv := ctx.Value(contextKey).(*contextValue) - log := cv.log - config.extraLogging(log, cv.originalStartTime, cv.retryCount, r, nil, err) - }). - OnSuccess(func(c *resty.Client, resp *resty.Response) { - ctx := resp.Request.Context() - cv := ctx.Value(contextKey).(*contextValue) - log := cv.log - config.extraLogging(log, cv.originalStartTime, cv.retryCount, resp.Request, resp, nil) - }) -} diff --git a/xopresty/resty_test.go b/xopresty/resty_test.go deleted file mode 100644 index e5a0b8c7..00000000 --- a/xopresty/resty_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package xopresty_test - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/xoplog/xop-go" - "github.com/xoplog/xop-go/xopmiddle" - "github.com/xoplog/xop-go/xopnum" - "github.com/xoplog/xop-go/xoprecorder" - "github.com/xoplog/xop-go/xopresty" - "github.com/xoplog/xop-go/xoptest" - - "github.com/muir/resty" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type exampleRequest struct { - Name string - Count int -} - -type exampleResult struct { - Score float64 - Comment string -} - -var cases = []struct { - name string - clientMod func(*resty.Client) *resty.Client - requestMod func(*resty.Request) *resty.Request - handler func(t *testing.T, log *xop.Logger, w http.ResponseWriter, r *http.Request) - restyOpts []xopresty.ClientOpt - expectError bool - expectedText []string -}{ - { - name: "with debugging and tracing", - clientMod: func(c *resty.Client) *resty.Client { - return c.SetDebug(true) - }, - requestMod: func(r *resty.Request) *resty.Request { - return r.EnableTrace() - }, - }, - { - name: "without debugging, without tracing", - }, - { - name: "with model", - requestMod: func(r *resty.Request) *resty.Request { - var res exampleResult - return r. - SetBody(exampleRequest{ - Name: "Joe", - Count: 38, - }).SetResult(&res). - SetHeader("Content-Type", "application/json"). - SetHeader("Accept", "application/json") - }, - handler: func(t *testing.T, log *xop.Logger, w http.ResponseWriter, r *http.Request) { - enc, err := json.Marshal(exampleResult{ - Score: 3.8, - Comment: "good progress", - }) - assert.NoError(t, err, "marshal") - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(enc) - log.Debug().Msg("sent response") - }, - expectedText: []string{ - `T1.1.1 MODEL:request {"Name":"Joe","Count":38}`, - `T1.1.1 MODEL:response {"Score":3.8,"Comment":"good progress"}`, - }, - }, -} - -func TestXopResty(t *testing.T) { - for _, tc := range cases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - tLog := xoptest.New(t) - seed := xop.NewSeed(xop.WithBase(tLog)) - log := seed.Request("client") - log.Info().Msg("i am the base log") - ctx := log.Sub().MinLevel(xopnum.TraceLevel).Logger().IntoContext(context.Background()) - - var called bool - inbound := xopmiddle.New(seed, func(r *http.Request) string { - return "handler:" + r.Method - }) - ts := httptest.NewServer(inbound.HandlerFuncMiddleware()(func(w http.ResponseWriter, r *http.Request) { - called = true - log := xop.FromContextOrPanic(r.Context()) - log.Info().Msg("in request handler") - if tc.handler == nil { - http.Error(w, "no handler", 500) - return - } - tc.handler(t, log, w, r) - })) - defer ts.Close() - - log.Done() - c := xopresty.Client(resty.New()) - if tc.clientMod != nil { - c = tc.clientMod(c) - } - r := c.R().SetContext(ctx) - if tc.requestMod != nil { - r = tc.requestMod(r) - } - - _, err := r.Get(ts.URL) - - requestSpan := tLog.Recorder().FindSpan(xoprecorder.NameEquals("GET handler:GET")) - - require.NotNil(t, requestSpan, "requestSpan") - assert.NotEmpty(t, requestSpan.EndTime, "client request span completed") - - if tc.expectError { - assert.Error(t, err, "expected error") - return - } - - farSideSpan := tLog.Recorder().FindSpan(xoprecorder.ShortEquals("T1.2")) - require.NotNil(t, farSideSpan, "farSideSpan") - assert.NotEmpty(t, farSideSpan.EndTime, "server endpoint span completed") - assert.NoError(t, err, "Get") - assert.True(t, called, "handler called") - - text := "T1.1.1 LINK:http.remote_trace " + farSideSpan.Bundle.Trace.String() - assert.Equalf(t, 1, tLog.Recorder().CountLines(xoprecorder.TextContains(text)), "count lines with '%s'", text) - - for _, text := range tc.expectedText { - assert.Equalf(t, 1, tLog.Recorder().CountLines(xoprecorder.TextContains(text)), "count lines with '%s'", text) - } - }) - } -}