From c1babda92da17e1dec590a46f16db045e1d17eaf Mon Sep 17 00:00:00 2001 From: Anton Kolesnikov Date: Tue, 8 Jun 2021 01:40:15 +0200 Subject: [PATCH] Windows support (agent) (#212) * Create stubs for non-windows code * Disable dotnet profiling options for Windows * Implement windows-specific parts * Fix configuration type * Add MSI package definition * Embed windows version info at build * Add agent run mode * Rework windows-dev build * Add target manager * Rework CLI * Allow pyroscope agent to run without a config file * Rework agent logging * Improve error messages Co-authored-by: Dmitry Filimonov --- .gitignore | 3 + Dockerfile.windows | 40 +++ Makefile | 4 + cmd/pyroscope/logging.go | 6 + cmd/pyroscope/main.go | 9 +- cmd/pyroscope/main_unix.go | 13 + cmd/pyroscope/main_windows.go | 25 ++ go.mod | 8 +- go.sum | 23 +- pkg/agent/cli/agent.go | 93 ----- pkg/agent/csock/csock.go | 88 ----- pkg/agent/csock/tcp.go | 17 - pkg/agent/csock/unix.go | 24 -- pkg/agent/debugspy/placeholder.go | 2 - pkg/agent/dotnetspy/placeholder.go | 2 - pkg/agent/dotnetspy/session.go | 20 +- pkg/agent/dotnetspy/session_unix.go | 7 + pkg/agent/dotnetspy/session_windows.go | 7 + .../ebpfspy/{ebpfspy.go => ebpfspy_linux.go} | 0 pkg/agent/ebpfspy/placeholder.go | 2 - .../ebpfspy/{session.go => session_linux.go} | 0 pkg/agent/phpspy/placeholder.go | 2 - pkg/agent/rbspy/placeholder.go | 2 - pkg/agent/target/service.go | 91 +++++ pkg/agent/target/service_unix.go | 11 + pkg/agent/target/service_windows.go | 45 +++ pkg/agent/target/target.go | 156 ++++++++ .../target_suite_test.go} | 6 +- pkg/agent/target/target_test.go | 43 +++ pkg/agent/upstream/remote/remote.go | 12 +- pkg/cli/agent.go | 97 +++++ pkg/cli/agent_unix.go | 13 + pkg/cli/agent_windows.go | 28 ++ pkg/cli/cli.go | 338 ++---------------- pkg/cli/cli_darwin.go | 36 ++ pkg/cli/cli_linux.go | 11 + pkg/cli/cli_windows.go | 30 ++ pkg/cli/config_parser.go | 66 ++++ pkg/cli/flags.go | 183 ++++++++++ pkg/cli/flags_test.go | 58 ++- pkg/cli/server_unix.go | 88 +++++ pkg/cli/server_windows.go | 11 + pkg/cli/testdata/agent.yml | 7 + pkg/cli/{ => testdata}/example.yml | 0 pkg/cli/usage.go | 1 - pkg/config/config.go | 29 +- pkg/exec/cli.go | 118 +----- pkg/exec/cli_darwin.go | 2 - pkg/exec/cli_linux.go | 2 - pkg/exec/cli_unix.go | 104 ++++++ pkg/exec/cli_windows.go | 15 + pkg/exec/exec_test.go | 6 +- pkg/storage/segment/segment.go | 6 +- pkg/testing/tmpdir.go | 4 +- pkg/util/disk/usage_test.go | 1 + pkg/util/disk/{usage.go => usage_unix.go} | 2 + pkg/util/disk/usage_windows.go | 41 +++ .../generate-windows-version-info/README.md | 9 + .../generate-windows-version-info/main.go | 92 +++++ scripts/windows/pyroscope.wsx | 152 ++++++++ scripts/windows/resources/app.ico | Bin 0 -> 34662 bytes 61 files changed, 1627 insertions(+), 684 deletions(-) create mode 100644 Dockerfile.windows create mode 100644 cmd/pyroscope/main_unix.go create mode 100644 cmd/pyroscope/main_windows.go delete mode 100644 pkg/agent/cli/agent.go delete mode 100644 pkg/agent/csock/csock.go delete mode 100644 pkg/agent/csock/tcp.go delete mode 100644 pkg/agent/csock/unix.go create mode 100644 pkg/agent/dotnetspy/session_unix.go create mode 100644 pkg/agent/dotnetspy/session_windows.go rename pkg/agent/ebpfspy/{ebpfspy.go => ebpfspy_linux.go} (100%) rename pkg/agent/ebpfspy/{session.go => session_linux.go} (100%) create mode 100644 pkg/agent/target/service.go create mode 100644 pkg/agent/target/service_unix.go create mode 100644 pkg/agent/target/service_windows.go create mode 100644 pkg/agent/target/target.go rename pkg/agent/{csock/csock_suite_test.go => target/target_suite_test.go} (57%) create mode 100644 pkg/agent/target/target_test.go create mode 100644 pkg/cli/agent.go create mode 100644 pkg/cli/agent_unix.go create mode 100644 pkg/cli/agent_windows.go create mode 100644 pkg/cli/cli_darwin.go create mode 100644 pkg/cli/cli_linux.go create mode 100644 pkg/cli/cli_windows.go create mode 100644 pkg/cli/config_parser.go create mode 100644 pkg/cli/flags.go create mode 100644 pkg/cli/server_unix.go create mode 100644 pkg/cli/server_windows.go create mode 100644 pkg/cli/testdata/agent.yml rename pkg/cli/{ => testdata}/example.yml (100%) create mode 100644 pkg/exec/cli_unix.go create mode 100644 pkg/exec/cli_windows.go rename pkg/util/disk/{usage.go => usage_unix.go} (94%) create mode 100644 pkg/util/disk/usage_windows.go create mode 100644 scripts/windows/generate-windows-version-info/README.md create mode 100644 scripts/windows/generate-windows-version-info/main.go create mode 100644 scripts/windows/pyroscope.wsx create mode 100644 scripts/windows/resources/app.ico diff --git a/.gitignore b/.gitignore index 4e04de9efc..9516b4f546 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,13 @@ node_modules pkged.go .DS_Store /Icon* +/out/ /scripts/packages/ .eslintcache *.pyc *.internal +*.syso +*.exe .idea .vscode vendor/ diff --git a/Dockerfile.windows b/Dockerfile.windows new file mode 100644 index 0000000000..0fbaeebe90 --- /dev/null +++ b/Dockerfile.windows @@ -0,0 +1,40 @@ +FROM golang:1.16.4-alpine3.12 as go-builder + +RUN apk add --no-cache make git zstd gcc g++ libc-dev musl-dev bash mingw-w64-gcc + +WORKDIR /opt/pyroscope/ + +COPY pkg ./pkg +COPY cmd ./cmd +COPY scripts ./scripts +COPY go.mod go.sum pyroscope.go ./ + +# Generate .syso object file. +RUN source scripts/packages/git-info && go run scripts/windows/generate-windows-version-info/main.go \ + -version "$GIT_TAG" \ + -icon scripts/windows/resources/app.ico \ + -out cmd/pyroscope/resource.syso + +## Build for Windows x64 only. +RUN GOOS=windows GOARCH=amd64 go build \ + -trimpath -ldflags "$(scripts/generate-build-flags.sh)" \ + -tags dotnetspy,debugspy \ + -o pyroscope.exe \ + ./cmd/pyroscope + +FROM harbottle/wix AS msi-builder + +COPY --from=go-builder /opt/pyroscope/pyroscope.exe pyroscope.exe +COPY scripts/windows/pyroscope.wsx pyroscope.wsx +COPY scripts/windows/resources resources + +# Build MSI package. +RUN candle -arch x64 -ext WixUtilExtension \ + -dPyroscopeSourceExecutable=pyroscope.exe \ + pyroscope.wsx && \ + light -sval -ext WixUtilExtension \ + pyroscope.wixobj + +FROM scratch AS msi-exporter +COPY --from=msi-builder /mnt/workspace/pyroscope.exe / +COPY --from=msi-builder /mnt/workspace/pyroscope.msi / diff --git a/Makefile b/Makefile index f11d01c326..88ede0589e 100644 --- a/Makefile +++ b/Makefile @@ -134,3 +134,7 @@ update-protobuf: .PHONY: docker-dev docker-dev: docker build . --tag pyroscope/pyroscope:dev + +.PHONY: windows-dev +windows-dev: + docker build -f Dockerfile.windows --output type=local,dest=out . diff --git a/cmd/pyroscope/logging.go b/cmd/pyroscope/logging.go index f37960cbdb..4b5152ebbb 100644 --- a/cmd/pyroscope/logging.go +++ b/cmd/pyroscope/logging.go @@ -3,7 +3,9 @@ package main import ( "log" "os" + "runtime" + "github.com/fatih/color" "github.com/sirupsen/logrus" ) @@ -13,4 +15,8 @@ func init() { logrus.SetFormatter(&logrus.TextFormatter{}) logrus.SetOutput(os.Stdout) logrus.SetLevel(logrus.DebugLevel) + + if runtime.GOOS == "windows" { + color.NoColor = true + } } diff --git a/cmd/pyroscope/main.go b/cmd/pyroscope/main.go index bd887e5345..4339db9026 100644 --- a/cmd/pyroscope/main.go +++ b/cmd/pyroscope/main.go @@ -1,17 +1,14 @@ package main import ( - "os" - "github.com/fatih/color" + "github.com/pyroscope-io/pyroscope/pkg/cli" "github.com/pyroscope-io/pyroscope/pkg/config" ) func main() { - cfg := &config.Config{} - err := cli.Start(cfg) - if err != nil { - os.Stderr.Write([]byte(color.RedString("Error: ") + err.Error() + "\n\n")) + if err := cli.Start(new(config.Config)); err != nil { + fatalf("%s %v\n\n", color.RedString("Error:"), err) } } diff --git a/cmd/pyroscope/main_unix.go b/cmd/pyroscope/main_unix.go new file mode 100644 index 0000000000..d57e20ab2a --- /dev/null +++ b/cmd/pyroscope/main_unix.go @@ -0,0 +1,13 @@ +// +build !windows + +package main + +import ( + "fmt" + "os" +) + +func fatalf(format string, args ...interface{}) { + _, _ = fmt.Fprintf(os.Stderr, format, args...) + os.Exit(1) +} diff --git a/cmd/pyroscope/main_windows.go b/cmd/pyroscope/main_windows.go new file mode 100644 index 0000000000..11d09f0f85 --- /dev/null +++ b/cmd/pyroscope/main_windows.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "os" + + "github.com/kardianos/service" + "golang.org/x/sys/windows/svc/eventlog" +) + +func fatalf(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + if service.Interactive() { + _, _ = fmt.Fprint(os.Stderr, msg) + os.Exit(1) + } + log, err := eventlog.Open("Pyroscope") + if err == nil { + err = log.Error(1, msg) + } + if err != nil { + panic(msg) + } + os.Exit(1) +} diff --git a/go.mod b/go.mod index e537c46d5a..0fe1362705 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( github.com/avast/retry-go v3.0.0+incompatible github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 + github.com/blang/semver v3.5.1+incompatible github.com/cheggaaa/pb/v3 v3.0.5 github.com/clarkduvall/hyperloglog v0.0.0-20171127014514-a0107a5d8004 github.com/cosmtrek/air v1.12.2 @@ -20,6 +21,8 @@ require ( github.com/iancoleman/strcase v0.1.2 github.com/ianlancetaylor/demangle v0.0.0-20200715173712-053cf528c12f // indirect github.com/imdario/mergo v0.3.11 // indirect + github.com/josephspurrier/goversioninfo v1.2.0 + github.com/kardianos/service v1.2.0 github.com/kisielk/godepgraph v0.0.0-20190626013829-57a7e4a651a9 github.com/kr/pretty v0.2.0 // indirect github.com/kyoh86/richgo v0.3.3 @@ -33,16 +36,17 @@ require ( github.com/onsi/ginkgo v1.16.2 github.com/onsi/gomega v1.12.0 github.com/pelletier/go-toml v1.8.1 // indirect - github.com/peterbourgon/ff v1.7.0 github.com/peterbourgon/ff/v3 v3.0.0 github.com/prometheus/client_golang v1.10.0 - github.com/pyroscope-io/dotnetdiag v1.1.0 + github.com/pyroscope-io/dotnetdiag v1.2.0 github.com/rivo/uniseg v0.2.0 // indirect github.com/sirupsen/logrus v1.7.0 github.com/twmb/murmur3 v1.1.5 github.com/wacul/ptr v1.0.0 // indirect + golang.org/x/sys v0.0.0-20210423082822-04245dca01da golang.org/x/tools v0.1.0 google.golang.org/protobuf v1.26.0 + gopkg.in/yaml.v2 v2.4.0 honnef.co/go/tools v0.0.1-2020.1.6 ) diff --git a/go.sum b/go.sum index a76d5eca5c..aa277475a9 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -13,6 +15,8 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= +github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -37,6 +41,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -206,6 +212,8 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josephspurrier/goversioninfo v1.2.0 h1:tpLHXAxLHKHg/dCU2AAYx08A4m+v9/CWg6+WUvTF4uQ= +github.com/josephspurrier/goversioninfo v1.2.0/go.mod h1:AGP2a+Y/OVJZ+s6XM4IwFUpkETwvn0orYurY8qpw1+0= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -214,6 +222,8 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kardianos/service v1.2.0 h1:bGuZ/epo3vrt8IPC7mnKQolqFeYJb7Cs8Rk4PSOBB/g= +github.com/kardianos/service v1.2.0/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/godepgraph v0.0.0-20190626013829-57a7e4a651a9 h1:ZkWH0x1yafBo+Y2WdGGdszlJrMreMXWl7/dqpEkwsIk= github.com/kisielk/godepgraph v0.0.0-20190626013829-57a7e4a651a9/go.mod h1:Gb5YEgxqiSSVrXKWQxDcKoCM94NO5QAwOwTaVmIUAMI= @@ -269,7 +279,6 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -325,8 +334,6 @@ github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterbourgon/ff v1.7.0 h1:hknvTgsh90jNBIjPq7xeq32Y9AmSbpXvjrFW4sJwW+A= -github.com/peterbourgon/ff v1.7.0/go.mod h1:/KKxnU5cBj4w21jEMj4Rway/kslRP6XAOHh7CH8AyAM= github.com/peterbourgon/ff/v3 v3.0.0 h1:eQzEmNahuOjQXfuegsKQTSTDbf4dNvr/eNLrmJhiH7M= github.com/peterbourgon/ff/v3 v3.0.0/go.mod h1:UILIFjRH5a/ar8TjXYLTkIvSvekZqPm5Eb/qbGk6CT0= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= @@ -366,8 +373,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/pyroscope-io/dotnetdiag v1.1.0 h1:BIo2cIAH+/iXQgbgUfpLil3E9ODY2HcnBBwQ4smjgAU= -github.com/pyroscope-io/dotnetdiag v1.1.0/go.mod h1:mUudCmW+j2ewS55/FHq0lHj0Xy2AHWc1UhaaRdfSve8= +github.com/pyroscope-io/dotnetdiag v1.2.0 h1:RuuCMjuJbW3vnv+XO/0492P5OndG/ZFlvzZQy3k+hi4= +github.com/pyroscope-io/dotnetdiag v1.2.0/go.mod h1:eFUEHCp4eD1TgcXMlJihC+R4MrqGf7nTRdWxNADbDHA= github.com/pyroscope-io/lfu-go v1.0.1 h1:yGE9tbsAYTr+Bb5KL25WJOh7FX0bnb01MDgIqPvxPAY= github.com/pyroscope-io/lfu-go v1.0.1/go.mod h1:3W9sGrDLhKFkHZPXkz6c5dAKrxcwkKbFFKnJtDukMDA= github.com/pyroscope-io/revive v1.0.6-0.20210330033039-4a71146f9dc1 h1:0v9lBNgdmVtpyyk9PP/DfpJlOHkXriu5YgNlrhQw5YE= @@ -413,8 +420,9 @@ github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twmb/murmur3 v1.1.5 h1:i9OLS9fkuLzBXjt6dptlAEyk58fJsSTXbRg3SgVyqgk= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= @@ -525,6 +533,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -612,6 +621,8 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/agent/cli/agent.go b/pkg/agent/cli/agent.go deleted file mode 100644 index c7e7513adf..0000000000 --- a/pkg/agent/cli/agent.go +++ /dev/null @@ -1,93 +0,0 @@ -package cli - -import ( - "os" - "time" - - "github.com/pyroscope-io/pyroscope/pkg/agent" - "github.com/pyroscope-io/pyroscope/pkg/agent/csock" - "github.com/pyroscope-io/pyroscope/pkg/agent/types" - "github.com/pyroscope-io/pyroscope/pkg/agent/upstream" - "github.com/pyroscope-io/pyroscope/pkg/agent/upstream/remote" - "github.com/pyroscope-io/pyroscope/pkg/config" - "github.com/pyroscope-io/pyroscope/pkg/util/id" - "github.com/sirupsen/logrus" -) - -type Agent struct { - cfg *config.Agent - cs *csock.CSock - activeProfiles map[int]*agent.ProfileSession - id id.ID - u upstream.Upstream -} - -func New(cfg *config.Agent) (*Agent, error) { - rc := remote.RemoteConfig{ - UpstreamThreads: cfg.UpstreamThreads, - UpstreamAddress: cfg.ServerAddress, - UpstreamRequestTimeout: cfg.UpstreamRequestTimeout, - } - upstream, err := remote.New(rc, logrus.StandardLogger()) - if err != nil { - return nil, err - } - return &Agent{ - cfg: cfg, - activeProfiles: make(map[int]*agent.ProfileSession), - u: upstream, - }, nil -} - -func (a *Agent) Start() error { - sockPath := a.cfg.UNIXSocketPath - cs, err := csock.NewUnixCSock(sockPath, a.controlSocketHandler) - if err != nil { - return err - } - a.cs = cs - defer os.Remove(sockPath) - - go agent.SelfProfile(100, a.u, "pyroscope.agent.cpu{}", logrus.StandardLogger()) - cs.Start() - return nil -} - -func (a *Agent) Stop() { - a.cs.Stop() -} - -func (a *Agent) controlSocketHandler(req *csock.Request) *csock.Response { - switch req.Command { - case "start": - profileID := int(a.id.Next()) - // TODO: pass withSubprocesses from somewhere - // TODO: pass appName from somewhere - // TODO: add sample rate - - sc := agent.SessionConfig{ - Upstream: a.u, - AppName: "testapp", - ProfilingTypes: types.DefaultProfileTypes, - SpyName: types.GoSpy, - SampleRate: types.DefaultSampleRate, - UploadRate: 10 * time.Second, - Pid: 0, - WithSubprocesses: false, - } - s := agent.NewSession(&sc, logrus.StandardLogger()) - a.activeProfiles[profileID] = s - s.Start() - return &csock.Response{ProfileID: profileID} - case "stop": - // TODO: "testapp.cpu{}" should come from the client - profileID := req.ProfileID - if s, ok := a.activeProfiles[profileID]; ok { - s.Stop() - delete(a.activeProfiles, profileID) - } - return &csock.Response{} - default: - return &csock.Response{} - } -} diff --git a/pkg/agent/csock/csock.go b/pkg/agent/csock/csock.go deleted file mode 100644 index 56eeea4a13..0000000000 --- a/pkg/agent/csock/csock.go +++ /dev/null @@ -1,88 +0,0 @@ -// Package csock implements a control socket with a simple human-readable API -package csock - -import ( - "encoding/json" - "io/ioutil" - "net" - "net/http" - "strings" - - "github.com/sirupsen/logrus" -) - -type CSock struct { - listener net.Listener - activeProfiles map[int]chan struct{} - callback func(r *Request) *Response -} - -// NewCSock is a generic initializer. In most cases you want to use NewTCPCSock or NewUnixCSock. -func NewCSock(l net.Listener, cb func(r *Request) *Response) *CSock { - sock := &CSock{ - listener: l, - activeProfiles: make(map[int]chan struct{}), - callback: cb, - } - - return sock -} - -type Request struct { - SpyName string `json:"spy_name"` - ClientName string `json:"client_name"` - ClientVersion string `json:"client_version"` - Command string `json:"command"` - Pid int `json:"pid"` - ProfileID int `json:"profile_id"` -} - -type Response struct { - ProfileID int `json:"profile_id"` -} - -func commandFromRequest(r *http.Request) string { - s := r.URL.Path - arr := strings.Split(s, "/") - l := len(arr) - if l == 0 { - return "" - } - return arr[l-1] -} - -func (c *CSock) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - logrus.Error(err) // TODO: handle - } - req := &Request{} - err = json.Unmarshal(buf, &req) - if err != nil { - logrus.Error(err) - } - - req.Command = commandFromRequest(r) - resp := c.callback(req) - rw.WriteHeader(200) - b, err := json.Marshal(resp) - if err != nil { - logrus.Error(err) // TODO: handle - } - _, err = rw.Write(b) - if err != nil { - logrus.Error(err) - } -} - -func (c *CSock) CanonicalAddr() string { - return c.listener.Addr().String() -} - -func (c *CSock) Start() error { - return http.Serve(c.listener, c) -} - -func (c *CSock) Stop() error { - return c.listener.Close() -} diff --git a/pkg/agent/csock/tcp.go b/pkg/agent/csock/tcp.go deleted file mode 100644 index 36854b2be2..0000000000 --- a/pkg/agent/csock/tcp.go +++ /dev/null @@ -1,17 +0,0 @@ -package csock - -import "net" - -func NewTCPCSock(addrStr string, cb func(r *Request) *Response) (*CSock, error) { - addr, err := net.ResolveTCPAddr("tcp", addrStr) - if err != nil { - return nil, err - } - - listener, err := net.ListenTCP("tcp", addr) - if err != nil { - return nil, err - } - - return NewCSock(listener, cb), nil -} diff --git a/pkg/agent/csock/unix.go b/pkg/agent/csock/unix.go deleted file mode 100644 index 5719da41b6..0000000000 --- a/pkg/agent/csock/unix.go +++ /dev/null @@ -1,24 +0,0 @@ -package csock - -import ( - "net" - "os" -) - -func NewUnixCSock(path string, cb func(r *Request) *Response) (*CSock, error) { - addr, err := net.ResolveUnixAddr("unix", path) - if err != nil { - return nil, err - } - - listener, err := net.ListenUnix("unix", addr) - if err != nil { - return nil, err - } - err = os.Chmod(path, os.ModePerm) - if err != nil { - return nil, err - } - - return NewCSock(listener, cb), nil -} diff --git a/pkg/agent/debugspy/placeholder.go b/pkg/agent/debugspy/placeholder.go index bc25f07db8..4243c2ba0b 100644 --- a/pkg/agent/debugspy/placeholder.go +++ b/pkg/agent/debugspy/placeholder.go @@ -1,3 +1 @@ -// +build !debugspy - package debugspy diff --git a/pkg/agent/dotnetspy/placeholder.go b/pkg/agent/dotnetspy/placeholder.go index 2a1f73c53c..86ceefac21 100644 --- a/pkg/agent/dotnetspy/placeholder.go +++ b/pkg/agent/dotnetspy/placeholder.go @@ -1,3 +1 @@ -// +build !dotnetspy - package dotnetspy diff --git a/pkg/agent/dotnetspy/session.go b/pkg/agent/dotnetspy/session.go index be80bfa4dc..74d8ad7c06 100644 --- a/pkg/agent/dotnetspy/session.go +++ b/pkg/agent/dotnetspy/session.go @@ -58,7 +58,6 @@ func (s *session) start() error { return err } - s.session = ns stream := nettrace.NewStream(ns) trace, err := stream.Open() if err != nil { @@ -66,15 +65,19 @@ func (s *session) start() error { return err } - p := profiler.NewSampleProfiler(trace, profiler.WithManagedCodeOnly()) + p := profiler.NewSampleProfiler(trace, profilerOptions...) stream.EventHandler = p.EventHandler stream.MetadataHandler = p.MetadataHandler stream.StackBlockHandler = p.StackBlockHandler stream.SequencePointBlockHandler = p.SequencePointBlockHandler + s.session = ns s.ch = make(chan line) go func() { - defer close(s.ch) + defer func() { + s.session = nil + close(s.ch) + }() for { switch err = stream.Next(); err { default: @@ -101,12 +104,11 @@ func (s *session) start() error { // and starts a new session, if not in stopped state. func (s *session) flush(cb func([]byte, uint64)) error { // Ignore call, if NetTrace session has not been established. - if s.session == nil { - return nil - } - _ = s.session.Close() - for v := range s.ch { - cb(v.name, uint64(v.val)) + if s.session != nil { + _ = s.session.Close() + for v := range s.ch { + cb(v.name, uint64(v.val)) + } } if s.stopped { return nil diff --git a/pkg/agent/dotnetspy/session_unix.go b/pkg/agent/dotnetspy/session_unix.go new file mode 100644 index 0000000000..3a52231a21 --- /dev/null +++ b/pkg/agent/dotnetspy/session_unix.go @@ -0,0 +1,7 @@ +// +build dotnetspy,!windows + +package dotnetspy + +import "github.com/pyroscope-io/dotnetdiag/nettrace/profiler" + +var profilerOptions = []profiler.Option{profiler.WithManagedCodeOnly()} diff --git a/pkg/agent/dotnetspy/session_windows.go b/pkg/agent/dotnetspy/session_windows.go new file mode 100644 index 0000000000..a370a0ffd9 --- /dev/null +++ b/pkg/agent/dotnetspy/session_windows.go @@ -0,0 +1,7 @@ +// +build dotnetspy + +package dotnetspy + +import "github.com/pyroscope-io/dotnetdiag/nettrace/profiler" + +var profilerOptions []profiler.Option diff --git a/pkg/agent/ebpfspy/ebpfspy.go b/pkg/agent/ebpfspy/ebpfspy_linux.go similarity index 100% rename from pkg/agent/ebpfspy/ebpfspy.go rename to pkg/agent/ebpfspy/ebpfspy_linux.go diff --git a/pkg/agent/ebpfspy/placeholder.go b/pkg/agent/ebpfspy/placeholder.go index 87427d0085..d7d75bb9c3 100644 --- a/pkg/agent/ebpfspy/placeholder.go +++ b/pkg/agent/ebpfspy/placeholder.go @@ -1,3 +1 @@ -// +build !ebpfspy - package ebpfspy diff --git a/pkg/agent/ebpfspy/session.go b/pkg/agent/ebpfspy/session_linux.go similarity index 100% rename from pkg/agent/ebpfspy/session.go rename to pkg/agent/ebpfspy/session_linux.go diff --git a/pkg/agent/phpspy/placeholder.go b/pkg/agent/phpspy/placeholder.go index 2662d4388c..42eb8210e2 100644 --- a/pkg/agent/phpspy/placeholder.go +++ b/pkg/agent/phpspy/placeholder.go @@ -1,3 +1 @@ -// +build !phpspy - package phpspy diff --git a/pkg/agent/rbspy/placeholder.go b/pkg/agent/rbspy/placeholder.go index 47ec0006a8..09ec5b3a4f 100644 --- a/pkg/agent/rbspy/placeholder.go +++ b/pkg/agent/rbspy/placeholder.go @@ -1,3 +1 @@ -// +build !rbspy - package rbspy diff --git a/pkg/agent/target/service.go b/pkg/agent/target/service.go new file mode 100644 index 0000000000..2add3dbbbf --- /dev/null +++ b/pkg/agent/target/service.go @@ -0,0 +1,91 @@ +package target + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/mitchellh/go-ps" + "github.com/sirupsen/logrus" + + "github.com/pyroscope-io/pyroscope/pkg/agent" + "github.com/pyroscope-io/pyroscope/pkg/agent/pyspy" + "github.com/pyroscope-io/pyroscope/pkg/agent/spy" + "github.com/pyroscope-io/pyroscope/pkg/agent/upstream/remote" + "github.com/pyroscope-io/pyroscope/pkg/config" +) + +var ( + ErrNotFound = errors.New("service is not found") + ErrNotRunning = errors.New("service is not running") +) + +type service struct { + logger *logrus.Logger + target config.Target + sc *agent.SessionConfig +} + +func newServiceTarget(logger *logrus.Logger, upstream *remote.Remote, t config.Target) *service { + return &service{ + logger: logger, + target: t, + sc: &agent.SessionConfig{ + Upstream: upstream, + AppName: t.ApplicationName, + ProfilingTypes: []spy.ProfileType{spy.ProfileCPU}, + SpyName: t.SpyName, + SampleRate: uint32(t.SampleRate), + UploadRate: 10 * time.Second, + WithSubprocesses: t.DetectSubprocesses, + // PID to be specified. + }, + } +} + +func (s *service) attach(ctx context.Context) { + logger := s.logger.WithFields(logrus.Fields{ + "service-name": s.target.ServiceName, + "app-name": s.sc.AppName, + "spy-name": s.sc.SpyName}) + pid, err := getPID(s.target.ServiceName) + if err == nil { + logger.WithField("pid", pid).Debug("starting session") + s.sc.Pid = pid + err = s.wait(ctx) + } + if err != nil { + logger.WithError(err).Error("failed to attach spy to service") + } else { + logger.Debug("session ended") + } +} + +func (s *service) wait(ctx context.Context) error { + // TODO: this is somewhat hacky, we need to find a better way to configure agents + pyspy.Blocking = s.target.PyspyBlocking + + session := agent.NewSession(s.sc, s.logger) + if err := session.Start(); err != nil { + return err + } + defer session.Stop() + + t := time.NewTicker(time.Second) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-t.C: + p, err := ps.FindProcess(s.sc.Pid) + if err != nil { + return fmt.Errorf("could not find process: %w", err) + } + if p == nil && err == nil { + return nil + } + } + } +} diff --git a/pkg/agent/target/service_unix.go b/pkg/agent/target/service_unix.go new file mode 100644 index 0000000000..1cb6631e09 --- /dev/null +++ b/pkg/agent/target/service_unix.go @@ -0,0 +1,11 @@ +// +build !windows + +package target + +import ( + "errors" +) + +func getPID(_ string) (int, error) { + return 0, errors.New("not implemented") +} diff --git a/pkg/agent/target/service_windows.go b/pkg/agent/target/service_windows.go new file mode 100644 index 0000000000..c1078661c8 --- /dev/null +++ b/pkg/agent/target/service_windows.go @@ -0,0 +1,45 @@ +package target + +import ( + "errors" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/svc/mgr" +) + +// https://docs.microsoft.com/en-us/windows/win32/api/winsvc/nf-winsvc-queryservicestatusex +func getPID(serviceName string) (int, error) { + m, err := mgr.Connect() + if err != nil { + return 0, err + } + defer m.Disconnect() + s, err := m.OpenService(serviceName) + switch { + case err == nil: + case errors.Is(err, syscall.ERROR_NOT_FOUND): + return 0, ErrNotFound + default: + return 0, err + } + defer s.Close() + // A variable that receives the number of bytes needed to store the status + // information, if the function fails with ERROR_INSUFFICIENT_BUFFER. + // Not used. + var needed uint32 + var t windows.SERVICE_STATUS_PROCESS + err = windows.QueryServiceStatusEx(s.Handle, + windows.SC_STATUS_PROCESS_INFO, + (*byte)(unsafe.Pointer(&t)), + uint32(unsafe.Sizeof(t)), + &needed) + if err != nil { + return 0, err + } + if t.ProcessId == 0 { + return 0, ErrNotRunning + } + return int(t.ProcessId), nil +} diff --git a/pkg/agent/target/target.go b/pkg/agent/target/target.go new file mode 100644 index 0000000000..b3d2380981 --- /dev/null +++ b/pkg/agent/target/target.go @@ -0,0 +1,156 @@ +package target + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" + + "github.com/pyroscope-io/pyroscope/pkg/agent/spy" + "github.com/pyroscope-io/pyroscope/pkg/agent/types" + "github.com/pyroscope-io/pyroscope/pkg/agent/upstream/remote" + "github.com/pyroscope-io/pyroscope/pkg/config" + "github.com/pyroscope-io/pyroscope/pkg/util/names" +) + +const ( + defaultBackoffPeriod = time.Second * 10 +) + +// Manager tracks targets and attaches spies to running processes. +type Manager struct { + ctx context.Context + cancel context.CancelFunc + + logger *logrus.Logger + remote *remote.Remote + config *config.Agent + stop chan struct{} + wg sync.WaitGroup + + resolve func(config.Target) (target, bool) + backoffPeriod time.Duration +} + +type target interface { + // attach blocks till the context cancellation or the target + // process exit, whichever occurs first. + attach(ctx context.Context) +} + +func NewManager(l *logrus.Logger, r *remote.Remote, c *config.Agent) *Manager { + mgr := Manager{ + logger: l, + remote: r, + config: c, + stop: make(chan struct{}), + backoffPeriod: defaultBackoffPeriod, + } + mgr.ctx, mgr.cancel = context.WithCancel(context.Background()) + mgr.resolve = mgr.resolveTarget + return &mgr +} + +func (mgr *Manager) canonise(t *config.Target) error { + if t.SpyName == types.GoSpy { + return fmt.Errorf("gospy can not profile other processes") + } + var found bool + for _, s := range spy.SupportedSpies { + if s == t.SpyName { + found = true + break + } + } + if !found { + return fmt.Errorf("spy %q is not supported", t.SpyName) + } + if t.SampleRate == 0 { + t.SampleRate = types.DefaultSampleRate + } + if t.ApplicationName == "" { + t.ApplicationName = t.SpyName + "." + names.GetRandomName(generateSeed(t.ServiceName, t.SpyName)) + logger := mgr.logger.WithField("spy-name", t.SpyName) + if t.ServiceName != "" { + logger = logger.WithField("service-name", t.ServiceName) + } + logger.Infof("we recommend specifying application name via 'application-name' parameter") + logger.Infof("for now we chose the name for you and it's %q", t.ApplicationName) + } + return nil +} + +func (mgr *Manager) Start() { + for _, t := range mgr.config.Targets { + var tgt target + var ok bool + err := mgr.canonise(&t) + if err == nil { + tgt, ok = mgr.resolve(t) + if !ok { + err = fmt.Errorf("unknown target type") + } + } + if err != nil { + mgr.logger. + WithField("app-name", t.ApplicationName). + WithField("spy-name", t.SpyName). + WithError(err).Error("failed to setup target") + continue + } + mgr.wg.Add(1) + go mgr.runTarget(tgt) + } +} + +func (mgr *Manager) Stop() { + mgr.cancel() + close(mgr.stop) + mgr.wg.Wait() +} + +func (mgr *Manager) resolveTarget(t config.Target) (target, bool) { + var tgt target + switch { + case t.ServiceName != "": + tgt = newServiceTarget(mgr.logger, mgr.remote, t) + default: + return nil, false + } + return tgt, true +} + +func (mgr *Manager) runTarget(t target) { + ticker := time.NewTicker(mgr.backoffPeriod) + defer func() { + ticker.Stop() + mgr.wg.Done() + }() + for { + select { + default: + case <-mgr.stop: + return + } + t.attach(mgr.ctx) + // Unless manager is stopped, run spy again after some backoff + // period regardless of the exit reason. + select { + case <-ticker.C: + case <-mgr.stop: + return + } + } +} + +func generateSeed(args ...string) string { + path, err := os.Getwd() + if err != nil { + path = "" + } + return path + "|" + strings.Join(args, "&") +} diff --git a/pkg/agent/csock/csock_suite_test.go b/pkg/agent/target/target_suite_test.go similarity index 57% rename from pkg/agent/csock/csock_suite_test.go rename to pkg/agent/target/target_suite_test.go index f13ad2d2f3..2c80fbbf35 100644 --- a/pkg/agent/csock/csock_suite_test.go +++ b/pkg/agent/target/target_suite_test.go @@ -1,4 +1,4 @@ -package csock_test +package target_test import ( "testing" @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" ) -func TestCsock(t *testing.T) { +func TestTarget(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Csock Suite") + RunSpecs(t, "Target Suite") } diff --git a/pkg/agent/target/target_test.go b/pkg/agent/target/target_test.go new file mode 100644 index 0000000000..faa4d3cfd2 --- /dev/null +++ b/pkg/agent/target/target_test.go @@ -0,0 +1,43 @@ +// +build debugspy + +package target + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + + "github.com/pyroscope-io/pyroscope/pkg/agent/upstream/remote" + "github.com/pyroscope-io/pyroscope/pkg/config" +) + +type fakeTarget struct{ attached int } + +func (t *fakeTarget) attach(_ context.Context) { t.attached++ } + +var _ = Describe("target", func() { + It("Attaches to targets", func() { + tgtMgr := NewManager(logrus.StandardLogger(), new(remote.Remote), &config.Agent{ + Targets: []config.Target{ + { + ServiceName: "my-service", + SpyName: "debugspy", + ApplicationName: "my.app", + }, + }, + }) + + t := new(fakeTarget) + tgtMgr.resolve = func(c config.Target) (target, bool) { return t, true } + tgtMgr.backoffPeriod = time.Millisecond * 10 + + tgtMgr.Start() + time.Sleep(time.Second) + tgtMgr.Stop() + + Expect(t.attached).ToNot(BeZero()) + }) +}) diff --git a/pkg/agent/upstream/remote/remote.go b/pkg/agent/upstream/remote/remote.go index 38ab3f93f5..e79dd5161b 100644 --- a/pkg/agent/upstream/remote/remote.go +++ b/pkg/agent/upstream/remote/remote.go @@ -38,6 +38,8 @@ type RemoteConfig struct { UpstreamThreads int UpstreamAddress string UpstreamRequestTimeout time.Duration + + ManualStart bool } func New(cfg RemoteConfig, logger agent.Logger) (*Remote, error) { @@ -65,12 +67,18 @@ func New(cfg RemoteConfig, logger agent.Logger) (*Remote, error) { return nil, ErrCloudTokenRequired } - // start goroutines for uploading profile data - remote.start() + if !cfg.ManualStart { + // start goroutines for uploading profile data + remote.start() + } return remote, nil } +func (r *Remote) Start() { + r.start() +} + func (r *Remote) start() { for i := 0; i < r.cfg.UpstreamThreads; i++ { go r.handleJobs() diff --git a/pkg/cli/agent.go b/pkg/cli/agent.go new file mode 100644 index 0000000000..9e0c015abc --- /dev/null +++ b/pkg/cli/agent.go @@ -0,0 +1,97 @@ +package cli + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/kardianos/service" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" + + "github.com/pyroscope-io/pyroscope/pkg/agent/target" + "github.com/pyroscope-io/pyroscope/pkg/agent/upstream/remote" + "github.com/pyroscope-io/pyroscope/pkg/config" +) + +type agentService struct { + remote *remote.Remote + tgtMgr *target.Manager +} + +func newAgentService(logger *logrus.Logger, config *config.Agent) (*agentService, error) { + rc := remote.RemoteConfig{ + AuthToken: config.AuthToken, + UpstreamThreads: config.UpstreamThreads, + UpstreamAddress: config.ServerAddress, + UpstreamRequestTimeout: config.UpstreamRequestTimeout, + ManualStart: true, + } + upstream, err := remote.New(rc, logger) + if err != nil { + return nil, fmt.Errorf("upstream configuration: %w", err) + } + s := agentService{ + tgtMgr: target.NewManager(logger, upstream, config), + remote: upstream, + } + return &s, nil +} + +func (svc *agentService) Start(_ service.Service) error { + svc.remote.Start() + svc.tgtMgr.Start() + return nil +} + +func (svc *agentService) Stop(_ service.Service) error { + svc.tgtMgr.Stop() + svc.remote.Stop() + return nil +} + +func loadTargets(c *config.Agent) error { + b, err := ioutil.ReadFile(c.Config) + switch { + case err == nil: + case os.IsNotExist(err): + return nil + default: + return err + } + var a config.Agent + if err = yaml.Unmarshal(b, &a); err != nil { + return err + } + c.Targets = a.Targets + return nil +} + +func createLogger(config *config.Agent) (*logrus.Logger, error) { + if config.NoLogging { + logrus.SetOutput(ioutil.Discard) + return logrus.StandardLogger(), nil + } + l, err := logrus.ParseLevel(config.LogLevel) + if err != nil { + return nil, fmt.Errorf("parsing log level: %w", err) + } + logrus.SetLevel(l) + if service.Interactive() || config.LogFilePath == "" { + return logrus.StandardLogger(), nil + } + f, err := ensureLogFile(config.LogFilePath) + if err != nil { + return nil, fmt.Errorf("log file: %w", err) + } + logrus.SetOutput(f) + return logrus.StandardLogger(), nil +} + +func ensureLogFile(p string) (*os.File, error) { + if err := os.MkdirAll(filepath.Dir(p), 0770); err != nil { + return nil, err + } + return os.OpenFile(p, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) +} diff --git a/pkg/cli/agent_unix.go b/pkg/cli/agent_unix.go new file mode 100644 index 0000000000..2a87fe0ff1 --- /dev/null +++ b/pkg/cli/agent_unix.go @@ -0,0 +1,13 @@ +// +build !windows + +package cli + +import ( + "fmt" + + "github.com/pyroscope-io/pyroscope/pkg/config" +) + +func startAgent(_ *config.Agent) error { + return fmt.Errorf("agent mode is supported only on Windows") +} diff --git a/pkg/cli/agent_windows.go b/pkg/cli/agent_windows.go new file mode 100644 index 0000000000..d98d3ed191 --- /dev/null +++ b/pkg/cli/agent_windows.go @@ -0,0 +1,28 @@ +package cli + +import ( + "fmt" + + "github.com/kardianos/service" + "github.com/pyroscope-io/pyroscope/pkg/config" +) + +func startAgent(config *config.Agent) error { + logger, err := createLogger(config) + if err != nil { + return fmt.Errorf("could not create logger: %w", err) + } + logger.Info("starting pyroscope agent") + if err = loadTargets(config); err != nil { + return fmt.Errorf("could not load targets: %w", err) + } + agent, err := newAgentService(logger, config) + if err != nil { + return fmt.Errorf("could not initialize agent: %w", err) + } + svc, err := service.New(agent, &service.Config{Name: "pyroscope"}) + if err != nil { + return fmt.Errorf("could not initialize system service: %w", err) + } + return svc.Run() +} diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 2c782bef6b..e956553445 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -5,229 +5,19 @@ import ( "flag" "fmt" "os" - "path/filepath" - "reflect" "runtime" - "strconv" - "strings" - "time" - "github.com/pyroscope-io/pyroscope/pkg/agent" - "github.com/pyroscope-io/pyroscope/pkg/agent/spy" - "github.com/pyroscope-io/pyroscope/pkg/agent/upstream/direct" - "github.com/pyroscope-io/pyroscope/pkg/analytics" + "github.com/peterbourgon/ff/v3" + "github.com/peterbourgon/ff/v3/ffcli" + "github.com/sirupsen/logrus" + "github.com/pyroscope-io/pyroscope/pkg/build" "github.com/pyroscope-io/pyroscope/pkg/config" "github.com/pyroscope-io/pyroscope/pkg/convert" "github.com/pyroscope-io/pyroscope/pkg/dbmanager" "github.com/pyroscope-io/pyroscope/pkg/exec" - "github.com/pyroscope-io/pyroscope/pkg/server" - "github.com/pyroscope-io/pyroscope/pkg/storage" - "github.com/pyroscope-io/pyroscope/pkg/util/atexit" - "github.com/pyroscope-io/pyroscope/pkg/util/bytesize" - "github.com/pyroscope-io/pyroscope/pkg/util/debug" - "github.com/pyroscope-io/pyroscope/pkg/util/metrics" - "github.com/pyroscope-io/pyroscope/pkg/util/slices" - "github.com/sirupsen/logrus" - - "github.com/iancoleman/strcase" - "github.com/peterbourgon/ff/ffyaml" - "github.com/peterbourgon/ff/v3" - "github.com/peterbourgon/ff/v3/ffcli" ) -const timeFormat = "2006-01-02T15:04:05Z0700" - -type arrayFlags []string - -func (i *arrayFlags) String() string { - return strings.Join(*i, ", ") -} - -func (i *arrayFlags) Set(value string) error { - *i = append(*i, value) - return nil -} - -type timeFlag time.Time - -func (tf *timeFlag) String() string { - v := time.Time(*tf) - return v.Format(timeFormat) -} - -func (tf *timeFlag) Set(value string) error { - t2, err := time.Parse(timeFormat, value) - if err != nil { - var i int - i, err = strconv.Atoi(value) - if err != nil { - return err - } - t2 = time.Unix(int64(i), 0) - } - - t := (*time.Time)(tf) - b, _ := t2.MarshalBinary() - t.UnmarshalBinary(b) - - return nil -} - -// this is mostly reflection magic -func PopulateFlagSet(obj interface{}, flagSet *flag.FlagSet, skip ...string) *SortedFlags { - v := reflect.ValueOf(obj).Elem() - t := reflect.TypeOf(v.Interface()) - num := t.NumField() - - installPrefix := getInstallPrefix() - supportedSpies := strings.Join(spy.SupportedExecSpies(), ", ") - - for i := 0; i < num; i++ { - field := t.Field(i) - fieldV := v.Field(i) - defaultValStr := field.Tag.Get("def") - descVal := field.Tag.Get("desc") - skipVal := field.Tag.Get("skip") - nameVal := field.Tag.Get("name") - if nameVal == "" { - nameVal = strcase.ToKebab(field.Name) - } - if skipVal == "true" || slices.StringContains(skip, nameVal) { - continue - } - - descVal = strings.ReplaceAll(descVal, "", supportedSpies) - - switch field.Type { - case reflect.TypeOf([]string{}): - val := fieldV.Addr().Interface().(*[]string) - val2 := (*arrayFlags)(val) - flagSet.Var(val2, nameVal, descVal) - case reflect.TypeOf(""): - val := fieldV.Addr().Interface().(*string) - defaultValStr := strings.ReplaceAll(defaultValStr, "", installPrefix) - flagSet.StringVar(val, nameVal, defaultValStr, descVal) - case reflect.TypeOf(true): - val := fieldV.Addr().Interface().(*bool) - flagSet.BoolVar(val, nameVal, defaultValStr == "true", descVal) - case reflect.TypeOf(time.Time{}): - valTime := fieldV.Addr().Interface().(*time.Time) - val := (*timeFlag)(valTime) - flagSet.Var(val, nameVal, descVal) - case reflect.TypeOf(time.Second): - val := fieldV.Addr().Interface().(*time.Duration) - var defaultVal time.Duration - if defaultValStr == "" { - defaultVal = time.Duration(0) - } else { - var err error - defaultVal, err = time.ParseDuration(defaultValStr) - if err != nil { - logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) - } - } - flagSet.DurationVar(val, nameVal, defaultVal, descVal) - case reflect.TypeOf(bytesize.Byte): - val := fieldV.Addr().Interface().(*bytesize.ByteSize) - var defaultVal bytesize.ByteSize - if defaultValStr != "" { - var err error - defaultVal, err = bytesize.Parse(defaultValStr) - if err != nil { - logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) - } - } - *val = defaultVal - flagSet.Var(val, nameVal, descVal) - case reflect.TypeOf(1): - val := fieldV.Addr().Interface().(*int) - var defaultVal int - if defaultValStr == "" { - defaultVal = 0 - } else { - var err error - defaultVal, err = strconv.Atoi(defaultValStr) - if err != nil { - logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) - } - } - flagSet.IntVar(val, nameVal, defaultVal, descVal) - case reflect.TypeOf(1.00): - val := fieldV.Addr().Interface().(*float64) - var defaultVal float64 - if defaultValStr == "" { - defaultVal = 0.00 - } else { - var err error - defaultVal, err = strconv.ParseFloat(defaultValStr, 64) - if err != nil { - logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) - } - } - flagSet.Float64Var(val, nameVal, defaultVal, descVal) - case reflect.TypeOf(uint64(1)): - val := fieldV.Addr().Interface().(*uint64) - var defaultVal uint64 - if defaultValStr == "" { - defaultVal = uint64(0) - } else { - var err error - defaultVal, err = strconv.ParseUint(defaultValStr, 10, 64) - if err != nil { - logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) - } - } - flagSet.Uint64Var(val, nameVal, defaultVal, descVal) - case reflect.TypeOf(uint(1)): - val := fieldV.Addr().Interface().(*uint) - var defaultVal uint - if defaultValStr == "" { - defaultVal = uint(0) - } else { - out, err := strconv.ParseUint(defaultValStr, 10, 64) - if err != nil { - logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) - } - defaultVal = uint(out) - } - flagSet.UintVar(val, nameVal, defaultVal, descVal) - default: - logrus.Fatalf("type %s is not supported", field.Type) - } - } - return NewSortedFlags(obj, flagSet) -} - -// on mac pyroscope is usually installed via homebrew. homebrew installs under a prefix -// this is logic to figure out what prefix it is -func getInstallPrefix() string { - if runtime.GOOS != "darwin" { - return "" - } - - executablePath, err := os.Executable() - if err != nil { - // TODO: figure out what kind of errors might happen, handle it - return "" - } - cellarPath := filepath.Clean(filepath.Join(resolvePath(executablePath), "../../../..")) - - if !strings.HasSuffix(cellarPath, "Cellar") { - // looks like it's not installed via homebrew - return "" - } - - return filepath.Clean(filepath.Join(cellarPath, "../")) -} - -func resolvePath(path string) string { - if res, err := filepath.EvalSymlinks(path); err == nil { - return res - } - return path -} - func generateRootCmd(cfg *config.Config) *ffcli.Command { // init the log formatter for logrus logrus.SetReportCaller(true) @@ -245,6 +35,7 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { var ( serverFlagSet = flag.NewFlagSet("pyroscope server", flag.ExitOnError) + agentFlagSet = flag.NewFlagSet("pyroscope agent", flag.ExitOnError) convertFlagSet = flag.NewFlagSet("pyroscope convert", flag.ExitOnError) execFlagSet = flag.NewFlagSet("pyroscope exec", flag.ExitOnError) connectFlagSet = flag.NewFlagSet("pyroscope connect", flag.ExitOnError) @@ -253,6 +44,7 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { ) serverSortedFlags := PopulateFlagSet(&cfg.Server, serverFlagSet) + agentSortedFlags := PopulateFlagSet(&cfg.Agent, agentFlagSet) convertSortedFlags := PopulateFlagSet(&cfg.Convert, convertFlagSet) execSortedFlags := PopulateFlagSet(&cfg.Exec, execFlagSet, "pid") connectSortedFlags := PopulateFlagSet(&cfg.Exec, connectFlagSet) @@ -260,7 +52,7 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { rootSortedFlags := PopulateFlagSet(cfg, rootFlagSet) options := []ff.Option{ - ff.WithConfigFileParser(ffyaml.Parser), + ff.WithConfigFileParser(parser), ff.WithEnvVarPrefix("PYROSCOPE"), ff.WithAllowMissingConfigFile(true), ff.WithConfigFileFlag("config"), @@ -275,6 +67,15 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { FlagSet: serverFlagSet, } + agentCmd := &ffcli.Command{ + UsageFunc: agentSortedFlags.printUsage, + Options: append(options, ff.WithIgnoreUndefined(true)), + Name: "agent", + ShortUsage: "pyroscope agent [flags]", + ShortHelp: "starts pyroscope agent.", + FlagSet: agentFlagSet, + } + convertCmd := &ffcli.Command{ UsageFunc: convertSortedFlags.printUsage, Options: options, @@ -311,30 +112,19 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { FlagSet: dbmanagerFlagSet, } - rootCmd := &ffcli.Command{ - UsageFunc: rootSortedFlags.printUsage, - Options: options, - ShortUsage: "pyroscope [flags] ", - FlagSet: rootFlagSet, - Subcommands: []*ffcli.Command{ - convertCmd, - serverCmd, - execCmd, - connectCmd, - dbmanagerCmd, - }, - } - serverCmd.Exec = func(ctx context.Context, args []string) error { l, err := logrus.ParseLevel(cfg.Server.LogLevel) if err != nil { return err } logrus.SetLevel(l) - return startServer(&cfg.Server) } + agentCmd.Exec = func(ctx context.Context, args []string) error { + return startAgent(&cfg.Agent) + } + convertCmd.Exec = func(ctx context.Context, args []string) error { logrus.SetOutput(os.Stderr) logger := func(s string) { @@ -342,6 +132,7 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { } return convert.Cli(&cfg.Convert, logger, args) } + execCmd.Exec = func(_ context.Context, args []string) error { if cfg.Exec.NoLogging { logrus.SetLevel(logrus.PanicLevel) @@ -354,7 +145,7 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { return nil } - return exec.Cli(&cfg.Exec, args) + return exec.Cli(context.Background(), &cfg.Exec, args) } connectCmd.Exec = func(ctx context.Context, args []string) error { @@ -369,7 +160,7 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { return nil } - return exec.Cli(&cfg.Exec, args) + return exec.Cli(context.Background(), &cfg.Exec, args) } dbmanagerCmd.Exec = func(ctx context.Context, args []string) error { @@ -378,6 +169,22 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { } return dbmanager.Cli(&cfg.DbManager, &cfg.Server, args) } + + rootCmd := &ffcli.Command{ + UsageFunc: rootSortedFlags.printUsage, + Options: options, + ShortUsage: "pyroscope [flags] ", + FlagSet: rootFlagSet, + Subcommands: []*ffcli.Command{ + convertCmd, + serverCmd, + agentCmd, + execCmd, + connectCmd, + dbmanagerCmd, + }, + } + rootCmd.Exec = func(ctx context.Context, args []string) error { if cfg.Version || len(args) > 0 && args[0] == "version" { fmt.Println(gradientBanner()) @@ -396,70 +203,3 @@ func generateRootCmd(cfg *config.Config) *ffcli.Command { func Start(cfg *config.Config) error { return generateRootCmd(cfg).ParseAndRun(context.Background(), os.Args[1:]) } - -func startServer(cfg *config.Server) error { - // new a storage with configuration - s, err := storage.New(cfg) - if err != nil { - return fmt.Errorf("new storage: %v", err) - } - atexit.Register(func() { s.Close() }) - - // new a direct upstream - u := direct.New(s) - - // uploading the server profile self - if err := agent.SelfProfile(uint32(cfg.SampleRate), u, "pyroscope.server", logrus.StandardLogger()); err != nil { - return fmt.Errorf("start self profile: %v", err) - } - - // debuging the RAM and disk usages - go reportDebuggingInformation(cfg, s) - - // new server - c, err := server.New(cfg, s) - if err != nil { - return fmt.Errorf("new server: %v", err) - } - atexit.Register(func() { c.Stop() }) - - // start the analytics - if !cfg.AnalyticsOptOut { - analyticsService := analytics.NewService(cfg, s, c) - go analyticsService.Start() - atexit.Register(func() { analyticsService.Stop() }) - } - // if you ever change this line, make sure to update this homebrew test: - // https://github.com/pyroscope-io/homebrew-brew/blob/main/Formula/pyroscope.rb#L94 - logrus.Info("starting HTTP server") - - // start the server - return c.Start() -} - -func reportDebuggingInformation(cfg *config.Server, s *storage.Storage) { - t := time.NewTicker(1 * time.Second) - i := 0 - for range t.C { - if logrus.IsLevelEnabled(logrus.DebugLevel) { - maps := map[string]map[string]interface{}{ - "mem": debug.MemUsage(), - "disk": debug.DiskUsage(cfg.StoragePath), - "cache": s.CacheStats(), - } - - for dataType, data := range maps { - for k, v := range data { - if iv, ok := v.(bytesize.ByteSize); ok { - v = int64(iv) - } - metrics.Gauge(dataType+"."+k, v) - } - if i%30 == 0 { - logrus.WithFields(data).Debug(dataType + " stats") - } - } - } - i++ - } -} diff --git a/pkg/cli/cli_darwin.go b/pkg/cli/cli_darwin.go new file mode 100644 index 0000000000..ce00d00186 --- /dev/null +++ b/pkg/cli/cli_darwin.go @@ -0,0 +1,36 @@ +package cli + +import ( + "os" + "path/filepath" + "runtime" + "strings" +) + +func defaultAgentConfigPath() string { + return filepath.Join(getInstallPrefix(), "/etc/pyroscope/agent.yml") +} + +func defaultAgentLogFilePath() string { return "" } + +// on mac pyroscope is usually installed via homebrew. homebrew installs under a prefix +// this is logic to figure out what prefix it is +func getInstallPrefix() string { + if runtime.GOOS != "darwin" { + return "" + } + + executablePath, err := os.Executable() + if err != nil { + // TODO: figure out what kind of errors might happen, handle it + return "" + } + cellarPath := filepath.Clean(filepath.Join(resolvePath(executablePath), "../../../..")) + + if !strings.HasSuffix(cellarPath, "Cellar") { + // looks like it's not installed via homebrew + return "" + } + + return filepath.Clean(filepath.Join(cellarPath, "../")) +} diff --git a/pkg/cli/cli_linux.go b/pkg/cli/cli_linux.go new file mode 100644 index 0000000000..c4fa88d143 --- /dev/null +++ b/pkg/cli/cli_linux.go @@ -0,0 +1,11 @@ +// +build !windows + +package cli + +func defaultAgentConfigPath() string { + return "/etc/pyroscope/agent.yml" +} + +func defaultAgentLogFilePath() string { return "" } + +func getInstallPrefix() string { return "" } diff --git a/pkg/cli/cli_windows.go b/pkg/cli/cli_windows.go new file mode 100644 index 0000000000..79dac25e41 --- /dev/null +++ b/pkg/cli/cli_windows.go @@ -0,0 +1,30 @@ +package cli + +import ( + "os" + "path/filepath" +) + +func defaultAgentConfigPath() string { + return filepath.Join(getInstallPrefix(), `Pyroscope Agent`, "agent.yml") +} + +func defaultAgentLogFilePath() string { + return filepath.Join(getDataDirectory(), `Pyroscope Agent`, "pyroscope-agent.log") +} + +func getInstallPrefix() string { + return `C:\Program Files\Pyroscope` +} + +func getDataDirectory() string { + p, ok := os.LookupEnv("PROGRAMDATA") + if ok { + return filepath.Join(p, `Pyroscope`) + } + e, err := os.Executable() + if err == nil { + return filepath.Dir(e) + } + return filepath.Dir(os.Args[0]) +} diff --git a/pkg/cli/config_parser.go b/pkg/cli/config_parser.go new file mode 100644 index 0000000000..d012effbbb --- /dev/null +++ b/pkg/cli/config_parser.go @@ -0,0 +1,66 @@ +package cli + +import ( + "io" + "strconv" + + "gopkg.in/yaml.v2" + + "github.com/peterbourgon/ff/v3/ffyaml" +) + +// parser is a parser for YAML file format that silently ignores fields that +// cannot be converted to a string, e.g.: maps and structs. Flags and their +// values are read from the key/value pairs defined in the config file. +// Undefined (skipped) flags will cause parse error unless WithIgnoreUndefined +// option is set to true. +// +// Due to the fact that ff package does not support YAML with fields of +// map/struct type, those need to be decoded separately. +func parser(r io.Reader, set func(name, value string) error) error { + var m map[string]interface{} + d := yaml.NewDecoder(r) + if err := d.Decode(&m); err != nil && err != io.EOF { + return ffyaml.ParseError{Inner: err} + } + for key, val := range m { + for _, value := range valsToStrs(val) { + if err := set(key, value); err != nil { + return err + } + } + } + return nil +} + +func valsToStrs(val interface{}) []string { + if vals, ok := val.([]interface{}); ok { + ss := make([]string, len(vals)) + for i := range vals { + ss[i] = valToStr(vals[i]) + } + return ss + } + return []string{valToStr(val)} +} + +func valToStr(val interface{}) string { + switch v := val.(type) { + case byte: + return string([]byte{v}) + case string: + return v + case bool: + return strconv.FormatBool(v) + case uint64: + return strconv.FormatUint(v, 10) + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + case float64: + return strconv.FormatFloat(v, 'g', -1, 64) + default: + return "" + } +} diff --git a/pkg/cli/flags.go b/pkg/cli/flags.go new file mode 100644 index 0000000000..73e4fa63c1 --- /dev/null +++ b/pkg/cli/flags.go @@ -0,0 +1,183 @@ +package cli + +import ( + "flag" + "reflect" + "strconv" + "strings" + "time" + + "github.com/iancoleman/strcase" + "github.com/pyroscope-io/pyroscope/pkg/agent/spy" + "github.com/pyroscope-io/pyroscope/pkg/util/bytesize" + "github.com/pyroscope-io/pyroscope/pkg/util/slices" + "github.com/sirupsen/logrus" +) + +const timeFormat = "2006-01-02T15:04:05Z0700" + +type arrayFlags []string + +func (i *arrayFlags) String() string { + return strings.Join(*i, ", ") +} + +func (i *arrayFlags) Set(value string) error { + *i = append(*i, value) + return nil +} + +type timeFlag time.Time + +func (tf *timeFlag) String() string { + v := time.Time(*tf) + return v.Format(timeFormat) +} + +func (tf *timeFlag) Set(value string) error { + t2, err := time.Parse(timeFormat, value) + if err != nil { + var i int + i, err = strconv.Atoi(value) + if err != nil { + return err + } + t2 = time.Unix(int64(i), 0) + } + + t := (*time.Time)(tf) + b, _ := t2.MarshalBinary() + t.UnmarshalBinary(b) + + return nil +} + +func PopulateFlagSet(obj interface{}, flagSet *flag.FlagSet, skip ...string) *SortedFlags { + v := reflect.ValueOf(obj).Elem() + t := reflect.TypeOf(v.Interface()) + num := t.NumField() + + installPrefix := getInstallPrefix() + supportedSpies := strings.Join(spy.SupportedExecSpies(), ", ") + + for i := 0; i < num; i++ { + field := t.Field(i) + fieldV := v.Field(i) + if !(fieldV.IsValid() && fieldV.CanSet()) { + continue + } + + defaultValStr := field.Tag.Get("def") + descVal := field.Tag.Get("desc") + skipVal := field.Tag.Get("skip") + nameVal := field.Tag.Get("name") + if nameVal == "" { + nameVal = strcase.ToKebab(field.Name) + } + if skipVal == "true" || slices.StringContains(skip, nameVal) { + continue + } + + descVal = strings.ReplaceAll(descVal, "", supportedSpies) + + switch field.Type { + case reflect.TypeOf([]string{}): + val := fieldV.Addr().Interface().(*[]string) + val2 := (*arrayFlags)(val) + flagSet.Var(val2, nameVal, descVal) + case reflect.TypeOf(""): + val := fieldV.Addr().Interface().(*string) + defaultValStr = strings.ReplaceAll(defaultValStr, "", installPrefix) + defaultValStr = strings.ReplaceAll(defaultValStr, "", defaultAgentConfigPath()) + defaultValStr = strings.ReplaceAll(defaultValStr, "", defaultAgentLogFilePath()) + flagSet.StringVar(val, nameVal, defaultValStr, descVal) + case reflect.TypeOf(true): + val := fieldV.Addr().Interface().(*bool) + flagSet.BoolVar(val, nameVal, defaultValStr == "true", descVal) + case reflect.TypeOf(time.Time{}): + valTime := fieldV.Addr().Interface().(*time.Time) + val := (*timeFlag)(valTime) + flagSet.Var(val, nameVal, descVal) + case reflect.TypeOf(time.Second): + val := fieldV.Addr().Interface().(*time.Duration) + var defaultVal time.Duration + if defaultValStr == "" { + defaultVal = time.Duration(0) + } else { + var err error + defaultVal, err = time.ParseDuration(defaultValStr) + if err != nil { + logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) + } + } + flagSet.DurationVar(val, nameVal, defaultVal, descVal) + case reflect.TypeOf(bytesize.Byte): + val := fieldV.Addr().Interface().(*bytesize.ByteSize) + var defaultVal bytesize.ByteSize + if defaultValStr != "" { + var err error + defaultVal, err = bytesize.Parse(defaultValStr) + if err != nil { + logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) + } + } + *val = defaultVal + flagSet.Var(val, nameVal, descVal) + case reflect.TypeOf(1): + val := fieldV.Addr().Interface().(*int) + var defaultVal int + if defaultValStr == "" { + defaultVal = 0 + } else { + var err error + defaultVal, err = strconv.Atoi(defaultValStr) + if err != nil { + logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) + } + } + flagSet.IntVar(val, nameVal, defaultVal, descVal) + case reflect.TypeOf(1.00): + val := fieldV.Addr().Interface().(*float64) + var defaultVal float64 + if defaultValStr == "" { + defaultVal = 0.00 + } else { + var err error + defaultVal, err = strconv.ParseFloat(defaultValStr, 64) + if err != nil { + logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) + } + } + flagSet.Float64Var(val, nameVal, defaultVal, descVal) + case reflect.TypeOf(uint64(1)): + val := fieldV.Addr().Interface().(*uint64) + var defaultVal uint64 + if defaultValStr == "" { + defaultVal = uint64(0) + } else { + var err error + defaultVal, err = strconv.ParseUint(defaultValStr, 10, 64) + if err != nil { + logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) + } + } + flagSet.Uint64Var(val, nameVal, defaultVal, descVal) + case reflect.TypeOf(uint(1)): + val := fieldV.Addr().Interface().(*uint) + var defaultVal uint + if defaultValStr == "" { + defaultVal = uint(0) + } else { + out, err := strconv.ParseUint(defaultValStr, 10, 64) + if err != nil { + logrus.Fatalf("invalid default value: %q (%s)", defaultValStr, nameVal) + } + defaultVal = uint(out) + } + flagSet.UintVar(val, nameVal, defaultVal, descVal) + default: + logrus.Fatalf("type %s is not supported", field.Type) + } + } + return NewSortedFlags(obj, flagSet) +} diff --git a/pkg/cli/flags_test.go b/pkg/cli/flags_test.go index 337c43f882..cae3bdb22d 100644 --- a/pkg/cli/flags_test.go +++ b/pkg/cli/flags_test.go @@ -7,9 +7,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/peterbourgon/ff/ffyaml" "github.com/peterbourgon/ff/v3" "github.com/peterbourgon/ff/v3/ffcli" + + "github.com/pyroscope-io/pyroscope/pkg/config" "github.com/pyroscope-io/pyroscope/pkg/util/bytesize" ) @@ -70,7 +71,7 @@ var _ = Describe("flags", func() { exampleCommand := &ffcli.Command{ FlagSet: exampleFlagSet, Options: []ff.Option{ - ff.WithConfigFileParser(ffyaml.Parser), + ff.WithConfigFileParser(parser), ff.WithConfigFileFlag("config"), }, Exec: func(_ context.Context, args []string) error { @@ -79,7 +80,7 @@ var _ = Describe("flags", func() { } err := exampleCommand.ParseAndRun(context.Background(), []string{ - "-config", "example.yml", + "-config", "testdata/example.yml", }) Expect(err).ToNot(HaveOccurred()) @@ -92,7 +93,7 @@ var _ = Describe("flags", func() { Expect(cfg.FooBytes).To(Equal(100 * bytesize.MB)) }) - It("arguments take precendence", func() { + It("arguments take precedence", func() { exampleFlagSet := flag.NewFlagSet("example flag set", flag.ExitOnError) cfg := FlagsStruct{} PopulateFlagSet(&cfg, exampleFlagSet) @@ -100,7 +101,7 @@ var _ = Describe("flags", func() { exampleCommand := &ffcli.Command{ FlagSet: exampleFlagSet, Options: []ff.Option{ - ff.WithConfigFileParser(ffyaml.Parser), + ff.WithConfigFileParser(parser), ff.WithConfigFileFlag("config"), }, Exec: func(_ context.Context, args []string) error { @@ -109,13 +110,58 @@ var _ = Describe("flags", func() { } err := exampleCommand.ParseAndRun(context.Background(), []string{ - "-config", "example.yml", + "-config", "testdata/example.yml", "-foo", "test-val-4", }) Expect(err).ToNot(HaveOccurred()) Expect(cfg.Foo).To(Equal("test-val-4")) }) + + It("agent configuration", func() { + exampleFlagSet := flag.NewFlagSet("example flag set", flag.ExitOnError) + var cfg config.Agent + PopulateFlagSet(&cfg, exampleFlagSet) + + exampleCommand := &ffcli.Command{ + FlagSet: exampleFlagSet, + Options: []ff.Option{ + ff.WithIgnoreUndefined(true), + ff.WithConfigFileParser(parser), + ff.WithConfigFileFlag("config"), + }, + Exec: func(_ context.Context, args []string) error { + return nil + }, + } + + err := exampleCommand.ParseAndRun(context.Background(), []string{ + "-config", "testdata/agent.yml", + }) + + Expect(err).ToNot(HaveOccurred()) + Expect(cfg).To(Equal(config.Agent{ + Config: "testdata/agent.yml", + LogLevel: "debug", + NoLogging: false, + ServerAddress: "http://localhost:4040", + AuthToken: "", + UpstreamThreads: 4, + UpstreamRequestTimeout: 10 * time.Second, + })) + + Expect(loadTargets(&cfg)).ToNot(HaveOccurred()) + Expect(cfg.Targets).To(Equal([]config.Target{ + { + ServiceName: "foo", + SpyName: "debugspy", + ApplicationName: "foo.app", + SampleRate: 0, + DetectSubprocesses: false, + PyspyBlocking: false, + }, + })) + }) }) }) }) diff --git a/pkg/cli/server_unix.go b/pkg/cli/server_unix.go new file mode 100644 index 0000000000..8a447f81f4 --- /dev/null +++ b/pkg/cli/server_unix.go @@ -0,0 +1,88 @@ +// +build !windows + +package cli + +import ( + "fmt" + "time" + + "github.com/pyroscope-io/pyroscope/pkg/util/bytesize" + "github.com/pyroscope-io/pyroscope/pkg/util/debug" + "github.com/pyroscope-io/pyroscope/pkg/util/metrics" + "github.com/sirupsen/logrus" + + "github.com/pyroscope-io/pyroscope/pkg/agent" + "github.com/pyroscope-io/pyroscope/pkg/agent/upstream/direct" + "github.com/pyroscope-io/pyroscope/pkg/analytics" + "github.com/pyroscope-io/pyroscope/pkg/config" + "github.com/pyroscope-io/pyroscope/pkg/server" + "github.com/pyroscope-io/pyroscope/pkg/storage" + "github.com/pyroscope-io/pyroscope/pkg/util/atexit" +) + +func startServer(cfg *config.Server) error { + // new a storage with configuration + s, err := storage.New(cfg) + if err != nil { + return fmt.Errorf("new storage: %v", err) + } + atexit.Register(func() { s.Close() }) + + // new a direct upstream + u := direct.New(s) + + // uploading the server profile self + if err := agent.SelfProfile(uint32(cfg.SampleRate), u, "pyroscope.server", logrus.StandardLogger()); err != nil { + return fmt.Errorf("start self profile: %v", err) + } + + // debuging the RAM and disk usages + go reportDebuggingInformation(cfg, s) + + // new server + c, err := server.New(cfg, s) + if err != nil { + return fmt.Errorf("new server: %v", err) + } + atexit.Register(func() { c.Stop() }) + + // start the analytics + if !cfg.AnalyticsOptOut { + analyticsService := analytics.NewService(cfg, s, c) + go analyticsService.Start() + atexit.Register(func() { analyticsService.Stop() }) + } + // if you ever change this line, make sure to update this homebrew test: + // https://github.com/pyroscope-io/homebrew-brew/blob/main/Formula/pyroscope.rb#L94 + logrus.Info("starting HTTP server") + + // start the server + return c.Start() +} + +func reportDebuggingInformation(cfg *config.Server, s *storage.Storage) { + t := time.NewTicker(1 * time.Second) + i := 0 + for range t.C { + if logrus.IsLevelEnabled(logrus.DebugLevel) { + maps := map[string]map[string]interface{}{ + "mem": debug.MemUsage(), + "disk": debug.DiskUsage(cfg.StoragePath), + "cache": s.CacheStats(), + } + + for dataType, data := range maps { + for k, v := range data { + if iv, ok := v.(bytesize.ByteSize); ok { + v = int64(iv) + } + metrics.Gauge(dataType+"."+k, v) + } + if i%30 == 0 { + logrus.WithFields(data).Debug(dataType + " stats") + } + } + } + i++ + } +} diff --git a/pkg/cli/server_windows.go b/pkg/cli/server_windows.go new file mode 100644 index 0000000000..152c647a2d --- /dev/null +++ b/pkg/cli/server_windows.go @@ -0,0 +1,11 @@ +package cli + +import ( + "fmt" + + "github.com/pyroscope-io/pyroscope/pkg/config" +) + +func startServer(_ *config.Server) error { + return fmt.Errorf("server mode is not supported on Windows") +} diff --git a/pkg/cli/testdata/agent.yml b/pkg/cli/testdata/agent.yml new file mode 100644 index 0000000000..8d4a090e1b --- /dev/null +++ b/pkg/cli/testdata/agent.yml @@ -0,0 +1,7 @@ +--- +log-level: debug + +targets: + - service-name: foo + application-name: foo.app + spy-name: debugspy diff --git a/pkg/cli/example.yml b/pkg/cli/testdata/example.yml similarity index 100% rename from pkg/cli/example.yml rename to pkg/cli/testdata/example.yml diff --git a/pkg/cli/usage.go b/pkg/cli/usage.go index 273cdc0e22..9483d4bfef 100644 --- a/pkg/cli/usage.go +++ b/pkg/cli/usage.go @@ -28,7 +28,6 @@ func init() { // disabled these commands for now, they are not documented and confuse people var hiddenCommands = []string{ - "agent", "convert", "dbmanager", } diff --git a/pkg/config/config.go b/pkg/config/config.go index cff92e1116..70f35ca697 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,17 +17,30 @@ type Config struct { } type Agent struct { - Config string `def:"/etc/pyroscope/agent.yml" desc:"location of config file"` - LogLevel string `def:"info" desc:"log level: debug|info|warn|error"` + Config string `def:"" desc:"location of config file"` + + LogFilePath string `def:"" desc:"log file path"` + LogLevel string `def:"info" desc:"log level: debug|info|warn|error"` + NoLogging bool `def:"false" desc:"disables logging from pyroscope"` - // AgentCMD []string - AgentSpyName string `desc:"name of the spy you want to use"` // TODO: add options - AgentPID int `def:"-1" desc:"pid of the process you want to spy on"` ServerAddress string `def:"http://localhost:4040" desc:"address of the pyroscope server"` AuthToken string `def:"" desc:"authorization token used to upload profiling data"` - UpstreamThreads int `def:"4"` - UpstreamRequestTimeout time.Duration `def:"10s"` - UNIXSocketPath string `def:"/var/run/pyroscope-agent.sock" desc:"path to a UNIX socket file"` + UpstreamThreads int `def:"4" desc:"number of upload threads"` + UpstreamRequestTimeout time.Duration `def:"10s" desc:"profile upload timeout"` + + Targets []Target `skip:"true"` +} + +type Target struct { + ServiceName string `yaml:"service-name"` + + SpyName string `yaml:"spy-name"` + ApplicationName string `yaml:"application-name"` + SampleRate uint `yaml:"sample-rate"` + DetectSubprocesses bool `yaml:"detect-subprocesses"` + + // Spy-specific settings. + PyspyBlocking bool `yaml:"pyspy-blocking"` } type Server struct { diff --git a/pkg/exec/cli.go b/pkg/exec/cli.go index 2b0bacc7ff..bd3c324923 100644 --- a/pkg/exec/cli.go +++ b/pkg/exec/cli.go @@ -1,16 +1,13 @@ package exec import ( + "context" "errors" "fmt" "os" "os/exec" - "os/user" "path" - "regexp" - "strconv" "strings" - "syscall" "time" "github.com/fatih/color" @@ -31,7 +28,7 @@ var disableMacOSChecks bool var disableLinuxChecks bool // Cli is command line interface for both exec and connect commands -func Cli(cfg *config.Exec, args []string) error { +func Cli(ctx context.Context, cfg *config.Exec, args []string) error { // isExec = true means we need to start the process first (pyroscope exec) // isExec = false means the process is already there (pyroscope connect) isExec := cfg.Pid == 0 @@ -94,34 +91,14 @@ func Cli(cfg *config.Exec, args []string) error { pid := cfg.Pid var cmd *exec.Cmd if isExec { - cmd = exec.Command(args[0], args[1:]...) + cmd = exec.CommandContext(ctx, args[0], args[1:]...) cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout cmd.Stdin = os.Stdin - cmd.SysProcAttr = &syscall.SysProcAttr{} - - // permissions drop - if isRoot() && !cfg.NoRootDrop && os.Getenv("SUDO_UID") != "" && os.Getenv("SUDO_GID") != "" { - creds, err := generateCredentialsDrop() - if err != nil { - logrus.Errorf("failed to drop permissions, %q", err) - } else { - cmd.SysProcAttr.Credential = creds - } - } - - if cfg.UserName != "" || cfg.GroupName != "" { - creds, err := generateCredentials(cfg.UserName, cfg.GroupName) - if err != nil { - logrus.Errorf("failed to generate credentials: %q", err) - } else { - cmd.SysProcAttr.Credential = creds - } + if err := adjustCmd(cmd, *cfg); err != nil { + logrus.Error(err) } - - cmd.SysProcAttr.Setpgid = true - err := cmd.Start() - if err != nil { + if err := cmd.Start(); err != nil { return err } pid = cmd.Process.Pid @@ -170,7 +147,7 @@ func Cli(cfg *config.Exec, args []string) error { if isExec { waitForSpawnedProcessToExit(cmd) } else { - waitForProcessToExit(pid) + waitForProcessToExit(ctx, pid) } return nil @@ -206,18 +183,23 @@ func waitForSpawnedProcessToExit(cmd *exec.Cmd) { } } -func waitForProcessToExit(pid int) { +func waitForProcessToExit(ctx context.Context, pid int) { // pid == -1 means we're profiling whole system if pid == -1 { select {} // revive:disable-line:empty-block This block has to be empty } - t := time.NewTicker(time.Second) - for range t.C { - p, err := ps.FindProcess(pid) - if p == nil || err != nil { - logrus.WithField("err", err).Debug("could not find subprocess, it might be dead") + defer t.Stop() + for { + select { + case <-ctx.Done(): return + case <-t.C: + p, err := ps.FindProcess(pid) + if p == nil || err != nil { + logrus.WithField("err", err).Debug("could not find subprocess, it might be dead") + return + } } } } @@ -253,11 +235,6 @@ func stringsContains(arr []string, element string) bool { return false } -func isRoot() bool { - u, err := user.Current() - return err == nil && u.Username == "root" -} - func generateSeed(args []string) string { cwd, err := os.Getwd() if err != nil { @@ -265,62 +242,3 @@ func generateSeed(args []string) string { } return cwd + "|" + strings.Join(args, "&") } - -func generateCredentialsDrop() (*syscall.Credential, error) { - sudoUser := os.Getenv("SUDO_USER") - sudoUID := os.Getenv("SUDO_UID") - sudoGid := os.Getenv("SUDO_GID") - - logrus.Infof("dropping permissions, running command as %q (%s/%s)", sudoUser, sudoUID, sudoGid) - - uid, err := strconv.Atoi(sudoUID) - if err != nil { - return nil, err - } - gid, err := strconv.Atoi(sudoGid) - if err != nil { - return nil, err - } - - return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil -} - -var digitCheck = regexp.MustCompile(`^[0-9]+$`) - -func generateCredentials(userName, groupName string) (*syscall.Credential, error) { - c := syscall.Credential{} - - var u *user.User - var g *user.Group - var err error - - if userName != "" { - if digitCheck.MatchString(userName) { - u, err = user.LookupId(userName) - } else { - u, err = user.Lookup(userName) - } - if err != nil { - return nil, err - } - - uid, _ := strconv.Atoi(u.Uid) - c.Uid = uint32(uid) - } - - if groupName != "" { - if digitCheck.MatchString(groupName) { - g, err = user.LookupGroupId(groupName) - } else { - g, err = user.LookupGroup(groupName) - } - if err != nil { - return nil, err - } - - gid, _ := strconv.Atoi(g.Gid) - c.Gid = uint32(gid) - } - - return &c, nil -} diff --git a/pkg/exec/cli_darwin.go b/pkg/exec/cli_darwin.go index fc55b6948d..8d5273ba53 100644 --- a/pkg/exec/cli_darwin.go +++ b/pkg/exec/cli_darwin.go @@ -1,5 +1,3 @@ -// +build darwin - package exec import "errors" diff --git a/pkg/exec/cli_linux.go b/pkg/exec/cli_linux.go index c3e4c2b017..f76a0504e5 100644 --- a/pkg/exec/cli_linux.go +++ b/pkg/exec/cli_linux.go @@ -1,5 +1,3 @@ -// +build linux - package exec import ( diff --git a/pkg/exec/cli_unix.go b/pkg/exec/cli_unix.go new file mode 100644 index 0000000000..e50764d5a9 --- /dev/null +++ b/pkg/exec/cli_unix.go @@ -0,0 +1,104 @@ +// +build !windows + +package exec + +import ( + "os" + "os/exec" + "os/user" + "regexp" + "strconv" + "syscall" + + "github.com/sirupsen/logrus" + + "github.com/pyroscope-io/pyroscope/pkg/config" +) + +func adjustCmd(cmd *exec.Cmd, cfg config.Exec) error { + cmd.SysProcAttr = &syscall.SysProcAttr{} + // permissions drop + if isRoot() && !cfg.NoRootDrop && os.Getenv("SUDO_UID") != "" && os.Getenv("SUDO_GID") != "" { + creds, err := generateCredentialsDrop() + if err != nil { + logrus.Errorf("failed to drop permissions, %q", err) + } else { + cmd.SysProcAttr.Credential = creds + } + } + + if cfg.UserName != "" || cfg.GroupName != "" { + creds, err := generateCredentials(cfg.UserName, cfg.GroupName) + if err != nil { + logrus.Errorf("failed to generate credentials: %q", err) + } else { + cmd.SysProcAttr.Credential = creds + } + } + cmd.SysProcAttr.Setpgid = true + return nil +} + +func isRoot() bool { + u, err := user.Current() + return err == nil && u.Username == "root" +} + +func generateCredentialsDrop() (*syscall.Credential, error) { + sudoUser := os.Getenv("SUDO_USER") + sudoUID := os.Getenv("SUDO_UID") + sudoGid := os.Getenv("SUDO_GID") + + logrus.Infof("dropping permissions, running command as %q (%s/%s)", sudoUser, sudoUID, sudoGid) + + uid, err := strconv.Atoi(sudoUID) + if err != nil { + return nil, err + } + gid, err := strconv.Atoi(sudoGid) + if err != nil { + return nil, err + } + + return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil +} + +var digitCheck = regexp.MustCompile(`^[0-9]+$`) + +func generateCredentials(userName, groupName string) (*syscall.Credential, error) { + c := syscall.Credential{} + + var u *user.User + var g *user.Group + var err error + + if userName != "" { + if digitCheck.MatchString(userName) { + u, err = user.LookupId(userName) + } else { + u, err = user.Lookup(userName) + } + if err != nil { + return nil, err + } + + uid, _ := strconv.Atoi(u.Uid) + c.Uid = uint32(uid) + } + + if groupName != "" { + if digitCheck.MatchString(groupName) { + g, err = user.LookupGroupId(groupName) + } else { + g, err = user.LookupGroup(groupName) + } + if err != nil { + return nil, err + } + + gid, _ := strconv.Atoi(g.Gid) + c.Gid = uint32(gid) + } + + return &c, nil +} diff --git a/pkg/exec/cli_windows.go b/pkg/exec/cli_windows.go new file mode 100644 index 0000000000..671a1459a0 --- /dev/null +++ b/pkg/exec/cli_windows.go @@ -0,0 +1,15 @@ +package exec + +import ( + "os/exec" + + "github.com/pyroscope-io/pyroscope/pkg/config" +) + +func performOSChecks(_ string) error { + return nil +} + +func adjustCmd(cmd *exec.Cmd, cfg config.Exec) error { + return nil +} diff --git a/pkg/exec/exec_test.go b/pkg/exec/exec_test.go index d1a508d82a..39f5c85210 100644 --- a/pkg/exec/exec_test.go +++ b/pkg/exec/exec_test.go @@ -4,6 +4,8 @@ package exec import ( + "context" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pyroscope-io/pyroscope/pkg/config" @@ -18,14 +20,14 @@ var _ = Describe("Cli", func() { Describe("Cli", func() { Context("no arguments", func() { It("returns error", func() { - err := Cli(&(*cfg).Exec, []string{}) + err := Cli(context.Background(), &(*cfg).Exec, []string{}) Expect(err).To(MatchError("no arguments passed")) }) }) Context("simple case", func() { It("returns nil", func() { (*cfg).Exec.SpyName = "debugspy" - err := Cli(&(*cfg).Exec, []string{"ls"}) + err := Cli(context.Background(), &(*cfg).Exec, []string{"ls"}) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/storage/segment/segment.go b/pkg/storage/segment/segment.go index 06a59a83bc..d4f6ec745b 100644 --- a/pkg/storage/segment/segment.go +++ b/pkg/storage/segment/segment.go @@ -3,6 +3,8 @@ package segment import ( "fmt" "math/big" + "os" + "path/filepath" "sync" "time" ) @@ -238,7 +240,7 @@ func (s *Segment) Put(st, et time.Time, samples uint64, cb func(depth int, t tim v.add(sn, r, true) cb(depth, tm, r, addons) }) - v.print(fmt.Sprintf("/tmp/0-put-%s-%s.html", st.String(), et.String())) + v.print(filepath.Join(os.TempDir(), fmt.Sprintf("0-put-%s-%s.html", st.String(), et.String()))) } // TODO: simplify arguments @@ -257,7 +259,7 @@ func (s *Segment) Get(st, et time.Time, cb func(depth int, samples, writes uint6 v.add(sn, r, true) cb(depth, sn.samples, sn.writes, t, r) }) - v.print(fmt.Sprintf("/tmp/0-get-%s-%s.html", st.String(), et.String())) + v.print(filepath.Join(os.TempDir(), fmt.Sprintf("0-get-%s-%s.html", st.String(), et.String()))) } // TODO: this should be refactored diff --git a/pkg/testing/tmpdir.go b/pkg/testing/tmpdir.go index 3f48f514e9..6850b6d4df 100644 --- a/pkg/testing/tmpdir.go +++ b/pkg/testing/tmpdir.go @@ -33,7 +33,7 @@ func DirStats(path string) (directories, files int, size bytesize.ByteSize) { func TmpDir(cb func(name string)) { defer ginkgo.GinkgoRecover() - path, err := ioutil.TempDir("/tmp", "pyroscope-test-dir") + path, err := ioutil.TempDir("", "pyroscope-test-dir") if err != nil { panic(err) } @@ -54,7 +54,7 @@ func (t *TmpDirectory) Close() { func TmpDirSync() *TmpDirectory { defer ginkgo.GinkgoRecover() - path, err := ioutil.TempDir("/tmp", "pyroscope-test-dir") + path, err := ioutil.TempDir("", "pyroscope-test-dir") if err != nil { panic(err) } diff --git a/pkg/util/disk/usage_test.go b/pkg/util/disk/usage_test.go index d34e130001..3186c601b6 100644 --- a/pkg/util/disk/usage_test.go +++ b/pkg/util/disk/usage_test.go @@ -3,6 +3,7 @@ package disk import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/pyroscope-io/pyroscope/pkg/config" "github.com/pyroscope-io/pyroscope/pkg/testing" ) diff --git a/pkg/util/disk/usage.go b/pkg/util/disk/usage_unix.go similarity index 94% rename from pkg/util/disk/usage.go rename to pkg/util/disk/usage_unix.go index d2f23c76b8..6ee51a7d21 100644 --- a/pkg/util/disk/usage.go +++ b/pkg/util/disk/usage_unix.go @@ -1,3 +1,5 @@ +// +build !windows + package disk import ( diff --git a/pkg/util/disk/usage_windows.go b/pkg/util/disk/usage_windows.go new file mode 100644 index 0000000000..64859fed98 --- /dev/null +++ b/pkg/util/disk/usage_windows.go @@ -0,0 +1,41 @@ +package disk + +import ( + "os" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" + + "github.com/pyroscope-io/pyroscope/pkg/util/bytesize" +) + +var ( + kernel32 = windows.NewLazySystemDLL("kernel32.dll") + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx + getDiskFreeSpaceEx = kernel32.NewProc("GetDiskFreeSpaceExW") +) + +func FreeSpace(path string) (bytesize.ByteSize, error) { + dirPath, err := syscall.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + + var ( + freeBytesAvailableToCaller uint64 + totalNumberOfBytes uint64 + totalNumberOfFreeBytes uint64 + ) + + ret, _, err := getDiskFreeSpaceEx.Call( + uintptr(unsafe.Pointer(dirPath)), + uintptr(unsafe.Pointer(&freeBytesAvailableToCaller)), + uintptr(unsafe.Pointer(&totalNumberOfBytes)), + uintptr(unsafe.Pointer(&totalNumberOfFreeBytes))) + if ret == 0 { + return 0, os.NewSyscallError("GetDiskFreeSpaceEx", err) + } + + return bytesize.ByteSize(freeBytesAvailableToCaller), nil +} diff --git a/scripts/windows/generate-windows-version-info/README.md b/scripts/windows/generate-windows-version-info/README.md new file mode 100644 index 0000000000..49e7b38394 --- /dev/null +++ b/scripts/windows/generate-windows-version-info/README.md @@ -0,0 +1,9 @@ +# Windows Version Info + +This tool generates `syso` file which is required for windows build. +In particular, version info is used in MSI build. + +`go` embeds .syso object files at build, therefore the file should +be generated in advance. + +Refer for details: https://docs.microsoft.com/en-us/windows/win32/menurc/versioninfo-resource?redirectedfrom=MSDN diff --git a/scripts/windows/generate-windows-version-info/main.go b/scripts/windows/generate-windows-version-info/main.go new file mode 100644 index 0000000000..bd4e7536a5 --- /dev/null +++ b/scripts/windows/generate-windows-version-info/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/blang/semver" + "github.com/josephspurrier/goversioninfo" +) + +func main() { + var version string + flag.StringVar(&version, "version", "", "Version in semver format.") + + var outputPath string + flag.StringVar(&outputPath, "out", "", "Output file path.") + + var iconPath string + flag.StringVar(&iconPath, "icon", "", "Icon file path.") + flag.Parse() + + if version == "" { + fatalf("version is required") + } + version = strings.Trim(version, `"`) + v, err := semver.Parse(strings.TrimPrefix(version, "v")) + if err != nil { + fatalf("invalid version %q: %v", version, err) + } + + if outputPath == "" { + fatalf("output path is required") + } + + versionInfo := goversioninfo.VersionInfo{ + FixedFileInfo: goversioninfo.FixedFileInfo{ + FileVersion: goversioninfo.FileVersion{ + Major: int(v.Major), + Minor: int(v.Minor), + Patch: int(v.Patch), + Build: 0, + }, + ProductVersion: goversioninfo.FileVersion{ + Major: int(v.Major), + Minor: int(v.Minor), + Patch: int(v.Patch), + Build: 0, + }, + FileFlagsMask: "3f", + FileFlags: "00", + FileOS: "040004", + FileType: "01", + FileSubType: "00", + }, + StringFileInfo: goversioninfo.StringFileInfo{ + Comments: "", + CompanyName: "Pyroscope, Inc", + FileDescription: "Pyroscope continuous profiling platform agent", + FileVersion: version, + InternalName: "pyroscope.exe", + LegalCopyright: "Copyright (c) 2021 Pyroscope, Inc", + LegalTrademarks: "", + OriginalFilename: "", + PrivateBuild: "", + ProductName: "Pyroscope Agent", + ProductVersion: version, + SpecialBuild: "", + }, + VarFileInfo: goversioninfo.VarFileInfo{ + Translation: goversioninfo.Translation{ + LangID: goversioninfo.LngUSEnglish, + CharsetID: goversioninfo.CsUnicode, + }, + }, + IconPath: iconPath, + ManifestPath: "", + } + + versionInfo.Build() + versionInfo.Walk() + + if err = versionInfo.WriteSyso(outputPath, "amd64"); err != nil { + fatalf("failed to write output file %s: %v", outputPath, err) + } +} + +func fatalf(format string, args ...interface{}) { + _, _ = fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/scripts/windows/pyroscope.wsx b/scripts/windows/pyroscope.wsx new file mode 100644 index 0000000000..4c98716882 --- /dev/null +++ b/scripts/windows/pyroscope.wsx @@ -0,0 +1,152 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Privileged + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/windows/resources/app.ico b/scripts/windows/resources/app.ico new file mode 100644 index 0000000000000000000000000000000000000000..88f56e5eb10de2d1988c2c46ea1f6456bb8aae69 GIT binary patch literal 34662 zcmcG$1ymhTmNr@!*Py}O-8I1>3Bd^#+#v*);O+#60KtO0LvWYi?k>UI?IQQxbWis` zZ(jGCo}TsJwLa>w&)(m+_pYi_bMxiza zs0r!4v_!TH5m;o?!FRT8y^#3rP z5+?;fMR*M`5q1%>5DjXx5RE3Y5|1^r5>F08F~UkT)&jMqK<9L!KEy9k%|Vnn>Hm=H ze`;hS83&k&x-r>^$1T~2rzhA+<{sEd<{|7P3;zXwY;%uLpBYx-5knGe7Z5F8(f?HL z@BS1xF#reYG63bGhVqtikS;xNlCD5FNmn79WNT2YL%99}f5v~YLFb?`_iQASpZ^ds z5w@e!;gvvpk??oke`P@VG7+l12(Xh((m*x4xX3o3TAL7V@+~N~|KJ&lOK$SbEpGDP zn^61tf7qaTtfVp%w<24E0a!^p|Jxk6 z$X5U^vPDu}%EM`XsxycH)g?rL`U=8Nb(%_#yMa2~%#JnOB;-8aCOkITA#(f&(DC?i zvygjVJueObEWq$n9Y+dK{fW7N=6wc%axHL?FH&-oul~3Cg4DMFv`#~)?vo((4dfNg z9Ym1&dW?hQh;+1#9&4;sxL~qVT{DXLeKjm8noW-ziGdsxjo||JLa41 zCB}vJdGf#1{5RddAIIB;=O(*o2slYjSp{is5C6sky933!G$= zgdAiO|5TrYY#f04;6cyb%-`am^{-N6fQfo)=y{e$t;LsPeP|C)xJb{qCfczG<_GlV4mXP+Gu>~Y`ojM(2dMt? zcn3WR2gx}rH2440f12wdC^shb9Q-qRfQhI9Ai{Kn5um=v`J*pLdk+zyzk~2HoLVXi zL4c8Fk)WOBc*xFD3}m`n3fhO)|DPP7*Ixt(x<2sIpPCBL-97!i4(iKd2EtM}HjGu8F>2HNxW zk!GRj{yJU~Sjtm)cHxU)u2=U@Jbd>MUQTEY%r|R1G`kGEw1@w{+J7cl>Bj++Ul4$c z>&%l|@cx-o1PtMj1Vh-~-QTdtTzsW@e}Dsl0HEPoQnFvw%wm1DJmO=GEaZP|H17_u zp*}%uZ||=;B)||(VK9VS09pg*x&I3&Hyi!bf2;q;fn8u9VB^~);1Ip(V1EmSu*!lV zEQ(+VvkLf;QT?`y?&Fy;iP||M{M#pNxHnJOB&z2OP@5riuATV<_<=hDI0cMnn*CLm%I*G%kt_yUCs6wZIf50B(FXZlnY zSE1COAD(DUp*_;Q&Y+Q7LZqujE4$v4KGE!OHP5;p)cYTDAc|Jl&y`k`h z!skE6u9w#22FnI*x`S?AnYl8uD*z{C9zbzDEFo=Pm(3_+1D7wW9UMMoqBDLRqzbPD1|^UN+1xd9B?I;|2QJ%*SUX%=6~gb z`F#(-j9o*+D7w!_uZBQgR6RdpStMdm*Ki|qfqz5ma-q4~A|IBgIBBkN2a3OTH5Xzc+V|JmjL`WVBZ62Ne%0AV;) zgD@Qb{qNn7JLscx9To!d1wpSY7@ubV@OcCP(MLEy^bP>X|KN-Q4gJ+o~y1J8doUecz`+Kh))r!~kmMGim3Z8x$>X@bOm0B}G-bAwa ztLIKPo!u*>7wt_)dhlSlMzr=~Kwi)bR5G{gT@Om@36D)8T`L+1gT1iA@ zKj*a<+c#KW!w;pezh-zpM2Fd`yB5LwUN`FZ(t#6Y2@os8+WfE;giji#gA^a9;YSB* z9q2p?w14G^@DGXuNHH`UcwMhtBypO>H_2)rxy!V~rSU5szTKxz4R%jhT~E&={7UffJ& zpSE_tJ=mqPWTg?1gptGbV{gS|?A+-z{B9=yV5VP_O+)eHgn`p{3Nxuf{oHmfwlXJq z53*8Pe*USUPNYYji!S%)sQw--2ZHGWn)6iiY_%G{*QJC`n@-pEkn9Fk0kaI+I!})u zA1Ec>$V2Y-KPcfCbP=8+g<=CK)vGeNBpX#w2bq|l1BK_c0T0ELK z2^AV_y@b~5@|us2Ve%xby5LVXMS<_5;?UQ5#3X66vf|d&k-1BjidC8fnycc{nN1&` z#MsU;$S?>YlcxQ#NG(M9?BxZr&D7a%8##xC4x~gakc}hb;OvMfNMwEZY)g8!y`4>s zc^@*@f);uZ2t4chF}7r3++-yfF4;Bxb6ZE=UM_1nyNAEr76G8+b(g3%gy{2DVn!h;3pA>gr<2)JI5?;*fU{F+E77l*Ogy- zD(yZwqo2B3BneJL>va)i^#J)CRLFj7ZG|<8`kPv=-9*Zc09?>vd@2(03Z6GI?#N&7&lG zR)ruEz$W%^FPYH1S?^ikP1 zKO`KQx3^O9Aj5i%^TnH3yP0LDQ_wZVJWWi8rCy!MhVIi3L#A+mjSV@eLyiNdB%BiybFtaXX@77E5@9|N* z%KT>cf#E%%0=MSfj-PQauUdvaUQD=sdokJ?Z90iZ61g%{P_?ao^BGft-dnM+2j z&WE&y&MYIEQ4}>voVL|qnBY&W&UiO`D|xq-wV(WV6FK35k3#DONEdYfny+68ky#Ue{lhMk=|T?h0e$B3Ui9Q zGZqYUU6H=Cke2Td9azKsH3=_jyUGjnrT8B!o|golhmm6Ou>7*dG-))+ z_SS+&5)CsK6V6f$>@q#YikPFX5EHG!p`@NVtl!b;EA0#K@4Ks1z*{16Wv8nflg0KC z%qain+x!hpce%Rbx-H4rv+hEQV8SB~O2J!%&Y&QkA+qlYZ_M&uOv?FPO#7j|RRWPL zdkJpy?Ni~ncZ8;hI5wL9SPL=(?p6--#@CC2p|^?2-B7_4Uz@8;X)(q{sw9~+Nc>Y5 z(fBrEq&LAMRW-UFk6nyKc$RZOpTZ&N$PILaECFXI4f7s9q1j^f{xa7MyW0^**oHQZ zKnBeKdxO0EYGXn7)nR1VdLR3AIjhUmmtwe6CDg&VhbhedlzYTQB;cF%;}E6`WmX5H z%s3u016N>%8ky4snmjhQWX2H{K(zuVH;Cnj!q?8Ob%K@Vmc}I+VhjV0SgI}o_wS2a3m&Ia)%m9ChOm@)q zpcIBNa*k9ONDdnbaC)#8jQeS(->%wPHaVmoYDG0-vq#XZC>cRj3?|X`sMV3`IuT>eiO87pBajPc(}E8L10BHUAr?~BtcyUEu~-J zi97M>h~ybaGj;JN7>77IKM(rrg2&i~^t5SE*szIS&2dP5GJH7#Um1^;;EbMZ1bvLZ z_swV*4Mkj2P}gadZY`YcTkAwJ?ds}K*QFZULp10o5YQn+$Knw@N7+Kfck2{eNT6!R zP++hu{#^r--ZIf)G5i2sYUnUeb+=)F<>y1a=dE``a}T)g8M4`02xnPEuD-pK2>jyqKhjJn|9@q6~+5GiKA`w)8eF8EqIe@?+{#~DTV})6FTwa{~)snD}yRSZP zNm@n{-Lv0Y=S^LOusGO_I9HYRVl<^s58FDxerYK<$1wOyDSN?`5w&C=F4R?s7A80H_Qq)FmRRl-OySMid9n2jbN`G)S+#iT%HFbJ0{_IEM zkS~m=>McoZSmrK~L6*qJhH?lPi`Pb+uBZmPxAUy9PSmQ|XyrLv5uE~Z?cNMR) z=tQhAJT#C}qwu;Vp;2onnk*69D5U|((SD2Y-Sn*?!u7n{AbT}!e5Wxp$nXBrIZNBD zt@CA*Gc_?+-!H2ZWvToO9B^i+$kYzv3u#LJio$dIyg?Q4dwE*ui?d2ZFwl1`&o=GW z_&WD|W~ey91+1EqL3)syT!{RF+xdnC0}YFQ;o7UYWUTyc#6|DB5Af9ogt1#HN^D{% z=3Z9 zz5GejA&))eklJu@xCjoRq)Iy z6pJxOM$P$B3@N+Ldr0=G=EDbH#=tlfI~2QZ9sD1Q60;7C8=u$9W{kXETs)dAo&X$k7W3>4j zOd629)~c2*NCAiMAlSx3n7CK<5>YKqg*~wOFwmz!gnjNT)ja>gk`BZ$@arVcrkclo`*uAw$XYKZ?qCUt|3Iy>L6r zmbQ~{5)*nbZ=l82yn12<+i$H>$f6=4DMafsC5EWf_L*Z^*-$|0*%XyKKq~~0Lyha< zpj(26>@3O}AdFnRGYRzDAlQ-UP%*xmJqanJ>diPfSxhX;UZd2W^OH}MkIxXT< zX&R^7mW1duH*k}<@vKCI-HNwoZ#w~MjBXy z;EjTQ##5bNUk(DmsYhvmbyLnsDGEMBMtU>Swl?gC+snms$AThKME7p`>oysJzo>UnQ_WjMS|<=DdKHMs zW|kZ|$jN7(CWYl%42dHI`y0sr~T(qR;awQmV^E`^v;aH44sY~lTmE1~dX z{Y7IcTqsQdXHOJtO@Ql(w2llx_I`kD%W1LwC93J$;!28Qo>JM|tD)!p zymH|TEi3_~4sb=iY{a^Wa`oH~n$@YTPErigtqPKZB3*)>5s~C{YHk^Bbq^X{XU(h* z;sQRHdre=ZJrbg-_B87@lBo1ki-z@VCnpL}BCSHi<8H#x-ENYHrpt1%XP?gP2(OKI zn4(!0=vmr^&YzMnX}Pw?=PA~EDxU6na5;9@ zbA1(-z{Pmgt`+v_dqM`ceCt(<`|+ko(AbFS2e|0rVkX^XTO_Mg@(-_V(hhl_JsF(8 zuAS{iLRSHpPiJ|Tt7RoKV){#Tkn>(b)X~(F=TNFE!metL!dsl`@4hRh8VnB@R(M`Z z_Y38cgKk%9`u$tu@#5bw%^Jq_0D_C!p&r@D4mxv$Rpc+QD4m zTv2>8-e)@Xi3)pPz_rhWM6+*!40w0D!chXLjtmk+Xu}z}O<*JoCR|W-d{prDjiF^s zpctVjYRH)BH4gJ@H+;Tg{enMsAxnPQKiwnt73Ulk)ZYW6Mr6JBey?8m%Jr0ejaPQ* zA!mgC0#6oz?qUR_lEWery_u#LM13dM$^gXN_yUH(R zve5_}qLo*NG0;Zl`C?aB=@4%mt!HkeGH#AK#6bJm9ot9!)Xy-e!3buS*y?D~`7gdi zkrZC~%*l60*-*mcrZncKw71KdCV9{}>{>~Grhbml67yBfx^UyHZ`g1g8q!|9FWBh= zedCHrgSNl`_Y(@!-2?ATgNyCQlIgo(cQqZgEF~3%!ixU;4+63kIX0qb)csgUM}Cr; z#~0-)(nsu9-!!q_3ifrUWq6w@m!A{((hyVPuJ_)9maBV1`c_ZaD;zE{z%QL+-(_9JZ8gOFzg6p3n31so$7OwTec#%R5k;5r2_!tR4OZu{Kzq&nvZ)+Ci7~8$? z`Q0G}k;*55zK${2oxY5e)-1doa%xT;uGK-ZJqZ-W^-fLfC8hCG?tNHNAj=marpF3T zbtw)QTtC8&Kr~70cv=Y+hgF&vc7c@JZU)_Ynp))OgDYjT5$F&+2{Z$LBw@n$8)r%Y zL^)uYNd4#G^SeWg(`b>WR>xuw(dQ3KVWJSi;5axXnl?vq>b83Bon47E-JIWaTq7}+ zEbnxA>4jzou?-QIGCOYcUUW*_cwd6W{TAH5^HF=HmG3v`&aYp~4N0)W<$X~0dtzOI z?>kRUJvf_x{cJeF6YZpu5UxMKk$D?UK(PJzi0$V=lhWEC*ZM_V$p(BUYh5bG2#@qK z9J*ryGpn8}?2+Mpf+=kGW>X6ThyOmAHx*!B^ksIyM&;^mrKeivQUk$^5u1iN-1*_( zwlKB9WtYrnQEqxd9t=AvC&s7e14qHWUHPU+--EM}L&`Ojh_ZFwa*PMVO}lv?M`4Vr zyfRy^-=&|EzdZoX_9ps~TW`7Hq4y`uiIxsLF#L0mse+1a-8ZEs(#2WzM@I}Z+fr-N z%RKrRwM2Yao-ULQ60~;{Iez`LbRh?^Y?fNNE=TbInoOsCT4MDs!^^Um>Xcq9S*K~L8+=1gcJ5bp?$4P?9a)jQ zIzHc?G)|+m%lg2>QXV{Wr0q?;K@~02c`V3`vgIioO*L})wAkWRX+MOQ4r7h+DR(WG zj<9Ju22JaBj+kx}AdNW%N3-}^GQGZ7N=5*@-E|t#Ygsf4n(js*Og{n8*2!-#krsUMB+=Lr1p?=E2Ytl2hQ(#zslrKVfo}# zh4yf~QT*DQkLF40?1BaA9*O}PO%hZnAKkFLH8(d^`6gRmM8sG6`VKP_3k?R_mKL`B z?mM)6{CxwD^Q4k84gKcK_X*aq_jXR*xGiLE*x`IA=|1mvKu6Vd_I`_DbOG!dB|gUZ zm7Tf2w=Po=P3rn(($SQ+bpc=94tzks+t4R=)xozE$*jG|Tbmu?ij*(2XcdpjVM=_TriUI0P-!piejY3hGI-Pt)e)j` z`e)UI8zPA%=Gkw(^jXLU$D;X0Sv#QnVaI$(7Pv68|<_T7lFz;!?*%V#8K$ z(C-%DgK6MLSvBlyG_WtFO?n4xxW=f|H>A9}zo8Qo@Tk&McLgA(gQDVFBkjH-BeH!o z)g~42m!iPBE-O-h{i|e`SYD$gbWK+mk#e23%Nmrdo5PioXx>p%BE%qpZO6o%DPk8> z-&v-!T0mb9*{nOcp=+Ovz`jtDL5d4vFa6X}E|T6llb+)IKJUU0CD}%a&@+njk(zM- zIEbg~zROVExxt1L^_c|qm`xVCA!jJ#O#18Ip(*U6Dt_t^@3WYZiU8+GnSpk}$*qjj zNt*A4eY8|KEfwQfWYSatj-)TO;jT6$Jr5QpcCdda%nc8I`UInWwdHbV^dslRC}9wE zbpzUm&mtqoe%FN0da#$==;G7ZS4AsXm^ME;TzMJv`y|Ym(QK7I2h^-kD4L3mM(dm4 zqYL7Ngs8+tQR#RpeAz!NKxR*=cE`9PLfK6J_Xq zDY+}zhLYZ=S(msM!OG}?z-P66tH_T-E*i5oDuUMV2Fp)x-xKs_ZovaeSq{sBB_u?-D*VOQyg+n&-pGnoDo@wBydQbmjBqw(p`C9!}@vsE5h6rSAy;|u4;KqSc_nJ zn}qScs?Dv#mYeiZ&nZvazA@pvooVf@ueu~cm!HxK+Jryr(S)| z*ixgVlYNTbH2WUSAOIwqNMMw3Q~MCW?LK?;jd5srOYpagP2l=CGQq)|@)$WCrZn1@ z^dw)1eo ze%o#Mm5v+OfNm>e+jk5{Yx!7UMpcZy?S@|_u+DKvN0WX8%3S@42tbhX?G2lkewl1< z$?4~Y{12P_?zD9P-o)7$*D&{DUyG7QGmY<29c&1U0Yw8LZ>im-w%1#u^M~iAcRzNR zM!Y%**+4ovD$wsQYs?|j=3^PHSsym)3j;Bc(X8yKgO_iU(S^ACIyl^$yztmf$QdYT z=N|@jECo#0yu)G_qv4k1GNugG(>09JLmndQzzdW97G(SC{N{&(p~%Xa6;{GtXY;GG zezA*7FLo*Wsh-ht;sBRgIdj(YBL3z2@~kQPs$-2-r#%{ky{itwL->zJ8s}OXaNN2& z&v$7DYKMbAZ|9nGnM14dsHzE1YZwPRKZsZDrEgEepm(&9tva+-OiW1+z!Jo6KNi}FzdK@* z-*;`9nz!E_{@U~UaOnKOI4Frd<3zxiumkLXq55``C3^9!Q80^LwP+Ik+rV4Y`B1kG z_#{IZH@xfNXqR=RtsnfV2*idj?HrM{Tc?DP5&c;u;2FTi@9G3Z?OJEQH(ENHSO21( zs5YfMULh)A3O$<7JMv(bs(dl&YiEGaUtSYg6)!&i{wpR0@)lpmq1nBTk-q-X$kLca z^$qD^*KAu)(Ot5R&J?z4`sRo%wnNvLXU-0^BkSt1 z+9dSnm()q!>KIG4X$PENn{fR4MNeckwKs#2nrTxu;x$P_8)~F6h&Rj=wtv#@I=)3h zRbXc%6L5;)7&b$IQ;C^K;L3PdIbXHCA>NU-8U3BH-d$5{-K$uUI=>tpWl(Ih+;25jWUtNo8)u$1Ug2-b$ja^usehp$1XgR zx=ZT}@blK7Uj%63Bvol`N2V`sP3F!+ELEh9Fkhf`S-+8Wx&xSter&d{_*@@1r; zmBUKeR=;W8Je(shTU(6ajzmu5M574k(-`T=d%1eZZ1PN5(X#`B9qOsJ$cu?(uR!0% z!!UnjCeS}MxT?u8rUN-XlkFdoL{Ham{|MEjW7jiAI0Q} zM>lThEwMU}VQ2%bRGio=blHrPwH;dupjf=IH=#5<+x$ph9aGxFBdV~RR`ABxO zA3hmkrmE&$SpIlQqpdaZXVPUqtT%r1+)9|Q+!z-5-67A zjZas#yOWKCOMysr9IGt22r7znR;Y2_DaS#_7#woXc3R_y%ADj9^D@r4=aCP6W zfeLM^7>+=CLzD_e%oqZA-L!;8L3l>yuc{{K@5wlKcm}~IxnyX@xK69QNbsRces}2v z9b(#azIxq@wfw~p@UHHYXrsB0c3;XO*I{JhlxSE#Ar|soK)!`*{_EfR>*`~vC5&@T z_Z>G=6lkMA1%p`-9b3F+$Utu5mvFi9I*S^6rq{Cl$MlYZ)%yGoEmpZpA7Pp(r6iDg z2q}9BubpT|U>~S@Ptv`LT2o&eJ=|XJ_buX>H5V6FMUwxZy*m-HqmmwOG=B@wcO3rC z^l&z`PcBX@5rXc&&y1lLawi4;6qNOsnL?}QnZwSW{J9QVk0Y=k$ptdIutCB0mkYgA zmo9ew$BB=n&m@}+R;FE9(WRHHAUJ`H_VMmj7sJ7MeZKme0UzDvae2jF-%n1s?LUh` zrV>F7n(qEhxy#YvYx9!qr9b#>{oXzOl$ic5Td#?mOesYrV0JU`NOg=L@qo8RMT(%f zB{mMzKZ>1Zt^-|l8n3BzB_xpcuvT63g}t`L>pwL>diwR4@BP#gupguY25sX)AK?SO9KC9eFVvIvWS=f5 zD@%7lD?vc3#b0S!@MOuiJ(MO$eL`8&D0}~a(HC*k7yYKmr?R;jUn~tL%-c(cZ-5pU zw5|zwK0IjMhMp4BE?jhL`l1!-D8UKztoU!`$a6j_Ph;fDd5&~na;mDX?E-+(KVRA{m0E1Ozy zs$eDSeA6718%PZvERA7;MhA5KeFJH;4ZB}```NS8U}3#H89)E*69Rtmn8&+_%~{GY zdALCbTPU%8Q5<8r__B627(JofjW^||7BDnuBC}Mas+_a@)Oe)HpqU?w(W9)*pS=3ND2pW~!l+hY4I&!|{s`QN2&P^@lfp8k zr$EQ_wOJ8bP2uwXqVMxtTR3q>xA*%syN5#iaj(LVfG_*ZoIzB6QAx{*>0BbBhz_GH zaqwqSZ!0&e@y;~`w<(C2sHUkKpHztDLIvk^FH9ye%*rvI&-wO`X^i%&ynDpM5}`4^ zzU!Mni~T8KuDTmKdUw~5U%@i`V5gx)rTr)tq{5}Ps@7x8@_Xl^xSjwxRS7XXm^7~H zm>m9&ETPxEMyg>F47c;Kgi5P!Hb&l5J13lFHVf4^5P97%2a49|dmIsAWWPCoSulb^ zZ*>Cb`A&=V)@Pq+CZTteT)mP)D5g>!HIa#&mM zp51=aRLif8M$&$rEX91RaVyJlQJzOn6PT zmV9t6ko6raKH|iNaLnU(Ot2`JE%A{Z&SVCpibNGR$-kpfcE?tpb|y&2>fd!ys_`|) zfu%5ZKl-zk-&jCe8f-)YzBo&pX-pSR264rD*r|S{bTA+=ra(D_P%q$eG4@oC(u$7< z?JeR(LUN}#P9_LmyPn^y;0Q+1yIw=56AwpUC#2-=#I z+Wuu-m{bYa0%Y9hN&2M6PrFO20$@+Y$vv+tSJ&*2vv00WRR5!{F^n%afU70 zzz1KPL4kE+3b%eXB{7pRPjZ$a9=nu@^xrW3u@*?seSHIp9V$Gl_k_A?)t;_dOEWjB zcUPJmPFipdvsy139$v0alft__5bTxB*h#owwNxZjvRj%I3C*lKsxG&z4T z&e}35w+{_XUUVpWWV%=QMOs9^-MmUi^0*_IkO9Z-bbNtL3kS=()fUfzP#NQoeItjr-+wK35n&a`TZ01G+%inRJD7&3m+aBQKD$&%8GX zB(W^9G$D0xYkT|4=iET{E{JU9=&^ri^X|H8HH@qcxtn|meBM{*{hfyX*~`}B@WNs3 z9Ocbd?xEtHOkk`I6@b@u9-Ro_OERSVO>Twy?)MmpBW3`_Da#vq5@|YpNI`fxM_b+X zaTr-weDAkgm`B|&q^qHC?rZ%zt`1>3eTu*3S+=CXJ6ssfueiQY_o}WRu+YxECci&M zM-ryB%Ur_sxf^>bYj^+2+u|6jDEpX|<0BLt;<3xq=yi$jA;i7gkSms4tg6l|r)+}M z8T;ciH@_81H%Ek>Ez`n8N(8wuZuH3-@!`s&8QOkA!0#=ayoQI?ra}FO4AcW#RGB09 z$O$S9cTESsObYt&cFD#v1vJUMZo(I=ASwC7Ao{M>jJ0?vIIIJ7M_BX1#|I(Z1 zx#gw5hO~MtRgsU`Gsiw?$n*O+<+{lAWp2zikGFfgrm5GsVf|PNr%|Et^FXWYbVP#* zM|g$1Vdn=WJj@q4M~(MnwuJBF$R!$l#l4o`ly$zF1^SZN&33N*P4esfNKYR#tp!qmlSy(05#o~gWLK}nTa z$C$5a)=_V9@HtfJs)wqV<_YCM9x#Z00XrjeI`zRc_jfbfFW>phE=1LDs<`}G|7HfL z#RaCCv3BejBbIzNxi`bFvT##&UFZ=KSP3WDjOiyhMv$$Br>2M1cXo4LiCJ629&i3) ztZ(@Nazow(CiCVdIrQKr!GX+8dQ6V#+CS3*Kx;km#t5(9Wrs%FOW%IaN9~FO#R1Y3zophCw)!RfCsEy4DFcxy(Ex-{CnF*8h%bGq(MJ@$ z{9JF`BvBHrjayfgnUgD-*jv!z#xQ_SZdZOd-ud{=Q&LVUU&MfMjGlN5%N{T~POnvx zL%Eb3jvM#&`pRo=IPpe4t-c1(7EXK42FFTL_{SN~H_eaIT#p{^o4@MN$=1H{t;$)v zTqiFsFnq_QZ117@s}|qwX)T6OP4Mf3f~MzNt&w+{5-=z~U6jy60bL|&tCWY-_##{0 zf~&rrrkUA$_7kF--NZ)=e{L|ig?rU!Zu#PRx9oCQyJ1;|&J0Pi!t|@wi0~>V(udj4 z0XUK={(;G%F4_Youd!;m)P#v_EmvnBW=ksSg|2*-621;90DE4C*PI@`?-Iu(+ARqm`2o`ZQJvw-nBKhhfD8fic?JHhQ^`>Xfr;FpfXZEvh|kh z9(?g)>A(MsOoP!z^6u6kF`q-{EQZ?HE)MSE7?kKcVL{NWh5iI7^R^n2^z8H<(AcaUiFT-@9?y0-yrT_ zCI$cAqetrb-ovYrr>@bieuFs5Ya9!()O6Q?)5uT)OHcUqdR@G)aA>G=ZrxpJkvF7J zY;S)3%1Lj1$IgoS?Vi?x5xL(YXFaB5Ecq;b8jfCqo#JN63gD}!3X+!{R+_Gh=4F|; ze}9d(Jm(FU-aD<;&Pf~T%kMr20_ZA5dhQnl*=;x^*u&#$%C3X5?+|ieNlTylMfj(T zI#dM=lyH6Br1>*eD#8)lJsO^lEld1&M9FOW@>lzjg>w?*C?;l6n6X?OGa$>suSIV3 zPEf5VdOwFcYH)g?y$J6xz%_9egzZoe*+ln~rW&Q)aTxhBmhvFpnC)({Rws|)TlJ1@ zWp)`1Hr5%yr0c0_zj;Ic(L78nK7aW|tTO!D$`cGe2M59z-DqipR&nC# zxuqx^ZHamh-PFL9FNoY6lqJIYT|R{;1$m<4jahi#t%iWC;3<=ZOMi8d2y8(A2Ig?otFKOIe)=pEm8U`> zc21h&78An-dnTmhweoNBSBkLrRCsvaO{LDrAZ;3|r?uvSS+W2X1^7E@P1YS|HdDXf zBId~lZJlFx!)YWf&QHrx63U)kdWzI(+=1W3cC$Xo}&+QU?auIZ&8Wz={ZmLwB@ zIEiMemD$OsK12VbLW?e2#mDz?TlQ3sXcMRaigBS54Nns<%T;qF)!wgSE)4ag`^u)_ z*LRwPS4QtU0$$1gnj&pQBHsDV;oG@k7}vFO^6Ix|)z-D!;H*ukBik(k9-HDFJ`3Jc zb6sT9*l{d@w@7{Go;2acH(G4o>J}+FyzDC~3|uoHsORBm6Khcr(`QG(OjCjy?+2qL zF|jG@7h9j*?~$9X1~TsfW+89SI?q?$Yik?j+ZVK4pLQ_iO&<0y3kfC9fjUC6x=Tm@ zAe36tPl~>Vnz4lzZO;+{;CRJTQHLY{YKr9uO0&pOx|S_sQ9wgiCpvKGgK+xLGPn6! zS)zvCn@Qgv`UbNa1meadY@xyC4u&bjfB00SMzpoSqY#H5)< zGyOYyEwAnxnjvdqGH{vw8Zt*(My>NkBkI`6%-?bPf9JCrlCxd71r)H&2_%<5~uE!f9uLg`{IDG;8@zLFTmlE?i-O~H9xzHJm z10TVXI}#_i2buO^*--_gIJXHx+fIYEZ$sn=qm-a4!r7trgQMsAfPraF`%Wztw{z>g zH`t;oyg^GfuTdQs2*4J;62#cX4~+8+FH z9#GbdSzo%Ud*0LD-rTZWV&`5jmy4VJ+TiD~!S@D$5`dR;#M-gqjI(TSyj2_<-W>d< z#VnLZ(2-Y69^P0oj9M3ktjcZD-VQ<`iP zdLFN;2WWhzX5m+uL;{i$c(?nGWbeRoTaFljOQ zV(i+LTr-nliiqa^_cBdR$6i+urxBo}*B&6s41<`!YIXo$i?WuDb@)&bMs})f06k%j zy54&dfh`gGKn#ZO$lbd}!?pmUmXXXPPRv(?=*O0T1N1KkJgVR#yj6srv}AlP@MZCO zLF=&EE)fTAiI084fS8%k#8$Cw!y7hcABxLE^L5Zu4uJOzgpT2L8kaRiBJ42lg`Dmyk$g~b07}A)?G-?V8-Lg zOTcI|ojT8@+Z!d>SxBo3R`b#l>h0Pt{eCn~?g2D#r@1D0ph;s<_IICbaVyB*5 z>T*U6hBLq2$Ao79a^|>PzNGAE;CZckVNBt-#o1)zpPpvi*6RlmhT9F07hdqNHfWa> zdsUDijk22TO%X^2$aaEbFp@Nqvfjoca{buq!iBM)lBlrBR^%U#liv8m60{?Qa?D67 z%n19`6xLu$X^Q+^LSR6!;DuD>?<%%d_D5+*o9-IdW>u?>@-->sY+u=>*6sQ;LC1o( zANw?Hl6?#v338j#JjY9YGY$MnjCe21g}8{-+MT_fkc!(t>}aXxUPo0$0+n9W%GK>| z)@H~-`-Dc7VSZ_Ah~Ko6Jf({0Lb0_UrJ%K;N242ON52B94ax-ME!TB?0rq)83cu~) z$`6@r3FYx*1~y}>g_N3!vy;~`N1xf_VNL7Z%|=?)ZV*#YYBdN9v&Mp4W`peHtp)wL z#&B-sDGiV^x$}%wY;mJ&oi;mnY=ow;{-46Gu{*Ek>z|lSu9%I{*k;q%c4OO4W7}%b z*tTsnP8v2wWBY&I&lh;sI_tchnX~5X*?Z3q!w218P{flg{xaCdloR8y_|#N2+1FJLntvG&LEf9sp^)&SNbCU#Ubo5bN4l5jnpyb zq^CN~ffgiQ8_%b{1#@UZcn`W;^iUfLR5qTULA`6Z2jnK*> zg%Ns~>662Fy3UKORwdtm+E}jEUUefHeMcbL*kLiX0nBLx3xgLm2U1MGjJDP%dv$pK zW?320vpXGF$^j_oD|}V>@59nfKfe$Bq&hA4El|MlUHW*#6ClvHPV@=jYg=O@h<5Ba zvPTT?r#JNMjj6C6uVvp3kQm2Vi5*e#D4-6M`wM?Bj&31$6t08K?G#u*O^i!k-Bz4Z z#rMKkR9|m>y0v+yF}gnZ<(|{0tq1kvW%Od!UaLx=_({0xL{E4S@>}26<{o*kA9@5~ z^<=_8MOtzZmabi;ngE7CyO{A@1%QuB0Xo-uW0dqgOG|Y;7EejXf$>`lf%GV?ZDwAt z`-EU<`{t=J;_kP&;aSR)!rDwF-eUc)zvi8wUGH^*{b920LY1!k9NPTHiy8yOBSkFM zSJdh@5i)p#O?wC3}np*y> zrQhyLSv1uozTRKZ63Nrxs-yh5bi);5%nV!aEvwD7?En%U~2tdRoRW)+*JYtguo-cG3x5$ z^gqWlDQwv*;6JM9FcEQYhIGwBM@MKQ@`oCfpERI5q8G0YaoF}KZ&_INi*Ef&;M(QS zFHxMP_$NDf<mp?{KAq?1-ON;=MP1eX%~qH^|6v!E*lsBS&Ml0s%M4s(4px7>tq6)4?2Dbd@< zd=~T{=y_%_soCPX2_o)NLrAp;j6DZyP(<&$3OIxJArx}5?7j7odjysOgc|R+HOV59 zIqo8e0b%Gi`bOHuzx`P;#$m-5Z+^L{x79o8bU-Ki@d#bQkE`A4=Im5Z*0XE_2Bgu^ zXB^hcgdju*C*@@6${*F0Q`X7sR>_SKM5|~l$7=pwG}(Jpl%h={s%}nQrea8cCRj%! zC<~A>BU$1|Ki1-)CZBvN8iCnU(FMX*{v$@tW0AwK$ z466V2>#mC3U}=QJUG{R;1qnJDzr{no|%3NP@V1WGsh(vtx zf1hY z9gsjN5^rI%dD1#GJdk*|i>)0j0${RJj~2^WQ(^cux1A3)uQ7^4x)QjF?9ckHM91ye z_!+)NvseAPDi1~V@%Xwwq!D|v9&Re4dl~SqRe!2Dj-K5wg~!n*A#Q`QFMG;)mP<-WZ9)WN99a-f0)0v@l9HK z!z~_oxeEb6QYHkuB>t;xB3&E*fIsfYW%=#6Fr5hSj|AR&Nkm_q~4 zpVIZhYfHB&-w6&7J%5r{D(OiH&zoWGAfUXP@~W{jiS1VOtrsp6_D3UIv zHLqkS)OcjL^8S4^o44}C$C^8Gx%~x=rP!$XDJd9aP~XY#G7V!bnJC^g{5<7>siR!C z`S483iNSo{Py>(ztj|-rwxh@#B(3Z6m95UZX!X%+sTE5(K@nr_4-+KwSA&syf^&F3 z*pz-==vP&XJ*^%Vn9mcySL^jedv`|ttCWFn@ zuhO*JpoMf|3cJ8jZb2xXWKeC`!!}&Xi6?8luQ^E@CY{n61VSe-SC5F{qiX_Gn6318 z@7=?(mNRefjm5Gx95*Q?@7!dhwVsfHPZSg78;^%Z$W58ZJyT~TI@o#_vD=dL2$_33 zyEI~6%fImT=ldx|6q=GM2fg+1JkP{(nd?eQ@PI@H@oV8GgQ)eq$g5;%_-%BhRVDJW zscN0z5D6}02`eXQ=cH7KWH!9J$k0|(N%T@};DozJl!Elxi~c5lT=QB|UuX6ZoH~cd zESpkIOiY^OM|yYi+@*rx%UQ7>*A-XdL1u^n&B3u73Eb_jyOS)hSdhkQ-A_>({q=f) zu^3awIuXJEqzml78|!yH)U@Fg2*4l4qC@5CnuqBlQU3@f^FQi=ssjT2LSENyo;o=V zDgI%I3^a*V=3j=0Q3#BFTse!99!t_-Q<}jQ)?v?(u1!cqMrZo_4(DjUH*v2NefM#H z%E1NQ4X)7b@``@G4$BB;xe!bI^!&W^K~kM0mcn3W%zUT{j;&VnV@e6mArRoQL0nFf)nq?iWe+*MIK&3Z|3scGg(oqT5NZ|+xp>hE$KLlpGCE|1nr2Yv9 zI4RctRLS#!H5sx;eN)3F{o)*KQ`Tu*9O64mu~W>!MG>C zjf7)a`Fc#l)l*a|d8c)jh653#()1dRr1=X?8s%s+NrK>pu(xs;<6ws12dYSVasI>o zg4Dae3?gO=?rnNTKfzeI>!hq(scPq$L3AJgU?&@;T3IU9MPMFyMB3Kq{c?!|Q6kyi z9WDrAs?-I_O^h_~MjLW;_DiWf~L#gLTid3Iy2b%kr1y3w@*4;J67&AnLYs-E z#pci@)1Mjy3fKhSXCNqqllvn+P;jZNu#5X-lajE(r)@e28wxMo|104r?BFN(Pr{S5 zPOxMkgCkfe<;HkS;a^6Be2sTmhgHtlQUg5-P3LTbVkp&k!RHNC`;EN+fNzNY~NFB>HU;7%YQcMJaTWg zha<8%tw_pCf|Tph8-*r0>0p2|l0!r3i+Bz+2Q}8V^3bu)`ca0Q9fql+wKlBOoH>=h zIR&XXk+RnYwPQ$z5W=|?j_M;LAw0vfgAclEs0p~{xz=dD4&1m=-7r}9`g z*}y0@WRf~$Jo_i3u}=rx(;L3Q5wk@Zpd4BZ4k`;(x^`o@2dI8l;eY=2n}K@8Rzs$Z z^rUd9mZDQ0|4Oyh2)s`t(nvfzC9&$PFJ-m9HJLba)mf>8J8t0sl59YXrb0u4H0M~7 zyZe6;Oc?r>NSq-J10dV=^@I?nJjg0(#}5X~p;aZ(d1Dehm0UjT;OBo!CAxW=O`Spf zW`2j_9{LJRM|2T!zSbq9DMBqC{+UN}p^PIjFd3vvTFwG7^LS3p;_~8u#A8K);e8h& zEyc`A#Q?RkG9PhZfDS2Qp64c%gpZZ)D1%H3wBjL6CN^cH`qN`ziM;|KAuo6{_RJlk z3-RJnasm`-hhXNZ`=eiP_DJ6p5>#ImT&oNFg5$YM6QFh(eGvfULN1zfl&cGdu_yfu z=Q-aTT3Kyyq_h=Bq%x4q+hxoc!ra+CnMn~wL|YU@KP&feKRu6aV9&j?Ldyyoe7N{_ z5i2L|FgyMx`mdB3Ify%V!B_yV53`8l$KH09mmfE!cT~ylQzQYT2-$Bi!N39XpF|+b zQitM==;i!F-eMJG#L?rH7+U&*(72gO7-MiXx^;EhBDepJlSm9HGN})PnbkDC79X$9 z(sQIC(8b4Tc-G|y0?5kFtFxU*JT$N2%oKV-@>=lUzk47zETBKaEb|sz1`! zlK&}EcQkDlpA_0e?uFiwGHik;Su`#Nou3mO@_ivn0?QNuVRsZLM+m#62{tUj;R?YF z8-a=wZJn3m_M<(j3_twlE{nO)HgBoC^t=rlHrE<#StWwx#E<^wY}9>wX@Om?TwdPz zz$slEGKJEv^7_j6E&@(nSW%7d&}UnvW1vawGOjkRByVCpdg}ND&krh;43%ZTV?y_* zmTAabD;2VJMDyWFS|@&=C~v!Nm3FCIx#`#W5BsO@ZE1Y89@PTuJV&&C&>)cECcHD8 z&AdJ_eB|$2cwe(ntVKpa>e71~XkWJAJwvMWGQ>FJ9{1uWt|*baQcz8*TDos1^?oIf z2=XFpLf;HRS8{<2<1t$8_K3>cv2?cBE z7H9A2fv!TtXn+*CtHU}eCHg|s`vRlx94A-oA!-KO_gQ)|=u;*WyMkktVIG2GFB{36 z%?Vt~=@nT2bA28y_Fj*gX|sL@bTi%5ji0$W1fNYc7z_=q+ZD%((l>lI5?NAD z5Me)~PpdJCWqp>~ukR9gllB~1GVxwJP5t$dCK8lQN+(R=jU7M=kEBc z2Tyez@#y#h4h3&aJVf)!V}Ex7^zQ$N8d7?S=&lvT{ldw{JBIfTE@so)0~p5O0r>CX z_ok60%hXrAEVcc1gh^J4N&mF<_R|g}^On2UU0aVZ_;Q}eQ9ky13~$497Bbtt7stz= z0H0m{}=?2aXwXrNx&R(=Hl zi2$|hSf}QR92lgF*wM3l!9dM7UpsgYeOSNBH=hpFNKq7nac`TDmI(&cdjtJHTj zXI){XwFYX~Hz(F~&3q?*!I-~Nw01lZ9%=A{1iE&Qh4{PA->J>huU2fvcC$kgb%1{M z0d?A|FkAiS6(%rZt2o@5SuH4q-cF;YDH9b~Asg~dr5VJ>sq58L)(^dOj@Wn?c(Yg5 z49khQvtu$=1`9C2z%D6AJq{6{TCIJYBqW0bSV_Yxd4%-Ia39T)iy$F-XFNV9Sx$9D zybllOv^?aHw?)a{|Gv5<1_0-e`w8`lfd>I~*C$u3#E}=zLKXGwv0C&I6oK`YyEBQz zDh7d};PmC|D?iue}nL7l^A3&M}2U zKW5iB%LSiX?8RrYYC?l1mIuFLCi}dL?{q%G7tF)`{)HTW9W7o1VQbiXyFS=D%j z&9*qNm3n!GQ3u=K6KWXuYkSr4d4$DDMakdB!j5T~5aZ;W!JWdPkf*LD_}?A*W&188 z<^FkLkIkSNtQK8FxIE zR?XesWSemyk)`BQ8&Srh*H2A7>*7Y&Yb10rV| z!0LLimZs56jseq8Hd?;cL6yxu^^(R0uwj0ls7vcFVLwPK)9xH*h%+eKq!!I&Z*k)Y z<`{d3*oeFnOzs|Tx^JISC3Fi8{q(=ojEIdiOED?v^SUSObjy>fb7;4%N%gc4AEI?T zn1Ix?OgrDwOg{R?X&SG+$0dOj-Qk0x<&k~STn)z7oYC(;$II#<|^XuBb*BJf%rRfvPL!{a~d&1Ur3GWO8AJ{ zF7%(dx_)rK{vj~CxM!$^)$JxX3^@_2D%>vU!>3mhklUC2F@r4CC7GY_&97l&Bxq#d z$IJ$1b^Iq`V2qReuE)LQ%zH?IzKM;)vjH$Q9%uFs@^YZ$7Oosbe-nm~?0O6Jadr!M>jOrRsKnY{lA$NAZ*;L7*Cq zGXsETo>#3}ToS!S6P9I*BDAT^U(8f%27py*&2}@kj>qf%0dmnM3o8r4&rGe{C8s(f$ImF`Z(> zVkytmcwdWR!zecnc&! z?1lxuNsG7XKdm4|W?Fq#g_akEC9fH$T&=I1awlFVe41{g;$Pb0C_hAG)c?ez8e!b? zrq!}{XS-0m=q;5v9`4M9mc$aG25FdqfNR!}5iaEU+PC#O5peM~vea9@D5bDVAHB5i zE>VjwgwnD@!iErr;M||E(+mBxd}U?XTxNqPVI{Skx3}8ZsmUo$^Q1yoe>ULf${%@+Gx8^5uZZi|uvkbS<4C zfsj5ctg*!mT!ujON;Ws~WAJre@U8E6mz+MC$4cCvvsa1|cG>r^luxU-cWRbUNLRNlwD=f8=hhM} z9Inm5Kb4UINr@k4ix^9!=6XNqj*Qq}2; z9Z&a7X|BxktkNK?$pDvi?9tY;tvNmWKlKQ)S$s)e?J<5$Ku~fBRdi( zZ|px_N0}0BrC=2r;AOZOT`Mbn(sa3|0$lr5v(y0KEXLYUWIH_J)rBWX z5PN6tCqDsTO3Wj`&U5;Pa;UVKA^+CLZA3u~W}E+TP`imWXW=Y6TAAz+Qd4v6+t)y! zO-U5;Xx_V%gYuVZNpvviozkpalOEhOi+=z8x^17!{TM||biKeyQNSP}U;c7Cl)PxibEAsaqACvvF0DI$14d?sM_s6a8t2@YF%U}?~loB zKk5@x4I3%>{^Xqh*neVb5q$5(Nr^le*_A*ghG3uY+A03^v*4+E z7Pn(*=8+v>XTQ#HxA9YG3%2&t=gIx)%(m)#y|#Z#8N7&eI`koR-~C#O)>kmRkom-x zo`~MdcIBc6VtH5}w13uaxlbRyLBe_%nyb|bGc67#oz+LC!!t36!W+fnxJG>YsX#8C z=B`ub>rQ^jhle?g&@W9@dv-%SMwkhBHI@Ha2477Dy>+x~nHMP^ww;_XIm%V&lS1MN2UxGT&&lsvC{BC+?r*U#wcEeC1fLn(v2tl{OkPswB z)Ka-f>tk=<6fSiq%c5+lweN1192aARc31eq>V58Al_KRq`0P4I6X zrKa00O;rx+k5If}I^AoYs3FiBB-TDl(=iqqbUTj7CFW8Qa=}=8j!pe;e7CY2OG5); za%tZ+1?oXNd+>AXqZ6ogd+bw}_UDX81cdFX{ns-DAJXf)xfe^~V1k3VNJ{J>B>g*( zZT5Y%ItD1EjL5lipA{|w8G*;izuCrjj|YE0-ZuL#mtx?+{U#nBP0o&;^(Y)TjmqzX zSf?>10q4}IawUvcMB}VGBcy-7Hap89kQ;T-P`EbTz z4;M#2_Cu=1s$)O*?OPkYbidS;>}{CsHL+58LPpn^2y3wnUPum7e|^1Y{R-l(yC*k0 znIX2>aNf@D{!Qcge$}6x3LK2%cdPOdAcP|l*YR=2-=h&*n8hXw%0~AzsBZrgFEorp zdg#(Wn)%nH{Bg=MxC?eS5Y~0Iae-hCA%Ppu3-eZNwu5SHKeq5ZY5|1R4u29S)0H2T zxqgzXK4vp-FP6)#R#jh;va^Q?_vx?4g#QYHk_IIJ2_$4t!Rf#6rcK<;F6+z#mH{rulD3~}@f&ST<)j(PX@V`~^4Hh2p=8_ndp44fWMO6WF0&9T*#8$8|9v`x4Nrpo=&Fx|-O_ z9lXh-0bMz&<{fL&MHHmaELxm<23qTCLgWWIENl?*2oM;lrikEKwPEVTq+r^ z-x5i8O7NHGM)}MlzeCBk*__#kkJYG2rfd<+FWj~X&{~iS_kkHQzebyK5dgw?%KUf{ zcISwLs`fboX#t$GGu?_O^ahhpa4|cH`mrqZYv%b-h=g??zpdl(8ggWV*OqCC;zVq! z#rH}AFqv1?Gj?`9;9@^7q3r#MaMFr`0pEM)dXZzjtjdb5xo5YmKP&0Kh@#Gb*=g|r zt4F>WgtCaP3iVj7UHl>v8{dG*$ORAjoqr1R+Xl~``=0Um#k~4AAe^OONb=27h3q;@ zaKd8@l%Iit=iYRYd-OMJp|!<}h%B|q=GV#lHvf=o2SqSyvj#JMKCxXue3u^ZA3B4V z;Z2B>aO++u-6nBPYWu&DYbCc!5}G%ioBY)W_HQC28C&!Hp!YHnBNd1_e>y6HNP-RY zjGVveYX7m%aDqvDaLrWTL#?x$Oy;9ix2_-(f($!R;T}FubEmX770>Ajz^k# zej4t~kJvPWtp|+5i2Sz8JU$BCA1mT6WQRaIbIIDbD)1#(P!24iXNNy%y-JPa==~`f zjZ%;)Vmd*!@Q_S_q?mo`%p?|xBJ_$=DQ$q9J)25)oLI*iG>_#?dzv1&mqam6Ev^(0b_T8DGApn~Y^#MA&m=4q6LkGTR3Y&S zp!?nS^(HFm4kT-25*=62%)>vs9DzeFFm^0^j>0N+!z}onQeE(;MXt`lHZ1)Y(!w}d zQm+Ssh^^+#G6`a2j;4*37Hgxa7UO7|Ggz1Y&!1mNgO47~O0N{=DNMhGOKDVyV&SPM zx&4mI(`DoS)bMYFgq**_iYOvBO1R**K zH$9&bZQVAzl-750n!`{AWAQLc(PfM7>IU5(1<_I-ba-f_RDE*DS5nOnuX3B9*XVh3 zC|8@R1Lv3UnOy3Q%@)Z!AFJ5HjepI|-_i0bw|>82B@$lXET#5jcQ3Ij8+I~BCZ9Cv z_8AI9Fu4))_+-_fqg%9z>{08uBU2sQm{ zGy#|>pRdY<8D68aI_@5?c0g89muUA^#qGXb2zQ5$BB57h+UI$DSK^H>&ts0AKV3Dlx0gBSKh+%pd2ma7z&K&L zDMj{KlOxG45dw!4gDknCU@h*ZdsQ6^>?-P11?+)LOoXVr`Z+6XB%#q@L8qK|p8?JZ z(KUU!+q0){mF`r;=c#8syBuvFEzec)%aK=q|CH*=6HUfCNye@-esvOHfJ#pfvUMA~bFVm;G8#m%o#A2oW32 z!QxF)!{&6i8W6+H3PtH~GZEDjw1>5jN8gft`8@vx+$QF)q~^TWKA-|dzR6mBax8l~ zQ>}bx64l>2jgZ`mig^DryrN#kBGQk3ZPha4974S(q;Um9(29A&a+k?DVRJcpy?#tl z^~rxqJ$!vYXLo+a8e)o1xsJ(ZdDpMhMEQZi)?3_HU!T54BF?n+w5|T!N@Lahn=G$B z)%8lqYyaQgZ?nNv@saS>;2<>W_eQz-Aevp+`0jmCS4TK36*fPOWFYZ5>52wBTnOXT zvp_@Yea(9v>jzUr1N`=vL740KnkrNRcWL>M*bGr8tFVrk02s!?K-z<`Y8eRKXcx1e zl7scJ(70Q36di6$4z7oAkQ^^Hzaec64e_gr?LNy2%s0MoAtG<2QIg)^npR$m#f-fJd7^9#+9u7 zZ+}1-*K>>w31RHAA{r%(RWJ;0C>N28_OnR>i>$K`lgo z^G(Gw7nUw8g0|jnZiN{I%L~L-da5z^DoQ-Uw@$@;<*LqWViHUxG+ zrR=l{#8-8%`&1J!N&*wIj5R@r>bg{v2qn=8TxW=}4~gEds8pc&m)8D?%P7nBt$R4~ z(_m=>`vpn*nWJ^>?;JMC9pN18>?fX9i&Oa_159QCin^Lz8^w-W!zQ=hh~PHB!A3Tm zxbIt8I9n*Vu-P5g3P-foODgQ@P**-jW-g00_53!Rjj%!=N8Nmo=WwRkX<%N07fSQ_ zSN0hN@55GfwJ0zz|fB| z`dNKV(3SkWN9*Emze3R4-S`sA#0op5gLXSa*nn{zkx#GAELQU*=XZENp0q<&GA?Fq zBfMcO=0AdjaiKh7$dv4#5sZ)p2k!ML;6MWlngB%}ZLfzXq0yRe*w&FpNy)m0Q?gRL z?-lnCO`eGp=35VC&HTg(Bp#*TKi(>fx+BtS^#qNmdc41#o*!OBxygwfo|FyYjR1f_ z#z|jEJBsVZNe%C|xep9lIl|~Ty9&3a`Qw)NlBa5`EouKagZntQrmoO#K>sqfSn~uJ zr7;Tmq?&(XMb-5dQ0|$XpbA|>A#NV7Q`ma-(cM&^_K*i-a1>_> zUw6@DvKRHCKU1ecTxvI$t8n}4kX>s}g??HR_R^*kwWm5WO3+Z0mtof+UTU6F@mn)( zJX1;Bk9uM!m;-4f<0AJ@_S}<8lM>!yBPpn{4C3=^4tJ0Mx$^fx+RQ1{#})yJI5o$% zhnWI+4A{A^2BtOZQH&FkjTz5%mbw|_7zechY6QZ(Qlg^iYx?rHe&#JX9`^}PICnGa zH_L3Xj{$7S;N5#su;D@R$JjYua71~Df?>{5MBQ?~EMGgm=}ZIEHKltoIy&=AUFhNtYb=Xt_o5v|X|n)bNl;Yyg8 z1&}LoYQy-nt@JZ4FPuWNzNH_=zr0QZ9IkjCr`_SU$wJ+&-zCG!Mr7eRdMS=$Om1I3 zHkO}Xn#ya`$OF?HZL-KJ=oMh5Ent7!6oVg!1=&7OzwX&}vsIqPpHJUs!%x`rx8u>Q zEv7zd4XgJQ4;O2acXvw{}O}J>cw%eNh|-GPoVYB+`6Kh$;9rKci%h&Hsk07zB1=mmr`!>n-%Bq8x?MPurJMH>?C?{|M3;a5Mn+) zNQoF?qB{hD77{ZU)2Cr8sx4h9pEea8{HbW6=xD)2%Dl zCIiHRp$C6Wjg7j;5{zwm;@Lk*j`SsY)F*ir&ZC<*LK}>Hs{>1UqegOndvG*t9EC)a zik}gy%>3fFA^J-;keq?J9w&#EiaGOrIFMHSGbTl74K*MOpC(et%UCJk?n=mA|MezI zB6<%uZZsq6KZ($p1fo}-xu7TAGf$pbzj%o3`Bvqji1Dw5rhl-=-ex-xl`KAAlZgI` z#As88h@NWHBr*FiW#n-bY_HgmPf(zb>!jhsg^?fT@X zlgE2sVlmVAUEJtYR;ofNe32zSQt27UhK!a+ls|tXTYp=v*b+B39<@~cB3NdC9{YEA zpv^gKG19WH{n>iYT;SwCnD^=wNu?$z1O} zOvWJA4jUzzyPfnbJEAQg_v@q%p$abDkZbE;k@2FRqj>+{DrnAl=xd3PvT0MN!kmeY z({vL7!6A%dd04Q426MRf@#v+S3RjNIt*h<;5 z{`V5qTcN(w$WGW}W>N&zzlO5DtVEkv7L_dzHdE&3KX<%uB7i@u~NKssd|AWt#;YW%^6cM%a8a7>9N5EJjB4R)ui=zB(-sip2R#f#s zah|eV!d29?Uk)y~DHFf#02fVidc@mQB?j(hD|hGD3t3}{4BMh$ z)n&2#E%wW%=`PKKs{u*slYh~;Wy?vyzOY}m$<-w+sQ~WZ=7t*{&^WygC9)=%5=(b5 z;(FNp$u4Zpq0VUo-x;Bh0*z8rG)r6WIi#i624-^*2*b-_lu3R|ZU41Inl;6Pu#0#~ z1f7aRy6ItEHFQGG)59SOT-`wySo)Rgl{pViTljnojed@Gg<2mXIk~_(nD#AuOak63 zLAFi)msMvk^0REY?_hU;q+oGm1j8#Ytxs>?rv8P@%klET>k@9y@iK0U|3DPN#9(4p z_HCYwx5ww_m5p{JzJ-O((DwwxYU_F8G^nTlcb1d9-BGM9tKdAD7wvALx!@TucQI#AR5@n zu#mR5Cegiw0kiABoQ9wITCB9|wV5rTjFZ1;eJJ23-7IqsT}23gk|o8ZzRs@jns{Nb zF3Va;aW*}pP0?f^2)K%URA!Ym@&{Nu;EOWG%Y2K-MxN88&SYYt^`bX(zsny}Wd9ZL z+Ia$J6qAs8)@&*a_C@(!aYJ4+oh|~q{^($6-D7+W;`ideIwfA_)$^T5yt2M(JZVd5 zY?cYgv=faDXd<(?rtFte#*KY~A-KZ#zL}sg=D~L&!!C4?hpi56v#~X8^I_QA+2@i+ z$LasGtNF_cmdn@F)npUetI-yanOlMn7$QtRHL8>s_xhE95s-b7w$~W3ItITyke%g; zjUl1{KbrzFij!U0Ksh}-6uoXLHMFsG@{H->tKY>GazzjZ8#BRAFZyuXPzd0(Ln1NG z2QE)0rhK9;oejK{5ha41!wrWRnmH+;G%Cs40q33Hd{Onjfw8dIs*eMQy%igoO@B9#2bX*Ky5Z!CEmSY`Q6Eg6hIe&$kMw^#s6Y1^F;% zRlM#QGzD;^Y{{4jN*4D`+?LkpZIHeTUaIJhhfrEG&OpK%P6ZdIN-!Q6VGPA2bcizu zA$tELE5ao-_p`Pnx%DTYZ{Z`}s>H1%a?9+mn7aiA_sFwUW~()l5~^PcHr|fSC}k!Hbpf zrv&B$f2)X4d?HOGk%lKhA>Bb&lBGORK$u;$#NY7kk9O4=uGaFta!~>a9bvuI`!Q-a-Y< zdZDQWv!hpf^V~!Ro%{0o;NCJk6%ZlW8WG_{74f+&7`)SzCa@aQ%OZ%z(GkHi#j>WY zY5?{kd<0W4+04~*5i62Y5sYPKrPQJw>O)rGMDZ9h{HDE@a#opjS85qm$k6;+0MZVL z3^}V$bV;DBv>G!{5qbtyFdjy#>!f7nS+U)%6Du$L@O`(QuMBOuyLM