From 9757f947331cab65d8fe73a4ab59e4be41a8bec3 Mon Sep 17 00:00:00 2001 From: Matt Conway Date: Fri, 10 Nov 2023 08:48:57 -0500 Subject: [PATCH] add facility to get a live stacktrace of kubetruth process --- Gemfile | 8 +- Gemfile.lock | 223 +++++++++++++-------------------------- bin/stackdump | 5 + lib/kubetruth.rb | 3 +- lib/kubetruth/sigdump.rb | 29 +++++ 5 files changed, 115 insertions(+), 153 deletions(-) create mode 100755 bin/stackdump create mode 100644 lib/kubetruth/sigdump.rb diff --git a/Gemfile b/Gemfile index 60240a4..b64bd92 100644 --- a/Gemfile +++ b/Gemfile @@ -4,10 +4,7 @@ source "https://rubygems.org" group :development do gem "rake" gem "pry" - gem "pry-byebug" - gem "ruby-debug-ide" - gem "debase" - gem "solargraph" + gem "debug" end # test dependencies @@ -23,7 +20,8 @@ end # application runtime dependencies gem "gem_logger" gem "logging" -gem "activesupport" +gem 'sigdump' +gem "activesupport", '~> 7.0', '<= 7.0.8' gem "clamp" gem "cloudtruth-client", path: "client" gem "kubeclient" diff --git a/Gemfile.lock b/Gemfile.lock index ccc3f25..a3debe9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,10 +1,3 @@ -GIT - remote: https://github.com/wr0ngway/yaml-safe_load_stream.git - revision: 3c8bcd30369ac87c42ef04e6e2f6727f44936233 - branch: ruby_3 - specs: - yaml-safe_load_stream (0.1.2) - PATH remote: client specs: @@ -15,217 +8,156 @@ PATH GEM remote: https://rubygems.org/ specs: - activesupport (6.1.4) + activesupport (7.0.8) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - zeitwerk (~> 2.3) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) - ast (2.4.2) - async (1.29.1) + addressable (2.8.5) + public_suffix (>= 2.0.2, < 6.0) + async (1.31.0) console (~> 1.10) nio4r (~> 2.3) timers (~> 4.1) - backport (1.2.0) - benchmark (0.2.1) - byebug (11.1.3) + base64 (0.2.0) clamp (1.3.2) - codecov (0.5.2) + codecov (0.6.0) simplecov (>= 0.15, < 0.22) coderay (1.1.3) - concurrent-ruby (1.1.9) - console (1.13.1) + concurrent-ruby (1.2.2) + console (1.23.2) + fiber-annotation fiber-local crack (0.4.5) rexml - debase (0.2.5.beta2) - debase-ruby_core_source (>= 0.10.12) - debase-ruby_core_source (0.10.12) - diff-lcs (1.4.4) + debug (1.8.0) + irb (>= 1.5.0) + reline (>= 0.3.1) + diff-lcs (1.5.0) docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - dotenv (2.7.6) - e2mmap (0.1.0) - faraday (1.5.1) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0.1) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.1) - faraday-patron (~> 1.0) - multipart-post (>= 1.2, < 3) + dotenv (2.8.1) + faraday (2.7.11) + base64 + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) faraday-cookie_jar (0.0.7) faraday (>= 0.8.0) http-cookie (~> 1.0.0) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) faraday-multipart (1.0.4) multipart-post (~> 2) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - ffi (1.15.3) + faraday-net_http (3.0.2) + ffi (1.16.3) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake + fiber-annotation (0.2.0) fiber-local (1.0.0) gem_logger (0.3.0) activesupport hashdiff (1.0.1) - http (4.4.1) - addressable (~> 2.3) + http (5.1.1) + addressable (~> 2.8) http-cookie (~> 1.0) http-form_data (~> 2.2) - http-parser (~> 1.2.0) + llhttp-ffi (~> 0.4.0) http-accept (1.7.0) - http-cookie (1.0.4) + http-cookie (1.0.5) domain_name (~> 0.5) http-form_data (2.3.0) - http-parser (1.2.3) - ffi-compiler (>= 1.0, < 2.0) - i18n (1.8.10) + i18n (1.14.1) concurrent-ruby (~> 1.0) - jaro_winkler (1.5.4) - json (2.6.3) - jsonpath (1.1.0) + io-console (0.6.0) + irb (1.8.3) + rdoc + reline (>= 0.3.8) + jsonpath (1.1.5) multi_json - kramdown (2.4.0) - rexml - kramdown-parser-gfm (1.1.0) - kramdown (~> 2.0) - kubeclient (4.9.2) - http (>= 3.0, < 5.0) + kubeclient (4.11.0) + http (>= 3.0, < 6.0) jsonpath (~> 1.0) recursive-open-struct (~> 1.1, >= 1.1.1) rest-client (~> 2.0) - liquid (5.0.1) + liquid (5.4.0) little-plugger (1.1.4) - logging (2.3.0) + llhttp-ffi (0.4.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logging (2.3.1) little-plugger (~> 1.1) multi_json (~> 1.14) method_source (1.0.0) - mime-types (3.3.1) + mime-types (3.5.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0704) - minitest (5.14.4) + mime-types-data (3.2023.1003) + minitest (5.20.0) multi_json (1.15.0) - multipart-post (2.1.1) + multipart-post (2.3.0) netrc (0.11.0) - nio4r (2.5.7) - nokogiri (1.14.2-arm64-darwin) - racc (~> 1.4) - nokogiri (1.14.2-x86_64-darwin) - racc (~> 1.4) - parallel (1.22.1) - parser (3.2.2.0) - ast (~> 2.4.1) - pry (0.14.1) + nio4r (2.5.9) + pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - pry-byebug (3.8.0) - byebug (~> 11.0) - pry (~> 0.10) - public_suffix (4.0.6) - racc (1.6.2) - rainbow (3.1.1) - rake (13.0.6) - rbs (2.8.4) + psych (5.1.1.1) + stringio + public_suffix (5.0.3) + rake (13.1.0) + rdoc (6.6.0) + psych (>= 4.0.0) recursive-open-struct (1.1.3) - regexp_parser (2.7.0) + reline (0.4.0) + io-console (~> 0.5) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - reverse_markdown (2.1.1) - nokogiri - rexml (3.2.5) - rspec (3.10.0) - rspec-core (~> 3.10.0) - rspec-expectations (~> 3.10.0) - rspec-mocks (~> 3.10.0) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.1) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-mocks (3.10.2) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-support (3.10.2) - rubocop (1.50.0) - json (~> 2.3) - parallel (~> 1.10) - parser (>= 3.2.0.0) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.28.0, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.28.0) - parser (>= 3.2.1.0) - ruby-debug-ide (0.7.1.beta3) - rake (>= 0.8.1) - ruby-progressbar (1.13.0) - ruby2_keywords (0.0.4) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + ruby2_keywords (0.0.5) + sigdump (0.2.5) simplecov (0.21.2) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) - simplecov_json_formatter (0.1.3) - solargraph (0.49.0) - backport (~> 1.2) - benchmark - bundler (~> 2.0) - diff-lcs (~> 1.4) - e2mmap - jaro_winkler (~> 1.5) - kramdown (~> 2.3) - kramdown-parser-gfm (~> 1.1) - parser (~> 3.0) - rbs (~> 2.0) - reverse_markdown (~> 2.0) - rubocop (~> 1.38) - thor (~> 1.0) - tilt (~> 2.0) - yard (~> 0.9, >= 0.9.24) - thor (1.2.1) - tilt (2.1.0) - timers (4.3.3) - tzinfo (2.0.4) + simplecov_json_formatter (0.1.4) + stringio (3.0.9) + timers (4.3.5) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) unf (0.1.4) unf_ext - unf_ext (0.0.7.7) - unicode-display_width (2.4.2) - vcr (6.0.0) - webmock (3.13.0) - addressable (>= 2.3.6) + unf_ext (0.0.8.2) + vcr (6.2.0) + webmock (3.19.1) + addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - yard (0.9.32) - zeitwerk (2.4.2) PLATFORMS arm64-darwin-21 - x86_64-darwin-20 DEPENDENCIES - activesupport + activesupport (~> 7.0, <= 7.0.8) async clamp cloudtruth-client! codecov - debase + debug dotenv faraday-cookie_jar gem_logger @@ -233,15 +165,12 @@ DEPENDENCIES liquid logging pry - pry-byebug rake rspec - ruby-debug-ide + sigdump simplecov - solargraph vcr webmock - yaml-safe_load_stream! BUNDLED WITH 2.2.32 diff --git a/bin/stackdump b/bin/stackdump new file mode 100755 index 0000000..90099ef --- /dev/null +++ b/bin/stackdump @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e +kubetruth_pid=$(ps ax | grep '[k]ubetruth' | awk '{print $1}') +kill -CONT "${kubetruth_pid}" +cat /tmp/sigdump-${kubetruth_pid}.log diff --git a/lib/kubetruth.rb b/lib/kubetruth.rb index 92eba6b..0e70d84 100644 --- a/lib/kubetruth.rb +++ b/lib/kubetruth.rb @@ -1,10 +1,11 @@ require 'active_support/core_ext/hash/keys' require 'active_support/core_ext/string/inflections' -# prevent our use of ActiveSupport causing an error with json adapters require 'active_support/json' require 'yaml' require_relative 'kubetruth/logging' +require_relative 'kubetruth/sigdump' + # Need to setup logging before loading any other files Kubetruth::Logging.setup_logging(level: :info, color: false) diff --git a/lib/kubetruth/sigdump.rb b/lib/kubetruth/sigdump.rb new file mode 100644 index 0000000..688f065 --- /dev/null +++ b/lib/kubetruth/sigdump.rb @@ -0,0 +1,29 @@ +require 'sigdump/setup' +require 'async' + +# Adds the reporting of async tasks during a sigdump +module Sigdump + class << self + + def dump_async(task, io) + io.write " Async Task #{task.description} status=#{task.status}\n" + if task.backtrace + task.backtrace.each {|bt| + io.write " #{bt}\n" + } + end + io.flush + end + + alias_method :original_dump_backtrace, :dump_backtrace + def dump_backtrace(thread, io) + original_dump_backtrace(thread, io) + + ObjectSpace.each_object(Async::Task) do |task| + dump_async(task, io) if task + end + + end + + end +end