diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 259518b00..98824c01b 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -16,6 +16,10 @@ "ImportPath": "github.com/PuerkitoBio/urlesc", "Rev": "5bd2802263f21d8788851d5305584c82a5c75d7e" }, + { + "ImportPath": "github.com/c2h5oh/datasize", + "Rev": "54516c931ae99c3c74637b9ea2390cf9a6327f26" + }, { "ImportPath": "github.com/davecgh/go-spew/spew", "Rev": "5215b55f46b2b919f50a1df0eaa5886afe4e3b3d" diff --git a/cmd/sonobuoy/app/master.go b/cmd/sonobuoy/app/master.go index 92c545603..bd3d23517 100644 --- a/cmd/sonobuoy/app/master.go +++ b/cmd/sonobuoy/app/master.go @@ -22,6 +22,7 @@ import ( "github.com/heptio/sonobuoy/pkg/config" "github.com/heptio/sonobuoy/pkg/discovery" "github.com/heptio/sonobuoy/pkg/errlog" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -47,7 +48,7 @@ func runMaster(cmd *cobra.Command, args []string) { cfg, err := config.LoadConfig() if err != nil { - errlog.LogError(err) + errlog.LogError(errors.Wrap(err, "error loading sonobuoy configuration")) os.Exit(1) } diff --git a/docs/configuration.md b/docs/configuration.md index 17f0ac51b..f003e8d47 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,6 +81,11 @@ See [Parameter Reference][3] for a more detailed description of each setting. "LabelSelector": "", "Namespaces": ".*" }, + "Limits": { + "PodLogs": { + "LimitTime": "24h" + } + }, "Server": { "advertiseaddress": "", "bindaddress": "0.0.0.0", @@ -105,6 +110,8 @@ See [Parameter Reference][3] for a more detailed description of each setting. | Resources | String Array | An array containing all possible resources | *See the [sample JSON][2] above for a list of all available resource types.*

Indicates to Sonobuoy what type of data it should be recording | | Filters.LabelSelector | String | "" | Uses standard Kubernetes [label selector syntax][14] to filter which resource objects are recorded | | Filters.Namespaces | String | ".*" | Uses regex on namespaces to filter which resource objects are recorded | +| Limits.PodLogs.LimitTime | String | "" | Limits how far back in time to gather Pod Logs, leave blank for no limit (e.g. "24h", "60m". See https://golang.org/pkg/time/#ParseDuration for details.) | +| Limits.PodLogs.LimitSize | String | "" | Limits the size of Pod Logs to gather, per container, leave blank for no limit (e.g. "10 MB", "1 GB", etc.) | | Server.advertiseaddress | String | `$SONOBUOY_ADVERTISE_IP` || the current server's `os.Hostname()`| *Only used if Sonobuoy dispatches agent pods to collect node-specific information*

The IP address that remote Sonobuoy agents send information back to, in order for disparate data to be aggregated into a single report | | Server.bindaddress | String | "0.0.0.0" | *See `Server.advertiseaddress` for context.*

If data aggregation is required, an HTTP server is started to handle the worker requests. This is the address that server binds to. | | Server.bindport | Int | 8080 | The port for the HTTP server mentioned in *Server.bindaddress*. | diff --git a/pkg/config/config.go b/pkg/config/config.go index 20b3ad8ef..a15ba288c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -18,7 +18,9 @@ package config import ( "path" + "time" + "github.com/c2h5oh/datasize" "github.com/heptio/sonobuoy/pkg/buildinfo" "github.com/heptio/sonobuoy/pkg/plugin" "github.com/satori/go.uuid" @@ -111,6 +113,8 @@ type Config struct { /////////////////////////////////////////////// Filters FilterOptions `json:"Filters" mapstructure:"Filters"` + Limits LimitConfig `json:"Limits" mapstructure:"Limits"` + /////////////////////////////////////////////// // plugin configurations settings /////////////////////////////////////////////// @@ -121,6 +125,18 @@ type Config struct { LoadedPlugins []plugin.Interface // this is assigned when plugins are loaded. } +// LimitConfig is a configuration on the limits of sizes of various responses. +type LimitConfig struct { + PodLogs SizeOrTimeLimitConfig `json:"PodLogs" mapstructure:"PodLogs"` +} + +// SizeOrTimeLimitConfig represents configuration that limits the size of +// something either by a total disk size, or by a length of time. +type SizeOrTimeLimitConfig struct { + LimitSize string `json:"LimitSize" mapstructure:"LimitSize"` + LimitTime string `json:"LimitTime" mapstructure:"LimitTime"` +} + // FilterResources is a utility function used to parse Resources func (cfg *Config) FilterResources(filter []string) map[string]bool { results := make(map[string]bool) @@ -141,6 +157,52 @@ func (cfg *Config) OutputDir() string { return path.Join(cfg.ResultsDir, cfg.UUID) } +// SizeLimitBytes returns how many bytes the configuration is set to limit, +// returning defaultVal if not set. +func (c SizeOrTimeLimitConfig) SizeLimitBytes(defaultVal int64) int64 { + val, defaulted, err := c.sizeLimitBytes() + + // Ignore error, since we should have already caught it in validation + if err != nil || defaulted { + return defaultVal + } + + return val +} + +func (c SizeOrTimeLimitConfig) sizeLimitBytes() (val int64, defaulted bool, err error) { + str := c.LimitSize + if str == "" { + return 0, true, nil + } + + var bs datasize.ByteSize + err = bs.UnmarshalText([]byte(str)) + return int64(bs.Bytes()), false, err +} + +// TimeLimitDuration returns the duration the configuration is set to limit, returning defaultVal if not set. +func (c SizeOrTimeLimitConfig) TimeLimitDuration(defaultVal time.Duration) time.Duration { + val, defaulted, err := c.timeLimitDuration() + + // Ignore error, since we should have already caught it in validation + if err != nil || defaulted { + return defaultVal + } + + return val +} + +func (c SizeOrTimeLimitConfig) timeLimitDuration() (val time.Duration, defaulted bool, err error) { + str := c.LimitTime + if str == "" { + return 0, true, nil + } + + val, err = time.ParseDuration(str) + return val, false, err +} + // NewWithDefaults returns a newly-constructed Config object with default values. func NewWithDefaults() *Config { var cfg Config diff --git a/pkg/config/loader.go b/pkg/config/loader.go index 29c832f01..fb49b081f 100644 --- a/pkg/config/loader.go +++ b/pkg/config/loader.go @@ -19,6 +19,7 @@ package config import ( "fmt" "os" + "strings" "github.com/heptio/sonobuoy/pkg/buildinfo" "github.com/heptio/sonobuoy/pkg/plugin" @@ -89,10 +90,37 @@ func LoadConfig() (*Config, error) { // 5 - Load any plugins we have err = loadAllPlugins(cfg) + if err != nil { + return nil, err + } + + // 6 - Return any validation errors + validationErrs := cfg.Validate() + if len(validationErrs) > 0 { + errstrs := make([]string, len(validationErrs)) + for i := range validationErrs { + errstrs[i] = validationErrs[i].Error() + } + + return nil, errors.Errorf("invalid configuration: %v", strings.Join(errstrs, ", ")) + } return cfg, err } +// Validate returns a list of errors for the configuration, if any are found. +func (cfg *Config) Validate() (errors []error) { + if _, defaulted, err := cfg.Limits.PodLogs.sizeLimitBytes(); err != nil && !defaulted { + errors = append(errors, err) + } + + if _, defaulted, err := cfg.Limits.PodLogs.timeLimitDuration(); err != nil && !defaulted { + errors = append(errors, err) + } + + return errors +} + // LoadClient creates a kube-clientset, using given sonobuoy configuration func LoadClient(cfg *Config) (kubernetes.Interface, error) { var config *rest.Config diff --git a/pkg/discovery/pods.go b/pkg/discovery/pods.go index b53fa211c..a691f3ccf 100644 --- a/pkg/discovery/pods.go +++ b/pkg/discovery/pods.go @@ -20,10 +20,11 @@ import ( "io/ioutil" "os" "path" + "time" - "github.com/sirupsen/logrus" "github.com/heptio/sonobuoy/pkg/config" "github.com/pkg/errors" + "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -44,6 +45,8 @@ func gatherPodLogs(kubeClient kubernetes.Interface, ns string, opts metav1.ListO } logrus.Info("Collecting Pod Logs...") + limitBytes := cfg.Limits.PodLogs.SizeLimitBytes(0) + limitTime := int64(cfg.Limits.PodLogs.TimeLimitDuration(0) / time.Second) // 2 - Foreach pod, dump each of its containers' logs in a tree in the following location: // pods/:podname/logs/:containername.txt @@ -52,7 +55,9 @@ func gatherPodLogs(kubeClient kubernetes.Interface, ns string, opts metav1.ListO body, err := kubeClient.CoreV1().Pods(ns).GetLogs( pod.Name, &v1.PodLogOptions{ - Container: container.Name, + Container: container.Name, + LimitBytes: &limitBytes, + SinceSeconds: &limitTime, }, ).Do().Raw() diff --git a/vendor/github.com/c2h5oh/datasize/.gitignore b/vendor/github.com/c2h5oh/datasize/.gitignore new file mode 100644 index 000000000..daf913b1b --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/vendor/github.com/c2h5oh/datasize/.travis.yml b/vendor/github.com/c2h5oh/datasize/.travis.yml new file mode 100644 index 000000000..a6ebc037c --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/.travis.yml @@ -0,0 +1,11 @@ +sudo: false + +language: go +go: + - 1.4 + - 1.5 + - 1.6 + - tip + +script: + - go test -v diff --git a/vendor/github.com/c2h5oh/datasize/LICENSE b/vendor/github.com/c2h5oh/datasize/LICENSE new file mode 100644 index 000000000..f2ba916e6 --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Maciej Lisiewski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/c2h5oh/datasize/README.md b/vendor/github.com/c2h5oh/datasize/README.md new file mode 100644 index 000000000..188bd8f18 --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/README.md @@ -0,0 +1,66 @@ +# datasize [![Build Status](https://travis-ci.org/c2h5oh/datasize.svg?branch=master)](https://travis-ci.org/c2h5oh/datasize) + +Golang helpers for data sizes + + +### Constants +Just like `time` package provides `time.Second`, `time.Day` constants `datasize` provides: +* `datasize.B` 1 byte +* `datasize.KB` 1 kilobyte +* `datasize.MB` 1 megabyte +* `datasize.GB` 1 gigabyte +* `datasize.TB` 1 terabyte +* `datasize.PB` 1 petabyte +* `datasize.EB` 1 exabyte + +### Helpers +Just like `time` package provides `duration.Nanoseconds() uint64 `, `duration.Hours() float64` helpers `datasize` has +* `ByteSize.Bytes() uint64` +* `ByteSize.Kilobytes() float4` +* `ByteSize.Megabytes() float64` +* `ByteSize.Gigabytes() float64` +* `ByteSize.Terabytes() float64` +* `ByteSize.Petebytes() float64` +* `ByteSize.Exabytes() float64` + +Warning: see limitations at the end of this document about a possible precission loss + +### Parsing strings +`datasize.ByteSize` implements `TestUnmarshaler` interface and will automatically parse human readable strings into correct values where it is used: +* `"10 MB"` -> `10* datasize.MB` +* `"10240 g"` -> `10 * datasize.TB` +* `"2000"` -> `2000 * datasize.B` +* `"1tB"` -> `datasize.TB` +* `"5 peta"` -> `5 * datasize.PB` +* `"28 kilobytes"` -> `28 * datasize.KB` +* `"1 gigabyte"` -> `1 * datasize.GB` + +You can also do it manually: +```go +var v datasize.ByteSize +err := v.UnmarshalText([]byte("100 mb")) +``` + +### Printing +`Bytesize.String()` uses largest unit allowing an integer value: + * `(102400 * datasize.MB).String()` -> `"100GB"` + * `(datasize.MB + datasize.KB).String()` -> `"1025KB"` + +Use `%d` format string to get value in bytes without a unit + +### JSON and other encoding +Both `TextMarshaler` and `TextUnmarshaler` interfaces are implemented - JSON will just work. Other encoders will work provided they use those interfaces. + +### Human readable +`ByteSize.HumanReadable()` or `ByteSize.HR()` returns a string with 1-3 digits, followed by 1 decimal place, a space and unit big enough to get 1-3 digits + + * `(102400 * datasize.MB).String()` -> `"100.0 GB"` + * `(datasize.MB + 512 * datasize.KB).String()` -> `"1.5 MB"` + +### Limitations +* The underlying data type for `data.ByteSize` is `uint64`, so values outside of 0 to 2^64-1 range will overflow +* size helper functions (like `ByteSize.Kilobytes()`) return `float64`, which can't represent all possible values of `uint64` accurately: + * if the returned value is supposed to have no fraction (ie `(10 * datasize.MB).Kilobytes()`) accuracy loss happens when value is more than 2^53 larger than unit: `.Kilobytes()` over 8 petabytes, `.Megabytes()` over 8 exabytes + * if the returned value is supposed to have a fraction (ie `(datasize.PB + datasize.B).Megabytes()`) in addition to the above note accuracy loss may occur in fractional part too - larger integer part leaves fewer bytes to store fractional part, the smaller the remainder vs unit the move bytes are required to store the fractional part +* Parsing a string with `Mb`, `Tb`, etc units will return a syntax error, because capital followed by lower case is commonly used for bits, not bytes +* Parsing a string with value exceeding 2^64-1 bytes will return 2^64-1 and an out of range error diff --git a/vendor/github.com/c2h5oh/datasize/datasize.go b/vendor/github.com/c2h5oh/datasize/datasize.go new file mode 100644 index 000000000..675478816 --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/datasize.go @@ -0,0 +1,217 @@ +package datasize + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +type ByteSize uint64 + +const ( + B ByteSize = 1 + KB = B << 10 + MB = KB << 10 + GB = MB << 10 + TB = GB << 10 + PB = TB << 10 + EB = PB << 10 + + fnUnmarshalText string = "UnmarshalText" + maxUint64 uint64 = (1 << 64) - 1 + cutoff uint64 = maxUint64 / 10 +) + +var ErrBits = errors.New("unit with capital unit prefix and lower case unit (b) - bits, not bytes ") + +func (b ByteSize) Bytes() uint64 { + return uint64(b) +} + +func (b ByteSize) KBytes() float64 { + v := b / KB + r := b % KB + return float64(v) + float64(r)/float64(KB) +} + +func (b ByteSize) MBytes() float64 { + v := b / MB + r := b % MB + return float64(v) + float64(r)/float64(MB) +} + +func (b ByteSize) GBytes() float64 { + v := b / GB + r := b % GB + return float64(v) + float64(r)/float64(GB) +} + +func (b ByteSize) TBytes() float64 { + v := b / TB + r := b % TB + return float64(v) + float64(r)/float64(TB) +} + +func (b ByteSize) PBytes() float64 { + v := b / PB + r := b % PB + return float64(v) + float64(r)/float64(PB) +} + +func (b ByteSize) EBytes() float64 { + v := b / EB + r := b % EB + return float64(v) + float64(r)/float64(EB) +} + +func (b ByteSize) String() string { + switch { + case b == 0: + return fmt.Sprint("0B") + case b%EB == 0: + return fmt.Sprintf("%dEB", b/EB) + case b%PB == 0: + return fmt.Sprintf("%dPB", b/PB) + case b%TB == 0: + return fmt.Sprintf("%dTB", b/TB) + case b%GB == 0: + return fmt.Sprintf("%dGB", b/GB) + case b%MB == 0: + return fmt.Sprintf("%dMB", b/MB) + case b%KB == 0: + return fmt.Sprintf("%dKB", b/KB) + default: + return fmt.Sprintf("%dB", b) + } +} + +func (b ByteSize) HR() string { + return b.HumanReadable() +} + +func (b ByteSize) HumanReadable() string { + switch { + case b > EB: + return fmt.Sprintf("%.1f EB", b.EBytes()) + case b > PB: + return fmt.Sprintf("%.1f PB", b.PBytes()) + case b > TB: + return fmt.Sprintf("%.1f TB", b.TBytes()) + case b > GB: + return fmt.Sprintf("%.1f GB", b.GBytes()) + case b > MB: + return fmt.Sprintf("%.1f MB", b.MBytes()) + case b > KB: + return fmt.Sprintf("%.1f KB", b.KBytes()) + default: + return fmt.Sprintf("%d B", b) + } +} + +func (b ByteSize) MarshalText() ([]byte, error) { + return []byte(b.String()), nil +} + +func (b *ByteSize) UnmarshalText(t []byte) error { + var val uint64 + var unit string + + // copy for error message + t0 := t + + var c byte + var i int + +ParseLoop: + for i < len(t) { + c = t[i] + switch { + case '0' <= c && c <= '9': + if val > cutoff { + goto Overflow + } + + c = c - '0' + val *= 10 + + if val > val+uint64(c) { + // val+v overflows + goto Overflow + } + val += uint64(c) + i++ + + default: + if i == 0 { + goto SyntaxError + } + break ParseLoop + } + } + + unit = strings.TrimSpace(string(t[i:])) + switch unit { + case "Kb", "Mb", "Gb", "Tb", "Pb", "Eb": + goto BitsError + } + unit = strings.ToLower(unit) + switch unit { + case "", "b", "byte": + // do nothing - already in bytes + + case "k", "kb", "kilo", "kilobyte", "kilobytes": + if val > maxUint64/uint64(KB) { + goto Overflow + } + val *= uint64(KB) + + case "m", "mb", "mega", "megabyte", "megabytes": + if val > maxUint64/uint64(MB) { + goto Overflow + } + val *= uint64(MB) + + case "g", "gb", "giga", "gigabyte", "gigabytes": + if val > maxUint64/uint64(GB) { + goto Overflow + } + val *= uint64(GB) + + case "t", "tb", "tera", "terabyte", "terabytes": + if val > maxUint64/uint64(TB) { + goto Overflow + } + val *= uint64(TB) + + case "p", "pb", "peta", "petabyte", "petabytes": + if val > maxUint64/uint64(PB) { + goto Overflow + } + val *= uint64(PB) + + case "E", "EB", "e", "eb", "eB": + if val > maxUint64/uint64(EB) { + goto Overflow + } + val *= uint64(EB) + + default: + goto SyntaxError + } + + *b = ByteSize(val) + return nil + +Overflow: + *b = ByteSize(maxUint64) + return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrRange} + +SyntaxError: + *b = 0 + return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrSyntax} + +BitsError: + *b = 0 + return &strconv.NumError{fnUnmarshalText, string(t0), ErrBits} +} diff --git a/vendor/github.com/c2h5oh/datasize/datasize_test.go b/vendor/github.com/c2h5oh/datasize/datasize_test.go new file mode 100644 index 000000000..4cd283a11 --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/datasize_test.go @@ -0,0 +1,73 @@ +package datasize_test + +import ( + "testing" + + . "github.com/c2h5oh/datasize" +) + +func TestMarshalText(t *testing.T) { + table := []struct { + in ByteSize + out string + }{ + {0, "0B"}, + {B, "1B"}, + {KB, "1KB"}, + {MB, "1MB"}, + {GB, "1GB"}, + {TB, "1TB"}, + {PB, "1PB"}, + {EB, "1EB"}, + {400 * TB, "400TB"}, + {2048 * MB, "2GB"}, + {B + KB, "1025B"}, + {MB + 20*KB, "1044KB"}, + {100*MB + KB, "102401KB"}, + } + + for _, tt := range table { + b, _ := tt.in.MarshalText() + s := string(b) + + if s != tt.out { + t.Errorf("MarshalText(%d) => %s, want %s", tt.in, s, tt.out) + } + } +} + +func TestUnmarshalText(t *testing.T) { + table := []struct { + in string + err bool + out ByteSize + }{ + {"0", false, ByteSize(0)}, + {"0B", false, ByteSize(0)}, + {"0 KB", false, ByteSize(0)}, + {"1", false, B}, + {"1K", false, KB}, + {"2MB", false, 2 * MB}, + {"5 GB", false, 5 * GB}, + {"20480 G", false, 20 * TB}, + {"50 eB", true, ByteSize((1 << 64) - 1)}, + {"200000 pb", true, ByteSize((1 << 64) - 1)}, + {"10 Mb", true, ByteSize(0)}, + {"g", true, ByteSize(0)}, + {"10 kB ", false, 10 * KB}, + {"10 kBs ", true, ByteSize(0)}, + } + + for _, tt := range table { + var s ByteSize + err := s.UnmarshalText([]byte(tt.in)) + + if (err != nil) != tt.err { + t.Errorf("UnmarshalText(%s) => %v, want no error", tt.in, err) + } + + if s != tt.out { + t.Errorf("UnmarshalText(%s) => %d bytes, want %d bytes", tt.in, s, tt.out) + } + } +}