diff --git a/docs/snapshot.md b/docs/snapshot.md index 0f7ff483e..e1f36cb09 100644 --- a/docs/snapshot.md +++ b/docs/snapshot.md @@ -1,7 +1,5 @@ # Sonobuoy Snapshot Layout -> NOTE: This documentation is a draft proposal for the structure of a Sonobuoy tarball and how it should be changed. This documentation does not represent the current state of a Sonobuoy tarball. - - [Filename](#filename) - [Contents](#contents) - [/resources](#resources) @@ -50,8 +48,6 @@ This looks like the following: ### /podlogs -> NOTE: pod logs are currently distributed throughout the tarball in `/resources/ns//pods//.txt`. The proposal is to move them under a different directory so that "types" within a directory are consistent - The `/podlogs` directory contains logs for each pod found during the Sonobuoy run, similarly to what you would get with `kubectl logs -n `. - `/podlogs///.log` - Contains the logs for the each container, for each pod in each namespace. @@ -68,6 +64,12 @@ The `/plugins` directory contains output for each plugin selected for this Sonob - `/plugins//results/.` - For plugins that run once on every node to collect node-specific data (ones that use the DaemonSet driver, for instance), this will contain the results for this plugin, for each node, using the format that the plugin expects. See [file formats][2] for details. +Some plugins can include several files as part of their results. To do this, a plugin will submit a `.tar.gz` file, the contents of which are extracted into the following: + +- `/plugins//results/` - For plugins that collect cluster-wide data into a `.tar.gz` file + +- `/plugins///` - For plugins that collect per-node data into a `.tar.gz` file + This looks like the following: ![tarball plugins screenshot][7] @@ -77,7 +79,6 @@ This looks like the following: The `/meta` directory contains metadata about this Sonobuoy run, including configuration and query runtime. - `/meta/query-time.json` - Contains metadata about how long each query took, example: `{"queryobj":"Pods","time":12.345ms"}` - > NOTE: this file is currently distributed throughout the tarball in `/resources/.../results.json`. Proposal is to move it here, and create other `.json` files under `/meta` if we start capturing other things than query time. - `/meta/config.json` - A copy of the Sonobuoy configuration that was set up when this run was created, but with unspecified values filled in with explicit defaults, and with a `UUID` field in the root JSON, set to a randomly generated UUID created for that Sonobuoy run. This looks like the following: @@ -86,8 +87,6 @@ This looks like the following: ### /serverversion.json -> NOTE: this is currently `/serverversion/serverversion.json`, proposal here is to just make it `/serverversion.json` - `/serverversion.json` contains the output from querying the server's version, including the major and minor version, git commit, etc. ## File formats diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 927b0735a..8aa656a4f 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -20,6 +20,7 @@ import ( "encoding/json" "io/ioutil" "os" + "path" "time" "github.com/golang/glog" @@ -47,16 +48,19 @@ func Run(kubeClient kubernetes.Interface, cfg *config.Config) (errCount uint) { // 1. Get the list of namespaces and apply the regex filter on the namespace nslist := FilterNamespaces(kubeClient, cfg.Filters.Namespaces) - // 2. Create the directory which will store the results - outpath := cfg.ResultsDir + "/" + cfg.UUID - err := os.MkdirAll(outpath, 0755) + // 2. Create the directory which will store the results, including the + // `meta` directory inside it (which we always need regardless of + // config) + outpath := path.Join(cfg.ResultsDir, cfg.UUID) + metapath := path.Join(outpath, MetaLocation) + err := os.MkdirAll(metapath, 0755) if err != nil { panic(err.Error()) } // 3. Dump the config.json we used to run our test if blob, err := json.Marshal(cfg); err == nil { - if err = ioutil.WriteFile(outpath+"/config.json", blob, 0644); err != nil { + if err = ioutil.WriteFile(path.Join(metapath, "config.json"), blob, 0644); err != nil { panic(err.Error()) } } @@ -67,20 +71,26 @@ func Run(kubeClient kubernetes.Interface, cfg *config.Config) (errCount uint) { ) // 5. Run the queries + recorder := NewQueryRecorder() trackErrorsFor("querying cluster resources")( - QueryClusterResources(kubeClient, cfg), + QueryClusterResources(kubeClient, recorder, cfg), ) for _, ns := range nslist { trackErrorsFor("querying resources under namespace " + ns)( - QueryNSResources(kubeClient, ns, cfg), + QueryNSResources(kubeClient, recorder, ns, cfg), ) } - // 6. Clean up after the plugins + // 6. Dump the query times + trackErrorsFor("recording query times")( + recorder.DumpQueryData(path.Join(metapath, "query-time.json")), + ) + + // 7. Clean up after the plugins pluginaggregation.Cleanup(kubeClient, cfg.LoadedPlugins) - // 7. tarball up results YYYYMMDDHHMM_sonobuoy_UID.tar.gz + // 8. tarball up results YYYYMMDDHHMM_sonobuoy_UID.tar.gz tb := cfg.ResultsDir + "/" + t.Format("200601021504") + "_sonobuoy_" + cfg.UUID + ".tar.gz" err = tarx.Compress(tb, outpath, &tarx.CompressOptions{Compression: tarx.Gzip}) if err == nil { diff --git a/pkg/discovery/pods.go b/pkg/discovery/pods.go index a097e2e97..267f1dbd8 100644 --- a/pkg/discovery/pods.go +++ b/pkg/discovery/pods.go @@ -30,9 +30,9 @@ import ( ) const ( - // PodsLocation is the location within the results tarball where pod + // PodLogsLocation is the location within the results tarball where pod // information is stored. - PodsLocation = "pods" + PodLogsLocation = "podlogs" ) // gatherPodLogs will loop through collecting pod logs and placing them into a directory tree @@ -60,7 +60,7 @@ func gatherPodLogs(kubeClient kubernetes.Interface, ns string, opts metav1.ListO return errors.WithStack(err) } - outdir := path.Join(cfg.OutputDir(), NSResourceLocation, ns, PodsLocation, pod.Name, "logs") + outdir := path.Join(cfg.OutputDir(), PodLogsLocation, ns, pod.Name, "logs") if err = os.MkdirAll(outdir, 0755); err != nil { return errors.WithStack(err) } diff --git a/pkg/discovery/queries.go b/pkg/discovery/queries.go index e3f609fd0..02f741dbf 100644 --- a/pkg/discovery/queries.go +++ b/pkg/discovery/queries.go @@ -44,19 +44,14 @@ type UntypedListQuery func() ([]interface{}, error) const ( // NSResourceLocation is the place under which namespaced API resources (pods, etc) are stored NSResourceLocation = "resources/ns" - // NonNSResourceLocation is the place under which non-namespaced API resources (nodes, etc) are stored - NonNSResourceLocation = "resources/non-ns" + // ClusterResourceLocation is the place under which non-namespaced API resources (nodes, etc) are stored + ClusterResourceLocation = "resources/cluster" // HostsLocation is the place under which host information (configz, healthz) is stored HostsLocation = "hosts" + // MetaLocation is the place under which snapshot metadata (query times, config) is stored + MetaLocation = "meta" ) -// queryData captures the results of the run for post-processing -type queryData struct { - QueryObj string `json:"queryobj,omitempty"` - ElapsedTime string `json:"time,omitempty"` - Error error `json:"error,omitempty"` -} - // objListQuery performs a list query and serialize the results func objListQuery(outpath string, file string, f ObjQuery) (time.Duration, error) { start := time.Now() @@ -107,23 +102,10 @@ func untypedListQuery(outpath string, file string, f UntypedListQuery) (time.Dur return duration, err } -// recordResults will write out the execution results of a query. -func recordResults(f *os.File, name string, duration time.Duration, recerr error) error { - summary := &queryData{ - QueryObj: name, - ElapsedTime: duration.String(), - Error: recerr, - } - if err := SerializeObjAppend(f, summary); err != nil { - return err - } - return nil -} - // timedQuery Wraps the execution of the function with a recorded timed snapshot -func timedQuery(f *os.File, name string, fn func() (time.Duration, error)) error { +func timedQuery(recorder *QueryRecorder, name string, ns string, fn func() (time.Duration, error)) { duration, fnErr := fn() - return recordResults(f, name, duration, fnErr) + recorder.RecordQuery(name, ns, duration, fnErr) } // queryNsResource performs the appropriate namespace-scoped query according to its input args @@ -211,7 +193,7 @@ func queryNonNsResource(resourceKind string, kubeClient kubernetes.Interface) (r // QueryNSResources will query namespace-specific resources in the cluster, // writing them out to /resources/ns//*.json // TODO: Eliminate dependencies from config.Config and pass in data -func QueryNSResources(kubeClient kubernetes.Interface, ns string, cfg *config.Config) error { +func QueryNSResources(kubeClient kubernetes.Interface, recorder *QueryRecorder, ns string, cfg *config.Config) error { glog.Infof("Running ns query (%v)", ns) // 1. Create the parent directory we will use to store the results @@ -220,22 +202,7 @@ func QueryNSResources(kubeClient kubernetes.Interface, ns string, cfg *config.Co return errors.WithStack(err) } - // 2. Create the results output file. - f, err := os.Create(outdir + "/results.json") - if err != nil { - return errors.WithStack(err) - } - defer func() { - f.WriteString("{}]") - f.Close() - }() - - _, err = f.WriteString("[") - if err != nil { - return errors.WithStack(err) - } - - // 3. Setup label filter if there is one. + // 2. Setup label filter if there is one. opts := metav1.ListOptions{} if len(cfg.Filters.LabelSelector) > 0 { if _, err := labels.Parse(cfg.Filters.LabelSelector); err != nil { @@ -247,17 +214,14 @@ func QueryNSResources(kubeClient kubernetes.Interface, ns string, cfg *config.Co resources := cfg.FilterResources(config.NamespacedResources) - // 4. Execute the ns-query + // 3. Execute the ns-query for resourceKind := range resources { // We use annotations to tag resources as being namespaced vs not, skip any // that aren't "ns" if resourceKind != "PodLogs" { lister := func() (runtime.Object, error) { return queryNsResource(ns, resourceKind, opts, kubeClient) } query := func() (time.Duration, error) { return objListQuery(outdir+"/", resourceKind+".json", lister) } - err = timedQuery(f, resourceKind, query) - if err != nil { - return err - } + timedQuery(recorder, resourceKind, ns, query) } } @@ -269,7 +233,7 @@ func QueryNSResources(kubeClient kubernetes.Interface, ns string, cfg *config.Co } duration := time.Since(start) - recordResults(f, "podlogs", duration, err) + recorder.RecordQuery("PodLogs", ns, duration, err) } return nil @@ -278,44 +242,26 @@ func QueryNSResources(kubeClient kubernetes.Interface, ns string, cfg *config.Co // QueryClusterResources queries non-namespace resources in the cluster, writing // them out to /resources/non-ns/*.json // TODO: Eliminate dependencies from config.Config and pass in data -func QueryClusterResources(kubeClient kubernetes.Interface, cfg *config.Config) error { +func QueryClusterResources(kubeClient kubernetes.Interface, recorder *QueryRecorder, cfg *config.Config) error { glog.Infof("Running non-ns query") resources := cfg.FilterResources(config.ClusterResources) // 1. Create the parent directory we will use to store the results - outdir := path.Join(cfg.OutputDir(), NonNSResourceLocation) + outdir := path.Join(cfg.OutputDir(), ClusterResourceLocation) if len(resources) > 0 { if err := os.MkdirAll(outdir, 0755); err != nil { return errors.WithStack(err) } } - // 2. Create the results output file. - f, err := os.Create(outdir + "/results.json") - if err != nil { - return errors.WithStack(err) - } - defer func() { - f.WriteString("{}]") - f.Close() - }() - - _, err = f.WriteString("[") - if err != nil { - return errors.WithStack(err) - } - - // 3. Execute the non-ns-query + // 2. Execute the non-ns-query for resourceKind := range resources { // Eliminate special cases. if resourceKind != "ServerVersion" { lister := func() (runtime.Object, error) { return queryNonNsResource(resourceKind, kubeClient) } query := func() (time.Duration, error) { return objListQuery(outdir+"/", resourceKind+".json", lister) } - err = timedQuery(f, resourceKind, query) - if err != nil { - return errors.WithStack(err) - } + timedQuery(recorder, resourceKind, "", query) } } @@ -326,23 +272,17 @@ func QueryClusterResources(kubeClient kubernetes.Interface, cfg *config.Config) // NOTE: Node data collection is an aggregated time b/c propagating that detail back up // is odd and would pollute some of the output. start := time.Now() - gatherErr := gatherNodeData(kubeClient, cfg) + err := gatherNodeData(kubeClient, cfg) duration := time.Since(start) - err = recordResults(f, "podlogs", duration, gatherErr) - if err != nil { - return errors.WithStack(err) - } + recorder.RecordQuery("Nodes", "", duration, err) } if resources["ServerVersion"] { objqry := func() (interface{}, error) { return kubeClient.Discovery().ServerVersion() } query := func() (time.Duration, error) { - return untypedQuery(cfg.OutputDir()+"/serverversion", "serverversion.json", objqry) - } - err = timedQuery(f, "serverversion", query) - if err != nil { - return err + return untypedQuery(cfg.OutputDir(), "serverversion.json", objqry) } + timedQuery(recorder, "serverversion", "", query) } return nil diff --git a/pkg/discovery/queryrecorder.go b/pkg/discovery/queryrecorder.go new file mode 100644 index 000000000..9bd48273d --- /dev/null +++ b/pkg/discovery/queryrecorder.go @@ -0,0 +1,78 @@ +/* +Copyright 2017 Heptio Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "encoding/json" + "os" + "path" + "time" +) + +type QueryRecorder struct { + queries []*queryData +} + +func NewQueryRecorder() *QueryRecorder { + return &QueryRecorder{ + queries: make([]*queryData, 0), + } +} + +// queryData captures the results of the run for post-processing +type queryData struct { + QueryObj string `json:"queryobj,omitempty"` + Namespace string `json:"namespace,omitempty"` + ElapsedTime string `json:"time,omitempty"` + Error error `json:"error,omitempty"` +} + +func (q *QueryRecorder) RecordQuery(name string, namespace string, duration time.Duration, recerr error) { + summary := &queryData{ + QueryObj: name, + Namespace: namespace, + ElapsedTime: duration.String(), + Error: recerr, + } + + q.queries = append(q.queries, summary) +} + +func (q *QueryRecorder) DumpQueryData(filepath string) error { + // Ensure the leading path is created + err := os.MkdirAll(path.Dir(filepath), 0755) + if err != nil { + return err + } + + // Format the query data as JSON + data, err := json.Marshal(q.queries) + if err != nil { + return err + } + + // Create the file + f, err := os.Create(filepath) + if err != nil { + return err + } + defer f.Close() + + // Write the data + _, err = f.Write(data) + return err +}