Issue #47: Collect files opened by envoy pre-termination (lsof) (#68)

* create function to write iostats to file

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* rename variables

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* adjust test file to test for presence of iostats.json

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* standardize output of iostats.json file to be an array of objects
- same format as output of networkInterfaces: allows standardized testing

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* ignore golint: rangeValCopy

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* handle error

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* change variable name to start with smaller letters

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* fix lint error

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* avoid copying of objects in loop

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* create initial files

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* cleanup

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* enable OpenFilesDataCollection

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* unit test for lsof.go

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* resolve dirty commits

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* resolve dirty commit

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* rename variables

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* remove govet nolint

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* Create and expose the method to get pid of the child process

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* get pid of envoy from Runtime instance method
- no longer needs to search through all processes to find the envoy
instance

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

* remove directive to ignore lint

Signed-off-by: Hengyuan <jhy8890948@gmail.com>

pkg/binary/envoy/debug/lsof.go

@@ -0,0 +1,115 @@
+// Copyright 2019 Tetrate
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debug
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/shirou/gopsutil/process"
+
+	"github.com/tetratelabs/getenvoy/pkg/binary"
+	"github.com/tetratelabs/getenvoy/pkg/binary/envoy"
+	"github.com/tetratelabs/log"
+)
+
+// OpenFileStat defines the structure of statistics about a single opened file
+type OpenFileStat struct {
+	// string enclosed in `` are known as struct tags
+	// this particular tag is used by json.Marshal() to encode Command field in a specific manner
+	// process dependent information
+	Command string `json:"command"`
+	Pid     string `json:"pid"`
+	User    string `json:"user"`
+	Fd      string `json:"fd"`
+	// non process dependent information
+	Type   string `json:"type"`   // type of node associated with the file / FIFO for FIFO special file
+	Device string `json:"device"` // device numbers associated with the file, separated by commas
+	Size   string `json:"size"`
+	Node   string `json:"node"` // inode number of a local file/ Internet protocol type
+	Name   string `json:"name"` // name of the mount point and file system on which the file resides
+}
+
+// EnableOpenFilesDataCollection is a preset option that registers collection of statistics of files opened by envoy instance(s)
+func EnableOpenFilesDataCollection(r *envoy.Runtime) {
+	if err := os.Mkdir(filepath.Join(r.DebugStore(), "lsof"), os.ModePerm); err != nil {
+		log.Errorf("error in creating a directory to write open file data of envoy to: %v", err)
+	}
+	r.RegisterPreTermination(retrieveOpenFilesData)
+}
+
+// retrieveOpenFilesData writes statistics of open files associated with envoy instance(s) to a json file
+// if succeeded, return nil, else return an error instance
+func retrieveOpenFilesData(r binary.Runner) error {
+	// get pid of envoy instance
+	pid, err := r.GetPid()
+	if err != nil {
+		return fmt.Errorf("error in getting pid of envoy instance: %v", err)
+	}
+	envoyProcess, err := process.NewProcess(int32(pid))
+	if err != nil {
+		return fmt.Errorf("error in creating an envoy instance: %v", err)
+	}
+
+	f, err := os.Create(filepath.Join(r.DebugStore(), "lsof/lsof.json"))
+	if err != nil {
+		return fmt.Errorf("error in creating a file to write open file statistics to: %v", err)
+	}
+	defer f.Close() //nolint
+
+	result := make([]OpenFileStat, 0)
+	// print open file stats for all envoy instances
+	// relevant fields of the process
+	username, _ := envoyProcess.Username()
+	name, _ := envoyProcess.Name()
+
+	openFiles, err := envoyProcess.OpenFiles()
+	if err != nil {
+		return fmt.Errorf("error in getting open file statistics: %v", err)
+	}
+
+	for _, stat := range openFiles {
+		ofStat := OpenFileStat{
+			Command: name,
+			Pid:     fmt.Sprint(pid),
+			User:    username,
+			Fd:      fmt.Sprint(stat.Fd),
+			Name:    stat.Path,
+		}
+
+		var fstat syscall.Stat_t
+		if errSyscall := syscall.Stat(stat.Path, &fstat); errSyscall != nil {
+			// continue if the path is invalid
+			result = append(result, ofStat)
+			continue
+		}
+
+		ofStat.Node = fmt.Sprint(fstat.Ino)
+		ofStat.Size = fmt.Sprint(fstat.Size)
+		result = append(result, ofStat)
+	}
+
+	out, err := json.Marshal(result)
+	if err != nil {
+		return fmt.Errorf("unable to convert to json representation: %v", err)
+	}
+
+	fmt.Fprintln(f, string(out))
+
+	return nil
+}

pkg/binary/envoy/debug/lsof_test.go

@@ -0,0 +1,63 @@
+// Copyright 2019 Tetrate
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debug
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/tetratelabs/getenvoy/pkg/binary/envoy"
+	"github.com/tetratelabs/getenvoy/pkg/binary/envoytest"
+)
+
+func TestGetOpenFileStats(t *testing.T) {
+	t.Run("creates non-empty files", func(t *testing.T) {
+		r, _ := envoy.NewRuntime(EnableOpenFilesDataCollection)
+		defer os.RemoveAll(r.DebugStore() + ".tar.gz")
+		defer os.RemoveAll(r.DebugStore())
+		envoytest.RunKill(r, filepath.Join("testdata", "null.yaml"), time.Second*10)
+
+		files := [...]string{"lsof/lsof.json"}
+
+		for _, file := range files {
+			path := filepath.Join(r.DebugStore(), file)
+			f, err := os.Stat(path)
+			if err != nil {
+				t.Errorf("error stating %v: %v", path, err)
+			}
+			if f.Size() < 1 {
+				t.Errorf("file %v was empty", path)
+			}
+			if strings.HasSuffix(file, ".json") {
+				raw, err := ioutil.ReadFile(path)
+				if err != nil {
+					t.Errorf("error to read the file %v: %v", path, err)
+				}
+				var is []interface{}
+				if err := json.Unmarshal(raw, &is); err != nil {
+					t.Errorf("error to unmarshal json string, %v: \"%v\"", err, raw)
+				}
+				if len(is) < 1 {
+					t.Errorf("unmarshaled content is empty, expected to be a non-empty array: \"%v\"", raw)
+				}
+			}
+		}
+	})
+}

pkg/binary/envoy/runtime.go

@@ -91,6 +91,14 @@ func (r *Runtime) Status() int {
 	}
 }
 
+// GetPid returns the pid of the child process
+func (r *Runtime) GetPid() (int, error) {
+	if r.cmd == nil || r.cmd.Process == nil {
+		return 0, fmt.Errorf("envoy process not yet started")
+	}
+	return r.cmd.Process.Pid, nil
+}
+
 func (r *Runtime) envoyReady() bool {
 	// Once we have seen its ready once stop spamming the ready endpoint.
 	// If we expand the interface to support ready <-> not ready then

pkg/binary/interface.go

@@ -32,6 +32,7 @@ type Runner interface {
 	RegisterDone()
 	SendSignal(signal os.Signal)
 	Status() int
+	GetPid() (int, error)
 	AppendArgs([]string)
 	Wait(int)
 	WaitWithContext(context.Context, int)

pkg/cmd/run.go

@@ -80,6 +80,7 @@ getenvoy run standard:1.11.1 -- --help
 				debug.EnableEnvoyAdminDataCollection,
 				debug.EnableEnvoyLogCollection,
 				debug.EnableNodeCollection,
+				debug.EnableOpenFilesDataCollection,
 				controlplaneFunc(),
 			)
 			if err != nil {