From 326001d24d36fb52ef7466525a5eabd81c84f245 Mon Sep 17 00:00:00 2001 From: Hamza El-Saawy <84944216+helsaawy@users.noreply.github.com> Date: Mon, 14 Mar 2022 14:02:34 -0400 Subject: [PATCH] Scrubbing annotations from logs (#1324) Updated scrubbing for guest side, and added scrubbing for annotations --- cmd/gcs/main.go | 4 ++ internal/guest/bridge/bridge.go | 15 +++++- internal/log/scrub.go | 46 +++++++++++++------ internal/uvm/create_lcow.go | 4 ++ .../Microsoft/hcsshim/internal/log/scrub.go | 46 +++++++++++++------ .../hcsshim/internal/uvm/create_lcow.go | 4 ++ 6 files changed, 91 insertions(+), 28 deletions(-) diff --git a/cmd/gcs/main.go b/cmd/gcs/main.go index cad6948cd1..0926ef4d4b 100644 --- a/cmd/gcs/main.go +++ b/cmd/gcs/main.go @@ -19,6 +19,7 @@ import ( "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" "github.com/Microsoft/hcsshim/internal/guest/runtime/runc" "github.com/Microsoft/hcsshim/internal/guest/transport" + "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oc" "github.com/cenkalti/backoff/v4" "github.com/containerd/cgroups" @@ -176,6 +177,7 @@ func main() { rootMemReserveBytes := flag.Uint64("root-mem-reserve-bytes", 75*1024*1024, "the amount of memory reserved for the orchestration, the rest will be assigned to containers") gcsMemLimitBytes := flag.Uint64("gcs-mem-limit-bytes", 50*1024*1024, "the maximum amount of memory the gcs can use") disableTimeSync := flag.Bool("disable-time-sync", false, "If true do not run chronyd time synchronization service inside the UVM") + scrubLogs := flag.Bool("scrub-logs", false, "If true, scrub potentially sensitive information from logging") flag.Usage = func() { fmt.Fprintf(os.Stderr, "\nUsage of %s:\n", os.Args[0]) @@ -225,6 +227,8 @@ func main() { logrus.SetLevel(level) + log.SetScrubbing(*scrubLogs) + baseLogPath := "/run/gcs/c" logrus.Info("GCS started") diff --git a/internal/guest/bridge/bridge.go b/internal/guest/bridge/bridge.go index 2d2e56c768..6b371950b0 100644 --- a/internal/guest/bridge/bridge.go +++ b/internal/guest/bridge/bridge.go @@ -301,8 +301,19 @@ func (b *Bridge) ListenAndServe(bridgeIn io.ReadCloser, bridgeOut io.WriteCloser trace.StringAttribute("activityID", base.ActivityID), trace.StringAttribute("cid", base.ContainerID)) - log.G(ctx).WithField("message", string(message)).Debug("request read message") - + entry := log.G(ctx) + if entry.Logger.GetLevel() >= logrus.DebugLevel { + s := string(message) + switch header.Type { + case prot.ComputeSystemCreateV1: + b, err := log.ScrubBridgeCreate(message) + s = string(b) + if err != nil { + entry.WithError(err).Warning("could not scrub bridge payload") + } + } + entry.WithField("message", s).Debug("request read message") + } requestChan <- &Request{ Context: ctx, Header: header, diff --git a/internal/log/scrub.go b/internal/log/scrub.go index f66913941f..b906a16bb4 100644 --- a/internal/log/scrub.go +++ b/internal/log/scrub.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "strings" "sync/atomic" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" @@ -14,7 +15,7 @@ import ( type genMap = map[string]interface{} type scrubberFunc func(genMap) error -const ScrubbedReplacement = "" +const _scrubbedReplacement = "" var ( ErrUnknownType = errors.New("encoded object is of unknown type") @@ -53,32 +54,50 @@ func ScrubProcessParameters(s string) (string, error) { if err := json.Unmarshal(b, &pp); err != nil { return "", err } - pp.Environment = map[string]string{ScrubbedReplacement: ScrubbedReplacement} + pp.Environment = map[string]string{_scrubbedReplacement: _scrubbedReplacement} buf := bytes.NewBuffer(b[:0]) if err := encode(buf, pp); err != nil { return "", err } - return buf.String(), nil + return strings.TrimSpace(s), nil } // ScrubBridgeCreate scrubs requests sent over the bridge of type // internal/gcs/protocol.containerCreate wrapping an internal/hcsoci.linuxHostedSystem func ScrubBridgeCreate(b []byte) ([]byte, error) { - return scrubBytes(b, scrubLinuxHostedSystem) + return scrubBytes(b, scrubBridgeCreate) } -func scrubLinuxHostedSystem(m genMap) error { +func scrubBridgeCreate(m genMap) error { if !isRequestBase(m) { return ErrUnknownType } - if m, ok := index(m, "ContainerConfig"); ok { - if m, ok := index(m, "OciSpecification"); ok { - if m, ok := index(m, "process"); ok { - if _, ok := m["env"]; ok { - m["env"] = []string{ScrubbedReplacement} - return nil - } + if ss, ok := m["ContainerConfig"]; ok { + // ContainerConfig is a json encoded struct passed as a regular string field + s, ok := ss.(string) + if !ok { + return ErrUnknownType + } + b, err := scrubBytes([]byte(s), scrubLinuxHostedSystem) + if err != nil { + return err + } + m["ContainerConfig"] = string(b) + return nil + } + return ErrUnknownType +} + +func scrubLinuxHostedSystem(m genMap) error { + if m, ok := index(m, "OciSpecification"); ok { + if _, ok := m["annotations"]; ok { + m["annotations"] = map[string]string{_scrubbedReplacement: _scrubbedReplacement} + } + if m, ok := index(m, "process"); ok { + if _, ok := m["env"]; ok { + m["env"] = []string{_scrubbedReplacement} + return nil } } } @@ -135,7 +154,8 @@ func scrubBytes(b []byte, scrub scrubberFunc) ([]byte, error) { if err := encode(buf, m); err != nil { return nil, err } - return buf.Bytes(), nil + + return bytes.TrimSpace(buf.Bytes()), nil } func encode(buf *bytes.Buffer, v interface{}) error { diff --git a/internal/uvm/create_lcow.go b/internal/uvm/create_lcow.go index 1211f78ceb..e98b0f0690 100644 --- a/internal/uvm/create_lcow.go +++ b/internal/uvm/create_lcow.go @@ -657,6 +657,10 @@ func makeLCOWDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ *hcs opts.ExecCommandLine = fmt.Sprintf("%s --disable-time-sync", opts.ExecCommandLine) } + if log.IsScrubbingEnabled() { + opts.ExecCommandLine += " --scrub-logs" + } + initArgs += " " + opts.ExecCommandLine if opts.ProcessDumpLocation != "" { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/log/scrub.go b/test/vendor/github.com/Microsoft/hcsshim/internal/log/scrub.go index f66913941f..b906a16bb4 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/log/scrub.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/log/scrub.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "strings" "sync/atomic" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" @@ -14,7 +15,7 @@ import ( type genMap = map[string]interface{} type scrubberFunc func(genMap) error -const ScrubbedReplacement = "" +const _scrubbedReplacement = "" var ( ErrUnknownType = errors.New("encoded object is of unknown type") @@ -53,32 +54,50 @@ func ScrubProcessParameters(s string) (string, error) { if err := json.Unmarshal(b, &pp); err != nil { return "", err } - pp.Environment = map[string]string{ScrubbedReplacement: ScrubbedReplacement} + pp.Environment = map[string]string{_scrubbedReplacement: _scrubbedReplacement} buf := bytes.NewBuffer(b[:0]) if err := encode(buf, pp); err != nil { return "", err } - return buf.String(), nil + return strings.TrimSpace(s), nil } // ScrubBridgeCreate scrubs requests sent over the bridge of type // internal/gcs/protocol.containerCreate wrapping an internal/hcsoci.linuxHostedSystem func ScrubBridgeCreate(b []byte) ([]byte, error) { - return scrubBytes(b, scrubLinuxHostedSystem) + return scrubBytes(b, scrubBridgeCreate) } -func scrubLinuxHostedSystem(m genMap) error { +func scrubBridgeCreate(m genMap) error { if !isRequestBase(m) { return ErrUnknownType } - if m, ok := index(m, "ContainerConfig"); ok { - if m, ok := index(m, "OciSpecification"); ok { - if m, ok := index(m, "process"); ok { - if _, ok := m["env"]; ok { - m["env"] = []string{ScrubbedReplacement} - return nil - } + if ss, ok := m["ContainerConfig"]; ok { + // ContainerConfig is a json encoded struct passed as a regular string field + s, ok := ss.(string) + if !ok { + return ErrUnknownType + } + b, err := scrubBytes([]byte(s), scrubLinuxHostedSystem) + if err != nil { + return err + } + m["ContainerConfig"] = string(b) + return nil + } + return ErrUnknownType +} + +func scrubLinuxHostedSystem(m genMap) error { + if m, ok := index(m, "OciSpecification"); ok { + if _, ok := m["annotations"]; ok { + m["annotations"] = map[string]string{_scrubbedReplacement: _scrubbedReplacement} + } + if m, ok := index(m, "process"); ok { + if _, ok := m["env"]; ok { + m["env"] = []string{_scrubbedReplacement} + return nil } } } @@ -135,7 +154,8 @@ func scrubBytes(b []byte, scrub scrubberFunc) ([]byte, error) { if err := encode(buf, m); err != nil { return nil, err } - return buf.Bytes(), nil + + return bytes.TrimSpace(buf.Bytes()), nil } func encode(buf *bytes.Buffer, v interface{}) error { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_lcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_lcow.go index 1211f78ceb..e98b0f0690 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_lcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_lcow.go @@ -657,6 +657,10 @@ func makeLCOWDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ *hcs opts.ExecCommandLine = fmt.Sprintf("%s --disable-time-sync", opts.ExecCommandLine) } + if log.IsScrubbingEnabled() { + opts.ExecCommandLine += " --scrub-logs" + } + initArgs += " " + opts.ExecCommandLine if opts.ProcessDumpLocation != "" {