Scrubbing annotations from logs (#1324)

Updated scrubbing for guest side, and added scrubbing for annotations

cmd/gcs/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2"
 	"github.com/Microsoft/hcsshim/internal/guest/runtime/runc"
 	"github.com/Microsoft/hcsshim/internal/guest/transport"
+	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/cenkalti/backoff/v4"
 	"github.com/containerd/cgroups"
@@ -176,6 +177,7 @@ func main() {
 	rootMemReserveBytes := flag.Uint64("root-mem-reserve-bytes", 75*1024*1024, "the amount of memory reserved for the orchestration, the rest will be assigned to containers")
 	gcsMemLimitBytes := flag.Uint64("gcs-mem-limit-bytes", 50*1024*1024, "the maximum amount of memory the gcs can use")
 	disableTimeSync := flag.Bool("disable-time-sync", false, "If true do not run chronyd time synchronization service inside the UVM")
+	scrubLogs := flag.Bool("scrub-logs", false, "If true, scrub potentially sensitive information from logging")
 
 	flag.Usage = func() {
 		fmt.Fprintf(os.Stderr, "\nUsage of %s:\n", os.Args[0])
@@ -225,6 +227,8 @@ func main() {
 
 	logrus.SetLevel(level)
 
+	log.SetScrubbing(*scrubLogs)
+
 	baseLogPath := "/run/gcs/c"
 
 	logrus.Info("GCS started")

internal/guest/bridge/bridge.go

@@ -301,8 +301,19 @@ func (b *Bridge) ListenAndServe(bridgeIn io.ReadCloser, bridgeOut io.WriteCloser
 					trace.StringAttribute("activityID", base.ActivityID),
 					trace.StringAttribute("cid", base.ContainerID))
 
-				log.G(ctx).WithField("message", string(message)).Debug("request read message")
-
+				entry := log.G(ctx)
+				if entry.Logger.GetLevel() >= logrus.DebugLevel {
+					s := string(message)
+					switch header.Type {
+					case prot.ComputeSystemCreateV1:
+						b, err := log.ScrubBridgeCreate(message)
+						s = string(b)
+						if err != nil {
+							entry.WithError(err).Warning("could not scrub bridge payload")
+						}
+					}
+					entry.WithField("message", s).Debug("request read message")
+				}
 				requestChan <- &Request{
 					Context:     ctx,
 					Header:      header,

internal/log/scrub.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"strings"
 	"sync/atomic"
 
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
@@ -14,7 +15,7 @@ import (
 type genMap = map[string]interface{}
 type scrubberFunc func(genMap) error
 
-const ScrubbedReplacement = "<scrubbed>"
+const _scrubbedReplacement = "<scrubbed>"
 
 var (
 	ErrUnknownType = errors.New("encoded object is of unknown type")
@@ -53,32 +54,50 @@ func ScrubProcessParameters(s string) (string, error) {
 	if err := json.Unmarshal(b, &pp); err != nil {
 		return "", err
 	}
-	pp.Environment = map[string]string{ScrubbedReplacement: ScrubbedReplacement}
+	pp.Environment = map[string]string{_scrubbedReplacement: _scrubbedReplacement}
 
 	buf := bytes.NewBuffer(b[:0])
 	if err := encode(buf, pp); err != nil {
 		return "", err
 	}
-	return buf.String(), nil
+	return strings.TrimSpace(s), nil
 }
 
 // ScrubBridgeCreate scrubs requests sent over the bridge of type
 // internal/gcs/protocol.containerCreate wrapping an internal/hcsoci.linuxHostedSystem
 func ScrubBridgeCreate(b []byte) ([]byte, error) {
-	return scrubBytes(b, scrubLinuxHostedSystem)
+	return scrubBytes(b, scrubBridgeCreate)
 }
 
-func scrubLinuxHostedSystem(m genMap) error {
+func scrubBridgeCreate(m genMap) error {
 	if !isRequestBase(m) {
 		return ErrUnknownType
 	}
-	if m, ok := index(m, "ContainerConfig"); ok {
-		if m, ok := index(m, "OciSpecification"); ok {
-			if m, ok := index(m, "process"); ok {
-				if _, ok := m["env"]; ok {
-					m["env"] = []string{ScrubbedReplacement}
-					return nil
-				}
+	if ss, ok := m["ContainerConfig"]; ok {
+		// ContainerConfig is a json encoded struct passed as a regular string field
+		s, ok := ss.(string)
+		if !ok {
+			return ErrUnknownType
+		}
+		b, err := scrubBytes([]byte(s), scrubLinuxHostedSystem)
+		if err != nil {
+			return err
+		}
+		m["ContainerConfig"] = string(b)
+		return nil
+	}
+	return ErrUnknownType
+}
+
+func scrubLinuxHostedSystem(m genMap) error {
+	if m, ok := index(m, "OciSpecification"); ok {
+		if _, ok := m["annotations"]; ok {
+			m["annotations"] = map[string]string{_scrubbedReplacement: _scrubbedReplacement}
+		}
+		if m, ok := index(m, "process"); ok {
+			if _, ok := m["env"]; ok {
+				m["env"] = []string{_scrubbedReplacement}
+				return nil
 			}
 		}
 	}
@@ -135,7 +154,8 @@ func scrubBytes(b []byte, scrub scrubberFunc) ([]byte, error) {
 	if err := encode(buf, m); err != nil {
 		return nil, err
 	}
-	return buf.Bytes(), nil
+
+	return bytes.TrimSpace(buf.Bytes()), nil
 }
 
 func encode(buf *bytes.Buffer, v interface{}) error {

internal/uvm/create_lcow.go

@@ -657,6 +657,10 @@ func makeLCOWDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ *hcs
 		opts.ExecCommandLine = fmt.Sprintf("%s --disable-time-sync", opts.ExecCommandLine)
 	}
 
+	if log.IsScrubbingEnabled() {
+		opts.ExecCommandLine += " --scrub-logs"
+	}
+
 	initArgs += " " + opts.ExecCommandLine
 
 	if opts.ProcessDumpLocation != "" {