Merged PR 5137749: update hcsshim to 057bebe

Related work items: microsoft#1062, microsoft#1087, microsoft#1089, microsoft#1095, microsoft#1104, microsoft#1112, microsoft#1117, microsoft#1118, microsoft#1125, microsoft#1137, microsoft#1139, microsoft#1140, microsoft#1141, microsoft#1142, microsoft#1143, microsoft#1145, microsoft#1146, microsoft#1150, microsoft#1151, microsoft#1153, microsoft#1154, microsoft#1155, microsoft#1156, microsoft#1157, microsoft#1158, microsoft#1159, microsoft#1161, microsoft#1162, microsoft#1163, microsoft#1164, microsoft#1165, microsoft#1166, microsoft#1167, microsoft#1168, microsoft#1169, microsoft#1171, microsoft#1172, microsoft#1173, microsoft#1174, microsoft#1178

.github/workflows/ci.yml

@@ -4,8 +4,8 @@ on:
   - pull_request
 
 env:
-  GOPROXY: off
   GOFLAGS: -mod=vendor
+  GOPROXY: off
 
 jobs:
   lint:
@@ -19,8 +19,46 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.38.0
-          args: --timeout=5m
+          version: v1.42.0 # Has fixes for stylecheck configuration https://github.com/golangci/golangci-lint/pull/2017/files
+          args: --timeout=5m -v
+
+  verify-main-vendor:
+    runs-on: 'windows-2019'
+    env:
+      GOPROXY: "https://proxy.golang.org,direct"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.15.0'
+      - name: Validate main modules
+        shell: powershell
+        run: |
+          $currentPath = (Get-Location).Path
+          $process = Start-Process powershell.exe -PassThru -Verb runAs -Wait -ArgumentList $currentPath/scripts/Verify-GoModules.ps1, $currentPath
+          if ($process.ExitCode -ne 0) {
+            Write-Error "Main modules are not up to date. Please validate your go version >= this job's and run `go mod vendor` followed by `go mod tidy` in the repo root path."
+          }
+          exit $process.ExitCode
+          
+  verify-test-vendor:
+    runs-on: 'windows-2019'
+    env:
+      GOPROXY: "https://proxy.golang.org,direct"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.15.0'
+      - name: Validate test modules
+        shell: powershell
+        run: |
+          $currentPath = (Get-Location).Path
+          $process = Start-Process powershell.exe -PassThru -Verb runAs -Wait -ArgumentList $currentPath/scripts/Verify-GoModules.ps1, $currentPath, "test"
+          if ($process.ExitCode -ne 0) {
+            Write-Error "Test package modules are not up to date. Please validate your go version >= this job's and run `go mod vendor` followed by `go mod tidy` in hcsshim/test directory."
+          }
+          exit $process.ExitCode
 
   test:
     runs-on: 'windows-2019'

.golangci.yml

@@ -0,0 +1,96 @@
+linters:
+  enable:
+    - stylecheck
+
+linters-settings:
+  stylecheck:
+    # https://staticcheck.io/docs/checks
+    checks: ["all"]
+
+
+issues:
+  # This repo has a LOT of generated schema files, operating system bindings, and other things that ST1003 from stylecheck won't like
+  # (screaming case Windows api constants for example). There's also some structs that we *could* change the initialisms to be Go
+  # friendly (Id -> ID) but they're exported and it would be a breaking change. This makes it so that most new code, code that isn't
+  # supposed to be a pretty faithful mapping to an OS call/constants, or non-generated code still checks if we're following idioms,
+  # while ignoring the things that are just noise or would be more of a hassle than it'd be worth to change.
+  exclude-rules:
+    - path: layer.go
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: hcsshim.go
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\hcs\\schema2\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\wclayer\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: hcn\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\hcs\\schema1\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\hns\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: ext4\\internal\\compactext4\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: ext4\\internal\\format\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\guestrequest\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\guest\\prot\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\windevice\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\winapi\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\vmcompute\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\regstate\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"
+
+    - path: internal\\hcserror\\
+      linters:
+        - stylecheck
+      Text: "ST1003:"

README.md

@@ -62,15 +62,35 @@ C:\> ctr.exe run --runtime io.containerd.runhcs.v1 --rm mcr.microsoft.com/window
 
 ## Contributing
 
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+This project welcomes contributions and suggestions. Most contributions require you to agree to a
 Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
 the rights to use your contribution. For details, visit https://cla.microsoft.com.
 
 When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
 a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
 provided by the bot. You will only need to do this once across all repos using our CLA.
 
-We also ask that contributors [sign their commits](https://git-scm.com/docs/git-commit) using `git commit -s` or `git commit --signoff` to certify they either authored the work themselves or otherwise have permission to use it in this project.
+We also require that contributors [sign their commits](https://git-scm.com/docs/git-commit) using `git commit -s` or `git commit --signoff` to
+certify they either authored the work themselves or otherwise have permission to use it in this project. Please see https://developercertificate.org/ for
+more info, as well as to make sure that you can attest to the rules listed. Our CI uses the [DCO Github app](https://github.com/apps/dco) to ensure
+that all commits in a given PR are signed-off.
+
+### Test Directory (Important to note)
+
+This project has tried to trim some dependencies from the root Go modules file that would be cumbersome to get transitively included if this
+project is being vendored/used as a library. Some of these dependencies were only being used for tests, so the /test directory in this project also has
+its own go.mod file where these are now included to get around this issue. Our tests rely on the code in this project to run, so the test Go modules file
+has a relative path replace directive to pull in the latest hcsshim code that the tests actually touch from this project
+(which is the repo itself on your disk).
+
+```
+replace (
+	github.com/Microsoft/hcsshim => ../
+)
+```
+
+Because of this, for most code changes you may need to run `go mod vendor` + `go mod tidy` in the /test directory in this repository, as the
+CI in this project will check if the files are out of date and will fail if this is true.
 
 
 ## Code of Conduct

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -18,7 +18,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 
-	"github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
 	runhcsopts "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
 	"github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
 	"github.com/Microsoft/hcsshim/internal/cmd"
@@ -587,7 +586,7 @@ func (ht *hcsTask) DeleteExec(ctx context.Context, eid string) (int, uint32, tim
 	return int(status.Pid), status.ExitStatus, status.ExitedAt, nil
 }
 
-func (ht *hcsTask) Pids(ctx context.Context) ([]options.ProcessDetails, error) {
+func (ht *hcsTask) Pids(ctx context.Context) ([]runhcsopts.ProcessDetails, error) {
 	// Map all user created exec's to pid/exec-id
 	pidMap := make(map[int]string)
 	ht.execs.Range(func(key, value interface{}) bool {
@@ -606,7 +605,7 @@ func (ht *hcsTask) Pids(ctx context.Context) ([]options.ProcessDetails, error) {
 	}
 
 	// Copy to pid/exec-id pair's
-	pairs := make([]options.ProcessDetails, len(props.ProcessList))
+	pairs := make([]runhcsopts.ProcessDetails, len(props.ProcessList))
 	for i, p := range props.ProcessList {
 		pairs[i].ImageName = p.ImageName
 		pairs[i].CreatedAt = p.CreateTimestamp
@@ -790,10 +789,19 @@ func (ht *hcsTask) closeHost(ctx context.Context) {
 }
 
 func (ht *hcsTask) ExecInHost(ctx context.Context, req *shimdiag.ExecProcessRequest) (int, error) {
+	cmdReq := &cmd.CmdProcessRequest{
+		Args:     req.Args,
+		Workdir:  req.Workdir,
+		Terminal: req.Terminal,
+		Stdin:    req.Stdin,
+		Stdout:   req.Stdout,
+		Stderr:   req.Stderr,
+	}
+
 	if ht.host == nil {
-		return cmd.ExecInShimHost(ctx, req)
+		return cmd.ExecInShimHost(ctx, cmdReq)
 	}
-	return cmd.ExecInUvm(ctx, ht.host, req)
+	return cmd.ExecInUvm(ctx, ht.host, cmdReq)
 }
 
 func (ht *hcsTask) DumpGuestStacks(ctx context.Context) string {

cmd/containerd-shim-runhcs-v1/task_wcow_podsandbox.go

@@ -240,10 +240,18 @@ func (wpst *wcowPodSandboxTask) waitParentExit() {
 }
 
 func (wpst *wcowPodSandboxTask) ExecInHost(ctx context.Context, req *shimdiag.ExecProcessRequest) (int, error) {
+	cmdReq := &cmd.CmdProcessRequest{
+		Args:     req.Args,
+		Workdir:  req.Workdir,
+		Terminal: req.Terminal,
+		Stdin:    req.Stdin,
+		Stdout:   req.Stdout,
+		Stderr:   req.Stderr,
+	}
 	if wpst.host == nil {
 		return 0, errTaskNotIsolated
 	}
-	return cmd.ExecInUvm(ctx, wpst.host, req)
+	return cmd.ExecInUvm(ctx, wpst.host, cmdReq)
 }
 
 func (wpst *wcowPodSandboxTask) DumpGuestStacks(ctx context.Context) string {

cmd/dmverity-vhd/main.go

@@ -27,7 +27,7 @@ const (
 	imageFlag     = "image"
 	verboseFlag   = "verbose"
 	outputDirFlag = "out-dir"
-	maxVHDSize    = 128 * 1024 * 1024 * 1024
+	maxVHDSize    = dmverity.RecommendedVHDSizeGB
 )
 
 func init() {

cmd/gcs/main.go

@@ -84,6 +84,7 @@ func readMemoryEvents(startTime time.Time, efdFile *os.File, cgName string, thre
 func main() {
 	startTime := time.Now()
 	logLevel := flag.String("loglevel", "debug", "Logging Level: debug, info, warning, error, fatal, panic.")
+	coreDumpLoc := flag.String("core-dump-location", "", "The location/format where process core dumps will be written to.")
 	kmsgLogLevel := flag.Uint("kmsgLogLevel", uint(kmsg.Warning), "Log all kmsg entries with a priority less than or equal to the supplied level.")
 	logFile := flag.String("logfile", "", "Logging Target: An optional file name/path. Omit for console output.")
 	logFormat := flag.String("log-format", "text", "Logging Format: text or json")
@@ -144,6 +145,19 @@ func main() {
 
 	logrus.Info("GCS started")
 
+	// Set the process core dump location. This will be global to all containers as it's a kernel configuration.
+	// If no path is specified core dumps will just be placed in the working directory of wherever the process
+	// was invoked to a file named "core".
+	if *coreDumpLoc != "" {
+		if err := ioutil.WriteFile(
+			"/proc/sys/kernel/core_pattern",
+			[]byte(*coreDumpLoc),
+			0644,
+		); err != nil {
+			logrus.WithError(err).Fatal("failed to set core dump location")
+		}
+	}
+
 	// Continuously log /dev/kmsg
 	go kmsg.ReadForever(kmsg.LogLevel(*kmsgLogLevel))
 

cmd/gcstools/commoncli/common.go

@@ -20,7 +20,7 @@ func SetFlagsForLogging() []*string {
 // SetupLogging creates the logger from the command line parameters.
 func SetupLogging(args ...*string) error {
 	if len(args) < 1 {
-		return fmt.Errorf("Invalid log params")
+		return fmt.Errorf("invalid log params")
 	}
 	level, err := logrus.ParseLevel(*args[1])
 	if err != nil {