Add crio compatibility to WorkloadAttestor k8s - fixes #3183

pkg/agent/plugin/workloadattestor/k8s/k8s.go

@@ -202,22 +202,31 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque
  return nil, err
  }
 
+ var attestResponse *workloadattestorv1.AttestResponse
  for _, item := range list.Items {
  item := item
  if isNotPod(item.UID, podUID) {
  continue
  }
 
- status, lookup := lookUpContainerInPod(containerID, item.Status)
+ lookupStatus, lookup := lookUpContainerInPod(containerID, item.Status)
  switch lookup {
  case containerInPod:
- return &workloadattestorv1.AttestResponse{
- SelectorValues: getSelectorValuesFromPodInfo(&item, status),
- }, nil
+ if attestResponse != nil {
+ log.Warn("Two pods found with same container Id")
+ return nil, status.Error(codes.Internal, "two pods found with same container Id")
+ }
+ attestResponse = &workloadattestorv1.AttestResponse{
+ SelectorValues: getSelectorValuesFromPodInfo(&item, lookupStatus),
+ }
  case containerNotInPod:
  }
  }
 
+ if attestResponse != nil {
+ return attestResponse, nil
+ }
+
  // if the container was not located after the maximum number of attempts then the search is over.
  if attempt >= config.MaxPollAttempts {
  log.Warn("Container id not found; giving up")

pkg/agent/plugin/workloadattestor/k8s/k8s_posix.go

@@ -4,6 +4,7 @@
 package k8s
 
 import (
+ "log"
  "regexp"
  "strings"
  "unicode"
@@ -69,22 +70,64 @@ func getPodUIDAndContainerIDFromCGroups(cgroups []cgroups.Cgroup) (types.UID, st
  return podUID, containerID, nil
 }
 
-// cgroupRE is the regex used to parse out the pod UID and container ID from a
-// cgroup name. It assumes that any ".scope" suffix has been trimmed off
-// beforehand. CAUTION: we used to verify that the pod and container id were
-// descendants of a kubepods directory, however, as of Kubernetes 1.21, cgroups
-// namespaces are in use and therefore we can no longer discern if that is the
-// case from within SPIRE agent container (since the container itself is
-// namespaced). As such, the regex has been relaxed to simply find the pod UID
-// followed by the container ID with allowances for arbitrary punctuation, and
-// container runtime prefixes, etc.
-var cgroupRE = regexp.MustCompile(`` +
- // "pod"-prefixed Pod UID (with punctuation separated groups) followed by punctuation
- `[[:punct:]]pod([[:xdigit:]]{8}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{12})[[:punct:]]` +
- // zero or more punctuation separated "segments" (e.g. "docker-")
- `(?:[[:^punct:]]+[[:punct:]])*` +
- // non-punctuation end of string, i.e., the container ID
- `([[:^punct:]]+)$`)
+// regexes listed here have to exlusively match a cgroup path
+// the regexes must include two named groups "poduid" and "containerid"
+// if the regex needs to exclude certain substrings, the "mustnotmatch" group can be used
+var cgroupREs = []*regexp.Regexp{
+ // the regex used to parse out the pod UID and container ID from a
+ // cgroup name. It assumes that any ".scope" suffix has been trimmed off
+ // beforehand. CAUTION: we used to verify that the pod and container id were
+ // descendants of a kubepods directory, however, as of Kubernetes 1.21, cgroups
+ // namespaces are in use and therefore we can no longer discern if that is the
+ // case from within SPIRE agent container (since the container itself is
+ // namespaced). As such, the regex has been relaxed to simply find the pod UID
+ // followed by the container ID with allowances for arbitrary punctuation, and
+ // container runtime prefixes, etc.
+ regexp.MustCompile(`` +
+ // "pod"-prefixed Pod UID (with punctuation separated groups) followed by punctuation
+ `[[:punct:]]pod(?P<poduid>[[:xdigit:]]{8}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{12})[[:punct:]]` +
+ // zero or more punctuation separated "segments" (e.g. "docker-")
+ `(?:[[:^punct:]]+[[:punct:]])*` +
+ // non-punctuation end of string, i.e., the container ID
+ `(?P<containerid>[[:^punct:]]+)$`),
+
+ // This regex applies for container runtimes, that won't put the PodUID into
+ // the cgroup name.
+ // Currently only cri-o in combination with kubeedge is known for this abnormally.
+ regexp.MustCompile(`` +
+ // intentionally empty poduid group
+ `(?P<poduid>)` +
+ // mustnotmatch group: cgroup path must not include a poduid
+ `(?P<mustnotmatch>pod[[:xdigit:]]{8}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{4}[[:punct:]]?[[:xdigit:]]{12}[[:punct:]])?` +
+ // /crio-
+ `(?:[[:^punct:]]*/*)*crio[[:punct:]]` +
+ // non-punctuation end of string, i.e., the container ID
+ `(?P<containerid>[[:^punct:]]+)$`),
+}
+
+func reSubMatchMap(r *regexp.Regexp, str string) map[string]string {
+ match := r.FindStringSubmatch(str)
+ if match == nil {
+ return nil
+ }
+ subMatchMap := make(map[string]string)
+ for i, name := range r.SubexpNames() {
+ if i != 0 {
+ subMatchMap[name] = match[i]
+ }
+ }
+ return subMatchMap
+}
+
+func isValidCGroupPathMatches(matches map[string]string) bool {
+ if matches == nil {
+ return false
+ }
+ if matches["mustnotmatch"] != "" {
+ return false
+ }
+ return true
+}
 
 func getPodUIDAndContainerIDFromCGroupPath(cgroupPath string) (types.UID, string, bool) {
  // We are only interested in kube pods entries, for example:
@@ -93,15 +136,30 @@ func getPodUIDAndContainerIDFromCGroupPath(cgroupPath string) (types.UID, string
  // - /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod2c48913c-b29f-11e7-9350-020968147796.slice/docker-9bca8d63d5fa610783847915bcff0ecac1273e5b4bed3f6fa1b07350e0135961.scope
  // - /kubepods-besteffort-pod72f7f152_440c_66ac_9084_e0fc1d8a910c.slice:cri-containerd:b2a102854b4969b2ce98dc329c86b4fb2b06e4ad2cc8da9d8a7578c9cd2004a2"
  // - /../../pod2c48913c-b29f-11e7-9350-020968147796/9bca8d63d5fa610783847915bcff0ecac1273e5b4bed3f6fa1b07350e0135961
-
+ // - 0::/../crio-45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814.scope
  // First trim off any .scope suffix. This allows for a cleaner regex since
  // we don't have to muck with greediness. TrimSuffix is no-copy so this
  // is cheap.
  cgroupPath = strings.TrimSuffix(cgroupPath, ".scope")
 
- matches := cgroupRE.FindStringSubmatch(cgroupPath)
- if matches != nil {
- return canonicalizePodUID(matches[1]), matches[2], true
+ var matchResults map[string]string
+ for _, regex := range cgroupREs {
+ matches := reSubMatchMap(regex, cgroupPath)
+ if isValidCGroupPathMatches(matches) {
+ if matchResults != nil {
+ log.Printf("More than one regex matches for cgroup %s", cgroupPath)
+ return "", "", false
+ }
+ matchResults = matches
+ }
+ }
+
+ if matchResults != nil {
+ var podUID types.UID
+ if matchResults["poduid"] != "" {
+ podUID = canonicalizePodUID(matchResults["poduid"])
+ }
+ return podUID, matchResults["containerid"], true
  }
  return "", "", false
 }
@@ -119,5 +177,5 @@ func canonicalizePodUID(uid string) types.UID {
 }
 
 func isNotPod(itemPodUID, podUID types.UID) bool {
- return itemPodUID != podUID
+ return podUID != "" && itemPodUID != podUID
 }

pkg/agent/plugin/workloadattestor/k8s/k8s_posix_test.go

@@ -20,10 +20,13 @@ import (
 )
 
 const (
- kindPodListFilePath = "testdata/kind_pod_list.json"
+ kindPodListFilePath = "testdata/kind_pod_list.json"
+ crioPodListFilePath = "testdata/crio_pod_list.json"
+ crioPodListDuplicateContainerIDFilePath = "testdata/crio_pod_list_duplicate_containerId.json"
 
  cgPidInPodFilePath = "testdata/cgroups_pid_in_pod.txt"
  cgPidInKindPodFilePath = "testdata/cgroups_pid_in_kind_pod.txt"
+ cgPidInCrioPodFilePath = "testdata/cgroups_pid_in_crio_pod.txt"
  cgInitPidInPodFilePath = "testdata/cgroups_init_pid_in_pod.txt"
  cgPidNotInPodFilePath = "testdata/cgroups_pid_not_in_pod.txt"
  cgSystemdPidInPodFilePath = "testdata/systemd_cgroups_pid_in_pod.txt"
@@ -51,6 +54,25 @@ var (
  {Type: "k8s", Value: "sa:default"},
  }
 
+ testCrioPodSelectors = []*common.Selector{
+ {Type: "k8s", Value: "container-image:gcr.io/spiffe-io/spire-agent:0.8.1"},
+ {Type: "k8s", Value: "container-image:gcr.io/spiffe-io/spire-agent@sha256:1e4c481d76e9ecbd3d8684891e0e46aa021a30920ca04936e1fdcc552747d941"},
+ {Type: "k8s", Value: "container-name:workload-api-client"},
+ {Type: "k8s", Value: "node-name:a37b7d23-d32a-4932-8f33-40950ac16ee9"},
+ {Type: "k8s", Value: "ns:sfh-199"},
+ {Type: "k8s", Value: "pod-image-count:1"},
+ {Type: "k8s", Value: "pod-image:gcr.io/spiffe-io/spire-agent:0.8.1"},
+ {Type: "k8s", Value: "pod-image:gcr.io/spiffe-io/spire-agent@sha256:1e4c481d76e9ecbd3d8684891e0e46aa021a30920ca04936e1fdcc552747d941"},
+ {Type: "k8s", Value: "pod-init-image-count:0"},
+ {Type: "k8s", Value: "pod-label:app:sample-workload"},
+ {Type: "k8s", Value: "pod-label:pod-template-hash:6658cb9566"},
+ {Type: "k8s", Value: "pod-name:sample-workload-6658cb9566-5n4b4"},
+ {Type: "k8s", Value: "pod-owner-uid:ReplicaSet:349d135e-3781-43e3-bc25-c900aedf1d0c"},
+ {Type: "k8s", Value: "pod-owner:ReplicaSet:sample-workload-6658cb9566"},
+ {Type: "k8s", Value: "pod-uid:a2830d0d-b0f0-4ff0-81b5-0ee4e299cf80"},
+ {Type: "k8s", Value: "sa:default"},
+ }
+
  testInitPodSelectors = []*common.Selector{
  {Type: "k8s", Value: "container-image:docker-pullable://quay.io/coreos/flannel@sha256:1b401bf0c30bada9a539389c3be652b58fe38463361edf488e6543c8761d4970"},
  {Type: "k8s", Value: "container-image:quay.io/coreos/flannel:v0.9.0-amd64"},
@@ -89,6 +111,13 @@ func (s *Suite) TestAttestWithPidInKindPod() {
  s.requireAttestSuccessWithKindPod(p)
 }
 
+func (s *Suite) TestAttestWithPidInCrioPod() {
+ s.startInsecureKubelet()
+ p := s.loadInsecurePlugin()
+
+ s.requireAttestSuccessWithCrioPod(p)
+}
+
 func (s *Suite) TestAttestWithPidNotInPod() {
  s.startInsecureKubelet()
  p := s.loadInsecurePlugin()
@@ -99,6 +128,13 @@ func (s *Suite) TestAttestWithPidNotInPod() {
  s.Require().Empty(selectors)
 }
 
+func (s *Suite) TestAttestFailDuplicateContainerId() {
+ s.startInsecureKubelet()
+ p := s.loadInsecurePlugin()
+
+ s.requireAttestFailWithDuplicateContainerID(p)
+}
+
 func (s *Suite) TestAttestWithPidInPodSystemdCgroups() {
  s.startInsecureKubelet()
  p := s.loadInsecurePlugin()
@@ -141,6 +177,18 @@ func (s *Suite) requireAttestSuccessWithKindPod(p workloadattestor.WorkloadAttes
  s.requireAttestSuccess(p, testKindPodSelectors)
 }
 
+func (s *Suite) requireAttestSuccessWithCrioPod(p workloadattestor.WorkloadAttestor) {
+ s.addPodListResponse(crioPodListFilePath)
+ s.addCgroupsResponse(cgPidInCrioPodFilePath)
+ s.requireAttestSuccess(p, testCrioPodSelectors)
+}
+
+func (s *Suite) requireAttestFailWithDuplicateContainerID(p workloadattestor.WorkloadAttestor) {
+ s.addPodListResponse(crioPodListDuplicateContainerIDFilePath)
+ s.addCgroupsResponse(cgPidInCrioPodFilePath)
+ s.requireAttestFailure(p, codes.Internal, "two pods found with same container Id")
+}
+
 func (s *Suite) requireAttestSuccessWithPodSystemdCgroups(p workloadattestor.WorkloadAttestor) {
  s.addPodListResponse(podListFilePath)
  s.addCgroupsResponse(cgSystemdPidInPodFilePath)
@@ -202,6 +250,15 @@ func TestGetContainerIDFromCGroups(t *testing.T) {
  expectContainerID: "9bca8d63d5fa610783847915bcff0ecac1273e5b4bed3f6fa1b07350e0135961",
  expectCode: codes.OK,
  },
+ {
+ name: "cri-o",
+ cgroupPaths: []string{
+ "0::/../crio-45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814.scope",
+ },
+ expectPodUID: "",
+ expectContainerID: "45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814",
+ expectCode: codes.OK,
+ },
  {
  name: "more than one container ID in cgroups",
  cgroupPaths: []string{
@@ -314,6 +371,18 @@ func TestGetPodUIDAndContainerIDFromCGroupPath(t *testing.T) {
  expectPodUID: "72f7f152-440c-66ac-9084-e0fc1d8a910c",
  expectContainerID: "b2a102854b4969b2ce98dc329c86b4fb2b06e4ad2cc8da9d8a7578c9cd2004a2",
  },
+ {
+ name: "cri-o in combination with kubeedge",
+ cgroupPath: "0::/../crio-45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814.scope",
+ expectPodUID: "",
+ expectContainerID: "45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814",
+ },
+ {
+ name: "cri-o in combination with minikube",
+ cgroupPath: "9:devices:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod561fd272_d131_47ef_a01b_46a997a778f3.slice/crio-030ded69d4c98fcf69c988f75a5eb3a1b4357e1432bd5510c936a40d7e9a1198.scope",
+ expectPodUID: "561fd272-d131-47ef-a01b-46a997a778f3",
+ expectContainerID: "030ded69d4c98fcf69c988f75a5eb3a1b4357e1432bd5510c936a40d7e9a1198",
+ },
  {
  name: "uid generateds by kubernetes",
  cgroupPath: "/kubepods/pod2732ca68f6358eba7703fb6f82a25c94",

pkg/agent/plugin/workloadattestor/k8s/testdata/cgroups_pid_in_crio_pod.txt

@@ -0,0 +1 @@
+0::/../crio-09bc3d7ade839efec32b6bec4ec79d099027a668ddba043083ec21d3c3b8f1e6.scope