Skip to content

Commit

Permalink
closes spiffe#3183
Browse files Browse the repository at this point in the history
Signed-off-by: Christian Görg <christian.goerg@trumpf.com>
  • Loading branch information
goergch committed Jul 13, 2022
1 parent 12cf52e commit a4a56f2
Show file tree
Hide file tree
Showing 5 changed files with 566 additions and 15 deletions.
37 changes: 31 additions & 6 deletions pkg/agent/plugin/workloadattestor/k8s/k8s_posix.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,21 +190,31 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque
return nil, err
}

var attestResponse *workloadattestorv1.AttestResponse = nil
for _, item := range list.Items {
item := item
if item.UID != podUID {

// podUID can be empty, when cgroup contains only containerID
if item.UID != podUID && podUID != "" {
continue
}

status, lookup := lookUpContainerInPod(containerID, item.Status)
lookupStatus, lookup := lookUpContainerInPod(containerID, item.Status)
switch lookup {
case containerInPod:
return &workloadattestorv1.AttestResponse{
SelectorValues: getSelectorValuesFromPodInfo(&item, status),
}, nil
if attestResponse != nil {
log.Warn("Two pods found with same container Id")
return nil, status.Error(codes.Aborted, "Two pods found with same container Id")
}
attestResponse = &workloadattestorv1.AttestResponse{
SelectorValues: getSelectorValuesFromPodInfo(&item, lookupStatus),
}
case containerNotInPod:
}
}
if attestResponse != nil {
return attestResponse, nil
}

// if the container was not located after the maximum number of attempts then the search is over.
if attempt >= config.MaxPollAttempts {
Expand Down Expand Up @@ -582,14 +592,24 @@ var cgroupRE = regexp.MustCompile(`` +
// non-punctuation end of string, i.e., the container ID
`([[:^punct:]]+)$`)

// cgroupNoPodUidRE is the backup regex, when cgroupRE does not match
// This regex applies for container runtimes, that won't put the PodUID into
// the cgroup name.
// Currently only cri-o is known for this abnormaly.
var cgroupNoPodUidRE = regexp.MustCompile(`` +
// /crio-
`[[:punct:]]crio[[:punct:]]` +
// non-punctuation end of string, i.e., the container ID
`([[:^punct:]]+)$`)

func getPodUIDAndContainerIDFromCGroupPath(cgroupPath string) (types.UID, string, bool) {
// We are only interested in kube pods entries, for example:
// - /kubepods/burstable/pod2c48913c-b29f-11e7-9350-020968147796/9bca8d63d5fa610783847915bcff0ecac1273e5b4bed3f6fa1b07350e0135961
// - /docker/8d461fa5765781bcf5f7eb192f101bc3103d4b932e26236f43feecfa20664f96/kubepods/besteffort/poddaa5c7ee-3484-4533-af39-3591564fd03e/aff34703e5e1f89443e9a1bffcc80f43f74d4808a2dd22c8f88c08547b323934
// - /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod2c48913c-b29f-11e7-9350-020968147796.slice/docker-9bca8d63d5fa610783847915bcff0ecac1273e5b4bed3f6fa1b07350e0135961.scope
// - /kubepods-besteffort-pod72f7f152_440c_66ac_9084_e0fc1d8a910c.slice:cri-containerd:b2a102854b4969b2ce98dc329c86b4fb2b06e4ad2cc8da9d8a7578c9cd2004a2"
// - /../../pod2c48913c-b29f-11e7-9350-020968147796/9bca8d63d5fa610783847915bcff0ecac1273e5b4bed3f6fa1b07350e0135961

// - 0::/../crio-45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814.scope
// First trim off any .scope suffix. This allows for a cleaner regex since
// we don't have to muck with greediness. TrimSuffix is no-copy so this
// is cheap.
Expand All @@ -598,6 +618,11 @@ func getPodUIDAndContainerIDFromCGroupPath(cgroupPath string) (types.UID, string
matches := cgroupRE.FindStringSubmatch(cgroupPath)
if matches != nil {
return canonicalizePodUID(matches[1]), matches[2], true
} else {
matches := cgroupNoPodUidRE.FindStringSubmatch(cgroupPath)
if matches != nil {
return "", matches[1], true
}
}
return "", "", false
}
Expand Down
81 changes: 72 additions & 9 deletions pkg/agent/plugin/workloadattestor/k8s/k8s_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,18 @@ import (
const (
pid = 123

podListFilePath = "testdata/pod_list.json"
kindPodListFilePath = "testdata/kind_pod_list.json"
podListNotRunningFilePath = "testdata/pod_list_not_running.json"

cgPidInPodFilePath = "testdata/cgroups_pid_in_pod.txt"
cgPidInKindPodFilePath = "testdata/cgroups_pid_in_kind_pod.txt"
cgInitPidInPodFilePath = "testdata/cgroups_init_pid_in_pod.txt"
cgPidNotInPodFilePath = "testdata/cgroups_pid_not_in_pod.txt"
cgSystemdPidInPodFilePath = "testdata/systemd_cgroups_pid_in_pod.txt"
podListFilePath = "testdata/pod_list.json"
kindPodListFilePath = "testdata/kind_pod_list.json"
crioPodListFilePath = "testdata/crio_pod_list.json"
crioPodListDuplicateContainerIdFilePath = "testdata/crio_pod_list_duplicate_containerId.json"
podListNotRunningFilePath = "testdata/pod_list_not_running.json"

cgPidInPodFilePath = "testdata/cgroups_pid_in_pod.txt"
cgPidInKindPodFilePath = "testdata/cgroups_pid_in_kind_pod.txt"
cgPidInCrioPodFilePath = "testdata/cgroups_pid_in_crio_pod.txt"
cgInitPidInPodFilePath = "testdata/cgroups_init_pid_in_pod.txt"
cgPidNotInPodFilePath = "testdata/cgroups_pid_not_in_pod.txt"
cgSystemdPidInPodFilePath = "testdata/systemd_cgroups_pid_in_pod.txt"

certPath = "cert.pem"
keyPath = "key.pem"
Expand Down Expand Up @@ -110,6 +113,25 @@ FwOGLt+I3+9beT0vo+pn9Rq0squewFYe3aJbwpkyfP2xOovQCdm4PC8y
{Type: "k8s", Value: "sa:default"},
}

testCrioPodSelectors = []*common.Selector{
{Type: "k8s", Value: "container-image:gcr.io/spiffe-io/spire-agent:0.8.1"},
{Type: "k8s", Value: "container-image:gcr.io/spiffe-io/spire-agent@sha256:1e4c481d76e9ecbd3d8684891e0e46aa021a30920ca04936e1fdcc552747d941"},
{Type: "k8s", Value: "container-name:workload-api-client"},
{Type: "k8s", Value: "node-name:a37b7d23-d32a-4932-8f33-40950ac16ee9"},
{Type: "k8s", Value: "ns:sfh-199"},
{Type: "k8s", Value: "pod-image-count:1"},
{Type: "k8s", Value: "pod-image:gcr.io/spiffe-io/spire-agent:0.8.1"},
{Type: "k8s", Value: "pod-image:gcr.io/spiffe-io/spire-agent@sha256:1e4c481d76e9ecbd3d8684891e0e46aa021a30920ca04936e1fdcc552747d941"},
{Type: "k8s", Value: "pod-init-image-count:0"},
{Type: "k8s", Value: "pod-label:app:sample-workload"},
{Type: "k8s", Value: "pod-label:pod-template-hash:6658cb9566"},
{Type: "k8s", Value: "pod-name:sample-workload-6658cb9566-5n4b4"},
{Type: "k8s", Value: "pod-owner-uid:ReplicaSet:349d135e-3781-43e3-bc25-c900aedf1d0c"},
{Type: "k8s", Value: "pod-owner:ReplicaSet:sample-workload-6658cb9566"},
{Type: "k8s", Value: "pod-uid:a2830d0d-b0f0-4ff0-81b5-0ee4e299cf80"},
{Type: "k8s", Value: "sa:default"},
}

testInitPodSelectors = []*common.Selector{
{Type: "k8s", Value: "container-image:docker-pullable://quay.io/coreos/flannel@sha256:1b401bf0c30bada9a539389c3be652b58fe38463361edf488e6543c8761d4970"},
{Type: "k8s", Value: "container-image:quay.io/coreos/flannel:v0.9.0-amd64"},
Expand Down Expand Up @@ -188,6 +210,20 @@ func (s *Suite) TestAttestWithPidInKindPod() {
s.requireAttestSuccessWithKindPod(p)
}

func (s *Suite) TestAttestWithPidInCrioPod() {
s.startInsecureKubelet()
p := s.loadInsecurePlugin()

s.requireAttestSuccessWithCrioPod(p)
}

func (s *Suite) TestAttestFailDuplicateContainerId() {
s.startInsecureKubelet()
p := s.loadInsecurePlugin()

s.requireAttestFailWithCrioPod(p)
}

func (s *Suite) TestAttestWithPidInPodSystemdCgroups() {
s.startInsecureKubelet()
p := s.loadInsecurePlugin()
Expand Down Expand Up @@ -796,6 +832,18 @@ func (s *Suite) requireAttestSuccessWithKindPod(p workloadattestor.WorkloadAttes
s.requireAttestSuccess(p, testKindPodSelectors)
}

func (s *Suite) requireAttestSuccessWithCrioPod(p workloadattestor.WorkloadAttestor) {
s.addPodListResponse(crioPodListFilePath)
s.addCgroupsResponse(cgPidInCrioPodFilePath)
s.requireAttestSuccess(p, testCrioPodSelectors)
}

func (s *Suite) requireAttestFailWithCrioPod(p workloadattestor.WorkloadAttestor) {
s.addPodListResponse(crioPodListDuplicateContainerIdFilePath)
s.addCgroupsResponse(cgPidInCrioPodFilePath)
s.requireAttestFailure(p, codes.Aborted, "Two pods found with same container Id")
}

func (s *Suite) requireAttestSuccessWithPodSystemdCgroups(p workloadattestor.WorkloadAttestor) {
s.addPodListResponse(podListFilePath)
s.addCgroupsResponse(cgSystemdPidInPodFilePath)
Expand Down Expand Up @@ -909,6 +957,15 @@ func TestGetContainerIDFromCGroups(t *testing.T) {
expectContainerID: "9bca8d63d5fa610783847915bcff0ecac1273e5b4bed3f6fa1b07350e0135961",
expectCode: codes.OK,
},
{
name: "cri-o",
cgroupPaths: []string{
"0::/../crio-45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814.scope",
},
expectPodUID: "",
expectContainerID: "45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814",
expectCode: codes.OK,
},
{
name: "more than one container ID in cgroups",
cgroupPaths: []string{
Expand Down Expand Up @@ -1021,6 +1078,12 @@ func TestGetPodUIDAndContainerIDFromCGroupPath(t *testing.T) {
expectPodUID: "72f7f152-440c-66ac-9084-e0fc1d8a910c",
expectContainerID: "b2a102854b4969b2ce98dc329c86b4fb2b06e4ad2cc8da9d8a7578c9cd2004a2",
},
{
name: "cri-o",
cgroupPath: "0::/../crio-45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814.scope",
expectPodUID: "",
expectContainerID: "45490e76e0878aaa4d9808f7d2eefba37f093c3efbba9838b6d8ab804d9bd814",
},
{
name: "uid generateds by kubernetes",
cgroupPath: "/kubepods/pod2732ca68f6358eba7703fb6f82a25c94",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0::/../crio-09bc3d7ade839efec32b6bec4ec79d099027a668ddba043083ec21d3c3b8f1e6.scope
157 changes: 157 additions & 0 deletions pkg/agent/plugin/workloadattestor/k8s/testdata/crio_pod_list.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
{
"apiVersion": "v1",
"items": [
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"creationTimestamp": "2019-09-20T06:13:48Z",
"generateName": "sample-workload-6658cb9566-",
"labels": {
"app": "sample-workload",
"pod-template-hash": "6658cb9566"
},
"name": "sample-workload-6658cb9566-5n4b4",
"namespace": "sfh-199",
"ownerReferences": [
{
"apiVersion": "apps/v1",
"blockOwnerDeletion": true,
"controller": true,
"kind": "ReplicaSet",
"name": "sample-workload-6658cb9566",
"uid": "349d135e-3781-43e3-bc25-c900aedf1d0c"
}
],
"resourceVersion": "17021",
"selfLink": "/api/v1/namespaces/sfh-199/pods/sample-workload-6658cb9566-5n4b4",
"uid": "a2830d0d-b0f0-4ff0-81b5-0ee4e299cf80"
},
"spec": {
"containers": [
{
"args": [
"api",
"watch"
],
"command": [
"/opt/spire/bin/spire-agent"
],
"image": "gcr.io/spiffe-io/spire-agent:0.8.1",
"imagePullPolicy": "IfNotPresent",
"name": "workload-api-client",
"resources": {},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"volumeMounts": [
{
"mountPath": "/tmp/spire-agent/public",
"name": "spire-agent-socket",
"readOnly": true
},
{
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
"name": "default-token-qfslv",
"readOnly": true
}
]
}
],
"dnsPolicy": "ClusterFirst",
"enableServiceLinks": true,
"nodeName": "a37b7d23-d32a-4932-8f33-40950ac16ee9",
"priority": 0,
"restartPolicy": "Always",
"schedulerName": "default-scheduler",
"securityContext": {},
"serviceAccount": "default",
"serviceAccountName": "default",
"terminationGracePeriodSeconds": 30,
"tolerations": [
{
"effect": "NoExecute",
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"tolerationSeconds": 300
},
{
"effect": "NoExecute",
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"tolerationSeconds": 300
}
],
"volumes": [
{
"hostPath": {
"path": "/run/spire-agent/public",
"type": "Directory"
},
"name": "spire-agent-socket"
},
{
"name": "default-token-qfslv",
"secret": {
"defaultMode": 420,
"secretName": "default-token-qfslv"
}
}
]
},
"status": {
"conditions": [
{
"lastProbeTime": null,
"lastTransitionTime": "2019-09-20T06:13:48Z",
"status": "True",
"type": "Initialized"
},
{
"lastProbeTime": null,
"lastTransitionTime": "2019-09-20T06:13:49Z",
"status": "True",
"type": "Ready"
},
{
"lastProbeTime": null,
"lastTransitionTime": "2019-09-20T06:13:49Z",
"status": "True",
"type": "ContainersReady"
},
{
"lastProbeTime": null,
"lastTransitionTime": "2019-09-20T06:13:48Z",
"status": "True",
"type": "PodScheduled"
}
],
"containerStatuses": [
{
"containerID": "containerd://09bc3d7ade839efec32b6bec4ec79d099027a668ddba043083ec21d3c3b8f1e6",
"image": "gcr.io/spiffe-io/spire-agent:0.8.1",
"imageID": "gcr.io/spiffe-io/spire-agent@sha256:1e4c481d76e9ecbd3d8684891e0e46aa021a30920ca04936e1fdcc552747d941",
"lastState": {},
"name": "workload-api-client",
"ready": true,
"restartCount": 0,
"state": {
"running": {
"startedAt": "2019-09-20T06:13:49Z"
}
}
}
],
"hostIP": "172.17.0.2",
"phase": "Running",
"podIP": "10.244.0.8",
"qosClass": "BestEffort",
"startTime": "2019-09-20T06:13:48Z"
}
}
],
"kind": "List",
"metadata": {
"resourceVersion": "",
"selfLink": ""
}
}
Loading

0 comments on commit a4a56f2

Please sign in to comment.