Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@VSpike
VSpike committed Dec 3, 2021
1 parent 3c1f9f9 commit e53f80f
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 27 deletions.
20 changes: 12 additions & 8 deletions cmd/workloads-manager/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ import (
var (
scheme = runtime.NewScheme()

app = kingpin.New("workloads-manager", "Manages workloads.crd.gocardless.com resources").Version(cmd.VersionStanza())
contextName = app.Flag("context-name", "Distinct name for the context this controller runs within. Usually the user-facing name of the kubernetes context for the cluster").Envar("CONTEXT_NAME").String()
pubsubProjectId = app.Flag("pubsub-project-id", "ID for the project containing the Pub/Sub topic for console event publishing").Envar("PUBSUB_PROJECT_ID").String()
pubsubTopicId = app.Flag("pubsub-topic-id", "ID of the topic to publish lifecycle event messages").Envar("PUBSUB_TOPIC_ID").String()
app = kingpin.New("workloads-manager", "Manages workloads.crd.gocardless.com resources").Version(cmd.VersionStanza())
contextName = app.Flag("context-name", "Distinct name for the context this controller runs within. Usually the user-facing name of the kubernetes context for the cluster").Envar("CONTEXT_NAME").String()
pubsubProjectId = app.Flag("pubsub-project-id", "ID for the project containing the Pub/Sub topic for console event publishing").Envar("PUBSUB_PROJECT_ID").String()
pubsubTopicId = app.Flag("pubsub-topic-id", "ID of the topic to publish lifecycle event messages").Envar("PUBSUB_TOPIC_ID").String()
enableSessionRecording = app.Flag("session-recording", "Enable session recording features").Envar("ENABLE_SESSION_RECORDING").Default("false").Bool()
sessionSidecarImage = app.Flag("session-sidecar-image", "Container image to use for the session recording sidecar container").Envar("SESSION_SIDECAR_IMAGE").Default("").String()

commonOpts = cmd.NewCommonOptions(app).WithMetrics(app)
)
Expand Down Expand Up @@ -71,10 +73,12 @@ func main() {

// controller
if err = (&consolecontroller.ConsoleReconciler{
Client: mgr.GetClient(),
LifecycleRecorder: lifecycleRecorder,
Log: ctrl.Log.WithName("controllers").WithName("console"),
Scheme: mgr.GetScheme(),
Client: mgr.GetClient(),
LifecycleRecorder: lifecycleRecorder,
Log: ctrl.Log.WithName("controllers").WithName("console"),
Scheme: mgr.GetScheme(),
EnableSessionRecording: *enableSessionRecording,
SessionSidecarImage: *sessionSidecarImage,
}).SetupWithManager(ctx, mgr); err != nil {
app.Fatalf("failed to create controller: %v", err)
}
Expand Down
107 changes: 97 additions & 10 deletions controllers/workloads/console/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -47,6 +48,7 @@ const (
EventUnknownOutcome = "UnknownOutcome"
EventInvalidSpecification = "InvalidSpecification"
EventTemplateUnsupported = "TemplateUnsupported"
EventMissingSidecarImage = "MissingSidecarImage"

// Console log keys

Expand All @@ -67,7 +69,10 @@ const (
DefaultTTLAfterFinished = 24 * time.Hour

// Console session recording
TLogDataMount = "/var/log/tlog"
SessionRecVolMount = "/var/log/session"
SessionRecVolName = "session-data"
SidewrapShutdownDelay = 60
SidewrapGracePeriod = 30
)

type IgnoreCreatePredicate struct {
Expand All @@ -83,6 +88,11 @@ type ConsoleReconciler struct {
LifecycleRecorder workloadsv1alpha1.LifecycleEventRecorder
Log logr.Logger
Scheme *runtime.Scheme
// Enable injection of console session recording using tlog
EnableSessionRecording bool
// The image reference for the sidecar to inject to stream session
// recording data (if Session Recording is enabled)
SessionSidecarImage string
}

func (r *ConsoleReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
Expand Down Expand Up @@ -594,41 +604,115 @@ func requeueAfterInterval(logger logr.Logger, interval time.Duration) reconcile.
return reconcile.Result{Requeue: true, RequeueAfter: interval}
}

func modifyPodTemplate(podTemplate *corev1.PodTemplateSpec) *corev1.PodTemplateSpec {
func sessionRecordFileName(ix int) string {
return fmt.Sprintf("%s/output%0d", SessionRecVolMount, ix)
}

func buildSidecarContainer(image string) corev1.Container {
return corev1.Container{
Name: "session-streamer",
Image: image,
ImagePullPolicy: corev1.PullIfNotPresent,
VolumeMounts: []corev1.VolumeMount{
{
Name: SessionRecVolName,
MountPath: SessionRecVolMount,
ReadOnly: false,
},
},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("512Mi"),
corev1.ResourceCPU: resource.MustParse("512m"),
},
Requests: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("512Mi"),
corev1.ResourceCPU: resource.MustParse("512m"),
},
},
Env: []corev1.EnvVar{
{
Name: "KUBERNETES_POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.name",
},
},
},
{
Name: "KUBERNETES_NAMESPACE",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
{
Name: "SIDECAR_SHUTDOWN_DELAY",
Value: fmt.Sprint(SidewrapShutdownDelay),
},
{
Name: "SIDECAR_GRACE_PERIOD",
Value: fmt.Sprint(SidewrapGracePeriod),
},
},
}
}

func (r *ConsoleReconciler) modifyPodTemplate(logger logr.Logger, podTemplate *corev1.PodTemplateSpec) *corev1.PodTemplateSpec {

mutatedTemplate := podTemplate.DeepCopy()

// Add volume mount for tlog data
// Add volume mount for session recording data
mutatedTemplate.Spec.Volumes = append(
mutatedTemplate.Spec.Volumes,
corev1.Volume{
Name: "tlog-data",
Name: SessionRecVolName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
)

// Modify each container command and args to start the
// session recording wrapper which spawns the original
// command
for ix := range mutatedTemplate.Spec.Containers {
ctr := &mutatedTemplate.Spec.Containers[ix]
execCommand := []string{"--"}
execCommand = append(execCommand, ctr.Command...)
execCommand = append(execCommand, ctr.Args...)

ctr.Command = []string{"tlog-rec"}
args := []string{"-o", fmt.Sprintf("%s/output%0d", TLogDataMount, ix)}
args := []string{"-o", sessionRecordFileName(ix)}
args = append(args, execCommand...)
ctr.Args = args

ctr.VolumeMounts = append(
ctr.VolumeMounts,
corev1.VolumeMount{
Name: "tlog-data",
MountPath: TLogDataMount,
Name: SessionRecVolName,
MountPath: SessionRecVolMount,
},
)
}

if len(r.SessionSidecarImage) > 0 {
mutatedTemplate.Spec.Containers = append(
mutatedTemplate.Spec.Containers,
buildSidecarContainer(r.SessionSidecarImage),
)
// The grace period for the pod must be longer than Sidewrap's delay
// and grace periods combined
var gracePeriod int64 = SidewrapShutdownDelay + SidewrapGracePeriod + 30
mutatedTemplate.Spec.TerminationGracePeriodSeconds = &gracePeriod
} else {
msg := "Unable to add session recording sidecar due to missing image setting"
logger.Info(
msg,
"event", EventMissingSidecarImage,
"error", msg,
)
}
return mutatedTemplate
}

Expand All @@ -640,7 +724,7 @@ func (r *ConsoleReconciler) buildJob(logger logr.Logger, name types.NamespacedNa

numContainers := len(jobTemplate.Spec.Containers)

// If there's no containers in the spec then the controller will be emitting
// If there are no containers in the spec then the controller will be emitting
// warnings anyway, as the job will be rejected
if numContainers > 0 {
container := &jobTemplate.Spec.Containers[0]
Expand Down Expand Up @@ -700,7 +784,10 @@ func (r *ConsoleReconciler) buildJob(logger logr.Logger, name types.NamespacedNa
jobTemplate.ObjectMeta.Labels,
)

podTemplate := modifyPodTemplate((*corev1.PodTemplateSpec)(jobTemplate))
podTemplate := (*corev1.PodTemplateSpec)(jobTemplate)
if r.EnableSessionRecording {
podTemplate = r.modifyPodTemplate(logger, podTemplate)
}

return &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Expand Down
16 changes: 8 additions & 8 deletions controllers/workloads/console/integration/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ var _ = Describe("Console", func() {
Template: workloadsv1alpha1.PodTemplatePreserveMetadataSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
{
Image: "alpine:latest",
Name: "console-container-0",
Command: []string{"/bin/sh", "-c", "sleep 100"},
Expand Down Expand Up @@ -389,19 +389,19 @@ var _ = Describe("Console", func() {
Expect(role.Rules).To(
Equal(
[]rbacv1.PolicyRule{
rbacv1.PolicyRule{
{
Verbs: []string{"create"},
APIGroups: []string{""},
Resources: []string{"pods/exec", "pods/attach"},
ResourceNames: []string{podName},
},
rbacv1.PolicyRule{
{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"pods/log"},
ResourceNames: []string{podName},
},
rbacv1.PolicyRule{
{
Verbs: []string{"get", "delete"},
APIGroups: []string{""},
Resources: []string{"pods"},
Expand Down Expand Up @@ -431,9 +431,9 @@ var _ = Describe("Console", func() {
)
Expect(drb.Spec.Subjects).To(
ConsistOf([]rbacv1.Subject{
rbacv1.Subject{Kind: "User", Name: csl.Spec.User},
rbacv1.Subject{Kind: "User", Name: "add-user@example.com"},
rbacv1.Subject{Kind: "GoogleGroup", Name: "group@example.com"},
{Kind: "User", Name: csl.Spec.User},
{Kind: "User", Name: "add-user@example.com"},
{Kind: "GoogleGroup", Name: "group@example.com"},
}),
)

Expand Down Expand Up @@ -624,7 +624,7 @@ var _ = Describe("Console", func() {
)
Expect(drb.Spec.Subjects).To(
ConsistOf([]rbacv1.Subject{
rbacv1.Subject{Kind: "User", Name: "authorising-user-2@example.com"},
{Kind: "User", Name: "authorising-user-2@example.com"},
}),
)

Expand Down
2 changes: 1 addition & 1 deletion controllers/workloads/console/integration/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func TestSuite(t *testing.T) {
}

var _ = BeforeSuite(func(done Done) {
logf.SetLogger(zap.LoggerTo(GinkgoWriter, true))
logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))

By("bootstrapping test environment")
testEnv = &envtest.Environment{
Expand Down

0 comments on commit e53f80f

Please sign in to comment.