From a2f7901166398fcfbfa69b971a7a9205b6835fec Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Wed, 28 Dec 2022 19:05:04 +0400 Subject: [PATCH] feat: include Kubernetes controlplane endpoint as one of the endpoints These endpoints are used for workers to find the addresses of the controlplane nodes to connect to `trustd` to issue certificates of `apid`. These endpoints today come from two sources: * discovery service data * Kubernetes API server endpoints This PR adds to the list static entry based on the Kubernetes control plane endpoint in the machine config. E.g. if the loadbalancer is used for the controlplane endpoint, and that loadbalancer also proxies requests for port 50001 (trustd), this static endpoint will provide workers with connectivity to trustd even if the discovery service is disabled, and Kubernetes API is not up. If this endpoint doesn't provide any trustd API, Talos will still try other endpoints. Talos does server certificate validation when calling trustd, so including malicious endpoints doesn't cause any harm, as malicious endpoint can't provider proper server certificate. Signed-off-by: Andrey Smirnov (cherry picked from commit 80fed319408be9e493141fb2c01e5731708835c7) --- .../pkg/controllers/k8s/static_endpoint.go | 92 +++++++++++++++++++ .../controllers/k8s/static_endpoint_test.go | 62 +++++++++++++ .../machined/pkg/controllers/secrets/api.go | 27 +++++- .../runtime/v1alpha2/v1alpha2_controller.go | 1 + pkg/machinery/resources/k8s/endpoint.go | 3 + 5 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 internal/app/machined/pkg/controllers/k8s/static_endpoint.go create mode 100644 internal/app/machined/pkg/controllers/k8s/static_endpoint_test.go diff --git a/internal/app/machined/pkg/controllers/k8s/static_endpoint.go b/internal/app/machined/pkg/controllers/k8s/static_endpoint.go new file mode 100644 index 0000000000..ed7e2a7682 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/static_endpoint.go @@ -0,0 +1,92 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/slices" + "github.com/siderolabs/go-pointer" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// StaticEndpointController injects endpoints based on machine configuration. +type StaticEndpointController struct{} + +// Name implements controller.Controller interface. +func (ctrl *StaticEndpointController) Name() string { + return "k8s.StaticEndpointController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *StaticEndpointController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: pointer.To(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *StaticEndpointController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.EndpointType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *StaticEndpointController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + machineConfig, err := safe.ReaderGet[*config.MachineConfig](ctx, r, resource.NewMetadata(config.NamespaceName, config.MachineConfigType, config.V1Alpha1ID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine config: %w", err) + } + + cpHostname := machineConfig.Config().Cluster().Endpoint().Hostname() + + var resolver net.Resolver + + addrs, err := resolver.LookupNetIP(ctx, "ip", cpHostname) + if err != nil { + return fmt.Errorf("error resolving %q: %w", cpHostname, err) + } + + addrs = slices.Map(addrs, netip.Addr.Unmap) + + if err = safe.WriterModify(ctx, r, k8s.NewEndpoint(k8s.ControlPlaneNamespaceName, k8s.ControlPlaneKubernetesEndpointsID), func(endpoint *k8s.Endpoint) error { + endpoint.TypedSpec().Addresses = addrs + + return nil + }); err != nil { + return fmt.Errorf("error modifying endpoint: %w", err) + } + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/static_endpoint_test.go b/internal/app/machined/pkg/controllers/k8s/static_endpoint_test.go new file mode 100644 index 0000000000..ad86e9f2cc --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/static_endpoint_test.go @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "net/netip" + "net/url" + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type StaticEndpointControllerSuite struct { + ctest.DefaultSuite +} + +func (suite *StaticEndpointControllerSuite) TestReconcile() { + u, err := url.Parse("https://[2001:db8::1]:6443/") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ControlPlaneKubernetesEndpointsID}, + func(endpoint *k8s.Endpoint, assert *assert.Assertions) { + assert.Equal([]netip.Addr{netip.MustParseAddr("2001:db8::1")}, endpoint.TypedSpec().Addresses) + }) +} + +func TestStaticEndpointControllerSuite(t *testing.T) { + suite.Run(t, &StaticEndpointControllerSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&k8sctrl.StaticEndpointController{})) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/api.go b/internal/app/machined/pkg/controllers/secrets/api.go index ef8f18ae55..0545967a45 100644 --- a/internal/app/machined/pkg/controllers/secrets/api.go +++ b/internal/app/machined/pkg/controllers/secrets/api.go @@ -354,7 +354,32 @@ func (ctrl *APIController) generateWorker(ctx context.Context, r controller.Runt var ca []byte - ca, serverCert.Crt, err = remoteGen.IdentityContext(ctx, serverCSR) + // run the CSR generation in a goroutine, so we can abort the request if the inputs change + errCh := make(chan error) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + ca, serverCert.Crt, err = remoteGen.IdentityContext(ctx, serverCSR) + errCh <- err + }() + + select { + case <-r.EventCh(): + // there's an update to the inputs, terminate the attempt, and let the controller handle the retry + cancel() + + // re-queue the reconcile event, so that controller retries with new inputs + r.QueueReconcile() + + // wait for the goroutine to finish, ignoring the error (should be context.Canceled) + <-errCh + + return nil + case err = <-errCh: + } + if err != nil { return fmt.Errorf("failed to sign API server CSR: %w", err) } diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go index 61130acdf4..b26db51c01 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go @@ -139,6 +139,7 @@ func (ctrl *Controller) Run(ctx context.Context, drainer *runtime.Drainer) error &k8s.NodenameController{}, &k8s.RenderConfigsStaticPodController{}, &k8s.RenderSecretsStaticPodController{}, + &k8s.StaticEndpointController{}, &k8s.StaticPodConfigController{}, &k8s.StaticPodServerController{}, &kubeaccess.ConfigController{}, diff --git a/pkg/machinery/resources/k8s/endpoint.go b/pkg/machinery/resources/k8s/endpoint.go index c46c4419df..ebb631c91a 100644 --- a/pkg/machinery/resources/k8s/endpoint.go +++ b/pkg/machinery/resources/k8s/endpoint.go @@ -26,6 +26,9 @@ const ControlPlaneAPIServerEndpointsID = resource.ID("kube-apiserver") // ControlPlaneDiscoveredEndpointsID is resource ID for cluster discovery based Endpoints. const ControlPlaneDiscoveredEndpointsID = resource.ID("discovery") +// ControlPlaneKubernetesEndpointsID is resource ID for control plane endpoint-based Endpoints. +const ControlPlaneKubernetesEndpointsID = resource.ID("controlplane") + // Endpoint resource holds definition of rendered secrets. type Endpoint = typed.Resource[EndpointSpec, EndpointRD]