Server Token Rotation (k3s-io#8578)

* Consolidate NewCertCommands
* Add support for user defined new token
* Add E2E testlets
* Ensure agent token also changes

Signed-off-by: Derek Nola <derek.nola@suse.com>

cmd/cert/main.go

@@ -15,11 +15,9 @@ import (
 func main() {
  app := cmds.NewApp()
  app.Commands = []cli.Command{
- cmds.NewCertCommand(
- cmds.NewCertSubcommands(
- cert.Rotate,
- cert.RotateCA,
- ),
+ cmds.NewCertCommands(
+ cert.Rotate,
+ cert.RotateCA,
  ),
  }
 

cmd/k3s/main.go

@@ -57,6 +57,7 @@ func main() {
  tokenCommand,
  tokenCommand,
  tokenCommand,
+ tokenCommand,
  ),
  cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand,
  cmds.NewEtcdSnapshotSubcommands(
@@ -74,11 +75,9 @@ func main() {
  secretsencryptCommand,
  secretsencryptCommand),
  ),
- cmds.NewCertCommand(
- cmds.NewCertSubcommands(
- certCommand,
- certCommand,
- ),
+ cmds.NewCertCommands(
+ certCommand,
+ certCommand,
  ),
  cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)),
  }

cmd/server/main.go

@@ -54,6 +54,7 @@ func main() {
  token.Delete,
  token.Generate,
  token.List,
+ token.Rotate,
  ),
  cmds.NewEtcdSnapshotCommand(etcdsnapshot.Save,
  cmds.NewEtcdSnapshotSubcommands(
@@ -71,11 +72,9 @@ func main() {
  secretsencrypt.Rotate,
  secretsencrypt.Reencrypt),
  ),
- cmds.NewCertCommand(
- cmds.NewCertSubcommands(
- cert.Rotate,
- cert.RotateCA,
- ),
+ cmds.NewCertCommands(
+ cert.Rotate,
+ cert.RotateCA,
  ),
  cmds.NewCompletionCommand(completion.Run),
  }

cmd/token/main.go

@@ -20,6 +20,7 @@ func main() {
  token.Delete,
  token.Generate,
  token.List,
+ token.Rotate,
  ),
  }
 

main.go

@@ -48,11 +48,9 @@ func main() {
  secretsencrypt.Rotate,
  secretsencrypt.Reencrypt),
  ),
- cmds.NewCertCommand(
- cmds.NewCertSubcommands(
- cert.Rotate,
- cert.RotateCA,
- ),
+ cmds.NewCertCommands(
+ cert.Rotate,
+ cert.RotateCA,
  ),
  cmds.NewCompletionCommand(completion.Run),
  }

pkg/cli/cmds/certs.go

@@ -54,33 +54,29 @@ var (
  }
 )
 
-func NewCertCommand(subcommands []cli.Command) cli.Command {
+func NewCertCommands(rotate, rotateCA func(ctx *cli.Context) error) cli.Command {
  return cli.Command{
  Name: CertCommand,
  Usage: "Manage K3s certificates",
  SkipFlagParsing: false,
  SkipArgReorder: true,
- Subcommands: subcommands,
- }
-}
-
-func NewCertSubcommands(rotate, rotateCA func(ctx *cli.Context) error) []cli.Command {
- return []cli.Command{
- {
- Name: "rotate",
- Usage: "Rotate " + version.Program + " component certificates on disk",
- SkipFlagParsing: false,
- SkipArgReorder: true,
- Action: rotate,
- Flags: CertRotateCommandFlags,
- },
- {
- Name: "rotate-ca",
- Usage: "Write updated " + version.Program + " CA certificates to the datastore",
- SkipFlagParsing: false,
- SkipArgReorder: true,
- Action: rotateCA,
- Flags: CertRotateCACommandFlags,
+ Subcommands: []cli.Command{
+ {
+ Name: "rotate",
+ Usage: "Rotate " + version.Program + " component certificates on disk",
+ SkipFlagParsing: false,
+ SkipArgReorder: true,
+ Action: rotate,
+ Flags: CertRotateCommandFlags,
+ },
+ {
+ Name: "rotate-ca",
+ Usage: "Write updated " + version.Program + " CA certificates to the datastore",
+ SkipFlagParsing: false,
+ SkipArgReorder: true,
+ Action: rotateCA,
+ Flags: CertRotateCACommandFlags,
+ },
  },
  }
 }

pkg/cli/cmds/token.go

@@ -3,6 +3,7 @@ package cmds
 import (
  "time"
 
+ "github.com/k3s-io/k3s/pkg/version"
  "github.com/urfave/cli"
 )
 
@@ -12,7 +13,9 @@ const TokenCommand = "token"
 type Token struct {
  Description string
  Kubeconfig string
+ ServerURL string
  Token string
+ NewToken string
  Output string
  Groups cli.StringSlice
  Usages cli.StringSlice
@@ -32,7 +35,7 @@ var (
  }
 )
 
-func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) error) cli.Command {
+func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Context) error) cli.Command {
  return cli.Command{
  Name: TokenCommand,
  Usage: "Manage bootstrap tokens",
@@ -92,6 +95,32 @@ func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) erro
  SkipArgReorder: true,
  Action: list,
  },
+ {
+ Name: "rotate",
+ Usage: "Rotate original server token with a new bootstrap token",
+ Flags: append(TokenFlags,
+ &cli.StringFlag{
+ Name: "token,t",
+ Usage: "Existing token used to join a server or agent to a cluster",
+ Destination: &TokenConfig.Token,
+ EnvVar: version.ProgramUpper + "_TOKEN",
+ },
+ &cli.StringFlag{
+ Name: "server, s",
+ Usage: "(cluster) Server to connect to",
+ Destination: &TokenConfig.ServerURL,
+ EnvVar: version.ProgramUpper + "_URL",
+ Value: "https://127.0.0.1:6443",
+ },
+ &cli.StringFlag{
+ Name: "new-token",
+ Usage: "New token that replaces existing token",
+ Destination: &TokenConfig.NewToken,
+ }),
+ SkipFlagParsing: false,
+ SkipArgReorder: true,
+ Action: rotate,
+ },
  },
  }
 }

pkg/cli/token/token.go

@@ -1,18 +1,23 @@
 package token
 
 import (
+ "bytes"
  "context"
  "encoding/json"
  "fmt"
  "os"
+ "path/filepath"
  "strings"
  "text/tabwriter"
  "time"
 
+ "github.com/erikdubbelboer/gspt"
  "github.com/k3s-io/k3s/pkg/cli/cmds"
  "github.com/k3s-io/k3s/pkg/clientaccess"
  "github.com/k3s-io/k3s/pkg/kubeadm"
+ "github.com/k3s-io/k3s/pkg/server"
  "github.com/k3s-io/k3s/pkg/util"
+ "github.com/k3s-io/k3s/pkg/version"
  "github.com/pkg/errors"
  "github.com/urfave/cli"
  "gopkg.in/yaml.v2"
@@ -22,6 +27,7 @@ import (
  "k8s.io/client-go/tools/clientcmd"
  bootstrapapi "k8s.io/cluster-bootstrap/token/api"
  bootstraputil "k8s.io/cluster-bootstrap/token/util"
+ "k8s.io/utils/pointer"
 )
 
 func Create(app *cli.Context) error {
@@ -139,6 +145,50 @@ func generate(app *cli.Context, cfg *cmds.Token) error {
  return nil
 }
 
+func Rotate(app *cli.Context) error {
+ if err := cmds.InitLogging(); err != nil {
+ return err
+ }
+ fmt.Println("\033[33mWARNING\033[0m: Recommended to keep a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.")
+ info, err := serverAccess(&cmds.TokenConfig)
+ if err != nil {
+ return err
+ }
+ b, err := json.Marshal(server.TokenRotateRequest{
+ NewToken: pointer.String(cmds.TokenConfig.NewToken),
+ })
+ if err != nil {
+ return err
+ }
+ if err = info.Put("/v1-"+version.Program+"/token", b); err != nil {
+ return err
+ }
+ // wait for etcd db propagation delay
+ time.Sleep(1 * time.Second)
+ fmt.Println("Token rotated, restart k3s with new token")
+ return nil
+}
+
+func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) {
+ // hide process arguments from ps output, since they likely contain tokens.
+ gspt.SetProcTitle(os.Args[0] + " token")
+
+ dataDir, err := server.ResolveDataDir("")
+ if err != nil {
+ return nil, err
+ }
+
+ if cfg.Token == "" {
+ fp := filepath.Join(dataDir, "token")
+ tokenByte, err := os.ReadFile(fp)
+ if err != nil {
+ return nil, err
+ }
+ cfg.Token = string(bytes.TrimRight(tokenByte, "\n"))
+ }
+ return clientaccess.ParseAndValidateToken(cfg.ServerURL, cfg.Token, clientaccess.WithUser("server"))
+}
+
 func List(app *cli.Context) error {
  if err := cmds.InitLogging(); err != nil {
  return err

pkg/cluster/storage.go

@@ -21,6 +21,49 @@ import (
 // After this many attempts, the lock is deleted and the counter reset.
 const maxBootstrapWaitAttempts = 5
 
+func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error {
+
+ token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
+ if err != nil {
+ return err
+ }
+
+ normalizedToken, err := normalizeToken(token)
+ if err != nil {
+ return err
+ }
+
+ storageClient, err := client.New(config.Runtime.EtcdConfig)
+ if err != nil {
+ return err
+ }
+ defer storageClient.Close()
+
+ tokenKey := storageKey(normalizedToken)
+
+ var bootstrapList []client.Value
+ if err := wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) {
+ bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0)
+ if err != nil {
+ return false, err
+ }
+ return true, nil
+ }); err != nil {
+ return err
+ }
+
+ normalizedOldToken, err := normalizeToken(oldToken)
+ if err != nil {
+ return err
+ }
+ // reuse the existing migration function to reencrypt bootstrap data with new token
+ if err := migrateTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil {
+ return err
+ }
+
+ return nil
+}
+
 // Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
 // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token.
 // This is used when bootstrapping a cluster from a managed database or external etcd cluster.
@@ -225,7 +268,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
  logrus.Warn("found multiple bootstrap keys in storage")
  }
  // check for empty string key and for old token format with k10 prefix
- if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
+ if err := migrateTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
  return nil, false, err
  }
 
@@ -236,6 +279,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
  }
  for _, bootstrapKV := range bootstrapList {
  // ensure bootstrap is stored in the current token's key
+ logrus.Debugf("checking bootstrap key %s against %s", string(bootstrapKV.Key), tokenKey)
  if string(bootstrapKV.Key) == tokenKey {
  return &bootstrapKV, false, nil
  }
@@ -277,21 +321,24 @@ func normalizeToken(token string) (string, error) {
  return password, nil
 }
 
-// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is
+// migrateTokens will list all keys that has prefix /bootstrap and will check for key that is
 // hashed with empty string and keys that is hashed with old token format before normalizing
 // then migrate those and resave only with the normalized token
-func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
+func migrateTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
  oldTokenKey := storageKey(oldToken)
 
  for _, bootstrapKV := range bootstrapList {
  // checking for empty string bootstrap key
+ logrus.Debug("Comparing ", string(bootstrapKV.Key), " to ", oldTokenKey)
  if string(bootstrapKV.Key) == emptyStringKey {
  logrus.Warn("Bootstrap data encrypted with empty string, deleting and resaving with token")
  if err := doMigrateToken(ctx, storageClient, bootstrapKV, "", emptyStringKey, token, tokenKey); err != nil {
  return err
  }
  } else if string(bootstrapKV.Key) == oldTokenKey && oldTokenKey != tokenKey {
- logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
+ if emptyStringKey != "" {
+ logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
+ }
  if err := doMigrateToken(ctx, storageClient, bootstrapKV, oldToken, oldTokenKey, token, tokenKey); err != nil {
  return err
  }

pkg/daemons/control/deps/deps.go

@@ -243,6 +243,7 @@ func genUsers(config *config.Control) error {
  return err
  }
 
+ // if no token is provided on bootstrap, we generate a random token
  serverPass, err := getServerPass(passwd, config)
  if err != nil {
  return err

pkg/server/router.go

@@ -85,6 +85,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
  serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
  serverAuthed.Path("/db/info").Handler(nodeAuthed)
  serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime))
+ serverAuthed.Path(prefix + "/token").Handler(tokenRequestHandler(ctx, serverConfig))
 
  systemAuthed := mux.NewRouter().SkipClean(true)
  systemAuthed.NotFoundHandler = serverAuthed