diff --git a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go
index 47178c7b8..9f26fc6e9 100644
--- a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go
+++ b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go
@@ -93,7 +93,9 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
return nil, err
}
- if secureComms {
+ if secureComms || cfg.daemonConfig.SecureComms {
+ var inbounds, outbounds []string
+
ppssh.Singleton()
host, port, err := net.SplitHostPort(cfg.listenAddr)
if err != nil {
@@ -103,15 +105,32 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
logger.Printf("Address %s is changed to 127.0.0.1:%s since secure-comms is enabled.", cfg.listenAddr, port)
cfg.listenAddr = "127.0.0.1:" + port
}
- inbounds := append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
- outbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
// Create a Client that will approach the api-server-rest service at the podns
// To obtain secrets from KBS, we approach the api-server-rest service which then approaches the CDH asking for a secret resource
// the CDH than contact the KBS (possibly after approaching Attestation Agent for a token) and the KBS serves the requested key
// The communication between the CDH (and Attestation Agent) and the KBS is performed via an SSH tunnel named "KBS"
apic := apic.NewApiClient(API_SERVER_REST_PORT, cfg.podNamespace)
- services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppssh.GetSecret(apic.GetKey), sshutil.SSHPORT))
+
+ ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(apic.GetKey))
+
+ if secureComms {
+ // CoCo
+ ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY)
+ ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY)
+ } else {
+ // non-CoCo
+ ppSecrets.SetKey(ppssh.WN_PUBLIC_KEY, cfg.daemonConfig.WnPublicKey)
+ ppSecrets.SetKey(ppssh.PP_PRIVATE_KEY, cfg.daemonConfig.PpPrivateKey)
+ }
+
+ inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
+ inbounds = append(inbounds, strings.Split(cfg.daemonConfig.SecureCommsInbounds, ",")...)
+
+ outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
+ outbounds = append(outbounds, strings.Split(cfg.daemonConfig.SecureCommsOutbounds, ",")...)
+
+ services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppSecrets, sshutil.SSHPORT))
} else {
if !disableTLS {
cfg.tlsConfig = &tlsConfig
diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go
index 6bdbadd63..9a33c6394 100644
--- a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go
+++ b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go
@@ -12,6 +12,7 @@ import (
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor"
+ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy"
daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan"
@@ -28,7 +29,7 @@ const (
)
type daemonConfig struct {
- serverConfig adaptor.ServerConfig
+ serverConfig cloud.ServerConfig
networkConfig
}
@@ -86,12 +87,15 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
}
var (
- disableTLS bool
- tlsConfig tlsutil.TLSConfig
- secureComms bool
- secureCommsInbounds string
- secureCommsOutbounds string
- secureCommsKbsAddr string
+ disableTLS bool
+ tlsConfig tlsutil.TLSConfig
+ secureComms bool
+ secureCommsNoTrustee bool
+ secureCommsInbounds string
+ secureCommsOutbounds string
+ secureCommsPpInbounds string
+ secureCommsPpOutbounds string
+ secureCommsKbsAddr string
)
cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) {
@@ -112,8 +116,11 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing")
flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing")
flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods")
- flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels")
- flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels")
+ flags.BoolVar(&secureCommsNoTrustee, "secure-comms-no-trustee", false, "Deliver the keys to peer pods using userdata instead of Trustee")
+ flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "WN Inbound tags for secure communication tunnels")
+ flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "WN Outbound tags for secure communication tunnels")
+ flags.StringVar(&secureCommsPpInbounds, "secure-comms-pp-inbounds", "", "PP Inbound tags for secure communication tunnels")
+ flags.StringVar(&secureCommsPpOutbounds, "secure-comms-pp-outbounds", "", "PP Outbound tags for secure communication tunnels")
flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a Trustee Service for Secure-Comms")
flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection")
@@ -139,8 +146,11 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
}
cfg.serverConfig.SecureComms = true
+ cfg.serverConfig.SecureCommsTrustee = !secureCommsNoTrustee
cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds
cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds
+ cfg.serverConfig.SecureCommsPpInbounds = secureCommsPpInbounds
+ cfg.serverConfig.SecureCommsPpOutbounds = secureCommsPpOutbounds
cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr
} else {
if !disableTLS {
diff --git a/src/cloud-api-adaptor/docs/ScPortForwarding.png b/src/cloud-api-adaptor/docs/ScPortForwarding.png
new file mode 100644
index 000000000..9178866e4
Binary files /dev/null and b/src/cloud-api-adaptor/docs/ScPortForwarding.png differ
diff --git a/src/cloud-api-adaptor/docs/SecureComms.md b/src/cloud-api-adaptor/docs/SecureComms.md
index 85a501b53..5c667e51d 100644
--- a/src/cloud-api-adaptor/docs/SecureComms.md
+++ b/src/cloud-api-adaptor/docs/SecureComms.md
@@ -2,31 +2,95 @@
Here, we describe how to set up the Secure Communications (`Secure Comms` for short) feature of the `cloud-api-adaptor` (`Adaptor`) and `agent-protocol-forwarder` (`Forwarder`).
-The `Secure Comms` feature establishes an SSH channel by the `Adaptor` as an SSH client and the `Forwarder` as an SSH server. All control communications then use this SSH channel between the `Adaptor` and `Forwarder`. Secure Comms use a two-phase approach:
+The `Secure Comms` feature establishes an SSH channel by the `Adaptor`, as an SSH client, and the `Forwarder`, as an SSH server. All control communications then use this SSH channel between the `Adaptor` and `Forwarder`. This feature is an alternative to using `TLS encryption` feature between the `Adaptor` and `Forwarder`.
+
+Both `Secure Comms` and `TLS encryption` supports a method of delivering keys to encrypt the control-plane traffic between the `Adaptor` and the `Forwarder` via user-data during pod VM creation. Delivering the keys via the user-data may allow a malicious entity on the cloud provider side to gain access to the keys and modify any communication between the Worker Nodes and Peer Pods. For this reason, `Secure Comms` also supports an alternate mode where the keys are delivered to the Peer Pod via `Trustee`. The keys are delivered to the Peer Pod after an attestation phase (See diagram below). Delivering the keys via `Trustee` protects the Peer Pod from a malicious entity on the cloud provider side.
+We name this mode of operation as `Trustee Mode`. The option of delivering keys via user-data is named as `NoTrustee Mode`.
+
+
+ 
+
+
+
+To deliver the secrets via `Trustee`, `Secure Comms` under `Trustee Mode` uses a two-phase approach:
- Attestation Phase: In this phase, an SSH channel is created to enable the Peer Pod to attest against an attestation service. The attestation traffic is tunneled from the Peer Pod to the CAA and from there sent to a KBS as configured at the CAA. Once attestation is achieved, the Forwarder obtains Peer Pod keys from the KBS. The Peer Pod keys are dynamically added to the KBS by the CAA prior to starting the Peer Pod.
- Kubernetes Phase: In this phase, an SSH channel is created to enable the runtime to communicate with the Kata Agent running in the Peer Pod. This phase uses the keys obtained during the Attestation Phase, ensuring that the communication is secured.
Additional tunnels can be established via the SSH channel by configuring the Adaptor and Forwarder as described below.
## Secure Comms Architecture
-`Secure Comms` secures the communication between the cluster Worker Nodes and Peer Pods. The Worker Nodes are assumed to be protected by a firewall and can be kept unreachable from outside of the cluster of Worker Nodes. To communicate with Peer Pods, an SSH channel is created from the Worker Node allocated to run the Peer Pod and the Peer Pod VM. This SSH channel can then be used for all communication between the Peer Pod and the cluster and any back end services running at the cluster or elsewhere. As shown in the diagram, once the SSH channel is created, clients at the Peer Pod VM side such as `KBC`, `Attestation Agent`, etc. may connect to services offered by the cluster side such as `KBS`, `Attestation Services`, etc. At the same time, clients located at the cluster side such as the `Runtime shim`, may connect to services offered by the Peer Pod VM side such as the `Kata Agent`.
+`Secure Comms` uses a Go SSH Lib to secure the communication between the cluster Worker Nodes and Peer Pods. To communicate with the Peer Pods, an SSH channel is created between the `Adaptor` in the Worker Node and the `Forwarder` in the Peer Pod.
+The Worker Node is the client part of the channel, and the Peer Pod is the server part of the SSH channel. All communications between the Peer Pod and any services available on the Worker Node side can be configured to use the SSH channel.
+
+Once an SSH channel is established, a Port Forwarding layer, implemented on top of the Go SSH Lib, is used to secure any port to port communication initiated between the `Adaptor` and `Forwarder` and vice versa (see diagram). A client (1) seeking to communicate to the other side, comunicates with a local Inbound component (2) instead. The local Inbound act as a local server for the client. Once a connection is made to the Inbound, a tunnel (3) is created between the local Inbound and a remote Outbound component (4) via the SSH channel. The remote Outbound than acts as a client on behalf of the real client (1) and connect to the remote server (5) to establish an end to end session between the real client (1) and real server (5).
+
+
+ 
+
+Once the SSH channel is created, clients at the Peer Pod side such as `Attestation Agent` connects to the services offered by the cluster side such as `KBS` using the channel. Likewise, clients located at the cluster side such as the `Kata shim`, connects to the services on the Peer Pod side such as the `Kata Agent` using the same channel.
+
+As shown in the following diagram, using `SecureComms`, each Peer Pod is consequently served using a single communication channel implemented as an SSH channel. This single TCP session is initiated from the Worker Node and terminated at the Peer Pod. It is used for any control communication between the Worker Node and Peer Pod and vice versa. This consolidation of communication between a Worker Node and Peer Pod via a single SSH channel improves the operational aspects of Worker Node and Peer Pod communication, especially when Worker Node and Peer Pod are in different networks. The Worker Node network does not need to open ingress ports to expose services to Peer Pod. For example, when the communication between CDH (in Peer Pod) and the Trustee (in the Worker Node network) is via the SSH channel, the Trustee service port does not need to be open to enable communication with CDH (Peer Pod).
+
-SecureComms uses the following sequence of steps:
+`SecureComms` in `Trustee Mode` uses the following sequence of steps:
- Worker Node creates keys for the Peer Pod and updates Trustee
- Worker Node creates the Peer Pod VM
- Worker Node establishes an "Attestation Phase" SSH channel with the Peer Pod VM
- Peer Pod VM attests and obtain keys from Trustee. The Peer Pod then signal to the Worker Node that the "Attestation Phase" has ended and terminates the "Attestation Phase" SSH channel.
-- Worker Node establishes an "Kubernetes Phase" SSH channel with the Peer Pod VM - Both sides verify the identity of the other side using the keys delivered via Trustee.
+- Worker Node establishes a "Kubernetes Phase" SSH channel with the Peer Pod VM - Both sides verify the identity of the other side using the keys delivered via Trustee.
+
+`SecureComms` in `NoTrustee Mode` uses the following sequence of steps:
+- Worker Node creates keys for the Peer Pod
+- Worker Node creates the Peer Pod VM and deliver the keys via daemonConfig as part of the userdata of the Cloud API.
+- Peer Pod VM obtain keys from daemonConfig.
+- Worker Node establishes a "Kubernetes Phase" SSH channel with the Peer Pod VM - Both sides verify the identity of the other side using the keys delivered via daemonConfig.
-Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects the `Runtime shim` to the `Kata Agent` and may also connect other services required by the Peer Pod VM or by containers running inside the Peer Pod.
+Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects the `Kata shim` to the `Kata Agent` and may also connect other services required by the Peer Pod VM or by containers running inside the Peer Pod.
See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details.
+## Setup for testing and for non-CoCo peerpods with NoTrustee mode
+
+### Deploy CAA
+Use any of the option for installing CAA depending on the cloud driver used.
+
+
+### Activate Secure-Comms feature from CAA side
+Make the following parameter changes to the `peer-pods-cm` configMap in the `confidential-containers-system` namespace.
+- Activate Secure-Comms from CAA side by setting the `SECURE_COMMS` parameter to `"true"`.
+- Deactivate Secure-Comms use of Trustee by setting the `SECURE_COMMS_NO_TRUSTEE` parameter to `"true"`.
+
+Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
+```sh
+apiVersion: v1
+data:
+ ...
+ SECURE_COMMS: "true"
+ SECURE_COMMS_NO_TRUSTEE: "true"
+ ...
+```
+
+You may also include additional Inbounds and Outbounds configurations to the Adaptor side using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points.
+You may also add Inbounds and Outbounds configurations to the Forwarder (I.e. Peer Pod, PP) side using the `SECURE_COMMS_PP_INBOUNDS` and `SECURE_COMMS_PP_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
+
+Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
+```sh
+apiVersion: v1
+data:
+ ...
+ SECURE_COMMS: "true"
+ SECURE_COMMS_NO_TRUSTEE: "true"
+ SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
+ SECURE_COMMS_PP_INBOUNDS: "KUBERNETES_PHASE:mytunnel:podns:6666"
+
+ ...
+```
+
## Setup for CoCo with Trustee
### Deploy CAA
@@ -160,7 +224,7 @@ Testing securecomms as a standalone can be done by using:
cd src/cloud-api-adaptor
go run ./test/securecomms/double/main.go
```
-This will create a client and a server and mimic the connection between a CAA client to a PP Server
+This will create a client and a server and mimic the connection between a CAA client to a Peer Pod Server
Successful connection result in exit code of 0
Alternatively, the client and server can be separately executed in independent terminals using `./test/securecomms/double/wnssh.go` and `./test/securecomms/ppssh/main.go` respectively.
@@ -172,6 +236,4 @@ To facilitate end-to-end testing, the libvirt github workflow `e2e_libvirt.yaml`
## Future Plans
- Add DeleteResource() support in KBS, KBC, api-server-rest, than cleanup resources added by Secure Comms to KBS whenever a Peer Pod fail to be created or when a Peer Pod is terminated.
-- Add support for running the vxlan tunnel traffic via a Secure Comms SSH tunnel
-- Add support for non-confidential Peer Pods which do not go via an Attestation Phase.
- Add support for KBS identities allowing a Peer Pod to register its own identity in KBS and replace the current Secure Comms mechanism which delivers a private key to the Peer Pod via the KBS
diff --git a/src/cloud-api-adaptor/docs/SecureComms_TLS.png b/src/cloud-api-adaptor/docs/SecureComms_TLS.png
new file mode 100644
index 000000000..6b90ea51e
Binary files /dev/null and b/src/cloud-api-adaptor/docs/SecureComms_TLS.png differ
diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh
index 3810e6bb9..8fc9af4a4 100755
--- a/src/cloud-api-adaptor/entrypoint.sh
+++ b/src/cloud-api-adaptor/entrypoint.sh
@@ -22,8 +22,11 @@ optionals+=""
[[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} "
[[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify "
[[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms "
+[[ "${SECURE_COMMS_NO_TRUSTEE}" == "true" ]] && optionals+="-secure-comms-no-trustee "
[[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} "
[[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} "
+[[ "${SECURE_COMMS_PP_INBOUNDS}" ]] && optionals+="-secure-comms-pp-inbounds ${SECURE_COMMS_PP_INBOUNDS} "
+[[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} "
[[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
[[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
index e59717ba1..b956f8284 100644
--- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
+++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
@@ -16,6 +16,7 @@ import (
"path/filepath"
"strings"
"sync"
+ "time"
"github.com/containerd/containerd/pkg/cri/annotations"
pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor"
@@ -28,6 +29,7 @@ import (
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util"
+ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil"
provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers"
putil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util/cloudinit"
@@ -45,6 +47,25 @@ type InitData struct {
Data map[string]string `toml:"data,omitempty"`
}
+type ServerConfig struct {
+ TLSConfig *tlsutil.TLSConfig
+ SocketPath string
+ PauseImage string
+ PodsDir string
+ ForwarderPort string
+ ProxyTimeout time.Duration
+ Initdata string
+ EnableCloudConfigVerify bool
+ SecureComms bool
+ SecureCommsTrustee bool
+ SecureCommsInbounds string
+ SecureCommsOutbounds string
+ SecureCommsPpInbounds string
+ SecureCommsPpOutbounds string
+ SecureCommsKbsAddress string
+ PeerPodsLimitPerNode int
+}
+
var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix)
func (s *cloudService) addSandbox(sid sandboxID, sandbox *sandbox) error {
@@ -84,15 +105,20 @@ func (s *cloudService) removeSandbox(id sandboxID) error {
}
func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode,
- secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir,
- daemonPort, initdata, sshport string) Service {
+ serverConfig *ServerConfig, sshport string) Service {
var err error
var sshClient *wnssh.SshClient
- if secureComms {
- inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(secureCommsInbounds, ",")...)
- outbounds := append([]string{"BOTH_PHASES:KBS:" + kbsAddress}, strings.Split(secureCommsOutbounds, ",")...)
- sshClient, err = wnssh.InitSshClient(inbounds, outbounds, kbsAddress, sshport)
+ if serverConfig.SecureComms {
+ inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(serverConfig.SecureCommsInbounds, ",")...)
+
+ var outbounds []string
+ outbounds = append(outbounds, strings.Split(serverConfig.SecureCommsOutbounds, ",")...)
+ if serverConfig.SecureCommsTrustee {
+ outbounds = append(outbounds, "BOTH_PHASES:KBS:"+serverConfig.SecureCommsKbsAddress)
+ }
+
+ sshClient, err = wnssh.InitSshClient(inbounds, outbounds, serverConfig.SecureCommsTrustee, serverConfig.SecureCommsKbsAddress, sshport)
if err != nil {
log.Fatalf("InitSshClient %v", err)
}
@@ -102,9 +128,7 @@ func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNo
provider: provider,
proxyFactory: proxyFactory,
sandboxes: map[sandboxID]*sandbox{},
- podsDir: podsDir,
- daemonPort: daemonPort,
- initdata: initdata,
+ serverConfig: serverConfig,
workerNode: workerNode,
sshClient: sshClient,
}
@@ -222,7 +246,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err)
}
- podDir := filepath.Join(s.podsDir, string(sid))
+ podDir := filepath.Join(s.serverConfig.PodsDir, string(sid))
if err := os.MkdirAll(podDir, os.ModePerm); err != nil {
return nil, fmt.Errorf("creating a pod directory: %s, %w", podDir, err)
}
@@ -266,6 +290,23 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
daemonConfig.TLSServerKey = string(keyPEM)
}
+ var sshCi *wnssh.SshClientInstance
+
+ if s.sshClient != nil {
+ var ppPrivateKey []byte
+ sshCi, ppPrivateKey = s.sshClient.InitPP(context.Background(), string(sid))
+ if sshCi == nil {
+ return nil, fmt.Errorf("failed sshClient.InitPP")
+ }
+ if !s.serverConfig.SecureCommsTrustee {
+ daemonConfig.WnPublicKey = s.sshClient.GetWnPublicKey()
+ daemonConfig.PpPrivateKey = ppPrivateKey
+ daemonConfig.SecureCommsOutbounds = s.serverConfig.SecureCommsPpOutbounds
+ daemonConfig.SecureCommsInbounds = s.serverConfig.SecureCommsPpInbounds
+ daemonConfig.SecureComms = true
+ }
+ }
+
daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ")
if err != nil {
return nil, fmt.Errorf("generating JSON data: %w", err)
@@ -306,19 +347,19 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
logger.Printf("initdata in Pod annotation: %s", initdataStr)
if initdataStr == "" {
- logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.initdata)
- initdataStr = s.initdata
+ logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.serverConfig.Initdata)
+ initdataStr = s.serverConfig.Initdata
}
if initdataStr != "" {
decodedBytes, err := base64.StdEncoding.DecodeString(initdataStr)
if err != nil {
- return nil, fmt.Errorf("Error base64 decode initdata: %w", err)
+ return nil, fmt.Errorf("error base64 decode initdata: %w", err)
}
initdata := InitData{}
err = toml.Unmarshal(decodedBytes, &initdata)
if err != nil {
- return nil, fmt.Errorf("Error unmarshalling initdata: %w", err)
+ return nil, fmt.Errorf("error unmarshalling initdata: %w", err)
}
cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
@@ -328,14 +369,15 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
}
sandbox := &sandbox{
- id: sid,
- podName: pod,
- podNamespace: namespace,
- netNSPath: netNSPath,
- agentProxy: agentProxy,
- podNetwork: podNetworkConfig,
- cloudConfig: cloudConfig,
- spec: vmSpec,
+ id: sid,
+ podName: pod,
+ podNamespace: namespace,
+ netNSPath: netNSPath,
+ agentProxy: agentProxy,
+ podNetwork: podNetworkConfig,
+ cloudConfig: cloudConfig,
+ spec: vmSpec,
+ sshClientInst: sshCi,
}
if err := s.addSandbox(sid, sandbox); err != nil {
@@ -379,24 +421,16 @@ func (s *cloudService) StartVM(ctx context.Context, req *pb.StartVMRequest) (res
logger.Printf("created an instance %s for sandbox %s", instance.Name, sid)
instanceIP := instance.IPs[0].String()
- forwarderPort := s.daemonPort
+ forwarderPort := s.serverConfig.ForwarderPort
if s.sshClient != nil {
- ci := s.sshClient.InitPP(context.Background(), string(sid), instance.IPs)
- if ci == nil {
- return nil, fmt.Errorf("failed sshClient.InitPP")
- }
-
- if err := ci.Start(); err != nil {
+ if err := sandbox.sshClientInst.Start(instance.IPs); err != nil {
return nil, fmt.Errorf("failed SshClientInstance.Start: %w", err)
}
// Set agentProxy
instanceIP = "127.0.0.1"
- forwarderPort = ci.GetPort("KATAAGENT")
-
- // Set ci in sandbox
- sandbox.sshClientInst = ci
+ forwarderPort = sandbox.sshClientInst.GetPort("KATAAGENT")
}
if err := s.workerNode.Setup(sandbox.netNSPath, instance.IPs, sandbox.podNetwork); err != nil {
diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go
index e1da1658b..c3992135f 100644
--- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go
+++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go
@@ -117,7 +117,14 @@ func TestCloudService(t *testing.T) {
podsDir: dir,
}
- s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", "", dir, forwarder.DefaultListenPort, "", "")
+ cfg := &ServerConfig{
+ SecureComms: false,
+ PodsDir: dir,
+ ForwarderPort: forwarder.DefaultListenPort,
+ }
+
+ // false, "", "", "", "", "", dir, forwarder.DefaultListenPort, ""
+ s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, cfg, "")
assert.NotNil(t, s)
@@ -163,7 +170,12 @@ func TestCloudServiceWithSecureComms(t *testing.T) {
// create a podvm
gkc := test.NewGetKeyClient("9019")
ctx2, cancel := context.WithCancel(context.Background())
- sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:9019"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7111"}, ppssh.GetSecret(gkc.GetKey), sshport)
+
+ ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(gkc.GetKey))
+ ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY)
+ ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY)
+
+ sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:9019"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7111"}, ppSecrets, sshport)
_ = sshServer.Start(ctx2)
defer func() {
cancel()
@@ -175,7 +187,15 @@ func TestCloudServiceWithSecureComms(t *testing.T) {
podsDir: dir,
}
- s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, true, "", "", "127.0.0.1:9009", dir, forwarder.DefaultListenPort, "", sshport)
+ cfg := &ServerConfig{
+ SecureComms: true,
+ SecureCommsTrustee: true,
+ PodsDir: dir,
+ ForwarderPort: forwarder.DefaultListenPort,
+ SecureCommsKbsAddress: "127.0.0.1:9009",
+ }
+
+ s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, cfg, sshport)
assert.NotNil(t, s)
diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go
index 6d71b6933..e91fff077 100644
--- a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go
+++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go
@@ -31,12 +31,10 @@ type cloudService struct {
workerNode podnetwork.WorkerNode
sandboxes map[sandboxID]*sandbox
cond *sync.Cond
- podsDir string
- daemonPort string
mutex sync.Mutex
ppService *k8sops.PeerPodService
- initdata string
sshClient *wnssh.SshClient
+ serverConfig *ServerConfig
}
type sandboxID string
diff --git a/src/cloud-api-adaptor/pkg/adaptor/server.go b/src/cloud-api-adaptor/pkg/adaptor/server.go
index 5539777a5..0eae22fef 100644
--- a/src/cloud-api-adaptor/pkg/adaptor/server.go
+++ b/src/cloud-api-adaptor/pkg/adaptor/server.go
@@ -10,7 +10,6 @@ import (
"os"
"path/filepath"
"sync"
- "time"
"github.com/containerd/ttrpc"
pbHypervisor "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor"
@@ -21,7 +20,6 @@ import (
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/vminfo"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/sshutil"
- "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil"
pbPodVMInfo "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/proto/podvminfo"
provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers"
)
@@ -33,22 +31,6 @@ const (
DefaultPodsDir = "/run/peerpod/pods"
)
-type ServerConfig struct {
- TLSConfig *tlsutil.TLSConfig
- SocketPath string
- PauseImage string
- PodsDir string
- ForwarderPort string
- ProxyTimeout time.Duration
- Initdata string
- EnableCloudConfigVerify bool
- SecureComms bool
- SecureCommsInbounds string
- SecureCommsOutbounds string
- SecureCommsKbsAddress string
- PeerPodsLimitPerNode int
-}
-
type Server interface {
Start(ctx context.Context) error
Shutdown() error
@@ -68,14 +50,12 @@ type server struct {
PeerPodsLimitPerNode int
}
-func NewServer(provider provider.Provider, cfg *ServerConfig, workerNode podnetwork.WorkerNode) Server {
+func NewServer(provider provider.Provider, cfg *cloud.ServerConfig, workerNode podnetwork.WorkerNode) Server {
logger.Printf("server config: %#v", cfg)
agentFactory := proxy.NewFactory(cfg.PauseImage, cfg.TLSConfig, cfg.ProxyTimeout)
- cloudService := cloud.NewService(provider, agentFactory, workerNode,
- cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds,
- cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, cfg.Initdata, sshutil.SSHPORT)
+ cloudService := cloud.NewService(provider, agentFactory, workerNode, cfg, sshutil.SSHPORT)
vmInfoService := vminfo.NewService(cloudService)
return &server{
diff --git a/src/cloud-api-adaptor/pkg/adaptor/server_test.go b/src/cloud-api-adaptor/pkg/adaptor/server_test.go
index baa9fc095..da51ae12b 100644
--- a/src/cloud-api-adaptor/pkg/adaptor/server_test.go
+++ b/src/cloud-api-adaptor/pkg/adaptor/server_test.go
@@ -13,6 +13,7 @@ import (
"testing"
"time"
+ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler"
provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers"
@@ -150,7 +151,7 @@ func newServer(t *testing.T, socketPath, podsDir string) Server {
port := startAgentServer(t)
provider := &mockProvider{}
- serverConfig := &ServerConfig{
+ serverConfig := &cloud.ServerConfig{
SocketPath: socketPath,
PodsDir: podsDir,
ForwarderPort: port,
diff --git a/src/cloud-api-adaptor/pkg/adaptor/shim_test.go b/src/cloud-api-adaptor/pkg/adaptor/shim_test.go
index 688973cc4..264ee587c 100644
--- a/src/cloud-api-adaptor/pkg/adaptor/shim_test.go
+++ b/src/cloud-api-adaptor/pkg/adaptor/shim_test.go
@@ -17,6 +17,7 @@ import (
"testing"
"time"
+ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud"
daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder/interceptor"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork"
@@ -86,7 +87,7 @@ func TestShim(t *testing.T) {
workerNode = podnetwork.NewWorkerNode(t, "", 0, 0)
}
- serverConfig := &ServerConfig{
+ serverConfig := &cloud.ServerConfig{
SocketPath: helperSocketPath,
PodsDir: podsDir,
ForwarderPort: port,
diff --git a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go
index fd8a694d0..d175cea85 100644
--- a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go
+++ b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go
@@ -42,6 +42,12 @@ type Config struct {
TLSServerKey string `json:"tls-server-key,omitempty"`
TLSServerCert string `json:"tls-server-cert,omitempty"`
TLSClientCA string `json:"tls-client-ca,omitempty"`
+
+ PpPrivateKey []byte `json:"sc-pp-prv,omitempty"`
+ WnPublicKey []byte `json:"sc-wn-pub,omitempty"`
+ SecureCommsInbounds string `json:"sc-inbounds,omitempty"`
+ SecureCommsOutbounds string `json:"sc-outbounds,omitempty"`
+ SecureComms bool `json:"sc,omitempty"`
}
type Daemon interface {
diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go
index 1707f6473..48bc6221e 100644
--- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go
+++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go
@@ -5,7 +5,6 @@ import (
)
type PpSecrets struct {
- keys []string
secrets map[string][]byte
getSecret GetSecret
}
@@ -14,44 +13,54 @@ type GetSecret func(name string) ([]byte, error)
func NewPpSecrets(getSecret GetSecret) *PpSecrets {
return &PpSecrets{
- keys: []string{},
secrets: make(map[string][]byte),
getSecret: getSecret,
}
}
-func (fs *PpSecrets) AddKey(key string) {
- fs.keys = append(fs.keys, key)
+func (sec *PpSecrets) AddKey(key string) {
+ if _, ok := sec.secrets[key]; ok {
+ return
+ }
+ sec.secrets[key] = nil
}
-func (fs *PpSecrets) GetKey(key string) []byte {
- return fs.secrets[key]
+func (sec *PpSecrets) GetKey(key string) []byte {
+ return sec.secrets[key]
}
-func (fs *PpSecrets) Go() {
- sleeptime := time.Duration(1)
+func (sec *PpSecrets) SetKey(key string, keydata []byte) {
+ sec.secrets[key] = keydata
+}
- for len(fs.keys) > 0 {
- key := fs.keys[0]
- logger.Printf("PpSecrets obtaining key %s", key)
+func (sec *PpSecrets) Go() {
+ sleeptime := time.Duration(1)
- data, err := fs.getSecret(key)
- if err == nil && len(data) > 0 {
- logger.Printf("PpSecrets %s success", key)
- fs.secrets[key] = data
- fs.keys = fs.keys[1:]
+ for key, keydata := range sec.secrets {
+ if keydata != nil {
continue
}
- if err != nil {
- logger.Printf("PpSecrets %s getSecret err: %v", key, err)
- } else {
- logger.Printf("PpSecrets %s getSecret returned an empty secret", key)
- }
+ logger.Printf("PpSecrets obtaining key %s", key)
+
+ // loop until we get a valid key
+ for {
+ keydata, err := sec.getSecret(key)
+ if err == nil && len(keydata) > 0 {
+ logger.Printf("PpSecrets %s success", key)
+ sec.secrets[key] = keydata
+ break
+ }
+ if err != nil {
+ logger.Printf("PpSecrets %s getSecret err: %v", key, err)
+ } else {
+ logger.Printf("PpSecrets %s getSecret returned an empty secret", key)
+ }
- time.Sleep(sleeptime * time.Second)
- sleeptime *= 2
- if sleeptime > 30 {
- sleeptime = 30
+ time.Sleep(sleeptime * time.Second)
+ sleeptime *= 2
+ if sleeptime > 30 {
+ sleeptime = 30
+ }
}
}
}
diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go
index d13a24383..08feb291d 100644
--- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go
+++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go
@@ -30,7 +30,7 @@ type SshServer struct {
outbounds sshproxy.Outbounds
wg sync.WaitGroup
readyCh chan struct{}
- getSecret GetSecret
+ ppSecrets *PpSecrets
sshport string
listener net.Listener
ctx context.Context
@@ -42,9 +42,9 @@ type SshServer struct {
// Structure of an inbound tag: "::"
// Structure of an outbound tag: ":::"
// Phase may be "A" (Attestation), "K" (Kubernetes), or "B" (Both)
-func NewSshServer(inbound_strings, outbounds_strings []string, getSecret GetSecret, sshport string) *SshServer {
+func NewSshServer(inbound_strings, outbounds_strings []string, ppSecrets *PpSecrets, sshport string) *SshServer {
s := &SshServer{
- getSecret: getSecret,
+ ppSecrets: ppSecrets,
sshport: sshport,
readyCh: make(chan struct{}),
}
@@ -119,11 +119,8 @@ func (s *SshServer) attestationPhase() *ssh.ServerConfig {
for ctx.Err() == nil {
logger.Printf("Attestation phase: getting keys from KBS\n")
- ppSecrets := NewPpSecrets(s.getSecret)
- ppSecrets.AddKey(WN_PUBLIC_KEY)
- ppSecrets.AddKey(PP_PRIVATE_KEY)
- ppSecrets.Go() // wait for the keys
- config, err := initKubernetesPhaseSshConfig(ppSecrets)
+ s.ppSecrets.Go() // wait for the keys
+ config, err := initKubernetesPhaseSshConfig(s.ppSecrets)
if err == nil {
logger.Printf("Attestation phase: InitKubernetesPhaseSshConfig is ready\n")
peer.Upgrade()
diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go
index 467723e92..a38b9f817 100644
--- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go
+++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go
@@ -183,7 +183,12 @@ func TestPpssh(t *testing.T) {
// Forwarder Initialization
ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
- sshServer := NewSshServer([]string{"BOTH_PHASES:KBS:9002"}, []string{"KUBERNETES_PHASE:ABC:127.0.0.1:7105"}, GetSecret(getKey), sshport)
+
+ ppSecrets := NewPpSecrets(GetSecret(getKey))
+ ppSecrets.AddKey(WN_PUBLIC_KEY)
+ ppSecrets.AddKey(PP_PRIVATE_KEY)
+
+ sshServer := NewSshServer([]string{"BOTH_PHASES:KBS:9002"}, []string{"KUBERNETES_PHASE:ABC:127.0.0.1:7105"}, ppSecrets, sshport)
_ = sshServer.Start(ctx)
clientSshPeer, conn := getAttestationClient(t, sshport)
clientSshPeer.AddTags(inbounds, outbounds)
diff --git a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go
index d252d609f..d23b54804 100644
--- a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go
+++ b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go
@@ -25,6 +25,7 @@ type SshClient struct {
inboundStrings []string
outboundStrings []string
sshport string
+ wnPublicKey []byte
}
type SshClientInstance struct {
@@ -51,7 +52,7 @@ func PpSecretName(sid string) string {
// Structure of an inbound tag: "::"
// Structure of an outbound tag: ":::"
// Phase may be "A" (Attestation), "K" (Kubernetes), or "B" (Both)
-func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string, sshport string) (*SshClient, error) {
+func InitSshClient(inbound_strings, outbound_strings []string, secureCommsTrustee bool, kbsAddress string, sshport string) (*SshClient, error) {
logger.Printf("Using PP SecureComms: InitSshClient version %s", sshutil.PpSecureCommsVersion)
// Read WN Secret
@@ -72,22 +73,25 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string
return nil, fmt.Errorf("unable to parse private key: %v", err)
}
- kbscPrivateKey, _, err := kubemgr.KubeMgr.ReadSecret(sshutil.KBS_CLIENT_SECRET)
- if err != nil {
- return nil, fmt.Errorf("failed to read KBS client secret: %w", err)
- }
+ var kc *KbsClient
+ if secureCommsTrustee {
+ kbscPrivateKey, _, err := kubemgr.KubeMgr.ReadSecret(sshutil.KBS_CLIENT_SECRET)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read KBS client secret: %w", err)
+ }
- kc := InitKbsClient(kbsAddress)
- err = kc.SetPemSecret(kbscPrivateKey)
- if err != nil {
- return nil, fmt.Errorf("KbsClient - %v", err)
- }
+ kc = InitKbsClient(kbsAddress)
+ err = kc.SetPemSecret(kbscPrivateKey)
+ if err != nil {
+ return nil, fmt.Errorf("KbsClient - %v", err)
+ }
- wnSecretPath := "default/sshclient/publicKey"
- logger.Printf("Updating KBS with secret for: %s", wnSecretPath)
- err = kc.PostResource(wnSecretPath, wnPublicKey)
- if err != nil {
- return nil, fmt.Errorf("failed to PostResource WN Secret: %v", err)
+ wnSecretPath := "default/sshclient/publicKey"
+ logger.Printf("Updating KBS with secret for: %s", wnSecretPath)
+ err = kc.PostResource(wnSecretPath, wnPublicKey)
+ if err != nil {
+ return nil, fmt.Errorf("failed to PostResource WN Secret: %v", err)
+ }
}
sshClient := &SshClient{
@@ -96,6 +100,7 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string
inboundStrings: inbound_strings,
outboundStrings: outbound_strings,
sshport: sshport,
+ wnPublicKey: wnPublicKey,
}
return sshClient, nil
@@ -123,9 +128,13 @@ func (ci *SshClientInstance) DisconnectPP(sid string) {
kubemgr.KubeMgr.DeleteSecret(PpSecretName(sid))
}
-func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) *SshClientInstance {
+func (c *SshClient) GetWnPublicKey() []byte {
+ return c.wnPublicKey
+}
+
+func (c *SshClient) InitPP(ctx context.Context, sid string) (ci *SshClientInstance, ppPrivateKey []byte) {
// Create peerPod Secret named peerPodId
- var ppPublicKey, ppPrivateKey []byte
+ var ppPublicKey []byte
var err error
var kubernetesPhase bool
@@ -136,43 +145,39 @@ func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr)
ppPrivateKey, ppPublicKey, err = kubemgr.KubeMgr.CreateSecret(PpSecretName(sid))
if err != nil {
logger.Printf("Failed to create PP secret: %v", err)
- return nil
+ return
}
} else {
// we already have a store secret for this PP
kubernetesPhase = true
}
- // >>> Update the KBS about the SID's Secret !!! <<<
- sidSecretPath := fmt.Sprintf("default/pp-%s/privateKey", sid)
- logger.Printf("Updating KBS with secret for: %s", sidSecretPath)
- err = c.kc.PostResource(sidSecretPath, ppPrivateKey)
- if err != nil {
- logger.Printf("Failed to PostResource PP Secret: %v", err)
- return nil
- }
+ if c.kc != nil {
+ // >>> Update the KBS about the SID's Secret !!! <<<
+ sidSecretPath := fmt.Sprintf("default/pp-%s/privateKey", sid)
+ logger.Printf("Updating KBS with secret for: %s", sidSecretPath)
+ err = c.kc.PostResource(sidSecretPath, ppPrivateKey)
+ if err != nil {
+ logger.Printf("Failed to PostResource PP Secret: %v", err)
+ return
+ }
+ }
var ppSshPublicKeyBytes []byte
if len(ppPublicKey) > 0 {
ppSshPublicKey, _, _, _, err := ssh.ParseAuthorizedKey(ppPublicKey)
if err != nil {
logger.Printf("Unable to ParseAuthorizedKey serverPublicKey: %v", err)
- return nil
+ return
}
ppSshPublicKeyBytes = ppSshPublicKey.Marshal()
}
- ppAddr := make([]string, len(ipAddr))
- for i, ip := range ipAddr {
- ppAddr[i] = ip.String() + ":" + c.sshport
- }
-
ctx, cancel := context.WithCancel(ctx)
- ci := &SshClientInstance{
+ ci = &SshClientInstance{
sid: sid,
ppPublicKey: ppSshPublicKeyBytes,
- ppAddr: ppAddr,
sshClient: c,
ctx: ctx,
cancel: cancel,
@@ -188,10 +193,17 @@ func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr)
logger.Fatalf("Failed to parse outbound tag %v: %v", c.outboundStrings, err)
}
- return ci
+ return
}
-func (ci *SshClientInstance) Start() error {
+func (ci *SshClientInstance) Start(ipAddr []netip.Addr) error {
+ ppAddr := make([]string, len(ipAddr))
+ for i, ip := range ipAddr {
+ ppAddr[i] = ip.String() + ":" + ci.sshClient.sshport
+ }
+
+ ci.ppAddr = ppAddr
+
if !ci.kubernetesPhase {
// Attestation phase
logger.Println("Attestation phase: starting")
diff --git a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go
index cce5d6678..8f9169048 100644
--- a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go
+++ b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go
@@ -26,7 +26,7 @@ func TestSshProxyReverseKBS(t *testing.T) {
test.CreatePKCS8Secret(t)
// CAA Initialization
- sshClient, err := InitSshClient([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, []string{"BOTH_PHASES:KBS:9001", "KUBERNETES_PHASE:KUBEAPI:26443", "KUBERNETES_PHASE:DNS:8053"}, "127.0.0.1:9001", sshport)
+ sshClient, err := InitSshClient([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, []string{"BOTH_PHASES:KBS:9001", "KUBERNETES_PHASE:KUBEAPI:26443", "KUBERNETES_PHASE:DNS:8053"}, true, "127.0.0.1:9001", sshport)
if err != nil {
log.Fatalf("InitSshClient %v", err)
}
@@ -34,7 +34,7 @@ func TestSshProxyReverseKBS(t *testing.T) {
////////// CAA StartVM
ipAddr, _ := netip.ParseAddr("127.0.0.1") // ipAddr of the VM
ipAddrs := []netip.Addr{ipAddr}
- ci := sshClient.InitPP(context.Background(), "sid", ipAddrs)
+ ci, _ := sshClient.InitPP(context.Background(), "sid")
if ci == nil {
log.Fatalf("failed InitiatePeerPodTunnel")
}
@@ -48,12 +48,17 @@ func TestSshProxyReverseKBS(t *testing.T) {
// create a podvm
gkc := test.NewGetKeyClient("7030")
ctx2, cancel2 := context.WithCancel(context.Background())
- sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7030", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7121"}, ppssh.GetSecret(gkc.GetKey), sshport)
+
+ ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(gkc.GetKey))
+ ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY)
+ ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY)
+
+ sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7030", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7121"}, ppSecrets, sshport)
_ = sshServer.Start(ctx2)
// Forwarder Initialization
- if err := ci.Start(); err != nil {
+ if err := ci.Start(ipAddrs); err != nil {
log.Fatalf("failed ci.Start: %s", err)
}
diff --git a/src/cloud-api-adaptor/test/securecomms/ppssh.go b/src/cloud-api-adaptor/test/securecomms/ppssh.go
index b9a15d82d..337f3f39e 100644
--- a/src/cloud-api-adaptor/test/securecomms/ppssh.go
+++ b/src/cloud-api-adaptor/test/securecomms/ppssh.go
@@ -20,7 +20,12 @@ func PP() {
// Forwarder Initialization
ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
- sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7000", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7131"}, ppssh.GetSecret(getKey), sshutil.SSHPORT)
+
+ ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(getKey))
+ ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY)
+ ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY)
+
+ sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7000", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7131"}, ppSecrets, sshutil.SSHPORT)
_ = sshServer.Start(ctx)
time.Sleep(1 * time.Minute)
cancel()
diff --git a/src/cloud-api-adaptor/test/securecomms/wnssh.go b/src/cloud-api-adaptor/test/securecomms/wnssh.go
index 026125e03..d4683f12f 100644
--- a/src/cloud-api-adaptor/test/securecomms/wnssh.go
+++ b/src/cloud-api-adaptor/test/securecomms/wnssh.go
@@ -21,7 +21,7 @@ func WN() bool {
test.HttpServer("26443")
// CAA Initialization
- sshClient, err := wnssh.InitSshClient([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, []string{"BOTH_PHASES:KBS:9004", "KUBERNETES_PHASE:KUBEAPI:26443", "KUBERNETES_PHASE:DNS:8053"}, "127.0.0.1:9004", sshutil.SSHPORT)
+ sshClient, err := wnssh.InitSshClient([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, []string{"BOTH_PHASES:KBS:9004", "KUBERNETES_PHASE:KUBEAPI:26443", "KUBERNETES_PHASE:DNS:8053"}, true, "127.0.0.1:9004", sshutil.SSHPORT)
if err != nil {
log.Fatalf("InitSshClient %v", err)
}
@@ -30,7 +30,7 @@ func WN() bool {
ipAddr, _ := netip.ParseAddr("127.0.0.1") // ipAddr of the VM
ipAddrs := []netip.Addr{ipAddr}
ctx := context.Background()
- ci := sshClient.InitPP(ctx, "sid", ipAddrs)
+ ci, _ := sshClient.InitPP(ctx, "sid")
if ci == nil {
log.Fatalf("failed InitiatePeerPodTunnel")
}
@@ -41,7 +41,7 @@ func WN() bool {
log.Fatalf("failed find port")
}
- if err := ci.Start(); err != nil {
+ if err := ci.Start(ipAddrs); err != nil {
log.Fatalf("failed ci.Start: %s", err)
}