Skip to content

V0.6.3 #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: v0.6.3
Choose a base branch
from
Open

V0.6.3 #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
44f30c2
update image tag and support fip annotation
Goend Aug 4, 2022
d4a1fa5
update use trust openstack auth
Goend Aug 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ endif

# Release variables

STAGING_REGISTRY := gcr.io/k8s-staging-capi-openstack
STAGING_REGISTRY := hub.ecns.io/test
STAGING_BUCKET ?= artifacts.k8s-staging-capi-openstack.appspot.com
BUCKET ?= $(STAGING_BUCKET)
PROD_REGISTRY ?= k8s.gcr.io/capi-openstack
Expand Down
54 changes: 47 additions & 7 deletions controllers/openstackmachine_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,19 @@ import (
"time"

"github.com/go-logr/logr"
networkport "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"k8s.io/utils/pointer"
infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha5"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/compute"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/loadbalancer"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/provider"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/scope"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
capierrors "sigs.k8s.io/cluster-api/errors"
"sigs.k8s.io/cluster-api/util"
Expand All @@ -46,13 +53,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"

infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha5"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/compute"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/loadbalancer"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/provider"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/scope"
)

// OpenStackMachineReconciler reconciles a OpenStackMachine object.
Expand Down Expand Up @@ -388,6 +388,46 @@ func (r *OpenStackMachineReconciler) reconcileNormal(ctx context.Context, scope
}
}

// when machine have machinedeployment.clusters.x-k8s.io/fip=enable annotation
// we should give the fip to machine
if machine.Annotations["machinedeployment.clusters.x-k8s.io/fip"] == "enable" && len(addresses)==1 {
fp, err := networkingService.GetOrCreateFloatingIP(openStackMachine, openStackCluster, clusterName, "")
if err != nil {
handleUpdateMachineError(scope.Logger, openStackMachine, errors.Errorf("Floating IP cannot be created: %v", err))
return ctrl.Result{}, nil
}
var port = new(networkport.Port)
// we should get machine network name if we define network
if len(openStackMachine.Spec.Networks) > 0 {
nets, err := instanceStatus.NetworkStatus()
if err != nil {
err = errors.Errorf("failed to get openstackmachine template: %v", err)
handleUpdateMachineError(scope.Logger, openStackMachine, err)
return ctrl.Result{}, nil
}
scope.Logger.Info(nets.Addresses()[0].Address)
pos, err := networkingService.GetPortFromInstanceIP(*openStackMachine.Spec.InstanceID, nets.Addresses()[0].Address)
if err != nil {
err = errors.Errorf("getting management port for machine deployment machine %s: %v", err)
handleUpdateMachineError(scope.Logger, openStackMachine, err)
return ctrl.Result{}, nil
}
port = &pos[0]
} else {
port, err = computeService.GetManagementPort(openStackCluster, instanceStatus)
if err != nil {
err = errors.Errorf("getting management port for control plane machine %s: %v", machine.Name, err)
handleUpdateMachineError(scope.Logger, openStackMachine, err)
return ctrl.Result{}, nil
}
}
err = networkingService.AssociateFloatingIP(openStackMachine, fp, port.ID)
if err != nil {
handleUpdateMachineError(scope.Logger, openStackMachine, errors.Errorf("Floating IP %s cannot be associated: %v by port id %s", fp,err,port.ID))
return ctrl.Result{}, nil
}
}

scope.Logger.Info("Reconciled Machine create successfully")
return ctrl.Result{}, nil
}
Expand Down
101 changes: 88 additions & 13 deletions pkg/cloud/services/provider/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"crypto/tls"
"crypto/x509"
"fmt"
"github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts"
"net/http"

"github.com/gophercloud/gophercloud"
Expand All @@ -37,13 +38,58 @@ import (
infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha5"
)

type NewAuthInfo struct {
clientconfig.AuthInfo
TrustID string `yaml:"trust_id,omitempty" json:"trust_id,omitempty"`
}

// NewCloud represents an entry in a clouds.yaml/public-clouds.yaml/secure.yaml file.
type NewCloud struct {
Cloud string `yaml:"cloud,omitempty" json:"cloud,omitempty"`
Profile string `yaml:"profile,omitempty" json:"profile,omitempty"`
AuthInfo *NewAuthInfo `yaml:"auth,omitempty" json:"auth,omitempty"`
AuthType clientconfig.AuthType `yaml:"auth_type,omitempty" json:"auth_type,omitempty"`
RegionName string `yaml:"region_name,omitempty" json:"region_name,omitempty"`
Regions []clientconfig.Region `yaml:"regions,omitempty" json:"regions,omitempty"`

// EndpointType and Interface both specify whether to use the public, internal,
// or admin interface of a service. They should be considered synonymous, but
// EndpointType will take precedence when both are specified.
EndpointType string `yaml:"endpoint_type,omitempty" json:"endpoint_type,omitempty"`
Interface string `yaml:"interface,omitempty" json:"interface,omitempty"`

// API Version overrides.
IdentityAPIVersion string `yaml:"identity_api_version,omitempty" json:"identity_api_version,omitempty"`
VolumeAPIVersion string `yaml:"volume_api_version,omitempty" json:"volume_api_version,omitempty"`

// Verify whether or not SSL API requests should be verified.
Verify *bool `yaml:"verify,omitempty" json:"verify,omitempty"`

// CACertFile a path to a CA Cert bundle that can be used as part of
// verifying SSL API requests.
CACertFile string `yaml:"cacert,omitempty" json:"cacert,omitempty"`

// ClientCertFile a path to a client certificate to use as part of the SSL
// transaction.
ClientCertFile string `yaml:"cert,omitempty" json:"cert,omitempty"`

// ClientKeyFile a path to a client key to use as part of the SSL
// transaction.
ClientKeyFile string `yaml:"key,omitempty" json:"key,omitempty"`
}
Copy link

@chestack chestack Aug 15, 2022 •
edited by Goend
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个 不能复用 "github.com/gophercloud/utils/openstack/clientconfig" 的 type Cloud struct {} 吗

Copy link
Author

@Goend Goend Aug 16, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about add fip options into openstackmachine spec

add annotations to machine deployment, machinedeployment.clusters.x-k8s.io/fip: "enable",and it is ok for openstackmachine

Copy link
Author

@Goend Goend Aug 16, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

关于trust 的使用有几个问题

1. trust 何时创建，magnum里是创建cluster时 创建trust

2. 因为 token会过期，password会修改，使用trust是为了解决 后续 访问openstack API的问题，场景主要是 cloud provider使用

[root@zxy-clustern0 ~]# cat /etc/kubernetes/cloud-config
[Global]
auth-url=http://keystone.openstack.svc.cluster.local:80/v3
user-id=d8431d5559f54edd8dc31811921ae6d6
password=4wmU2NG4PAFAz7vN89
trust-id=8d4b24f9cbc047e2bd4370d4d3d5a2fd
ca-file=/etc/kubernetes/ca-bundle.crt
region=RegionOne
[LoadBalancer]
use-octavia=True
subnet-id=
floating-network-id=
create-monitor=yes
monitor-delay=1m
monitor-timeout=30s
monitor-max-retries=3
[BlockStorage]
bs-version=v2
ignore-volume-az=True

trust 当前没有创建逻辑 如果需要 需要在capi中补充 即创建cluster资源时 需要创建trust用户
这里认证是解决创建虚拟机时使用trust用户认证 master虚拟机内部的cloud-config 生成是由KubeadmControlPlane资源下spec.files下的content内容决定的 work虚拟机是由KubeadmConfigTemplate 下template.spec.files下的content内容决定的 使用clusterctl命令行工具 在生成配置时会帮我们注入 但这里 我们应该自己完成这个配置的组装和base64并填写到这里

Copy link
Author

@Goend Goend Aug 16, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个 不能复用 "github.com/gophercloud/utils/openstack/clientconfig" 的 type Cloud struct {} 吗

"github.com/gophercloud/utils/openstack/clientconfig" 的 type Cloud struct {} 中AuthInfo 是默认不包含trust id，我将其补充了进去 外层使用NewCloud。并使用此字段的有无来决定认证的方式 只使用原来的cloud struct我无法解析应该位于auth下的trust信息 应该是不行
示例

  • 现在
[root@node-3 ~]# cat cloud_config.yaml 
clouds:
  test-trust:
    identity_api_version: 3
    auth:
      auth_url: http://keystone.openstack.svc.cluster.local/v3
      user_id: 8b924ec3792b452987cff3cadd44d35f
      password: tocn8v4JbjT8AuJx8D
      trust_id: 63b279acae8e46fe8c3a5f7384147935
    region_name: RegionOne
  • 原来
clouds:
  test:
    identity_api_version: 3
    auth:
      auth_url: http://keystone.openstack.svc.cluster.local/v3
      project_domain_name: Default
      user_domain_name: Default
      project_name: admin
      username: admin
      password: test@passw0rd
    region_name: RegionOne


type NewClouds struct {
Clouds map[string]NewCloud `yaml:"clouds" json:"clouds"`
}

const (
cloudsSecretKey = "clouds.yaml"
caSecretKey = "cacert"
)


func NewClientFromMachine(ctx context.Context, ctrlClient client.Client, openStackMachine *infrav1.OpenStackMachine) (*gophercloud.ProviderClient, *clientconfig.ClientOpts, string, error) {
var cloud clientconfig.Cloud
var cloud NewCloud
var caCert []byte

if openStackMachine.Spec.IdentityRef != nil {
Expand All @@ -57,7 +103,7 @@ func NewClientFromMachine(ctx context.Context, ctrlClient client.Client, openSta
}

func NewClientFromCluster(ctx context.Context, ctrlClient client.Client, openStackCluster *infrav1.OpenStackCluster) (*gophercloud.ProviderClient, *clientconfig.ClientOpts, string, error) {
var cloud clientconfig.Cloud
var cloud NewCloud
var caCert []byte

if openStackCluster.Spec.IdentityRef != nil {
Expand All @@ -70,10 +116,10 @@ func NewClientFromCluster(ctx context.Context, ctrlClient client.Client, openSta
return NewClient(cloud, caCert)
}

func NewClient(cloud clientconfig.Cloud, caCert []byte) (*gophercloud.ProviderClient, *clientconfig.ClientOpts, string, error) {
func NewClient(cloud NewCloud, caCert []byte) (*gophercloud.ProviderClient, *clientconfig.ClientOpts, string, error) {
clientOpts := new(clientconfig.ClientOpts)
if cloud.AuthInfo != nil {
clientOpts.AuthInfo = cloud.AuthInfo
clientOpts.AuthInfo = &cloud.AuthInfo.AuthInfo
clientOpts.AuthType = cloud.AuthType
clientOpts.RegionName = cloud.RegionName
}
Expand All @@ -84,11 +130,11 @@ func NewClient(cloud clientconfig.Cloud, caCert []byte) (*gophercloud.ProviderCl
}
opts.AllowReauth = true


provider, err := openstack.NewClient(opts.IdentityEndpoint)
if err != nil {
return nil, nil, "", fmt.Errorf("create providerClient err: %v", err)
}

config := &tls.Config{
RootCAs: x509.NewCertPool(),
MinVersion: tls.VersionTLS12,
Expand All @@ -101,17 +147,46 @@ func NewClient(cloud clientconfig.Cloud, caCert []byte) (*gophercloud.ProviderCl
}

provider.HTTPClient.Transport = &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: config}
if klog.V(6).Enabled() {
provider.HTTPClient.Transport = &osclient.RoundTripper{
Rt: provider.HTTPClient.Transport,
Logger: &defaultLogger{},
provider.HTTPClient.Transport = &osclient.RoundTripper{
Rt: provider.HTTPClient.Transport,
Logger: &defaultLogger{},
}
if cloud.AuthInfo.TrustID!="" {
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

判断条件在这里

tokenauth:=tokens.AuthOptions{}
tokenauth.IdentityEndpoint=opts.IdentityEndpoint
tokenauth.UserID=opts.UserID
tokenauth.Username=opts.Username
tokenauth.Password=opts.Password
tokenauth.DomainID=opts.DomainID
tokenauth.DomainName=opts.DomainName
tokenauth.ApplicationCredentialID=opts.ApplicationCredentialID
tokenauth.ApplicationCredentialName=opts.ApplicationCredentialName
tokenauth.ApplicationCredentialSecret=opts.ApplicationCredentialSecret
tokenauth.AllowReauth=opts.AllowReauth
if opts.Scope!=nil {
tokenauth.Scope.ProjectID=opts.Scope.ProjectID
tokenauth.Scope.ProjectName=opts.Scope.ProjectName
tokenauth.Scope.DomainName=opts.Scope.DomainName
tokenauth.Scope.DomainID=opts.Scope.DomainID
}
authOptsExt := trusts.AuthOptsExt{
TrustID: cloud.AuthInfo.TrustID,
AuthOptionsBuilder: &tokenauth,
}
err = openstack.AuthenticateV3(provider, authOptsExt, gophercloud.EndpointOpts{})
if err != nil {
return nil, nil, "", fmt.Errorf("providerClient authentication err: %v", err)
}
projectID, err := getProjectIDFromAuthResult(provider.GetAuthResult())
if err != nil {
return nil, nil, "", err
}
return provider,clientOpts,projectID,nil
}
err = openstack.Authenticate(provider, *opts)
if err != nil {
return nil, nil, "", fmt.Errorf("providerClient authentication err: %v", err)
}

projectID, err := getProjectIDFromAuthResult(provider.GetAuthResult())
if err != nil {
return nil, nil, "", err
Expand All @@ -128,8 +203,8 @@ func (defaultLogger) Printf(format string, args ...interface{}) {
}

// getCloudFromSecret extract a Cloud from the given namespace:secretName.
func getCloudFromSecret(ctx context.Context, ctrlClient client.Client, secretNamespace string, secretName string, cloudName string) (clientconfig.Cloud, []byte, error) {
emptyCloud := clientconfig.Cloud{}
func getCloudFromSecret(ctx context.Context, ctrlClient client.Client, secretNamespace string, secretName string, cloudName string) (NewCloud, []byte, error) {
emptyCloud := NewCloud{}

if secretName == "" {
return emptyCloud, nil, nil
Expand All @@ -153,7 +228,7 @@ func getCloudFromSecret(ctx context.Context, ctrlClient client.Client, secretNam
return emptyCloud, nil, fmt.Errorf("OpenStack credentials secret %v did not contain key %v",
secretName, cloudsSecretKey)
}
var clouds clientconfig.Clouds
var clouds NewClouds
if err = yaml.Unmarshal(content, &clouds); err != nil {
return emptyCloud, nil, fmt.Errorf("failed to unmarshal clouds credentials stored in secret %v: %v", secretName, err)
}
Expand Down