Support for external machine controller (driver)

LICENSE.md

@@ -299,6 +299,12 @@ Alibaba Cloud Go Software Development Kit.
 https://github.com/aliyun/alibaba-cloud-sdk-go.  
 Apache 2 license (https://github.com/aliyun/alibaba-cloud-sdk-go/blob/master/LICENSE)  
 
+gRPC-Go.  
+https://github.com/grpc/grpc-go.  
+Copyright 2017 gRPC authors.  
+Apache 2 license (https://github.com/grpc/grpc-go/blob/master/LICENSE)  
+
+
 ------
 ## BSD 3-clause "New" or "Revised" License
 

Makefile

@@ -42,7 +42,7 @@ start:
 			--machine-safety-apiserver-statuscheck-period=1m \
 			--machine-safety-orphan-vms-period=30m \
 			--machine-safety-overshooting-period=1m \
-			--v=2
+			--v=3
 
 #################################################################
 # Rules related to binary build, Docker image build and release #

README.md

@@ -15,7 +15,7 @@ kubectl create/get/delete machine vm1
 
 Nodes/Machines/VMs are different terminologies used to represent similar things. We use these terms in the following way
 
-1. VM: A virtual machine running on any cloud provider.
+1. VM: A virtual machine running on any cloud provider. It could also refer to a physical machine (PM) in case of a bare metal setup.
 1. Node: Native kubernetes node objects. The objects you get to see when you do a *"kubectl get nodes"*. Although nodes can be either physical/virtual machines, for the purposes of our discussions it refers to a VM.
 1. Machine: A VM that is provisioned/managed by the Machine Controller Manager.
 

cmd/machine-controller-manager/app/controllermanager.go

@@ -34,8 +34,9 @@ import (
 
 	machinescheme "github.com/gardener/machine-controller-manager/pkg/client/clientset/versioned/scheme"
 	machineinformers "github.com/gardener/machine-controller-manager/pkg/client/informers/externalversions"
-	machinecontroller "github.com/gardener/machine-controller-manager/pkg/controller"
-	corecontroller "github.com/gardener/machine-controller-manager/pkg/util/controller"
+	mcmcontroller "github.com/gardener/machine-controller-manager/pkg/controller"
+	corecontroller "github.com/gardener/machine-controller-manager/pkg/util/clientbuilder/core"
+	machinecontroller "github.com/gardener/machine-controller-manager/pkg/util/clientbuilder/machine"
 	coreinformers "k8s.io/client-go/informers"
 	kubescheme "k8s.io/client-go/kubernetes/scheme"
 
@@ -63,12 +64,11 @@ const (
 	controllerManagerAgentName = "machine-controller-manager"
 )
 
-var openStackGVR = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "openstackmachineclasses"}
-var awsGVR = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "awsmachineclasses"}
-var azureGVR = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "azuremachineclasses"}
-var gcpGVR = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "gcpmachineclasses"}
-var alicloudGVR = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "alicloudmachineclasses"}
-var packetGVR = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "packetmachineclasses"}
+var (
+	machineGVR           = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "machines"}
+	machineSetGVR        = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "machinesets"}
+	machineDeploymentGVR = schema.GroupVersionResource{Group: "machine.sapcloud.io", Version: "v1alpha1", Resource: "machinedeployments"}
+)
 
 // Run runs the MCMServer.  This should never exit.
 func Run(s *options.MCMServer) error {
@@ -224,7 +224,7 @@ func StartControllers(s *options.MCMServer,
 		klog.Fatal(err)
 	}
 
-	if availableResources[awsGVR] || availableResources[azureGVR] || availableResources[gcpGVR] || availableResources[openStackGVR] || availableResources[alicloudGVR] || availableResources[packetGVR] {
+	if availableResources[machineGVR] || availableResources[machineSetGVR] || availableResources[machineDeploymentGVR] {
 		klog.V(5).Infof("Creating shared informers; resync interval: %v", s.MinResyncPeriod)
 
 		controlMachineInformerFactory := machineinformers.NewFilteredSharedInformerFactory(
@@ -250,7 +250,7 @@ func StartControllers(s *options.MCMServer,
 		machineSharedInformers := controlMachineInformerFactory.Machine().V1alpha1()
 
 		klog.V(5).Infof("Creating controllers...")
-		machineController, err := machinecontroller.NewController(
+		mcmcontroller, err := mcmcontroller.NewController(
 			s.Namespace,
 			controlMachineClient,
 			controlCoreClient,
@@ -283,10 +283,10 @@ func StartControllers(s *options.MCMServer,
 		targetCoreInformerFactory.Start(stop)
 
 		klog.V(5).Info("Running controller")
-		go machineController.Run(int(s.ConcurrentNodeSyncs), stop)
+		go mcmcontroller.Run(int(s.ConcurrentNodeSyncs), stop)
 
 	} else {
-		return fmt.Errorf("unable to start machine controller: API GroupVersion %q or %q or %q or %q or %q or %q is not available; found %#v", awsGVR, azureGVR, gcpGVR, openStackGVR, alicloudGVR, packetGVR, availableResources)
+		return fmt.Errorf("unable to start machine controller: API GroupVersion %q or %q or %q is not available; \nFound: %#v", machineGVR, machineSetGVR, machineDeploymentGVR, availableResources)
 	}
 
 	select {}

docs/README.md

@@ -1,18 +1,25 @@
 # Documentation Index
 
-## Development
+## Using Out-of-Tree (External) provider support (Recommended)
 
+#### Development
+* [Adding support for a new cloud provider](development/cp_support_new.md)
+
+## Using In-Tree provider support (:warning: DEPRECATED!)
+
+##### Development
+
+* [Adding support for a new cloud provider](development/cp_support_old.md)
 * [Setting up a local development environment](development/local_setup.md)
 * [Testing and Dependency Management](development/testing_and_dependencies.md)
-* [Adding support for a new cloud provider](development/new_cp_support.md)
 
-## Usage
+#### Usage
 
 * [Setting up your usage environment](usage/prerequisite.md)
 * [Creating/Deleting machines (VM)](usage/machine.md)
 * [Maintaining machine replicas using machines-sets](usage/machine_set.md)
 * [Updating machines using machines-deployments](usage/machine_deployment.md)
 
-## Deployment
+#### Deployment
 
-* [Deploying the Machine Controller Manager into a Kubernetes cluster](deployment/kubernetes.md)
+* [Deploying the MCM into a Kubernetes cluster using IN-TREE providers](deployment/kubernetes.md)

docs/deployment/kubernetes.md

@@ -37,19 +37,19 @@ $ make docker-image
 $ make push
 ```
 
-- Now you can deploy this docker image to your cluster. A sample development [file is given at](/kubernetes/deployment/deployment.yaml). By default, the deployment manages the cluster it is running in. Optionally, the kubeconfig could also be passed as a flag as described in  `/kubernetes/deployment/deployment.yaml`. This is done when you want your controller running outside the cluster to be managed from.
+- Now you can deploy this docker image to your cluster. A sample development [file is given at](/kubernetes/deployment/in-tree/deployment.yaml). By default, the deployment manages the cluster it is running in. Optionally, the kubeconfig could also be passed as a flag as described in  `/kubernetes/deployment/in-tree/deployment.yaml`. This is done when you want your controller running outside the cluster to be managed from.
 ```bash
-$ kubectl apply -f kubernetes/deployment/deployment.yaml
+$ kubectl apply -f kubernetes/deployment/in-tree/deployment.yaml
 ```
 - Also deploy the required clusterRole and clusterRoleBindings
 ```bash
-$ kubectl apply -f kubernetes/deployment/clusterrole.yaml
-$ kubectl apply -f kubernetes/deployment/clusterrolebinding.yaml
+$ kubectl apply -f kubernetes/deployment/in-tree/clusterrole.yaml
+$ kubectl apply -f kubernetes/deployment/in-tree/clusterrolebinding.yaml
 ```
 
 ## Configuring optional parameters while deploying
 
-Machine-controller-manager supports several configurable parameters while deploying. Refer to [the following lines](/kubernetes/deployment/deployment.yaml#L21-L30), to know how each parameter can be configured, and what it's purpose is for.
+Machine-controller-manager supports several configurable parameters while deploying. Refer to [the following lines](/kubernetes/deployment/in-tree/deployment.yaml#L21-L30), to know how each parameter can be configured, and what it's purpose is for.
 
 ## Usage
 

docs/development/cp_support_new.md

@@ -0,0 +1,144 @@
+# Adding support for a new provider
+
+Steps to be followed while implementing a new (hyperscale) provider are mentioned below. This is the easiest way to add a new provider support using a blueprint code.
+
+However, you may also develop your machine controller from scratch which would provide you more flexibility. However make sure that your custom machine controller adhere's to the `Machine.Status` struct defined in the [MachineAPIs](/pkg/apis/machine/types.go) to make sure the MCM is able to act with higher level controllers like MachineSet and MachineDeployment controller. The key is the `Machine.Status.CurrentStatus.Phase` key that indicates the status of the machine object.
+
+Our strong recommendation would be to follow the steps below as this provides most flexibility required to support machine management for adding new providers. And if you feel to extend the functionality feel free to update our [machine controller libraries](/pkg/util/provider).
+
+## Setting up your repository
+
+1. Create a new empty repository named `machine-controller-manager-provider-{provider-name}` on github username/project. Do not initialize this repository with a README.
+1. Copy the remote repository `URL` (HTTPS/SSH) to this repository which is displayed once you create this repository.
+1. Now on your local system, create directories as required. {your-github-username} given below could also be {github-project} depending on where you have created the new repository.
+    ```bash
+    mkdir -p $GOPATH/src/github.com/{your-github-username}
+    ```
+1. Navigate to this created directory.
+    ```bash
+    cd $GOPATH/src/github.com/{your-github-username}
+    ```
+1. Clone [this repository](https://github.com/gardener/machine-controller-manager-provider-sampleprovider) on your local machine.
+    ```bash
+    git clone git@github.com:gardener/machine-controller-manager-provider-sampleprovider.git
+    ```
+1. Rename the directory from `machine-controller-manager-provider-sampleprovider` to `machine-controller-manager-provider-{provider-name}`.
+    ```bash
+    mv machine-controller-manager-provider-sampleprovider machine-controller-manager-provider-{provider-name}
+    ```
+1. Navigate into the newly created directory.
+    ```bash
+    cd machine-controller-manager-provider-{provider-name}
+    ```
+1. Update the remote `origin` URL to the newly created repository's URL you had copied above.
+    ```bash
+    git remote set-url origin git@github.com:{your-github-username}/machine-controller-manager-provider-{provider-name}.git
+    ```
+1. Rename github project from `gardener` to `{github-org/your-github-username}` where ever you have cloned the repository above. Use the hack script given below to do the same.
+    ```bash
+    make rename-project PROJECT_NAME={github-org/your-github-username}
+    eg:
+        make rename-project PROJECT_NAME=gardener (or)
+        make rename-project PROJECT_NAME=githubusername
+    ```
+1. Rename all files and code from `SampleProvider` to your desired `{provider-name}`. Use the hack script given below to do the same. {provider-name} is case sensitive.
+    ```bash
+    make rename-provider PROVIDER_NAME={provider-name}
+    eg:
+        make rename-provider PROVIDER_NAME=AmazonWebServices (or)
+        make rename-provider PROVIDER_NAME=AWS
+    ```
+1. Now commit your changes and push it upstream.
+    ```bash
+    git add -A
+    git commit -m "Renamed SampleProvide to {provider-name}"
+    git push origin master
+    ```
+
+## Code changes required
+
+The contract between he Machine Controller Manager (MCM) and the Machine Controller (MC) AKA driver has been [documented here](machine_error_codes.md) and the [machine error codes can be found here](/pkg/util/provider/machinecodes/codes/codes.go). You may refer to them for any queries.
+
+:warning:
+- Keep in mind that, **there should to be a unique way to map between machine objects and VMs**. This can be done by mapping machine object names with VM-Name/ tags/ other metadata.
+- Optionally there should also be a unique way to map a VM to it's machine class object. This can be done by tagging VM objects with tags/resource-groups associated with the machine class.
+
+#### Steps to integrate
+
+1. Update the `pkg/provider/apis/provider_spec.go` specification file to reflect the structure of the `ProviderSpec` blob. It typically contains the machine template details in the `MachineClass` object. Follow the sample spec provided already in the file. A sample provider specification can be found [here](https://github.com/gardener/machine-controller-manager-provider-aws/blob/master/pkg/aws/apis/aws_provider_spec.go).
+1. Fill in the methods described at `pkg/provider/core.go` to manage VMs on your cloud provider. Comments are provided above each method to help you fill them up with desired `REQUEST` and `RESPONSE` parameters.
+    - A sample provider implementation for these methods can be found [here](https://github.com/gardener/machine-controller-manager-provider-aws/blob/master/pkg/aws/core.go).
+    - Fill in the required methods `CreateMachine()`, and `DeleteMachine()` methods.
+    - Optionally fill in methods like `GetMachineStatus()`, `ListMachines()`, and `GetVolumeIDs()`. You may choose to fill these, once the working of the required methods seem to be working.
+        - `GetVolumeIDs()` expects VolumeIDs to be decoded from the volumeSpec based on the cloud provider.
+1. Perform validation of APIs that you have described and make it a part of your methods as required at each requests.
+1. Write unit tests to make it work with your implementation by running `make test`.
+    ```bash
+    make test
+    ```
+1. Re-generate the vendors, to update any new vendors imported.
+    ```bash
+    make revendor
+    ```
+1. Update the sample YAML files on `kubernetes/` directory to provide sample files through which the working of the machine controller can be tested.
+1. Update `README.md` to reflect any additional changes
+
+## Testing your code changes
+
+Make sure `$TARGET_KUBECONFIG` points to the cluster where you wish to manage machines. `$CONTROL_NAMESPACE` represents the namespaces where MCM is looking for machine CR objects, and `$CONTROL_KUBECONFIG` points to the cluster which holds these machine CRs.
+
+1. On the first terminal running at `$GOPATH/src/github.com/{github-org/your-github-username}/machine-controller-manager-provider-{provider-name}`,
+    - Run the machine controller (driver) using the command below.
+        ```bash
+        make start
+        ```
+1. On the second terminal pointing to `$GOPATH/src/github.com/gardener`,
+    - Clone the [latest MCM code](https://github.com/gardener/machine-controller-manager)
+        ```bash
+        git clone git@github.com:gardener/machine-controller-manager.git
+        ```
+    - Navigate to the newly created directory.
+        ```bash
+        cd machine-controller-manager
+        ```
+    - Deploy the required CRDs from the machine-controller-manager repo,
+        ```bash
+        kubectl apply -f kubernetes/crds.yaml
+        ```
+    - Run the machine-controller-manager in the `cmi-client` branch
+        ```bash
+        make start
+        ```
+1. On the third terminal pointing to `$GOPATH/src/github.com/{github-org/your-github-username}/machine-controller-manager-provider-{provider-name}`
+    - Fill in the object files given below and deploy them as described below.
+    - Deploy the `machine-class`
+        ```bash
+        kubectl apply -f kubernetes/machine-class.yaml
+        ```
+    - Deploy the `kubernetes secret` if required.
+        ```bash
+        kubectl apply -f kubernetes/secret.yaml
+        ```
+    - Deploy the `machine` object and make sure it joins the cluster successfully.
+        ```bash
+        kubectl apply -f kubernetes/machine.yaml
+        ```
+    - Once machine joins, you can test by deploying a machine-deployment.
+    - Deploy the `machine-deployment` object and make sure it joins the cluster successfully.
+        ```bash
+        kubectl apply -f kubernetes/machine-deployment.yaml
+        ```
+    - Make sure to delete both the `machine` and `machine-deployment` object after use.
+        ```bash
+        kubectl delete -f kubernetes/machine.yaml
+        kubectl delete -f kubernetes/machine-deployment.yaml
+        ```
+
+## Releasing your docker image
+
+1. Make sure you have logged into gcloud/docker using the CLI.
+2. To release your docker image, run the following.
+```bash
+    make release IMAGE_REPOSITORY=<link-to-image-repo>
+```
+3. A sample kubernetes deploy file can be found at `kubernetes/deployment.yaml`. Update the same (with your desired MCM and MC images) to deploy your MCM pod.

docs/development/new_cp_support.md → docs/development/cp_support_old.md

@@ -15,5 +15,5 @@ For adding support for a new cloud provider in the Machine Controller Manager, f
 1. Update `pkg/controller/controller.go` to add new providerMachineClassLister, providerMachineClassQueue, awsMachineClassSynced into the controller struct. Also initialize them in NewController() method.
 1. Add a new file `pkg/controller/providermachineclass.go` that allows re-queuing of machines which refer to an modified providerMachineClass.
 1. Update `pkg/controller/controller.go` to extend `WaitForCacheSync` and `.Shutdown()` similar to other cloud providers.
-1. Update the example ClusterRole in `kubernetes/deployment/clusterrole.yaml` to allow operations on your new machine class.
+1. Update the example ClusterRole in `kubernetes/deployment/in-tree/clusterrole.yaml` to allow operations on your new machine class.
 1. Update `pkg/controller/controller.go`, `pkg/controller/secret.go`, `pkg/controller/secret_util.go` to add event handlers to add/remove finalizers referenced by your machine Class. Refer [this commit](https://github.com/gardener/machine-controller-manager/pull/104/commits/013f70726b1057aed1cf7fe0f0449922ab9a256a).

docs/development/local_setup.md

@@ -77,9 +77,11 @@ I1227 11:08:19.963638   55523 controllermanager.go:204] Starting shared informer
 I1227 11:08:20.766085   55523 controller.go:247] Starting machine-controller-manager
 ```
 
+:warning: The file `dev/target-kubeconfig.yaml` points to the cluster whose nodes you want to manage. `dev/control-kubeconfig.yaml` points to the cluster from where you want to manage the nodes from. However, `dev/control-kubeconfig.yaml` is optional.
+
 The Machine Controller Manager should now be ready to manage the VMs in your kubernetes cluster.
 
-:warning: The file `dev/target-kubeconfig.yaml` points to the cluster whose nodes you want to manage. `dev/control-kubeconfig.yaml` points to the cluster from where you want to manage the nodes from. However, `dev/control-kubeconfig.yaml` is optional.
+:warning: This is assuming that your MCM is built to manage machines for any in-tree supported providers. There is a new way to deploy and manage out of tree (external) support for providers whose development can be [found here](cp_support_new.md)
 
 ## Testing Machine Classes