Configure RBAC on all HA nodes (#1371)

* Configure RBAC on all HA nodes

* Fix apiserver service name

microk8s-resources/actions/disable.rbac.sh

@@ -9,7 +9,13 @@ echo "Disabling RBAC"
 
 echo "Reconfiguring apiserver"
 refresh_opt_in_config "authorization-mode" "AlwaysAllow" kube-apiserver
-run_with_sudo preserve_env snapctl restart "${SNAP_NAME}.daemon-apiserver"
+if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ]
+then
+  restart_service "apiserver"
+else
+  run_with_sudo preserve_env snapctl restart "${SNAP_NAME}.daemon-apiserver"
+fi
+
 apiserver=$(wait_for_service apiserver)
 if [[ $apiserver == fail ]]
 then

microk8s-resources/actions/enable.rbac.sh

@@ -8,6 +8,12 @@ echo "Enabling RBAC"
 
 echo "Reconfiguring apiserver"
 refresh_opt_in_config "authorization-mode" "RBAC,Node" kube-apiserver
-run_with_sudo preserve_env snapctl restart "${SNAP_NAME}.daemon-apiserver"
+
+if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ]
+then
+  restart_service "apiserver"
+else
+  run_with_sudo preserve_env snapctl restart "${SNAP_NAME}.daemon-apiserver"
+fi
 
 echo "RBAC is enabled"

scripts/cluster/agent.py

@@ -21,6 +21,7 @@
     is_token_expired,
     get_dqlite_port,
     get_cluster_agent_port,
+    get_arg,
 )
 
 from flask import Flask, jsonify, request, abort, Response
@@ -193,24 +194,6 @@ def getCA():
     return ca
 
 
-def get_arg(key, file):
-    """
-    Get an argument from an arguments file
-
-    :param key: the argument we look for
-    :param file: the arguments file to search in
-    :returns: the value of the argument or None(if the key doesn't exist)
-    """
-    filename = "{}/args/{}".format(snapdata_path, file)
-    with open(filename) as fp:
-        for _, line in enumerate(fp):
-            if line.startswith(key):
-                args = line.split(' ')
-                args = args[-1].split('=')
-                return args[-1].rstrip()
-    return None
-
-
 def is_valid(token_line, token_type=cluster_tokens_file):
     """
     Check whether a token is valid
@@ -590,6 +573,7 @@ def join_node_dqlite():
     remove_token_from_file(token, cluster_tokens_file)
     api_port = get_arg('--secure-port', 'kube-apiserver')
     node_addr = request.remote_addr
+    api_authz = get_arg('--authorization-mode', 'kube-apiserver')
     node_name = get_node_ep(hostname, node_addr)
     if node_name != hostname:
         kubelet_args = read_kubelet_args_file(node_name)
@@ -606,6 +590,7 @@ def join_node_dqlite():
         voters=voters,
         callback_token=callback_token,
         apiport=api_port,
+        apiauthz=api_authz,
         kubelet_args=kubelet_args,
         call_address=node_addr,
         admin_token=get_token('admin'),

scripts/cluster/common/utils.py

@@ -146,3 +146,22 @@ def get_cluster_agent_port():
                 if len(port_parse) > 1:
                     cluster_agent_port = port_parse[1].rstrip()
     return cluster_agent_port
+
+
+def get_arg(key, file):
+    """
+    Get an argument from an arguments file
+
+    :param key: the argument we look for
+    :param file: the arguments file to search in
+    :returns: the value of the argument or None(if the key doesn't exist)
+    """
+    snapdata_path = os.environ.get('SNAP_DATA')
+    filename = "{}/args/{}".format(snapdata_path, file)
+    with open(filename) as fp:
+        for _, line in enumerate(fp):
+            if line.startswith(key):
+                args = line.split(' ')
+                args = args[-1].split('=')
+                return args[-1].rstrip()
+    return None

scripts/cluster/join.py

@@ -21,6 +21,7 @@
     is_node_running_dqlite,
     get_dqlite_port,
     get_cluster_agent_port,
+    get_arg,
 )
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -33,6 +34,7 @@
 callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path)
 server_cert_file_via_env = "${SNAP_DATA}/certs/server.remote.crt"
 server_cert_file = "{}/certs/server.remote.crt".format(snapdata_path)
+default_api_port = "16443"
 
 CLUSTER_API_V2 = "cluster/api/v2.0"
 cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path)
@@ -248,6 +250,15 @@ def update_kubelet(token, ca, master_ip, api_port):
     subprocess.check_call("snapctl restart microk8s.daemon-kubelet".split())
 
 
+def update_apiserver(api_authz):
+    """
+    Configure the apiserver. Note this method does not restart the api server
+
+    :param api_authz: the authorization mode
+    """
+    set_arg('--authorization-mode', api_authz, 'kube-apiserver')
+
+
 def store_remote_ca(ca):
     """
     Store the remote ca
@@ -802,6 +813,9 @@ def join_dqlite(connection_parts):
     store_cert("ca.crt", info["ca"])
     store_cert("ca.key", info["ca_key"])
     store_cert("serviceaccount.key", info["service_account_key"])
+    api_port = get_arg('--secure-port', 'kube-apiserver')
+    if not api_port:
+        api_port = default_api_port
     # triplets of [username in known_tokens.csv, username in kubeconfig, kubeconfig filename name]
     for component in [
         ("kube-proxy", "kubeproxy", "proxy.config"),
@@ -814,12 +828,13 @@ def join_dqlite(connection_parts):
             print("Error, could not locate {} token. Joining cluster failed.".format(component[0]))
             exit(3)
         assert token is not None
-        # TODO make this configurable
         create_kubeconfig(
-            component_token, info["ca"], "127.0.0.1", "16443", component[2], component[1]
+            component_token, info["ca"], "127.0.0.1", api_port, component[2], component[1]
         )
     if "admin_token" in info:
         replace_admin_token(info["admin_token"])
+    if "apiauthz" in info:
+        update_apiserver(info["apiauthz"])
     create_admin_kubeconfig(info["ca"], info["admin_token"])
     set_arg("--hostname-override", hostname_override, "kube-proxy")
     store_base_kubelet_args(info["kubelet_args"])