rbd-mirror: major refactor

- Use config-key store to add cluster peer.
- Support multiple pools mirroring.

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>

docs/source/index.rst

@@ -300,6 +300,16 @@ ceph-ansible provides a set of playbook in ``infrastructure-playbooks`` director
    day-2/purge
    day-2/upgrade
 
+RBD Mirroring
+-------------
+
+Ceph-ansible provides the role ``ceph-rbd-mirror`` that can setup an RBD mirror replication.
+
+.. toctree::
+   :maxdepth: 1
+
+   rbdmirror/index
+
 Contribution
 ============
 

docs/source/rbdmirror/index.rst

@@ -0,0 +1,60 @@
+RBD Mirroring
+=============
+
+There's not so much to do from the primary cluster side in order to setup an RBD mirror replication.
+``ceph_rbd_mirror_configure`` has to be set to ``true`` to make ceph-ansible create the mirrored pool
+defined in ``ceph_rbd_mirror_pool`` and the keyring that is going to be used to add the rbd mirror peer.
+
+group_vars from the primary cluster:
+
+.. code-block:: yaml
+
+      ceph_rbd_mirror_configure: true
+      ceph_rbd_mirror_pool: rbd
+
+Optionnally, you can tell ceph-ansible to set the name and the secret of the keyring you want to create:
+
+.. code-block:: yaml
+
+      ceph_rbd_mirror_local_user: client.rbd-mirror-peer  # 'client.rbd-mirror-peer' is the default value.
+      ceph_rbd_mirror_local_user_secret: AQC+eM1iKKBXFBAAVpunJvqpkodHSYmljCFCnw==
+
+This secret will be needed to add the rbd mirror peer from the secondary cluster.
+If you do not enforce it as shown above, you can get it from a monitor by running the following command:
+``ceph auth get {{ ceph_rbd_mirror_local_user }}``
+
+
+.. code-block:: shell
+
+    $ sudo ceph auth get client.rbd-mirror-peer
+
+Once your variables are defined, you can run the playbook (you might want to run with --limit option):
+
+.. code-block:: shell
+
+    $ ansible-playbook -vv -i hosts site-container.yml --limit rbdmirror0
+
+
+The configuration of the rbd mirror replication strictly speaking is done on the secondary cluster.
+The rbd-mirror daemon pulls the data from the primary cluster. This is where the rbd mirror peer addition has to be done.
+The configuration is similar with what was done on the primary cluster, it just needs few additional variables.
+
+``ceph_rbd_mirror_remote_user`` : This user must match the name defined in the variable ``ceph_rbd_mirror_local_user`` from the primary cluster.
+``ceph_rbd_mirror_remote_mon_hosts`` : This must a comma separated list of the monitor addresses from the primary cluster.
+``ceph_rbd_mirror_remote_key`` : This must be the same value as the user (``{{ ceph_rbd_mirror_local_user }}``) keyring secret from the primary cluster.
+
+group_vars from the secondary cluster:
+
+.. code-block:: yaml
+
+   ceph_rbd_mirror_configure: true
+   ceph_rbd_mirror_pool: rbd
+   ceph_rbd_mirror_remote_user: client.rbd-mirror-peer  # This must match the value defined in {{ ceph_rbd_mirror_local_user }} on primary cluster.
+   ceph_rbd_mirror_remote_mon_hosts: 1.2.3.4
+   ceph_rbd_mirror_remote_key: AQC+eM1iKKBXFBAAVpunJvqpkodHSYmljCFCnw==  # This must match the secret of the registered keyring of the user defined in {{ ceph_rbd_mirror_local_user }} on primary cluster.
+
+Once you variables are defined, you can run the playbook (you might want to run with --limit option):
+
+.. code-block:: shell
+
+    $ ansible-playbook -vv -i hosts site-container.yml --limit rbdmirror0

group_vars/rbdmirrors.yml.sample

@@ -18,29 +18,15 @@ dummy:
 # valid for Luminous and later releases.
 #copy_admin_key: false
 
-# NOTE: deprecated generic local user id for pre-Luminous releases
-#ceph_rbd_mirror_local_user: "admin"
-
 
 #################
 # CONFIGURATION #
 #################
 
+#ceph_rbd_mirror_local_user: client.rbd-mirror-peer
 #ceph_rbd_mirror_configure: false
-#ceph_rbd_mirror_pool: ""
 #ceph_rbd_mirror_mode: pool
-
-# NOTE (leseb): the following variable needs the name of the remote cluster.
-# The name of this cluster must be different than your local cluster simply
-# because we need to have both keys and ceph.conf inside /etc/ceph.
-# Thus if cluster names are identical we can not have them under /etc/ceph
-#ceph_rbd_mirror_remote_cluster: ""
-
-# NOTE: the rbd-mirror daemon needs a user to authenticate with the
-# remote cluster. By default, this key should be available under
-# /etc/ceph/<remote_cluster>.client.<remote_user>.keyring
-#ceph_rbd_mirror_remote_user: ""
-
+#ceph_rbd_mirror_remote_cluster: remote
 
 ##########
 # DOCKER #

roles/ceph-rbd-mirror/defaults/main.yml

@@ -10,29 +10,15 @@
 # valid for Luminous and later releases.
 copy_admin_key: false
 
-# NOTE: deprecated generic local user id for pre-Luminous releases
-ceph_rbd_mirror_local_user: "admin"
-
 
 #################
 # CONFIGURATION #
 #################
 
+ceph_rbd_mirror_local_user: client.rbd-mirror-peer
 ceph_rbd_mirror_configure: false
-ceph_rbd_mirror_pool: ""
 ceph_rbd_mirror_mode: pool
-
-# NOTE (leseb): the following variable needs the name of the remote cluster.
-# The name of this cluster must be different than your local cluster simply
-# because we need to have both keys and ceph.conf inside /etc/ceph.
-# Thus if cluster names are identical we can not have them under /etc/ceph
-ceph_rbd_mirror_remote_cluster: ""
-
-# NOTE: the rbd-mirror daemon needs a user to authenticate with the
-# remote cluster. By default, this key should be available under
-# /etc/ceph/<remote_cluster>.client.<remote_user>.keyring
-ceph_rbd_mirror_remote_user: ""
-
+ceph_rbd_mirror_remote_cluster: remote
 
 ##########
 # DOCKER #

roles/ceph-rbd-mirror/tasks/configure_mirroring.yml

@@ -1,18 +1,161 @@
 ---
+- name: cephx tasks
+  when:
+    - cephx | bool
+  block:
+    - name: get client.bootstrap-rbd-mirror from ceph monitor
+      ceph_key:
+        name: client.bootstrap-rbd-mirror
+        cluster: "{{ cluster }}"
+        output_format: plain
+        state: info
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+      register: _bootstrap_rbd_mirror_key
+      delegate_to: "{{ groups.get(mon_group_name)[0] }}"
+      run_once: true
+      no_log: "{{ no_log_on_ceph_key_tasks }}"
+
+    - name: copy ceph key(s)
+      copy:
+        dest: "/var/lib/ceph/bootstrap-rbd-mirror/{{ cluster }}.keyring"
+        content: "{{ _bootstrap_rbd_mirror_key.stdout + '\n' }}"
+        owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        mode: "{{ ceph_keyring_permissions }}"
+      no_log: "{{ no_log_on_ceph_key_tasks }}"
+
+    - name: create rbd-mirror keyrings
+      ceph_key:
+        name: "{{ item.name }}"
+        cluster: "{{ cluster }}"
+        user: client.admin
+        user_key: "/etc/ceph/{{ cluster }}.client.admin.keyring"
+        caps:
+          mon: "profile rbd-mirror"
+          osd: "profile rbd"
+        dest: "{{ item.dest }}"
+        secret: "{{ item.secret | default(omit) }}"
+        import_key: true
+        owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        mode: "{{ ceph_keyring_permissions }}"
+      no_log: "{{ no_log_on_ceph_key_tasks }}"
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+      delegate_to: "{{ groups.get(mon_group_name)[0] }}"
+      loop:
+        - { name: "client.rbd-mirror.{{ ansible_facts['hostname'] }}",
+            dest: "/etc/ceph/{{ cluster }}.client.rbd-mirror.{{ ansible_facts['hostname'] }}.keyring" }
+        - { name: "{{ ceph_rbd_mirror_local_user }}",
+            dest: "/etc/ceph/{{ cluster }}.{{ ceph_rbd_mirror_local_user }}.keyring",
+            secret: "{{ ceph_rbd_mirror_local_user_secret | default('') }}" }
+
+    - name: get "client.rbd-mirror.{{ ansible_facts['hostname'] }}" from ceph monitor
+      ceph_key:
+        name: "client.rbd-mirror.{{ ansible_facts['hostname'] }}"
+        cluster: "{{ cluster }}"
+        output_format: plain
+        state: info
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+      register: _rbd_mirror_key
+      delegate_to: "{{ groups.get(mon_group_name)[0] }}"
+      run_once: true
+      no_log: "{{ no_log_on_ceph_key_tasks }}"
+
+    - name: copy ceph key
+      copy:
+        dest: "/etc/ceph/{{ cluster }}.client.rbd-mirror.{{ ansible_facts['hostname'] }}.keyring"
+        content: "{{ _rbd_mirror_key.stdout + '\n' }}"
+        owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        mode: "{{ ceph_keyring_permissions }}"
+      no_log: false
+
+- name: start and add the rbd-mirror service instance
+  service:
+    name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_facts['hostname'] }}"
+    state: started
+    enabled: yes
+    masked: no
+  changed_when: false
+  when:
+    - not containerized_deployment | bool
+    - ceph_rbd_mirror_remote_user is defined
+
+- name: set_fact ceph_rbd_mirror_pools
+  set_fact:
+    ceph_rbd_mirror_pools:
+      - name: "{{ ceph_rbd_mirror_pool }}"
+  when: ceph_rbd_mirror_pools is undefined
+
+- name: create pool if it doesn't exist
+  ceph_pool:
+    name: "{{ item.name }}"
+    cluster: "{{ cluster }}"
+    pg_num: "{{ item.pg_num | default(omit) }}"
+    pgp_num: "{{ item.pgp_num | default(omit) }}"
+    size: "{{ item.size | default(omit) }}"
+    min_size: "{{ item.min_size | default(omit) }}"
+    pool_type: "{{ item.type | default('replicated') }}"
+    rule_name: "{{ item.rule_name | default(omit) }}"
+    erasure_profile: "{{ item.erasure_profile | default(omit) }}"
+    pg_autoscale_mode: "{{ item.pg_autoscale_mode | default(omit) }}"
+    target_size_ratio: "{{ item.target_size_ratio | default(omit) }}"
+    application: "{{ item.application | default('rbd') }}"
+  delegate_to: "{{ groups[mon_group_name][0] }}"
+  loop: "{{ ceph_rbd_mirror_pools }}"
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+
 - name: enable mirroring on the pool
-  command: "{{ container_exec_cmd | default('') }} rbd --cluster {{ cluster }} --keyring /etc/ceph/{{ cluster }}.client.rbd-mirror.{{ ansible_facts['hostname'] }}.keyring --name client.rbd-mirror.{{ ansible_facts['hostname'] }} mirror pool enable {{ ceph_rbd_mirror_pool }} {{ ceph_rbd_mirror_mode }}"
+  command: "{{ rbd_cmd }} --cluster {{ cluster }} mirror pool enable {{ item.name }} {{ ceph_rbd_mirror_mode }}"
   register: result
   changed_when: false
-  retries: 90
+  retries: 60
   delay: 1
   until: result is succeeded
+  loop: "{{ ceph_rbd_mirror_pools }}"
+  delegate_to: "{{ groups[mon_group_name][0] }}"
 
-- name: list mirroring peer
-  command: "{{ container_exec_cmd | default('') }} rbd --cluster {{ cluster }} --keyring /etc/ceph/{{ cluster }}.client.rbd-mirror.{{ ansible_facts['hostname'] }}.keyring --name client.rbd-mirror.{{ ansible_facts['hostname'] }} mirror pool info {{ ceph_rbd_mirror_pool }}"
-  changed_when: false
-  register: mirror_peer
+- name: add mirroring peer
+  when: ceph_rbd_mirror_remote_user is defined
+  block:
+    - name: list mirroring peer
+      command: "{{ rbd_cmd }} --cluster {{ cluster }} mirror pool info {{ item.name }}"
+      changed_when: false
+      register: mirror_peer
+      loop: "{{ ceph_rbd_mirror_pools }}"
+      delegate_to: "{{ groups[mon_group_name][0] }}"
 
-- name: add a mirroring peer
-  command: "{{ container_exec_cmd | default('') }} rbd --cluster {{ cluster }} --keyring /etc/ceph/{{ cluster }}.client.rbd-mirror.{{ ansible_facts['hostname'] }}.keyring --name client.rbd-mirror.{{ ansible_facts['hostname'] }} mirror pool peer add {{ ceph_rbd_mirror_pool }} {{ ceph_rbd_mirror_remote_user }}@{{ ceph_rbd_mirror_remote_cluster }}"
-  changed_when: false
-  when: ceph_rbd_mirror_remote_user not in mirror_peer.stdout
+    - name: create a temporary file
+      tempfile:
+        path: /etc/ceph
+        state: file
+        suffix: _ceph-ansible
+      register: tmp_file
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+
+    - name: write secret to temporary file
+      copy:
+        dest: "{{ tmp_file.path }}"
+        content: "{{ ceph_rbd_mirror_remote_key }}"
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+
+    - name: add a mirroring peer
+      command: "{{ rbd_cmd }} --cluster {{ cluster }} mirror pool peer add {{ item.item.name }} {{ ceph_rbd_mirror_remote_user }}@{{ ceph_rbd_mirror_remote_cluster }} --remote-mon-host {{ ceph_rbd_mirror_remote_mon_hosts }} --remote-key-file {{ tmp_file.path }}"
+      changed_when: false
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      loop: "{{ mirror_peer.results }}"
+      when: ceph_rbd_mirror_remote_user not in item.stdout
+
+    - name: rm temporary file
+      file:
+        path: "{{ tmp_file.path }}"
+        state: absent
+      delegate_to: "{{ groups[mon_group_name][0] }}"