From 91433063e5942893f08dffc199229cca2f4cb667 Mon Sep 17 00:00:00 2001 From: Stephan Eckweiler Date: Thu, 16 Feb 2023 12:04:00 +0000 Subject: [PATCH 1/4] Add support for ClickHouse Keeper --- README.md | 23 +++++++++++++++++++++- defaults/main.yml | 1 + tasks/configure/sys.yml | 14 ++++++++++++++ templates/clickhousekeeper-servers.j2 | 28 +++++++++++++++++++++++++++ vars/main.yml | 7 +++++-- 5 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 templates/clickhousekeeper-servers.j2 diff --git a/README.md b/README.md index 8cf552b..6ff6950 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ F: You can manage listen ports clickhouse_http_port: 8123 clickhouse_tcp_port: 9000 clickhouse_interserver_http: 9009 +clickhouse_keeper_port: 9181 ``` F: You can add listen ips on top of defaults: ```yaml @@ -225,6 +226,26 @@ clickhouse_merge_tree_config: parts_to_throw_insert: 600 ``` +F: You can configure ClickHouse Keeper, as an alternative to Zookeeper. For a list of available parameters, see [Configuring ClickHouse Keeper](https://clickhouse.com/docs/en/guides/sre/keeper/clickhouse-keeper). The items in the following variable will end up in the `raft_configuration`. Mutually exclusive with `clickhouse_zookeeper_nodes`. +```yaml +clickhouse_keeper_id: 1 # This machine's ID +clickhousekeeper_nodes: + - id: 1 + hostname: "host1" + port: 9234 # use Raft Port here + - id: 2 + hostname: "host2" + port: 9234 +``` + +The following vars for ClickHouse Keeper have sane defaults, but are adjustable: +```yaml +clickhouse_keeper_port: 9181 +clickhouse_path_chk: /var/lib/clickhouse/coordination +clickhouse_path_chk_log: /var/lib/clickhouse/coordination/log +clickhouse_path_chk_snap: /var/lib/clickhouse/coordination/snapshots +``` + Example Playbook ---------------- @@ -386,7 +407,7 @@ Tag | Action install | Only installation of packages config_sys | Only configuration system configs(users.xml and config.xml) config_db | Only add&remove databases -config_sys / Only regenerate dicts +config_sys | Only regenerate dicts config | config_sys+config_db License diff --git a/defaults/main.yml b/defaults/main.yml index dab6945..fb67a00 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -35,6 +35,7 @@ clickhouse_listen_host: "{{ clickhouse_listen_host_default + clickhouse_listen_h clickhouse_http_port: 8123 clickhouse_tcp_port: 9000 +clickhouse_keeper_port: 9181 # clickhouse_https_port: 8443 # clickhouse_tcp_secure_port: 9440 diff --git a/tasks/configure/sys.yml b/tasks/configure/sys.yml index 7bc2735..3a36448 100644 --- a/tasks/configure/sys.yml +++ b/tasks/configure/sys.yml @@ -11,6 +11,9 @@ - "{{ clickhouse_path_configdir }}" - "{{ clickhouse_path_tmp }}" - "{{ clickhouse_path_data }}" + - "{{ clickhouse_path_chk }}" + - "{{ clickhouse_path_chk_snap }}" + - "{{ clickhouse_path_chk_log }}" notify: restart-ch become: true @@ -84,6 +87,17 @@ become: true when: clickhouse_zookeeper_nodes is defined +- name: Config | Generate Clickhouse Keeper servers config + template: + src: clickhousekeeper-servers.j2 + dest: "{{ clickhouse_path_configdir }}/config.d/clickhousekeeper-servers.xml" + owner: "{{ clickhouse_user | default('clickhouse') }}" + group: "{{ clickhouse_group | default('clickhouse') }}" + mode: "u=rw,og=r" + notify: restart-ch + become: true + when: clickhousekeeper_nodes is defined + - name: Config | Fix interserver_http_port and intersever_https_port collision lineinfile: path: "{{ clickhouse_path_configdir }}/config.xml" diff --git a/templates/clickhousekeeper-servers.j2 b/templates/clickhousekeeper-servers.j2 new file mode 100644 index 0000000..6900333 --- /dev/null +++ b/templates/clickhousekeeper-servers.j2 @@ -0,0 +1,28 @@ + +{{ ansible_managed | comment('xml') }} + + + {{ clickhouse_keeper_port }} + {{ clickhouse_keeper_id }} + {{ clickhouse_path_chk_log }} + {{ clickhouse_path_chk_snap }} + + +{% for server in clickhousekeeper_nodes %} + +{% for key, value in server.items() %} + <{{ key }}>{{ value }} +{% endfor %} + +{% endfor %} + + + +{% for server in clickhousekeeper_nodes %} + + {{ server.hostname }} + {{ clickhouse_keeper_port }} + +{% endfor %} + + diff --git a/vars/main.yml b/vars/main.yml index e842bbf..4bed358 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -8,6 +8,9 @@ clickhouse_package: clickhouse_path_configdir: "/etc/clickhouse-server" clickhouse_path_logdir: "/var/log/clickhouse-server" -clickhouse_path_data: "{{ clickhouse_path_base }}/clickhouse/" -clickhouse_path_tmp: "{{ clickhouse_path_base }}/clickhouse/tmp/" +clickhouse_path_data: "{{ clickhouse_path_base }}/clickhouse" +clickhouse_path_tmp: "{{ clickhouse_path_base }}/clickhouse/tmp" +clickhouse_path_chk: "{{ clickhouse_path_data }}/coordination" +clickhouse_path_chk_snap: "{{ clickhouse_path_chk }}/snapshots" +clickhouse_path_chk_log: "{{ clickhouse_path_chk }}/log" clickhouse_service: 'clickhouse-server.service' From f19420496881205733b703c9a97d78babfd26e48 Mon Sep 17 00:00:00 2001 From: "oksana.lipkovska" Date: Fri, 10 Mar 2023 16:52:53 +0200 Subject: [PATCH 2/4] edited when for Generate Clickhouse Keeper servers config to generate config only if it's present in clickhousekeeper_nodes section --- tasks/configure/sys.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/configure/sys.yml b/tasks/configure/sys.yml index 3a36448..a9bf2b0 100644 --- a/tasks/configure/sys.yml +++ b/tasks/configure/sys.yml @@ -96,7 +96,7 @@ mode: "u=rw,og=r" notify: restart-ch become: true - when: clickhousekeeper_nodes is defined + when: (clickhousekeeper_nodes is defined) and (clickhousekeeper_nodes | selectattr('id', '==', clickhouse_keeper_id)) - name: Config | Fix interserver_http_port and intersever_https_port collision lineinfile: From 4f9ef6ad0e536eda5f08591efe82f4ca2da06fd4 Mon Sep 17 00:00:00 2001 From: "oksana.lipkovska" Date: Tue, 14 Mar 2023 16:30:41 +0200 Subject: [PATCH 3/4] separated zookeeper and keeper configs to put first one on all nodes and second one only on nodes that will run keeper --- tasks/configure/sys.yml | 11 +++++++++++ templates/clickhousekeeper-servers.j2 | 8 -------- templates/zookeeper-config.j2 | 12 ++++++++++++ templates/zookeeper-servers.j2 | 2 +- 4 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 templates/zookeeper-config.j2 diff --git a/tasks/configure/sys.yml b/tasks/configure/sys.yml index a9bf2b0..78f21ad 100644 --- a/tasks/configure/sys.yml +++ b/tasks/configure/sys.yml @@ -87,6 +87,17 @@ become: true when: clickhouse_zookeeper_nodes is defined +- name: Config | Generate zookeeper config for all nodes + template: + src: zookeeper-config.j2 + dest: "{{ clickhouse_path_configdir }}/config.d/zookeeper-config.xml" + owner: "{{ clickhouse_user | default('clickhouse') }}" + group: "{{ clickhouse_group | default('clickhouse') }}" + mode: "u=rw,og=r" + notify: restart-ch + become: true + when: clickhousekeeper_nodes is defined + - name: Config | Generate Clickhouse Keeper servers config template: src: clickhousekeeper-servers.j2 diff --git a/templates/clickhousekeeper-servers.j2 b/templates/clickhousekeeper-servers.j2 index 6900333..211d36c 100644 --- a/templates/clickhousekeeper-servers.j2 +++ b/templates/clickhousekeeper-servers.j2 @@ -17,12 +17,4 @@ {% endfor %} - -{% for server in clickhousekeeper_nodes %} - - {{ server.hostname }} - {{ clickhouse_keeper_port }} - -{% endfor %} - diff --git a/templates/zookeeper-config.j2 b/templates/zookeeper-config.j2 new file mode 100644 index 0000000..76db324 --- /dev/null +++ b/templates/zookeeper-config.j2 @@ -0,0 +1,12 @@ + +{{ ansible_managed | comment('xml') }} + + +{% for server in clickhousekeeper_nodes %} + + {{ server.hostname }} + {{ clickhouse_keeper_port }} + +{% endfor %} + + \ No newline at end of file diff --git a/templates/zookeeper-servers.j2 b/templates/zookeeper-servers.j2 index 2f2839c..babf441 100644 --- a/templates/zookeeper-servers.j2 +++ b/templates/zookeeper-servers.j2 @@ -13,4 +13,4 @@ {% endfor %} - + \ No newline at end of file From 2ae6d4dc7254faf86a72d2c852c743e60ae67f6b Mon Sep 17 00:00:00 2001 From: Stephan Eckweiler Date: Thu, 16 Mar 2023 10:02:16 +0000 Subject: [PATCH 4/4] Document automation of shard macro --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 6ff6950..3bf93da 100644 --- a/README.md +++ b/README.md @@ -335,6 +335,31 @@ clickhouse_macros: replica: "db_host_1" ``` +To automate detection of `shard` macro value based on `clickhouse_clusters` dict +(given that `ansible_hostname` resolves to `db-host-?`): +```yaml +- hosts: clickhouse_cluster + remote_user: root + vars: + clickhouse_clusters: + your_cluster_name: + shard_1: + - { host: "db-host-1", port: 9000 } + - { host: "db-host-2", port: 9000 } + shard_2: + - { host: "db-host-3", port: 9000 } + - { host: "db-host-4", port: 9000 } + clickhouse_macros: + shard: "{{ this_shard }}" + replica: "{{ ansible_hostname }}" + tasks: + - name: get shard of host + set_fact: + this_shard: "{{ item.0.key }}" + loop: "{{ clickhouse_clusters.your_cluster_name | dict2items | subelements('value') }}" + when: item.1.host == ansible_hostname +``` + Security harden the cluster. You can configure the cluster with extra settings which enables - HTTPS port