From aa20b24f7df91c2d6dd8f282c38990fd16be9f3b Mon Sep 17 00:00:00 2001 From: liubo Date: Tue, 18 Feb 2020 16:21:46 +0800 Subject: [PATCH] release-3.1: support tiflash (#1148) * Support Tiflash (#1119) * add tiflash config * Add tiflash * add deploy tiflash * update * fix some error * fix typo * optimize labels for tiflash * optimize format * reset some config (#1096) * Merge multiple configuration files and add status check (#1102) * update tiflash version (#1118) * Update roles/tiflash/tasks/main.yml * create status dir for all services Co-authored-by: Liangliang Gu * rename theflash (#1128) * refactor tiflash config (#1136) * add tiflash proxy status port config (#1140) * tiflash add metrics port (#1144) * add tiflash conf in prometheus for tiflash dashboard (#1146) * add tiflash conf in prometheus for tiflash dashboard Co-authored-by: Liangliang Gu --- conf/tiflash-learner.yml | 46 +++++++ conf/tiflash.yml | 55 ++++++++ deploy.yml | 7 ++ excessive_rolling_update.yml | 53 ++++++++ group_vars/tiflash_servers.yml | 9 ++ inventory.ini | 3 + roles/common_dir/tasks/main.yml | 1 - roles/local/tasks/binary_deployment.yml | 27 ++++ roles/local/templates/binary_packages.yml.j2 | 5 + roles/prometheus/templates/prometheus.yml.j2 | 28 +++++ roles/tiflash/defaults/main.yml | 21 ++++ roles/tiflash/meta/main.yml | 4 + roles/tiflash/tasks/binary_deployment.yml | 23 ++++ roles/tiflash/tasks/main.yml | 66 ++++++++++ roles/tiflash/tasks/supervise_deployment.yml | 8 ++ roles/tiflash/tasks/systemd_deployment.yml | 8 ++ .../templates/run_tiflash_binary.sh.j2 | 21 ++++ roles/tiflash/templates/tiflash.toml.j2 | 118 ++++++++++++++++++ .../tiflash/templates/tiflash_learner.toml.j2 | 90 +++++++++++++ roles/tiflash/vars/tiflash-learner.yml | 46 +++++++ roles/tiflash/vars/tiflash.yml | 55 ++++++++ rolling_update.yml | 53 ++++++++ start.yml | 32 +++++ stop.yml | 21 ++++ unsafe_cleanup.yml | 9 ++ 25 files changed, 808 insertions(+), 1 deletion(-) create mode 100644 conf/tiflash-learner.yml create mode 100644 conf/tiflash.yml create mode 100644 group_vars/tiflash_servers.yml create mode 100644 roles/tiflash/defaults/main.yml create mode 100644 roles/tiflash/meta/main.yml create mode 100644 roles/tiflash/tasks/binary_deployment.yml create mode 100644 roles/tiflash/tasks/main.yml create mode 100644 roles/tiflash/tasks/supervise_deployment.yml create mode 100644 roles/tiflash/tasks/systemd_deployment.yml create mode 100644 roles/tiflash/templates/run_tiflash_binary.sh.j2 create mode 100644 roles/tiflash/templates/tiflash.toml.j2 create mode 100644 roles/tiflash/templates/tiflash_learner.toml.j2 create mode 100644 roles/tiflash/vars/tiflash-learner.yml create mode 100644 roles/tiflash/vars/tiflash.yml diff --git a/conf/tiflash-learner.yml b/conf/tiflash-learner.yml new file mode 100644 index 000000000..cd42a16de --- /dev/null +++ b/conf/tiflash-learner.yml @@ -0,0 +1,46 @@ +# TiKV config template +# Human-readable big numbers: +# File size(based on byte): KB, MB, GB, TB, PB +# e.g.: 1_048_576: "1MB" +# Time(based on ms): ms, s, m, h +# e.g.: 78_000: "1.3m" + +readpool: + storage: + + coprocessor: + +server: + +storage: + +pd: +# This section will be overwritten by command line parameters + +metric: +#address: "172.16.30.31:9531" +#interval: "15s" +#job: "tikv" + +raftstore: + +coprocessor: + +rocksdb: + wal-dir: "" + + defaultcf: + + lockcf: + + writecf: + +raftdb: + defaultcf: + +security: + ca-path: "" + cert-path: "" + key-path: "" + +import: diff --git a/conf/tiflash.yml b/conf/tiflash.yml new file mode 100644 index 000000000..26d917a5d --- /dev/null +++ b/conf/tiflash.yml @@ -0,0 +1,55 @@ +--- + +global: + display_name: "TiFlash" + default_profile: "default" + mark_cache_size: 5368709120 + listen_host: "0.0.0.0" +flash: + flash_cluster: + refresh_interval: 20 + update_rule_interval: 5 + master_ttl: 60 + proxy: + +status: + +logger: + count: 20 + size: "1000M" + level: "debug" +application: + runAsDaemon: true + +raft: + storage_engine: "tmt" + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 diff --git a/deploy.yml b/deploy.yml index 6c72f4eea..271420aeb 100644 --- a/deploy.yml +++ b/deploy.yml @@ -131,6 +131,13 @@ roles: - tikv +- name: deploying TiFlash cluster + hosts: tiflash_servers + tags: + - tiflash + roles: + - tiflash + - name: deploying pump cluster hosts: pump_servers tags: diff --git a/excessive_rolling_update.yml b/excessive_rolling_update.yml index b5ef331ba..ee1ee81bd 100644 --- a/excessive_rolling_update.yml +++ b/excessive_rolling_update.yml @@ -381,6 +381,59 @@ - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml" +- name: rolling update TiFlash cluster + hosts: tiflash_servers + any_errors_fatal: true + serial: 1 + tags: + - tiflash + + pre_tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh + when: process_supervision == 'supervise' + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" + + roles: + - tiflash + + post_tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh + when: process_supervision == 'supervise' + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) - name: rolling update pump cluster hosts: pump_servers diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml new file mode 100644 index 000000000..f7efd3c44 --- /dev/null +++ b/group_vars/tiflash_servers.yml @@ -0,0 +1,9 @@ +--- + +tcp_port: 9000 +http_port: 8123 +interserver_http_port: 9009 +flash_service_port: 3930 +flash_proxy_port: 20170 +flash_proxy_status_port: 20292 +metrics_port: 8234 diff --git a/inventory.ini b/inventory.ini index 56f6cddc9..8f1be185e 100644 --- a/inventory.ini +++ b/inventory.ini @@ -49,6 +49,9 @@ [drainer_servers] +## Tiflash Part +[tiflash_servers] + ## Group variables [pd_servers:vars] # location_labels = ["zone","rack","host"] diff --git a/roles/common_dir/tasks/main.yml b/roles/common_dir/tasks/main.yml index 3dcd01016..78ed16ccd 100644 --- a/roles/common_dir/tasks/main.yml +++ b/roles/common_dir/tasks/main.yml @@ -12,7 +12,6 @@ file: path={{ item }} state=directory mode=0755 with_items: - "{{ status_dir }}" - when: deployment_method == 'supervise' or 'tikv_servers' in group_names - name: create deploy binary directory file: path={{ item }} state=directory mode=0755 diff --git a/roles/local/tasks/binary_deployment.yml b/roles/local/tasks/binary_deployment.yml index 60a802208..71240765b 100644 --- a/roles/local/tasks/binary_deployment.yml +++ b/roles/local/tasks/binary_deployment.yml @@ -48,6 +48,21 @@ - has_outbound_network - not deploy_without_tidb|default(false) +- name: download TiFlash packages + get_url: + url: "{{ item.url }}" + dest: "{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz" + checksum: "{{ item.checksum | default(omit) }}" + force: yes + validate_certs: no + register: get_url_result + until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: "{{ tiflash_packages }}" + when: + - has_outbound_network + - name: unarchive third party binary shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf args: @@ -62,6 +77,13 @@ warn: no when: not deploy_without_tidb|default(false) +- name: unarchive tiflash + shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs tar xzf + args: + chdir: "{{ downloads_dir }}" + warn: no + with_items: "{{ tiflash_packages }}" + - name: cp monitoring binary shell: > cp -v {{ downloads_dir }}/{{ item }}-*/{{ item }} "{{ resources_dir }}/bin/{{ item }}" @@ -81,3 +103,8 @@ shell: > cp -rfv {{ downloads_dir }}/tispark-sample-data "{{ resources_dir }}/bin/" when: not deploy_without_tidb|default(false) + +- name: cp tiflash directory + shell: > + cp -rfv {{ downloads_dir }}/{{ item.name }}-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash" + with_items: "{{ tiflash_packages }}" diff --git a/roles/local/templates/binary_packages.yml.j2 b/roles/local/templates/binary_packages.yml.j2 index 7a8a9a999..2f9dd362c 100644 --- a/roles/local/templates/binary_packages.yml.j2 +++ b/roles/local/templates/binary_packages.yml.j2 @@ -55,3 +55,8 @@ tispark_packages: version: latest url: http://download.pingcap.org/tispark-sample-data.tar.gz {% endif %} + +tiflash_packages: + - name: tiflash + version: {{ tidb_version }} + url: http://download.pingcap.org/tiflash-{{ tidb_version }}-linux-amd64.tar.gz diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index 5994b4cba..d04942b11 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -109,6 +109,18 @@ alerting: {% set _ = tikv_status_addrs.append("%s:%s" % (tikv_ip, tikv_status_port)) -%} {% endfor -%} +{% set tiflash_metrics_addrs = [] -%} +{% set tiflash_http_addrs = [] -%} +{% for host in groups.tiflash_servers -%} + {% set tiflash_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set tiflash_metrics_port = hostvars[host].metrics_port -%} + {% set tiflash_proxy_status_port = hostvars[host].flash_proxy_status_port -%} + {% set tiflash_http_port = hostvars[host].http_port -%} + {% set _ = tiflash_metrics_addrs.append("%s:%s" % (tiflash_ip, tiflash_metrics_port)) -%} + {% set _ = tiflash_metrics_addrs.append("%s:%s" % (tiflash_ip, tiflash_proxy_status_port)) -%} + {% set _ = tiflash_http_addrs.append("%s:%s" % (tiflash_ip, tiflash_http_port)) -%} +{% endfor -%} + {% set pd_addrs = [] -%} {% for host in groups.pd_servers -%} {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} @@ -176,6 +188,16 @@ scrape_configs: - '{{ tikv_status_addr }}' {% endfor %} +{% if tiflash_metrics_addrs %} + - job_name: 'tiflash' + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: +{% for tiflash_metrics_addr in tiflash_metrics_addrs %} + - '{{ tiflash_metrics_addr }}' +{% endfor %} +{% endif %} + - job_name: "pd" honor_labels: true # don't overwrite job & instance labels static_configs: @@ -292,6 +314,12 @@ scrape_configs: labels: group: 'tikv' - targets: +{% for tiflash_http_addr in tiflash_http_addrs %} + - '{{ tiflash_http_addr }}' +{% endfor %} + labels: + group: 'tiflash' + - targets: {% for pd_addr in pd_addrs %} - '{{ pd_addr }}' {% endfor %} diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml new file mode 100644 index 000000000..eca043bb1 --- /dev/null +++ b/roles/tiflash/defaults/main.yml @@ -0,0 +1,21 @@ +--- + +tiflash_dir: "{{ deploy_dir }}/tiflash" +tmp_path: "{{ deploy_dir }}/tiflash/data/tmp" +path: "{{ deploy_dir }}/tiflash/data/db" +cluster_manager_path: "{{ deploy_dir }}/bin/tiflash/flash_cluster_manager" +cluster_manager_log: "{{ deploy_dir }}/log/tiflash_cluster_manager.log" +tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log" +tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log" +tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log" +flash_proxy_config: "{{ deploy_dir }}/conf/tiflash-learner.toml" +data_dir: "{{ deploy_dir }}/tiflash/data/flash" +tiflash_conf_dir: "{{ deploy_dir }}/conf" +tiflash_log_dir: "{{ deploy_dir }}/log" +tiflash_scripts_dir: "{{ deploy_dir }}/scripts" +tcp_port: 9000 +http_port: 8123 +flash_service_port: 3930 +flash_proxy_port: 20170 +flash_proxy_status_port: 20292 +metrics_port: 8234 diff --git a/roles/tiflash/meta/main.yml b/roles/tiflash/meta/main.yml new file mode 100644 index 000000000..beff6a6bf --- /dev/null +++ b/roles/tiflash/meta/main.yml @@ -0,0 +1,4 @@ +--- + +dependencies: + - role: common_dir diff --git a/roles/tiflash/tasks/binary_deployment.yml b/roles/tiflash/tasks/binary_deployment.yml new file mode 100644 index 000000000..8305d14dd --- /dev/null +++ b/roles/tiflash/tasks/binary_deployment.yml @@ -0,0 +1,23 @@ +--- + +- name: deploy tiflash binary + copy: src="{{ resources_dir }}/bin/tiflash" dest="{{ deploy_dir }}/bin/" mode=0755 backup=yes + register: tiflash_binary + +- name: create run script + template: + src: "{{ item }}_{{ role_name }}_binary.sh.j2" + dest: "{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh" + mode: "0755" + backup: yes + with_items: + - run + vars: + role_status_dir: status/{{ role_name }} + register: tiflash_script + +- name: backup script file + command: mv "{{ tiflash_script.backup_file }}" "{{ backup_dir }}" + when: tiflash_script.changed and tiflash_script.backup_file is defined + +- include_tasks: "systemd_deployment.yml" \ No newline at end of file diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml new file mode 100644 index 000000000..052035d8a --- /dev/null +++ b/roles/tiflash/tasks/main.yml @@ -0,0 +1,66 @@ +--- + +- name: create tiflash directories + file: path={{ item }} state=directory mode=0755 + with_items: + - "{{ tiflash_dir }}" + - "{{ tmp_path }}" + - "{{ path }}" + - "{{ data_dir }}" + - "{{ tiflash_conf_dir }}" + - "{{ tiflash_log_dir }}" + - "{{ tiflash_scripts_dir }}" + +- name: "load tiflash config: tidb-ansible/conf/tiflash.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash.yml name=tiflash_conf_custom + +- name: load tiflash default config + include_vars: file=tiflash.yml name=tiflash_conf_default + +- name: generate tiflash dynamic config + set_fact: + tiflash_conf_generated: + flash: + flash_cluster: + cluster_manager_path: "{{ cluster_manager_path }}" + log: "{{ cluster_manager_log }}" + +- name: combine tiflash config + set_fact: + tiflash_conf: "{{ tiflash_conf_custom | with_default_dicts(tiflash_conf_generated, tiflash_conf_default) | update_default_dicts }}" + +- debug: var=tiflash_conf + +- name: create tiflash config file + template: src=tiflash.toml.j2 dest={{ deploy_dir }}/conf/tiflash.toml mode=0644 backup=yes + register: tiflash_conf_st + +- name: backup tiflash conf file + command: mv "{{ tiflash_conf_st.backup_file }}" "{{ backup_dir }}" + when: tiflash_conf_st.changed and tiflash_conf_st.backup_file is defined + +- name: "load tiflash learner config: tidb-ansible/conf/tiflash-learner.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash-learner.yml name=tiflash_learner_conf_custom + +- name: load tiflash learner default config + include_vars: file=tiflash-learner.yml name=tiflash_learner_conf_default + +- name: generate tiflash learner config + set_fact: + tiflash_learner_conf: "{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_default) | update_default_dicts }}" + +- debug: var=tiflash_learner_conf + +- name: create tiflash learner config file + template: src=tiflash_learner.toml.j2 dest={{ deploy_dir }}/conf/tiflash-learner.toml mode=0644 backup=yes + register: tiflash_learner_conf_st + +- name: backup tiflash conf file + command: mv "{{ tiflash_learner_conf_st.backup_file }}" "{{ backup_dir }}" + when: tiflash_learner_conf_st.changed and tiflash_learner_conf_st.backup_file is defined + +- include_tasks: "binary_deployment.yml" + +- name: prepare firewalld white list + set_fact: + firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp', flash_proxy_status_port ~ '/tcp', metrics_port ~ '/tcp'] + firewalld_ports }}" diff --git a/roles/tiflash/tasks/supervise_deployment.yml b/roles/tiflash/tasks/supervise_deployment.yml new file mode 100644 index 000000000..d2aa0d711 --- /dev/null +++ b/roles/tiflash/tasks/supervise_deployment.yml @@ -0,0 +1,8 @@ +--- + +- name: deploy supervise + include_role: + name: supervise + vars: + this_role_name: tiflash + service_name: tiflash-{{ tcp_port }} \ No newline at end of file diff --git a/roles/tiflash/tasks/systemd_deployment.yml b/roles/tiflash/tasks/systemd_deployment.yml new file mode 100644 index 000000000..2b31ffc56 --- /dev/null +++ b/roles/tiflash/tasks/systemd_deployment.yml @@ -0,0 +1,8 @@ +--- + +- name: deploy systemd + include_role: + name: systemd + vars: + this_role_name: tiflash + service_name: tiflash-{{ tcp_port }} \ No newline at end of file diff --git a/roles/tiflash/templates/run_tiflash_binary.sh.j2 b/roles/tiflash/templates/run_tiflash_binary.sh.j2 new file mode 100644 index 000000000..6d31b426f --- /dev/null +++ b/roles/tiflash/templates/run_tiflash_binary.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash +set -e +ulimit -n 1000000 + +# WARNING: This file was auto-generated. Do not edit! +# All your edit might be overwritten! +cd "{{ deploy_dir }}" || exit 1 + +export RUST_BACKTRACE=1 + +export TZ=${TZ:-/etc/localtime} +export LD_LIBRARY_PATH={{ deploy_dir }}/bin/tiflash:$LD_LIBRARY_PATH + +echo -n 'sync ... ' +stat=$(time sync) +echo ok +echo $stat + +echo $$ > "status/{{ role_name }}.pid" + +exec bin/tiflash/tiflash server --config-file conf/tiflash.toml diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2 new file mode 100644 index 000000000..3a94f3809 --- /dev/null +++ b/roles/tiflash/templates/tiflash.toml.j2 @@ -0,0 +1,118 @@ +{% for item, value in tiflash_conf.global | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} +tmp_path = "{{ tmp_path }}" +path = "{{ path }}" +tcp_port = {{ tcp_port }} +http_port = {{ http_port }} + +[flash] +{% set all_tidb = [] -%} +{% for host in groups.tidb_servers -%} + {% set tidb_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set tidb_port = hostvars[host].tidb_status_port -%} + {% set _ = all_tidb.append("%s:%s" % (tidb_ip, tidb_port)) -%} +{% endfor %} +tidb_status_addr = "{{ all_tidb | join(',') }}" +service_addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" +{% for item, value in tiflash_conf.flash | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[flash.{{ item }}] +{% if item == 'proxy' %} +config = "{{ flash_proxy_config }}" +{% endif %} +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[status] +metrics_port = {{ metrics_port }} +{% for item, value in tiflash_conf.profiles | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% endif %} +{% endfor %} + +[logger] +errorlog = "{{ tiflash_errlog }}" +log = "{{ tiflash_server_log }}" +{% for item, value in tiflash_conf.logger | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[application] +{% for item, value in tiflash_conf.application | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[raft] +{% set all_pd = [] -%} +{% for host in groups.pd_servers -%} + {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set pd_port = hostvars[host].pd_client_port -%} + {% set _ = all_pd.append("%s:%s" % (pd_ip, pd_port)) -%} +{% endfor %} +pd_addr = "{{ all_pd | join(',') }}" +{% for item, value in tiflash_conf.raft | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[quotas] +{% for item, value in tiflash_conf.quotas | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[quotas.{{ item }}] +{% for sub_item, sub_value in value | dictsort_by_value_type -%} +{% if sub_value is not mapping -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% else %} + +[quotas.{{ item }}.{{ sub_item }}] +{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} +{{ sub_sub_item }} = {{ sub_sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + +[users] +{% for item, value in tiflash_conf.users | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[users.{{ item }}] +{% for sub_item, sub_value in value | dictsort_by_value_type -%} +{% if sub_value is not mapping -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% else %} + +[users.{{ item }}.{{ sub_item }}] +{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} +{{ sub_sub_item }} = {{ sub_sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + +[profiles] +{% for item, value in tiflash_conf.profiles | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[profiles.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2 new file mode 100644 index 000000000..0181cd9f0 --- /dev/null +++ b/roles/tiflash/templates/tiflash_learner.toml.j2 @@ -0,0 +1,90 @@ +log-file = "{{ tiflash_tikv_log }}" + +[readpool] +{% for item, value in tiflash_learner_conf.readpool | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[readpool.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[server] +labels = { "tiflash_http_port" = "{{ http_port }}" } +engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" +addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}" +advertise-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}" +status-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_status_port }}" +{% for item, value in tiflash_learner_conf.server | dictsort -%} +{% if item == "labels" %} +{{ item }} = {{ value | tikv_server_labels_format }} +{% else %} +{{ item }} = {{ value | to_json }} +{% endif %} +{% endfor %} + +[storage] +data-dir = "{{ data_dir }}" +{% for item, value in tiflash_learner_conf.storage | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[pd] +{% for item, value in tiflash_learner_conf.pd | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[metric] +{% for item, value in tiflash_learner_conf.metric | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[raftstore] +{% for item, value in tiflash_learner_conf.raftstore | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[coprocessor] +{% for item, value in tiflash_learner_conf.coprocessor | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[rocksdb] +{% for item, value in tiflash_learner_conf.rocksdb | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[rocksdb.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[raftdb] +{% for item, value in tiflash_learner_conf.raftdb | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[raftdb.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[security] +{% for item, value in tiflash_learner_conf.security | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[import] +{% for item, value in tiflash_learner_conf.import | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} diff --git a/roles/tiflash/vars/tiflash-learner.yml b/roles/tiflash/vars/tiflash-learner.yml new file mode 100644 index 000000000..cd42a16de --- /dev/null +++ b/roles/tiflash/vars/tiflash-learner.yml @@ -0,0 +1,46 @@ +# TiKV config template +# Human-readable big numbers: +# File size(based on byte): KB, MB, GB, TB, PB +# e.g.: 1_048_576: "1MB" +# Time(based on ms): ms, s, m, h +# e.g.: 78_000: "1.3m" + +readpool: + storage: + + coprocessor: + +server: + +storage: + +pd: +# This section will be overwritten by command line parameters + +metric: +#address: "172.16.30.31:9531" +#interval: "15s" +#job: "tikv" + +raftstore: + +coprocessor: + +rocksdb: + wal-dir: "" + + defaultcf: + + lockcf: + + writecf: + +raftdb: + defaultcf: + +security: + ca-path: "" + cert-path: "" + key-path: "" + +import: diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml new file mode 100644 index 000000000..26d917a5d --- /dev/null +++ b/roles/tiflash/vars/tiflash.yml @@ -0,0 +1,55 @@ +--- + +global: + display_name: "TiFlash" + default_profile: "default" + mark_cache_size: 5368709120 + listen_host: "0.0.0.0" +flash: + flash_cluster: + refresh_interval: 20 + update_rule_interval: 5 + master_ttl: 60 + proxy: + +status: + +logger: + count: 20 + size: "1000M" + level: "debug" +application: + runAsDaemon: true + +raft: + storage_engine: "tmt" + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 diff --git a/rolling_update.yml b/rolling_update.yml index 19247209c..a85a14a33 100644 --- a/rolling_update.yml +++ b/rolling_update.yml @@ -381,6 +381,59 @@ - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml" +- name: rolling update TiFlash cluster + hosts: tiflash_servers + any_errors_fatal: true + serial: 1 + tags: + - tiflash + + pre_tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh + when: process_supervision == 'supervise' + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" + + roles: + - tiflash + + post_tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh + when: process_supervision == 'supervise' + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) - name: rolling update pump cluster hosts: pump_servers diff --git a/start.yml b/start.yml index a620e3d42..e4f700b60 100644 --- a/start.yml +++ b/start.yml @@ -334,6 +334,38 @@ - enable_tls|default(false) - wait_replication|default(false) +- hosts: tiflash_servers + tags: + - tiflash + tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh + when: process_supervision == 'supervise' + with_items: + - tiflash + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started enabled=no + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) + - hosts: pump_servers tags: - pump diff --git a/stop.yml b/stop.yml index 860ae7bdc..9e2053374 100644 --- a/stop.yml +++ b/stop.yml @@ -228,6 +228,27 @@ msg: "the pump port {{ pump_port }} is not down" when: enable_binlog|default(false) +- hosts: tiflash_servers + tags: + - tiflash + tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh + when: process_supervision == 'supervise' + with_items: + - tiflash + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" - hosts: tikv_servers tags: diff --git a/unsafe_cleanup.yml b/unsafe_cleanup.yml index 1a1f3d864..ed5c14e00 100644 --- a/unsafe_cleanup.yml +++ b/unsafe_cleanup.yml @@ -77,6 +77,15 @@ with_items: - pump-{{ pump_port }}.service +- hosts: tiflash_servers + tasks: + - name: clean systemd config + file: path="/etc/systemd/system/{{ item }}" state=absent + become: true + when: process_supervision == 'systemd' + with_items: + - tiflash-{{ tcp_port }}.service + - hosts: tikv_servers tasks: - name: clean systemd config