From 33f08e3843cf0cc787196f11d422f34afe23c925 Mon Sep 17 00:00:00 2001
From: liubo <liubo@pingcap.com>
Date: Thu, 16 Jan 2020 15:02:25 +0800
Subject: [PATCH 1/6] Support Tiflash (#1119)

* add tiflash config

* Add tiflash

* add deploy tiflash

* update

* fix some error

* fix typo

* optimize labels for tiflash

* optimize format

* reset some config (#1096)

* Merge multiple configuration files and add status check (#1102)

* update tiflash version (#1118)

* Update roles/tiflash/tasks/main.yml

* create status dir for all services

Co-authored-by: Liangliang Gu <marsishandsome@gmail.com>
---
 conf/tiflash-learner.yml                      |  55 +++++++++
 conf/tiflash.yml                              |  52 ++++++++
 deploy.yml                                    |   7 ++
 excessive_rolling_update.yml                  |  53 ++++++++
 group_vars/tiflash_servers.yml                |   7 ++
 inventory.ini                                 |   3 +
 roles/common_dir/tasks/main.yml               |   1 -
 roles/local/tasks/binary_deployment.yml       |  27 ++++
 roles/local/templates/binary_packages.yml.j2  |   5 +
 roles/tiflash/defaults/main.yml               |  20 +++
 roles/tiflash/meta/main.yml                   |   4 +
 roles/tiflash/tasks/binary_deployment.yml     |  23 ++++
 roles/tiflash/tasks/main.yml                  |  66 ++++++++++
 roles/tiflash/tasks/supervise_deployment.yml  |   8 ++
 roles/tiflash/tasks/systemd_deployment.yml    |   8 ++
 .../templates/run_tiflash_binary.sh.j2        |  21 ++++
 roles/tiflash/templates/tiflash.toml.j2       | 115 ++++++++++++++++++
 .../tiflash/templates/tiflash_learner.toml.j2 |  84 +++++++++++++
 roles/tiflash/vars/tiflash-learner.yml        |  55 +++++++++
 roles/tiflash/vars/tiflash.yml                |  52 ++++++++
 rolling_update.yml                            |  53 ++++++++
 start.yml                                     |  32 +++++
 stop.yml                                      |  21 ++++
 unsafe_cleanup.yml                            |   9 ++
 24 files changed, 780 insertions(+), 1 deletion(-)
 create mode 100644 conf/tiflash-learner.yml
 create mode 100644 conf/tiflash.yml
 create mode 100644 group_vars/tiflash_servers.yml
 create mode 100644 roles/tiflash/defaults/main.yml
 create mode 100644 roles/tiflash/meta/main.yml
 create mode 100644 roles/tiflash/tasks/binary_deployment.yml
 create mode 100644 roles/tiflash/tasks/main.yml
 create mode 100644 roles/tiflash/tasks/supervise_deployment.yml
 create mode 100644 roles/tiflash/tasks/systemd_deployment.yml
 create mode 100644 roles/tiflash/templates/run_tiflash_binary.sh.j2
 create mode 100644 roles/tiflash/templates/tiflash.toml.j2
 create mode 100644 roles/tiflash/templates/tiflash_learner.toml.j2
 create mode 100644 roles/tiflash/vars/tiflash-learner.yml
 create mode 100644 roles/tiflash/vars/tiflash.yml

diff --git a/conf/tiflash-learner.yml b/conf/tiflash-learner.yml
new file mode 100644
index 000000000..32875ba1e
--- /dev/null
+++ b/conf/tiflash-learner.yml
@@ -0,0 +1,55 @@
+# TiKV config template
+#  Human-readable big numbers:
+#   File size(based on byte): KB, MB, GB, TB, PB
+#    e.g.: 1_048_576: "1MB"
+#   Time(based on ms): ms, s, m, h
+#    e.g.: 78_000: "1.3m"
+
+readpool:
+  storage:
+
+  coprocessor:
+
+server:
+
+storage:
+
+pd:
+# This section will be overwritten by command line parameters
+
+metric:
+#address: "172.16.30.31:9531"
+#interval: "15s"
+#job: "tikv"
+
+raftstore:
+  raftdb-path: ""
+  sync-log: true
+  max-leader-missing-duration: "22s"
+  abnormal-leader-missing-duration: "21s"
+  peer-stale-state-check-interval: "20s"
+
+coprocessor:
+
+rocksdb:
+  wal-dir: ""
+
+  defaultcf:
+    block-cache-size: "10GB"
+
+  lockcf:
+    block-cache-size: "4GB"
+
+  writecf:
+    block-cache-size: "4GB"
+
+raftdb:
+  defaultcf:
+    block-cache-size: "1GB"
+
+security:
+  ca-path: ""
+  cert-path: ""
+  key-path: ""
+
+import:
diff --git a/conf/tiflash.yml b/conf/tiflash.yml
new file mode 100644
index 000000000..3382a164e
--- /dev/null
+++ b/conf/tiflash.yml
@@ -0,0 +1,52 @@
+---
+
+global:
+  display_name: "TiFlash"
+  default_profile: "default"
+  mark_cache_size: 5368709120
+  listen_host: "0.0.0.0"
+flash:
+  flash_cluster:
+    refresh_interval: 20
+    update_rule_interval: 5
+    master_ttl: 60
+  proxy:
+
+logger:
+  count: 10
+  size: "1000M"
+  level: "trace"
+application:
+  runAsDaemon: true
+
+raft:
+
+quotas:
+  default:
+    interval:
+      result_rows: 0
+      read_rows: 0
+      execution_time: 0
+      queries: 0
+      errors: 0
+      duration: 3600
+users:
+  readonly:
+    quota: "default"
+    profile: "readonly"
+    password: ""
+    networks:
+      ip: "::/0"
+  default:
+    quota: "default"
+    profile: "default"
+    password: ""
+    networks:
+      ip: "::/0"
+profiles:
+  readonly:
+    readonly: 1
+  default:
+    load_balancing: "random"
+    use_uncompressed_cache: 0
+    max_memory_usage: 10000000000
diff --git a/deploy.yml b/deploy.yml
index 6c72f4eea..271420aeb 100644
--- a/deploy.yml
+++ b/deploy.yml
@@ -131,6 +131,13 @@
   roles:
     - tikv
 
+- name: deploying TiFlash cluster
+  hosts: tiflash_servers
+  tags:
+    - tiflash
+  roles:
+    - tiflash
+
 - name: deploying pump cluster
   hosts: pump_servers
   tags:
diff --git a/excessive_rolling_update.yml b/excessive_rolling_update.yml
index b5ef331ba..ee1ee81bd 100644
--- a/excessive_rolling_update.yml
+++ b/excessive_rolling_update.yml
@@ -381,6 +381,59 @@
 
     - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml"
 
+- name: rolling update TiFlash cluster
+  hosts: tiflash_servers
+  any_errors_fatal: true
+  serial: 1
+  tags:
+    - tiflash
+
+  pre_tasks:
+    - name: stop TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh
+      when: process_supervision == 'supervise'
+
+    - name: stop TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=stopped
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is down
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: stopped
+        msg: "the TiFlash port {{ http_port }} is not down"
+
+  roles:
+    - tiflash
+
+  post_tasks:
+    - name: start TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh
+      when: process_supervision == 'supervise'
+
+    - name: start TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=started
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is up
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: started
+        msg: "the TiFlash port {{ http_port }} is not up"
+
+    - name: wait until the TiFlash status page is available
+      uri:
+        url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()"
+        return_content: yes
+      register: tiflash_http_result
+      until: tiflash_http_result.status == 200
+      retries: 12
+      delay: 5
+      when: not enable_tls|default(false)
 
 - name: rolling update pump cluster
   hosts: pump_servers
diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml
new file mode 100644
index 000000000..255fd6264
--- /dev/null
+++ b/group_vars/tiflash_servers.yml
@@ -0,0 +1,7 @@
+---
+
+tcp_port: 9000
+http_port: 8123
+interserver_http_port: 9009
+flash_service_port: 3930
+flash_proxy_port: 20170
diff --git a/inventory.ini b/inventory.ini
index 56f6cddc9..8f1be185e 100644
--- a/inventory.ini
+++ b/inventory.ini
@@ -49,6 +49,9 @@
 
 [drainer_servers]
 
+## Tiflash Part
+[tiflash_servers]
+
 ## Group variables
 [pd_servers:vars]
 # location_labels = ["zone","rack","host"]
diff --git a/roles/common_dir/tasks/main.yml b/roles/common_dir/tasks/main.yml
index 3dcd01016..78ed16ccd 100644
--- a/roles/common_dir/tasks/main.yml
+++ b/roles/common_dir/tasks/main.yml
@@ -12,7 +12,6 @@
   file: path={{ item }} state=directory mode=0755
   with_items:
     - "{{ status_dir }}"
-  when: deployment_method == 'supervise' or 'tikv_servers' in group_names
 
 - name: create deploy binary directory
   file: path={{ item }} state=directory mode=0755
diff --git a/roles/local/tasks/binary_deployment.yml b/roles/local/tasks/binary_deployment.yml
index 60a802208..71240765b 100644
--- a/roles/local/tasks/binary_deployment.yml
+++ b/roles/local/tasks/binary_deployment.yml
@@ -48,6 +48,21 @@
     - has_outbound_network
     - not deploy_without_tidb|default(false)
 
+- name: download TiFlash packages
+  get_url:
+    url: "{{ item.url }}"
+    dest: "{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz"
+    checksum: "{{ item.checksum | default(omit) }}"
+    force: yes
+    validate_certs: no
+  register: get_url_result
+  until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items: "{{ tiflash_packages }}"
+  when:
+    - has_outbound_network
+
 - name: unarchive third party binary
   shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf
   args:
@@ -62,6 +77,13 @@
     warn: no
   when: not deploy_without_tidb|default(false)
 
+- name: unarchive tiflash
+  shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs tar xzf
+  args:
+    chdir: "{{ downloads_dir }}"
+    warn: no
+  with_items: "{{ tiflash_packages }}"
+
 - name: cp monitoring binary
   shell: >
     cp -v {{ downloads_dir }}/{{ item }}-*/{{ item }} "{{ resources_dir }}/bin/{{ item }}"
@@ -81,3 +103,8 @@
   shell: >
     cp -rfv {{ downloads_dir }}/tispark-sample-data "{{ resources_dir }}/bin/"
   when: not deploy_without_tidb|default(false)
+
+- name: cp tiflash directory
+  shell: >
+    cp -rfv {{ downloads_dir }}/{{ item.name }}-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash"
+  with_items: "{{ tiflash_packages }}"
diff --git a/roles/local/templates/binary_packages.yml.j2 b/roles/local/templates/binary_packages.yml.j2
index 7a8a9a999..2f9dd362c 100644
--- a/roles/local/templates/binary_packages.yml.j2
+++ b/roles/local/templates/binary_packages.yml.j2
@@ -55,3 +55,8 @@ tispark_packages:
     version: latest
     url: http://download.pingcap.org/tispark-sample-data.tar.gz
 {% endif %}
+
+tiflash_packages:
+  - name: tiflash
+    version: {{ tidb_version }}
+    url: http://download.pingcap.org/tiflash-{{ tidb_version }}-linux-amd64.tar.gz
diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml
new file mode 100644
index 000000000..a27addf5c
--- /dev/null
+++ b/roles/tiflash/defaults/main.yml
@@ -0,0 +1,20 @@
+---
+
+tiflash_dir: "{{ deploy_dir }}/tiflash"
+tmp_path: "{{ deploy_dir }}/tiflash/data/tmp"
+path: "{{ deploy_dir }}/tiflash/data/db"
+cluster_manager_path: "{{ deploy_dir }}/bin/tiflash/flash_cluster_manager"
+cluster_manager_log: "{{ deploy_dir }}/log/tiflash_cluster_manager.log"
+tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log"
+tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log"
+tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log"
+flash_proxy_config: "{{ deploy_dir }}/conf/tiflash-learner.toml"
+data_dir: "{{ deploy_dir }}/tiflash/data/flash"
+tiflash_conf_dir: "{{ deploy_dir }}/conf"
+tiflash_log_dir: "{{ deploy_dir }}/log"
+tiflash_scripts_dir: "{{ deploy_dir }}/scripts"
+tcp_port: 9000
+http_port: 8123
+interserver_http_port: 9009
+flash_service_port: 3930
+flash_proxy_port: 20170
diff --git a/roles/tiflash/meta/main.yml b/roles/tiflash/meta/main.yml
new file mode 100644
index 000000000..beff6a6bf
--- /dev/null
+++ b/roles/tiflash/meta/main.yml
@@ -0,0 +1,4 @@
+---
+
+dependencies:
+  - role: common_dir
diff --git a/roles/tiflash/tasks/binary_deployment.yml b/roles/tiflash/tasks/binary_deployment.yml
new file mode 100644
index 000000000..8305d14dd
--- /dev/null
+++ b/roles/tiflash/tasks/binary_deployment.yml
@@ -0,0 +1,23 @@
+---
+
+- name: deploy tiflash binary
+  copy: src="{{ resources_dir }}/bin/tiflash" dest="{{ deploy_dir }}/bin/" mode=0755 backup=yes
+  register: tiflash_binary
+
+- name: create run script
+  template:
+    src: "{{ item }}_{{ role_name }}_binary.sh.j2"
+    dest: "{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh"
+    mode: "0755"
+    backup: yes
+  with_items:
+    - run
+  vars:
+    role_status_dir: status/{{ role_name }}
+  register: tiflash_script
+
+- name: backup script file
+  command: mv "{{ tiflash_script.backup_file }}" "{{ backup_dir }}"
+  when: tiflash_script.changed and tiflash_script.backup_file is defined
+
+- include_tasks: "systemd_deployment.yml"
\ No newline at end of file
diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml
new file mode 100644
index 000000000..4c660ec3e
--- /dev/null
+++ b/roles/tiflash/tasks/main.yml
@@ -0,0 +1,66 @@
+---
+
+- name: create tiflash directories
+  file: path={{ item }} state=directory mode=0755
+  with_items:
+  - "{{ tiflash_dir }}"
+  - "{{ tmp_path }}"
+  - "{{ path }}"
+  - "{{ data_dir }}"
+  - "{{ tiflash_conf_dir }}"
+  - "{{ tiflash_log_dir }}"
+  - "{{ tiflash_scripts_dir }}"
+
+- name: "load tiflash config: tidb-ansible/conf/tiflash.yml"
+  include_vars: file={{ playbook_dir }}/conf/tiflash.yml name=tiflash_conf_custom
+
+- name: load tiflash default config
+  include_vars: file=tiflash.yml name=tiflash_conf_default
+
+- name: generate tiflash dynamic config
+  set_fact:
+    tiflash_conf_generated:
+      flash:
+        flash_cluster:
+          cluster_manager_path: "{{ cluster_manager_path }}"
+          log: "{{ cluster_manager_log }}"
+
+- name: combine tiflash config
+  set_fact:
+    tiflash_conf: "{{ tiflash_conf_custom | with_default_dicts(tiflash_conf_generated, tiflash_conf_default) | update_default_dicts }}"
+
+- debug: var=tiflash_conf
+
+- name: create tiflash config file
+  template: src=tiflash.toml.j2 dest={{ deploy_dir }}/conf/tiflash.toml mode=0644 backup=yes
+  register: tiflash_conf_st
+
+- name: backup tiflash conf file
+  command: mv "{{ tiflash_conf_st.backup_file }}" "{{ backup_dir }}"
+  when: tiflash_conf_st.changed and tiflash_conf_st.backup_file is defined
+
+- name: "load tiflash learner config: tidb-ansible/conf/tiflash-learner.yml"
+  include_vars: file={{ playbook_dir }}/conf/tiflash-learner.yml name=tiflash_learner_conf_custom
+
+- name: load tiflash learner default config
+  include_vars: file=tiflash-learner.yml name=tiflash_learner_conf_default
+
+- name: generate tiflash learner config
+  set_fact:
+    tiflash_learner_conf: "{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_default) | update_default_dicts }}"
+
+- debug: var=tiflash_learner_conf
+
+- name: create tiflash learner config file
+  template: src=tiflash_learner.toml.j2 dest={{ deploy_dir }}/conf/tiflash-learner.toml mode=0644 backup=yes
+  register: tiflash_learner_conf_st
+
+- name: backup tiflash conf file
+  command: mv "{{ tiflash_learner_conf_st.backup_file }}" "{{ backup_dir }}"
+  when: tiflash_learner_conf_st.changed and tiflash_learner_conf_st.backup_file is defined
+
+- include_tasks: "binary_deployment.yml"
+
+- name: prepare firewalld white list
+  set_fact:
+    firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', interserver_http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}"
diff --git a/roles/tiflash/tasks/supervise_deployment.yml b/roles/tiflash/tasks/supervise_deployment.yml
new file mode 100644
index 000000000..d2aa0d711
--- /dev/null
+++ b/roles/tiflash/tasks/supervise_deployment.yml
@@ -0,0 +1,8 @@
+---
+
+- name: deploy supervise
+  include_role:
+    name: supervise
+  vars:
+    this_role_name: tiflash
+    service_name: tiflash-{{ tcp_port }}
\ No newline at end of file
diff --git a/roles/tiflash/tasks/systemd_deployment.yml b/roles/tiflash/tasks/systemd_deployment.yml
new file mode 100644
index 000000000..2b31ffc56
--- /dev/null
+++ b/roles/tiflash/tasks/systemd_deployment.yml
@@ -0,0 +1,8 @@
+---
+
+- name: deploy systemd
+  include_role:
+    name: systemd
+  vars:
+    this_role_name: tiflash
+    service_name: tiflash-{{ tcp_port }}
\ No newline at end of file
diff --git a/roles/tiflash/templates/run_tiflash_binary.sh.j2 b/roles/tiflash/templates/run_tiflash_binary.sh.j2
new file mode 100644
index 000000000..168172bae
--- /dev/null
+++ b/roles/tiflash/templates/run_tiflash_binary.sh.j2
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+ulimit -n 1000000
+
+# WARNING: This file was auto-generated. Do not edit!
+#          All your edit might be overwritten!
+cd "{{ deploy_dir }}" || exit 1
+
+export RUST_BACKTRACE=1
+
+export TZ=${TZ:-/etc/localtime}
+export LD_LIBRARY_PATH={{ deploy_dir }}/bin/tiflash:$LD_LIBRARY_PATH
+
+echo -n 'sync ... '
+stat=$(time sync)
+echo ok
+echo $stat
+
+echo $$ > "status/{{ role_name }}.pid"
+
+exec bin/tiflash/theflash server --config-file conf/tiflash.toml
diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2
new file mode 100644
index 000000000..0a89ca99c
--- /dev/null
+++ b/roles/tiflash/templates/tiflash.toml.j2
@@ -0,0 +1,115 @@
+{% for item, value in tiflash_conf.global | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+tmp_path = "{{ tmp_path }}"
+path = "{{ path }}"
+tcp_port = {{ tcp_port }}
+http_port = {{ http_port }}
+interserver_http_port = {{ interserver_http_port }}
+
+[flash]
+{% set all_tidb = [] -%}
+{% for host in groups.tidb_servers -%}
+  {% set tidb_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
+  {% set tidb_port = hostvars[host].tidb_status_port -%}
+  {% set _ = all_tidb.append("%s:%s" % (tidb_ip, tidb_port)) -%}
+{% endfor %}
+tidb_status_addr = "{{ all_tidb | join(',') }}"
+service_addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}"
+{% for item, value in tiflash_conf.flash | dictsort_by_value_type -%}
+{% if value is not mapping -%}
+{{ item }} = {{ value | to_json }}
+{% else %}
+
+[flash.{{ item }}]
+{% if item == 'proxy' %}
+addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}"
+advertise-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}"
+data-dir = "{{ data_dir }}"
+config = "{{ flash_proxy_config }}"
+log-file = "{{ tiflash_tikv_log }}"
+{% endif %}
+{% for sub_item, sub_value in value | dictsort -%}
+{{ sub_item }} = {{ sub_value | to_json }}
+{% endfor %}
+{% endif %}
+{% endfor %}
+
+[logger]
+errorlog = "{{ tiflash_errlog }}"
+log = "{{ tiflash_server_log }}"
+{% for item, value in tiflash_conf.logger | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[application]
+{% for item, value in tiflash_conf.application | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[raft]
+{% set all_pd = [] -%}
+{% for host in groups.pd_servers -%}
+  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
+  {% set pd_port = hostvars[host].pd_client_port -%}
+  {% set _ = all_pd.append("%s:%s" % (pd_ip, pd_port)) -%}
+{% endfor %}
+pd_addr = "{{ all_pd | join(',') }}"
+{% for item, value in tiflash_conf.raft | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[quotas]
+{% for item, value in tiflash_conf.quotas | dictsort_by_value_type -%}
+{% if value is not mapping -%}
+{{ item }} = {{ value | to_json }}
+{% else %}
+
+[quotas.{{ item }}]
+{% for sub_item, sub_value in value | dictsort_by_value_type -%}
+{% if sub_value is not mapping -%}
+{{ sub_item }} = {{ sub_value | to_json }}
+{% else %}
+
+[quotas.{{ item }}.{{ sub_item }}]
+{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%}
+{{ sub_sub_item }} = {{ sub_sub_value | to_json }}
+{% endfor %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endfor %}
+
+[users]
+{% for item, value in tiflash_conf.users | dictsort_by_value_type -%}
+{% if value is not mapping -%}
+{{ item }} = {{ value | to_json }}
+{% else %}
+
+[users.{{ item }}]
+{% for sub_item, sub_value in value | dictsort_by_value_type -%}
+{% if sub_value is not mapping -%}
+{{ sub_item }} = {{ sub_value | to_json }}
+{% else %}
+
+[users.{{ item }}.{{ sub_item }}]
+{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%}
+{{ sub_sub_item }} = {{ sub_sub_value | to_json }}
+{% endfor %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endfor %}
+
+[profiles]
+{% for item, value in tiflash_conf.profiles | dictsort_by_value_type -%}
+{% if value is not mapping -%}
+{{ item }} = {{ value | to_json }}
+{% else %}
+
+[profiles.{{ item }}]
+{% for sub_item, sub_value in value | dictsort -%}
+{{ sub_item }} = {{ sub_value | to_json }}
+{% endfor %}
+{% endif %}
+{% endfor %}
diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2
new file mode 100644
index 000000000..881791bbf
--- /dev/null
+++ b/roles/tiflash/templates/tiflash_learner.toml.j2
@@ -0,0 +1,84 @@
+[readpool]
+{% for item, value in tiflash_learner_conf.readpool | dictsort_by_value_type -%}
+{% if value is not mapping -%}
+{{ item }} = {{ value | to_json }}
+{% else %}
+
+[readpool.{{ item }}]
+{% for sub_item, sub_value in value | dictsort -%}
+{{ sub_item }} = {{ sub_value | to_json }}
+{% endfor %}
+{% endif %}
+{% endfor %}
+
+[server]
+labels = { "tiflash_http_port" = "{{ http_port }}" }
+engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}"
+{% for item, value in tiflash_learner_conf.server | dictsort -%}
+{% if item == "labels" %}
+{{ item }} = {{ value | tikv_server_labels_format }}
+{% else %}
+{{ item }} = {{ value | to_json }}
+{% endif %}
+{% endfor %}
+
+[storage]
+{% for item, value in tiflash_learner_conf.storage | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[pd]
+{% for item, value in tiflash_learner_conf.pd | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[metric]
+{% for item, value in tiflash_learner_conf.metric | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[raftstore]
+{% for item, value in tiflash_learner_conf.raftstore | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[coprocessor]
+{% for item, value in tiflash_learner_conf.coprocessor | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[rocksdb]
+{% for item, value in tiflash_learner_conf.rocksdb | dictsort_by_value_type -%}
+{% if value is not mapping -%}
+{{ item }} = {{ value | to_json }}
+{% else %}
+
+[rocksdb.{{ item }}]
+{% for sub_item, sub_value in value | dictsort -%}
+{{ sub_item }} = {{ sub_value | to_json }}
+{% endfor %}
+{% endif %}
+{% endfor %}
+
+[raftdb]
+{% for item, value in tiflash_learner_conf.raftdb | dictsort_by_value_type -%}
+{% if value is not mapping -%}
+{{ item }} = {{ value | to_json }}
+{% else %}
+
+[raftdb.{{ item }}]
+{% for sub_item, sub_value in value | dictsort -%}
+{{ sub_item }} = {{ sub_value | to_json }}
+{% endfor %}
+{% endif %}
+{% endfor %}
+
+[security]
+{% for item, value in tiflash_learner_conf.security | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
+
+[import]
+{% for item, value in tiflash_learner_conf.import | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
diff --git a/roles/tiflash/vars/tiflash-learner.yml b/roles/tiflash/vars/tiflash-learner.yml
new file mode 100644
index 000000000..32875ba1e
--- /dev/null
+++ b/roles/tiflash/vars/tiflash-learner.yml
@@ -0,0 +1,55 @@
+# TiKV config template
+#  Human-readable big numbers:
+#   File size(based on byte): KB, MB, GB, TB, PB
+#    e.g.: 1_048_576: "1MB"
+#   Time(based on ms): ms, s, m, h
+#    e.g.: 78_000: "1.3m"
+
+readpool:
+  storage:
+
+  coprocessor:
+
+server:
+
+storage:
+
+pd:
+# This section will be overwritten by command line parameters
+
+metric:
+#address: "172.16.30.31:9531"
+#interval: "15s"
+#job: "tikv"
+
+raftstore:
+  raftdb-path: ""
+  sync-log: true
+  max-leader-missing-duration: "22s"
+  abnormal-leader-missing-duration: "21s"
+  peer-stale-state-check-interval: "20s"
+
+coprocessor:
+
+rocksdb:
+  wal-dir: ""
+
+  defaultcf:
+    block-cache-size: "10GB"
+
+  lockcf:
+    block-cache-size: "4GB"
+
+  writecf:
+    block-cache-size: "4GB"
+
+raftdb:
+  defaultcf:
+    block-cache-size: "1GB"
+
+security:
+  ca-path: ""
+  cert-path: ""
+  key-path: ""
+
+import:
diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml
new file mode 100644
index 000000000..3382a164e
--- /dev/null
+++ b/roles/tiflash/vars/tiflash.yml
@@ -0,0 +1,52 @@
+---
+
+global:
+  display_name: "TiFlash"
+  default_profile: "default"
+  mark_cache_size: 5368709120
+  listen_host: "0.0.0.0"
+flash:
+  flash_cluster:
+    refresh_interval: 20
+    update_rule_interval: 5
+    master_ttl: 60
+  proxy:
+
+logger:
+  count: 10
+  size: "1000M"
+  level: "trace"
+application:
+  runAsDaemon: true
+
+raft:
+
+quotas:
+  default:
+    interval:
+      result_rows: 0
+      read_rows: 0
+      execution_time: 0
+      queries: 0
+      errors: 0
+      duration: 3600
+users:
+  readonly:
+    quota: "default"
+    profile: "readonly"
+    password: ""
+    networks:
+      ip: "::/0"
+  default:
+    quota: "default"
+    profile: "default"
+    password: ""
+    networks:
+      ip: "::/0"
+profiles:
+  readonly:
+    readonly: 1
+  default:
+    load_balancing: "random"
+    use_uncompressed_cache: 0
+    max_memory_usage: 10000000000
diff --git a/rolling_update.yml b/rolling_update.yml
index 19247209c..a85a14a33 100644
--- a/rolling_update.yml
+++ b/rolling_update.yml
@@ -381,6 +381,59 @@
 
     - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml"
 
+- name: rolling update TiFlash cluster
+  hosts: tiflash_servers
+  any_errors_fatal: true
+  serial: 1
+  tags:
+    - tiflash
+
+  pre_tasks:
+    - name: stop TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh
+      when: process_supervision == 'supervise'
+
+    - name: stop TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=stopped
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is down
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: stopped
+        msg: "the TiFlash port {{ http_port }} is not down"
+
+  roles:
+    - tiflash
+
+  post_tasks:
+    - name: start TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh
+      when: process_supervision == 'supervise'
+
+    - name: start TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=started
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is up
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: started
+        msg: "the TiFlash port {{ http_port }} is not up"
+
+    - name: wait until the TiFlash status page is available
+      uri:
+        url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()"
+        return_content: yes
+      register: tiflash_http_result
+      until: tiflash_http_result.status == 200
+      retries: 12
+      delay: 5
+      when: not enable_tls|default(false)
 
 - name: rolling update pump cluster
   hosts: pump_servers
diff --git a/start.yml b/start.yml
index a620e3d42..e4f700b60 100644
--- a/start.yml
+++ b/start.yml
@@ -334,6 +334,38 @@
         - enable_tls|default(false)
         - wait_replication|default(false)
 
+- hosts: tiflash_servers
+  tags:
+    - tiflash
+  tasks:
+    - name: start TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh
+      when: process_supervision == 'supervise'
+      with_items:
+        - tiflash
+
+    - name: start TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=started enabled=no
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is up
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: started
+        msg: "the TiFlash port {{ http_port }} is not up"
+
+    - name: wait until the TiFlash status page is available
+      uri:
+        url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()"
+        return_content: yes
+      register: tiflash_http_result
+      until: tiflash_http_result.status == 200
+      retries: 12
+      delay: 5
+      when: not enable_tls|default(false)
+
 - hosts: pump_servers
   tags:
     - pump
diff --git a/stop.yml b/stop.yml
index 860ae7bdc..9e2053374 100644
--- a/stop.yml
+++ b/stop.yml
@@ -228,6 +228,27 @@
         msg: "the pump port {{ pump_port }} is not down"
       when: enable_binlog|default(false)
 
+- hosts: tiflash_servers
+  tags:
+    - tiflash
+  tasks:
+    - name: stop TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh
+      when: process_supervision == 'supervise'
+      with_items:
+        - tiflash
+
+    - name: stop TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=stopped
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is down
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: stopped
+        msg: "the TiFlash port {{ http_port }} is not down"
 
 - hosts: tikv_servers
   tags:
diff --git a/unsafe_cleanup.yml b/unsafe_cleanup.yml
index 1a1f3d864..ed5c14e00 100644
--- a/unsafe_cleanup.yml
+++ b/unsafe_cleanup.yml
@@ -77,6 +77,15 @@
       with_items:
         - pump-{{ pump_port }}.service
 
+- hosts: tiflash_servers
+  tasks:
+    - name: clean systemd config
+      file: path="/etc/systemd/system/{{ item }}" state=absent
+      become: true
+      when: process_supervision == 'systemd'
+      with_items:
+        - tiflash-{{ tcp_port }}.service
+
 - hosts: tikv_servers
   tasks:
     - name: clean systemd config

From 8f1b9b745fd5a6ac191b6fe9e10547a38ef94844 Mon Sep 17 00:00:00 2001
From: Liangliang Gu <marsishandsome@gmail.com>
Date: Tue, 4 Feb 2020 15:18:11 +0800
Subject: [PATCH 2/6] rename theflash (#1128)

---
 roles/tiflash/templates/run_tiflash_binary.sh.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/tiflash/templates/run_tiflash_binary.sh.j2 b/roles/tiflash/templates/run_tiflash_binary.sh.j2
index 168172bae..6d31b426f 100644
--- a/roles/tiflash/templates/run_tiflash_binary.sh.j2
+++ b/roles/tiflash/templates/run_tiflash_binary.sh.j2
@@ -18,4 +18,4 @@ echo $stat
 
 echo $$ > "status/{{ role_name }}.pid"
 
-exec bin/tiflash/theflash server --config-file conf/tiflash.toml
+exec bin/tiflash/tiflash server --config-file conf/tiflash.toml

From bee519758d7a731c038759dbeda367265687d664 Mon Sep 17 00:00:00 2001
From: Liangliang Gu <marsishandsome@gmail.com>
Date: Wed, 12 Feb 2020 15:53:53 +0800
Subject: [PATCH 3/6] refactor tiflash config (#1136)

---
 conf/tiflash-learner.yml                        | 9 ---------
 conf/tiflash.yml                                | 5 +++--
 roles/tiflash/defaults/main.yml                 | 1 -
 roles/tiflash/tasks/main.yml                    | 2 +-
 roles/tiflash/templates/tiflash.toml.j2         | 5 -----
 roles/tiflash/templates/tiflash_learner.toml.j2 | 5 +++++
 roles/tiflash/vars/tiflash-learner.yml          | 9 ---------
 roles/tiflash/vars/tiflash.yml                  | 5 +++--
 8 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/conf/tiflash-learner.yml b/conf/tiflash-learner.yml
index 32875ba1e..cd42a16de 100644
--- a/conf/tiflash-learner.yml
+++ b/conf/tiflash-learner.yml
@@ -23,11 +23,6 @@ metric:
 #job: "tikv"
 
 raftstore:
-  raftdb-path: ""
-  sync-log: true
-  max-leader-missing-duration: "22s"
-  abnormal-leader-missing-duration: "21s"
-  peer-stale-state-check-interval: "20s"
 
 coprocessor:
 
@@ -35,17 +30,13 @@ rocksdb:
   wal-dir: ""
 
   defaultcf:
-    block-cache-size: "10GB"
 
   lockcf:
-    block-cache-size: "4GB"
 
   writecf:
-    block-cache-size: "4GB"
 
 raftdb:
   defaultcf:
-    block-cache-size: "1GB"
 
 security:
   ca-path: ""
diff --git a/conf/tiflash.yml b/conf/tiflash.yml
index 3382a164e..d2ca99f14 100644
--- a/conf/tiflash.yml
+++ b/conf/tiflash.yml
@@ -13,13 +13,14 @@ flash:
   proxy:
 
 logger:
-  count: 10
+  count: 20
   size: "1000M"
-  level: "trace"
+  level: "debug"
 application:
   runAsDaemon: true
 
 raft:
+  storage_engine: "tmt"
 
 quotas:
   default:
diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml
index a27addf5c..e1bf4aea8 100644
--- a/roles/tiflash/defaults/main.yml
+++ b/roles/tiflash/defaults/main.yml
@@ -15,6 +15,5 @@ tiflash_log_dir: "{{ deploy_dir }}/log"
 tiflash_scripts_dir: "{{ deploy_dir }}/scripts"
 tcp_port: 9000
 http_port: 8123
-interserver_http_port: 9009
 flash_service_port: 3930
 flash_proxy_port: 20170
diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml
index 4c660ec3e..e737502bb 100644
--- a/roles/tiflash/tasks/main.yml
+++ b/roles/tiflash/tasks/main.yml
@@ -63,4 +63,4 @@
 
 - name: prepare firewalld white list
   set_fact:
-    firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', interserver_http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}"
+    firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}"
diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2
index 0a89ca99c..e93d72d24 100644
--- a/roles/tiflash/templates/tiflash.toml.j2
+++ b/roles/tiflash/templates/tiflash.toml.j2
@@ -5,7 +5,6 @@ tmp_path = "{{ tmp_path }}"
 path = "{{ path }}"
 tcp_port = {{ tcp_port }}
 http_port = {{ http_port }}
-interserver_http_port = {{ interserver_http_port }}
 
 [flash]
 {% set all_tidb = [] -%}
@@ -23,11 +22,7 @@ service_addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_servic
 
 [flash.{{ item }}]
 {% if item == 'proxy' %}
-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}"
-advertise-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}"
-data-dir = "{{ data_dir }}"
 config = "{{ flash_proxy_config }}"
-log-file = "{{ tiflash_tikv_log }}"
 {% endif %}
 {% for sub_item, sub_value in value | dictsort -%}
 {{ sub_item }} = {{ sub_value | to_json }}
diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2
index 881791bbf..a58f3bd18 100644
--- a/roles/tiflash/templates/tiflash_learner.toml.j2
+++ b/roles/tiflash/templates/tiflash_learner.toml.j2
@@ -1,3 +1,5 @@
+log-file = "{{ tiflash_tikv_log }}"
+
 [readpool]
 {% for item, value in tiflash_learner_conf.readpool | dictsort_by_value_type -%}
 {% if value is not mapping -%}
@@ -14,6 +16,8 @@
 [server]
 labels = { "tiflash_http_port" = "{{ http_port }}" }
 engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}"
+addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}"
+advertise-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}"
 {% for item, value in tiflash_learner_conf.server | dictsort -%}
 {% if item == "labels" %}
 {{ item }} = {{ value | tikv_server_labels_format }}
@@ -23,6 +27,7 @@ engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service
 {% endfor %}
 
 [storage]
+data-dir = "{{ data_dir }}"
 {% for item, value in tiflash_learner_conf.storage | dictsort -%}
 {{ item }} = {{ value | to_json }}
 {% endfor %}
diff --git a/roles/tiflash/vars/tiflash-learner.yml b/roles/tiflash/vars/tiflash-learner.yml
index 32875ba1e..cd42a16de 100644
--- a/roles/tiflash/vars/tiflash-learner.yml
+++ b/roles/tiflash/vars/tiflash-learner.yml
@@ -23,11 +23,6 @@ metric:
 #job: "tikv"
 
 raftstore:
-  raftdb-path: ""
-  sync-log: true
-  max-leader-missing-duration: "22s"
-  abnormal-leader-missing-duration: "21s"
-  peer-stale-state-check-interval: "20s"
 
 coprocessor:
 
@@ -35,17 +30,13 @@ rocksdb:
   wal-dir: ""
 
   defaultcf:
-    block-cache-size: "10GB"
 
   lockcf:
-    block-cache-size: "4GB"
 
   writecf:
-    block-cache-size: "4GB"
 
 raftdb:
   defaultcf:
-    block-cache-size: "1GB"
 
 security:
   ca-path: ""
diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml
index 3382a164e..d2ca99f14 100644
--- a/roles/tiflash/vars/tiflash.yml
+++ b/roles/tiflash/vars/tiflash.yml
@@ -13,13 +13,14 @@ flash:
   proxy:
 
 logger:
-  count: 10
+  count: 20
   size: "1000M"
-  level: "trace"
+  level: "debug"
 application:
   runAsDaemon: true
 
 raft:
+  storage_engine: "tmt"
 
 quotas:
   default:

From d7c5d3709785eadc2c5d506483eef23c3c8f37d7 Mon Sep 17 00:00:00 2001
From: Liangliang Gu <marsishandsome@gmail.com>
Date: Fri, 14 Feb 2020 14:45:57 +0800
Subject: [PATCH 4/6] add tiflash proxy status port config (#1140)

---
 group_vars/tiflash_servers.yml                  | 1 +
 roles/tiflash/defaults/main.yml                 | 1 +
 roles/tiflash/tasks/main.yml                    | 2 +-
 roles/tiflash/templates/tiflash_learner.toml.j2 | 1 +
 4 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml
index 255fd6264..e30fea252 100644
--- a/group_vars/tiflash_servers.yml
+++ b/group_vars/tiflash_servers.yml
@@ -5,3 +5,4 @@ http_port: 8123
 interserver_http_port: 9009
 flash_service_port: 3930
 flash_proxy_port: 20170
+flash_proxy_status_port: 20292
diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml
index e1bf4aea8..e27684b61 100644
--- a/roles/tiflash/defaults/main.yml
+++ b/roles/tiflash/defaults/main.yml
@@ -17,3 +17,4 @@ tcp_port: 9000
 http_port: 8123
 flash_service_port: 3930
 flash_proxy_port: 20170
+flash_proxy_status_port: 20292
diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml
index e737502bb..208180dae 100644
--- a/roles/tiflash/tasks/main.yml
+++ b/roles/tiflash/tasks/main.yml
@@ -63,4 +63,4 @@
 
 - name: prepare firewalld white list
   set_fact:
-    firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}"
+    firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp', flash_proxy_status_port ~ '/tcp'] + firewalld_ports }}"
diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2
index a58f3bd18..0181cd9f0 100644
--- a/roles/tiflash/templates/tiflash_learner.toml.j2
+++ b/roles/tiflash/templates/tiflash_learner.toml.j2
@@ -18,6 +18,7 @@ labels = { "tiflash_http_port" = "{{ http_port }}" }
 engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}"
 addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}"
 advertise-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}"
+status-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_status_port }}"
 {% for item, value in tiflash_learner_conf.server | dictsort -%}
 {% if item == "labels" %}
 {{ item }} = {{ value | tikv_server_labels_format }}

From c4254805b2485db6e2a1b759bcb7ca72f77f3b98 Mon Sep 17 00:00:00 2001
From: Liangliang Gu <marsishandsome@gmail.com>
Date: Mon, 17 Feb 2020 10:51:40 +0800
Subject: [PATCH 5/6] tiflash add metrics port (#1144)

---
 conf/tiflash.yml                        | 2 ++
 group_vars/tiflash_servers.yml          | 1 +
 roles/tiflash/defaults/main.yml         | 1 +
 roles/tiflash/tasks/main.yml            | 2 +-
 roles/tiflash/templates/tiflash.toml.j2 | 8 ++++++++
 roles/tiflash/vars/tiflash.yml          | 2 ++
 6 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/conf/tiflash.yml b/conf/tiflash.yml
index d2ca99f14..26d917a5d 100644
--- a/conf/tiflash.yml
+++ b/conf/tiflash.yml
@@ -12,6 +12,8 @@ flash:
     master_ttl: 60
   proxy:
 
+status:
+
 logger:
   count: 20
   size: "1000M"
diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml
index e30fea252..f7efd3c44 100644
--- a/group_vars/tiflash_servers.yml
+++ b/group_vars/tiflash_servers.yml
@@ -6,3 +6,4 @@ interserver_http_port: 9009
 flash_service_port: 3930
 flash_proxy_port: 20170
 flash_proxy_status_port: 20292
+metrics_port: 8234
diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml
index e27684b61..eca043bb1 100644
--- a/roles/tiflash/defaults/main.yml
+++ b/roles/tiflash/defaults/main.yml
@@ -18,3 +18,4 @@ http_port: 8123
 flash_service_port: 3930
 flash_proxy_port: 20170
 flash_proxy_status_port: 20292
+metrics_port: 8234
diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml
index 208180dae..052035d8a 100644
--- a/roles/tiflash/tasks/main.yml
+++ b/roles/tiflash/tasks/main.yml
@@ -63,4 +63,4 @@
 
 - name: prepare firewalld white list
   set_fact:
-    firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp', flash_proxy_status_port ~ '/tcp'] + firewalld_ports }}"
+    firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp', flash_proxy_status_port ~ '/tcp', metrics_port ~ '/tcp'] + firewalld_ports }}"
diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2
index e93d72d24..3a94f3809 100644
--- a/roles/tiflash/templates/tiflash.toml.j2
+++ b/roles/tiflash/templates/tiflash.toml.j2
@@ -30,6 +30,14 @@ config = "{{ flash_proxy_config }}"
 {% endif %}
 {% endfor %}
 
+[status]
+metrics_port = {{ metrics_port }}
+{% for item, value in tiflash_conf.profiles | dictsort_by_value_type -%}
+{% if value is not mapping -%}
+{{ item }} = {{ value | to_json }}
+{% endif %}
+{% endfor %}
+
 [logger]
 errorlog = "{{ tiflash_errlog }}"
 log = "{{ tiflash_server_log }}"
diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml
index d2ca99f14..26d917a5d 100644
--- a/roles/tiflash/vars/tiflash.yml
+++ b/roles/tiflash/vars/tiflash.yml
@@ -12,6 +12,8 @@ flash:
     master_ttl: 60
   proxy:
 
+status:
+
 logger:
   count: 20
   size: "1000M"

From 46ce173e89c1ae9f0081441656422ac29abc938c Mon Sep 17 00:00:00 2001
From: liubo <liubo@pingcap.com>
Date: Tue, 18 Feb 2020 11:09:34 +0800
Subject: [PATCH 6/6] add tiflash conf in prometheus for tiflash dashboard
 (#1146)

* add tiflash conf in prometheus for tiflash dashboard
---
 roles/prometheus/templates/prometheus.yml.j2 | 28 ++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2
index 5994b4cba..d04942b11 100644
--- a/roles/prometheus/templates/prometheus.yml.j2
+++ b/roles/prometheus/templates/prometheus.yml.j2
@@ -109,6 +109,18 @@ alerting:
   {% set _ = tikv_status_addrs.append("%s:%s" % (tikv_ip, tikv_status_port)) -%}
 {% endfor -%}
 
+{% set tiflash_metrics_addrs = [] -%}
+{% set tiflash_http_addrs = [] -%}
+{% for host in groups.tiflash_servers -%}
+  {% set tiflash_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
+  {% set tiflash_metrics_port = hostvars[host].metrics_port -%}
+  {% set tiflash_proxy_status_port = hostvars[host].flash_proxy_status_port -%}
+  {% set tiflash_http_port = hostvars[host].http_port -%}
+  {% set _ = tiflash_metrics_addrs.append("%s:%s" % (tiflash_ip, tiflash_metrics_port)) -%}
+  {% set _ = tiflash_metrics_addrs.append("%s:%s" % (tiflash_ip, tiflash_proxy_status_port)) -%}
+  {% set _ = tiflash_http_addrs.append("%s:%s" % (tiflash_ip, tiflash_http_port)) -%}
+{% endfor -%}
+
 {% set pd_addrs = [] -%}
 {% for host in groups.pd_servers -%}
   {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
@@ -176,6 +188,16 @@ scrape_configs:
       - '{{ tikv_status_addr }}'
 {% endfor %}
 
+{% if tiflash_metrics_addrs %}
+  - job_name: 'tiflash'
+    honor_labels: true # don't overwrite job & instance labels
+    static_configs:
+    - targets:
+{% for tiflash_metrics_addr in tiflash_metrics_addrs %}
+      - '{{ tiflash_metrics_addr }}'
+{% endfor %}
+{% endif %}
+
   - job_name: "pd"
     honor_labels: true # don't overwrite job & instance labels
     static_configs:
@@ -292,6 +314,12 @@ scrape_configs:
       labels:
         group: 'tikv'
     - targets:
+{% for tiflash_http_addr in tiflash_http_addrs %}
+      - '{{ tiflash_http_addr }}'
+{% endfor %}
+      labels:
+        group: 'tiflash'
+    - targets:
 {% for pd_addr in pd_addrs %}
       - '{{ pd_addr }}'
 {% endfor %}