diff --git a/.circleci/config.yml b/.circleci/config.yml index 4eb876c4dd91..949e1028799d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,11 +17,18 @@ jobs: echo 'export NODE_PATH=~/.npm-global/lib/node_modules:$NODE_PATH' >> $BASH_ENV - run: - name: "Check git conflicts" + name: "Check file encoding" command: | git remote add upstream https://github.com/pingcap/docs.git git fetch upstream - python3 scripts/check-conflicts.py $(git diff-tree --name-only --no-commit-id -r upstream/master..HEAD -- '*.md' '*.yml' '*.yaml') + wget https://raw.githubusercontent.com/CharLotteiu/pingcap-docs-checks/main/check-file-encoding.py + python3 check-file-encoding.py $(git diff-tree --name-only --no-commit-id -r upstream/master..HEAD -- '*.md' ':(exclude).github/*') + + - run: + name: "Check git conflicts" + command: | + wget https://raw.githubusercontent.com/CharLotteiu/pingcap-docs-checks/main/check-conflicts.py + python3 check-conflicts.py $(git diff-tree --name-only --no-commit-id -r upstream/master..HEAD -- '*.md' '*.yml' '*.yaml') - run: name: "Install markdownlint" @@ -43,10 +50,23 @@ jobs: command: | scripts/verify-link-anchors.sh + - run: + name: "Check control characters" + command: | + wget https://raw.githubusercontent.com/CharLotteiu/pingcap-docs-checks/main/check-control-char.py + python3 check-control-char.py $(git diff-tree --name-only --no-commit-id -r upstream/master..HEAD -- '*.md' ':(exclude).github/*') + - run: name: "Check unclosed tags" command: | - python3 scripts/check-tags.py $(git diff-tree --name-only --no-commit-id -r upstream/master..HEAD -- '*.md' ':(exclude).github/*') + wget https://raw.githubusercontent.com/CharLotteiu/pingcap-docs-checks/main/check-tags.py + python3 check-tags.py $(git diff-tree --name-only --no-commit-id -r upstream/master..HEAD -- '*.md' ':(exclude).github/*') + + - run: + name: "Check manual line breaks" + command: | + wget https://raw.githubusercontent.com/CharLotteiu/pingcap-docs-checks/main/check-manual-line-breaks.py + python3 check-manual-line-breaks.py $(git diff-tree --name-only --no-commit-id -r upstream/master..HEAD -- '*.md' ':(exclude).github/*') build: docker: diff --git a/.github/ISSUE_TEMPLATE/change-request.md b/.github/ISSUE_TEMPLATE/change-request.md index 7727f7280945..0bf865746090 100644 --- a/.github/ISSUE_TEMPLATE/change-request.md +++ b/.github/ISSUE_TEMPLATE/change-request.md @@ -5,8 +5,9 @@ about: Any suggestion? Want to add something new? ## Change Request -**This repository is ONLY used to solve issues related to DOCS. -For other issues (related to TiDB, PD, etc), please move to [other repositories](https://github.com/pingcap/).** + Please answer the following questions before submitting your issue. Thanks! diff --git a/.github/ISSUE_TEMPLATE/error-report.md b/.github/ISSUE_TEMPLATE/error-report.md index 99871a841fe8..d1a986f5edd5 100644 --- a/.github/ISSUE_TEMPLATE/error-report.md +++ b/.github/ISSUE_TEMPLATE/error-report.md @@ -5,8 +5,9 @@ about: Typos, grammatical errors, terminology misuse, ambiguity, etc. ## Error Report -**This repository is ONLY used to solve issues related to DOCS. -For other issues (related to TiDB, PD, etc), please move to [other repositories](https://github.com/pingcap/).** + Please answer the following questions before submitting your issue. Thanks! diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index bf050f373603..1a1da6dfeb77 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -5,8 +5,9 @@ about: Usage question that isn't answered in docs or discussion ## Question -**This repository is ONLY used to solve issues related to DOCS. -For other issues (related to TiDB, PD, etc), please move to [other repositories](https://github.com/pingcap/).** + Before submitting your question, make sure you have: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 64ce221d0d00..dae71379e682 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,6 +13,8 @@ - [ ] master (the latest development version) +- [ ] v5.1 (TiDB 5.1 versions) +- [ ] v5.0 (TiDB 5.0 versions) - [ ] v4.0 (TiDB 4.0 versions) - [ ] v3.1 (TiDB 3.1 versions) - [ ] v3.0 (TiDB 3.0 versions) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 867334f01eba..dbc0619e656c 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -11,11 +11,19 @@ jobs: steps: - uses: actions/checkout@v2 - + + - name: Extract branch, repo and sha + shell: bash + id: extract_info + run: | + echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" + echo "::set-output name=repo::$(cut -d'/' -f2 <<< ${{ github.repository }})" + echo "::set-output name=sha::$(sha=${{ github.sha }}; echo ${sha:0:6})" + - name: Repository Dispatch uses: peter-evans/repository-dispatch@v1.1.1 with: token: ${{ secrets.PR_TRIGGER_BUILD_TOKEN }} repository: pingcap/website-docs/ - event-type: triggered + event-type: ${{ steps.extract_info.outputs.repo }}/${{ steps.extract_info.outputs.branch }}-${{ steps.extract_info.outputs.sha }} client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "repo": "${{ github.repository }}"}' diff --git a/.github/workflows/link-check.yaml b/.github/workflows/link-check.yaml index 3b4c932dda19..5e4a64bbaca3 100644 --- a/.github/workflows/link-check.yaml +++ b/.github/workflows/link-check.yaml @@ -1,31 +1,31 @@ -name: external-link-check +# name: external-link-check -on: [pull_request] +# on: [pull_request] -jobs: - base: - runs-on: ubuntu-18.04 - steps: - - name: Check out code - uses: actions/checkout@v2 +# jobs: +# base: +# runs-on: ubuntu-18.04 +# steps: +# - name: Check out code +# uses: actions/checkout@v2 - # NOTE: - # - the check process can be very fast, - # but if `-c` is too large then `Too Many Requests (HTTP error 429)` may be reported from `https://github.com/*`. - # - we hardcode `--document-root` to `/github/workspace` in the container now. - # - we use `http://172.*` as sample addresses in some docs, so we need to exclude them. - - name: Check external links - id: lc - uses: peter-evans/link-checker@v1 - with: - args: -c 32 -d /github/workspace -r -x "http://172.*|https://github.com/.*/pull/.*|https://github.com/.*/issues/.*|https://github.com/[^.\n]*$|http://127.0.0.1.*|http://localhost.*|http://.*:.*|https://static.googleusercontent.com/media/research.google.com/.*|https://www.crunchbase.com/organization/.*|http://www.brendangregg.com.*|https://blog.csdn.net.*|http://mysql-cocoa.sourceforge.net.*|https://tools.ietf.org.*|http://code.google.com.*|https://spark.apache.org.*|https://www.caasdata.com/|https://www.linkedin.com/company/.*|https://sea-group.org/.*|https://octave.sourceforge.io/database/index.html|https://www.teld.cn/|http://gnade.sourceforge.net/|https://en.wikipedia.org/wiki/WeBank_\(China|https://twitter.com/PingCAP|.*.md" . +# # NOTE: +# # - the check process can be very fast, +# # but if `-c` is too large then `Too Many Requests (HTTP error 429)` may be reported from `https://github.com/*`. +# # - we hardcode `--document-root` to `/github/workspace` in the container now. +# # - we use `http://172.*` as sample addresses in some docs, so we need to exclude them. +# - name: Check external links +# id: lc +# uses: peter-evans/link-checker@v1 +# with: +# args: -c 32 -d /github/workspace -r -x "http://172.*|https://github.com/.*/pull/.*|https://github.com/.*/issues/.*|https://github.com/[^.\n]*$|http://127.0.0.1.*|http://localhost.*|http://.*:.*|https://static.googleusercontent.com/media/research.google.com/.*|https://www.crunchbase.com/organization/.*|http://www.brendangregg.com.*|https://blog.csdn.net.*|http://mysql-cocoa.sourceforge.net.*|https://tools.ietf.org.*|http://code.google.com.*|https://spark.apache.org.*|https://www.caasdata.com/|https://www.linkedin.com/company/.*|https://sea-group.org/.*|https://octave.sourceforge.io/database/index.html|https://www.teld.cn/|http://gnade.sourceforge.net/|https://en.wikipedia.org/wiki/WeBank_\(China|https://twitter.com/PingCAP|http://www.webgraphviz.com/|http://octave.sourceforge.net/database/index.html|http://www.ymt.com/|http://www.alhem.net/project/mysql/|https://www.reddit.com/r/TiDB|http://www.189.cn/sh/|https://dev.mysql.com/doc/refman/5.7/en/connector-odbc-info.html|https://asktug.com/_/tidb-performance-map/#/|https://commons.apache.org/proper/commons-dbcp/|https://tomcat.apache.org/tomcat-7.0-doc/jdbc-pool.html|https://en.wikipedia.org/wiki/Plane_\(Unicode|.*.md" . - - name: Fail if there were external link errors - if: ${{ steps.lc.outputs.exit_code }} - run: | - echo "Please check the broken links reported in the previous step \"Check external links\"." - exit ${{ steps.lc.outputs.exit_code }} - # Debug via SSH if previous steps failed - #- name: Set up tmate session - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v2 +# - name: Fail if there were external link errors +# if: ${{ steps.lc.outputs.exit_code }} +# run: | +# echo "Please check the broken links reported in the previous step \"Check external links\"." +# exit ${{ steps.lc.outputs.exit_code }} +# # Debug via SSH if previous steps failed +# #- name: Set up tmate session +# # if: ${{ failure() }} +# # uses: mxschmitt/action-tmate@v2 diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index 9e1c085503ca..ef0bce505c69 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -3,10 +3,7 @@ on: push: branches: - master - - release-2.1 - - release-3.0 - - release-3.1 - - release-4.0 + - release-* paths: - media/** jobs: @@ -36,4 +33,4 @@ jobs: pip3 install awscli printf "%s\n" ${{ secrets.AWS_ACCESS_KEY }} ${{ secrets.AWS_SECRET_KEY }} ${{ secrets.AWS_REGION }} "json" | aws configure - name: Upload - run: cloud-assets-utils verify-and-sync -qiniu true -qiniu-bucket ${{ secrets.QINIU_BUCKET_NAME }} -aws true -aws-bucket ${{ secrets.AWS_BUCKET_NAME }} media -replace-first-path-to images/docs + run: cloud-assets-utils verify-and-sync -qiniu true -qiniu-bucket ${{ secrets.QINIU_BUCKET_NAME }} -aws true -aws-bucket ${{ secrets.AWS_BUCKET_NAME }} media -replace-first-path-to images/docs -cdn-refresh https://download.pingcap.com/ diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml new file mode 100644 index 000000000000..db5203798e1e --- /dev/null +++ b/.github/workflows/rebase.yml @@ -0,0 +1,19 @@ +name: Automatic Rebase +on: + issue_comment: + types: [created] +jobs: + rebase: + name: Rebase + if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') + runs-on: ubuntu-latest + steps: + - name: Checkout the latest code + uses: actions/checkout@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 # otherwise, you will fail to push refs to dest repo + - name: Automatic Rebase + uses: cirrus-actions/rebase@1.4 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml deleted file mode 100644 index 65a8c313b9a4..000000000000 --- a/.github/workflows/stale.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: "Close stale issues/prs" -on: - schedule: - - cron: "0 0 * * *" - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'This issue is stale because it has been open for 60 days with no activity. If no comment has been made and the `lifecycle/stale` label is not removed, this issue will be closed in 15 days.' - close-issue-message: 'This issue will be closed because it has been stale for 15 days with no activity. If you still have any question about this issue, feel free to reopen it or create a new one.' - days-before-stale: 60 - days-before-close: 15 - stale-issue-label: 'lifecycle/stale' - exempt-issue-labels: 'lifecycle/frozen,good-first-issue,help-wanted' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca163cd7e048..476b6d6d559e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,11 +29,13 @@ Before you contribute, please take a quick look at some general information abou ### Learn about docs versions -Currently, we maintain four versions of TiDB documentation, each with a separate branch: +Currently, we maintain six versions of TiDB documentation, each with a separate branch: | Docs branch name | Version description | | :--- | :--- | | `master` branch | the latest development version | +| `release-5.1` branch | the 5.1 version | +| `release-5.0` branch | the 5.0 stable version | | `release-4.0` branch | the 4.0 stable version | | `release-3.1` branch | the 3.1 stable version | | `release-3.0` branch | the 3.0 stable version | @@ -50,7 +52,8 @@ Currently, we maintain four versions of TiDB documentation, each with a separate - If your changes apply to only one docs version, just submit a PR to the corresponding version branch. - If your changes apply to multiple docs versions, you don't have to submit a PR to each branch. Instead, after you submit your PR, trigger the sre-bot to submit a PR to other version branches by adding one or several of the following labels as needed. Once the current PR is merged, sre-bot will start to work. - + - `needs-cherry-pick-5.1` label: sre-bot will submit a PR to the `release-5.1` branch. + - `needs-cherry-pick-5.0` label: sre-bot will submit a PR to the `release-5.0` branch. - `needs-cherry-pick-4.0` label: sre-bot will submit a PR to the `release-4.0` branch. - `needs-cherry-pick-3.1` label: sre-bot will submit a PR to the `release-3.1` branch. - `needs-cherry-pick-3.0` label: sre-bot will submit a PR to the `release-3.0` branch. @@ -61,7 +64,7 @@ Currently, we maintain four versions of TiDB documentation, each with a separate ## How to contribute -Please perform the following steps to create your Pull Request to this repository. If don't like to use command, you can also use [GitHub Desktop](https://desktop.github.com/), which is easier to get started. +Please perform the following steps to create your Pull Request to this repository. If don't like to use commands, you can also use [GitHub Desktop](https://desktop.github.com/), which is easier to get started. > **Note:** > diff --git a/README.md b/README.md index 58d3b4486260..3049740008c1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Welcome to TiDB documentation! -This repository stores all the source files of [TiDB Docs at the PingCAP website](https://docs.pingcap.com/tidb/v4.0), while the [pingcap/docs-cn](https://github.com/pingcap/docs-cn) repository stores all the source files of [TiDB Documentation in Chinese](https://docs.pingcap.com/zh/tidb/v4.0). +This repository stores all the source files of [TiDB Docs at the PingCAP website](https://docs.pingcap.com/tidb/stable), while the [pingcap/docs-cn](https://github.com/pingcap/docs-cn) repository stores all the source files of [TiDB Documentation in Chinese](https://docs.pingcap.com/zh/tidb/stable). If you find documentation issues, feel free to [create an Issue](https://github.com/pingcap/docs/issues/new/choose) to let us know or directly [create a Pull Request](/CONTRIBUTING.md#how-to-contribute) to help fix or update it. @@ -24,6 +24,8 @@ Currently, we maintain the following versions of TiDB documentation in different | Branch name | TiDB docs version | | :---------|:----------| | [`master`](https://github.com/pingcap/docs/tree/master) | The latest development version | +| [`release-5.1`](https://github.com/pingcap/docs/tree/release-5.1) | 5.1 version | +| [`release-5.0`](https://github.com/pingcap/docs/tree/release-5.0) | 5.0 stable version | | [`release-4.0`](https://github.com/pingcap/docs/tree/release-4.0) | 4.0 stable version | | [`release-3.1`](https://github.com/pingcap/docs/tree/release-3.1) | 3.1 stable version | | [`release-3.0`](https://github.com/pingcap/docs/tree/release-3.0) | 3.0 stable version | diff --git a/TOC.md b/TOC.md index bee1d538bd59..316a3a506b19 100644 --- a/TOC.md +++ b/TOC.md @@ -7,14 +7,13 @@ + About TiDB + [TiDB Introduction](/overview.md) - + [What's New in TiDB 4.0](/whats-new-in-tidb-4.0.md) - + [TiDB 4.0 Experimental Features](/experimental-features-4.0.md) + + [TiDB 5.1 Release Notes](/releases/release-5.1.0.md) + [Basic Features](/basic-features.md) + + [Experimental Features](/experimental-features.md) + Benchmarks - + [v4.0 Sysbench Performance Test](/benchmark/benchmark-sysbench-v4-vs-v3.md) - + [v4.0 TPC-H Performance Test](/benchmark/v4.0-performance-benchmarking-with-tpch.md) - + [v4.0 TPC-C Performance Test](/benchmark/v4.0-performance-benchmarking-with-tpcc.md) - + [Interaction Test on Online Workloads and `ADD INDEX`](/benchmark/online-workloads-and-add-index-operations.md) + + [v5.1 Sysbench Performance Test Report](/benchmark/benchmark-sysbench-v5.1.0-vs-v5.0.2.md) + + [v5.1 TPC-C Performance Test Report](/benchmark/v5.1-performance-benchmarking-with-tpcc.md) + + [v5.1 MPP mode TPC-H 100GB Performance Test](/benchmark/v5.1-performance-benchmarking-with-tpch.md) + [MySQL Compatibility](/mysql-compatibility.md) + [TiDB Limitations](/tidb-limitations.md) + [TiDB Adopters](/adopters.md) @@ -26,7 +25,7 @@ + Deploy + [Software and Hardware Requirements](/hardware-and-software-requirements.md) + [Environment Configuration Checklist](/check-before-deployment.md) - + Topology Patterns + + Plan Cluster Topology + [Minimal Topology](/minimal-deployment-topology.md) + [TiFlash Topology](/tiflash-deployment-topology.md) + [TiCDC Topology](/ticdc-deployment-topology.md) @@ -35,20 +34,17 @@ + [Cross-DC Topology](/geo-distributed-deployment-topology.md) + [Hybrid Topology](/hybrid-deployment-topology.md) + Install and Start - + Linux OS - + [Use TiUP (Recommended)](/production-deployment-using-tiup.md) - + [Use TiUP Offline (Recommended)](/production-offline-deployment-using-tiup.md) - + [Deploy in Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable) - + [Use TiDB Ansible](/online-deployment-using-ansible.md) - + [Use TiDB Ansible Offline](/offline-deployment-using-ansible.md) + + [Use TiUP (Recommended)](/production-deployment-using-tiup.md) + + [Deploy in Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable) + [Verify Cluster Status](/post-installation-check.md) - + Benchmarks Methods + + Test Cluster Performance + [Test TiDB Using Sysbench](/benchmark/benchmark-tidb-using-sysbench.md) - + [Run TPC-C Test on TiDB](/benchmark/benchmark-tidb-using-tpcc.md) + + [Test TiDB Using TPC-C](/benchmark/benchmark-tidb-using-tpcc.md) + Migrate + [Overview](/migration-overview.md) + Migrate from MySQL - + [Migrate Using Dumpling and TiDB Lightning](/migrate-from-mysql-dumpling-files.md) + + [Migrate from Amazon Aurora MySQL Using TiDB Lightning](/migrate-from-aurora-using-lightning.md) + + [Migrate from MySQL SQL Files Using TiDB Lightning](/migrate-from-mysql-dumpling-files.md) + [Migrate from Amazon Aurora MySQL Using DM](/migrate-from-aurora-mysql-database.md) + Migrate from CSV Files + [Use TiDB Lightning](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) @@ -57,26 +53,21 @@ + Maintain + Upgrade + [Use TiUP (Recommended)](/upgrade-tidb-using-tiup.md) - + [Use TiUP Offline (Recommended)](/upgrade-tidb-using-tiup-offline.md) + [Use TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.1/upgrade-a-tidb-cluster) - + [Use TiDB Ansible](/upgrade-tidb-using-ansible.md) + Scale + [Use TiUP (Recommended)](/scale-tidb-using-tiup.md) - + [Use TiDB Ansible](/scale-tidb-using-ansible.md) + [Use TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.1/scale-a-tidb-cluster) + Backup and Restore + Use BR Tool (Recommended) - + [Use BR Tool](/br/backup-and-restore-tool.md) + + [BR Tool Overview](/br/backup-and-restore-tool.md) + + [Use BR Command-line for Backup and Restoration](/br/use-br-command-line-tool.md) + [BR Use Cases](/br/backup-and-restore-use-cases.md) - + [BR Storages](/br/backup-and-restore-storages.md) - + [Use Dumpling and TiDB Lightning (Recommended)](/backup-and-restore-using-dumpling-lightning.md) - + [Use Mydumper and TiDB Lightning](/backup-and-restore-using-mydumper-lightning.md) - + [Read Historical Data](/read-historical-data.md) + + [External Storages](/br/backup-and-restore-storages.md) + + [BR FAQ](/br/backup-and-restore-faq.md) + [Configure Time Zone](/configure-time-zone.md) + [Daily Checklist](/daily-check.md) + [Maintain TiFlash](/tiflash/maintain-tiflash.md) + [Maintain TiDB Using TiUP](/maintain-tidb-using-tiup.md) - + [Maintain TiDB Using Ansible](/maintain-tidb-using-ansible.md) + [Modify Configuration Online](/dynamic-config.md) + Monitor and Alert + [Monitoring Framework Overview](/tidb-monitoring-framework.md) @@ -114,6 +105,14 @@ + [Overview](/sql-tuning-overview.md) + Understanding the Query Execution Plan + [Overview](/explain-overview.md) + + [`EXPLAIN` Walkthrough](/explain-walkthrough.md) + + [Indexes](/explain-indexes.md) + + [Joins](/explain-joins.md) + + [MPP Queries](/explain-mpp.md) + + [Subqueries](/explain-subqueries.md) + + [Aggregation](/explain-aggregation.md) + + [Views](/explain-views.md) + + [Partitions](/explain-partitions.md) + SQL Optimization Process + [Overview](/sql-optimization-concepts.md) + Logic Optimization @@ -141,8 +140,13 @@ + Tutorials + [Multiple Data Centers in One City Deployment](/multi-data-centers-in-one-city-deployment.md) + [Three Data Centers in Two Cities Deployment](/three-data-centers-in-two-cities-deployment.md) + + Read Historical Data + + Use Stale Read (Recommended) + + [Usage Scenarios of Stale Read](/stale-read.md) + + [Perform Stale Read Using `As OF TIMESTAMP`](/as-of-timestamp.md) + + [Use the `tidb_snapshot` System Variable](/read-historical-data.md) + Best Practices - + [Use TiDB](/tidb-best-practices.md) + + [Use TiDB](/best-practices/tidb-best-practices.md) + [Java Application Development](/best-practices/java-app-best-practices.md) + [Use HAProxy](/best-practices/haproxy-best-practices.md) + [Highly Concurrent Write](/best-practices/high-concurrency-best-practices.md) @@ -150,6 +154,7 @@ + [PD Scheduling](/best-practices/pd-scheduling-best-practices.md) + [TiKV Performance Tuning with Massive Regions](/best-practices/massive-regions-best-practices.md) + [Three-node Hybrid Deployment](/best-practices/three-nodes-hybrid-deployment.md) + + [Local Read Under Three Data Centers Deployment](/best-practices/three-dc-local-read.md) + [Use Placement Rules](/configure-placement-rules.md) + [Use Load Base Split](/configure-load-base-split.md) + [Use Store Limit](/configure-store-limit.md) @@ -158,8 +163,10 @@ + [Use Cases](/ecosystem-tool-user-case.md) + [Download](/download-ecosystem-tools.md) + Backup & Restore (BR) - + [Use BR Tool](/br/backup-and-restore-tool.md) + + [BR Tool Overview](/br/backup-and-restore-tool.md) + + [Use BR Command-line for Backup and Restoration](/br/use-br-command-line-tool.md) + [BR Use Cases](/br/backup-and-restore-use-cases.md) + + [External Storages](/br/backup-and-restore-storages.md) + [BR FAQ](/br/backup-and-restore-faq.md) + TiDB Binlog + [Overview](/tidb-binlog/tidb-binlog-overview.md) @@ -193,7 +200,6 @@ + [Backends](/tidb-lightning/tidb-lightning-backends.md) + [Web Interface](/tidb-lightning/tidb-lightning-web-interface.md) + [Monitor](/tidb-lightning/monitor-tidb-lightning.md) - + [Troubleshoot](/troubleshoot-tidb-lightning.md) + [FAQ](/tidb-lightning/tidb-lightning-faq.md) + [Glossary](/tidb-lightning/tidb-lightning-glossary.md) + TiCDC @@ -201,17 +207,16 @@ + [Deploy](/ticdc/deploy-ticdc.md) + [Maintain](/ticdc/manage-ticdc.md) + [Troubleshoot](/ticdc/troubleshoot-ticdc.md) + + [Monitor](/ticdc/monitor-ticdc.md) + [TiCDC Open Protocol](/ticdc/ticdc-open-protocol.md) + [Integrate TiDB with Confluent Platform](/ticdc/integrate-confluent-using-ticdc.md) + + [Glossary](/ticdc/ticdc-glossary.md) + [Dumpling](/dumpling-overview.md) + sync-diff-inspector + [Overview](/sync-diff-inspector/sync-diff-inspector-overview.md) + [Data Check for Tables with Different Schema/Table Names](/sync-diff-inspector/route-diff.md) + [Data Check in Sharding Scenarios](/sync-diff-inspector/shard-diff.md) + [Data Check for TiDB Upstream/Downstream Clusters](/sync-diff-inspector/upstream-downstream-diff.md) - + [Loader](/loader-overview.md) - + [Mydumper](/mydumper-overview.md) - + [Syncer](/syncer-overview.md) + TiSpark + [Quick Start](/get-started-with-tispark.md) + [User Guide](/tispark-overview.md) @@ -227,12 +232,14 @@ + [PD](/grafana-pd-dashboard.md) + [TiKV](/grafana-tikv-dashboard.md) + [TiFlash](/tiflash/monitor-tiflash.md) + + [TiCDC](/ticdc/monitor-ticdc.md) + Secure + [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) + [Enable TLS Between TiDB Components](/enable-tls-between-components.md) + [Generate Self-signed Certificates](/generate-self-signed-certificates.md) + [Encryption at Rest](/encryption-at-rest.md) + [Enable Encryption for Disk Spill](/enable-disk-spill-encrypt.md) + + [Log Redaction](/log-redaction.md) + Privileges + [Security Compatibility with MySQL](/security-compatibility-with-mysql.md) + [Privilege Management](/privilege-management.md) @@ -365,10 +372,12 @@ + [`SHUTDOWN`](/sql-statements/sql-statement-shutdown.md) + [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) + [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) + + [`TABLE`](/sql-statements/sql-statement-table.md) + [`TRACE`](/sql-statements/sql-statement-trace.md) + [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) + [`UPDATE`](/sql-statements/sql-statement-update.md) + [`USE`](/sql-statements/sql-statement-use.md) + + [`WITH`](/sql-statements/sql-statement-with.md) + Data Types + [Overview](/data-type-overview.md) + [Default Values](/data-type-default-values.md) @@ -395,6 +404,7 @@ + [Precision Math](/functions-and-operators/precision-math.md) + [Set Operations](/functions-and-operators/set-operators.md) + [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) + + [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) + [Clustered Indexes](/clustered-indexes.md) + [Constraints](/constraints.md) + [Generated Columns](/generated-columns.md) @@ -415,6 +425,9 @@ + INFORMATION_SCHEMA + [Overview](/information-schema/information-schema.md) + [`ANALYZE_STATUS`](/information-schema/information-schema-analyze-status.md) + + [`CLIENT_ERRORS_SUMMARY_BY_HOST`](/information-schema/client-errors-summary-by-host.md) + + [`CLIENT_ERRORS_SUMMARY_BY_USER`](/information-schema/client-errors-summary-by-user.md) + + [`CLIENT_ERRORS_SUMMARY_GLOBAL`](/information-schema/client-errors-summary-global.md) + [`CHARACTER_SETS`](/information-schema/information-schema-character-sets.md) + [`CLUSTER_CONFIG`](/information-schema/information-schema-cluster-config.md) + [`CLUSTER_HARDWARE`](/information-schema/information-schema-cluster-hardware.md) @@ -425,7 +438,9 @@ + [`COLLATIONS`](/information-schema/information-schema-collations.md) + [`COLLATION_CHARACTER_SET_APPLICABILITY`](/information-schema/information-schema-collation-character-set-applicability.md) + [`COLUMNS`](/information-schema/information-schema-columns.md) + + [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) + [`DDL_JOBS`](/information-schema/information-schema-ddl-jobs.md) + + [`DEADLOCKS`](/information-schema/information-schema-deadlocks.md) + [`ENGINES`](/information-schema/information-schema-engines.md) + [`INSPECTION_RESULT`](/information-schema/information-schema-inspection-result.md) + [`INSPECTION_RULES`](/information-schema/information-schema-inspection-rules.md) @@ -446,6 +461,7 @@ + [`TIDB_HOT_REGIONS`](/information-schema/information-schema-tidb-hot-regions.md) + [`TIDB_INDEXES`](/information-schema/information-schema-tidb-indexes.md) + [`TIDB_SERVERS_INFO`](/information-schema/information-schema-tidb-servers-info.md) + + [`TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) + [`TIFLASH_REPLICA`](/information-schema/information-schema-tiflash-replica.md) + [`TIKV_REGION_PEERS`](/information-schema/information-schema-tikv-region-peers.md) + [`TIKV_REGION_STATUS`](/information-schema/information-schema-tikv-region-status.md) @@ -475,6 +491,9 @@ + [Use Diagnostics](/dashboard/dashboard-diagnostics-usage.md) + [Search Logs Page](/dashboard/dashboard-log-search.md) + [Profile Instances Page](/dashboard/dashboard-profiling.md) + + Session Management and Configuration + + [Share Session](/dashboard/dashboard-session-share.md) + + [Configure SSO](/dashboard/dashboard-session-sso.md) + [FAQ](/dashboard/dashboard-faq.md) + CLI + [tikv-ctl](/tikv-control.md) @@ -502,12 +521,15 @@ + [Overview](/tiflash/tiflash-overview.md) + [Use TiFlash](/tiflash/use-tiflash.md) + TiUP - + [Documentation Guide](/tiup/tiup-documentation-guide.md) + + [Documentation Map](/tiup/tiup-documentation-guide.md) + [Overview](/tiup/tiup-overview.md) + [Terminology and Concepts](/tiup/tiup-terminology-and-concepts.md) + [Manage TiUP Components](/tiup/tiup-component-management.md) + [FAQ](/tiup/tiup-faq.md) + [Troubleshooting Guide](/tiup/tiup-troubleshooting-guide.md) + + [Reference Guide](/tiup/tiup-reference.md) + + [Cluster Topology Reference](/tiup/tiup-cluster-topology-reference.md) + + [Mirror Reference Guide](/tiup/tiup-mirror-reference.md) + TiUP Components + [tiup-playground](/tiup/tiup-playground.md) + [tiup-cluster](/tiup/tiup-cluster.md) @@ -528,8 +550,21 @@ + [Glossary](/glossary.md) + Release Notes + [All Releases](/releases/release-notes.md) - + [TiDB Roadmap](/roadmap.md) + + v5.1 + + [5.1.1](/releases/release-5.1.1.md) + + [5.1.0](/releases/release-5.1.0.md) + + v5.0 + + [5.0.3](/releases/release-5.0.3.md) + + [5.0.2](/releases/release-5.0.2.md) + + [5.0.1](/releases/release-5.0.1.md) + + [5.0 GA](/releases/release-5.0.0.md) + + [5.0.0-rc](/releases/release-5.0.0-rc.md) + v4.0 + + [4.0.14](/releases/release-4.0.14.md) + + [4.0.13](/releases/release-4.0.13.md) + + [4.0.12](/releases/release-4.0.12.md) + + [4.0.11](/releases/release-4.0.11.md) + + [4.0.10](/releases/release-4.0.10.md) + [4.0.9](/releases/release-4.0.9.md) + [4.0.8](/releases/release-4.0.8.md) + [4.0.7](/releases/release-4.0.7.md) diff --git a/_index.md b/_index.md index 72bbdd3727f8..206090fa55e0 100644 --- a/_index.md +++ b/_index.md @@ -8,7 +8,7 @@ aliases: ['/docs/dev/'] [TiDB](https://github.com/pingcap/tidb) (/’taɪdiːbi:/, "Ti" stands for Titanium) is an open-source, distributed, NewSQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. TiDB can be deployed on-premise or in-cloud. -Designed for the cloud, TiDB provides flexible scalability, reliability and security on the cloud platform. Users can elastically scale TiDB to meet the requirements of their changing workloads. [TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.1/tidb-operator-overview) helps manage TiDB on Kubernetes and automates operating tasks, which makes TiDB easier to deploy on any cloud that provides managed Kubernetes. [TiDB Cloud](https://pingcap.com/products/tidbcloud) (Beta), the fully-managed TiDB service, is the easiest, most economical, and most resilient way to unlock the full power of [TiDB in the cloud](https://docs.pingcap.com/tidbcloud/beta), allowing you to deploy and run TiDB clusters with just a few clicks. +Designed for the cloud, TiDB provides flexible scalability, reliability and security on the cloud platform. Users can elastically scale TiDB to meet the requirements of their changing workloads. [TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.1/tidb-operator-overview) helps manage TiDB on Kubernetes and automates operating tasks, which makes TiDB easier to deploy on any cloud that provides managed Kubernetes. [TiDB Cloud](https://pingcap.com/products/tidbcloud) (Public Preview), the fully-managed TiDB service, is the easiest, most economical, and most resilient way to unlock the full power of [TiDB in the cloud](https://docs.pingcap.com/tidbcloud/beta), allowing you to deploy and run TiDB clusters with just a few clicks. @@ -16,11 +16,10 @@ Designed for the cloud, TiDB provides flexible scalability, reliability and secu - [TiDB Introduction](/overview.md) - [Basic Features](/basic-features.md) -- [What's New in TiDB 4.0](/whats-new-in-tidb-4.0.md) +- [TiDB 5.1 Release Notes](/releases/release-5.1.0.md) - [Compatibility with MySQL](/mysql-compatibility.md) - [Usage Limitations](/tidb-limitations.md) - [TiDB Adopters](/adopters.md) -- [Credits](/credits.md) @@ -47,7 +46,8 @@ Designed for the cloud, TiDB provides flexible scalability, reliability and secu Migrate Data - [Migration Overview](/migration-overview.md) -- [Migrate from Aurora MySQL Database](/migrate-from-aurora-mysql-database.md) +- [Migrate full data from Aurora](/migrate-from-aurora-using-lightning.md) +- [Migrate continuously from Aurora/MySQL Database](/migrate-from-aurora-mysql-database.md) - [Migrate from CSV Files](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) - [Migrate from MySQL SQL Files](/migrate-from-mysql-dumpling-files.md) diff --git a/adopters.md b/adopters.md index 067d63dc1d1a..aafbe2586ac8 100644 --- a/adopters.md +++ b/adopters.md @@ -12,8 +12,6 @@ This is a list of TiDB adopters in various industries. | :--- | :--- | :--- | |[China Telecom Bestpay](https://www.crunchbase.com/organization/orange-finance)|Mobile Payment|[English](https://pingcap.com/case-studies/how-we-process-data-five-times-more-efficiently-using-a-scale-out-mysql-alternative/); [Chinese](https://pingcap.com/cases-cn/user-case-bestpay/)| |[VNG](https://en.wikipedia.org/wiki/VNG_Corporation)|Mobile Payment|English [#1](https://pingcap.com/case-studies/tidb-at-zalopay-infrastructure-lesson-learned/), [#2](https://pingcap.com/case-studies/zalopay-using-a-scale-out-mysql-alternative-to-serve-millions-of-users)| -|[Paytm Labs](https://www.crunchbase.com/organization/paytm-labs)|Mobile Payment|[English](https://pingcap.com/case-studies/gaining-3x-throughput-by-migrating-from-aurora-to-a-scale-out-mysql-alternative)| -|[PayPay](https://www.crunchbase.com/organization/paypay#section-overviewy)|Mobile Payment|[English](https://pingcap.com/case-studies/japan-largest-mobile-payment-company-migrates-from-aurora-to-a-scale-out-database); [Chinese](https://pingcap.com/cases-cn/user-case-paypay/)| |[Ping++](https://www.crunchbase.com/organization/ping-5)|Mobile Payment|[Chinese](https://pingcap.com/cases-cn/user-case-ping++/)| |[LianLian Tech](https://www.crunchbase.com/organization/lianlian-pay)|Mobile Payment|| |[U-Next](https://www.crunchbase.com/organization/u-next)|Media and Entertainment|[English](https://pingcap.com/case-studies/running-a-scale-out-database-on-arm-as-mysql-alternative/); [Chinese](https://pingcap.com/cases-cn/user-case-unext/)| @@ -21,18 +19,22 @@ This is a list of TiDB adopters in various industries. |[iQiyi](https://en.wikipedia.org/wiki/IQiyi)|Media and Entertainment|[English](https://pingcap.com/case-studies/tidb-in-iqiyi/); [Chinese](https://pingcap.com/cases-cn/user-case-iqiyi/)| |[BookMyShow](https://www.crunchbase.com/organization/bookmyshow)|Media and Entertainment|[English](https://pingcap.com/case-studies/tidb-in-bookmyshow/)| |[Yiguo.com](https://www.crunchbase.com/organization/shanghai-yiguo-electron-business)|E-commerce|[English](https://www.datanami.com/2018/02/22/hybrid-database-capturing-perishable-insights-yiguo/); [Chinese](https://pingcap.com/cases-cn/user-case-yiguo)| -|[Shopee](https://en.wikipedia.org/wiki/Shopee)|E-commerce|[English](https://pingcap.com/case-studies/tidb-in-shopee/); [Chinese](https://pingcap.com/cases-cn/user-case-shopee/)| +|[Shopee](https://en.wikipedia.org/wiki/Shopee)|E-commerce|English [#1](https://pingcap.com/case-studies/tidb-in-shopee/), [#2](https://pingcap.com/case-studies/choosing-right-database-for-your-applications); [Chinese](https://pingcap.com/cases-cn/user-case-shopee/)| |[Zhuan Zhuan](https://www.crunchbase.com/organization/zhuan-zhuan)|E-commerce|English [#1](https://pingcap.com/case-studies/tidb-in-zhuanzhuan/), [#2](https://pingcap.com/case-studies/scale-out-database-powers-china-letgo-with-reduced-maintenance-costs); Chinese [#1](https://pingcap.com/cases-cn/user-case-zhuanzhuan/), [#2](https://pingcap.com/cases-cn/user-case-zhuanzhuan-2/), [#3](https://pingcap.com/cases-cn/user-case-zhuanzhuan-3/)| |[Xiaohongshu](https://en.wikipedia.org/wiki/Xiaohongshu)|E-commerce|[English](https://pingcap.com/case-studies/how-we-use-a-scale-out-htap-database-for-real-time-analytics-and-complex-queries); Chinese [#1](https://pingcap.com/cases-cn/user-case-xiaohongshu/), [#2](https://pingcap.com/cases-cn/user-case-xiaohongshu-2/)| +|[Meituan](https://www.crunchbase.com/organization/meituan)|E-commerce|English [#1](https://pingcap.com/case-studies/migrating-from-mysql-to-a-scale-out-database-to-serve-our-290-million-monthly-users), [#2](https://pingcap.com/case-studies/how-we-use-a-mysql-alternative-to-avoid-sharding-and-provide-strong-consistency); [Chinese](https://pingcap.com/cases-cn/user-case-meituan/)| |[Happigo.com](https://www.crunchbase.com/organization/happigo-com)|E-commerce|| -|[Yimutian](http://www.ymt.com/)|E-commerce|| +|[Yimutian](https://www.crunchbase.com/organization/yimutian)|E-commerce|| |[Youju Tech](https://nn.yjyz.com/)|E-commerce|| |[Maizuo](https://www.crunchbase.com/organization/maizhuo)|E-commerce|| |[Mogujie](https://www.crunchbase.com/organization/mogujie)|E-commerce|| -|[Zhihu](https://en.wikipedia.org/wiki/Zhihu)|Knowledge Sharing|[English](https://pingcap.com/case-studies/lesson-learned-from-queries-over-1.3-trillion-rows-of-data-within-milliseconds-of-response-time-at-zhihu/); [Chinese](https://pingcap.com/cases-cn/user-case-zhihu/)| -|[JD Cloud & AI](https://www.crunchbase.com/organization/jd-cloud)|Cloud Computing|[English](https://pingcap.com/case-studies/lesson-learned-from-40-k-qps-and-20-billion-rows-of-data-in-a-single-scale-out-cluster/); [Chinese](https://pingcap.com/cases-cn/user-case-jingdongyun/)| +|[Zhihu](https://en.wikipedia.org/wiki/Zhihu)|Knowledge Sharing|English [#1](https://pingcap.com/case-studies/lesson-learned-from-queries-over-1.3-trillion-rows-of-data-within-milliseconds-of-response-time-at-zhihu/), [#2](https://pingcap.com/case-studies/horizontally-scaling-hive-metastore-database-by-migrating-from-mysql-to-tidb); [Chinese](https://pingcap.com/cases-cn/user-case-zhihu/)| +|[PatSnap](https://www.crunchbase.com/organization/patsnap)|Artificial Intelligence| [English](https://pingcap.com/case-studies/why-we-chose-a-scale-out-data-warehouse-for-real-time-analytics); [Chinese](https://pingcap.com/cases-cn/user-case-zhihuiya/)| +|[JD Cloud](https://www.crunchbase.com/organization/jd-cloud)|Cloud Computing|[English](https://pingcap.com/case-studies/lesson-learned-from-40-k-qps-and-20-billion-rows-of-data-in-a-single-scale-out-cluster/); [Chinese](https://pingcap.com/cases-cn/user-case-jingdongyun/)| |[Mobike](https://en.wikipedia.org/wiki/Mobike)|Ridesharing|[English](https://pingcap.com/case-studies/tidb-in-mobike); Chinese [#1](https://pingcap.com/cases-cn/user-case-mobike/), [#2](https://pingcap.com/cases-cn/user-case-mobike-2/)| -|[Xiaomi](https://en.wikipedia.org/wiki/Xiaomi)|Consumer Electronics|[Chinese](https://pingcap.com/cases-cn/user-case-xiaomi/)| +|[Autohome](https://www.crunchbase.com/organization/autohome)|Automobile|[English](https://pingcap.com/case-studies/reduce-real-time-query-latency-from-0.5s-to-0.01s-with-scale-out-htap-database); [Chinese](https://pingcap.com/cases-cn/user-case-qichezhijia/)| +|[Chehaoduo](https://www.crunchbase.com/organization/guazi-com)|Automobile|[English](https://pingcap.com/case-studies/top-car-trading-platform-chooses-scale-out-database-as-mysql-alternative)| +|[Xiaomi](https://en.wikipedia.org/wiki/Xiaomi)|Consumer Electronics|[English](https://pingcap.com/case-studies/tidb-in-xiaomi); [Chinese](https://pingcap.com/cases-cn/user-case-xiaomi/)| |[LY.com](https://www.crunchbase.com/organization/ly-com)|Travel|[Chinese](https://pingcap.com/cases-cn/user-case-tongcheng/)| |[Qunar.com](https://www.crunchbase.com/organization/qunar-com)|Travel|[Chinese](https://pingcap.com/cases-cn/user-case-qunar/)| |[Hulu](https://www.hulu.com)|Entertainment|| @@ -42,16 +44,20 @@ This is a list of TiDB adopters in various industries. |[Bank of Beijing](https://en.wikipedia.org/wiki/Bank_of_Beijing)|Banking|[English](https://pingcap.com/case-studies/how-we-use-a-distributed-database-to-achieve-horizontal-scaling-without-downtime); Chinese [#1](https://pingcap.com/cases-cn/user-case-beijing-bank/), [#2](https://pingcap.com/cases-cn/user-case-beijing-bank-2/)| |[WeBank](https://en.wikipedia.org/wiki/WeBank_(China))|Banking|[English](https://pingcap.com/case-studies/how-we-reduced-batch-processing-time-by-58-percent-with-a-scale-out-mysql-alternative/); Chinese [#1](https://pingcap.com/cases-cn/user-case-webank/), [#2](https://pingcap.com/cases-cn/user-case-webank-2/)| |[Bank of China](https://en.wikipedia.org/wiki/Bank_of_China) | Banking | [English](https://pingcap.com/case-studies/how-bank-of-china-uses-a-scale-out-database-to-support-zabbix-monitoring-at-scale); [Chinese](https://pingcap.com/cases-cn/user-case-bank-of-china/) | +|[China Zheshang Bank](https://en.wikipedia.org/wiki/China_Zheshang_Bank)|Banking|[English](https://pingcap.com/case-studies/reduce-query-latency-from-seconds-to-milliseconds-with-a-scale-out-database); [Chinese](https://pingcap.com/cases-cn/user-case-zheshang-bank/)| |[Industrial and Commercial Bank of China](https://en.wikipedia.org/wiki/Industrial_and_Commercial_Bank_of_China)|Banking|| |[Ping An Life Insurance](https://www.bloomberg.com/profile/company/OPAHWZ:CH)|Insurance|[English](https://pingcap.com/case-studies/how-chinas-insurance-giant-improved-agile-application-performance-with-a-newsql-database); [Chinese](https://pingcap.com/cases-cn/user-case-pingan/)| |[Yimian Data](https://www.crunchbase.com/organization/yimian-data)|Big Data|[Chinese](https://pingcap.com/cases-cn/user-case-yimian)| |[CAASDATA](https://www.caasdata.com/)|Big Data|[Chinese](https://pingcap.com/cases-cn/user-case-kasi/)| +|[58.com](https://www.crunchbase.com/organization/58-com)|Advertising|[English](https://pingcap.com/case-studies/no-sharding-no-etl-use-scale-out-mysql-alternative-to-store-160-tb-of-data); [Chinese](https://pingcap.com/cases-cn/user-case-58/)| |[Mobikok](https://www.linkedin.com/company/shenzhen-keke-network-technology-co.-ltd./)|AdTech|[Chinese](https://pingcap.com/cases-cn/user-case-mobikok/)| +|[Ninja Van](https://www.crunchbase.com/organization/ninja-van-2)| Logistics|[English](https://pingcap.com/case-studies/choose-a-mysql-alternative-over-vitess-and-crdb-to-scale-out-our-databases-on-k8s)| |[ZTO Express](https://www.crunchbase.com/organization/zto-express)| Logistics|[English](https://pingcap.com/case-studies/why-we-migrated-from-exadata-to-a-scale-out-htap-database-for-near-real-time-analytics); [Chinese](https://pingcap.com/cases-cn/user-case-zto-express/)| |[G7 Networks](https://www.crunchbase.com/organization/g7)| Logistics|[Chinese](https://pingcap.com/cases-cn/user-case-g7/)| |[Hive-Box](http://www.fcbox.com/en/pc/index.html#/)|Logistics|[Chinese](https://pingcap.com/cases-cn/user-case-fengchao/)| |[NetEase Games](https://www.linkedin.com/company/netease-games)|Gaming|[English](https://pingcap.com/case-studies/why-we-chose-tidb-over-other-mysql-based-and-newsql-storage-solutions); [Chinese](https://pingcap.com/cases-cn/user-case-wangyihuyu/)| |[GAEA](http://www.gaea.com/en/)|Gaming|[English](https://pingcap.com/case-studies/2017-05-22-Comparison-between-MySQL-and-TiDB-with-tens-of-millions-of-data-per-day); [Chinese](https://pingcap.com/cases-cn/user-case-gaea-ad/)| +|[Kunlun](https://www.crunchbase.com/organization/kunlun)|Gaming|[English](https://pingcap.com/case-studies/empowering-your-gaming-application-with-a-scale-out-newsql-database)| |[YOOZOO Games](https://www.crunchbase.com/organization/yoozoo-games)|Gaming|[Chinese](https://pingcap.com/cases-cn/user-case-youzu/)| |[Seasun Games](https://www.crunchbase.com/organization/seasun)|Gaming|[Chinese](https://pingcap.com/cases-cn/user-case-xishanju/)| |[FUNYOURS JAPAN](http://company.funyours.co.jp/)|Gaming|[Chinese](https://pingcap.com/cases-cn/user-case-funyours-japan/)| @@ -71,7 +77,6 @@ This is a list of TiDB adopters in various industries. |[Keruyun](https://www.crunchbase.com/organization/keruyun-technology-beijing-co-ltd)|SaaS|[Chinese](https://pingcap.com/cases-cn/user-case-keruyun/)| |[LinkDoc Technology](https://www.crunchbase.com/organization/linkdoc-technology)|HealthTech|[Chinese](https://pingcap.com/cases-cn/user-case-linkdoc/)| |[Chunyu Yisheng](https://www.crunchbase.com/organization/chunyu)|HealthTech|| -|[Meituan-Dianping](https://en.wikipedia.org/wiki/Meituan-Dianping)|Food Delivery|[English](https://pingcap.com/case-studies/tidb-in-meituan-dianping/); [Chinese](https://pingcap.com/cases-cn/user-case-meituan/)| |[Qutoutiao](https://www.crunchbase.com/organization/qutoutiao)|Social Network|| |[Jinri Toutiao](https://en.wikipedia.org/wiki/Toutiao)|Mobile News Platform|[Chinese](https://pingcap.com/cases-cn/user-case-toutiao/)| |[360 Finance](https://www.crunchbase.com/organization/360-finance)|FinTech|[Chinese](https://pingcap.com/cases-cn/user-case-360/)| diff --git a/alert-rules.md b/alert-rules.md index 548068f18a4c..783ffa15f0f0 100644 --- a/alert-rules.md +++ b/alert-rules.md @@ -10,7 +10,13 @@ aliases: ['/docs/dev/alert-rules/','/docs/dev/reference/alert-rules/'] This document describes the alert rules for different components in a TiDB cluster, including the rule descriptions and solutions of the alert items in TiDB, TiKV, PD, TiDB Binlog, Node_exporter and Blackbox_exporter. -According to the severity level, alert rules are divided into three categories (from high to low): emergency-level, critical-level and warning-level. +According to the severity level, alert rules are divided into three categories (from high to low): emergency-level, critical-level, and warning-level. This division of severity levels applies to all alert items of each component below. + +| Severity level | Description | +| :-------- | :----- | +| Emergency-level | The highest severity level at which the service is unavailable. Emergency-level alerts are often caused by a service or node failure. **Manual intervention is required immediately**. | +| Critical-level | Decreased service availability. For the critical-level alerts, a close watch on the abnormal metrics is required. | +| Warning-level | Warning-level alerts are a reminder for an issue or error. | ## TiDB alert rules @@ -18,8 +24,6 @@ This section gives the alert rules for the TiDB component. ### Emergency-level alerts -Emergency-level alerts are often caused by a service or node failure. Manual intervention is required immediately. - #### `TiDB_schema_error` * Alert rule: @@ -79,8 +83,6 @@ Emergency-level alerts are often caused by a service or node failure. Manual int ### Critical-level alerts -For the critical-level alerts, a close watch on the abnormal metrics is required. - #### `TiDB_server_panic_total` * Alert rule: @@ -97,8 +99,6 @@ For the critical-level alerts, a close watch on the abnormal metrics is required ### Warning-level alerts -Warning-level alerts are a reminder for an issue or error. - #### `TiDB_memory_abnormal` * Alert rule: @@ -193,8 +193,6 @@ This section gives the alert rules for the PD component. ### Emergency-level alerts -Emergency-level alerts are often caused by a service or node failure. Manual intervention is required immediately. - #### `PD_cluster_offline_tikv_nums` * Alert rule: @@ -212,8 +210,6 @@ Emergency-level alerts are often caused by a service or node failure. Manual int ### Critical-level alerts -For the critical-level alerts, a close watch on the abnormal metrics is required. - #### `PD_etcd_write_disk_latency` * Alert rule: @@ -247,8 +243,6 @@ For the critical-level alerts, a close watch on the abnormal metrics is required ### Warning-level alerts -Warning-level alerts are a reminder for an issue or error. - #### `PD_cluster_lost_connect_tikv_nums` * Alert rule: @@ -413,8 +407,6 @@ This section gives the alert rules for the TiKV component. ### Emergency-level alerts -Emergency-level alerts are often caused by a service or node failure. Manual intervention is required immediately. - #### `TiKV_memory_used_too_fast` * Alert rule: @@ -433,26 +425,20 @@ Emergency-level alerts are often caused by a service or node failure. Manual int * Alert rule: - `sum(increase(tidb_tikvclient_gc_action_result{type="success"}[6h])) < 1` - - > **Note:** - > - > In TiDB 3.* versions, the `tidb_tikvclient_gc_action_result` metric exists but does not have a value. It's because distributed garbage collection (GC) is introduced in the TiDB 3.0 version but will not be performed in TiDB. + `sum(increase(tikv_gcworker_gc_tasks_vec{task="gc"}[1d])) < 1 and sum(increase(tikv_gc_compaction_filter_perform[1d])) < 1` * Description: - GC is not performed successfully in a Region within 6 hours, which indicates that GC is not working properly. If GC does not run in a short term, it will not cause much trouble; but if GC keeps down, more and more versions are retained, which slows down the query. + GC is not performed successfully on a TiKV instance within 24 hours, which indicates that GC is not working properly. If GC does not run in a short term, it will not cause much trouble; but if GC keeps down, more and more versions are retained, which slows down the query. * Solution: - 1. Perform `select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME = "tikv_gc_leader_desc"` to locate the `tidb-server` corresponding to the GC leader; + 1. Perform `SELECT VARIABLE_VALUE FROM mysql.tidb WHERE VARIABLE_NAME = "tikv_gc_leader_desc"` to locate the `tidb-server` corresponding to the GC leader; 2. View the log of the `tidb-server`, and grep gc_worker tidb.log; 3. If you find that the GC worker has been resolving locks (the last log is "start resolve locks") or deleting ranges (the last log is “start delete {number} ranges”) during this time, it means the GC process is running normally. Otherwise, contact [support@pingcap.com](mailto:support@pingcap.com) to resolve this issue. ### Critical-level alerts -For the critical-level alerts, a close watch on the abnormal metrics is required. - #### `TiKV_server_report_failure_msg_total` * Alert rule: @@ -616,29 +602,8 @@ For the critical-level alerts, a close watch on the abnormal metrics is required The pressure on the apply Raft log thread is too high. It is often caused by a burst of writes. -#### `TiDB_tikvclient_gc_action_fail` (only happen when in special configurations) - -* Alert rule: - - `sum(increase(tidb_tikvclient_gc_action_result{type="fail”}[1m])) > 10` - - > **Note:** - > - > In TiDB 3.* versions, the `tidb_tikvclient_gc_action_result` metric exists but does not have a value. It's because distributed garbage collection (GC) is introduced in the TiDB 3.0 version but will not be performed in TiDB. - -* Description: - - There are many Regions where GC fails to work. - -* Solution: - - 1. It is normally because the GC concurrency is set too high. You can moderately lower the GC concurrency degree, and you need to first confirm that the failed GC is caused by the busy server. - 2. You can moderately lower the concurrency degree by running `update set VARIABLE_VALUE="{number}” where VARIABLE_NAME=”tikv_gc_concurrency”`. - ### Warning-level alerts -Warning-level alerts are a reminder for an issue or error. - #### `TiKV_leader_drops` * Alert rule: @@ -817,8 +782,6 @@ This section gives the alert rules for the Node_exporter host. ### Emergency-level alerts -Emergency-level alerts are often caused by a service or node failure. Manual intervention is required immediately. - #### `NODE_disk_used_more_than_80%` * Alert rule: @@ -846,7 +809,7 @@ Emergency-level alerts are often caused by a service or node failure. Manual int * Solution: - * Log in to the machine and run the `df -i` command to view the inode usage of the filesystem. + * Log in to the machine and run the `df -i` command to view the node usage of the filesystem. * Make a plan to increase the disk capacity or delete some data or increase cluster node depending on different situations. #### `NODE_disk_readonly` @@ -866,8 +829,6 @@ Emergency-level alerts are often caused by a service or node failure. Manual int ### Critical-level alerts -For the critical-level alerts, a close watch on the abnormal metrics is required. - #### `NODE_memory_used_more_than_80%` * Alert rule: @@ -885,8 +846,6 @@ For the critical-level alerts, a close watch on the abnormal metrics is required ### Warning-level alerts -Warning-level alerts are a reminder for an issue or error. - #### `NODE_node_overload` * Alert rule: @@ -970,8 +929,6 @@ This section gives the alert rules for the Blackbox_exporter TCP, ICMP, and HTTP ### Emergency-level alerts -Emergency-level alerts are often caused by a service or node failure. Manual intervention is required immediately. - #### `TiDB_server_is_down` * Alert rule: @@ -1150,8 +1107,6 @@ Emergency-level alerts are often caused by a service or node failure. Manual int ### Warning-level alerts -Warning-level alerts are a reminder for an issue or error. - #### `BLACKER_ping_latency_more_than_1s` * Alert rule: diff --git a/as-of-timestamp.md b/as-of-timestamp.md new file mode 100644 index 000000000000..ce63be0ff055 --- /dev/null +++ b/as-of-timestamp.md @@ -0,0 +1,274 @@ +--- +title: Read Historical Data Using the `AS OF TIMESTAMP` Clause +summary: Learn how to read historical data using the `AS OF TIMESTAMP` statement clause. +--- + +# Read Historical Data Using the `AS OF TIMESTAMP` Clause + +This document describes how to perform the [Stale Read](/stale-read.md) feature using the `AS OF TIMESTAMP` clause to read historical data in TiDB, including specific usage examples and strategies for saving historical data. + +> **Warning:** +> +> Currently, you cannot use Stale Read together with TiFlash. If your SQL query contains the `AS OF TIMESTAMP` clause and TiDB might read data from TiFlash replicas, you might encounter an error with a message like `ERROR 1105 (HY000): stale requests require tikv backend`. +> +> To fix the problem, disable TiFlash replicas for your Stale Read query. To do that, perform one of the following operations: +> +> - Use the `set session tidb_isolation_read_engines='tidb,tikv'` variable. +> - Use the [hint](/optimizer-hints.md#read_from_storagetiflasht1_name--tl_name--tikvt2_name--tl_name-) to enforce TiDB to read data from TiKV. + +TiDB supports reading historical data through a standard SQL interface, which is the `AS OF TIMESTAMP` SQL clause, without the need for special clients or drivers. After data is updated or deleted, you can read the historical data before the update or deletion using this SQL interface. + +> **Note:** +> +> When reading historical data, TiDB returns the data with the old table structure even if the current table structure is different. + +## Syntax + +You can use the `AS OF TIMESTAMP` clause in the following three ways: + +- [`SELECT ... FROM ... AS OF TIMESTAMP`](/sql-statements/sql-statement-select.md) +- [`START TRANSACTION READ ONLY AS OF TIMESTAMP`](/sql-statements/sql-statement-start-transaction.md) +- [`SET TRANSACTION READ ONLY AS OF TIMESTAMP`](/sql-statements/sql-statement-set-transaction.md) + +If you want to specify an exact point of time, you can set a datetime value or use a time function in the `AS OF TIMESTAMP` clause. The format of datetime is like "2016-10-08 16:45:26.999", with millisecond as the minimum time unit, but for most of the time, the time unit of second is enough for specifying a datetime, such as "2016-10-08 16:45:26". You can also get the current time to the millisecond using the `NOW(3)` function. If you want to read the data of several seconds ago, it is **recommended** to use an expression such as `NOW() - INTERVAL 10 SECOND`. + +If you want to specify a time range, you can use the `TIDB_BOUNDED_STALENESS()` function in the clause. When this function is used, TiDB selects a suitable timestamp within the specified time range. "Suitable" means there are no transactions that start before this timestamp and have not been committed on the accessed replica, that is, TiDB can perform read operations on the accessed replica and the read operations are not blocked. You need to use `TIDB_BOUNDED_STALENESS(t1, t2)` to call this function. `t1` and `t2` are the two ends of the time range, which can be specified using either datetime values or time functions. + +Here are some examples of the `AS OF TIMESTAMP` clause: + +- `AS OF TIMESTAMP '2016-10-08 16:45:26'`: Tells TiDB to read the latest data stored at 16:45:26 on October 8, 2016. +- `AS OF TIMESTAMP NOW() - INTERVAL 10 SECOND`: Tells TiDB to read the latest data stored 10 seconds ago. +- `AS OF TIMESTAMP TIDB_BOUNDED_STALENESS('2016-10-08 16:45:26', '2016-10-08 16:45:29')`: Tells TiDB to read the data as new as possible within the time range of 16:45:26 to 16:45:29 on October 8, 2016. +- `AS OF TIMESTAMP TIDB_BOUNDED_STALENESS(NOW() - INTERVAL 20 SECOND, NOW())`: Tells TiDB to read the data as new as possible within the time range of 20 seconds ago to the present. + +> **Note:** +> +> In addition to specifying a timestamp, the most common use of the `AS OF TIMESTAMP` clause is to read data that is several seconds old. If this approach is used, it is recommended to read historical data older than 5 seconds. +> +> You need to deploy the NTP service for your TiDB and PD nodes when you use Stale Read. This avoids the situation where the specified timestamp used by TiDB goes ahead of the latest TSO allocating progress (such as a timestamp several seconds ahead), or is later than the GC safe point timestamp. When the specified timestamp goes beyond the service scope, TiDB returns an error. + +## Usage examples + +This section describes different ways to use the `AS OF TIMESTAMP` clause with several examples. It first introduces how to prepare the data for recovery, and then shows how to use `AS OF TIMESTAMP` in `SELECT`, `START TRANSACTION READ ONLY AS OF TIMESTAMP`, and `SET TRANSACTION READ ONLY AS OF TIMESTAMP` respectively. + +### Prepare data sample + +To prepare data for recovery, create a table first and insert several rows of data: + +```sql +create table t (c int); +``` + +``` +Query OK, 0 rows affected (0.01 sec) +``` + +```sql +insert into t values (1), (2), (3); +``` + +``` +Query OK, 3 rows affected (0.00 sec) +``` + +View the data in the table: + +```sql +select * from t; +``` + +``` ++------+ +| c | ++------+ +| 1 | +| 2 | +| 3 | ++------+ +3 rows in set (0.00 sec) +``` + +View the current time: + +```sql +select now(); +``` + +``` ++---------------------+ +| now() | ++---------------------+ +| 2021-05-26 16:45:26 | ++---------------------+ +1 row in set (0.00 sec) +``` + +Update the data in a row: + +```sql +update t set c=22 where c=2; +``` + +``` +Query OK, 1 row affected (0.00 sec) +``` + +Confirm that the data of the row is updated: + +```sql +select * from t; +``` + +``` ++------+ +| c | ++------+ +| 1 | +| 22 | +| 3 | ++------+ +3 rows in set (0.00 sec) +``` + +### Read historical data using the `SELECT` statement + +You can use the [`SELECT ... FROM ... AS OF TIMESTAMP`](/sql-statements/sql-statement-select.md) statement to read data from a time point in the past. + +```sql +select * from t as of timestamp '2021-05-26 16:45:26'; +``` + +``` ++------+ +| c | ++------+ +| 1 | +| 2 | +| 3 | ++------+ +3 rows in set (0.00 sec) +``` + +> **Note:** +> +> When reading multiple tables using one `SELECT` statement, you need to make sure that the format of TIMESTAMP EXPRESSIONs is consistent. For example, `select * from t as of timestamp NOW() - INTERVAL 2 SECOND, c as of timestamp NOW() - INTERVAL 2 SECOND;`. In addition, you must specify the `AS OF` information for the relevant table in the `SELECT` statement; otherwise, the `SELECT` statement reads the latest data by default. + +### Read historical data using the `START TRANSACTION READ ONLY AS OF TIMESTAMP` statement + +You can use the [`START TRANSACTION READ ONLY AS OF TIMESTAMP`](/sql-statements/sql-statement-start-transaction.md) statement to start a read-only transaction based on a time point in the past. The transaction reads historical data of the given time. + +```sql +start transaction read only as of timestamp '2021-05-26 16:45:26'; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +```sql +select * from t; +``` + +``` ++------+ +| c | ++------+ +| 1 | +| 2 | +| 3 | ++------+ +3 rows in set (0.00 sec) +``` + +```sql +commit; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +After the transaction is committed, you can read the latest data. + +```sql +select * from t; +``` + +``` ++------+ +| c | ++------+ +| 1 | +| 22 | +| 3 | ++------+ +3 rows in set (0.00 sec) +``` + +> **Note:** +> +> If you start a transaction with the statement `START TRANSACTION READ ONLY AS OF TIMESTAMP`, it is a read-only transaction. Write operations are rejected in this transaction. + +### Read historical data using the `SET TRANSACTION READ ONLY AS OF TIMESTAMP` statement + +You can use the [`SET TRANSACTION READ ONLY AS OF TIMESTAMP`](/sql-statements/sql-statement-set-transaction.md) statement to set the next transaction as a read-only transaction based on a specified time point in the past. The transaction reads historical data of the given time. + +```sql +set transaction read only as of timestamp '2021-05-26 16:45:26'; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +```sql +begin; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +```sql +select * from t; +``` + +``` ++------+ +| c | ++------+ +| 1 | +| 2 | +| 3 | ++------+ +3 rows in set (0.00 sec) +``` + +```sql +commit; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +After the transaction is committed, you can read the latest data. + +```sql +select * from t; +``` + +``` ++------+ +| c | ++------+ +| 1 | +| 22 | +| 3 | ++------+ +3 rows in set (0.00 sec) +``` + +> **Note:** +> +> If you start a transaction with the statement `SET TRANSACTION READ ONLY AS OF TIMESTAMP`, it is a read-only transaction. Write operations are rejected in this transaction. diff --git a/auto-random.md b/auto-random.md index 67ede73209ab..d80743d3f915 100644 --- a/auto-random.md +++ b/auto-random.md @@ -142,7 +142,7 @@ This attribute supports forward compatibility, namely, downgrade compatibility. Pay attention to the following restrictions when you use `AUTO_RANDOM`: -- Specify this attribute for the primary key column **ONLY** of integer type. Otherwise, an error might occur. In addition, when the value of `alter-primary-key` is `true`, `AUTO_RANDOM` is not supported even on the integer primary key. +- Specify this attribute for the primary key column **ONLY** of integer type. Otherwise, an error might occur. In addition, when the attribute of the primary key is `NONCLUSTERED`, `AUTO_RANDOM` is not supported even on the integer primary key. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). - You cannot use `ALTER TABLE` to modify the `AUTO_RANDOM` attribute, including adding or removing this attribute. - You cannot change the column type of the primary key column that is specified with `AUTO_RANDOM` attribute. - You cannot specify `AUTO_RANDOM` and `AUTO_INCREMENT` for the same column at the same time. diff --git a/backup-and-restore-using-dumpling-lightning.md b/backup-and-restore-using-dumpling-lightning.md index 2159ef7ced61..abb685e3ca38 100644 --- a/backup-and-restore-using-dumpling-lightning.md +++ b/backup-and-restore-using-dumpling-lightning.md @@ -1,11 +1,15 @@ --- title: Use Dumpling and TiDB Lightning for Data Backup and Restoration summary: Introduce how to use Dumpling and TiDB Lightning to backup and restore full data of TiDB. -aliases: ['/docs/dev/export-or-backup-using-dumpling/','/tidb/dev/export-or-backup-using-dumpling'] +aliases: ['/docs/dev/export-or-backup-using-dumpling/','/tidb/dev/export-or-backup-using-dumpling','/docs/dev/backup-and-restore-using-mydumper-lightning/','/docs/dev/how-to/maintain/backup-and-restore/mydumper-lightning/','/docs/dev/how-to/maintain/backup-and-restore/','/tidb/dev/backup-and-restore-using-mydumper-lightning/'] --- # Use Dumpling and TiDB Lightning for Data Backup and Restoration +> **Warning:** +> +> It is no longer recommended to use Dumpling and TiDB Lightning for data backup and restoration. It is strongly recommended to use [BR tool](/br/backup-and-restore-tool.md) instead for a better tool experience. + This document introduces in detail how to use Dumpling and TiDB Lightning to backup and restore full data of TiDB. For incremental backup and replication to downstream, refer to [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md). Suppose that the TiDB server information is as follows: @@ -55,22 +59,22 @@ The steps to manually modify the GC time are as follows: {{< copyable "sql" >}} ```sql - SELECT * FROM mysql.tidb WHERE VARIABLE_NAME = 'tikv_gc_life_time'; + SHOW GLOBAL VARIABLES LIKE 'tidb_gc_life_time'; ``` ```sql - +-----------------------+------------------------------------------------------------------------------------------------+ - | VARIABLE_NAME | VARIABLE_VALUE | - +-----------------------+------------------------------------------------------------------------------------------------+ - | tikv_gc_life_time | 10m0s | - +-----------------------+------------------------------------------------------------------------------------------------+ - 1 rows in set (0.02 sec) + +-------------------+-------+ + | Variable_name | Value | + +-------------------+-------+ + | tidb_gc_life_time | 10m0s | + +-------------------+-------+ + 1 row in set (0.03 sec) ``` {{< copyable "sql" >}} ```sql - UPDATE mysql.tidb SET VARIABLE_VALUE = '720h' WHERE VARIABLE_NAME = 'tikv_gc_life_time'; + SET GLOBAL tidb_gc_life_time = '720h'; ``` 2. After executing the `dumpling` command, restore the GC value of the TiDB cluster to the initial value in step 1: @@ -78,7 +82,7 @@ The steps to manually modify the GC time are as follows: {{< copyable "sql" >}} ```sql - UPDATE mysql.tidb SET VARIABLE_VALUE = '10m' WHERE VARIABLE_NAME = 'tikv_gc_life_time'; + SET GLOBAL tidb_gc_life_time = '10m'; ``` ## Restore data into TiDB diff --git a/backup-and-restore-using-mydumper-lightning.md b/backup-and-restore-using-mydumper-lightning.md deleted file mode 100644 index a1914895cebd..000000000000 --- a/backup-and-restore-using-mydumper-lightning.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: Use Mydumper and TiDB Lightning for Backup and Restoration -aliases: ['/docs/dev/backup-and-restore-using-mydumper-lightning/','/docs/dev/how-to/maintain/backup-and-restore/mydumper-lightning/','/docs/dev/how-to/maintain/backup-and-restore/'] ---- - -# Use Mydumper and TiDB Lightning for Data Backup and Restoration - -This document describes how to perform full backup and restoration of the TiDB data using Mydumper and TiDB Lightning. For incremental backup and restoration, refer to [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md). - -Suppose that the TiDB server information is as follows: - -|Server Name|Server Address|Port|User|Password| -|:----|:-------|:----|:----|:--------| -|TiDB|127.0.0.1|4000|root|*| - -Use the following tools for data backup and restoration: - -- [Mydumper](/mydumper-overview.md): to export data from TiDB -- [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md): to import data into TiDB - -## Full backup and restoration using Mydumper/TiDB Lightning - -`mydumper` is a powerful data backup tool. For more information, refer to [`maxbube/mydumper`](https://github.com/maxbube/mydumper). - -Use [Mydumper](/mydumper-overview.md) to export data from TiDB and use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to import data into TiDB. - -> **Note:** -> -> It is recommended to download [Mydumper](/mydumper-overview.md) from the PingCAP website, because the R&D team has adapted `mydumper` for TiDB. It is not recommended to use `mysqldump` which is much slower for both backup and restoration. - -### Best practices for full backup and restoration using Mydumper/TiDB Lightning - -To quickly backup and restore data (especially large amounts of data), refer to the following recommendations: - -* Keep the exported data file as small as possible. It is recommended to use the `-F` option of Mydumper to set the file size. If you use TiDB Lightning to restore data, it is recommended that you set the value of `-F` to `256` (MB). If you use `loader` for restoration, it is recommended to set the value to `64` (MB). - -## Backup data from TiDB - -Use the following `mydumper` command to backup data from TiDB: - -{{< copyable "shell-regular" >}} - -```bash -./bin/mydumper -h 127.0.0.1 -P 4000 -u root -t 32 -F 256 -B test -T t1,t2 --skip-tz-utc -o ./var/test -``` - -In this command: - -- `-B test` means that the data is exported from the `test` database. -- `-T t1,t2` means that only the `t1` and `t2` tables are exported. -- `-t 32` means that 32 threads are used to export the data. -- `-F 256` means that a table is partitioned into chunks, and one chunk is 256MB. -- `--skip-tz-utc` means to ignore the inconsistency of time zone setting between MySQL and the data exporting machine and to disable automatic conversion. - -If `mydumper` returns the following error: - -``` -** (mydumper:27528): CRITICAL **: 13:25:09.081: Could not read data from testSchema.testTable: GC life time is shorter than transaction duration, transaction starts at 2019-08-05 21:10:01.451 +0800 CST, GC safe point is 2019-08-05 21:14:53.801 +0800 CST -``` - -Then execute two more commands: - -1. Before executing the `mydumper` command, query the [GC](/garbage-collection-overview.md) values of the TiDB cluster and adjust it to a suitable value using the MySQL client: - - {{< copyable "sql" >}} - - ```sql - SELECT * FROM mysql.tidb WHERE VARIABLE_NAME = 'tikv_gc_life_time'; - ``` - - ``` - +-----------------------+------------------------------------------------------------------------------------------------+ - | VARIABLE_NAME | VARIABLE_VALUE | - +-----------------------+------------------------------------------------------------------------------------------------+ - | tikv_gc_life_time | 10m0s | - +-----------------------+------------------------------------------------------------------------------------------------+ - 1 rows in set (0.02 sec) - ``` - - {{< copyable "sql" >}} - - ```sql - UPDATE mysql.tidb SET VARIABLE_VALUE = '720h' WHERE VARIABLE_NAME = 'tikv_gc_life_time'; - ``` - -2. After running the `mydumper` command, adjust GC value of the TiDB cluster to its original value in step 1. - - {{< copyable "sql" >}} - - ```sql - UPDATE mysql.tidb SET VARIABLE_VALUE = '10m' WHERE VARIABLE_NAME = 'tikv_gc_life_time'; - ``` - -## Restore data into TiDB - -To restore data into TiDB, use TiDB Lightning to import the exported data. See [TiDB Lightning Tutorial](/tidb-lightning/tidb-lightning-backends.md). diff --git a/basic-sql-operations.md b/basic-sql-operations.md index 19175527bc56..8cdf0169ca39 100644 --- a/basic-sql-operations.md +++ b/basic-sql-operations.md @@ -187,7 +187,7 @@ To insert data into a table, use the `INSERT` statement: {{< copyable "sql" >}} ```sql -INSERT INTO person VALUES("1","tom","20170912"); +INSERT INTO person VALUES(1,'tom','20170912'); ``` To insert a record containing data of some fields into a table, use the `INSERT` statement: @@ -195,7 +195,7 @@ To insert a record containing data of some fields into a table, use the `INSERT` {{< copyable "sql" >}} ```sql -INSERT INTO person(id,name) VALUES("2","bob"); +INSERT INTO person(id,name) VALUES('2','bob'); ``` To update some fields of a record in a table, use the `UPDATE` statement: diff --git a/benchmark/benchmark-sysbench-v4-vs-v3.md b/benchmark/benchmark-sysbench-v4-vs-v3.md index bbde9660aea7..9387c91b18cc 100644 --- a/benchmark/benchmark-sysbench-v4-vs-v3.md +++ b/benchmark/benchmark-sysbench-v4-vs-v3.md @@ -83,7 +83,6 @@ raftdb.max-background-jobs: 3 raftdb.allow-concurrent-memtable-write: true server.request-batch-enable-cross-command: false server.grpc-concurrency: 6 -readpool.unifiy-read-pool: true readpool.unified.min-thread-count: 5 readpool.unified.max-thread-count: 20 readpool.storage.normal-concurrency: 10 diff --git a/benchmark/benchmark-sysbench-v5-vs-v4.md b/benchmark/benchmark-sysbench-v5-vs-v4.md new file mode 100644 index 000000000000..7ba96dc5aacc --- /dev/null +++ b/benchmark/benchmark-sysbench-v5-vs-v4.md @@ -0,0 +1,223 @@ +--- +title: TiDB Sysbench Performance Test Report -- v5.0 vs. v4.0 +--- + +# TiDB Sysbench Performance Test Report -- v5.0 vs. v4.0 + +## Test purpose + +This test aims at comparing the Sysbench performance of TiDB v5.0 and TiDB v4.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment (AWS EC2） + +### Hardware configuration + +| Service type | EC2 type | Instance count | +|:----------|:----------|:----------| +| PD | m5.xlarge | 3 | +| TiKV | i3.4xlarge| 3 | +| TiDB | c5.4xlarge| 3 | +| Sysbench | c5.9xlarge| 1 | + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| PD | 4.0 and 5.0 | +| TiDB | 4.0 and 5.0 | +| TiKV | 4.0 and 5.0 | +| Sysbench | 1.0.20 | + +### Parameter configuration + +#### TiDB v4.0 configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV v4.0 configuration + +{{< copyable "" >}} + +```yaml +storage.scheduler-worker-pool-size: 5 +raftstore.store-pool-size: 3 +raftstore.apply-pool-size: 3 +rocksdb.max-background-jobs: 3 +raftdb.max-background-jobs: 3 +raftdb.allow-concurrent-memtable-write: true +server.request-batch-enable-cross-command: false +server.grpc-concurrency: 6 +readpool.unified.min-thread-count: 5 +readpool.unified.max-thread-count: 20 +readpool.storage.normal-concurrency: 10 +pessimistic-txn.pipelined: true +``` + +#### TiDB v5.0 configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV v5.0 configuration + +{{< copyable "" >}} + +```yaml +storage.scheduler-worker-pool-size: 5 +raftstore.store-pool-size: 3 +raftstore.apply-pool-size: 3 +rocksdb.max-background-jobs: 8 +raftdb.max-background-jobs: 4 +raftdb.allow-concurrent-memtable-write: true +server.grpc-concurrency: 6 +readpool.unified.min-thread-count: 5 +readpool.unified.max-thread-count: 20 +readpool.storage.normal-concurrency: 10 +pessimistic-txn.pipelined: true +server.enable-request-batch: false +``` + +#### TiDB v4.0 global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +``` + +#### TiDB v5.0 global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +set global tidb_enable_async_commit = 1; +set global tidb_enable_1pc = 1; +set global tidb_guarantee_linearizability = 0; +set global tidb_enable_clustered_index = 1; + +``` + +## Test plan + +1. Deploy TiDB v5.0 and v4.0 using TiUP. +2. Use Sysbench to import 16 tables, each table with 10 million rows of data. +3. Execute the `analyze table` statement on each table. +4. Back up the data used for restore before different concurrency tests, which ensures data consistency for each test. +5. Start the Sysbench client to perform the `point_select`, `read_write`, `update_index`, and `update_non_index` tests. Perform stress tests on TiDB via AWS NLB. In each type of test, the warm-up takes 1 minute and the test takes 5 minutes. +6. After each type of test is completed, stop the cluster, overwrite the cluster with the backup data in step 4, and restart the cluster. + +### Prepare test data + +Execute the following command to prepare the test data: + +{{< copyable "shell-regular" >}} + +```bash +sysbench oltp_common \ + --threads=16 \ + --rand-type=uniform \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=$aws_nlb_host \ + --mysql-port=$aws_nlb_port \ + --mysql-user=root \ + --mysql-password=password \ + prepare --tables=16 --table-size=10000000 +``` + +### Perform the test + +Execute the following command to perform the test. + +{{< copyable "shell-regular" >}} + +```bash +sysbench $testname \ + --threads=$threads \ + --time=300 \ + --report-interval=1 \ + --rand-type=uniform \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=$aws_nlb_host \ + --mysql-port=$aws_nlb_port \ + run --tables=16 --table-size=10000000 +``` + +## Test results + +### Point Select performance + +| Threads | v4.0 QPS | v4.0 95% latency (ms) | v5.0 QPS | v5.0 95% latency (ms) | QPS improvement | +|:----------|:----------|:----------|:----------|:----------|:----------| +| 150 | 159451.19 | 1.32 | 177876.25 | 1.23 | 11.56% | +| 300 | 244790.38 | 1.96 | 252675.03 | 1.82 | 3.22% | +| 600 | 322929.05 | 3.75 | 331956.84 | 3.36 | 2.80% | +| 900 | 364840.05 | 5.67 | 365655.04 | 5.09 | 0.22% | +| 1200 | 376529.18 | 7.98 | 366507.47 | 7.04 | -2.66% | +| 1500 | 368390.52 | 10.84 | 372476.35 | 8.90 | 1.11% | + +Compared with v4.0, the Point Select performance of TiDB v5.0 has increased by 2.7%. + +![Point Select](/media/sysbench_v5vsv4_point_select.png) + +### Update Non-index performance + +| Threads | v4.0 QPS | v4.0 95% latency (ms) | v5.0 QPS | v5.0 95% latency (ms) | QPS improvement | +|:----------|:----------|:----------|:----------|:----------|:----------| +| 150 | 17243.78 | 11.04 | 30866.23 | 6.91 | 79.00% | +| 300 | 25397.06 | 15.83 | 45915.39 | 9.73 | 80.79% | +| 600 | 33388.08 | 25.28 | 60098.52 | 16.41 | 80.00% | +| 900 | 38291.75 | 36.89 | 70317.41 | 21.89 | 83.64% | +| 1200 | 41003.46 | 55.82 | 76376.22 | 28.67 | 86.27% | +| 1500 | 44702.84 | 62.19 | 80234.58 | 34.95 | 79.48% | + +Compared with v4.0, the Update Non-index performance of TiDB v5.0 has increased by 81%. + +![Update Non-index](/media/sysbench_v5vsv4_update_non_index.png) + +### Update Index performance + +| Threads | v4.0 QPS | v4.0 95% latency (ms) | v5.0 QPS | v5.0 95% latency (ms) | QPS improvement | +|:----------|:----------|:----------|:----------|:----------|:----------| +| 150 | 11736.21 | 17.01 | 15631.34 | 17.01 | 33.19% | +| 300 | 15435.95 | 28.67 | 19957.06 | 22.69 | 29.29% | +| 600 | 18983.21 | 49.21 | 23218.14 | 41.85 | 22.31% | +| 900 | 20855.29 | 74.46 | 26226.76 | 53.85 | 25.76% | +| 1200 | 21887.64 | 102.97 | 28505.41 | 69.29 | 30.24% | +| 1500 | 23621.15 | 110.66 | 30341.06 | 82.96 | 28.45% | + +Compared with v4.0, the Update Index performance of TiDB v5.0 has increased by 28%. + +![Update Index](/media/sysbench_v5vsv4_update_index.png) + +### Read Write performance + +| Threads | v4.0 QPS | v4.0 95% latency (ms) | v5.0 QPS | v5.0 95% latency (ms) | QPS improvement | +|:----------|:----------|:----------|:----------|:----------|:----------| +| 150 | 59979.91 | 61.08 | 66098.57 | 55.82 | 10.20% | +| 300 | 77118.32 | 102.97 | 84639.48 | 90.78 | 9.75% | +| 600 | 90619.52 | 183.21 | 101477.46 | 167.44 | 11.98% | +| 900 | 97085.57 | 267.41 | 109463.46 | 240.02 | 12.75% | +| 1200 | 106521.61 | 331.91 | 115416.05 | 320.17 | 8.35% | +| 1500 | 116278.96 | 363.18 | 118807.5 | 411.96 | 2.17% | + +Compared with v4.0, the read-write performance of TiDB v5.0 has increased by 9%. + +![Read Write](/media/sysbench_v5vsv4_read_write.png) diff --git a/benchmark/benchmark-sysbench-v5.1.0-vs-v5.0.2.md b/benchmark/benchmark-sysbench-v5.1.0-vs-v5.0.2.md new file mode 100644 index 000000000000..5e413d27b2b3 --- /dev/null +++ b/benchmark/benchmark-sysbench-v5.1.0-vs-v5.0.2.md @@ -0,0 +1,185 @@ +--- +title: TiDB Sysbench Performance Test Report -- v5.1.0 vs. v5.0.2 +--- + +# TiDB Sysbench Performance Test Report -- v5.1.0 vs. v5.0.2 + +## Test overview + +This test aims at comparing the Sysbench performance of TiDB v5.1.0 and TiDB v5.0.2 in the Online Transactional Processing (OLTP) scenario. The results show that compared with v5.0.2, the Point Select performance of v5.1.0 is improved by 19.4%, and the performance of the Read Write and Update Index is slightly reduced. + +## Test environment (AWS EC2） + +### Hardware configuration + +| Service type | EC2 type | Instance count | +|:----------|:----------|:----------| +| PD | m5.xlarge | 3 | +| TiKV | i3.4xlarge| 3 | +| TiDB | c5.4xlarge| 3 | +| Sysbench | c5.9xlarge| 1 | + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| PD | v5.0.2 and v5.1.0 | +| TiDB | v5.0.2 and v5.1.0 | +| TiKV | v5.0.2 and v5.1.0 | +| Sysbench | 1.0.20 | + +### Parameter configuration + +TiDB v5.1.0 and TiDB v5.0.2 use the same configuration. + +#### TiDB parameter configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV parameter configuration + +{{< copyable "" >}} + +```yaml +storage.scheduler-worker-pool-size: 5 +raftstore.store-pool-size: 3 +raftstore.apply-pool-size: 3 +rocksdb.max-background-jobs: 8 +raftdb.max-background-jobs: 4 +raftdb.allow-concurrent-memtable-write: true +server.grpc-concurrency: 6 +readpool.unified.min-thread-count: 5 +readpool.unified.max-thread-count: 20 +readpool.storage.normal-concurrency: 10 +pessimistic-txn.pipelined: true +server.enable-request-batch: false +``` + +#### TiDB global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +set global tidb_enable_async_commit = 1; +set global tidb_enable_1pc = 1; +set global tidb_guarantee_linearizability = 0; +set global tidb_enable_clustered_index = 1; +``` + +## Test plan + +1. Deploy TiDB v5.1.0 and v5.0.2 using TiUP. +2. Use Sysbench to import 16 tables, each table with 10 million rows of data. +3. Execute the `analyze table` statement on each table. +4. Back up the data used for restore before different concurrency tests, which ensures data consistency for each test. +5. Start the Sysbench client to perform the `point_select`, `read_write`, `update_index`, and `update_non_index` tests. Perform stress tests on TiDB via HAProxy. The test takes 5 minutes. +6. After each type of test is completed, stop the cluster, overwrite the cluster with the backup data in step 4, and restart the cluster. + +### Prepare test data + +Execute the following command to prepare the test data: + +{{< copyable "shell-regular" >}} + +```bash +sysbench oltp_common \ + --threads=16 \ + --rand-type=uniform \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=$aws_nlb_host \ + --mysql-port=$aws_nlb_port \ + --mysql-user=root \ + --mysql-password=password \ + prepare --tables=16 --table-size=10000000 +``` + +### Perform the test + +Execute the following command to perform the test: + +{{< copyable "shell-regular" >}} + +```bash +sysbench $testname \ + --threads=$threads \ + --time=300 \ + --report-interval=1 \ + --rand-type=uniform \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=$aws_nlb_host \ + --mysql-port=$aws_nlb_port \ + run --tables=16 --table-size=10000000 +``` + +## Test results + +### Point Select performance + +| Threads | v5.0.2 QPS | v5.0.2 95% latency (ms) | v5.1.0 QPS | v5.1.0 95% latency (ms) | QPS improvement | +|:----------|:----------|:----------|:----------|:----------|:----------| +|150|137732.27|1.86|158861.67|2|15.34%| +|300|201420.58|2.91|238038.44|2.71|18.18%| +|600|303631.52|3.49|428573.21|2.07|41.15%| +|900|383628.13|3.55|464863.22|3.89|21.18%| +|1200|391451.54|5.28|413656.74|13.46|5.67%| +|1500|410276.93|7.43|471418.78|10.65|14.90%| + +Compared with v5.0.2, the Point Select performance of v5.1.0 is improved by 19.4%. + +![Point Select](/media/sysbench_v510vsv502_point_select.png) + +### Update Non-index performance + +| Threads | v5.0.2 QPS | v5.0.2 95% latency (ms) | v5.1.0 QPS | v5.1.0 95% latency (ms) | QPS improvement | +|:----------|:----------|:----------|:----------|:----------|:----------| +|150|29248.2|7.17|29362.7|8.13|0.39%| +|300|40316.09|12.52|39651.52|13.7|-1.65%| +|600|51011.11|22.28|47047.9|27.66|-7.77%| +|900|58814.16|27.66|59331.84|28.67|0.88%| +|1200|65286.52|32.53|67745.39|31.37|3.77%| +|1500|68300.86|39.65|67899.17|44.17|-0.59%| + +Compared with v5.0.2, the Update Non-index performance of v5.1.0 is reduced by 0.8%. + +![Update Non-index](/media/sysbench_v510vsv502_update_non_index.png) + +### Update Index performance + +| Threads | v5.0.2 QPS | v5.0.2 95% latency (ms) | v5.1.0 QPS | v5.1.0 95% latency (ms) | QPS improvement | +|:----------|:----------|:----------|:----------|:----------|:----------| +|150|15066.54|14.73|14829.31|14.73|-1.57%| +|300|18535.92|24.83|17401.01|29.72|-6.12%| +|600|22862.73|41.1|21923.78|44.98|-4.11%| +|900|25286.74|57.87|24916.76|58.92|-1.46%| +|1200|27566.18|70.55|27800.62|69.29|0.85%| +|1500|28184.76|92.42|28679.72|86|1.76%| + +Compared with v5.0.2, the Update Index performance of v5.1.0 is reduced by 1.8%. + +![Update Index](/media/sysbench_v510vsv502_update_index.png) + +### Read Write performance + +| Threads | v5.0.2 QPS | v5.0.2 95% latency (ms) | v5.1.0 QPS | v5.1.0 95% latency (ms) | QPS improvement | +|:----------|:----------|:----------|:----------|:----------|:----------| +|150|66415.33|56.84|66591.49|57.87|0.27%| +|300|82488.39|97.55|81226.41|101.13|-1.53%| +|600|99195.36|173.58|97357.86|179.94|-1.85%| +|900|107382.76|253.35|101665.95|267.41|-5.32%| +|1200|112389.23|337.94|107426.41|350.33|-4.42%| +|1500|113548.73|450.77|109805.26|442.73|-3.30%| + +Compared with v5.0.2, the Read Write performance of v5.1.0 is reduced by 2.7%. + +![Read Write](/media/sysbench_v510vsv502_read_write.png) diff --git a/benchmark/benchmark-tidb-using-sysbench.md b/benchmark/benchmark-tidb-using-sysbench.md index 902a35ff8d66..e3130f651b1f 100644 --- a/benchmark/benchmark-tidb-using-sysbench.md +++ b/benchmark/benchmark-tidb-using-sysbench.md @@ -5,54 +5,19 @@ aliases: ['/docs/dev/benchmark/benchmark-tidb-using-sysbench/','/docs/dev/benchm # How to Test TiDB Using Sysbench -In this test, Sysbench 1.0.14 and TiDB 3.0 Beta are used. It is recommended to use Sysbench 1.0 or later, which can be [downloaded here](https://github.com/akopytov/sysbench/releases/tag/1.0.14). - -## Test environment - -- [Hardware recommendations](/hardware-and-software-requirements.md) - -- The TiDB cluster is deployed according to the [TiDB Deployment Guide](https://pingcap.com/docs/v3.0/online-deployment-using-ansible/). Suppose there are 3 servers in total. It is recommended to deploy 1 TiDB instance, 1 PD instance and 1 TiKV instance on each server. As for disk space, supposing that there are 32 tables and 10M rows of data on each table, it is recommended that the disk space where TiKV's data directory resides is larger than 512 GB. - -The number of concurrent connections to a single TiDB cluster is recommended to be under 500. If you need to increase the concurrency pressure on the entire system, you can add TiDB instances to the cluster whose number depends on the pressure of the test. - -IDC machines: - -| Type | Name | -|:---- |:---- | -| OS | Linux (CentOS 7.3.1611) | -| CPU | 40 vCPUs, Intel® Xeon® CPU E5-2630 v4 @ 2.20GHz | -| RAM | 128GB | -| DISK | Intel Optane SSD P4800X 375G * 1 | -| NIC | 10Gb Ethernet | +It is recommended to use Sysbench 1.0 or later, which can be [downloaded here](https://github.com/akopytov/sysbench/releases/tag/1.0.14). ## Test plan -### TiDB version information - -| Component | GitHash | -|:---- |:---- | -| TiDB | 7a240818d19ae96e4165af9ea35df92466f59ce6 | -| TiKV | e26ceadcdfe94fb6ff83b5abb614ea3115394bcd | -| PD | 5e81548c3c1a1adab056d977e7767307a39ecb70 | - -### Cluster topology - -| Machine IP | Deployment instance | -|:---- |:---- | -| 172.16.30.31 | 3*sysbench | -| 172.16.30.33 | 1\*tidb 1\*pd 1\*tikv | -| 172.16.30.34 | 1\*tidb 1\*pd 1\*tikv | -| 172.16.30.35 | 1\*tidb 1\*pd 1\*tikv | - ### TiDB configuration -Higher log level means fewer logs to be printed and thus positively influences TiDB performance. Enable `prepared plan cache` in the TiDB configuration to lower the cost of optimizing execution plan. Specifically, you can add the following command in the TiDB configuration file: +Higher log level means fewer logs to be printed and thus positively influences TiDB performance. Enable `prepared plan cache` in the TiDB configuration to lower the cost of optimizing execution plan. Specifically, you can add the following command in the TiUP configuration file: -```toml -[log] -level = "error" -[prepared-plan-cache] -enabled = true +```yaml +server_configs: + tidb: + log.level: "error" + prepared-plan-cache.enabled: true ``` ### TiKV configuration @@ -63,22 +28,22 @@ There are multiple Column Families on TiKV cluster which are mainly used to stor Default CF : Write CF = 4 : 1 -Configuring the block cache of RocksDB on TiKV should be based on the machine’s memory size, in order to make full use of the memory. To deploy a TiKV cluster on a 40GB virtual machine, it is suggested to configure the block cache as follows: +Configuring the block cache of RocksDB on TiKV should be based on the machine’s memory size, in order to make full use of the memory. To deploy a TiKV cluster on a 40GB virtual machine, it is recommended to configure the block cache as follows: -```toml -log-level = "error" -[rocksdb.defaultcf] -block-cache-size = "24GB" -[rocksdb.writecf] -block-cache-size = "6GB" +```yaml +server_configs: + tikv: + log-level: "error" + rocksdb.defaultcf.block-cache-size: "24GB" + rocksdb.writecf.block-cache-size: "6GB" ``` -For TiDB 3.0 or later versions, you can also use the shared block cache to configure: +You can also configure TiKV to share block cache: -```toml -log-level = "error" -[storage.block-cache] -capacity = "30GB" +```yaml +server_configs: + tikv: + storage.block-cache.capacity: "30GB" ``` For more detailed information on TiKV performance tuning, see [Tune TiKV Performance](/tune-tikv-memory-performance.md). @@ -87,7 +52,7 @@ For more detailed information on TiKV performance tuning, see [Tune TiKV Perform > **Note:** > -> This test was performed without load balancing tools such as HAproxy. We run the Sysbench test on individual TiDB node and added the results up. The load balancing tools and the parameters of different versions might also impact the performance. +> The test in this document was performed without load balancing tools such as HAproxy. We run the Sysbench test on individual TiDB node and added the results up. The load balancing tools and the parameters of different versions might also impact the performance. ### Sysbench configuration @@ -123,6 +88,10 @@ db-driver=mysql ### Data import +> **Note:** +> +> If you enable the optimistic transaction model (TiDB uses the pessimistic transaction model by default), TiDB rolls back transactions when a concurrency conflict is found. Setting `tidb_disable_txn_auto_retry` to `off` turns on the automatic retry mechanism after meeting a transaction conflict, which can prevent Sysbench from quitting because of the transaction conflict error. + Before importing the data, it is necessary to make some settings to TiDB. Execute the following command in MySQL client: {{< copyable "sql" >}} @@ -131,7 +100,7 @@ Before importing the data, it is necessary to make some settings to TiDB. Execut set global tidb_disable_txn_auto_retry = off; ``` -Then exit the client. TiDB uses an optimistic transaction model that rolls back transactions when a concurrency conflict is found. Setting `tidb_disable_txn_auto_retry` to `off` turns on the automatic retry mechanism after meeting a transaction conflict, which can prevent Sysbench from quitting because of the transaction conflict error. +Then exit the client. Restart MySQL client and execute the following SQL statement to create a database `sbtest`: @@ -204,49 +173,6 @@ sysbench --config-file=config oltp_update_index --tables=32 --table-size=1000000 sysbench --config-file=config oltp_read_only --tables=32 --table-size=10000000 run ``` -## Test results - -32 tables are tested, each with 10M of data. - -Sysbench test was carried on each of the tidb-servers. And the final result was a sum of all the results. - -### oltp_point_select - -| Type | Thread | TPS | QPS | avg.latency(ms) | .95.latency(ms) | max.latency(ms) | -|:---- |:---- |:---- |:---- |:----------------|:----------------- |:---- | -| point_select | 3\*8 | 67502.55 | 67502.55 | 0.36 | 0.42 | 141.92 | -| point_select | 3\*16 | 120141.84 | 120141.84 | 0.40 | 0.52 | 20.99 | -| point_select | 3\*32 | 170142.92 | 170142.92 | 0.58 | 0.99 | 28.08 | -| point_select | 3\*64 | 195218.54 | 195218.54 | 0.98 | 2.14 | 21.82 | -| point_select | 3\*128 | 208189.53 | 208189.53 | 1.84 | 4.33 | 31.02 | - -![oltp_point_select](/media/oltp_point_select.png) - -### oltp_update_index - -| Type | Thread | TPS | QPS | avg.latency(ms) | .95.latency(ms) | max.latency(ms) | -|:---- |:---- |:---- |:---- |:----------------|:----------------- |:---- | -| oltp_update_index | 3\*8 | 9668.98 | 9668.98 | 2.51 | 3.19 | 103.88| -| oltp_update_index | 3\*16 | 12834.99 | 12834.99 | 3.79 | 5.47 | 176.90 | -| oltp_update_index | 3\*32 | 15955.77 | 15955.77 | 6.07 | 9.39 | 4787.14 | -| oltp_update_index | 3\*64 | 18697.17 | 18697.17 | 10.34 | 17.63 | 4539.04 | -| oltp_update_index | 3\*128 | 20446.81 | 20446.81 | 18.98 | 40.37 | 5394.75 | -| oltp_update_index | 3\*256 | 23563.03 | 23563.03 | 32.86 | 78.60 | 5530.69 | - -![oltp_update_index](/media/oltp_update_index.png) - -### oltp_read_only - -| Type | Thread | TPS | QPS | avg.latency(ms) | .95.latency(ms) | max.latency(ms) | -|:---- |:---- |:---- |:---- |:----------------|:----------------- |:---- | -| oltp_read_only | 3\*8 | 2411.00 | 38575.96 | 9.92 | 20.00 | 92.23 | -| oltp_read_only | 3\*16 | 3873.53 | 61976.50 | 12.25 | 16.12 | 56.94 | -| oltp_read_only | 3\*32 | 5066.88 | 81070.16 | 19.42 | 26.20 | 123.41 | -| oltp_read_only | 3\*64 | 5466.36 | 87461.81 | 34.65 | 63.20 | 231.19 | -| oltp_read_only | 3\*128 | 6684.16 | 106946.59 | 57.29 | 97.55 | 180.85 | - -![oltp_read_only](/media/oltp_read_only.png) - ## Common issues ### TiDB and TiKV are both properly configured under high concurrency, why is the overall performance still low? diff --git a/benchmark/benchmark-tidb-using-tpcc.md b/benchmark/benchmark-tidb-using-tpcc.md index 35d85fc98a20..95ad33f16f08 100644 --- a/benchmark/benchmark-tidb-using-tpcc.md +++ b/benchmark/benchmark-tidb-using-tpcc.md @@ -29,256 +29,71 @@ In this document, the testing uses 1,000 warehouses as an example to test TiDB. TPC-C uses tpmC (transactions per minute) to measure the maximum qualified throughput (MQTh, Max Qualified Throughput). The transactions are the NewOrder transactions and the final unit of measure is the number of new orders processed per minute. -This testing uses the open-source BenchmarkSQL 5.0 as the TPC-C testing tool and adds the support for the MySQL protocol. You can download the testing program by using the following command: +The test in this document is implemented based on [go-tpc](https://github.com/pingcap/go-tpc). You can download the test program using [TiUP](/tiup/tiup-overview.md) commands. {{< copyable "shell-regular" >}} ```shell -git clone -b 5.0-mysql-support-opt-2.1 https://github.com/pingcap/benchmarksql.git +tiup install bench ``` -To install Java and Ant, run the following command (take CentOS as an example): - -{{< copyable "shell-regular" >}} - -```shell -sudo yum install -y java ant -``` - -Enter the `benchmarksql` directory and run `ant` to build: - -{{< copyable "shell-regular" >}} - -```shell -cd benchmarksql && \ -ant -``` - -## Deploy the TiDB cluster - -For 1,000 warehouses, the TiDB cluster can be deployed on three machines, with one TiDB instance, one PD instance, and one TiKV instance on each machine. - -For example, the hardware configuration is as follows: - -| Type | Name | -| :--- | :---| -| OS | Linux (CentOS 7.3.1611) | -| CPU | 40 vCPUs, Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz | -| RAM | 128GB | -| DISK | Optane 500GB SSD | - -1. Because this type of CPU has a NUMA architecture, it is recommended to [bind the core using `numactl`](/check-before-deployment.md#install-the-numactl-tool). - -2. Execute the `lscpu` command to view the NUMA nodes. The result is similar to: - - ```text - NUMA node0 CPU(s): 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38 - NUMA node1 CPU(s): 1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39 - ``` - -3. Start TiDB by adding `numactl` to the `{tidb_deploy_path}/scripts/run_tidb.sh` start-up script: - - ```text - #!/bin/bash - set -e - - ulimit -n 1000000 - - # WARNING: This file was auto-generated. Do not edit! - # All your edit might be overwritten! - DEPLOY_DIR=/home/damon/deploy/tidb1-1 - - cd "${DEPLOY_DIR}" || exit 1 - - export TZ=Asia/Shanghai - - # You need to specify different cpunodebind and membind for different TiDB instances on the same machine to bind different Numa nodes. - exec numactl --cpunodebind=0 --membind=0 bin/tidb-server \ - -P 4111 \ - --status="10191" \ - --advertise-address="172.16.4.53" \ - --path="172.16.4.10:2490" \ - --config=conf/tidb.toml \ - --log-slow-query="/home/damon/deploy/tidb1-1/log/tidb_slow_query.log" \ - --log-file="/home/damon/deploy/tidb1-1/log/tidb.log" 2>> "/home/damon/deploy/tidb1-1/log/tidb_stderr.log" - ``` - - > **Note:** - > - > Direct modification of `run_tidb.sh` may be overwritten. So in a production environment, it is recommended to use TiUP if you need to bind the core. - -4. You can deploy an HAproxy to balance the loads on multiple TiDB nodes. It is recommended to configure `nbproc` as the number of CPU cores. - -## Edit the configuration - -### Configure TiDB - -```toml -[log] -level = "error" - -[performance] -# Sets the maximum number of CPU cores to use for a single TiDB instance according to NUMA configuration. -max-procs = 20 - -[prepared-plan-cache] -# Enables the prepared plan cache in TiDB configuration to reduce the overhead of optimizing your execution plan. -enabled = true -``` - -### Configure TiKV - -You can use the basic configuration at the beginning. Then after the test is run, you can adjust it based on the metrics on Grafana and the [Tune TiKV Thread Performance](/tune-tikv-thread-performance.md). - -### Configure BenchmarkSQL - -Edit the `benchmarksql/run/props.mysql` file: - -```text -conn=jdbc:mysql://{HAPROXY-HOST}:{HAPROXY-PORT}/tpcc?useSSL=false&useServerPrepStmts=true&useConfigs=maxPerformance -warehouses=1000 # Uses 1,000 warehouses. -terminals=500 # Uses 500 terminals. -loadWorkers=32 # The number of concurrent workers that load data. -``` +For detailed usage of the TiUP Bench component, see [TiUP Bench](/tiup/tiup-bench.md). ## Load data -**Loading data is usually the most time-consuming and problematic stage of the entire TPC-C test.** This section provides the following four steps to load data. - -First, use a MySQL client to connect to the TiDB server and run the following command: - -{{< copyable "sql" >}} - -```sql -create database tpcc; -``` +**Loading data is usually the most time-consuming and problematic stage of the entire TPC-C test.** This section provides the following command to load data. -Second, run the following BenchmarkSQL script in shell to create tables: +Execute the following TiUP command in Shell: {{< copyable "shell-regular" >}} ```shell -cd run && \ -./runSQL.sh props.mysql sql.mysql/tableCreates.sql && \ -./runSQL.sh props.mysql sql.mysql/indexCreates.sql +tiup bench tpcc -H 172.16.5.140 -P 4000 -D tpcc --warehouses 1000 prepare ``` -Third, use one of the following two ways to load data: +Based on different machine configurations, this loading process might take a few hours. If the cluster size is small, you can use a smaller `WAREHOUSE` value for the test. -* [Use BenchmarkSQL to load data directly](#use-benchmarksql-to-load-data-directly) -* [Use TiDB Lightning to load data](#use-tidb-lightning-to-load-data) +After the data is loaded, you can execute the `tiup bench tpcc -H 172.16.5.140 -P 4000 -D tpcc --warehouses 4 check` command to validate the data correctness. -### Use BenchmarkSQL to load data directly +## Run the test -Run the following script to load data: +Execute the following command to run the test: {{< copyable "shell-regular" >}} ```shell -./runLoader.sh props.mysql +tiup bench tpcc -H 172.16.5.140 -P 4000 -D tpcc --warehouses 1000 run ``` -This process might last for several hours depending on the machine configuration. - -### Use TiDB Lightning to load data - -The amount of loaded data increases as the number of warehouses increases. When you need to load more than 1000 warehouses of data, you can first use BenchmarkSQL to generate CSV files, and then quickly load the CSV files through TiDB Lightning (hereinafter referred to as Lightning). The CSV files can be reused multiple times, which saves the time required for each generation. - -Follow the steps below to use TiDB Lightning to load data: - -1. Modify the BenchmarkSQL configuration file. - - The CSV file of one warehouse requires 77 MB of disk space. To ensure sufficient disk space, add a line to the `benchmarksql/run/props.mysql` file: - - ```text - fileLocation=/home/user/csv/ # The absolute path of the directory where your CSV files are stored - ``` - - It is recommended that the CSV file names adhere to the naming rules in Lightning, that is, `{database}.{table}.csv`, because eventually you'll use Lightning to load data. Here you can modify the above configuration as follows: - - ```text - fileLocation=/home/user/csv/tpcc. # The absolute path of the directory where your CSV files are stored + the file name prefix (database) - ``` - - This will generate CSV files with a naming style such as `tpcc.bmsql_warehouse.csv`. - -2. Generate the CSV file. - - {{< copyable "shell-regular" >}} - - ```shell - ./runLoader.sh props.mysql - ``` - -3. Use Lightning to load data. - - To load data using Lightning, see [TiDB Lightning Deployment](/tidb-lightning/deploy-tidb-lightning.md). The following steps introduce how to use TiDB Ansible to deploy Lightning and use Lightning to load data. +During the test, test results are continuously printed on the console: - 1. Edit `inventory.ini`. - - It is recommended to manually specify the deployed IP address, the port, and the deployment directory to avoid anomalies caused by conflicts. For the disk space of `import_dir`, see [TiDB Lightning Deployment](/tidb-lightning/deploy-tidb-lightning.md). `data_source_dir` refers to the directory where the CSV files are stored as mentioned before. - - ```ini - [importer_server] - IS1 ansible_host=172.16.5.34 deploy_dir=/data2/is1 tikv_importer_port=13323 import_dir=/data2/import - [lightning_server] - LS1 ansible_host=172.16.5.34 deploy_dir=/data2/ls1 tidb_lightning_pprof_port=23323 data_source_dir=/home/user/csv - ``` - - 2. Edit `conf/tidb-lightning.yml`. - - ```yaml - mydumper: - no-schema: true - csv: - separator: ',' - delimiter: '' - header: false - not-null: false - 'null': 'NULL' - backslash-escape: true - trim-last-separator: false - ``` - - 3. Deploy Lightning and Importer. - - {{< copyable "shell-regular" >}} - - ```shell - ansible-playbook deploy.yml --tags=lightning - ``` - - 4. Start Lightning and Importer. +```text +[Current] NEW_ORDER - Takes(s): 4.6, Count: 5, TPM: 65.5, Sum(ms): 4604, Avg(ms): 920, 90th(ms): 1500, 99th(ms): 1500, 99.9th(ms): 1500 +[Current] ORDER_STATUS - Takes(s): 1.4, Count: 1, TPM: 42.2, Sum(ms): 256, Avg(ms): 256, 90th(ms): 256, 99th(ms): 256, 99.9th(ms): 256 +[Current] PAYMENT - Takes(s): 6.9, Count: 5, TPM: 43.7, Sum(ms): 2208, Avg(ms): 441, 90th(ms): 512, 99th(ms): 512, 99.9th(ms): 512 +[Current] STOCK_LEVEL - Takes(s): 4.4, Count: 1, TPM: 13.8, Sum(ms): 224, Avg(ms): 224, 90th(ms): 256, 99th(ms): 256, 99.9th(ms): 256 +... +``` - * Log into the server where Lightning and Importer are deployed. - * Enter the deployment directory. - * Execute `scripts/start_importer.sh` under the Importer directory to start Importer. - * Execute `scripts/start_lightning.sh` under the Lightning directory to begin to load data. +After the test is finished, the test summary results are printed: - Because you've used TiDB Ansible deployment method, you can see the loading progress of Lightning on the monitoring page, or check whether the loading process is completed through the log. +```text +[Summary] DELIVERY - Takes(s): 455.2, Count: 32, TPM: 4.2, Sum(ms): 44376, Avg(ms): 1386, 90th(ms): 2000, 99th(ms): 4000, 99.9th(ms): 4000 +[Summary] DELIVERY_ERR - Takes(s): 455.2, Count: 1, TPM: 0.1, Sum(ms): 953, Avg(ms): 953, 90th(ms): 1000, 99th(ms): 1000, 99.9th(ms): 1000 +[Summary] NEW_ORDER - Takes(s): 487.8, Count: 314, TPM: 38.6, Sum(ms): 282377, Avg(ms): 899, 90th(ms): 1500, 99th(ms): 1500, 99.9th(ms): 1500 +[Summary] ORDER_STATUS - Takes(s): 484.6, Count: 29, TPM: 3.6, Sum(ms): 8423, Avg(ms): 290, 90th(ms): 512, 99th(ms): 1500, 99.9th(ms): 1500 +[Summary] PAYMENT - Takes(s): 490.1, Count: 321, TPM: 39.3, Sum(ms): 144708, Avg(ms): 450, 90th(ms): 512, 99th(ms): 1000, 99.9th(ms): 1500 +[Summary] STOCK_LEVEL - Takes(s): 487.6, Count: 41, TPM: 5.0, Sum(ms): 9318, Avg(ms): 227, 90th(ms): 512, 99th(ms): 1000, 99.9th(ms): 1000 +``` -Fourth, after successfully loading data, you can run `sql.common/test.sql` to validate the correctness of the data. If all SQL statements return an empty result, then the data is correctly loaded. +After the test is finished, you can execute the `tiup bench tpcc -H 172.16.5.140 -P 4000 -D tpcc --warehouses 4 check` command to validate the data correctness. -## Run the test +## Clean up test data -Run the following BenchmarkSQL test script: +Execute the following command to clean up the test data: {{< copyable "shell-regular" >}} ```shell -nohup ./runBenchmark.sh props.mysql &> test.log & +tiup bench tpcc -H 172.16.5.140 -P 4000 -D tpcc --warehouses 4 cleanup ``` - -After the execution is finished, view the result using `test.log`: - -```text -07:09:53,455 [Thread-351] INFO jTPCC : Term-00, Measured tpmC (NewOrders) = 77373.25 -07:09:53,455 [Thread-351] INFO jTPCC : Term-00, Measured tpmTOTAL = 171959.88 -07:09:53,455 [Thread-351] INFO jTPCC : Term-00, Session Start = 2019-03-21 07:07:52 -07:09:53,456 [Thread-351] INFO jTPCC : Term-00, Session End = 2019-03-21 07:09:53 -07:09:53,456 [Thread-351] INFO jTPCC : Term-00, Transaction Count = 345240 -``` - -The value in the tpmC section is the testing result. - -After the test completes, you can also run `sql.common/test.sql` to validate the correctness of the data. If all SQL statements return an empty result, then the testing of the data is correctly performed. diff --git a/benchmark/v4.0-performance-benchmarking-with-tpcc.md b/benchmark/v4.0-performance-benchmarking-with-tpcc.md index d3c8d8c64288..a654c6103fd6 100644 --- a/benchmark/v4.0-performance-benchmarking-with-tpcc.md +++ b/benchmark/v4.0-performance-benchmarking-with-tpcc.md @@ -83,7 +83,6 @@ raftdb.max-background-jobs: 3 raftdb.allow-concurrent-memtable-write: true server.request-batch-enable-cross-command: false server.grpc-concurrency: 6 -readpool.unifiy-read-pool: true readpool.unified.min-thread-count: 5 readpool.unified.max-thread-count: 20 readpool.storage.normal-concurrency: 10 diff --git a/benchmark/v4.0-performance-benchmarking-with-tpch.md b/benchmark/v4.0-performance-benchmarking-with-tpch.md index cda6ab2b885e..801db52f05b6 100644 --- a/benchmark/v4.0-performance-benchmarking-with-tpch.md +++ b/benchmark/v4.0-performance-benchmarking-with-tpch.md @@ -163,6 +163,10 @@ To avoid TiKV and TiFlash racing for disk and I/O resources, mount the two NVMe ## Test result +> **Note:** +> +> The tables on which SQL statements are executed in this test only have primary keys and do not have secondary indexes. Therefore, the test result below is not influenced by indexes. + | Query ID | v3.0 | v4.0 TiKV Only | v4.0 TiKV/TiFlash Automatically | | :-------- | :----------- | :------------ | :-------------- | | 1 | 7.78 s | 7.45 s | 2.09 s | diff --git a/benchmark/v5.0-performance-benchmarking-with-tpcc.md b/benchmark/v5.0-performance-benchmarking-with-tpcc.md new file mode 100644 index 000000000000..62dcdbddc451 --- /dev/null +++ b/benchmark/v5.0-performance-benchmarking-with-tpcc.md @@ -0,0 +1,149 @@ +--- +title: TiDB TPC-C Performance Test Report -- v5.0 vs. v4.0 +--- + +# TiDB TPC-C Performance Test Report -- v5.0 vs. v4.0 + +## Test purpose + +This test aims at comparing the TPC-C performance of TiDB v5.0 and TiDB v4.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment (AWS EC2) + +### Hardware configuration + +| Service type | EC2 type | Instance count | +|:----------|:----------|:----------| +| PD | m5.xlarge | 3 | +| TiKV | i3.4xlarge| 3 | +| TiDB | c5.4xlarge| 3 | +| TPC-C | c5.9xlarge| 1 | + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| PD | 4.0 and 5.0 | +| TiDB | 4.0 and 5.0 | +| TiKV | 4.0 and 5.0 | +| BenchmarkSQL | None | + +### Parameter configuration + +#### TiDB v4.0 configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV v4.0 configuration + +{{< copyable "" >}} + +```yaml +pessimistic-txn.pipelined: true +raftdb.allow-concurrent-memtable-write: true +raftdb.max-background-jobs: 4 +raftstore.apply-max-batch-size: 2048 +raftstore.apply-pool-size: 3 +raftstore.store-max-batch-size: 2048 +raftstore.store-pool-size: 3 +readpool.storage.normal-concurrency: 10 +readpool.unified.max-thread-count: 20 +readpool.unified.min-thread-count: 5 +rocksdb.max-background-jobs: 8 +server.grpc-concurrency: 6 +server.request-batch-enable-cross-command: false +storage.scheduler-worker-pool-size: 20 +``` + +#### TiDB v5.0 configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV v5.0 configuration + +{{< copyable "" >}} + +```yaml +pessimistic-txn.pipelined: true +raftdb.allow-concurrent-memtable-write: true +raftdb.max-background-jobs: 4 +raftstore.apply-max-batch-size: 2048 +raftstore.apply-pool-size: 3 +raftstore.store-max-batch-size: 2048 +raftstore.store-pool-size: 3 +readpool.storage.normal-concurrency: 10 +readpool.unified.max-thread-count: 20 +readpool.unified.min-thread-count: 5 +rocksdb.max-background-jobs: 8 +server.grpc-concurrency: 6 +storage.scheduler-worker-pool-size: 20 +server.enable-request-batch: false +``` + +#### TiDB v4.0 global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +``` + +#### TiDB v5.0 global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +set global tidb_enable_async_commit = 1; +set global tidb_enable_1pc = 1; +set global tidb_guarantee_linearizability = 0; +set global tidb_enable_clustered_index = 1; +``` + +## Test plan + +1. Deploy TiDB v5.0 and v4.0 using TiUP. + +2. Use BenchmarkSQL to import the TPC-C 5000 Warehouse data. + + 1. Compile BenchmarkSQL： + + {{< copyable "bash" >}} + + ```bash + git clone https://github.com/pingcap/benchmarksql && cd benchmarksql && ant + ``` + + 2. Enter the `run` directory, edit the `props.mysql` file according to the actual situation, and modify the `conn`, `warehouses`, `loadWorkers`, `terminals`, and `runMins` configuration items. + + 3. Execute the `runSQL.sh ./props.mysql sql.mysql/tableCreates.sql` command. + + 4. Execute the `runSQL.sh ./props.mysql sql.mysql/indexCreates.sql` command. + + 5. Run MySQL client and execute the `analyze table` statement on every table. + +3. Execute the `runBenchmark.sh ./props.mysql` command. + +4. Extract the tpmC data of New Order from the result. + +## Test result + +According to the test statistics, the TPC-C performance of TiDB v5.0 has **increased by 36%** compared with that of TiDB v4.0. + +![TPC-C](/media/tpcc_v5vsv4_corrected_v2.png) diff --git a/benchmark/v5.1-performance-benchmarking-with-tpcc.md b/benchmark/v5.1-performance-benchmarking-with-tpcc.md new file mode 100644 index 000000000000..fa66d5db2f2b --- /dev/null +++ b/benchmark/v5.1-performance-benchmarking-with-tpcc.md @@ -0,0 +1,92 @@ +--- +title: TiDB TPC-C Performance Test Report -- v5.1.0 vs. v5.0.2 +--- + +# TiDB TPC-C Performance Test Report -- v5.1.0 vs. v5.0.2 + +## Test overview + +This test aims to compare the TPC-H performance of TiDB v5.1.0 and TiDB v5.0.2 in the online analytical processing (OLAP) scenario. The results show that compared with v5.0.2, the TPC-C performance of v5.1.0 is improved by 2.8%. + +## Test environment (AWS EC2) + +## Hardware configuration + +| Service type | EC2 type | Instance count | +|:----------|:----------|:----------| +| PD | m5.xlarge | 3 | +| TiKV | i3.4xlarge| 3 | +| TiDB | c5.4xlarge| 3 | +| TPC-C | c5.9xlarge| 1 | + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| PD | v5.0.2 and v5.1.0 | +| TiDB | v5.0.2 and v5.1.0 | +| TiKV | v5.0.2 and v5.1.0 | +| TiUP | 1.5.1 | + +### Parameter configuration + +TiDB v5.1.0 and TiDB v5.0.2 use the same configuration. + +#### TiDB parameter configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV parameter configuration + +{{< copyable "" >}} + +```yaml +pessimistic-txn.pipelined: true +raftdb.allow-concurrent-memtable-write: true +raftdb.max-background-jobs: 4 +raftstore.apply-max-batch-size: 2048 +raftstore.apply-pool-size: 3 +raftstore.store-max-batch-size: 2048 +raftstore.store-pool-size: 3 +readpool.storage.normal-concurrency: 10 +readpool.unified.max-thread-count: 20 +readpool.unified.min-thread-count: 5 +rocksdb.max-background-jobs: 8 +server.grpc-concurrency: 6 +storage.scheduler-worker-pool-size: 20 +server.enable-request-batch: false +``` + +#### TiDB global variable configuration + +{{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +set global tidb_enable_async_commit = 1; +set global tidb_enable_1pc = 1; +set global tidb_guarantee_linearizability = 0; +set global tidb_enable_clustered_index = 1; +``` + +## Test plan + +1. Deploy TiDB v5.1.0 and v5.0.2 using TiUP. +2. Create a database named `tpcc`: `create database tpcc;`. +3. Use BenchmarkSQL to import the TPC-C 5000 Warehouse data: `tiup bench tpcc prepare --warehouse 5000 --db tpcc -H 127.0.0.1 -p 4000`. +4. Execute the `tiup bench tpcc run -U root --db tpcc --host 127.0.0.1 --port 4000 --time 300s --warehouses 5000 --threads {{thread}}` command to perform stress tests on TiDB via HAProxy. +5. Extract the tpmC data of New Order from the result. + +## Test result + +Compared with v5.0.2, the TPC-C performance of v5.1.0 is **improved by 2.8%**. + +![TPC-C](/media/tpcc_v510_vs_v502.png) diff --git a/benchmark/v5.1-performance-benchmarking-with-tpch.md b/benchmark/v5.1-performance-benchmarking-with-tpch.md new file mode 100644 index 000000000000..39e6ccd6be72 --- /dev/null +++ b/benchmark/v5.1-performance-benchmarking-with-tpch.md @@ -0,0 +1,122 @@ +--- +title: TiDB TPC-H 100GB Performance Test Report -- v5.1 MPP mode vs. Greenplum 6.15.0 and Apache Spark 3.1.1 +summary: Compare the TPC-H 100GB performance of TiDB 5.1 MPP mode, Greenplum 6.15.0, and Apache Spark 3.1.1. +--- + +# TiDB TPC-H 100GB Performance Test Report -- TiDB v5.1 MPP mode vs. Greenplum 6.15.0 and Apache Spark 3.1.1 + +## Test overview + +This test aims at comparing the TPC-H 100GB performance of TiDB v5.1 in the MPP mode with that of Greenplum and Apache Spark, two mainstream analytics engines, in their latest versions. The test result shows that the performance of TiDB v5.1 in the MPP mode is two to three times faster than that of the other two solutions under TPC-H workload. + +In v5.0, TiDB introduces the MPP mode for [TiFlash](/tiflash/tiflash-overview.md), which significantly enhances TiDB's Hybrid Transactional and Analytical Processing (HTAP) capabilities. Test objects in this report are as follows: + ++ TiDB v5.1 columnar storage in the MPP mode ++ Greenplum 6.15.0 ++ Apache Spark 3.1.1 + Parquet + +## Test environment + +### Hardware prerequisite + ++ Node count: 3 ++ CPU: Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz, 40 cores ++ Memory: 189 GB ++ Disks: NVMe 3TB * 2 + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| TiDB | 5.1 | +| Greenplum | 6.15.0 | +| Apache Spark | 3.1.1 | + +### Parameter configuration + +#### TiDB v5.1 + +For the v5.1 cluster, TiDB uses the default parameter configuration except for the following configuration items. + +In the configuration file `users.toml` of TiFlash, configure `max_memory_usage` as follows: + +```toml +[profiles.default] +max_memory_usage = 10000000000000 +``` + +Set session variables with the following SQL statements: + +```sql +set @@tidb_isolation_read_engines='tiflash'; +set @@tidb_allow_mpp=1; +set @@tidb_mem_quota_query = 10 << 30; +``` + +All TPC-H test tables are replicated to TiFlash in columnar format, with no additional partitions or indexes. + +#### Greenplum + +Except for the initial 3 nodes, the Greenplum cluster is deployed using an additional master node. Each segment server contains 8 segments, which means 4 segments per NVMe SSD. So there are 24 segments in total. The storage format is append-only/column-oriented storage and partition keys are used as primary keys. + +{{< copyable "" >}} + +``` +log_statement = all +gp_autostats_mode = none +statement_mem = 2048MB +gp_vmem_protect_limit = 16384 +``` + +#### Apache Spark + +The test of Apache Spark uses Apache Parquet as the storage format and stores the data on HDFS. The HDFS system consists of three nodes. Each node has two assigned NVMe SSD disks as the data disks. The Spark cluster is deployed in standalone mode, using NVMe SSD disks as the local directory of `spark.local.dir` to speed up the shuffle spill, with no additional partitions or indexes. + +{{< copyable "" >}} + +``` +--driver-memory 20G +--total-executor-cores 120 +--executor-cores 5 +--executor-memory 15G +``` + +## Test result + +> **Note:** +> +> The following test results are the average data of three tests. All numbers are in seconds. + +| Query ID | TiDB v5.1 | Greenplum 6.15.0 | Apache Spark 3.1.1 + Parquet | +| :-------- | :----------- | :------------ | :-------------- | +| 1 | 8.08 | 64.1307 | 52.64 | +| 2 | 2.53 | 4.76612 | 11.83 | +| 3 | 4.84 | 15.62898 | 13.39 | +| 4 | 10.94 | 12.88318 | 8.54 | +| 5 | 12.27 | 23.35449 | 25.23 | +| 6 | 1.32 | 6.033 | 2.21 | +| 7 | 5.91 | 12.31266 | 25.45 | +| 8 | 6.71 | 11.82444 | 23.12 | +| 9 | 44.19 | 22.40144 | 35.2 | +| 10 | 7.13 | 12.51071 | 12.18 | +| 11 | 2.18 | 2.6221 | 10.99 | +| 12 | 2.88 | 7.97906 | 6.99 | +| 13 | 6.84 | 10.15873 | 12.26 | +| 14 | 1.69 | 4.79394 | 3.89 | +| 15 | 3.29 | 10.48785 | 9.82 | +| 16 | 5.04 | 4.64262 | 6.76 | +| 17 | 11.7 | 74.65243 | 44.65 | +| 18 | 12.87 | 64.87646 | 30.27 | +| 19 | 4.75 | 8.08625 | 4.7 | +| 20 | 8.89 | 15.47016 | 8.4 | +| 21 | 24.44 | 39.08594 | 34.83 | +| 22 | 1.23 | 7.67476 | 4.59 | + +![TPC-H](/media/tidb-v5-tpch-100-vs-gp-spark.png) + +In the performance diagram above: + +- Blue lines represent TiDB v5.1; +- Red lines represent Greenplum 6.15.0; +- Yellow lines represent Apache Spark 3.1.1. +- The y-axis represents the execution time of the query. The less the time is, the better the performance is. diff --git a/best-practices/grafana-monitor-best-practices.md b/best-practices/grafana-monitor-best-practices.md index 3683898b9ffd..5fb569a29d53 100644 --- a/best-practices/grafana-monitor-best-practices.md +++ b/best-practices/grafana-monitor-best-practices.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/best-practices/grafana-monitor-best-practices/','/docs/dev/ # Best Practices for Monitoring TiDB Using Grafana -When you [deploy a TiDB cluster using TiDB Ansible](/online-deployment-using-ansible.md), a set of [Grafana + Prometheus monitoring platform](/tidb-monitoring-framework.md) is deployed simultaneously to collect and display metrics for various components and machines in the TiDB cluster. This document describes best practices for monitoring TiDB using Grafana. It aims to help you use metrics to analyze the status of the TiDB cluster and diagnose problems. +When you [deploy a TiDB cluster using TiUP](/production-deployment-using-tiup.md) and have added Grafana and Prometheus in the topology configuration, a set of [Grafana + Prometheus monitoring platform](/tidb-monitoring-framework.md) is deployed simultaneously to collect and display metrics for various components and machines in the TiDB cluster. This document describes best practices for monitoring TiDB using Grafana. It aims to help you use metrics to analyze the status of the TiDB cluster and diagnose problems. ## Monitoring architecture @@ -17,7 +17,7 @@ When you [deploy a TiDB cluster using TiDB Ansible](/online-deployment-using-ans For TiDB 2.1.3 or later versions, TiDB monitoring supports the pull method. It is a good adjustment with the following benefits: - There is no need to restart the entire TiDB cluster if you need to migrate Prometheus. Before adjustment, migrating Prometheus requires restarting the entire cluster because the target address needs to be updated. -- You can deploy 2 separate sets of Grafana + Prometheus monitoring platforms (not highly available) to prevent a single point of monitoring. To do this, execute the deployment command of TiDB ansible twice with different IP addresses. +- You can deploy 2 separate sets of Grafana + Prometheus monitoring platforms (not highly available) to prevent a single point of monitoring. - The Pushgateway which might become a single point of failure is removed. ## Source and display of monitoring data @@ -203,4 +203,4 @@ curl -u user:pass 'http://__grafana_ip__:3000/api/datasources/proxy/1/api/v1/que ## Summary -The Grafana + Prometheus monitoring platform is a very powerful tool. Making good use of it can improve efficiency, saving you a lot of time on analyzing the status of the TiDB cluster. More importantly, it can help you diagnose problems. This tool is very useful in the operation and maintenance of TiDB clusters, especially when there is a large amount of data. \ No newline at end of file +The Grafana + Prometheus monitoring platform is a very powerful tool. Making good use of it can improve efficiency, saving you a lot of time on analyzing the status of the TiDB cluster. More importantly, it can help you diagnose problems. This tool is very useful in the operation and maintenance of TiDB clusters, especially when there is a large amount of data. diff --git a/best-practices/massive-regions-best-practices.md b/best-practices/massive-regions-best-practices.md index 13028bd8a730..907300b844e7 100644 --- a/best-practices/massive-regions-best-practices.md +++ b/best-practices/massive-regions-best-practices.md @@ -68,7 +68,7 @@ By default, `raftstore.store-pool-size` is configured to `2` in TiKV. If a bottl In the actual situation, read and write requests are not evenly distributed on every Region. Instead, they are concentrated on a few Regions. Then you can minimize the number of messages between the Raft leader and the followers for the temporarily idle Regions, which is the feature of Hibernate Region. In this feature, Raftstore does sent tick messages to the Raft state machines of idle Regions if not necessary. Then these Raft state machines will not be triggered to generate heartbeat messages, which can greatly reduce the workload of Raftstore. -Hibernate Region is enabled by default in [TiKV master](https://github.com/tikv/tikv/tree/master). You can enable this feature according to your needs. For the configuration of Hibernate Region, refer to [Configure Hibernate Region](https://github.com/tikv/tikv/blob/master/docs/reference/configuration/raftstore-config.md#hibernate-region). +Hibernate Region is enabled by default in [TiKV master](https://github.com/tikv/tikv/tree/master). You can configure this feature according to your needs. For details, refer to [Configure Hibernate Region](/tikv-configuration-file.md). ### Method 3: Enable `Region Merge` diff --git a/best-practices/pd-scheduling-best-practices.md b/best-practices/pd-scheduling-best-practices.md index 37fe27860cdb..91e0b0586b97 100644 --- a/best-practices/pd-scheduling-best-practices.md +++ b/best-practices/pd-scheduling-best-practices.md @@ -102,6 +102,12 @@ The processes of scale-down and failure recovery are basically the same. `replic Region merge refers to the process of merging adjacent small regions. It serves to avoid unnecessary resource consumption by a large number of small or even empty regions after data deletion. Region merge is performed by `mergeChecker`, which processes in a similar way to `replicaChecker`: PD continuously scans all regions in the background, and generates an operator when contiguous small regions are found. +Specifically, when a newly split Region exists for more than the value of [`split-merge-interval`](/pd-configuration-file.md#split-merge-interval) (`1h` by default), if any of the following conditions occurs, this Region triggers the Region merge scheduling: + +- The size of this Region is smaller than the value of the [`max-merge-region-size`](/pd-configuration-file.md#max-merge-region-size) (20 MiB by default) + +- The number of keys in this Region is smaller than the value of [`max-merge-region-keys`](/pd-configuration-file.md#max-merge-region-keys) (200,000 by default). + ## Query scheduling status You can check the status of scheduling system through metrics, pd-ctl and logs. This section briefly introduces the methods of metrics and pd-ctl. Refer to [PD Monitoring Metrics](/grafana-pd-dashboard.md) and [PD Control](/pd-control.md) for details. @@ -261,7 +267,7 @@ Similar to slow scheduling, the speed of region merge is most likely limited by - TiKV: Set `split-region-on-table` to `false`. You cannot modify the parameter dynamically. - PD - Set `key-type` to `"txn"` or `"raw"`. You can modify the parameter dynamically. - - Keep `key-type` as `table` and set `enable-cross-table-merge` to `true`. You can modify the parameter dynamically. + - Or keep `key-type` as `table` and set `enable-cross-table-merge` to `true`. You can modify the parameter dynamically. > **Note:** > diff --git a/best-practices/three-dc-local-read.md b/best-practices/three-dc-local-read.md new file mode 100644 index 000000000000..b404757df4ed --- /dev/null +++ b/best-practices/three-dc-local-read.md @@ -0,0 +1,29 @@ +--- +title: Local Read under Three Data Centers Deployment +summary: Learn how to use the Stale Read feature to read local data under three DCs deployment and thus reduce cross-center requests. +--- + +# Local Read under Three Data Centers Deployment + +In the model of three data centers, a Region has three replicas which are isolated in each data center. However, due to the requirement of strongly consistent read, TiDB must access the Leader replica of the corresponding data for every query. If the query is generated in a data center different from that of the Leader replica, TiDB needs to read data from another data center, thus causing the access latency to increase. + +This document describes how to use the [Stale Read](/stale-read.md) feature to avoid cross-center access and reduce the access latency at the expense of real-time data availability. + +## Deploy a TiDB cluster of three data centers + +For the three-data-center deployment method, refer to [Multiple Data Centers in One City Deployment](/multi-data-centers-in-one-city-deployment.md). + +Note that if both the TiKV and TiDB nodes have the configuration item `labels` configured, the TiKV and TiDB nodes in the same data center must have the same value for the `zone` label. For example, if a TiKV node and a TiDB node are both in the data center `dc-1`, then the two nodes need to be configured with the following label: + +``` +[labels] +zone=dc-1 +``` + +## Perform local read using Stale Read + +[Stale Read](/stale-read.md) is a mechanism that TiDB provides for the users to read historical data. Using this mechanism, you can read the corresponding historical data of a specific point in time or within a specified time range, and thus save the latency brought by data replication between storage nodes. When using Stale Read in some scenarios of geo-distributed deployment, TiDB accesses the replica in the current data center to read the corresponding data at the expense of some real-time performance, which avoids network latency brought by cross-center connection and reduces the access latency for the entire query process. + +When TiDB receives a Stale Read query, if the `zone` label of that TiDB node is configured, then TiDB sends the request to the TiKV node with the same `zone` label where the corresponding data replica resides. + +For how to perform Stale Read, see [Perform Stale Read using the `AS OF TIMESTAMP` clause](/as-of-timestamp.md). diff --git a/tidb-best-practices.md b/best-practices/tidb-best-practices.md similarity index 96% rename from tidb-best-practices.md rename to best-practices/tidb-best-practices.md index 466a22a9a4f9..dc5d4d322fd7 100644 --- a/tidb-best-practices.md +++ b/best-practices/tidb-best-practices.md @@ -38,14 +38,14 @@ TiDB provides complete distributed transactions and the model has some optimizat * Optimistic transaction model - TiDB's optimistic transaction model does not detect conflicts until the commit phase. If there are conflicts, the transaction needs retry. But this model is inefficient if the conflict is severe, because operations before retry are invalid and need to repeat. - + TiDB's optimistic transaction model does not detect conflicts until the commit phase. If there are conflicts, the transaction needs retry. But this model is inefficient if the conflict is severe, because operations before retry are invalid and need to repeat. + Assume that the database is used as a counter. High access concurrency might lead to severe conflicts, resulting in multiple retries or even timeouts. Therefore, in the scenario of severe conflicts, it is recommended to use the pessimistic transaction mode or to solve problems at the system architecture level, such as placing counter in Redis. Nonetheless, the optimistic transaction model is efficient if the access conflict is not very severe. * Pessimistic transaction model In TiDB, the pessimistic transaction model has almost the same behavior as in MySQL. The transaction applies a lock during the execution phase, which avoids retries in conflict situations and ensures a higher success rate. By applying the pessimistic locking, you can also lock data in advance using `SELECT FOR UPDATE`. - + However, if the application scenario has fewer conflicts, the optimistic transaction model has better performance. * Transaction size limit @@ -53,7 +53,7 @@ TiDB provides complete distributed transactions and the model has some optimizat As distributed transactions need to conduct two-phase commit and the bottom layer performs Raft replication, if a transaction is very large, the commit process would be quite slow, and the following Raft replication process is thus stuck. To avoid this problem, the transaction size is limited: - A transaction is limited to 5,000 SQL statements (by default) - - Each Key-Value entry is no more than 6 MB + - Each Key-Value entry is no more than 6 MB (by default) - The total size of Key-Value entries is no more than 10 GB. You can find similar limits in [Google Cloud Spanner](https://cloud.google.com/spanner/quotas). @@ -75,9 +75,9 @@ Simply put, TiDB performs the following operations: * A row of data is mapped to a Key-Value pair. The key is prefixed with `TableID` and suffixed with the row ID. * An index is mapped as a Key-Value pair. The key is prefixed with `TableID+IndexID` and suffixed with the index value. -The data or indexes in the same table have the same prefix. These Key-Values are at adjacent positions in the key space of TiKV. Therefore, when the amount of data to be written is large and all is written to one table, the write hotspot is created. The situation gets worse when some index values of the continuous written data is also continuous (e.g. fields that increase with time, like `update time`), which creates a few write hotspots and becomes the bottleneck of the entire system. +The data or indexes in the same table have the same prefix. These Key-Values are at adjacent positions in the key space of TiKV. Therefore, when the amount of data to be written is large and all is written to one table, the write hotspot is created. The situation gets worse when some index values of the continuous written data is also continuous (for example, fields that increase with time, like `update time`), which creates a few write hotspots and becomes the bottleneck of the entire system. -Similarly, if all data is read from a focused small range (e.g. the continuous tens or hundreds of thousands of rows of data), an access hotspot of data is likely to occur. +Similarly, if all data is read from a focused small range (for example, the continuous tens or hundreds of thousands of rows of data), an access hotspot of data is likely to occur. ### Secondary index @@ -88,7 +88,7 @@ Lots of MySQL experience is also applicable to TiDB. It is noted that TiDB has i * The more secondary indexes, the better? Secondary indexes can speed up queries, but adding an index has side effects. The previous section introduces the storage model of indexes. For each additional index, there will be one more Key-Value when inserting a piece of data. Therefore, the more indexes, the slower the writing speed and the more space it takes up. - + In addition, too many indexes affects the runtime of the optimizer, and inappropriate indexes mislead the optimizer. Thus, more secondary indexes does not mean better performance. * Which columns should create indexes? @@ -117,17 +117,17 @@ Lots of MySQL experience is also applicable to TiDB. It is noted that TiDB has i As data is distributed across many Regions, queries run in TiDB concurrently. But the concurrency by default is not high in case it consumes lots of system resources. Besides, the OLTP query usually does not involve a large amount of data and the low concurrency is enough. But for the OLAP query, the concurrency is high and TiDB modifies the query concurrency through the following system variables: - [`tidb_distsql_scan_concurrency`](/system-variables.md#tidb_distsql_scan_concurrency): - + The concurrency of scanning data, including scanning the table and index data. - [`tidb_index_lookup_size`](/system-variables.md#tidb_index_lookup_size): - + If it needs to access the index to get row IDs before accessing the table data, it uses a batch of row IDs as a single request to access the table data. This parameter sets the size of a batch. The larger batch increases latency, while the smaller one might lead to more queries. The proper size of this parameter is related to the amount of data that the query involves. Generally, no modification is required. - + - [`tidb_index_lookup_concurrency`](/system-variables.md#tidb_index_lookup_concurrency): If it needs to access the index to get row IDs before accessing the table data, the concurrency of getting data through row IDs every time is modified through this parameter. - + * Ensure the order of results through indexes You can use indexes to filter or sort data. Firstly, get row IDs according to the index order. Then, return the row content according to the return order of row IDs. In this way, the returned results are ordered according to the index column. It has been mentioned earlier that the model of scanning index and getting row is parallel + pipeline. If the row is returned according to the index order, a high concurrency between two queries does not reduce latency. Thus, the concurrency is low by default, but it can be modified through the [`tidb_index_serial_scan_concurrency`](/system-variables.md#tidb_index_serial_scan_concurrency) variable. @@ -181,7 +181,7 @@ You can control the concurrency of SQL execution through the `SET` statement and In addition, you can also use MySQL's standard index selection, the hint syntax, or control the optimizer to select indexes through `Use Index`/`Ignore Index hint`. -If the application scenario has both OLTP and OLAP workloads, you can send the OLTP request and OLAP request to different TiDB servers, diminishing the impact of OLAP on OLTP. It is recommended to use machines with high-performance hardware (e.g. more processor cores, larger memory) for the TiDB server that processes OLAP workloads. +If the application scenario has both OLTP and OLAP workloads, you can send the OLTP request and OLAP request to different TiDB servers, diminishing the impact of OLAP on OLTP. It is recommended to use machines with high-performance hardware (for example, more processor cores and larger memory) for the TiDB server that processes OLAP workloads. To completely isolate OLTP and OLAP workloads, it is recommended to run OLAP applications on TiFlash. TiFlash is a columnar storage engine with great performance on OLAP workloads. TiFlash can achieve physical isolation on the storage layer and guarantees consistent reads. diff --git a/br/backup-and-restore-faq.md b/br/backup-and-restore-faq.md index ae63f7995d3a..1bb6ec0021e2 100644 --- a/br/backup-and-restore-faq.md +++ b/br/backup-and-restore-faq.md @@ -12,7 +12,7 @@ This document lists the frequently asked questions (FAQs) and the solutions abou When you restore data, each node must have access to **all** backup files (SST files). By default, if `local` storage is used, you cannot restore data because the backup files are scattered among different nodes. Therefore, you have to copy the backup file of each TiKV node to the other TiKV nodes. -It is recommended to mount an NFS disk as a backup disk during backup. For details, see [Back up a single table to a network disk](/br/backup-and-restore-use-cases.md#back-up-a-single-table-to-a-network-disk-recommended). +It is recommended to mount an NFS disk as a backup disk during backup. For details, see [Back up a single table to a network disk](/br/backup-and-restore-use-cases.md#back-up-a-single-table-to-a-network-disk-recommended-in-production-environment). ## How much does it affect the cluster during backup using BR? @@ -22,11 +22,9 @@ To reduce the impact on the cluster, you can use the `--ratelimit` parameter to ## Does BR back up system tables? During data restoration, do they raise conflict? -The system libraries (`information_schema`, `performance_schema`, `mysql`) are filtered out during full backup. For more details, refer to the [Backup Principle](/br/backup-and-restore-tool.md#implementation-principles). +Before v5.1.0, BR filtered out data from the system schema `mysql` during the backup. Since v5.1.0, BR **backs up** all data by default, including the system schemas `mysql.*`. But the technical implementation of restoring the system tables in `mysql.*` is not complete yet, so the tables in the system schema `mysql` are **not** restored by default. For more details, refer to the [Back up and restore table data in the `mysql` system schema (experimental feature)](/br/backup-and-restore-tool.md#back-up-and-restore-table-data-in-the-mysql-system-schema-experimental-feature). -Because these system libraries do not exist in the backup files, no conflict occurs among system tables during data restoration. - -## What should I do to handle the `Permission denied` error, even if I have tried to run BR using root in vain? +## What should I do to handle the `Permission denied` or `No such file or directory` error, even if I have tried to run BR using root in vain? You need to confirm whether TiKV has access to the backup directory. To back up data, confirm whether TiKV has the write permission. To restore data, confirm whether it has the read permission. @@ -42,7 +40,7 @@ Running BR with the root access might fail due to the disk permission, because t ## What should I do to handle the `Io(Os...)` error? -Almost all of these problems are system call errors that occur when TiKV writes data to the disk. You can check the mounting method and the file system of the backup directory, and try to back up data to another folder or another hard disk. +Almost all of these problems are system call errors that occur when TiKV writes data to the disk. For example, if you encounter error messages such as `Io(Os {code: 13, kind: PermissionDenied...})` or `Io(Os {code: 2, kind: NotFound...})`, you can first check the mounting method and the file system of the backup directory, and try to back up data to another folder or another hard disk. For example, you might encounter the `Code: 22(invalid argument)` error when backing up data to the network disk built by `samba`. @@ -69,3 +67,31 @@ However, if you want to restore data from local storage, the number of replicas + Before v4.0.3, DDL jobs generated during the BR restore might cause unexpected DDL executions in TiCDC/Drainer. Therefore, if you need to perform restore on the upstream cluster of TiCDC/Drainer, add all tables restored using BR to the TiCDC/Drainer block list. You can use [`filter.rules`](https://github.com/pingcap/ticdc/blob/7c3c2336f98153326912f3cf6ea2fbb7bcc4a20c/cmd/changefeed.toml#L16) to configure the block list for TiCDC and use [`syncer.ignore-table`](/tidb-binlog/tidb-binlog-configuration-file.md#ignore-table) to configure the block list for Drainer. + +## Does BR back up the `SHARD_ROW_ID_BITS` and `PRE_SPLIT_REGIONS` information of a table? Does the restored table have multiple Regions? + +Yes. BR backs up the [`SHARD_ROW_ID_BITS` and `PRE_SPLIT_REGIONS`](/sql-statements/sql-statement-split-region.md#pre_split_regions) information of a table. The data of the restored table is also split into multiple Regions. + +## Why is the `region is unavailable` error reported for a SQL query after I use BR to restore the backup data? + +If the cluster backed up using BR has TiFlash, `TableInfo` stores the TiFlash information when BR restores the backup data. If the cluster to be restored does not have TiFlash, the `region is unavailable` error is reported. + +## Does BR support in-place full recovery of some historical backup? + +No. BR does not support in-place full recovery of some historical backup. + +## How can I use BR for incremental backup in the Kubernetes environment? + +To get the `commitTs` field of the last BR backup, run the `kubectl -n ${namespace} get bk ${name}` command using kubectl. You can use the content of this field as `--lastbackupts`. + +## How can I convert BR backupTS to Unix time? + +BR `backupTS` defaults to the latest timestamp obtained from PD before the backup starts. You can use `pd-ctl tso timestamp` to parse the timestamp to obtain an accurate value, or use `backupTS >> 18` to quickly obtain an estimated value. + +## After BR restores the backup data, do I need to execute the `ANALYZE` statement on the table to update the statistics of TiDB on the tables and indexes? + +BR does not back up statistics (except in v4.0.9). Therefore, after restoring the backup data, you need to manually execute `ANALYZE TABLE` or wait for TiDB to automatically execute `ANALYZE`. + +In v4.0.9, BR backs up statistics by default, which consumes too much memory. To ensure that the backup process goes well, the backup for statistics is disabled by default starting from v4.0.10. + +If you do not execute `ANALYZE` on the table, TiDB will fail to select the optimized execution plan due to inaccurate statistics. If query performance is not a key concern, you can ignore `ANALYZE`. diff --git a/br/backup-and-restore-storages.md b/br/backup-and-restore-storages.md index b2d5d73d88fd..3db43a531c6d 100644 --- a/br/backup-and-restore-storages.md +++ b/br/backup-and-restore-storages.md @@ -1,12 +1,12 @@ --- -title: BR Storages -summary: Describes the storage URL format used in BR. +title: External Storages +summary: Describes the storage URL format used in BR, TiDB Lightning, and Dumpling. aliases: ['/docs/dev/br/backup-and-restore-storages/'] --- -# BR Storages +# External Storages -BR supports reading and writing data on the local filesystem, as well as on Amazon S3 and Google Cloud Storage. These are distinguished by the URL scheme in the `--storage` parameter passed into BR. +Backup & Restore (BR), TiDB Lighting, and Dumpling support reading and writing data on the local filesystem and on Amazon S3. BR also supports reading and writing data on the Google Cloud Storage (GCS). These are distinguished by the URL scheme in the `--storage` parameter passed into BR, in the `-d` parameter passed into TiDB Lightning, and in the `--output` (`-o`) parameter passed into Dumpling. ## Schemes @@ -19,19 +19,49 @@ The following services are supported: | Google Cloud Storage (GCS) | gcs, gs | `gcs://bucket-name/prefix/of/dest/` | | Write to nowhere (for benchmarking only) | noop | `noop://` | -## Parameters +## URL parameters Cloud storages such as S3 and GCS sometimes require additional configuration for connection. You can specify parameters for such configuration. For example: -{{< copyable "shell-regular" >}} ++ Use Dumpling to export data to S3: -```shell -./br backup full -u 127.0.0.1:2379 -s 's3://bucket-name/prefix?region=us-west-2' -``` + {{< copyable "shell-regular" >}} + + ```bash + ./dumpling -u root -h 127.0.0.1 -P 3306 -B mydb -F 256MiB \ + -o 's3://my-bucket/sql-backup?region=us-west-2' + ``` + ++ Use TiDB Lightning to import data from S3: + + {{< copyable "shell-regular" >}} + + ```bash + ./tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ + -d 's3://my-bucket/sql-backup?region=us-west-2' + ``` + ++ Use TiDB Lightning to import data from S3 (using the path style in the request mode): + + {{< copyable "shell-regular" >}} -### S3 parameters + ```bash + ./tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ + -d 's3://my-bucket/sql-backup?force-path-style=true&endpoint=http://10.154.10.132:8088' + ``` -| Parameter | Description | ++ Use BR to back up data to GCS: + + {{< copyable "shell-regular" >}} + + ```bash + ./br backup full -u 127.0.0.1:2379 \ + -s 'gcs://bucket-name/prefix' + ``` + +### S3 URL parameters + +| URL parameter | Description | |----------:|---------| | `access-key` | The access key | | `secret-access-key` | The secret access key | @@ -46,30 +76,64 @@ Cloud storages such as S3 and GCS sometimes require additional configuration for > **Note:** > -> It is not recommended to pass in the access key and secret access key directly in the storage URL, because these keys are logged in plain text. BR tries to infer these keys from the environment in the following order: +> It is not recommended to pass in the access key and secret access key directly in the storage URL, because these keys are logged in plain text. The migration tools try to infer these keys from the environment in the following order: 1. `$AWS_ACCESS_KEY_ID` and `$AWS_SECRET_ACCESS_KEY` environment variables 2. `$AWS_ACCESS_KEY` and `$AWS_SECRET_KEY` environment variables -3. Shared credentials file on the BR node at the path specified by the `$AWS_SHARED_CREDENTIALS_FILE` environment variable -4. Shared credentials file on the BR node at `~/.aws/credentials` +3. Shared credentials file on the tool node at the path specified by the `$AWS_SHARED_CREDENTIALS_FILE` environment variable +4. Shared credentials file on the tool node at `~/.aws/credentials` 5. Current IAM role of the Amazon EC2 container 6. Current IAM role of the Amazon ECS task -### GCS parameters +### GCS URL parameters -| Parameter | Description | +| URL parameter | Description | |----------:|---------| -| `credentials-file` | The path to the credentials JSON file on the TiDB node | +| `credentials-file` | The path to the credentials JSON file on the tool node | | `storage-class` | Storage class of the uploaded objects (for example, `STANDARD`, `COLDLINE`) | | `predefined-acl` | Predefined ACL of the uploaded objects (for example, `private`, `project-private`) | -When `credentials-file` is not specified, BR will try to infer the credentials from the environment, in the following order: +When `credentials-file` is not specified, the migration tool will try to infer the credentials from the environment, in the following order: -1. Content of the file on the BR node at the path specified by the `$GOOGLE_APPLICATION_CREDENTIALS` environment variable -2. Content of the file on the BR node at `~/.config/gcloud/application_default_credentials.json` +1. Content of the file on the tool node at the path specified by the `$GOOGLE_APPLICATION_CREDENTIALS` environment variable +2. Content of the file on the tool node at `~/.config/gcloud/application_default_credentials.json` 3. When running in GCE or GAE, the credentials fetched from the metadata server. -## Sending credentials to TiKV +## Command-line parameters + +In addition to the URL parameters, BR and Dumpling also support specifying these configurations using command-line parameters. For example: + +{{< copyable "shell-regular" >}} + +```bash +./dumpling -u root -h 127.0.0.1 -P 3306 -B mydb -F 256MiB \ + -o 's3://my-bucket/sql-backup' \ + --s3.region 'us-west-2' +``` + +If you have specified URL parameters and command-line parameters at the same time, the URL parameters are overwritten by the command-line parameters. + +### S3 command-line parameters + +| Command-line parameter | Description | +|----------:|------| +| `--s3.region` | Amazon S3's service region, which defaults to `us-east-1`. | +| `--s3.endpoint` | The URL of custom endpoint for S3-compatible services. For example, `https://s3.example.com/`. | +| `--s3.storage-class` | The storage class of the upload object. For example, `STANDARD` and `STANDARD_IA`. | +| `--s3.sse` | The server-side encryption algorithm used to encrypt the upload. The value options are empty, `AES256` and `aws:kms`. | +| `--s3.sse-kms-key-id` | If `--s3.sse` is configured as `aws:kms`, this parameter is used to specify the KMS ID. | +| `--s3.acl` | The canned ACL of the upload object. For example, `private` and `authenticated-read`. | +| `--s3.provider` | The type of the S3-compatible service. The supported types are `aws`, `alibaba`, `ceph`, `netease` and `other`. | + +### GCS command-line parameters + +| Command-line parameter | Description | +|----------:|---------| +| `--gcs.credentials-file` | The path of the JSON-formatted credential on the tool node. | +| `--gcs.storage-class` | The storage type of the upload object, such as `STANDARD` and `COLDLINE`. | +| `--gcs.predefined-acl` | The pre-defined ACL of the upload object, such as `private` and `project-private`. | + +## BR sending credentials to TiKV By default, when using S3 and GCS destinations, BR will send the credentials to every TiKV nodes to reduce setup complexity. @@ -77,6 +141,16 @@ However, this is unsuitable on cloud environment, where every node has their own {{< copyable "shell-regular" >}} -```shell +```bash ./br backup full -c=0 -u pd-service:2379 -s 's3://bucket-name/prefix' ``` + +When using SQL statements to [back up](/sql-statements/sql-statement-backup.md) and [restore](/sql-statements/sql-statement-restore.md) data, you can add the `SEND_CREDENTIALS_TO_TIKV = FALSE` option: + +{{< copyable "sql" >}} + +```sql +BACKUP DATABASE * TO 's3://bucket-name/prefix' SEND_CREDENTIALS_TO_TIKV = FALSE; +``` + +This option is not supported in TiDB Lightning and Dumpling, because the two applications are currently standalone. diff --git a/br/backup-and-restore-tool.md b/br/backup-and-restore-tool.md index 63116362f25d..cd71e8a52739 100644 --- a/br/backup-and-restore-tool.md +++ b/br/backup-and-restore-tool.md @@ -1,40 +1,22 @@ --- -title: Use BR to Back up and Restore Data -summary: Learn how to back up and restore data of the TiDB cluster using BR. +title: BR Tool Overview +summary: Learn what is BR and how to use the tool. aliases: ['/docs/dev/br/backup-and-restore-tool/','/docs/dev/reference/tools/br/br/','/docs/dev/how-to/maintain/backup-and-restore/br/'] --- -# Use BR to Back up and Restore Data +# BR Tool Overview -[Backup & Restore](http://github.com/pingcap/br) (BR) is a command-line tool for distributed backup and restoration of the TiDB cluster data. Compared with [`dumpling`](/backup-and-restore-using-dumpling-lightning.md) and [`mydumper`](/backup-and-restore-using-mydumper-lightning.md), BR is more suitable for scenarios of huge data volume. This document describes the BR command line, detailed use examples, best practices, restrictions, and introduces the implementation principles of BR. +[BR](http://github.com/pingcap/br) (Backup & Restore) is a command-line tool for distributed backup and restoration of the TiDB cluster data. -## Usage restrictions +Compared with [`dumpling`](/backup-and-restore-using-dumpling-lightning.md), BR is more suitable for scenarios of huge data volume. -- BR only supports TiDB v3.1 and later versions. -- BR supports restore on clusters of different topologies. However, the online applications will be greatly impacted during the restore operation. It is recommended that you perform restore during the off-peak hours or use `rate-limit` to limit the rate. -- It is recommended that you execute multiple backup operations serially. Otherwise, different backup operations might interfere with each other. -- When BR restores data to the upstream cluster of TiCDC/Drainer, TiCDC/Drainer cannot replicate the restored data to the downstream. -- BR supports operations only between clusters with the same [`new_collations_enabled_on_first_bootstrap`](/character-set-and-collation.md#collation-support-framework) value because BR only backs up KV data. If the cluster to be backed up and the cluster to be restored use different collations, the data validation fails. Therefore, before restoring a cluster, make sure that the switch value from the query result of the `select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME='new_collation_enabled';` statement is consistent with that during the backup process. - - - For v3.1 clusters, the new collation framework is not supported, so you can see it as disabled. - - For v4.0 clusters, check whether the new collation is enabled by executing `SELECT VARIABLE_VALUE FROM mysql.tidb WHERE VARIABLE_NAME='new_collation_enabled';`. - - For example, assume that data is backed up from a v3.1 cluster and will be restored to a v4.0 cluster. The `new_collation_enabled` value of the v4.0 cluster is `true`, which means that the new collation is enabled in the cluster to be restored when this cluster is created. If you perform the restore in this situation, an error might occur. - -## Recommended deployment configuration - -- It is recommended that you deploy BR on the PD node. -- It is recommended that you mount a high-performance SSD to BR nodes and all TiKV nodes. A 10-gigabit network card is recommended. Otherwise, bandwidth is likely to be the performance bottleneck during the backup and restore process. - -> **Note:** -> -> If you do not mount a network disk or use other shared storage, the data backed up by BR will be generated on each TiKV node. Because BR only backs up leader replicas, you should estimate the space reserved for each node based on the leader size. -> -> Meanwhile, because TiDB v4.0 uses leader count for load balancing by default, leaders are greatly different in size, resulting in uneven distribution of backup data on each node. +This document describes BR's implementation principles, recommended deployment configuration, usage restrictions and several methods to use BR. ## Implementation principles -BR sends the backup or restoration commands to each TiKV node. After receiving these commands, TiKV performs the corresponding backup or restoration operations. Each TiKV node has a path in which the backup files generated in the backup operation are stored and from which the stored backup files are read during the restoration. +BR sends the backup or restoration commands to each TiKV node. After receiving these commands, TiKV performs the corresponding backup or restoration operations. + +Each TiKV node has a path in which the backup files generated in the backup operation are stored and from which the stored backup files are read during the restoration. ![br-arch](/media/br-arch.png) @@ -84,6 +66,7 @@ Two types of backup files are generated in the path where backup files are store - **The SST file**: stores the data that the TiKV node backed up. - **The `backupmeta` file**: stores the metadata of this backup operation, including the number, the key range, the size, and the Hash (sha256) value of the backup files. +- **The `backup.lock` file**: prevents multiple backup operations from storing data to the same directory. ### The format of the SST file name @@ -119,581 +102,126 @@ After the restoration operation is completed, BR performs a checksum calculation -## How to use BR - -Currently, you can use SQL statements or the command-line tool to back up and restore data. - -### Use SQL statements - -TiDB v4.0.2 and later versions support backup and restore operations using SQL statements. For details, see the [Backup syntax](/sql-statements/sql-statement-backup.md#backup) and the [Restore syntax](/sql-statements/sql-statement-restore.md#restore). - -### Use the command-line tool - -Also, you can use the command-line tool to perform backup and restore. First, you need to download the binary file of the BR tool. For details, see [download link](/download-ecosystem-tools.md#br-backup-and-restore). - -The following section takes the command-line tool as an example to introduce how to perform backup and restore operations. - -## Command-line description - -A `br` command consists of sub-commands, options, and parameters. - -* Sub-command: the characters without `-` or `--`. -* Option: the characters that start with `-` or `--`. -* Parameter: the characters that immediately follow behind and are passed to the sub-command or the option. - -This is a complete `br` command: - -{{< copyable "shell-regular" >}} - -```shell -br backup full --pd "${PDIP}:2379" -s "local:///tmp/backup" -``` - -Explanations for the above command are as follows: - -* `backup`: the sub-command of `br`. -* `full`: the sub-command of `backup`. -* `-s` (or `--storage`): the option that specifies the path where the backup files are stored. -* `"local:///tmp/backup"`: the parameter of `-s`. `/tmp/backup` is the path in the local disk where the backed up files of each TiKV node are stored. -* `--pd`: the option that specifies the Placement Driver (PD) service address. -* `"${PDIP}:2379"`: the parameter of `--pd`. - -> **Note:** -> -> - When the `local` storage is used, the backup data are scattered in the local file system of each node. -> -> - It is **not recommended** to back up to a local disk in the production environment because you **have to** manually aggregate these data to complete the data restoration. For more information, see [Restore Cluster Data](#restore-cluster-data). -> -> - Aggregating these backup data might cause redundancy and bring troubles to operation and maintenance. Even worse, if restoring data without aggregating these data, you can receive a rather confusing error message `SST file not found`. -> -> - It is recommended to mount the NFS disk on each node, or back up to the `S3` object storage. - -### Sub-commands - -A `br` command consists of multiple layers of sub-commands. Currently, BR has the following three sub-commands: - -* `br backup`: used to back up the data of the TiDB cluster. -* `br restore`: used to restore the data of the TiDB cluster. - -Each of the above three sub-commands might still include the following three sub-commands to specify the scope of an operation: - -* `full`: used to back up or restore all the cluster data. -* `db`: used to back up or restore the specified database of the cluster. -* `table`: used to back up or restore a single table in the specified database of the cluster. - -### Common options - -* `--pd`: used for connection, specifying the PD server address. For example, `"${PDIP}:2379"`. -* `-h` (or `--help`): used to get help on all sub-commands. For example, `br backup --help`. -* `-V` (or `--version`): used to check the version of BR. -* `--ca`: specifies the path to the trusted CA certificate in the PEM format. -* `--cert`: specifies the path to the SSL certificate in the PEM format. -* `--key`: specifies the path to the SSL certificate key in the PEM format. -* `--status-addr`: specifies the listening address through which BR provides statistics to Prometheus. - -## Back up cluster data - -To back up the cluster data, use the `br backup` command. You can add the `full` or `table` sub-command to specify the scope of your backup operation: the whole cluster or a single table. - -If the BR version is earlier than v4.0.3, and the backup duration might exceed the [`tikv_gc_life_time`](/garbage-collection-configuration.md#tikv_gc_life_time) configuration which is `10m0s` by default (`10m0s` means 10 minutes), increase the value of this configuration item. - -For example, set `tikv_gc_life_time` to `720h`: - -{{< copyable "sql" >}} - -```sql -mysql -h${TiDBIP} -P4000 -u${TIDB_USER} ${password_str} -Nse \ - "update mysql.tidb set variable_value='720h' where variable_name='tikv_gc_life_time'"; -``` - -Since v4.0.3, BR automatically adapts to GC and you do not need to manually adjust the `tikv_gc_life_time` value. +## Deploy and use BR -### Back up all the cluster data +### Recommended deployment configuration -To back up all the cluster data, execute the `br backup full` command. To get help on this command, execute `br backup full -h` or `br backup full --help`. - -**Usage example:** - -Back up all the cluster data to the `/tmp/backup` path of each TiKV node and write the `backupmeta` file to this path. +- It is recommended that you deploy BR on the PD node. +- It is recommended that you mount a high-performance SSD to BR nodes and all TiKV nodes. A 10-gigabit network card is recommended. Otherwise, bandwidth is likely to be the performance bottleneck during the backup and restore process. > **Note:** > -> + If the backup disk and the service disk are different, it has been tested that online backup reduces QPS of the read-only online service by about 15%-25% in case of full-speed backup. If you want to reduce the impact on QPS, use `--ratelimit` to limit the rate. +> - If you do not mount a network disk or use other shared storage, the data backed up by BR will be generated on each TiKV node. Because BR only backs up leader replicas, you should estimate the space reserved for each node based on the leader size. > -> + If the backup disk and the service disk are the same, the backup competes with the service for I/O resources. This might decrease the QPS of the read-only online service by more than half. Therefore, it is **highly not recommended** to back up the online service data to the TiKV data disk. - -{{< copyable "shell-regular" >}} - -```shell -br backup full \ - --pd "${PDIP}:2379" \ - --storage "local:///tmp/backup" \ - --ratelimit 120 \ - --log-file backupfull.log -``` - -Explanations for some options in the above command are as follows: - -* `--ratelimit`: specifies the maximum speed at which a backup operation is performed (MiB/s) on each TiKV node. -* `--log-file`: specifies writing the BR log to the `backupfull.log` file. - -A progress bar is displayed in the terminal during the backup. When the progress bar advances to 100%, the backup is complete. Then the BR also checks the backup data to ensure data safety. The progress bar is displayed as follows: - -```shell -br backup full \ - --pd "${PDIP}:2379" \ - --storage "local:///tmp/backup" \ - --ratelimit 120 \ - --log-file backupfull.log -Full Backup <---------/................................................> 17.12%. -``` - -### Back up a database +> - Because TiDB uses leader count for load balancing by default, leaders can greatly differ in size. This might resulting in uneven distribution of backup data on each node. -To back up a database in the cluster, execute the `br backup db` command. To get help on this command, execute `br backup db -h` or `br backup db --help`. +### Usage restrictions -**Usage example:** +The following are the limitations of using BR for backup and restoration: -Back up the data of the `test` database to the `/tmp/backup` path on each TiKV node and write the `backupmeta` file to this path. - -{{< copyable "shell-regular" >}} - -```shell -br backup db \ - --pd "${PDIP}:2379" \ - --db test \ - --storage "local:///tmp/backup" \ - --ratelimit 120 \ - --log-file backuptable.log -``` - -In the above command, `--db` specifies the name of the database to be backed up. For descriptions of other options, see [Back up all the cluster data](/br/backup-and-restore-tool.md#back-up-cluster-data). - -A progress bar is displayed in the terminal during the backup. When the progress bar advances to 100%, the backup is complete. Then the BR also checks the backup data to ensure data safety. - -### Back up a table - -To back up the data of a single table in the cluster, execute the `br backup table` command. To get help on this command, execute `br backup table -h` or `br backup table --help`. - -**Usage example:** - -Back up the data of the `test.usertable` table to the `/tmp/backup` path on each TiKV node and write the `backupmeta` file to this path. - -{{< copyable "shell-regular" >}} - -```shell -br backup table \ - --pd "${PDIP}:2379" \ - --db test \ - --table usertable \ - --storage "local:///tmp/backup" \ - --ratelimit 120 \ - --log-file backuptable.log -``` - -The `table` sub-command has two options: - -* `--db`: specifies the database name -* `--table`: specifies the table name. - -For descriptions of other options, see [Back up all cluster data](#back-up-cluster-data). - -A progress bar is displayed in the terminal during the backup operation. When the progress bar advances to 100%, the backup is complete. Then the BR also checks the backup data to ensure data safety. - -### Back up with table filter - -To back up multiple tables with more complex criteria, execute the `br backup full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. - -**Usage example:** - -The following command backs up the data of all tables in the form `db*.tbl*` to the `/tmp/backup` path on each TiKV node and writes the `backupmeta` file to this path. - -{{< copyable "shell-regular" >}} - -```shell -br backup full \ - --pd "${PDIP}:2379" \ - --filter 'db*.tbl*' \ - --storage "local:///tmp/backup" \ - --ratelimit 120 \ - --log-file backupfull.log -``` - -### Back up data to Amazon S3 backend - -If you back up the data to the Amazon S3 backend, instead of `local` storage, you need to specify the S3 storage path in the `storage` sub-command, and allow the BR node and the TiKV node to access Amazon S3. - -You can refer to the [AWS Official Document](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html) to create an S3 `Bucket` in the specified `Region`. You can also refer to another [AWS Official Document](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-folder.html) to create a `Folder` in the `Bucket`. - -Pass `SecretKey` and `AccessKey` of the account that has privilege to access the S3 backend to the BR node. Here `SecretKey` and `AccessKey` are passed as environment variables. Then pass the privilege to the TiKV node through BR. - -{{< copyable "shell-regular" >}} - -```shell -export AWS_ACCESS_KEY_ID=${AccessKey} -export AWS_SECRET_ACCESS_KEY=${SecretKey} -``` - -When backing up using BR, explicitly specify the parameters `--s3.region` and `--send-credentials-to-tikv`. `--s3.region` indicates the region where S3 is located, and `--send-credentials-to-tikv` means passing the privilege to access S3 to the TiKV node. - -{{< copyable "shell-regular" >}} - -```shell -br backup full \ - --pd "${PDIP}:2379" \ - --storage "s3://${Bucket}/${Folder}" \ - --s3.region "${region}" \ - --send-credentials-to-tikv=true \ - --log-file backuptable.log -``` - -### Back up incremental data +- When BR restores data to the upstream cluster of TiCDC/Drainer, TiCDC/Drainer cannot replicate the restored data to the downstream. +- BR supports operations only between clusters with the same [`new_collations_enabled_on_first_bootstrap`](/character-set-and-collation.md#collation-support-framework) value because BR only backs up KV data. If the cluster to be backed up and the cluster to be restored use different collations, the data validation fails. Therefore, before restoring a cluster, make sure that the switch value from the query result of the `select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME='new_collation_enabled';` statement is consistent with that during the backup process. -If you want to back up incrementally, you only need to specify the **last backup timestamp** `--lastbackupts`. +### Compatibility -The incremental backup has two limitations: +The compatibility issues of BR and the TiDB cluster are divided into the following categories: -- The incremental backup needs to be under a different path from the previous full backup. -- GC (Garbage Collection) safepoint must be before the `lastbackupts`. ++ Some versions of BR are not compatible with the interface of the TiDB cluster. ++ The KV format might change when some features are enabled or disabled. If these features are not consistently enabled or disabled during backup and restore, compatibility issues might occur. -To back up the incremental data between `(LAST_BACKUP_TS, current PD timestamp]`, execute the following command: +These features are as follows: -{{< copyable "shell-regular" >}} +| Features | Related issues | Solutions | +| ---- | ---- | ----- | +| Clustered index | [#565](https://github.com/pingcap/br/issues/565) | Make sure that the value of the `tidb_enable_clustered_index` global variable during restore is consistent with that during backup. Otherwise, data inconsistency might occur, such as `default not found` and inconsistent data index. | +| New collation | [#352](https://github.com/pingcap/br/issues/352) | Make sure that the value of the `new_collations_enabled_on_first_bootstrap` variable is consistent with that during backup. Otherwise, inconsistent data index might occur and checksum might fail to pass. | +| TiCDC enabled on the restore cluster | [#364](https://github.com/pingcap/br/issues/364#issuecomment-646813965) | Currently, TiKV cannot push down the BR-ingested SST files to TiCDC. Therefore, you need to disable TiCDC when using BR to restore data. | -```shell -br backup full\ - --pd ${PDIP}:2379 \ - -s local:///home/tidb/backupdata/incr \ - --lastbackupts ${LAST_BACKUP_TS} -``` +However, even after you have ensured that the above features are consistently enabled or disabled during backup and restore, compatibility issues might still occur due to the inconsistent internal versions or inconsistent interfaces between BR and TiKV/TiDB/PD. To avoid such cases, BR has the built-in version check. -To get the timestamp of the last backup, execute the `validate` command. For example: +#### Version check -{{< copyable "shell-regular" >}} +Before performing backup and restore, BR compares and checks the TiDB cluster version and the BR version. If there is a major-version mismatch (for example, BR v4.x and TiDB v5.x), BR prompts a reminder to exit. To forcibly skip the version check, you can set `--check-requirements=false`. -```shell -LAST_BACKUP_TS=`br validate decode --field="end-version" -s local:///home/tidb/backupdata` -``` +Note that skipping the version check might introduce incompatibility. The version compatibility information between BR and TiDB versions are as follows: -In the above example, for the incremental backup data, BR records the data changes and the DDL operations during `(LAST_BACKUP_TS, current PD timestamp]`. When restoring data, BR first restores DDL operations and then the data. +| Backup version (vertical) \ Restore version (horizontal) | Use BR nightly to restore TiDB nightly | Use BR v5.0 to restore TiDB v5.0| Use BR v4.0 to restore TiDB v4.0 | +| ---- | ---- | ---- | ---- | +| Use BR nightly to back up TiDB nightly | ✅ | ✅ | ❌ (If a table with the primary key of the non-integer clustered index type is restored to a TiDB v4.0 cluster, BR will cause data error without warning.) | +| Use BR v5.0 to back up TiDB v5.0 | ✅ | ✅ | ❌ (If a table with the primary key of the non-integer clustered index type is restored to a TiDB v4.0 cluster, BR will cause data error without warning.) +| Use BR v4.0 to back up TiDB v4.0 | ✅ | ✅ | ✅ (If TiKV >= v4.0.0-rc.1, and if BR contains the [#233](https://github.com/pingcap/br/pull/233) bug fix and TiKV does not contain the [#7241](https://github.com/tikv/tikv/pull/7241) bug fix, BR will cause the TiKV node to restart.) | +| Use BR nightly or v5.0 to back up TiDB v4.0 | ❌ (If the TiDB version is earlier than v4.0.9, the [#609](https://github.com/pingcap/br/issues/609) issue might occur.) | ❌ (If the TiDB version is earlier than v4.0.9, the [#609](https://github.com/pingcap/br/issues/609) issue might occur.) | ❌ (If the TiDB version is earlier than v4.0.9, the [#609](https://github.com/pingcap/br/issues/609) issue might occur.) | -### Back up Raw KV (experimental feature) +### Back up and restore table data in the `mysql` system schema (experimental feature) > **Warning:** > > This feature is experimental and not thoroughly tested. It is highly **not recommended** to use this feature in the production environment. -In some scenarios, TiKV might run independently of TiDB. Given that, BR also supports bypassing the TiDB layer and backing up data in TiKV. - -For example, you can execute the following command to back up all keys between `[0x31, 0x3130303030303030)` in the default CF to `$BACKUP_DIR`: - -{{< copyable "shell-regular" >}} - -```shell -br backup raw --pd $PD_ADDR \ - -s "local://$BACKUP_DIR" \ - --start 31 \ - --end 3130303030303030 \ - --format hex \ - --cf default -``` - -Here, the parameters of `--start` and `--end` are decoded using the method specified by `--format` before being sent to TiKV. Currently, the following methods are available: - -- "raw": The input string is directly encoded as a key in binary format. -- "hex": The default encoding method. The input string is treated as a hexadecimal number. -- "escape": First escape the input string, and then encode it into binary format. - -## Restore cluster data - -To restore the cluster data, use the `br restore` command. You can add the `full`, `db` or `table` sub-command to specify the scope of your restoration: the whole cluster, a database or a single table. - -> **Note:** -> -> If you use the local storage, you **must** copy all back up SST files to every TiKV node in the path specified by `--storage`. -> -> Even if each TiKV node eventually only need to read a part of the all SST files, they all need full access to the complete archive because: -> -> - Data are replicated into multiple peers. When ingesting SSTs, these files have to be present on *all* peers. This is unlike back up where reading from a single node is enough. -> - Where each peer is scattered to during restore is random. We don't know in advance which node will read which file. -> -> These can be avoided using shared storage, for example mounting an NFS on the local path, or using S3. With network storage, every node can automatically read every SST file, so these caveats no longer apply. - -### Restore all the backup data - -To restore all the backup data to the cluster, execute the `br restore full` command. To get help on this command, execute `br restore full -h` or `br restore full --help`. - -**Usage example:** - -Restore all the backup data in the `/tmp/backup` path to the cluster. - -{{< copyable "shell-regular" >}} - -```shell -br restore full \ - --pd "${PDIP}:2379" \ - --storage "local:///tmp/backup" \ - --ratelimit 128 \ - --log-file restorefull.log -``` - -Explanations for some options in the above command are as follows: - -* `--ratelimit`: specifies the maximum speed at which a restoration operation is performed (MiB/s) on each TiKV node. -* `--log-file`: specifies writing the BR log to the `restorefull.log` file. - -A progress bar is displayed in the terminal during the restoration. When the progress bar advances to 100%, the restoration is complete. Then the BR also checks the backup data to ensure data safety. - -```shell -br restore full \ - --pd "${PDIP}:2379" \ - --storage "local:///tmp/backup" \ - --log-file restorefull.log -Full Restore <---------/...............................................> 17.12%. -``` - -### Restore a database - -To restore a database to the cluster, execute the `br restore db` command. To get help on this command, execute `br restore db -h` or `br restore db --help`. - -**Usage example:** - -Restore a database backed up in the `/tmp/backup` path to the cluster. - -{{< copyable "shell-regular" >}} - -```shell -br restore db \ - --pd "${PDIP}:2379" \ - --db "test" \ - --storage "local:///tmp/backup" \ - --log-file restorefull.log -``` - -In the above command, `--db` specifies the name of the database to be restored. For descriptions of other options, see [Restore all backup data](#restore-all-the-backup-data)). - -### Restore a table - -To restore a single table to the cluster, execute the `br restore table` command. To get help on this command, execute `br restore table -h` or `br restore table --help`. - -**Usage example:** - -Restore a table backed up in the `/tmp/backup` path to the cluster. - -{{< copyable "shell-regular" >}} - -```shell -br restore table \ - --pd "${PDIP}:2379" \ - --db "test" \ - --table "usertable" \ - --storage "local:///tmp/backup" \ - --log-file restorefull.log -``` - -In the above command, `--table` specifies the name of the table to be restored. For descriptions of other options, see [Restore all backup data](#restore-all-the-backup-data) and [Restore a database](#restore-a-database). - -### Restore with table filter - -To restore multiple tables with more complex criteria, execute the `br restore full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. - -**Usage example:** - -The following command restores a subset of tables backed up in the `/tmp/backup` path to the cluster. +Before v5.1.0, BR filtered out data from the system schema `mysql` during the backup. Since v5.1.0, BR **backs up** all data by default, including the system schemas `mysql.*`. But the technical implementation of restoring the system tables in `mysql.*` is not complete yet, so the tables in the system schema `mysql` are **not** restored by default. -{{< copyable "shell-regular" >}} +If you want the data of a system table (for example, `mysql.usertable1`) to be restored to the system schema `mysql`, you can set the [`filter` parameter](/br/use-br-command-line-tool.md#back-up-with-table-filter) to filter the table name (`-f "mysql.usertable1"`). After the setting, the system table is first restored to the temporary schema, and then to the system schema through renaming. -```shell -br restore full \ - --pd "${PDIP}:2379" \ - --filter 'db*.tbl*' \ - --storage "local:///tmp/backup" \ - --log-file restorefull.log -``` - -### Restore data from Amazon S3 backend - -If you restore data from the Amazon S3 backend, instead of `local` storage, you need to specify the S3 storage path in the `storage` sub-command, and allow the BR node and the TiKV node to access Amazon S3. - -Pass `SecretKey` and `AccessKey` of the account that has privilege to access the S3 backend to the BR node. Here `SecretKey` and `AccessKey` are passed as environment variables. Then pass the privilege to the TiKV node through BR. - -{{< copyable "shell-regular" >}} - -```shell -export AWS_ACCESS_KEY_ID=${AccessKey} -export AWS_SECRET_ACCESS_KEY=${SecretKey} -``` - -When restoring data using BR, explicitly specify the parameters `--s3.region` and `--send-credentials-to-tikv`. `--s3.region` indicates the region where S3 is located, and `--send-credentials-to-tikv` means passing the privilege to access S3 to the TiKV node. - -`Bucket` and `Folder` in the `--storage` parameter represent the S3 bucket and the folder where the data to be restored is located. - -{{< copyable "shell-regular" >}} - -```shell -br restore full \ - --pd "${PDIP}:2379" \ - --storage "s3://${Bucket}/${Folder}" \ - --s3.region "${region}" \ - --send-credentials-to-tikv=true \ - --log-file restorefull.log -``` - -In the above command, `--table` specifies the name of the table to be restored. For descriptions of other options, see [Restore a database](#restore-a-database). +It should be noted that the following system tables cannot be restored correctly due to technical reasons. Even if `-f "mysql.*"` is specified, these tables will not be restored: -### Restore incremental data +- Tables related to statistics: "stats_buckets", "stats_extended", "stats_feedback", "stats_fm_sketch", "stats_histograms", "stats_meta", "stats_top_n" +- Tables related to privileges or the system: "tidb", "global_variables", "columns_priv", "db", "default_roles", "global_grants", "global_priv", "role_edges", "tables_priv", "user", "gc_delete_range", "Gc_delete_range_done", "schema_index_usage" -Restoring incremental data is similar to [restoring full data using BR](#restore-all-the-backup-data). Note that when restoring incremental data, make sure that all the data backed up before `last backup ts` has been restored to the target cluster. +### Minimum machine configuration required for running BR -### Restore Raw KV (experimental feature) - -> **Warning:** -> -> This feature is in the experiment, without being thoroughly tested. It is highly **not recommended** to use this feature in the production environment. - -Similar to [backing up Raw KV](#back-up-raw-kv-experimental-feature), you can execute the following command to restore Raw KV: - -{{< copyable "shell-regular" >}} - -```shell -br restore raw --pd $PD_ADDR \ - -s "local://$BACKUP_DIR" \ - --start 31 \ - --end 3130303030303030 \ - --format hex \ - --cf default -``` - -In the above example, all the backed up keys in the range `[0x31, 0x3130303030303030)` are restored to the TiKV cluster. The coding methods of these keys are identical to that of [keys during the backup process](#back-up-raw-kv-experimental-feature) - -### Online restore (experimental feature) - -> **Warning:** -> -> This feature is in the experiment, without being thoroughly tested. It also relies on the unstable `Placement Rules` feature of PD. It is highly **not recommended** to use this feature in the production environment. +The minimum machine configuration required for running BR is as follows: -During data restoration, writing too much data affects the performance of the online cluster. To avoid this effect as much as possible, BR supports [Placement rules](/configure-placement-rules.md) to isolate resources. In this case, downloading and importing SST are only performed on a few specified nodes (or "restore nodes" for short). To complete the online restore, take the following steps. +| CPU | Memory | Hard Disk Type | Network | +| --- | --- | --- | --- | +| 1 core | 4 GB | HDD | Gigabit network card | -1. Configure PD, and start Placement rules: +In general scenarios (less than 1000 tables for backup and restore), the CPU consumption of BR at runtime does not exceed 200%, and the memory consumption does not exceed 4 GB. However, when backing up and restoring a large number of tables, BR might consume more than 4 GB of memory. In a test of backing up 24000 tables, BR consumes about 2.7 GB of memory, and the CPU consumption remains below 100%. - {{< copyable "shell-regular" >}} +### Best practices - ```shell - echo "config set enable-placement-rules true" | pd-ctl - ``` - -2. Edit the configuration file of the "restore node" in TiKV, and specify "restore" to the `server` configuration item: - - {{< copyable "" >}} - - ``` - [server] - labels = { exclusive = "restore" } - ``` - -3. Start TiKV of the "restore node" and restore the backed up files using BR. Compared with the offline restore, you only need to add the `--online` flag: - - {{< copyable "shell-regular" >}} - - ``` - br restore full \ - -s "local://$BACKUP_DIR" \ - --pd $PD_ADDR \ - --online - ``` - -## Best practices +The following are some recommended operations for using BR for backup and restoration: +- It is recommended that you perform the backup operation during off-peak hours to minimize the impact on applications. +- BR supports restore on clusters of different topologies. However, the online applications will be greatly impacted during the restore operation. It is recommended that you perform restore during the off-peak hours or use `rate-limit` to limit the rate. +- It is recommended that you execute multiple backup operations serially. Running different backup operations in parallel reduces backup performance and also affects the online application. +- It is recommended that you execute multiple restore operations serially. Running different restore operations in parallel increases Region conflicts and also reduces restore performance. - It is recommended that you mount a shared storage (for example, NFS) on the backup path specified by `-s`, to make it easier to collect and manage backup files. - It is recommended that you use a storage hardware with high throughput, because the throughput of a storage hardware limits the backup and restoration speed. -- It is recommended that you perform the backup operation during off-peak hours to minimize the impact on applications. - -For more recommended practices of using BR, refer to [BR Use Cases](/br/backup-and-restore-use-cases.md). - -## Examples - -This section shows how to back up and restore the data of an existing cluster. You can estimate the performance of backup and restoration based on machine performance, configuration and data volume. - -### Data volume and machine configuration - -Suppose that the backup and restoration operations are performed on 10 tables in the TiKV cluster, each table with 5 million rows of data. The total data volume is 35 GB. - -```sql -MySQL [sbtest]> show tables; -+------------------+ -| Tables_in_sbtest | -+------------------+ -| sbtest1 | -| sbtest10 | -| sbtest2 | -| sbtest3 | -| sbtest4 | -| sbtest5 | -| sbtest6 | -| sbtest7 | -| sbtest8 | -| sbtest9 | -+------------------+ - -MySQL [sbtest]> select count(*) from sbtest1; -+----------+ -| count(*) | -+----------+ -| 5000000 | -+----------+ -1 row in set (1.04 sec) -``` -The table structure is as follows: - -```sql -CREATE TABLE `sbtest1` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `k` int(11) NOT NULL DEFAULT '0', - `c` char(120) NOT NULL DEFAULT '', - `pad` char(60) NOT NULL DEFAULT '', - PRIMARY KEY (`id`), - KEY `k_1` (`k`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=5138499 -``` - -Suppose that 4 TiKV nodes is used, each with the following configuration: - -| CPU | Memory | Disk | Number of replicas | -| :-------- | :------ | :---- | :------------------ | -| 16 cores | 32 GB | SSD | 3 | +### How to use BR -### Backup +Currently, the following methods are supported to run the BR tool: -Before the backup operation, check the following two items: +- Use SQL statements +- Use the command-line tool +- Use BR In the Kubernetes environment -- You have set `tikv_gc_life_time` set to a larger value so that the backup operation will not be interrupted because of data loss. -- No DDL statement is being executed on the TiDB cluster. +#### Use SQL statements -Then execute the following command to back up all the cluster data: +TiDB supports both [`BACKUP`](/sql-statements/sql-statement-backup.md#backup) and [`RESTORE`](/sql-statements/sql-statement-restore.md#restore) SQL statements. The progress of these operations can be monitored with the statement [`SHOW BACKUPS|RESTORES`](/sql-statements/sql-statement-show-backups.md). -{{< copyable "shell-regular" >}} +#### Use the command-line tool -```shell -bin/br backup full -s local:///tmp/backup --pd "${PDIP}:2379" --log-file backup.log -``` - -``` -[INFO] [collector.go:165] ["Full backup summary: total backup ranges: 2, total success: 2, total failed: 0, total take(s): 0.00, total kv: 4, total size(Byte): 133, avg speed(Byte/s): 27293.78"] ["backup total regions"=2] ["backup checksum"=1.640969ms] ["backup fast checksum"=227.885µs] -``` +The `br` command-line utility is available as a [separate download](/download-ecosystem-tools.md#br-backup-and-restore). For details, see [Use BR Command-line for Backup and Restoration](/br/use-br-command-line-tool.md). -### Restoration +#### In the Kubernetes environment -Before the restoration, make sure that the TiKV cluster to be restored is a new cluster. +In the Kubernetes environment, you can use the BR tool to back up TiDB cluster data to S3-compatible storage, Google Cloud Storage (GCS) and persistent volumes (PV), and restore them: -Then execute the following command to restore all the cluster data: +> **Note:** +> +> For Amazon S3 and Google Cloud Storage parameter descriptions, see the [External Storages](/br/backup-and-restore-storages.md#url-parameters) document. -{{< copyable "shell-regular" >}} +- [Back up Data to S3-Compatible Storage Using BR](https://docs.pingcap.com/tidb-in-kubernetes/stable/backup-to-aws-s3-using-br) +- [Restore Data from S3-Compatible Storage Using BR](https://docs.pingcap.com/tidb-in-kubernetes/stable/restore-from-aws-s3-using-br) +- [Back up Data to GCS Using BR](https://docs.pingcap.com/tidb-in-kubernetes/stable/backup-to-gcs-using-br) +- [Restore Data from GCS Using BR](https://docs.pingcap.com/tidb-in-kubernetes/stable/restore-from-gcs-using-br) +- [Back up Data to PV Using BR](https://docs.pingcap.com/tidb-in-kubernetes/stable/backup-to-pv-using-br) +- [Restore Data from PV Using BR](https://docs.pingcap.com/tidb-in-kubernetes/stable/restore-from-pv-using-br) -```shell -bin/br restore full -s local:///tmp/backup --pd "${PDIP}:2379" --log-file restore.log -``` +## Other documents about BR -``` -[INFO] [collector.go:165] ["Full Restore summary: total restore tables: 1, total success: 1, total failed: 0, total take(s): 0.26, total kv: 20000, total size(MB): 10.98, avg speed(MB/s): 41.95"] ["restore files"=3] ["restore ranges"=2] ["split region"=0.562369381s] ["restore checksum"=36.072769ms] -``` +- [Use BR Command-line](/br/use-br-command-line-tool.md) +- [BR Use Cases](/br/backup-and-restore-use-cases.md) +- [BR FAQ](/br/backup-and-restore-faq.md) +- [External Storages](/br/backup-and-restore-storages.md) diff --git a/br/backup-and-restore-use-cases.md b/br/backup-and-restore-use-cases.md index f0a2914a94ad..83170ee5356c 100644 --- a/br/backup-and-restore-use-cases.md +++ b/br/backup-and-restore-use-cases.md @@ -6,24 +6,33 @@ aliases: ['/docs/dev/br/backup-and-restore-use-cases/','/docs/dev/reference/tool # BR Use Cases -[Backup & Restore](/br/backup-and-restore-tool.md) (BR) is a command-line tool for distributed backup and restoration of the TiDB cluster data. This document describes the processes of operating BR in [four use cases](#use-cases) that aims to help you achieve the following goals: +[BR](/br/backup-and-restore-tool.md) is a tool for distributed backup and restoration of the TiDB cluster data. + +This document describes how to run BR in the following use cases: + +- Back up a single table to a network disk (recommended in production environment) +- Restore data from a network disk (recommended in production environment) +- Back up a single table to a local disk (recommended in testing environment) +- Restore data from a local disk (recommended in testing environment) + +This document aims to help you achieve the following goals: * Back up and restore data using a network disk or local disk correctly. * Get the status of a backup or restoration operation through monitoring metrics. * Learn how to tune performance during the operation. * Troubleshoot the possible anomalies during the backup operation. -> **Note:** -> -> Pay attention to the [usage restrictions](/br/backup-and-restore-tool.md#usage-restrictions) before using BR. - ## Audience -You are expected to have a basic understanding of [TiDB](https://docs.pingcap.com/tidb/v4.0) and [TiKV](https://tikv.org/). Before reading this document, it is recommended that you read [Use BR to Back up and Restore Data](/br/backup-and-restore-tool.md) first. +You are expected to have a basic understanding of TiDB and [TiKV](https://tikv.org/). + +Before reading on, make sure you have read [BR Tool Overview](/br/backup-and-restore-tool.md), especially [Usage Restrictions](/br/backup-and-restore-tool.md#usage-restrictions) and [Best Practices](/br/backup-and-restore-tool.md#best-practices). ## Prerequisites -This section introduces the recommended method of deploying TiDB, cluster versions, the hardware information of the TiKV cluster, and the cluster configuration for the use case demonstrations. You can estimate the performance of your backup or restoration operation based on your own hardware and configuration. +This section introduces the recommended method of deploying TiDB, cluster versions, the hardware information of the TiKV cluster, and the cluster configuration for the use case demonstrations. + +You can estimate the performance of your backup or restoration operation based on your own hardware and configuration. ### Deployment method @@ -31,14 +40,14 @@ It is recommended that you deploy the TiDB cluster using [TiUP](/tiup/tiup-clust ### Cluster versions -* TiDB: v4.0.2 -* TiKV: v4.0.2 -* PD: v4.0.2 -* BR: v4.0.2 +* TiDB: v5.0.0 +* TiKV: v5.0.0 +* PD: v5.0.0 +* BR: v5.0.0 > **Note:** > -> v4.0.2 was the latest version at the time this document was written. It is recommended that you use the latest version of [TiDB/TiKV/PD/BR](/releases/release-notes.md) and make sure that the BR version is **consistent with** the TiDB version. +> v5.0.0 was the latest version at the time this document was written. It is recommended that you use the latest version of [TiDB/TiKV/PD/BR](/releases/release-notes.md) and make sure that the BR version is **consistent with** the TiDB version. ### TiKV hardware information @@ -57,61 +66,48 @@ BR directly sends commands to the TiKV cluster and are not dependent on the TiDB ## Use cases -This document describes the following four use cases: +This document describes the following use cases: -* [Back up a single table to a network disk (recommended)](#back-up-a-single-table-to-a-network-disk-recommended) -* [Restore data from a network disk (recommended)](#restore-data-from-a-network-disk-recommended) -* [Back up a single table to a local disk](#back-up-a-single-table-to-a-local-disk) -* [Restore data from a local disk](#restore-data-from-a-local-disk) +* [Back up a single table to a network disk (recommended in production environment)](#back-up-a-single-table-to-a-network-disk-recommended-in-production-environment) +* [Restore data from a network disk (recommended in production environment)](#restore-data-from-a-network-disk-recommended-in-production-environment) +* [Back up a single table to a local disk (recommended in testing environment)](#back-up-a-single-table-to-a-local-disk-recommended-in-testing-environment) +* [Restore data from a local disk (recommended in testing environment)](#restore-data-from-a-local-disk-recommended-in-testing-environment) It is recommended that you use a network disk to back up and restore data. This spares you from collecting backup files and greatly improves the backup efficiency especially when the TiKV cluster is in a large scale. -> **Note:** -> -> Before the backup or restoration operation, you need to do some preparations. See [Preparation for backup](#preparation-for-backup) and [Preparation for restoration](#preparation-for-restoration) for details. - -### Preparation for backup - -For the detailed usage of the `br backup` command, refer to [BR command-line description](/br/backup-and-restore-tool.md#command-line-description). - -1. Before executing the `br backup` command, check the value of the [`tikv_gc_life_time`](/garbage-collection-configuration.md#tikv_gc_life_time) configuration item, and adjust the value appropriately in the MySQL client to make sure that [Garbage Collection](/garbage-collection-overview.md) (GC) does not run during the backup operation. +Before the backup or restoration operations, you need to do some preparations: - {{< copyable "sql" >}} +- [Preparation for backup](#preparation-for-backup) +- [Preparation for restoration](#preparation-for-restoration) - ```sql - SELECT * FROM mysql.tidb WHERE VARIABLE_NAME = 'tikv_gc_life_time'; - UPDATE mysql.tidb SET VARIABLE_VALUE = '720h' WHERE VARIABLE_NAME = 'tikv_gc_life_time'; - ``` - -2. After the backup operation, set the parameter back to the original value. +### Preparation for backup - {{< copyable "sql" >}} +The BR tool already supports self-adapting to GC. It automatically registers `backupTS` (the latest PD timestamp by default) to PD's `safePoint` to ensure that TiDB's GC Safe Point does not move forward during the backup, thus avoiding manually setting GC configurations. - ```sql - UPDATE mysql.tidb SET VARIABLE_VALUE = '10m' WHERE VARIABLE_NAME = 'tikv_gc_life_time'; - ``` +For the detailed usage of the `br backup` command, refer to [Use BR Command-line for Backup and Restoration](/br/use-br-command-line-tool.md). -> **Note:** -> -> Since v4.0.8, BR supports the self-adaptive GC. To avoid manually adjusting GC, register `backupTS` in `safePoint` in PD and make sure that `safePoint` does not move forward during the backup process. +1. Before executing the `br backup` command, ensure that no DDL is running on the TiDB cluster. +2. Ensure that the storage device where the backup will be created has sufficient space. ### Preparation for restoration -For the detailed usage of the `br restore` command, refer to [BR command-line description](/br/backup-and-restore-tool.md#command-line-description). - -> **Note:** -> -> Before executing the `br restore` command, check the new cluster to make sure that the table in the cluster does not have a duplicate name. +Before executing the [`br restore` command](/br/use-br-command-line-tool.md#br-command-line-description), check the new cluster to make sure that the table in the cluster does not have a duplicate name. -### Back up a single table to a network disk (recommended) +### Back up a single table to a network disk (recommended in production environment) Use the `br backup` command to back up the single table data `--db batchmark --table order_line` to the specified path `local:///br_data` in the network disk. #### Backup prerequisites -* Configure a high-performance SSD hard disk host as the NFS server to store data, and all BR nodes and TiKV nodes as NFS clients. Mount the same path (for example, `/br_data`) to the NFS server for NFS clients to access the server. +* [Preparation for backup](#preparation-for-backup) +* Configure a high-performance SSD hard disk host as the NFS server to store data, and all BR nodes, TiKV nodes, and TiFlash nodes as NFS clients. Mount the same path (for example, `/br_data`) to the NFS server for NFS clients to access the server. * The total transfer rate between the NFS server and all NFS clients must reach at least `the number of TiKV instances * 150MB/s`. Otherwise the network I/O might become the performance bottleneck. +> **Note:** +> +> * During data backup, because only the data of leader replicas are backed up, even if there is a TiFlash replica in the cluster, BR can complete the backup without mounting TiFlash nodes. +> * When restoring data, BR will restore the data of all replicas. Also, TiFlash nodes need access to the backup data for BR to complete the restore. Therefore, before the restore, you must mount TiFlash nodes to the NFS server. + #### Topology The following diagram shows the typology of BR: @@ -176,34 +172,45 @@ During the backup process, pay attention to the following metrics on the monitor #### Backup results explanation +When finishing the backup, BR outputs the backup summary to the console. + Before executing the backup command, a path in which the log is stored has been specified. You can get the statistical information of the backup operation from this log. Search "summary" in this log, you can see the following information: ``` -["Table backup summary: - total backup ranges: 4, - total success: 4, +["Full backup Success summary: + total backup ranges: 2, + total success: 2, total failed: 0, - total take(s): 986.43, - total kv: 5659888624, - total size(MB): 353227.18, - avg speed(MB/s): 358.09"] - ["backup total regions"=7196] - ["backup checksum"=6m28.291772955s] - ["backup fast checksum"=24.950298ms] + total take(Full backup time): 31.802912166s, + total take(real time): 49.799662427s, + total size(MB): 5997.49, + avg speed(MB/s): 188.58, + total kv: 120000000"] + ["backup checksum"=17.907153678s] + ["backup fast checksum"=349.333µs] + ["backup total regions"=43] + [BackupTS=422618409346269185] + [Size=826765915] ``` The above log includes the following information: -* Backup duration: `total take(s): 986.43` -* Data size: `total size(MB): 353227.18` -* Backup throughput: `avg speed(MB/s): 358.09` -* Backup checksum duration: `take=6m28.29s` +* Backup duration: `total take(Full backup time): 31.802912166s` +* Total runtime of the application: `total take(real time): 49.799662427s` +* Backup data size: `total size(MB): 5997.49` +* Backup throughput: `avg speed(MB/s): 188.58` +* Number of backed-up KV pairs: `total kv: 120000000` +* Backup checksum duration: `["backup checksum"=17.907153678s]` +* Total duration of calculating the checksum, KV pairs, and bytes of each table: `["backup fast checksum"=349.333µs]` +* Total number of backup Regions: `["backup total regions"=43]` +* The actual size of the backup data in the disk after compression: `[Size=826765915]` +* Snapshot timestamp of the backup data: `[BackupTS=422618409346269185]` -From the above information, the throughput of a single TiKV instance can be calculated: `avg speed(MB/s)`/`tikv_count` = `89`. +From the above information, the throughput of a single TiKV instance can be calculated: `avg speed(MB/s)`/`tikv_count` = `62.86`. #### Performance tuning -If the resource usage of TiKV does not become an obvious bottleneck during the backup process (for example, in the [Monitoring metrics for the backup](#monitoring-metrics-for-the-backup), the highest CPU usage rate of backup-worker is around `1500%` and the overall I/O usage rate is below `30%`), you can try to increase the value of `--concurrency` to tune the performance. But this performance tuning method is not suitable for the use cases of many small tables. See the following example: +If the resource usage of TiKV does not become an obvious bottleneck during the backup process (for example, in the [Monitoring metrics for the backup](#monitoring-metrics-for-the-backup), the highest CPU usage rate of backup-worker is around `1500%` and the overall I/O usage rate is below `30%`), you can try to increase the value of `--concurrency` (`4` by default) to tune the performance. But this performance tuning method is not suitable for the use cases of many small tables. See the following example: {{< copyable "shell-regular" >}} @@ -227,13 +234,13 @@ The tuned performance results are as follows (with the same data size): * Backup throughput: `avg speed(MB/s)` increased from `358.09` to `659.59` * Throughput of a single TiKV instance: `avg speed(MB/s)/tikv_count` increased from `89` to `164.89` -### Restore data from a network disk (recommended) +### Restore data from a network disk (recommended in production environment) Use the `br restore` command to restore the complete backup data to an offline cluster. Currently, BR does not support restoring data to an online cluster. #### Restoration prerequisites -None +* [Preparation for restoration](#preparation-for-restoration) #### Topology @@ -294,7 +301,8 @@ Before executing the restoration command, a path in which the log is stored has total restore tables: 1, total success: 1, total failed: 0, - total take(s): 961.37, + total take(Full restore time): 17m1.001611365s, + total take(real time): 16m1.371611365s, total kv: 5659888624, total size(MB): 353227.18, avg speed(MB/s): 367.42"] @@ -302,24 +310,28 @@ Before executing the restoration command, a path in which the log is stored has ["restore ranges"=6888] ["split region"=49.049182743s] ["restore checksum"=6m34.879439498s] + [Size=48693068713] ``` The above log includes the following information: -* Restoration duration: `total take(s):961.37` -* Data size: `total size(MB): 353227.18` -* Restoration throughput: `avg speed(MB/s): 367.42` +* Restore duration: `total take(Full restore time): 17m1.001611365s` +* Total runtime of the application: `total take(real time): 16m1.371611365s` +* Restore data size: `total size(MB): 353227.18` +* Restore KV pair number: `total kv: 5659888624` +* Restore throughput: `avg speed(MB/s): 367.42` * `Region Split` duration: `take=49.049182743s` -* Restoration checksum duration: `take=6m34.879439498s` +* Restore checksum duration: `restore checksum=6m34.879439498s` +* The actual size of the restored data in the disk: `[Size=48693068713]` From the above information, the following items can be calculated: * The throughput of a single TiKV instance: `avg speed(MB/s)`/`tikv_count` = `91.8` -* The average restoration speed of a single TiKV instance: `total size(MB)`/(`split time` + `restore time`)/`tikv_count` = `87.4` +* The average restore speed of a single TiKV instance: `total size(MB)`/(`split time` + `restore time`)/`tikv_count` = `87.4` #### Performance tuning -If the resource usage of TiKV does not become an obvious bottleneck during the restoration process, you can try to increase the value of `--concurrency` which is `128` by default. See the following example: +If the resource usage of TiKV does not become an obvious bottleneck during the restore process, you can try to increase the value of `--concurrency` which is `128` by default. See the following example: {{< copyable "shell-regular" >}} @@ -329,17 +341,18 @@ bin/br restore table --db batchmark --table order_line -s local:///br_data/ --pd The tuned performance results are as follows (with the same data size): -+ Restoration duration: `total take(s)` reduced from `961.37` to `443.49` -+ Restoration throughput: `avg speed(MB/s)` increased from `367.42` to `796.47` ++ Restore duration: `total take(s)` reduced from `961.37` to `443.49` ++ Restore throughput: `avg speed(MB/s)` increased from `367.42` to `796.47` + Throughput of a single TiKV instance: `avg speed(MB/s)`/`tikv_count` increased from `91.8` to `199.1` -+ Average restoration speed of a single TiKV instance: `total size(MB)`/(`split time` + `restore time`)/`tikv_count` increased from `87.4` to `162.3` ++ Average restore speed of a single TiKV instance: `total size(MB)`/(`split time` + `restore time`)/`tikv_count` increased from `87.4` to `162.3` -### Back up a single table to a local disk +### Back up a single table to a local disk (recommended in testing environment) Use the `br backup` command to back up the single table `--db batchmark --table order_line` to the specified path `local:///home/tidb/backup_local` in the local disk. #### Backup prerequisites +* [Preparation for backup](#preparation-for-backup) * Each TiKV node has a separate disk to store the backupSST file. * The `backup_endpoint` node has a separate disk to store the `backupmeta` file. * TiKV and the `backup_endpoint` node must have the same directory for the backup (for example, `/home/tidb/backup_local`). @@ -388,12 +401,13 @@ The information from the above log includes: From the above information, the throughput of a single TiKV instance can be calculated: `avg speed(MB/s)`/`tikv_count` = `160`. -### Restore data from a local disk +### Restore data from a local disk (recommended in testing environment) Use the `br restore` command to restore the complete backup data to an offline cluster. Currently, BR does not support restoring data to an online cluster. #### Restoration prerequisites +* [Preparation for restoration](#preparation-for-restoration) * The TiKV cluster and the backup data do not have a duplicate database or table. Currently, BR does not support table route. * Each TiKV node has a separate disk to store the backupSST file. * The `restore_endpoint` node has a separate disk to store the `backupmeta` file. @@ -444,17 +458,17 @@ From the above information, the following items can be calculated: * The throughput of a single TiKV instance: `avg speed(MB/s)`/`tikv_count` = `97.2` * The average restoration speed of a single TiKV instance: `total size(MB)`/(`split time` + `restore time`)/`tikv_count` = `92.4` -### Error handling +## Error handling during backup This section introduces the common errors occurred during the backup process. -#### `key locked Error` in the backup log +### `key locked Error` in the backup log Error message in the log: `log - ["backup occur kv error"][error="{\"KvError\":{\"locked\":` -If a key is locked during the backup process, BR tries to resolve the lock. A small number of this error does not affect the correctness of the backup. +If a key is locked during the backup process, BR tries to resolve the lock. A small number of these errors do not affect the correctness of the backup. -#### Backup failure +### Backup failure Error message in the log: `log - Error: msg:"Io(Custom { kind: AlreadyExists, error: \"[5_5359_42_123_default.sst] is already exists in /dir/backup_local/\" })"` diff --git a/br/use-br-command-line-tool.md b/br/use-br-command-line-tool.md new file mode 100644 index 000000000000..25fc603e680d --- /dev/null +++ b/br/use-br-command-line-tool.md @@ -0,0 +1,496 @@ +--- +title: Use BR Command-line for Backup and Restoration +summary: Learn how to use the BR command line to backup and restore cluster data. +--- + +# Use BR Command-line for Backup and Restoration + +This document describes how to back up and restore TiDB cluster data using the BR command line. + +Make sure you have read [BR Tool Overview](/br/backup-and-restore-tool.md), especially [Usage Restrictions](/br/backup-and-restore-tool.md#usage-restrictions) and [Best Practices](/br/backup-and-restore-tool.md#best-practices). + +## BR command-line description + +A `br` command consists of sub-commands, options, and parameters. + +* Sub-command: the characters without `-` or `--`. +* Option: the characters that start with `-` or `--`. +* Parameter: the characters that immediately follow behind and are passed to the sub-command or the option. + +This is a complete `br` command: + +{{< copyable "shell-regular" >}} + +```shell +br backup full --pd "${PDIP}:2379" -s "local:///tmp/backup" +``` + +Explanations for the above command are as follows: + +* `backup`: the sub-command of `br`. +* `full`: the sub-command of `backup`. +* `-s` (or `--storage`): the option that specifies the path where the backup files are stored. +* `"local:///tmp/backup"`: the parameter of `-s`. `/tmp/backup` is the path in the local disk where the backed up files of each TiKV node are stored. +* `--pd`: the option that specifies the Placement Driver (PD) service address. +* `"${PDIP}:2379"`: the parameter of `--pd`. + +> **Note:** +> +> - When the `local` storage is used, the backup data are scattered in the local file system of each node. +> +> - It is **not recommended** to back up to a local disk in the production environment because you **have to** manually aggregate these data to complete the data restoration. For more information, see [Restore Cluster Data](#use-br-command-line-to-restore-cluster-data). +> +> - Aggregating these backup data might cause redundancy and bring troubles to operation and maintenance. Even worse, if restoring data without aggregating these data, you can receive a rather confusing error message `SST file not found`. +> +> - It is recommended to mount the NFS disk on each node, or back up to the `S3` object storage. + +### Sub-commands + +A `br` command consists of multiple layers of sub-commands. Currently, BR has the following three sub-commands: + +* `br backup`: used to back up the data of the TiDB cluster. +* `br restore`: used to restore the data of the TiDB cluster. + +Each of the above three sub-commands might still include the following three sub-commands to specify the scope of an operation: + +* `full`: used to back up or restore all the cluster data. +* `db`: used to back up or restore the specified database of the cluster. +* `table`: used to back up or restore a single table in the specified database of the cluster. + +### Common options + +* `--pd`: used for connection, specifying the PD server address. For example, `"${PDIP}:2379"`. +* `-h` (or `--help`): used to get help on all sub-commands. For example, `br backup --help`. +* `-V` (or `--version`): used to check the version of BR. +* `--ca`: specifies the path to the trusted CA certificate in the PEM format. +* `--cert`: specifies the path to the SSL certificate in the PEM format. +* `--key`: specifies the path to the SSL certificate key in the PEM format. +* `--status-addr`: specifies the listening address through which BR provides statistics to Prometheus. + +## Use BR command-line to back up cluster data + +To back up the cluster data, use the `br backup` command. You can add the `full` or `table` sub-command to specify the scope of your backup operation: the whole cluster or a single table. + +### Back up all the cluster data + +To back up all the cluster data, execute the `br backup full` command. To get help on this command, execute `br backup full -h` or `br backup full --help`. + +**Usage example:** + +Back up all the cluster data to the `/tmp/backup` path of each TiKV node and write the `backupmeta` file to this path. + +> **Note:** +> +> + If the backup disk and the service disk are different, it has been tested that online backup reduces QPS of the read-only online service by about 15%-25% in case of full-speed backup. If you want to reduce the impact on QPS, use `--ratelimit` to limit the rate. +> +> + If the backup disk and the service disk are the same, the backup competes with the service for I/O resources. This might decrease the QPS of the read-only online service by more than half. Therefore, it is **highly not recommended** to back up the online service data to the TiKV data disk. + +{{< copyable "shell-regular" >}} + +```shell +br backup full \ + --pd "${PDIP}:2379" \ + --storage "local:///tmp/backup" \ + --ratelimit 120 \ + --log-file backupfull.log +``` + +Explanations for some options in the above command are as follows: + +* `--ratelimit`: specifies the maximum speed at which a backup operation is performed (MiB/s) on each TiKV node. +* `--log-file`: specifies writing the BR log to the `backupfull.log` file. + +A progress bar is displayed in the terminal during the backup. When the progress bar advances to 100%, the backup is complete. Then the BR also checks the backup data to ensure data safety. The progress bar is displayed as follows: + +```shell +br backup full \ + --pd "${PDIP}:2379" \ + --storage "local:///tmp/backup" \ + --ratelimit 120 \ + --log-file backupfull.log +Full Backup <---------/................................................> 17.12%. +``` + +### Back up a database + +To back up a database in the cluster, execute the `br backup db` command. To get help on this command, execute `br backup db -h` or `br backup db --help`. + +**Usage example:** + +Back up the data of the `test` database to the `/tmp/backup` path on each TiKV node and write the `backupmeta` file to this path. + +{{< copyable "shell-regular" >}} + +```shell +br backup db \ + --pd "${PDIP}:2379" \ + --db test \ + --storage "local:///tmp/backup" \ + --ratelimit 120 \ + --log-file backuptable.log +``` + +In the above command, `--db` specifies the name of the database to be backed up. For descriptions of other options, see [Back up all the cluster data](#use-br-command-line-to-back-up-cluster-data). + +A progress bar is displayed in the terminal during the backup. When the progress bar advances to 100%, the backup is complete. Then the BR also checks the backup data to ensure data safety. + +### Back up a table + +To back up the data of a single table in the cluster, execute the `br backup table` command. To get help on this command, execute `br backup table -h` or `br backup table --help`. + +**Usage example:** + +Back up the data of the `test.usertable` table to the `/tmp/backup` path on each TiKV node and write the `backupmeta` file to this path. + +{{< copyable "shell-regular" >}} + +```shell +br backup table \ + --pd "${PDIP}:2379" \ + --db test \ + --table usertable \ + --storage "local:///tmp/backup" \ + --ratelimit 120 \ + --log-file backuptable.log +``` + +The `table` sub-command has two options: + +* `--db`: specifies the database name +* `--table`: specifies the table name. + +For descriptions of other options, see [Back up all cluster data](#use-br-command-line-to-back-up-cluster-data). + +A progress bar is displayed in the terminal during the backup operation. When the progress bar advances to 100%, the backup is complete. Then the BR also checks the backup data to ensure data safety. + +### Back up with table filter + +To back up multiple tables with more complex criteria, execute the `br backup full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. + +**Usage example:** + +The following command backs up the data of all tables in the form `db*.tbl*` to the `/tmp/backup` path on each TiKV node and writes the `backupmeta` file to this path. + +{{< copyable "shell-regular" >}} + +```shell +br backup full \ + --pd "${PDIP}:2379" \ + --filter 'db*.tbl*' \ + --storage "local:///tmp/backup" \ + --ratelimit 120 \ + --log-file backupfull.log +``` + +### Back up data to Amazon S3 backend + +If you back up the data to the Amazon S3 backend, instead of `local` storage, you need to specify the S3 storage path in the `storage` sub-command, and allow the BR node and the TiKV node to access Amazon S3. + +You can refer to the [AWS Official Document](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html) to create an S3 `Bucket` in the specified `Region`. You can also refer to another [AWS Official Document](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-folder.html) to create a `Folder` in the `Bucket`. + +> **Note:** +> +> To complete one backup, TiKV and BR usually require the minimum privileges of `s3:ListBucket`, `s3:PutObject`, and `s3:AbortMultipartUpload`. + +Pass `SecretKey` and `AccessKey` of the account that has privilege to access the S3 backend to the BR node. Here `SecretKey` and `AccessKey` are passed as environment variables. Then pass the privilege to the TiKV node through BR. + +{{< copyable "shell-regular" >}} + +```shell +export AWS_ACCESS_KEY_ID=${AccessKey} +export AWS_SECRET_ACCESS_KEY=${SecretKey} +``` + +When backing up using BR, explicitly specify the parameters `--s3.region` and `--send-credentials-to-tikv`. `--s3.region` indicates the region where S3 is located, and `--send-credentials-to-tikv` means passing the privilege to access S3 to the TiKV node. + +{{< copyable "shell-regular" >}} + +```shell +br backup full \ + --pd "${PDIP}:2379" \ + --storage "s3://${Bucket}/${Folder}" \ + --s3.region "${region}" \ + --send-credentials-to-tikv=true \ + --log-file backuptable.log +``` + +### Back up incremental data + +If you want to back up incrementally, you only need to specify the **last backup timestamp** `--lastbackupts`. + +The incremental backup has two limitations: + +- The incremental backup needs to be under a different path from the previous full backup. +- GC (Garbage Collection) safepoint must be before the `lastbackupts`. + +To back up the incremental data between `(LAST_BACKUP_TS, current PD timestamp]`, execute the following command: + +{{< copyable "shell-regular" >}} + +```shell +br backup full\ + --pd ${PDIP}:2379 \ + -s local:///home/tidb/backupdata/incr \ + --lastbackupts ${LAST_BACKUP_TS} +``` + +To get the timestamp of the last backup, execute the `validate` command. For example: + +{{< copyable "shell-regular" >}} + +```shell +LAST_BACKUP_TS=`br validate decode --field="end-version" -s local:///home/tidb/backupdata | tail -n1` +``` + +In the above example, for the incremental backup data, BR records the data changes and the DDL operations during `(LAST_BACKUP_TS, current PD timestamp]`. When restoring data, BR first restores DDL operations and then the data. + +### Point-in-time recovery (experimental feature) + +Point-in-time recovery (PITR) allows you to restore data to a point in time of your choice. + +An example scenario would be to take a full backup every day and take incremental backups every 6 hours and then use TiCDC for PITR. Assume that on one day, the full backup was performed at 00:00 and the first incremental backup was performed at 06:00. If you want to restore the database to the state of 07:16, you can first restore the full backup (taken at 00:00) and the incremental backup (taken at 06:00), and then restore TiCDC logs that fill in the gap between 06:00 and 07:16. + +To peform the PITR, you can take the following steps: + +1. Restore a full backup using `br restore full`. +2. (optional) Restore incremental backup(s). +3. Use `br restore cdclog` to restore the transactions that happened after the last incremental backup. The complete command to execute is as follows: + + ```shell + br restore cdclog --storage local:///data/cdclog --start-ts $START_TS --end-ts $END_TS + ``` + + In the command above: + + - `local:///data/cdclog` is the location of the TiCDC logs. This might be on the local filesystem or on the external storage like S3. + - `$START_TS` is the end position of the restore from the last restored backup (either a full backup or an incremental backup). + - `$END_TS` is the point to which you want to restore your data. + +### Back up Raw KV (experimental feature) + +> **Warning:** +> +> This feature is experimental and not thoroughly tested. It is highly **not recommended** to use this feature in the production environment. + +In some scenarios, TiKV might run independently of TiDB. Given that, BR also supports bypassing the TiDB layer and backing up data in TiKV. + +For example, you can execute the following command to back up all keys between `[0x31, 0x3130303030303030)` in the default CF to `$BACKUP_DIR`: + +{{< copyable "shell-regular" >}} + +```shell +br backup raw --pd $PD_ADDR \ + -s "local://$BACKUP_DIR" \ + --start 31 \ + --end 3130303030303030 \ + --format hex \ + --cf default +``` + +Here, the parameters of `--start` and `--end` are decoded using the method specified by `--format` before being sent to TiKV. Currently, the following methods are available: + +- "raw": The input string is directly encoded as a key in binary format. +- "hex": The default encoding method. The input string is treated as a hexadecimal number. +- "escape": First escape the input string, and then encode it into binary format. + +## Use BR command-line to restore cluster data + +To restore the cluster data, use the `br restore` command. You can add the `full`, `db` or `table` sub-command to specify the scope of your restoration: the whole cluster, a database or a single table. + +> **Note:** +> +> If you use the local storage, you **must** copy all back up SST files to every TiKV node in the path specified by `--storage`. +> +> Even if each TiKV node eventually only need to read a part of the all SST files, they all need full access to the complete archive because: +> +> - Data are replicated into multiple peers. When ingesting SSTs, these files have to be present on *all* peers. This is unlike back up where reading from a single node is enough. +> - Where each peer is scattered to during restore is random. We don't know in advance which node will read which file. +> +> These can be avoided using shared storage, for example mounting an NFS on the local path, or using S3. With network storage, every node can automatically read every SST file, so these caveats no longer apply. + +### Restore all the backup data + +To restore all the backup data to the cluster, execute the `br restore full` command. To get help on this command, execute `br restore full -h` or `br restore full --help`. + +**Usage example:** + +Restore all the backup data in the `/tmp/backup` path to the cluster. + +{{< copyable "shell-regular" >}} + +```shell +br restore full \ + --pd "${PDIP}:2379" \ + --storage "local:///tmp/backup" \ + --ratelimit 128 \ + --log-file restorefull.log +``` + +Explanations for some options in the above command are as follows: + +* `--ratelimit`: specifies the maximum speed at which a restoration operation is performed (MiB/s) on each TiKV node. +* `--log-file`: specifies writing the BR log to the `restorefull.log` file. + +A progress bar is displayed in the terminal during the restoration. When the progress bar advances to 100%, the restoration is complete. Then the BR also checks the backup data to ensure data safety. + +```shell +br restore full \ + --pd "${PDIP}:2379" \ + --storage "local:///tmp/backup" \ + --log-file restorefull.log +Full Restore <---------/...............................................> 17.12%. +``` + +### Restore a database + +To restore a database to the cluster, execute the `br restore db` command. To get help on this command, execute `br restore db -h` or `br restore db --help`. + +**Usage example:** + +Restore a database backed up in the `/tmp/backup` path to the cluster. + +{{< copyable "shell-regular" >}} + +```shell +br restore db \ + --pd "${PDIP}:2379" \ + --db "test" \ + --storage "local:///tmp/backup" \ + --log-file restorefull.log +``` + +In the above command, `--db` specifies the name of the database to be restored. For descriptions of other options, see [Restore all backup data](#restore-all-the-backup-data)). + +### Restore a table + +To restore a single table to the cluster, execute the `br restore table` command. To get help on this command, execute `br restore table -h` or `br restore table --help`. + +**Usage example:** + +Restore a table backed up in the `/tmp/backup` path to the cluster. + +{{< copyable "shell-regular" >}} + +```shell +br restore table \ + --pd "${PDIP}:2379" \ + --db "test" \ + --table "usertable" \ + --storage "local:///tmp/backup" \ + --log-file restorefull.log +``` + +In the above command, `--table` specifies the name of the table to be restored. For descriptions of other options, see [Restore all backup data](#restore-all-the-backup-data) and [Restore a database](#restore-a-database). + +### Restore with table filter + +To restore multiple tables with more complex criteria, execute the `br restore full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. + +**Usage example:** + +The following command restores a subset of tables backed up in the `/tmp/backup` path to the cluster. + +{{< copyable "shell-regular" >}} + +```shell +br restore full \ + --pd "${PDIP}:2379" \ + --filter 'db*.tbl*' \ + --storage "local:///tmp/backup" \ + --log-file restorefull.log +``` + +### Restore data from Amazon S3 backend + +If you restore data from the Amazon S3 backend, instead of `local` storage, you need to specify the S3 storage path in the `storage` sub-command, and allow the BR node and the TiKV node to access Amazon S3. + +> **Note:** +> +> To complete one restore, TiKV and BR usually require the minimum privileges of `s3:ListBucket` and `s3:GetObject`. + +Pass `SecretKey` and `AccessKey` of the account that has privilege to access the S3 backend to the BR node. Here `SecretKey` and `AccessKey` are passed as environment variables. Then pass the privilege to the TiKV node through BR. + +{{< copyable "shell-regular" >}} + +```shell +export AWS_ACCESS_KEY_ID=${AccessKey} +export AWS_SECRET_ACCESS_KEY=${SecretKey} +``` + +When restoring data using BR, explicitly specify the parameters `--s3.region` and `--send-credentials-to-tikv`. `--s3.region` indicates the region where S3 is located, and `--send-credentials-to-tikv` means passing the privilege to access S3 to the TiKV node. + +`Bucket` and `Folder` in the `--storage` parameter represent the S3 bucket and the folder where the data to be restored is located. + +{{< copyable "shell-regular" >}} + +```shell +br restore full \ + --pd "${PDIP}:2379" \ + --storage "s3://${Bucket}/${Folder}" \ + --s3.region "${region}" \ + --send-credentials-to-tikv=true \ + --log-file restorefull.log +``` + +In the above command, `--table` specifies the name of the table to be restored. For descriptions of other options, see [Restore a database](#restore-a-database). + +### Restore incremental data + +Restoring incremental data is similar to [restoring full data using BR](#restore-all-the-backup-data). Note that when restoring incremental data, make sure that all the data backed up before `last backup ts` has been restored to the target cluster. + +### Restore Raw KV (experimental feature) + +> **Warning:** +> +> This feature is in the experiment, without being thoroughly tested. It is highly **not recommended** to use this feature in the production environment. + +Similar to [backing up Raw KV](#back-up-raw-kv-experimental-feature), you can execute the following command to restore Raw KV: + +{{< copyable "shell-regular" >}} + +```shell +br restore raw --pd $PD_ADDR \ + -s "local://$BACKUP_DIR" \ + --start 31 \ + --end 3130303030303030 \ + --format hex \ + --cf default +``` + +In the above example, all the backed up keys in the range `[0x31, 0x3130303030303030)` are restored to the TiKV cluster. The coding methods of these keys are identical to that of [keys during the backup process](#back-up-raw-kv-experimental-feature) + +### Online restore (experimental feature) + +> **Warning:** +> +> This feature is in the experiment, without being thoroughly tested. It also relies on the unstable `Placement Rules` feature of PD. It is highly **not recommended** to use this feature in the production environment. + +During data restoration, writing too much data affects the performance of the online cluster. To avoid this effect as much as possible, BR supports [Placement rules](/configure-placement-rules.md) to isolate resources. In this case, downloading and importing SST are only performed on a few specified nodes (or "restore nodes" for short). To complete the online restore, take the following steps. + +1. Configure PD, and start Placement rules: + + {{< copyable "shell-regular" >}} + + ```shell + echo "config set enable-placement-rules true" | pd-ctl + ``` + +2. Edit the configuration file of the "restore node" in TiKV, and specify "restore" to the `server` configuration item: + + {{< copyable "" >}} + + ``` + [server] + labels = { exclusive = "restore" } + ``` + +3. Start TiKV of the "restore node" and restore the backed up files using BR. Compared with the offline restore, you only need to add the `--online` flag: + + {{< copyable "shell-regular" >}} + + ``` + br restore full \ + -s "local://$BACKUP_DIR" \ + --pd $PD_ADDR \ + --online + ``` diff --git a/certificate-authentication.md b/certificate-authentication.md index b4cc5af9be5f..4563ee0a6c2d 100644 --- a/certificate-authentication.md +++ b/certificate-authentication.md @@ -55,9 +55,9 @@ It is recommended that you use [OpenSSL](https://www.openssl.org/) to create key ```bash Country Name (2 letter code) [AU]:US State or Province Name (full name) [Some-State]:California - Locality Name (eg, city) []:San Francisco - Organization Name (eg, company) [Internet Widgits Pty Ltd]:PingCAP Inc. - Organizational Unit Name (eg, section) []:TiDB + Locality Name (e.g. city) []:San Francisco + Organization Name (e.g. company) [Internet Widgits Pty Ltd]:PingCAP Inc. + Organizational Unit Name (e.g. section) []:TiDB Common Name (e.g. server FQDN or YOUR name) []:TiDB admin Email Address []:s@pingcap.com ``` @@ -83,9 +83,9 @@ It is recommended that you use [OpenSSL](https://www.openssl.org/) to create key ```bash Country Name (2 letter code) [AU]:US State or Province Name (full name) [Some-State]:California - Locality Name (eg, city) []:San Francisco - Organization Name (eg, company) [Internet Widgits Pty Ltd]:PingCAP Inc. - Organizational Unit Name (eg, section) []:TiKV + Locality Name (e.g. city) []:San Francisco + Organization Name (e.g. company) [Internet Widgits Pty Ltd]:PingCAP Inc. + Organizational Unit Name (e.g. section) []:TiKV Common Name (e.g. server FQDN or YOUR name) []:TiKV Test Server Email Address []:k@pingcap.com @@ -148,9 +148,9 @@ After generating the server key and certificate, you need to generate the key an ```bash Country Name (2 letter code) [AU]:US State or Province Name (full name) [Some-State]:California - Locality Name (eg, city) []:San Francisco - Organization Name (eg, company) [Internet Widgits Pty Ltd]:PingCAP Inc. - Organizational Unit Name (eg, section) []:TiDB + Locality Name (e.g. city) []:San Francisco + Organization Name (e.g. company) [Internet Widgits Pty Ltd]:PingCAP Inc. + Organizational Unit Name (e.g. section) []:TiDB Common Name (e.g. server FQDN or YOUR name) []:tpch-user1 Email Address []:zz@pingcap.com @@ -302,7 +302,7 @@ The user certificate information can be specified by `require subject`, `require create user 'u1'@'%' require san 'DNS:d1,URI:spiffe://example.org/myservice1,URI:spiffe://example.org/myservice2' ``` - The above configuration only allows the `u1` user to log in to TiDB using the certificate with the URI item `spiffe://example.org/myservice1` or `spiffe://example.org/myservice2` and the DNS item `d1`. + The above configuration only allows the `u1` user to log in to TiDB using the certificate with the URI item `spiffe://example.org/myservice1` or `spiffe://example.org/myservice2` and the DNS item `d1`. + `require cipher`: Checks the cipher method supported by the client. Use the following statement to check the list of supported cipher methods: diff --git a/character-set-and-collation.md b/character-set-and-collation.md index 8b90a1317434..96aa2faccc6b 100644 --- a/character-set-and-collation.md +++ b/character-set-and-collation.md @@ -109,7 +109,7 @@ SHOW COLLATION WHERE Charset = 'utf8mb4'; | utf8mb4_general_ci | utf8mb4 | 45 | | Yes | 1 | | utf8mb4_unicode_ci | utf8mb4 | 224 | | Yes | 1 | +--------------------+---------+------+---------+----------+---------+ -2 rows in set (0.00 sec) +3 rows in set (0.00 sec) ``` ## `utf8` and `utf8mb4` in TiDB @@ -173,7 +173,7 @@ Different databases can use different character sets and collations. Use the `ch {{< copyable "sql" >}} ```sql -CREATE SCHEMA test1 CHARACTER SET utf8mb4 COLLATE uft8mb4_general_ci; +CREATE SCHEMA test1 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; ``` ```sql @@ -200,7 +200,7 @@ SELECT @@character_set_database, @@collation_database; +--------------------------|----------------------+ | @@character_set_database | @@collation_database | +--------------------------|----------------------+ -| utf8mb4 | uft8mb4_general_ci | +| utf8mb4 | utf8mb4_general_ci | +--------------------------|----------------------+ 1 row in set (0.00 sec) ``` diff --git a/check-before-deployment.md b/check-before-deployment.md index 6cb8f9cfb51f..3fa071c58a27 100644 --- a/check-before-deployment.md +++ b/check-before-deployment.md @@ -189,6 +189,22 @@ To check whether the NTP service is installed and whether it synchronizes with t Active: active (running) since 一 2017-12-18 13:13:19 CST; 3s ago ``` + - If it returns `Unit ntpd.service could not be found.`, then try the following command to see whether your system is configured to use `chronyd` instead of `ntpd` to perform clock synchronization with NTP: + + {{< copyable "shell-regular" >}} + + ```bash + sudo systemctl status cronyd.service + ``` + + ``` + chronyd.service - NTP client/server + Loaded: loaded (/usr/lib/systemd/system/chronyd.service; enabled; vendor preset: enabled) + Active: active (running) since Mon 2021-04-05 09:55:29 EDT; 3 days ago + ``` + + If your system is configured to use `chronyd`, proceed to step 3. + 2. Run the `ntpstat` command to check whether the NTP service synchronizes with the NTP server. > **Note:** @@ -221,6 +237,48 @@ To check whether the NTP service is installed and whether it synchronizes with t Unable to talk to NTP daemon. Is it running? ``` +3. Run the `chronyc tracking` command to check wheter the Chrony service synchronizes with the NTP server. + + > **Note:** + > + > This only applies to systems that use Chrony instead of NTPd. + + {{< copyable "shell-regular" >}} + + ```bash + chronyc tracking + ``` + + - If the command returns `Leap status : Normal`, the synchronization process is normal. + + ``` + Reference ID : 5EC69F0A (ntp1.time.nl) + Stratum : 2 + Ref time (UTC) : Thu May 20 15:19:08 2021 + System time : 0.000022151 seconds slow of NTP time + Last offset : -0.000041040 seconds + RMS offset : 0.000053422 seconds + Frequency : 2.286 ppm slow + Residual freq : -0.000 ppm + Skew : 0.012 ppm + Root delay : 0.012706812 seconds + Root dispersion : 0.000430042 seconds + Update interval : 1029.8 seconds + Leap status : Normal + ``` + + - If the command returns the following result, an error occurs in the synchronization: + + ``` + Leap status : Not synchronised + ``` + + - If the command returns the following result, the `chronyd` service is not running normally: + + ``` + 506 Cannot talk to daemon + ``` + To make the NTP service start synchronizing as soon as possible, run the following command. Replace `pool.ntp.org` with your NTP server. {{< copyable "shell-regular" >}} @@ -241,13 +299,17 @@ sudo systemctl start ntpd.service && \ sudo systemctl enable ntpd.service ``` -## Check and disable THP +## Check and configure the optimal parameters of the operating system -It is **NOT** recommended to use Transparent Huge Pages (THP) for database applications, because databases often have sparse rather than continuous memory access patterns. If high-level memory fragmentation is serious, a higher latency will occur when THP pages are allocated. If the direct compaction is enabled for THP, the CPU usage will surge. Therefore, it is recommended to disable THP. +For TiDB in the production environment, it is recommended to optimize the operating system configuration in the following ways: -Take the following steps to check whether THP is disabled. If not, follow the steps to disable it: +1. Disable THP (Transparent Huge Pages). The memory access pattern of databases tends to be sparse rather than consecutive. If the high-level memory fragmentation is serious, higher latency will occur when THP pages are allocated. +2. Set the I/O Scheduler of the storage media to `noop`. For the high-speed SSD storage media, the kernel's I/O scheduling operations can cause performance loss. After the Scheduler is set to `noop`, the performance is better because the kernel directly sends I/O requests to the hardware without other operations. Also, the noop Scheduler is better applicable. +3. Choose the `performance` mode for the cpufrequ module which controls the CPU frequency. The performance is maximized when the CPU frequency is fixed at its highest supported operating frequency without dynamic adjustment. -1. Execute the following command to see whether THP is enabled or disabled. If `[always] madvise never` is output, THP is enabled. +Take the following steps to check the current operating system configuration and configure optimal parameters: + +1. Execute the following command to see whether THP is enabled or disabled: {{< copyable "shell-regular" >}} @@ -259,75 +321,235 @@ Take the following steps to check whether THP is disabled. If not, follow the st [always] madvise never ``` -2. Execute the `grubby` command to see the default kernel version: - > **Note:** > - > Before you execute `grubby`, you should first install the `grubby` package. + > If `[always] madvise never` is output, THP is enabled. You need to disable it. + +2. Execute the following command to see the I/O Scheduler of the disk where the data directory is located. Assume that you create data directories on both sdb and sdc disks: {{< copyable "shell-regular" >}} ```bash - grubby --default-kernel + cat /sys/block/sd[bc]/queue/scheduler ``` - ```bash - /boot/vmlinuz-3.10.0-957.el7.x86_64 ``` + noop [deadline] cfq + noop [deadline] cfq + ``` + + > **Note:** + > + > If `noop [deadline] cfq` is output, the I/O Scheduler for the disk is in the `deadline` mode. You need to change it to `noop`. -3. Execute `grubby --update-kernel` to modify the kernel configuration: +3. Execute the following command to see the `ID_SERIAL` of the disk: {{< copyable "shell-regular" >}} ```bash - grubby --args="transparent_hugepage=never" --update-kernel /boot/vmlinuz-3.10.0-957.el7.x86_64 + udevadm info --name=/dev/sdb | grep ID_SERIAL + ``` + + ``` + E: ID_SERIAL=36d0946606d79f90025f3e09a0c1f9e81 + E: ID_SERIAL_SHORT=6d0946606d79f90025f3e09a0c1f9e81 ``` > **Note:** > - > `--update-kernel` is followed by the actual default kernel version. + > If multiple disks are allocated with data directories, you need to execute the above command several times to record the `ID_SERIAL` of each disk. -4. Execute `grubby --info` to see the modified default kernel configuration: +4. Execute the following command to see the power policy of the cpufreq module: {{< copyable "shell-regular" >}} ```bash - grubby --info /boot/vmlinuz-3.10.0-957.el7.x86_64 + cpupower frequency-info --policy + ``` + + ``` + analyzing CPU 0: + current policy: frequency should be within 1.20 GHz and 3.10 GHz. + The governor "powersave" may decide which speed to use within this range. ``` > **Note:** > - > `--info` is followed by the actual default kernel version. + > If `The governor "powersave"` is output, the power policy of the cpufreq module is `powersave`. You need to modify it to `performance`. If you use a virtual machine or a cloud host, the output is usually `Unable to determine current policy`, and you do not need to change anything. - ```bash - index=0 - kernel=/boot/vmlinuz-3.10.0-957.el7.x86_64 - args="ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet LANG=en_US.UTF-8 transparent_hugepage=never" - root=/dev/mapper/centos-root - initrd=/boot/initramfs-3.10.0-957.el7.x86_64.img - title=CentOS Linux (3.10.0-957.el7.x86_64) 7 (Core) - ``` +5. Configure optimal parameters of the operating system: -5. Execute `reboot` to reboot the machine or modify the current kernel configuration: + + Method one: Use tuned (Recommended) - - To reboot the machine for the configuration change above to take effect, execute `reboot`: + 1. Execute the `tuned-adm list` command to see the tuned profile of the current operating system: - {{< copyable "shell-regular" >}} + {{< copyable "shell-regular" >}} - ```bash - reboot - ``` + ```bash + tuned-adm list + ``` - - If you do not want to reboot the machine, modify the current kernel configuration to take effect immediately: + ``` + Available profiles: + - balanced - General non-specialized tuned profile + - desktop - Optimize for the desktop use-case + - hpc-compute - Optimize for HPC compute workloads + - latency-performance - Optimize for deterministic performance at the cost of increased power consumption + - network-latency - Optimize for deterministic performance at the cost of increased power consumption, focused on low latency network performance + - network-throughput - Optimize for streaming network throughput, generally only necessary on older CPUs or 40G+ networks + - powersave - Optimize for low power consumption + - throughput-performance - Broadly applicable tuning that provides excellent performance across a variety of common server workloads + - virtual-guest - Optimize for running inside a virtual guest + - virtual-host - Optimize for running KVM guests + Current active profile: balanced + ``` - {{< copyable "shell-regular" >}} + The output `Current active profile: balanced` means that the tuned profile of the current operating system is `balanced`. It is recommended to optimize the configuration of the operating system based on the current profile. - ```bash - echo never > /sys/kernel/mm/transparent_hugepage/enabled - echo never > /sys/kernel/mm/transparent_hugepage/defrag - ``` + 2. Create a new tuned profile: + + {{< copyable "shell-regular" >}} + + ```bash + mkdir /etc/tuned/balanced-tidb-optimal/ + vi /etc/tuned/balanced-tidb-optimal/tuned.conf + ``` + + ``` + [main] + include=balanced + + [cpu] + governor=performance + + [vm] + transparent_hugepages=never + + [disk] + devices_udev_regex=(ID_SERIAL=36d0946606d79f90025f3e09a0c1fc035)|(ID_SERIAL=36d0946606d79f90025f3e09a0c1f9e81) + elevator=noop + ``` + + The output `include=balanced` means to add the optimization configuration of the operating system to the current `balanced` profile. + + 3. Apply the new tuned profile: + + {{< copyable "shell-regular" >}} + + ```bash + tuned-adm profile balanced-tidb-optimal + ``` + + + Method two: Configure using scripts. Skip this method if you already use method one. + + 1. Execute the `grubby` command to see the default kernel version: -6. Check the default kernel configuration that has taken effect after reboot or modification. If `always madvise [never]` is output, THP is disabled. + > **Note:** + > + > Install the `grubby` package first before you execute `grubby`. + + {{< copyable "shell-regular" >}} + + ```bash + grubby --default-kernel + ``` + + ```bash + /boot/vmlinuz-3.10.0-957.el7.x86_64 + ``` + + 2. Execute `grubby --update-kernel` to modify the kernel configuration: + + {{< copyable "shell-regular" >}} + + ```bash + grubby --args="transparent_hugepage=never" --update-kernel /boot/vmlinuz-3.10.0-957.el7.x86_64 + ``` + + > **Note:** + > + > `--update-kernel` is followed by the actual default kernel version. + + 3. Execute `grubby --info` to see the modified default kernel configuration: + + {{< copyable "shell-regular" >}} + + ```bash + grubby --info /boot/vmlinuz-3.10.0-957.el7.x86_64 + ``` + + > **Note:** + > + > `--info` is followed by the actual default kernel version. + + ``` + index=0 + kernel=/boot/vmlinuz-3.10.0-957.el7.x86_64 + args="ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet LANG=en_US.UTF-8 transparent_hugepage=never" + root=/dev/mapper/centos-root + initrd=/boot/initramfs-3.10.0-957.el7.x86_64.img + title=CentOS Linux (3.10.0-957.el7.x86_64) 7 (Core) + ``` + + 4. Modify the current kernel configuration to immediately disable THP: + + {{< copyable "shell-regular" >}} + + ```bash + echo never > /sys/kernel/mm/transparent_hugepage/enabled + echo never > /sys/kernel/mm/transparent_hugepage/defrag + ``` + + 5. Configure the I/O Scheduler in the udev script: + + {{< copyable "shell-regular" >}} + + ```bash + vi /etc/udev/rules.d/60-tidb-schedulers.rules + ``` + + ``` + ACTION=="add|change", SUBSYSTEM=="block", ENV{ID_SERIAL}=="36d0946606d79f90025f3e09a0c1fc035", ATTR{queue/scheduler}="noop" + ACTION=="add|change", SUBSYSTEM=="block", ENV{ID_SERIAL}=="36d0946606d79f90025f3e09a0c1f9e81", ATTR{queue/scheduler}="noop" + + ``` + + 6. Apply the udev script: + + {{< copyable "shell-regular" >}} + + ```bash + udevadm control --reload-rules + udevadm trigger --type=devices --action=change + ``` + + 7. Create a service to configure the CPU power policy: + + {{< copyable "shell-regular" >}} + + ```bash + cat >> /etc/systemd/system/cpupower.service << EOF + [Unit] + Description=CPU performance + [Service] + Type=oneshot + ExecStart=/usr/bin/cpupower frequency-set --governor performance + [Install] + WantedBy=multi-user.target + EOF + ``` + + 8. Apply the CPU power policy configuration service: + + {{< copyable "shell-regular" >}} + + ```bash + systemctl daemon-reload + systemctl enable cpupower.service + systemctl start cpupower.service + ``` + +6. Execute the following command to verify the THP status: {{< copyable "shell-regular" >}} @@ -335,50 +557,62 @@ Take the following steps to check whether THP is disabled. If not, follow the st cat /sys/kernel/mm/transparent_hugepage/enabled ``` - ```bash + ``` always madvise [never] ``` -7. If THP is still enabled, use tuned or ktune (two tools used for dynamic kernel debugging) to modify the THP configuration. Take the following steps: +7. Execute the following command to verify the I/O Scheduler of the disk where the data directory is located: - 1. Activate tuned: + {{< copyable "shell-regular" >}} - {{< copyable "shell-regular" >}} + ```bash + cat /sys/block/sd[bc]/queue/scheduler + ``` - ```bash - tuned-adm active - ``` + ``` + [noop] deadline cfq + [noop] deadline cfq + ``` - ```bash - Current active profile: virtual-guest - ``` +8. Execute the following command to see the power policy of the cpufreq module: - 2. Create a new custom profile: + {{< copyable "shell-regular" >}} - {{< copyable "shell-regular" >}} + ```bash + cpupower frequency-info --policy + ``` - ```bash - mkdir /etc/tuned/virtual-guest-no-thp - vi /etc/tuned/virtual-guest-no-thp/tuned.conf - ``` + ``` + analyzing CPU 0: + current policy: frequency should be within 1.20 GHz and 3.10 GHz. + The governor "performance" may decide which speed to use within this range. + ``` - ```bash - [main] - include=virtual-guest +9. Execute the following commands to modify the `sysctl` parameters: - [vm] - transparent_hugepages=never - ``` + {{< copyable "shell-regular" >}} - 3. Apply the new custom profile: + ```bash + echo "fs.file-max = 1000000">> /etc/sysctl.conf + echo "net.core.somaxconn = 32768">> /etc/sysctl.conf + echo "net.ipv4.tcp_tw_recycle = 0">> /etc/sysctl.conf + echo "net.ipv4.tcp_syncookies = 0">> /etc/sysctl.conf + echo "vm.overcommit_memory = 1">> /etc/sysctl.conf + sysctl -p + ``` - {{< copyable "shell-regular" >}} +10. Execute the following command to configure the user's `limits.conf` file: - ```bash - tuned-adm profile virtual-guest-no-thp - ``` + {{< copyable "shell-regular" >}} - 4. Check again whether THP is disabled. + ```bash + cat << EOF >>/etc/security/limits.conf + tidb soft nofile 1000000 + tidb hard nofile 1000000 + tidb soft stack 32768 + tidb hard stack 32768 + EOF + ``` ## Manually configure the SSH mutual trust and sudo without password @@ -405,11 +639,12 @@ This section describes how to manually configure the SSH mutual trust and sudo w tidb ALL=(ALL) NOPASSWD: ALL ``` -3. Use the `tidb` user to log in to the control machine, and run the following command. Replace `10.0.1.1` with the IP of your target machine, and enter the `tidb` user password of the target machine as prompted. After the command is executed, SSH mutual trust is already created. This applies to other machines as well. +3. Use the `tidb` user to log in to the control machine, and run the following command. Replace `10.0.1.1` with the IP of your target machine, and enter the `tidb` user password of the target machine as prompted. After the command is executed, SSH mutual trust is already created. This applies to other machines as well. Newly created `tidb` users do not have the `.ssh` directory. To create such a directory, execute the command that generates the RSA key. To deploy TiDB components on the control machine, configure mutual trust for the control machine and the control machine itself. {{< copyable "shell-regular" >}} ```bash + ssh-keygen -t rsa ssh-copy-id -i ~/.ssh/id_rsa.pub 10.0.1.1 ``` @@ -442,7 +677,7 @@ This section describes how to manually configure the SSH mutual trust and sudo w This section describes how to install the NUMA tool. In online environments, because the hardware configuration is usually higher than required, to better plan the hardware resources, multiple instances of TiDB or TiKV can be deployed on a single machine. In such scenarios, you can use NUMA tools to prevent the competition for CPU resources which might cause reduced performance. > **Note:** -> +> > - Binding cores using NUMA is a method to isolate CPU resources and is suitable for deploying multiple instances on highly configured physical machines. > - After completing deployment using `tiup cluster deploy`, you can use the `exec` command to perform cluster level management operations. diff --git a/choose-index.md b/choose-index.md index 26280ff192b7..b00824139bba 100644 --- a/choose-index.md +++ b/choose-index.md @@ -41,7 +41,7 @@ Skyline-pruning is a heuristic filtering rule for indexes. To judge an index, th - How many access conditions are covered by the indexed columns. An “access condition” is a where condition that can be converted to a column range. And the more access conditions an indexed column set covers, the better it is in this dimension. -For these three dimensions, if an index named idx_a is not worse than the index named idx_b in all three dimensions and one of the dimensions is better than Idx_b, then idx_a is preferred. +For these three dimensions, if an index named idx_a is not worse than the index named idx_b in all three dimensions and one of the dimensions is better than idx_b, then idx_a is preferred. ### Selection based on cost estimation @@ -63,7 +63,7 @@ According to these factors and the cost model, the optimizer selects an index wi 2. Statistics are accurate, and reading from TiFlash is faster, but why does the optimizer choose to read from TiKV? At present, the cost model of distinguishing TiFlash from TiKV is still rough. You can decrease the value of `tidb_opt_seek_factor` parameter, then the optimizer prefers to choose TiFlash. - + 3. The statistics are accurate. Index A needs to retrieve rows from tables, but it actually executes faster than Index B that does not retrieve rows from tables. Why does the optimizer choose Index B? In this case, the cost estimation may be too large for retrieving rows from tables. You can decrease the value of `tidb_opt_network_factor` parameter to reduce the cost of retrieving rows from tables. diff --git a/clustered-indexes.md b/clustered-indexes.md index d0874b18657c..aeec47817d7c 100644 --- a/clustered-indexes.md +++ b/clustered-indexes.md @@ -1,188 +1,226 @@ --- title: Clustered Indexes -summary: Learn how clustered indexes apply to TiDB. +summary: Learn the concept, user scenarios, usages, limitations, and compatibility of clustered indexes. --- # Clustered Indexes -The clustered index is an experimental feature introduced in TiDB 5.0.0-rc. This document provides multiple examples to explain how this feature makes a difference to the query performance of TiDB. To enable this feature and see the detailed operation guide, see [tidb_enable_clustered_index](/system-variables.md#tidb_enable_clustered_index). +TiDB supports the clustered index feature since v5.0. This feature controls how data is stored in tables containing primary keys. It provides TiDB the ability to organize tables in a way that can improve the performance of certain queries. -Clustered indexes provide TiDB the ability to organize tables in a way that can improve the performance of certain queries. The term _clustered_ in this context refers to the _organization of how data is stored_ and not _a group of database servers working together_. Some database management systems refer to clustered indexes as _index-organized tables_ (IOT). +The term _clustered_ in this context refers to the _organization of how data is stored_ and not _a group of database servers working together_. Some database management systems refer to clustered indexes as _index-organized tables_ (IOT). -TiDB supports clustering only by a table's `PRIMARY KEY`. With clustered indexes enabled, the terms _the_ `PRIMARY KEY` and _the clustered index_ might be used interchangeably. `PRIMARY KEY` refers to the constraint (a logical property), and clustered index describes the physical implementation of how the data is stored. +Currently, tables containing primary keys in TiDB are divided into the following two categories: -## Limited support before TiDB v5.0 +- `NONCLUSTERED`: The primary key of the table is non-clustered index. In tables with non-clustered indexes, the keys for row data consist of internal `_tidb_rowid` implicitly assigned by TiDB. Because primary keys are essentially unique indexes, tables with non-clustered indexes need at least two key-value pairs to store a row, which are: + - `_tidb_rowid` (key) - row data (value) + - Primary key data (key) - `_tidb_rowid` (value) +- `CLUSTERED`: The primary key of the table is clustered index. In tables with clustered indexes, the keys for row data consist of primary key data given by the user. Therefore, tables with clustered indexes need only one key-value pair to store a row, which is: + - Primary key data (key) - row data (value) -Before v5.0, TiDB has only limited support for clustered indexes, provided the following criteria are true: +> **Note:** +> +> TiDB supports clustering only by a table's `PRIMARY KEY`. With clustered indexes enabled, the terms _the_ `PRIMARY KEY` and _the clustered index_ might be used interchangeably. `PRIMARY KEY` refers to the constraint (a logical property), and clustered index describes the physical implementation of how the data is stored. -- The table contains a `PRIMARY KEY` -- The `PRIMARY KEY` is an `INTEGER` or `BIGINT` -- The `PRIMARY KEY` consists of only one column +## User scenarios -When any of these criteria are not met, TiDB will create a hidden 64-bit `handle` value to organize the table. Querying table rows by a clustered index is more efficient than by a non-clustered index because the query can be completed in a single step. In the following `EXPLAIN` outputs, a table that supports clustered indexes is compared with one that does not: +Compared to tables with non-clustered indexes, tables with clustered indexes offer greater performance and throughput advantages in the following scenarios: + ++ When data is inserted, the clustered index reduces one write of the index data from the network. ++ When a query with an equivalent condition only involves the primary key, the clustered index reduces one read of index data from the network. ++ When a query with a range condition only involves the primary key, the clustered index reduces multiple reads of index data from the network. ++ When a query with an equivalent or range condition only involves the primary key prefix, the clustered index reduces multiple reads of index data from the network. + +On the other hand, tables with clustered indexes have certain disadvantages. See the following: + +- There might be write hotspot issues when inserting a large number of primary keys with close values. +- The table data takes up more storage space if the data type of the primary key is larger than 64 bits, especially when there are multiple secondary indexes. + +## Usages + +## Create a table with clustered indexes + +Since TiDB v5.0, you can add non-reserved keywords `CLUSTERED` or `NONCLUSTERED` after `PRIMARY KEY` in a `CREATE TABLE` statement to specify whether the table's primary key is a clustered index. For example: ```sql -CREATE TABLE always_clusters_in_all_versions ( - id BIGINT NOT NULL PRIMARY KEY auto_increment, - b CHAR(100), - INDEX(b) -); - -CREATE TABLE does_not_cluster_by_default ( - guid CHAR(32) NOT NULL PRIMARY KEY, - b CHAR(100), - INDEX(b) -); - -INSERT INTO always_clusters_in_all_versions VALUES (1, 'aaa'), (2, 'bbb'); -INSERT INTO does_not_cluster_by_default VALUES ('02dd050a978756da0aff6b1d1d7c8aef', 'aaa'), ('35bfbc09cb3c93d8ef032642521ac042', 'bbb'); - -EXPLAIN SELECT * FROM always_clusters_in_all_versions WHERE id = 1; -EXPLAIN SELECT * FROM does_not_cluster_by_default WHERE guid = '02dd050a978756da0aff6b1d1d7c8aef'; +CREATE TABLE t (a BIGINT PRIMARY KEY CLUSTERED, b VARCHAR(255)); +CREATE TABLE t (a BIGINT PRIMARY KEY NONCLUSTERED, b VARCHAR(255)); +CREATE TABLE t (a BIGINT KEY CLUSTERED, b VARCHAR(255)); +CREATE TABLE t (a BIGINT KEY NONCLUSTERED, b VARCHAR(255)); +CREATE TABLE t (a BIGINT, b VARCHAR(255), PRIMARY KEY(a, b) CLUSTERED); +CREATE TABLE t (a BIGINT, b VARCHAR(255), PRIMARY KEY(a, b) NONCLUSTERED); ``` +Note that keywords `KEY` and `PRIMARY KEY` have the same meaning in the column definition. + +You can also use the [comment syntax](/comment-syntax.md) in TiDB to specify the type of the primary key. For example: + ```sql -Query OK, 0 rows affected (0.09 sec) +CREATE TABLE t (a BIGINT PRIMARY KEY /*T![clustered_index] CLUSTERED */, b VARCHAR(255)); +CREATE TABLE t (a BIGINT PRIMARY KEY /*T![clustered_index] NONCLUSTERED */, b VARCHAR(255)); +CREATE TABLE t (a BIGINT, b VARCHAR(255), PRIMARY KEY(a, b) /*T![clustered_index] CLUSTERED */); +CREATE TABLE t (a BIGINT, b VARCHAR(255), PRIMARY KEY(a, b) /*T![clustered_index] NONCLUSTERED */); +``` -Query OK, 0 rows affected (0.10 sec) +For statements that do not explicitly specify the keyword `CLUSTERED`/`NONCLUSTERED`, the default behavior is controlled by the system variable `@@global.tidb_enable_clustered_index`. Supported values for this variable are as follows: -Records: 2 Duplicates: 0 Warnings: 0 +- `OFF` indicates that primary keys are created as non-clustered indexes by default. +- `ON` indicates that primary keys are created as clustered indexes by default. +- `INT_ONLY` indicates that the behavior is controlled by the configuration item `alter-primary-key`. If `alter-primary-key` is set to `true`, primary keys are created as non-clustered indexes by default. If it is set to `false`, only the primary keys which consist of an integer column are created as clustered indexes. -Records: 2 Duplicates: 0 Warnings: 0 +The default value of `@@global.tidb_enable_clustered_index` is `INT_ONLY`. -+-------------+---------+------+---------------------------------------+---------------+ -| id | estRows | task | access object | operator info | -+-------------+---------+------+---------------------------------------+---------------+ -| Point_Get_1 | 1.00 | root | table:always_clusters_in_all_versions | handle:1 | -+-------------+---------+------+---------------------------------------+---------------+ -1 row in set (0.00 sec) +### Add or drop clustered indexes -+-------------+---------+------+--------------------------------------------------------+---------------+ -| id | estRows | task | access object | operator info | -+-------------+---------+------+--------------------------------------------------------+---------------+ -| Point_Get_1 | 1.00 | root | table:does_not_cluster_by_default, index:PRIMARY(guid) | | -+-------------+---------+------+--------------------------------------------------------+---------------+ -1 row in set (0.00 sec) +TiDB does not support adding or dropping clustered indexes after tables are created. Nor does it support the mutual conversion between clustered indexes and non-clustered indexes. For example: + +```sql +ALTER TABLE t ADD PRIMARY KEY(b, a) CLUSTERED; -- Currently not supported. +ALTER TABLE t DROP PRIMARY KEY; -- If the primary key is a clustered index, then not supported. +ALTER TABLE t DROP INDEX `PRIMARY`; -- If the primary key is a clustered index, then not supported. ``` -The two `EXPLAIN` results above look similar, but in the second example, TiDB must first read the `PRIMARY KEY` index on the `guid` column in order to find the `handle` value. This is more obvious in the following example where the `PRIMARY KEY` value is not in the index on `does_not_cluster_by_default.b`. TiDB must perform an extra lookup on the table rows (`└─TableFullScan_5`) to convert the `handle` value to the `PRIMARY KEY` value of `guid`: +### Add or drop non-clustered indexes + +TiDB supports adding or dropping non-clustered indexes after tables are created. You can explicitly specify the keyword `NONCLUSTERED` or omit it. For example: ```sql -EXPLAIN SELECT id FROM always_clusters_in_all_versions WHERE b = 'aaaa'; -EXPLAIN SELECT guid FROM does_not_cluster_by_default WHERE b = 'aaaa'; +ALTER TABLE t ADD PRIMARY KEY(b, a) NONCLUSTERED; +ALTER TABLE t ADD PRIMARY KEY(b, a); -- If you omit the keyword, the primary key is a non-clustered index by default. +ALTER TABLE t DROP PRIMARY KEY; +ALTER TABLE t DROP INDEX `PRIMARY`; ``` +### Check whether the primary key is a clustered index + +You can check whether the primary key of a table is a clustered index using one of the following methods: + +- Execute the command `SHOW CREATE TABLE`. +- Execute the command `SHOW INDEX FROM`. +- Query the `TIDB_PK_TYPE` column in the system table `information_schema.tables`. + +By running the command `SHOW CREATE TABLE`, you can see whether the attribute of `PRIMARY KEY` is `CLUSTERED` or `NONCLUSTERED`. For example: + ```sql -+--------------------------+---------+-----------+---------------------------------------------------+-------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+--------------------------+---------+-----------+---------------------------------------------------+-------------------------------------------------------+ -| Projection_4 | 0.00 | root | | test.always_clusters_in_all_versions.id | -| └─IndexReader_6 | 0.00 | root | | index:IndexRangeScan_5 | -| └─IndexRangeScan_5 | 0.00 | cop[tikv] | table:always_clusters_in_all_versions, index:b(b) | range:["aaaa","aaaa"], keep order:false, stats:pseudo | -+--------------------------+---------+-----------+---------------------------------------------------+-------------------------------------------------------+ -3 rows in set (0.01 sec) - -+---------------------------+---------+-----------+-----------------------------------+------------------------------------------------+ -| id | estRows | task | access object | operator info | -+---------------------------+---------+-----------+-----------------------------------+------------------------------------------------+ -| Projection_4 | 0.00 | root | | test.does_not_cluster_by_default.guid | -| └─TableReader_7 | 0.00 | root | | data:Selection_6 | -| └─Selection_6 | 0.00 | cop[tikv] | | eq(test.does_not_cluster_by_default.b, "aaaa") | -| └─TableFullScan_5 | 2.00 | cop[tikv] | table:does_not_cluster_by_default | keep order:false, stats:pseudo | -+---------------------------+---------+-----------+-----------------------------------+------------------------------------------------+ -4 rows in set (0.00 sec) +mysql> SHOW CREATE TABLE t; ++-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| t | CREATE TABLE `t` ( + `a` bigint(20) NOT NULL, + `b` varchar(255) DEFAULT NULL, + PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin | ++-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.01 sec) ``` -# Full support since TiDB v5.0 - -Since v5.0, TiDB provides full support for clustered indexes by any `PRIMARY KEY`. The following `EXPLAIN` output shows the previous example with clustered indexes enabled: +By running the command `SHOW INDEX FROM`, you can check whether the result in the column `Clustered` shows `YES` or `NO`. For example: ```sql -SET tidb_enable_clustered_index = 1; -CREATE TABLE will_now_cluster ( - guid CHAR(32) NOT NULL PRIMARY KEY, - b CHAR(100), - INDEX(b) -); - -INSERT INTO will_now_cluster VALUES (1, 'aaa'), (2, 'bbb'); -INSERT INTO will_now_cluster VALUES ('02dd050a978756da0aff6b1d1d7c8aef', 'aaa'), ('35bfbc09cb3c93d8ef032642521ac042', 'bbb'); - -EXPLAIN SELECT * FROM will_now_cluster WHERE guid = '02dd050a978756da0aff6b1d1d7c8aef'; -EXPLAIN SELECT guid FROM will_now_cluster WHERE b = 'aaaa'; +mysql> SHOW INDEX FROM t; ++-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+-----------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | Clustered | ++-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+-----------+ +| t | 0 | PRIMARY | 1 | a | A | 0 | NULL | NULL | | BTREE | | | YES | NULL | YES | ++-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+-----------+ +1 row in set (0.01 sec) ``` +You can also query the column `TIDB_PK_TYPE` in the system table `information_schema.tables` to see whether the result is `CLUSTERED` or `NONCLUSTERED`. For example: + ```sql -Query OK, 0 rows affected (0.00 sec) +mysql> SELECT TIDB_PK_TYPE FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't'; ++--------------+ +| TIDB_PK_TYPE | ++--------------+ +| CLUSTERED | ++--------------+ +1 row in set (0.03 sec) +``` -Query OK, 0 rows affected (0.11 sec) +## Limitations -Query OK, 2 rows affected (0.02 sec) -Records: 2 Duplicates: 0 Warnings: 0 +Currently, there are several different types of limitations for the clustered index feature. See the following: -Query OK, 2 rows affected (0.01 sec) -Records: 2 Duplicates: 0 Warnings: 0 +- Situations that are not supported and not in the support plan: + - Using clustered indexes together with the attribute [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md) is not supported. Also, the attribute [`PRE_SPLIT_REGIONS`](/sql-statements/sql-statement-split-region.md#pre_split_regions) does not take effect on tables with clustered indexes. + - Downgrading tables with clustered indexes is not supported. If you need to downgrade such tables, use logical backup tools to migrate data instead. +- Situations that are not supported yet but in the support plan: + - Adding, dropping, and altering clustered indexes using `ALTER TABLE` statements are not supported. +- Limitations for specific versions: + - In v5.0, using the clustered index feature together with TiDB Binlog is not supported. After TiDB Binlog is enabled, TiDB only allows creating a single integer column as the clustered index of a primary key. TiDB Binlog does not replicate data changes (such as insertion, deletion, and update) on existing tables with clustered indexes to the downstream. If you need to replicate tables with clustered indexes to the downstream, upgrade your cluster to v5.1 or use [TiCDC](/ticdc/ticdc-overview.md) for replication instead. -+-------------+---------+------+-------------------------------------------------------+---------------+ -| id | estRows | task | access object | operator info | -+-------------+---------+------+-------------------------------------------------------+---------------+ -| Point_Get_1 | 1.00 | root | table:will_now_cluster, clustered index:PRIMARY(guid) | | -+-------------+---------+------+-------------------------------------------------------+---------------+ -1 row in set (0.00 sec) +After TiDB Binlog is enabled, if the clustered index you create is not a single integer primary key, TiDB returns the following error: -+--------------------------+---------+-----------+------------------------------------+-------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+--------------------------+---------+-----------+------------------------------------+-------------------------------------------------------+ -| Projection_4 | 10.00 | root | | test.will_now_cluster.guid | -| └─IndexReader_6 | 10.00 | root | | index:IndexRangeScan_5 | -| └─IndexRangeScan_5 | 10.00 | cop[tikv] | table:will_now_cluster, index:b(b) | range:["aaaa","aaaa"], keep order:false, stats:pseudo | -+--------------------------+---------+-----------+------------------------------------+-------------------------------------------------------+ -3 rows in set (0.00 sec) +```sql +mysql> CREATE TABLE t (a VARCHAR(255) PRIMARY KEY CLUSTERED); +ERROR 8200 (HY000): Cannot create clustered index table when the binlog is ON ``` -Clustering by a composite `PRIMARY KEY` is also supported: +If you use clustered indexes together with the attribute `SHARD_ROW_ID_BITS`, TiDB reports the following error: ```sql -SET tidb_enable_clustered_index = 1; -CREATE TABLE composite_primary_key ( - key_a INT NOT NULL, - key_b INT NOT NULL, - b CHAR(100), - PRIMARY KEY (key_a, key_b) -); - -INSERT INTO composite_primary_key VALUES (1, 1, 'aaa'), (2, 2, 'bbb'); -EXPLAIN SELECT * FROM composite_primary_key WHERE key_a = 1 AND key_b = 2; +mysql> CREATE TABLE t (a VARCHAR(255) PRIMARY KEY CLUSTERED) SHARD_ROW_ID_BITS = 3; +ERROR 8200 (HY000): Unsupported shard_row_id_bits for table with primary key as row id ``` -```sql -Query OK, 0 rows affected (0.00 sec) +## Compatibility -Query OK, 0 rows affected (0.09 sec) +### Compatibility with earlier and later TiDB versions -Query OK, 2 rows affected (0.02 sec) -Records: 2 Duplicates: 0 Warnings: 0 +TiDB supports upgrading tables with clustered indexes but not downgrading such tables, which means that data in tables with clustered indexes on a later TiDB version is not available on an earlier one. -+-------------+---------+------+--------------------------------------------------------------------+---------------+ -| id | estRows | task | access object | operator info | -+-------------+---------+------+--------------------------------------------------------------------+---------------+ -| Point_Get_1 | 1.00 | root | table:composite_primary_key, clustered index:PRIMARY(key_a, key_b) | | -+-------------+---------+------+--------------------------------------------------------------------+---------------+ -1 row in set (0.00 sec) -``` +The clustered index feature is partially supported in TiDB v3.0 and v4.0. It is enabled by default when the following requirements are fully met: + +- The table contains a `PRIMARY KEY`. +- The `PRIMARY KEY` consists of only one column. +- The `PRIMARY KEY` is an `INTEGER`. + +Since TiDB v5.0, the clustered index feature is fully supported for all types of primary keys, but the default behavior is consistent with TiDB v3.0 and v4.0. To change the default behavior, you can configure the system variable `@@tidb_enable_clustered_index` to `ON` or `OFF`. For more details, see [Create a table with clustered indexes](#create-a-table-with-clustered-indexes). -This behavior is consistent with MySQL, where the InnoDB storage engine will by default cluster by any `PRIMARY KEY`. +### Compatibility with MySQL -## Storage considerations +TiDB specific comment syntax supports wrapping the keywords `CLUSTERED` and `NONCLUSTERED` in a comment. The result of `SHOW CREATE TABLE` also contains TiDB specific SQL comments. MySQL databases and TiDB databases of an earlier version will ignore these comments. -Because the `PRIMARY KEY` replaces a 64-bit `handle` value as the internal pointer to table rows, using clustered indexes might increase storage requirements. This is particularly impactful on tables that contain many secondary indexes. Consider the following example: +### Compatibility with TiDB ecosystem tools + +The clustered index feature is only compatible with the following ecosystem tools in v5.0 and later versions: + +- Backup and restore tools: BR, Dumpling, and TiDB Lightning. +- Data migration and replication tools: DM and TiCDC. + +However, you cannot convert a table with non-clustered indexes to a table with clustered indexes by backing up and restoring the table using the v5.0 BR tool, and vice versa. + +### Compatibility with other TiDB features + +For a table with a combined primary key or a single non-integer primary key, if you change the primary key from a non-clustered index to a clustered index, the keys of its row data change as well. Therefore, `SPLIT TABLE BY/BETWEEN` statements that are executable in TiDB versions earlier than v5.0 are no longer workable in v5.0 and later versions of TiDB. If you want to split a table with clustered indexes using `SPLIT TABLE BY/BETWEEN`, you need to provide the value of the primary key column, instead of specifying an integer value. See the following example: ```sql -CREATE TABLE t1 ( - guid CHAR(32) NOT NULL PRIMARY KEY, - b BIGINT, - INDEX(b) -); +mysql> create table t (a int, b varchar(255), primary key(a, b) clustered); +Query OK, 0 rows affected (0.01 sec) +mysql> split table t between (0) and (1000000) regions 5; +ERROR 1105 (HY000): Split table region lower value count should be 2 +mysql> split table t by (0), (50000), (100000); +ERROR 1136 (21S01): Column count doesn't match value count at row 0 +mysql> split table t between (0, 'aaa') and (1000000, 'zzz') regions 5; ++--------------------+----------------------+ +| TOTAL_SPLIT_REGION | SCATTER_FINISH_RATIO | ++--------------------+----------------------+ +| 4 | 1 | ++--------------------+----------------------+ +1 row in set (0.00 sec) +mysql> split table t by (0, ''), (50000, ''), (100000, ''); ++--------------------+----------------------+ +| TOTAL_SPLIT_REGION | SCATTER_FINISH_RATIO | ++--------------------+----------------------+ +| 3 | 1 | ++--------------------+----------------------+ +1 row in set (0.01 sec) ``` -Because the pointer to the `guid` is a `char(32)`, each index value for `b` will now require approximately `8 + 32 = 40 bytes` (a `BIGINT` value requires 8 bytes for storage). This compares to `8 + 8 = 16 bytes` for non-clustered tables. The exact storage requirements will differ after compression has been applied. +The attribute [`AUTO_RANDOM`](/auto-random.md) can only be used on clustered indexes. Otherwise, TiDB returns the following error: + +```sql +mysql> create table t (a bigint primary key nonclustered auto_random); +ERROR 8216 (HY000): Invalid auto random: column a is not the integer primary key, or the primary key is nonclustered +``` diff --git a/command-line-flags-for-pd-configuration.md b/command-line-flags-for-pd-configuration.md index 61b6cc79d4fb..bf0425f0304e 100644 --- a/command-line-flags-for-pd-configuration.md +++ b/command-line-flags-for-pd-configuration.md @@ -10,29 +10,29 @@ PD is configurable using command-line flags and environment variables. ## `--advertise-client-urls` -- The advertise URL list for client traffic from outside -- Default: ${client-urls} -- If the client cannot connect to PD through the default listening client URLs, you must manually set the advertise client URLs explicitly. -- For example, the internal IP address of Docker is 172.17.0.1, while the IP address of the host is 192.168.100.113 and the port mapping is set to `-p 2379:2379`. In this case, you can set `--advertise-client-urls` to `"http://192.168.100.113:2379"`. The client can find this service through `"http://192.168.100.113:2379"`. +- The list of advertise URLs for the client to access PD +- Default: `"${client-urls}"` +- In some situations such as in the Docker or NAT network environment, if a client cannot access PD through the default client URLs listened to by PD, you must manually set the advertise client URLs. +- For example, the internal IP address of Docker is `172.17.0.1`, while the IP address of the host is `192.168.100.113` and the port mapping is set to `-p 2379:2379`. In this case, you can set `--advertise-client-urls` to `"http://192.168.100.113:2379"`. The client can find this service through `"http://192.168.100.113:2379"`. ## `--advertise-peer-urls` -- The advertise URL list for peer traffic from outside -- Default: ${peer-urls} -- If the peer cannot connect to PD through the default listening peer URLs, you must manually set the advertise peer URLs explicitly. -- For example, the internal IP address of Docker is 172.17.0.1, while the IP address of the host is 192.168.100.113 and the port mapping is set to `-p 2380:2380`. In this case, you can set `--advertise-peer-urls` to `"http://192.168.100.113:2380"`. The other PD nodes can find this service through `"http://192.168.100.113:2380"`. +- The list of advertise URLs for other PD nodes (peers) to access a PD node +- Default: `"${peer-urls}"` +- In some situations such as in the Docker or NAT network environment, if the other nodes (peers) cannot access the PD node through the default peer URLs listened to by this PD node, you must manually set the advertise peer URLs. +- For example, the internal IP address of Docker is `172.17.0.1`, while the IP address of the host is `192.168.100.113` and the port mapping is set to `-p 2380:2380`. In this case, you can set `--advertise-peer-urls` to `"http://192.168.100.113:2380"`. The other PD nodes can find this service through `"http://192.168.100.113:2380"`. ## `--client-urls` -- The listening URL list for client traffic +- The list of client URLs to be listened to by PD - Default: `"http://127.0.0.1:2379"` -- To deploy a cluster, you must use `--client-urls` to specify the IP address of the current host, such as `"http://192.168.100.113:2379"`. If the cluster runs on Docker, specify the IP address of Docker as `"http://0.0.0.0:2379"`. +- When you deploy a cluster, you must specify the IP address of the current host as `--client-urls` (for example, `"http://192.168.100.113:2379"`). If the cluster runs on Docker, specify the IP address of Docker as `"http://0.0.0.0:2379"`. ## `--peer-urls` -- The listening URL list for peer traffic +- The list of peer URLs to be listened to by a PD node - Default: `"http://127.0.0.1:2380"` -- To deploy a cluster, you must use `--peer-urls` to specify the IP address of the current host, such as `"http://192.168.100.113:2380"`. If the cluster runs on Docker, specify the IP address of Docker as `"http://0.0.0.0:2380"`. +- When you deploy a cluster, you must specify `--peer-urls` as the IP address of the current host, such as `"http://192.168.100.113:2380"`. If the cluster runs on Docker, specify the IP address of Docker as `"http://0.0.0.0:2380"`. ## `--config` @@ -103,5 +103,5 @@ PD is configurable using command-line flags and environment variables. ## `--metrics-addr` -- The address of Prometheus Pushgateway, which does not push data to Promethus by default. +- The address of Prometheus Pushgateway, which does not push data to Prometheus by default. - Default: "" diff --git a/command-line-flags-for-tidb-configuration.md b/command-line-flags-for-tidb-configuration.md index 6db3a5dbfaf2..a6a7e05f50b5 100644 --- a/command-line-flags-for-tidb-configuration.md +++ b/command-line-flags-for-tidb-configuration.md @@ -97,11 +97,11 @@ When you start the TiDB cluster, you can use command-line options or environment ## `--path` -- The path to the data directory for local storage engine like "mocktikv" -- For `--store = tikv`, you must specify the path; for `--store = mocktikv`, the default value is used if you do not specify the path. +- The path to the data directory for local storage engine like "unistore" +- For `--store = tikv`, you must specify the path; for `--store = unistore`, the default value is used if you do not specify the path. - For the distributed storage engine like TiKV, `--path` specifies the actual PD address. Assuming that you deploy the PD server on 192.168.100.113:2379, 192.168.100.114:2379 and 192.168.100.115:2379, the value of `--path` is "192.168.100.113:2379, 192.168.100.114:2379, 192.168.100.115:2379". - Default: "/tmp/tidb" -- You can use `tidb-server --store=mocktikv --path=""` to enable a pure in-memory TiDB. +- You can use `tidb-server --store=unistore --path=""` to enable a pure in-memory TiDB. ## `--tmp-storage-path` @@ -162,8 +162,8 @@ When you start the TiDB cluster, you can use command-line options or environment ## `--store` - Specifies the storage engine used by TiDB in the bottom layer -- Default: "mocktikv" -- You can choose "mocktikv" or "tikv". ("mocktikv" is the local storage engine; "tikv" is a distributed storage engine) +- Default: "unistore" +- You can choose "unistore" or "tikv". ("unistore" is the local storage engine; "tikv" is a distributed storage engine) ## `--token-limit` diff --git a/command-line-flags-for-tikv-configuration.md b/command-line-flags-for-tikv-configuration.md index 1db9bdf7b0c4..dd6d7009c717 100644 --- a/command-line-flags-for-tikv-configuration.md +++ b/command-line-flags-for-tikv-configuration.md @@ -55,7 +55,7 @@ TiKV supports some readable unit conversions for command line parameters. - The path to the data directory - Default: "/tmp/tikv/store" -## `-L, --Log` +## `-L` - The log level - Default: "info" diff --git a/comment-syntax.md b/comment-syntax.md index 00a1d1265296..0d37b47bb2cf 100644 --- a/comment-syntax.md +++ b/comment-syntax.md @@ -127,14 +127,14 @@ In TiDB, you can also use another version: SELECT STRAIGHT_JOIN col1 FROM table1,table2 WHERE ... ``` -If the server version number is specified in the comment, for example, `/*!50110 KEY_BLOCK_SIZE=1024 */`, in MySQL it means that the contents in this comment are processed only when the MySQL version is or higher than 5.1.10. But in TiDB, the MySQL version number does not work and all contents in the comment are processed. **Do not leave any space inside the `/*T![` characters**. +If the server version number is specified in the comment, for example, `/*!50110 KEY_BLOCK_SIZE=1024 */`, in MySQL it means that the contents in this comment are processed only when the MySQL version is or higher than 5.1.10. But in TiDB, the MySQL version number does not work and all contents in the comment are processed. ## TiDB specific comment syntax TiDB has its own comment syntax (that is, TiDB specific comment syntax), which can be divided into the following two types: * `/*T! Specific code */`: This syntax can only be parsed and executed by TiDB, and be ignored in other databases. -* `/*T![feature_id] Specific code */`: This syntax is used to ensure compatibility between different versions of TiDB. TiDB can parse the SQL fragment in this comment only if it implements the corresponding feature of `feature_id` in the current version. For example, as the `AUTO_RANDOM` feature is introduced in v3.1.1, this version of TiDB can parse `/*T![auto_rand] auto_random */` into `auto_random`. Because the `AUTO_RANDOM` feature is not implemented in v3.0.0, the SQL statement fragment above is ignored. +* `/*T![feature_id] Specific code */`: This syntax is used to ensure compatibility between different versions of TiDB. TiDB can parse the SQL fragment in this comment only if it implements the corresponding feature of `feature_id` in the current version. For example, as the `AUTO_RANDOM` feature is introduced in v3.1.1, this version of TiDB can parse `/*T![auto_rand] auto_random */` into `auto_random`. Because the `AUTO_RANDOM` feature is not implemented in v3.0.0, the SQL statement fragment above is ignored. **Do not leave any space inside the `/*T![` characters**. ## Optimizer comment syntax @@ -150,6 +150,6 @@ For details about the optimizer hints that TiDB supports, see [Optimizer hints]( > **Note** > -> In MySQL client before 5.7.7, TiDB specific comment syntax and optimizer comment syntax are treated as comments and cleared by default. To use the two syntaxes in the old client, add the `--comments` option when you start the client. For example, `mysql -h 127.0.0.1 -P 4000 -uroot --comments`. +> In MySQL client, the TiDB-specific comment syntax is treated as comments and cleared by default. In MySQL client before 5.7.7, hints are also seen as comments and are cleared by default. It is recommended to use the `--comments` option when you start the client. For example, `mysql -h 127.0.0.1 -P 4000 -uroot --comments`. For more information, see [Comment Syntax](https://dev.mysql.com/doc/refman/5.7/en/comments.html). diff --git a/community.md b/community.md index 6fe1eff0d8dc..9c8df45371ed 100644 --- a/community.md +++ b/community.md @@ -4,7 +4,7 @@ summary: Learn about how to connect with us. aliases: ['/docs/dev/community/'] --- -# Connect with us +# Connect with Us - **Twitter**: [@PingCAP](https://twitter.com/PingCAP) - **Reddit**: diff --git a/config-templates/complex-cdc.yaml b/config-templates/complex-cdc.yaml index e4267e371912..9c4c48bcf56d 100644 --- a/config-templates/complex-cdc.yaml +++ b/config-templates/complex-cdc.yaml @@ -21,7 +21,7 @@ monitored: # # - PD: https://pingcap.com/docs/stable/reference/configuration/pd-server/configuration-file/ # # All configuration items use points to represent the hierarchy, e.g: # # readpool.storage.use-unified-pool -# # +# # # # You can overwrite this configuration via the instance-level `config` field. server_configs: @@ -40,6 +40,9 @@ server_configs: schedule.leader-schedule-limit: 4 schedule.region-schedule-limit: 2048 schedule.replica-schedule-limit: 64 + cdc: + # capture-session-ttl: 10 + # sorter.sort-dir: "/tmp/cdc_sort" pd_servers: - host: 10.0.1.4 @@ -93,15 +96,21 @@ cdc_servers: - host: 10.0.1.1 port: 8300 deploy_dir: "/tidb-deploy/cdc-8300" + data_dir: "/tidb-data/cdc-8300" log_dir: "/tidb-deploy/cdc-8300/log" + # gc-ttl: 86400 - host: 10.0.1.2 port: 8300 deploy_dir: "/tidb-deploy/cdc-8300" + data_dir: "/tidb-data/cdc-8300" log_dir: "/tidb-deploy/cdc-8300/log" + # gc-ttl: 86400 - host: 10.0.1.3 port: 8300 deploy_dir: "/tidb-deploy/cdc-8300" + data_dir: "/tidb-data/cdc-8300" log_dir: "/tidb-deploy/cdc-8300/log" + # gc-ttl: 86400 monitoring_servers: - host: 10.0.1.10 diff --git a/config-templates/complex-tidb-binlog.yaml b/config-templates/complex-tidb-binlog.yaml index 36db4f2ca451..f149a6d89fe9 100644 --- a/config-templates/complex-tidb-binlog.yaml +++ b/config-templates/complex-tidb-binlog.yaml @@ -92,35 +92,35 @@ pump_servers: - host: 10.0.1.1 ssh_port: 22 port: 8250 - deploy_dir: "/tidb-deploy/pump-8249" - data_dir: "/tidb-data/pump-8249" - # The following configs are used to overwrite the `server_configs.drainer` values. + deploy_dir: "/tidb-deploy/pump-8250" + data_dir: "/tidb-data/pump-8250" + # The following configs are used to overwrite the `server_configs.pump` values. config: gc: 7 - host: 10.0.1.2 ssh_port: 22 port: 8250 - deploy_dir: "/tidb-deploy/pump-8249" - data_dir: "/tidb-data/pump-8249" - # The following configs are used to overwrite the `server_configs.drainer` values. + deploy_dir: "/tidb-deploy/pump-8250" + data_dir: "/tidb-data/pump-8250" + # The following configs are used to overwrite the `server_configs.pump` values. config: gc: 7 - host: 10.0.1.3 ssh_port: 22 port: 8250 - deploy_dir: "/tidb-deploy/pump-8249" - data_dir: "/tidb-data/pump-8249" - # The following configs are used to overwrite the `server_configs.drainer` values. + deploy_dir: "/tidb-deploy/pump-8250" + data_dir: "/tidb-data/pump-8250" + # The following configs are used to overwrite the `server_configs.pump` values. config: gc: 7 drainer_servers: - host: 10.0.1.12 port: 8249 + deploy_dir: "/tidb-deploy/drainer-8249" data_dir: "/tidb-data/drainer-8249" # If drainer doesn't have a checkpoint, use initial commitTS as the initial checkpoint. # Will get a latest timestamp from pd if commit_ts is set to -1 (the default value). commit_ts: -1 - deploy_dir: "/tidb-deploy/drainer-8249" # The following configs are used to overwrite the `server_configs.drainer` values. config: syncer.db-type: "tidb" @@ -128,7 +128,23 @@ drainer_servers: syncer.to.user: "root" syncer.to.password: "" syncer.to.port: 4000 - + - host: 10.0.1.13 + port: 8249 + deploy_dir: "/tidb-deploy/drainer-8249" + data_dir: "/tidb-data/drainer-8249" + # If Drainer does not have a checkpoint, use the initial commitTS as the initial checkpoint. + # If commit_ts is set to -1 (the default value), you will get a latest timestamp from PD. + commit_ts: -1 + # The following configurations are used to overwrite the `server_configs.drainer` values. + config: + syncer.db-type: "kafka" + syncer.replicate-do-db: + - db1 + - db2 + syncer.to.kafka-addrs: "10.0.1.20:9092,10.0.1.21:9092,10.0.1.22:9092" + syncer.to.kafka-version: "0.10.0.0" + syncer.to.topic-name: "asyouwish" + monitoring_servers: - host: 10.0.1.10 # ssh_port: 22 diff --git a/config-templates/geo-redundancy-deployment.yaml b/config-templates/geo-redundancy-deployment.yaml index 9f18ac900a60..676145106757 100644 --- a/config-templates/geo-redundancy-deployment.yaml +++ b/config-templates/geo-redundancy-deployment.yaml @@ -1,6 +1,13 @@ # Tip: PD priority needs to be manually set using the PD-ctl client tool. such as, member Leader_priority PD-name numbers. # Global variables are applied to all deployments and used as the default value of # the deployments if a specific deployment value is missing. +# +# Abbreviations used in this example: +# sh: Shanghai Zone +# bj: Beijing Zone +# sha: Shanghai Datacenter A +# bja: Beijing Datacenter A +# bjb: Beijing Datacenter B global: user: "tidb" @@ -105,4 +112,4 @@ tikv_servers: monitoring_servers: - host: 10.0.1.16 grafana_servers: - - host: 10.0.1.16 \ No newline at end of file + - host: 10.0.1.16 diff --git a/configure-load-base-split.md b/configure-load-base-split.md index 1c85dcb0214c..1e264047bdb0 100644 --- a/configure-load-base-split.md +++ b/configure-load-base-split.md @@ -29,9 +29,9 @@ The Region split by Load Base Split will not be merged quickly. On the one hand, ## Usage -The Load Base Split feature is currently controlled by the `split.qps-threshold` parameter. If the sum of all types of read requests per second for a Region exceeds the value of `split.qps-threshold` for 10 seconds on end, split the Region. +The Load Base Split feature is currently controlled by the `split.qps-threshold` parameter (QPS threshold) and `split.byte-threshold` parameter (traffic threshold). If the sum of all types of read requests per second for a Region exceeds the QPS threshold or traffic threshold for 10 consecutive seconds, PD splits the Region. -Load Base Split is enabled by default, but the parameter is set to a rather high value, defaulting to `3000`. If you want to disable this feature, set the threshold high enough. +Load Base Split is enabled by default, but the parameter is set to a rather high value. `split.qps-threshold` defaults to `3000` and `split.byte-threshold` defaults to 30MB/s. If you want to disable this feature, set the two thresholds high enough at the same time. To modify the parameter, take either of the following two methods: diff --git a/configure-memory-usage.md b/configure-memory-usage.md index 589b8f6fa18e..414b22aa92ba 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -10,7 +10,7 @@ Currently, TiDB can track the memory quota of a single SQL query and take action ``` # Valid options: ["log", "cancel"] -oom-action = "log" +oom-action = "cancel" ``` - If the configuration item above uses "log", when the memory quota of a single SQL query exceeds the threshold value which is controlled by the `tidb_mem_quota_query` variable, TiDB prints an entry of log. Then the SQL query continues to be executed. If OOM occurs, you can find the corresponding SQL query in the log. @@ -26,16 +26,16 @@ mem-quota-query = 34359738368 In addition, you can control the memory quota of a query using the following session variables. Generally, you only need to configure `tidb_mem_quota_query`. Other variables are used for advanced configuration which most users do not need to care about. -| Variable Name | Description | Unit | Default Value | -|-----------------------------------|---------------------------------------------------|-------|-----------| -| tidb_mem_quota_query | Control the memory quota of a query | Byte | 32 << 30 | -| tidb_mem_quota_hashjoin | Control the memory quota of "HashJoinExec" | Byte | 32 << 30 | -| tidb_mem_quota_mergejoin | Control the memory quota of "MergeJoinExec" | Byte | 32 << 30 | -| tidb_mem_quota_sort | Control the memory quota of "SortExec" | Byte | 32 << 30 | -| tidb_mem_quota_topn | Control the memory quota of "TopNExec" | Byte | 32 << 30 | -| tidb_mem_quota_indexlookupreader | Control the memory quota of "IndexLookUpExecutor" | Byte | 32 << 30 | -| tidb_mem_quota_indexlookupjoin | Control the memory quota of "IndexLookUpJoin" | Byte | 32 << 30 | -| tidb_mem_quota_nestedloopapply | Control the memory quota of "NestedLoopApplyExec" | Byte | 32 << 30 | +| Variable Name | Description | Unit | Default Value | +| -------------------------------- | ------------------------------------------------- | ---- | ------------- | +| tidb_mem_quota_query | Control the memory quota of a query | Byte | 1 << 30 (1 GB) | +| tidb_mem_quota_hashjoin | Control the memory quota of "HashJoinExec" | Byte | 32 << 30 | +| tidb_mem_quota_mergejoin | Control the memory quota of "MergeJoinExec" | Byte | 32 << 30 | +| tidb_mem_quota_sort | Control the memory quota of "SortExec" | Byte | 32 << 30 | +| tidb_mem_quota_topn | Control the memory quota of "TopNExec" | Byte | 32 << 30 | +| tidb_mem_quota_indexlookupreader | Control the memory quota of "IndexLookUpExecutor" | Byte | 32 << 30 | +| tidb_mem_quota_indexlookupjoin | Control the memory quota of "IndexLookUpJoin" | Byte | 32 << 30 | +| tidb_mem_quota_nestedloopapply | Control the memory quota of "NestedLoopApplyExec" | Byte | 32 << 30 | Some usage examples: diff --git a/configure-time-zone.md b/configure-time-zone.md index e46147e0bea8..246e226ca16c 100644 --- a/configure-time-zone.md +++ b/configure-time-zone.md @@ -28,12 +28,12 @@ Each client has its own time zone setting, given by the session `time_zone` vari SET time_zone = timezone; ``` -You can use the following statement to view the current values of the global and client-specific time zones: +You can use the following statement to view the current values of the global, client-specific and system time zones: {{< copyable "sql" >}} ```sql -SELECT @@global.time_zone, @@session.time_zone; +SELECT @@global.time_zone, @@session.time_zone, @@global.system_time_zone; ``` To set the format of the value of the `time_zone`: diff --git a/connectors-and-apis.md b/connectors-and-apis.md index 1dbe3d6e6033..444af731e8eb 100644 --- a/connectors-and-apis.md +++ b/connectors-and-apis.md @@ -51,7 +51,7 @@ The third-party APIs are not developed by Oracle. The following table lists the | | MySQL wrapped | `libmysqlclient` | See [MySQL wrapped](http://www.alhem.net/project/mysql/) | | Go | go-sql-driver | Native Driver | See [Mysql Go API](https://github.com/go-sql-driver/mysql) | | Cocoa | MySQL-Cocoa | `libmysqlclient` | Compatible with the Objective-C Cocoa environment. See | -| D | MySQL for D | `libmysqlclient` | See [MySQL for D](http://www.steinmole.de/d/) | +| D | MySQL for D | `libmysqlclient` | See [MySQL for D](https://github.com/mysql-d/mysql-native) | | Eiffel | Eiffel MySQL | `libmysqlclient` | See [Section 27.14, “MySQL Eiffel Wrapper”](https://dev.mysql.com/doc/refman/5.7/en/apis-eiffel.html) | | Erlang | `erlang-mysql-driver` | `libmysqlclient` | See [`erlang-mysql-driver`](http://code.google.com/p/erlang-mysql-driver/) | | Haskell | Haskell MySQL Bindings | Native Driver | See [Brian O'Sullivan's pure Haskell MySQL bindings](http://www.serpentine.com/blog/software/mysql/) | diff --git a/constraints.md b/constraints.md index 473820f56e2b..564ff6745229 100644 --- a/constraints.md +++ b/constraints.md @@ -245,9 +245,31 @@ Query OK, 0 rows affected (0.10 sec) * Table `t3` failed to be created, because a table can only have one primary key. * Table `t4` was created successfully, because even though there can be only one primary key, TiDB supports defining multiple columns as the composite primary key. -In addition to the rules above, by default, TiDB has an additional restriction that once a table is successfully created, its primary key cannot be changed. If you need to add/remove the primary key, you need to set `alter-primary-key` to `true` in the TiDB configuration file, and restart the TiDB instance to make it effective. +In addition to the rules above, TiDB currently only supports adding and deleting the primary keys of the `NONCLUSTERED` type. For example: -When the add/delete primary key feature is enabled, TiDB allows adding/deleting primary key to the table. However, it should be noted that, if a table with an integer type primary key has been created before the feature is enabled, you cannot delete its primary key constraint even when you enable the add/delete primary key feature. +{{< copyable "sql" >}} + +```sql +CREATE TABLE t5 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b) CLUSTERED); +ALTER TABLE t5 DROP PRIMARY KEY; +``` + +``` +ERROR 8200 (HY000): Unsupported drop primary key when the table is using clustered index +``` + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t5 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b) NONCLUSTERED); +ALTER TABLE t5 DROP PRIMARY KEY; +``` + +``` +Query OK, 0 rows affected (0.10 sec) +``` + +For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). ## FOREIGN KEY diff --git a/credits.md b/credits.md index 1a8b9a31fedf..c303dd7af4c5 100644 --- a/credits.md +++ b/credits.md @@ -18,7 +18,7 @@ The TiDB developer community uses SIG (Special Interest Group) as a unit to mana |[Dashboard](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-dashboard)|[breeswish](https://github.com/breeswish)| |[docs](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-docs)|[lilin90](https://github.com/lilin90)| |[execution](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-exec)|[qw4990](https://github.com/qw4990)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[kennytm](https://github.com/kennytm)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[kennytm](https://github.com/kennytm)| |[tiup](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tiup)|[lonng](https://github.com/lonng)| |[web](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-web)|[wd0517](https://github.com/wd0517)| |[ddl](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-ddl)|[zimulala](https://github.com/zimulala)| @@ -62,16 +62,16 @@ The TiDB developer community uses SIG (Special Interest Group) as a unit to mana |[docs](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-docs)|[TomShawn](https://github.com/TomShawn)| |[docs](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-docs)|[toutdesuite](https://github.com/toutdesuite)| |[docs](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-docs)|[WangXiangUSTC](https://github.com/WangXiangUSTC)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[3pointer](https://github.com/3pointer)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[5kbpers](https://github.com/5kbpers)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[amyangfei](https://github.com/amyangfei)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[gmhdbjd](https://github.com/gmhdbjd)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[GregoryIan](https://github.com/GregoryIan)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[july2993](https://github.com/july2993)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[leoppro](https://github.com/leoppro)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[lichunzhu](https://github.com/lichunzhu)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[overvenus](https://github.com/overvenus)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[YuJuncen](https://github.com/YuJuncen)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[3pointer](https://github.com/3pointer)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[5kbpers](https://github.com/5kbpers)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[amyangfei](https://github.com/amyangfei)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[gmhdbjd](https://github.com/gmhdbjd)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[GregoryIan](https://github.com/GregoryIan)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[july2993](https://github.com/july2993)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[leoppro](https://github.com/leoppro)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[lichunzhu](https://github.com/lichunzhu)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[overvenus](https://github.com/overvenus)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[YuJuncen](https://github.com/YuJuncen)| |[tiup](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tiup)|[AstroProfundis](https://github.com/AstroProfundis)| |[tiup](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tiup)|[july2993](https://github.com/july2993)| |[tiup](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tiup)|[nrc](https://github.com/nrc)| @@ -92,7 +92,7 @@ The TiDB developer community uses SIG (Special Interest Group) as a unit to mana |[planner](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-planner)|[eurekaka](https://github.com/eurekaka)| |[planner](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-planner)|[francis0407](https://github.com/francis0407)| |[planner](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-planner)|[lzmhhh123](https://github.com/lzmhhh123)| -|[tools](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-tools)|[csuzhangxc](https://github.com/csuzhangxc)| +|[migrate](https://github.com/pingcap/community/tree/master/special-interest-groups/sig-migrate)|[csuzhangxc](https://github.com/csuzhangxc)| ## Reviewers diff --git a/dashboard/dashboard-diagnostics-access.md b/dashboard/dashboard-diagnostics-access.md index 97315586f6f9..ce064392e967 100644 --- a/dashboard/dashboard-diagnostics-access.md +++ b/dashboard/dashboard-diagnostics-access.md @@ -14,7 +14,7 @@ The cluster diagnostics feature in TiDB Dashboard diagnoses the problems that mi > **Note:** > -> The cluster diagnostics feature depends on Prometheus deployed in the cluster. For details about how to deploy this monitoring component, see [TiUP](/tiup/tiup-overview.md) or [TiDB Ansible](/online-deployment-using-ansible.md) deployment document. If no monitoring component is deployed in the cluster, the generated diagnostic report will indicate a failure. +> The cluster diagnostics feature depends on Prometheus deployed in the cluster. For details about how to deploy this monitoring component, see the [TiUP](/tiup/tiup-overview.md) deployment document. If no monitoring component is deployed in the cluster, the generated diagnostic report will indicate a failure. ## Access the page diff --git a/dashboard/dashboard-diagnostics-report.md b/dashboard/dashboard-diagnostics-report.md index af9660babcc3..bd334978aa97 100644 --- a/dashboard/dashboard-diagnostics-report.md +++ b/dashboard/dashboard-diagnostics-report.md @@ -31,7 +31,7 @@ In this report, some small buttons are described as follows: * **expand**: Click **expand** to see details about this monitoring metric. For example, the detailed information of `tidb_get_token` in the image above includes the monitoring information of each TiDB instance's latency. * **collapse**: Contrary to **expand**, the button is used to fold detailed monitoring information. -All monitoring metrics basically correspond to those on the TiDB Grafna monitoring dashboard. After a module is found to be abnormal, you can view more monitoring information on the TiDB Grafna. +All monitoring metrics basically correspond to those on the TiDB Grafana monitoring dashboard. After a module is found to be abnormal, you can view more monitoring information on the TiDB Grafana. In addition, the `TOTAL_TIME` and `TOTAL_COUNT` metrics in this report are monitoring data read from Prometheus, so calculation inaccuracy might exits in their statistics. diff --git a/dashboard/dashboard-diagnostics-usage.md b/dashboard/dashboard-diagnostics-usage.md index 9700161530b6..918270171d24 100644 --- a/dashboard/dashboard-diagnostics-usage.md +++ b/dashboard/dashboard-diagnostics-usage.md @@ -71,6 +71,7 @@ The result of another `go-ycsb` pressure test is shown in the image above. You c Generate a report that compares the system in the following two time ranges: T1: `2020-03-08 01:36:00` to `2020-03-08 01:41:00`. In this range, the system is normal, which is called a reference range. + T2: `2020-03-08 01:46:30` to `2020-03-08 01:51:30`. In this range, QPS began to decrease. After the report is generated, you can view this report on the **Compare Diagnose** page. @@ -101,6 +102,7 @@ The result of a `go-ycsb` pressure test is shown in the image above. You can see Generate a report that compares the system in the following two time ranges: T1: `2020-05-22 22:11:00` to `2020-05-22 22:14:00`. In this range, the system is normal, which is called a reference range. + T2: `2020-05-22 22:14:00` `2020-05-22 22:17:00`. In this range, QPS began to decrease. After generating the comparison report, check the **Max diff item** report. This report compares the monitoring items of the two time ranges above and sorts them according to the difference of the monitoring items. The result of this table is as follows: diff --git a/dashboard/dashboard-faq.md b/dashboard/dashboard-faq.md index fca8279c95d8..4755b616b5c6 100644 --- a/dashboard/dashboard-faq.md +++ b/dashboard/dashboard-faq.md @@ -30,7 +30,7 @@ If you have deployed TiDB using the `tiup cluster` or `tiup playground` command, The **QPS** and **Latency** sections on the **Overview** page require a cluster with Prometheus deployed. Otherwise, the error is shown. You can solve this problem by deploying a Prometheus instance in the cluster. -If you still encounter this problem when the Prometheus instance has been deployed, the possible reason is that your deployment tool is out of date (TiUP, TiDB Operator, or TiDB Ansible), and your tool does not automatically report metrics addresses, which makes TiDB Dashboard unable to query metrics. You can upgrade you deployment tool to the latest version and try again. +If you still encounter this problem when the Prometheus instance has been deployed, the possible reason is that your deployment tool is out of date (TiUP or TiDB Operator), and your tool does not automatically report metrics addresses, which makes TiDB Dashboard unable to query metrics. You can upgrade you deployment tool to the latest version and try again. If your deployment tool is TiUP, take the following steps to solve this problem. For other deployment tools, refer to the corresponding documents of those tools. @@ -57,4 +57,26 @@ If your deployment tool is TiUP, take the following steps to solve this problem. ### An `invalid connection` error is shown in **Top SQL Statements** and **Recent Slow Queries** on the Overview page -This error is possibly related to the `prepared-plan-cache` feature of TiDB. You can disable `prepared-plan-cache` by updating [TiDB Configuration file](/tidb-configuration-file.md#prepared-plan-cache). +The possible reason is that you have enabled the `prepared-plan-cache` feature of TiDB. As an experimental feature, when enabled, `prepared-plan-cache` might not function properly in specific TiDB versions, which could cause this problem in TiDB Dashboard (and other applications). You can disable `prepared-plan-cache` by updating [TiDB Configuration file](/tidb-configuration-file.md#prepared-plan-cache) to solve this problem. + +### An `unknown field` error is shown in **Slow Queries** page + +If the `unknown field` error appears on the **Slow Queries** page after the cluster upgrade, the error is related to a compatibility issue caused by the difference between TiDB Dashboard server fields (which might be updated) and user preferences fields (which are in the browser cache). This issue has been fixed. If your cluster is earlier than v5.0.3 or v4.0.14, perform the following steps to resolve the issue: + +To clear your browser cache, take the following steps: + +1. Open TiDB Dashboard page. + +2. Open Developer Tools. Different browsers have different ways of opening Developer Tools. After clicking the **Menu Bar**: + + - Firefox: Menu ➤ Web Developer ➤ Toggle Tools, or Tools ➤ Web Developer ➤ Toggle Tools. + - Chrome: More tools ➤ Developer tools. + - Safari: Develop ➤ Show Web Inspector. If you can't see the Develop menu, go to Safari ➤ Preferences ➤ Advanced, and check the Show Develop menu in menu bar checkbox. + + In the following example, Chrome is used. + + ![Opening DevTools from Chrome's main menu](/media/dashboard/dashboard-faq-devtools.png) + +3. Select the **Application** panel, expand the **Local Storage** menu and select the **TiDB Dashboard page domain**. Click the **Clear All** button. + + ![Clear the Local Storage](/media/dashboard/dashboard-faq-devtools-application.png) diff --git a/dashboard/dashboard-key-visualizer.md b/dashboard/dashboard-key-visualizer.md index 9d4044dc4af1..db7c1984187c 100644 --- a/dashboard/dashboard-key-visualizer.md +++ b/dashboard/dashboard-key-visualizer.md @@ -156,7 +156,7 @@ In the heatmap above, bright and dark colors are a fine-grained mix. This indica In the heatmap above, there is an alternating brightness and darkness along the X-axis (time) but the brightness is relatively even along the Y-axis (Region). This indicates that the reads and writes change periodically, which might occur in scenarios of periodically scheduled tasks. For example, the big data platform periodically extracts data from TiDB every day. In this kind of scenarios, pay attention to whether the resources are sufficient during peak usage. -## Concentrated reads or writes +### Concentrated reads or writes ![Concentrated](/media/dashboard/dashboard-keyviz-continue.png) diff --git a/dashboard/dashboard-ops-deploy.md b/dashboard/dashboard-ops-deploy.md index 68ffeebb552b..aa30eb8392e9 100644 --- a/dashboard/dashboard-ops-deploy.md +++ b/dashboard/dashboard-ops-deploy.md @@ -10,7 +10,7 @@ The TiDB Dashboard UI is built into the PD component for v4.0 or higher versions See the following documents to learn how to deploy a standard TiDB cluster: -+ [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md#deploy-a-local-test-environment-using-tiup-playground) ++ [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md) + [Deploy TiDB in Production Environment](/production-deployment-using-tiup.md) + [Kubernetes environment deployment](https://docs.pingcap.com/tidb-in-kubernetes/stable/access-dashboard) diff --git a/dashboard/dashboard-ops-reverse-proxy.md b/dashboard/dashboard-ops-reverse-proxy.md index 69fb92f6e6b4..0538b7f79db8 100644 --- a/dashboard/dashboard-ops-reverse-proxy.md +++ b/dashboard/dashboard-ops-reverse-proxy.md @@ -133,7 +133,7 @@ server_configs:
Modify configuration when deploying a new cluster using TiUP -If you are deploying a new cluster, you can add the configuration above to the `topology.yaml` TiUP topology file and deploy the cluster. For specific instruction, see [TiUP deployment document](/production-deployment-using-tiup.md#step-3-edit-the-initialization-configuration-file). +If you are deploying a new cluster, you can add the configuration above to the `topology.yaml` TiUP topology file and deploy the cluster. For specific instruction, see [TiUP deployment document](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file).
diff --git a/dashboard/dashboard-overview.md b/dashboard/dashboard-overview.md index cac38b5ca9b7..baa49107e5ba 100644 --- a/dashboard/dashboard-overview.md +++ b/dashboard/dashboard-overview.md @@ -65,7 +65,7 @@ The content displayed in this area is consistent with the more detailed [Slow Qu > **Note:** > -> This feature is available only in the cluster with slow query logs enabled. By default, slow query logs are enabled in the cluster deployed using TiUP or TiDB Ansible. +> This feature is available only in the cluster with slow query logs enabled. By default, slow query logs are enabled in the cluster deployed using TiUP. ## Instances diff --git a/dashboard/dashboard-session-share.md b/dashboard/dashboard-session-share.md new file mode 100644 index 000000000000..53a7fee12481 --- /dev/null +++ b/dashboard/dashboard-session-share.md @@ -0,0 +1,56 @@ +--- +title: Share TiDB Dashboard Sessions +summary: Learn how to share the current TiDB Dashboard session to other users. +--- + +# Share TiDB Dashboard Sessions + +You can share the current session of the TiDB Dashboard to other users so that they can access and operate the TiDB Dashboard without entering the user password. + +## Steps for the Inviter + +1. Sign into TiDB Dashboard. + +2. Click the username in the left sidebar to access the configuration page. + +3. Click **Share Current Session**. + + ![Sample Step](/media/dashboard/dashboard-session-share-settings-1.png) + + > **Note:** + > + > For security reasons, the shared session cannot be shared again. + +4. Adjust sharing settings in the popup dialog: + + - Expire in: How long the shared session will be effective. Signing out of the current session does not affect the effective time of the shared session. + + - Share as read-only privilege: The shared session only permits read operations but not write operations (such as modifying configurations). + +5. Click **Generate Authorization Code**. + + ![Sample Step](/media/dashboard/dashboard-session-share-settings-2.png) + +6. Provide the generated **Authorization Code** to the user to whom you want to share the session. + + ![Sample Step](/media/dashboard/dashboard-session-share-settings-3.png) + + > **Warning:** + > + > Keep your authorization code secure and do not send it to anyone who is untrusted. Otherwise, they will be able to access and operate TiDB Dashboard without your authorization. + +## Steps for the Invitee + +1. On the sign-in page of TiDB Dashboard, click **Use Alternative Authentication**. + + ![Sample Step](/media/dashboard/dashboard-session-share-signin-1.png) + +2. Click **Authorization Code** to use it to sign in. + + ![Sample Step](/media/dashboard/dashboard-session-share-signin-2.png) + +3. Enter the authorization code you have received from the inviter. + +4. Click **Sign In**. + + ![Sample Step](/media/dashboard/dashboard-session-share-signin-3.png) diff --git a/dashboard/dashboard-session-sso.md b/dashboard/dashboard-session-sso.md new file mode 100644 index 000000000000..597e3e8d5ade --- /dev/null +++ b/dashboard/dashboard-session-sso.md @@ -0,0 +1,151 @@ +--- +title: Configure SSO for TiDB Dashboard +summary: Learn how to enable SSO to sign into TiDB Dashboard. +--- + +# Configure SSO for TiDB Dashboard + +TiDB Dashboard supports [OIDC](https://openid.net/connect/)-based Single Sign-On (SSO). After enabling the SSO feature of TiDB Dashboard, the configured SSO service is used for your sign-in authentication and then you can access TiDB Dashboard without entering the SQL user password. + +## Configure OIDC SSO + +### Enable SSO + +1. Sign into TiDB Dashboard. + +2. Click the username in the left sidebar to access the configuration page. + +3. In the **Single Sign-On** section, select **Enable to use SSO when sign into TiDB Dashboard**. + +4. Fill the **OIDC Client ID** and the **OIDC Discovery URL** fields in the form. + + Generally, you can obtain the two fields from the SSO service provider: + + - OIDC Client ID is also called OIDC Token Issuer. + - OIDC Discovery URL is also called OIDC Token Audience. + +5. Click **Authorize Impersonation** and input the SQL password. + + TiDB Dashboard will store this SQL password and use it to impersonate a normal SQL sign-in after an SSO sign-in is finished. + + ![Sample Step](/media/dashboard/dashboard-session-sso-enable-1.png) + + > **Note:** + > + > The password you have entered will be encrypted and stored. The SSO sign-in will fail after the password of the SQL user is changed. In this case, you can re-enter the password to bring SSO back. + +6. Click **Authorize and Save**. + + ![Sample Step](/media/dashboard/dashboard-session-sso-enable-2.png) + +7. Click **Update** (Update) to save the configuration. + + ![Sample Step](/media/dashboard/dashboard-session-sso-enable-3.png) + +Now SSO sign-in has been enabled for TiDB Dashboard. + +> **Note:** +> +> For security reasons, some SSO services require additional configuration for the SSO service, such as the trusted sign-in and sign-out URIs. Refer to the documentation of the SSO service for further information. + +### Disable SSO + +You can disable the SSO, which will completely erase the stored SQL password: + +1. Sign into TiDB Dashboard. + +2. Click the username in the left sidebar to access the configuration page. + +3. In the **Single Sign-On** section, deselect **Enable to use SSO when sign into TiDB Dashboard**. + +4. Click **Update** (Update) to save the configuration. + + ![Sample Step](/media/dashboard/dashboard-session-sso-disable.png) + +### Re-enter the password after a password change + +The SSO sign-in will fail once the password of the SQL user is changed. In this case, you can bring back the SSO sign-in by re-entering the SQL password: + +1. Sign into TiDB Dashboard. + +2. Click the username in the left sidebar to access the configuration page. + +3. In the **Single Sign-On** section, Click **Authorize Impersonation** and input the updated SQL password. + + ![Sample Step](/media/dashboard/dashboard-session-sso-reauthorize.png) + +4. Click **Authorize and Save**. + +## Sign in via SSO + +Once SSO is configured for TiDB Dashboard, you can sign in via SSO by taking following steps: + +1. In the sign-in page of TiDB Dashboard, click **Sign in via Company Account**. + + ![Sample Step](/media/dashboard/dashboard-session-sso-signin.png) + +2. Sign into the system with SSO service configured. + +3. You are redirected back to TiDB Dashboard to finish the sign-in. + +## Example: Use Okta for TiDB Dashboard SSO sign-in + +[Okta](https://www.okta.com/) is an OIDC SSO identity service, which is compatible with the SSO feature of TiDB Dashboard. The steps below demonstrate how to configure Okta and TiDB Dashboard so that Okta can be used as the TiDB Dashboard SSO provider. + +### Step 1: Configure Okta + +First, create an Okta Application Integration to integrate SSO. + +1. Access the Okta administration site. + +2. Navigate from the left sidebar **Applications** > **Applications**. + +3. Click **Create App Integration**。 + + ![Sample Step](/media/dashboard/dashboard-session-sso-okta-1.png) + +4. In the poped up dialog, choose **OIDC - OpenID Connect** in **Sign-in method**. + +5. Choose **Single-Page Application** in **Application Type**. + +6. Click the **Next** button. + + ![Sample Step](/media/dashboard/dashboard-session-sso-okta-2.png) + +7. Fill **Sign-in redirect URIs** as follows: + + ``` + http://DASHBOARD_IP:PORT/dashboard/?sso_callback=1 + ``` + + Substitute `DASHBOARD_IP:PORT` with the actual domain (or IP address) and port that you use to access the TiDB Dashboard in the browser. + +8. Fill **Sign-out redirect URIs** as follows: + + ``` + http://DASHBOARD_IP:PORT/dashboard/ + ``` + + Similarly, substitute `DASHBOARD_IP:PORT` with the actual domain (or IP address) and port. + + ![Sample Step](/media/dashboard/dashboard-session-sso-okta-3.png) + +9. Configure what type of users in your organization is allowed for SSO sign-in in the **Assignments** field, and then click **Save** to save the configuration. + + ![Sample Step](/media/dashboard/dashboard-session-sso-okta-4.png) + +### Step 2: Obtain OIDC information and fill in TiDB Dashboard + +1. In the Application Integration just created in Okta, click **Sign On**. + + ![Sample Step 1](/media/dashboard/dashboard-session-sso-okta-info-1.png) + +2. Copy values of the **Issuer** and **Audience** fields from the **OpenID Connect ID Token** section. + + ![Sample Step 2](/media/dashboard/dashboard-session-sso-okta-info-2.png) + +3. Open the TiDB Dashboard configuration page, fill **OIDC Client ID** with **Issuer** obtained from the last step and fill **OIDC Discovery URL** with **Audience**. Then finish the authorization and save the configuration. For example: + + ![Sample Step 3](/media/dashboard/dashboard-session-sso-okta-info-3.png) + +Now TiDB Dashboard has been configured to use Okta SSO for sign-in. diff --git a/deploy-test-cluster-using-docker-compose.md b/deploy-test-cluster-using-docker-compose.md deleted file mode 100644 index bba4cbadcd06..000000000000 --- a/deploy-test-cluster-using-docker-compose.md +++ /dev/null @@ -1,184 +0,0 @@ ---- -title: TiDB Docker Compose Deployment -summary: Use Docker Compose to quickly deploy a TiDB testing cluster. -aliases: ['/docs/dev/deploy-test-cluster-using-docker-compose/','/docs/dev/how-to/get-started/deploy-tidb-from-docker-compose/'] ---- - -# TiDB Docker Compose Deployment - -> **Warning:** -> -> The Docker Compose deployment method provided in this document is no longer maintained. If you want to test TiDB, it is recommended to refer to [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md) for deployment. For **production environment**, **do not use** Docker Compose for deployment, but [deploy TiDB with TiUP](/production-deployment-using-tiup.md) or [TiDB Operator in Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable/deploy-tidb-operator). - -This document describes how to quickly deploy a TiDB testing cluster with a single command using [Docker Compose](https://docs.docker.com/compose/overview). - -With Docker Compose, you can use a YAML file to configure application services in multiple containers. Then, with a single command, you can create and start all the services from your configuration. - -## Prerequisites - -Make sure you have installed the following items on your machine: - -- [Git](https://git-scm.com/downloads) -- [Docker Compose](https://docs.docker.com/compose/install/) -- [MySQL Client](https://dev.mysql.com/downloads/mysql/) - -## Deploy TiDB using Docker Compose - -1. Download `tidb-docker-compose`. - - ```bash - git clone https://github.com/pingcap/tidb-docker-compose.git - ``` - -2. Change the directory to tidb-docker-compose and get the latest TiDB Docker Images: - - ```bash - cd tidb-docker-compose && docker-compose pull - ``` - -3. Start the TiDB cluster: - - ```bash - docker-compose up -d - ``` - -4. Use the MySQL client to connect to TiDB to read and write data: - - ``` - mysql -h 127.0.0.1 -P 4000 -u root - ``` - -## Monitor the cluster - -After you have successfully deployed a TiDB cluster, you can now monitor the TiDB cluster using one of the following methods: - -- Use Grafana to view the status of the cluster via [http://localhost:3000](http://localhost:3000) with the default account name and password: `admin` and `admin`. -- Use [TiDB-Vision](https://github.com/pingcap/tidb-vision), a cluster visualization tool, to see data transfer and load-balancing inside your cluster via [http://localhost:8010](http://localhost:8010). - -## Customize the cluster - -After the deployment is completed, the following components are deployed by default: - -- 3 PD instances, 3 TiKV instances, 1 TiDB instance -- Monitoring components: Prometheus, Pushgateway, Grafana -- Data visualization component: tidb-vision - -To customize the cluster, you can edit the `docker-compose.yml` file directly. It is recommended to generate `docker-compose.yml` using the [Helm](https://helm.sh) template engine, because manual editing is tedious and error-prone. - -1. Install Helm. - - [Helm](https://helm.sh) can be used as a template rendering engine. Make sure your Helm version >= 2.9.0 and < 3.0.0. - To use Helm, you only need to download its binary file: - - ```bash - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash - ``` - - For macOS, you can also install Helm using the following command in Homebrew: - - ```bash - brew install helm@2 - brew link --force helm@2 - ``` - -2. Download `tidb-docker-compose`. - - ```bash - git clone https://github.com/pingcap/tidb-docker-compose.git - ``` - -3. Customize the cluster. - - ```bash - cd tidb-docker-compose - vim compose/values.yaml # custom the cluster size, docker image, port mapping and so on - ``` - - You can modify the configuration in `values.yaml`, such as the cluster size, TiDB image version, and so on. - - [tidb-vision](https://github.com/pingcap/tidb-vision) is the data visualization interface of the TiDB cluster, used to visually display the PD scheduling on TiKV data. If you do not need this component, comment out the `tidbVision` section. - - For PD, TiKV, TiDB and tidb-vision, you can build Docker images from GitHub source code or local files for development and testing. - - - To build PD, TiKV or TiDB images from the locally compiled binary file, you need to comment out the `image` field and copy the compiled binary file to the corresponding `pd/bin/pd-server`, `tikv/bin/tikv-server`, `tidb/bin/tidb-server`. - - To build the tidb-vision image from local, you need to comment out the `image` field and copy the tidb-vision project to `tidb-vision/tidb-vision`. - -4. Generate the `docker-compose.yml` file. - - ```bash - helm template compose > generated-docker-compose.yml - ``` - -5. Create and start the cluster using the generated `docker-compose.yml` file. - - ```bash - docker-compose -f generated-docker-compose.yml pull # Get the latest Docker images - docker-compose -f generated-docker-compose.yml up -d - ``` - -6. Access the cluster. - - ```bash - mysql -h 127.0.0.1 -P 4000 -u root - ``` - - Access the Grafana monitoring interface: - - - Default address: - - Default account name: admin - - Default password: admin - - If tidb-vision is enabled, you can access the cluster data visualization interface: . - -## Access the Spark shell and load TiSpark - -Insert some sample data to the TiDB cluster: - -```bash -$ docker-compose exec tispark-master bash -$ cd /opt/spark/data/tispark-sample-data -$ mysql -h tidb -P 4000 -u root < dss.ddl -``` - -After the sample data is loaded into the TiDB cluster, you can access the Spark shell using `docker-compose exec tispark-master /opt/spark/bin/spark-shell`. - -```bash -$ docker-compose exec tispark-master /opt/spark/bin/spark-shell -... -Spark context available as 'sc' (master = local[*], app id = local-1527045927617). -Spark session available as 'spark'. -Welcome to - ____ __ - / __/__ ___ _____/ /__ - _\ \/ _ \/ _ `/ __/ '_/ - /___/ .__/\_,_/_/ /_/\_\ version 2.1.1 - /_/ - -Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_172) -Type in expressions to have them evaluated. -Type :help for more information. - -scala> import org.apache.spark.sql.TiContext -... -scala> val ti = new TiContext(spark) -... -scala> ti.tidbMapDatabase("TPCH_001") -... -scala> spark.sql("select count(*) from lineitem").show -+--------+ -|count(1)| -+--------+ -| 60175| -+--------+ -``` - -You can also access Spark with Python or R using the following commands: - -``` -docker-compose exec tispark-master /opt/spark/bin/pyspark -docker-compose exec tispark-master /opt/spark/bin/sparkR -``` - -For more details about TiSpark, see [TiSpark Quick Start Guide](/get-started-with-tispark.md). - -Here is [a 5-minute tutorial](https://pingcap.com/blog/how_to_spin_up_an_htap_database_in_5_minutes_with_tidb_tispark/) for macOS users that shows how to spin up a standard TiDB cluster using Docker Compose on your local computer. diff --git a/deploy-tidb-from-binary.md b/deploy-tidb-from-binary.md deleted file mode 100644 index 4db1d04e4e04..000000000000 --- a/deploy-tidb-from-binary.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -title: Local Deployment from Binary Tarball -summary: Use the binary to deploy a TiDB cluster. -aliases: ['/docs/dev/deploy-tidb-from-binary/','/docs/dev/how-to/get-started/deploy-tidb-from-binary/'] ---- - -# Local Deployment from Binary Tarball - -This guide provides installation instructions for all TiDB components on a single developer machine. It is intended for evaluation purposes, and does not match the recommended usage for production systems. - -See also [testing environment](/test-deployment-from-binary-tarball.md) and [production environment](/production-deployment-from-binary-tarball.md) deployment. - -The following local TCP ports will be used: - -| Component | Port | Protocol | Description | -| --------- | ----- | -------- | ----------- | -| TiDB | 4000 | TCP | the communication port for the application and DBA tools | -| TiDB | 10080 | TCP | the communication port to report TiDB status | -| TiKV | 20160 | TCP | the TiKV communication port | -| PD | 2379 | TCP | the communication port between TiDB and PD | -| PD | 2380 | TCP | the inter-node communication port within the PD cluster | - -## Prepare - -This guide is for deployment on Linux only. It is recommended to use RHEL/CentOS 7.3 or higher. TiKV requires you to raise the open files limit: - -```bash -tidbuser="tidb" - -cat << EOF > /tmp/tidb.conf -$tidbuser soft nofile 1000000 -$tidbuser hard nofile 1000000 -EOF - -sudo cp /tmp/tidb.conf /etc/security/limits.d/ -sudo sysctl -w fs.file-max=1000000 -``` - -See the [production deployment](/production-deployment-from-binary-tarball.md) optional kernel tuning parameters. - -## Create a database running user account - -1. Log in to the machine using the `root` user account and create a database running user account (`tidb`) using the following command: - - ```bash - # useradd tidb -m - ``` - -2. Switch the user from `root` to `tidb` by using the following command. You can use this `tidb` user account to deploy your TiDB cluster. - - ```bash - # su - tidb - ``` - -## Download the official binary package - -``` -# Download the package. -$ wget https://download.pingcap.org/tidb-latest-linux-amd64.tar.gz -$ wget https://download.pingcap.org/tidb-latest-linux-amd64.sha256 - -# Check the file integrity. If the result is OK, the file is correct. -$ sha256sum -c tidb-latest-linux-amd64.sha256 - -# Extract the package. -$ tar -xzf tidb-latest-linux-amd64.tar.gz -$ cd tidb-latest-linux-amd64 -``` - -## Start - -Follow the steps below to start PD, TiKV and TiDB: - -1. Start PD. - - ```bash - $ ./bin/pd-server --data-dir=pd \ - --log-file=pd.log & - ``` - -2. Start TiKV. - - ```bash - $ ./bin/tikv-server --pd="127.0.0.1:2379" \ - --data-dir=tikv \ - --log-file=tikv.log & - ``` - -3. Start TiDB. - - ```bash - $ ./bin/tidb-server --store=tikv \ - --path="127.0.0.1:2379" \ - --log-file=tidb.log & - ``` - -4. Use the MySQL client to connect to TiDB. - - ```sh - $ mysql -h 127.0.0.1 -P 4000 -u root -D test - ``` diff --git a/dev-guide/deploy.md b/dev-guide/deploy.md index 3abd6f965896..8a9a89e6796e 100644 --- a/dev-guide/deploy.md +++ b/dev-guide/deploy.md @@ -5,7 +5,7 @@ draft: true # Build for Deployment -The easiest way to deploy TiDB is to use Docker Compose, see [TiDB Docker Compose Deployment](/deploy-test-cluster-using-docker-compose.md). For more deployment methods, see [TiDB deployment methods](/_index.md). +The easiest way to deploy TiDB is to use Docker Compose, see . ## Before you begin diff --git a/download-ecosystem-tools.md b/download-ecosystem-tools.md index 54bd50a1c74b..d75f986b65bd 100644 --- a/download-ecosystem-tools.md +++ b/download-ecosystem-tools.md @@ -18,7 +18,7 @@ If you want to download the latest version of [TiDB Binlog](/tidb-binlog/tidb-bi > **Note:** > -> `{version}` in the above download link indicates the version number of TiDB. For example, the download link for `v4.0.8` is `https://download.pingcap.org/tidb-v4.0.8-linux-amd64.tar.gz`. +> `{version}` in the above download link indicates the version number of TiDB. For example, the download link for `v5.1.0` is `https://download.pingcap.org/tidb-v5.1.0-linux-amd64.tar.gz`. ## TiDB Lightning @@ -30,23 +30,23 @@ Download [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) by using t > **Note:** > -> `{version}` in the above download link indicates the version number of TiDB Lightning. For example, the download link for `v4.0.8` is `https://download.pingcap.org/tidb-toolkit-v4.0.8-linux-amd64.tar.gz`. +> `{version}` in the above download link indicates the version number of TiDB Lightning. For example, the download link for `v5.1.0` is `https://download.pingcap.org/tidb-toolkit-v5.1.0-linux-amd64.tar.gz`. ## BR (backup and restore) Download [BR](/br/backup-and-restore-tool.md) by using the download link in the following table: -| Package name | OS | Architecure | SHA256 checksum | +| Package name | OS | Architecture | SHA256 checksum | |:---|:---|:---|:---| | `http://download.pingcap.org/tidb-toolkit-{version}-linux-amd64.tar.gz` | Linux | amd64 | `http://download.pingcap.org/tidb-toolkit-{version}-linux-amd64.sha256` | > **Note:** > -> `{version}` in the above download link indicates the version number of BR. For example, the download link for `v4.0.8-beta` is `http://download.pingcap.org/tidb-toolkit-v4.0.8-beta-linux-amd64.tar.gz`. +> `{version}` in the above download link indicates the version number of BR. For example, the download link for `v5.0.0-beta` is `http://download.pingcap.org/tidb-toolkit-v5.0.0-beta-linux-amd64.tar.gz`. ## TiDB DM (Data Migration) -Download [DM](https://docs.pingcap.com/tidb-data-migration/v1.0/overview) by using the download link in the following table: +Download [DM](https://docs.pingcap.com/tidb-data-migration/stable/overview) by using the download link in the following table: | Package name | OS | Architecture | SHA256 checksum | |:---|:---|:---|:---| @@ -54,7 +54,7 @@ Download [DM](https://docs.pingcap.com/tidb-data-migration/v1.0/overview) by usi > **Note:** > -> `{version}` in the above download link indicates the version number of DM. For example, the download link for `v1.0.6` is `https://download.pingcap.org/dm-v1.0.6-linux-amd64.tar.gz`. You can check the published DM versions in the [DM Release](https://github.com/pingcap/dm/releases) page. +> `{version}` in the above download link indicates the version number of DM. For example, the download link for `v2.0.4` is `https://download.pingcap.org/dm-v2.0.4-linux-amd64.tar.gz`. You can check the published DM versions in the [DM Release](https://github.com/pingcap/dm/releases) page. ## Dumpling @@ -66,21 +66,18 @@ Download [Dumpling](/dumpling-overview.md) from the links below: > **Note:** > -> The `{version}` in the download link is the version number of Dumpling. For example, the link for downloading the `v4.0.2` version of Dumpling is `https://download.pingcap.org/tidb-toolkit-v4.0.2-linux-amd64.tar.gz`. You can view the currently released versions in [Dumpling Releases](https://github.com/pingcap/dumpling/releases). +> The `{version}` in the download link is the version number of Dumpling. For example, the link for downloading the `v5.1.0` version of Dumpling is `https://download.pingcap.org/tidb-toolkit-v5.1.0-linux-amd64.tar.gz`. You can view the currently released versions in [Dumpling Releases](https://github.com/pingcap/dumpling/releases). > > Dumpling supports arm64 linux. You can replace `amd64` in the download link with `arm64`, which means the `arm64` version of Dumpling. -## Syncer, Loader, and Mydumper +## sync-diff-inspector -If you want to download the latest version of [Syncer](/syncer-overview.md), [Loader](/loader-overview.md), or [Mydumper](/mydumper-overview.md), directly download the tidb-enterprise-tools package, because all these tools are included in this package. +Download [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) from the links below: | Package name | OS | Architecture | SHA256 checksum | |:---|:---|:---|:---| | [tidb-enterprise-tools-nightly-linux-amd64.tar.gz](https://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz) | Linux | amd64 | [tidb-enterprise-tools-nightly-linux-amd64.sha256](https://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.sha256) | -This enterprise tools package includes all the following tools: +## TiCDC -- Syncer -- Loader -- Mydumper -- [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) +To download [TiCDC](/ticdc/ticdc-overview.md), refer to [Deploy TiCDC](/ticdc/deploy-ticdc.md). diff --git a/dumpling-overview.md b/dumpling-overview.md index 7495a0f79dae..6f532d11f800 100644 --- a/dumpling-overview.md +++ b/dumpling-overview.md @@ -1,6 +1,7 @@ --- title: Dumpling Overview summary: Use the Dumpling tool to export data from TiDB. +aliases: ['/docs/dev/mydumper-overview/','/docs/dev/reference/tools/mydumper/','/tidb/dev/mydumper-overview/'] --- # Dumpling Overview @@ -9,6 +10,12 @@ This document introduces the data export tool - [Dumpling](https://github.com/pi For backups of SST files (key-value pairs) or backups of incremental data that are not sensitive to latency, refer to [BR](/br/backup-and-restore-tool.md). For real-time backups of incremental data, refer to [TiCDC](/ticdc/ticdc-overview.md). +> **Note:** +> +> PingCAP previously maintained a fork of the [mydumper project](https://github.com/maxbube/mydumper) with enhancements specific to TiDB. This fork has since been replaced by [Dumpling](/dumpling-overview.md), which has been rewritten in Go, and supports more optimizations that are specific to TiDB. It is strongly recommended that you use Dumpling instead of mydumper. +> +> For the overview of Mydumper, refer to [v4.0 Mydumper documentation](https://docs.pingcap.com/tidb/v4.0/backup-and-restore-using-mydumper-lightning). + ## Improvements of Dumpling compared with Mydumper 1. Support exporting data in multiple formats, including SQL and CSV @@ -28,7 +35,9 @@ For detailed usage of Dumpling, use the `--help` option or refer to [Option list When using Dumpling, you need to execute the export command on a running cluster. This document assumes that there is a TiDB instance on the `127.0.0.1:4000` host and that this TiDB instance has a root user without a password. -Dumpling is included in the tidb-toolkit installation package and can be [download here](/download-ecosystem-tools.md#dumpling). +You can get Dumpling using [TiUP](/tiup/tiup-overview.md) by running `tiup install dumpling`. Afterwards, you can use `tiup dumpling ...` to run Dumpling. + +Dumpling is also included in the tidb-toolkit installation package and can be [download here](/download-ecosystem-tools.md#dumpling). ## Export data from TiDB/MySQL @@ -51,12 +60,23 @@ dumpling \ -P 4000 \ -h 127.0.0.1 \ --filetype sql \ - --threads 32 \ + --t 8 \ -o /tmp/test \ + -r 200000 \ -F 256MiB ``` -In the above command, `-h`, `-P` and `-u` mean address, port and user, respectively. If password authentication is required, you can pass it to Dumpling with `-p $YOUR_SECRET_PASSWORD`. +In the command above: + ++ The `-h`, `-p`, and `-u` option respectively mean the address, the port, and the user. If a password is required for authentication, you can use `-p $YOUR_SECRET_PASSWORD` to pass the password to Dumpling. ++ The `-o` option specifies the export directory of the storage, which supports a local file path or a [URL of an external storage](/br/backup-and-restore-storages.md). ++ The `t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. ++ The `-r` option specifies the maximum number of rows in a single file. With this option specified, Dumpling enables the in-table concurrency to speed up the export and reduce the memory usage. ++ The `-F` option is used to specify the maximum size of a single file (the unit here is `MiB`; inputs like `5GiB` or `8KB` are also acceptable). It is recommended to keep its value to 256 MiB or less if you plan to use TiDB Lightning to load this file into a TiDB instance. + +> **Note:** +> +> If the size of a single exported table exceeds 10 GB, it is **strongly recommended to use** the `-r` and `-F` options. ### Export to CSV files @@ -159,9 +179,9 @@ export AWS_ACCESS_KEY_ID=${AccessKey} export AWS_SECRET_ACCESS_KEY=${SecretKey} ``` -Dumpling also supports reading credential files from `~/.aws/credentials`. For more Dumpling configuration, see the configuration of [BR storages](/br/backup-and-restore-storages.md), which is consistent with the Dumpling configuration. +Dumpling also supports reading credential files from `~/.aws/credentials`. For more Dumpling configuration, see the configuration of [External storages](/br/backup-and-restore-storages.md). -When you back up data using Dumpling, explicitly specify the `--s3.region` parameter, which means the region of the S3 storage: +When you back up data using Dumpling, explicitly specify the `--s3.region` parameter, which means the region of the S3 storage (for example, `ap-northeast-1`): {{< copyable "shell-regular" >}} @@ -170,6 +190,7 @@ When you back up data using Dumpling, explicitly specify the `--s3.region` param -u root \ -P 4000 \ -h 127.0.0.1 \ + -r 200000 \ -o "s3://${Bucket}/${Folder}" \ --s3.region "${region}" ``` @@ -191,7 +212,7 @@ By default, Dumpling exports all databases except system databases (including `m --where "id < 100" ``` -The above command exports the data that matches `id < 100` from each table. +The above command exports the data that matches `id < 100` from each table. Note that you cannot use the `--where` parameter together with `--sql`. #### Use the `--filter` option to filter data @@ -205,6 +226,7 @@ Dumpling can filter specific databases or tables by specifying the table filter -P 4000 \ -h 127.0.0.1 \ -o /tmp/test \ + -r 200000 \ --filter "employees.*" \ --filter "*.WorkOrder" ``` @@ -229,21 +251,20 @@ Examples: The exported file is stored in the `./export-` directory by default. Commonly used options are as follows: -- `-o` is used to select the directory where the exported files are stored. -- `-F` option is used to specify the maximum size of a single file (the unit here is `MiB`; inputs like `5GiB` or `8KB` are also acceptable). -- `-r` option is used to specify the maximum number of records (or the number of rows in the database) for a single file. When it is enabled, Dumpling enables concurrency in the table to improve the speed of exporting large tables. +- The `t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. +- The `-r` option specifies the maximum number of records (or the number of rows in the database) for a single file. When it is enabled, Dumpling enables concurrency in the table to improve the speed of exporting large tables. -With the above options specified, Dumpling can have a higher degree of parallelism. +With the above options specified, Dumpling can have a quicker speed of data export. ### Adjust Dumpling's data consistency options > **Note:** > -> In most scenarios, you do not need to adjust the default data consistency options of Dumpling. +> In most scenarios, you do not need to adjust the default data consistency options of Dumpling (the default value is `auto`). -Dumpling uses the `--consistency ` option to control the way in which data is exported for "consistency assurance". For TiDB, data consistency is guaranteed by getting a snapshot of a certain timestamp by default (i.e. `--consistency snapshot`). When using snapshot for consistency, you can use the `--snapshot` option to specify the timestamp to be backed up. You can also use the following levels of consistency: +Dumpling uses the `--consistency ` option to control the way in which data is exported for "consistency assurance". When using snapshot for consistency, you can use the `--snapshot` option to specify the timestamp to be backed up. You can also use the following levels of consistency: -- `flush`: Use [`FLUSH TABLES WITH READ LOCK`](https://dev.mysql.com/doc/refman/8.0/en/flush.html#flush-tables-with-read-lock) to ensure consistency. +- `flush`: Use [`FLUSH TABLES WITH READ LOCK`](https://dev.mysql.com/doc/refman/8.0/en/flush.html#flush-tables-with-read-lock) to temporarily interrupt the DML and DDL operations of the replica database, to ensure the global consistency of the backup connection, and to record the binlog position (POS) information. The lock is released after all backup connections start transactions. It is recommended to perform full backups during off-peak hours or on the MySQL replica database. - `snapshot`: Get a consistent snapshot of the specified timestamp and export it. - `lock`: Add read locks on all tables to be exported. - `none`: No guarantee for consistency. @@ -272,7 +293,7 @@ ls -lh /tmp/test | awk '{print $5 "\t" $9}' Dumpling can export the data of a certain [tidb_snapshot](/read-historical-data.md#how-tidb-reads-data-from-history-versions) with the `--snapshot` option specified. -The `--snapshot` option can be set to a TSO (the `Position` field output by the `SHOW MASTER STATUS` command) or a valid time of the `datetime` data type, for example: +The `--snapshot` option can be set to a TSO (the `Position` field output by the `SHOW MASTER STATUS` command) or a valid time of the `datetime` data type (in the form of `YYYY-MM-DD hh:mm:ss`), for example: {{< copyable "shell-regular" >}} @@ -287,20 +308,20 @@ The TiDB historical data snapshots when the TSO is `417773951312461825` and the When Dumpling is exporting a large single table from TiDB, Out of Memory (OOM) might occur because the exported data size is too large, which causes connection abort and export failure. You can use the following parameters to reduce the memory usage of TiDB: -+ Setting `--rows` to split the data to be exported into chunks. This reduces the memory overhead of TiDB's data scan and enables concurrent table data dump to improve export efficiency. -+ Reduce the value of `--tidb-mem-quota-query` to `8589934592` (8 GB) or lower. `--tidb-mem-quota-query` controls the memory usage of a TiDB query and its default value is 32 GB. ++ Setting `-r` to split the data to be exported into chunks. This reduces the memory overhead of TiDB's data scan and enables concurrent table data dump to improve export efficiency. ++ Reduce the value of `--tidb-mem-quota-query` to `8589934592` (8 GB) or lower. `--tidb-mem-quota-query` controls the memory usage of a single query statement in TiDB. + Adjust the `--params "tidb_distsql_scan_concurrency=5"` parameter. [`tidb_distsql_scan_concurrency`](/system-variables.md#tidb_distsql_scan_concurrency) is a session variable which controls the concurrency of the scan operations in TiDB. ### TiDB GC settings when exporting a large volume of data -When exporting data from TiDB, if the TiDB version is greater than v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. But for TiDB earlier than v4.0.0, you need to manually modify the GC time. +When exporting data from TiDB, if the TiDB version is greater than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. In other scenarios, if the data size is very large, to avoid export failure due to GC during the export process, you can extend the GC time in advance: {{< copyable "sql" >}} ```sql -update mysql.tidb set VARIABLE_VALUE = '720h' where VARIABLE_NAME = 'tikv_gc_life_time'; +SET GLOBAL tidb_gc_life_time = '720h'; ``` After your operation is completed, set the GC time back (the default value is `10m`): @@ -308,50 +329,50 @@ After your operation is completed, set the GC time back (the default value is `1 {{< copyable "sql" >}} ```sql -update mysql.tidb set VARIABLE_VALUE = '10m' where VARIABLE_NAME = 'tikv_gc_life_time'; +SET GLOBAL tidb_gc_life_time = '10m'; ``` -Finally, all the exported data can be imported back to TiDB using [Lightning](/tidb-lightning/tidb-lightning-backends.md). +Finally, all the exported data can be imported back to TiDB using [TiDB Lightning](/tidb-lightning/tidb-lightning-backends.md). ## Option list of Dumpling -| Options | Usage | Default value | -| --------| --- | --- | -| `-V` or `--version` | Output the Dumpling version and exit directly | -| `-B` or `--database` | Export specified databases | -| `-T` or `--tables-list` | Export specified tables | -| `-f` or `--filter` | Export tables that match the filter pattern. For the filter syntax, see [table-filter](/table-filter.md). | `"\*.\*"` (export all databases or tables) | -| `--case-sensitive` | whether table-filter is case-sensitive | false (case-insensitive) | -| `-h` or `--host` | The IP address of the connected database host | "127.0.0.1" | -| `-t` or `--threads` | The number of concurrent backup threads | 4 | -| `-r` or `--rows` | Divide the table into specified rows of data (generally applicable for concurrent operations of splitting a large table into multiple files. | -| `-L` or `--logfile` | Log output address. If it is empty, the log will be output to the console | "" | -| `--loglevel` | Log level {debug,info,warn,error,dpanic,panic,fatal} | "info" | -| `--logfmt` | Log output format {text,json} | "text" | -| `-d` or `--no-data` | Do not export data (suitable for scenarios where only the schema is exported) | -| `--no-header` | Export CSV files of the tables without generating header | -| `-W` or `--no-views` | Do not export the views | true | -| `-m` or `--no-schemas` | Do not export the schema with only the data exported | -| `-s` or `--statement-size` | Control the size of the `INSERT` statements; the unit is bytes | -| `-F` or `--filesize` | The file size of the divided tables. The unit must be specified such as `128B`, `64KiB`, `32MiB`, and `1.5GiB`. | -| `--filetype` | Exported file type (csv/sql) | "sql" | -| `-o` or `--output` | Exported file path | "./export-${time}" | -| `-S` or `--sql` | Export data according to the specified SQL statement. This command does not support concurrent export. | -| `--consistency` | flush: use FTWRL before the dump
snapshot: dump the TiDB data of a specific snapshot of a TSO
lock: execute `lock tables read` on all tables to be dumped
none: dump without adding locks, which cannot guarantee consistency
auto: MySQL defaults to using flush, TiDB defaults to using snapshot | "auto" | -| `--snapshot` | Snapshot TSO; valid only when `consistency=snapshot` | -| `--where` | Specify the scope of the table backup through the `where` condition | -| `-p` or `--password` | The password of the connected database host | -| `-P` or `--port` | The port of the connected database host | 4000 | -| `-u` or `--user` | The username of the connected database host | "root" | -| `--dump-empty-database` | Export the `CREATE DATABASE` statements of the empty databases | true | -| `--ca` | The address of the certificate authority file for TLS connection | -| `--cert` | The address of the client certificate file for TLS connection | -| `--key` | The address of the client private key file for TLS connection | -| `--csv-delimiter` | Delimiter of character type variables in CSV files | '"' | -| `--csv-separator` | Separator of each value in CSV files | ',' | -| `--csv-null-value` | Representation of null values in CSV files | "\\N" | -| `--escape-backslash` | Use backslash (`\`) to escape special characters in the export file | true | -| `--output-filename-template` | The filename templates represented in the format of [golang template](https://golang.org/pkg/text/template/#hdr-Arguments)
Support the `{{.DB}}`, `{{.Table}}`, and `{{.Index}}` arguments
The three arguments represent the database name, table name, and chunk ID of the data file | '{{.DB}}.{{.Table}}.{{.Index}}' | -| `--status-addr` | Dumpling's service address, including the address for Prometheus to pull metrics and pprof debugging | ":8281" | -| `--tidb-mem-quota-query` | The memory limit of exporting SQL statements by a single line of Dumpling command, the unit is byte, and the default value is 32 GB | 34359738368 | -| `--params` | Specifies the session variable for the connection of the database to be exported. The required format is `"character_set_client=latin1,character_set_connection=latin1"` | +| Options | Usage | Default value | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | +| `-V` or `--version` | Output the Dumpling version and exit directly | +| `-B` or `--database` | Export specified databases | +| `-T` or `--tables-list` | Export specified tables | +| `-f` or `--filter` | Export tables that match the filter pattern. For the filter syntax, see [table-filter](/table-filter.md). | `[\*.\*,!/^(mysql|sys|INFORMATION_SCHEMA|PERFORMANCE_SCHEMA|METRICS_SCHEMA|INSPECTION_SCHEMA)$/.\*]` (export all databases or tables excluding system schemas) | +| `--case-sensitive` | whether table-filter is case-sensitive | false (case-insensitive) | +| `-h` or `--host` | The IP address of the connected database host | "127.0.0.1" | +| `-t` or `--threads` | The number of concurrent backup threads | 4 | +| `-r` or `--rows` | Divide the table into specified rows of data (generally applicable for concurrent operations of splitting a large table into multiple files. | +| `-L` or `--logfile` | Log output address. If it is empty, the log will be output to the console | "" | +| `--loglevel` | Log level {debug,info,warn,error,dpanic,panic,fatal} | "info" | +| `--logfmt` | Log output format {text,json} | "text" | +| `-d` or `--no-data` | Do not export data (suitable for scenarios where only the schema is exported) | +| `--no-header` | Export CSV files of the tables without generating header | +| `-W` or `--no-views` | Do not export the views | true | +| `-m` or `--no-schemas` | Do not export the schema with only the data exported | +| `-s` or `--statement-size` | Control the size of the `INSERT` statements; the unit is bytes | +| `-F` or `--filesize` | The file size of the divided tables. The unit must be specified such as `128B`, `64KiB`, `32MiB`, and `1.5GiB`. | +| `--filetype` | Exported file type (csv/sql) | "sql" | +| `-o` or `--output` | The path of exported local files or [the URL of the external storage](/br/backup-and-restore-storages.md) | "./export-${time}" | +| `-S` or `--sql` | Export data according to the specified SQL statement. This command does not support concurrent export. | +| `--consistency` | flush: use FTWRL before the dump
snapshot: dump the TiDB data of a specific snapshot of a TSO
lock: execute `lock tables read` on all tables to be dumped
none: dump without adding locks, which cannot guarantee consistency
auto: use --consistency flush for MySQL; use --consistency snapshot for TiDB | "auto" | +| `--snapshot` | Snapshot TSO; valid only when `consistency=snapshot` | +| `--where` | Specify the scope of the table backup through the `where` condition | +| `-p` or `--password` | The password of the connected database host | +| `-P` or `--port` | The port of the connected database host | 4000 | +| `-u` or `--user` | The username of the connected database host | "root" | +| `--dump-empty-database` | Export the `CREATE DATABASE` statements of the empty databases | true | +| `--ca` | The address of the certificate authority file for TLS connection | +| `--cert` | The address of the client certificate file for TLS connection | +| `--key` | The address of the client private key file for TLS connection | +| `--csv-delimiter` | Delimiter of character type variables in CSV files | '"' | +| `--csv-separator` | Separator of each value in CSV files | ',' | +| `--csv-null-value` | Representation of null values in CSV files | "\\N" | +| `--escape-backslash` | Use backslash (`\`) to escape special characters in the export file | true | +| `--output-filename-template` | The filename templates represented in the format of [golang template](https://golang.org/pkg/text/template/#hdr-Arguments)
Support the `{{.DB}}`, `{{.Table}}`, and `{{.Index}}` arguments
The three arguments represent the database name, table name, and chunk ID of the data file | '{{.DB}}.{{.Table}}.{{.Index}}' | +| `--status-addr` | Dumpling's service address, including the address for Prometheus to pull metrics and pprof debugging | ":8281" | +| `--tidb-mem-quota-query` | The memory limit of exporting SQL statements by a single line of Dumpling command, and the unit is byte. For v4.0.10 or later versions, if you do not set this parameter, TiDB uses the value of the `mem-quota-query` configuration item as the memory limit value by default. For versions earlier than v4.0.10, the parameter value defaults to 32 GB. | 34359738368 | +| `--params` | Specifies the session variable for the connection of the database to be exported. The required format is `"character_set_client=latin1,character_set_connection=latin1"` | diff --git a/dynamic-config.md b/dynamic-config.md index 6d9930d86ce6..af0791432b65 100644 --- a/dynamic-config.md +++ b/dynamic-config.md @@ -33,7 +33,6 @@ show config; | Type | Instance | Name | Value | +------+-----------------+-----------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | tidb | 127.0.0.1:4001 | advertise-address | 127.0.0.1 | -| tidb | 127.0.0.1:4001 | alter-primary-key | false | | tidb | 127.0.0.1:4001 | binlog.binlog-socket | | | tidb | 127.0.0.1:4001 | binlog.enable | false | | tidb | 127.0.0.1:4001 | binlog.ignore-error | false | @@ -66,18 +65,22 @@ When using the `set config` statement, you can modify the configuration of a sin - Modify the configuration of all TiKV instances: - {{< copyable "sql" >}} +> **Note:** +> +> It is recommended to wrap variable names in backticks. - ```sql - set config tikv split.qps-threshold=1000 - ``` +{{< copyable "sql" >}} + +```sql +set config tikv `split.qps-threshold`=1000 +``` - Modify the configuration of a single TiKV instance: {{< copyable "sql" >}} ```sql - set config "127.0.0.1:20180" split.qps-threshold=1000 + set config "127.0.0.1:20180" `split.qps-threshold`=1000 ``` If the modification is successful, `Query OK` is returned: @@ -91,7 +94,7 @@ If an error occurs during the batch modification, a warning is returned: {{< copyable "sql" >}} ```sql -set config tikv log-level='warn'; +set config `tikv log-level`='warn'; ``` ```sql @@ -192,9 +195,10 @@ The following TiKV configuration items can be modified online: | `{db-name}.{cf-name}.titan.blob-run-mode` | The mode of processing blob files | | `storage.block-cache.capacity` | The size of shared block cache (supported since v4.0.3) | | `backup.num-threads` | The number of backup threads (supported since v4.0.3) | -| `split.qps-threshold` | The threshold to execute `load-base-split` on a Region. If the read QPS is higher than this value for 10 consecutive seconds, this Region should be split. | -| `split.split-balance-score` | The parameter of `load-base-split`, which ensures the load of the two split Regions is as balanced as possible | -| `split.split-contained-score` | The parameter of `load-base-split`, which reduces the cross-Region visits after split as much as possible | +| `split.qps-threshold` | The threshold to execute `load-base-split` on a Region. If the QPS of read requests for a Region exceeds `qps-threshold` for a consecutive period of time, this Region should be split.| +| `split.byte-threshold` | The threshold to execute `load-base-split` on a Region. If the traffic of read requests for a Region exceeds the `byte-threshold` for a consecutive period of time, this Region should be split. | +| `split.split-balance-score` | The parameter of `load-base-split`, which ensures the load of the two split Regions is as balanced as possible. The smaller the value is, the more balanced the load is. But setting it too small might cause split failure. | +| `split.split-contained-score` | The parameter of `load-base-split`. The smaller the value, the fewer cross-Region visits after Region split. | In the table above, parameters with the `{db-name}` or `{db-name}.{cf-name}` prefix are configurations related to RocksDB. The optional values of `db-name` are `rocksdb` and `raftdb`. @@ -212,7 +216,7 @@ You can modify the PD configurations using the following statement: {{< copyable "sql" >}} ```sql -set config pd log.level='info' +set config pd `log.level`='info' ``` If the modification is successful, `Query OK` is returned: diff --git a/ecosystem-tool-user-case.md b/ecosystem-tool-user-case.md index f338369cc14e..a497aa144836 100644 --- a/ecosystem-tool-user-case.md +++ b/ecosystem-tool-user-case.md @@ -26,11 +26,11 @@ If the full data volume is large (at the TB level), you can first use [Dumpling] If you need to back up a TiDB cluster or restore backed up data to the cluster, use [BR](/br/backup-and-restore-tool.md) (Backup & Restore). -In addition, BR can also be used to perform [incremental backup](/br/backup-and-restore-tool.md#back-up-incremental-data) and [incremental restore](/br/backup-and-restore-tool.md#restore-incremental-data) of TiDB cluster data. +In addition, BR can also be used to perform [incremental backup](/br/use-br-command-line-tool.md#back-up-incremental-data) and [incremental restore](/br/use-br-command-line-tool.md#restore-incremental-data) of TiDB cluster data. -## Migrate data from TiDB +## Migrate data to TiDB -If you need to migrate data from a TiDB cluster to MySQL or to another TiDB cluster, use [Dumpling](/dumpling-overview.md) to export full data from TiDB as SQL dump files, and then use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to import data to MySQL or another TiDB cluster. +If you need to migrate data from a TiDB cluster to another TiDB cluster, use [Dumpling](/dumpling-overview.md) to export full data from TiDB as SQL dump files, and then use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to import data to another TiDB cluster. If you also need to migrate incremental data, use [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md). diff --git a/ecosystem-tool-user-guide.md b/ecosystem-tool-user-guide.md index d40ebeb8fc35..30512618774c 100644 --- a/ecosystem-tool-user-guide.md +++ b/ecosystem-tool-user-guide.md @@ -18,6 +18,10 @@ The following are the basics of Dumpling: - Supported TiDB versions: all versions - Kubernetes support: No +> **Note:** +> +> PingCAP previously maintained a fork of the [mydumper project](https://github.com/maxbube/mydumper) with enhancements specific to TiDB. This fork has since been replaced by [Dumpling](/dumpling-overview.md), which has been rewritten in Go, and supports more optimizations that are specific to TiDB. It is strongly recommended that you use Dumpling instead of mydumper. + ## Full data import [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) (Lightning) is a tool used for the full import of large amounts of data into a TiDB cluster. Currently, TiDB Lightning supports reading SQL dump exported via Dumpling or CSV data source. @@ -64,7 +68,7 @@ The following are the basics of TiDB Binlog: ### Data migration -[TiDB Data Migration](https://docs.pingcap.com/tidb-data-migration/stable) (DM) is an integrated data replication task management platform that supports the full data migration and the incremental data migration from MySQL/MariaDB to TiDB. +[TiDB Data Migration](https://docs.pingcap.com/tidb-data-migration/stable) (DM) is an integrated data replication task management platform that supports the full data migration and the incremental data replication from MySQL/MariaDB to TiDB. The following are the basics of DM: @@ -79,7 +83,7 @@ If the data volume is at the TB level, take the following steps: 1. Use [Dumpling](/dumpling-overview.md) to export the full data from MySQL/MariaDB. 2. Use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to import the data exported in Step 1 to the TiDB cluster. -3. Use DM to migrate the incremental data from MySQL/MariaDB to TiDB. +3. Use DM to replicate the incremental data from MySQL/MariaDB to TiDB. > **Note:** > diff --git a/enable-tls-between-clients-and-servers.md b/enable-tls-between-clients-and-servers.md index 417c4954375a..cf2bda67b464 100644 --- a/enable-tls-between-clients-and-servers.md +++ b/enable-tls-between-clients-and-servers.md @@ -4,7 +4,7 @@ summary: Use the encrypted connection to ensure data security. aliases: ['/docs/dev/enable-tls-between-clients-and-servers/','/docs/dev/how-to/secure/enable-tls-clients/','/docs/dev/encrypted-connections-with-tls-protocols/'] --- -# Enable TLS Between TiDB Clients and Servers +# Enable TLS between TiDB Clients and Servers Non-encrypted connection between TiDB's server and client is used by default, which enables third parties that monitor channel traffic to know the data sent and received between the server and the client, including but not limited to query content, query results, and so on. If a channel is untrustworthy (such as if the client is connected to the TiDB server via a public network), then a non-encrypted connection is prone to information leakage. In this case, for security reasons, it is recommended to use an encrypted connection. @@ -183,4 +183,4 @@ The newly loaded certificate, key, and CA take effect on the connection that is ### See also -- [Enable TLS Between TiDB Components](/enable-tls-between-components.md)。 +- [Enable TLS Between TiDB Components](/enable-tls-between-components.md). diff --git a/enable-tls-between-components.md b/enable-tls-between-components.md index da29f4014559..292f60cfd1cf 100644 --- a/enable-tls-between-components.md +++ b/enable-tls-between-components.md @@ -27,45 +27,49 @@ Currently, it is not supported to only enable encrypted transmission of some spe 2. Configure certificates. - To enable mutual authentication among TiDB components, configure the certificates of TiDB, TiKV, and PD as follows. + To enable mutual authentication among TiDB components, configure the certificates of TiDB, TiKV, and PD as follows. - - TiDB + - TiDB Configure in the configuration file or command-line arguments: ```toml [security] - # Path of file that contains list of trusted SSL CAs for connection with cluster components. + # Path of the file that contains list of trusted SSL CAs for connection with cluster components. cluster-ssl-ca = "/path/to/ca.pem" - # Path of file that contains X509 certificate in PEM format for connection with cluster components. + # Path of the file that contains X509 certificate in PEM format for connection with cluster components. cluster-ssl-cert = "/path/to/tidb-server.pem" - # Path of file that contains X509 key in PEM format for connection with cluster components. + # Path of the file that contains X509 key in PEM format for connection with cluster components. cluster-ssl-key = "/path/to/tidb-server-key.pem" ``` - - TiKV + - TiKV Configure in the configuration file or command-line arguments, and set the corresponding URL to `https`: ```toml [security] - # set the path for certificates. Empty string means disabling secure connections. + ## The path for certificates. An empty string means that secure connections are disabled. + # Path of the file that contains a list of trusted SSL CAs. If it is set, the following settings `cert_path` and `key_path` are also needed. ca-path = "/path/to/ca.pem" + # Path of the file that contains X509 certificate in PEM format. cert-path = "/path/to/tikv-server.pem" + # Path of the file that contains X509 key in PEM format. key-path = "/path/to/tikv-server-key.pem" ``` - - PD + - PD Configure in the configuration file or command-line arguments, and set the corresponding URL to `https`: ```toml [security] - # Path of file that contains list of trusted SSL CAs. If set, following four settings shouldn't be empty + ## The path for certificates. An empty string means that secure connections are disabled. + # Path of the file that contains a list of trusted SSL CAs. If it is set, the following settings `cert_path` and `key_path` are also needed. cacert-path = "/path/to/ca.pem" - # Path of file that contains X509 certificate in PEM format. + # Path of the file that contains X509 certificate in PEM format. cert-path = "/path/to/pd-server.pem" - # Path of file that contains X509 key in PEM format. + # Path of the file that contains X509 key in PEM format. key-path = "/path/to/pd-server-key.pem" ``` @@ -75,11 +79,12 @@ Currently, it is not supported to only enable encrypted transmission of some spe ```toml [security] - # Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty + ## The path for certificates. An empty string means that secure connections are disabled. + # Path of the file that contains a list of trusted SSL CAs. If it is set, the following settings `cert_path` and `key_path` are also needed. ca_path = "/path/to/ca.pem" - # Path of file that contains X509 certificate in PEM format. + # Path of the file that contains X509 certificate in PEM format. cert_path = "/path/to/tiflash-server.pem" - # Path of file that contains X509 key in PEM format. + # Path of the file that contains X509 key in PEM format. key_path = "/path/to/tiflash-server-key.pem" ``` @@ -87,9 +92,11 @@ Currently, it is not supported to only enable encrypted transmission of some spe ```toml [security] - # Sets the path for certificates. The empty string means that secure connections are disabled. + # Path of the file that contains a list of trusted SSL CAs. If it is set, the following settings `cert_path` and `key_path` are also needed. ca-path = "/path/to/ca.pem" + # Path of the file that contains X509 certificate in PEM format. cert-path = "/path/to/tiflash-server.pem" + # Path of the file that contains X509 key in PEM format. key-path = "/path/to/tiflash-server-key.pem" ``` @@ -103,7 +110,7 @@ Currently, it is not supported to only enable encrypted transmission of some spe cdc server --pd=https://127.0.0.1:2379 --log-file=ticdc.log --addr=0.0.0.0:8301 --advertise-addr=127.0.0.1:8301 --ca=/path/to/ca.pem --cert=/path/to/ticdc-cert.pem --key=/path/to/ticdc-key.pem ``` - Now, encrypted transmission among TiDB components is enabled. + Now, encrypted transmission among TiDB components is enabled. > **Note:** > diff --git a/encryption-at-rest.md b/encryption-at-rest.md index 6063f1187401..251a31f12d3b 100644 --- a/encryption-at-rest.md +++ b/encryption-at-rest.md @@ -8,7 +8,7 @@ aliases: ['/docs/dev/encryption at rest/'] Encryption at rest means that data is encrypted when it is stored. For databases, this feature is also referred to as TDE (transparent data encryption). This is opposed to encryption in flight (TLS) or encryption in use (rarely used). Different things could be doing encryption at rest (SSD drive, file system, cloud vendor, etc), but by having TiKV do the encryption before storage this helps ensure that attackers must authenticate with the database to gain access to data. For example, when an attacker gains access to the physical machine, data cannot be accessed by copying files on disk. -TiKV supports encryption at rest starting from v4.0.0. The feature allows TiKV to transparently encrypt data files using [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) in [CTR](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation) mode. To enable encryption at rest, an encryption key must be provided by user and this key is called master key. The master key can be provided via AWS KMS (recommended), or specifying a key stored as plaintext in a file. TiKV automatically rotates data keys that it used to encrypt actual data files. Manually rotating the master key can be done occasionally. Note that encryption at rest only encrypts data at rest (i.e. on disk) and not while data is transferred over network. It is advised to use TLS together with encryption at rest. +TiKV supports encryption at rest starting from v4.0.0. The feature allows TiKV to transparently encrypt data files using [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) in [CTR](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation) mode. To enable encryption at rest, an encryption key must be provided by user and this key is called master key. The master key can be provided via AWS KMS (recommended), or specifying a key stored as plaintext in a file. TiKV automatically rotates data keys that it used to encrypt actual data files. Manually rotating the master key can be done occasionally. Note that encryption at rest only encrypts data at rest (namely, on disk) and not while data is transferred over network. It is advised to use TLS together with encryption at rest. Also from v4.0.0, BR supports S3 server-side encryption (SSE) when backing up to S3. A customer owned AWS KMS key can also be used together with S3 server-side encryption. @@ -33,9 +33,9 @@ TiKV currently supports encrypting data using AES128, AES192 or AES256, in CTR m The same master key can be shared by multiple instances of TiKV. The recommended way to provide a master key in production is via AWS KMS. Create a customer master key (CMK) through AWS KMS, and then provide the CMK key ID to TiKV in the configuration file. The TiKV process needs access to the KMS CMK while it is running, which can be done by using an [IAM role](https://aws.amazon.com/iam/). If TiKV fails to get access to the KMS CMK, it will fail to start or restart. Refer to AWS documentation for [KMS](https://docs.aws.amazon.com/kms/index.html) and [IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html) usage. -Alternatively, if using custom key is desired, supplying the master key via file is also supported. The file needs to contain a 256 bits (or 32 bytes) key encoded as hex string. The file should end with a newline (i.e. `\n`) and contain nothing else. Persisting the key on disk, however, leaks the key, so the key file is only suitable to be stored on the `tempfs` in RAM. +Alternatively, if using custom key is desired, supplying the master key via file is also supported. The file must contain a 256 bits (or 32 bytes) key encoded as hex string, end with a newline (namely, `\n`), and contain nothing else. Persisting the key on disk, however, leaks the key, so the key file is only suitable to be stored on the `tempfs` in RAM. -Data keys are generated by TiKV and passed to the underlying storage engine (i.e. RocksDB). All files written by RocksDB, including SST files, WAL files, and the MANIFEST file, are encrypted by the current data key. Other temporary files used by TiKV that may include user data are also encrypted using the same data key. Data keys are automatically rotated by TiKV every week by default, but the period is configurable. On key rotation, TiKV does not rewrite all existing files to replace the key, but RocksDB compaction are expected to rewrite old data into new data files, with the most recent data key, if the cluster gets constant write workload. TiKV keeps track of the key and encryption method used to encrypt each of the files and use the information to decrypt the content on reads. +Data keys are generated by TiKV and passed to the underlying storage engine (namely, RocksDB). All files written by RocksDB, including SST files, WAL files, and the MANIFEST file, are encrypted by the current data key. Other temporary files used by TiKV that may include user data are also encrypted using the same data key. Data keys are automatically rotated by TiKV every week by default, but the period is configurable. On key rotation, TiKV does not rewrite all existing files to replace the key, but RocksDB compaction are expected to rewrite old data into new data files, with the most recent data key, if the cluster gets constant write workload. TiKV keeps track of the key and encryption method used to encrypt each of the files and use the information to decrypt the content on reads. Regardless of data encryption method, data keys are encrypted using AES256 in GCM mode for additional authentication. This required the master key to be 256 bits (32 bytes), when passing from file instead of KMS. @@ -51,7 +51,7 @@ data-key-rotation-period = 7d Possible values for `data-encryption-method` are "aes128-ctr", "aes192-ctr", "aes256-ctr" and "plaintext". The default value is "plaintext", which means encryption is not turned on. `data-key-rotation-period` defines how often TiKV rotates the data key. Encryption can be turned on for a fresh TiKV cluster, or an existing TiKV cluster, though only data written after encryption is enabled is guaranteed to be encrypted. To disable encryption, remove `data-encryption-method` in the configuration file, or reset it to "plaintext", and restart TiKV. To change encryption method, update `data-encryption-method` in the configuration file and restart TiKV. -The master key has to be specified if encryption is enabled (i.e. `data-encryption-method` is not "plaintext"). To specify a AWS KMS CMK as master key, add the `encryption.master-key` section after the `encryption` section: +The master key has to be specified if encryption is enabled (that is,`data-encryption-method` is not "plaintext"). To specify a AWS KMS CMK as master key, add the `encryption.master-key` section after the `encryption` section: ``` [security.encryption.master-key] @@ -130,7 +130,7 @@ To enable S3 server-side encryption when backup to S3 using BR, pass `--s3.sse` ./br backup full --pd --storage "s3:///" --s3.sse aws:kms ``` -To use a custom AWS KMS CMK that you created and owned, pass `--s3.sse-kms-key-id` in addition. In this case, both the BR process and all the TiKV nodes in the cluster would need access to the KMS CMK (e.g. via AWS IAM), and the KMS CMK needs to be in the same AWS region as the S3 bucket used to store the backup. It is advised to grant access to the KMS CMK to BR process and TiKV nodes via AWS IAM. Refer to AWS documentation for usage of [IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html). For example: +To use a custom AWS KMS CMK that you created and owned, pass `--s3.sse-kms-key-id` in addition. In this case, both the BR process and all the TiKV nodes in the cluster would need access to the KMS CMK (for example, via AWS IAM), and the KMS CMK needs to be in the same AWS region as the S3 bucket used to store the backup. It is advised to grant access to the KMS CMK to BR process and TiKV nodes via AWS IAM. Refer to AWS documentation for usage of [IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html). For example: ``` ./br backup full --pd --storage "s3:///" --s3.region --s3.sse aws:kms --s3.sse-kms-key-id 0987dcba-09fe-87dc-65ba-ab0987654321 diff --git a/error-codes.md b/error-codes.md index 856ed792973b..6867b930ba12 100644 --- a/error-codes.md +++ b/error-codes.md @@ -10,8 +10,7 @@ This document describes the problems encountered during the use of TiDB and prov ## Error codes -TiDB is compatible with the error codes in MySQL, and in most cases returns the same error code as MySQL. For a list of error codes for MySQL, see [Server Error Message Reference](https://dev.mysql.com/doc/refman/5.7/en/server-error-reference.html). -In addition, TiDB has the following unique error codes: +TiDB is compatible with the error codes in MySQL, and in most cases returns the same error code as MySQL. For a list of error codes for MySQL, see [Server Error Message Reference](https://dev.mysql.com/doc/refman/5.7/en/server-error-reference.html). In addition, TiDB has the following unique error codes: > **Note:** > @@ -52,7 +51,7 @@ In addition, TiDB has the following unique error codes: * Error Number: 8018 When you reload a plugin, if the plugin has not been loaded before, this error is returned. - + You can execute an initial load of the plugin. * Error Number: 8019 @@ -74,7 +73,7 @@ In addition, TiDB has the following unique error codes: * Error Number: 8022 The transaction commit fails and has been rolled back. - + The application can safely retry the whole transaction. * Error Number: 8023 @@ -89,9 +88,9 @@ In addition, TiDB has the following unique error codes: * Error Number: 8025 - The single Key-Value pair being written is too large. The largest single Key-Value pair supported in TiDB is 6 MB. - - If a pair exceeds this limit, you need to manually deal with this row of data to meet the 6 MB limit. + The single Key-Value pair being written is too large. The largest single Key-Value pair supported in TiDB is 6 MB by default. + + If a pair exceeds this limit, you need to properly adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration value to relax the limit. * Error Number: 8026 @@ -99,7 +98,7 @@ In addition, TiDB has the following unique error codes: * Error Number: 8027 - The table schema version is outdated. TiDB uses the F1 online schema change algorithm to execute DDL statements. When the table schema version of the TiDB server is earlier than that of the entire system, this error is returned if you execute a SQL statement. + The table schema version is outdated. TiDB applies schema changes online. When the table schema version of the TiDB server is earlier than that of the entire system, this error is returned if you execute a SQL statement. When this error occurs, check the network between the TiDB server and the PD Leader. @@ -149,9 +148,9 @@ In addition, TiDB has the following unique error codes: * Error Number: 8048 - An unsupported database isolation level is set. - - If you cannot modify the codes because you are using a third-party tool or framework, consider using `tidb_skip_isolation_level_check` to bypass this check. + An unsupported database isolation level is set. + + If you cannot modify the codes because you are using a third-party tool or framework, consider using [`tidb_skip_isolation_level_check`](/system-variables.md#tidb_skip_isolation_level_check) to bypass this check. {{< copyable "sql" >}} @@ -167,27 +166,21 @@ In addition, TiDB has the following unique error codes: * Error Number: 8051 - Unknown data type is encountered when TiDB parses the Exec argument list sent by the client. - + Unknown data type is encountered when TiDB parses the Exec argument list sent by the client. + If you encounter this error, check the client. If the client is normal, [contact PingCAP](mailto:info@pingcap.com) for support. * Error Number: 8052 - The serial number of the data packet from the client is incorrect. - + The serial number of the data packet from the client is incorrect. + If you encounter this error, check the client. If the client is normal, [contact PingCAP](mailto:info@pingcap.com) for support. * Error Number: 8055 - The current snapshot is too old. The data may have been garbage collected. You can increase the value of `tikv_gc_life_time` to avoid this problem. The new version of TiDB automatically reserves data for long-running transactions. Usually this error does not occur. - - See [garbage collection overview](/garbage-collection-overview.md) and [garbage collection configuration](/garbage-collection-configuration.md). - - {{< copyable "sql" >}} + The current snapshot is too old. The data may have been garbage collected. You can increase the value of [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) to avoid this problem. TiDB automatically reserves data for long-running transactions. Usually this error does not occur. - ```sql - update mysql.tidb set VARIABLE_VALUE="24h" where VARIABLE_NAME="tikv_gc_life_time"; - ``` + See [garbage collection overview](/garbage-collection-overview.md) and [garbage collection configuration](/garbage-collection-configuration.md). * Error Number: 8059 @@ -241,20 +234,20 @@ In addition, TiDB has the following unique error codes: * Error Number: 8102 - Unable to read the plugin definition information. - + Unable to read the plugin definition information. + Check the configuration related to the plugin. * Error Number: 8103 - The plugin name is incorrect. - + The plugin name is incorrect. + Check the configuration of the plugin. * Error Number: 8104 - The plugin version does not match. - + The plugin version does not match. + Check the configuration of the plugin. * Error Number: 8105 @@ -263,20 +256,20 @@ In addition, TiDB has the following unique error codes: * Error Number: 8106 - The plugin defines a system variable whose name does not begin with the plugin name. - + The plugin defines a system variable whose name does not begin with the plugin name. + Contact the developer of the plugin to modify. * Error Number: 8107 - The loaded plugin does not specify a version, or the specified version is too low. - + The loaded plugin does not specify a version, or the specified version is too low. + Check the configuration of the plugin. * Error Number: 8108 - Unsupported execution plan type. This error is an internal error. - + Unsupported execution plan type. This error is an internal error. + If you encounter this error, [contact PingCAP](mailto:info@pingcap.com) for support. * Error Number: 8109 @@ -311,14 +304,14 @@ In addition, TiDB has the following unique error codes: * Error Number: 8120 - The `start tso` of transactions cannot be obtained. - + The `start tso` of transactions cannot be obtained. + Check the state/monitor/log of the PD server and the network between the TiDB server and the PD server. * Error Number: 8121 - Privilege check fails. - + Privilege check fails. + Check the privilege configuration of the database. * Error Number: 8122 @@ -327,8 +320,8 @@ In addition, TiDB has the following unique error codes: * Error Number: 8123 - An SQL query with aggregate functions returns non-aggregated columns, which violates the `only_full_group_by` mode. - + An SQL query with aggregate functions returns non-aggregated columns, which violates the `only_full_group_by` mode. + Modify the SQL statement or disable the `only_full_group_by` mode. * Error Number: 8129 @@ -347,8 +340,8 @@ In addition, TiDB has the following unique error codes: * Error Number: 8215 - `ADMIN REPAIR TABLE` fails. - + `ADMIN REPAIR TABLE` fails. + If you encounter this error, [contact PingCAP](mailto:info@pingcap.com) for support. * Error Number: 8216 @@ -359,14 +352,14 @@ In addition, TiDB has the following unique error codes: * Error Number: 8223 - This error occurs when detecting that the data is not consistent with the index. - + This error occurs when detecting that the data is not consistent with the index. + If you encounter this error, [contact PingCAP](mailto:info@pingcap.com) for support. * Error Number: 8224 - The DDL job cannot be found. - + The DDL job cannot be found. + Check whether the job id specified by the `restore` operation exists. * Error Number: 8225 @@ -386,9 +379,9 @@ In addition, TiDB has the following unique error codes: * Error Number: 8228 Unsupported types are specified when using `setval` on Sequence. - + See [Sequence documentation](/sql-statements/sql-statement-create-sequence.md#examples) to find the example of the function. - + * Error Number: 8229 The transaction exceeds the survival time. @@ -397,7 +390,7 @@ In addition, TiDB has the following unique error codes: * Error Number: 8230 - TiDB currently does not support using Sequence as the default value on newly added columns, and reports this error if you use it. + TiDB currently does not support using Sequence as the default value on newly added columns, and reports this error if you use it. * Error Number: 9001 @@ -449,16 +442,28 @@ In addition, TiDB has the following unique error codes: * Error Number: 9008 - Too many requests are sent to TiKV at the same time. The number exceeds limit. - + Too many requests are sent to TiKV at the same time. The number exceeds limit. + Increase `tidb_store_limit` or set it to `0` to remove the limit on the traffic of requests. * Error Number: 9010 TiKV cannot process this raft log. - + Check the state/monitor/log of the TiKV server. +* Error Number: 9012 + + The TiFlash request timed out. + + Check the state/monitor/log of the TiFlash server and the network between the TiDB server and TiFlash server. + +* Error Number: 9013 + + The TiFlash server is busy and this usually occurs when the workload is too high. + + Check the state/monitor/log of the TiFlash server. + ## Troubleshooting See the [troubleshooting](/troubleshoot-tidb-cluster.md) and [FAQ](/faq/tidb-faq.md) documents. diff --git a/experimental-features-4.0.md b/experimental-features-4.0.md deleted file mode 100644 index c29facf53232..000000000000 --- a/experimental-features-4.0.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: TiDB 4.0 Experimental Features -summary: Learn the experimental features of TiDB v4.0. ---- - -# TiDB 4.0 Experimental Features - -This document introduces the experimental features of TiDB v4.0. It is **NOT** recommended to use these features in the production environment. - -## Scheduling - -+ Cascading Placement Rules is an experimental feature of the Placement Driver (PD) introduced in v4.0. It is a replica rule system that guides PD to generate corresponding schedules for different types of data. By combining different scheduling rules, you can finely control the attributes of any continuous data range, such as the number of replicas, the storage location, the host type, whether to participate in Raft election, and whether to act as the Raft leader. See [Cascading Placement Rules](/configure-placement-rules.md) for details. -+ Elastic scheduling is an experimental feature based on Kubernetes, which enables TiDB to dynamically scale out and scale in clusters. This feature can effectively mitigate the high workload during peak hours of an application and saves unnecessary overhead. See [Enable TidbCluster Auto-scaling](https://docs.pingcap.com/tidb-in-kubernetes/stable/enable-tidb-cluster-auto-scaling) for details. - -## SQL feature - -Support the expression index feature. The expression index is also called the function-based index. When you create an index, the index fields do not have to be a specific column but can be an expression calculated from one or more columns. This feature is useful for quickly accessing the calculation-based tables. See [Expression index](/sql-statements/sql-statement-create-index.md) for details. - -## Service-level features - -+ TiDB instances support caching the calculation results that the operator has pushed down to TiKV in the unit of Region, which improves the efficiency of SQL executions in the following scenarios. See [Coprocessor Cache](/coprocessor-cache.md) for details. - - The SQL statements are the same. - - The SQL statements contain a changing condition (limited to the primary key of tables or partitions), and the other parts are consistent. - - The SQL statements contain multiple changing conditions and the other parts are consistent. The changing conditions exactly match a compound index column. -+ Support persisting configuration parameters in PD and dynamically modifying configuration items to improve product usability. diff --git a/experimental-features.md b/experimental-features.md new file mode 100644 index 000000000000..aa55505635c1 --- /dev/null +++ b/experimental-features.md @@ -0,0 +1,61 @@ +--- +title: TiDB Experimental Features +summary: Learn the experimental features of TiDB. +aliases: ['/tidb/dev/experimental-features-4.0/'] +--- + +# TiDB Experimental Features + +This document introduces the experimental features of TiDB in different versions. It is **NOT** recommended to use these features in the production environment. + +## Stability + ++ TiFlash limits the use of I/O resources by compressing or sorting data, mitigating the contention for I/O resources between background tasks and front-end data reading and writing (Introduced in v5.0) ++ Improve the stability of the optimizer's choice of indexes (Introduced in v5.0) + + Extend the statistics feature by collecting the multi-column order dependency information. + + Refactor the statistics module, including deleting the `TopN` value from `CMSKetch` and the histogram, and adding NDV information for histogram buckets of each table index. + +## Scheduling + ++ Cascading Placement Rules feature. It is a replica rule system that guides PD to generate corresponding schedules for different types of data. By combining different scheduling rules, you can finely control the attributes of any continuous data range, such as the number of replicas, the storage location, the host type, whether to participate in Raft election, and whether to act as the Raft leader. See [Cascading Placement Rules](/configure-placement-rules.md) for details. (Introduced in v4.0) ++ Elastic scheduling feature. It enables the TiDB cluster to dynamically scale out and in on Kubernetes based on real-time workloads, which effectively reduces the stress during your application's peak hours and saves overheads. See [Enable TidbCluster Auto-scaling](https://docs.pingcap.com/tidb-in-kubernetes/stable/enable-tidb-cluster-auto-scaling) for details. (Introduced in v4.0) + +## SQL + ++ List Partition (Introduced in v5.0) ++ List COLUMNS Partition (Introduced in v5.0) ++ [Dynamic Pruning Mode for Partitioned Tables](/partitioned-table.md#dynamic-pruning-mode). (Introduced in v5.1) ++ The expression index feature. The expression index is also called the function-based index. When you create an index, the index fields do not have to be a specific column but can be an expression calculated from one or more columns. This feature is useful for quickly accessing the calculation-based tables. See [Expression index](/sql-statements/sql-statement-create-index.md) for details. (Introduced in v4.0) ++ [Generated Columns](/generated-columns.md). ++ [User-Defined Variables](/user-defined-variables.md). ++ [JSON data type](/data-type-json.md) and [JSON functions](/functions-and-operators/json-functions.md). ++ [View](/information-schema/information-schema-views.md). + +## Configuration management + ++ Persistently store configuration parameters in PD, and support dynamically modifying configuration items. (Introduced in v4.0) ++ [SHOW CONFIG](/sql-statements/sql-statement-show-config.md) (Introduced in v4.0) + +## Data sharing and subscription + ++ [Integrate TiCDC with Kafka Connect (Confluent Platform)](/ticdc/integrate-confluent-using-ticdc.md) (Introduced in v5.0) ++ [The cyclic replication feature of TiCDC](/ticdc/manage-ticdc.md#cyclic-replication) (Introduced in v5.0) + +## Storage + ++ [Disable Titan](/storage-engine/titan-configuration.md#disable-titan-experimental). ++ [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental). ++ TiFlash supports distributing the new data of the storage engine on multiple hard drives to share the I/O pressure. (Introduced in v4.0) + +## Backup and restoration + ++ [Back up Raw KV](/br/use-br-command-line-tool.md#back-up-raw-kv-experimental-feature). + +## Garbage collection + ++ [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50). + +## Diagnostics + ++ [SQL diagnostics](/information-schema/information-schema-sql-diagnostics.md). ++ [Cluster diagnostics](/dashboard/dashboard-diagnostics-access.md). diff --git a/explain-joins.md b/explain-joins.md index f37fe4ce9525..a50a7b01992d 100644 --- a/explain-joins.md +++ b/explain-joins.md @@ -180,7 +180,7 @@ Query OK, 0 rows affected (3.65 sec) ### Variations of Index Join -An index join operation using the hint [`INL_JOIN`](/optimizer-hints.md#inl_joint1_name--tl_name-) creates a hash table of the intermediate results before joining on the outer table. TiDB also supports creating a hash table on the outer table using the hint [`INL_HASH_JOIN`](/optimizer-hints.md#inl_hash_join). If the column sets on the inner table match the columns of the outer table, the [`INL_MERGE_JOIN`](/optimizer-hints.md#inl_merge_join) index join can apply. Each of these variations of index join is automatically selected by the SQL Optimizer. +An index join operation using the hint [`INL_JOIN`](/optimizer-hints.md#inl_joint1_name--tl_name-) creates a hash table of the intermediate results before joining on the outer table. TiDB also supports creating a hash table on the outer table using the hint [`INL_HASH_JOIN`](/optimizer-hints.md#inl_hash_join). Each of these variations of index join is automatically selected by the SQL Optimizer. ### Configuration diff --git a/explain-mpp.md b/explain-mpp.md new file mode 100644 index 000000000000..bc1bf1ddcc2f --- /dev/null +++ b/explain-mpp.md @@ -0,0 +1,173 @@ +--- +title: Explain Statements in the MPP Mode +summary: Learn about the execution plan information returned by the EXPLAIN statement in TiDB. +--- + +# Explain Statements in the MPP Mode + +TiDB supports using the [MPP mode](/tiflash/use-tiflash.md#use-the-mpp-mode) to execute queries. In the MPP mode, the TiDB optimizer generates execution plans for MPP. Note that the MPP mode is only available for tables that have replicas on [TiFlash](/tiflash/tiflash-overview.md). + +The examples in this document are based on the following sample data: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t1 (id int, value int); +INSERT INTO t1 values(1,2),(2,3),(1,3); +ALTER TABLE t1 set tiflash replica 1; +ANALYZE TABLE t1; +SET tidb_allow_mpp = 1; +``` + +## MPP query fragments and MPP tasks + +In the MPP mode, a query is logically sliced into multiple query fragments. Take the following statement as an example: + +{{< copyable "sql" >}} + +```sql +EXPLAIN SELECT COUNT(*) FROM t1 GROUP BY id; +``` + +This query is divided into two fragments in the MPP mode. One for the first-stage aggregation and the other for the second-stage aggregation, also the final aggregation. When this query is executed, each query fragment is instantiated into one or more MPP tasks. + +## Exchange operators + +`ExchangeReceiver` and `ExchangeSender`are two exchange operators specific for MPP execution plans. The `ExchangeReceiver` operator reads data from downstream query fragments and the `ExchangeSender` operator sends data from downstream query fragments to upstream query fragments. In the MPP mode, the root operator of each MPP query fragment is `ExchangeSender`, meaning that query fragments are delimited by the `ExchangeSender` operator. + +The following is a simple MPP execution plan: + +{{< copyable "sql" >}} + +```sql +EXPLAIN SELECT COUNT(*) FROM t1 GROUP BY id; +``` + +```sql ++------------------------------------+---------+-------------------+---------------+----------------------------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------------+---------+-------------------+---------------+----------------------------------------------------+ +| TableReader_31 | 2.00 | root | | data:ExchangeSender_30 | +| └─ExchangeSender_30 | 2.00 | batchCop[tiflash] | | ExchangeType: PassThrough | +| └─Projection_26 | 2.00 | batchCop[tiflash] | | Column#4 | +| └─HashAgg_27 | 2.00 | batchCop[tiflash] | | group by:test.t1.id, funcs:sum(Column#7)->Column#4 | +| └─ExchangeReceiver_29 | 2.00 | batchCop[tiflash] | | | +| └─ExchangeSender_28 | 2.00 | batchCop[tiflash] | | ExchangeType: HashPartition, Hash Cols: test.t1.id | +| └─HashAgg_9 | 2.00 | batchCop[tiflash] | | group by:test.t1.id, funcs:count(1)->Column#7 | +| └─TableFullScan_25 | 3.00 | batchCop[tiflash] | table:t1 | keep order:false | ++------------------------------------+---------+-------------------+---------------+----------------------------------------------------+ +``` + +The above execution plan contains two query fragments: + +* The first is `[TableFullScan_25, HashAgg_9, ExchangeSender_28]`, which is mainly responsible for the first-stage aggregation. +* The second is `[ExchangeReceiver_29, HashAgg_27, Projection_26, ExchangeSender_30]`, which is mainly responsible for the second-stage aggregation. + +The `operator info` column of the `ExchangeSender` operator shows the exchange type information. Currently, there are three exchange types. See the following: + +* HashPartition: The `ExchangeSender` operator firstly partitions data according to the Hash values and then distributes data to the `ExchangeReceiver` operator of upstream MPP tasks. This exchange type is often used for Hash Aggregation and Shuffle Hash Join algorithms. +* Broadcast: The `ExchangeSender` operator distributes data to upstream MPP tasks through broadcast. This exchange type is often used for Broadcast Join. +* PassThrough: The `ExchangeSender` operator sends data to the only upstream MPP task, which is different from the Broadcast type. This exchange type is often used when returning data to TiDB. + +In the example execution plan, the exchange type of the operator `ExchangeSender_28` is HashPartition, meaning that it performs the Hash Aggregation algorithm. The exchange type of the operator `ExchangeSender_30` is PassThrough, meaning that it is used to return data to TiDB. + +MPP is also often applied to join operations. The MPP mode in TiDB supports the following two join algorithms: + +* Shuffle Hash Join: Shuffle the data input from the join operation using the HashPartition exchange type. Then, upstream MPP tasks join data within the same partition. +* Broadcast Join: Broadcast data of the small table in the join operation to each node, after which each node joins the data separately. + +The following is a typical execution plan for Shuffle Hash Join: + +{{< copyable "sql" >}} + +```sql +SET tidb_opt_broadcast_join=0; +SET tidb_broadcast_join_threshold_count=0; +SET tidb_broadcast_join_threshold_size=0; +EXPLAIN SELECT COUNT(*) FROM t1 a JOIN t1 b ON a.id = b.id; +``` + +```sql ++----------------------------------------+---------+--------------+---------------+----------------------------------------------------+ +| id | estRows | task | access object | operator info | ++----------------------------------------+---------+--------------+---------------+----------------------------------------------------+ +| StreamAgg_14 | 1.00 | root | | funcs:count(1)->Column#7 | +| └─TableReader_48 | 9.00 | root | | data:ExchangeSender_47 | +| └─ExchangeSender_47 | 9.00 | cop[tiflash] | | ExchangeType: PassThrough | +| └─HashJoin_44 | 9.00 | cop[tiflash] | | inner join, equal:[eq(test.t1.id, test.t1.id)] | +| ├─ExchangeReceiver_19(Build) | 6.00 | cop[tiflash] | | | +| │ └─ExchangeSender_18 | 6.00 | cop[tiflash] | | ExchangeType: HashPartition, Hash Cols: test.t1.id | +| │ └─Selection_17 | 6.00 | cop[tiflash] | | not(isnull(test.t1.id)) | +| │ └─TableFullScan_16 | 6.00 | cop[tiflash] | table:a | keep order:false | +| └─ExchangeReceiver_23(Probe) | 6.00 | cop[tiflash] | | | +| └─ExchangeSender_22 | 6.00 | cop[tiflash] | | ExchangeType: HashPartition, Hash Cols: test.t1.id | +| └─Selection_21 | 6.00 | cop[tiflash] | | not(isnull(test.t1.id)) | +| └─TableFullScan_20 | 6.00 | cop[tiflash] | table:b | keep order:false | ++----------------------------------------+---------+--------------+---------------+----------------------------------------------------+ +12 rows in set (0.00 sec) +``` + +In the above execution plan: + +* The query fragment `[TableFullScan_20, Selection_21, ExchangeSender_22]` reads data from table b and shuffles data to upstream MPP tasks. +* The query fragment `[TableFullScan_16, Selection_17, ExchangeSender_18]` reads data from table a and shuffles data to upstream MPP tasks. +* The query fragment `[ExchangeReceiver_19, ExchangeReceiver_23, HashJoin_44, ExchangeSender_47]` joins all data and returns it to TiDB. + +A typical execution plan for Broadcast Join is as follows: + +{{< copyable "sql" >}} + +```sql +EXPLAIN SELECT COUNT(*) FROM t1 a JOIN t1 b ON a.id = b.id; +``` + +```sql ++----------------------------------------+---------+--------------+---------------+------------------------------------------------+ +| id | estRows | task | access object | operator info | ++----------------------------------------+---------+--------------+---------------+------------------------------------------------+ +| StreamAgg_15 | 1.00 | root | | funcs:count(1)->Column#7 | +| └─TableReader_47 | 9.00 | root | | data:ExchangeSender_46 | +| └─ExchangeSender_46 | 9.00 | cop[tiflash] | | ExchangeType: PassThrough | +| └─HashJoin_43 | 9.00 | cop[tiflash] | | inner join, equal:[eq(test.t1.id, test.t1.id)] | +| ├─ExchangeReceiver_20(Build) | 6.00 | cop[tiflash] | | | +| │ └─ExchangeSender_19 | 6.00 | cop[tiflash] | | ExchangeType: Broadcast | +| │ └─Selection_18 | 6.00 | cop[tiflash] | | not(isnull(test.t1.id)) | +| │ └─TableFullScan_17 | 6.00 | cop[tiflash] | table:a | keep order:false | +| └─Selection_22(Probe) | 6.00 | cop[tiflash] | | not(isnull(test.t1.id)) | +| └─TableFullScan_21 | 6.00 | cop[tiflash] | table:b | keep order:false | ++----------------------------------------+---------+--------------+---------------+------------------------------------------------+ +``` + +In the above execution plan: + +* The query fragment `[TableFullScan_17, Selection_18, ExchangeSender_19]` reads data from the small table (table a) and broadcasts the data to each node that contains data from the large table (table b). +* The query fragment `[TableFullScan_21, Selection_22, ExchangeReceiver_20, HashJoin_43, ExchangeSender_46]` joins all data and returns it to TiDB. + +## `EXPLAIN ANALYZE` statements in the MPP mode + +The `EXPLAIN ANALYZE` statement is similar to `EXPLAIN`, but it also outputs some runtime information. + +The following is the output of a simple `EXPLAIN ANALYZE` example: + +{{< copyable "sql" >}} + +```sql +EXPLAIN ANALYZE SELECT COUNT(*) FROM t1 GROUP BY id; +``` + +```sql ++------------------------------------+---------+---------+-------------------+---------------+---------------------------------------------------------------------------------------------+----------------------------------------------------------------+--------+------+ +| id | estRows | actRows | task | access object | execution info | operator info | memory | disk | ++------------------------------------+---------+---------+-------------------+---------------+---------------------------------------------------------------------------------------------+----------------------------------------------------------------+--------+------+ +| TableReader_31 | 4.00 | 2 | root | | time:44.5ms, loops:2, cop_task: {num: 1, max: 0s, proc_keys: 0, copr_cache_hit_ratio: 0.00} | data:ExchangeSender_30 | N/A | N/A | +| └─ExchangeSender_30 | 4.00 | 2 | batchCop[tiflash] | | tiflash_task:{time:16.5ms, loops:1, threads:1} | ExchangeType: PassThrough, tasks: [2, 3, 4] | N/A | N/A | +| └─Projection_26 | 4.00 | 2 | batchCop[tiflash] | | tiflash_task:{time:16.5ms, loops:1, threads:1} | Column#4 | N/A | N/A | +| └─HashAgg_27 | 4.00 | 2 | batchCop[tiflash] | | tiflash_task:{time:16.5ms, loops:1, threads:1} | group by:test.t1.id, funcs:sum(Column#7)->Column#4 | N/A | N/A | +| └─ExchangeReceiver_29 | 4.00 | 2 | batchCop[tiflash] | | tiflash_task:{time:14.5ms, loops:1, threads:20} | | N/A | N/A | +| └─ExchangeSender_28 | 4.00 | 0 | batchCop[tiflash] | | tiflash_task:{time:9.49ms, loops:0, threads:0} | ExchangeType: HashPartition, Hash Cols: test.t1.id, tasks: [1] | N/A | N/A | +| └─HashAgg_9 | 4.00 | 0 | batchCop[tiflash] | | tiflash_task:{time:9.49ms, loops:0, threads:0} | group by:test.t1.id, funcs:count(1)->Column#7 | N/A | N/A | +| └─TableFullScan_25 | 6.00 | 0 | batchCop[tiflash] | table:t1 | tiflash_task:{time:9.49ms, loops:0, threads:0} | keep order:false | N/A | N/A | ++------------------------------------+---------+---------+-------------------+---------------+---------------------------------------------------------------------------------------------+----------------------------------------------------------------+--------+------+ +``` + +Compared to the output of `EXPLAIN`, the `operator info` column of the operator `ExchangeSender` also shows `tasks`, which records the id of the MPP task that the query fragment instantiates into. In addition, each MPP operator has a `threads` field in the `execution info` column, which records the concurrency of operations when TiDB executes this operator. If the cluster consists of multiple nodes, this concurrency is the result of adding up the concurrency of all nodes. diff --git a/explain-overview.md b/explain-overview.md index 857b1cbc930c..1d6548683e84 100644 --- a/explain-overview.md +++ b/explain-overview.md @@ -6,116 +6,65 @@ aliases: ['/docs/dev/query-execution-plan/','/docs/dev/reference/performance/und # `EXPLAIN` Overview -Based on the latest statistics of your tables, the TiDB optimizer chooses the most efficient query execution plan, which consists of a series of operators. This document details the execution plan in TiDB. - -## Introduction - -You can use the `EXPLAIN` command in TiDB to view the execution plan. The result of the `EXPLAIN` statement provides information about how TiDB executes SQL queries: - -- `EXPLAIN` works together with statements such as `SELECT` and `DELETE`. -- When you execute the `EXPLAIN` statement, TiDB returns the final optimized physical execution plan. In other words, `EXPLAIN` displays the complete information about how TiDB executes the SQL statement, such as in which order, how tables are joined, and what the expression tree looks like. -- For more information about each column of `EXPLAIN`, see [`EXPLAIN` Output Format](/sql-statements/sql-statement-explain.md). - -The results of `EXPLAIN` shed light on how to index the data tables so that the execution plan can use the index to speed up the execution of SQL statements. You can also use `EXPLAIN` to check if the optimizer chooses the optimal order to join tables. - -## Operator execution order - -The execution plan in TiDB has a tree structure, with each node of the tree as an operator. Considering the concurrent execution of multiple threads in each operator, all operators consume CPU and memory resources to process data during the execution of a SQL statement. From this point of view, there is no execution order for the operator. - -However, from the perspective of which operators process a row of data first, the execution of a piece of data is in order. The following rule roughly summarizes this order: - -**`Build` is always executed before `Probe` and always appears before `Probe`.** +> **Note:** +> +> When you use the MySQL client to connect to TiDB, to read the output result in a clearer way without line wrapping, you can use the `pager less -S` command. Then, after the `EXPLAIN` result is output, you can press the right arrow → button on your keyboard to horizontally scroll through the output. -The first half of this rule means: if an operator has multiple child nodes, the operator with the `Build` keyword at the end of the child node ID is always executed before the operator with the `Probe` keyword. The second half means: when TiDB shows the execution plan, the `Build` side always appears first, followed by the `Probe` side. +SQL is a declarative language. It describes what the results of a query should look like, **not the methodology** to actually retrieve those results. TiDB considers all the possible ways in which a query could be executed, including using what order to join tables and whether any potential indexes can be used. The process of _considering query execution plans_ is known as SQL optimization. -The following examples illustrate this rule: +The `EXPLAIN` statement shows the selected execution plan for a given statement. That is, after considering hundreds or thousands of ways in which the query could be executed, TiDB believes that this _plan_ will consume the least resources and execute in the shortest amount of time: {{< copyable "sql" >}} ```sql -explain select * from t use index(idx_a) where a = 1; +CREATE TABLE t (id INT NOT NULL PRIMARY KEY auto_increment, a INT NOT NULL, pad1 VARCHAR(255), INDEX(a)); +INSERT INTO t VALUES (1, 1, 'aaa'),(2,2, 'bbb'); +EXPLAIN SELECT * FROM t WHERE a = 1; ``` ```sql -+-------------------------------+---------+-----------+-------------------------+---------------------------------------------+ -| id | estRows | task | access object | operator info | -+-------------------------------+---------+-----------+-------------------------+---------------------------------------------+ -| IndexLookUp_7 | 10.00 | root | | | -| ├─IndexRangeScan_5(Build) | 10.00 | cop[tikv] | table:t, index:idx_a(a) | range:[1,1], keep order:false, stats:pseudo | -| └─TableRowIDScan_6(Probe) | 10.00 | cop[tikv] | table:t | keep order:false, stats:pseudo | -+-------------------------------+---------+-----------+-------------------------+---------------------------------------------+ -3 rows in set (0.00 sec) -``` +Query OK, 0 rows affected (0.96 sec) -The `IndexLookUp_7` operator has two child nodes: `IndexRangeScan_5 (Build)` and `TableRowIDScan_6 (Probe)`. `IndexRangeScan_5 (Build)` appears first. +Query OK, 2 rows affected (0.02 sec) +Records: 2 Duplicates: 0 Warnings: 0 -To get a piece of data, first, you need to execute `IndexRangeScan_5 (Build)` to get a RowID. Then, use `TableRowIDScan_6 (Probe)` to get a complete row of data based on the RowID. - -The implication of the above rule is: for nodes at the same level, the operator that appears first might be executed first, and the operator that appears last might be executed last. This can be illustrated in the following example: - -{{< copyable "sql" >}} - -```sql -explain select * from t t1 use index(idx_a) join t t2 use index() where t1.a = t2.a; -``` - -```sql -+----------------------------------+----------+-----------+--------------------------+------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+----------------------------------+----------+-----------+--------------------------+------------------------------------------------------------------+ -| HashJoin_22 | 12487.50 | root | | inner join, inner:TableReader_26, equal:[eq(test.t.a, test.t.a)] | -| ├─TableReader_26(Build) | 9990.00 | root | | data:Selection_25 | -| │ └─Selection_25 | 9990.00 | cop[tikv] | | not(isnull(test.t.a)) | -| │ └─TableFullScan_24 | 10000.00 | cop[tikv] | table:t2 | keep order:false, stats:pseudo | -| └─IndexLookUp_29(Probe) | 9990.00 | root | | | -| ├─IndexFullScan_27(Build) | 9990.00 | cop[tikv] | table:t1, index:idx_a(a) | keep order:false, stats:pseudo | -| └─TableRowIDScan_28(Probe) | 9990.00 | cop[tikv] | table:t1 | keep order:false, stats:pseudo | -+----------------------------------+----------+-----------+--------------------------+------------------------------------------------------------------+ -7 rows in set (0.00 sec) ++-------------------------------+---------+-----------+---------------------+---------------------------------------------+ +| id | estRows | task | access object | operator info | ++-------------------------------+---------+-----------+---------------------+---------------------------------------------+ +| IndexLookUp_10 | 10.00 | root | | | +| ├─IndexRangeScan_8(Build) | 10.00 | cop[tikv] | table:t, index:a(a) | range:[1,1], keep order:false, stats:pseudo | +| └─TableRowIDScan_9(Probe) | 10.00 | cop[tikv] | table:t | keep order:false, stats:pseudo | ++-------------------------------+---------+-----------+---------------------+---------------------------------------------+ +3 rows in set (0.00 sec) ``` -To complete the `HashJoin_22` operation, you need to execute `TableReader_26(Build)`, and then execute `IndexLookUp_29(Probe)`. +`EXPLAIN` does not execute the actual query. [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) can be used to execute the query and show `EXPLAIN` information. This can be useful in diagnosing cases where the execution plan selected is suboptimal. For additional examples of using `EXPLAIN`, see the following documents: -When executing `IndexLookUp_29(Probe)`, you need to execute `IndexFullScan_27(Build)` and `TableRowIDScan_28(Probe)` one by one. Therefore, from the perspective of the whole execution link, `TableRowIDScan_28(Probe)` is the last one awoken to be executed. +* [Indexes](/explain-indexes.md) +* [Joins](/explain-joins.md) +* [Subqueries](/explain-subqueries.md) +* [Aggregation](/explain-aggregation.md) +* [Views](/explain-views.md) +* [Partitions](/explain-partitions.md) -### Task overview - -Currently, calculation tasks of TiDB can be divided into two categories: cop tasks and root tasks. The cop tasks are performed by the Coprocessor in TiKV, and the root tasks are performed in TiDB. +## Understand EXPLAIN output -One of the goals of SQL optimization is to push the calculation down to TiKV as much as possible. The Coprocessor in TiKV supports most of the built-in SQL functions (including the aggregate functions and the scalar functions), SQL `LIMIT` operations, index scans, and table scans. However, all `Join` operations can only be performed as root tasks in TiDB. +The following describes the output of the `EXPLAIN` statement above: -### Access Object overview +* `id` describes the name of an operator, or sub-task that is required to execute the SQL statement. See [Operator overview](#operator-overview) for additional details. +* `estRows` shows an estimate of the number of rows TiDB expects to process. This number might be based on dictionary information, such as when the access method is based on a primary or unique key, or it could be based on statistics such as a CMSketch or histogram. +* `task` shows where an operator is performing the work. See [Task overview](#task-overview) for additional details. +* `access object` shows the table, partition and index that is being accessed. The parts of the index are also shown, as in the case above that the column `a` from the index was used. This can be useful in cases where you have composite indexes. +* `operator info` shows additional details about the access. See [Operator info overview](#operator-info-overview) for additional details. -Access Object is the data item accessed by the operator, including `table`, `partition`, and `index` (if any). Only operators that directly access the data have this information. +### Operator overview -### Range query +An operator is a particular step that is executed as part of returning query results. The operators that perform table scans (of the disk or the TiKV Block Cache) are listed as follows: -In the `WHERE`/`HAVING`/`ON` conditions, the TiDB optimizer analyzes the result returned by the primary key query or the index key query. For example, these conditions might include comparison operators of the numeric and date type, such as `>`, `<`, `=`, `>=`, `<=`, and the character type such as `LIKE`. - -> **Note:** -> -> - Currently, TiDB only supports the case where a comparison operator is connected by a column (at one end) and a constant value (at the other end), or the case that the calculation result is a constant. You cannot use query conditions like `year(birth_day) < 1992` as the index. -> - It is recommended to compare data of the same type; otherwise, additional `cast` operations are introduced, which causes the index to be unavailable. For example, regarding the condition `user_id = 123456`, if `user_id` is a string, you must define `123456` as a string constant. - -You can also use `AND` (intersection) and `OR` (union) to combine the range query conditions of one column. For a multi-dimensional composite index, you can use conditions in multiple columns. For example, regarding the composite index `(a, b, c)`: - -- When `a` is an equivalent query, continue to figure out the query range of `b`; when `b` is also an equivalent query, continue to figure out the query range of `c`. -- Otherwise, if `a` is a non-equivalent query, you can only figure out the range of `a`. - -## Operator overview - -Different operators output different information after the `EXPLAIN` statement is executed. This section focuses on the execution plan of different operators, ranging from table scans, table aggregation, to table join. - -You can use optimizer hints to control the behavior of the optimizer, and thereby controlling the selection of the physical operators. For example, `/*+ HASH_JOIN(t1, t2) */` means that the optimizer uses the `Hash Join` algorithm. For more details, see [Optimizer Hints](/optimizer-hints.md). - -### Read the execution plan of table scans - -The operators that perform table scans (of the disk or the TiKV Block Cache) are listed as follows: - -- **TableFullScan**: Full table scan. -- **TableRangeScan**: Table scans with the specified range. -- **TableRowIDScan**: Accurately scans the table data based on the RowID passed down from the upper layer. -- **IndexFullScan**: Another type of "full table scan", except that the index data is scanned, rather than the table data. +- **TableFullScan**: Full table scan +- **TableRangeScan**: Table scans with the specified range +- **TableRowIDScan**: Scans the table data based on the RowID. Usually follows an index read operation to retrieve the matching data rows. +- **IndexFullScan**: Similar to a "full table scan", except that an index is scanned, rather than the table data. - **IndexRangeScan**: Index scans with the specified range. TiDB aggregates the data or calculation results scanned from TiKV/TiFlash. The data aggregation operators can be divided into the following categories: @@ -125,381 +74,34 @@ TiDB aggregates the data or calculation results scanned from TiKV/TiFlash. The d - **IndexLookUp**: First aggregates the RowID (in TiKV) scanned by the `Build` side. Then at the `Probe` side, accurately reads the data from TiKV based on these RowIDs. At the `Build` side, there are operators like `IndexFullScan` or `IndexRangeScan`; at the `Probe` side, there is the `TableRowIDScan` operator. - **IndexMerge**: Similar to `IndexLookUp`. `IndexMerge` can be seen as an extension of `IndexLookupReader`. `IndexMerge` supports reading multiple indexes at the same time. There are many `Build`s and one `Probe`. The execution process of `IndexMerge` the same as that of `IndexLookUp`. -#### Table data and index data - -Table data refers to the original data of a table stored in TiKV. For each row of the table data, its `key` is a 64-bit integer called `RowID`. If a primary key of a table is an integer, TiDB uses the value of the primary key as the `RowID` of the table data; otherwise, the system automatically generates `RowID`. The `value` of the table data is encoded from all the data in this row. When reading table data, data is returned in the order of increasing `RowID`s. - -Similar to the table data, the index data is also stored in TiKV. Its `key` is the ordered bytes encoded from the index column. `value` is the `RowID` corresponding to a row of index data. The non-index column of the row can be read using `RowID`. When reading index data, TiKV returns data in ascending order of index columns. If there are multiple index columns, firstly, ensure that the first column is incremented; if the i-th column is equivalent, make sure that the (i+1)-th column is incremented. - -#### `IndexLookUp` example - -{{< copyable "sql" >}} - -```sql -explain select * from t use index(idx_a); -``` - -```sql -+-------------------------------+----------+-----------+-------------------------+--------------------------------+ -| id | estRows | task | access object | operator info | -+-------------------------------+----------+-----------+-------------------------+--------------------------------+ -| IndexLookUp_6 | 10000.00 | root | | | -| ├─IndexFullScan_4(Build) | 10000.00 | cop[tikv] | table:t, index:idx_a(a) | keep order:false, stats:pseudo | -| └─TableRowIDScan_5(Probe) | 10000.00 | cop[tikv] | table:t | keep order:false, stats:pseudo | -+-------------------------------+----------+-----------+-------------------------+--------------------------------+ -3 rows in set (0.00 sec) -``` - -The `IndexLookUp_6` operator has two child nodes: `IndexFullScan_4(Build)` and `TableRowIDScan_5(Probe)`. - -+ `IndexFullScan_4(Build)` performs an index full scan and scans all the data of index `a`. Because it is a full scan, this operation gets the `RowID` of all the data in the table. -+ `TableRowIDScan_5(Probe)` scans all table data using `RowID`s. - -This execution plan is not as efficient as using `TableReader` to perform a full table scan, because `IndexLookUp` performs an extra index scan (which comes with additional overhead), apart from the table scan. - -For table scan operations, the operator info column in the `explain` table shows whether the data is sorted. In the above example, the `keep order:false` in the `IndexFullScan` operator indicates that the data is unsorted. The `stats:pseudo` in the operator info means that there is no statistics, or that the statistics will not be used for estimation because it is outdated. For other scan operations, the operator info involves similar information. - -#### `TableReader` example - -{{< copyable "sql" >}} - -```sql -explain select * from t where a > 1 or b >100; -``` - -```sql -+-------------------------+----------+-----------+---------------+----------------------------------------+ -| id | estRows | task | access object | operator info | -+-------------------------+----------+-----------+---------------+----------------------------------------+ -| TableReader_7 | 8000.00 | root | | data:Selection_6 | -| └─Selection_6 | 8000.00 | cop[tikv] | | or(gt(test.t.a, 1), gt(test.t.b, 100)) | -| └─TableFullScan_5 | 10000.00 | cop[tikv] | table:t | keep order:false, stats:pseudo | -+-------------------------+----------+-----------+---------------+----------------------------------------+ -3 rows in set (0.00 sec) -``` - -In the above example, the child node of the `TableReader_7` operator is `Selection_6`. The subtree rooted at this child node is seen as a `Cop Task` and is delivered to the corresponding TiKV. This `Cop Task` uses the `TableFullScan_5` operator to perform the table scan. `Selection` represents the selection condition in the SQL statement, namely, the `WHERE`/`HAVING`/`ON` clause. - -`TableFullScan_5` performs a full table scan, and the load on the cluster increases accordingly, which might affect other queries running in the cluster. If an appropriate index is built and the `IndexMerge` operator is used, these will greatly improve query performance and reduce load on the cluster. - -#### `IndexMerge` example +While the structure appears as a tree, executing the query does not strictly require the child nodes to be completed before the parent nodes. TiDB supports intra-query parallelism, so a more accurate way to describe the execution is that the child nodes _flow into_ their parent nodes. Parent, child and sibling operators _might_ potentially be executing parts of the query in parallel. -`IndexMerge` is a method introduced in TiDB v4.0 to access tables. Using this method, the TiDB optimizer can use multiple indexes per table and merge the results returned by each index. In some scenarios, this method makes the query more efficient by avoiding full table scans. +In the previous example, the `├─IndexRangeScan_8(Build)` operator finds the internal `RowID` for rows that match the `a(a)` index. The `└─TableRowIDScan_9(Probe)` operator then retrieves these rows from the table. -```sql -mysql> explain select * from t where a = 1 or b = 1; -+-------------------------+----------+-----------+---------------+--------------------------------------+ -| id | estRows | task | access object | operator info | -+-------------------------+----------+-----------+---------------+--------------------------------------+ -| TableReader_7 | 8000.00 | root | | data:Selection_6 | -| └─Selection_6 | 8000.00 | cop[tikv] | | or(eq(test.t.a, 1), eq(test.t.b, 1)) | -| └─TableFullScan_5 | 10000.00 | cop[tikv] | table:t | keep order:false, stats:pseudo | -+-------------------------+----------+-----------+---------------+--------------------------------------+ -mysql> set @@tidb_enable_index_merge = 1; -mysql> explain select * from t use index(idx_a, idx_b) where a > 1 or b > 1; -+--------------------------------+---------+-----------+-------------------------+------------------------------------------------+ -| id | estRows | task | access object | operator info | -+--------------------------------+---------+-----------+-------------------------+------------------------------------------------+ -| IndexMerge_16 | 6666.67 | root | | | -| ├─IndexRangeScan_13(Build) | 3333.33 | cop[tikv] | table:t, index:idx_a(a) | range:(1,+inf], keep order:false, stats:pseudo | -| ├─IndexRangeScan_14(Build) | 3333.33 | cop[tikv] | table:t, index:idx_b(b) | range:(1,+inf], keep order:false, stats:pseudo | -| └─TableRowIDScan_15(Probe) | 6666.67 | cop[tikv] | table:t | keep order:false, stats:pseudo | -+--------------------------------+---------+-----------+-------------------------+------------------------------------------------+ -``` - -In the above query, the filter condition is a `WHERE` clause that uses `OR` as the connector. Without `IndexMerge`, you can use only one index per table. `a = 1` cannot be pushed down to the index `a`; neither can `b = 1` be pushed down to the index `b`. The full table scan is inefficient when a huge volume of data exists in `t`. To handle such a scenario, `IndexMerge` is introduced in TiDB to access tables. +#### Range query -`IndexMerge` allows the optimizer to use multiple indexes per table, and merge the results returned by each index to generate the execution plan of the latter `IndexMerge` in the figure above. Here the `IndexMerge_16` operator has three child nodes, among which `IndexRangeScan_13` and `IndexRangeScan_14` get all the `RowID`s that meet the conditions based on the result of range scan, and then the `TableRowIDScan_15` operator accurately reads all the data that meets the conditions according to these `RowID`s. - -For the scan operation that is performed on a specific range of data, such as `IndexRangeScan`/`TableRangeScan`, the `operator info` column in the result has additional information about the scan range compared with other scan operations like `IndexFullScan`/`TableFullScan`. In the above example, the `range:(1,+inf]` in the `IndexRangeScan_13` operator indicates that the operator scans the data from 1 to positive infinity. +In the `WHERE`/`HAVING`/`ON` conditions, the TiDB optimizer analyzes the result returned by the primary key query or the index key query. For example, these conditions might include comparison operators of the numeric and date type, such as `>`, `<`, `=`, `>=`, `<=`, and the character type such as `LIKE`. > **Note:** > -> At present, the `IndexMerge` feature is disabled by default in TiDB 4.0.0-rc.1. In addition, the currently supported scenarios of `IndexMerge` in TiDB 4.0 are limited to the disjunctive normal form (expressions connected by `or`). The conjunctive normal form (expressions connected by `and`) will be supported in later versions. Enable the `IndexMerge` in one of two ways: -> -> - Set the `tidb_enable_index_merge` system variable to 1; -> -> - Use the SQL Hint [`USE_INDEX_MERGE`](/optimizer-hints.md#use_index_merget1_name-idx1_name--idx2_name-) in the query. -> -> SQL Hint has a higher priority than system variables. - -### Read the aggregated execution plan - -Aggregation algorithms in TiDB include the following categories: - -- [Hash Aggregate](#hash-aggregate-example) -- [Stream Aggregate](#stream-aggregate-example) - -#### `Hash Aggregate` example - -The `Hash Aggregation` operator is optimized in multi-threaded concurrency. It is quick to execute at the cost of more memory usage. The following is an example of `Hash Aggregate`: - -{{< copyable "sql" >}} - -```sql -explain select /*+ HASH_AGG() */ count(*) from t; -``` +> - In order to use an index, the condition must be _sargable_. For example, the condition `YEAR(date_column) < 1992` can not use an index, but `date_column < '1992-01-01` can. +> - It is recommended to compare data of the same type and [character set and collation](/character-set-and-collation.md). Mixing types may require additional `cast` operations, or prevent indexes from being used. +> - You can also use `AND` (intersection) and `OR` (union) to combine the range query conditions of one column. For a multi-dimensional composite index, you can use conditions in multiple columns. For example, regarding the composite index `(a, b, c)`: +> - When `a` is an equivalent query, continue to figure out the query range of `b`; when `b` is also an equivalent query, continue to figure out the query range of `c`. +> - Otherwise, if `a` is a non-equivalent query, you can only figure out the range of `a`. -```sql -+---------------------------+----------+-----------+---------------+---------------------------------+ -| id | estRows | task | access object | operator info | -+---------------------------+----------+-----------+---------------+---------------------------------+ -| HashAgg_11 | 1.00 | root | | funcs:count(Column#7)->Column#4 | -| └─TableReader_12 | 1.00 | root | | data:HashAgg_5 | -| └─HashAgg_5 | 1.00 | cop[tikv] | | funcs:count(1)->Column#7 | -| └─TableFullScan_8 | 10000.00 | cop[tikv] | table:t | keep order:false, stats:pseudo | -+---------------------------+----------+-----------+---------------+---------------------------------+ -4 rows in set (0.00 sec) -``` - -Generally speaking, `Hash Aggregate` is executed in two stages. - -- One is on the Coprocessor of TiKV/TiFlash, with the intermediate results of the aggregation function calculated when the table scan operator reads the data. -- The other is at the TiDB layer, with the final result calculated through aggregating the intermediate results of all Coprocessor Tasks. - -The operator info column in the `explain` table also records other information about `Hash Aggregation`. You need to pay attention to what aggregate function that `Hash Aggregation` uses. In the above example, the operator info of the `Hash Aggregation` operator is `funcs:count(Column#7)->Column#4`. It means that `Hash Aggregation` uses the aggregate function `count` for calculation. The operator info of the `Stream Aggregation` operator in the following example is the same with this one. - -#### `Stream Aggregate` example - -The `Stream Aggregation` operator usually takes up less memory than `Hash Aggregate`. In some scenarios, `Stream Aggregation` executes faster than `Hash Aggregate`. In the case of a large amount of data or insufficient system memory, it is recommended to use the `Stream Aggregate` operator. An example is as follows: - -{{< copyable "sql" >}} - -```sql -explain select /*+ STREAM_AGG() */ count(*) from t; -``` - -```sql -+----------------------------+----------+-----------+---------------+---------------------------------+ -| id | estRows | task | access object | operator info | -+----------------------------+----------+-----------+---------------+---------------------------------+ -| StreamAgg_16 | 1.00 | root | | funcs:count(Column#7)->Column#4 | -| └─TableReader_17 | 1.00 | root | | data:StreamAgg_8 | -| └─StreamAgg_8 | 1.00 | cop[tikv] | | funcs:count(1)->Column#7 | -| └─TableFullScan_13 | 10000.00 | cop[tikv] | table:t | keep order:false, stats:pseudo | -+----------------------------+----------+-----------+---------------+---------------------------------+ -4 rows in set (0.00 sec) -``` - -Similar to `Hash Aggregate`, `Stream Aggregate` is executed in two stages. - -- One is on the Coprocessor of TiKV/TiFlash, with the intermediate results of the aggregation function calculated when the table scan operator reads the data. -- The other is at the TiDB layer, with the final result calculated through aggregating the intermediate results of all Coprocessor Tasks. - -### Read the `Join` execution plan - -The `Join` algorithms in TiDB consist of the following categories: - -- [Hash Join](#hash-join-example) -- [Merge Join](#merge-join-example) -- [Index Join (Index Nested Loop Join)](#index-join-example) -- [Index Hash Join (Index Nested Loop Hash Join)](#index-hash-join-example) -- [Index Merge Join (Index Nested Loop Merge Join)](#index-merge-join-example) - -The following are examples of the execution processes of these `Join` algorithms. - -#### `Hash Join` example - -The `Hash Join` operator uses multi-thread. Its execution speed is fast at the cost of more memory usage. An example of `Hash Join` is as follows: - -{{< copyable "sql" >}} - -```sql -EXPLAIN SELECT /*+ HASH_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id = t2.id; -``` - -```sql -+-----------------------------+----------+-----------+------------------------+------------------------------------------------+ -| id | estRows | task | access object | operator info | -+-----------------------------+----------+-----------+------------------------+------------------------------------------------+ -| HashJoin_30 | 12487.50 | root | | inner join, equal:[eq(test.t1.id, test.t2.id)] | -| ├─IndexReader_35(Build) | 9990.00 | root | | index:IndexFullScan_34 | -| │ └─IndexFullScan_34 | 9990.00 | cop[tikv] | table:t2, index:id(id) | keep order:false, stats:pseudo | -| └─IndexReader_33(Probe) | 9990.00 | root | | index:IndexFullScan_32 | -| └─IndexFullScan_32 | 9990.00 | cop[tikv] | table:t1, index:id(id) | keep order:false, stats:pseudo | -+-----------------------------+----------+-----------+------------------------+------------------------------------------------+ -5 rows in set (0.01 sec) -``` - -The execution process of `Hash Join` is as follows: - -1. Cache the data of the `Build` side in memory. -2. Construct a Hash Table on the `Build` side based on the cached data. -3. Read the data at the `Probe` side. -4. Use the data of the `Probe` side to probe the Hash Table. -5. Return qualified data to the user. - -The operator info column in the `explain` table also records other information about `Hash Join`, including whether the query is Inner Join or Outer Join, and what are the conditions of Join. In the above example, the query is an Inner Join, where the Join condition `equal:[eq(test.t1.id, test.t2.id)]` partly corresponds with the query statement `where t1.id = t2. id`. The operator info of the other Join operators in the following examples is similar to this one. - -#### `Merge Join` example - -The `Merge Join` operator usually uses less memory than `Hash Join`. However, `Merge Join` might take longer to be executed. When the amount of data is large, or the system memory is insufficient, it is recommended to use `Merge Join`. The following is an example: - -{{< copyable "sql" >}} - -```sql -EXPLAIN SELECT /*+ MERGE_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id = t2.id; -``` - -```sql -+-----------------------------+----------+-----------+------------------------+-------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+-----------------------------+----------+-----------+------------------------+-------------------------------------------------------+ -| MergeJoin_7 | 12487.50 | root | | inner join, left key:test.t1.id, right key:test.t2.id | -| ├─IndexReader_12(Build) | 9990.00 | root | | index:IndexFullScan_11 | -| │ └─IndexFullScan_11 | 9990.00 | cop[tikv] | table:t2, index:id(id) | keep order:true, stats:pseudo | -| └─IndexReader_10(Probe) | 9990.00 | root | | index:IndexFullScan_9 | -| └─IndexFullScan_9 | 9990.00 | cop[tikv] | table:t1, index:id(id) | keep order:true, stats:pseudo | -+-----------------------------+----------+-----------+------------------------+-------------------------------------------------------+ -5 rows in set (0.01 sec) -``` - -The execution of the `Merge Join` operator is as follows: - -1. Read all the data of a Join Group from the `Build` side into the memory -2. Read the data of the `Probe` side. -3. Compare whether each row of data on the `Probe` side matches a complete Join Group on the `Build` side. Apart from equivalent conditions, there are non-equivalent conditions. Here "match" mainly refers to checking whether non-equivalent conditions are met. Join Group refers to the data with the same value among all Join Keys. - -#### `Index Join` example - -If the result set (obtained after the outer tables are filtered by the `WHERE` condition) is small, it is recommended to use `Index Join`. Here "small" means data is less than 10,000 rows. - -{{< copyable "sql" >}} - -```sql -EXPLAIN SELECT /*+ INL_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id = t2.id; -``` - -```sql -+-----------------------------+----------+-----------+------------------------+--------------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+-----------------------------+----------+-----------+------------------------+--------------------------------------------------------------------------------+ -| IndexJoin_11 | 12487.50 | root | | inner join, inner:IndexReader_10, outer key:test.t1.id, inner key:test.t2.id | -| ├─IndexReader_31(Build) | 9990.00 | root | | index:IndexFullScan_30 | -| │ └─IndexFullScan_30 | 9990.00 | cop[tikv] | table:t1, index:id(id) | keep order:false, stats:pseudo | -| └─IndexReader_10(Probe) | 1.00 | root | | index:Selection_9 | -| └─Selection_9 | 1.00 | cop[tikv] | | not(isnull(test.t2.id)) | -| └─IndexRangeScan_8 | 1.00 | cop[tikv] | table:t2, index:id(id) | range: decided by [eq(test.t2.id, test.t1.id)], keep order:false, stats:pseudo | -+-----------------------------+----------+-----------+------------------------+--------------------------------------------------------------------------------+ -6 rows in set (0.00 sec) -``` - -#### `Index Hash Join` example - -`Index Hash Join` uses the same conditions as `Index Join`. However, `Index Hash Join` saves more memory in some scenarios. - -{{< copyable "sql" >}} - -```sql -EXPLAIN SELECT /*+ INL_HASH_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id = t2.id; -``` - -```sql -+-----------------------------+----------+-----------+------------------------+--------------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+-----------------------------+----------+-----------+------------------------+--------------------------------------------------------------------------------+ -| IndexHashJoin_18 | 12487.50 | root | | inner join, inner:IndexReader_10, outer key:test.t1.id, inner key:test.t2.id | -| ├─IndexReader_31(Build) | 9990.00 | root | | index:IndexFullScan_30 | -| │ └─IndexFullScan_30 | 9990.00 | cop[tikv] | table:t1, index:id(id) | keep order:false, stats:pseudo | -| └─IndexReader_10(Probe) | 1.00 | root | | index:Selection_9 | -| └─Selection_9 | 1.00 | cop[tikv] | | not(isnull(test.t2.id)) | -| └─IndexRangeScan_8 | 1.00 | cop[tikv] | table:t2, index:id(id) | range: decided by [eq(test.t2.id, test.t1.id)], keep order:false, stats:pseudo | -+-----------------------------+----------+-----------+------------------------+--------------------------------------------------------------------------------+ -6 rows in set (0.00 sec) -``` - -#### `Index Merge Join` example - -`Index Merge Join` is used in similar scenarios as Index Join. However, the index prefix used by the inner table is the inner table column collection in the join keys. `Index Merge Join` saves more memory than `INL_JOIN`. - -{{< copyable "sql" >}} - -```sql -EXPLAIN SELECT /*+ INL_MERGE_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id = t2.id; -``` - -```sql -+-----------------------------+----------+-----------+------------------------+-------------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+-----------------------------+----------+-----------+------------------------+-------------------------------------------------------------------------------+ -| IndexMergeJoin_16 | 12487.50 | root | | inner join, inner:IndexReader_14, outer key:test.t1.id, inner key:test.t2.id | -| ├─IndexReader_31(Build) | 9990.00 | root | | index:IndexFullScan_30 | -| │ └─IndexFullScan_30 | 9990.00 | cop[tikv] | table:t1, index:id(id) | keep order:false, stats:pseudo | -| └─IndexReader_14(Probe) | 1.00 | root | | index:Selection_13 | -| └─Selection_13 | 1.00 | cop[tikv] | | not(isnull(test.t2.id)) | -| └─IndexRangeScan_12 | 1.00 | cop[tikv] | table:t2, index:id(id) | range: decided by [eq(test.t2.id, test.t1.id)], keep order:true, stats:pseudo | -+-----------------------------+----------+-----------+------------------------+-------------------------------------------------------------------------------+ -6 rows in set (0.00 sec) -``` - -## Optimization example - -For more details, refer to [bikeshare example database](/import-example-data.md). - -{{< copyable "sql" >}} - -```sql -EXPLAIN SELECT count(*) FROM trips WHERE start_date BETWEEN '2017-07-01 00:00:00' AND '2017-07-01 23:59:59'; -``` - -```sql -+------------------------------+----------+-----------+---------------+------------------------------------------------------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+------------------------------+----------+-----------+---------------+------------------------------------------------------------------------------------------------------------------------+ -| StreamAgg_20 | 1.00 | root | | funcs:count(Column#13)->Column#11 | -| └─TableReader_21 | 1.00 | root | | data:StreamAgg_9 | -| └─StreamAgg_9 | 1.00 | cop[tikv] | | funcs:count(1)->Column#13 | -| └─Selection_19 | 8166.73 | cop[tikv] | | ge(bikeshare.trips.start_date, 2017-07-01 00:00:00.000000), le(bikeshare.trips.start_date, 2017-07-01 23:59:59.000000) | -| └─TableFullScan_18 | 19117643.00 | cop[tikv] | table:trips | keep order:false, stats:pseudo | -+------------------------------+----------+-----------+---------------+------------------------------------------------------------------------------------------------------------------------+ -``` - -The execution process of the above example can be illustrated as follows: - -1. Coprocessor reads the data on the trips table (executed by `TableScan_18`). -2. Find data that meets the `start_date BETWEEN '2017-07-01 00:00:00' AND '2017-07-01 23:59:59'` condition (executed by `Selection_19`). -3. Calculate the number of rows that satisfy the condition, and return the result to TiDB (executed by `StreamAgg_9`). -4. TiDB aggregates the results returned by each Coprocessor (executed by `TableReader_21`). -5. TiDB calculates the number of rows of all data (`StreamAgg_20`), and finally returns the results to the client. - -In the above query, TiDB estimates the number of rows in the output of `TableScan_18` as 19117643.00, based on the statistics of the `trips` table. The number of rows that meet the `start_date BETWEEN '2017-07-01 00:00:00' AND '2017-07-01 23:59:59'` condition is 8168.73. After the aggregation operation, there is only 1 result. - -The execution as illustrated in the above example is not efficient enough, though most of the calculation logic is pushed down to the TiKV Coprocessor. You can add an appropriate index to eliminate the full table scan on `trips` by `TableScan_18`, thereby accelerating the execution of the query: - -{{< copyable "sql" >}} - -```sql -ALTER TABLE trips ADD INDEX (start_date); -``` - -{{< copyable "sql" >}} - -```sql -EXPLAIN SELECT count(*) FROM trips WHERE start_date BETWEEN '2017-07-01 00:00:00' AND '2017-07-01 23:59:59'; -``` +### Task overview -```sql -+-----------------------------+---------+-----------+-------------------------------------------+---------------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+-----------------------------+---------+-----------+-------------------------------------------+---------------------------------------------------------------------------------+ -| StreamAgg_17 | 1.00 | root | | funcs:count(Column#13)->Column#11 | -| └─IndexReader_18 | 1.00 | root | | index:StreamAgg_9 | -| └─StreamAgg_9 | 1.00 | cop[tikv] | | funcs:count(1)->Column#13 | -| └─IndexRangeScan_16 | 8166.73 | cop[tikv] | table:trips, index:start_date(start_date) | range:[2017-07-01 00:00:00,2017-07-01 23:59:59], keep order:false, stats:pseudo | -+-----------------------------+---------+-----------+-------------------------------------------+---------------------------------------------------------------------------------+ -4 rows in set (0.00 sec) -``` +Currently, calculation tasks of TiDB can be divided into two categories: cop tasks and root tasks. A `cop[tikv]` task indicates that the operator is performed inside the TiKV coprocessor. A `root` task indicates that it will be completed inside of TiDB. -After adding the index, use `IndexScan_24` to directly read the data that meets the `start_date BETWEEN '2017-07-01 00:00:00' AND '2017-07-01 23:59:59'` condition. The estimated number of rows to be scanned decreases from 19117643.00 to 8166.73. In the test environment, the execution time of this query decreases from 50.41 seconds to 0.01 seconds. +One of the goals of SQL optimization is to push the calculation down to TiKV as much as possible. The Coprocessor in TiKV supports most of the built-in SQL functions (including the aggregate functions and the scalar functions), SQL `LIMIT` operations, index scans, and table scans. However, all `Join` operations can only be performed as root tasks in TiDB. -## Operator-related system variables +### Operator info overview -Based on MySQL, TiDB defines some special system variables and syntax to optimize performance. Some system variables are related to specific operators, such as the concurrency of the operator, the upper limit of the operator memory, and whether to use partitioned tables. These can be controlled by system variables, thereby affecting the efficiency of each operator. +The `operator info` can show useful information such as which conditions were able to be pushed down: -## See also +* `range: [1,1]` shows that the predicate from the where clause of the query (`a = 1`) was pushed right down to TiKV (the task is of `cop[tikv]`). +* `keep order:false` shows that the semantics of this query did not require TiKV to return the results in order. If the query were to be modified to require an order (such as `SELECT * FROM t WHERE a = 1 ORDER BY id`), then this condition would be `keep order:true`. +* `stats:pseudo` shows that the estimates shown in `estRows` might not be accurate. TiDB periodically updates statistics as part of a background operation. A manual update can also be performed by running `ANALYZE TABLE t`. -* [EXPLAIN](/sql-statements/sql-statement-explain.md) -* [EXPLAIN ANALYZE](/sql-statements/sql-statement-explain-analyze.md) -* [ANALYZE TABLE](/sql-statements/sql-statement-analyze-table.md) -* [TRACE](/sql-statements/sql-statement-trace.md) -* [System Variables](/system-variables.md) +Different operators output different information after the `EXPLAIN` statement is executed. You can use optimizer hints to control the behavior of the optimizer, and thereby controlling the selection of the physical operators. For example, `/*+ HASH_JOIN(t1, t2) */` means that the optimizer uses the `Hash Join` algorithm. For more details, see [Optimizer Hints](/optimizer-hints.md). diff --git a/explain-subqueries.md b/explain-subqueries.md index 9c01ca8a3954..35780dacfc2e 100644 --- a/explain-subqueries.md +++ b/explain-subqueries.md @@ -54,21 +54,22 @@ EXPLAIN SELECT * FROM t1 WHERE id IN (SELECT t1_id FROM t2); ``` ```sql -+--------------------------------+----------+-----------+------------------------------+---------------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+--------------------------------+----------+-----------+------------------------------+---------------------------------------------------------------------------------+ -| IndexMergeJoin_19 | 45.00 | root | | inner join, inner:TableReader_14, outer key:test.t2.t1_id, inner key:test.t1.id | -| ├─HashAgg_38(Build) | 45.00 | root | | group by:test.t2.t1_id, funcs:firstrow(test.t2.t1_id)->test.t2.t1_id | -| │ └─IndexReader_39 | 45.00 | root | | index:HashAgg_31 | -| │ └─HashAgg_31 | 45.00 | cop[tikv] | | group by:test.t2.t1_id, | -| │ └─IndexFullScan_37 | 90000.00 | cop[tikv] | table:t2, index:t1_id(t1_id) | keep order:false | -| └─TableReader_14(Probe) | 1.00 | root | | data:TableRangeScan_13 | -| └─TableRangeScan_13 | 1.00 | cop[tikv] | table:t1 | range: decided by [test.t2.t1_id], keep order:true | -+--------------------------------+----------+-----------+------------------------------+---------------------------------------------------------------------------------+ -7 rows in set (0.00 sec) ++----------------------------------+----------+-----------+------------------------------+---------------------------------------------------------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++----------------------------------+----------+-----------+------------------------------+---------------------------------------------------------------------------------------------------------------------------+ +| IndexJoin_14 | 5.00 | root | | inner join, inner:IndexLookUp_13, outer key:test.t2.t1_id, inner key:test.t1.id, equal cond:eq(test.t2.t1_id, test.t1.id) | +| ├─StreamAgg_49(Build) | 5.00 | root | | group by:test.t2.t1_id, funcs:firstrow(test.t2.t1_id)->test.t2.t1_id | +| │ └─IndexReader_50 | 5.00 | root | | index:StreamAgg_39 | +| │ └─StreamAgg_39 | 5.00 | cop[tikv] | | group by:test.t2.t1_id, | +| │ └─IndexFullScan_31 | 50000.00 | cop[tikv] | table:t2, index:t1_id(t1_id) | keep order:true | +| └─IndexLookUp_13(Probe) | 1.00 | root | | | +| ├─IndexRangeScan_11(Build) | 1.00 | cop[tikv] | table:t1, index:PRIMARY(id) | range: decided by [eq(test.t1.id, test.t2.t1_id)], keep order:false | +| └─TableRowIDScan_12(Probe) | 1.00 | cop[tikv] | table:t1 | keep order:false | ++----------------------------------+----------+-----------+------------------------------+---------------------------------------------------------------------------------------------------------------------------+ +8 rows in set (0.00 sec) ``` -The result above shows that TiDB performs an index join operation (merge variant) that starts by reading the index on `t2.t1_id`. The values of `t1_id` are deduplicated inside TiKV first as a part of the `└─HashAgg_31` operator task, and then deduplicated again in TiDB as a part of the `├─HashAgg_38(Build)` operator task. The deduplication is performed by the aggregation function `firstrow(test.t2.t1_id)`. The result is then joined against the `t1` table's `PRIMARY KEY`. +The result above shows that TiDB performs an index join operation that starts by reading the index on `t2.t1_id`. The values of `t1_id` are deduplicated inside TiKV first as a part of the `└─HashAgg_31` operator task, and then deduplicated again in TiDB as a part of the `├─HashAgg_38(Build)` operator task. The deduplication is performed by the aggregation function `firstrow(test.t2.t1_id)`. The result is then joined against the `t1` table's `PRIMARY KEY`. ## Inner join (unique subquery) @@ -79,23 +80,24 @@ EXPLAIN SELECT * FROM t1 WHERE id IN (SELECT t1_id FROM t3); ``` ```sql -+-----------------------------+---------+-----------+------------------------------+---------------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+-----------------------------+---------+-----------+------------------------------+---------------------------------------------------------------------------------+ -| IndexMergeJoin_20 | 999.00 | root | | inner join, inner:TableReader_15, outer key:test.t3.t1_id, inner key:test.t1.id | -| ├─IndexReader_39(Build) | 999.00 | root | | index:IndexFullScan_38 | -| │ └─IndexFullScan_38 | 999.00 | cop[tikv] | table:t3, index:t1_id(t1_id) | keep order:false | -| └─TableReader_15(Probe) | 1.00 | root | | data:TableRangeScan_14 | -| └─TableRangeScan_14 | 1.00 | cop[tikv] | table:t1 | range: decided by [test.t3.t1_id], keep order:true | -+-----------------------------+---------+-----------+------------------------------+---------------------------------------------------------------------------------+ -5 rows in set (0.00 sec) ++----------------------------------+---------+-----------+-----------------------------+---------------------------------------------------------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++----------------------------------+---------+-----------+-----------------------------+---------------------------------------------------------------------------------------------------------------------------+ +| IndexJoin_17 | 1978.13 | root | | inner join, inner:IndexLookUp_16, outer key:test.t3.t1_id, inner key:test.t1.id, equal cond:eq(test.t3.t1_id, test.t1.id) | +| ├─TableReader_44(Build) | 1978.00 | root | | data:TableFullScan_43 | +| │ └─TableFullScan_43 | 1978.00 | cop[tikv] | table:t3 | keep order:false | +| └─IndexLookUp_16(Probe) | 1.00 | root | | | +| ├─IndexRangeScan_14(Build) | 1.00 | cop[tikv] | table:t1, index:PRIMARY(id) | range: decided by [eq(test.t1.id, test.t3.t1_id)], keep order:false | +| └─TableRowIDScan_15(Probe) | 1.00 | cop[tikv] | table:t1 | keep order:false | ++----------------------------------+---------+-----------+-----------------------------+---------------------------------------------------------------------------------------------------------------------------+ +6 rows in set (0.01 sec) ``` Semantically because `t3.t1_id` is guaranteed unique, it can be executed directly as an `INNER JOIN`. ## Semi join (correlated subquery) -In the previous two examples, TiDB is able to perform an `INNER JOIN` operation after the data inside the subquery is made unique (via `HashAgg`) or guaranteed unique. Both joins are performed using an Index Join (merge variant). +In the previous two examples, TiDB is able to perform an `INNER JOIN` operation after the data inside the subquery is made unique (via `HashAgg`) or guaranteed unique. Both joins are performed using an Index Join. In this example, TiDB chooses a different execution plan: @@ -104,17 +106,18 @@ EXPLAIN SELECT * FROM t1 WHERE id IN (SELECT t1_id FROM t2 WHERE t1_id != t1.int ``` ```sql -+-----------------------------+-----------+-----------+------------------------------+--------------------------------------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+-----------------------------+-----------+-----------+------------------------------+--------------------------------------------------------------------------------------------------------+ -| MergeJoin_9 | 45446.40 | root | | semi join, left key:test.t1.id, right key:test.t2.t1_id, other cond:ne(test.t2.t1_id, test.t1.int_col) | -| ├─IndexReader_24(Build) | 180000.00 | root | | index:IndexFullScan_23 | -| │ └─IndexFullScan_23 | 180000.00 | cop[tikv] | table:t2, index:t1_id(t1_id) | keep order:true | -| └─TableReader_22(Probe) | 56808.00 | root | | data:Selection_21 | -| └─Selection_21 | 56808.00 | cop[tikv] | | ne(test.t1.id, test.t1.int_col) | -| └─TableFullScan_20 | 71010.00 | cop[tikv] | table:t1 | keep order:true | -+-----------------------------+-----------+-----------+------------------------------+--------------------------------------------------------------------------------------------------------+ -6 rows in set (0.00 sec) ++----------------------------------+---------+-----------+-----------------------------+-------------------------------------------------------------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++----------------------------------+---------+-----------+-----------------------------+-------------------------------------------------------------------------------------------------------------------------------+ +| IndexJoin_14 | 1582.40 | root | | anti semi join, inner:IndexLookUp_13, outer key:test.t3.t1_id, inner key:test.t1.id, equal cond:eq(test.t3.t1_id, test.t1.id) | +| ├─TableReader_35(Build) | 1978.00 | root | | data:TableFullScan_34 | +| │ └─TableFullScan_34 | 1978.00 | cop[tikv] | table:t3 | keep order:false | +| └─IndexLookUp_13(Probe) | 1.00 | root | | | +| ├─IndexRangeScan_10(Build) | 1.00 | cop[tikv] | table:t1, index:PRIMARY(id) | range: decided by [eq(test.t1.id, test.t3.t1_id)], keep order:false | +| └─Selection_12(Probe) | 1.00 | cop[tikv] | | lt(test.t1.int_col, 100) | +| └─TableRowIDScan_11 | 1.00 | cop[tikv] | table:t1 | keep order:false | ++----------------------------------+---------+-----------+-----------------------------+-------------------------------------------------------------------------------------------------------------------------------+ +7 rows in set (0.00 sec) ``` From the result above, you can see that TiDB uses a `Semi Join` algorithm. Semi-join differs from inner join: semi-join only permits the first value on the right key (`t2.t1_id`), which means that the duplicates are eliminated as a part of the join operator task. The join algorithm is also Merge Join, which is like an efficient zipper-merge as the operator reads data from both the left and the right side in sorted order. diff --git a/expression-syntax.md b/expression-syntax.md index 782010a16bd9..09489d4f9597 100644 --- a/expression-syntax.md +++ b/expression-syntax.md @@ -20,18 +20,32 @@ The expressions can be divided into the following types: The following rules are the expression syntax, which is based on the [parser.y](https://github.com/pingcap/parser/blob/master/parser.y) rules of TiDB parser. For the navigable version of the following syntax diagram, refer to [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/#Expression). -**Expression:** - -![Expression](/media/sqlgram/Expression.png) - -**PredicateExpr:** - -![PredicateExpr](/media/sqlgram/PredicateExpr.png) - -**BitExpr:** - -![BitExpr](/media/sqlgram/BitExpr.png) - -**SimpleExpr:** - -![SimpleExpr](/media/sqlgram/SimpleExpr.png) +```ebnf+diagram +Expression ::= + ( singleAtIdentifier assignmentEq | 'NOT' | Expression ( logOr | 'XOR' | logAnd ) ) Expression +| 'MATCH' '(' ColumnNameList ')' 'AGAINST' '(' BitExpr FulltextSearchModifierOpt ')' +| PredicateExpr ( IsOrNotOp 'NULL' | CompareOp ( ( singleAtIdentifier assignmentEq )? PredicateExpr | AnyOrAll SubSelect ) )* ( IsOrNotOp ( trueKwd | falseKwd | 'UNKNOWN' ) )? + +PredicateExpr ::= + BitExpr ( BetweenOrNotOp BitExpr 'AND' BitExpr )* ( InOrNotOp ( '(' ExpressionList ')' | SubSelect ) | LikeOrNotOp SimpleExpr LikeEscapeOpt | RegexpOrNotOp SimpleExpr )? + +BitExpr ::= + BitExpr ( ( '|' | '&' | '<<' | '>>' | '*' | '/' | '%' | 'DIV' | 'MOD' | '^' ) BitExpr | ( '+' | '-' ) ( BitExpr | "INTERVAL" Expression TimeUnit ) ) +| SimpleExpr + +SimpleExpr ::= + SimpleIdent ( ( '->' | '->>' ) stringLit )? +| FunctionCallKeyword +| FunctionCallNonKeyword +| FunctionCallGeneric +| SimpleExpr ( 'COLLATE' CollationName | pipes SimpleExpr ) +| WindowFuncCall +| Literal +| paramMarker +| Variable +| SumExpr +| ( '!' | '~' | '-' | '+' | 'NOT' | 'BINARY' ) SimpleExpr +| 'EXISTS'? SubSelect +| ( ( '(' ( ExpressionList ',' )? | 'ROW' '(' ExpressionList ',' ) Expression | builtinCast '(' Expression 'AS' CastType | ( 'DEFAULT' | 'VALUES' ) '(' SimpleIdent | 'CONVERT' '(' Expression ( ',' CastType | 'USING' CharsetName ) ) ')' +| 'CASE' ExpressionOpt WhenClause+ ElseOpt 'END' +``` diff --git a/faq/deploy-and-maintain-faq.md b/faq/deploy-and-maintain-faq.md index 561cbd189c79..0e8a79dbe915 100644 --- a/faq/deploy-and-maintain-faq.md +++ b/faq/deploy-and-maintain-faq.md @@ -89,7 +89,7 @@ Check the time difference between the machine time of the monitor and the time w | Variable | Description | | ---- | ------- | | `cluster_name` | the name of a cluster, adjustable | -| `tidb_version` | the version of TiDB, configured by default in TiDB Ansible branches | +| `tidb_version` | the version of TiDB | | `deployment_method` | the method of deployment, binary by default, Docker optional | | `process_supervision` | the supervision way of processes, systemd by default, supervise optional | | `timezone` | the timezone of the managed node, adjustable, `Asia/Shanghai` by default, used with the `set_timezone` variable | @@ -104,25 +104,9 @@ Check the time difference between the machine time of the monitor and the time w | `enable_slow_query_log` | to record the slow query log of TiDB into a single file: ({{ deploy_dir }}/log/tidb_slow_query.log). False by default, to record it into the TiDB log | | `deploy_without_tidb` | the Key-Value mode, deploy only PD, TiKV and the monitoring service, not TiDB; set the IP of the tidb_servers host group to null in the `inventory.ini` file | -### Deploy TiDB offline using TiDB Ansible（not recommended since TiDB v4.0) - -> **Warning:** -> -> It is not recommended to deploy TiDB using TiDB Ansible since TiDB v4.0. [Use TiUP to deploy TiDB](/production-deployment-using-tiup.md) instead. - -If the central control machine cannot access the Internet, you can [deploy TiDB offline using TiDB Ansible](https://docs.pingcap.com/tidb/stable/offline-deployment-using-ansible). - -### How to deploy TiDB quickly using Docker Compose on a single machine? - -You can use Docker Compose to build a TiDB cluster locally, including the cluster monitoring components. You can also customize the version and number of instances for each component. The configuration file can also be customized. You can only use this deployment method for testing and development environment. For details, see [TiDB Docker Compose Deployment](/deploy-test-cluster-using-docker-compose.md). - ### How to separately record the slow query log in TiDB? How to locate the slow query SQL statement? -1. The slow query definition for TiDB is in the `conf/tidb.yml` configuration file of `tidb-ansible`. The `slow-threshold: 300` parameter is used to configure the threshold value of the slow query (unit: millisecond). - - The slow query log is recorded in `tidb.log` by default. If you want to generate a slow query log file separately, set `enable_slow_query_log` in the `inventory.ini` configuration file to `True`. - - Then run `ansible-playbook rolling_update.yml --tags=tidb` to perform a rolling update on the `tidb-server` instance. After the update is finished, the `tidb-server` instance will record the slow query log in `tidb_slow_query.log`. +1. The slow query definition for TiDB is in the TiDB configuration file. The `slow-threshold: 300` parameter is used to configure the threshold value of the slow query (unit: millisecond). 2. If a slow query occurs, you can locate the `tidb-server` instance where the slow query is and the slow query time point using Grafana and find the SQL statement information recorded in the log on the corresponding node. @@ -156,30 +140,10 @@ The Direct mode wraps the Write request into the I/O command and sends this comm ./fio -ioengine=psync -bs=32k -fdatasync=1 -thread -rw=randrw -percentage_random=100,0 -size=10G -filename=fio_randread_write_test.txt -name='fio mixed randread and sequential write test' -iodepth=4 -runtime=60 -numjobs=4 -group_reporting --output-format=json --output=fio_randread_write_test.json ``` -### Error `UNREACHABLE! "msg": "Failed to connect to the host via ssh: "` when deploying TiDB using TiDB Ansible - -Two possible reasons and solutions: - -- The SSH mutual trust is not configured as required. It’s recommended to follow [the steps described in the official document](/online-deployment-using-ansible.md#step-5-configure-the-ssh-mutual-trust-and-sudo-rules-on-the-control-machine) and check whether it is successfully configured using `ansible -i inventory.ini all -m shell -a 'whoami' -b`. -- If it involves the scenario where a single server is assigned multiple roles, for example, the mixed deployment of multiple components or multiple TiKV instances are deployed on a single server, this error might be caused by the SSH reuse mechanism. You can use the option of `ansible … -f 1` to avoid this error. - ## Cluster management ### Daily management -#### What are the common operations of TiDB Ansible? - -| Job | Playbook | -|:----------------------------------|:-----------------------------------------| -| Start the cluster | `ansible-playbook start.yml` | -| Stop the cluster | `ansible-playbook stop.yml` | -| Destroy the cluster | `ansible-playbook unsafe_cleanup.yml` (If the deployment directory is a mount point, an error will be reported, but implementation results will remain unaffected) | -| Clean data (for test) | `ansible-playbook unsafe_cleanup_data.yml` | -| Apply rolling updates | `ansible-playbook rolling_update.yml` | -| Apply rolling updates to TiKV | `ansible-playbook rolling_update.yml --tags=tikv` | -| Apply rolling updates to components except PD | `ansible-playbook rolling_update.yml --skip-tags=pd` | -| Apply rolling updates to the monitoring components | `ansible-playbook rolling_update_monitor.yml` | - #### How to log into TiDB? You can log into TiDB like logging into MySQL. For example: @@ -206,7 +170,7 @@ By default, TiDB/PD/TiKV outputs standard error in the logs. If a log file is sp #### How to safely stop TiDB? -If the cluster is deployed using TiDB Ansible, you can use the `ansible-playbook stop.yml` command to stop the TiDB cluster. If the cluster is not deployed using TiDB Ansible, `kill` all the services directly. The components of TiDB will do `graceful shutdown`. +Kill all the services using `kill` directly. The components of TiDB will do `graceful shutdown`. #### Can `kill` be executed in TiDB? @@ -227,11 +191,11 @@ Take `Release Version: v1.0.3-1-ga80e796` as an example of version number descri - `-1` indicates the current version has one commit. - `ga80e796` indicates the version `git-hash`. -#### What's the difference between various TiDB master versions? How to avoid using the wrong TiDB Ansible version? +#### What's the difference between various TiDB master versions? The TiDB community is highly active. After the 1.0 GA release, the engineers have been keeping optimizing and fixing bugs. Therefore, the TiDB version is updated quite fast. If you want to keep informed of the latest version, see [TiDB Weekly update](https://pingcap.com/weekly/). -It is not recommended to deploy the TiDB cluster using TiDB Ansible. [Deploy TiDB using TiUP](/production-deployment-using-tiup.md) instead. TiDB has a unified management of the version number after the 1.0 GA release. You can view the version number using the following two methods: +It is recommeneded to [deploy TiDB using TiUP](/production-deployment-using-tiup.md). TiDB has a unified management of the version number after the 1.0 GA release. You can view the version number using the following two methods: - `select tidb_version()` - `tidb-server -V` @@ -277,6 +241,13 @@ Two reasons: - The two results are calculated in different ways. `information_schema.tables.data_length` is an estimated value by calculating the averaged length of each row, while the store size on the TiKV monitoring panel sums up the length of the data files (the SST files of RocksDB) in a single TiKV instance. - `information_schema.tables.data_length` is a logical value, while the store size is a physical value. The redundant data generated by multiple versions of the transaction is not included in the logical value, while the redundant data is compressed by TiKV in the physical value. +#### Why does the transaction not use the Async Commit or the one-phase commit feature? + +In the following situations, even you have enabled the [Async Commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) feature and the [one-phase commit](/system-variables.md#tidb_enable_1pc-new-in-v50) feature using the system variables, TiDB will not use these features: + +- If you have enabled TiDB Binlog, restricted by the implementation of TiDB Binlog, TiDB does not use the Async Commit or one-phase commit feature. +- TiDB uses the Async Commit or one-phase commit features only when no more than 256 key-value pairs are written in the transaction and the total size of keys is no more than 4 KB. This is because, for transactions with a large amount of data to write, using Async Commit cannot greatly improve the performance. + ### PD management #### The `TiKV cluster is not bootstrapped` message is displayed when I access PD @@ -316,11 +287,6 @@ The offline node usually indicates the TiKV node. You can determine whether the 1. Manually stop the relevant services on the offline node. 2. Delete the `node_exporter` data of the corresponding node from the Prometheus configuration file. -3. Delete the data of the corresponding node from Ansible `inventory.ini`. - -#### Why couldn't I connect to the PD server using `127.0.0.1` when I was using the PD Control? - -If your TiDB cluster is deployed using TiDB Ansible, the PD external service port is not bound to `127.0.0.1`, so PD Control does not recognize `127.0.0.1` and you can only connect to it using the local IP address. ### TiDB server management diff --git a/faq/migration-tidb-faq.md b/faq/migration-tidb-faq.md index 501f5780e4c4..7d6559264735 100644 --- a/faq/migration-tidb-faq.md +++ b/faq/migration-tidb-faq.md @@ -5,7 +5,15 @@ summary: Learn about the FAQs related to data migration. # Migration FAQs -This document summarizes the FAQs related to TiDB data migration. +This document summarizes the frequently asked questions (FAQs) related to TiDB data migration. + +For the frequently asked questions about migration-related tools, click the corresponding links in the list below: + +- [Backup & Restore FAQ](/br/backup-and-restore-faq.md) +- [TiDB Binlog FAQ](/tidb-binlog/tidb-binlog-faq.md) +- [TiDB Lightning FAQs](/tidb-lightning/tidb-lightning-faq.md) +- [TiDB Data Migration (DM) FAQs](https://docs.pingcap.com/tidb-data-migration/stable/faq) +- [Troubleshoot TiCDC](/ticdc/troubleshoot-ticdc.md) ## Full data export and import @@ -13,17 +21,62 @@ This document summarizes the FAQs related to TiDB data migration. Because TiDB supports most MySQL syntax, generally you can migrate your applications to TiDB without changing a single line of code in most cases. -### If I accidentally import the MySQL user table into TiDB, or forget the password and cannot log in, how to deal with it? +### Data import and export is slow, and many retries and EOF errors appear in the log of each component without other errors -Restart the TiDB service, add the `-skip-grant-table=true` parameter in the configuration file. Log into the cluster without password and recreate the user, or recreate the `mysql.user` table. For the specific table schema, search the official documentation. +If no other logical errors occur, retries and EOF errors might be caused by network issues. It is recommended to first use tools to check the network connectivity. In the following example, [iperf](https://iperf.fr/) is used for troubleshooting: + ++ Execute the following command on the server-side node where the retries and EOF errors occur: + + {{< copyable "shell-regular" >}} + + ```shell + iperf3 -s + ``` + ++ Execute the following command on the client-side node where the retries and EOF errors occur: + + {{< copyable "shell-regular" >}} -### Can TiDB provide services while Loader is running? + ```shell + iperf3 -c + ``` + +The following example is the output of a client node with a good network connection: + +```shell +$ iperf3 -c 192.168.196.58 +Connecting to host 192.168.196.58, port 5201 +[ 5] local 192.168.196.150 port 55397 connected to 192.168.196.58 port 5201 +[ ID] Interval Transfer Bitrate +[ 5] 0.00-1.00 sec 18.0 MBytes 150 Mbits/sec +[ 5] 1.00-2.00 sec 20.8 MBytes 175 Mbits/sec +[ 5] 2.00-3.00 sec 18.2 MBytes 153 Mbits/sec +[ 5] 3.00-4.00 sec 22.5 MBytes 188 Mbits/sec +[ 5] 4.00-5.00 sec 22.4 MBytes 188 Mbits/sec +[ 5] 5.00-6.00 sec 22.8 MBytes 191 Mbits/sec +[ 5] 6.00-7.00 sec 20.8 MBytes 174 Mbits/sec +[ 5] 7.00-8.00 sec 20.1 MBytes 168 Mbits/sec +[ 5] 8.00-9.00 sec 20.8 MBytes 175 Mbits/sec +[ 5] 9.00-10.00 sec 21.8 MBytes 183 Mbits/sec +- - - - - - - - - - - - - - - - - - - - - - - - - +[ ID] Interval Transfer Bitrate +[ 5] 0.00-10.00 sec 208 MBytes 175 Mbits/sec sender +[ 5] 0.00-10.00 sec 208 MBytes 174 Mbits/sec receiver + +iperf Done. +``` + +If the output shows low network bandwidth and high bandwidth fluctuations, a large number of retries and EOF errors might appear in each component log. In this case, you need to consult your network service provider to improve the network quality. + +If the output of each metric looks good, try to update each component. If the problem persists after the updating, you can [contact us](https://tidbcommunity.slack.com/archives/CH7TTLL7P). + +### If I accidentally import the MySQL user table into TiDB, or forget the password and cannot log in, how to deal with it? -TiDB can provide services while Loader is running because Loader inserts the data logically. But do not perform the related DDL operations. +Restart the TiDB service, add the `-skip-grant-table=true` parameter in the configuration file. Log into the cluster without password and recreate the user, or recreate the `mysql.user` table. For the specific table schema, search the official documentation. ### How to export the data in TiDB? -Currently, TiDB does not support `select into outfile`. You can use the following methods to export the data in TiDB: +You can use the following methods to export the data in TiDB: - See [MySQL uses mysqldump to export part of the table data](https://blog.csdn.net/xin_yu_xin/article/details/7574662) in Chinese and export data using mysqldump and the `WHERE` clause. - Use the MySQL client to export the results of `select` to a file. @@ -62,63 +115,30 @@ Two solutions: - You can also increase the limited number of statements in a single TiDB transaction, but this will consume more memory. -### Does TiDB have a function like the Flashback Query in Oracle? Does it support DDL? - - Yes, it does. And it supports DDL as well. For details, see [how TiDB reads data from history versions](/read-historical-data.md). - -## Migrate the data online - -### Syncer infrastructure - -See [Parsing TiDB online data synchronization tool Syncer](https://pingcap.com/blog-cn/tidb-syncer/) in Chinese. +### Why does Dumpling return `The local disk space is insufficient` error or cause the upstream database to run out of memory when exporting a table? -#### Syncer user guide +This issue might have the following causes: -See [Syncer User Guide](/syncer-overview.md). ++ The database's primary keys are not evenly distributed (for example, when you enable [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md)). ++ The upstream database is TiDB and the exported table is a partitioned table. -#### How to configure to monitor Syncer status? +For the above cases, Dumpling splits excessively large data chunk for the export and sends queries with excessively large results. To address the issue, you can [contact us](https://tidbcommunity.slack.com/archives/CH7TTLL7P) to get the nightly version of Dumpling. -Download and import [Syncer Json](https://github.com/pingcap/docs/blob/master/etc/Syncer.json) to Grafana. Edit the Prometheus configuration file and add the following content: +### Does TiDB have a function like the Flashback Query in Oracle? Does it support DDL? -``` -- job_name: 'syncer_ops' // task name - static_configs: - - targets: [’10.10.1.1:10096’] // Syncer monitoring address and port, informing Prometheus to pull the data of Syncer -``` + Yes, it does. And it supports DDL as well. For details, see [how TiDB reads data from history versions](/read-historical-data.md). -Restart Prometheus. +## Migrate the data online -#### Is there a current solution to replicating data from TiDB to other databases like HBase and Elasticsearch? +### Is there a current solution to replicating data from TiDB to other databases like HBase and Elasticsearch? No. Currently, the data replication depends on the application itself. -#### Does Syncer support replicating only some of the tables when Syncer is replicating data? - -Yes. For details, see [Syncer User Guide](/syncer-overview.md). - -#### Do frequent DDL operations affect the replication speed of Syncer? - -Frequent DDL operations may affect the replication speed. For Syncer, DDL operations are executed serially. When DDL operations are executed during data replication, data will be replicated serially and thus the replication speed will be slowed down. - -#### If the machine that Syncer is in is broken and the directory of the `syncer.meta` file is lost, what should I do? - -When you replicate data using Syncer GTID, the `syncer.meta` file is constantly updated during the replication process. The current version of Syncer does not contain the design for high availability. The `syncer.meta` configuration file of Syncer is directly stored on the hard disks, which is similar to other tools in the MySQL ecosystem, such as Mydumper. - -Two solutions: - -- Put the `syncer.meta` file in a relatively secure disk. For example, use disks with RAID 1. -- Restore the location information of history replication according to the monitoring data that Syncer reports to Prometheus regularly. But the location information might be inaccurate due to the delay when a large amount of data is replicated. - -#### If the downstream TiDB data is not consistent with the MySQL data during the replication process of Syncer, will DML operations cause exits? - -- If the data exists in the upstream MySQL but does not exist in the downstream TiDB, when the upstream MySQL performs the `UPDATE` or `DELETE` operation on this row of data, Syncer will not report an error and the replication process will not exit, and this row of data does not exist in the downstream. -- If a conflict exists in the primary key indexes or the unique indexes in the downstream, preforming the `UPDATE` operation will cause an exit and performing the `INSERT` operation will not cause an exit. - ## Migrate the traffic ### How to migrate the traffic quickly? -It is recommended to build a multi-source MySQL -> TiDB real-time replication environment using Syncer tool. You can migrate the read and write traffic in batches by editing the network configuration as needed. Deploy a stable network LB (HAproxy, LVS, F5, DNS, etc.) on the upper layer, in order to implement seamless migration by directly editing the network configuration. +It is recommended to migrate application data from MySQL to TiDB using [TiDB Data Migration](https://docs.pingcap.com/tidb-data-migration/v2.0/overview) tool. You can migrate the read and write traffic in batches by editing the network configuration as needed. Deploy a stable network LB (HAproxy, LVS, F5, DNS, etc.) on the upper layer, in order to implement seamless migration by directly editing the network configuration. ### Is there a limit for the total write and read capacity in TiDB? @@ -126,7 +146,7 @@ The total read capacity has no limit. You can increase the read capacity by addi ### The error message `transaction too large` is displayed -Due to the limitation of the underlying storage engine, each key-value entry (one row) in TiDB should be no more than 6MB. +Due to the limitation of the underlying storage engine, each key-value entry (one row) in TiDB should be no more than 6MB. You can adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration value up to 120MB. Distributed transactions need two-phase commit and the bottom layer performs the Raft replication. If a transaction is very large, the commit process would be quite slow and the write conflict is more likely to occur. Moreover, the rollback of a failed transaction leads to an unnecessary performance penalty. To avoid these problems, we limit the total size of key-value entries to no more than 100MB in a transaction by default. If you need larger transactions, modify the value of `txn-total-size-limit` in the TiDB configuration file. The maximum value of this configuration item is up to 10G. The actual limitation is also affected by the physical memory of the machine. @@ -160,15 +180,5 @@ If the amount of data that needs to be deleted at a time is very large, this loo ### How to improve the data loading speed in TiDB? -- The [Lightning](/tidb-lightning/tidb-lightning-overview.md) tool is developed for distributed data import. It should be noted that the data import process does not perform a complete transaction process for performance reasons. Therefore, the ACID constraint of the data being imported during the import process cannot be guaranteed. The ACID constraint of the imported data can only be guaranteed after the entire import process ends. Therefore, the applicable scenarios mainly include importing new data (such as a new table or a new index) or the full backup and restoring (truncate the original table and then import data). +- The [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) tool is developed for distributed data import. It should be noted that the data import process does not perform a complete transaction process for performance reasons. Therefore, the ACID constraint of the data being imported during the import process cannot be guaranteed. The ACID constraint of the imported data can only be guaranteed after the entire import process ends. Therefore, the applicable scenarios mainly include importing new data (such as a new table or a new index) or the full backup and restoring (truncate the original table and then import data). - Data loading in TiDB is related to the status of disks and the whole cluster. When loading data, pay attention to metrics like the disk usage rate of the host, TiClient Error, Backoff, Thread CPU and so on. You can analyze the bottlenecks using these metrics. - -### What should I do if it is slow to reclaim storage space after deleting data? - -You can configure concurrent GC to increase the speed of reclaiming storage space. The default concurrency is 1, and you can modify it to at most 50% of the number of TiKV instances using the following command: - -{{< copyable "sql" >}} - -```sql -update mysql.tidb set VARIABLE_VALUE="3" where VARIABLE_NAME="tikv_gc_concurrency"; -``` diff --git a/faq/sql-faq.md b/faq/sql-faq.md index bbbbee41e4d6..c8cbd05be770 100644 --- a/faq/sql-faq.md +++ b/faq/sql-faq.md @@ -134,13 +134,7 @@ Deleting a large amount of data leaves a lot of useless keys, affecting the quer ## What should I do if it is slow to reclaim storage space after deleting data? -You can configure concurrent GC to increase the speed of reclaiming storage space. The default concurrency is 1, and you can modify it to at most 50% of the number of TiKV instances using the following command: - -{{< copyable "sql" >}} - -```sql -update mysql.tidb set VARIABLE_VALUE="3" where VARIABLE_NAME="tikv_gc_concurrency"; -``` +Because TiDB uses Multiversion concurrency control (MVCC), deleting data does not immediately reclaim space. Garbage collection is delayed so that concurrent transactions are able to see earlier versions of rows. This can be configured via the [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) (default: `10m0s`) system variable. ## Does `SHOW PROCESSLIST` display the system process ID? diff --git a/faq/tidb-faq.md b/faq/tidb-faq.md index 1b707e75561e..f757286e9ad4 100644 --- a/faq/tidb-faq.md +++ b/faq/tidb-faq.md @@ -124,12 +124,12 @@ The accessed Region is not available. A Raft Group is not available, with possib #### ERROR 9006 (HY000): GC life time is shorter than transaction duration -The interval of `GC Life Time` is too short. The data that should have been read by long transactions might be deleted. You can add `GC Life Time` using the following command: +The interval of `GC Life Time` is too short. The data that should have been read by long transactions might be deleted. You can adjust [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) using the following command: {{< copyable "sql" >}} ```sql -update mysql.tidb set variable_value='30m' where variable_name='tikv_gc_life_time'; +SET GLOBAL tidb_gc_life_time = '30m'; ``` > **Note:** @@ -140,15 +140,11 @@ update mysql.tidb set variable_value='30m' where variable_name='tikv_gc_life_tim Check whether `tidb_disable_txn_auto_retry` is set to `on`. If so, set it to `off`; if it is already `off`, increase the value of `tidb_retry_limit` until the error no longer occurs. -#### ERROR 1105 (HY000): client has multi-statement capability disabled +#### ERROR 8130 (HY000): client has multi-statement capability disabled -This error might occur after upgrading from an earlier version of TiDB. To prevent against SQL injection attacks, TiDB now prevents multiple queries being executed in the same `COM_QUERY` call by default. +This error might occur after upgrading from an earlier version of TiDB. To reduce the impact of SQL injection attacks, TiDB now prevents multiple queries from being executed in the same `COM_QUERY` call by default. -Check the documentation for your client driver for instructions on how to enable multiple statements. i.e: - -* [go-sql-driver](https://github.com/go-sql-driver/mysql#multistatements) (`multiStatements`) -* [Connector/J](https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-configuration-properties.html) (`allowMultiQueries`) -* PHP [mysqli](https://dev.mysql.com/doc/apis-php/en/apis-php-mysqli.quickstart.multiple-statement.html) (`mysqli_multi_query`) +The system variable [`tidb_multi_statement_mode`](/system-variables.md#tidb_multi_statement_mode-new-in-v4011) can be used to control this behavior. ### MySQL native error messages diff --git a/functions-and-operators/aggregate-group-by-functions.md b/functions-and-operators/aggregate-group-by-functions.md index c03228d8e4b2..4094fa952290 100644 --- a/functions-and-operators/aggregate-group-by-functions.md +++ b/functions-and-operators/aggregate-group-by-functions.md @@ -10,17 +10,17 @@ This document describes details about the supported aggregate functions in TiDB. ## Supported aggregate functions -This section describes the supported MySQL group (aggregate) functions in TiDB. +This section describes the supported MySQL `GROUP BY` aggregate functions in TiDB. | Name | Description | |:--------------------------------------------------------------------------------------------------------------|:--------------------------------------------------| -| [`COUNT()`](https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_count) | Return a count of the number of rows returned | -| [`COUNT(DISTINCT)`](https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_count-distinct) | Return the count of a number of different values | -| [`SUM()`](https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_sum) | Return the sum | -| [`AVG()`](https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_avg) | Return the average value of the argument | -| [`MAX()`](https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_max) | Return the maximum value | -| [`MIN()`](https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_min) | Return the minimum value | -| [`GROUP_CONCAT()`](https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_group-concat) | Return a concatenated string | +| [`COUNT()`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_count) | Return a count of the number of rows returned | +| [`COUNT(DISTINCT)`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_count-distinct) | Return the count of a number of different values | +| [`SUM()`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_sum) | Return the sum | +| [`AVG()`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_avg) | Return the average value of the argument | +| [`MAX()`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_max) | Return the maximum value | +| [`MIN()`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_min) | Return the minimum value | +| [`GROUP_CONCAT()`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_group-concat) | Return a concatenated string | | [`VARIANCE()`, `VAR_POP()`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_var-pop) | Return the population standard variance| | [`STD()`，`STDDEV()`，`STDDEV_POP`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_std) | Return the population standard deviation | | [`VAR_SAMP()`](https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_var-samp) | Return the sample variance | diff --git a/functions-and-operators/expressions-pushed-down.md b/functions-and-operators/expressions-pushed-down.md index d6c6e16d4be2..33484d8b5d4a 100644 --- a/functions-and-operators/expressions-pushed-down.md +++ b/functions-and-operators/expressions-pushed-down.md @@ -16,7 +16,7 @@ When TiDB reads data from TiKV, TiDB tries to push down some expressions (includ | [Comparison functions and operators](/functions-and-operators/operators.md#comparison-functions-and-operators) | <, <=, =, != (`<>`), >, >=, [`<=>`](https://dev.mysql.com/doc/refman/5.7/en/comparison-operators.html#operator_equal-to), [`IN()`](https://dev.mysql.com/doc/refman/5.7/en/comparison-operators.html#function_in), IS NULL, LIKE, IS TRUE, IS FALSE, [`COALESCE()`](https://dev.mysql.com/doc/refman/5.7/en/comparison-operators.html#function_coalesce) | | [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | +, -, *, /, [`ABS()`](https://dev.mysql.com/doc/refman/5.7/en/mathematical-functions.html#function_abs), [`CEIL()`](https://dev.mysql.com/doc/refman/5.7/en/mathematical-functions.html#function_ceil), [`CEILING()`](https://dev.mysql.com/doc/refman/5.7/en/mathematical-functions.html#function_ceiling), [`FLOOR()`](https://dev.mysql.com/doc/refman/5.7/en/mathematical-functions.html#function_floor) | | [Control flow functions](/functions-and-operators/control-flow-functions.md) | [`CASE`](https://dev.mysql.com/doc/refman/5.7/en/control-flow-functions.html#operator_case), [`IF()`](https://dev.mysql.com/doc/refman/5.7/en/control-flow-functions.html#function_if), [`IFNULL()`](https://dev.mysql.com/doc/refman/5.7/en/control-flow-functions.html#function_ifnull) | -| [JSON functions](/functions-and-operators/json-functions.md) | [JSON_TYPE(json_val)][json_type],
[JSON_EXTRACT(json_doc, path[, path] ...)][json_extract],
[JSON_UNQUOTE(json_val)][json_unquote],
[JSON_OBJECT(key, val[, key, val] ...)][json_object],
[JSON_ARRAY([val[, val] ...])][json_array],
[JSON_MERGE(json_doc, json_doc[, json_doc] ...)][json_merge],
[JSON_SET(json_doc, path, val[, path, val] ...)][json_set],
[JSON_INSERT(json_doc, path, val[, path, val] ...)][json_insert],
[JSON_REPLACE(json_doc, path, val[, path, val] ...)][json_replace],
[JSON_REMOVE(json_doc, path[, path] ...)][json_remove] | +| [JSON functions](/functions-and-operators/json-functions.md) | [JSON_TYPE(json_val)][json_type],
[JSON_EXTRACT(json_doc, path[, path] ...)][json_extract],
[JSON_OBJECT(key, val[, key, val] ...)][json_object],
[JSON_ARRAY([val[, val] ...])][json_array],
[JSON_MERGE(json_doc, json_doc[, json_doc] ...)][json_merge],
[JSON_SET(json_doc, path, val[, path, val] ...)][json_set],
[JSON_INSERT(json_doc, path, val[, path, val] ...)][json_insert],
[JSON_REPLACE(json_doc, path, val[, path, val] ...)][json_replace],
[JSON_REMOVE(json_doc, path[, path] ...)][json_remove] | | [Date and time functions](/functions-and-operators/date-and-time-functions.md) | [`DATE_FORMAT()`](https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_date-format) | ## Blocklist specific expressions @@ -145,23 +145,43 @@ tidb> explain select * from t where a < 2 and a > 2; | IS FALSE | isfalse | [json_extract]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-extract + [json_short_extract]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#operator_json-column-path + [json_short_extract_unquote]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#operator_json-inline-path + [json_unquote]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-unquote + [json_type]: https://dev.mysql.com/doc/refman/5.7/en/json-attribute-functions.html#function_json-type + [json_set]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-set + [json_insert]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-insert + [json_replace]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-replace + [json_remove]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-remove + [json_merge]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-merge + [json_merge_preserve]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-merge-preserve + [json_object]: https://dev.mysql.com/doc/refman/5.7/en/json-creation-functions.html#function_json-object + [json_array]: https://dev.mysql.com/doc/refman/5.7/en/json-creation-functions.html#function_json-array + [json_keys]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-keys + [json_length]: https://dev.mysql.com/doc/refman/5.7/en/json-attribute-functions.html#function_json-length + [json_valid]: https://dev.mysql.com/doc/refman/5.7/en/json-attribute-functions.html#function_json-valid + [json_quote]: https://dev.mysql.com/doc/refman/5.7/en/json-creation-functions.html#function_json-quote + [json_contains]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-contains + [json_contains_path]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-contains-path + [json_arrayagg]: https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_json-arrayagg + [json_depth]: https://dev.mysql.com/doc/refman/5.7/en/json-attribute-functions.html#function_json-depth diff --git a/functions-and-operators/information-functions.md b/functions-and-operators/information-functions.md index 98877381ab3a..575518326a0e 100644 --- a/functions-and-operators/information-functions.md +++ b/functions-and-operators/information-functions.md @@ -24,7 +24,6 @@ TiDB supports most of the [information functions](https://dev.mysql.com/doc/refm | [`SYSTEM_USER()`](https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_system-user) | Synonym for `USER()` | | [`USER()`](https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_user) | Return the user name and host name provided by the client | | [`VERSION()`](https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_version) | Return a string that indicates the MySQL server version | -| `TIDB_VERSION()` | Return a string that indicates the TiDB server version | ## Unsupported functions diff --git a/functions-and-operators/json-functions.md b/functions-and-operators/json-functions.md index f34d88dcd629..6b0947ef00c4 100644 --- a/functions-and-operators/json-functions.md +++ b/functions-and-operators/json-functions.md @@ -84,27 +84,51 @@ The following JSON functions are unsupported in TiDB. You can track the progress * [JSON Data Type](/data-type-json.md) [json_extract]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-extract + [json_short_extract]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#operator_json-column-path + [json_short_extract_unquote]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#operator_json-inline-path + [json_unquote]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-unquote + [json_type]: https://dev.mysql.com/doc/refman/5.7/en/json-attribute-functions.html#function_json-type + [json_set]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-set + [json_insert]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-insert + [json_replace]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-replace + [json_remove]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-remove + [json_merge]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-merge + [json_merge_preserve]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-merge-preserve + [json_object]: https://dev.mysql.com/doc/refman/5.7/en/json-creation-functions.html#function_json-object + [json_array]: https://dev.mysql.com/doc/refman/5.7/en/json-creation-functions.html#function_json-array + [json_keys]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-keys + [json_length]: https://dev.mysql.com/doc/refman/5.7/en/json-attribute-functions.html#function_json-length + [json_valid]: https://dev.mysql.com/doc/refman/5.7/en/json-attribute-functions.html#function_json-valid + [json_quote]: https://dev.mysql.com/doc/refman/5.7/en/json-creation-functions.html#function_json-quote + [json_contains]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-contains + [json_contains_path]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-contains-path -[json_arrayagg]: https://dev.mysql.com/doc/refman/5.7/en/group-by-functions.html#function_json-arrayagg + +[json_arrayagg]: https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_json-arrayagg + [json_depth]: https://dev.mysql.com/doc/refman/5.7/en/json-attribute-functions.html#function_json-depth + [json_search]: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-search + [json_append]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-append + [json_array_append]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-array-append + [json_array_insert]: https://dev.mysql.com/doc/refman/5.7/en/json-modification-functions.html#function_json-array-insert diff --git a/functions-and-operators/tidb-functions.md b/functions-and-operators/tidb-functions.md new file mode 100644 index 000000000000..07a20be367a3 --- /dev/null +++ b/functions-and-operators/tidb-functions.md @@ -0,0 +1,259 @@ +--- +title: TiDB Specific Functions +summary: Learn about the usage of TiDB specific functions. +--- + +# TiDB Specific Functions + +This document introduces the functions that are specific to TiDB. + +## TIDB_BOUNDED_STALENESS + +The `TIDB_BOUNDED_STALENESS` function is an internal function of TiDB. + +## TIDB_DECODE_KEY + +The `TIDB_DECODE_KEY` function can be used to decode a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and in logging outputs. + +### Synopsis + +```ebnf+diagram +TableStmt ::= + "TIDB_DECODE_KEY(" STR ")" +``` + +### Examples + +In the following example, the table `t1` has a hidden `rowid` that is generated by TiDB. The `TIDB_DECODE_KEY` is used in the statement. From the result, you can see that the hidden `rowid` is decoded and output, which is a typical result for the non-clustered primary key. + +{{< copyable "sql" >}} + +```sql +SELECT START_KEY, TIDB_DECODE_KEY(START_KEY) FROM information_schema.tikv_region_status WHERE table_name='t1' AND REGION_ID=2\G +``` + +```sql +*************************** 1. row *************************** + START_KEY: 7480000000000000FF3B5F728000000000FF1DE3F10000000000FA +TIDB_DECODE_KEY(START_KEY): {"_tidb_rowid":1958897,"table_id":"59"} +1 row in set (0.00 sec) +``` + +In the following example, the table `t2` has a compound clustered primary key. From the JSON output, you can see a `handle` that contains the name and value for both of the columns that are part of the primary key. + +{{< copyable "sql" >}} + +```sql +show create table t2\G +``` + +```sql +*************************** 1. row *************************** + Table: t2 +Create Table: CREATE TABLE `t2` ( + `id` binary(36) NOT NULL, + `a` tinyint(3) unsigned NOT NULL, + `v` varchar(512) DEFAULT NULL, + PRIMARY KEY (`a`,`id`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +1 row in set (0.001 sec) +``` + +{{< copyable "sql" >}} + +```sql +select * from information_schema.tikv_region_status where table_name='t2' limit 1\G +``` + +```sql +*************************** 1. row *************************** + REGION_ID: 48 + START_KEY: 7480000000000000FF3E5F720400000000FF0000000601633430FF3338646232FF2D64FF3531632D3131FF65FF622D386337352DFFFF3830653635303138FFFF61396265000000FF00FB000000000000F9 + END_KEY: + TABLE_ID: 62 + DB_NAME: test + TABLE_NAME: t2 + IS_INDEX: 0 + INDEX_ID: NULL + INDEX_NAME: NULL + EPOCH_CONF_VER: 1 + EPOCH_VERSION: 38 + WRITTEN_BYTES: 0 + READ_BYTES: 0 + APPROXIMATE_SIZE: 136 + APPROXIMATE_KEYS: 479905 + REPLICATIONSTATUS_STATE: NULL +REPLICATIONSTATUS_STATEID: NULL +1 row in set (0.005 sec) +``` + +{{< copyable "sql" >}} + +```sql +select tidb_decode_key('7480000000000000FF3E5F720400000000FF0000000601633430FF3338646232FF2D64FF3531632D3131FF65FF622D386337352DFFFF3830653635303138FFFF61396265000000FF00FB000000000000F9'); +``` + +```sql ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| tidb_decode_key('7480000000000000FF3E5F720400000000FF0000000601633430FF3338646232FF2D64FF3531632D3131FF65FF622D386337352DFFFF3830653635303138FFFF61396265000000FF00FB000000000000F9') | ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| {"handle":{"a":"6","id":"c4038db2-d51c-11eb-8c75-80e65018a9be"},"table_id":62} | ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.001 sec) +``` + +### MySQL compatibility + +The `TIDB_DECODE_KEY` function is TiDB-specific and not compatible with MySQL. + +## TIDB_DECODE_PLAN + +The `TIDB_DECODE_PLAN` function can be used to decode a TiDB execution plan. These plans can be found in the slow query log. + +### Synopsis + +```ebnf+diagram +TableStmt ::= + "TIDB_DECODE_PLAN(" STR ")" +``` + +### Example + +{{< copyable "sql" >}} + +```sql +SELECT tidb_decode_plan('8QIYMAkzMV83CQEH8E85LjA0CWRhdGE6U2VsZWN0aW9uXzYJOTYwCXRpbWU6NzEzLjHCtXMsIGxvb3BzOjIsIGNvcF90YXNrOiB7bnVtOiAxLCBtYXg6IDU2OC41wgErRHByb2Nfa2V5czogMCwgcnBjXxEpAQwFWBAgNTQ5LglZyGNvcHJfY2FjaGVfaGl0X3JhdGlvOiAwLjAwfQkzLjk5IEtCCU4vQQoxCTFfNgkxXzAJMwm2SGx0KHRlc3QudC5hLCAxMDAwMCkNuQRrdgmiAHsFbBQzMTMuOMIBmQnEDDk2MH0BUgEEGAoyCTQzXzUFVwX1oGFibGU6dCwga2VlcCBvcmRlcjpmYWxzZSwgc3RhdHM6cHNldWRvCTk2ISE2aAAIMTUzXmYA')\G +``` + +```sql +*************************** 1. row *************************** + tidb_decode_plan('8QIYMAkzMV83CQEH8E85LjA0CWRhdGE6U2VsZWN0aW9uXzYJOTYwCXRpbWU6NzEzLjHCtXMsIGxvb3BzOjIsIGNvcF90YXNrOiB7bnVtOiAxLCBtYXg6IDU2OC41wgErRHByb2Nfa2V5czogMCwgcnBjXxEpAQwFWBAgNTQ5LglZyGNvcHJfY2FjaGVfaGl0X3JhdGlvOiAwLjAwfQkzLjk5IEtCCU4vQQoxCTFfNgkxXz: id task estRows operator info actRows execution info memory disk + TableReader_7 root 319.04 data:Selection_6 960 time:713.1µs, loops:2, cop_task: {num: 1, max: 568.5µs, proc_keys: 0, rpc_num: 1, rpc_time: 549.1µs, copr_cache_hit_ratio: 0.00} 3.99 KB N/A + └─Selection_6 cop[tikv] 319.04 lt(test.t.a, 10000) 960 tikv_task:{time:313.8µs, loops:960} N/A N/A + └─TableFullScan_5 cop[tikv] 960 table:t, keep order:false, stats:pseudo 960 tikv_task:{time:153µs, loops:960} N/A N/A +``` + +### MySQL compatibility + +The `TIDB_DECODE_PLAN` function is TiDB-specific and not compatible with MySQL. + +## TIDB_IS_DDL_OWNER + +The `TIDB_IS_DDL_OWNER` function can be used to check whether or not the TiDB instance you are connected to is the one that is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. + +### Synopsis + +```ebnf+diagram +TableStmt ::= + "TIDB_IS_DDL_OWNER())" +``` + +### Example + +{{< copyable "sql" >}} + +```sql +SELECT tidb_is_ddl_owner(); +``` + +```sql ++---------------------+ +| tidb_is_ddl_owner() | ++---------------------+ +| 1 | ++---------------------+ +1 row in set (0.00 sec) +``` + +### MySQL compatibility + +The `TIDB_IS_DDL_OWNER` function is TiDB-specific and not compatible with MySQL. + +### See also + +- [ADMIN SHOW DDL](/sql-statements/sql-statement-admin-show-ddl.md) +- [ADMIN CANCEL DDL](/sql-statements/sql-statement-admin-cancel-ddl.md) + +## TIDB_PARSE_TSO + +The `TIDB_PARSE_TSO` function can be used to extract the physical timestamp from a TiDB TSO timestamp. + +TSO stands for Time Stamp Oracle and is a monotonically increasing timestamp given out by PD (Placement Driver) for every transaction. + +A TSO is a number that consists of two parts: + +- A physical timestamp +- A logical counter + +### Synopsis + +```ebnf+diagram +TableStmt ::= + "TIDB_PARSE_TSO(" NUM ")" +``` + +### Example + +{{< copyable "sql" >}} + +```sql +BEGIN; +SELECT TIDB_PARSE_TSO(@@tidb_current_ts); +ROLLBACK; +``` + +```sql ++-----------------------------------+ +| TIDB_PARSE_TSO(@@tidb_current_ts) | ++-----------------------------------+ +| 2021-05-26 11:33:37.776000 | ++-----------------------------------+ +1 row in set (0.0012 sec) +``` + +Here `TIDB_PARSE_TSO` is used to extract the physical timestamp from the timestamp number that is available in the `tidb_current_ts` session variable. Because timestamps are given out per transaction, this function is running in a transaction. + +### MySQL compatibility + +The `TIDB_PARSE_TSO` function is TiDB-specific and not compatible with MySQL. + +### See also + +- [`tidb_current_ts`](/system-variables.md#tidb_current_ts) + +## TIDB_VERSION + +The `TIDB_VERSION` function can be used to get the version and build details of the TiDB server that you are connected to. You can use this function when reporting issues on GitHub. + +### Synopsis + +```ebnf+diagram +TableStmt ::= + "TIDB_VERSION()" +``` + +### Example + +{{< copyable "sql" >}} + +```sql +SELECT TIDB_VERSION()\G +``` + +```sql +*************************** 1. row *************************** +TIDB_VERSION(): Release Version: v5.1.0-alpha-13-gd5e0ed0aa-dirty +Edition: Community +Git Commit Hash: d5e0ed0aaed72d2f2dfe24e9deec31cb6cb5fdf0 +Git Branch: master +UTC Build Time: 2021-05-24 14:39:20 +GoVersion: go1.13 +Race Enabled: false +TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306 +Check Table Before Drop: false +1 row in set (0.00 sec) +``` + +### MySQL compatibility + +The `TIDB_VERSION` function is TiDB-specific and not compatible with MySQL. If MySQL compatibility is required, you can also use `VERSION` to get version information, but the result does not contain build details. diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index a4f2f7207f2b..240a48a8c0e3 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -1,134 +1,18 @@ --- -title: GC Configuration +title: Garbage Collection Configuration summary: Learn about GC configuration parameters. aliases: ['/docs/dev/garbage-collection-configuration/','/docs/dev/reference/garbage-collection/configuration/'] --- -# GC Configuration +# Garbage Collection Configuration -The GC (Garbage Collection) configuration and operational status are recorded in the `mysql.tidb` system table. You can use SQL statements to query or modify them: +Garbage collection is configured via the following system variables: -{{< copyable "sql" >}} - -```sql -select VARIABLE_NAME, VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME like "tikv_gc%"; -``` - -```sql -+--------------------------+----------------------------------------------------------------------------------------------------+ -| VARIABLE_NAME | VARIABLE_VALUE | -+--------------------------+----------------------------------------------------------------------------------------------------+ -| tikv_gc_leader_uuid | 5afd54a0ea40005 | -| tikv_gc_leader_desc | host:tidb-cluster-tidb-0, pid:215, start at 2019-07-15 11:09:14.029668932 +0000 UTC m=+0.463731223 | -| tikv_gc_leader_lease | 20190715-12:12:14 +0000 | -| tikv_gc_enable | true | -| tikv_gc_run_interval | 10m0s | -| tikv_gc_life_time | 10m0s | -| tikv_gc_last_run_time | 20190715-12:09:14 +0000 | -| tikv_gc_safe_point | 20190715-11:59:14 +0000 | -| tikv_gc_auto_concurrency | true | -| tikv_gc_mode | distributed | -+--------------------------+----------------------------------------------------------------------------------------------------+ -13 rows in set (0.00 sec) -``` - -For example, the following statement makes GC keep history data for the most recent 24 hours: - -```sql -update mysql.tidb set VARIABLE_VALUE="24h" where VARIABLE_NAME="tikv_gc_life_time"; -``` - -> **Note:** -> -> In addition to the following GC configuration parameters, the `mysql.tidb` system table also contains records that store the status of the storage components in a TiDB cluster, among which GC related ones are included, as listed below: -> -> - `tikv_gc_leader_uuid`, `tikv_gc_leader_desc` and `tikv_gc_leader_lease`: Records the information of the GC leader -> - `tikv_gc_last_run_time`: The duration of the latest GC (updated at the beginning of each round of GC) -> - `tikv_gc_safe_point`: The current safe point (updated at the beginning of each round of GC) - -## `tikv_gc_enable` - -- Enables or disables GC -- Default: `true` - -## `tikv_gc_run_interval` - -- Specifies the GC interval, in the format of Go Duration, for example, `"1h30m"`, and `"15m"` -- Default: `"10m0s"` - -## `tikv_gc_life_time` - -- The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point. -- Default: `"10m0s"` - - > **Note:** - > - > - In scenarios of frequent updates, a large value (days or even months) for `tikv_gc_life_time` may cause potential issues, such as: - > - Larger storage use - > - A large amount of history data may affect performance to a certain degree, especially for range queries such as `select count(*) from t` - > - If there is any transaction that has been running longer than `tikv_gc_life_time`, during GC, the data since `start_ts` is retained for this transaction to continue execution. For example, if `tikv_gc_life_time` is configured to 10 minutes, among all transactions being executed, the transaction that starts earliest has been running for 15 minutes, GC will retain data of the recent 15 minutes. - -## `tikv_gc_mode` - -- Specifies the GC mode. Possible values are: - - - `"distributed"` (default): Distributed GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader on the TiDB side uploads the safe point to PD. Each TiKV node obtains the safe point respectively and performs GC on all leader Regions on the current node. This mode is supported from TiDB 3.0. - - - `"central"`: Central GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader sends GC requests to all Regions. This mode is adopted by TiDB 2.1 or earlier versions. Starting from TiDB 5.0, this mode is not supported. Clusters set to this mode automatically switch to the `distributed` mode. - -## `tikv_gc_auto_concurrency` - -- Controls whether to let TiDB automatically specify the GC concurrency, or the maximum number of GC threads allowed concurrently. - - When `tikv_gc_mode` is set to `"distributed"`, GC concurrency works in the [Resolve Locks](/garbage-collection-overview.md#resolve-locks) step. When `tikv_gc_mode` is set to `"central"`, it is applied to both the Resolve Locks and [Do GC](/garbage-collection-overview.md#do-gc) steps. - - - `true`(default): Automatically use the number of TiKV nodes in the cluster as the GC concurrency - - `false`: Use the value of [`tikv_gc_concurrency`](#tikv_gc_concurrency) as the GC concurrency - -## `tikv_gc_concurrency` - -- Specifies the GC concurrency manually. This parameter works only when you set [`tikv_gc_auto_concurrency`](#tikv_gc_auto_concurrency) to `false`. -- Default: 2 - -## `tikv_gc_scan_lock_mode` (**experimental feature**) - -> **Warning:** -> -> Green GC is still an experimental feature. It is recommended **NOT** to use it in the production environment. - -This parameter specifies the way of scanning locks in the Resolve Locks step of GC, that is, whether to enable Green GC (experimental feature) or not. In the Resolve Locks step of GC, TiKV needs to scan all locks in the cluster. With Green GC disabled, TiDB scans locks by Regions. Green GC provides the "physical scanning" feature, which means that each TiKV node can bypass the Raft layer to directly scan data. This feature can effectively mitigate the impact of GC wakening up all Regions when the [Hibernate Region](/tikv-configuration-file.md#raftstorehibernate-regions-experimental) feature is enabled, thus improving the execution speed in the Resolve Locks step. - -- `"legacy"` (default): Uses the old way of scanning, that is, disable Green GC. -- `"physical"`: Uses the physical scanning method, that is, enable Green GC. - -> **Note:** -> -> The configuration of Green GC is hidden. Execute the following statement when you enable Green GC for the first time: -> -> {{< copyable "sql" >}} -> -> ```sql -> insert into mysql.tidb values ('tikv_gc_scan_lock_mode', 'legacy', ''); -> ``` - -## Notes on GC process changes - -Since TiDB 3.0, some configuration options have changed with support for the distributed GC mode and concurrent Resolve Locks processing. The changes are shown in the following table: - -| Version/Configuration | Resolve Locks | Do GC | -|-------------------|---------------|----------------| -| 2.x | Serial | Concurrent | -| 3.0
`tikv_gc_mode = centered`
`tikv_gc_auto_concurrency = false` | Concurrent | Concurrent | -| 3.0
`tikv_gc_mode = centered`
`tikv_gc_auto_concurrency = true` | Auto-concurrent | Auto-concurrent | -| 3.0
`tikv_gc_mode = distributed`
`tikv_gc_auto_concurrency = false` | Concurrent | Distributed | -| 3.0
`tikv_gc_mode = distributed`
`tikv_gc_auto_concurrency = true`
(default) | Auto-concurrent | Distributed | - -- Serial: requests are sent from TiDB Region by Region. -- Concurrent: requests are sent to each Region concurrently based on the number of threads specified in the `tikv_gc_concurrency`. -- Auto-concurrent: requests are sent to each Region concurrently with the number of TiKV nodes as concurrency value. -- Distributed: no need for TiDB to send requests to TiKV to trigger GC because each TiKV handles GC on its own. - -In addition, if Green GC (experimental feature) is enabled, that is, setting the value of [`tikv_gc_scan_lock_mode`](#tikv_gc_scan_lock_mode-experimental-feature) to `physical`, the processing of Resolve Lock is not affected by the concurrency configuration above. +* [`tidb_gc_enable`](/system-variables.md#tidb_gc_enable-new-in-v50) +* [`tidb_gc_run_interval`](/system-variables.md#tidb_gc_run_interval-new-in-v50) +* [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) +* [`tidb_gc_concurrency`](/system-variables.md#tidb_gc_concurrency-new-in-v50) +* [`tidb_gc_scan_lock_mode`](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) ## GC I/O limit @@ -144,9 +28,17 @@ You can dynamically modify this configuration using tikv-ctl: tikv-ctl --host=ip:port modify-tikv-config -m server -n gc.max_write_bytes_per_sec -v 10MB ``` -## GC in Compaction Filter +## Changes in TiDB 5.0 + +In previous releases of TiDB, garbage collection was configured via the `mysql.tidb` system table. While changes to this table continue to be supported, it is recommended to use the system variables provided. This helps ensure that any changes to configuration can be validated, and prevent unexpected behavior ([#20655](https://github.com/pingcap/tidb/issues/20655)). + +The `CENTRAL` garbage collection mode is no longer supported. The `DISTRIBUTED` GC mode (which has been the default since TiDB 3.0) will automatically be used in its place. This mode is more efficient, since TiDB no longer needs to send requests to each TiKV region to trigger garbage collection. + +For information on changes in previous releases, refer to earlier versions of this document using the _TIDB version selector_ in the left hand menu. + +### GC in Compaction Filter -Since v5.0, TiDB introduces the mechanism of GC in Compaction Filter. Based on the distributed GC mode, the mechanism uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. This new GC machanism helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it avoids a large number of left tombstone marks which degrade the sequential scan performance. This mechanism is enabled by default and is also automatically enabled after a rolling upgrade. The following example shows how to enable this GC mechanism in the TiKV configuration file: +Based on the `DISTRIBUTED` GC mode, the mechanism of GC in Compaction Filter uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. This new GC mechanism helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it avoids a large number of left tombstone marks which degrade the sequential scan performance. The following example shows how to enable the mechanism in the TiKV configuration file: {{< copyable "" >}} diff --git a/garbage-collection-overview.md b/garbage-collection-overview.md index eb85a8923d31..b4ac57ac6363 100644 --- a/garbage-collection-overview.md +++ b/garbage-collection-overview.md @@ -18,7 +18,7 @@ GC runs periodically on TiDB. For each GC, TiDB firstly calculates a timestamp c 2. Delete Ranges. During this step, the obsolete data of the entire range generated from the `DROP TABLE`/`DROP INDEX` operation is quickly cleared. 3. Do GC. During this step, each TiKV node scans data on it and deletes unneeded old versions of each key. -In the default configuration, GC is triggered every 10 minutes. Each GC retains data of the recent 10 minutes, which means that the the GC life time is 10 minutes by default (safe point = the current time - GC life time). If one round of GC has been running for too long, before this round of GC is completed, the next round of GC will not start even if it is time to trigger the next GC. In addition, for long-duration transactions to run properly after exceeding the GC life time, the safe point does not exceed the start time (start_ts) of the ongoing transactions. +In the default configuration, GC is triggered every 10 minutes. Each GC retains data of the recent 10 minutes, which means that the GC life time is 10 minutes by default (safe point = the current time - GC life time). If one round of GC has been running for too long, before this round of GC is completed, the next round of GC will not start even if it is time to trigger the next GC. In addition, for long-duration transactions to run properly after exceeding the GC life time, the safe point does not exceed the start time (start_ts) of the ongoing transactions. ## Implementation details @@ -28,7 +28,14 @@ The TiDB transaction model is implemented based on [Google's Percolator](https:/ The Resolve Locks step clears the locks before the safe point. This means that if the primary key of a lock is committed, this lock needs to be committed; otherwise, it needs to be rolled back. If the primary key is still locked (not committed or rolled back), this transaction is seen as timing out and rolled back. -In the Resolve Lock step, the GC leader sends requests to all Regions to scan obsolete locks, checks the primary key statuses of scanned locks, and sends requests to commit or roll back the corresponding transaction. By default, this process is performed concurrently, and the concurrency number is the same as the number of TiKV nodes. +The Resolve Locks step is implemented in either of the following two ways, which can be configured using the system variable [`tidb_gc_scan_lock_mode`](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50): + +> **Warning:** +> +> Currently, `PHYSICAL` (Green GC) is an experimental feature. It is not recommended that you use it in the production environment. + +- `LEGACY` (default): The GC leader sends requests to all Regions to scan obsolete locks, checks the primary key statuses of scanned locks, and sends requests to commit or roll back the corresponding transaction. +- `PHYSICAL`: TiDB bypasses the Raft layer and directly scans data on each TiKV node. ### Delete Ranges @@ -42,4 +49,4 @@ In this step, TiDB only needs to send the safe point to PD, and then the whole r > **Note:** > -> In TiDB v2.1 or earlier versions, the Do GC step is implemented by TiDB sending requests to each Region. In v3.0 or later versions, you can modify the `tikv_gc_mode` to use the previous GC mechanism. For more details, refer to [GC Configuration](/garbage-collection-configuration.md#tikv_gc_mode). +> Starting with TiDB 5.0, the Do GC step will always use the `DISTRIBUTED` gc mode. This replaces the earlier `CENTRAL` gc mode, which was implemented by TiDB servers sending GC requests to each Region. diff --git a/generate-self-signed-certificates.md b/generate-self-signed-certificates.md index a9c79d8e5a38..f19265356c0e 100644 --- a/generate-self-signed-certificates.md +++ b/generate-self-signed-certificates.md @@ -4,7 +4,7 @@ summary: Use `openssl` to generate self-signed certificates. aliases: ['/docs/dev/generate-self-signed-certificates/','/docs/dev/how-to/secure/generate-self-signed-certificates/'] --- -# Generate Self-signed Certificates +# Generate Self-Signed Certificates This document provides an example of using `openssl` to generate a self-signed certificate. You can also generate certificates and keys that meet requirements according to your demands. diff --git a/geo-distributed-deployment-topology.md b/geo-distributed-deployment-topology.md index 2eb54c131208..68e4484968f3 100644 --- a/geo-distributed-deployment-topology.md +++ b/geo-distributed-deployment-topology.md @@ -4,23 +4,25 @@ summary: Learn the geo-distributed deployment topology of TiDB. aliases: ['/docs/dev/geo-distributed-deployment-topology/'] --- -# Geo-distributed Deployment topology +# Geo-Distributed Deployment Topology -This document takes the typical architecture of three data centers (DC) in two cities as an example, and introduces the geo-distributed deployment architecture and the key configuration. +This document takes the typical architecture of three data centers (DC) in two cities as an example, and introduces the geo-distributed deployment architecture and the key configuration. The cities used in this example are Shanghai (referred to as `sha`) and Beijing (referred to as `bja` and `bjb`). ## Topology information | Instance | Count | Physical machine configuration | BJ IP | SH IP | Configuration | | :-- | :-- | :-- | :-- | :-- | :-- | -| TiDB | 3 | 16 VCore 32GB * 1 | 10.0.1.1
10.0.1.2
10.0.1.3
10.0.1.4 | 10.0.1.5 | Default port
Global directory configuration | -| PD | 3 | 4 VCore 8GB * 1 | 10.0.1.6
10.0.1.7
10.0.1.8
10.0.1.9 | 10.0.1.10 | Default port
Global directory configuration | -| TiKV | 3 | 16 VCore 32GB 2TB (nvme ssd) * 1 | 10.0.1.11
10.0.1.12
10.0.1.13
10.0.1.14 | 10.0.1.15 | Default port
Global directory configuration | +| TiDB | 5 | 16 VCore 32GB * 1 | 10.0.1.1
10.0.1.2
10.0.1.3
10.0.1.4 | 10.0.1.5 | Default port
Global directory configuration | +| PD | 5 | 4 VCore 8GB * 1 | 10.0.1.6
10.0.1.7
10.0.1.8
10.0.1.9 | 10.0.1.10 | Default port
Global directory configuration | +| TiKV | 5 | 16 VCore 32GB 2TB (nvme ssd) * 1 | 10.0.1.11
10.0.1.12
10.0.1.13
10.0.1.14 | 10.0.1.15 | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.16 | | Default port
Global directory configuration | ### Topology templates - [The geo-distributed topology template](https://github.com/pingcap/docs/blob/master/config-templates/geo-redundancy-deployment.yaml) +For detailed descriptions of the configuration items in the above TiDB cluster topology file, see [Topology Configuration File for Deploying TiDB Using TiUP](/tiup/tiup-cluster-topology-reference.md). + ### Key parameters This section describes the key parameter configuration of the TiDB geo-distributed deployment. @@ -37,7 +39,7 @@ This section describes the key parameter configuration of the TiDB geo-distribut - The label configuration: - Since TiKV is deployed across different data centers, if the physical machines go down, the Region Group might lose three of the default five replicas, which causes the cluster unavailability. To address this issue, you can configure the labels to enable the smart scheduling of PD, which ensures that the Region Group does not allow three replicas to be located in TiKV instances on the same machine in the same cabinet of the same data center. + Since TiKV is deployed across different data centers, if the physical machines go down, the Raft Group might lose three of the default five replicas, which causes the cluster unavailability. To address this issue, you can configure the labels to enable the smart scheduling of PD, which ensures that the Raft Group does not allow three replicas to be located in TiKV instances on the same machine in the same cabinet of the same data center. - The TiKV configuration: @@ -86,3 +88,5 @@ This section describes the key parameter configuration of the TiDB geo-distribut > > - You do not need to manually create the `tidb` user in the configuration file. The TiUP cluster component automatically creates the `tidb` user on the target machines. You can customize the user, or keep the user consistent with the control machine. > - If you configure the deployment directory as a relative path, the cluster will be deployed in the home directory of the user. + +[Schedule Replicas by Topology Labels](/schedule-replicas-by-topology-labels.md) further explains the use of labels and the number of Raft Group replicas. diff --git a/get-started-with-tidb-lightning.md b/get-started-with-tidb-lightning.md index 7ffc12b2901e..a87108f10863 100644 --- a/get-started-with-tidb-lightning.md +++ b/get-started-with-tidb-lightning.md @@ -44,17 +44,17 @@ After executing this command, the full backup data is exported to the `/data/my_ ### Step 1: Deploy TiDB cluster -Before the data import, you need to deploy a TiDB cluster (later than v2.0.9). In this tutorial, TiDB v4.0.3 is used. For the deployment method, refer to [TiDB Introduction](/overview.md). +Before the data import, you need to deploy a TiDB cluster. In this tutorial, TiDB v5.0.0 is used as an example. For the deployment method, refer to [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). ### Step 2: Download TiDB Lightning installation package Download the TiDB Lightning installation package from the following link: -- **v4.0.3**: [tidb-toolkit-v4.0.3-linux-amd64.tar.gz](https://download.pingcap.org/tidb-toolkit-v4.0.3-linux-amd64.tar.gz) +- **v5.0.0**: [tidb-toolkit-v5.0.0-linux-amd64.tar.gz](https://download.pingcap.org/tidb-toolkit-v5.0.0-linux-amd64.tar.gz) > **Note:** > -> Choose the same version of TiDB Lightning as that of the TiDB cluster. +> TiDB Lightning is compatible with TiDB clusters of earlier versions. It is recommended that you download the latest stable version of the TiDB Lightning installation package. ### Step 3: Start `tidb-lightning` @@ -73,12 +73,15 @@ Download the TiDB Lightning installation package from the following link: backend = "local" # Sets the directory for temporarily storing the sorted key-value pairs. # The target directory must be empty. - "sorted-kv-dir" = "/mnt/ssd/sorted-kv-dir" + sorted-kv-dir = "/mnt/ssd/sorted-kv-dir" [mydumper] # Local source data directory data-source-dir = "/data/my_datasource/" + # Configures the wildcard rule. By default, all tables in the mysql, sys, INFORMATION_SCHEMA, PERFORMANCE_SCHEMA, METRICS_SCHEMA, and INSPECTION_SCHEMA system databases are filtered. + # If this item is not configured, the "cannot find schema" error occurs when system tables are imported. + filter = ['*.*', '!mysql.*', '!sys.*', '!INFORMATION_SCHEMA.*', '!PERFORMANCE_SCHEMA.*', '!METRICS_SCHEMA.*', '!INSPECTION_SCHEMA.*'] [tidb] # Information of the target cluster host = "172.16.31.2" @@ -104,7 +107,7 @@ Download the TiDB Lightning installation package from the following link: After the import is completed, TiDB Lightning exits automatically. If the import is successful, you can find `tidb lightning exit` in the last line of the log file. -If any error occurs, refer to [TiDB Lightning Troubleshooting](/troubleshoot-tidb-lightning.md). +If any error occurs, refer to [TiDB Lightning FAQs](/tidb-lightning/tidb-lightning-faq.md). ## Summary diff --git a/get-started-with-tispark.md b/get-started-with-tispark.md index 2ed6276c87b3..363b235b2381 100644 --- a/get-started-with-tispark.md +++ b/get-started-with-tispark.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/get-started-with-tispark/','/docs/dev/how-to/get-started/ti # TiSpark Quick Start Guide -To make it easy to [try TiSpark](/tispark-overview.md), the TiDB cluster installed using TiDB Ansible integrates Spark, TiSpark jar package and TiSpark sample data by default. +To make it easy to [try TiSpark](/tispark-overview.md), the TiDB cluster installed using TiUP integrates Spark and TiSpark jar package by default. ## Deployment information diff --git a/glossary.md b/glossary.md index 23ede910ab98..c98ba483a6e0 100644 --- a/glossary.md +++ b/glossary.md @@ -28,6 +28,10 @@ Leader/Follower/Learner each corresponds to a role in a Raft group of [peers](#r ## O +### Old value + +The "original value" in the incremental change log output by TiCDC. You can specify whether the incremental change log output by TiCDC contains the "original value". + ### Operator An operator is a collection of actions that applies to a Region for scheduling purposes. Operators perform scheduling tasks such as "migrate the leader of Region 2 to Store 5" and "migrate replicas of Region 2 to Store 1, 4, 5". diff --git a/grafana-overview-dashboard.md b/grafana-overview-dashboard.md index 4cd8bc9467ce..887bd68b22c6 100644 --- a/grafana-overview-dashboard.md +++ b/grafana-overview-dashboard.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/grafana-overview-dashboard/','/docs/dev/reference/key-monit # Key Metrics -If you use TiDB Ansible or TiUP to deploy the TiDB cluster, the monitoring system (Prometheus & Grafana) is deployed at the same time. For more information, see [TiDB Monitoring Framework Overview](/tidb-monitoring-framework.md). +If you use TiUP to deploy the TiDB cluster, the monitoring system (Prometheus & Grafana) is deployed at the same time. For more information, see [TiDB Monitoring Framework Overview](/tidb-monitoring-framework.md). The Grafana dashboard is divided into a series of sub dashboards which include Overview, PD, TiDB, TiKV, Node\_exporter, Disk Performance, and so on. A lot of metrics are there to help you diagnose. @@ -16,58 +16,58 @@ For routine operations, you can get an overview of the component (PD, TiDB, TiKV To understand the key metrics displayed on the Overview dashboard, check the following table: -Service | Panel Name | Description | Normal Range ----- | ---------------- | ---------------------------------- | -------------- -Services Port Status | Services Up | The online nodes number of each service. | -PD | PD role | The role of the current PD. | -PD | Storage capacity | The total storage capacity of the TiDB cluster. | -PD | Current storage size | The occupied storage capacity of the TiDB cluster, including the space occupied by TiKV replicas. | -PD | Normal stores | The number of nodes in the normal state. | -PD | Abnormal stores | The number of nodes in the abnormal state. | 0 | -PD | Number of Regions | The total number of Regions in the current cluster. Note that the number of Regions has nothing to do with the number of replicas. | -PD | 99% completed_cmds_duration_seconds | The 99th percentile duration to complete a pd-server request. | less than 5ms -PD | Handle\_requests\_duration\_seconds | The network duration of a PD request. | -PD | Region health | The state of each Region. | Generally, the number of pending peers is less than 100, and that of the missing peers cannot always be greater than `0`. | -PD | Hot write Region's leader distribution | The total number of leaders who are the write hotspots on each TiKV instance. | -PD | Hot read Region's leader distribution | The total number of leaders who are the read hotspots on each TiKV instance. | -PD | Region heartbeat report | The count of heartbeats reported to PD per instance. | -PD | 99% Region heartbeat latency | The heartbeat latency per TiKV instance (P99). | -TiDB | Statement OPS | The number of different types of SQL statements executed per second, which is counted according to `SELECT`, `INSERT`, `UPDATE`, and other types of statements. | -TiDB | Duration | The execution time.
1. The duration between the time that the client's network request is sent to TiDB and the time that the request is returned to the client after TiDB has executed the request. In general, client requests are sent in the form of SQL statements; however, this duration can include the execution time of commands such as `COM_PING`, `COM_SLEEP`, `COM_STMT_FETCH`, and `COM_SEND_LONG_DATA`.
2. Because TiDB supports Multi-Query, TiDB supports sending multiple SQL statements at one time, such as `select 1; select 1; select 1;`. In this case, the total execution time of this query includes the execution time of all SQL statements. | -TiDB | QPS By Instance | The QPS on each TiDB instance, which is classified according to the success or failure of command execution results. | -TiDB | Failed Query OPM | The statistics of error types (such as syntax errors and primary key conflicts) based on the errors occurred when executing SQL statements per second on each TiDB instance. The module in which the error occurs and the error code are included. | -TiDB | Connection Count | The connection number of each TiDB instance. | -TiDB | Memory Usage | The memory usage statistics of each TiDB instance, which is divided into the memory occupied by processes and the memory applied by Golang on the heap. | -TiDB | Transaction OPS | The number of transactions executed per second. | -TiDB | Transaction Duration | The execution time of a transaction | -TiDB | KV Cmd OPS | The number of executed KV commands. | -TiDB | KV Cmd Duration 99 | The execution time of the KV command. | -TiDB | PD TSO OPS | The number of TSO that TiDB obtains from PD per second. | -TiDB | PD TSO Wait Duration | The duration that TiDB waits for PD to return TSO. | -TiDB | TiClient Region Error OPS | The number of Region related errors returned by TiKV. | -TiDB | Lock Resolve OPS | The number of TiDB operations that resolve locks. When TiDB's read or write request encounters a lock, it tries to resolve the lock. | -TiDB | KV Backoff OPS | The number of errors returned by TiKV. | -TiKV | leader | The number of leaders on each TiKV node. | -TiKV | region | The number of Regions on each TiKV node. | -TiKV | CPU | The CPU usage ratio on each TiKV node. | -TiKV | Memory | The memory usage on each TiKV node. | -TiKV | store size | The size of storage space used by each TiKV instance. | -TiKV | cf size | The size of each column family (CF for short). | -TiKV | channel full | The number of "channel full" errors on each TiKV instance. | 0 | -TiKV | server report failures | The number of error messages reported by each TiKV instance. | 0 | -TiKV | scheduler pending commands | The number of pending commands on each TiKV instance. | -TiKV | coprocessor executor count | The number of coprocessor operations received by TiKV per second. Each type of coprocessor is counted separately. | -TiKV | coprocessor request duration | The time consumed to process read requests of coprocessor. | -TiKV | raft store CPU | The CPU usage ratio of the raftstore thread | The default number of threads is 2 (configured by `raftstore.store-pool-size`). A value of over 80% for a single thread indicates that the CPU usage ratio is very high.| -TiKV | Coprocessor CPU | The CPU usage ratio of the coprocessor thread. | -System Info | Vcores | The number of CPU cores. | -System Info | Memory | The total memory. | -System Info | CPU Usage | The CPU usage ratio, 100% at a maximum. | -System Info | Load [1m] | The overload within 1 minute. | -System Info | Memory Available | The size of the available memory. | -System Info | Network Traffic | The statistics of the network traffic. | -System Info | TCP Retrans | The frequency of the TOC retransmission. | -System Info | IO Util | The disk usage ratio, 100% at a maximum; generally you need to consider adding a new node when the usage ratio is up to 80% ~ 90%. | +| Service | Panel Name | Description | Normal Range | +| ---- | ---------------- | ---------------------------------- | -------------- | +| Services Port Status | Services Up | The online nodes number of each service. | +| PD | PD role | The role of the current PD. | +| PD | Storage capacity | The total storage capacity of the TiDB cluster. | +| PD | Current storage size | The occupied storage capacity of the TiDB cluster, including the space occupied by TiKV replicas. | +| PD | Normal stores | The number of nodes in the normal state. | +| PD | Abnormal stores | The number of nodes in the abnormal state. | 0 | +| PD | Number of Regions | The total number of Regions in the current cluster. Note that the number of Regions has nothing to do with the number of replicas. | +| PD | 99% completed_cmds_duration_seconds | The 99th percentile duration to complete a pd-server request. | less than 5ms +| PD | Handle\_requests\_duration\_seconds | The network duration of a PD request. | +| PD | Region health | The state of each Region. | Generally, the number of pending peers is less than 100, and that of the missing peers cannot always be greater than `0`. | +| PD | Hot write Region's leader distribution | The total number of leaders who are the write hotspots on each TiKV instance. | +| PD | Hot read Region's leader distribution | The total number of leaders who are the read hotspots on each TiKV instance. | +| PD | Region heartbeat report | The count of heartbeats reported to PD per instance. | +| PD | 99% Region heartbeat latency | The heartbeat latency per TiKV instance (P99). | +| TiDB | Statement OPS | The number of different types of SQL statements executed per second, which is counted according to `SELECT`, `INSERT`, `UPDATE`, and other types of statements. | +| TiDB | Duration | The execution time.
1. The duration between the time that the client's network request is sent to TiDB and the time that the request is returned to the client after TiDB has executed the request. In general, client requests are sent in the form of SQL statements; however, this duration can include the execution time of commands such as `COM_PING`, `COM_SLEEP`, `COM_STMT_FETCH`, and `COM_SEND_LONG_DATA`.
2. Because TiDB supports Multi-Query, TiDB supports sending multiple SQL statements at one time, such as `select 1; select 1; select 1;`. In this case, the total execution time of this query includes the execution time of all SQL statements. | +| TiDB | CPS By Instance | CPS By Instance: the command statistics on each TiDB instance, which is classified according to the success or failure of command execution results. | +| TiDB | Failed Query OPM | The statistics of error types (such as syntax errors and primary key conflicts) based on the errors occurred when executing SQL statements per second on each TiDB instance. The module in which the error occurs and the error code are included. | +| TiDB | Connection Count | The connection number of each TiDB instance. | +| TiDB | Memory Usage | The memory usage statistics of each TiDB instance, which is divided into the memory occupied by processes and the memory applied by Golang on the heap. | +| TiDB | Transaction OPS | The number of transactions executed per second. | +| TiDB | Transaction Duration | The execution time of a transaction | +| TiDB | KV Cmd OPS | The number of executed KV commands. | +| TiDB | KV Cmd Duration 99 | The execution time of the KV command. | +| TiDB | PD TSO OPS | The number of TSO that TiDB obtains from PD per second. | +| TiDB | PD TSO Wait Duration | The duration that TiDB waits for PD to return TSO. | +| TiDB | TiClient Region Error OPS | The number of Region related errors returned by TiKV. | +| TiDB | Lock Resolve OPS | The number of TiDB operations that resolve locks. When TiDB's read or write request encounters a lock, it tries to resolve the lock. | +| TiDB | KV Backoff OPS | The number of errors returned by TiKV. | +| TiKV | leader | The number of leaders on each TiKV node. | +| TiKV | region | The number of Regions on each TiKV node. | +| TiKV | CPU | The CPU usage ratio on each TiKV node. | +| TiKV | Memory | The memory usage on each TiKV node. | +| TiKV | store size | The size of storage space used by each TiKV instance. | +| TiKV | cf size | The size of each column family (CF for short). | +| TiKV | channel full | The number of "channel full" errors on each TiKV instance. | 0 | +| TiKV | server report failures | The number of error messages reported by each TiKV instance. | 0 | +| TiKV | scheduler pending commands | The number of pending commands on each TiKV instance. | +| TiKV | coprocessor executor count | The number of coprocessor operations received by TiKV per second. Each type of coprocessor is counted separately. | +| TiKV | coprocessor request duration | The time consumed to process read requests of coprocessor. | +| TiKV | raft store CPU | The CPU usage ratio of the raftstore thread | The default number of threads is 2 (configured by `raftstore.store-pool-size`). A value of over 80% for a single thread indicates that the CPU usage ratio is very high.| +| TiKV | Coprocessor CPU | The CPU usage ratio of the coprocessor thread. | +| System Info | Vcores | The number of CPU cores. | +| System Info | Memory | The total memory. | +| System Info | CPU Usage | The CPU usage ratio, 100% at a maximum. | +| System Info | Load [1m] | The overload within 1 minute. | +| System Info | Memory Available | The size of the available memory. | +| System Info | Network Traffic | The statistics of the network traffic. | +| System Info | TCP Retrans | The frequency of the TOC retransmission. | +| System Info | IO Util | The disk usage ratio, 100% at a maximum; generally you need to consider adding a new node when the usage ratio is up to 80% ~ 90%. | ## Interface of the Overview dashboard diff --git a/grafana-pd-dashboard.md b/grafana-pd-dashboard.md index aa36392d5f26..68c346e86532 100644 --- a/grafana-pd-dashboard.md +++ b/grafana-pd-dashboard.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/grafana-pd-dashboard/','/docs/dev/reference/key-monitoring- # Key Monitoring Metrics of PD -If you use TiUP or TiDB Ansible to deploy the TiDB cluster, the monitoring system (Prometheus & Grafana) is deployed at the same time. For more information, see [Overview of the Monitoring Framework](/tidb-monitoring-framework.md). +If you use TiUP to deploy the TiDB cluster, the monitoring system (Prometheus & Grafana) is deployed at the same time. For more information, see [Overview of the Monitoring Framework](/tidb-monitoring-framework.md). The Grafana dashboard is divided into a series of sub dashboards which include Overview, PD, TiDB, TiKV, Node\_exporter, Disk Performance, and so on. A lot of metrics are there to help you diagnose. @@ -40,7 +40,7 @@ The following is the description of PD Dashboard metrics items: ## Operator -- Schedule operator create: The number of newly created operators per type +- Schedule operator create: The number of newly created operators per type - Schedule operator check: The number of checked operator per type. It mainly checks whether the current step is finished; if yes, it returns the next step to be executed - Schedule operator finish: The number of finished operators per type - Schedule operator timeout: The number of timeout operators per type @@ -131,10 +131,6 @@ The following is the description of PD Dashboard metrics items: ![PD Dashboard - etcd metrics](/media/pd-dashboard-etcd-v2.png) -Cluster | Label distribution | The distribution status of the labels in the cluster | -Cluster | pd_cluster_metadata | The metadata of the PD cluster including cluster ID, the timestamp, and the generated ID. | -Cluster | Region health | The health status of Regions indicated via count of unusual Regions including pending peers, down peers, extra peers, offline peers, missing peers, learner peers and incorrect namespaces | The number of pending peers should be less than `100`. The missing peers should not be persistently greater than `0`. - ## TiDB - PD Server TSO handle time and Client recv time: The duration between PD receiving the TSO request and the PD client getting the TSO response diff --git a/grafana-tidb-dashboard.md b/grafana-tidb-dashboard.md index a7305847c34f..52e04b900ce6 100644 --- a/grafana-tidb-dashboard.md +++ b/grafana-tidb-dashboard.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/grafana-tidb-dashboard/','/docs/dev/reference/key-monitorin # TiDB Monitoring Metrics -If you use TiDB Ansible or TiUP to deploy the TiDB cluster, the monitoring system (Prometheus & Grafana) is deployed at the same time. For the monitoring architecture, see [TiDB Monitoring Framework Overview](/tidb-monitoring-framework.md). +If you use TiUP to deploy the TiDB cluster, the monitoring system (Prometheus & Grafana) is deployed at the same time. For the monitoring architecture, see [TiDB Monitoring Framework Overview](/tidb-monitoring-framework.md). The Grafana dashboard is divided into a series of sub dashboards which include Overview, PD, TiDB, TiKV, Node\_exporter, Disk Performance, and so on. The TiDB dashboard consists of the TiDB panel and the TiDB Summary panel. The differences between the two panels are different in the following aspects: @@ -23,11 +23,12 @@ To understand the key metrics displayed on the TiDB dashboard, check the followi - Duration: execution time - The duration between the time that the client's network request is sent to TiDB and the time that the request is returned to the client after TiDB has executed it. In general, client requests are sent in the form of SQL statements, but can also include the execution time of commands such as `COM_PING`, `COM_SLEEP`, `COM_STMT_FETCH`, and `COM_SEND_LONG_DATA` - Because TiDB supports Multi-Query, it supports sending multiple SQL statements at one time, such as `select 1; select 1; select 1;`. In this case, the total execution time of this query includes the execution time of all SQL statements - - QPS: the number of SQL statements executed per second on all TiDB instances. The execution results are classified into `OK` (successful) and `Error` (failed) - - Statement OPS: the number of different types of SQL statements executed per second, which is counted according to `SELECT`, `INSERT`, `UPDATE`, and other types of statements - - QPS By Instance: the QPS on each TiDB instance, which is classified according to the success or failure of command execution results + - Command Per Second: the number of commands processed by TiDB per second, which is classified according to the success or failure of command execution results + - QPS: the number of SQL statements executed per second on all TiDB instances, which is counted according to `SELECT`, `INSERT`, `UPDATE`, and other types of statements + - CPS By Instance: the command statistics on each TiDB instance, which is classified according to the success or failure of command execution results - Failed Query OPM: the statistics of error types (such as syntax errors and primary key conflicts) according to the errors occurred when executing SQL statements per second on each TiDB instance. It contains the module in which the error occurs and the error code - - Slow query: the statistics of the processing time of slow queries (the time cost of the entire slow query, the time cost of Coprocessor，and the waiting time for Coprocessor scheduling) + - Slow query: the statistics of the processing time of slow queries (the time cost of the entire slow query, the time cost of Coprocessor，and the waiting time for Coprocessor scheduling). Slow queries are classified into internal and general SQL statements + - Connection Idle Duration: the duration of idle connections - 999/99/95/80 Duration: the statistics of the execution time for different types of SQL statements (different percentiles) - Query Detail @@ -41,39 +42,40 @@ To understand the key metrics displayed on the TiDB dashboard, check the followi - CPU Usage: the statistics of CPU usage of each TiDB instance - Connection Count: the number of clients connected to each TiDB instance - Open FD Count: the statistics of opened file descriptors of each TiDB instance - - Goroutine Count: the number of Goroutines on each TiDB instance - - Go GC Duration: Golang GC time on each TiDB instance - - Go Threads: the number of threads on each TiDB instance - - Go GC Count: the number of times that Golang GC is executed on each TiDB instance - - Go GC CPU Usage: the statistics of CPU used by Golang GC for each TiDB instance + - Disconnection Count: the number of clients disconnected to each TiDB instance - Events OPM: the statistics of key events, such as "start", "close", "graceful-shutdown","kill", "hang", and so on - - Keep Alive OPM: the number of times that the metrics are refreshed every minute on each TiDB instance. It usually needs no attention. + - Goroutine Count: the number of Goroutines on each TiDB instance - Prepare Statement Count: the number of `Prepare` statements that are executed on each TiDB instance and the total count of them + - Keep Alive OPM: the number of times that the metrics are refreshed every minute on each TiDB instance. It usually needs no attention. + - Panic And Critical Error: the number of panics and critical errors occurred in TiDB - Time Jump Back OPS: the number of times that the operating system rewinds every second on each TiDB instance - - Write Binlog Error：the number of times that the binlog write failure occurs every second on each TiDB instance - - Handshake Error OPS: the number of times that a handshake error occurs every second on each TiDB instance - Get Token Duration: the time cost of getting Token on each connection + - Skip Binlog Count: the number of binlog write failures in TiDB + - Client Data Traffic: data traffic statistics of TiDB and the client - Transaction - Transaction OPS: the number of transactions executed per second - - Duration: the execution time of a transaction + - Duration: the execution duration of a transaction - Transaction Statement Num: the number of SQL statements in a transaction - Transaction Retry Num: the number of times that a transaction retries - - Session Retry Error OPS: the number of errors encountered during the transaction retry per second + - Session Retry Error OPS: the number of errors encountered during the transaction retry per second. This metric includes two error types: retry failure and exceeding the maximum number of retries + - Commit Token Wait Duration: the wait duration in the flow control queue during the transaction commit. If the wait duration is long, it means that the transaction to commit is too large and the flow is controlled. If the system still has resources available, you can speed up the commit process by increasing the `committer-concurrency` value in the TiDB configuration file - KV Transaction OPS: the number of transactions executed per second within each TiDB instance - A user transaction might trigger multiple transaction executions in TiDB, including reading internal metadata, atomic retries of the user transaction, and so on - TiDB's internally scheduled tasks also operate on the database through transactions, which are also included in this panel - KV Transaction Duration: the time spent on executing transactions within each TiDB - - Commit Token Wait Duration: the wait duration in the flow control queue during the transaction commit. If the wait duration is long, it means that the transaction to commit is too large and the flow is controlled. If the system still has resources available, you can speed up the commit process by increasing the `committer-concurrency` value in the TiDB configuration file - - Transaction Max Write KV Num: the maximum number of key-value pairs written by a single transaction - - Transaction Max Write Size Bytes: the maximum key-value pair size written by a single transaction - - Transaction Regions Num 90: 90% of the number of Regions written by a single transaction - - Send HeartBeat Duration: the interval at which transactions send heartbeats - - TTL Lifetime Reach Counter: the number of transactions that reach the upper limit of TTL. The default value of the TTL upper limit is 10 minutes. It means that 10 minutes have passed since the first lock of a pessimistic transaction or the first prewrite of an optimistic transaction. The default value of the upper limit of TTL is 10 minutes. The upper limit of TTL life can be changed by modifying `max-txn-TTL` in the TiDB configuration file + - Transaction Regions Num: the number of Regions operated in the transaction + - Transaction Write KV Num Rate and Sum: the rate at which KVs are written and the sum of these written KVs in the transaction + - Transaction Write KV Num: the number of KVs operated in the transaction - Statement Lock Keys: the number of locks for a single statement + - Send HeartBeat Duration: the duration for the transaction to send heartbeats + - Transaction Write Size Bytes Rate and sum: the rate at which bytes are written and the sum of these written bytes in the transaction + - Transaction Write Size Bytes: the size of the data written in the transaction - Acquire Pessimistic Locks Duration: the time consumed by adding locks - - Pessimistic Statement Retry OPS: the number of retry attempts for pessimistic statements. When the statement tries to add lock, it might encounter a write conflict. At this time, the statement will acquire a new snapshot and add lock again + - TTL Lifetime Reach Counter: the number of transactions that reach the upper limit of TTL. The default value of the TTL upper limit is 1 hour. It means that 1 hour has passed since the first lock of a pessimistic transaction or the first prewrite of an optimistic transaction. The default value of the upper limit of TTL is 1 hour. The upper limit of TTL life can be changed by modifying `max-txn-TTL` in the TiDB configuration file - Load Safepoint OPS: the number of times that `Safepoint` is loaded. `Safepoint` is to ensure that the data before `Safepoint` is not read when the transaction reads data, thus ensuring data safety. The data before `Safepoint` might be cleaned up by the GC + - Pessimistic Statement Retry OPS: the number of retry attempts for pessimistic statements. When the statement tries to add lock, it might encounter a write conflict. At this time, the statement will acquire a new snapshot and add lock again + - Async Commit Transaction Counter: the number of transactions that have async commit enabled, which has two statuses: successful and failed - Executor - Parse Duration: the statistics of the parsing time of SQL statements diff --git a/grafana-tikv-dashboard.md b/grafana-tikv-dashboard.md index 9c3eb3b7dea2..03bca8a699a3 100644 --- a/grafana-tikv-dashboard.md +++ b/grafana-tikv-dashboard.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/grafana-tikv-dashboard/','/docs/dev/reference/key-monitorin # Key Monitoring Metrics of TiKV -If you use TiUP or TiDB Ansible to deploy the TiDB cluster, the monitoring system (Prometheus/Grafana) is deployed at the same time. For more information, see [Overview of the Monitoring Framework](/tidb-monitoring-framework.md). +If you use TiUP to deploy the TiDB cluster, the monitoring system (Prometheus/Grafana) is deployed at the same time. For more information, see [Overview of the Monitoring Framework](/tidb-monitoring-framework.md). The Grafana dashboard is divided into a series of sub dashboards which include Overview, PD, TiDB, TiKV, Node\_exporter, and so on. A lot of metrics are there to help you diagnose. @@ -77,9 +77,8 @@ This document provides a detailed description of these key metrics on the **TiKV - Storage ReadPool CPU: The CPU utilization of the `storage read pool` thread - Coprocessor CPU: The CPU utilization of the `coprocessor` thread - RocksDB CPU: The CPU utilization of the RocksDB thread -- Split check CPU: The CPU utilization of the `split check` thread - GC worker CPU: The CPU utilization of the `GC worker` thread -- Snapshot worker CPU: The CPU utilization of the `snapshot worker` thread +- BackGround worker CPU: The CPU utilization of the `background worker` thread ## PD @@ -224,20 +223,17 @@ This document provides a detailed description of these key metrics on the **TiKV ## GC -- MVCC versions: The number of versions for each key -- MVCC delete versions: The number of versions deleted by GC for each key - GC tasks: The count of GC tasks processed by gc_worker - GC tasks Duration: The time consumed when executing GC tasks -- GC keys (write CF): The count of keys in write CF affected during GC -- TiDB GC worker actions: The count of TiDB GC worker actions - TiDB GC seconds: The GC duration -- GC speed: The number of keys deleted by GC per second -- TiKV AutoGC Working: The status of Auto GC +- TiDB GC worker actions: The count of TiDB GC worker actions - ResolveLocks Progress: The progress of the first phase of GC (Resolve Locks) - TiKV Auto GC Progress: The progress of the second phase of GC +- GC speed: The number of keys deleted by GC per second - TiKV Auto GC SafePoint: The value of TiKV GC safe point. The safe point is the current GC timestamp - GC lifetime: The lifetime of TiDB GC - GC interval: The interval of TiDB GC +- GC in Compaction Filter: The count of filtered versions in the compaction filter of write CF. ## Snapshot diff --git a/hardware-and-software-requirements.md b/hardware-and-software-requirements.md index cbaeccf4a213..094722a101c2 100644 --- a/hardware-and-software-requirements.md +++ b/hardware-and-software-requirements.md @@ -12,9 +12,9 @@ As an open source distributed NewSQL database with high performance, TiDB can be | Linux OS Platform | Version | | :-----------------------:| :----------: | -| Red Hat Enterprise Linux | 7.3 or later | -| CentOS | 7.3 or later | -| Oracle Enterprise Linux | 7.3 or later | +| Red Hat Enterprise Linux | 7.3 or later 7.x releases | +| CentOS | 7.3 or later 7.x releases | +| Oracle Enterprise Linux | 7.3 or later 7.x releases | | Ubuntu LTS | 16.04 or later | > **Note:** @@ -22,6 +22,11 @@ As an open source distributed NewSQL database with high performance, TiDB can be > - For Oracle Enterprise Linux, TiDB supports the Red Hat Compatible Kernel (RHCK) and does not support the Unbreakable Enterprise Kernel provided by Oracle Enterprise Linux. > - A large number of TiDB tests have been run on the CentOS 7.3 system, and in our community there are a lot of best practices in which TiDB is deployed on the Linux operating system. Therefore, it is recommended to deploy TiDB on CentOS 7.3 or later. > - The support for the Linux operating systems above includes the deployment and operation in physical servers as well as in major virtualized environments like VMware, KVM and XEN. +> - Red Hat Enterprise Linux 8.0, CentOS 8 Stream, and Oracle Enterprise Linux 8.0 are not supported yet as the testing of these platforms is in progress. +> - Support for CentOS 8 Linux is not planned because its upstream support ends on December 31, 2021. +> - Support for Ubuntu 16.04 will be removed in future versions of TiDB. Upgrading to Ubuntu 18.04 or later is strongly recommended. + +Other Linux OS versions such as Debian Linux and Fedora Linux might work but are not officially supported. ## Software recommendations @@ -30,7 +35,7 @@ As an open source distributed NewSQL database with high performance, TiDB can be | Software | Version | | :--- | :--- | | sshpass | 1.06 or later | -| TiUP | 0.6.2 or later | +| TiUP | 1.5.0 or later | > **Note:** > @@ -42,6 +47,7 @@ As an open source distributed NewSQL database with high performance, TiDB can be | :--- | :--- | | sshpass | 1.06 or later | | numa | 2.0.12 or later | +| tar | any | ## Server recommendations diff --git a/hybrid-deployment-topology.md b/hybrid-deployment-topology.md index 3a6769c35e9e..cc54fba32496 100644 --- a/hybrid-deployment-topology.md +++ b/hybrid-deployment-topology.md @@ -17,7 +17,7 @@ The deployment machine has multiple CPU processors with sufficient memory. To im | Instance | Count | Physical machine configuration | IP | Configuration | | :-- | :-- | :-- | :-- | :-- | | TiDB | 6 | 32 VCore 64GB | 10.0.1.1
10.0.1.2
10.0.1.3 | Configure NUMA to bind CPU cores | -| PD | 3 | 16 VCore 32 GB | 10.0.1.4
10.0.1.5
10.0.1.6 | Configure the `location_lables` parameter | +| PD | 3 | 16 VCore 32 GB | 10.0.1.4
10.0.1.5
10.0.1.6 | Configure the `location_labels` parameter | | TiKV | 6 | 32 VCore 64GB | 10.0.1.7
10.0.1.8
10.0.1.9 | 1. Separate the instance-level port and status_port;
2. Configure the global parameters `readpool`, `storage` and `raftstore`;
3. Configure labels of the instance-level host;
4. Configure NUMA to bind CPU cores | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.10 | Default configuration | @@ -26,6 +26,8 @@ The deployment machine has multiple CPU processors with sufficient memory. To im - [The simple template for the hybrid deployment](https://github.com/pingcap/docs-cn/blob/master/config-templates/simple-multi-instance.yaml) - [The complex template for the hybrid deployment](https://github.com/pingcap/docs/blob/master/config-templates/complex-multi-instance.yaml) +For detailed descriptions of the configuration items in the above TiDB cluster topology file, see [Topology Configuration File for Deploying TiDB Using TiUP](/tiup/tiup-cluster-topology-reference.md). + ### Key parameters This section introduces the key parameters when you deploy multiple instances on a single machine, which is mainly used in scenarios when multiple instances of TiDB and TiKV are deployed on a single machine. You need to fill in the results into the configuration template according to the calculation methods provided below. @@ -37,7 +39,7 @@ This section introduces the key parameters when you deploy multiple instances on - Enable `readpool.storage` and `readpool.coprocessor`: ```yaml - readpool.storage.use-unified-pool: false + readpool.storage.use-unified-pool: true readpool.coprocessor.use-unified-pool: true ``` @@ -69,7 +71,7 @@ This section introduces the key parameters when you deploy multiple instances on - The label scheduling configuration - Since multiple instances of TiKV are deployed on a single machine, if the physical machines go down, the Region Group might lose two of the default three replicas, which causes the cluster unavailability. To address this issue, you can use the label to enable the smart scheduling of PD, which ensures that the Region Group has more than two replicas in multiple TiKV instances on the same machine. + Since multiple instances of TiKV are deployed on a single machine, if the physical machines go down, the Raft Group might lose two of the default three replicas, which causes the cluster unavailability. To address this issue, you can use the label to enable the smart scheduling of PD, which ensures that the Raft Group has more than two replicas in multiple TiKV instances on the same machine. - The TiKV configuration diff --git a/identify-slow-queries.md b/identify-slow-queries.md index 694ec72ab8ee..e251894fb04a 100644 --- a/identify-slow-queries.md +++ b/identify-slow-queries.md @@ -121,7 +121,7 @@ TiKV Coprocessor Task fields: * [`tidb_slow_log_threshold`](/system-variables.md#tidb_slow_log_threshold): Sets the threshold for the slow log. The SQL statement whose execution time exceeds this threshold is recorded in the slow log. The default value is 300 (ms). * [`tidb_query_log_max_len`](/system-variables.md#tidb_query_log_max_len): Sets the maximum length of the SQL statement recorded in the slow log. The default value is 4096 (byte). -* [`tidb_log_desensitization`](/system-variables.md#tidb_log_desensitization): Determines whether to desensitize user data using `?` in the SQL statement recorded in the slow log. The default value is `0`, which means to disable the feature. +* [tidb_redact_log](/system-variables.md#tidb_redact_log): Determines whether to desensitize user data using `?` in the SQL statement recorded in the slow log. The default value is `0`, which means to disable the feature. * [`tidb_enable_collect_execution_info`](/system-variables.md#tidb_enable_collect_execution_info): Determines whether to record the physical execution information of each operator in the execution plan. The default value is `1`. This feature impacts the performance by approximately 3%. After enabling this feature, you can view the `Plan` information as follows: ```sql @@ -542,10 +542,7 @@ admin show slow top [internal | all] N admin show slow recent 10 ``` -`top N` shows the slowest N query records recently (within a few days). -If the `internal` option is provided, the returned results would be the inner SQL executed by the system; -If the `all` option is provided, the returned results would be the user's SQL combinated with inner SQL; -Otherwise, this command would only return the slow query records from the user's SQL. +`top N` shows the slowest N query records recently (within a few days). If the `internal` option is provided, the returned results would be the inner SQL executed by the system; If the `all` option is provided, the returned results would be the user's SQL combinated with inner SQL; Otherwise, this command would only return the slow query records from the user's SQL. {{< copyable "sql" >}} diff --git a/information-schema/client-errors-summary-by-host.md b/information-schema/client-errors-summary-by-host.md new file mode 100644 index 000000000000..147bff917299 --- /dev/null +++ b/information-schema/client-errors-summary-by-host.md @@ -0,0 +1,88 @@ +--- +title: CLIENT_ERRORS_SUMMARY_BY_HOST +summary: Learn about the `CLIENT_ERRORS_SUMMARY_BY_HOST` information_schema table. +--- + +# CLIENT_ERRORS_SUMMARY_BY_HOST + +The table `CLIENT_ERRORS_SUMMARY_BY_HOST` provides a summary of SQL errors and warnings that have been returned to clients that connect to a TiDB server. These include: + +* Malformed SQL statements. +* Division by zero errors. +* The attempt to insert out-of-range or duplicate key values. +* Permission errors. +* A table that does not exist. + +These errors are returned to the client via the MySQL server protocol, where applications are expected to take appropriate action. The `information_schema`.`CLIENT_ERRORS_SUMMARY_BY_HOST` table provides a useful method to inspect errors in the scenario where applications are not correctly handling (or logging) errors returned by the TiDB server. + +Because `CLIENT_ERRORS_SUMMARY_BY_HOST` summarizes the errors on a per-remote-host basis, it can be useful to diagnose scenarios where one application server is generating more errors than other servers. Possible scenarios include: + +* An outdated MySQL client library. +* An outdated application (possibly this server was missed when rolling out a new deployment). +* Incorrect usage of the "host" portion of user permissions. +* Unreliable network connectivity generating more timeouts or disconnected connections. + +The summarized counts can be reset using the statement `FLUSH CLIENT_ERRORS_SUMMARY`. The summary is local to each TiDB server and is only retained in memory. Summaries will be lost if the TiDB server restarts. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC CLIENT_ERRORS_SUMMARY_BY_HOST; +``` + +```sql ++---------------+---------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++---------------+---------------+------+------+---------+-------+ +| HOST | varchar(255) | NO | | NULL | | +| ERROR_NUMBER | bigint(64) | NO | | NULL | | +| ERROR_MESSAGE | varchar(1024) | NO | | NULL | | +| ERROR_COUNT | bigint(64) | NO | | NULL | | +| WARNING_COUNT | bigint(64) | NO | | NULL | | +| FIRST_SEEN | timestamp | YES | | NULL | | +| LAST_SEEN | timestamp | YES | | NULL | | ++---------------+---------------+------+------+---------+-------+ +7 rows in set (0.00 sec) +``` + +Field description: + +* `HOST`: The remote host of the client. +* `ERROR_NUMBER`: The MySQL-compatible error number that was returned. +* `ERROR_MESSAGE`: The error message which matches the error number (in prepared statement form). +* `ERROR_COUNT`: The number of times this error was returned to the client host. +* `WARNING_COUNT`: The number of times this warning was returned to the client host. +* `FIRST_SEEN`: The first time this error (or warning) was seen from the client host. +* `LAST_SEEN`: The most recent time this error (or warning) was seen from the client host. + +The following example shows a warning being generated when the client connects to a local TiDB server. The summary is reset after executing `FLUSH CLIENT_ERRORS_SUMMARY`: + +{{< copyable "sql" >}} + +```sql +SELECT 0/0; +SELECT * FROM CLIENT_ERRORS_SUMMARY_BY_HOST; +FLUSH CLIENT_ERRORS_SUMMARY; +SELECT * FROM CLIENT_ERRORS_SUMMARY_BY_HOST; +``` + +```sql ++-----+ +| 0/0 | ++-----+ +| NULL | ++-----+ +1 row in set, 1 warning (0.00 sec) + ++-----------+--------------+---------------+-------------+---------------+---------------------+---------------------+ +| HOST | ERROR_NUMBER | ERROR_MESSAGE | ERROR_COUNT | WARNING_COUNT | FIRST_SEEN | LAST_SEEN | ++-----------+--------------+---------------+-------------+---------------+---------------------+---------------------+ +| 127.0.0.1 | 1365 | Division by 0 | 0 | 1 | 2021-03-18 12:51:54 | 2021-03-18 12:51:54 | ++-----------+--------------+---------------+-------------+---------------+---------------------+---------------------+ +1 row in set (0.00 sec) + +Query OK, 0 rows affected (0.00 sec) + +Empty set (0.00 sec) +``` diff --git a/information-schema/client-errors-summary-by-user.md b/information-schema/client-errors-summary-by-user.md new file mode 100644 index 000000000000..8e095be9e06c --- /dev/null +++ b/information-schema/client-errors-summary-by-user.md @@ -0,0 +1,87 @@ +--- +title: CLIENT_ERRORS_SUMMARY_BY_USER +summary: Learn about the `CLIENT_ERRORS_SUMMARY_BY_USER` information_schema table. +--- + +# CLIENT_ERRORS_SUMMARY_BY_USER + +The table `CLIENT_ERRORS_SUMMARY_BY_USER` provides a summary of SQL errors and warnings that have been returned to clients that connect to a TiDB server. These include: + +* Malformed SQL statements. +* Division by zero errors. +* The attempt to insert out-of-range or duplicate key values. +* Permission errors. +* A table that does not exist. + +Client errors are returned to the client via the MySQL server protocol, where applications are expected to take appropriate action. The `information_schema`.`CLIENT_ERRORS_SUMMARY_BY_USER` table provides an useful method to inspect errors in the scenario where applications are not correctly handling (or logging) errors returned by the TiDB server. + +Because `CLIENT_ERRORS_SUMMARY_BY_USER` summarizes the errors on a per-user basis, it can be useful to diagnose scenarios where one user server is generating more errors than other servers. Possible scenarios include: + +* Permission errors. +* Missing tables, or relational objects. +* Incorrect SQL syntax, or incompatibilities between the application and the version of TiDB. + +The summarized counts can be reset with the statement `FLUSH CLIENT_ERRORS_SUMMARY`. The summary is local to each TiDB server and is only retained in memory. Summaries will be lost if the TiDB server restarts. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC CLIENT_ERRORS_SUMMARY_BY_USER; +``` + +```sql ++---------------+---------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++---------------+---------------+------+------+---------+-------+ +| USER | varchar(64) | NO | | NULL | | +| ERROR_NUMBER | bigint(64) | NO | | NULL | | +| ERROR_MESSAGE | varchar(1024) | NO | | NULL | | +| ERROR_COUNT | bigint(64) | NO | | NULL | | +| WARNING_COUNT | bigint(64) | NO | | NULL | | +| FIRST_SEEN | timestamp | YES | | NULL | | +| LAST_SEEN | timestamp | YES | | NULL | | ++---------------+---------------+------+------+---------+-------+ +7 rows in set (0.00 sec) +``` + +Field description: + +* `USER`: The authenticated user. +* `ERROR_NUMBER`: The MySQL-compatible error number that was returned. +* `ERROR_MESSAGE`: The error message which matches the error number (in prepared statement form). +* `ERROR_COUNT`: The number of times this error was returned to the user. +* `WARNING_COUNT`: The number of times this warning was returned to the user. +* `FIRST_SEEN`: The first time this error (or warning) was sent to the user. +* `LAST_SEEN`: The most recent time this error (or warning) was sent to the user. + +The following example shows a warning being generated when the client connects to a local TiDB server. The summary is reset after executing `FLUSH CLIENT_ERRORS_SUMMARY`: + +{{< copyable "sql" >}} + +```sql +SELECT 0/0; +SELECT * FROM CLIENT_ERRORS_SUMMARY_BY_USER; +FLUSH CLIENT_ERRORS_SUMMARY; +SELECT * FROM CLIENT_ERRORS_SUMMARY_BY_USER; +``` + +```sql ++-----+ +| 0/0 | ++-----+ +| NULL | ++-----+ +1 row in set, 1 warning (0.00 sec) + ++------+--------------+---------------+-------------+---------------+---------------------+---------------------+ +| USER | ERROR_NUMBER | ERROR_MESSAGE | ERROR_COUNT | WARNING_COUNT | FIRST_SEEN | LAST_SEEN | ++------+--------------+---------------+-------------+---------------+---------------------+---------------------+ +| root | 1365 | Division by 0 | 0 | 1 | 2021-03-18 13:05:36 | 2021-03-18 13:05:36 | ++------+--------------+---------------+-------------+---------------+---------------------+---------------------+ +1 row in set (0.00 sec) + +Query OK, 0 rows affected (0.00 sec) + +Empty set (0.00 sec) +``` diff --git a/information-schema/client-errors-summary-global.md b/information-schema/client-errors-summary-global.md new file mode 100644 index 000000000000..dc19ae1a8d31 --- /dev/null +++ b/information-schema/client-errors-summary-global.md @@ -0,0 +1,79 @@ +--- +title: CLIENT_ERRORS_SUMMARY_GLOBAL +summary: Learn about the `CLIENT_ERRORS_SUMMARY_GLOBAL` information_schema table. +--- + +# CLIENT_ERRORS_SUMMARY_GLOBAL + +The table `CLIENT_ERRORS_SUMMARY_GLOBAL` provides a global summary of all SQL errors and warnings that have been returned to clients that connect to a TiDB server. These include: + +* Malformed SQL statements. +* Division by zero errors. +* The attempt to insert out-of-range of duplicate key values. +* Permission errors. +* A table does not exist. + +Client errors are returned to the client via the MySQL server protocol, where applications are expected to take appropriate action. The `information_schema`.`CLIENT_ERRORS_SUMMARY_GLOBAL` table provides a high-level overview, and is useful in the scenario where applications are not correctly handling (or logging) errors returned by the TiDB server. + +The summarized counts can be reset with the statement `FLUSH CLIENT_ERRORS_SUMMARY`. The summary is local to each TiDB server and is only retained in memory. Summaries will be lost if the TiDB server restarts. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC CLIENT_ERRORS_SUMMARY_GLOBAL; +``` + +```sql ++---------------+---------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++---------------+---------------+------+------+---------+-------+ +| ERROR_NUMBER | bigint(64) | NO | | NULL | | +| ERROR_MESSAGE | varchar(1024) | NO | | NULL | | +| ERROR_COUNT | bigint(64) | NO | | NULL | | +| WARNING_COUNT | bigint(64) | NO | | NULL | | +| FIRST_SEEN | timestamp | YES | | NULL | | +| LAST_SEEN | timestamp | YES | | NULL | | ++---------------+---------------+------+------+---------+-------+ +6 rows in set (0.00 sec) +``` + +Field description: + +* `ERROR_NUMBER`: The MySQL-compatible error number that was returned. +* `ERROR_MESSAGE`: The error message which matches the error number (in prepared statement form). +* `ERROR_COUNT`: The number of times this error was returned. +* `WARNING_COUNT`: The number of times this warning was returned. +* `FIRST_SEEN`: The first time this error (or warning) was sent. +* `LAST_SEEN`: The most recent time this error (or warning) was sent. + +The following example shows a warning being generated when connecting to a local TiDB server. The summary is reset after executing `FLUSH CLIENT_ERRORS_SUMMARY`: + +{{< copyable "sql" >}} + +```sql +SELECT 0/0; +SELECT * FROM CLIENT_ERRORS_SUMMARY_GLOBAL; +FLUSH CLIENT_ERRORS_SUMMARY; +SELECT * FROM CLIENT_ERRORS_SUMMARY_GLOBAL; +``` + +```sql ++-----+ +| 0/0 | ++-----+ +| NULL | ++-----+ +1 row in set, 1 warning (0.00 sec) + ++--------------+---------------+-------------+---------------+---------------------+---------------------+ +| ERROR_NUMBER | ERROR_MESSAGE | ERROR_COUNT | WARNING_COUNT | FIRST_SEEN | LAST_SEEN | ++--------------+---------------+-------------+---------------+---------------------+---------------------+ +| 1365 | Division by 0 | 0 | 1 | 2021-03-18 13:10:51 | 2021-03-18 13:10:51 | ++--------------+---------------+-------------+---------------+---------------------+---------------------+ +1 row in set (0.00 sec) + +Query OK, 0 rows affected (0.00 sec) + +Empty set (0.00 sec) +``` diff --git a/information-schema/information-schema-data-lock-waits.md b/information-schema/information-schema-data-lock-waits.md new file mode 100644 index 000000000000..71cba4cda575 --- /dev/null +++ b/information-schema/information-schema-data-lock-waits.md @@ -0,0 +1,86 @@ +--- +title: DATA_LOCK_WAITS +summary: Learn the `DATA_LOCK_WAITS` information_schema table. +--- + +# DATA_LOCK_WAITS + +The `DATA_LOCK_WAITS` table shows the ongoing pessimistic locks waiting on all TiKV nodes in the cluster. + +> **Warning:** +> +> Currently, this is an experimental feature. The definition and behavior of the table structure might have major changes in future releases. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC data_lock_waits; +``` + +```sql ++------------------------+---------------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++------------------------+---------------------+------+------+---------+-------+ +| KEY | varchar(64) | NO | | NULL | | +| TRX_ID | bigint(21) unsigned | NO | | NULL | | +| CURRENT_HOLDING_TRX_ID | bigint(21) unsigned | NO | | NULL | | +| SQL_DIGEST | varchar(64) | YES | | NULL | | ++------------------------+---------------------+------+------+---------+-------+ +``` + +The meaning of each column field in the `DATA_LOCK_WAITS` table is as follows: + +* `KEY`: The KEY that is waiting for the lock and displayed in the form of hexadecimal string. +* `TRX_ID`: The ID of the transaction that is waiting for the lock. This ID is also the `start_ts` of the transaction. +* `CURRENT_HOLDING_TRX_ID`: The ID of the transaction that currently holds the lock. This ID is also the `start_ts` of the transaction. +* `SQL_DIGEST`: The digest of the SQL statement that is currently blocked in the lock-waiting transaction. + +> **Warning:** +> +> * The information in this table is obtained in real time from all TiKV nodes during the query. Currently, even if the `WHERE` condition is added, TiDB might still collect information from all TiKV nodes. If the cluster is large and the load is high, querying this table might cause a potential risk of performance jitter. Therefore, use this table according to your actual situation. +> * The information from different TiKV nodes is NOT guaranteed to be the snapshot at the same point in time. + +## Example + +{{< copyable "sql" >}} + +```sql +select * from information_schema.data_lock_waits\G +``` + +```sql +*************************** 1. row *************************** + KEY: 7480000000000000355f728000000000000002 + TRX_ID: 425405024158875649 +CURRENT_HOLDING_TRX_ID: 425405016242126849 + SQL_DIGEST: f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb22 +2 rows in set (0.01 sec) +``` + +The above query result shows that the transaction of the ID `425405024158875649` was trying to obtain the pessimistic lock on the key `7480000000000000355f728000000000000002` when the statement with digest `"f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb22"` was being executed, but the lock on this key was held by the transaction of the ID `425405016242126849`. + +## SQL Digest + +The `DATA_LOCK_WAITS` table records the SQL digest but not the original SQL statement. + +SQL digest is the hash value of the normalized SQL statement. To find the original SQL statement corresponding to the SQL digest, perform one of the following operations: + +- For the statements executed on the current TiDB node in the recent period of time, you can find the corresponding original SQL statement in the `STATEMENTS_SUMMARY` or `STATEMENTS_SUMMARY_HISTORY` table according to the SQL digest. +- For the statements executed on all TiDB nodes in the entire cluster in the recent period of time, you can find the corresponding SQL statement in the `CLUSTER_STATEMENTS_SUMMARY` or `CLUSTER_STATEMENTS_SUMMARY_HISTORY` table according to the SQL digest. + +{{< copyable "sql" >}} + +```sql +select digest, digest_text from information_schema.statements_summary where digest = "f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb2"; +``` + +```sql ++------------------------------------------------------------------+---------------------------------------+ +| digest | digest_text | ++------------------------------------------------------------------+---------------------------------------+ +| f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb2 | update `t` set `v` = ? where `id` = ? | ++------------------------------------------------------------------+---------------------------------------+ +``` + +For detailed description of SQL digest, `STATEMENTS_SUMMARY`, `STATEMENTS_SUMMARY_HISTORY`, `CLUSTER_STATEMENTS_SUMMARY`, and `CLUSTER_STATEMENTS_SUMMARY_HISTORY` tables, see [Statement Summary Tables](/statement-summary-tables.md). diff --git a/information-schema/information-schema-deadlocks.md b/information-schema/information-schema-deadlocks.md new file mode 100644 index 000000000000..6133ac398819 --- /dev/null +++ b/information-schema/information-schema-deadlocks.md @@ -0,0 +1,200 @@ +--- +title: DEADLOCKS +summary: Learn the `DEADLOCKS` information_schema table. +--- + +# DEADLOCKS + +The `DEADLOCKS` table shows the information of the several deadlock errors that have occurred recently on the current TiDB node. + +> **Warning:** +> +> Currently, this is an experimental feature. The definition and behavior of the table structure might have major changes in future releases. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC deadlocks; +``` + +```sql ++--------------------+---------------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++--------------------+---------------------+------+------+---------+-------+ +| DEADLOCK_ID | bigint(21) | NO | | NULL | | +| OCCUR_TIME | timestamp(6) | YES | | NULL | | +| RETRYABLE | tinyint(1) | NO | | NULL | | +| TRY_LOCK_TRX_ID | bigint(21) unsigned | NO | | NULL | | +| CURRENT_SQL_DIGEST | varchar(64) | YES | | NULL | | +| KEY | text | YES | | NULL | | +| TRX_HOLDING_LOCK | bigint(21) unsigned | NO | | NULL | | ++--------------------+---------------------+------+------+---------+-------+ +``` + +The `DEADLOCKS` table uses multiple rows to show the same deadlock event, and each row displays the information about one of the transactions involved in the deadlock event. If the TiDB node records multiple deadlock errors, each error is distinguished using the `DEADLOCK_ID` column. The same `DEADLOCK_ID` indicates the same deadlock event. Note that `DEADLOCK_ID` **does not guarantee global uniqueness and will not be persisted**. It only shows the same deadlock event in the same result set. + +The meaning of each column field in the `DEADLOCKS` table is as follows: + +* `DEADLOCK_ID`: The ID of the deadlock event. When multiple deadlock errors exist in the table, you can use this column to distinguish rows that belong to different deadlock errors. +* `OCCUR_TIME`: The time when the deadlock error occurs. +* `RETRYABLE`: Whether the deadlock error can be retried. Currently, TiDB does not support collecting the information of the retryable deadlock error, so the value of this field is always `0`. For the description of retryable deadlock errors, see the [Retryable deadlock errors](#retryable-deadlock-errors) section. +* `TRY_LOCK_TRX_ID`: The ID of the transaction that tries to acquire lock. This ID is also the `start_ts` of the transaction. +* `CURRENT_SQL_DIGEST`: The digest of the SQL statement currently being executed in the lock-acquiring transaction. +* `KEY`: The blocked key that the transaction tries to lock. The value of this field is displayed in the form of hexadecimal string. +* `TRX_HOLDING_LOCK`: The ID of the transaction that currently holds the lock on the key and causes blocking. This ID is also the `start_ts` of the transaction. + +To adjust the maximum number of deadlock events that can be recorded in the `DEADLOCKS` table, adjust the [`pessimistic-txn.deadlock-history-capacity`](/tidb-configuration-file.md#deadlock-history-capacity) configuration in the TiDB configuration file. By default, the information of the recent 10 deadlock events is recorded in the table. + +## Example 1 + +Assume that the table definition and the initial data are as follows: + +{{< copyable "sql" >}} + +```sql +create table t (id int primary key, v int); +insert into t values (1, 10), (2, 20); +``` + +Execute the two transactions in the following order: + +| Transaction 1 | Transaction 2 | Description | +|--------------------------------------|--------------------------------------|----------------------| +| `update t set v = 11 where id = 1;` | | | +| | `update t set v = 21 where id = 2;` | | +| `update t set v = 12 where id = 2;` | | Transaction 1 gets blocked. | +| | `update t set v = 22 where id = 1;` | Transaction 2 reports a deadlock error. | + +Next, transaction 2 reports a deadlock error. At this time, query the `DEADLOCKS` table: + +{{< copyable "sql" >}} + +```sql +select * from information_schema.deadlocks; +``` + +The expected output is as follows: + +```sql ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +| DEADLOCK_ID | OCCUR_TIME | RETRYABLE | TRY_LOCK_TRX_ID | CURRENT_SQL_DIGEST | KEY | TRX_HOLDING_LOCK | ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +| 1 | 2021-06-04 08:22:38.765699 | 0 | 425405959304904707 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000002 | 425405959304904708 | +| 1 | 2021-06-04 08:22:38.765699 | 0 | 425405959304904708 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000001 | 425405959304904707 | ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +``` + +Two rows of data are generated in the `DEADLOCKS` table. The `DEADLOCK_ID` field of both rows is `1`, which means that the information in both rows belongs to the same deadlock error. The first row shows that the transaction of the ID `425405959304904707` is blocked on the key of `"7480000000000000385F728000000000000002"` by the transaction of the ID `"425405959304904708"`. The second row shows that the transaction of the ID `"425405959304904708"` is blocked on the key of `"7480000000000000385F728000000000000001"` by the transaction of the ID `425405959304904707`, which constitutes mutual blocking and forms a deadlock. + +## Example 2 + +Assume that you query the `DEADLOCKS` table and get the following result: + +```sql ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +| DEADLOCK_ID | OCCUR_TIME | RETRYABLE | TRY_LOCK_TRX_ID | CURRENT_SQL_DIGEST | KEY | TRX_HOLDING_LOCK | ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +| 1 | 2021-06-04 08:22:38.765699 | 0 | 425405959304904707 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000002 | 425405959304904708 | +| 1 | 2021-06-04 08:22:38.765699 | 0 | 425405959304904708 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000001 | 425405959304904707 | +| 2 | 2021-06-04 08:22:56.795410 | 0 | 425405961664462853 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000002 | 425405961664462854 | +| 2 | 2021-06-04 08:22:56.795410 | 0 | 425405961664462854 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000003 | 425405961664462855 | +| 2 | 2021-06-04 08:22:56.795410 | 0 | 425405961664462855 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000001 | 425405961664462853 | ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +``` + +The `DEADLOCK_ID` column in the above query result shows that the first two rows together represent the information of a deadlock error, and the two transactions that wait for each other form the deadlock. The next three rows together represent another deadlock information, and the three transactions that wait in a cycle form the deadlock. + +## Retryable deadlock errors + +> **Note:** +> +> Currently, TiDB does not support collecting retryable deadlock errors in the `DEADLOCKS` table. + +When transaction A is blocked by a lock already held by transaction B, and transaction B is directly or indirectly blocked by the lock held by the current transaction A, a deadlock error will occur. In this deadlock, there might be two cases: + ++ Case 1: Transaction B might be (directly or indirectly) blocked by a lock generated by a statement that has been executed after transaction A starts and before transaction A gets blocked. ++ Case 2: Transaction B might also be blocked by the statement currently being executed in transaction A. + +In case 1, TiDB will report a deadlock error to the client of transaction A and terminate the transaction. + +In case 2, the statement currently being executed in transaction A will be automatically retried in TiDB. For example, suppose that transaction A executes the following statement: + +{{< copyable "sql" >}} + +```sql +update t set v = v + 1 where id = 1 or id = 2; +``` + +Transaction B executes the following two statements successively. + +{{< copyable "sql" >}} + +```sql +update t set v = 4 where id = 2; +update t set v = 2 where id = 1; +``` + +Then if transaction A locks the two rows with `id = 1` and `id = 2`, and the two transactions run in the following sequence: + +1. Transaction A locks the row with `id = 1`. +2. Transaction B executes the first statement and locks the row with `id = 2`. +3. Transaction B executes the second statement and tries to lock the row with `id = 1`, which is blocked by transaction A. +4. Transaction A tries to lock the row with `id = 2` and is blocked by transaction B, which forms a deadlock. + +For this case, because the statement of transaction A that blocks other transactions is also the statement currently being executed, the pessimistic lock on the current statement can be resolved (so that transaction B can continue to run), and the current statement can be retried. TiDB uses the key hash internally to determine whether this is the case. + +When a retryable deadlock occurs, the internal automatic retry will not cause a transaction error, so it is transparent to the client. However, if this situation occurs frequently, the performance might be affected. When this occurs, you can see `single statement deadlock, retry statement` in the TiDB log. + +## CLUSTER_DEADLOCKS + +The `CLUSTER_DEADLOCKS` table returns information about the recent deadlock errors on each TiDB node in the entire cluster, which is the information of the `DEADLOCKS` table on each node combined together. `CLUSTER_DEADLOCKS` also contains an additional `INSTANCE` column to display the IP address and port of the node to distinguish between different TiDB nodes. + +Note that, because `DEADLOCK_ID` does not guarantee global uniqueness, in the query result of the `CLUSTER_DEADLOCKS` table, you need to use the `INSTANCE` and `DEADLOCK_ID` together to distinguish the information of different deadlock errors in the result set. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC cluster_deadlocks; +``` + +```sql ++--------------------+---------------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++--------------------+---------------------+------+------+---------+-------+ +| INSTANCE | varchar(64) | YES | | NULL | | +| DEADLOCK_ID | bigint(21) | NO | | NULL | | +| OCCUR_TIME | timestamp(6) | YES | | NULL | | +| RETRYABLE | tinyint(1) | NO | | NULL | | +| TRY_LOCK_TRX_ID | bigint(21) unsigned | NO | | NULL | | +| CURRENT_SQL_DIGEST | varchar(64) | YES | | NULL | | +| KEY | text | YES | | NULL | | +| TRX_HOLDING_LOCK | bigint(21) unsigned | NO | | NULL | | ++--------------------+---------------------+------+------+---------+-------+ +``` + +## SQL Digest + +The `DEADLOCKS` table records the SQL digest but not the original SQL statement. + +SQL digest is the hash value after the SQL normalization. To find the original SQL statement corresponding to the SQL digest, perform one of the following operations: + +- For the statements executed on the current TiDB node in the recent period of time, you can find the corresponding original SQL statement in the `STATEMENTS_SUMMARY` or `STATEMENTS_SUMMARY_HISTORY` table according to the SQL digest. +- For the statements executed on all TiDB nodes in the entire cluster in the recent period of time, you can find the corresponding SQL statement in the `CLUSTER_STATEMENTS_SUMMARY` or `CLUSTER_STATEMENTS_SUMMARY_HISTORY` table according to the SQL digest. + +{{< copyable "sql" >}} + +```sql +select digest, digest_text from information_schema.statements_summary where digest = "f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb2"; +``` + +```sql ++------------------------------------------------------------------+---------------------------------------+ +| digest | digest_text | ++------------------------------------------------------------------+---------------------------------------+ +| f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb2 | update `t` set `v` = ? where `id` = ? | ++------------------------------------------------------------------+---------------------------------------+ +``` + +For detailed description of SQL digest, `STATEMENTS_SUMMARY`, `STATEMENTS_SUMMARY_HISTORY`, `CLUSTER_STATEMENTS_SUMMARY`, and `CLUSTER_STATEMENTS_SUMMARY_HISTORY` tables, see [Statement Summary Tables](/statement-summary-tables.md). diff --git a/information-schema/information-schema-tidb-servers-info.md b/information-schema/information-schema-tidb-servers-info.md index 04de17417489..6ee0bc32253e 100644 --- a/information-schema/information-schema-tidb-servers-info.md +++ b/information-schema/information-schema-tidb-servers-info.md @@ -5,7 +5,7 @@ summary: Learn the `TIDB_SERVERS_INFO` information_schema table. # TIDB_SERVERS_INFO -The `TIDB_SERVERS_INFO` table provides information about TiDB servers in the TiDB Cluster (i.e. tidb-server processes). +The `TIDB_SERVERS_INFO` table provides information about TiDB servers in the TiDB Cluster (namely, tidb-server processes). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-tidb-trx.md b/information-schema/information-schema-tidb-trx.md new file mode 100644 index 000000000000..1a9008bc49f9 --- /dev/null +++ b/information-schema/information-schema-tidb-trx.md @@ -0,0 +1,137 @@ +--- +title: TIDB_TRX +summary: Learn the `TIDB_TRX` information_schema table. +--- + +# TIDB_TRX + +The `TIDB_TRX` table provides information about the transactions currently being executed on the TiDB node. + +> **Warning:** +> +> * Currently, this is an experimental feature. The definition and behavior of the table structure might have major changes in future releases. +> * Currently, the `TIDB_TRX` table does not support displaying information of TiDB's internal transactions. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC tidb_trx; +``` + +```sql ++--------------------+---------------------------------------------------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++--------------------+---------------------------------------------------------+------+------+---------+-------+ +| ID | bigint(21) unsigned | NO | PRI | NULL | | +| START_TIME | timestamp(6) | YES | | NULL | | +| CURRENT_SQL_DIGEST | varchar(64) | YES | | NULL | | +| STATE | enum('Normal','LockWaiting','Committing','RollingBack') | YES | | NULL | | +| WAITING_START_TIME | timestamp(6) | YES | | NULL | | +| MEM_BUFFER_KEYS | bigint(64) | YES | | NULL | | +| MEM_BUFFER_BYTES | bigint(64) | YES | | NULL | | +| SESSION_ID | bigint(21) unsigned | YES | | NULL | | +| USER | varchar(16) | YES | | NULL | | +| DB | varchar(64) | YES | | NULL | | +| ALL_SQL_DIGESTS | text | YES | | NULL | | ++--------------------+---------------------------------------------------------+------+------+---------+-------+ +``` + +The meaning of each column field in the `TIDB_TRX` table is as follows: + +* `ID`: The transaction ID, which is the `start_ts` (start timestamp) of the transaction. +* `START_TIME`: The start time of the transaction, which is the physical time corresponding to the `start_ts` of the transaction. +* `CURRENT_SQL_DIGEST`: The digest of the SQL statement currently being executed in the transaction. +* `STATE`: The current state of the transaction. The possible values ​​include: + * `Normal`: The transaction is being executed normally or in an idle state. + * `LockWaiting`: The transaction is waiting for the pessimistic lock to be acquired. Note that the transaction enters this state at the beginning of the pessimistic locking operation, no matter whether it is blocked by other transactions or not. + * `Committing`: The transaction is in the process of commit. + * `RollingBack`: The transaction is being rolled back. +* `WAITING_START_TIME`: When the value of `STATE` is `LockWaiting`, this column shows the start time of the waiting. +* `MEM_BUFFER_KEYS`: The number of keys written into the memory buffer by the current transaction. +* `MEM_BUFFER_BYTES`: The total number of key-value bytes written into the memory buffer by the current transaction. +* `SESSION_ID`: The ID of the session to which this transaction belongs. +* `USER`: The name of the user who performs the transaction. +* `DB`: The current default database name of the session in which the transaction is executed. +* `ALL_SQL_DIGESTS`: The digest list of statements that have been executed in this transaction. For each transaction, the first 50 statements at most are recorded. + +## Example + +{{< copyable "sql" >}} + +```sql +select * from information_schema.tidb_trx\G +``` + +```sql +*************************** 1. row *************************** + ID: 425403705115541506 + START_TIME: 2021-06-04 05:59:10.691000 +CURRENT_SQL_DIGEST: NULL + STATE: Normal +WAITING_START_TIME: NULL + MEM_BUFFER_KEYS: 2 + MEM_BUFFER_BYTES: 48 + SESSION_ID: 7 + USER: root + DB: test + ALL_SQL_DIGESTS: [e6f07d43b5c21db0fbb9a31feac2dc599787763393dd5acbfad80e247eb02ad5, 04fa858fa491c62d194faec2ab427261cc7998b3f1ccf8f6844febca504cb5e9, f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb2] +1 row in set (0.00 sec) +``` + +The query result of the above example indicates that a transaction is being executed on the current node (the `STATE` is `Normal`), and this transaction is currently idle (`CURRENT_SQL_DIGEST` is `NULL`). This transaction has executed three statements (there are three records in the `ALL_SQL_DIGESTS` list and they are the digests of the three executed statements). + +## CLUSTER_TIDB_TRX + +The `TIDB_TRX` table only provides information about the transactions that are being executed on a single TiDB node. If you want to view the information of the transactions that are being executed on all TiDB nodes in the entire cluster, you need to query the `CLUSTER_TIDB_TRX` table. Compared with the query result of the `TIDB_TRX` table, the query result of the `CLUSTER_TIDB_TRX` table contains an extra `INSTANCE` field. The `INSTANCE` field displays the IP address and port of each node in the cluster, which is used to distinguish the TiDB node where the transaction is located. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC cluster_tidb_trx; +``` + +```sql ++--------------------+---------------------------------------------------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++--------------------+---------------------------------------------------------+------+------+---------+-------+ +| INSTANCE | varchar(64) | YES | | NULL | | +| ID | bigint(21) unsigned | NO | PRI | NULL | | +| START_TIME | timestamp(6) | YES | | NULL | | +| CURRENT_SQL_DIGEST | varchar(64) | YES | | NULL | | +| STATE | enum('Normal','LockWaiting','Committing','RollingBack') | YES | | NULL | | +| WAITING_START_TIME | timestamp(6) | YES | | NULL | | +| MEM_BUFFER_KEYS | bigint(64) | YES | | NULL | | +| MEM_BUFFER_BYTES | bigint(64) | YES | | NULL | | +| SESSION_ID | bigint(21) unsigned | YES | | NULL | | +| USER | varchar(16) | YES | | NULL | | +| DB | varchar(64) | YES | | NULL | | +| ALL_SQL_DIGESTS | text | YES | | NULL | | ++--------------------+---------------------------------------------------------+------+------+---------+-------+ +``` + +## SQL Digest + +The `TIDB_TRX` table only records SQL digests, not the original SQL statement. + +SQL digest is the hash value after the SQL normalization. To find the original SQL statement corresponding to the SQL digest, perform one of the following operations: + +- For the statements executed on the current TiDB node in the recent period of time, you can find the corresponding orginal SQL statement from the SQL digest in `STATEMENTS_SUMMARY` or `STATEMENTS_SUMMARY_HISTORY`. +- For the statements executed on all TiDB nodes in the entire cluster in the recent period of time, you can find the corresponding SQL statement from the SQL digest in `CLUSTER_STATEMENTS_SUMMARY` or `CLUSTER_STATEMENTS_SUMMARY_HISTORY`. + +{{< copyable "sql" >}} + +```sql +select digest, digest_text from information_schema.statements_summary where digest = "f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb2"; +``` + +```sql ++------------------------------------------------------------------+---------------------------------------+ +| digest | digest_text | ++------------------------------------------------------------------+---------------------------------------+ +| f7530877a35ae65300c42250abd8bc731bbaf0a7cabc05dab843565230611bb2 | update `t` set `v` = ? where `id` = ? | ++------------------------------------------------------------------+---------------------------------------+ +``` + +For detailed description of SQL digest, `STATEMENTS_SUMMARY`, `STATEMENTS_SUMMARY_HISTORY`, `CLUSTER_STATEMENTS_SUMMARY`, and `CLUSTER_STATEMENTS_SUMMARY_HISTORY` tables, see [Statement Summary Tables](/statement-summary-tables.md). diff --git a/information-schema/information-schema.md b/information-schema/information-schema.md index 5fbf191e7788..d2d763dffe50 100644 --- a/information-schema/information-schema.md +++ b/information-schema/information-schema.md @@ -52,7 +52,11 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | Table Name | Description | |-----------------------------------------------------------------------------------------|-------------| | [`ANALYZE_STATUS`](/information-schema/information-schema-analyze-status.md) | Provides information about tasks to collect statistics. | +| [`CLIENT_ERRORS_SUMMARY_BY_HOST`](/information-schema/client-errors-summary-by-host.md) | Provides a summary of errors and warnings generated by client requests and returned to clients. | +| [`CLIENT_ERRORS_SUMMARY_BY_USER`](/information-schema/client-errors-summary-by-user.md) | Provides a summary of errors and warnings generated by clients. | +| [`CLIENT_ERRORS_SUMMARY_GLOBAL`](/information-schema/client-errors-summary-global.md) | Provides a summary of errors and warnings generated by clients. | | [`CLUSTER_CONFIG`](/information-schema/information-schema-cluster-config.md) | Provides details about configuration settings for the entire TiDB cluster. | +| `CLUSTER_DEADLOCKS` | Provides a cluster-level view of the `DEADLOCKS` table. | | [`CLUSTER_HARDWARE`](/information-schema/information-schema-cluster-info.md) | Provides details on the underlying physical hardware discovered on each TiDB component. | | [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. | | [`CLUSTER_LOAD`](/information-schema/information-schema-cluster-load.md) | Provides current load information for TiDB servers in the cluster. | @@ -60,9 +64,12 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | `CLUSTER_PROCESSLIST` | Provides a cluster-level view of the `PROCESSLIST` table. | | `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. | | `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. | -| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `CLUSTER_STATEMENTS_SUMMARY_HISTORY` table. | +| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. | +| `CLUSTER_TIDB_TRX` | Provides a cluster-level view of the `TIDB_TRX` table. | | [`CLUSTER_SYSTEMINFO`](/information-schema/information-schema-cluster-systeminfo.md) | Provides details about kernel parameter configuration for servers in the cluster. | +| [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) | Provides the lock-waiting information on the TiKV server. | | [`DDL_JOBS`](/information-schema/information-schema-ddl-jobs.md) | Provides similar output to `ADMIN SHOW DDL JOBS` | +| [`DEADLOCKS`](/information-schema/information-schema-deadlocks.md) | Provides the information of several deadlock errors that have recently occurred. | | [`INSPECTION_RESULT`](/information-schema/information-schema-inspection-result.md) | Triggers internal diagnostics checks. | | [`INSPECTION_RULES`](/information-schema/information-schema-inspection-rules.md) | A list of internal diagnostic checks performed. | | [`INSPECTION_SUMMARY`](/information-schema/information-schema-inspection-summary.md) | A summarized report of important monitoring metrics. | @@ -76,7 +83,8 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) | Provides details about table sizes in storage. | | [`TIDB_HOT_REGIONS`](/information-schema/information-schema-tidb-hot-regions.md) | Provides statistics about which regions are hot. | | [`TIDB_INDEXES`](/information-schema/information-schema-tidb-indexes.md) | Provides index information about TiDB tables. | -| [`TIDB_SERVERS_INFO`](/information-schema/information-schema-tidb-servers-info.md) | Provides a list of TiDB servers (i.e. tidb-server component) | +| [`TIDB_SERVERS_INFO`](/information-schema/information-schema-tidb-servers-info.md) | Provides a list of TiDB servers (namely, tidb-server component) | +| [`TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) | Provides the information of the transactions that are being executed on the TiDB node. | | [`TIFLASH_REPLICA`](/information-schema/information-schema-tiflash-replica.md) | Provides details about TiFlash replicas. | | [`TIKV_REGION_PEERS`](/information-schema/information-schema-tikv-region-peers.md) | Provides details about where regions are stored. | | [`TIKV_REGION_STATUS`](/information-schema/information-schema-tikv-region-status.md) | Provides statistics about regions. | diff --git a/literal-values.md b/literal-values.md index 8901423750ee..6c81663b5a29 100644 --- a/literal-values.md +++ b/literal-values.md @@ -32,9 +32,9 @@ If the `ANSI_QUOTES` SQL MODE is enabled, string literals can be quoted only wit The string is divided into the following two types: + Binary string: It consists of a sequence of bytes, whose charset and collation are both `binary`, and uses **byte** as the unit when compared with each other. -+ Non-binary string: It consists of a sequence of characters and has various charsets and collations other than `binary`. When compared with each other, non-binary strings use **characters** as the unit. A charater might contian multiple bytes, depending on the charset. ++ Non-binary string: It consists of a sequence of characters and has various charsets and collations other than `binary`. When compared with each other, non-binary strings use **characters** as the unit. A character might contain multiple bytes, depending on the charset. -A string literal may have an optional `character set introducer` and `COLLATE clause`, to designate it as a string that uses a specific character set and collation. +A string literal may have an optional `character set introducer` and `COLLATE clause`, to designate it as a string that uses a specific character set and collation. ``` [_charset_name]'string' [COLLATE collation_name] @@ -95,12 +95,12 @@ Date and time literal values can be represented in several formats, such as quot TiDB supports the following date formats: * `'YYYY-MM-DD'` or `'YY-MM-DD'`: The `-` delimiter here is not strict. It can be any punctuation. For example, `'2017-08-24'`, `'2017&08&24'`, `'2012@12^31'` are all valid date formats. The only special punctuation is '.', which is is treated as a decimal point to separate the integer and fractional parts. Date and time can be separated by `T` or a white space. For example, `2017-8-24 10:42:00` and `2017-8-24T10:42:00` represents the same date and time. -* `'YYYYMMDDHHMMSS'` or `'YYMMDDHHMMSS'`: For example, `'20170824104520'` and `'170824104520'` are regarded as `'2017-08-24 10:45:20'`. However, if you provide a value out of range, such as `'170824304520'`, it is not treated as a valid date. +* `'YYYYMMDDHHMMSS'` or `'YYMMDDHHMMSS'`: For example, `'20170824104520'` and `'170824104520'` are regarded as `'2017-08-24 10:45:20'`. However, if you provide a value out of range, such as `'170824304520'`, it is not treated as a valid date. Note that incorrect formats such as `YYYYMMDD HHMMSS`, `YYYYMMDD HH:MM:DD`, or `YYYY-MM-DD HHMMSS` will fail to insert. * `YYYYMMDDHHMMSS` or `YYMMDDHHMMSS`: Note that these formats have no single or double quotes, but a number. For example, `20170824104520` is interpreted as `'2017-08-24 10:45:20'`. DATETIME or TIMESTAMP values can be followed by a fractional part, used to represent microseconds precision (6 digits). The fractional part should always be separated from the rest of the time by a decimal point `.`. -The year value containing only two digits is ambiguous. It is recommended to use the four-digit year format. TiDB interpretes the two-digit year value according to the following rules: +The year value containing only two digits is ambiguous. It is recommended to use the four-digit year format. TiDB interprets the two-digit year value according to the following rules: * If the year value is in the range of `70-99`, it is converted to `1970-1999`. * If the year value is in the range of `00-69`, it is converted to `2000-2069`. @@ -165,7 +165,7 @@ X'1z' (z is not a hexadecimal legal digit) 0X12AC (0X must be written as 0x) ``` -Hexadecimal literals written using `X'val'` notation must contain an even number of digits. To avoid the syntax error, pad the value with a leading zero: +Hexadecimal literals written using `X'val'` notation must contain an even number of digits. If the length of `val` is an odd number (for example, `X'A'` or `X'11A'`), to avoid the syntax error, pad the value with a leading zero: ```sql mysql> select X'aff'; diff --git a/loader-overview.md b/loader-overview.md deleted file mode 100644 index 1873f3b9f213..000000000000 --- a/loader-overview.md +++ /dev/null @@ -1,193 +0,0 @@ ---- -title: Loader Instructions -summary: Use Loader to load data to TiDB. -aliases: ['/docs/dev/loader-overview/','/docs/dev/reference/tools/loader/','/docs/dev/load-misuse-handling/','/docs/dev/reference/tools/error-case-handling/load-misuse-handling/','/tidb/dev/load-misuse-handling'] ---- - -# Loader Instructions - -> **Warning:** -> -> Loader is no longer maintained. Its features are completely superseded by [TiDB Lightning TiDB-backend](/tidb-lightning/tidb-lightning-backends.md#tidb-lightning-tidb-backend). It is strongly recommend to use TiDB Lightning instead. - -## What is Loader? - -Loader is a data import tool to load data to TiDB. - -It can be [downloaded](/download-ecosystem-tools.md) as part of the tidb-enterprise-tools package. - -## Why did we develop Loader? - -Since tools like mysqldump will take us days to migrate massive amounts of data, we used the [Mydumper/myloader suite](https://github.com/maxbube/mydumper) to multi-thread export and import data. During the process, we found that Mydumper works well. However, as myloader lacks functions of error retry and savepoint, it is inconvenient for us to use. Therefore, we developed loader, which reads the output data files of Mydumper and imports data to TiDB through the MySQL protocol. - -## What can Loader do? - -+ Multi-threaded data import - -+ Support table-level concurrent import and scattered hotspot write - -+ Support concurrent import of a single large table and scattered hotspot write - -+ Support Mydumper data format - -+ Support error retry - -+ Support savepoint - -+ Improve the speed of importing data through system variable - -## Usage - -> **Note:** -> -> - Do not import the `mysql` system database from the MySQL instance to the downstream TiDB instance. -> - If Mydumper uses the `-m` parameter, the data is exported without the table schema and the loader can not import the data. -> - If you use the default `checkpoint-schema` parameter, after importing the data of a database, run `drop database tidb_loader` before you begin to import the next database. -> - It is recommended to specify the `checkpoint-schema = "tidb_loader"` parameter when importing data. - -### Parameter description - -``` - -L string: the log level setting, which can be set as debug, info, warn, error, fatal (default: "info") - - -P int: the port of TiDB (default: 4000) - - -V boolean: prints version and exit - - -c string: config file - - -checkpoint-schema string: the database name of checkpoint. In the execution process, loader will constantly update this database. After recovering from an interruption, loader will get the process of the last run through this database. (default: "tidb_loader") - - -d string: the storage directory of data that need to import (default: "./") - - -h string: the host IP of TiDB (default: "127.0.0.1") - - -p string: the account and password of TiDB - - -status-addr string: It can be used by Prometheus to pull Loader metrics, and is also the pprof address of Loader (default: ":8272") - - -t int: the number of threads. Each worker restores one file at a time. (default: 16) - - -u string: the user name of TiDB (default: "root") -``` - -### Configuration file - -Apart from command-line parameters, you can also use configuration files. The format is shown as below: - -```toml -# Loader log level, which can be set as "debug", "info", "warn", "error", and "fatal" (default: "info") -log-level = "info" - -# Loader log file -log-file = "loader.log" - -# Directory of the dump to import (default: "./") -dir = "./" - -# It can be used by Prometheus to pull Loader metrics, and is also the pprof address of Loader (default: ":8272"). -status-addr = ":8272" - -# The checkpoint data is saved to TiDB, and the schema name is defined here. -checkpoint-schema = "tidb_loader" - -# Number of threads restoring concurrently for worker pool (default: 16). Each worker restores one file at a time. -pool-size = 16 - -# The target database information -[db] -host = "127.0.0.1" -user = "root" -password = "" -port = 4000 - -# `sql_mode` of session level used to connect to the database when loading data. If `sql-mode` is not provided or set to "@DownstreamDefault", the global `sql_mode` for downstream is used. -# sql-mode = "" -# `max-allowed-packet` sets the maximum data packet allowed for database connection, which corresponds to the `max_allowed_packet` in system parameters. If it is set to 0, the global `max_allowed_packet` for downstream is used. -max-allowed-packet = 67108864 - -# The sharding replicating rules support wildcharacter. -# 1. The asterisk character (*, also called "star") matches zero or more characters, -# for example, "doc*" matches "doc" and "document" but not "dodo"; -# asterisk character must be in the end of the wildcard word, -# and there is only one asterisk in one wildcard word. -# 2. The question mark '?' matches exactly one character. -# [[route-rules]] -# pattern-schema = "shard_db_*" -# pattern-table = "shard_table_*" -# target-schema = "shard_db" -# target-table = "shard_table" -``` - -### Usage example - -Command line parameter: - -{{< copyable "shell-regular" >}} - -``` -./bin/loader -d ./test -h 127.0.0.1 -u root -P 4000 -``` - -Or use configuration file "config.toml": - -{{< copyable "shell-regular" >}} - -``` -./bin/loader -c=config.toml -``` - -## FAQ - -### The scenario of replicating data from sharded tables - -Loader supports importing data from sharded tables into one table within one database according to the route-rules. Before replicating, check the following items: - -- Whether the sharding rules can be represented using the `route-rules` syntax. -- Whether the sharded tables contain monotone increasing primary keys, or whether there are conflicts in the unique indexes or the primary keys after the combination. - -To combine tables, start the `route-rules` parameter in the configuration file of Loader: - -- To use the table combination function, it is required to fill the `pattern-schema` and `target-schema`. -- If the `pattern-table` and `target-table` are NULL, the table name is not combined or converted. - -```toml -[[route-rules]] -pattern-schema = "example_db" -pattern-table = "table_*" -target-schema = "example_db" -target-table = "table" -``` - -## Error: ```Try adjusting the `max_allowed_packet` variable``` - -The following error is reported during full data import: - -``` -packet for query is too large. Try adjusting the 'max_allowed_packet' variable -``` - -### Reasons - -* Both MySQL client and MySQL/TiDB Server have `max_allowed_packet` quotas. If any of the `max_allowed_packet` quotas is violated, the client receives a corresponding error message. Currently, the latest version of Loader and TiDB Server all have a default `max_allowed_packet` quota of `64M`. - * Please use the latest version, or the latest stable version of the tool. See the [download page](/download-ecosystem-tools.md). -* The full data import processing module in Loader or DM does not support splitting `dump sqls` files. This is because Mydumper has the simple code implementation, as shown in the code comment `/* Poor man's data dump code */`. To support correctly splitting `dump sqls` files, you need to implement a sound parser based on TiDB parser, but you will encounter the following issues: - * A large amount of workload. - * The task is difficult. It is not easy to ensure the correctness. - * Significant performance reduction. - -### Solutions - -* For the above reasons, it is recommended to use the `-s, --statement-size` option which Mydumper offers to control the size of `Insert Statement`: `Attempted size of INSERT statement in bytes, default 1000000`. - - According to the default configuration of `--statement-size`, the size of `Insert Statement` that Mydumper generates is as close as `1M`. This default configuration ensures that this error will not occur in most cases. - - Sometimes the following `WARN` log appears during the dump process. The `WARN` log itself does not affect the dump process but indicates that the dumped table might be a wide table. - - ``` - Row bigger than statement_size for xxx - ``` - -* If a single row of a wide table exceeds 64M, you need to modify and enable the following two configurations: - * Execute `set @@global.max_allowed_packet=134217728` (`134217728 = 128M`) in TiDB Server. - * Add `max-allowed-packet=128M` to db configuration in the Loader configuration file according to your situation, and then restart the progress or task. diff --git a/log-redaction.md b/log-redaction.md new file mode 100644 index 000000000000..dcd33c11f9dd --- /dev/null +++ b/log-redaction.md @@ -0,0 +1,50 @@ +--- +title: Log Redaction +summary: Learn the log redaction in TiDB components. +--- + +# Log Redaction + +When TiDB provides detailed log information, it might print sensitive data (for example, user data) in the log, which causes data security risks. To avoid such risks, each component (TiDB, TiKV, and PD) provides a configuration item that enables log redaction to shield user data values. + +## Log redaction in TiDB side + +To enable log redaction in the TiDB side, set the value of [`global.tidb_redact_log`](/system-variables.md#tidb_redact_log) to `1`. This configuration value defaults to `0`, which means that log redaction is disabled. + +You can use the `set` syntax to set the global variable `tidb_redact_log`: + +{{< copyable "sql" >}} + +```sql +set @@global.tidb_redact_log=1; +``` + +After the setting, all logs generated in new sessions are redacted: + +```sql +create table t (a int, unique key (a)); +Query OK, 0 rows affected (0.00 sec) + +insert into t values (1),(1); +ERROR 1062 (23000): Duplicate entry '1' for key 'a' +``` + +The error log for the `INSERT` statement above is printed as follows: + +``` +[2020/10/20 11:45:49.539 +08:00] [INFO] [conn.go:800] ["command dispatched failed"] [conn=5] [connInfo="id:5, addr:127.0.0.1:57222 status:10, collation:utf8_general_ci, user:root"] [command=Query] [status="inTxn:0, autocommit:1"] [sql="insert into t values ( ? ) , ( ? )"] [txn_mode=OPTIMISTIC] [err="[kv:1062]Duplicate entry '?' for key 'a'"] +``` + +From the error log above, you can see that all sensitive information is shielded using `?` after `tidb_redact_log` is enabled. In this way, data security risks are avoided. + +## Log redaction in TiKV side + +To enable log redaction in the TiKV side, set the value of [`security.redact-info-log`](/tikv-configuration-file.md#redact-info-log-new-in-v408) to `true`. This configuration value defaults to `false`, which means that log redaction is disabled. + +## Log redaction in PD side + +To enable log redaction in the PD side, set the value of [`security.redact-info-log`](/pd-configuration-file.md#redact-info-log-new-in-v50) to `true`. This configuration value defaults to `false`, which means that log redaction is disabled. + +## Log redaction in TiFlash side + +To enable log redaction in the TiFlash side, set both the [`security.redact_info_log`](/tiflash/tiflash-configuration.md#configure-the-tiflashtoml-file) value in tiflash-server and the [`security.redact-info-log`](/tiflash/tiflash-configuration.md#configure-the-tiflash-learnertoml-file) value in tiflash-learner to `true`. Both configuration values default to `false`, which means that log redaction is disabled. diff --git a/maintain-tidb-using-ansible.md b/maintain-tidb-using-ansible.md deleted file mode 100644 index 1f5c2234dce3..000000000000 --- a/maintain-tidb-using-ansible.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: TiDB Ansible Common Operations -summary: Learn some common operations when using TiDB Ansible to administer a TiDB cluster. -aliases: ['/docs/dev/maintain-tidb-using-ansible/','/docs/dev/how-to/deploy/orchestrated/ansible-operations/'] ---- - -# TiDB Ansible Common Operations - -This guide describes the common operations when you administer a TiDB cluster using TiDB Ansible. - -## Start a cluster - -```bash -$ ansible-playbook start.yml -``` - -This operation starts all the components in the entire TiDB cluster in order, which include PD, TiDB, TiKV, and the monitoring components. - -## Stop a cluster - -```bash -$ ansible-playbook stop.yml -``` - -This operation stops all the components in the entire TiDB cluster in order, which include PD, TiDB, TiKV, and the monitoring components. - -## Clean up cluster data - -``` -$ ansible-playbook unsafe_cleanup_data.yml -``` - -This operation stops the TiDB, Pump, TiKV and PD services, and cleans up the data directory of Pump, TiKV and PD. - -## Destroy a cluster - -``` -$ ansible-playbook unsafe_cleanup.yml -``` - -This operation stops the cluster and cleans up the data directory. - -> **Note:** -> -> If the deployment directory is a mount point, an error will be reported, but implementation results remain unaffected, so you can ignore it. diff --git a/maintain-tidb-using-tiup.md b/maintain-tidb-using-tiup.md index 8c7d8593bfcd..03f50ee1662d 100644 --- a/maintain-tidb-using-tiup.md +++ b/maintain-tidb-using-tiup.md @@ -109,11 +109,11 @@ When the cluster is in operation, if you need to modify the parameters of a comp log.slow-threshold: 300 ``` - For the parameter format, see the [TiUP parameter template](https://github.com/pingcap/tiup/blob/master/examples/topology.example.yaml). + For the parameter format, see the [TiUP parameter template](https://github.com/pingcap/tiup/blob/master/embed/templates/examples/topology.example.yaml). **Use `.` to represent the hierarchy of the configuration items**. - For more information on the configuration parameters of components, refer to [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/v4.0.0-rc/config/config.toml.example), [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/v4.0.0-rc/etc/config-template.toml), and [PD `config.toml.example`](https://github.com/pingcap/pd/blob/v4.0.0-rc/conf/config.toml). + For more information on the configuration parameters of components, refer to [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/master/config/config.toml.example), [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/master/etc/config-template.toml), and [PD `config.toml.example`](https://github.com/tikv/pd/blob/master/conf/config.toml). 3. Rolling update the configuration and restart the corresponding components by running the `reload` command: @@ -125,7 +125,7 @@ When the cluster is in operation, if you need to modify the parameters of a comp ### Example -If you want to set the transaction size limit parameter (`txn-total-size-limit` in the [performance](https://github.com/pingcap/tidb/blob/v4.0.0-rc/config/config.toml.example) module) to `1G` in tidb-server, edit the configuration as follows: +If you want to set the transaction size limit parameter (`txn-total-size-limit` in the [performance](https://github.com/pingcap/tidb/blob/master/config/config.toml.example) module) to `1G` in tidb-server, edit the configuration as follows: ``` server_configs: diff --git a/media/dashboard/dashboard-faq-devtools-application.png b/media/dashboard/dashboard-faq-devtools-application.png new file mode 100644 index 000000000000..812e672bafd3 Binary files /dev/null and b/media/dashboard/dashboard-faq-devtools-application.png differ diff --git a/media/dashboard/dashboard-faq-devtools.png b/media/dashboard/dashboard-faq-devtools.png new file mode 100644 index 000000000000..db2413d913f5 Binary files /dev/null and b/media/dashboard/dashboard-faq-devtools.png differ diff --git a/media/dashboard/dashboard-session-share-settings-1.png b/media/dashboard/dashboard-session-share-settings-1.png new file mode 100644 index 000000000000..6551d803039c Binary files /dev/null and b/media/dashboard/dashboard-session-share-settings-1.png differ diff --git a/media/dashboard/dashboard-session-share-settings-2.png b/media/dashboard/dashboard-session-share-settings-2.png new file mode 100644 index 000000000000..6c90439886cf Binary files /dev/null and b/media/dashboard/dashboard-session-share-settings-2.png differ diff --git a/media/dashboard/dashboard-session-share-settings-3.png b/media/dashboard/dashboard-session-share-settings-3.png new file mode 100644 index 000000000000..372a4ef41049 Binary files /dev/null and b/media/dashboard/dashboard-session-share-settings-3.png differ diff --git a/media/dashboard/dashboard-session-share-signin-1.png b/media/dashboard/dashboard-session-share-signin-1.png new file mode 100644 index 000000000000..00a29527f800 Binary files /dev/null and b/media/dashboard/dashboard-session-share-signin-1.png differ diff --git a/media/dashboard/dashboard-session-share-signin-2.png b/media/dashboard/dashboard-session-share-signin-2.png new file mode 100644 index 000000000000..006e85b115b7 Binary files /dev/null and b/media/dashboard/dashboard-session-share-signin-2.png differ diff --git a/media/dashboard/dashboard-session-share-signin-3.png b/media/dashboard/dashboard-session-share-signin-3.png new file mode 100644 index 000000000000..96cc51e59b06 Binary files /dev/null and b/media/dashboard/dashboard-session-share-signin-3.png differ diff --git a/media/dashboard/dashboard-session-sso-disable.png b/media/dashboard/dashboard-session-sso-disable.png new file mode 100644 index 000000000000..7740f1579ca9 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-disable.png differ diff --git a/media/dashboard/dashboard-session-sso-enable-1.png b/media/dashboard/dashboard-session-sso-enable-1.png new file mode 100644 index 000000000000..8b810c0f41a8 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-enable-1.png differ diff --git a/media/dashboard/dashboard-session-sso-enable-2.png b/media/dashboard/dashboard-session-sso-enable-2.png new file mode 100644 index 000000000000..5a721c39a965 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-enable-2.png differ diff --git a/media/dashboard/dashboard-session-sso-enable-3.png b/media/dashboard/dashboard-session-sso-enable-3.png new file mode 100644 index 000000000000..244a2ed84aff Binary files /dev/null and b/media/dashboard/dashboard-session-sso-enable-3.png differ diff --git a/media/dashboard/dashboard-session-sso-okta-1.png b/media/dashboard/dashboard-session-sso-okta-1.png new file mode 100644 index 000000000000..bbea4f0b7759 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-okta-1.png differ diff --git a/media/dashboard/dashboard-session-sso-okta-2.png b/media/dashboard/dashboard-session-sso-okta-2.png new file mode 100644 index 000000000000..6a89ed6444ae Binary files /dev/null and b/media/dashboard/dashboard-session-sso-okta-2.png differ diff --git a/media/dashboard/dashboard-session-sso-okta-3.png b/media/dashboard/dashboard-session-sso-okta-3.png new file mode 100644 index 000000000000..186931fbf5ec Binary files /dev/null and b/media/dashboard/dashboard-session-sso-okta-3.png differ diff --git a/media/dashboard/dashboard-session-sso-okta-4.png b/media/dashboard/dashboard-session-sso-okta-4.png new file mode 100644 index 000000000000..a33b5c33fe13 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-okta-4.png differ diff --git a/media/dashboard/dashboard-session-sso-okta-info-1.png b/media/dashboard/dashboard-session-sso-okta-info-1.png new file mode 100644 index 000000000000..a0d88c6c9b4c Binary files /dev/null and b/media/dashboard/dashboard-session-sso-okta-info-1.png differ diff --git a/media/dashboard/dashboard-session-sso-okta-info-2.png b/media/dashboard/dashboard-session-sso-okta-info-2.png new file mode 100644 index 000000000000..d36c5c96018f Binary files /dev/null and b/media/dashboard/dashboard-session-sso-okta-info-2.png differ diff --git a/media/dashboard/dashboard-session-sso-okta-info-3.png b/media/dashboard/dashboard-session-sso-okta-info-3.png new file mode 100644 index 000000000000..fe3d85df0b6f Binary files /dev/null and b/media/dashboard/dashboard-session-sso-okta-info-3.png differ diff --git a/media/dashboard/dashboard-session-sso-reauthorize.png b/media/dashboard/dashboard-session-sso-reauthorize.png new file mode 100644 index 000000000000..c87891131378 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-reauthorize.png differ diff --git a/media/dashboard/dashboard-session-sso-signin.png b/media/dashboard/dashboard-session-sso-signin.png new file mode 100644 index 000000000000..b213419429e7 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-signin.png differ diff --git a/media/sysbench_v510vsv502_point_select.png b/media/sysbench_v510vsv502_point_select.png new file mode 100644 index 000000000000..257ecde394d5 Binary files /dev/null and b/media/sysbench_v510vsv502_point_select.png differ diff --git a/media/sysbench_v510vsv502_read_write.png b/media/sysbench_v510vsv502_read_write.png new file mode 100644 index 000000000000..dffc83b07dd7 Binary files /dev/null and b/media/sysbench_v510vsv502_read_write.png differ diff --git a/media/sysbench_v510vsv502_update_index.png b/media/sysbench_v510vsv502_update_index.png new file mode 100644 index 000000000000..353beb19c613 Binary files /dev/null and b/media/sysbench_v510vsv502_update_index.png differ diff --git a/media/sysbench_v510vsv502_update_non_index.png b/media/sysbench_v510vsv502_update_non_index.png new file mode 100644 index 000000000000..96a1fbf1d8ed Binary files /dev/null and b/media/sysbench_v510vsv502_update_non_index.png differ diff --git a/media/sysbench_v5vsv4_point_select.png b/media/sysbench_v5vsv4_point_select.png new file mode 100644 index 000000000000..bd8aeea8e270 Binary files /dev/null and b/media/sysbench_v5vsv4_point_select.png differ diff --git a/media/sysbench_v5vsv4_read_write.png b/media/sysbench_v5vsv4_read_write.png new file mode 100644 index 000000000000..419b187ecc9f Binary files /dev/null and b/media/sysbench_v5vsv4_read_write.png differ diff --git a/media/sysbench_v5vsv4_update_index.png b/media/sysbench_v5vsv4_update_index.png new file mode 100644 index 000000000000..33e1a327670b Binary files /dev/null and b/media/sysbench_v5vsv4_update_index.png differ diff --git a/media/sysbench_v5vsv4_update_non_index.png b/media/sysbench_v5vsv4_update_non_index.png new file mode 100644 index 000000000000..9a5d270e5443 Binary files /dev/null and b/media/sysbench_v5vsv4_update_non_index.png differ diff --git a/media/ticdc-state-transfer.png b/media/ticdc-state-transfer.png new file mode 100644 index 000000000000..48770525e803 Binary files /dev/null and b/media/ticdc-state-transfer.png differ diff --git a/media/ticdc/ticdc-dashboard-changefeed-1.png b/media/ticdc/ticdc-dashboard-changefeed-1.png new file mode 100644 index 000000000000..46108a472373 Binary files /dev/null and b/media/ticdc/ticdc-dashboard-changefeed-1.png differ diff --git a/media/ticdc/ticdc-dashboard-changefeed-2.png b/media/ticdc/ticdc-dashboard-changefeed-2.png new file mode 100644 index 000000000000..7215fb8c2bb0 Binary files /dev/null and b/media/ticdc/ticdc-dashboard-changefeed-2.png differ diff --git a/media/ticdc/ticdc-dashboard-changefeed-3.png b/media/ticdc/ticdc-dashboard-changefeed-3.png new file mode 100644 index 000000000000..4910cb10416f Binary files /dev/null and b/media/ticdc/ticdc-dashboard-changefeed-3.png differ diff --git a/media/ticdc/ticdc-dashboard-events-1.png b/media/ticdc/ticdc-dashboard-events-1.png new file mode 100644 index 000000000000..280aa2949283 Binary files /dev/null and b/media/ticdc/ticdc-dashboard-events-1.png differ diff --git a/media/ticdc/ticdc-dashboard-events-2.png b/media/ticdc/ticdc-dashboard-events-2.png new file mode 100644 index 000000000000..2a4d8e443046 Binary files /dev/null and b/media/ticdc/ticdc-dashboard-events-2.png differ diff --git a/media/ticdc/ticdc-dashboard-events-3.png b/media/ticdc/ticdc-dashboard-events-3.png new file mode 100644 index 000000000000..de7bd82dfefc Binary files /dev/null and b/media/ticdc/ticdc-dashboard-events-3.png differ diff --git a/media/ticdc/ticdc-dashboard-overview.png b/media/ticdc/ticdc-dashboard-overview.png new file mode 100644 index 000000000000..3426319e8f00 Binary files /dev/null and b/media/ticdc/ticdc-dashboard-overview.png differ diff --git a/media/ticdc/ticdc-dashboard-server.png b/media/ticdc/ticdc-dashboard-server.png new file mode 100644 index 000000000000..fd73a4d346ea Binary files /dev/null and b/media/ticdc/ticdc-dashboard-server.png differ diff --git a/media/ticdc/ticdc-dashboard-tikv-1.png b/media/ticdc/ticdc-dashboard-tikv-1.png new file mode 100644 index 000000000000..d4646c3aaa3e Binary files /dev/null and b/media/ticdc/ticdc-dashboard-tikv-1.png differ diff --git a/media/ticdc/ticdc-dashboard-tikv-2.png b/media/ticdc/ticdc-dashboard-tikv-2.png new file mode 100644 index 000000000000..5820b38c0263 Binary files /dev/null and b/media/ticdc/ticdc-dashboard-tikv-2.png differ diff --git a/media/tidb-architecture-v3.1.png b/media/tidb-architecture-v3.1.png new file mode 100644 index 000000000000..0f3caa2e56f3 Binary files /dev/null and b/media/tidb-architecture-v3.1.png differ diff --git a/media/tidb-architecture.png b/media/tidb-architecture.png index c9f9ecbb593e..51d4f57aa7dd 100644 Binary files a/media/tidb-architecture.png and b/media/tidb-architecture.png differ diff --git a/media/tidb-lightning-architecture.png b/media/tidb-lightning-architecture.png index 9506e6f852ea..9c96dba22495 100644 Binary files a/media/tidb-lightning-architecture.png and b/media/tidb-lightning-architecture.png differ diff --git a/media/tidb-v5-tpch-100-vs-gp-spark.png b/media/tidb-v5-tpch-100-vs-gp-spark.png new file mode 100644 index 000000000000..c76f4ede8c37 Binary files /dev/null and b/media/tidb-v5-tpch-100-vs-gp-spark.png differ diff --git a/media/tpcc_v510_vs_v502.png b/media/tpcc_v510_vs_v502.png new file mode 100644 index 000000000000..f5b4ad8cb10f Binary files /dev/null and b/media/tpcc_v510_vs_v502.png differ diff --git a/media/tpcc_v5vsv4.png b/media/tpcc_v5vsv4.png new file mode 100644 index 000000000000..f47c8e2f678a Binary files /dev/null and b/media/tpcc_v5vsv4.png differ diff --git a/media/tpcc_v5vsv4_corrected.png b/media/tpcc_v5vsv4_corrected.png new file mode 100644 index 000000000000..5b253557e815 Binary files /dev/null and b/media/tpcc_v5vsv4_corrected.png differ diff --git a/media/tpcc_v5vsv4_corrected_v2.png b/media/tpcc_v5vsv4_corrected_v2.png new file mode 100644 index 000000000000..d921c6899981 Binary files /dev/null and b/media/tpcc_v5vsv4_corrected_v2.png differ diff --git a/migrate-from-aurora-mysql-database.md b/migrate-from-aurora-mysql-database.md index 8a65a21ddcc6..1560aca80571 100644 --- a/migrate-from-aurora-mysql-database.md +++ b/migrate-from-aurora-mysql-database.md @@ -1,9 +1,9 @@ --- -title: Migrate from MySQL (Amazon Aurora) +title: Migrate from Amazon Aurora MySQL Using DM summary: Learn how to migrate from MySQL (using a case of Amazon Aurora) to TiDB by using TiDB Data Migration (DM). aliases: ['/docs/dev/migrate-from-aurora-mysql-database/','/docs/dev/how-to/migrate/from-mysql-aurora/','/docs/dev/how-to/migrate/from-aurora/'] --- -# Migrate from MySQL (Amazon Aurora) +# Migrate from Amazon Aurora MySQL Using DM To migrate data from MySQL (Amazon Aurora) to TiDB, refer to [Migrate from MySQL (Amazon Aurora)](https://docs.pingcap.com/tidb-data-migration/v2.0/migrate-from-mysql-aurora). diff --git a/migrate-from-aurora-using-lightning.md b/migrate-from-aurora-using-lightning.md new file mode 100644 index 000000000000..26106d74a839 --- /dev/null +++ b/migrate-from-aurora-using-lightning.md @@ -0,0 +1,116 @@ +--- +title: Migrate from Amazon Aurora MySQL Using TiDB Lightning +summary: Learn how to migrate full data from Amazon Aurora MySQL to TiDB using TiDB Lightning. +--- + +# Migrate from Amazon Aurora MySQL Using TiDB Lightning + +This document introduces how to migrate full data from Amazon Aurora MySQL to TiDB using TiDB Lightning. + +## Step 1: Export full data from Aurora to Amazon S3 + +Refer to [AWS Documentation - Exporting DB snapshot data to Amazon S3](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_ExportSnapshot.html) to export the snapshot data of Aurora to Amazon S3. + +## Step 2: Deploy TiDB Lightning + +For detailed deployment methods, see [Deploy TiDB Lightning](/tidb-lightning/deploy-tidb-lightning.md). + +## Step 3: Configure the data source of TiDB Lightning + +Based on different deployment methods, edit the `tidb-lighting.toml` configuration file as follows: + +1. Configure `data-source-dir` under `[mydumper]` as the S3 Bucket path of exported data in [step 1](#step-1-export-full-data-from-aurora-to-amazon-s3). + + ``` + [mydumper] + # Data source directory + data-source-dir = "s3://bucket-name/data-path" + ``` + +2. Configure the target TiDB cluster as follows: + + ``` + [tidb] + # The target cluster information. Fill in one address of tidb-server. + host = "172.16.31.1" + port = 4000 + user = "root" + password = "" + # The PD address of the cluster. + pd-addr = "127.0.0.1:2379" + ``` + +3. Configure the backend mode: + + ``` + [tikv-importer] + # Uses Local-backend. + backend = "local" + # The storage path of local temporary files. Ensure that the corresponding directory does not exist or is empty and that the disk capacity is large enough for storage. + sorted-kv-dir = "/path/to/local-temp-dir" + ``` + +4. Configure the file routing. + + ``` + [mydumper] + no-schema = true + + [[mydumper.files]] + # Uses single quoted strings to avoid escaping. + pattern = '(?i)^(?:[^/]*/)*([a-z0-9_]+)\.([a-z0-9_]+)/(?:[^/]*/)*(?:[a-z0-9\-_.]+\.(parquet))$' + schema = '$1' + table = '$2' + type = '$3' + ``` + +> **Note:** +> +> - The above example uses the Local-backend for best performance. You can also choose TiDB-backend or Importer-backend according to your need. For detailed introduction of the three backend modes, see [TiDB Lightning Backends](/tidb-lightning/tidb-lightning-backends.md). +> - Because the path for exporting snapshot data from Aurora is different from the default file naming format supported by TiDB Lightning, you need to set additional file routing configuration. +> - If TLS is enabled in the target TiDB cluster, you also need to configure TLS. + +For other configurations, see [TiDB Lightning Configuration](/tidb-lightning/tidb-lightning-configuration.md). + +## Step 4: Create table schema + +Because the snapshot data exported from Aurora to S3 does not contain the SQL statement file used to create database tables, you need to manually export and import the table creation statements corresponding to the database tables into TiDB. You can use Dumpling and TiDB Lightning to create all table schemas: + +1. Use Dumpling to export table schema files: + + ``` + ./dumpling --host 127.0.0.1 --port 4000 --user root --password password --no-data --output ./schema --filter "mydb.*" + ``` + + > **Note:** + > + > - Set the parameters of the data source address and the path of output files according to your actual situation. + > - If you need to export all database tables, you do not need to set the `--filter` parameter. If you only need to export some of the database tables, configure `--filter` according to [table-filter](https://github.com/pingcap/tidb-tools/blob/master/pkg/table-filter/README.md). + +2. Use TiDB Lightning to create table schemas: + + ``` + ./tidb-lightning -config tidb-lightning.toml -d ./schema -no-schema=false + ``` + + In this example, TiDB Lightning is only used to create table schemas, so you need to execute the above command quickly. At a regular speed, ten table creation statements can be executed in one second. + +> **Note:** +> +> If the number of database tables to create is relatively small, you can manually create the corresponding databases and tables in TiDB directly, or use other tools such as mysqldump to export the schema and then import it into TiDB. + +## Step 5: Run TiDB Lightning to import data + +Run TiDB Lightning to start the import operation. If you start TiDB Lightning by using `nohup` directly in the command line, the program might exit because of the `SIGHUP` signal. Therefore, it is recommended to write `nohup` in a script. For example: + +```bash +# !/bin/bash +export AWS_ACCESS_KEY_ID=${AccessKey} +export AWS_SECRET_ACCESS_KEY=${SecretKey} +nohup ./tidb-lightning -config tidb-lightning.toml > nohup.out & +``` + +When the import operation is started, view the progress by the following two ways: + +- `grep` the keyword `progress` in logs, which is updated every 5 minutes by default. +- Access the monitoring dashboard. See [Monitor TiDB Lightning](/tidb-lightning/monitor-tidb-lightning.md) for details. diff --git a/migrate-from-mysql-dumpling-files.md b/migrate-from-mysql-dumpling-files.md index fe9f26e1a734..a15af57be99f 100644 --- a/migrate-from-mysql-dumpling-files.md +++ b/migrate-from-mysql-dumpling-files.md @@ -1,10 +1,10 @@ --- -title: Migrate Data from MySQL SQL Files +title: Migrate from MySQL SQL Files Using TiDB Lightning summary: Learn how to migrate data from MySQL SQL files to TiDB using TiDB Lightning. aliases: ['/docs/dev/migrate-from-mysql-mydumper-files/','/tidb/dev/migrate-from-mysql-mydumper-files/'] --- -# Migrate Data from MySQL SQL Files +# Migrate from MySQL SQL Files Using TiDB Lightning This document describes how to migrate data from MySQL SQL files to TiDB using TiDB Lightning. For details on how to generate MySQL SQL files, refer to [Dumpling](/dumpling-overview.md). @@ -47,6 +47,23 @@ This document takes the TiDB-backend as an example. Create the `tidb-lightning.t password = "" ``` +3. Add the necessary parameter for the backend. This document uses the TiDB-backend mode. Here, "backend" can also be set to "local" or "importer" according to your actual application scenario. For details, refer to [Backend Mode](/tidb-lightning/tidb-lightning-backends.md). + + ``` + [tikv-importer] + backend = "tidb" + ``` + +4. Add necessary parameters for importing the TiDB cluster. + + ``` + [tidb] + host = "{{tidb-host}}" + port = {{tidb-port}} + user = "{{tidb-user}}" + password = "{{tidb-password}}" + ``` + For other configurations, see [TiDB Lightning Configuration](/tidb-lightning/tidb-lightning-configuration.md). ## Step 3: Run TiDB Lightning to import data diff --git a/migration-overview.md b/migration-overview.md index 29f2a0802fa1..c9b2ecc69cb5 100644 --- a/migration-overview.md +++ b/migration-overview.md @@ -8,6 +8,10 @@ aliases: ['/docs/dev/migration-overview/'] This document describes how to migrate data to TiDB, including migrating data from MySQL and from CSV/SQL files. +## Migrate from Aurora to TiDB + +In a cloud environment, you can directly migrate full data to TiDB by exporting snapshot from Aurora. For details, see [Migrate from Amazon Aurora MySQL Using TiDB Lightning](/migrate-from-aurora-using-lightning.md). + ## Migrate from MySQL to TiDB To migrate data from MySQL to TiDB, it is recommended to use one of the following methods: diff --git a/minimal-deployment-topology.md b/minimal-deployment-topology.md index 973cf3ca98f7..c17babeb8707 100644 --- a/minimal-deployment-topology.md +++ b/minimal-deployment-topology.md @@ -22,6 +22,8 @@ This document describes the minimal deployment topology of TiDB clusters. - [The simple template for the minimal topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-mini.yaml) - [The complex template for the minimal topology](https://github.com/pingcap/docs/blob/master/config-templates/complex-mini.yaml) +For detailed descriptions of the configuration items in the above TiDB cluster topology file, see [Topology Configuration File for Deploying TiDB Using TiUP](/tiup/tiup-cluster-topology-reference.md). + > **Note:** > > - You do not need to manually create the `tidb` user in the configuration file. The TiUP cluster component automatically creates the `tidb` user on the target machines. You can customize the user, or keep the user consistent with the control machine. diff --git a/mydumper-overview.md b/mydumper-overview.md deleted file mode 100644 index 4a2d34cc9725..000000000000 --- a/mydumper-overview.md +++ /dev/null @@ -1,201 +0,0 @@ ---- -title: Mydumper Instructions -summary: Use Mydumper to export data from TiDB. -aliases: ['/docs/dev/mydumper-overview/','/docs/dev/reference/tools/mydumper/'] ---- - -# Mydumper Instructions - -> **Warning:** -> -> PingCAP previously maintained a fork of the [mydumper project](https://github.com/maxbube/mydumper) with enhancements specific to TiDB. This fork has since been replaced by [Dumpling](/dumpling-overview.md), which has been rewritten in Go, and supports more optimizations that are specific to TiDB. It is strongly recommended that you use Dumpling instead of mydumper. - -## What is Mydumper? - -[Mydumper](https://github.com/pingcap/mydumper) is a fork project optimized for TiDB. You can use this tool for logical backups of **MySQL** or **TiDB**. - -It can be [downloaded](/download-ecosystem-tools.md) as part of the Enterprise Tools package. - -### What enhancements does it contain over regular Mydumper? - -+ To ensure backup consistency for TiDB, this optimized Mydumper tool sets the value of [tidb_snapshot](/read-historical-data.md#how-tidb-reads-data-from-history-versions) to specify the point in time when the data is backed up instead of using `FLUSH TABLES WITH READ LOCK`. - -+ This tool uses the hidden `_tidb_rowid` column of TiDB to optimize the performance of concurrently exporting data from a single table. - -## Usage - -### New parameter description - -`-z` or `--tidb-snapshot`: sets the `tidb_snapshot` to be used for the backup. The default value is the current TSO (the `Position` field output from `SHOW MASTER STATUS`). Set this parameter to the TSO or a valid `datetime` such as `-z "2016-10-08 16:45:26"`. - -### Required privileges - -- SELECT -- RELOAD -- LOCK TABLES -- REPLICATION CLIENT - -### Usage example - -Execute the following command to back up data from TiDB. You can add command line parameters to the command as needed: - -{{< copyable "shell-regular" >}} - -```bash -./bin/mydumper -h 127.0.0.1 -u root -P 4000 -``` - -## Dump table data concurrently - -This section introduces the working principle and parameters of Mydumper. This section also gives an example of Mydumper command, and explains the performance evaluation and the TiDB versions that support the `_tidb_rowid` index. - -### Working principle - -Mydumper first calculates `min(_tidb_rowid)` and `max(_tidb_rowid)`, and segments the table into chunks according to the value specified by `-r`. Then, Mydumper assigns these chunks to different threads and dumps these chunks concurrently. - -### Parameters - -- `-t` or `--threads`: specifies the number of concurrent threads (`4` by default). -- `-r` or `--rows`: specifies the maximum number of rows in a chunk. If this parameter is specified, Mydumper ignores the value of `--chunk-filesize`. - -### Example - -The following is a complete Mydumper command: - -{{< copyable "shell-regular" >}} - -```shell -./bin/mydumper -h 127.0.0.1 -u root -P 4000 -r 10000 -t 4 -``` - -### Performance evaluation - -Do a performance evaluation before you perform the dump operation. Because the concurrent scanning brings pressure on the TiDB and TiKV clusters, you need to evaluate and test the impact that the dump operation might have on the database clusters and applications. - -### TiDB versions that support the `_tidb_rowid` index - -Because concurrent table data dump uses the implicit `_tidb_rowid` row of TiDB, TiDB versions that support the `_tidb_rowid` index can fully take advantage of the concurrent dump. - -The following TiDB versions supports the `_tidb_rowid` index: - -- v2.1.3 and later v2.1 versions -- v3.0 and v3.1 -- the latest unpublished version (by default) - -## FAQ - -### How to resolve the error that occurs when the `--tidb-snapshot` option is used to export data? - -In this situation, you need to add a `--skip-tz-utc` option. Otherwise, Mydumper will pre-configure the UTC time zone and convert the time zone when `tidb-snapshot` is configured, which causes this error. - -### How to determine if the Mydumper I am using is the PingCAP optimized version? - -Execute the following command： - -{{< copyable "shell-regular" >}} - -```bash -./bin/mydumper -V -``` - -If the output contains `git_hash` (`d3e6fec8b069daee772d0dbaa47579f67a5947e7` in the following example), you are using the PingCAP optimized version of Mydumper: - -``` -mydumper 0.9.5 (d3e6fec8b069daee772d0dbaa47579f67a5947e7), built against MySQL 5.7.24 -``` - -### How to resolve the "invalid mydumper files for there are no `-schema-create.sql` files found" error when using Loader to restore the data backed up by Mydumper? - -Check whether the `-T` or `--tables-list` option is used when using Mydumper to back up data. If these options are used, Mydumper does not generate a file that includes a `CREATE DATABASE` SQL statement. - -**Solution**: Create the `{schema-name}-schema-create.sql` file in the directory for data backup of Mydumper. Write "CREATE DATABASE `{schema-name}`" to the file, and then run Loader. - -### Why is the TIMESTAMP type of data exported using Mydumper inconsistent with that in the database? - -Check whether the time zone of the server that is running Mydumper is consistent with that of the database. Mydumper converts the TIMESTAMP type of data according to the time zone of its server. You can add the `--skip-tz-utc` option to disable the conversion of dates and times. - -### How to configure the `-F,--chunk-filesize` option of Mydumper? - -Mydumper splits the data of each table into multiple chunks according to the value of this option during backup. Each chunk is saved in a file with a size of about `chunk-filesize`. In this way, data is split into multiple files and you can use the parallel processing of Loader/TiDB lightning to improve the import speed. If you later use **Loader** to restore the backup files, it is recommended to set the value of this option to `64` (in MB); If you use **TiDB Lightning** to restore files, `256` (in MB) is recommended. - -### How to configure the `-s --statement-size` option of Mydumper? - -Mydumper uses this option to control the size of `Insert Statement` which defaults to `10000000` (about 1 MB). Use this option to avoid the following errors when restoring data: - -```log -packet for query is too large. Try adjusting the 'max_allowed_packet' variable -``` - -The default value meets the requirements in most cases, but **if it is a wide table, the size of a single row of data might exceed the limit of `statement-size`, and Mydumper reports the following warning**: - -```log -Row bigger than statement_size for xxx -``` - -If you restore the data in this situation, Mydumper still reports the `packet for query is too large` error. To solve this problem, modify the following two configurations (take `128 MB` as an example): - -* Execute `set @@global.max_allowed_packet=134217728` (`134217728` = `128 MB`) in TiDB server. -* Add the `max-allowed-packet=128M` line to the DB configuration of Loader or DM task's configuration file according to your situation. Then, restart the process or task. - -### How to set the `-l, --long-query-guard` option of Mydumper? - -Set the value of this option to the estimated time required for a backup. If Mydumper runs longer than this value, it reports an error and exits. It is recommended to set the value to `7200` (in seconds) for the first time of your backup and then modify it according to your actual backup time. - -### How to set the `--tidb-force-priority` option of Mydumper? - -This option can only be set when backing up TiDB’s data. It can be set to `LOW_PRIORITY`, `DELAYED`, or `HIGH_PRIORITY`. If you do not want data backup to affect online services, it is recommended to set this option to `LOW_PRIORITY`; if the backup has a higher priority, `HIGH_PRIORITY` is recommended. - -### How to resolve the "GC life time is short than transaction duration" error when using Mydumper to back up TiDB's data? - -Mydumper uses the `tidb_snapshot` system variable to ensure data consistency when backing up TiDB's data. This error is reported if the historical data of a snapshot is cleared by TiDB's Garbage Collection (GC) during backup. To solve this problem, perform the following steps: - -1. Before using Mydumper to back up data, use MySQL client to check the value of `tikv_gc_life_time` in the TiDB cluster and set it to an appropriate value: - - {{< copyable "sql" >}} - - ```sql - SELECT * FROM mysql.tidb WHERE VARIABLE_NAME = 'tikv_gc_life_time'; - ``` - - ``` - +-----------------------+------------------------------------------------------------------------------------------------+ - | VARIABLE_NAME | VARIABLE_VALUE | - +-----------------------+------------------------------------------------------------------------------------------------+ - | tikv_gc_life_time | 10m0s | - +-----------------------+------------------------------------------------------------------------------------------------+ - 1 rows in set (0.02 sec) - ``` - - {{< copyable "sql" >}} - - ```sql - update mysql.tidb set VARIABLE_VALUE = '720h' where VARIABLE_NAME = 'tikv_gc_life_time'; - ``` - -2. Set the value of `tikv_gc_life_time` to the initial one after the backup is complete: - - {{< copyable "sql" >}} - - ```sql - update mysql.tidb set VARIABLE_VALUE = '10m0s' where VARIABLE_NAME = 'tikv_gc_life_time'; - ``` - -### Do I need to configure the `--tidb-rowid` option of Mydumper? - -If this option is set to true, the exported data contains the data of TiDB's hidden columns. Using hidden columns when restoring data to TiDB might cause data inconsistency. Currently, it is not recommended to use this option. - -### How to resolve the "Segmentation Fault" error? - -This bug has been fixed. If the error persists, you can upgrade to the latest version of Mydumper. - -### How to resolve the "Error dumping table ({schema}.{table}) data: line ...... (total length ...)" error? - -This error occurs when Mydumper parses SQL statements. In this situation, use the latest version of Mydumper. If this error persists, you can file an issue to [mydumper/issues](https://github.com/pingcap/mydumper/issues). - -### How to resolve the "Failed to set tidb_snapshot: parsing time \"20190901-10:15:00 +0800\" as \"20190901-10:15:00 +0700 MST\": cannot parse \"\" as \"MST\"" error? - -Check whether the version of TiDB is lower than 2.1.11. If so, upgrade to TiDB 2.1.11 or later versions. - -### Do you plan to make these changes available to upstream Mydumper? - -Yes, we intend to make our changes available to upstream Mydumper. See [PR #155](https://github.com/maxbube/mydumper/pull/155). diff --git a/mysql-compatibility.md b/mysql-compatibility.md index 04bb79141156..5505f1e9560d 100644 --- a/mysql-compatibility.md +++ b/mysql-compatibility.md @@ -10,6 +10,10 @@ TiDB is fully compatible with the MySQL 5.7 protocol and the common features and However, some features of MySQL are not supported. This could be because there is now a better way to solve the problem (such as XML functions superseded by JSON), or a lack of current demand versus effort required (such as stored procedures and functions). Some features might also be difficult to implement as a distributed system. +- In addition, TiDB does not support the MySQL replication protocol, but provides specific tools to replicate data with MySQL. + - Replicate data from MySQL: [TiDB Data Migration (DM)](https://docs.pingcap.com/tidb-data-migration/stable/overview) is a tool that supports the full data migration and the incremental data replication from MySQL/MariaDB into TiDB. + - Replicate data to MySQL: [TiCDC](/ticdc/ticdc-overview.md) is a tool for replicating the incremental data of TiDB by pulling TiKV change logs. TiCDC uses the [MySQL sink](/ticdc/ticdc-overview.md#sink-support) to replicate the incremental data of TiDB to MySQL. + > **Note:** > > This page refers to general differences between MySQL and TiDB. Refer to the dedicated pages for [Security](/security-compatibility-with-mysql.md) and [Pessimistic Transaction Model](/pessimistic-transaction.md#difference-with-mysql-innodb) compatibility. @@ -35,6 +39,7 @@ However, some features of MySQL are not supported. This could be because there i + `CHECK TABLE` syntax [#4673](https://github.com/pingcap/tidb/issues/4673) + `CHECKSUM TABLE` syntax [#1895](https://github.com/pingcap/tidb/issues/1895) + `GET_LOCK` and `RELEASE_LOCK` functions [#14994](https://github.com/pingcap/tidb/issues/14994) ++ [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) with the `REPLACE` keyword [#24515](https://github.com/pingcap/tidb/issues/24515) ## Features that are different from MySQL @@ -50,8 +55,6 @@ However, some features of MySQL are not supported. This could be because there i > **Note:** > -> + To use the `tidb_allow_remove_auto_inc` system variable, your TiDB version must be >= v2.1.18 or >= v3.0.4. -> + The `AUTO_ID_CACHE` table attribute requires that your TiDB version >= v3.0.14 or >= v3.1.2 or >= v4.0.0-rc.2. > + If you have not specified the primary key when creating a table, TiDB uses `_tidb_rowid` to identify the row. The allocation of this value shares an allocator with the auto-increment column (if such a column exists). If you specify an auto-increment column as the primary key, TiDB uses this column to identify the row. In this situation, the following situation might happen: ```sql @@ -92,13 +95,10 @@ See also: [TiDB SQL Grammar](https://pingcap.github.io/sqlgram/#functioncallkeyw In TiDB, all supported DDL changes are performed online. Compared with DDL operations in MySQL, the DDL operations in TiDB have the following major restrictions: * Multiple operations cannot be completed in a single `ALTER TABLE` statement. For example, it is not possible to add multiple columns or indexes in a single statement. Otherwise, the `Unsupported multi schema change` error might be output. -* Different types of indexes (`HASH|BTREE|RTREE|FULLTEXT`) are not supported, and will be parsed and ignored when specified. -* Adding/Dropping the primary key is unsupported unless [`alter-primary-key`](/tidb-configuration-file.md#alter-primary-key) is enabled. -* Changing the field type to its superset is unsupported. For example, TiDB does not support changing the field type from `INTEGER` to `VARCHAR`, or from `TIMESTAMP` to `DATETIME`. Otherwise, the error information `Unsupported modify column: type %d not match origin %d` might be output. -* Change/Modify data type does not currently support "lossy changes", such as changing from BIGINT to INT. -* Change/Modify decimal columns does not support changing the precision. -* Change/Modify integer columns does not permit changing the `UNSIGNED` attribute. +* `ALTER TABLE` in TiDB does not support the changes of some data types. For example, TiDB does not support the change from the `DECIMAL` type to the `DATE` type. If a data type change is unsupported, TiDB reports the `Unsupported modify column: type %d not match origin %d` error. Refer to [`ALTER TABLE`](/sql-statements/sql-statement-modify-column.md) for more details. * The `ALGORITHM={INSTANT,INPLACE,COPY}` syntax functions only as an assertion in TiDB, and does not modify the `ALTER` algorithm. See [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) for further details. +* Adding/Dropping the primary key of the `CLUSTERED` type is unsupported. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). +* Different types of indexes (`HASH|BTREE|RTREE|FULLTEXT`) are not supported, and will be parsed and ignored when specified. * Table Partitioning supports Hash, Range, and `Add`/`Drop`/`Truncate`/`Coalesce`. The other partition operations are ignored. The `Warning: Unsupported partition type, treat as normal table` error might be output. The following Table Partition syntaxes are not supported: - `PARTITION BY LIST` - `PARTITION BY KEY` @@ -178,3 +178,10 @@ The following column types are supported by MySQL, but **NOT** by TiDB: + FLOAT4/FLOAT8 + `SQL_TSI_*` (including SQL_TSI_MONTH, SQL_TSI_WEEK, SQL_TSI_DAY, SQL_TSI_HOUR, SQL_TSI_MINUTE and SQL_TSI_SECOND, excluding SQL_TSI_YEAR) + +### Incompatibility caused by deprecated features + +TiDB does not implement certain features that have been marked as deprecated in MySQL, including: + +* Specifying precision for floating point types. MySQL 8.0 [deprecates](https://dev.mysql.com/doc/refman/8.0/en/floating-point-types.html) this feature, and it is recommended to use the `DECIMAL` type instead. +* The `ZEROFILL` attribute. MySQL 8.0 [deprecates](https://dev.mysql.com/doc/refman/8.0/en/numeric-type-attributes.html) this feature, and it is recommended to instead pad numeric values in your application. diff --git a/offline-deployment-using-ansible.md b/offline-deployment-using-ansible.md deleted file mode 100644 index 341158cf0c9d..000000000000 --- a/offline-deployment-using-ansible.md +++ /dev/null @@ -1,186 +0,0 @@ ---- -title: Deploy TiDB Offline Using TiDB Ansible -summary: Use TiDB Ansible to deploy a TiDB cluster offline. -aliases: ['/docs/dev/offline-deployment-using-ansible/','/docs/dev/how-to/deploy/orchestrated/offline-ansible/'] ---- - -# Deploy TiDB Offline Using TiDB Ansible - -> **Warning:** -> -> For production environments, it is recommended that you [deploy TiDB using TiUP offline](/production-offline-deployment-using-tiup.md). Since v4.0, PingCAP no longer provides support for deploying TiDB using TiDB Ansible (deprecated). If you really need to use it for deployment, be aware of any risk. You can [import the TiDB cluster deployed by TiDB Ansible to TiUP](/upgrade-tidb-using-tiup.md#import-the-tidb-ansible-cluster-to-tiup). -> -> If you only want to try out TiDB and explore new features, refer to [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md). - -This guide describes how to deploy a TiDB cluster offline using TiDB Ansible. - -## Prepare - -Before you start, make sure that you have: - -1. A download machine - - - The machine must have access to the Internet in order to download TiDB Ansible, TiDB and related packages. - - For Linux operating system, it is recommended to install CentOS 7.3 or later. - -2. Several target machines and one control machine - - - For system requirements and configuration, see [Prepare the environment](/online-deployment-using-ansible.md#prepare). - - It is acceptable without access to the Internet. - -## Step 1: Install system dependencies on the control machine - -Take the following steps to install system dependencies on the control machine installed with the CentOS 7 system. - -1. Download the [`pip`](https://download.pingcap.org/ansible-system-rpms.el7.tar.gz) offline installation package on the download machine and then upload it to the control machine. - - > **Note:** - > - > This offline installation package includes `pip` and `sshpass`, and only supports the CentOS 7 system. - -2. Install system dependencies on the control machine. - - {{< copyable "shell-root" >}} - - ```bash - tar -xzvf ansible-system-rpms.el7.tar.gz && - cd ansible-system-rpms.el7 && - chmod u+x install_ansible_system_rpms.sh && - ./install_ansible_system_rpms.sh - ``` - -3. After the installation is finished, you can use `pip -V` to check whether it is successfully installed. - - {{< copyable "shell-root" >}} - - ```bash - pip -V - ``` - - ``` - pip 8.1.2 from /usr/lib/python2.7/site-packages (python 2.7) - ``` - -> **Note:** -> -> If `pip` is already installed to your system, make sure that the version is 8.1.2 or later. Otherwise, compatibility error occurs when you install TiDB Ansible and its dependencies offline. - -## Step 2: Create the `tidb` user on the control machine and generate the SSH key - -See [Create the `tidb` user on the control machine and generate the SSH key](/online-deployment-using-ansible.md#step-2-create-the-tidb-user-on-the-control-machine-and-generate-the-ssh-key). - -## Step 3: Install TiDB Ansible and its dependencies offline on the control machine - -Currently, all the versions of TiDB Ansible from 2.4 to 2.7.11 are supported. The versions of TiDB Ansible and the related dependencies are listed in the `tidb-ansible/requirements.txt` file. The following installation steps take Ansible 2.5 as an example. - -1. Download [Ansible 2.5 offline installation package](https://download.pingcap.org/ansible-2.5.0-pip.tar.gz) on the download machine and then upload it to the control machine. - -2. Install TiDB Ansible and its dependencies offline. - - {{< copyable "shell-root" >}} - - ```bash - tar -xzvf ansible-2.5.0-pip.tar.gz && - cd ansible-2.5.0-pip/ && - chmod u+x install_ansible.sh && - ./install_ansible.sh - ``` - -3. View the version of TiDB Ansible. - - After TiDB Ansible is installed, you can view the version using `ansible --version`. - - {{< copyable "shell-root" >}} - - ```bash - ansible --version - ``` - - ``` - ansible 2.5.0 - ``` - -## Step 4: Download TiDB Ansible and TiDB packages on the download machine - -1. Install TiDB Ansible on the download machine. - - Use the following method to install Ansible online on the download machine installed with the CentOS 7 system. After TiDB Ansible is installed, you can view the version using `ansible --version`. - - {{< copyable "shell-root" >}} - - ```bash - yum install epel-release && - yum install ansible curl && - ansible --version - ``` - - ``` - ansible 2.5.0 - ``` - - > **Note:** - > - > Make sure that the version of Ansible is 2.5, otherwise a compatibility issue occurs. - -2. Download TiDB Ansible. - - Use the following command to download the corresponding version of TiDB Ansible from the [TiDB Ansible project](https://github.com/pingcap/tidb-ansible) GitHub repo. The default folder name is `tidb-ansible`. - - {{< copyable "shell-regular" >}} - - ```bash - git clone https://github.com/pingcap/tidb-ansible.git - ``` - - > **Note:** - > - > It is required to use the corresponding tidb-ansible version when you deploy and upgrade the TiDB cluster. If you deploy TiDB using a mismatched version of tidb-ansible (such as using tidb-ansible v2.1.4 to deploy TiDB v2.1.6), an error might occur. - -3. Run the `local_prepare.yml` playbook, and download TiDB binary online to the download machine. - - {{< copyable "shell-regular" >}} - - ```bash - cd tidb-ansible && - ansible-playbook local_prepare.yml - ``` - -4. After running the above command, copy the `tidb-ansible` folder to the `/home/tidb` directory of the control machine. The ownership authority of the file must be the `tidb` user. - -## Step 5: Configure the SSH mutual trust and sudo rules on the control machine - -See [Configure the SSH mutual trust and sudo rules on the control machine](/online-deployment-using-ansible.md#step-5-configure-the-ssh-mutual-trust-and-sudo-rules-on-the-control-machine). - -## Step 6: Install the NTP service on the target machines - -See [Install the NTP service on the target machines](/online-deployment-using-ansible.md#step-6-install-the-ntp-service-on-the-target-machines). - -> **Note:** -> -> If the time and time zone of all your target machines are same, the NTP service is on and is normally synchronizing time, you can skip this step. See [How to check whether the NTP service is normal](/online-deployment-using-ansible.md#how-to-check-whether-the-ntp-service-is-normal). - -## Step 7: Configure the CPUfreq governor mode on the target machine - -See [Configure the CPUfreq governor mode on the target machine](/online-deployment-using-ansible.md#step-7-configure-the-cpufreq-governor-mode-on-the-target-machine). - -## Step 8: Mount the data disk ext4 filesystem with options on the target machines - -See [Mount the data disk ext4 filesystem with options on the target machines](/online-deployment-using-ansible.md#step-8-mount-the-data-disk-ext4-filesystem-with-options-on-the-target-machines). - -## Step 9: Edit the `inventory.ini` file to orchestrate the TiDB cluster - -See [Edit the `inventory.ini` file to orchestrate the TiDB cluster](/online-deployment-using-ansible.md#step-9-edit-the-inventoryini-file-to-orchestrate-the-tidb-cluster). - -## Step 10: Deploy the TiDB cluster - -1. You do not need to run the playbook in `ansible-playbook local_prepare.yml`. - -2. See [Deploy the TiDB cluster](/online-deployment-using-ansible.md#step-11-deploy-the-tidb-cluster). - -## Test the TiDB cluster - -See [Test the TiDB cluster](/online-deployment-using-ansible.md#test-the-tidb-cluster). - -> **Note:** -> -> By default, TiDB periodically shares usage details with PingCAP to help understand how to improve the product. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). diff --git a/online-deployment-using-ansible.md b/online-deployment-using-ansible.md deleted file mode 100644 index 30d3121d5425..000000000000 --- a/online-deployment-using-ansible.md +++ /dev/null @@ -1,992 +0,0 @@ ---- -title: Deploy TiDB Using TiDB Ansible -summary: Use TiDB Ansible to deploy a TiDB cluster. -aliases: ['/docs/dev/online-deployment-using-ansible/','/docs/dev/how-to/deploy/orchestrated/ansible/'] ---- - -# Deploy TiDB Using TiDB Ansible - -> **Warning:** -> -> For production environments, it is recommended that you [deploy TiDB using TiUP](/production-deployment-using-tiup.md). Since v4.0, PingCAP no longer provides support for deploying TiDB using TiDB Ansible (deprecated). If you really need to use it for deployment, be aware of any risk. You can [import the TiDB cluster deployed by TiDB Ansible to TiUP](/upgrade-tidb-using-tiup.md#import-the-tidb-ansible-cluster-to-tiup). -> -> If you only want to try out TiDB and explore new features, refer to [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md). - -This guide describes how to deploy a TiDB cluster using TiDB Ansible. For the production environment, it is recommended to deploy TiDB using TiUP. - -## Overview - -Ansible is an IT automation tool that can configure systems, deploy software, and orchestrate more advanced IT tasks such as continuous deployments or zero downtime rolling updates. - -[TiDB Ansible](https://github.com/pingcap/tidb-ansible) is a TiDB cluster deployment tool developed by PingCAP, based on Ansible playbook. TiDB Ansible enables you to quickly deploy a new TiDB cluster which includes PD, TiDB, TiKV, and the cluster monitoring modules. - -You can use the TiDB Ansible configuration file to set up the cluster topology and complete all the following operation tasks: - -- Initialize operating system parameters -- Deploy the whole TiDB cluster -- [Start the TiDB cluster](/maintain-tidb-using-ansible.md#start-a-cluster) -- [Stop the TiDB cluster](/maintain-tidb-using-ansible.md#stop-a-cluster) -- [Modify component configuration](/upgrade-tidb-using-ansible.md#edit-the-configuration-file-of-tidb-cluster-components) -- [Scale the TiDB cluster](/scale-tidb-using-ansible.md) -- [Upgrade the component version](/upgrade-tidb-using-ansible.md#step-5-perform-a-rolling-update-to-tidb-cluster-components) -- [Enable the cluster binlog](/tidb-binlog/tidb-binlog-overview.md) -- [Clean up data of the TiDB cluster](/maintain-tidb-using-ansible.md#clean-up-cluster-data) -- [Destroy the TiDB cluster](/maintain-tidb-using-ansible.md#destroy-a-cluster) - -## Prepare - -Before you start, make sure you have: - -1. Several target machines that meet the following requirements: - - - 4 or more machines - - A standard TiDB cluster contains 6 machines. You can use 4 machines for testing. For more details, see [Software and Hardware Recommendations](/hardware-and-software-requirements.md). - - - x86_64 architecture (AMD64) with CentOS 7.3 (64 bit) or later; Or x86_64 architecture (ARM64) with CentOS 7.6 1810 or later. - - Network between machines - - > **Note:** - > - > When you deploy TiDB using TiDB Ansible, **use SSD disks for the data directory of TiKV and PD nodes**. Otherwise, it cannot pass the check. If you only want to try TiDB out and explore the features, it is recommended to [deploy TiDB using Docker Compose](/deploy-test-cluster-using-docker-compose.md) on a single machine. - -2. A control machine that meets the following requirements: - - > **Note:** - > - > The control machine can be one of the target machines. - - - CentOS 7.3 (64 bit) or later with Python 2.7 installed - - Access to the Internet - -## Step 1: Install system dependencies on the control machine - -Log in to the control machine using the `root` user account, and run the corresponding command according to your operating system. - -- If you use a control machine installed with CentOS 7, run the following command: - - {{< copyable "shell-root" >}} - - ```shell - yum -y install epel-release git curl sshpass && \ - yum -y install python2-pip - ``` - -- If you use a control machine installed with Ubuntu, run the following command: - - {{< copyable "shell-root" >}} - - ```shell - apt-get -y install git curl sshpass python-pip - ``` - -## Step 2: Create the `tidb` user on the control machine and generate the SSH key - -Make sure you have logged in to the control machine using the `root` user account, and then run the following command. - -1. Create the `tidb` user. - - {{< copyable "shell-root" >}} - - ```shell - useradd -m -d /home/tidb tidb - ``` - -2. Set a password for the `tidb` user account. - - {{< copyable "shell-root" >}} - - ```shell - passwd tidb - ``` - -3. Configure sudo without password for the `tidb` user account by adding `tidb ALL=(ALL) NOPASSWD: ALL` to the end of the sudo file: - - {{< copyable "shell-root" >}} - - ```shell - visudo - ``` - - ``` - tidb ALL=(ALL) NOPASSWD: ALL - ``` - -4. Generate the SSH key. - - Execute the `su` command to switch the user from `root` to `tidb`. - - {{< copyable "shell-root" >}} - - ```shell - su - tidb - ``` - - Create the SSH key for the `tidb` user account and hit the Enter key when `Enter passphrase` is prompted. After successful execution, the SSH private key file is `/home/tidb/.ssh/id_rsa`, and the SSH public key file is `/home/tidb/.ssh/id_rsa.pub`. - - {{< copyable "shell-regular" >}} - - ```shell - ssh-keygen -t rsa - ``` - - ``` - Generating public/private rsa key pair. - Enter file in which to save the key (/home/tidb/.ssh/id_rsa): - Created directory '/home/tidb/.ssh'. - Enter passphrase (empty for no passphrase): - Enter same passphrase again: - Your identification has been saved in /home/tidb/.ssh/id_rsa. - Your public key has been saved in /home/tidb/.ssh/id_rsa.pub. - The key fingerprint is: - SHA256:eIBykszR1KyECA/h0d7PRKz4fhAeli7IrVphhte7/So tidb@172.16.10.49 - The key's randomart image is: - +---[RSA 2048]----+ - |=+o+.o. | - |o=o+o.oo | - | .O.=.= | - | . B.B + | - |o B * B S | - | * + * + | - | o + . | - | o E+ . | - |o ..+o. | - +----[SHA256]-----+ - ``` - -## Step 3: Download TiDB Ansible to the control machine - -Log in to the control machine using the `tidb` user account and enter the `/home/tidb` directory. Run the following command to download TiDB Ansible from the master branch of the [TiDB Ansible project](https://github.com/pingcap/tidb-ansible). The default folder name is `tidb-ansible`. - -{{< copyable "shell-regular" >}} - -```shell -git clone https://github.com/pingcap/tidb-ansible.git -``` - -> **Note:** -> -> - To deploy and upgrade TiDB clusters, use the corresponding version of `tidb-ansible`. If you only modify the version in the `inventory.ini` file, errors might occur. -> - It is required to download `tidb-ansible` to the `/home/tidb` directory using the `tidb` user account. If you download it to the `/root` directory, a privilege issue occurs. - -If you have questions regarding which version to use, email to info@pingcap.com for more information or [file an issue](https://github.com/pingcap/tidb-ansible/issues/new). - -## Step 4: Install TiDB Ansible and its dependencies on the control machine - -Make sure you have logged in to the control machine using the `tidb` user account. - -It is required to use `pip` to install Ansible and its dependencies, otherwise a compatibility issue occurs. Currently, the release-2.0, release-2.1, release-3.1, and master branches of TiDB Ansible are compatible with Ansible 2.4 ~ 2.7.11 (2.4 ≤ Ansible ≤ 2.7.11). - -1. Install TiDB Ansible and the dependencies on the control machine: - - {{< copyable "shell-regular" >}} - - ```bash - cd /home/tidb/tidb-ansible && \ - sudo pip install -r ./requirements.txt - ``` - - The version information of Ansible and dependencies is in the `tidb-ansible/requirements.txt` file. - -2. View the version of TiDB Ansible: - - {{< copyable "shell-regular" >}} - - ```bash - ansible --version - ``` - - ``` - ansible 2.7.11 - ``` - -## Step 5: Configure the SSH mutual trust and sudo rules on the control machine - -Make sure you have logged in to the control machine using the `tidb` user account. - -1. Add the IPs of your target machines to the `[servers]` section of the `hosts.ini` file. - - {{< copyable "shell-regular" >}} - - ```bash - cd /home/tidb/tidb-ansible && \ - vi hosts.ini - ``` - - ``` - [servers] - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - 172.16.10.4 - 172.16.10.5 - 172.16.10.6 - - [all:vars] - username = tidb - ntp_server = pool.ntp.org - ``` - -2. Run the following command and input the `root` user account password of your target machines. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook -i hosts.ini create_users.yml -u root -k - ``` - - This step creates the `tidb` user account on the target machines, configures the sudo rules and the SSH mutual trust between the control machine and the target machines. - -To configure the SSH mutual trust and sudo without password manually, see [How to manually configure the SSH mutual trust and sudo without password](#how-to-manually-configure-the-ssh-mutual-trust-and-sudo-without-password). - -## Step 6: Install the NTP service on the target machines - -> **Note:** -> -> If the time and time zone of all your target machines are same, the NTP service is on and is normally synchronizing time, you can ignore this step. See [How to check whether the NTP service is normal](#how-to-check-whether-the-ntp-service-is-normal). - -Make sure you have logged in to the control machine using the `tidb` user account, run the following command: - -{{< copyable "shell-regular" >}} - -```bash -cd /home/tidb/tidb-ansible && \ -ansible-playbook -i hosts.ini deploy_ntp.yml -u tidb -b -``` - -The NTP service is installed and started using the software repository that comes with the system on the target machines. The default NTP server list in the installation package is used. The related `server` parameter is in the `/etc/ntp.conf` configuration file. - -To make the NTP service start synchronizing as soon as possible, the system executes the `ntpdate` command to set the local date and time by polling `ntp_server` in the `hosts.ini` file. The default server is `pool.ntp.org`, and you can also replace it with your NTP server. - -## Step 7: Configure the CPUfreq governor mode on the target machine - -For details about CPUfreq, see [the CPUfreq Governor documentation](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/power_management_guide/cpufreq_governors). - -Set the CPUfreq governor mode to `performance` to make full use of CPU performance. - -### Check the governor modes supported by the system - -You can run the `cpupower frequency-info --governors` command to check the governor modes which the system supports: - -{{< copyable "shell-root" >}} - -```bash -cpupower frequency-info --governors -``` - -``` -analyzing CPU 0: - available cpufreq governors: performance powersave -``` - -Taking the above code for example, the system supports the `performance` and `powersave` modes. - -> **Note:** -> -> As the following shows, if it returns `Not Available`, it means that the current system does not support CPUfreq configuration and you can skip this step. - -{{< copyable "shell-root" >}} - -```bash -cpupower frequency-info --governors -``` - -``` -analyzing CPU 0: - available cpufreq governors: Not Available -``` - -### Check the current governor mode - -You can run the `cpupower frequency-info --policy` command to check the current CPUfreq governor mode: - -{{< copyable "shell-root" >}} - -```bash -cpupower frequency-info --policy -``` - -``` -analyzing CPU 0: - current policy: frequency should be within 1.20 GHz and 3.20 GHz. - The governor "powersave" may decide which speed to use - within this range. -``` - -As the above code shows, the current mode is `powersave` in this example. - -### Change the governor mode - -You can use either of the following two methods to change the governor mode. In the above example, the current governor mode is `powersave` and the following commands change it to `performance`. - -- Use the `cpupower frequency-set --governor` command to change the current mode: - - {{< copyable "shell-root" >}} - - ```shell - cpupower frequency-set --governor performance - ``` - -- Run the following command to set the mode on the target machine in batches: - - {{< copyable "shell-regular" >}} - - ```shell - ansible -i hosts.ini all -m shell -a "cpupower frequency-set --governor performance" -u tidb -b - ``` - -## Step 8: Mount the data disk ext4 filesystem with options on the target machines - -Log in to the target machines using the `root` user account. - -Format your data disks to the ext4 filesystem and add the `nodelalloc` and `noatime` mount options to the filesystem. It is required to add the `nodelalloc` option, or else the Ansible deployment cannot pass the test. The `noatime` option is optional. - -> **Note:** -> -> If your data disks have been formatted to ext4 and have added the mount options, you can uninstall it by running the `umount /dev/nvme0n1p1` command, follow the steps starting from editing the `/etc/fstab` file, and add the options again to the filesystem. - -Take the `/dev/nvme0n1` data disk as an example: - -1. View the data disk. - - {{< copyable "shell-root" >}} - - ```shell - fdisk -l - ``` - - ``` - Disk /dev/nvme0n1: 1000 GB - ``` - -2. Create the partition table. - - {{< copyable "shell-root" >}} - - ```shell - parted -s -a optimal /dev/nvme0n1 mklabel gpt -- mkpart primary ext4 1 -1 - ``` - - > **Note:** - > - > Use the `lsblk` command to view the device number of the partition: for a nvme disk, the generated device number is usually `nvme0n1p1`; for a regular disk (for example, `/dev/sdb`), the generated device number is usually `sdb1`. - -3. Format the data disk to the ext4 filesystem. - - {{< copyable "shell-root" >}} - - ```shell - mkfs.ext4 /dev/nvme0n1p1 - ``` - -4. View the partition UUID of the data disk. - - In this example, the UUID of `nvme0n1p1` is `c51eb23b-195c-4061-92a9-3fad812cc12f`. - - {{< copyable "shell-root" >}} - - ```shell - lsblk -f - ``` - - ``` - NAME FSTYPE LABEL UUID MOUNTPOINT - sda - ├─sda1 ext4 237b634b-a565-477b-8371-6dff0c41f5ab /boot - ├─sda2 swap f414c5c0-f823-4bb1-8fdf-e531173a72ed - └─sda3 ext4 547909c1-398d-4696-94c6-03e43e317b60 / - sr0 - nvme0n1 - └─nvme0n1p1 ext4 c51eb23b-195c-4061-92a9-3fad812cc12f - ``` - -5. Edit the `/etc/fstab` file and add the mount options. - - {{< copyable "shell-root" >}} - - ```shell - vi /etc/fstab - ``` - - ``` - UUID=c51eb23b-195c-4061-92a9-3fad812cc12f /data1 ext4 defaults,nodelalloc,noatime 0 2 - ``` - -6. Mount the data disk. - - {{< copyable "shell-root" >}} - - ```shell - mkdir /data1 && \ - mount -a - ``` - -7. Check using the following command. - - {{< copyable "shell-root" >}} - - ```shell - mount -t ext4 - ``` - - ``` - /dev/nvme0n1p1 on /data1 type ext4 (rw,noatime,nodelalloc,data=ordered) - ``` - - If the filesystem is ext4 and `nodelalloc` is included in the mount options, you have successfully mount the data disk ext4 filesystem with options on the target machines. - -## Step 9: Edit the `inventory.ini` file to orchestrate the TiDB cluster - -Log in to the control machine using the `tidb` user account, and edit the `tidb-ansible/inventory.ini` file to orchestrate the TiDB cluster. The standard TiDB cluster contains 6 machines: 2 TiDB instances, 3 PD instances, and 3 TiKV instances. - -- Deploy at least 3 instances for TiKV. -- Do not deploy TiKV together with TiDB or PD on the same machine. -- Use the first TiDB machine as the monitoring machine. - -> **Note:** -> -> It is required to use the internal IP address to deploy. If the SSH port of the target machines is not the default 22 port, you need to add the `ansible_port` variable. For example, `TiDB1 ansible_host=172.16.10.1 ansible_port=5555`. - -You can choose one of the following two types of cluster topology according to your scenario: - -- [The cluster topology of a single TiKV instance on each TiKV node](#option-1-use-the-cluster-topology-of-a-single-tikv-instance-on-each-tikv-node) - - In most cases, it is recommended to deploy one TiKV instance on each TiKV node for better performance. However, if the CPU and memory of your TiKV machines are much better than the required in [Hardware and Software Requirements](/hardware-and-software-requirements.md), and you have more than two disks in one node or the capacity of one SSD is larger than 2 TB, you can deploy no more than 2 TiKV instances on a single TiKV node. - -- [The cluster topology of multiple TiKV instances on each TiKV node](#option-2-use-the-cluster-topology-of-multiple-tikv-instances-on-each-tikv-node) - -### Option 1: Use the cluster topology of a single TiKV instance on each TiKV node - -| Name | Host IP | Services | -|:------|:------------|:-----------| -| node1 | 172.16.10.1 | PD1, TiDB1 | -| node2 | 172.16.10.2 | PD2, TiDB2 | -| node3 | 172.16.10.3 | PD3 | -| node4 | 172.16.10.4 | TiKV1 | -| node5 | 172.16.10.5 | TiKV2 | -| node6 | 172.16.10.6 | TiKV3 | - -```ini -[tidb_servers] -172.16.10.1 -172.16.10.2 - -[pd_servers] -172.16.10.1 -172.16.10.2 -172.16.10.3 - -[tikv_servers] -172.16.10.4 -172.16.10.5 -172.16.10.6 - -[monitoring_servers] -172.16.10.1 - -[grafana_servers] -172.16.10.1 - -[monitored_servers] -172.16.10.1 -172.16.10.2 -172.16.10.3 -172.16.10.4 -172.16.10.5 -172.16.10.6 -``` - -### Option 2: Use the cluster topology of multiple TiKV instances on each TiKV node - -Take two TiKV instances on each TiKV node as an example: - -| Name | Host IP | Services | -|:------|:------------|:-----------| -| node1 | 172.16.10.1 | PD1, TiDB1 | -| node2 | 172.16.10.2 | PD2, TiDB2 | -| node3 | 172.16.10.3 | PD3 | -| node4 | 172.16.10.4 | TiKV1-1, TiKV1-2 | -| node5 | 172.16.10.5 | TiKV2-1, TiKV2-2 | -| node6 | 172.16.10.6 | TiKV3-1, TiKV3-2 | - -```ini -[tidb_servers] -172.16.10.1 -172.16.10.2 - -[pd_servers] -172.16.10.1 -172.16.10.2 -172.16.10.3 - -# Note: To use labels in TiKV, you must also configure location_labels for PD at the same time. -[tikv_servers] -TiKV1-1 ansible_host=172.16.10.4 deploy_dir=/data1/deploy tikv_port=20171 tikv_status_port=20181 labels="host=tikv1" -TiKV1-2 ansible_host=172.16.10.4 deploy_dir=/data2/deploy tikv_port=20172 tikv_status_port=20182 labels="host=tikv1" -TiKV2-1 ansible_host=172.16.10.5 deploy_dir=/data1/deploy tikv_port=20171 tikv_status_port=20181 labels="host=tikv2" -TiKV2-2 ansible_host=172.16.10.5 deploy_dir=/data2/deploy tikv_port=20172 tikv_status_port=20182 labels="host=tikv2" -TiKV3-1 ansible_host=172.16.10.6 deploy_dir=/data1/deploy tikv_port=20171 tikv_status_port=20181 labels="host=tikv3" -TiKV3-2 ansible_host=172.16.10.6 deploy_dir=/data2/deploy tikv_port=20172 tikv_status_port=20182 labels="host=tikv3" - -[monitoring_servers] -172.16.10.1 - -[grafana_servers] -172.16.10.1 - -[monitored_servers] -172.16.10.1 -172.16.10.2 -172.16.10.3 -172.16.10.4 -172.16.10.5 -172.16.10.6 - -...... - -# Note: For labels in TiKV to work, you must also configure location_labels for PD when deploying the cluster. -[pd_servers:vars] -location_labels = ["host"] -``` - -**Edit the parameters in the service configuration file:** - -1. For the cluster topology of multiple TiKV instances on each TiKV node, you need to edit the `capacity` parameter under `block-cache-size` in `tidb-ansible/conf/tikv.yml`: - - ```yaml - storage: - block-cache: - capacity: "1GB" - ``` - - > **Note:** - > - > + The number of TiKV instances is the number of TiKV processes on each server. - > + Recommended configuration: `capacity` = MEM_TOTAL \* 0.5 / the number of TiKV instances - -2. For the cluster topology of multiple TiKV instances on each TiKV node, you need to edit the `high-concurrency`, `normal-concurrency` and `low-concurrency` parameters in the `tidb-ansible/conf/tikv.yml` file: - - ```yaml - readpool: - coprocessor: - # Notice: if CPU_NUM > 8, default thread pool size for coprocessors - # will be set to CPU_NUM * 0.8. - # high-concurrency: 8 - # normal-concurrency: 8 - # low-concurrency: 8 - ``` - - > **Note:** - > - > Recommended configuration: the number of TiKV instances \* the parameter value = the number of CPU cores \* 0.8. - -3. If multiple TiKV instances are deployed on a same physical disk, edit the `capacity` parameter in `conf/tikv.yml`: - - ```yaml - raftstore: - capacity: 0 - ``` - - > **Note:** - > - > Recommended configuration: `capacity` = total disk capacity / the number of TiKV instances. For example, `capacity: "100GB"`. - -## Step 10: Edit variables in the `inventory.ini` file - -This step describes how to edit the variable of deployment directory and other variables in the `inventory.ini` file. - -### Configure the deployment directory - -Edit the `deploy_dir` variable to configure the deployment directory. - -The global variable is set to `/home/tidb/deploy` by default, and it applies to all services. If the data disk is mounted on the `/data1` directory, you can set it to `/data1/deploy`. For example: - -```ini -## Global variables -[all:vars] -deploy_dir = /data1/deploy -``` - -> **Note:** -> -> To separately set the deployment directory for a service, you can configure the host variable while configuring the service host list in the `inventory.ini` file. It is required to add the first column alias, to avoid confusion in scenarios of mixed services deployment. - -```ini -TiKV1-1 ansible_host=172.16.10.4 deploy_dir=/data1/deploy -``` - -### Edit other variables (Optional) - -To enable the following control variables, use the capitalized `True`. To disable the following control variables, use the capitalized `False`. - -| Variable Name | Description | -| :---- | :------- | -| cluster_name | the name of a cluster, adjustable | -| cpu_architecture | CPU architecture. `amd64` by default, `arm64` optional | -| tidb_version | the version of TiDB, configured by default in TiDB Ansible branches | -| process_supervision | the supervision way of processes, `systemd` by default, `supervise` optional | -| timezone | the global default time zone configured when a new TiDB cluster bootstrap is initialized; you can edit it later using the global `time_zone` system variable and the session `time_zone` system variable as described in [Time Zone Support](/configure-time-zone.md); the default value is `Asia/Shanghai` and see [the list of time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) for more optional values | -| enable_firewalld | to enable the firewall, closed by default; to enable it, add the ports in [network requirements](/hardware-and-software-requirements.md#network-requirements) to the allowlist | -| enable_ntpd | to monitor the NTP service of the managed node, True by default; do not close it | -| set_hostname | to edit the hostname of the managed node based on the IP, False by default | -| enable_binlog | whether to deploy Pump and enable the binlog, False by default, dependent on the Kafka cluster; see the `zookeeper_addrs` variable | -| zookeeper_addrs | the zookeeper address of the binlog Kafka cluster | -| deploy_without_tidb | the Key-Value mode, deploy only PD, TiKV and the monitoring service, not TiDB; set the IP of the tidb_servers host group to null in the `inventory.ini` file | -| alertmanager_target | optional: If you have deployed `alertmanager` separately, you can configure this variable using the `alertmanager_host:alertmanager_port` format | -| grafana_admin_user | the username of Grafana administrator; default `admin` | -| grafana_admin_password | the password of Grafana administrator account; default `admin`; used to import Dashboard and create the API key using TiDB Ansible; update this variable if you have modified it through Grafana web | -| collect_log_recent_hours | to collect the log of recent hours; default the recent 2 hours | -| enable_bandwidth_limit | to set a bandwidth limit when pulling the diagnostic data from the target machines to the control machine; used together with the `collect_bandwidth_limit` variable | -| collect_bandwidth_limit | the limited bandwidth when pulling the diagnostic data from the target machines to the control machine; unit: Kbit/s; default 10000, indicating 10Mb/s; for the cluster topology of multiple TiKV instances on each TiKV node, you need to divide the number of the TiKV instances on each TiKV node | -| prometheus_storage_retention | the retention time of the monitoring data of Prometheus (30 days by default); this is a new configuration in the `group_vars/monitoring_servers.yml` file in 2.1.7, 3.0 and the later tidb-ansible versions | - -## Step 11: Deploy the TiDB cluster - -When `ansible-playbook` runs Playbook, the default concurrent number is 5. If many target machines are deployed, you can add the `-f` parameter to specify the concurrency, such as `ansible-playbook deploy.yml -f 10`. - -The following example uses `tidb` as the user who runs the service. - -1. Edit the `tidb-ansible/inventory.ini` file to make sure `ansible_user = tidb`. - - ```ini - ## Connection - # ssh via normal user - ansible_user = tidb - ``` - - > **Note:** - > - > Do not configure `ansible_user` to `root`, because `tidb-ansible` limits the user that runs the service to the normal user. - - Run the following command and if all servers return `tidb`, then the SSH mutual trust is successfully configured: - - {{< copyable "shell-regular" >}} - - ```shell - ansible -i inventory.ini all -m shell -a 'whoami' - ``` - - Run the following command and if all servers return `root`, then sudo without password of the `tidb` user is successfully configured: - - {{< copyable "shell-regular" >}} - - ```shell - ansible -i inventory.ini all -m shell -a 'whoami' -b - ``` - -2. Run the `local_prepare.yml` playbook and download TiDB binary to the control machine. - - {{< copyable "shell-regular" >}} - - ```shell - ansible-playbook local_prepare.yml - ``` - -3. Initialize the system environment and modify the kernel parameters. - - {{< copyable "shell-regular" >}} - - ```shell - ansible-playbook bootstrap.yml - ``` - -4. Deploy the TiDB cluster software. - - {{< copyable "shell-regular" >}} - - ```shell - ansible-playbook deploy.yml - ``` - - > **Note:** - > - > You can use the `Report` button on the Grafana Dashboard to generate the PDF file. This function depends on the `fontconfig` package and English fonts. To use this function, log in to the `grafana_servers` machine and install it using the following command: - > - > {{< copyable "shell-regular" >}} - > - > ```bash - > sudo yum install fontconfig open-sans-fonts - > ``` - -5. Start the TiDB cluster. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook start.yml - ``` - -## Test the TiDB cluster - -Because TiDB is compatible with MySQL, you must use the MySQL client to connect to TiDB directly. It is recommended to configure load balancing to provide uniform SQL interface. - -1. Connect to the TiDB cluster using the MySQL client. - - {{< copyable "shell-regular" >}} - - ```shell - mysql -u root -h 172.16.10.1 -P 4000 - ``` - - > **Note:** - > - > The default port of TiDB service is `4000`. - -2. Access the monitoring platform using a web browser. - - - Address: - - Default account and password: `admin`; `admin` - -> **Note:** -> -> By default, TiDB periodically shares usage details with PingCAP to help understand how to improve the product. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). - -## Deployment FAQs - -This section lists the common questions about deploying TiDB using TiDB Ansible. - -### How to customize the port? - -Edit the `inventory.ini` file and add the following host variable after the IP of the corresponding service: - -| Component | Variable Port | Default Port | Description | -|:--------------|:-------------------|:-------------|:-------------------------| -| TiDB | tidb_port | 4000 | the communication port for the application and DBA tools | -| TiDB | tidb_status_port | 10080 | the communication port to report TiDB status | -| TiKV | tikv_port | 20160 | the TiKV communication port | -| TiKV | tikv_status_port | 20180 | the communication port to report the TiKV status | -| PD | pd_client_port | 2379 | the communication port between TiDB and PD | -| PD | pd_peer_port | 2380 | the inter-node communication port within the PD cluster | -| Pump | pump_port | 8250 | the pump communication port | -| Prometheus | prometheus_port | 9090 | the communication port for the Prometheus service | -| Pushgateway | pushgateway_port | 9091 | the aggregation and report port for TiDB, TiKV, and PD monitor | -| Node_exporter | node_exporter_port | 9100 | the communication port to report the system information of every TiDB cluster node | -| Grafana | grafana_port | 3000 | the port for the external Web monitoring service and client (Browser) access | -| Kafka_exporter | kafka_exporter_port | 9308 | the communication port for Kafka_exporter, used to monitor the binlog Kafka cluster | - -### How to customize the deployment directory? - -Edit the `inventory.ini` file and add the following host variable after the IP of the corresponding service: - -| Component | Variable Directory | Default Directory | Description | -|:--------------|:----------------------|:------------------------------|:-----| -| Global | deploy_dir | /home/tidb/deploy | the deployment directory | -| TiDB | tidb_log_dir | {{ deploy_dir }}/log | the TiDB log directory | -| TiKV | tikv_log_dir | {{ deploy_dir }}/log | the TiKV log directory | -| TiKV | tikv_data_dir | {{ deploy_dir }}/data | the data directory | -| TiKV | wal_dir | "" | the rocksdb write-ahead log directory, consistent with the TiKV data directory when the value is null | -| TiKV | raftdb_path | "" | the raftdb directory, being tikv_data_dir/raft when the value is null | -| PD | pd_log_dir | {{ deploy_dir }}/log | the PD log directory | -| PD | pd_data_dir | {{ deploy_dir }}/data.pd | the PD data directory | -| Pump | pump_log_dir | {{ deploy_dir }}/log | the Pump log directory | -| Pump | pump_data_dir | {{ deploy_dir }}/data.pump | the Pump data directory | -| Prometheus | prometheus_log_dir | {{ deploy_dir }}/log | the Prometheus log directory | -| Prometheus | prometheus_data_dir | {{ deploy_dir }}/data.metrics | the Prometheus data directory | -| Pushgateway | pushgateway_log_dir | {{ deploy_dir }}/log | the pushgateway log directory | -| Node_exporter | node_exporter_log_dir | {{ deploy_dir }}/log | the node_exporter log directory | -| Grafana | grafana_log_dir | {{ deploy_dir }}/log | the Grafana log directory | -| Grafana | grafana_data_dir | {{ deploy_dir }}/data.grafana | the Grafana data directory | - -### How to check whether the NTP service is normal? - -1. Run the following command. If it returns `running`, then the NTP service is running: - - {{< copyable "shell-regular" >}} - - ```shell - sudo systemctl status ntpd.service - ``` - - ``` - ntpd.service - Network Time Service - Loaded: loaded (/usr/lib/systemd/system/ntpd.service; disabled; vendor preset: disabled) - Active: active (running) since 一 2017-12-18 13:13:19 CST; 3s ago - ``` - -2. Run the ntpstat command. If it returns `synchronised to NTP server` (synchronizing with the NTP server), then the synchronization process is normal. - - {{< copyable "shell-regular" >}} - - ```shell - ntpstat - ``` - - ``` - synchronised to NTP server (85.199.214.101) at stratum 2 - time correct to within 91 ms - polling server every 1024 s - ``` - -> **Note:** -> -> For the Ubuntu system, you need to install the `ntpstat` package. - -- The following condition indicates the NTP service is not synchronizing normally: - - {{< copyable "shell-regular" >}} - - ```shell - ntpstat - ``` - - ``` - unsynchronised - ``` - -- The following condition indicates the NTP service is not running normally: - - {{< copyable "shell-regular" >}} - - ```shell - ntpstat - ``` - - ``` - Unable to talk to NTP daemon. Is it running? - ``` - -- To make the NTP service start synchronizing as soon as possible, run the following command. You can replace `pool.ntp.org` with other NTP servers. - - {{< copyable "shell-regular" >}} - - ```shell - sudo systemctl stop ntpd.service && \ - sudo ntpdate pool.ntp.org && \ - sudo systemctl start ntpd.service - ``` - -- To install the NTP service manually on the CentOS 7 system, run the following command: - - {{< copyable "shell-regular" >}} - - ```shell - sudo yum install ntp ntpdate && \ - sudo systemctl start ntpd.service && \ - sudo systemctl enable ntpd.service - ``` - -### How to modify the supervision method of a process from `supervise` to `systemd`? - -Run the following command: - -{{< copyable "shell-root" >}} - -```shell -process supervision, [systemd, supervise] -``` - -``` -process_supervision = systemd -``` - -For versions earlier than TiDB 1.0.4, the TiDB Ansible supervision method of a process is `supervise` by default. The previously installed cluster can remain the same. If you need to change the supervision method to `systemd`, stop the cluster and run the following command: - -{{< copyable "shell-regular" >}} - -```shell -ansible-playbook stop.yml && \ -ansible-playbook deploy.yml -D && \ -ansible-playbook start.yml -``` - -### How to manually configure the SSH mutual trust and sudo without password? - -1. Log in to the target machine respectively using the `root` user account, create the `tidb` user and set the login password. - - {{< copyable "shell-root" >}} - - ```shell - useradd tidb && \ - passwd tidb - ``` - -2. To configure sudo without password, run the following command, and add `tidb ALL=(ALL) NOPASSWD: ALL` to the end of the file: - - {{< copyable "shell-root" >}} - - ```shell - visudo - ``` - - ``` - tidb ALL=(ALL) NOPASSWD: ALL - ``` - -3. Use the `tidb` user to log in to the control machine, and run the following command. Replace `172.16.10.61` with the IP of your target machine, and enter the `tidb` user password of the target machine as prompted. Successful execution indicates that SSH mutual trust is already created. This applies to other machines as well. - - {{< copyable "shell-regular" >}} - - ```shell - ssh-copy-id -i ~/.ssh/id_rsa.pub 172.16.10.61 - ``` - -4. Log in to the control machine using the `tidb` user account, and log in to the IP of the target machine using SSH. If you do not need to enter the password and can successfully log in, then the SSH mutual trust is successfully configured. - - {{< copyable "shell-regular" >}} - - ```shell - ssh 172.16.10.61 - ``` - - ``` - [tidb@172.16.10.61 ~]$ - ``` - -5. After you login to the target machine using the `tidb` user, run the following command. If you do not need to enter the password and can switch to the `root` user, then sudo without password of the `tidb` user is successfully configured. - - {{< copyable "shell-regular" >}} - - ```shell - sudo -su root - ``` - - ``` - [root@172.16.10.61 tidb]# - ``` - -### Error: You need to install jmespath prior to running json_query filter - -1. See [Install TiDB Ansible and its dependencies on the control machine](#step-4-install-tidb-ansible-and-its-dependencies-on-the-control-machine) and use `pip` to install TiDB Ansible and the corresponding dependencies in the control machine. The `jmespath` dependent package is installed by default. - -2. Run the following command to check whether `jmespath` is successfully installed: - - {{< copyable "shell-regular" >}} - - ```bash - pip show jmespath - ``` - -3. Enter `import jmespath` in the Python interactive window of the control machine. - - - If no error displays, the dependency is successfully installed. - - If the `ImportError: No module named jmespath` error displays, the Python `jmespath` module is not successfully installed. - - {{< copyable "shell-regular" >}} - - ```shell - python - ``` - - ``` - Python 2.7.5 (default, Nov 6 2016, 00:28:07) - [GCC 4.8.5 20150623 (Red Hat 4.8.5-11)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - ``` - - {{< copyable "shell-regular" >}} - - ```shell - import jmespath - ``` - -### The `zk: node does not exist` error when starting Pump/Drainer - -Check whether the `zookeeper_addrs` configuration in `inventory.ini` is the same with the configuration in the Kafka cluster, and whether the namespace is filled in. The description about namespace configuration is as follows: - -```ini -# ZooKeeper connection string (see ZooKeeper docs for details). -# ZooKeeper address of the Kafka cluster. Example: -# zookeeper_addrs = "192.168.0.11:2181,192.168.0.12:2181,192.168.0.13:2181" -# You can also append an optional chroot string to the URLs to specify the root directory for all Kafka znodes. Example: -# zookeeper_addrs = "192.168.0.11:2181,192.168.0.12:2181,192.168.0.13:2181/kafka/123" -``` diff --git a/optimistic-transaction.md b/optimistic-transaction.md index 2d6756b8c6b3..2403dbc608fa 100644 --- a/optimistic-transaction.md +++ b/optimistic-transaction.md @@ -40,7 +40,7 @@ To support distributed transactions, TiDB adopts two-phase commit (2PC) in optim 1. TiDB selects a Primary Key from the data to be written. 2. TiDB receives the information of Region distribution from PD, and groups all keys by Region accordingly. 3. TiDB sends prewrite requests to all TiKV nodes involved. Then, TiKV checks whether there are conflict or expired versions. Valid data is locked. - 4. TiDB receives all requests in the prewrite phase and the prewrite is successful. + 4. TiDB receives all responses in the prewrite phase and the prewrite is successful. 5. TiDB receives a commit version number from PD and marks it as `commit_ts`. 6. TiDB initiates the second commit to the TiKV node where Primary Key is located. TiKV checks the data, and cleans the locks left in the prewrite phase. 7. TiDB receives the message that reports the second phase is successfully finished. diff --git a/optimizer-hints.md b/optimizer-hints.md index 516aa8cbd5f2..a98ef4946108 100644 --- a/optimizer-hints.md +++ b/optimizer-hints.md @@ -119,10 +119,6 @@ The parameter(s) given in `INL_JOIN()` is the candidate table for the inner tabl The `INL_HASH_JOIN(t1_name [, tl_name])` hint tells the optimizer to use the index nested loop hash join algorithm. The conditions for using this algorithm are the same with the conditions for using the index nested loop join algorithm. The difference between the two algorithms is that `INL_JOIN` creates a hash table on the joined inner table, but `INL_HASH_JOIN` creates a hash table on the joined outer table. `INL_HASH_JOIN` has a fixed limit on memory usage, while the memory used by `INL_JOIN` depends on the number of rows matched in the inner table. -### INL_MERGE_JOIN - -The `INL_MERGE_JOIN(t1_name [, tl_name])` hint tells the optimizer to use the index nested loop merge join algorithm. This hint is used in the same scenario as in that of `INL_JOIN`. Compared with `INL_JOIN` and `INL_HASH_JOIN`, it saves more memory but requires more strict usage conditions: the column sets of the inner table in join keys is the prefix of the inner table index, or the index of the inner table is the prefix of the column sets of the inner table in join keys. - ### HASH_JOIN(t1_name [, tl_name ...]) The `HASH_JOIN(t1_name [, tl_name ...])` hint tells the optimizer to use the hash join algorithm for the given table(s). This algorithm allows the query to be executed concurrently with multiple threads, which achieves a higher processing speed but consumes more memory. For example: @@ -314,7 +310,7 @@ In addition to this hint, setting the `tidb_replica_read` environment variable t The `IGNORE_PLAN_CACHE()` hint reminds the optimizer not to use the Plan Cache when handling the current `prepare` statement. -This hint is used to temporarily disable the Plan Cache for a certain type of queries. +This hint is used to temporarily disable the Plan Cache for a certain type of queries when [prepare-plan-cache](/tidb-configuration-file.md#prepared-plan-cache) is enabled. In the following example, the Plan Cache is forcibly disabled when executing the `prepare` statement. diff --git a/overview.md b/overview.md index 1238e01e578c..4c5a66ce0ca4 100644 --- a/overview.md +++ b/overview.md @@ -47,3 +47,10 @@ aliases: ['/docs/dev/key-features/','/tidb/dev/key-features','/docs/dev/overview - **Data aggregation and secondary processing scenarios** The application data of most companies are scattered in different systems. As the application grows, the decision-making leaders need to understand the business status of the entire company to make decisions in time. In this case, the company needs to aggregate the scattered data into the same system and execute secondary processing to generate a T+0 or T+1 report. The traditional solution is to use ETL and Hadoop, but the Hadoop system is complicated, with high operations and maintenance cost and storage cost. Compared with Hadoop, TiDB is much simpler. You can replicate data into TiDB using ETL tools or data migration tools provided by TiDB. Reports can be directly generated using SQL statements. + +## See also + +- [TiDB Architecture](/tidb-architecture.md) +- [TiDB Storage](/tidb-storage.md) +- [TiDB Computing](/tidb-computing.md) +- [TiDB Scheduling](/tidb-scheduling.md) diff --git a/partitioned-table.md b/partitioned-table.md index bccfbf796f5b..5d9b28c9873f 100644 --- a/partitioned-table.md +++ b/partitioned-table.md @@ -10,9 +10,9 @@ This document introduces TiDB's implementation of partitioning. ## Partitioning types -This section introduces the types of partitioning which are available in TiDB. Currently, TiDB supports Range partitioning and Hash partitioning. +This section introduces the types of partitioning in TiDB. Currently, TiDB supports [Range partitioning](#range-partitioning), [List partitioning](#list-partitioning), [List COLUMNS partitioning](#list-columns-partitioning), and [Hash partitioning](#hash-partitioning). -Range partitioning is used to resolve the performance issues caused by a large amount of deletions in the application, and it supports fast drop partition operations. Hash partitioning is used to scatter the data when there are a large amount of writes. +Range partitioning, List partitioning and List COLUMNS partitioning are used to resolve the performance issues caused by a large amount of deletions in the application, and support fast drop partition operations. Hash partitioning is used to scatter the data when there are a large amount of writes. ### Range partitioning @@ -28,8 +28,8 @@ CREATE TABLE employees ( fname VARCHAR(30), lname VARCHAR(30), hired DATE NOT NULL DEFAULT '1970-01-01', - separated DATE NOT NULL DEFAULT '9999-12-31', - job_code INT NOT NULL, + separated DATE DEFAULT '9999-12-31', + job_code INT, store_id INT NOT NULL ); ``` @@ -44,8 +44,8 @@ CREATE TABLE employees ( fname VARCHAR(30), lname VARCHAR(30), hired DATE NOT NULL DEFAULT '1970-01-01', - separated DATE NOT NULL DEFAULT '9999-12-31', - job_code INT NOT NULL, + separated DATE DEFAULT '9999-12-31', + job_code INT, store_id INT NOT NULL ) @@ -59,7 +59,7 @@ PARTITION BY RANGE (store_id) ( In this partition scheme, all rows corresponding to employees whose `store_id` is 1 through 5 are stored in the `p0` partition while all employees whose `store_id` is 6 through 10 are stored in `p1`. Range partitioning requires the partitions to be ordered, from lowest to highest. -If you insert a row of data `(72, 'Mitchell', 'Wilson', '1998-06-25', NULL, 13)`, it falls in the `p2` partition. But if you insert a record whose `store_id` is larger than 20, an error is reported because TiDB can not know which partition this record should be inserted into. In this case, you can use `MAXVALUE` when creating a table: +If you insert a row of data `(72, 'Tom', 'John', '2015-06-25', NULL, NULL, 15)`, it falls in the `p2` partition. But if you insert a record whose `store_id` is larger than 20, an error is reported because TiDB can not know which partition this record should be inserted into. In this case, you can use `MAXVALUE` when creating a table: {{< copyable "sql" >}} @@ -69,8 +69,8 @@ CREATE TABLE employees ( fname VARCHAR(30), lname VARCHAR(30), hired DATE NOT NULL DEFAULT '1970-01-01', - separated DATE NOT NULL DEFAULT '9999-12-31', - job_code INT NOT NULL, + separated DATE DEFAULT '9999-12-31', + job_code INT, store_id INT NOT NULL ) @@ -94,8 +94,8 @@ CREATE TABLE employees ( fname VARCHAR(30), lname VARCHAR(30), hired DATE NOT NULL DEFAULT '1970-01-01', - separated DATE NOT NULL DEFAULT '9999-12-31', - job_code INT NOT NULL, + separated DATE DEFAULT '9999-12-31', + job_code INT, store_id INT NOT NULL ) @@ -118,7 +118,7 @@ CREATE TABLE employees ( fname VARCHAR(30), lname VARCHAR(30), hired DATE NOT NULL DEFAULT '1970-01-01', - separated DATE NOT NULL DEFAULT '9999-12-31', + separated DATE DEFAULT '9999-12-31', job_code INT, store_id INT ) @@ -164,6 +164,194 @@ Range partitioning is particularly useful when one or more of the following cond * You want to use a column that contains time or date values, or containing values arising from some other series. * You need to frequently run queries on the columns used for partitioning. For example, when executing a query like `EXPLAIN SELECT COUNT(*) FROM employees WHERE separated BETWEEN '2000-01-01' AND '2000-12-31' GROUP BY store_id;`, TiDB can quickly know that only the data in the `p2` partition needs to be scanned, because the other partitions do not match the `WHERE` condition. +### List partitioning + +> **Warning:** +> +> List partitioning is an experimental feature. It is not recommended that you use it in the production environment. + +Before creating a List partitioned table, you need to set the value of the session variable `tidb_enable_list_partition` to `ON`. + +{{< copyable "sql" >}} + +```sql +set @@session.tidb_enable_list_partition = ON +``` + +Also, make sure that `tidb_enable_table_partition` is set to `ON`, which is the default setting. + +List partitioning is similar to Range partitioning. Unlike Range partitioning, in List partitioning, the partitioning expression values for all rows in each partition are in a given value set. This value set defined for each partition can have any number of values but cannot have duplicate values. You can use the `PARTITION ... VALUES IN (...)` clause to define a value set. + +Suppose that you want to create a personnel record table. You can create a table as follows: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE employees ( + id INT NOT NULL, + hired DATE NOT NULL DEFAULT '1970-01-01', + store_id INT +); +``` + +Suppose that there are 20 stores distributed in 4 districts, as shown in the table below: + +``` +| Region | Store ID Numbers | +| ------- | -------------------- | +| North | 1, 2, 3, 4, 5 | +| East | 6, 7, 8, 9, 10 | +| West | 11, 12, 13, 14, 15 | +| Central | 16, 17, 18, 19, 20 | +``` + +If you want to store the personnel data of employees of the same region in the same partition, you can create a List partitioned table based on `store_id`: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE employees ( + id INT NOT NULL, + hired DATE NOT NULL DEFAULT '1970-01-01', + store_id INT +) +PARTITION BY LIST (store_id) ( + PARTITION pNorth VALUES IN (1, 2, 3, 4, 5), + PARTITION pEast VALUES IN (6, 7, 8, 9, 10), + PARTITION pWest VALUES IN (11, 12, 13, 14, 15), + PARTITION pCentral VALUES IN (16, 17, 18, 19, 20) +); +``` + +After creating the partitions as above, you can easily add or delete records related to a specific region in the table. For example, suppose that all stores in the East region (East) are sold to another company. Then all the row data related to the store employees of this region can be deleted by executing `ALTER TABLE employees TRUNCATE PARTITION pEast`, which is much more efficient than the equivalent statement `DELETE FROM employees WHERE store_id IN (6, 7, 8, 9, 10)`. + +You can also execute `ALTER TABLE employees DROP PARTITION pEast` to delete all related rows, but this statement also deletes the `pEast` partition from the table definition. In this situation, you must execute the `ALTER TABLE ... ADD PARTITION` statement to recover the original partitioning scheme of the table. + +Unlike Range partitioning, List partitioning does not have a similar `MAXVALUE` partition to store all values that do not belong to other partitions. Instead, all expected values of the partition expression must be included in the `PARTITION ... VALUES IN (...)` clause. If the value to be inserted in an `INSERT` statement does not match the column value set of any partition, the statement fails to execute and an error is reported. See the following example: + +```sql +test> CREATE TABLE t ( + -> a INT, + -> b INT + -> ) + -> PARTITION BY LIST (a) ( + -> PARTITION p0 VALUES IN (1, 2, 3), + -> PARTITION p1 VALUES IN (4, 5, 6) + -> ); +Query OK, 0 rows affected (0.11 sec) + +test> INSERT INTO t VALUES (7, 7); +ERROR 1525 (HY000): Table has no partition for value 7 +``` + +To ignore the error type above, you can use the `IGNORE` keyword. After using this keyword, if a row contains values that do not match the column value set of any partition, this row will not be inserted. Instead, any row with matched values is inserted, and no error is reported: + +```sql +test> TRUNCATE t; +Query OK, 1 row affected (0.00 sec) + +test> INSERT IGNORE INTO t VALUES (1, 1), (7, 7), (8, 8), (3, 3), (5, 5); +Query OK, 3 rows affected, 2 warnings (0.01 sec) +Records: 5 Duplicates: 2 Warnings: 2 + +test> select * from t; ++------+------+ +| a | b | ++------+------+ +| 5 | 5 | +| 1 | 1 | +| 3 | 3 | ++------+------+ +3 rows in set (0.01 sec) +``` + +### List COLUMNS partitioning + +> **Warning:** +> +> List COLUMNS partitioning is an experimental feature. It is not recommended that you use it in the production environment. + +List COLUMNS partitioning is a variant of List partitioning. You can use multiple columns as partition keys. Besides the integer data type, you can also use the columns in the string, `DATE`, and `DATETIME` data types as partition columns. + +Suppose that you want to divide the store employees from the following 12 cities into 4 regions, as shown in the following table: + +``` +| Region | Cities | +| :----- | ------------------------------ | +| 1 | LosAngeles,Seattle, Houston | +| 2 | Chicago, Columbus, Boston | +| 3 | NewYork, LongIsland, Baltimore | +| 4 | Atlanta, Raleigh, Cincinnati | +``` + +You can use List COLUMNS partitioning to create a table and store each row in the partition that corresponds to the employee's city, as shown below: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE employees_1 ( + id INT NOT NULL, + fname VARCHAR(30), + lname VARCHAR(30), + hired DATE NOT NULL DEFAULT '1970-01-01', + separated DATE DEFAULT '9999-12-31', + job_code INT, + store_id INT, + city VARCHAR(15) +) +PARTITION BY LIST COLUMNS(city) ( + PARTITION pRegion_1 VALUES IN('LosAngeles', 'Seattle', 'Houston'), + PARTITION pRegion_2 VALUES IN('Chicago', 'Columbus', 'Boston'), + PARTITION pRegion_3 VALUES IN('NewYork', 'LongIsland', 'Baltimore'), + PARTITION pRegion_4 VALUES IN('Atlanta', 'Raleigh', 'Cincinnati') +); +``` + +Unlike List partitioning, in List COLUMNS partitioning, you do not need to use the expression in the `COLUMNS()` clause to convert column values to integers. + +List COLUMNS partitioning can also be implemented using columns of the `DATE` and `DATETIME` types, as shown in the following example. This example uses the same names and columns as the previous `employees_1` table, but uses List COLUMNS partitioning based on the `hired` column: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE employees_2 ( + id INT NOT NULL, + fname VARCHAR(30), + lname VARCHAR(30), + hired DATE NOT NULL DEFAULT '1970-01-01', + separated DATE DEFAULT '9999-12-31', + job_code INT, + store_id INT, + city VARCHAR(15) +) +PARTITION BY LIST COLUMNS(hired) ( + PARTITION pWeek_1 VALUES IN('2020-02-01', '2020-02-02', '2020-02-03', + '2020-02-04', '2020-02-05', '2020-02-06', '2020-02-07'), + PARTITION pWeek_2 VALUES IN('2020-02-08', '2020-02-09', '2020-02-10', + '2020-02-11', '2020-02-12', '2020-02-13', '2020-02-14'), + PARTITION pWeek_3 VALUES IN('2020-02-15', '2020-02-16', '2020-02-17', + '2020-02-18', '2020-02-19', '2020-02-20', '2020-02-21'), + PARTITION pWeek_4 VALUES IN('2020-02-22', '2020-02-23', '2020-02-24', + '2020-02-25', '2020-02-26', '2020-02-27', '2020-02-28') +); +``` + +In addition, you can also add multiple columns in the `COLUMNS()` clause. For example: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t ( + id int, + name varchar(10) +) +PARTITION BY LIST COLUMNS(id,name) ( + partition p0 values IN ((1,'a'),(2,'b')), + partition p1 values IN ((3,'c'),(4,'d')), + partition p3 values IN ((5,'e'),(null,null)) +); +``` + ### Hash partitioning Hash partitioning is used to make sure that data is evenly scattered into a certain number of partitions. With Range partitioning, you must specify the range of the column values for each partition when you use Range partitioning, while you just need to specify the number of partitions when you use Hash partitioning. @@ -180,7 +368,7 @@ CREATE TABLE employees ( fname VARCHAR(30), lname VARCHAR(30), hired DATE NOT NULL DEFAULT '1970-01-01', - separated DATE NOT NULL DEFAULT '9999-12-31', + separated DATE DEFAULT '9999-12-31', job_code INT, store_id INT ) @@ -201,7 +389,7 @@ CREATE TABLE employees ( fname VARCHAR(30), lname VARCHAR(30), hired DATE NOT NULL DEFAULT '1970-01-01', - separated DATE NOT NULL DEFAULT '9999-12-31', + separated DATE DEFAULT '9999-12-31', job_code INT, store_id INT ) @@ -220,7 +408,7 @@ In conclusion, when the expression has a form that is closer to `y = cx`, it is In theory, pruning is also possible for expressions involving more than one column value, but determining which of such expressions are suitable can be quite difficult and time-consuming. For this reason, the use of hashing expressions involving multiple columns is not particularly recommended. -When using `PARTITIION BY HASH`, TiDB decides which partition the data should fall into based on the modulus of the result of the expression. In other words, if a partitioning expression is `expr` and the number of partitions is `num`, `MOD(expr, num)` decides the partition in which the data is stored. Assume that `t1` is defined as follows: +When using `PARTITION BY HASH`, TiDB decides which partition the data should fall into based on the modulus of the result of the expression. In other words, if a partitioning expression is `expr` and the number of partitions is `num`, `MOD(expr, num)` decides the partition in which the data is stored. Assume that `t1` is defined as follows: {{< copyable "sql" >}} @@ -386,12 +574,26 @@ You can see that the inserted record `(NULL, 'mothra')` falls into the same part > **Note:** > `NULL` values by Hash partitions in TiDB are handled in the same way as described in [How MySQL Partitioning Handles NULL](https://dev.mysql.com/doc/refman/8.0/en/partitioning-handling-nulls.html), which, however, is not consistent with the actual behavior of MySQL. In other words, MySQL's implementation in this case is not consistent with its documentation. -> +> > In this case, the actual behavior of TiDB is in line with the description of this document. ## Partition management -You can add, drop, merge, split, redefine partitions by using `ALTER TABLE` statements. +For `LIST` and `RANGE` partitioned tables, you can add and drop partitions using the `ALTER TABLE ADD PARTITION ()` or `ALTER TABLE
DROP PARTITION ` statement. + +For `LIST` and `RANGE` partitioned tables, `REORGANIZE PARTITION` is not yet supported. + +For `HASH` partitioned tables, `COALESCE PARTITION` and `ADD PARTITION` are not yet supported. + +`EXCHANGE PARTITION` works by swapping a partition and a non-partitioned table, similar to how renaming a table like `RENAME TABLE t1 TO t1_tmp, t2 TO t1, t1_tmp TO t2` works. + +For example, `ALTER TABLE partitioned_table EXCHANGE PARTITION p1 WITH TABLE non_partitioned_table` swaps the `non_partitioned_table` table in the `p1` partition with the `partitioned_table` table. + +Ensure that all rows that you are exchanging into the partition match the partition definition; otherwise, these rows will not be found and cause unexpected issues. + +> **Warning:** +> +> `EXCHANGE PARTITION` is an experimental feature. It is not recommended to use it in a production environment. To enable it, set the `tidb_enable_exchange_partition` system variable to `ON`. ### Range partition management @@ -830,7 +1032,22 @@ PARTITIONS 4; ERROR 1491 (HY000): A PRIMARY KEY must include all columns in the table's partitioning function ``` -The `CREATE TABLE` statement fails because both `col1` and `col3` are included in the proposed partitioning key, but neither of these columns is part of both of unique keys on the table. +The `CREATE TABLE` statement fails because both `col1` and `col3` are included in the proposed partitioning key, but neither of these columns is part of both of unique keys on the table. After the following modifications, the `CREATE TABLE` statement becomes valid: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t3 ( + col1 INT NOT NULL, + col2 DATE NOT NULL, + col3 INT NOT NULL, + col4 INT NOT NULL, + UNIQUE KEY (col1, col2, col3), + UNIQUE KEY (col1, col3) +) +PARTITION BY HASH(col1 + col3) + PARTITIONS 4; +``` The following table cannot be partitioned at all, because there is no way to include in a partitioning key any columns that belong to both unique keys: @@ -876,7 +1093,31 @@ PARTITION BY HASH( YEAR(col2) ) PARTITIONS 4; ``` -In both cases, the primary key does not include all columns referenced in the partitioning expression. +In the above examples, the primary key does not include all columns referenced in the partitioning expression. After adding the missing column in the primary key, the `CREATE TABLE` statement becomes valid: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t5 ( + col1 INT NOT NULL, + col2 DATE NOT NULL, + col3 INT NOT NULL, + col4 INT NOT NULL, + PRIMARY KEY(col1, col2, col3) +) +PARTITION BY HASH(col3) +PARTITIONS 4; +CREATE TABLE t6 ( + col1 INT NOT NULL, + col2 DATE NOT NULL, + col3 INT NOT NULL, + col4 INT NOT NULL, + PRIMARY KEY(col1, col2, col3), + UNIQUE KEY(col2) +) +PARTITION BY HASH( YEAR(col2) ) +PARTITIONS 4; +``` If a table has neither unique keys nor primary keys, then this restriction does not apply. @@ -949,7 +1190,7 @@ YEARWEEK() ### Compatibility with MySQL -Currently, TiDB only supports Range partitioning and Hash partitioning. Other partitioning types that are available in MySQL such as list partitioning and key partitioning are not supported yet in TiDB. +Currently, TiDB supports Range partitioning, List partitioning, List COLUMNS partitioning, and Hash partitioning. Other partitioning types that are available in MySQL such as key partitioning are not supported yet in TiDB. For a table partitioned by `RANGE COLUMNS`, currently TiDB only supports using a single partitioning column. @@ -1049,6 +1290,198 @@ select * from t; 5 rows in set (0.00 sec) ``` -The `tidb_enable_table_partition` environment variable controls whether to enable the partitioned table feature. If this variable is set to `off`, the partition information will be ignored when a table is created, and this table will be created as a normal table. +The `tidb_enable_list_partition` environment variable controls whether to enable the partitioned table feature. If this variable is set to `OFF`, the partition information will be ignored when a table is created, and this table will be created as a normal table. + +This variable is only used in table creation. After the table is created, modify this variable value takes no effect. For details, see [system variables](/system-variables.md#tidb_enable_list_partition-new-in-v50). + +### Dynamic pruning mode + +> **Warning:** +> +> This is still an experimental feature. It is **NOT** recommended that you use it in the production environment. + +TiDB accesses partitioned tables in one of the two modes: `dynamic` mode and `static` mode. Currently, `static` mode is used by default. If you want to enable `dynamic` mode, you need to manually set the `tidb_partition_prune_mode` variable to `dynamic`. + +{{< copyable "sql" >}} + +```sql +set @@session.tidb_partition_prune_mode = 'dynamic' +``` + +In `static` mode, TiDB accesses each partition separately using multiple operators, and then merges the results using `Union`. The following example is a simple read operation where TiDB merges the results of two corresponding partitions using `Union`: + +{{< copyable "sql" >}} + +```sql +mysql> create table t1(id int, age int, key(id)) partition by range(id) ( + -> partition p0 values less than (100), + -> partition p1 values less than (200), + -> partition p2 values less than (300), + -> partition p3 values less than (400)); +Query OK, 0 rows affected (0.01 sec) + +mysql> explain select * from t1 where id < 150; ++------------------------------+----------+-----------+------------------------+--------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------+----------+-----------+------------------------+--------------------------------+ +| PartitionUnion_9 | 6646.67 | root | | | +| ├─TableReader_12 | 3323.33 | root | | data:Selection_11 | +| │ └─Selection_11 | 3323.33 | cop[tikv] | | lt(test.t1.id, 150) | +| │ └─TableFullScan_10 | 10000.00 | cop[tikv] | table:t1, partition:p0 | keep order:false, stats:pseudo | +| └─TableReader_18 | 3323.33 | root | | data:Selection_17 | +| └─Selection_17 | 3323.33 | cop[tikv] | | lt(test.t1.id, 150) | +| └─TableFullScan_16 | 10000.00 | cop[tikv] | table:t1, partition:p1 | keep order:false, stats:pseudo | ++------------------------------+----------+-----------+------------------------+--------------------------------+ +7 rows in set (0.00 sec) +``` + +In `dynamic` mode, each operator supports direct access to multiple partitions, so TiDB no longer uses `Union`. + +{{< copyable "sql" >}} + +```sql +mysql> set @@session.tidb_partition_prune_mode = 'dynamic'; +Query OK, 0 rows affected (0.00 sec) + +mysql> explain select * from t1 where id < 150; ++-------------------------+----------+-----------+-----------------+--------------------------------+ +| id | estRows | task | access object | operator info | ++-------------------------+----------+-----------+-----------------+--------------------------------+ +| TableReader_7 | 3323.33 | root | partition:p0,p1 | data:Selection_6 | +| └─Selection_6 | 3323.33 | cop[tikv] | | lt(test.t1.id, 150) | +| └─TableFullScan_5 | 10000.00 | cop[tikv] | table:t1 | keep order:false, stats:pseudo | ++-------------------------+----------+-----------+-----------------+--------------------------------+ +3 rows in set (0.00 sec) +``` + +From the above query results, you can see that the `Union` operator in the execution plan disappears while the partition pruning still takes effect and the execution plan only accesses `p0` and `p1`. + +`dynamic` mode makes execution plans simpler and clearer. Omitting the Union operation can improve the execution efficiency and avoid the problem of Union concurrent execution. In addition, `dynamic` mode also solves two problems that cannot be solved in `static` mode: + ++ Plan Cache cannot be used. (See example 1 and 2) ++ Execution plans with IndexJoin cannot be used. (See example 3 and 4) + +**Example 1**：In the following example, the Plan Cache feature is enabled in the configuration file and the same query is executed twice in `static` mode: + +{{< copyable "sql" >}} + +```sql +mysql> set @a=150; +Query OK, 0 rows affected (0.00 sec) + +mysql> set @@tidb_partition_prune_mode = 'static'; +Query OK, 0 rows affected (0.00 sec) + +mysql> prepare stmt from 'select * from t1 where id < ?'; +Query OK, 0 rows affected (0.00 sec) + +mysql> execute stmt using @a; +Empty set (0.00 sec) + +mysql> execute stmt using @a; +Empty set (0.00 sec) + +-- In static mode, when the same query is executed twice, the cache cannot be hit at the second time. +mysql> select @@last_plan_from_cache; ++------------------------+ +| @@last_plan_from_cache | ++------------------------+ +| 0 | ++------------------------+ +1 row in set (0.00 sec) +``` + +The `last_plan_from_cache` variable can show whether the last query hits the Plan Cache or not. From example 1, you can see that in `static` mode, even if the same query is executed multiple times on the partitioned table, the Plan Cache is not hit. + +**Example 2**: In the following example, the same operations are performed in `dynamic` mode as done in example 1: + +{{< copyable "sql" >}} + +```sql +mysql> set @@tidb_partition_prune_mode = 'dynamic'; +Query OK, 0 rows affected (0.00 sec) + +mysql> prepare stmt from 'select * from t1 where id < ?'; +Query OK, 0 rows affected (0.00 sec) + +mysql> execute stmt using @a; +Empty set (0.00 sec) + +mysql> execute stmt using @a; +Empty set (0.00 sec) + +-- In dynamic mode, the cache can be hit at the second time. +mysql> select @@last_plan_from_cache; ++------------------------+ +| @@last_plan_from_cache | ++------------------------+ +| 1 | ++------------------------+ +1 row in set (0.00 sec) +``` + +From example 2, you can see that in `dynamic` mode, querying the partitioned table hits the Plan Cache. + +**Example 3**: In the following example, a query is performed in `static` mode using the execution plan with IndexJoin: + +{{< copyable "sql" >}} + +```sql +mysql> create table t2(id int, code int); +Query OK, 0 rows affected (0.01 sec) + +mysql> set @@tidb_partition_prune_mode = 'static'; +Query OK, 0 rows affected (0.00 sec) + +mysql> explain select /*+ TIDB_INLJ(t1, t2) */ t1.* from t1, t2 where t2.code = 0 and t2.id = t1.id; ++--------------------------------+----------+-----------+------------------------+------------------------------------------------+ +| id | estRows | task | access object | operator info | ++--------------------------------+----------+-----------+------------------------+------------------------------------------------+ +| HashJoin_13 | 12.49 | root | | inner join, equal:[eq(test.t1.id, test.t2.id)] | +| ├─TableReader_42(Build) | 9.99 | root | | data:Selection_41 | +| │ └─Selection_41 | 9.99 | cop[tikv] | | eq(test.t2.code, 0), not(isnull(test.t2.id)) | +| │ └─TableFullScan_40 | 10000.00 | cop[tikv] | table:t2 | keep order:false, stats:pseudo | +| └─PartitionUnion_15(Probe) | 39960.00 | root | | | +| ├─TableReader_18 | 9990.00 | root | | data:Selection_17 | +| │ └─Selection_17 | 9990.00 | cop[tikv] | | not(isnull(test.t1.id)) | +| │ └─TableFullScan_16 | 10000.00 | cop[tikv] | table:t1, partition:p0 | keep order:false, stats:pseudo | +| ├─TableReader_24 | 9990.00 | root | | data:Selection_23 | +| │ └─Selection_23 | 9990.00 | cop[tikv] | | not(isnull(test.t1.id)) | +| │ └─TableFullScan_22 | 10000.00 | cop[tikv] | table:t1, partition:p1 | keep order:false, stats:pseudo | +| ├─TableReader_30 | 9990.00 | root | | data:Selection_29 | +| │ └─Selection_29 | 9990.00 | cop[tikv] | | not(isnull(test.t1.id)) | +| │ └─TableFullScan_28 | 10000.00 | cop[tikv] | table:t1, partition:p2 | keep order:false, stats:pseudo | +| └─TableReader_36 | 9990.00 | root | | data:Selection_35 | +| └─Selection_35 | 9990.00 | cop[tikv] | | not(isnull(test.t1.id)) | +| └─TableFullScan_34 | 10000.00 | cop[tikv] | table:t1, partition:p3 | keep order:false, stats:pseudo | ++--------------------------------+----------+-----------+------------------------+------------------------------------------------+ +17 rows in set, 1 warning (0.00 sec) +``` + +From example 3, you can see that even if the `TIDB_INLJ` hint is used, the query on the partitioned table cannot select the execution plan with IndexJoin. + +**Example 4**: In the following example, the query is performed in `dynamic` mode using the execution plan with IndexJoin: + +{{< copyable "sql" >}} + +```sql +mysql> set @@tidb_partition_prune_mode = 'dynamic'; +Query OK, 0 rows affected (0.00 sec) -This variable is only used in table creation. After the table is created, modify this variable value takes no effect. For details, see [system variables](/system-variables.md#tidb_enable_table_partition). +mysql> explain select /*+ TIDB_INLJ(t1, t2) */ t1.* from t1, t2 where t2.code = 0 and t2.id = t1.id; ++---------------------------------+----------+-----------+------------------------+---------------------------------------------------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++---------------------------------+----------+-----------+------------------------+---------------------------------------------------------------------------------------------------------------------+ +| IndexJoin_11 | 12.49 | root | | inner join, inner:IndexLookUp_10, outer key:test.t2.id, inner key:test.t1.id, equal cond:eq(test.t2.id, test.t1.id) | +| ├─TableReader_16(Build) | 9.99 | root | | data:Selection_15 | +| │ └─Selection_15 | 9.99 | cop[tikv] | | eq(test.t2.code, 0), not(isnull(test.t2.id)) | +| │ └─TableFullScan_14 | 10000.00 | cop[tikv] | table:t2 | keep order:false, stats:pseudo | +| └─IndexLookUp_10(Probe) | 1.25 | root | partition:all | | +| ├─Selection_9(Build) | 1.25 | cop[tikv] | | not(isnull(test.t1.id)) | +| │ └─IndexRangeScan_7 | 1.25 | cop[tikv] | table:t1, index:id(id) | range: decided by [eq(test.t1.id, test.t2.id)], keep order:false, stats:pseudo | +| └─TableRowIDScan_8(Probe) | 1.25 | cop[tikv] | table:t1 | keep order:false, stats:pseudo | ++---------------------------------+----------+-----------+------------------------+---------------------------------------------------------------------------------------------------------------------+ +8 rows in set (0.00 sec) +``` + +From example 4, you can see that in `dynamic` mode, the execution plan with IndexJoin is selected when you execute the query. diff --git a/pd-configuration-file.md b/pd-configuration-file.md index 3de64be6e9f2..c3ddade22d85 100644 --- a/pd-configuration-file.md +++ b/pd-configuration-file.md @@ -77,17 +77,6 @@ This document only describes parameters that are not included in command-line pa + Default value: `3` + Unit: second -### `tso-save-interval` - -+ The interval for PD to allocate TSOs for persistent storage in etcd -+ Default value: `3` -+ Unit: second - -### `enable-prevote` - -+ Enables or disables `raft prevote` -+ Default value: `true` - ### `quota-backend-bytes` + The storage size of the meta-information database, which is 8GiB by default @@ -109,21 +98,6 @@ This document only describes parameters that are not included in command-line pa + Determines whether to force PD to start as a new cluster and modify the number of Raft members to `1` + Default value: `false` -### `tick-interval` - -+ The tick period of etcd Raft -+ Default value: `100ms` - -### `election-interval` - -+ The timeout for the etcd leader election -+ Default value: `3s` - -### `use-region-storage` - -+ Enables or disables independent Region storage -+ Default value: `false` - ## security Configuration items related to security @@ -143,6 +117,12 @@ Configuration items related to security + The path of the PEM file that contains the X509 key + Default value: "" +### `redact-info-log` New in v5.0 + ++ Controls whether to enable log redaction in the PD log. ++ When you set the configuration value to `true`, user data is redacted in the PD log. ++ Default value: `false` + ## `log` Configuration items related to log @@ -215,7 +195,7 @@ Configuration items related to scheduling ### `max-snapshot-count` -+ Control the maximum number of snapshots that a single store receives or sends at the same time. PD schedulers depend on this configuration to prevent the resources used for normal traffic from being preempted. ++ Controls the maximum number of snapshots that a single store receives or sends at the same time. PD schedulers depend on this configuration to prevent the resources used for normal traffic from being preempted. + Default value value: `3` ### `max-pending-peer-count` @@ -238,6 +218,16 @@ Configuration items related to scheduling + The number of Region scheduling tasks performed at the same time + Default value: `2048` +### `hot-region-schedule-limit` + ++ Controls the hot Region scheduling tasks that are running at the same time. It is independent of the Region scheduling. ++ Default value: `4` + +### `hot-region-cache-hits-threshold` + ++ The threshold used to set the number of minutes required to identify a hot Region. PD can participate in the hotspot scheduling only after the Region is in the hotspot state for more than this number of minutes. ++ Default value: `3` + ### `replica-schedule-limit` + The number of Replica scheduling tasks performed at the same time @@ -250,14 +240,14 @@ Configuration items related to scheduling ### `high-space-ratio` -+ The threshold ratio below which the capacity of the store is sufficient. This configuration takes effect only when `region-score-formula-version` is set to `v1`. ++ The threshold ratio below which the capacity of the store is sufficient. If the space occupancy ratio of the store is smaller than this threshold value, PD ignores the remaining space of the store when performing scheduling, and balances load mainly based on the Region size. This configuration takes effect only when `region-score-formula-version` is set to `v1`. + Default value: `0.7` + Minimum value: greater than `0` + Maximum value: less than `1` ### `low-space-ratio` -+ The threshold ratio above which the capacity of the store is insufficient ++ The threshold ratio above which the capacity of the store is insufficient. If the space occupancy ratio of a store exceeds this threshold value, PD avoids migrating data to this store as much as possible. Meanwhile, to avoid the disk space of the corresponding store being exhausted, PD performs scheduling mainly based on the remaining space of the store. + Default value: `0.8` + Minimum value: greater than `0` + Maximum value: less than `1` @@ -279,35 +269,10 @@ Configuration items related to scheduling + Default value: `v2` + Optional values: `v1` and `v2` -### `disable-remove-down-replica` - -+ Determines whether to disable the feature that automatically removes `DownReplica`. When this parameter is set to `true`, PD does not automatically clean up the copy in the down state. -+ Default value: `false` - -### `disable-replace-offline-replica` - -+ Determines whether to disable the feature that migrates `OfflineReplica`. When this parameter is set to `true`, PD does not migrate the replicas in the offline state. -+ Default value: `false` - -### `disable-make-up-replica` - -+ Determines whether to disable the feature that automatically supplements replicas. When this parameter is set to `true`, PD does not supplement replicas for the Region with insufficient replicas. -+ Default value: `false` +### `enable-joint-consensus` New in v5.0 -### `disable-remove-extra-replica` - -+ Determines whether to disable the feature that removes extra replicas. When this parameter is set to `true`, PD does not remove the extra replicas from the Region with excessive replicas. -+ Default value: `false` - -### `disable-location-replacement` - -+ Determines whether to disable isolation level check. When this parameter is set to `true`, PD does not increase the isolation level of the Region replicas through scheduling. -+ Default value: `false` - -### `store-balance-rate` - -+ Determines the maximum number of operations related to adding peers within a minute -+ Default value: `15` ++ Controls whether to use Joint Consensus for replica scheduling. If this configuration is disabled, PD schedules one replica at a time. ++ Default value: `true` ## `replication` @@ -315,7 +280,7 @@ Configuration items related to replicas ### `max-replicas` -+ The number of replicas ++ The number of replicas, that is, the sum of the number of leaders and followers. The default value `3` means 1 leader and 2 followers. When this configuration is modified online, PD will schedule Regions in the background so that the number of replicas matches this configuration. + Default value: `3` ### `location-labels` @@ -342,6 +307,11 @@ Configuration items related to replicas + See [Placement Rules](/configure-placement-rules.md). + An experimental feature of TiDB 4.0. +### `flow-round-by-digit` New in TiDB 5.1 + ++ Default value: 3 ++ PD rounds the lowest digits of the flow number, which reduces the update of statistics caused by the changes of the Region flow information. This configuration item is used to specify the number of lowest digits to round for the Region flow information. For example, the flow `100512` will be rounded to `101000` because the default value is `3`. This configuration replaces `trace-region-flow`. + ## `label-property` Configuration items related to labels diff --git a/pd-control.md b/pd-control.md index 0442be3234c9..fe003c7457ba 100644 --- a/pd-control.md +++ b/pd-control.md @@ -10,9 +10,13 @@ As a command line tool of PD, PD Control obtains the state information of the cl ## Install PD Control +> **Note:** +> +> It is recommended that the version of the Control tool you use is consistent with the version of the cluster. + ### Use TiUP command -To use PD Control, execute the `tiup ctl pd -u http://: [-i]` command. +To use PD Control, execute the `tiup ctl: pd -u http://: [-i]` command. ### Download TiDB installation package @@ -24,7 +28,7 @@ If you want to download the latest version of `pd-ctl`, directly download the Ti > **Note:** > -> `{version}` indicates the version number of TiDB. For example, if `{version}` is `v4.0.0-rc.2`, the package download link is `https://download.pingcap.org/tidb-v4.0.0-rc.2-linux-amd64.tar.gz`. You can also download the latest unpublished version by replacing `{version}` with `latest`. +> `{version}` indicates the version number of TiDB. For example, if `{version}` is `v5.1.0`, the package download link is `https://download.pingcap.org/tidb-v5.1.0-linux-amd64.tar.gz`. ### Compile from source code @@ -175,7 +179,7 @@ Usage: } >> config show cluster-version // Display the current version of the cluster, which is the current minimum version of TiKV nodes in the cluster and does not correspond to the binary version. -"4.0.0" +"5.1.0" ``` - `max-snapshot-count` controls the maximum number of snapshots that a single store receives or sends out at the same time. The scheduler is restricted by this configuration to avoid taking up normal application resources. When you need to improve the speed of adding replicas or balancing, increase this value. @@ -239,7 +243,7 @@ Usage: - `patrol-region-interval` controls the execution frequency that `replicaChecker` checks the health status of Regions. A shorter interval indicates a higher execution frequency. Generally, you do not need to adjust it. ```bash - >> config set patrol-region-interval 10ms // Set the execution frequency of replicaChecker to 10ms + >> config set patrol-region-interval 50ms // Set the execution frequency of replicaChecker to 50ms ``` - `max-store-down-time` controls the time that PD decides the disconnected store cannot be restored if exceeded. If PD does not receive heartbeats from a store within the specified period of time, PD adds replicas in other nodes. @@ -278,7 +282,7 @@ Usage: >> config set hot-region-schedule-limit 4 // 4 tasks of hot Region scheduling at the same time at most ``` -- `hot-region-cache-hits-threshold` is used to set the threshold of a hot Region. A Region is considered as hot only if the number of its cache hits exceeds this threshold. +- `hot-region-cache-hits-threshold` is used to set the number of minutes required to identify a hot Region. PD can participate in the hotspot scheduling only after the Region is in the hotspot state for more than this number of minutes. - `tolerant-size-ratio` controls the size of the balance buffer area. When the score difference between the leader or Region of the two stores is less than specified multiple times of the Region size, it is considered in balance by PD. @@ -322,10 +326,20 @@ Usage: - `enable-debug-metrics` is used to enable the metrics for debugging. When you set it to `true`, PD enables some metrics such as `balance-tolerant-size`. -- `enable-placement-rules` is used to enable placement rules. +- `enable-placement-rules` is used to enable placement rules, which is enabled by default in v5.0 and later versions. - `store-limit-mode` is used to control the mode of limiting the store speed. The optional modes are `auto` and `manual`. In `auto` mode, the stores are automatically balanced according to the load (experimental). +- PD rounds the lowest digits of the flow number, which reduces the update of statistics caused by the changes of the Region flow information. This configuration item is used to specify the number of lowest digits to round for the Region flow information. For example, the flow `100512` will be rounded to `101000` because the default value is `3`. This configuration replaces `trace-region-flow`. + +- For example, set the value of `flow-round-by-digit` to `4`: + + {{< copyable "" >}} + + ```bash + config set flow-round-by-digit 4 + ``` + #### `config placement-rules [disable | enable | load | save | show | rule-group]` For the usage of `config placement-rules [disable | enable | load | save | show | rule-group]`, see [Configure placement rules](/configure-placement-rules.md#configure-rules). @@ -838,6 +852,34 @@ logic: 120102 ... ``` +### Query all nodes whose status is not `Up` + +{{< copyable "" >}} + +```bash +» store --jq='.stores[].store | select(.state_name!="Up") | { id, address, state_name}' +``` + +``` +{"id":1,"address":"127.0.0.1:20161""state_name":"Offline"} +{"id":5,"address":"127.0.0.1:20162""state_name":"Offline"} +... +``` + +### Query all TiFlash nodes + +{{< copyable "" >}} + +```bash +» store --jq='.stores[].store | select(.labels | length>0 and contains([{"key":"engine","value":"tiflash"}])) | { id, address, state_name}' +``` + +``` +{"id":1,"address":"127.0.0.1:20161""state_name":"Up"} +{"id":5,"address":"127.0.0.1:20162""state_name":"Up"} +... +``` + ### Query the distribution status of the Region replicas ```bash diff --git a/pd-recover.md b/pd-recover.md index 7e1434861dec..255fa2f22f00 100644 --- a/pd-recover.md +++ b/pd-recover.md @@ -27,7 +27,7 @@ To download the latest version of PD Recover, directly download the TiDB package > **Note:** > -> `{version}` indicates the version number of TiDB. For example, if `{version}` is `v4.0.0`, the package download link is `https://download.pingcap.org/tidb-v4.0.0-linux-amd64.tar.gz`. You can also download the latest unpublished version by replacing `{version}` with `latest`. +> `{version}` indicates the version number of TiDB. For example, if `{version}` is `v5.1.0`, the package download link is `https://download.pingcap.org/tidb-v5.1.0-linux-amd64.tar.gz`. ## Quick Start @@ -89,7 +89,7 @@ The allocated ID value you specify must be larger than the currently largest all #### Get allocated ID from the monitor (recommended) To get allocated ID from the monitor, you need to make sure that the metrics you are viewing are the metrics of **the last PD leader**, and you can get the largest allocated ID from the **Current ID allocation** panel in PD dashboard. - + #### Get allocated ID from PD log To get the allocated ID from the PD log, you need to make sure that the log you are viewing is the log of **the last PD leader**, and you can get the maximum allocated ID by running the following command: diff --git a/pessimistic-transaction.md b/pessimistic-transaction.md index 636e75bc48a8..a51438c7acf3 100644 --- a/pessimistic-transaction.md +++ b/pessimistic-transaction.md @@ -103,7 +103,7 @@ Pessimistic transactions in TiDB behave similarly to those in MySQL. See the min 6. The data read by `EMBEDDED SELECT` in the statement is not locked. -7. Open transactions in TiDB do not block garbage collection (GC). By default, this limits the maximum execution time of pessimistic transactions to 10 minutes. You can modify this limit by editing `max-txn-ttl` under `[performance]` in the TiDB configuration file. +7. Open transactions in TiDB do not block garbage collection (GC). By default, this limits the maximum execution time of pessimistic transactions to 1 hour. You can modify this limit by editing `max-txn-ttl` under `[performance]` in the TiDB configuration file. ## Isolation level @@ -131,17 +131,17 @@ If the application logic relies on the locking or lock waiting mechanisms, or if ![Pipelined pessimistic lock](/media/pessimistic-transaction-pipelining.png) -This feature is disabled by default. To enable it, modify the TiKV configuration: +This feature is enabled by default. To disable it, modify the TiKV configuration: ```toml [pessimistic-txn] -pipelined = true +pipelined = false ``` -If the TiKV cluster is v4.0.9 or later, you can also dynamically enable this feature by [modifying TiKV configuration online](/dynamic-config.md#modify-tikv-configuration-online): +If the TiKV cluster is v4.0.9 or later, you can also dynamically disable this feature by [modifying TiKV configuration online](/dynamic-config.md#modify-tikv-configuration-online): {{< copyable "sql" >}} ```sql -set config tikv pessimistic-txn.pipelined='true'; +set config tikv pessimistic-txn.pipelined='false'; ``` diff --git a/post-installation-check.md b/post-installation-check.md index 282535b0dcfe..a44ba6300031 100644 --- a/post-installation-check.md +++ b/post-installation-check.md @@ -6,17 +6,25 @@ aliases: ['/docs/dev/post-installation-check/'] # Check Cluster Status -This document describes how to check the cluster status via [TiDB Dashboard](/dashboard/dashboard-intro.md) and Grafana, and how to log in to the TiDB database to perform simple DML and DDL operations and SQL queries. +After a TiDB cluster is deployed, you need to check whether the cluster runs normally. This document introduces how to check the cluster status using TiUP commands, [TiDB Dashboard](/dashboard/dashboard-intro.md) and Grafana, and how to log into the TiDB database to perform simple SQL operations. ## Check the TiDB cluster status -This section describes how to check the TiDB cluster status using [TiDB Dashboard](/dashboard/dashboard-intro.md) and Grafana. +This section describes how to check the TiDB cluster status using TiUP commands, [TiDB Dashboard](/dashboard/dashboard-intro.md), and Grafana. -### Use TiDB Dashboard +### Use TiUP -> **Warning:** -> -> TiDB Dashboard is still an experimental feature. It is **NOT** recommended that you use it in the production environment. +Use the `tiup cluster display ` command to check the cluster status. For example: + +{{< copyable "shell-regular" >}} + +```shell +tiup cluster display tidb-test +``` + +Expected output: If the `Status` information of each node is `Up`, the cluster runs normally. + +### Use TiDB Dashboard 1. Log in to TiDB Dashboard at `${pd-ip}:${pd-port}/dashboard`. The username and password is the same as that of the TiDB `root` user. If you have modified the `root` password, enter the modified password. The password is empty by default. @@ -53,7 +61,7 @@ The following information indicates successful login: ```sql Welcome to the MySQL monitor. Commands end with ; or \g. Your MySQL connection id is 3 -Server version: 5.7.25-TiDB-v4.0.0 TiDB Server (Apache License 2.0) Community Edition, MySQL 5.7 compatible +Server version: 5.7.25-TiDB-v5.0.0 TiDB Server (Apache License 2.0) Community Edition, MySQL 5.7 compatible Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective @@ -76,7 +84,7 @@ Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. ```sql *************************** 1. row *************************** - tidb_version(): Release Version: v4.0.0 + tidb_version(): Release Version: v5.0.0 Edition: Community Git Commit Hash: 689a6b6439ae7835947fcaccf329a3fc303986cb Git Branch: HEAD @@ -141,7 +149,7 @@ Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. {{< copyable "sql" >}} ```sql - insert into `tab_tidb` values (1,'TiDB',5,'TiDB-v4.0.0'); + insert into `tab_tidb` values (1,'TiDB',5,'TiDB-v5.0.0'); ``` Expected output: @@ -164,7 +172,7 @@ Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. +----+------+-----+-------------+ | id | name | age | version | +----+------+-----+-------------+ - | 1 | TiDB | 5 | TiDB-v4.0.0 | + | 1 | TiDB | 5 | TiDB-v5.0.0 | +----+------+-----+-------------+ 1 row in set (0.00 sec) ``` diff --git a/predicate-push-down.md b/predicate-push-down.md index c2be2f77c630..b574db739fed 100644 --- a/predicate-push-down.md +++ b/predicate-push-down.md @@ -108,7 +108,7 @@ explain select * from t left join s on t.a = s.a where s.a is null; 6 rows in set (0.00 sec) ``` -In this query，there is a predicate `s.a is null` on the inner table `s`。 +In this query，there is a predicate `s.a is null` on the inner table `s`. From the `explain` results，we can see that the predicate is not pushed below join operator. This is because the outer join fills the inner table with `NULL` values when the `on` condition isn't satisfied, and the predicate `s.a is null` is used to filter the results after the join. If it is pushed down to the inner table below join, the execution plan is not equivalent to the original one. diff --git a/privilege-management.md b/privilege-management.md index e3384fc2d98a..9fe98feafb09 100644 --- a/privilege-management.md +++ b/privilege-management.md @@ -6,7 +6,10 @@ aliases: ['/docs/dev/privilege-management/','/docs/dev/reference/security/privil # Privilege Management -TiDB supports MySQL 5.7's privilege management system, including the syntax and privilege types. Starting with TiDB 3.0, support for SQL Roles is also available. +TiDB supports MySQL 5.7's privilege management system, including the syntax and privilege types. The following features from MySQL 8.0 are also supported: + +* SQL Roles, starting with TiDB 3.0. +* Dynamic privileges, starting with TiDB 5.1. This document introduces privilege-related TiDB operations, privileges required for TiDB operations and implementation of the privilege system. @@ -48,7 +51,7 @@ Empty set (0.00 sec) mysql> GRANT ALL PRIVILEGES ON test.* TO 'idontexist'; ERROR 1105 (HY000): You are not allowed to create a user with GRANT -mysql> SELECT user,host,password FROM mysql.user WHERE user='idontexist'; +mysql> SELECT user,host,authentication_string FROM mysql.user WHERE user='idontexist'; Empty set (0.00 sec) ``` @@ -72,13 +75,13 @@ Empty set (0.00 sec) mysql> GRANT ALL PRIVILEGES ON test.* TO 'idontexist'; Query OK, 1 row affected (0.05 sec) -mysql> SELECT user,host,password FROM mysql.user WHERE user='idontexist'; -+------------+------+----------+ -| user | host | password | -+------------+------+----------+ -| idontexist | % | | -+------------+------+----------+ -1 row in set (0.00 sec) +mysql> SELECT user,host,authentication_string FROM mysql.user WHERE user='idontexist'; ++------------+------+-----------------------+ +| user | host | authentication_string | ++------------+------+-----------------------+ +| idontexist | % | | ++------------+------+-----------------------+ +1 row in set (0.01 sec) ``` You can use fuzzy matching in `GRANT` to grant privileges to databases. @@ -181,35 +184,63 @@ SHOW GRANTS FOR `rw_user`@`192.168.%`; +------------------------------------------------------------------+ ``` +### Dynamic privileges + +Since v5.1, TiDB features support dynamic privileges, a feature borrowed from MySQL 8.0. Dynamic privileges are intended to replace the `SUPER` privilege by implementing more fine-grained access to certain operations. For example, using dynamic privileges, system administrators can create a user account that can only perform `BACKUP` and `RESTORE` operations. + +Dynamic privileges include: + +* `BACKUP_ADMIN` +* `RESTORE_ADMIN` +* `ROLE_ADMIN` +* `CONNECTION_ADMIN` +* `SYSTEM_VARIABLES_ADMIN` + +To see the full set of dynamic privileges, execute the `SHOW PRIVILEGES` statement. Because plugins are permitted to add new privileges, the list of privileges that are assignable might differ based on your TiDB installation. + ## Privileges required for TiDB operations -You can check privileges of TiDB users in the `INFORMATION_SCHEMA.USER_PRIVILEGES` table. - -| Privilege type | Privilege variable | Privilege description | -| :------------ | :------------ | :---------------------- | -| ALL | `AllPriv` | All the privileges | -| Drop | `DropPriv` | Deletes a schema or table | -| Index | `IndexPriv` | Creates or deletes an index | -| Alter | `AlterPriv` | Executes the `ALTER` statement | -| Super | `SuperPriv` | All the privileges | -| Grant | `GrantPriv` | Grants another user a privilege | -| Create | `CreatePriv` | Creates a schema or table | -| Select | `SelectPriv` | Reads the table data | -| Insert | `InsertPriv` | Inserts data to a table | -| Update | `UpdatePriv` | Updates the table data | -| Delete | `DeletePriv` | Deleted the table data | -| Reload | `ReloadPriv` | Executes the `FLUSH` statement | -| Config | `ConfigPriv` | Dynamically reloads configuration | -| Trigger | `TriggerPriv` | / | -| Process | `ProcessPriv` | Displays the running task | -| Execute | `ExecutePriv` | Executes the `EXECUTE` statement | -| Drop Role | `DropRolePriv` | Executes `DROP ROLE` | -| Show View | `ShowViewPriv` | Executes `SHOW CREATE VIEW` | -| References | `ReferencesPriv` | / | -| Create View | `CreateViewPriv` | Creates a View | -| Create User | `CreateUserPriv` | Creates a user | -| Create Role | `CreateRolePriv` | Executes `CREATE ROLE` | -| Show Databases | `ShowDBPriv` | Shows the table status in the database | +You can check privileges of TiDB users in the `INFORMATION_SCHEMA.USER_PRIVILEGES` table. For example: + +```sql +mysql> SELECT * FROM INFORMATION_SCHEMA.USER_PRIVILEGES WHERE grantee = "'root'@'%'"; ++------------+---------------+-------------------------+--------------+ +| GRANTEE | TABLE_CATALOG | PRIVILEGE_TYPE | IS_GRANTABLE | ++------------+---------------+-------------------------+--------------+ +| 'root'@'%' | def | Select | YES | +| 'root'@'%' | def | Insert | YES | +| 'root'@'%' | def | Update | YES | +| 'root'@'%' | def | Delete | YES | +| 'root'@'%' | def | Create | YES | +| 'root'@'%' | def | Drop | YES | +| 'root'@'%' | def | Process | YES | +| 'root'@'%' | def | References | YES | +| 'root'@'%' | def | Alter | YES | +| 'root'@'%' | def | Show Databases | YES | +| 'root'@'%' | def | Super | YES | +| 'root'@'%' | def | Execute | YES | +| 'root'@'%' | def | Index | YES | +| 'root'@'%' | def | Create User | YES | +| 'root'@'%' | def | Create Tablespace | YES | +| 'root'@'%' | def | Trigger | YES | +| 'root'@'%' | def | Create View | YES | +| 'root'@'%' | def | Show View | YES | +| 'root'@'%' | def | Create Role | YES | +| 'root'@'%' | def | Drop Role | YES | +| 'root'@'%' | def | CREATE TEMPORARY TABLES | YES | +| 'root'@'%' | def | LOCK TABLES | YES | +| 'root'@'%' | def | CREATE ROUTINE | YES | +| 'root'@'%' | def | ALTER ROUTINE | YES | +| 'root'@'%' | def | EVENT | YES | +| 'root'@'%' | def | SHUTDOWN | YES | +| 'root'@'%' | def | RELOAD | YES | +| 'root'@'%' | def | FILE | YES | +| 'root'@'%' | def | CONFIG | YES | +| 'root'@'%' | def | REPLICATION CLIENT | YES | +| 'root'@'%' | def | REPLICATION SLAVE | YES | ++------------+---------------+-------------------------+--------------+ +31 rows in set (0.00 sec) +``` ### ALTER @@ -222,6 +253,10 @@ You can check privileges of TiDB users in the `INFORMATION_SCHEMA.USER_PRIVILEGE > > In MySQL 5.7 documentation, users need `INSERT` and `CREATE` privileges to perform the `ALTER` operation on a table. But in reality for MySQL 5.7.25, only the `ALTER` privilege is required in this case. Currently, the `ALTER` privilege in TiDB is consistent with the actual behavior in MySQL. +### BACKUP + +Requires the `SUPER` or `BACKUP_ADMIN` privilege. + ### CREATE DATABASE Requires the `CREATE` privilege for the database. @@ -280,6 +315,8 @@ Requires the `INSERT` and `SELECT` privileges for the table. `SHOW GRANTS` requires the `SELECT` privilege to the `mysql` database. If the target user is current user, `SHOW GRANTS` does not require any privilege. +`SHOW PROCESSLIST` requires `SUPER` to show connections belonging to other users. + ### CREATE ROLE/USER `CREATE ROLE` requires the `CREATE ROLE` privilege. @@ -302,17 +339,17 @@ Requires the `GRANT` privilege with the privileges granted by `GRANT`. Requires additional `CREATE USER` privilege to create a user implicitly. -`GRANT ROLE` requires `SUPER` privilege. +`GRANT ROLE` requires `SUPER` or `ROLE_ADMIN` privilege. ### REVOKE Requires the `GRANT` privilege and those privileges targeted by the `REVOKE` statement. -`REVOKE ROLE` requires `SUPER` privilege. +`REVOKE ROLE` requires `SUPER` or `ROLE_ADMIN` privilege. ### SET GLOBAL -Requires `SUPER` privilege to set global variables. +Requires `SUPER` or `SYSTEM_VARIABLES_ADMIN` privilege to set global variables. ### ADMIN @@ -324,7 +361,7 @@ Requires `SUPER` privilege. ### KILL -Requires `SUPER` privilege to kill other user sessions. +Requires `SUPER` or `CONNECTION_ADMIN` privilege to kill other user sessions. ## Implementation of the privilege system @@ -355,11 +392,11 @@ In this record, `Host` and `User` determine that the connection request sent by > **Note:** > -> It is recommended to only update the privilege tables via the supplied syntax such as `GRANT`, `CREATE USER` and `DROP USER`. Making direct edits to the underlying privilege tables will not automatially update the privilege cache, leading to unpredictable behavior until `FLUSH PRIVILEGES` is executed. +> It is recommended to only update the privilege tables via the supplied syntax such as `GRANT`, `CREATE USER` and `DROP USER`. Making direct edits to the underlying privilege tables will not automatically update the privilege cache, leading to unpredictable behavior until `FLUSH PRIVILEGES` is executed. ### Connection verification -When the client sends a connection request, TiDB server will verify the login operation. TiDB server first checks the `mysql.user` table. If a record of `User` and `Host` matches the connection request, TiDB server then verifies the `Password`. +When the client sends a connection request, TiDB server will verify the login operation. TiDB server first checks the `mysql.user` table. If a record of `User` and `Host` matches the connection request, TiDB server then verifies the `authentication_string`. User identity is based on two pieces of information: `Host`, the host that initiates the connection, and `User`, the user name. If the user name is not empty, the exact match of user named is a must. @@ -388,19 +425,3 @@ Manually editing tables such as `mysql.user` with statements such as `INSERT`, ` ```sql FLUSH PRIVILEGES; ``` - -### Limitations and constraints - -Currently, the following privileges are not checked yet because they are less frequently used: - -- FILE -- USAGE -- SHUTDOWN -- EXECUTE -- PROCESS -- INDEX -- ... - -> **Note:** -> -> Column-level privileges are not implemented at this stage. diff --git a/production-deployment-from-binary-tarball.md b/production-deployment-from-binary-tarball.md deleted file mode 100644 index 5aaaf0eba08b..000000000000 --- a/production-deployment-from-binary-tarball.md +++ /dev/null @@ -1,207 +0,0 @@ ---- -title: Production Deployment from Binary Tarball -summary: Use the binary to deploy a TiDB cluster. -aliases: ['/docs/dev/production-deployment-from-binary-tarball/','/docs/dev/how-to/deploy/from-tarball/production-environment/'] ---- - -# Production Deployment from Binary Tarball - -This guide provides installation instructions from a binary tarball on Linux. A complete TiDB cluster contains PD, TiKV, and TiDB. To start the database service, follow the order of PD -> TiKV -> TiDB. To stop the database service, follow the order of stopping TiDB -> TiKV -> PD. - -See also [local deployment](/deploy-tidb-from-binary.md) and [testing environment](/test-deployment-from-binary-tarball.md) deployment. - -## Prepare - -Before you start, see [TiDB architecture](/tidb-architecture.md) and [Software and Hardware Recommendations](/hardware-and-software-requirements.md). Make sure the following requirements are satisfied: - -### Operating system - -For the operating system, it is recommended to use RHEL/CentOS 7.3 or higher. The following additional requirements are recommended: - -| Configuration | Description | -| :-- | :-------------------- | -| Supported Platform | RHEL/CentOS 7.3+ ([more details](/hardware-and-software-requirements.md)) | -| File System | ext4 is recommended | -| Swap Space | Should be disabled | -| Disk Block Size | Set the system disk `Block` size to `4096` | - -### Network and firewall - -| Configuration | Description | -| :-- | :------------------- | -| Firewall/Port | Check whether the ports required by TiDB are accessible between the nodes | - -### Operating system parameters - -| Configuration | Description | -| :-- | :-------------------------- | -| Nice Limits | For system users, set the default value of `nice` in TiDB to `0` | -| min_free_kbytes | The setting for `vm.min_free_kbytes` in `sysctl.conf` needs to be high enough | -| User Open Files Limit | For database administrators, set the number of TiDB open files to `1000000` | -| System Open File Limits | Set the number of system open files to `1000000` | -| User Process Limits | For TiDB users, set the `nproc` value to `4096` in `limits.conf` | -| Address Space Limits | For TiDB users, set the space to `unlimited` in `limits.conf` | -| File Size Limits | For TiDB users, set the `fsize` value to `unlimited` in `limits.conf` | -| Disk Readahead | Set the value of the `readahead` data disk to `4096` at a minimum | -| NTP service | Configure the NTP time synchronization service for each node | -| SELinux | Turn off the SELinux service for each node | -| CPU Frequency Scaling | It is recommended to turn on CPU overclocking | -| Transparent Hugepages | For Red Hat 7+ and CentOS 7+ systems, it is required to set the Transparent Hugepages to `always` | -| I/O Scheduler | Set the I/O Scheduler of data disks to the `deadline` mode | -| vm.swappiness | Set `vm.swappiness = 0` in `sysctl.conf` | -| net.core.somaxconn | Set `net.core.somaxconn = 32768` in `sysctl.conf` | -| net.ipv4.tcp_syncookies | Set `net.ipv4.tcp_syncookies = 0` in `sysctl.conf` | - -### Database running user settings - -| Configuration | Description | -| :-- | :---------------------------- | -| LANG environment | Set `LANG = en_US.UTF8` | -| TZ time zone | Set the TZ time zone of all nodes to the same value | - -## TiDB components and default ports - -Before you deploy a TiDB cluster, see the [required components](#tidb-database-components-required) and [optional components](#tidb-database-components-optional). - -### TiDB database components (required) - -See the following table for the default ports for the TiDB components: - -| Component | Default Port | Protocol | Description | -| :-- | :-- | :-- | :----------- | -| ssh | 22 | TCP | the sshd service | -| TiDB| 4000 | TCP | the communication port for the application and DBA tools | -| TiDB| 10080 | TCP | the communication port to report TiDB status | -| TiKV| 20160 | TCP | the TiKV communication port | -| PD | 2379 | TCP | the communication port between TiDB and PD | -| PD | 2380 | TCP | the inter-node communication port within the PD cluster | - -### TiDB database components (optional) - -See the following table for the default ports for the optional TiDB components: - -| Component | Default Port | Protocol | Description | -| :-- | :-- | :-- | :------------------------ | -| Prometheus | 9090| TCP | the communication port for the Prometheus service | -| Pushgateway | 9091 | TCP | the aggregation and report port for TiDB, TiKV, and PD monitor | -| Node_exporter| 9100| TCP | the communication port to report the system information of every TiDB cluster node | -| Grafana | 3000 | TCP | the port for the external Web monitoring service and client (Browser) access | -| alertmanager | 9093 | TCP | the port for the alert service | - -## Create a database running user account - -1. Log in to the machine using the `root` user account and create a database running user account (`tidb`) using the following command: - - ```bash - # useradd tidb -m - ``` - -2. Switch the user from `root` to `tidb` by using the following command. You can use this `tidb` user account to deploy your TiDB cluster. - - ```bash - # su - tidb - ``` - -## Download the official binary package - -``` -# Download the package. -$ wget https://download.pingcap.org/tidb-latest-linux-amd64.tar.gz -$ wget https://download.pingcap.org/tidb-latest-linux-amd64.sha256 - -# Check the file integrity. If the result is OK, the file is correct. -$ sha256sum -c tidb-latest-linux-amd64.sha256 - -# Extract the package. -$ tar -xzf tidb-latest-linux-amd64.tar.gz -$ cd tidb-latest-linux-amd64 -``` - -## Multiple nodes cluster deployment - -For the production environment, multiple nodes cluster deployment is recommended. Before you begin, see [Software and Hardware Recommendations](/hardware-and-software-requirements.md). - -Assuming that you have six nodes, you can deploy 3 PD instances, 3 TiKV instances, and 1 TiDB instance. See the following table for details: - -| Name | Host IP | Services | -| :-- | :-- | :-------------- | -| Node1 | 192.168.199.113| PD1, TiDB | -| Node2 | 192.168.199.114| PD2 | -| Node3 | 192.168.199.115| PD3 | -| Node4 | 192.168.199.116| TiKV1 | -| Node5 | 192.168.199.117| TiKV2 | -| Node6 | 192.168.199.118| TiKV3 | - -Follow the steps below to start PD, TiKV, and TiDB: - -1. Start PD on Node1, Node2, and Node3 in sequence. - - ```bash - $ ./bin/pd-server --name=pd1 \ - --data-dir=pd \ - --client-urls="http://192.168.199.113:2379" \ - --peer-urls="http://192.168.199.113:2380" \ - --initial-cluster="pd1=http://192.168.199.113:2380,pd2=http://192.168.199.114:2380,pd3=http://192.168.199.115:2380" \ - -L "info" \ - --log-file=pd.log & - - $ ./bin/pd-server --name=pd2 \ - --data-dir=pd \ - --client-urls="http://192.168.199.114:2379" \ - --peer-urls="http://192.168.199.114:2380" \ - --initial-cluster="pd1=http://192.168.199.113:2380,pd2=http://192.168.199.114:2380,pd3=http://192.168.199.115:2380" \ - -L "info" \ - --log-file=pd.log & - - $ ./bin/pd-server --name=pd3 \ - --data-dir=pd \ - --client-urls="http://192.168.199.115:2379" \ - --peer-urls="http://192.168.199.115:2380" \ - --initial-cluster="pd1=http://192.168.199.113:2380,pd2=http://192.168.199.114:2380,pd3=http://192.168.199.115:2380" \ - -L "info" \ - --log-file=pd.log & - ``` - -2. Start TiKV on Node4, Node5 and Node6. - - ```bash - $ ./bin/tikv-server --pd="192.168.199.113:2379,192.168.199.114:2379,192.168.199.115:2379" \ - --addr="192.168.199.116:20160" \ - --status-addr="192.168.199.116:20180" \ - --data-dir=tikv \ - --log-file=tikv.log & - - $ ./bin/tikv-server --pd="192.168.199.113:2379,192.168.199.114:2379,192.168.199.115:2379" \ - --addr="192.168.199.117:20160" \ - --status-addr="192.168.199.117:20180" \ - --data-dir=tikv \ - --log-file=tikv.log & - - $ ./bin/tikv-server --pd="192.168.199.113:2379,192.168.199.114:2379,192.168.199.115:2379" \ - --addr="192.168.199.118:20160" \ - --status-addr="192.168.199.118:20180" \ - --data-dir=tikv \ - --log-file=tikv.log & - ``` - -3. Start TiDB on Node1. - - ```bash - $ ./bin/tidb-server --store=tikv \ - --path="192.168.199.113:2379,192.168.199.114:2379,192.168.199.115:2379" \ - --log-file=tidb.log & - ``` - -4. Use the MySQL client to connect to TiDB. - - ```sh - $ mysql -h 192.168.199.113 -P 4000 -u root -D test - ``` - -> **Note:** -> -> - If you start TiKV or deploy PD in the production environment, it is highly recommended to specify the path for the configuration file using the `--config` parameter. If the parameter is not set, TiKV or PD does not read the configuration file. -> - To tune TiKV, see [Performance Tuning for TiKV](/tune-tikv-memory-performance.md). -> - If you use `nohup` to start the cluster in the production environment, write the startup commands in a script and then run the script. If not, the `nohup` process might abort because it receives exceptions when the Shell command exits. For more information, see [The TiDB/TiKV/PD process aborts unexpectedly](/troubleshoot-tidb-cluster.md#the-tidbtikvpd-process-aborts-unexpectedly). - -For the deployment and use of TiDB monitoring services, see [Deploy Monitoring Services for the TiDB Cluster](/deploy-monitoring-services.md) and [TiDB Monitoring API](/tidb-monitoring-api.md). diff --git a/production-deployment-using-tiup.md b/production-deployment-using-tiup.md index 0f4304375c87..73bc55696759 100644 --- a/production-deployment-using-tiup.md +++ b/production-deployment-using-tiup.md @@ -1,7 +1,7 @@ --- title: Deploy a TiDB Cluster Using TiUP summary: Learn how to easily deploy a TiDB cluster using TiUP. -aliases: ['/docs/dev/production-deployment-using-tiup/','/docs/dev/how-to/deploy/orchestrated/tiup/','/docs/dev/tiflash/deploy-tiflash/','/docs/dev/reference/tiflash/deploy/','/tidb/dev/deploy-tidb-from-dbdeployer/','/docs/dev/deploy-tidb-from-dbdeployer/','/docs/dev/how-to/get-started/deploy-tidb-from-dbdeployer/','/tidb/dev/deploy-tidb-from-homebrew/','/docs/dev/deploy-tidb-from-homebrew/','/docs/dev/how-to/get-started/deploy-tidb-from-homebrew/'] +aliases: ['/docs/dev/production-deployment-using-tiup/','/docs/dev/how-to/deploy/orchestrated/tiup/','/docs/dev/tiflash/deploy-tiflash/','/docs/dev/reference/tiflash/deploy/','/tidb/dev/deploy-tidb-from-dbdeployer/','/docs/dev/deploy-tidb-from-dbdeployer/','/docs/dev/how-to/get-started/deploy-tidb-from-dbdeployer/','/tidb/dev/deploy-tidb-from-homebrew/','/docs/dev/deploy-tidb-from-homebrew/','/docs/dev/how-to/get-started/deploy-tidb-from-homebrew/','/tidb/dev/production-offline-deployment-using-tiup','/docs/dev/production-offline-deployment-using-tiup/','/tidb/dev/deploy-tidb-from-binary','/tidb/dev/production-deployment-from-binary-tarball','/tidb/dev/test-deployment-from-binary-tarball','/tidb/dev/deploy-test-cluster-using-docker-compose','/tidb/dev/test-deployment-using-docker'] --- # Deploy a TiDB Cluster Using TiUP @@ -19,6 +19,10 @@ Make sure that you have read the following documents: ## Step 2: Install TiUP on the control machine +You can install TiUP on the control machine in either of the two ways: online deployment and offline deployment. + +### Method 1: Deploy TiUP online + Log in to the control machine using a regular user account (take the `tidb` user as an example). All the following TiUP installation and cluster management operations can be performed by the `tidb` user. 1. Install TiUP by executing the following command: @@ -73,13 +77,124 @@ Log in to the control machine using a regular user account (take the `tidb` user tiup --binary cluster ``` -## Step 3: Edit the initialization configuration file +### Method 2: Deploy TiUP offline + +Perform the following steps in this section to deploy a TiDB cluster offline using TiUP: + +#### Step 1: Prepare the TiUP offline component package + +To prepare the TiUP offline component package, manually pack an offline component package using `tiup mirror clone`. + +1. Install the TiUP package manager online. + + 1. Install the TiUP tool: + + {{< copyable "shell-regular" >}} + + ```shell + curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh + ``` + + 2. Redeclare the global environment variables: + + {{< copyable "shell-regular" >}} + + ```shell + source .bash_profile + ``` + + 3. Confirm whether TiUP is installed: + + {{< copyable "shell-regular" >}} + + ```shell + which tiup + ``` + +2. Pull the mirror using TiUP. + + 1. Pull the needed components on a machine that has access to the Internet: + + {{< copyable "shell-regular" >}} + + ```shell + tiup mirror clone tidb-community-server-${version}-linux-amd64 ${version} --os=linux --arch=amd64 + ``` + + The command above creates a directory named `tidb-community-server-${version}-linux-amd64` in the current directory, which contains the component package necessary for starting a cluster. + + 2. Pack the component package by using the `tar` command and send the package to the control machine in the isolated environment: + + {{< copyable "shell-regular" >}} + + ```bash + tar czvf tidb-community-server-${version}-linux-amd64.tar.gz tidb-community-server-${version}-linux-amd64 + ``` + + `tidb-community-server-${version}-linux-amd64.tar.gz` is an independent offline environment package. + +#### Step 2: Deploy the offline TiUP component + +After sending the package to the control machine of the target cluster, install the TiUP component by running the following commands: + +{{< copyable "shell-regular" >}} + +```bash +tar xzvf tidb-community-server-${version}-linux-amd64.tar.gz && \ +sh tidb-community-server-${version}-linux-amd64/local_install.sh && \ +source /home/tidb/.bash_profile +``` + +The `local_install.sh` script automatically executes the `tiup mirror set tidb-community-server-${version}-linux-amd64` command to set the current mirror address to `tidb-community-server-${version}-linux-amd64`. + +To switch the mirror to another directory, you can manually execute the `tiup mirror set ` command. To switch the mirror to the online environment, you can execute the `tiup mirror set https://tiup-mirrors.pingcap.com` command. + +## Step 3: Initialize cluster topology file According to the intended cluster topology, you need to manually create and edit the cluster initialization configuration file. -The following examples cover six common scenarios. You need to create a YAML configuration file (named `topology.yaml` for example) according to the topology description and templates in the corresponding links. For other scenarios, edit the configuration accordingly. +To create the cluster initialization configuration file, you can create a YAML-formatted configuration file on the control machine using TiUP: -The following topology documents provide a cluster configuration template for each of the following common scenarios: +{{< copyable "shell-regular" >}} + +```shell +tiup cluster template > topology.yaml +``` + +> **Note:** +> +> For the hybrid deployment scenarios, you can also execute `tiup cluster template --full > topology.yaml` to create the recommended topology template. For the geo-distributed deployment scenarios, you can execute `tiup cluster template --multi-dc > topology.yaml` to create the recommended topology template. + +Execute `vi topology.yaml` to see the configuration file content: + +```shell +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" +server_configs: {} +pd_servers: + - host: 10.0.1.4 + - host: 10.0.1.5 + - host: 10.0.1.6 +tidb_servers: + - host: 10.0.1.7 + - host: 10.0.1.8 + - host: 10.0.1.9 +tikv_servers: + - host: 10.0.1.1 + - host: 10.0.1.2 + - host: 10.0.1.3 +monitoring_servers: + - host: 10.0.1.4 +grafana_servers: + - host: 10.0.1.4 +alertmanager_servers: + - host: 10.0.1.4 +``` + +The following examples cover six common scenarios. You need to modify the configuration file (named `topology.yaml`) according to the topology description and templates in the corresponding links. For other scenarios, edit the configuration template accordingly. - [Minimal deployment topology](/minimal-deployment-topology.md) @@ -113,7 +228,7 @@ The following topology documents provide a cluster configuration template for ea > > - For parameters that should be globally effective, configure these parameters of corresponding components in the `server_configs` section of the configuration file. > - For parameters that should be effective on a specific node, configure these parameters in the `config` of this node. -> - Use `.` to indicate the subcategory of the configuration, such as `log.slow-threshold`. For more formats, see [TiUP configuration template](https://github.com/pingcap/tiup/blob/master/examples/topology.example.yaml). +> - Use `.` to indicate the subcategory of the configuration, such as `log.slow-threshold`. For more formats, see [TiUP configuration template](https://github.com/pingcap/tiup/blob/master/embed/templates/examples/topology.example.yaml). > - For more parameter description, see [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/master/config/config.toml.example), [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/master/etc/config-template.toml), [PD `config.toml.example`](https://github.com/pingcap/pd/blob/master/conf/config.toml), and [TiFlash configuration](/tiflash/tiflash-configuration.md). ## Step 4: Execute the deployment command @@ -131,20 +246,31 @@ The following topology documents provide a cluster configuration template for ea > - The user name configured in `topology.yaml` already exists on the target machine. > - You have used the `--skip-create-user` option in the command line to explicitly skip the step of creating the user. +Before you execute the `deploy` command, use the `check` and `check --apply` commands to detect and automatically repair the potential risks in the cluster: + +{{< copyable "shell-regular" >}} + +```shell +tiup cluster check ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] +tiup cluster check ./topology.yaml --apply --user root [-p] [-i /home/root/.ssh/gcp_rsa] +``` + +Then execute the `deploy` command to deploy the TiDB cluster: + {{< copyable "shell-regular" >}} ```shell -tiup cluster deploy tidb-test v4.0.0 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] +tiup cluster deploy tidb-test v5.1.0 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] ``` In the above command: - The name of the deployed TiDB cluster is `tidb-test`. -- The version of the TiDB cluster is `v4.0.0`. You can see other supported versions by running `tiup list tidb`. +- You can see the latest supported versions by running `tiup list tidb`. This document takes `v5.1.0` as an example. - The initialization configuration file is `topology.yaml`. - `--user root`: Log in to the target machine through the `root` key to complete the cluster deployment, or you can use other users with `ssh` and `sudo` privileges to complete the deployment. - `[-i]` and `[-p]`: optional. If you have configured login to the target machine without password, these parameters are not required. If not, choose one of the two parameters. `[-i]` is the private key of the `root` user (or other users specified by `--user`) that has access to the target machine. `[-p]` is used to input the user password interactively. -- If you need to specify the user group name to be created on the target machine, see [this example](https://github.com/pingcap/tiup/blob/master/examples/topology.example.yaml#L7). +- If you need to specify the user group name to be created on the target machine, see [this example](https://github.com/pingcap/tiup/blob/master/embed/templates/examples/topology.example.yaml#L7). At the end of the output log, you will see ```Deployed cluster `tidb-test` successfully```. This indicates that the deployment is successful. @@ -159,10 +285,10 @@ tiup cluster list TiUP supports managing multiple TiDB clusters. The command above outputs information of all the clusters currently managed by TiUP, including the name, deployment user, version, and secret key information: ```log -Starting /home/tidb/.tiup/components/cluster/v1.0.0/cluster list +Starting /home/tidb/.tiup/components/cluster/v1.5.0/cluster list Name User Version Path PrivateKey ---- ---- ------- ---- ---------- -tidb-test tidb v4.0.0 /home/tidb/.tiup/storage/cluster/clusters/tidb-test /home/tidb/.tiup/storage/cluster/clusters/tidb-test/ssh/id_rsa +tidb-test tidb v5.1.0 /home/tidb/.tiup/storage/cluster/clusters/tidb-test /home/tidb/.tiup/storage/cluster/clusters/tidb-test/ssh/id_rsa ``` ## Step 6: Check the status of the deployed TiDB cluster @@ -189,25 +315,7 @@ If the output log includes ```Started cluster `tidb-test` successfully```, the s ## Step 8: Verify the running status of the TiDB cluster -- Check the TiDB cluster status using TiUP: - - {{< copyable "shell-regular" >}} - - ```shell - tiup cluster display tidb-test - ``` - - If the `Status` is `Up` in the output, the cluster status is normal. - -- Log in to the database by running the following command: - - {{< copyable "shell-regular" >}} - - ```shell - mysql -u root -h 10.0.1.4 -P 4000 - ``` - -In addition, you also need to verify the status of the monitoring system, [TiDB Dashboard](/dashboard/dashboard-intro.md), and the execution of simple SQL commands. For the specific operations, see [Verify Cluster Status](/post-installation-check.md). +For the specific operations, see [Verify Cluster Status](/post-installation-check.md). ## What's next diff --git a/production-offline-deployment-using-tiup.md b/production-offline-deployment-using-tiup.md deleted file mode 100644 index 1cedf6f56e4a..000000000000 --- a/production-offline-deployment-using-tiup.md +++ /dev/null @@ -1,266 +0,0 @@ ---- -title: Deploy a TiDB Cluster Offline Using TiUP -summary: Introduce how to deploy a TiDB cluster offline using TiUP. -aliases: ['/docs/dev/production-offline-deployment-using-tiup/'] ---- - -# Deploy a TiDB Cluster Offline Using TiUP - -This document describes how to deploy a TiDB cluster offline using TiUP. - -## Step 1: Prepare the TiUP offline component package - -### Option 1: Download the official TiUP offline component package - -Download the offline mirror package of the TiDB server (including the TiUP offline component package) from the [Download TiDB](https://pingcap.com/download/) page. - -### Option 2: Manually pack an offline component package using `tiup mirror clone` - -The steps are below. - -- Install the TiUP package manager online. - - 1. Install the TiUP tool: - - {{< copyable "shell-regular" >}} - - ```shell - curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh - ``` - - 2. Redeclare the global environment variables: - - {{< copyable "shell-regular" >}} - - ```shell - source .bash_profile - ``` - - 3. Confirm whether TiUP is installed: - - {{< copyable "shell-regular" >}} - - ```shell - which tiup - ``` - -- Pull the mirror using TiUP - - 1. Pull the needed components on a machine that has access to the Internet: - - {{< copyable "shell-regular" >}} - - ```shell - tiup mirror clone tidb-community-server-${version}-linux-amd64 ${version} --os=linux --arch=amd64 - ``` - - The command above creates a directory named `tidb-community-server-${version}-linux-amd64` in the current directory, which contains the component package necessary for starting a cluster. - - 2. Pack the component package by using the `tar` command and send the package to the control machine in the isolated environment: - - {{< copyable "shell-regular" >}} - - ```bash - tar czvf tidb-community-server-${version}-linux-amd64.tar.gz tidb-community-server-${version}-linux-amd64 - ``` - - `tidb-community-server-${version}-linux-amd64.tar.gz` is an independent offline environment package. - -## Step 2: Deploy the offline TiUP component - -After sending the package to the control machine of the target cluster, install the TiUP component by running the following command: - -{{< copyable "shell-regular" >}} - -```bash -tar xzvf tidb-community-server-${version}-linux-amd64.tar.gz -sh tidb-community-server-${version}-linux-amd64/local_install.sh -source /home/tidb/.bash_profile -``` - -The `local_install.sh` script automatically executes the `tiup mirror set tidb-community-server-${version}-linux-amd64` command to set the current mirror address to `tidb-community-server-${version}-linux-amd64`. - -To switch the mirror to another directory, you can manually execute the `tiup mirror set ` command. To switch the mirror to the online environment, you can execute the `tiup mirror set https://tiup-mirrors.pingcap.com` command. - -## Step 3: Mount the TiKV data disk - -> **Note:** -> -> It is recommended to use the EXT4 file system format for the data directory of the target machines that deploy TiKV. Compared with the XFS file system format, the EXT4 file system format has more deployment cases of TiDB clusters. For the production environment, use the EXT4 file system format. - -Log in to the target machines using the `root` user account. - -Format your data disks to the ext4 filesystem and add the `nodelalloc` and `noatime` mount options to the filesystem. It is required to add the `nodelalloc` option, or else the TiUP deployment cannot pass the test. The `noatime` option is optional. - -> **Note:** -> -> If your data disks have been formatted to ext4 and have added the mount options, you can uninstall it by running the `umount /dev/nvme0n1p1` command, follow the steps starting from editing the `/etc/fstab` file, and add the options again to the filesystem. - -Take the `/dev/nvme0n1` data disk as an example: - -1. View the data disk: - - {{< copyable "shell-root" >}} - - ```bash - fdisk -l - ``` - - ``` - Disk /dev/nvme0n1: 1000 GB - ``` - -2. Create the partition table: - - {{< copyable "shell-root" >}} - - ```bash - parted -s -a optimal /dev/nvme0n1 mklabel gpt -- mkpart primary ext4 1 -1 - ``` - - > **Note:** - > - > Use the `lsblk` command to view the device number of the partition: for a `nvme` disk, the generated device number is usually `nvme0n1p1`; for a regular disk (for example, `/dev/sdb`), the generated device number is usually `sdb1`. - -3. Format the data disk to the ext4 filesystem: - - {{< copyable "shell-root" >}} - - ```bash - mkfs.ext4 /dev/nvme0n1p1 - ``` - -4. View the partition UUID of the data disk: - - In this example, the UUID of `nvme0n1p1` is `c51eb23b-195c-4061-92a9-3fad812cc12f`. - - {{< copyable "shell-root" >}} - - ```bash - lsblk -f - ``` - - ``` - NAME FSTYPE LABEL UUID MOUNTPOINT - sda - ├─sda1 ext4 237b634b-a565-477b-8371-6dff0c41f5ab /boot - ├─sda2 swap f414c5c0-f823-4bb1-8fdf-e531173a72ed - └─sda3 ext4 547909c1-398d-4696-94c6-03e43e317b60 / - sr0 - nvme0n1 - └─nvme0n1p1 ext4 c51eb23b-195c-4061-92a9-3fad812cc12f - ``` - -5. Edit the `/etc/fstab` file and add the mount options: - - {{< copyable "shell-root" >}} - - ```bash - vi /etc/fstab - ``` - - ``` - UUID=c51eb23b-195c-4061-92a9-3fad812cc12f /data1 ext4 defaults,nodelalloc,noatime 0 2 - ``` - -6. Mount the data disk: - - {{< copyable "shell-root" >}} - - ```bash - mkdir /data1 && \ - mount -a - ``` - -7. Check whether the steps above take effect by using the following command: - - {{< copyable "shell-root" >}} - - ```bash - mount -t ext4 - ``` - - If the filesystem is ext4 and `nodelalloc` is included in the mount options, you have successfully mount the data disk ext4 filesystem with options on the target machines. - - ``` - /dev/nvme0n1p1 on /data1 type ext4 (rw,noatime,nodelalloc,data=ordered) - ``` - -## Step 4: Edit the initialization configuration file `topology.yaml` - -You need to manually create and edit the cluster initialization configuration file. For the full configuration template, refer to the [TiUP configuration parameter template](https://github.com/pingcap/tiup/blob/master/examples/topology.example.yaml). - -Create a YAML configuration file on the control machine, such as `topology.yaml`: - -{{< copyable "shell-regular" >}} - -```shell -cat topology.yaml -``` - -```yaml -# # Global variables are applied to all deployments and used as the default value of -# # the deployments if a specific deployment value is missing. -global: - user: "tidb" - ssh_port: 22 - deploy_dir: "/tidb-deploy" - data_dir: "/tidb-data" - -server_configs: - pd: - replication.enable-placement-rules: true - -pd_servers: - - host: 10.0.1.4 - - host: 10.0.1.5 - - host: 10.0.1.6 -tidb_servers: - - host: 10.0.1.7 - - host: 10.0.1.8 - - host: 10.0.1.9 -tikv_servers: - - host: 10.0.1.1 - - host: 10.0.1.2 - - host: 10.0.1.3 -tiflash_servers: - - host: 10.0.1.10 - data_dir: /data1/tiflash/data,/data2/tiflash/data -cdc_servers: - - host: 10.0.1.6 - - host: 10.0.1.7 - - host: 10.0.1.8 -monitoring_servers: - - host: 10.0.1.4 -grafana_servers: - - host: 10.0.1.4 -alertmanager_servers: - - host: 10.0.1.4 -``` - -## Step 5: Deploy the TiDB cluster - -Execute the following command to deploy the TiDB cluster: - -{{< copyable "shell-regular" >}} - -```bash -tiup cluster deploy tidb-test v4.0.0 topology.yaml --user tidb [-p] [-i /home/root/.ssh/gcp_rsa] -tiup cluster start tidb-test -``` - -> **Parameter description:** -> -> - The name of the cluster deployed by the TiUP cluster is `tidb-test`. -> - The deployment version is `v4.0.0`. To obtain other supported versions, run `tiup list tidb`. -> - The initialization configuration file is `topology.yaml`. -> - `–user tidb`: log in to the target machine using the `tidb` user account to complete the cluster deployment. The `tidb` user needs to have `ssh` and `sudo` privileges of the target machine. You can use other users with `ssh` and `sudo` privileges to complete the deployment. -> - `[-i]` and `[-p]`: optional. If you have configured login to the target machine without password, these parameters are not required. If not, choose one of the two parameters. `[-i]` is the private key of the `root` user (or other users specified by `-user`) that has access to the target machine. `[-p]` is used to input the user password interactively. - -If you see the ``Deployed cluster `tidb-test` successfully`` output at the end of the log, the deployment is successful. - -After the deployment, see [Deploy and Maintain TiDB Using TiUP](/tiup/tiup-cluster.md) for the cluster operations. - -> **Note:** -> -> By default, TiDB and TiUP share usage details with PingCAP to help understand how to improve the product. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). diff --git a/quick-start-with-tidb.md b/quick-start-with-tidb.md index 56cac9c9468d..70b24fdc8814 100644 --- a/quick-start-with-tidb.md +++ b/quick-start-with-tidb.md @@ -6,21 +6,131 @@ aliases: ['/docs/dev/quick-start-with-tidb/','/docs/dev/test-deployment-using-do # Quick Start Guide for the TiDB Database Platform -> **Warning:** +This guide walks you through the quickest way to get started with TiDB. You will be using TiUP, a package manager in the TiDB ecosystem, to help you run any TiDB cluster component with only a single line of command. + +> **Note:** > -> For production environments, **do not** deploy TiDB with the following two methods. Instead, it is recommended that you [deploy TiDB using TiUP](/production-deployment-using-tiup.md). +> The deployment method provided in this guide is **ONLY FOR** quick start, **NOT FOR** production. +> +> - To deploy an on-premises production cluster, see [production installation guide](/production-deployment-using-tiup.md). +> - To deploy TiDB in Kubernetes, see [Get Started with TiDB in Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable/get-started). +> - To manage TiDB in the cloud, see [TiDB Cloud Quick Start](https://docs.pingcap.com/tidbcloud/beta/tidb-cloud-quickstart). + + +
+ +## Deploy a local test environment on Mac OS + +> **Note:** +> +> Currently, some TiDB components do not have a released version that supports the Apple M1 chip. Therefore, the `tiup playground` command currently cannot be executed on the local Mac machine that uses the Apple M1 chip. + +As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB instances, 3 TiKV instances, 3 PD instances, and optional TiFlash instances. With TiUP Playground, you can quickly build the test cluster by taking the following steps: + +1. Download and install TiUP: + + {{< copyable "shell-regular" >}} -This document describes how to try out TiDB, a distributed SQL database, in a few minutes. You can choose any of the following two methods to get started with the TiDB database platform: + ```shell + curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh + ``` -- [Deploy a local test environment using TiUP Playground](#deploy-a-local-test-environment-using-tiup-playground) -- [Set up a test environment on a single machine using TiUP Cluster](#set-up-a-test-environment-on-a-single-machine-using-tiup-cluster) +2. Declare the global environment variable: -## Deploy a local test environment using TiUP Playground + > **Note:** + > + > After the installation, TiUP displays the absolute path of the corresponding `profile` file. You need to modify the following `source` command according to the path. -- Scenario: Quickly deploy a TiDB cluster on a local Mac or Linux OS machine. You can see the basic architecture of TiDB and how TiDB, TiKV, PD, monitoring, and other components run in a TiDB cluster. -- Time required: 1 minute + {{< copyable "shell-regular" >}} + + ```shell + source .bash_profile + ``` -As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB instances, 3 TiKV instances, and 3 PD instances. With TiUP Playground, you can quickly build the above test cluster by taking the following steps: +3. Start the cluster in the current session: + + - If you want to start a TiDB cluster of the latest version with 1 TiDB instance, 1 TiKV instance, 1 PD instance, and 1 TiFlash instance, run the following command: + + {{< copyable "shell-regular" >}} + + ```shell + tiup playground + ``` + + - If you want to specify the TiDB version and the number of the instances of each component, run a command like this: + + {{< copyable "shell-regular" >}} + + ```shell + tiup playground v5.1.0 --db 2 --pd 3 --kv 3 --monitor + ``` + + The command downloads a version cluster to the local machine and starts it, such as v5.1.0. `--monitor` means that the monitoring component is also deployed. + + To view the latest version, run `tiup list tidb`. + + This command returns the access methods of the cluster: + + ```log + CLUSTER START SUCCESSFULLY, Enjoy it ^-^ + To connect TiDB: mysql --host 127.0.0.1 --port 4000 -u root + To connect TiDB: mysql --host 127.0.0.1 --port 4001 -u root + To view the dashboard: http://127.0.0.1:2379/dashboard + To view the monitor: http://127.0.0.1:9090 + ``` + + > **Note:** + > + > For the playground operated in this way, after the test deployment is finished, TiUP will clean up the original cluster data. You will get a new cluster after re-running the command. + > If you want the data to be persisted on storage，run `tiup --tag playground ...`. For details, refer to [TiUP Reference Guide](/tiup/tiup-reference.md#-t---tag). + +4. Start a new session to access TiDB: + + + Use the TiUP client to connect to TiDB. + + {{< copyable "shell-regular" >}} + + ```shell + tiup client + ``` + + + You can also use the MySQL client to connect to TiDB. + + {{< copyable "shell-regular" >}} + + ```shell + mysql --host 127.0.0.1 --port 4000 -u root + ``` + +5. Access the Prometheus dashboard of TiDB at . + +6. Access the [TiDB Dashboard](/dashboard/dashboard-intro.md) at . The default username is `root`, with an empty password. + +7. (Optional) [Load data to TiFlash](/tiflash/use-tiflash.md) for analysis. + +8. Clean up the cluster after the test deployment: + + 1. Stop the process by pressing `ctrl-c`. + + 2. Run the following command: + + {{< copyable "shell-regular" >}} + + ```shell + tiup clean --all + ``` + +> **Note:** +> +> TiUP Playground listens on `127.0.0.1` by default, and the service is only locally accessible. If you want the service to be externally accessible, specify the listening address using the `--host` parameter to bind the network interface card (NIC) to an externally accessible IP address. + +
+ +
+ +## Deploy a local test environment on Linux OS + +As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB instances, 3 TiKV instances, 3 PD instances, and optional TiFlash instances. With TiUP Playground, you can quickly build the test cluster by taking the following steps: 1. Download and install TiUP: @@ -44,7 +154,7 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in 3. Start the cluster in the current session: - - If you want to start a TiDB cluster of the latest version with 1 TiDB instance, 1 TiKV instance, and 1 PD instance, run the following command: + - If you want to start a TiDB cluster of the latest version with 1 TiDB instance, 1 TiKV instance, 1 PD instance, and 1 TiFlash instance, run the following command: {{< copyable "shell-regular" >}} @@ -57,10 +167,10 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v4.0.0 --db 2 --pd 3 --kv 3 --monitor + tiup playground v5.1.0 --db 2 --pd 3 --kv 3 --monitor ``` - The command downloads a v4.0.0 cluster to the local machine and starts it. `--monitor` means that the monitoring component is also deployed. + The command downloads a version cluster to the local machine and starts it, such as v5.1.0. `--monitor` means that the monitoring component is also deployed. To view the latest version, run `tiup list tidb`. @@ -74,17 +184,22 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in To view the monitor: http://127.0.0.1:9090 ``` + > **Note:** + > + > For the playground operated in this way, after the test deployment is finished, TiUP will clean up the original cluster data. You will get a new cluster after re-running the command. + > If you want the data to be persisted on storage，run `tiup --tag playground ...`. For details, refer to [TiUP Reference Guide](/tiup/tiup-reference.md#-t---tag). + 4. Start a new session to access TiDB: - 1. Install the MySQL client. If it is already installed, skip this step. + + Use the TiUP client to connect to TiDB. {{< copyable "shell-regular" >}} ```shell - yum -y install mysql + tiup client ``` - 2. Use the MySQL client to connect to TiDB. + + You can also use the MySQL client to connect to TiDB. {{< copyable "shell-regular" >}} @@ -96,7 +211,9 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in 6. Access the [TiDB Dashboard](/dashboard/dashboard-intro.md) at . The default username is `root`, with an empty password. -7. After the test, you can clean the cluster by taking the following steps: +7. (Optional) [Load data to TiFlash](/tiflash/use-tiflash.md) for analysis. + +8. Clean up the cluster after the test deployment: 1. Stop the process by pressing `ctrl-c`. @@ -110,7 +227,7 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in > **Note:** > -> TiUP Playground listens on `127.0.0.1` by default, and the service is only locally accessible; if you want the service to be externally accessible, you can specify `0.0.0.0` as the listening address using the `--host` parameter or bind the network interface card (NIC) to an externally accessible IP address. +> TiUP Playground listens on `127.0.0.1` by default, and the service is only locally accessible. If you want the service to be externally accessible, specify the listening address using the `--host` parameter to bind the network interface card (NIC) to an externally accessible IP address. ## Set up a test environment on a single machine using TiUP cluster @@ -128,6 +245,10 @@ Prepare a target machine that meets the following requirements: The smallest TiDB cluster topology is as follows: +**Note:** +> +> The IP address of the following instances only serves as an example IP. In your actual deployment, you need to replace the IP with your actual IP. + | Instance | Count | IP | Configuration | |:-- | :-- | :-- | :-- | | TiKV | 3 | 10.0.1.1
10.0.1.1
10.0.1.1 | Avoid conflict between the port and the directory | @@ -140,7 +261,7 @@ Other requirements for the target machine: - The `root` user and its password is required - [Stop the firewall service of the target machine](/check-before-deployment.md#check-and-stop-the-firewall-service-of-target-machines), or open the port needed by the TiDB cluster nodes -- Currently, TiUP only supports deploying TiDB on the x86_64 (AMD64) architecture (the ARM architecture will be supported in TiDB 4.0 GA): +- Currently, TiUP supports deploying TiDB on the x86_64 (AMD64 and ARM) architectures: - It is recommended to use CentOS 7.3 or later versions on AMD64 - It is recommended to use CentOS 7.6 1810 on ARM @@ -321,22 +442,27 @@ Other requirements for the target machine: tiup cluster display ``` +
+ + + ## What's next - If you have just deployed a TiDB cluster for the local test environment: - Learn [Basic SQL operations in TiDB](/basic-sql-operations.md) - [Migrate data to TiDB](/migration-overview.md) - - Learn [TiDB key features and scenarios](/overview.md) - - Learn [TiDB's architecture](/tidb-architecture.md) - - Learn [TiDB's compatibility with MySQL](/mysql-compatibility.md) - If you are ready to deploy a TiDB cluster for the production environment: - - [Deploy TiDB online using TiUP](/production-deployment-using-tiup.md) - - [Deploy TiDB offline using TiUP](/production-offline-deployment-using-tiup.md) + - [Deploy TiDB using TiUP](/production-deployment-using-tiup.md) - [Deploy TiDB on Cloud using TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/stable) +- If you're looking for analytics solution with TiFlash: + + - [Use TiFlash](/tiflash/use-tiflash.md) + - [TiFlash Overview](/tiflash/tiflash-overview.md) + > **Note:** > > By default, TiDB, TiUP and TiDB Dashboard share usage details with PingCAP to help understand how to improve the product. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). diff --git a/read-historical-data.md b/read-historical-data.md index 55cf5ec4f9f7..340deaf2b28a 100644 --- a/read-historical-data.md +++ b/read-historical-data.md @@ -1,35 +1,41 @@ --- -title: Read Historical Data -summary: Learn about how TiDB reads data from history versions. +title: Read Historical Data Using the System Variable `tidb_snapshot` +summary: Learn about how TiDB reads data from history versions using the system variable `tidb_snapshot`. aliases: ['/docs/dev/read-historical-data/','/docs/dev/how-to/get-started/read-historical-data/'] --- -# Read Historical Data +# Read Historical Data Using the System Variable `tidb_snapshot` -This document describes how TiDB reads data from the history versions, how TiDB manages the data versions, as well as an example to show how to use the feature. +This document describes how to read data from the history versions using the system variable `tidb_snapshot`, including specific usage examples and strategies for saving historical data. + +> **Note:** +> +> You can also use the [Stale Read](/stale-read.md) feature to read historical data, which is more recommended. ## Feature description -TiDB implements a feature to read history data using the standard SQL interface directly without special clients or drivers. By using this feature: +TiDB implements a feature to read history data using the standard SQL interface directly without special clients or drivers. -- Even when data is updated or removed, its history versions can be read using the SQL interface. -- Even if the table structure changes after the data is updated, TiDB can use the old structure to read the history data. +> **Note:** +> +> - Even when data is updated or removed, its history versions can be read using the SQL interface. +> - When reading historical data, TiDB returns the data with the old table structure even if the current table structure is different. ## How TiDB reads data from history versions -The `tidb_snapshot` system variable is introduced to support reading history data. About the `tidb_snapshot` variable: +The [`tidb_snapshot`](/system-variables.md#tidb_snapshot) system variable is introduced to support reading history data. About the `tidb_snapshot` variable: -- The variable is valid in the `Session` scope. -- Its value can be modified using the `Set` statement. +- The variable is valid in the `SESSION` scope. +- Its value can be modified using the `SET` statement. - The data type for the variable is text. - The variable accepts TSO (Timestamp Oracle) and datetime. TSO is a globally unique time service, which is obtained from PD. The acceptable datetime format is "2016-10-08 16:45:26.999". Generally, the datetime can be set using second precision, for example "2016-10-08 16:45:26". -- When the variable is set, TiDB creates a Snapshot using its value as the timestamp, just for the data structure and there is no any overhead. After that, all the `Select` operations will read data from this Snapshot. +- When the variable is set, TiDB creates a Snapshot using its value as the timestamp, just for the data structure and there is no any overhead. After that, all the `SELECT` operations will read data from this Snapshot. > **Note:** > > Because the timestamp in TiDB transactions is allocated by Placement Driver (PD), the version of the stored data is also marked based on the timestamp allocated by PD. When a Snapshot is created, the version number is based on the value of the `tidb_snapshot` variable. If there is a large difference between the local time of the TiDB server and the PD server, use the time of the PD server. -After reading data from history versions, you can read data from the latest version by ending the current Session or using the `Set` statement to set the value of the `tidb_snapshot` variable to "" (empty string). +After reading data from history versions, you can read data from the latest version by ending the current Session or using the `SET` statement to set the value of the `tidb_snapshot` variable to "" (empty string). ## How TiDB manages the data versions @@ -37,10 +43,10 @@ TiDB implements Multi-Version Concurrency Control (MVCC) to manage data versions In TiDB, Garbage Collection (GC) runs periodically to remove the obsolete data versions. For GC details, see [TiDB Garbage Collection (GC)](/garbage-collection-overview.md) -Pay special attention to the following two variables: +Pay special attention to the following: -- `tikv_gc_life_time`: It is used to configure the retention time of the history version. You can modify it manually. -- `tikv_gc_safe_point`: It records the current `safePoint`. You can safely create the snapshot to read the history data using the timestamp that is later than `safePoint`. `safePoint` automatically updates every time GC runs. +- [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50): This system variable is used to configure the retention time of earlier modifications (default: `10m0s`). +- The output of `SELECT * FROM mysql.tidb WHERE variable_name = 'tikv_gc_safe_point'`. This is the current `safePoint` where you can read historical data up to. It is updated every time the garbage collection process is run. ## Example @@ -152,3 +158,21 @@ Pay special attention to the following two variables: > **Note:** > > You should use `@@` instead of `@` before `tidb_snapshot` because `@@` is used to denote the system variable while `@` is used to denote the user variable. + +## How to restore historical data + +Before you restore data from an older version, make sure that Garbage Collection (GC) does not clear the history data while you are working on it. This can be done by setting the `tidb_gc_life_time` variable as the following example shows. Do not forget to set the variable back to the previous value after the restore. + +```sql +SET GLOBAL tidb_gc_life_time="60m"; +``` + +> **Note:** +> +> Increasing the GC life time from the default 10 minutes to half an hour or more will result in additional versions of rows being retained, which might require more disk space. This might also affect the performance of certain operations such as scans when TiDB needs to skip these additional versions of the same rows during data reads. + +To restore data from an older version, you can use one of the following methods: + +- For simple cases, use `SELECT` after setting the `tidb_snapshot` variable and copy-paste the output, or use `SELECT ... INTO LOCAL OUTFLE` and use `LOAD DATA` to import the data later on. + +- Use [Dumpling](/dumpling-overview.md#export-historical-data-snapshot-of-tidb) to export a historical snapshot. Dumpling performs well in exporting larger sets of data. diff --git a/releases/release-2.0.11.md b/releases/release-2.0.11.md index 1b3212ad86bd..63f60a9a15cb 100644 --- a/releases/release-2.0.11.md +++ b/releases/release-2.0.11.md @@ -17,5 +17,4 @@ On January 03, 2019, TiDB 2.0.11 is released. The corresponding TiDB Ansible 2.0 ## TiKV -- Fix two issues about Region merge -[#4003](https://github.com/tikv/tikv/pull/4003), [#4004](https://github.com/tikv/tikv/pull/4004) +- Fix two issues about Region merge [#4003](https://github.com/tikv/tikv/pull/4003), [#4004](https://github.com/tikv/tikv/pull/4004) diff --git a/releases/release-2.0.6.md b/releases/release-2.0.6.md index 174c65cf5aac..1c389a9079f5 100644 --- a/releases/release-2.0.6.md +++ b/releases/release-2.0.6.md @@ -20,7 +20,7 @@ On August 6, 2018, TiDB 2.0.6 is released. Compared with TiDB 2.0.5, this releas - Set the upper limit of placeholders count in the `PREPARE` statement to 65535, compatible with MySQL [#7250](https://github.com/pingcap/tidb/pull/7250) - Bug Fixes - Fix the issue that the `DROP USER` statement is incompatible with MySQL behavior in some cases [#7014](https://github.com/pingcap/tidb/pull/7014) - - Fix the issue that statements like `INSERT`/`LOAD DATA` meet OOM aftering opening `tidb_batch_insert` [#7092](https://github.com/pingcap/tidb/pull/7092) + - Fix the issue that statements like `INSERT`/`LOAD DATA` meet OOM after opening `tidb_batch_insert` [#7092](https://github.com/pingcap/tidb/pull/7092) - Fix the issue that the statistics fail to automatically update when the data of a table keeps updating [#7093](https://github.com/pingcap/tidb/pull/7093) - Fix the issue that the firewall breaks inactive gPRC connections [#7099](https://github.com/pingcap/tidb/pull/7099) - Fix the issue that prefix index returns a wrong result in some scenarios [#7126](https://github.com/pingcap/tidb/pull/7126) diff --git a/releases/release-2.1.12.md b/releases/release-2.1.12.md index af37d6c7d005..9f5b97a691a1 100644 --- a/releases/release-2.1.12.md +++ b/releases/release-2.1.12.md @@ -35,8 +35,7 @@ TiDB Ansible version: 2.1.12 ## PD -- Fix the issue that etcd leader election is blocked in extreme conditions -[#1576](https://github.com/pingcap/pd/pull/1576) +- Fix the issue that etcd leader election is blocked in extreme conditions [#1576](https://github.com/pingcap/pd/pull/1576) ## TiKV diff --git a/releases/release-2.1.2.md b/releases/release-2.1.2.md index b9ec87586e50..810feece1c73 100644 --- a/releases/release-2.1.2.md +++ b/releases/release-2.1.2.md @@ -19,8 +19,7 @@ On December 22, 2018, TiDB 2.1.2 is released. The corresponding TiDB Ansible 2.1 ## PD -- Fix the Region information update issue about Region merge -[#1377](https://github.com/pingcap/pd/pull/1377) +- Fix the Region information update issue about Region merge [#1377](https://github.com/pingcap/pd/pull/1377) ## TiKV diff --git a/releases/release-3.0.0-rc.3.md b/releases/release-3.0.0-rc.3.md index de11cfa0f889..fe2040a8cbf0 100644 --- a/releases/release-3.0.0-rc.3.md +++ b/releases/release-3.0.0-rc.3.md @@ -105,7 +105,7 @@ On June 21, 2019, TiDB 3.0.0-rc.3 is released. The corresponding TiDB Ansible ve + tikv-ctl - Add the `bad-regions` command to support checking more abnormal conditions [#4862](https://github.com/tikv/tikv/pull/4862) - - Add a feature of forcely executing the `tombstone` command [#4862](https://github.com/tikv/tikv/pull/4862) + - Add a feature of forcibly executing the `tombstone` command [#4862](https://github.com/tikv/tikv/pull/4862) + Misc - Add the `dist_release` compiling command [#4841](https://github.com/tikv/tikv/pull/4841) diff --git a/releases/release-3.0.3.md b/releases/release-3.0.3.md index 341c74738344..0d59196f71cb 100644 --- a/releases/release-3.0.3.md +++ b/releases/release-3.0.3.md @@ -22,7 +22,7 @@ TiDB Ansible version: 3.0.3 - Fix the issue that incorrect results for `ifnull` might be returned when Max/Min is used for type inferring with `NotNullFlag` retained [#11641](https://github.com/pingcap/tidb/pull/11641) - Fix the potential error that occurs when comparing bit type data in string form [#11660](https://github.com/pingcap/tidb/pull/11660) - Decrease the concurrency for data that requires sequential read to lower the possibility of OOM [#11679](https://github.com/pingcap/tidb/pull/11679) - - Fix the issue that incorrect type inferring might be caused when multiple parameters are unsigned for some built-in functions (e.g. `if`, `coalesce`) [#11621](https://github.com/pingcap/tidb/pull/11621) + - Fix the issue that incorrect type inferring might be caused when multiple parameters are unsigned for some built-in functions (for example, `if` and `coalesce`) [#11621](https://github.com/pingcap/tidb/pull/11621) - Fix the incompatibility with MySQL when the `Div` function handles unsigned decimal types [#11813](https://github.com/pingcap/tidb/pull/11813) - Fix the issue that panic might occur when executing SQL statements that modify the status of Pump/Drainer [#11827](https://github.com/pingcap/tidb/pull/11827) - Fix the issue that panic might occur for `select ... for update` when Autocommit = 1 and there is no `begin` statement [#11736](https://github.com/pingcap/tidb/pull/11736) diff --git a/releases/release-3.0.4.md b/releases/release-3.0.4.md index c260ae514bda..2c0298d52c18 100644 --- a/releases/release-3.0.4.md +++ b/releases/release-3.0.4.md @@ -35,6 +35,7 @@ TiDB Ansible version: 3.0.4 - Contributors Our thanks go to the following contributors from the community for helping this release: + - [sduzh](https://github.com/sduzh) - [lizhenda](https://github.com/lizhenda) diff --git a/releases/release-4.0.10.md b/releases/release-4.0.10.md new file mode 100644 index 000000000000..3b04eac7c854 --- /dev/null +++ b/releases/release-4.0.10.md @@ -0,0 +1,111 @@ +--- +title: TiDB 4.0.10 Release Notes +--- + +# TiDB 4.0.10 Release Notes + +Release date: January 15, 2021 + +TiDB version: 4.0.10 + +## New Features + ++ PD + + - Add the `enable-redact-log` configuration item to redact user data from logs [#3266](https://github.com/pingcap/pd/pull/3266) + ++ TiFlash + + - Add the `security.redact_info_log` configuration item to redact user data from logs + +## Improvements + ++ TiDB + + - Make the size limit of a key-value entry in transaction configurable using `txn-entry-size-limit` [#21843](https://github.com/pingcap/tidb/pull/21843) + ++ PD + + - Optimize the `store-state-filter` metrics to show more information [#3100](https://github.com/tikv/pd/pull/3100) + - Upgrade the `go.etcd.io/bbolt` dependency to v1.3.5 [#3331](https://github.com/tikv/pd/pull/3331) + ++ Tools + + + TiCDC + + - Enable the old value feature for the `maxwell` protocol [#1144](https://github.com/pingcap/ticdc/pull/1144) + - Enable the unified sorter feature by default [#1230](https://github.com/pingcap/ticdc/pull/1230) + + + Dumpling + + - Support checking unrecognized arguments and printing the current progress during dumping [#228](https://github.com/pingcap/dumpling/pull/228) + + + TiDB Lightning + + - Support retrying the error that occurs when reading from S3 [#533](https://github.com/pingcap/tidb-lightning/pull/533) + +## Bug Fixes + ++ TiDB + + - Fix a concurrency bug that might cause the batch client timeout [#22336](https://github.com/pingcap/tidb/pull/22336) + - Fix the issue of duplicate bindings caused by concurrent baseline capture [#22295](https://github.com/pingcap/tidb/pull/22295) + - Make the baseline capture bound to the SQL statement work when the log level is `'debug'` [#22293](https://github.com/pingcap/tidb/pull/22293) + - Correctly release GC locks when Region merge occurs [#22267](https://github.com/pingcap/tidb/pull/22267) + - Return correct values for user variables of the `datetime` type [#22143](https://github.com/pingcap/tidb/pull/22143) + - Fix the issue of using index merge when there are multiple table filters [#22124](https://github.com/pingcap/tidb/pull/22124) + - Fix the `wrong precision` issue in TiFlash caused by the `prepare` plan cache [#21960](https://github.com/pingcap/tidb/pull/21960) + - Fix the issue of incorrect results caused by schema change [#21596](https://github.com/pingcap/tidb/pull/21596) + - Avoid unnecessary column flag changes in `ALTER TABLE` [#21474](https://github.com/pingcap/tidb/pull/21474) + - Set the database name for table aliases of query blocks used in optimizer hints [#21380](https://github.com/pingcap/tidb/pull/21380) + - Generate the proper optimizer hint for `IndexHashJoin` and `IndexMergeJoin` [#21020](https://github.com/pingcap/tidb/pull/21020) + ++ TiKV + + - Fix the wrong mapping between ready and peer [#9409](https://github.com/tikv/tikv/pull/9409) + - Fix the issue that some logs are not redacted when `security.redact-info-log` is set to `true` [#9314](https://github.com/tikv/tikv/pull/9314) + ++ PD + + - Fix the issue that the ID allocation is not monotonic [#3308](https://github.com/tikv/pd/pull/3308) [#3323](https://github.com/tikv/pd/pull/3323) + - Fix the issue that the PD client might be blocked in some cases [#3285](https://github.com/pingcap/pd/pull/3285) + ++ TiFlash + + - Fix the issue that TiFlash fails to start because TiFlash fails to process the TiDB schema of an old version + - Fix the issue that TiFlash fails to start due to incorrect handling of `cpu_time` on the RedHat system + - Fix the issue that TiFlash fails to start when `path_realtime_mode` is set to `true` + - Fix an issue of incorrect results when calling the `substr` function with three parameters + - Fix the issue that TiFlash does not support changing the `Enum` type even if the change is lossless + ++ Tools + + + TiCDC + + - Fix the `maxwell` protocol issues, including the issue of `base64` data output and the issue of outputting TSO to unix timestamp [#1173](https://github.com/pingcap/ticdc/pull/1173) + - Fix a bug that outdated metadata might cause the newly created changefeed abnormal [#1184](https://github.com/pingcap/ticdc/pull/1184) + - Fix the issue of creating the receiver on the closed notifier [#1199](https://github.com/pingcap/ticdc/pull/1199) + - Fix a bug that the TiCDC owner might consume too much memory in the etcd watch client [#1227](https://github.com/pingcap/ticdc/pull/1227) + - Fix the issue that `max-batch-size` does not take effect [#1253](https://github.com/pingcap/ticdc/pull/1253) + - Fix the issue of cleaning up stale tasks before the capture information is constructed [#1280](https://github.com/pingcap/ticdc/pull/1280) + - Fix the issue that the recycling of db conn is block because `rollback` is not called in MySQL sink [#1285](https://github.com/pingcap/ticdc/pull/1285) + + + Dumpling + + - Avoid TiDB out of memory (OOM) by setting the default behavior of `tidb_mem_quota_query` [#233](https://github.com/pingcap/dumpling/pull/233) + + + Backup & Restore (BR) + + - Fix the issue that BR v4.0.9 cannot restore the files backed up using BR v4.0.8 on GCS [#688](https://github.com/pingcap/br/pull/688) + - Fix the issue that BR panics when the GCS storage URL has no prefix [#673](https://github.com/pingcap/br/pull/673) + - Disable backup statistics by default to avoid BR OOM [#693](https://github.com/pingcap/br/pull/693) + + + TiDB Binlog + + - Fix the issue that when the `AMEND TRANSACTION` feature is enabled, Drainer might choose the incorrect schema version to generate SQL statements [#1033](https://github.com/pingcap/tidb-binlog/pull/1033) + + + TiDB Lightning + + - Fix a bug that the Region is not split because the Region key is incorrectly encoded [#531](https://github.com/pingcap/tidb-lightning/pull/531) + - Fix the issue that the failure of `CREATE TABLE` might be lost when multiple tables are created [#530](https://github.com/pingcap/tidb-lightning/pull/530) + - Fix the issue of `column count mismatch` when using the TiDB-backend [#535](https://github.com/pingcap/tidb-lightning/pull/535) diff --git a/releases/release-4.0.11.md b/releases/release-4.0.11.md new file mode 100644 index 000000000000..c85859832e24 --- /dev/null +++ b/releases/release-4.0.11.md @@ -0,0 +1,169 @@ +--- +title: TiDB 4.0.11 Release Notes +--- + +# TiDB 4.0.11 Release Notes + +Release date: February 26, 2021 + +TiDB version: 4.0.11 + +## New Features + ++ TiDB + + - Support the `utf8_unicode_ci` and `utf8mb4_unicode_ci` collations [#22558](https://github.com/pingcap/tidb/pull/22558) + ++ TiKV + + - Support the `utf8mb4_unicode_ci` collation [#9577](https://github.com/tikv/tikv/pull/9577) + - Support the `cast_year_as_time` collation [#9299](https://github.com/tikv/tikv/pull/9299) + ++ TiFlash + + - Add a Coprocessor thread pool to queue Coprocessor requests for execution, which avoids out of memory (OOM) in some cases, and add the `cop_pool_size` and `batch_cop_pool_size` configuration items with the default values of `NumOfPhysicalCores * 2` + +## Improvements + ++ TiDB + + - Reorder inner joins that are simplified from outer joins [#22402](https://github.com/pingcap/tidb/pull/22402) + - Support multiple clusters in Grafana dashboards [#22534](https://github.com/pingcap/tidb/pull/22534) + - Add a workaround for the issue of multiple statements [#22468](https://github.com/pingcap/tidb/pull/22468) + - Divide the metrics of slow query into `internal` and `general` [#22405](https://github.com/pingcap/tidb/pull/22405) + - Add interface for `utf8_unicode_ci` and `utf8mb4_unicode_ci` collations [#22099](https://github.com/pingcap/tidb/pull/22099) + ++ TiKV + + - Add metrics of server information for DBaaS [#9591](https://github.com/tikv/tikv/pull/9591) + - Support multiple clusters in Grafana dashboards [#9572](https://github.com/tikv/tikv/pull/9572) + - Report RocksDB metrics to TiDB [#9316](https://github.com/tikv/tikv/pull/9316) + - Record the suspension time for Coprocessor tasks [#9277](https://github.com/tikv/tikv/pull/9277) + - Add thresholds of key counts and key size for Load Base Split [#9354](https://github.com/tikv/tikv/pull/9354) + - Check whether the file exists before data import [#9544](https://github.com/tikv/tikv/pull/9544) + - Improve Fast Tune panels [#9180](https://github.com/tikv/tikv/pull/9180) + ++ PD + + - Support multiple clusters in Grafana dashboards [#3398](https://github.com/pingcap/pd/pull/3398) + ++ TiFlash + + - Optimize the performance of the `date_format` function + - Optimize the memory consumption of handling ingest SST + - Optimize the retrying logic in Batch Coprocessor to reduce the probability of Region error + ++ Tools + + + TiCDC + + - Add the version information in the `capture` metadata and add the CLI version of a `changefeed` in the `changefeed` metadata [#1342](https://github.com/pingcap/ticdc/pull/1342) + + + TiDB Lightning + + - Create tables in parallel to improve import performance [#502](https://github.com/pingcap/tidb-lightning/pull/502) + - Skip splitting Regions to improve import performance if the engine's total size is smaller than the Region size [#524](https://github.com/pingcap/tidb-lightning/pull/524) + - Add a import progress bar and optimize the accuracy of restore progress [#506](https://github.com/pingcap/tidb-lightning/pull/506) + +## Bug Fixes + ++ TiDB + + - Fix the issue of abnormal `unicode_ci` constant propagation [#22614](https://github.com/pingcap/tidb/pull/22614) + - Fix the issue that might cause wrong collation and coercibility [#22602](https://github.com/pingcap/tidb/pull/22602) + - Fix the issue that might cause wrong collation results [#22599](https://github.com/pingcap/tidb/pull/22599) + - Fix the issue of constant substitution for different collations [#22582](https://github.com/pingcap/tidb/pull/22582) + - Fix a bug that the `like` function might return wrong result when using collation [#22531](https://github.com/pingcap/tidb/pull/22531) + - Fix the issue of incorrect `duration` type inference in `least` and `greatest` functions [#22580](https://github.com/pingcap/tidb/pull/22580) + - Fix a bug that occurs when the `like` function handles a single character wildcard (`_`) followed by a multiple character wildcard (`%`) [#22575](https://github.com/pingcap/tidb/pull/22575) + - Fix the type inference error of the TiDB's built-in functions (`least` and `greatest`) [#22562](https://github.com/pingcap/tidb/pull/22562) + - Fix a bug that makes the `like` function get the wrong result if the pattern string is a unicode string [#22529](https://github.com/pingcap/tidb/pull/22529) + - Fix a bug that the point get query does not get the snapshot data when the `@@tidb_snapshot` variable is set [#22527](https://github.com/pingcap/tidb/pull/22527) + - Fix the potential panic that occurs when generating hints from joins [#22518](https://github.com/pingcap/tidb/pull/22518) + - Fix the issue that strings are incorrectly converted to the `BIT` type [#22420](https://github.com/pingcap/tidb/pull/22420) + - Fix the `index out of range` error that occurs when inserting values to the `tidb_rowid` column [#22359](https://github.com/pingcap/tidb/pull/22359) + - Fix a bug that the cached plan is incorrectly used [#22353](https://github.com/pingcap/tidb/pull/22353) + - Fix the runtime panic in the `WEIGHT_STRING` function when the length of the binary/char string is too large [#22332](https://github.com/pingcap/tidb/pull/22332) + - Forbid using the generated column when the number of function parameters is invalid [#22174](https://github.com/pingcap/tidb/pull/22174) + - Correctly set the process information before building the execution plan [#22148](https://github.com/pingcap/tidb/pull/22148) + - Fix the issue of inaccurate runtime statistics of `IndexLookUp` [#22136](https://github.com/pingcap/tidb/pull/22136) + - Add cache for the memory usage information when the cluster is deployed in a container [#22116](https://github.com/pingcap/tidb/pull/22116) + - Fix the issue of the decoding plan errors [#22022](https://github.com/pingcap/tidb/pull/22022) + - Report errors for using invalid window specifications [#21976](https://github.com/pingcap/tidb/pull/21976) + - Report errors when the `PREPARE` statement is nested with `EXECUTE`, `DEALLOCATE` or `PREPARE` [#21972](https://github.com/pingcap/tidb/pull/21972) + - Fix the issue that no error is reported when the `INSERT IGNORE` statement is used on a non-existing partition [#21971](https://github.com/pingcap/tidb/pull/21971) + - Unify the encoding of `EXPLAIN` results and slow log [#21964](https://github.com/pingcap/tidb/pull/21964) + - Fix the issue of unknown columns in join when using the aggregate operator [#21957](https://github.com/pingcap/tidb/pull/21957) + - Fix the wrong type inference in the `ceiling` function [#21936](https://github.com/pingcap/tidb/pull/21936) + - Fix the issue that the `Double` type column ignores its decimal [#21916](https://github.com/pingcap/tidb/pull/21916) + - Fix the issue that the correlated aggregation is calculated in subqueries [#21877](https://github.com/pingcap/tidb/pull/21877) + - Report errors for the JSON object with key length >= 65536 [#21870](https://github.com/pingcap/tidb/pull/21870) + - Fix the issue that the `dyname` function is incompatible with MySQL [#21850](https://github.com/pingcap/tidb/pull/21850) + - Fix the issue that the `to_base64` function returns `NULL` when the input data is too long [#21813](https://github.com/pingcap/tidb/pull/21813) + - Fix the failure of comparing multiple fields in the subquery [#21808](https://github.com/pingcap/tidb/pull/21808) + - Fix the issue that occurs when comparing the float type in JSON [#21785](https://github.com/pingcap/tidb/pull/21785) + - Fix the issue that occurs when comparing the types of JSON objects [#21718](https://github.com/pingcap/tidb/pull/21718) + - Fix the issue that the coercibility value of the `cast` function is incorrectly set [#21714](https://github.com/pingcap/tidb/pull/21714) + - Fix an unexpected panic when using the `IF` function [#21711](https://github.com/pingcap/tidb/pull/21711) + - Fix the issue that the `NULL` result returned from JSON search is incompatible with MySQL [#21700](https://github.com/pingcap/tidb/pull/21700) + - Fix the issue that occurs when checking the `only_full_group_by` mode using `ORDER BY` and `HAVING` [#21697](https://github.com/pingcap/tidb/pull/21697) + - Fix the issue that the units of `Day` and `Time` are incompatible with MySQL [#21676](https://github.com/pingcap/tidb/pull/21676) + - Fix the issue that the default values of `LEAD` and `LAG` cannot adapt to the field type [#21665](https://github.com/pingcap/tidb/pull/21665) + - Perform a check to ensure that the `LOAD DATA` statement can only load data into base tables [#21638](https://github.com/pingcap/tidb/pull/21638) + - Fix the issue that occurs when `addtime` and `subtime` functions handle invalid arguments [#21635](https://github.com/pingcap/tidb/pull/21635) + - Change the round rule for approximate values to "round to the nearest even number" [#21628](https://github.com/pingcap/tidb/pull/21628) + - Fix the issue that `WEEK()` does not recognize `@@GLOBAL.default_week_format` until it has been explicitly read [#21623](https://github.com/pingcap/tidb/pull/21623) + ++ TiKV + + - Fix the issue that TiKV is failed to build with `PROST=1` [#9604](https://github.com/tikv/tikv/pull/9604) + - Fix the unmatched memory diagnostics [#9589](https://github.com/tikv/tikv/pull/9589) + - Fix the issue that the end key of a partial RawKV-restore range is inclusive [#9583](https://github.com/tikv/tikv/pull/9583) + - Fix the issue of TiKV panic that occurs when loading the old value of a key of a rolled-back transaction during TiCDC's incremental scan [#9569](https://github.com/tikv/tikv/pull/9569) + - Fix the configuration glitch of old values when changefeeds with different settings connect to one Region [#9565](https://github.com/tikv/tikv/pull/9565) + - Fix a crash issue that occurs when running a TiKV cluster on a machine with a network interface that lacks the MAC address (introduced in v4.0.9) [#9516](https://github.com/tikv/tikv/pull/9516) + - Fix the issue of TiKV OOM when backing up a huge Region [#9448](https://github.com/tikv/tikv/pull/9448) + - Fix the issue that `region-split-check-diff` cannot be customized [#9530](https://github.com/tikv/tikv/pull/9530) + - Fix the issue of TiKV panic when the system time goes back [#9542](https://github.com/tikv/tikv/pull/9542) + ++ PD + + - Fix the issue that member health metrics are incorrectly displayed [#3368](https://github.com/pingcap/pd/pull/3368) + - Forbid removing the tombstone store that still has peers [#3352](https://github.com/pingcap/pd/pull/3352) + - Fix the issue that the store limit cannot be persisted [#3403](https://github.com/pingcap/pd/pull/3403) + - Fix the limit constriction of the scatter range scheduler [#3401](https://github.com/pingcap/pd/pull/3401) + ++ TiFlash + + - Fix a bug that the `min`/`max` result is wrong for the decimal type + - Fix a bug that TiFlash might crash when reading data + - Fix the issue that some data written after DDL operations might be lost after data compaction + - Fix the issue that TiFlash incorrectly handles decimal constants in Coprocessor + - Fix the potential crash during the learner read process + - Fix the inconsistent behaviors of division by `0` or `NULL` between TiDB and TiFlash + ++ Tools + + + TiCDC + + - Fix a bug that the TiCDC service might unexpectedly exit when `ErrTaskStatusNotExists` and the closing of `capture` session occur at the same time [#1240](https://github.com/pingcap/ticdc/pull/1240) + - Fix the old value switch issue that a `changefeed` might be affected by another `changefeed` [#1347](https://github.com/pingcap/ticdc/pull/1347) + - Fix a bug that the TiCDC service might hang when processing a new `changefeed` with the invalid `sort-engine` parameter [#1309](https://github.com/pingcap/ticdc/pull/1309) + - Fix the issue of panic that occurs when getting the debugging information on non-owner nodes [#1349](https://github.com/pingcap/ticdc/pull/1349) + - Fix the issue that the `ticdc_processor_num_of_tables` and `ticdc_processor_table_resolved_ts` metrics are not properly updated when adding or removing tables [#1351](https://github.com/pingcap/ticdc/pull/1351) + - Fix the issue of potential data loss if a processor crashes when adding a table [#1363](https://github.com/pingcap/ticdc/pull/1363) + - Fix a bug that the owner might lead to abnormal TiCDC server exits during table migrations [#1352](https://github.com/pingcap/ticdc/pull/1352) + - Fix a bug that TiCDC does not exit in time after the service GC safepoint is lost [#1367](https://github.com/pingcap/ticdc/pull/1367) + - Fix a bug that the KV client might skip creating the event feed [#1336](https://github.com/pingcap/ticdc/pull/1336) + - Fix a bug that the atomicity of transactions is broken when the transactions are replicated to the downstream [#1375](https://github.com/pingcap/ticdc/pull/1375) + + + Backup & Restore (BR) + + - Fix the issue that TiKV might be caused to generate a big Region after BR restores the backup [#702](https://github.com/pingcap/br/pull/702) + - Fix the issue that BR restores a table's Auto ID even if the table does not have Auto ID [#720](https://github.com/pingcap/br/pull/720) + + + TiDB Lightning + + - Fix a bug that `column count mismatch` might be triggered when using the TiDB-backend [#535](https://github.com/pingcap/tidb-lightning/pull/535) + - Fix a bug that TiDB-backend panics if the column count of the source file and the column count of the target table mismatch [#528](https://github.com/pingcap/tidb-lightning/pull/528) + - Fix a bug that TiKV might unexpectedly panic during TiDB Lightning's data import [#554](https://github.com/pingcap/tidb-lightning/pull/554) diff --git a/releases/release-4.0.12.md b/releases/release-4.0.12.md new file mode 100644 index 000000000000..60d5e8344d25 --- /dev/null +++ b/releases/release-4.0.12.md @@ -0,0 +1,137 @@ +--- +title: TiDB 4.0.12 Release Notes +--- + +# TiDB 4.0.12 Release Notes + +Release date: April 2, 2021 + +TiDB version: 4.0.12 + +## New Features + ++ TiFlash + + - Add tools to check the status of `tiflash replica` for online rolling updates + +## Improvements + ++ TiDB + + - Refine the output information of the `EXPLAIN` statement for the `batch cop` mode [#23164](https://github.com/pingcap/tidb/pull/23164) + - Add the warning information for expressions that cannot be pushed to the storage layer in the output of the `EXPLAIN` statement [#23020](https://github.com/pingcap/tidb/pull/23020) + - Migrate a part of the DDL package code from `Execute`/`ExecRestricted` to the safe API (2) [#22935](https://github.com/pingcap/tidb/pull/22935) + - Migrate a part of the DDL package code from `Execute`/`ExecRestricted` to the safe API (1) [#22929](https://github.com/pingcap/tidb/pull/22929) + - Add `optimization-time` and `wait-TS-time` into the slow log [#22918](https://github.com/pingcap/tidb/pull/22918) + - Support querying `partition_id` from the `infoschema.partitions` table [#22489](https://github.com/pingcap/tidb/pull/22489) + - Add `last_plan_from_binding` to help the users know whether a SQL statement's execution plan is matched with the hints in the binding [#21430](https://github.com/pingcap/tidb/pull/21430) + - Scatter truncated tables without the `pre-split` option [#22872](https://github.com/pingcap/tidb/pull/22872) + - Add three format specifiers for the `str_to_date` expression [#22812](https://github.com/pingcap/tidb/pull/22812) + - Record the `PREPARE` execution failure as `Failed Query OPM` in the metrics monitor [#22672](https://github.com/pingcap/tidb/pull/22672) + - Do not report errors for the `PREPARE` execution if `tidb_snapshot` is set [#22641](https://github.com/pingcap/tidb/pull/22641) + ++ TiKV + + - Prevent a large number of reconnections in a short period of time [#9879](https://github.com/tikv/tikv/pull/9879) + - Optimize the write operations and Batch Get in the scenarios of many tombstones [#9729](https://github.com/tikv/tikv/pull/9729) + - Change the default value of `leader-transfer-max-log-lag` to `128` to increase the success rate of leader transfer [#9605](https://github.com/tikv/tikv/pull/9605) + ++ PD + + - Update the Region cache only when `pending-peers` or `down-peers` changes, which reduces the pressure of updating heartbeats [#3471](https://github.com/pingcap/pd/pull/3471) + - Prevent the Regions in `split-cache` from becoming the target of merge [#3459](https://github.com/pingcap/pd/pull/3459) + ++ TiFlash + + - Optimize the configuration file and remove useless items + - Reduce the size of TiFlash binary files + - Use an adaptive aggressive GC strategy to reduce memory usage + ++ Tools + + + TiCDC + + - Add a double confirmation when users create or resume the changefeed with the `start-ts` or `checkpoint-ts` 1 day before the current timestamp [#1497](https://github.com/pingcap/ticdc/pull/1497) + - Add Grafana panels for the Old Value feature [#1571](https://github.com/pingcap/ticdc/pull/1571) + + + Backup & Restore (BR) + + - Log the `HTTP_PROXY` and `HTTPS_PROXY` environmental variables [#827](https://github.com/pingcap/br/pull/827) + - Improve the backup performance when there are many tables [#745](https://github.com/pingcap/br/pull/745) + - Report errors if the service safe point check fails [#826](https://github.com/pingcap/br/pull/826) + - Add the `cluster_version` and `br_version` information in `backupmeta` [#803](https://github.com/pingcap/br/pull/803) + - Add retry for external storage errors to increase the success rate of backup [#851](https://github.com/pingcap/br/pull/851) + - Reduce memory usage during backup [#886](https://github.com/pingcap/br/pull/886) + + + TiDB Lightning + + - Check the TiDB cluster version before running TiDB Lightning to avoid unexpected errors [#787](https://github.com/pingcap/br/pull/787) + - Fail fast when TiDB Lightning meets the `cancel` error [#867](https://github.com/pingcap/br/pull/867) + - Add `tikv-importer.engine-mem-cache-size` and `tikv-importer.local-writer-mem-cache-size` configuration items to balance between memory usage and performance [#866](https://github.com/pingcap/br/pull/866) + - Run `batch split region` in parallel for TiDB Lightning's Local-backend to increase the import speed [#868](https://github.com/pingcap/br/pull/868) + - When using TiDB Lightning to import data from a S3 storage, TiDB Lightning no longer requires the `s3:ListBucket` permission [#919](https://github.com/pingcap/br/pull/919) + - When resuming from a checkpoint, TiDB Lightning keeps using the original engine [#924](https://github.com/pingcap/br/pull/924) + +## Bug Fixes + ++ TiDB + + - Fix the issue that the `get` variable expression goes wrong when the session variable is hexadecimal literals[#23372](https://github.com/pingcap/tidb/pull/23372) + - Fix the issue that wrong collation is used when creating the fast execution plan for the `Enum` or `Set` type [#23292](https://github.com/pingcap/tidb/pull/23292) + - Fix the possible wrong result of the `nullif` expression when it is used with `is-null` [#23279](https://github.com/pingcap/tidb/pull/23279) + - Fix the issue that the auto-analysis is triggered outside its time range [#23219](https://github.com/pingcap/tidb/pull/23219) + - Fix the issue that the `CAST` function might ignore errors for the `point get` plan [#23211](https://github.com/pingcap/tidb/pull/23211) + - Fix a bug that prevents SPM from taking effect when `CurrentDB` is empty [#23209](https://github.com/pingcap/tidb/pull/23209) + - Fix the issue of possible wrong table filters for the IndexMerge plan [#23165](https://github.com/pingcap/tidb/pull/23165) + - Fix the issue of unexpected `NotNullFlag` in the returning types of the `NULL` constant [#23135](https://github.com/pingcap/tidb/pull/23135) + - Fix a bug that collation might not be handled by the text type [#23092](https://github.com/pingcap/tidb/pull/23092) + - Fix the issue that the range partition might incorrectly handle the `IN` expression [#23074](https://github.com/pingcap/tidb/pull/23074) + - Fix the issue that after marking a TiKV store as tombstone, starting new TiKV stores with different StoreIDs with the same IP address and port keeps returning the `StoreNotMatch` error [#23071](https://github.com/pingcap/tidb/pull/23071) + - Do not adjust the `INT` type when it is `NULL` and compared with `YEAR` [#22844](https://github.com/pingcap/tidb/pull/22844) + - Fix the issue of lost connection when loading data on tables with the `auto_random` column [#22736](https://github.com/pingcap/tidb/pull/22736) + - Fix the issue of DDL hangover when the DDL operation meets panic in the cancelling path [#23297](https://github.com/pingcap/tidb/pull/23297) + - Fix the wrong key range of index scan when comparing the `YEAR` column with `NULL` [#23104](https://github.com/pingcap/tidb/pull/23104) + - Fix the issue that a successfully created view is failed to use [#23083](https://github.com/pingcap/tidb/pull/23083) + ++ TiKV + + - Fix the issue that the `IN` expression does not properly handle unsigned/signed integers [#9850](https://github.com/tikv/tikv/pull/9850) + - Fix the issue that the ingest operation is not re-entrant [#9779](https://github.com/tikv/tikv/pull/9779) + - Fix the issue that the space is missed when converting JSON to string in TiKV coprocessor [#9666](https://github.com/tikv/tikv/pull/9666) + ++ PD + + - Fix a bug that the isolation level is wrong when the store lacks the label [#3474](https://github.com/pingcap/pd/pull/3474) + ++ TiFlash + + - Fix the issue of incorrect execution results when the default value of the `binary` type column contains leading or tailing zero bytes + - Fix a bug that TiFlash fails to synchronize schema if the name of the database contains special characters + - Fix the issue of incorrect results when handling the `IN` expression with decimal values + - Fix a bug that the metric for the opened file count shown in Grafana is high + - Fix a bug that TiFlash does not support the `Timestamp` literal + - Fix the potential not responding issue while handling the `FROM_UNIXTIME` expression + - Fix the issue of incorrect results when casting string as integer + - Fix a bug that the `like` function might return wrong results + ++ Tools + + + TiCDC + + - Fix a disorder issue of the `resolved ts` event [#1464](https://github.com/pingcap/ticdc/pull/1464) + - Fix a data loss issue caused by wrong table scheduling due to the network problem [#1508](https://github.com/pingcap/ticdc/pull/1508) + - Fix a bug of untimely release of resources after a processor is stopped [#1547](https://github.com/pingcap/ticdc/pull/1547) + - Fix a bug that the transaction counter is not correctly updated, which might cause database connection leak [#1524](https://github.com/pingcap/ticdc/pull/1524) + - Fix the issue that multiple owners can co-exist when PD has jitter, which might lead to table missing [#1540](https://github.com/pingcap/ticdc/pull/1540) + + + Backup & Restore (BR) + + - Fix a bug that `WalkDir` for the s3 storage returns `nil` if the target path is bucket name [#733](https://github.com/pingcap/br/pull/733) + - Fix a bug that the `status` port is not served with TLS [#839](https://github.com/pingcap/br/pull/839) + + + TiDB Lightning + + - Fix the error that TiKV Importer might ignore that the file has already existed [#848](https://github.com/pingcap/br/pull/848) + - Fix a bug that the TiDB Lightning might use the wrong timestamp and read the wrong data [#850](https://github.com/pingcap/br/pull/850) + - Fix a bug that TiDB Lightning's unexpected exit might cause damaged checkpoint file [#889](https://github.com/pingcap/br/pull/889) + - Fix the issue of possible data error that occurs because the `cancel` error is ignored [#874](https://github.com/pingcap/br/pull/874) diff --git a/releases/release-4.0.13.md b/releases/release-4.0.13.md new file mode 100644 index 000000000000..7d0d7980f763 --- /dev/null +++ b/releases/release-4.0.13.md @@ -0,0 +1,142 @@ +--- +title: TiDB 4.0.13 Release Notes +--- + +# TiDB 4.0.13 Release Notes + +Release date: May 28, 2021 + +TiDB version: 4.0.13 + +## New Features + ++ TiDB + + - Support changing an `AUTO_INCREMENT` column to an `AUTO_RANDOM` one [#24608](https://github.com/pingcap/tidb/pull/24608) + - Add the `infoschema.client_errors_summary` tables to help users keep track of the errors that have been returned to clients [#23267](https://github.com/pingcap/tidb/pull/23267) + +## Improvements + ++ TiDB + + - Avoid frequently reading the `mysql.stats_histograms` table if the cached statistics is up-to-date to avoid high CPU usage [#24352](https://github.com/pingcap/tidb/pull/24352) + ++ TiKV + + - Make the calculation process of `store used size` more precise [#9904](https://github.com/tikv/tikv/pull/9904) + - Set more Regions in the `EpochNotMatch` message to reduce Region misses [#9731](https://github.com/tikv/tikv/pull/9731) + - Speed up freeing the memory accumulated in the long-running cluster [#10035](https://github.com/tikv/tikv/pull/10035) + ++ PD + + - Optimize the metrics of TSO processing time to help users determine whether the TSO processing time at the PD side is too long [#3524](https://github.com/pingcap/pd/pull/3524) + - Update the dashboard version to v2021.03.12.1 [#3469](https://github.com/pingcap/pd/pull/3469) + ++ TiFlash + + - Automatically clean archived data to free up disk space + ++ Tools + + + Backup & Restore (BR) + + - Support backing up user tables created in the `mysql` schema [#1077](https://github.com/pingcap/br/pull/1077) + - Update `checkVersion` to check the cluster data and the backup data [#1090](https://github.com/pingcap/br/pull/1090) + - Tolerate a small number of TiKV node failures during backup [#1062](https://github.com/pingcap/br/pull/1062) + + + TiCDC + + - Implement the processor flow control to avoid memory overflow (OOM) [#1751](https://github.com/pingcap/ticdc/pull/1751) + - Support cleaning up stale temporary files in Unified Sorter and prevent multiple `cdc server` instances from sharing the same `sort-dir` directory [#1741](https://github.com/pingcap/ticdc/pull/1741) + - Add the HTTP handler for the failpoint [#1732](https://github.com/pingcap/ticdc/pull/1732) + +## Bug Fixes + ++ TiDB + + - Fix the panic issue that occurs when the `UPDATE` statement with a subquery updates the generated column [#24658](https://github.com/pingcap/tidb/pull/24658) + - Fix the issue that causes duplicate query results when using the multi-column index for data reads [#24634](https://github.com/pingcap/tidb/pull/24634) + - Fix the issue that causes wrong query result when using the `BIT` type constant as the divisor in the DIV expression [#24266](https://github.com/pingcap/tidb/pull/24266) + - Fix the issue that the `NO_ZERO_IN_DATE` SQL mode does not take effect for the default column value set in DDL statements [#24185](https://github.com/pingcap/tidb/pull/24185) + - Fix an issue which causes wrong query results when using `UNION` between a `BIT` type column and an `INTEGER` type column [#24026](https://github.com/pingcap/tidb/pull/24026) + - Fix the issue that the `TableDual` plans are mistakenly created when comparing the `BINARY` type and the `CHAR` type [#23917](https://github.com/pingcap/tidb/pull/23917) + - Fix the issue that the `insert ignore on duplicate` statement might unexpectedly delete table records [#23825](https://github.com/pingcap/tidb/pull/23825) + - Fix the issue that the Audit plugin causes TiDB panic [#23819](https://github.com/pingcap/tidb/pull/23819) + - Fix the issue that the `HashJoin` operator incorrectly processes the collation [#23812](https://github.com/pingcap/tidb/pull/23812) + - Fix the issue of disconnection that occurs when `batch_point_get` incorrectly handles abnormal values in the pessimistic transaction [#23778](https://github.com/pingcap/tidb/pull/23778) + - Fix the issue of inconsistent indexes that occurs when the `tidb_row_format_version` configuration value is set to `1` and the `enable_new_collation` value is set to `true` [#23772](https://github.com/pingcap/tidb/pull/23772) + - Fix a bug that occurs when comparing the `INTEGER` type column with the `STRING` constant value [#23705](https://github.com/pingcap/tidb/pull/23705) + - Fix the error that occurs when the `BIT` type column is passed into the `approx_percent` function [#23702](https://github.com/pingcap/tidb/pull/23702) + - Fix a bug that causes TiDB to mistakenly report the `TiKV server timeout` error when executing TiFlash batch requests [#23700](https://github.com/pingcap/tidb/pull/23700) + - Fix the issue that the `IndexJoin` operator returns wrong results on the prefix column index [#23691](https://github.com/pingcap/tidb/pull/23691) + - Fix the issue which causes wrong query results because the collation on the `BINARY` type column is not properly handled [#23598](https://github.com/pingcap/tidb/pull/23598) + - Fix the issue of query panic that occurs when the `UPDATE` statement contains the join query with the `HAVING` clause [#23575](https://github.com/pingcap/tidb/pull/23575) + - Fix the issue that causes TiFlash to return wrong results when using the `NULL` constant in the comparison expression [#23474](https://github.com/pingcap/tidb/pull/23474) + - Fix the issue of wrong results when comparing the `YEAR` type column with the `STRING` constant [#23335](https://github.com/pingcap/tidb/pull/23335) + - Fix the issue that `group_concat` panics when `session.group_concat_max_len` is set too small [#23257](https://github.com/pingcap/tidb/pull/23257) + - Fix the issue of wrong query results that occurs when using the `BETWEEN` expression for the `TIME` type column [#23233](https://github.com/pingcap/tidb/pull/23233) + - Fix the issue of privilege check in the `DELETE` statements [#23215](https://github.com/pingcap/tidb/pull/23215) + - Fix the issue that no error is reported when inserting invalid strings to the `DECIMAL` type column [#23196](https://github.com/pingcap/tidb/pull/23196) + - Fix the issue of parsing error occurred when inserting data to the `DECIMAL` type columns [#23152](https://github.com/pingcap/tidb/pull/23152) + - Fix the issue that the `USE_INDEX_MERGE` hint does not take effect [#22924](https://github.com/pingcap/tidb/pull/22924) + - Fix a bug that the query returns wrong results when using `ENUM` or `SET` columns in the `WHERE` clause as an filter [#22814](https://github.com/pingcap/tidb/pull/22814) + - Fix a bug that the query returns wrong results when using the clustered index and the new collation at the same time [#21408](https://github.com/pingcap/tidb/pull/21408) + - Fix the panic that occurs when executing `ANALYZE` with `enable_new_collation` enabled [#21299](https://github.com/pingcap/tidb/pull/21299) + - Fix the issue that SQL views does not correctly handle the default roles associated with the SQL DEFINER [#24531](https://github.com/pingcap/tidb/pull/24531) + - Fix the issue that cancelling DDL jobs gets stuck [#24445](https://github.com/pingcap/tidb/pull/24445) + - Fix the issue that the `concat` function incorrectly handles the collation [#24300](https://github.com/pingcap/tidb/pull/24300) + - Fix a bug that the query returns wrong results when the `SELECT` field has an `IN` subquery and the subquery's outer side contains `NULL` tuples [#24022](https://github.com/pingcap/tidb/pull/24022) + - Fix a bug that TiFlash is chosen wrongly by the optimizer when `TableScan` is in descending order [#23974](https://github.com/pingcap/tidb/pull/23974) + - Fix a bug that the `point_get` plan returns the column name that is inconsistent with that of MySQL [#23970](https://github.com/pingcap/tidb/pull/23970) + - Fix the issue that executing the `show table status` statement on a database with a upper-cased name returns wrong results [#23958](https://github.com/pingcap/tidb/pull/23958) + - Fix a bug that the users who do not have the `INSERT` and `DELETE` privileges on a table at the same time can perform the `REPLACE` operation [#23938](https://github.com/pingcap/tidb/pull/23938) + - Fix the issue that the results of the `concat`/`make_set`/`insert` expressions are wrong because the collation is incorrectly handled [#23878](https://github.com/pingcap/tidb/pull/23878) + - Fix the panic that occurs when executing a query on the table that has `RANGE` partitions [#23689](https://github.com/pingcap/tidb/pull/23689) + - Fix the issue: In the cluster of an earlier version, if the `tidb_enable_table_partition` variable is set to `false`, the tables that contain partitions are handled as non-partitioned tables. Executing `batch point get` queries on this table, when the cluster is upgraded to a later version, causes connection panic. [#23682](https://github.com/pingcap/tidb/pull/23682) + - Fix the issue that when TiDB is configured to listen on TCP and UNIX sockets, the remote hosts over the TCP connection are not correctly validated for connection [#23513](https://github.com/pingcap/tidb/pull/23513) + - Fix a bug that the non-default collation causes wrong query results [#22923](https://github.com/pingcap/tidb/pull/22923) + - Fix a bug that the **Coprocessor Cache** panel of Grafana does not work [#22617](https://github.com/pingcap/tidb/pull/22617) + - Fix the error that occurs when the optimizer accesses the statistic cache [#22565](https://github.com/pingcap/tidb/pull/22565) + ++ TiKV + + - Fix a bug that TiKV cannot start if the `file_dict` file is not fully written into the disk that has been full [#9963](https://github.com/tikv/tikv/pull/9963) + - Limit TiCDC's scan speed at 128MB/s by default [#9983](https://github.com/tikv/tikv/pull/9983) + - Reduce the memory usage of TiCDC's initial scan [#10133](https://github.com/tikv/tikv/pull/10133) + - Support the back pressure for TiCDC's scan speed [#10142](https://github.com/tikv/tikv/pull/10142) + - Fix a potential OOM issue by avoiding unnecessary reads to get TiCDC old values [#10031](https://github.com/tikv/tikv/pull/10031) + - Fix a TiCDC OOM issue caused by reading old values [#10197](https://github.com/tikv/tikv/pull/10197) + - Add a timeout mechanism for S3 storages to avoid the client hanging without responses [#10132](https://github.com/tikv/tikv/pull/10132) + ++ TiFlash + + - Fix the issue that number of `delta-merge-tasks` is not reported to Prometheus + - Fix the TiFlash panic issue that occurs during `Segment Split` + - Fix the issue that the `Region write Duration (write blocks)` panel in Grafana is shown in a wrong place + - Fix the potential issue that the storage engine fails to remove data + - Fix the issue of incorrect results when casting the `TIME` type to the `INTEGER` type + - Fix a bug that the behavior of the `bitwise` operator is different from that of TiDB + - Fix the issue of incorrect results when casting the `STRING` type to the `INTEGER` type + - Fix the issue that consecutive and fast writes might make TiFlash out of memory + - Fix the potential issue that the exception of null pointer might be raised during the table GC + - Fix the TiFlash panic issue that occurs when writing data to dropped tables + - Fix the TiFlash panic issue that occurs during BR restore + - Fix a bug that the weights of some characters are wrong when using the general CI collation + - Fix the potential issue that data will be lost in tombstoned tables + - Fix the issue of incorrect results when comparing the string which contains zero bytes + - Fix the issue that the logical function returns wrong results if the input column contains null constants + - Fix the issue that the logical function only accepts the numeric type + - Fix the issue of incorrect results that occurs when the timestamp value is `1970-01-01` and the timezone offset is negative + - Fix the issue that hash value of `Decimal256` is not stable + ++ Tools + + + TiCDC + + - Fix the deadlock issue caused by the flow control when the sorter's input channel has been blocked [#1779](https://github.com/pingcap/ticdc/pull/1779) + - Fix the issue that the TiKV GC safe point is blocked due to the stagnation of TiCDC changefeed checkpoint [#1756](https://github.com/pingcap/ticdc/pull/1756) + - Revert the update in `explicit_defaults_for_timestamp` which requires the `SUPER` privilege when replicating data to MySQL [#1749](https://github.com/pingcap/ticdc/pull/1749) + + + TiDB Lightning + + - Fix a bug that TiDB Lightning's TiDB-backend cannot load any data when autocommit is disabled diff --git a/releases/release-4.0.14.md b/releases/release-4.0.14.md new file mode 100644 index 000000000000..69e680504a18 --- /dev/null +++ b/releases/release-4.0.14.md @@ -0,0 +1,166 @@ +--- +title: TiDB 4.0.14 Release Notes +--- + +# TiDB 4.0.14 Release Notes + +Release date: July 27, 2021 + +TiDB version: 4.0.14 + +## Compatibility changes + ++ TiDB + + - Change the default value of `tidb_multi_statement_mode` from `WARN` to `OFF` in v4.0. It is recommended to use the multi-statement feature of your client library instead. See [the documentation on `tidb_multi_statement_mode`](/system-variables.md#tidb_multi_statement_mode-new-in-v4011) for details. [#25749](https://github.com/pingcap/tidb/pull/25749) + - Upgrade Grafana dashboard from v6.1.16 to v7.5.7 to solve two security vulnerabilities. See the [Grafana blog post](https://grafana.com/blog/2020/06/03/grafana-6.7.4-and-7.0.2-released-with-important-security-fix/) for details. + - Change the default value of the `tidb_stmt_summary_max_stmt_count` variable from `200` to `3000` [#25872](https://github.com/pingcap/tidb/pull/25872) + ++ TiKV + + - Change the default value of `merge-check-tick-interval` from `10` to `2` to speed up the Region merge process [#9676](https://github.com/tikv/tikv/pull/9676) + +## Feature enhancements + ++ TiKV + + - Add a metric `pending` to monitor the number of pending PD heartbeats, which helps locate the issue of slow PD threads [#10008](https://github.com/tikv/tikv/pull/10008) + - Support using the virtual-host addressing mode to make BR support the S3-compatible storage [#10242](https://github.com/tikv/tikv/pull/10242) + ++ TiDB Dashboard + + - Support OIDC SSO. By setting the OIDC-compatible SSO services (such as Okta and Auth0), users can log into TiDB Dashboard without entering the SQL password. [#960](https://github.com/pingcap/tidb-dashboard/pull/960) + - Add the **Debug API** UI, which is an alternative method to the command line to call several common TiDB and PD internal APIs for advanced debugging [#927](https://github.com/pingcap/tidb-dashboard/pull/927) + +## Improvements + ++ TiDB + + - Change the `LOCK` record into the `PUT` record for the index keys using `point get` or `batch point get` for `UPDATE` reads [#26223](https://github.com/pingcap/tidb/pull/26223) + - Support the MySQL system variable `init_connect` and its associated features [#26031](https://github.com/pingcap/tidb/pull/26031) + - Support the stable result mode to make the query results more stable [#26003](https://github.com/pingcap/tidb/pull/26003) + - Support pushing down the built-in function `json_unquote()` to TiKV [#25721](https://github.com/pingcap/tidb/pull/25721) + - Make the SQL Plan Management (SPM) not affected by the character set [#23295](https://github.com/pingcap/tidb/pull/23295) + ++ TiKV + + - Shutdown the status server first to make sure that the client can correctly check the shutdown status [#10504](https://github.com/tikv/tikv/pull/10504) + - Always respond to stale peers to make sure that these peers are cleared quicker [#10400](https://github.com/tikv/tikv/pull/10400) + - Limit the TiCDC sink's memory consumption [#10147](https://github.com/tikv/tikv/pull/10147) + - When a Region is too large, use the even split to speed up the split process [#10275](https://github.com/tikv/tikv/pull/10275) + ++ PD + + - Reduce the conflicts among multiple schedulers that run at the same time [#3858](https://github.com/pingcap/pd/pull/3858) [#3854](https://github.com/tikv/pd/pull/3854) + ++ TiDB Dashboard + + - Update TiDB Dashboard to v2021.07.17.1 [#3882](https://github.com/pingcap/pd/pull/3882) + - Support sharing the current session as a read-only session to avoid further modification to it [#960](https://github.com/pingcap/tidb-dashboard/pull/960) + ++ Tools + + + Backup & Restore (BR) + + - Speed up restore by merging small backup files [#655](https://github.com/pingcap/br/pull/655) + + + Dumpling + + - Always split tables using `_tidb_rowid` when the upstream is a TiDB v3.x cluster, which helps reduce TiDB's memory usage [#306](https://github.com/pingcap/dumpling/pull/306) + + + TiCDC + + - Improve the error message returned when a PD endpoint misses the certificate [#1973](https://github.com/pingcap/ticdc/issues/1973) + - Make the sorter I/O errors more user-friendly [#1976](https://github.com/pingcap/ticdc/pull/1976) + - Add a concurrency limit on the Region incremental scan in the KV client to reduce the pressure of TiKV [#1926](https://github.com/pingcap/ticdc/pull/1926) + - Add metrics for the table memory consumption [#1884](https://github.com/pingcap/ticdc/pull/1884) + - Add `capture-session-ttl` to the TiCDC server configuration [#2169](https://github.com/pingcap/ticdc/pull/2169) + +## Bug fixes + ++ TiDB + + - Fix the issue that the `SELECT` result is incompatible with MySQL when joining a subquery with a `WHERE` clause evaluated to `false` [#24865](https://github.com/pingcap/tidb/issues/24865) + - Fix the calculation error of the `ifnull` function that occurs when the argument is the `ENUM` or `SET` type [#24944](https://github.com/pingcap/tidb/issues/24944) + - Fix the wrong aggregate pruning in some cases [#25202](https://github.com/pingcap/tidb/issues/25202) + - Fix the incorrect result of the merge join operation that might occur when the column is the `SET` type [#25669](https://github.com/pingcap/tidb/issues/25669) + - Fix the issue that TiDB returns wrong results for cartesian join [#25591](https://github.com/pingcap/tidb/issues/25591) + - Fix the panic issue that occurs when `SELECT ... FOR UPDATE` works on a join operation and the join uses a partitioned table [#20028](https://github.com/pingcap/tidb/issues/20028) + - Fix the issue that the cached `prepared` plan is incorrectly used for `point get` [#24741](https://github.com/pingcap/tidb/issues/24741) + - Fix the issue that the `LOAD DATA` statement can abnormally import non-utf8 data [#25979](https://github.com/pingcap/tidb/issues/25979) + - Fix a potential memory leak issue that occurs when accessing the statistics via an HTTP API [#24650](https://github.com/pingcap/tidb/pull/24650) + - Fix a security issue that occurs when executing the `ALTER USER` statement [#25225](https://github.com/pingcap/tidb/issues/25225) + - Fix a bug that the `TIKV_REGION_PEERS` table cannot correctly handle the `DOWN` status [#24879](https://github.com/pingcap/tidb/issues/24879) + - Fix the issue that invalid strings are not truncated when parsing `DateTime` [#22231](https://github.com/pingcap/tidb/issues/22231) + - Fix the issue that the `select into outfile` statement might have no result when the column type is `YEAR` [#22159](https://github.com/pingcap/tidb/issues/22159) + - Fix the issue that the query result might be wrong when `NULL` is in the `UNION` subquery [#26532](https://github.com/pingcap/tidb/issues/26532) + - Fix the issue that the projection operator in execution might cause panic in some cases [#26534](https://github.com/pingcap/tidb/pull/26534) + ++ TiKV + + - Fix the issue that the duration calculation might panic on certain platforms [#related-issue](https://github.com/rust-lang/rust/issues/86470#issuecomment-877557654) + - Fix the wrong function that casts `DOUBLE` to `DOUBLE` [#25200](https://github.com/pingcap/tidb/issues/25200) + - Fix the issue that the panic log might be lost when using the async logger [#8998](https://github.com/tikv/tikv/issues/8998) + - Fix the panic issue that occurs when building a snapshot twice if encryption is enabled [#9786](https://github.com/tikv/tikv/issues/9786) [#10407](https://github.com/tikv/tikv/issues/10407) + - Fix the wrong arguments type of the `json_unquote()` function in the coprocessor [#10176](https://github.com/tikv/tikv/issues/10176) + - Fix the issues of suspicious warnings during shutdown and the non-deterministic response from Raftstore [#10353](https://github.com/tikv/tikv/issues/10353) [#10307](https://github.com/tikv/tikv/issues/10307) + - Fix the issue of backup threads leak [#10287](https://github.com/tikv/tikv/issues/10287) + - Fix the issue that Region split might panic and corrupt the metadata if the split process is too slow and Region merge is on-going [#8456](https://github.com/tikv/tikv/issues/8456) [#8783](https://github.com/tikv/tikv/issues/8783) + - Fix the issue that the Region heartbeats prevent TiKV from splitting large Regions in some situations [#10111](https://github.com/tikv/tikv/issues/10111) + - Fix the wrong statistics caused by the format inconsistency of CM Sketch between TiKV and TiDB [#25638](https://github.com/pingcap/tidb/issues/25638) + - Fix the wrong statistics of the `apply wait duration` metric [#9893](https://github.com/tikv/tikv/issues/9893) + - Fix the "Missing Blob" error after using `delete_files_in_range` in Titan [#10232](https://github.com/tikv/tikv/pull/10232) + ++ PD + + - Fix a bug that the scheduler might reappear after executing the delete operation [#2572](https://github.com/tikv/pd/issues/2572) + - Fix the data race issue that might occur when the scheduler is started before the temporary configuration is loaded [#3771](https://github.com/tikv/pd/issues/3771) + - Fix a PD panic issue that might occur during the Region scattering operation [#3761](https://github.com/pingcap/pd/pull/3761) + - Fix the issue that the priority of some operators is not set correctly [#3703](https://github.com/pingcap/pd/pull/3703) + - Fix a PD panic issue that might occur when deleting the `evict-leader` scheduler from a non-existent store [#3660](https://github.com/tikv/pd/issues/3660) + - Fix the issue that the PD Leader re-election is slow when there are many stores [#3697](https://github.com/tikv/pd/issues/3697) + ++ TiDB Dashboard + + - Fix the issue that the **Profiling** UI cannot profile all TiDB instances [#944](https://github.com/pingcap/tidb-dashboard/pull/944) + - Fix the issue that the **Statements** UI does not display "Plan Count" [#939](https://github.com/pingcap/tidb-dashboard/pull/939) + - Fix the issue that the **Slow Query** UI might display the "unknown field" error after cluster upgrade [#902](https://github.com/pingcap/tidb-dashboard/issues/902) + ++ TiFlash + + - Fix the potential panic issue that occurs when compiling DAG requests + - Fix the panic issue that occurs when the read load is heavy + - Fix the issue that TiFlash keeps restarting because of the split failure in column storage + - Fix a potential bug that TiFlash cannot delete the delta data + - Fix the incorrect results that occur when cloning the shared delta index concurrently + - Fix a bug that TiFlash fails to restart in the case of incomplete data + - Fix the issue that the old dm files cannot be removed automatically + - Fix the panic issue that occurs when executing the `SUBSTRING` function with specific arguments + - Fix the issue of incorrect results when casting the `INTEGER` type to the `TIME` type + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the data restore from the `mysql` schema might fail [#1142](https://github.com/pingcap/br/pull/1142) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning fails to parse the `DECIMAL` type data in Parquet files [#1276](https://github.com/pingcap/br/pull/1276) + - Fix the EOF error reported when TiDB Lightning splits the imported large CSV files [#1133](https://github.com/pingcap/br/issues/1133) + - Fix a bug that an excessively large base value is generated when TiDB Lightning imports tables with the `auto_increment` column of the `FLOAT` or `DOUBLE` type [#1185](https://github.com/pingcap/br/pull/1185) + - Fix the issue of TiDB Lightning panic that occurs when generating KV data larger than 4 GB [#1128](https://github.com/pingcap/br/pull/1128) + + + Dumpling + + - When using Dumpling to export data to the S3 storage, the `s3:ListBucket` permission is no longer required on the entire bucket. The permission is required only on the data source prefix. [#898](https://github.com/pingcap/br/issues/898) + + + TiCDC + + - Fix the issue of extra partition dispatching after adding new table partitions [#2205](https://github.com/pingcap/ticdc/pull/2205) + - Fix the panic issue that occurs when TiCDC fails to read `/proc/meminfo` [#2023](https://github.com/pingcap/ticdc/pull/2023) + - Reduce TiCDC's runtime memory consumption [#2011](https://github.com/pingcap/ticdc/pull/2011) [#1957](https://github.com/pingcap/ticdc/pull/1957) + - Fix a bug that some MySQL connection might leak after MySQL sink meets the error and pauses [#1945](https://github.com/pingcap/ticdc/pull/1945) + - Fix the issue that TiCDC changefeed cannot be created when start TS is less than current TS minus GC TTL [#1839](https://github.com/pingcap/ticdc/issues/1839) + - Reduce memory `malloc` in sort heap to avoid too much CPU overhead [#1853](https://github.com/pingcap/ticdc/issues/1853) + - Fix a bug that the replication task might stop when moving a table [#1827](https://github.com/pingcap/ticdc/pull/1827) diff --git a/releases/release-4.0.5.md b/releases/release-4.0.5.md index 1c44f4a3f93a..1755c397b38b 100644 --- a/releases/release-4.0.5.md +++ b/releases/release-4.0.5.md @@ -104,7 +104,7 @@ TiDB version: 4.0.5 - Fix the `should ensure all columns have the same length` error that occurs because the `ErrTruncate/Overflow` error is incorrectly handled in the `builtinCastRealAsDecimalSig` function [#18967](https://github.com/pingcap/tidb/pull/18967) - Fix the issue that the `pre_split_regions` table option does not work in the partitioned table [#18837](https://github.com/pingcap/tidb/pull/18837) - - Fixe the issue that might cause a large transaction to be terminated prematurely [#18813](https://github.com/pingcap/tidb/pull/18813) + - Fix the issue that might cause a large transaction to be terminated prematurely [#18813](https://github.com/pingcap/tidb/pull/18813) - Fix the issue that using the `collation` functions get wrong query results [#18735](https://github.com/pingcap/tidb/pull/18735) - Fix the bug that the `getAutoIncrementID()` function does not consider the `tidb_snapshot` session variable, which might cause the dumper tool to fail with the `table not exist` error [#18692](https://github.com/pingcap/tidb/pull/18692) - Fix the `unknown column error` for SQL statement like `select a from t having t.a` [#18434](https://github.com/pingcap/tidb/pull/18434) diff --git a/releases/release-4.0.9.md b/releases/release-4.0.9.md index 4fbb286714be..944f33dac658 100644 --- a/releases/release-4.0.9.md +++ b/releases/release-4.0.9.md @@ -105,6 +105,7 @@ TiDB version: 4.0.9 - Support configuring both the `-F` and `-r` arguments at the same time [#177](https://github.com/pingcap/dumpling/pull/177) - Exclude system databases in `--filter` by default [#194](https://github.com/pingcap/dumpling/pull/194) - Support the `--transactional-consistency` parameter and support rebuilding MySQL connections during retry [#199](https://github.com/pingcap/dumpling/pull/199) + - Support using the `-c,--compress` parameter to specify the compression algorithm used by Dumpling. An empty string means no compression. [#202](https://github.com/pingcap/dumpling/pull/202) + TiDB Lightning diff --git a/releases/release-5.0.0-rc.md b/releases/release-5.0.0-rc.md new file mode 100644 index 000000000000..46beaf29bb81 --- /dev/null +++ b/releases/release-5.0.0-rc.md @@ -0,0 +1,189 @@ +--- +title: TiDB 5.0 RC Release Notes +--- + +# TiDB 5.0 RC Release Notes + +Release date: January 12, 2021 + +TiDB version: 5.0.0-rc + +TiDB v5.0.0-rc is the predecessor version of TiDB v5.0. In v5.0, PingCAP will be dedicated to helping enterprises quickly build applications based on TiDB, freeing them from worries about database performance, performance jitter, security, high availability, disaster recovery, troubleshooting SQL performance, and so on. + +In v5.0, the key new features or improvements are as follows: + ++ Clustered index. When this feature is enabled, database performance is improved. For example, in the TPC-C tpmC test, TiDB's performance, with clustered index enabled, improves by 39%. ++ Async commit. When this feature is enabled, the write latency is reduced. For example, in the Sysbench olpt-insert test, the write latency of TiDB, with async commit enabled, is reduced by 37.3%. ++ Reduced jitters. This is achieved by improving the optimizer stability and by limiting system tasks' usages of I/O, network, CPU, and memory resources. For example, in the 72-hour performance test, the standard deviation of Sysbench TPS jitter is reduced from 11.09% to 3.36%. ++ Raft Joint Consensus algorithm, which ensures the system availability during the Region membership change. ++ Optimized `EXPLAIN` features and invisible index, which helps Database Administrators (DBAs) debug SQL statements more efficiently. ++ Guaranteed reliability for enterprise data. You can back up data from TiDB to AWS S3 storage and Google Cloud GCS, or restore data from these cloud storage platforms. ++ Improved performance of data import from or data export to AWS S3 storage or TiDB/MySQL, which helps enterprises quickly build applications on the cloud. For example, in the TPC-C test, the performance of importing 1 TiB data improves by 40%, from 254 GiB/h to 366 GiB/h. + +## SQL + +### Support clustered index (experimental) + +When the clustered index feature is enabled, TiDB performance improves significantly (for example in the TPC-C tpmC test, TiDB's performance, with clustered index enabled, improves by 39%) in the following cases: + ++ When data is inserted, the clustered index reduces one write of the index data from the network. ++ When a query with an equivalent condition only involves the primary key, the clustered index reduces one read of index data from the network. ++ When a query with a range condition only involves the primary key, the clustered index reduces multiple reads of index data from the network. ++ When a query with an equivalent or range condition involves the primary key prefix, the clustered index reduces multiple reads of index data from the network. + +Clustered index defines the physical storage order of data in a table. The data in the table is sorted only according to the definition of the clustered index. Each table has only one clustered index. + +Users can enable the clustered index feature by modifying the `tidb_enable_clustered_index` variable. When enabled, the feature takes effect only on newly created tables and applies to the primary key that has multiple columns or is non-integer types in a single column. If the primary key is an integer type in a single column, or if the table has no primary key, the data is sorted in the same way as before, without being affected by the clustered index. + +For example, to check whether a table (`tbl_name`) has a clustered index, execute `select tidb_pk_type from information_schema.tables where table_name = '{tbl_name}'`. + ++ [User document](/system-variables.md#tidb_enable_clustered_index-new-in-v50) ++ Related issue: [#4841](https://github.com/pingcap/tidb/issues/4841) + +### Support invisible indexes + +When users tune performance or select optimal indexes, they can set an index to be `Visible` or `Invisible` by using SQL statements. This setting can avoid performing resource-consuming operations, such as `DROP INDEX` and `ADD INDEX`. + +To modify the visibility of an index, use the `ALTER INDEX` statement. After the modification, the optimizer decides whether to add this index to the index list based on the index visibility. + ++ [User document](/sql-statements/sql-statement-alter-index.md) ++ Related issue: [#9246](https://github.com/pingcap/tidb/issues/9246) + +### Support `EXCEPT` and `INTERSECT` operators + +The `INTERSECT` operator is a set operator, which returns the intersection of the result sets of two or more queries. To some extent, it is an alternative to the `InnerJoin` operator. + +The `EXCEPT` operator is a set operator, which combines the result sets of two queries and returns elements that are in the first query result but not in the second. + ++ [User document](/functions-and-operators/set-operators.md) ++ Related issue: [#18031](https://github.com/pingcap/tidb/issues/18031) + +## Transaction + +### Increase the success rate of executing pessimistic transactions + +In the pessimistic transaction mode, if the tables involved in a transaction contain concurrent DDL operations or `SCHEMA VERSION` changes, the system automatically updates the transaction's `SCHEMA VERSION` to the latest to avoid the transaction being interrupted by DDL operations and to ensure the successful transaction commit. If the transaction is interrupted, the client receives the `Information schema is changed` error message. + ++ [User document](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407) ++ Related issue: [#18005](https://github.com/pingcap/tidb/issues/18005) + +## Character set and collation + +Support case-insensitive comparison sort for character sets. + ++ [User document](/character-set-and-collation.md#new-framework-for-collations) ++ Related issue: [#17596](https://github.com/pingcap/tidb/issues/17596) + +## Security + +### Support desensitizing error messages and log files + +TiDB now supports desensitizing error messages and log files to avoid leaking sensitive information such as ID information and credit card number. + +Users can enable the desensitization feature for different components: + ++ For the TiDB side, set the `tidb_redact_log=1` variable using SQL statements in tidb-server. ++ For the TiKV side, set the `security.redact-info-log = true` configuration in tikv-server. ++ For the PD side, set the `security.redact-info-log = true` configuration in pd-server. [#2852](https://github.com/tikv/pd/issues/2852) [#3011](https://github.com/tikv/pd/pull/3011) ++ For the TiFlash side, set the `security.redact_info_log = true` configuration in tiflash-server and set `security.redact-info-log = true` in tiflash-learner. + +[User document](/log-redaction.md) + +Related issue: [#18566](https://github.com/pingcap/tidb/issues/18566) + +## Performance improvements + +### Support async commit (experimental) + +Enabling the async commit feature can significantly reduce the latency of transactions. For example, with this feature enabled, the latency of transactions in the Sysbench oltp-insert test is 37.3% lower than that when this feature is disabled. + +Previously without the async commit feature, the statements being written were only returned to the client after the two-phase transaction commit finished. Now the async commit feature supports returning the result to the client after the first phase of the two-phase commit finishes. The second phase is then performed asynchronously in the background, thus reducing the latency of transaction commit. + +However, when async commit is enabled, the external consistency of transactions can be guaranteed **only** when `tidb_guarantee_external_consistency = ON` is set. With async commit enabled, the performance might drop. + +Users can enable this feature by setting the global variable `tidb_enable_async_commit = ON`. + ++ [User document](/system-variables.md#tidb_enable_async_commit-new-in-v50) ++ Related issue: [#8316](https://github.com/tikv/tikv/issues/8316) + +### Improve the optimizer's stability in index selection (experimental) + +The optimizer's ability to always select a relatively suitable index greatly determines whether the latency of queries is stable. We have improved and refactored the statistics module to ensure that, for the same SQL statements, the optimizer does not select a different index from multiple candidate indexes each time due to missing or inaccurate statistics. The main improvements to help the optimizer select a relatively suitable index are as follows: + ++ Add more information to the statistics module, such as the multi-column NDV, the multi-column order dependency, and the multi-column function dependency. ++ Refactor the statistics module. + + Delete `TopN` values from `CMSKetch`. + + Refactor the search logic of `TopN`. + + Delete the `TopN` information from the histogram and create an index of the histogram for easy maintenance of Bucket NDV. + +Related issue: [#18065](https://github.com/pingcap/tidb/issues/18065) + +### Optimize performance jitter caused by imperfect scheduling or imperfect I/O flow control + +The TiDB scheduling process occupies resources such as I/O, network, CPU, and memory. If TiDB does not control the scheduled tasks, QPS and delay might cause performance jitter due to resource preemption. After the following optimizations, in the 72-hour test, the standard deviation of Sysbench TPS jitter is reduced from 11.09% to 3.36%. + ++ Reduce the redundant scheduling issues caused by fluctuations of node capacity (always near the waterline) and caused by PD's `store-limit` configuration value set too large. This is achieved by introducing a new set of scheduling calculation formulas enabled via the `region-score-formula-version = v2` configuration item. [#3269](https://github.com/tikv/pd/pull/3269) ++ Enable the cross-Region merge feature by modifying `enable-cross-table-merge = true` to reduce the number of empty Regions. [#3129](https://github.com/tikv/pd/pull/3129) ++ Data compaction in the TiKV background occupies a lot of I/O resources. The system automatically adjusts the compaction rate to balance the contention for I/O resources between background tasks and foreground reads and writes. After enabling this feature via the `rate-limiter-auto-tuned` configuration item, the delay jitter is greatly reduced. [#18011](https://github.com/pingcap/tidb/issues/18011) ++ When TiKV performs garbage collection (GC) and data compaction, partitions occupy CPU and I/O resources. Overlapping data exists during the execution of these two tasks. To reduce I/O usage, the GC Compaction Filter feature combines these two tasks into one and executes them in the same task. This feature is still experimental and you can enable it via `gc.enable-compaction-filter = ture`. [#18009](https://github.com/pingcap/tidb/issues/18009) ++ When TiFlash compresses or sorts data, it occupies a lot of I/O resources. The system alleviates contention for resources by limiting the compression and data sorting's use of I/O resources. This feature is still experimental and you can enable it via `bg_task_io_rate_limit`. + +Related issue: [#18005](https://github.com/pingcap/tidb/issues/18005) + +### Improve the stability of TiFlash in Real-time BI / Data Warehousing scenarios + ++ Limit the memory usage of DeltaIndex to avoid system out of memory (OOM) caused by excessive memory usage in the scenarios of huge data volume. ++ Limit the I/O write traffic used by the background data sorting task to reduce the impact on the foreground tasks. ++ Add new thread pools to queue coprocessor tasks, which avoids system OOM caused by excessive memory usage when processing coprocessors in high concurrency. + +### Other performance optimizations + ++ Improve the execution performance of `delete * from table where id **Note:** + > + > The scope of the variable is changed from session to global, and the default value is changed from `20000` to `0`. If the application relies on the original default value, you need to use the `set global` statement to modify the variable to the original value after the upgrade. + ++ Control temporary tables’ syntax compatibility using the [`tidb_enable_noop_functions`](/system-variables.md#tidb_enable_noop_functions-new-in-v40) system variable. When this variable value is `OFF`, the `CREATE TEMPORARY TABLE` syntax returns an error. ++ Add the following system variables to directly control the garbage collection-related parameters: + - [`tidb_gc_concurrency`](/system-variables.md#tidb_gc_concurrency-new-in-v50) + - [`tidb_gc_enable`](/system-variables.md#tidb_gc_enable-new-in-v50) + - [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) + - [`tidb_gc_run_interval`](/system-variables.md#tidb_gc_run_interval-new-in-v50) + - [`tidb_gc_scan_lock_mode`](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) ++ Change the default value of [`enable-joint-consensus`](/pd-configuration-file.md#enable-joint-consensus-new-in-v50) from `false` to `true`, which enables the Joint Consensus feature by default. ++ Change the value of [`tidb_enable_amend_pessimistic_txn`](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407) from `0` or `1` to `ON` or `OFF`. ++ Change the default value of [`tidb_enable_clustered_index`](/system-variables.md#tidb_enable_clustered_index-new-in-v50) from `OFF` to `INT_ONLY` with the following new meanings: + + `ON`: clustered index is enabled. Adding or deleting non-clustered indexes is supported. + + `OFF`: clustered index is disabled. Adding or deleting non-clustered indexes is supported. + + `INT_ONLY`: the default value. The behavior is consistent with that before v5.0. You can control whether to enable clustered index for the INT type together with `alter-primary-key = false`. + + > **Note:** + > + > The `INT_ONLY` value of `tidb_enable_clustered_index` in 5.0 GA has the same meaning as the `OFF` value in 5.0 RC. After upgrading from a 5.0 RC cluster with the `OFF` setting to 5.0 GA, it will be displayed as `INT_ONLY`. + +### Configuration file parameters + ++ Add the [`index-limit`](/tidb-configuration-file.md#index-limit-new-in-v50) configuration item for TiDB. Its value defaults to `64` and ranges between `[64,512]`. A MySQL table supports 64 indexes at most. If its value exceeds the default setting and more than 64 indexes are created for a table, when the table schema is re-imported into MySQL, an error will be reported. ++ Add the [`enable-enum-length-limit`](/tidb-configuration-file.md#enable-enum-length-limit-new-in-v50) configuration item for TiDB to be compatible and consistent with MySQL’s ENUM/SET length (ENUM length < 255). The default value is `true`. ++ Replace the `pessimistic-txn.enable` configuration item with the [`tidb_txn_mode`](/system-variables.md#tidb_txn_mode) environment variable. ++ Replace the `performance.max-memory` configuration item with [`performance.server-memory-quota`](/tidb-configuration-file.md#server-memory-quota-new-in-v409) ++ Replace the `tikv-client.copr-cache.enable` configuration item with [`tikv-client.copr-cache.capacity-mb`](/tidb-configuration-file.md#capacity-mb). If the item’s value is `0.0`, this feature is disabled. If the item’s value is greater than `0.0`, this feature is enabled. Its default value is `1000.0`. ++ Replace the `rocksdb.auto-tuned` configuration item with [`rocksdb.rate-limiter-auto-tuned`](/tikv-configuration-file.md#rate-limiter-auto-tuned-new-in-v50). ++ Delete the `raftstore.sync-log` configuration item. By default, written data is forcibly spilled to the disk. Before v5.0, you can explicitly disable `raftstore.sync-log`. Since v5.0, the configuration value is forcibly set to `true`. ++ Change the default value of the `gc.enable-compaction-filter` configuration item from `false` to `true`. ++ Change the default value of the `enable-cross-table-merge` configuration item from `false` to `true`. ++ Change the default value of the [`rate-limiter-auto-tuned`](/tikv-configuration-file.md#rate-limiter-auto-tuned-new-in-v50) configuration item from `false` to `true`. + +### Others + ++ Forbid conversion between `VARCHAR` type and `CHAR` type during the column type change to avoid data correctness issues. + +## New features + +### SQL + +#### List partitioning (**Experimental**) + +[User document](/partitioned-table.md#list-partitioning) + +With the list partitioning feature, you can effectively query and maintain tables with a large amount of data. + +With this feature enabled, partitions and how data is distributed among partitions are defined according to the `PARTITION BY LIST(expr) PARTITION part_name VALUES IN (...)` expression. The partitioned tables’ data set supports at most 1024 distinct integer values. You can define the values using the `PARTITION ... VALUES IN (...)` clause. + +To enable list partitioning, set the session variable [`tidb_enable_list_partition`](/system-variables.md#tidb_enable_list_partition-new-in-v50) to `ON`. + +#### List COLUMNS partitioning (**Experimental**) + +[User document](/partitioned-table.md#list-columns-partitioning) + +List COLUMNS partitioning is a variant of list partitioning. You can use multiple columns as partition keys. Besides the integer data type, you can also use the columns in the string, `DATE`, and `DATETIME` data types as partition columns. + +To enable List COLUMNS partitioning, set the session variable [`tidb_enable_list_partition`](/system-variables.md#tidb_enable_list_partition-new-in-v50) to `ON`. + +#### Invisible indexes + +[User document](/sql-statements/sql-statement-alter-index.md), [#9246](https://github.com/pingcap/tidb/issues/9246) + +When you tune performance or select optimal indexes, you can set an index to be `Visible` or `Invisible` by using SQL statements. This setting can avoid performing resource-consuming operations, such as `DROP INDEX` and `ADD INDEX`. + +To modify the visibility of an index, use the `ALTER INDEX` statement. After the modification, the optimizer decides whether to add this index to the index list based on the index visibility. + +#### `EXCEPT` and `INTERSECT` operators + +[User document](/functions-and-operators/set-operators.md), [#18031](https://github.com/pingcap/tidb/issues/18031) + +The `INTERSECT` operator is a set operator, which returns the intersection of the result sets of two or more queries. To some extent, it is an alternative to the `Inner Join` operator. + +The `EXCEPT` operator is a set operator, which combines the result sets of two queries and returns elements that are in the first query result but not in the second. + +### Transaction + +[User document](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407), [#18005](https://github.com/pingcap/tidb/issues/18005) + +In the pessimistic transaction mode, if the tables involved in a transaction contain concurrent DDL operations or `SCHEMA VERSION` changes, the system automatically updates the transaction's `SCHEMA VERSION` to the latest to ensure the successful transaction commit, and to avoid that the client receives the `Information schema is changed` error when the transaction is interrupted by DDL operations or `SCHEMA VERSION` changes. + +This feature is disabled by default. To enable the feature, modify the value of [`tidb_enable_amend_pessimistic_txn`](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407) system variable. This feature is introduced in v4.0.7 and has the following issues fixed in v5.0: + ++ The compatibility issue that occurs when TiDB Binlog executes `Add Column` operations ++ The data inconsistency issue that occurs when using the feature together with the unique index ++ The data inconsistency issue that occurs when using the feature together with the added index + +Currently, this feature still has the following incompatibility issues: + ++ Transaction’s semantics might change when there are concurrent transactions ++ Known compatibility issue that occurs when using the feature together with TiDB Binlog ++ Incompatibility with `Change Column` + +### Character set and collation + +- Support the `utf8mb4_unicode_ci` and `utf8_unicode_ci` collations. [User document](/character-set-and-collation.md#new-framework-for-collations), [#17596](https://github.com/pingcap/tidb/issues/17596) +- Support the case-insensitive comparison sort for collations + +### Security + +[User document](/log-redaction.md), [#18566](https://github.com/pingcap/tidb/issues/18566) + +To meet security compliance requirements (such as *General Data Protection Regulation*, or GDPR), the system supports desensitizing information (such as ID and credit card number) in the output error messages and logs, which can avoid leaking sensitive information. + +TiDB supports desensitizing the output log information. To enable this feature, use the following switches: + ++ The global variable [`tidb_redact_log`](/system-variables.md#tidb_redact_log). Its default value is `0`, which means that desensitization is disabled. To enable desensitization for tidb-server logs, set the variable value to `1`. ++ The configuration item `security.redact-info-log`. Its default value is `false`, which means that desensitization is disabled. To enable desensitization for tikv-server logs, set the variable value to `true`. ++ The configuration item `security.redact-info-log`. Its default value is `false`, which means that desensitization is disabled. To enable desensitization for pd-server logs, set the variable value to `true`. ++ The configuration item `security.redact_info_log` for tiflash-server and `security.redact-info-log` for tiflash-learner. Their default values are both `false`, which means that desensitization is disabled. To enable desensitization for tiflash-server and tiflash-learner logs, set the values of both variables to `true`. + +This feature is introduced in v5.0. To use the feature, enable the system variable and all configuration items above. + +## Performance optimization + +### MPP architecture + +[User document](/tiflash/use-tiflash.md) + +TiDB introduces the MPP architecture through TiFlash nodes. This architecture allows multiple TiFlash nodes to share the execution workload of large join queries. + +When the MPP mode is on, TiDB determines whether to send a query to the MPP engine for computation based on the calculation cost. In the MPP mode, TiDB distributes the computation of table joins to each running TiFlash node by redistributing the join key during data calculation (`Exchange` operation), and thus accelerates the calculation. Furthermore, with the aggregation computing feature that TiFlash has already supported, TiDB can pushdown the computation of a query to the TiFlash MPP cluster. Then the distributed environment can help accelerate the entire execution process and dramatically increase the speed of analytic queries. + +In the TPC-H 100 benchmark test, TiFlash MPP delivers significant processing speed over analytic engines of traditional analytic databases and SQL on Hadoop. With this architecture, you can perform large-scale analytic queries directly on the latest transaction data, with a higher performance than traditional offline analytic solutions. According to the benchmark, with the same cluster resource, TiDB 5.0 MPP shows 2 to 3 times of speedup over Greenplum 6.15.0 and Apache Spark 3.1.1, and some queries have 8 times better performance. + +Currently, the main features that the MPP mode does not support are as follows (For details, refer to [Use TiFlash](/tiflash/use-tiflash.md)): + ++ Table partitioning ++ Window Function ++ Collation ++ Some built-in functions ++ Reading data from TiKV ++ OOM spill ++ Union ++ Full Outer Join + +### Clustered index + +[User document](/clustered-indexes.md), [#4841](https://github.com/pingcap/tidb/issues/4841) + +When you are designing table structures or analyzing database behaviors, it is recommended to use the clustered index feature if you find that some columns with primary keys are often grouped and sorted, queries on these columns often return a certain range of data or a small amount of data with different values, and the corresponding data does not cause read or write hotspot issues. + +Clustered indexes, also known as _index-organized tables_ in some database management systems, is a storage structure associated with the data of a table. When creating a clustered index, you can specify one or more columns from the table as the keys for the index. TiDB stores these keys in a specific structure, which allows TiDB to quickly and efficiently find the rows associated with the keys, thus improves the performance of querying and writing data. + +When the clustered index feature is enabled, the TiDB performance improves significantly (for example in the TPC-C tpmC test, the performance of TiDB, with clustered index enabled, improves by 39%) in the following cases: + ++ When data is inserted, the clustered index reduces one write of the index data from the network. ++ When a query with an equivalent condition only involves the primary key, the clustered index reduces one read of index data from the network. ++ When a query with a range condition only involves the primary key, the clustered index reduces multiple reads of index data from the network. ++ When a query with an equivalent or range condition involves the primary key prefix, the clustered index reduces multiple reads of index data from the network. + +Each table can either use a clustered or non-clustered index to sort and store data. The differences of these two storage structures are as follows: + ++ When creating a clustered index, you can specify one or more columns in the table as the key value of the index. A clustered index sorts and stores the data of a table according to the key value. Each table can have only one clustered index. If a table has a clustered index, it is called a clustered index table. Otherwise, it is called a non-clustered index table. ++ When you create a non-clustered index, the data in the table is stored in an unordered structure. You do not need to explicitly specify the key value of the non-clustered index, because TiDB automatically assigns a unique ROWID to each row of data. During a query, the ROWID is used to locate the corresponding row. Because there are at least two network I/O operations when you query or insert data, the performance is degraded compared with clustered indexes. + +When table data is modified, the database system automatically maintains clustered indexes and non-clustered indexes for you. + +All primary keys are created as non-clustered indexes by default. You can create a primary key as a clustered index or non-clustered index in either of the following two ways: + ++ Specify the keyword `CLUSTERED | NONCLUSTERED` in the statement when creating a table, then the system creates the table in the specified way. The syntax is as follows: + +```sql +CREATE TABLE `t` (`a` VARCHAR(255), `b` INT, PRIMARY KEY (`a`, `b`) CLUSTERED); +``` + +Or + +```sql +CREATE TABLE `t` (`a` VARCHAR(255) PRIMARY KEY CLUSTERED, `b` INT); +``` + +You can execute the statement `SHOW INDEX FROM tbl-name` to query whether a table has a clustered index. + ++ Configure the system variable `tidb_enable_clustered_index` to control the clustered index feature. Supported values are `ON`, `OFF`, and `INT_ONLY`. + + `ON`: Indicates that the clustered index feature is enabled for all types of primary keys. Adding and dropping non-clustered indexes are supported. + + `OFF`: Indicates that the clustered index feature is disabled for all types of primary keys. Adding and dropping non-clustered indexes are supported. + + `INT_ONLY`: The default value. If the variable is set to `INT_ONLY` and `alter-primary-key` is set to `false`, the primary keys which consist of single integer columns are created as clustered indexes by default. The behavior is consistent with that of TiDB v5.0 and earlier versions. + +If a `CREATE TABLE` statement contains the keyword `CLUSTERED | NONCLUSTERED`, the statement overrides the configuration of the system variable and the configuration item. + +You are recommended to use the clustered index feature by specifying the keyword `CLUSTERED | NONCLUSTERED` in statements. In this way, it is more flexible for TiDB to use all data types of clustered and non-clustered indexes in the system at the same time as required. + +It is not recommended to use `tidb_enable_clustered_index = INT_ONLY`, because `INT_ONLY` is temporarily used to make this feature compatible and will be deprecated in the future. + +Limitations for the clustered index are as follows: + ++ Mutual conversion between clustered indexes and non-clustered indexes is not supported. ++ Dropping clustered indexes is not supported. ++ Adding, dropping, and altering clustered indexes using `ALTER TABLE` statements are not supported. ++ Reorganizing and re-creating a clustered index is not supported. ++ Enabling or disabling indexes is not supported, which means the invisible index feature is not effective for clustered indexes. ++ Creating a `UNIQUE KEY` as a clustered index is not supported. ++ Using the clustered index feature together with TiDB Binlog is not supported. After TiDB Binlog is enabled, TiDB only supports creating a single integer primary key as a clustered index. TiDB Binlog does not replicate data changes of existing tables with clustered indexes to the downstream. ++ Using the clustered index feature together with the attributes `SHARD_ROW_ID_BITS` and `PRE_SPLIT_REGIONS` is not supported. ++ If the cluster is upgraded to a later version then rolls back, you need to downgrade newly-added tables by exporting table data before the rollback and importing the data after the rollback. Other tables are not affected. + +### Async Commit + +[User document](/system-variables.md#tidb_enable_async_commit-new-in-v50), [#8316](https://github.com/tikv/tikv/issues/8316) + +The client of the database will wait for the database system to complete the transaction commit in two phases (2PC) synchronously. The transaction returns the result to the client after the first phase commit is successful, and the system executes the second phase commit operation in the background asynchronously to reduce the transaction commit latency. If the transaction write involves only one Region, the second phase is omitted directly, and the transaction becomes a one-phase commit. + +After the Async Commit feature is enabled, with the same hardware and configuration, when Sysbench is set to test the Update index with 64 threads, the average latency decreases by 41.7% from 12.04ms to 7.01ms. + +When Async Commit feature is enabled, to reduce one network interaction latency and improve the performance of data writes, database application developers are recommended to consider reducing the consistency of transactions from linear consistency to [causal consistency](/transaction-overview.md#causal-consistency). The SQL statement to enable causal consistency is `START TRANSACTION WITH CAUSAL CONSISTENCY`. + +After the causal consistency is enabled, with the same hardware and configuration, when Sysbench is set to test oltp_write_only with 64 threads, the average latency decreased by 5.6% from 11.86ms to 11.19ms. + +After the consistency of transactions is reduced from the linear consistency to causal consistency, if there is no interdependence between multiple transactions in the application, the transactions do not have a globally consistent order. + +**The Async Commit feature is enabled by default for newly created v5.0 clusters.** + +This feature is disabled by default for clusters upgraded from earlier versions to v5.0. You can enable this feature by executing the `set global tidb_enable_async_commit = ON;` and `set global tidb_enable_1pc = ON;` statements. + +The limitation for the Async Commit feature is as follows: + ++ Direct downgrade is not supported. + +### Enable the Coprocessor cache feature by default + +[User document](/tidb-configuration-file.md#tikv-clientcopr-cache-new-in-v400), [#18028](https://github.com/pingcap/tidb/issues/18028) + +In 5.0 GA, the Coprocessor cache feature is enabled by default. After this feature is enabled, to reduce the latency of reading data, TiDB caches the calculation results of the operators pushed down to tikv-server in tidb-server. + +To disable the Coprocessor cache feature, you can modify the `capacity-mb` configuration item of `tikv-client.copr-cache` to `0.0`. + +### Improve the execution performance of `delete * from table where id ) ... ON DUPLICATE KEY UPDATE` to read data from non-listed partitions [#24746](https://github.com/pingcap/tidb/issues/24746) + - Fix the potential `index out of range` error when a SQL statement contains both `GROUP BY` and `UNION` [#24281](https://github.com/pingcap/tidb/issues/24281) + - Fix the issue that the `CONCAT` function incorrectly handles the collation [#24296](https://github.com/pingcap/tidb/issues/24296) + - Fix the issue that the `collation_server` global variable does not take effect in new sessions [#24156](https://github.com/pingcap/tidb/pull/24156) + ++ TiKV + + - Fix a TiCDC OOM issue caused by reading old values [#9996](https://github.com/tikv/tikv/issues/9996) [#9981](https://github.com/tikv/tikv/issues/9981) + - Fix the issue of empty value in the secondary index for the clustered primary key column when collation is `latin1_bin` [#24548](https://github.com/pingcap/tidb/issues/24548) + - Add the `abort-on-panic` configuration, which allows TiKV to generate the core dump file when panic occurs. Users still need to correctly configure the environment to enable core dump [#10216](https://github.com/tikv/tikv/pull/10216) + - Fix the performance regression issue of `point get` queries that occurs when TiKV is not busy [#10046](https://github.com/tikv/tikv/issues/10046) + ++ PD + + - Fix the issue that the PD Leader re-election is slow when there are many stores [#3697](https://github.com/tikv/pd/issues/3697) + - Fix the panic issue that occurs when removing the evict leader scheduler from a non-existent store [#3660](https://github.com/tikv/pd/issues/3660) + - Fix the issue that the statistics are not updated after offline peers are merged [#3611](https://github.com/tikv/pd/issues/3611) + ++ TiFlash + + - Fix the issue of incorrect results when cloning shared delta index concurrently + - Fix the potential issue that TiFlash fails to restart with incomplete data + - Fix the issue that old dm files are not removed automatically + - Fix the potential panic that occurs when the Compaction Filter feature is enabled + - Fix the potential issue that `ExchangeSender` sends duplicated data + - Fix the issue that TiFlash cannot resolve the lock fallen back from async commit + - Fix the issue of incorrect results returned when the casted result of the `TIMEZONE` type contains the `TIMESTAMP` type + - Fix the TiFlash panic issue that occurs during Segment Split + - Fix the issue that the execution information about the non-root MPP task is not accurate + ++ Tools + + + TiCDC + + - Fix the issue that the time zone information is lost in the Avro output [#1712](https://github.com/pingcap/ticdc/pull/1712) + - Support cleaning up stale temporary files in Unified Sorter and forbid sharing the `sort-dir` directory [#1742](https://github.com/pingcap/ticdc/pull/1742) + - Fix a deadlock bug in the KV client that occurs when many stale Regions exist [#1599](https://github.com/pingcap/ticdc/issues/1599) + - Fix the wrong help information in the `--cert-allowed-cn` flag [#1697](https://github.com/pingcap/ticdc/pull/1697) + - Revert the update for `explicit_defaults_for_timestamp` which requires the `SUPER` privilege when replicating data to MySQL [#1750](https://github.com/pingcap/ticdc/pull/1750) + - Support the sink flow control to reduce the risk of memory overflow [#1840](https://github.com/pingcap/ticdc/pull/1840) + - Fix a bug that the replication task might stop when moving a table [#1828](https://github.com/pingcap/ticdc/pull/1828) + - Fix the issue that the TiKV GC safe point is blocked due to the stagnation of TiCDC changefeed checkpoint [#1759](https://github.com/pingcap/ticdc/pull/1759) + + + Backup & Restore (BR) + + - Fix the issue that the `DELETE` events are lost during the log restore [#1063](https://github.com/pingcap/br/issues/1063) + - Fix a bug that causes BR to send too many useless RPC requests to TiKV [#1037](https://github.com/pingcap/br/pull/1037) + - Fix the issue that no error is output when a backup operation fails [#1043](https://github.com/pingcap/br/pull/1043) + + + TiDB Lightning + + - Fix the issue of TiDB Lightning panic that occurs when generating KV data [#1127](https://github.com/pingcap/br/pull/1127) + - Fix the issue that TiDB Lightning in the TiDB-backend mode cannot load any data when the autocommit is disabled [#1104](https://github.com/pingcap/br/issues/1104) + - Fix a bug that the batch split Region fails due to the total key size exceeding the raft entry limit during the data import [#969](https://github.com/pingcap/br/issues/969) diff --git a/releases/release-5.0.3.md b/releases/release-5.0.3.md new file mode 100644 index 000000000000..dd0cd804340e --- /dev/null +++ b/releases/release-5.0.3.md @@ -0,0 +1,144 @@ +--- +title: TiDB 5.0.3 Release Notes +--- + +# TiDB 5.0.3 Release Notes + +Release date: July 2, 2021 + +TiDB version: 5.0.3 + +## Compatibility Changes + ++ TiDB + + - After a v4.0 cluster is upgraded to v5.0 or a later version (dev or v5.1), the default value of the `tidb_multi_statement_mode` variable changes from `WARN` to `OFF` + - TiDB is now compatible with MySQL 5.7's noop variable `innodb_default_row_format`. Setting this variable will have no effect. [#23541](https://github.com/pingcap/tidb/issues/23541) + +## Feature Enhancements + ++ Tools + + + TiCDC + + - Add an HTTP API to get the changefeed information and the health information of the node [#1955](https://github.com/pingcap/ticdc/pull/1955) + - Add the SASL/SCRAM support for the kafka sink [#1942](https://github.com/pingcap/ticdc/pull/1942) + - Make TiCDC support `--data-dir` at the server level [#2070](https://github.com/pingcap/ticdc/pull/2070) + +## Improvements + ++ TiDB + + - Support pushing down the `TopN` operator to TiFlash [#25162](https://github.com/pingcap/tidb/pull/25162) + - Support pushing down the built-in function `json_unquote()` to TiKV [#24415](https://github.com/pingcap/tidb/issues/24415) + - Support removing the union branch from the dual table [#25614](https://github.com/pingcap/tidb/pull/25614) + - Support pushing down the built-in function `replace()` to TiFlash [#25565](https://github.com/pingcap/tidb/pull/25565) + - Support pushing down the built-in functions `unix_timestamp()`, `concat()`, `year()`, `day()`, `datediff()`, `datesub()`, and `concat_ws()` to TiFlash [#25564](https://github.com/pingcap/tidb/pull/25564) + - Optimize the aggregate operator's cost factor [#25241](https://github.com/pingcap/tidb/pull/25241) + - Support pushing down the `Limit` operator to TiFlash [#25159](https://github.com/pingcap/tidb/pull/25159) + - Support pushing down the built-in function `str_to_date` to TiFlash [#25148](https://github.com/pingcap/tidb/pull/25148) + - Allow the MPP outer join to choose the build table based on the table row count [#25142](https://github.com/pingcap/tidb/pull/25142) + - Support pushing down the built-in functions `left()`, `right()`, and `abs()` to TiFlash [#25133](https://github.com/pingcap/tidb/pull/25133) + - Support pushing down the Broadcast Cartesian join to TiFlash [#25106](https://github.com/pingcap/tidb/pull/25106) + - Support pushing down the `Union All` operator to TiFlash [#25051](https://github.com/pingcap/tidb/pull/25051) + - Support balancing the MPP query workload among different TiFlash nodes based on Regions [#24724](https://github.com/pingcap/tidb/pull/24724) + - Support invalidating stale Regions in the cache after the MPP query is executed [#24432](https://github.com/pingcap/tidb/pull/24432) + - Improve the MySQL compatibility of the built-in function `str_to_date` for the format specifiers `%b/%M/%r/%T` [#25767](https://github.com/pingcap/tidb/pull/25767) + ++ TiKV + + - Limit the TiCDC sink's memory consumption [#10305](https://github.com/tikv/tikv/pull/10305) + - Add the memory-bounded upper limit for the TiCDC old value cache [#10313](https://github.com/tikv/tikv/pull/10313) + ++ PD + + - Update TiDB Dashboard to v2021.06.15.1 [#3798](https://github.com/pingcap/pd/pull/3798) + ++ TiFlash + + - Support casting the `STRING` type to the `DOUBLE` type + - Support the `STR_TO_DATE()` function + - Optimize the non-joined data in right outer join using multiple threads + - Support the Cartesian join + - Support the `LEFT()` and `RIGHT()` functions + - Support automatically invalidating stale Regions in MPP queries + - Support the `ABS()` function + ++ Tools + + + TiCDC + + - Refine gRPC's reconnection logic and increase the KV client's throughput [#1586](https://github.com/pingcap/ticdc/issues/1586) [#1501](https://github.com/pingcap/ticdc/issues/1501#issuecomment-820027078) [#1682](https://github.com/pingcap/ticdc/pull/1682) [#1393](https://github.com/pingcap/ticdc/issues/1393) [#1847](https://github.com/pingcap/ticdc/pull/1847) [#1905](https://github.com/pingcap/ticdc/issues/1905) [#1904](https://github.com/pingcap/ticdc/issues/1904) + - Make the sorter I/O errors more user-friendly + +## Bug Fixes + ++ TiDB + + - Fix the issue that an incorrect result is returned when using merge join on the `SET` type column [#25669](https://github.com/pingcap/tidb/issues/25669) + - Fix the data corruption issue in the `IN` expression's arguments [#25591](https://github.com/pingcap/tidb/issues/25591) + - Avoid the sessions of GC being affected by global variables [#24976](https://github.com/pingcap/tidb/issues/24976) + - Fix the panic issue that occurs when using `limit` in the window function queries [#25344](https://github.com/pingcap/tidb/issues/25344) + - Fix the wrong value returned when querying a partitioned table using `Limit` [#24636](https://github.com/pingcap/tidb/issues/24636) + - Fix the issue that `IFNULL` does not correctly take effect on the `ENUM` or `SET` type column [#24944](https://github.com/pingcap/tidb/issues/24944) + - Fix the wrong results caused by changing the `count` in the join subqueries to `first_row` [#24865](https://github.com/pingcap/tidb/issues/24865) + - Fix the query hang issue that occurs when `ParallelApply` is used under the `TopN` operator [#24930](https://github.com/pingcap/tidb/issues/24930) + - Fix the issue that more results than expected are returned when executing SQL statements using multi-column prefix indexes [#24356](https://github.com/pingcap/tidb/issues/24356) + - Fix the issue that the `<=>` operator cannot correctly take effect [#24477](https://github.com/pingcap/tidb/issues/24477) + - Fix the data race issue of the parallel `Apply` operator [#23280](https://github.com/pingcap/tidb/issues/23280) + - Fix the issue that the `index out of range` error is reported when sorting the IndexMerge results of the PartitionUnion operator [#23919](https://github.com/pingcap/tidb/issues/23919) + - Fix the issue that setting the `tidb_snapshot` variable to an unexpectedly large value might damage the transaction isolation [#25680](https://github.com/pingcap/tidb/issues/25680) + - Fix the issue that the ODBC-styled constant (for example, `{d '2020-01-01'}`) cannot be used as the expression [#25531](https://github.com/pingcap/tidb/issues/25531) + - Fix the issue that `SELECT DISTINCT` converted to `Batch Get` causes incorrect results [#25320](https://github.com/pingcap/tidb/issues/25320) + - Fix the issue that backing off queries from TiFlash to TiKV cannot be triggered [#23665](https://github.com/pingcap/tidb/issues/23665) [#24421](https://github.com/pingcap/tidb/issues/24421) + - Fix the `index-out-of-range` error that occurs when checking `only_full_group_by` [#23839](https://github.com/pingcap/tidb/issues/23839)) + - Fix the issue that the result of index join in correlated subqueries is wrong [#25799](https://github.com/pingcap/tidb/issues/25799) + ++ TiKV + + - Fix the wrong `tikv_raftstore_hibernated_peer_state` metric [#10330](https://github.com/tikv/tikv/issues/10330) + - Fix the wrong arguments type of the `json_unquote()` function in the coprocessor [#10176](https://github.com/tikv/tikv/issues/10176) + - Skip clearing callback during graceful shutdown to avoid breaking ACID in some cases [#10353](https://github.com/tikv/tikv/issues/10353) [#10307](https://github.com/tikv/tikv/issues/10307) + - Fix a bug that the read index is shared for replica reads on a Leader [#10347](https://github.com/tikv/tikv/issues/10347) + - Fix the wrong function that casts `DOUBLE` to `DOUBLE` [#25200](https://github.com/pingcap/tidb/issues/25200) ++ PD + + - Fix the data race issue that occurs when loading TTL configurations after the scheduler is started [#3771](https://github.com/tikv/pd/issues/3771) + - Fix a bug that the `is_learner` field of the `TIKV_REGION_PEERS` table in TiDB is incorrect [#3372](https://github.com/tikv/pd/issues/3372) [#24293](https://github.com/pingcap/tidb/issues/24293) + - Fix the issue that when all TiKV nodes in a zone are offline or down, PD does not schedule replicas to other zones [#3705](https://github.com/tikv/pd/issues/3705) + - Fix the issue that PD might get panic after the scatter Region scheduler is added [#3762](https://github.com/tikv/pd/pull/3762) + ++ TiFlash + + - Fix the issue that TiFlash keeps restarting because of the split failure + - Fix the potential issue that TiFlash cannot delete the delta data + - Fix a bug that TiFlash adds wrong padding for non-binary characters in the `CAST` function + - Fix the issue of incorrect results when handling aggregation queries with complex `GROUP BY` columns + - Fix the TiFlash panic issue that occurs under heavy write pressure + - Fix the panic that occurs when the right jon key is not nullalbe and the left join key is nullable + - Fix the potential issue that the `read-index` requests take a long time + - Fix the panic issue that occurs when the read load is heavy + - Fix the panic issue that might occur when the `Date_Format` function is called with the `STRING` type argument and `NULL` values + ++ Tools + + + TiCDC + + - Fix the issue that TiCDC owner exits when refreshing the checkpoint [#1902](https://github.com/pingcap/ticdc/issues/1902) + - Fix a bug that some MySQL connection might leak after MySQL sink meets the error and pauses [#1946](https://github.com/pingcap/ticdc/pull/1946) + - Fix the panic issue that occurs when TiCDC fails to read `/proc/meminfo` [#2024](https://github.com/pingcap/ticdc/pull/2024) + - Reduce TiCDC's runtime memory consumption [#2012](https://github.com/pingcap/ticdc/pull/2012) [#1958](https://github.com/pingcap/ticdc/pull/1958) + - Fix a bug that might cause TiCDC server panic due to the late calculation of resolved ts [#1576](https://github.com/pingcap/ticdc/issues/1576) + - Fix the potential deadlock issue for the processor [#2142](https://github.com/pingcap/ticdc/pull/2142) + + + Backup & Restore (BR) + + - Fix a bug that all system tables are filtered during restore [#1197](https://github.com/pingcap/br/issues/1197) [#1201](https://github.com/pingcap/br/issues/1201) + - Fix the issue that Backup & Restore reports the error of "file already exists" when TDE is enabled during the restore [#1179](https://github.com/pingcap/br/issues/1179) + + + TiDB Lightning + + - Fix the TiDB Lightning panic issue for some special data [#1213](https://github.com/pingcap/br/issues/1213) + - Fix the EOF error reported when TiDB Lightning splits the imported large CSV files [#1133](https://github.com/pingcap/br/issues/1133) + - Fix a bug that an excessively large base value is generated when TiDB Lightning imports tables with the `auto_increment` column of the `FLOAT` or `DOUBLE` type [#1186](https://github.com/pingcap/br/pull/1186) + - Fix the issue that TiDB fails to parse the `DECIMAL` type data in Parquet files [#1277](https://github.com/pingcap/br/pull/1277) diff --git a/releases/release-5.1.0.md b/releases/release-5.1.0.md new file mode 100644 index 000000000000..fd50b983d733 --- /dev/null +++ b/releases/release-5.1.0.md @@ -0,0 +1,348 @@ +--- +title: TiDB 5.1 Release Notes +--- + +# TiDB 5.1 Release Notes + +Release date: June 24, 2021 + +TiDB version: 5.1.0 + +In v5.1, the key new features or improvements are as follows: + +- Support the Common Table Expression (CTE) feature of MySQL 8.0 to improve the readability and execution efficiency of SQL statements. +- Support changing column types online to improve code development flexibility. +- Introduce a new statistics type to improve query stability, which is enabled as an experimental feature by default. +- Support the dynamic privilege feature of MySQL 8.0 to implement more fine-grained control over certain operations. +- Support directly reading data from the local replica using the Stale Read feature to reduce read latency and improve query performance (Experimental). +- Add the Lock View feature to facilitate database administrators (DBAs) to observe transaction locking events and troubleshoot deadlock problems (Experimental). +- Add TiKV write rate limiter for background tasks to ensure that the latency of read and write requests is stable. + +## Compatibility changes + +> **Note:** +> +> When upgrading from an earlier TiDB version to v5.1, if you want to know the compatibility change notes of all intermediate versions, you can check the [Release Notes](/releases/release-notes.md) for the corresponding version. + +### System variables + +| Variable name | Change type | Description | +|:----------|:-----------|:-----------| +| [`cte_max_recursion_depth`](/system-variables.md#cte_max_recursion_depth) | Newly added | Controls the maximum recursion depth in Common Table Expressions. | +| [`init_connect`](/system-variables.md#init_connect) | Newly added | Controls the initial connection to a TiDB server. | +| [`tidb_analyze_version`](/system-variables.md#tidb_analyze_version-new-in-v510) | Newly added | Controls how TiDB collects statistics. The default value of this variable is `2`. This is an experimental feature. | +| [`tidb_enable_enhanced_security`](/system-variables.md#tidb_enable_enhanced_security) | Newly added | Indicates whether the TiDB server you are connected to has the Security Enhanced Mode (SEM) enabled. This variable setting cannot be changed without restarting the TiDB server. | +| [`tidb_enforce_mpp`](/system-variables.md#tidb_enforce_mpp-new-in-v51) | Newly added | Controls whether to ignore the optimizer's cost estimation and to forcibly use the MPP mode for query execution. The data type of this variable is `BOOL` and the default value is `false`. | +| [`tidb_partition_prune_mode`](/system-variables.md#tidb_partition_prune_mode-new-in-v51) | Newly added | Specifies whether to enable dynamic pruning mode for partitioned tables. This feature is experimental. The default value of this variable is `static`, which means the dynamic pruning mode for partitioned tables is disabled by default. | + +### Configuration file parameters + +| Configuration file | Configuration item | Change type | Description | +|:----------|:-----------|:-----------|:-----------| +| TiDB configuration file | [`security.enable-sem`](/tidb-configuration-file.md#enable-sem) | Newly added | Controls whether to enable the Security Enhanced Mode (SEM). The default value of this configuration item is `false`, which means the SEM is disabled. | +| TiDB configuration file | [`performance.committer-concurrency`](/tidb-configuration-file.md#committer-concurrency) | Modified | Controls the concurrency number for requests related to commit operations in the commit phase of a single transaction. The default value is changed from `16` to `128`. | +| TiDB configuration file | [`performance.tcp-no-delay`](/tidb-configuration-file.md#tcp-no-delay) | Newly added | Determines whether to enable TCP_NODELAY at the TCP layer. The default value is `true`, which means TCP_NODELAY is enabled. | +| TiDB configuration file | [`performance.enforce-mpp`](/tidb-configuration-file.md#enforce-mpp) | Newly added | Controls whether TiDB ignores cost estimates of Optimizer at the instance level and enforces the MPP mode. The default value is `false`. This configuration item controls the initial value of the system variable [`tidb_enforce_mpp`](/system-variables.md#tidb_enforce_mpp-new-in-v51). | +| TiDB configuration file | [`pessimistic-txn.deadlock-history-capacity`](/tidb-configuration-file.md#deadlock-history-capacity) | Newly added | Sets the maximum number of deadlock events that can be recorded in the [`INFORMATION_SCHEMA.DEADLOCKS`](/information-schema/information-schema-deadlocks.md) table of a single TiDB server. The default value is `10`. | +| TiKV configuration file | [`abort-on-panic`](/tikv-configuration-file.md#abort-on-panic) | Newly added | Sets whether the `abort` process allows the system to generate core dump files when TiKV panics. The default value is `false`, which means it is not allowed to generate core dump files. | +| TiKV configuration file | [`hibernate-regions`](/tikv-configuration-file.md#hibernate-regions) | Modified | The default value is changed from `false` to `true`. If a Region is idle for a long time, it is automatically set as hibernated. | +| TiKV configuration file | [`old-value-cache-memory-quota`](/tikv-configuration-file.md#old-value-cache-memory-quota) | Newly added | Sets the upper limit of memory usage by TiCDC old values. The default value is `512MB`. | +| TiKV configuration file | [`sink-memory-quota`](/tikv-configuration-file.md#sink-memory-quota) | Newly added | Sets the upper limit of memory usage by TiCDC data change events. The default value is `512MB`. | +| TiKV configuration file | [`incremental-scan-threads`](/tikv-configuration-file.md#incremental-scan-threads) | Newly added | Sets the number of threads for the task of incrementally scanning historical data. The default value is `4`, which means there are four threads for the task. | +| TiKV configuration file | [`incremental-scan-concurrency`](/tikv-configuration-file.md#incremental-scan-concurrency) | Newly added | Sets the maximum number of concurrent executions for the tasks of incrementally scanning historical data. The default value is `6`, which means that six tasks can be concurrently executed at most. | +| TiKV configuration file | [`soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) | Modified | The soft limit on the pending compaction bytes. The default value is changed from `"64GB"` to `"192GB"`. | +| TiKV configuration file | [`storage.io-rate-limit`](/tikv-configuration-file.md#storageio-rate-limit) | Newly added | Controls the I/O rate of TiKV writes. The default value of `storage.io-rate-limit.max-bytes-per-sec` is `"0MB"`. | +| TiKV configuration file | [`resolved-ts.enable`](/tikv-configuration-file.md#enable) | Newly added | Determines whether to maintain the `resolved-ts` for all Region leaders. The default value is `true`. | +| TiKV configuration file | [`resolved-ts.advance-ts-interval`](/tikv-configuration-file.md#advance-ts-interval) | Newly added | The interval at which the `resolved-ts` is forwarded. The default value is `"1s"`. You can change the value dynamically. | +| TiKV configuration file | [`resolved-ts.scan-lock-pool-size`](/tikv-configuration-file.md#scan-lock-pool-size) | Newly added | The number of threads that TiKV uses to scan the MVCC (multi-version concurrency control) lock data when initializing the `resolved-ts`. The default value is `2`. | + +### Others + +- Upgrade the Go compiler version of TiDB from go1.13.7 to go1.16.4, which improves the TiDB performance. If you are a TiDB developer, upgrade your Go compiler version to ensure a smooth compilation. +- Avoid creating tables with clustered indexes in the cluster that uses TiDB Binlog during the TiDB rolling upgrade. +- Avoid executing statements like `alter table ... modify column` or `alter table ... change column` during the TiDB rolling upgrade. +- Since v5.1, setting the replica of system tables, when building TiFlash replicas for each table, is no longer supported. Before upgrading the cluster, you need to clear the relevant system table replicas; otherwise, the upgrade will fail. +- Deprecate the `--sort-dir` parameter in the `cdc cli changefeed` command of TiCDC. Instead, you can set `--sort-dir` in the `cdc server` command. [#1795](https://github.com/pingcap/ticdc/pull/1795) + +## New features + +### SQL + +- Support the Common Table Expression (CTE) feature of MySQL 8.0. + + This feature empowers TiDB with the capability of querying hierarchical data recursively or non-recursively and meets the needs of using tree queries to implement application logics in multiple sectors such as human resources, manufacturing, financial markets, and education. + + In TiDB, you can apply the `WITH` statement to use Common Table Expressions. [User document](/sql-statements/sql-statement-with.md), [#17472](https://github.com/pingcap/tidb/issues/17472) + +- Support the dynamic privilege feature of MySQL 8.0. + + Dynamic privileges are used to limit the `SUPER` privilege and provide TiDB with more flexible privilege configuration for more fine-grained access control. For example, you can use dynamic privileges to create a user account that can only perform `BACKUP` and `RESTORE` operations. + + The supported dynamic privileges are as follows: + + - `BACKUP_ADMIN` + - `RESTORE_ADMIN` + - `ROLE_ADMIN` + - `CONNECTION_ADMIN` + - `SYSTEM_VARIABLES_ADMIN` + + You can also use plugins to add new privileges. To check out all supported privileges, execute the `SHOW PRIVILEGES` statement. [User document](/privilege-management.md) + +- Add a new configuration item for the Security Enhanced Mode (SEM), which divides the TiDB administrator privileges in a finer-grained way. + + The Security Enhanced Mode is disabled by default. To enable it, see the [user document](/system-variables.md#tidb_enable_enhanced_security). + +- Enhance the capability of changing column types online. Support changing the column type online using the `ALTER TABLE` statement, including but not limited to: + + - Changing `VARCHAR` to `BIGINT` + - Modifying the `DECIMAL` precision + - Compressing the length of `VARCHAR(10)` to `VARCHAR(5)` + + [User document](/sql-statements/sql-statement-modify-column.md) + +- Introduce a new SQL syntax `AS OF TIMESTAMP` to perform Stale Read, a new experimental feature used to read historical data from a specified point in time or from a specified time range. + + [User document](/stale-read.md), [#21094](https://github.com/pingcap/tidb/issues/21094) + + The examples of `AS OF TIMESTAMP` are as follows: + + ```sql + SELECT * FROM t AS OF TIMESTAMP '2020-09-06 00:00:00'; + START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00'; + SET TRANSACTION READ ONLY as of timestamp '2020-09-06 00:00:00'; + ``` + +- Introduce a new statistics type `tidb_analyze_version = 2` (Experimental). + + `tidb_analyze_version` is set to `2` by default, which avoids the large errors that might occur in the large data volume caused by hash conflicts in Version 1 and maintains the estimation accuracy in most scenarios. + + [User document](/statistics.md) + +### Transaction + ++ Support the Lock View feature (Experimental) + + The Lock View feature provides more information about lock conflicts and lock waits of pessimistic locks, which helps DBAs to observe transaction locking conditions and troubleshoot deadlock problems. [#24199](https://github.com/pingcap/tidb/issues/24199) + + User document: + + - View the pessimistic locks and other locks that currently occur on all TiKV nodes in the clusters: [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) + - View several deadlock errors that recently occurred on the TiDB nodes: [`DEADLOCKS`](/information-schema/information-schema-deadlocks.md) + - View the transaction information executed currently on the TiDB nodes: [`TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) + +### Performance + ++ Stale read of data replicas (Experimental) + + Read local replicas data directly to reduce read latency and improve the query performance + + [User document](/stale-read.md), [#21094](https://github.com/pingcap/tidb/issues/21094) + ++ Enable the Hibernate Region feature by default. + + If a Region is in an inactive state for a long time, it is automatically set to a silent state, which reduces the system overhead of the heartbeat information between the Leader and the Follower. + + [User document](/tikv-configuration-file.md#hibernate-regions), [#10266](https://github.com/tikv/tikv/pull/10266) + +### Stability + ++ Solve the replication stability issue of TiCDC + + - Improve TiCDC memory usage to avoid OOM in the following scenarios + - If large amounts of data is accumulated during the replication interruption, exceeding 1TB, the re-replication causes OOM problems. + - Large amounts of data writes cause OOM problems in TiCDC. + - Reduce the possibility of TiCDC replication interruption in the following scenarios: + + [project#11](https://github.com/pingcap/ticdc/projects/11) + + - Replication interruption when the network is unstable + - Replication interruption when some TiKV/PD/TiCDC nodes are down + ++ TiFlash storage memory control + + Optimize the speed and memory usage of Region snapshot generation and reduce the possibility of OOM + ++ Add a write rate limiter for TiKV background tasks (TiKV Write Rate Limiter) + + To ensure the duration stability of read and write requests, TiKV Write Rate Limiter smoothes the write traffic of TiKV background tasks such as GC and Compaction. The default value of TiKV background task write rate limiter is "0MB". It is recommended to set this value to the optimal I/O bandwidth of the disk, such as the maximum I/O bandwidth specified by the cloud disk manufacturer. + + [User document](/tikv-configuration-file.md#storageio-rate-limit), [#9156](https://github.com/tikv/tikv/issues/9156) + ++ Solve scheduling stability issues when multiple scaling tasks are performed at the same time + +### Telemetry + +TiDB adds the running status of TiDB cluster requests in telemetry, including execution status, failure status, etc. + +To learn more about the information and how to disable this behavior, refer to [Telemetry](https://docs.pingcap.com/zh/tidb/stable/telemetry). + +## Improvements + ++ TiDB + + - Support the built-in function `VITESS_HASH()` [#23915](https://github.com/pingcap/tidb/pull/23915) + - Support pushing down data of the enumerated type to TiKV to improve performance when using enumerated types in `WHERE` clauses [#23619](https://github.com/pingcap/tidb/issues/23619) + - Optimize the calculation of Window Function to solve TiDB OOM problems when paging data with ROW_NUMBER() [#23807](https://github.com/pingcap/tidb/issues/23807) + - Optimize the calculation of `UNION ALL` to solve the TiDB OOM problems when using `UNION ALL` to join a large number of `SELECT` statements [#21441](https://github.com/pingcap/tidb/issues/21441) + - Optimize the dynamic pruning mode of partitioned tables to improve performance and stability [#24150](https://github.com/pingcap/tidb/issues/24150) + - Fix the `Region is Unavailable` issue that occurs in multiple scenarios [project#62](https://github.com/pingcap/tidb/projects/62) + - Fix multiple `Region is Unavailable` issues that might occur in frequent scheduling situations + - Fix `Region is Unavailable` issue that might occur in some high stress write situations + - Avoid frequently reading the `mysql.stats_histograms` table if the cached statistics is up-to-date to avoid high CPU usage [#24317](https://github.com/pingcap/tidb/pull/24317) + ++ TiKV + + - Use `zstd` to compress Region snapshots, preventing large space differences between nodes in case of heavy scheduling or scaling [#10005](https://github.com/tikv/tikv/pull/10005) + - Solve OOM issues in multiple cases [#10183](https://github.com/tikv/tikv/issues/10183) + + - Add memory usage tracking for each module + - Solve the OOM issue caused by oversized Raft entries cache + - Solve the OOM issue caused by stacked GC tasks + - Solve the OOM issue caused by fetching too many Raft entries from the Raft log to memory at one time + + - Split Regions more evenly to mitigate the issue that the growth of Region size exceeds the splitting speed when there are hotspot writes [#9785](https://github.com/tikv/tikv/issues/9785) + ++ TiFlash + + - Support `Union All`, `TopN`, and `Limit` functions + - Support the Cartesian product including left outer join and semi anti join in MPP mode + - Optimize lock operations to avoid that running DDL statements and read operations are blocked by each other + - Optimize cleanup of expired data by TiFlash + - Support further filtering of query filters on `timestamp` columns at the TiFlash storage level + - Improve the startup and scalability speed of TiFlash when a large number of tables are in a cluster + - Improve TiFlash compatibility when running on unknown CPUs + ++ PD + + - Avoid unexpected statistics after adding the `scatter region` scheduler [#3602](https://github.com/pingcap/pd/pull/3602) + - Solve multiple scheduling issues in the scaling process + + - Optimize the generation process of replica snapshots to solve slow scheduling issues during scaling [#3563](https://github.com/tikv/pd/issues/3563) [#10059](https://github.com/tikv/tikv/pull/10059) [#10001](https://github.com/tikv/tikv/pull/10001) + - Solve slow scheduling issues caused by heartbeat pressure due to traffic changes [#3693](https://github.com/tikv/pd/issues/3693) [#3739](https://github.com/tikv/pd/issues/3739) [#3728](https://github.com/tikv/pd/issues/3728) [#3751](https://github.com/tikv/pd/issues/3751) + - Reduce the space discrepancies of large clusters due to scheduling, and optimize the scheduling formula to prevent the bursting issue (which is similar to heterogeneous space clusters) caused by large compression rate discrepancies [#3592](https://github.com/tikv/pd/issues/3592) [#10005](https://github.com/tikv/tikv/pull/10005) + ++ Tools + + + Backup & Restore (BR) + + - Support backing up and restoring system tables in the `mysql` schema [#1143](https://github.com/pingcap/br/pull/1143) [#1078](https://github.com/pingcap/br/pull/1078) + - Support the S3-compatible storage that is based on the virtual-host addressing mode [#10243](https://github.com/tikv/tikv/pull/10243) + - Optimize the format of backupmeta to reduce memory usage [#1171](https://github.com/pingcap/br/pull/1171) + + + TiCDC + + - Improve the descriptions of some log messages to be clearer and more useful for diagnosing problems [#1759](https://github.com/pingcap/ticdc/pull/1759) + - Support the back pressure feature to allow the TiCDC scanning speed to sense the downstream processing capacity [#10151](https://github.com/tikv/tikv/pull/10151) + - Reduce memory usage when TiCDC performs the initial scan [#10133](https://github.com/tikv/tikv/pull/10133) + - Improve the cache hit rate for the TiCDC Old Value in pessimistic transactions [#10089](https://github.com/tikv/tikv/pull/10089) + + + Dumpling + + - Improve the logic of exporting data from TiDB v4.0 to avoid TiDB becoming out of memory (OOM) [#273](https://github.com/pingcap/dumpling/pull/273) + + - Fix the issue that no error is output when a backup operation fails [#280](https://github.com/pingcap/dumpling/pull/280) + + + TiDB Lightning + + - Improve data importing speed. The optimization results show that the speed of importing TPC-C data is increased by 30%, and the speed of importing large tables (2TB+) with more indexes (5 indexes) is increased by more than 50%. [#753](https://github.com/pingcap/br/pull/753) + - Add a pre-check on the data to be imported and also on the target cluster before importing, and report errors to reject the import process if it does not meet the import requirements [#999](https://github.com/pingcap/br/pull/999) + - Optimize the timing of checkpoint updates on the Local backend to improve performance of restarting from breakpoints[#1080](https://github.com/pingcap/br/pull/1080) + +## Bug Fixes + ++ TiDB + + - Fix the issue that the execution result of project elimination might be wrong when the projection result is empty [#23887](https://github.com/pingcap/tidb/issues/23887) + - Fix the issue of wrong query results when a column contains `NULL` values in some cases [#23891](https://github.com/pingcap/tidb/issues/23891) + - Forbid generating MPP plans when the scan contains virtual columns [#23886](https://github.com/pingcap/tidb/issues/23886) + - Fix the wrong reuse of `PointGet` and `TableDual` in Plan Cache [#23187](https://github.com/pingcap/tidb/issues/23187) [#23144](https://github.com/pingcap/tidb/issues/23144) [#23304](https://github.com/pingcap/tidb/issues/23304) [#23290](https://github.com/pingcap/tidb/issues/23290) + - Fix the error that occurs when the optimizer builds the `IndexMerge` plan for clustered indexes [#23906](https://github.com/pingcap/tidb/issues/23906) + - Fix the type inference of the BIT-type errors [#23832](https://github.com/pingcap/tidb/issues/23832) + - Fix the issue that some optimizer hints do not take effect when the `PointGet` operator exists [#23570](https://github.com/pingcap/tidb/issues/23570) + - Fix the issue that DDL operations might fail when rolling back due to an error [#23893](https://github.com/pingcap/tidb/issues/23893) + - Fix the issue that the index range of the binary literal constant is incorrectly built [#23672](https://github.com/pingcap/tidb/issues/23672) + - Fix the potential wrong results of the `IN` clause in some cases [#23889](https://github.com/pingcap/tidb/issues/23889) + - Fix the wrong results of some string functions [#23759](https://github.com/pingcap/tidb/issues/23759) + - Users now need both `INSERT` and `DELETE` privileges on a table to perform `REPLACE` operations [#23909](https://github.com/pingcap/tidb/issues/23909) + - Users now need both `INSERT` and `DELETE` privileges on a table to perform `REPLACE` operations [#24070](https://github.com/pingcap/tidb/pull/24070) + - Fix the wrong `TableDual` plans caused by incorrectly comparing binaries and bytes[#23846](https://github.com/pingcap/tidb/issues/23846) + - Fix the panic issue caused by using the prefix index and index join in some cases [#24547](https://github.com/pingcap/tidb/issues/24547) [#24716](https://github.com/pingcap/tidb/issues/24716) [#24717](https://github.com/pingcap/tidb/issues/24717) + - Fix the issue that the prepared plan cache of `point get` is incorrectly used by the `point get` statement in the transaction [#24741](https://github.com/pingcap/tidb/issues/24741) + - Fix the issue of writing the wrong prefix index value when the collation is `ascii_bin` or `latin1_bin` [#24569](https://github.com/pingcap/tidb/issues/24569) + - Fix the issue that the ongoing transaction might be interrupted by the GC worker [#24591](https://github.com/pingcap/tidb/issues/24591) + - Fix a bug that the point query might get wrong on the clustered index when `new-collation` is enabled but `new-row-format` is disabled [#24541](https://github.com/pingcap/tidb/issues/24541) + - Refactor the conversion of partition keys for shuffle hash join [#24490](https://github.com/pingcap/tidb/pull/24490) + - Fix the panic issue that occurs when building the plan for queries that contain the `HAVING` clause [#24045](https://github.com/pingcap/tidb/issues/24045) + - Fix the issue that the column pruning improvement causes the `Apply` and `Join` operators' results to go wrong [#23887](https://github.com/pingcap/tidb/issues/23887) + - Fix a bug that the primary lock fallen back from async commit cannot be resolved [#24384](https://github.com/pingcap/tidb/issues/24384) + - Fix a GC issue of statistics that might cause duplicated fm-sketch records [#24357](https://github.com/pingcap/tidb/pull/24357) + - Avoid unnecessary pessimistic rollback when the pessimistic locking receives the `ErrKeyExists` error [#23799](https://github.com/pingcap/tidb/issues/23799) + - Fix the issue that numeric literals cannot be recognized when the sql_mode contains `ANSI_QUOTES` [#24429](https://github.com/pingcap/tidb/issues/24429) + - Forbid statements such as `INSERT INTO table PARTITION () ... ON DUPLICATE KEY UPDATE` to read data from non-listed partitions [#24746](https://github.com/pingcap/tidb/issues/24746) + - Fix the potential `index out of range` error when a SQL statement contains both `GROUP BY` and `UNION` [#24281](https://github.com/pingcap/tidb/issues/24281) + - Fix the issue that the `CONCAT` function incorrectly handles the collation [#24296](https://github.com/pingcap/tidb/issues/24296) + - Fix the issue that the `collation_server` global variable does not take effect in new sessions [#24156](https://github.com/pingcap/tidb/pull/24156) + ++ TiKV + + - Fix the issue that the coprocessor fails to properly handle the signed or unsigned integer types in the `IN` expression [#9821](https://github.com/tikv/tikv/issues/9821) + - Fix the issue of many empty Regions after batch ingesting SST files [#964](https://github.com/pingcap/br/issues/964) + - Fix a bug that TiKV cannot start up after the file dictionary file is damaged [#9886](https://github.com/tikv/tikv/issues/9886) + - Fix a TiCDC OOM issue caused by reading old values [#9996](https://github.com/tikv/tikv/issues/9996) [#9981](https://github.com/tikv/tikv/issues/9981) + - Fix the issue of empty value in the secondary index for the clustered primary key column when collation is `latin1_bin` [#24548](https://github.com/pingcap/tidb/issues/24548) + - Add the `abort-on-panic` configuration, which allows TiKV to generate the core dump file when panic occurs. Users still need to correctly configure the environment to enable core dump [#10216](https://github.com/tikv/tikv/pull/10216) + - Fix the performance regression issue of `point get` queries that occurs when TiKV is not busy [#10046](https://github.com/tikv/tikv/issues/10046) + ++ PD + + - Fix the issue that the PD Leader re-election is slow when there are many stores [#3697](https://github.com/tikv/pd/issues/3697) + + - Fix the panic issue that occurs when removing the evict leader scheduler from a non-existent store [#3660](https://github.com/tikv/pd/issues/3660) + - Fix the issue that the statistics are not updated after offline peers are merged [#3611](https://github.com/tikv/pd/issues/3611) + ++ TiFlash + + - Fix the issue of incorrect results when casting the time type to the integer type + - Fix a bug that the `receiver` cannot find corresponding tasks within 10 seconds + - Fix the issue that there might be invalid iterators in `cancelMPPQuery` + - Fix a bug that the behavior of the `bitwise` operator is different from that of TiDB + - Fix the alert issue caused by overlapping ranges when using the `prefix key` + - Fix the issue of incorrect results when casting the string type to the integer type + - Fix the issue that consecutive and fast writes might make TiFlash out of memory + - Fix the potential issue that the exception of null pointer might be raised during the table GC + - Fix the TiFlash panic issue that occurs when writing data to dropped tables + - Fix the issue that TiFlash might panic during BR restore + - Fix the issue of incorrect results when cloning shared delta index concurrently + - Fix the potential panic that occurs when the Compaction Filter feature is enabled + - Fix the issue that TiFlash cannot resolve the lock fallen back from async commit + - Fix the issue of incorrect results returned when the casted result of the `TIMEZONE` type contains the `TIMESTAMP` type + - Fix the TiFlash panic issue that occurs during Segment Split + ++ Tools + + + TiDB Lightning + + - Fix the issue of TiDB Lightning panic that occurs when generating KV data [#1127](https://github.com/pingcap/br/pull/1127) + - Fix a bug that the batch split Region fails due to the total key size exceeding the raft entry limit during the data import [#969](https://github.com/pingcap/br/issues/969) + - Fix the issue that when importing CSV files, if the last line of the file does not contain line break characters (`\r\n`), an error will be reported [#1133](https://github.com/pingcap/br/issues/1133) + - Fix the issue that if a table to be imported includes an auto-increment column of the double type, the auto_increment value becomes abnormal [#1178](https://github.com/pingcap/br/pull/1178) + + + Backup & Restore (BR) + - Fix the issue of backup interruption caused by the failure of a few TiKV nodes [#980](https://github.com/pingcap/br/issues/980) + + + TiCDC + + - Fix the concurrency issue in Unified Sorter and filter the unhelpful error messages [#1678](https://github.com/pingcap/ticdc/pull/1678) + - Fix a bug that the creation of redundant directories might interrupt the replication with MinIO [#1463](https://github.com/pingcap/ticdc/issues/1463) + - Set the default value of the `explicit_defaults_for_timestamp` session variable to ON to make the MySQL 5.7 downstream keep the same behavior with the upstream TiDB [#1585](https://github.com/pingcap/ticdc/issues/1585) + - Fix the issue that the incorrect handling of `io.EOF` might cause replication interruption [#1633](https://github.com/pingcap/ticdc/issues/1633) + - Correct the TiKV CDC endpoint CPU metric in the TiCDC dashboard [#1645](https://github.com/pingcap/ticdc/pull/1645) + - Increase `defaultBufferChanSize` to avoid replication blocking in some cases [#1259](https://github.com/pingcap/ticdc/issues/1259) + - Fix the issue that the time zone information is lost in the Avro output [#1712](https://github.com/pingcap/ticdc/pull/1712) + - Support cleaning up stale temporary files in Unified Sorter and forbid sharing the `sort-dir` directory [#1742](https://github.com/pingcap/ticdc/pull/1742) + - Fix a deadlock bug in the KV client that occurs when many stale Regions exist [#1599](https://github.com/pingcap/ticdc/issues/1599) + - Fix the wrong help information in the `--cert-allowed-cn` flag [#1697](https://github.com/pingcap/ticdc/pull/1697) + - Revert the update for `explicit_defaults_for_timestamp` which requires the SUPER privilege when replicating data to MySQL [#1750](https://github.com/pingcap/ticdc/pull/1750) + - Support the sink flow control to reduce the risk of memory overflow [#1840](https://github.com/pingcap/ticdc/pull/1840) + - Fix a bug that the replication task might stop when moving a table [#1828](https://github.com/pingcap/ticdc/pull/1828) + - Fix the issue that the TiKV GC safe point is blocked due to the stagnation of TiCDC changefeed checkpoint [#1759](https://github.com/pingcap/ticdc/pull/1759) diff --git a/releases/release-5.1.1.md b/releases/release-5.1.1.md new file mode 100644 index 000000000000..9bde0fd1acd1 --- /dev/null +++ b/releases/release-5.1.1.md @@ -0,0 +1,156 @@ +--- +title: TiDB 5.1.1 Release Notes +--- + +# TiDB 5.1.1 Release Notes + +Release Date: July 30, 2021 + +TiDB version: 5.1.1 + +## Compatibility changes + ++ TiDB + + - For TiDB clusters upgrade from v4.0 to v5.1, the default value of `tidb_multi_statement_mode` is `OFF`. It is recommended to use the multi-statement feature of your client library instead. See [the documentation on `tidb_multi_statement_mode`](/system-variables.md#tidb_multi_statement_mode-new-in-v4011) for details. [#25751](https://github.com/pingcap/tidb/pull/25751) + - Change the default value of the `tidb_stmt_summary_max_stmt_count` variable from `200` to `3000` [#25874](https://github.com/pingcap/tidb/pull/25874) + - Require the `SUPER` privilege to access the `table_storage_stats` table [#26352](https://github.com/pingcap/tidb/pull/26352) + - Require the `SELECT` privilege on `mysql.user` to access the `information_schema.user_privileges` table to show other user's privileges [#26311](https://github.com/pingcap/tidb/pull/26311) + - Require the `CONFIG` privilege to access the `information_schema.cluster_hardware` table [#26297](https://github.com/pingcap/tidb/pull/26297) + - Require the `PROCESS` privilege to access the `information_schema.cluster_info` table [#26297](https://github.com/pingcap/tidb/pull/26297) + - Require the `PROCESS` privilege to access the `information_schema.cluster_load` table [#26297](https://github.com/pingcap/tidb/pull/26297) + - Require the `PROCESS` privilege to access the `information_schema.cluster_systeminfo` table [#26297](https://github.com/pingcap/tidb/pull/26297) + - Require the `PROCESS` privilege to access the `information_schema.cluster_log` table [#26297](https://github.com/pingcap/tidb/pull/26297) + - Require the `CONFIG` privilege to access the `information_schema.cluster_config` table [#26150](https://github.com/pingcap/tidb/pull/26150) + +## Feature enhancements + ++ TiDB Dashboard + + - Support OIDC SSO. By setting the OIDC-compatible SSO services (such as Okta and Auth0), users can log into TiDB Dashboard without entering the SQL password. [#3883](https://github.com/tikv/pd/pull/3883) + ++ TiFlash + + - Support the `HAVING()` function in DAG requests + +## Improvements + ++ TiDB + + - Announce the general availability (GA) of the Stale Read feature + - Avoid allocation for `paramMarker` to speed up data insertion [#26076](https://github.com/pingcap/tidb/pull/26076) + - Support the stable result mode to make the query results more stable [#25995](https://github.com/pingcap/tidb/pull/25995) + - Support pushing down the built-in function `json_unquote()` to TiKV [#26265](https://github.com/pingcap/tidb/pull/26265) + - Support retrying MPP queries [#26480](https://github.com/pingcap/tidb/pull/26480) + - Change the `LOCK` record into the `PUT` record for the index keys using `point get` or `batch point get` for `UPDATE` reads [#26225](https://github.com/pingcap/tidb/pull/26225) + - Forbid creating views from stale queries [#26200](https://github.com/pingcap/tidb/pull/26200) + - Thoroughly push down the `COUNT(DISTINCT)` aggregation function in the MPP mode [#26194](https://github.com/pingcap/tidb/pull/26194) + - Check the availability of TiFlash before launching MPP queries [#26192](https://github.com/pingcap/tidb/pull/26192) + - Do not allow setting the read timestamp to a future time [#25763](https://github.com/pingcap/tidb/pull/25763) + - Print log warnings when aggregation functions cannot be pushed down in `EXPLAIN` statements [#25737](https://github.com/pingcap/tidb/pull/25737) + - Add the `statements_summary_evicted` table to record the evicted count information of a cluster [#25587](https://github.com/pingcap/tidb/pull/25587) + - Improve the MySQL compatibility of the built-in function `str_to_date` for the format specifiers `%b/%M/%r/%T` [#25768](https://github.com/pingcap/tidb/pull/25768) + ++ TiKV + + - Make the prewrite requests as idempotent as possible to reduce the chance of undetermined errors [#10586](https://github.com/tikv/tikv/pull/10586) + - Prevent the risk of stack overflow when handling many expired commands [#10502](https://github.com/tikv/tikv/pull/10502) + - Avoid excessive commit request retrying by not using the Stale Read request's `start_ts` to update `max_ts` [#10451](https://github.com/tikv/tikv/pull/10451) + - Handle read ready and write ready separately to reduce read latency [#10592](https://github.com/tikv/tikv/pull/10592) + - Reduce the impact on data import speed when the I/O rate limiting is enabled [#10390](https://github.com/tikv/tikv/pull/10390) + - Improve the load balance between Raft gRPC connections [#10495](https://github.com/tikv/tikv/pull/10495) + ++ Tools + + + TiCDC + + - Remove `file sorter` [#2327](https://github.com/pingcap/ticdc/pull/2327) + - Improve the error message returned when a PD endpoint misses the certificate [#1973](https://github.com/pingcap/ticdc/issues/1973) + + + TiDB Lightning + + - Add a retry mechanism for restoring schemas [#1294](https://github.com/pingcap/br/pull/1294) + + + Dumpling + + - Always split tables using `_tidb_rowid` when the upstream is a TiDB v3.x cluster, which helps reduce TiDB's memory usage [#295](https://github.com/pingcap/dumpling/issues/295) + - Reduce the frequency of accessing the database metadata to improve Dumpling's performance and stability [#315](https://github.com/pingcap/dumpling/pull/315) + +## Bug fixes + ++ TiDB + + - Fix the data loss issue that might occur when changing the column type with `tidb_enable_amend_pessimistic_txn=on` [#26203](https://github.com/pingcap/tidb/issues/26203) + - Fix the issue that the behavior of the `last_day` function is incompatible in the SQL mode [#26001](https://github.com/pingcap/tidb/pull/26001) + - Fix the panic issue that might occur when `LIMIT` is on top of window functions [#25344](https://github.com/pingcap/tidb/issues/25344) + - Fix the issue that committing pessimistic transactions might cause write conflict [#25964](https://github.com/pingcap/tidb/issues/25964) + - Fix the issue that the result of index join in correlated subqueries is wrong [#25799](https://github.com/pingcap/tidb/issues/25799) + - Fix a bug that the successfully committed optimistic transactions might report commit errors [#10468](https://github.com/tikv/tikv/issues/10468) + - Fix the issue that an incorrect result is returned when using merge join on the `SET` type column [#25669](https://github.com/pingcap/tidb/issues/25669) + - Fix a bug that the index keys in a pessimistic transaction might be repeatedly committed [#26359](https://github.com/pingcap/tidb/issues/26359) + - Fix the risk of integer overflow when the optimizer is locating partitions [#26227](https://github.com/pingcap/tidb/issues/26227) + - Fix the issue that invalid values might be written when casting `DATE` to timestamp [#26292](https://github.com/pingcap/tidb/issues/26292) + - Fix the issue that the Coprocessor Cache metrics are not displayed on Grafana [#26338](https://github.com/pingcap/tidb/issues/26338) + - Fix the issue of annoying logs caused by telemetry [#25760](https://github.com/pingcap/tidb/issues/25760) [#25785](https://github.com/pingcap/tidb/issues/25785) + - Fix a bug on the query range of prefix index [#26029](https://github.com/pingcap/tidb/issues/26029) + - Fix the issue that concurrently truncating the same partition hangs DDL executions [#26229](https://github.com/pingcap/tidb/issues/26229) + - Fix the issue of duplicate `ENUM` items [#25955](https://github.com/pingcap/tidb/issues/25955) + - Fix a bug that the CTE iterator is not correctly closed [#26112](https://github.com/pingcap/tidb/issues/26112) + - Fix the issue that the `LOAD DATA` statement might abnormally import non-utf8 data [#25979](https://github.com/pingcap/tidb/issues/25979) + - Fix the panic issue that might occur when using the window function on the unsigned integer columns [#25956](https://github.com/pingcap/tidb/issues/25956) + - Fix the issue that TiDB might panic when resolving async commit locks [#25778](https://github.com/pingcap/tidb/issues/25778) + - Fix the issue that Stale Read is not fully compatible with the `PREPARE` statements [#25800](https://github.com/pingcap/tidb/pull/25800) + - Fix the issue that the ODBC-styled constant (for example, `{d '2020-01-01'}`) cannot be used as the expression [#25531](https://github.com/pingcap/tidb/issues/25531) + - Fix an error that occurs when running TiDB alone [#25555](https://github.com/pingcap/tidb/pull/25555) + ++ TiKV + + - Fix the issue that the duration calculation might panic on certain platforms [#10569](https://github.com/tikv/tikv/pull/10569) + - Fix the issue that Load Base Split mistakenly uses the unencoded keys of `batch_get_command` [#10542](https://github.com/tikv/tikv/issues/10542) + - Fix the issue that changing the `resolved-ts.advance-ts-interval` configuration online cannot take effect immediately [#10426](https://github.com/tikv/tikv/issues/10426) + - Fix the issue of follower metadata corruption in rare cases with more than 4 replicas [#10225](https://github.com/tikv/tikv/issues/10225) + - Fix the panic issue that occurs when building a snapshot twice if encryption is enabled [#9786](https://github.com/tikv/tikv/issues/9786) [#10407](https://github.com/tikv/tikv/issues/10407) + - Fix the wrong `tikv_raftstore_hibernated_peer_state` metric [#10330](https://github.com/tikv/tikv/issues/10330) + - Fix the wrong arguments type of the `json_unquote()` function in the coprocessor [#10176](https://github.com/tikv/tikv/issues/10176) + - Fix a bug that the index keys in a pessimistic transaction might be repeatedly committed [#10468](https://github.com/tikv/tikv/issues/10468#issuecomment-869491061) + - Fix the issue that the `ReadIndex` request returns stale result right after the leader is transferred [#9351](https://github.com/tikv/tikv/issues/9351) + ++ PD + + - Fix the issue the expected scheduling cannot be generated when the conflict occurs due to multiple schedulers running at the same time [#3807](https://github.com/tikv/pd/issues/3807) [#3778](https://github.com/tikv/pd/issues/3778) + - Fix the issue that the scheduler might appear again even if the scheduler is already deleted [#2572](https://github.com/tikv/pd/issues/2572) + ++ TiFlash + + - Fix the potential panic issue that occurs when running table scan tasks + - Fix a bug that TiFlash raises the error about `duplicated region` when handling DAQ requests + - Fix the panic issue that occurs when the read load is heavy + - Fix the potential panic issue that occurs when executing the `DateFormat` function + - Fix the potential memory leak issue that occurs when executing MPP tasks + - Fix the issue of unexpected results when executing the aggregation functions `COUNT` or `COUNT DISTINCT` + - Fix a potential bug that TiFlash cannot restore data when deployed on multiple disks + - Fix the issue that TiDB Dashboard cannot display the disk information of TiFlash correctly + - Fix the potential panic issue that occurs when deconstructing `SharedQueryBlockInputStream` + - Fix the potential panic issue that occurs when deconstructing `MPPTask` + - Fix the potential issue of data inconsistency after synchronizing data via snapshot + ++ Tools + + + TiCDC + + - Fix the support for the new collation feature [#2301](https://github.com/pingcap/ticdc/issues/2301) + - Fix the issue that an unsynchronized access to a shared map at runtime might cause panic [#2300](https://github.com/pingcap/ticdc/pull/2300) + - Fix the potential DDL loss issue that occurs when the owner crashes while executing the DDL statement [#2290](https://github.com/pingcap/ticdc/pull/2290) + - Fix the issue of trying to resolve locks in TiDB prematurely [#2188](https://github.com/pingcap/ticdc/issues/2188) + - Fix a bug that might cause data loss if a TiCDC node is killed immediately after a table migration [#2033](https://github.com/pingcap/ticdc/pull/2033) + - Fix the handling logic of `changefeed update` on `--sort-dir` and `--start-ts` [#1921](https://github.com/pingcap/ticdc/pull/1921) + + + Backup & Restore (BR) + + - Fix the issue that the size of the data to restore is incorrectly calculated [#1270](https://github.com/pingcap/br/issues/1270) + - Fix the issue of missed DDL events that occurs when restoring from cdclog [#870](https://github.com/pingcap/br/issues/870) + + + TiDB Lightning + + - Fix the issue that TiDB fails to parse the `DECIMAL` type data in Parquet files [#1275](https://github.com/pingcap/br/pull/1275) + - Fix the issue of integer overflow when calculating key intervals [#1291](https://github.com/pingcap/br/issues/1291) [#1290](https://github.com/pingcap/br/issues/1290) diff --git a/releases/release-notes.md b/releases/release-notes.md index b3dfa69a7194..ad60e6a83a44 100644 --- a/releases/release-notes.md +++ b/releases/release-notes.md @@ -5,8 +5,26 @@ aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] # TiDB Release Notes +## 5.1 + +- [5.1.1](/releases/release-5.1.1.md) +- [5.1.0](/releases/release-5.1.0.md) + +## 5.0 + +- [5.0.3](/releases/release-5.0.3.md) +- [5.0.2](/releases/release-5.0.2.md) +- [5.0.1](/releases/release-5.0.1.md) +- [5.0 GA](/releases/release-5.0.0.md) +- [5.0.0-rc](/releases/release-5.0.0-rc.md) + ## 4.0 +- [4.0.14](/releases/release-4.0.14.md) +- [4.0.13](/releases/release-4.0.13.md) +- [4.0.12](/releases/release-4.0.12.md) +- [4.0.11](/releases/release-4.0.11.md) +- [4.0.10](/releases/release-4.0.10.md) - [4.0.9](/releases/release-4.0.9.md) - [4.0.8](/releases/release-4.0.8.md) - [4.0.7](/releases/release-4.0.7.md) diff --git a/roadmap.md b/roadmap.md deleted file mode 100644 index 21e75f47378a..000000000000 --- a/roadmap.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: TiDB Roadmap -aliases: ['/docs/ROADMAP/','/docs/roadmap/','/docs/stable/roadmap/','/docs/v4.0/roadmap/','/tidb/stable/roadmap','/docs/v3.1/roadmap/','/tidb/v3.1/roadmap','/docs/v3.0/roadmap/','/tidb/v3.0/roadmap','/docs/v2.1/roadmap/','/tidb/v2.1/roadmap'] ---- - - - -# TiDB Roadmap - -## Improve system stability - -- [ ] Create bindings for `UPDATE`/`DELETE`/`INSERT` queries [#15827](https://github.com/pingcap/tidb/issues/15827) -- [ ] Optimize transaction commits to avoid commit failures caused by DDL execution [#18098](https://github.com/pingcap/tidb/issues/18098) -- [ ] Reduce latency jitter [#18005](https://github.com/pingcap/tidb/issues/18005) - -## Improve system performance and reduce latency - -- [ ] Optimize the performance and efficiency of bulk deletion [#18028](https://github.com/pingcap/tidb/issues/18028) -- [ ] Improve memory management [#17479](https://github.com/pingcap/tidb/issues/17479) -- [ ] Improve the accuracy and robustness of index selection [#18065](https://github.com/pingcap/tidb/issues/18065) -- [ ] Improve the performance of partition pruning and data access on the partitioned table [#18016](https://github.com/pingcap/tidb/issues/18016) -- [ ] Async Commit. This feature means that the statement being written can return to the client as soon as possible after the prewrite stage finishes, which reduces system latency. [#8316](https://github.com/tikv/tikv/issues/8316) -- [ ] Clustered index [#4841](https://github.com/pingcap/tidb/issues/4841) -- [ ] Support cross-region deployment and geo-partition [#18273](https://github.com/pingcap/tidb/issues/18273) - -## Improve system security - -### Authentication - -- [ ] Transport Layer Security (TLS) for TiFlash [#18080](https://github.com/pingcap/tidb/issues/18080) -- [ ] TLS for internal communication in the TiDB cluster [#529](https://github.com/pingcap/tiup/issues/529) -- [ ] SSH LDAP extension for TiUP [#528](https://github.com/pingcap/tiup/issues/528) - -### Transparent Data Encryption (TDE) - -- [ ] Transparent Data Encryption (TDE) for TiFlash [#18082](https://github.com/pingcap/tidb/issues/18082) -- [ ] TDE for PD [#18262](https://github.com/pingcap/tidb/issues/18262) - -### Mask - -- [ ] Desensitize the TiDB general log [#18034](https://github.com/pingcap/tidb/issues/18034) - -## Cost-effectiveness - -- [ ] Optimize the performance and stability of TiDB running on AWS i3.xlarge/i3.2xlarge [#18025](https://github.com/pingcap/tidb/issues/18025) -- [ ] Optimize the performance and stability of TiDB running on non-NVMe SSD or on cloud disk (such as AWS EBS gp2) [#18024](https://github.com/pingcap/tidb/issues/18024) - -## New features - -- [ ] Point-in-time recovery [#325](https://github.com/pingcap/br/issues/325) -- [ ] Change column types [#17526](https://github.com/pingcap/tidb/issues/17526) -- [ ] Easier to discover performance issues and diagnose the causes [#18867](https://github.com/pingcap/tidb/issues/18867) -- [ ] Support the collations of `utf8mb4_unicode_ci` and `utf8_unicode_ci` [#17596](https://github.com/pingcap/tidb/issues/17596) -- [ ] Make TiCDC a complete alternative to TiDB Binlog [#690](https://github.com/pingcap/ticdc/issues/690) - - [ ] Support distinguishing `UPDATE` and `INSERT` in a row changed event. - - [ ] Support providing old values in the row changed event, including old values before the `DELETE` or `UPDATE` execution. -- [ ] Support snapshot-level consistent replication for disaster recovery [#691](https://github.com/pingcap/ticdc/issues/691) - - [ ] Support MySQL sink in replicating the upstream to a snapshot-level consistent state when the upstream meets a disaster -- [ ] Manage TiCDC using API [#736](https://github.com/pingcap/ticdc/issues/736) -- [ ] Support the SQL-based `import` command [#18089](https://github.com/pingcap/tidb/issues/18089) -- [ ] Support Avro sink and make TiCDC compatible with Kafka connect [#660](https://github.com/pingcap/ticdc/issues/660) -- [ ] Support Spark 3.0 [#1173](https://github.com/pingcap/tispark/issues/1173) -- [ ] Support `EXCEPT`/`INTERSECT` operators [#18031](https://github.com/pingcap/tidb/issues/18031) -- [ ] Support migrating the RDS (such as MySQL/Aurora) on cloud to TiDB [#18629](https://github.com/pingcap/tidb/issues/18629) - -## TiDB Operator - -See [TiDB Operator Roadmap](https://docs.pingcap.com/tidb-in-kubernetes/dev/roadmap). diff --git a/scale-tidb-using-ansible.md b/scale-tidb-using-ansible.md deleted file mode 100644 index 0c4bcc8231b9..000000000000 --- a/scale-tidb-using-ansible.md +++ /dev/null @@ -1,556 +0,0 @@ ---- -title: Scale the TiDB Cluster Using TiDB Ansible -summary: Use TiDB Ansible to increase/decrease the capacity of a TiDB/TiKV/PD node. -aliases: ['/docs/dev/scale-tidb-using-ansible/','/docs/dev/how-to/scale/with-ansible/'] ---- - -# Scale the TiDB Cluster Using TiDB Ansible - -The capacity of a TiDB cluster can be increased or decreased without affecting the online services. - -> **Warning:** -> -> In decreasing the capacity, if your cluster has a mixed deployment of other services, do not perform the following procedures. The following examples assume that the removed nodes have no mixed deployment of other services. - -Assume that the topology is as follows: - -| Name | Host IP | Services | -| ---- | ------- | -------- | -| node1 | 172.16.10.1 | PD1 | -| node2 | 172.16.10.2 | PD2 | -| node3 | 172.16.10.3 | PD3, Monitor | -| node4 | 172.16.10.4 | TiDB1 | -| node5 | 172.16.10.5 | TiDB2 | -| node6 | 172.16.10.6 | TiKV1 | -| node7 | 172.16.10.7 | TiKV2 | -| node8 | 172.16.10.8 | TiKV3 | -| node9 | 172.16.10.9 | TiKV4 | - -## Increase the capacity of a TiDB/TiKV node - -For example, if you want to add two TiDB nodes (node101, node102) with the IP addresses `172.16.10.101` and `172.16.10.102`, take the following steps: - -1. Edit the `inventory.ini` file and the `hosts.ini` file, and append the node information. - - - Edit the `inventory.ini` file: - - ```ini - [tidb_servers] - 172.16.10.4 - 172.16.10.5 - 172.16.10.101 - 172.16.10.102 - - [pd_servers] - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - - [tikv_servers] - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - - [monitored_servers] - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - 172.16.10.4 - 172.16.10.5 - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - 172.16.10.101 - 172.16.10.102 - - [monitoring_servers] - 172.16.10.3 - - [grafana_servers] - 172.16.10.3 - ``` - - Now the topology is as follows: - - | Name | Host IP | Services | - | ---- | ------- | -------- | - | node1 | 172.16.10.1 | PD1 | - | node2 | 172.16.10.2 | PD2 | - | node3 | 172.16.10.3 | PD3, Monitor | - | node4 | 172.16.10.4 | TiDB1 | - | node5 | 172.16.10.5 | TiDB2 | - | **node101** | **172.16.10.101**|**TiDB3** | - | **node102** | **172.16.10.102**|**TiDB4** | - | node6 | 172.16.10.6 | TiKV1 | - | node7 | 172.16.10.7 | TiKV2 | - | node8 | 172.16.10.8 | TiKV3 | - | node9 | 172.16.10.9 | TiKV4 | - - - Edit the `hosts.ini` file: - - ```ini - [servers] - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - 172.16.10.4 - 172.16.10.5 - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - 172.16.10.101 - 172.16.10.102 - [all:vars] - username = tidb - ntp_server = pool.ntp.org - ``` - -2. Initialize the newly added node. - - 1. Configure the SSH mutual trust and sudo rules of the target machine on the control machine: - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook -i hosts.ini create_users.yml -l 172.16.10.101,172.16.10.102 -u root -k - ``` - - 2. Install the NTP service on the target machine: - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook -i hosts.ini deploy_ntp.yml -u tidb -b - ``` - - 3. Initialize the node on the target machine: - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook bootstrap.yml -l 172.16.10.101,172.16.10.102 - ``` - - > **Note:** - > - > If an alias is configured in the `inventory.ini` file, for example, `node101 ansible_host=172.16.10.101`, use `-l` to specify the alias when executing `ansible-playbook`. For example, `ansible-playbook bootstrap.yml -l node101,node102`. This also applies to the following steps. - -3. Deploy the newly added node: - - ``` - ansible-playbook deploy.yml -l 172.16.10.101,172.16.10.102 - ``` - -4. Start the newly added node: - - ``` - ansible-playbook start.yml -l 172.16.10.101,172.16.10.102 - ``` - -5. Update the Prometheus configuration and restart the cluster: - - ``` - ansible-playbook rolling_update_monitor.yml --tags=prometheus - ``` - -6. Monitor the status of the entire cluster and the newly added node by opening a browser to access the monitoring platform: `http://172.16.10.3:3000`. - -You can use the same procedure to add a TiKV node. But to add a PD node, some configuration files need to be manually updated. - -## Increase the capacity of a PD node - -For example, if you want to add a PD node (node103) with the IP address `172.16.10.103`, take the following steps: - -1. Edit the `inventory.ini` file and append the node information to the end of the `[pd_servers]` group: - - ```ini - [tidb_servers] - 172.16.10.4 - 172.16.10.5 - - [pd_servers] - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - 172.16.10.103 - - [tikv_servers] - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - - [monitored_servers] - 172.16.10.4 - 172.16.10.5 - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - 172.16.10.103 - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - - [monitoring_servers] - 172.16.10.3 - - [grafana_servers] - 172.16.10.3 - ``` - - Now the topology is as follows: - - | Name | Host IP | Services | - | ---- | ------- | -------- | - | node1 | 172.16.10.1 | PD1 | - | node2 | 172.16.10.2 | PD2 | - | node3 | 172.16.10.3 | PD3, Monitor | - | **node103** | **172.16.10.103** | **PD4** | - | node4 | 172.16.10.4 | TiDB1 | - | node5 | 172.16.10.5 | TiDB2 | - | node6 | 172.16.10.6 | TiKV1 | - | node7 | 172.16.10.7 | TiKV2 | - | node8 | 172.16.10.8 | TiKV3 | - | node9 | 172.16.10.9 | TiKV4 | - -2. Initialize the newly added node: - - ``` - ansible-playbook bootstrap.yml -l 172.16.10.103 - ``` - -3. Deploy the newly added node: - - ``` - ansible-playbook deploy.yml -l 172.16.10.103 - ``` - -4. Login the newly added PD node and edit the starting script: - - ``` - {deploy_dir}/scripts/run_pd.sh - ``` - - 1. Remove the `--initial-cluster="xxxx" \` configuration. - - > **Note:** - > - > You cannot add the `#` character at the beginning of the line. Otherwise, the following configuration cannot take effect. - - 2. Add `--join="http://172.16.10.1:2379" \`. The IP address (`172.16.10.1`) can be any of the existing PD IP address in the cluster. - - 3. Start the PD service in the newly added PD node: - - ``` - {deploy_dir}/scripts/start_pd.sh - ``` - - > **Note:** - > - > Before start, you need to ensure that the `health` status of the newly added PD node is "true", using [PD Control](/pd-control.md). Otherwise, the PD service might fail to start and an error message `["join meet error"] [error="etcdserver: unhealthy cluster"]` is returned in the log. - - 4. Use `pd-ctl` to check whether the new node is added successfully: - - ``` - ./pd-ctl -u "http://172.16.10.1:2379" - ``` - - > **Note:** - > - > `pd-ctl` is a command used to check the number of PD nodes. - -5. Start the monitoring service: - - ``` - ansible-playbook start.yml -l 172.16.10.103 - ``` - - > **Note:** - > - > If you use an alias (inventory_name), use the `-l` option to specify the alias. - -6. Update the cluster configuration: - - ``` - ansible-playbook deploy.yml - ``` - -7. Restart Prometheus, and enable the monitoring of PD nodes used for increasing the capacity: - - ``` - ansible-playbook stop.yml --tags=prometheus - ansible-playbook start.yml --tags=prometheus - ``` - -8. Monitor the status of the entire cluster and the newly added node by opening a browser to access the monitoring platform: `http://172.16.10.3:3000`. - -> **Note:** -> -> The PD Client in TiKV caches the list of PD nodes. Currently, the list is updated only if the PD leader is switched or the TiKV server is restarted to load the latest configuration. To avoid TiKV caching an outdated list, there should be at least two existing PD members in the PD cluster after increasing or decreasing the capacity of a PD node. If this condition is not met, transfer the PD leader manually to update the list of PD nodes. - -## Decrease the capacity of a TiDB node - -For example, if you want to remove a TiDB node (node5) with the IP address `172.16.10.5`, take the following steps: - -1. Stop all services on node5: - - ``` - ansible-playbook stop.yml -l 172.16.10.5 - ``` - -2. Edit the `inventory.ini` file and remove the node information: - - ```ini - [tidb_servers] - 172.16.10.4 - #172.16.10.5 # the removed node - - [pd_servers] - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - - [tikv_servers] - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - - [monitored_servers] - 172.16.10.4 - #172.16.10.5 # the removed node - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - - [monitoring_servers] - 172.16.10.3 - - [grafana_servers] - 172.16.10.3 - ``` - - Now the topology is as follows: - - | Name | Host IP | Services | - | ---- | ------- | -------- | - | node1 | 172.16.10.1 | PD1 | - | node2 | 172.16.10.2 | PD2 | - | node3 | 172.16.10.3 | PD3, Monitor | - | node4 | 172.16.10.4 | TiDB1 | - | **node5** | **172.16.10.5** | **TiDB2 removed** | - | node6 | 172.16.10.6 | TiKV1 | - | node7 | 172.16.10.7 | TiKV2 | - | node8 | 172.16.10.8 | TiKV3 | - | node9 | 172.16.10.9 | TiKV4 | - -3. Update the Prometheus configuration and restart the cluster: - - ``` - ansible-playbook rolling_update_monitor.yml --tags=prometheus - ``` - -4. Monitor the status of the entire cluster by opening a browser to access the monitoring platform: `http://172.16.10.3:3000`. - -## Decrease the capacity of a TiKV node - -For example, if you want to remove a TiKV node (node9) with the IP address `172.16.10.9`, take the following steps: - -1. Remove the node from the cluster using `pd-ctl`: - - 1. View the store ID of node9: - - ``` - ./pd-ctl -u "http://172.16.10.1:2379" -d store - ``` - - 2. Remove node9 from the cluster, assuming that the store ID is 10: - - ``` - ./pd-ctl -u "http://172.16.10.1:2379" -d store delete 10 - ``` - -2. Use `pd-ctl` to check whether the node is successfully removed: - - ``` - ./pd-ctl -u "http://172.16.10.1:2379" -d store 10 - ``` - - > **Note:** - > - > It takes some time to remove the node. If the status of the node you remove becomes Tombstone, then this node is successfully removed. - -3. After the node is successfully removed, stop the services on node9: - - ``` - ansible-playbook stop.yml -l 172.16.10.9 - ``` - -4. Edit the `inventory.ini` file and remove the node information: - - ```ini - [tidb_servers] - 172.16.10.4 - 172.16.10.5 - - [pd_servers] - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - - [tikv_servers] - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - #172.16.10.9 # the removed node - - [monitored_servers] - 172.16.10.4 - 172.16.10.5 - 172.16.10.1 - 172.16.10.2 - 172.16.10.3 - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - #172.16.10.9 # the removed node - - [monitoring_servers] - 172.16.10.3 - - [grafana_servers] - 172.16.10.3 - ``` - - Now the topology is as follows: - - | Name | Host IP | Services | - | ---- | ------- | -------- | - | node1 | 172.16.10.1 | PD1 | - | node2 | 172.16.10.2 | PD2 | - | node3 | 172.16.10.3 | PD3, Monitor | - | node4 | 172.16.10.4 | TiDB1 | - | node5 | 172.16.10.5 | TiDB2 | - | node6 | 172.16.10.6 | TiKV1 | - | node7 | 172.16.10.7 | TiKV2 | - | node8 | 172.16.10.8 | TiKV3 | - | **node9** | **172.16.10.9** | **TiKV4 removed** | - -5. Update the Prometheus configuration and restart the cluster: - - ``` - ansible-playbook rolling_update_monitor.yml --tags=prometheus - ``` - -6. Monitor the status of the entire cluster by opening a browser to access the monitoring platform: `http://172.16.10.3:3000`. - -## Decrease the capacity of a PD node - -For example, if you want to remove a PD node (node2) with the IP address `172.16.10.2`, take the following steps: - -1. Remove the node from the cluster using `pd-ctl`: - - 1. View the name of node2: - - ``` - ./pd-ctl -u "http://172.16.10.1:2379" -d member - ``` - - 2. Remove node2 from the cluster, assuming that the name is pd2: - - ``` - ./pd-ctl -u "http://172.16.10.1:2379" -d member delete name pd2 - ``` - -2. Use Grafana or `pd-ctl` to check whether the node is successfully removed: - - ``` - ./pd-ctl -u "http://172.16.10.1:2379" -d member - ``` - -3. After the node is successfully removed, stop the services on node2: - - ``` - ansible-playbook stop.yml -l 172.16.10.2 - ``` - - > **Note:** - > - > In this example, you can only stop the PD service on node2. If there are any other services deployed with the IP address `172.16.10.2`, use the `-t` option to specify the service (such as `-t tidb`). - -4. Edit the `inventory.ini` file and remove the node information: - - ```ini - [tidb_servers] - 172.16.10.4 - 172.16.10.5 - - [pd_servers] - 172.16.10.1 - #172.16.10.2 # the removed node - 172.16.10.3 - - [tikv_servers] - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - - [monitored_servers] - 172.16.10.4 - 172.16.10.5 - 172.16.10.1 - #172.16.10.2 # the removed node - 172.16.10.3 - 172.16.10.6 - 172.16.10.7 - 172.16.10.8 - 172.16.10.9 - - [monitoring_servers] - 172.16.10.3 - - [grafana_servers] - 172.16.10.3 - ``` - - Now the topology is as follows: - - | Name | Host IP | Services | - | ---- | ------- | -------- | - | node1 | 172.16.10.1 | PD1 | - | **node2** | **172.16.10.2** | **PD2 removed** | - | node3 | 172.16.10.3 | PD3, Monitor | - | node4 | 172.16.10.4 | TiDB1 | - | node5 | 172.16.10.5 | TiDB2 | - | node6 | 172.16.10.6 | TiKV1 | - | node7 | 172.16.10.7 | TiKV2 | - | node8 | 172.16.10.8 | TiKV3 | - | node9 | 172.16.10.9 | TiKV4 | - -5. Update the cluster configuration: - - ``` - ansible-playbook deploy.yml - ``` - -6. Restart Prometheus, and disable the monitoring of PD nodes used for increasing the capacity: - - ``` - ansible-playbook stop.yml --tags=prometheus - ansible-playbook start.yml --tags=prometheus - ``` - -7. To monitor the status of the entire cluster, open a browser to access the monitoring platform: `http://172.16.10.3:3000`. - -> **Note:** -> -> The PD Client in TiKV caches the list of PD nodes. Currently, the list is updated only if the PD leader is switched or the TiKV server is restarted to load the latest configuration. To avoid TiKV caching an outdated list, there should be at least two existing PD members in the PD cluster after increasing or decreasing the capacity of a PD node. If this condition is not met, transfer the PD leader manually to update the list of PD nodes. diff --git a/scale-tidb-using-tiup.md b/scale-tidb-using-tiup.md index ab498d7a71f4..761de3d818d2 100644 --- a/scale-tidb-using-tiup.md +++ b/scale-tidb-using-tiup.md @@ -8,7 +8,7 @@ aliases: ['/docs/dev/scale-tidb-using-tiup/','/docs/dev/how-to/scale/with-tiup/' The capacity of a TiDB cluster can be increased or decreased without interrupting the online services. -This document describes how to scale the TiDB, TiKV, PD, TiCDC, or TiFlash cluster using TiUP. If you have not installed TiUP, refer to the steps in [Install TiUP on the control machine](/upgrade-tidb-using-tiup.md#install-tiup-on-the-control-machine) and import the cluster into TiUP before you use TiUP to scale the TiDB cluster. +This document describes how to scale the TiDB, TiKV, PD, TiCDC, or TiFlash cluster using TiUP. If you have not installed TiUP, refer to the steps in [Install TiUP on the control machine](/production-deployment-using-tiup.md#step-2-install-tiup-on-the-control-machine). To view the current cluster name list, run `tiup cluster list`. @@ -143,8 +143,8 @@ If you want to add a TiFlash node to the `10.0.1.4` host, take the following ste > > When adding a TiFlash node to an existing TiDB cluster, you need to note the following things: > -> 1. Confirm that the current TiDB version supports using TiFlash, otherwise upgrade your TiDB cluster to v4.0.0-rc or higher. -> 2. Download [pd-ctl](https://download.pingcap.org/tidb-v4.0.0-rc.2-linux-amd64.tar.gz) and execute the `config set enable-placement-rules true` command to enable the PD's Placement Rules. +> 1. Confirm that the current TiDB version supports using TiFlash. Otherwise, upgrade your TiDB cluster to v5.0 or later versions. +> 2. Execute the `tiup ctl: pd -u http://: config set enable-placement-rules true` command to enable the Placement Rules feature. Or execute the corresponding command in [pd-ctl](/pd-control.md). 1. Add the node information to the `scale-out.yaml` file: @@ -256,9 +256,9 @@ If you want to remove a TiKV node from the `10.0.1.5` host, take the following s ``` ``` - Starting /root/.tiup/components/cluster/v0.4.6/cluster display + Starting /root/.tiup/components/cluster/v1.5.0/cluster display TiDB Cluster: - TiDB Version: v4.0.0-rc + TiDB Version: v5.1.0 ID Role Host Ports Status Data Dir Deploy Dir -- ---- ---- ----- ------ -------- ---------- 10.0.1.3:8300 cdc 10.0.1.3 8300 Up - deploy/cdc-8300 @@ -366,9 +366,13 @@ In special cases (such as when a node needs to be forcibly taken down), or if th {{< copyable "shell-regular" >}} ```shell - tiup ctl pd -u store + tiup ctl: pd -u http://: store ``` + > **Note:** + > + > If multiple PD instances exist in the cluster, you only need to specify the IP address:port of an active PD instance in the above command. + 2. Remove the TiFlash node in pd-ctl: * Enter `store delete ` in pd-ctl (`` is the store ID of the TiFlash node found in the previous step. @@ -378,21 +382,15 @@ In special cases (such as when a node needs to be forcibly taken down), or if th {{< copyable "shell-regular" >}} ```shell - tiup ctl pd -u store delete + tiup ctl: pd -u http://: store delete ``` + > **Note:** + > + > If multiple PD instances exist in the cluster, you only need to specify the IP address:port of an active PD instance in the above command. + 3. Wait for the store of the TiFlash node to disappear or for the `state_name` to become `Tombstone` before you stop the TiFlash process. - If, after waiting for a long time, the node still fails to disappear or the `state_name` fails to become `Tombstone`, consider using the following command to force the node out of the cluster. - - **Note that the command will directly discard the replicas on the TiFlash node, which might cause the query to fail.** - - {{< copyable "shell-regular" >}} - - ```shell - curl -X POST 'http:///pd/api/v1/store//state?state=Tombstone' - ``` - 4. Manually delete TiFlash data files (whose location can be found in the `data_dir` directory under the TiFlash configuration of the cluster topology file). 5. Manually update TiUP's cluster configuration file (delete the information of the TiFlash node that goes down in edit mode). diff --git a/schedule-replicas-by-topology-labels.md b/schedule-replicas-by-topology-labels.md index 546bab38743a..fb8895af77c0 100644 --- a/schedule-replicas-by-topology-labels.md +++ b/schedule-replicas-by-topology-labels.md @@ -91,7 +91,7 @@ The `location-level` configuration is an array of strings, which needs to corres ### Configure a cluster using TiUP (recommended) -When using TiUP to deploy a cluster, you can configure the TiKV location in the [initialization configuration file](/production-deployment-using-tiup.md#step-3-edit-the-initialization-configuration-file). TiUP will generate the corresponding TiKV and PD configuration files during deployment. +When using TiUP to deploy a cluster, you can configure the TiKV location in the [initialization configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file). TiUP will generate the corresponding TiKV and PD configuration files during deployment. In the following example, a two-layer topology of `zone/host` is defined. The TiKV nodes of the cluster are distributed among three zones, each zone with two hosts. In z1, two TiKV instances are deployed per host. In z2 and z3, one TiKV instance is deployed per host. In the following example, `tikv-n` represents the IP address of the `n`th TiKV node. @@ -148,33 +148,6 @@ tikv_servers: For details, see [Geo-distributed Deployment topology](/geo-distributed-deployment-topology.md). -
- Configure a cluster using TiDB Ansible - -When using TiDB Ansible to deploy a cluster, you can directly configure the TiKV location in the `inventory.ini` file. `tidb-ansible` will generate the corresponding TiKV and PD configuration files during deployment. - -In the following example, a two-layer topology of `zone/host` is defined. The TiKV nodes of the cluster are distributed among three zones, each zone with two hosts. In z1, two TiKV instances are deployed per host. In z2 and z3, one TiKV instance is deployed per host. - -``` -[tikv_servers] -# z1 -tikv-1 labels="zone=z1,host=h1" -tikv-2 labels="zone=z1,host=h1" -tikv-3 labels="zone=z1,host=h2" -tikv-4 labels="zone=z1,host=h2" -# z2 -tikv-5 labels="zone=z2,host=h1" -tikv-6 labels="zone=z2,host=h2" -# z3 -tikv-7 labels="zone=z3,host=h1" -tikv-8 labels="zone=z3,host=h2" - -[pd_servers:vars] -location_labels = ["zone", "host"] -``` - -
- ## PD schedules based on topology label PD schedules replicas according to the label layer to make sure that different replicas of the same data are scattered as much as possible. @@ -185,7 +158,7 @@ Assume that the number of cluster replicas is 3 (`max-replicas=3`). Because ther Then, assume that the number of cluster replicas is 5 (`max-replicas=5`). Because there are only 3 zones in total, PD cannot guarantee the isolation of each replica at the zone level. In this situation, the PD scheduler will ensure replica isolation at the host level. In other words, multiple replicas of a Region might be distributed in the same zone but not on the same host. -In the case of the 5-replica configuration, if z3 fails or is isolated as a whole, and cannot be recovered after a period of time (controlled by `max-store-down-time`), PD will make up the 5 replicas through scheduling. At this time, only 3 hosts are available. This means that host-level isolation cannot be guaranteed and that multiple replicas might be scheduled to the same host. But if the `isolation-level` value is set to `zone` instead of being left empty, this specifies the minimum physical isolation requirements for Region replicas. That is to say, PD will ensure that replicas of the same Region are scattered among different zones. PD will not perform corresponding scheduling even if following this isolation restriction does not meet the requirement of `max-replicas` for multiple replicas. +In the case of the 5-replica configuration, if z3 fails or is isolated as a whole, and cannot be recovered after a period of time (controlled by `max-store-down-time`), PD will make up the 5 replicas through scheduling. At this time, only 3 hosts are available. This means that host-level isolation cannot be guaranteed and that multiple replicas might be scheduled to the same host. But if the `isolation-level` value is set to `zone` instead of being left empty, this specifies the minimum physical isolation requirements for Region replicas. That is to say, PD will ensure that replicas of the same Region are scattered among different zones. PD will not perform corresponding scheduling even if following this isolation restriction does not meet the requirement of `max-replicas` for multiple replicas. For example, a TiKV cluster is distributed across three data zones z1, z2, and z3. Each Region has three replicas as required, and PD distributes the three replicas of the same Region to these three data zones respectively. If a power outage occurs in z1 and cannot be recovered after a period of time, PD determines that the Region replicas on z1 are no longer available. However, because `isolation-level` is set to `zone`, PD needs to strictly guarantee that different replicas of the same Region will not be scheduled on the same data zone. Because both z2 and z3 already have replicas, PD will not perform any scheduling under the minimum isolation level restriction of `isolation-level`, even if there are only two replicas at this moment. diff --git a/scripts/check-conflicts.py b/scripts/check-conflicts.py deleted file mode 100644 index 4e72765306cd..000000000000 --- a/scripts/check-conflicts.py +++ /dev/null @@ -1,43 +0,0 @@ -import re -import sys -import os - -lineNum = 0 -flag = 0 -pos = [] -single = [] -mark = 0 - -for filename in sys.argv[1:]: - single = [] - lineNum = 0 - if os.path.isfile(filename): - with open(filename,'r') as file: - for line in file: - lineNum += 1 - if re.match(r'<{7}.*\n', line): - flag = 1 - single.append(lineNum) - elif re.match(r'={7}\n', line) : - flag = 2 - elif re.match(r'>{7}', line) and flag == 2: - single.append(lineNum) - pos.append(single) - single = [] - flag = 0 - else: - continue - - - if len(pos): - mark = 1 - print("\n" + filename + ": this file has conflicts in the following lines:\n") - for conflict in pos: - if len(conflict) == 2: - print("CONFLICTS: line " + str(conflict[0]) + " to line " + str(conflict[1]) + "\n") - - pos = [] - -if mark: - print("The above conflicts will cause website build failure. Please fix them.") - exit(1) diff --git a/scripts/check-tags.py b/scripts/check-tags.py deleted file mode 100755 index 86711c109c54..000000000000 --- a/scripts/check-tags.py +++ /dev/null @@ -1,154 +0,0 @@ -import re -import sys -import os - -# reference: https://stackoverflow.com/questions/35761133/python-how-to-check-for-open-and-close-tags -def stack_tag(tag, stack): - t = tag[1:-1] - first_space = t.find(' ') - #print(t) - if t[-1:] == '/': - self_closed_tag = True - elif t[:1] != '/': - # Add tag to stack - if first_space == -1: - stack.append(t) - # print("TRACE open", stack) - else: - stack.append(t[:first_space]) - # print("TRACE open", stack) - else: - if first_space != -1: - t = t[1:first_space] - else: - t = t[1:] - - if len(stack) == 0: - # print("No blocks are open; tried to close", t) - closed_tag = True - else: - if stack[-1] == t: - # Close the block - stack.pop() - # print("TRACE close", t, stack) - else: - # print("Tried to close", t, "but most recent open block is", stack[-1]) - if t in stack: - stack.remove(t) - # print("Prior block closed; continuing") - - # if len(stack): - # print("Blocks still open at EOF:", stack) - return stack - -def tag_is_wrapped(pos, content): - tag_start = pos[0] - tag_end = pos[1] - content_previous = content[:tag_start][::-1] # reverse content_previous - content_later = content[tag_end:] - - left_wraps_findall = re.findall(r'`', content_previous) - left_single_backtick = len(left_wraps_findall) % 2 - right_wraps_findall = re.findall(r'`', content_later) - right_single_backtick = len(right_wraps_findall) % 2 - # print(left_single_backtick, right_single_backtick) - - if left_single_backtick != 0 and right_single_backtick != 0: - # print(content_previous.find('`'), content_later.find('`')) - # print(content_previous) - # print(content_later) - return True - else: - # print(content_previous.find('`'), content_later.find('`')) - # print(content_previous) - # print(content_later) - return False - -def filter_frontmatter(content): - # if there is frontmatter, remove it - if content.startswith('---'): - collect = [] - content_finditer = re.finditer(r'---\n', content) - for i in content_finditer: - meta_pos = i.span()[1] - collect.append(meta_pos) - - # if the number of "---" >= 2 - if len(collect) >= 2: - filter_point = collect[1] - content = content[filter_point:] - - return content - -def filter_backticks(content, filename): - # remove content wrapped by backticks - backticks = [] - content_findall = re.findall(r'```', content) - if len(content_findall): - content_finditer = re.finditer(r'```', content) - for i in content_finditer: - pos = i.span() - backticks.append(pos) - # e.g. backticks = [[23, 26],[37, 40],[123, 126],[147, 150]] - if len(backticks) % 2 != 0: - # print(len(content_findall)) - # print(backticks) - # print(backticks[0][0], backticks[0][1]) - print(filename, ": Some of your code blocks ``` ``` are not closed. Please close them.") - exit(1) - elif len(backticks) != 0: - backticks_start = backticks[0][0] - backticks_end = backticks[1][1] - # print(backticks_start, backticks_end) - content = content.replace(content[backticks_start:backticks_end],'') - content = filter_backticks(content, filename) - return content - -status_code = 0 - -# print(sys.argv[1:]) -for filename in sys.argv[1:]: - # print("Checking " + filename + "......\n") - if os.path.isfile(filename): - file = open(filename, "r" ) - content = file.read() - file.close() - - content = filter_frontmatter(content) - content = filter_backticks(content, filename) - # print(content) - result_findall = re.findall(r'<([^\n`>]*)>', content) - if len(result_findall) == 0: - # print("The edited markdown file " + filename + " has no tags!\n") - continue - else: - result_finditer = re.finditer(r'<([^\n`>]*)>', content) - stack = [] - for i in result_finditer: - # print(i.group(), i.span()) - tag = i.group() - pos = i.span() - - if tag[:4] == '': - continue - elif content[pos[0]-2:pos[0]] == '{{' and content[pos[1]:pos[1]+2] == '}}': - # print(tag) # filter copyable shortcodes - continue - elif tag[:5] == '' for i in stack] - print("ERROR: " + filename + ' has unclosed tags: ' + ', '.join(stack) + '.\n') - status_code = 1 - -if status_code: - print("HINT: Unclosed tags will cause website build failure. Please fix the reported unclosed tags. You can use backticks `` to wrap them or close them. Thanks.") - exit(1) diff --git a/security-compatibility-with-mysql.md b/security-compatibility-with-mysql.md index fce6496e8e1a..766ae1203de1 100644 --- a/security-compatibility-with-mysql.md +++ b/security-compatibility-with-mysql.md @@ -8,9 +8,29 @@ aliases: ['/docs/dev/security-compatibility-with-mysql/','/docs/dev/reference/se TiDB supports similar security functionality to MySQL 5.7, with the following exceptions: -- Only the `mysql_native_password` password-based and certificate-based authentication is supported -- External authentication (such as with LDAP) is not currently supported - Column level permissions are not supported - Password expiry, as well as password last-changed tracking and password lifetime are not supported [#9709](https://github.com/pingcap/tidb/issues/9709) - The permission attributes `max_questions`, `max_updated`, `max_connections`, `max_user_connections` are not supported - Password validation is not currently supported [#9741](https://github.com/pingcap/tidb/issues/9741) + +## Authentication plugin status + +TiDB supports multiple authentication methods. These methods can be specified on a per user basis using [`CREATE USER`](/sql-statements/sql-statement-create-user.md) and [`ALTER USER`](/sql-statements/sql-statement-create-user.md). These methods are compatible with the authentication methods of MySQL with the same names. + +You can use one of the following supported authentication methods in the table. To specify a default method that the server advertises when the client-server connection is being established, set the [`default_authentication_plugin`](/system-variables.md#default_authentication_plugin) variable. + +The support for TLS authentication is configured differently. For detailed information, see [Enable TLS between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md). + +| Authentication Method | Supported | +| :------------------------| :--------------- | +| `mysql_native_password` | Yes | +| `sha256_password` | No | +| `caching_sha2_password` | Yes, since 5.2.0 | +| `auth_socket` | No | +| [TLS Certificates] | Yes | +| LDAP | No | +| PAM | No | +| ed25519 (MariaDB) | No | +| GSSAPI (MariaDB) | No | + +[TLS Certificates]: /enable-tls-between-clients-and-servers.md diff --git a/sql-mode.md b/sql-mode.md index f372a84bd8e7..551503788970 100644 --- a/sql-mode.md +++ b/sql-mode.md @@ -8,9 +8,7 @@ aliases: ['/docs/dev/sql-mode/','/docs/dev/reference/sql/sql-mode/'] TiDB servers operate in different SQL modes and apply these modes differently for different clients. SQL mode defines the SQL syntaxes that TiDB supports and the type of data validation check to perform, as described below: -+ Before TiDB is started, modify the `--sql-mode="modes"` configuration item to set SQL mode. - -+ After TiDB is started, modify `SET [ SESSION | GLOBAL ] sql_mode='modes'` to set SQL mode. +After TiDB is started, modify `SET [ SESSION | GLOBAL ] sql_mode='modes'` to set SQL mode. Ensure that you have `SUPER` privilege when setting SQL mode at `GLOBAL` level, and your setting at this level only affects the connections established afterwards. Changes to SQL mode at `SESSION` level only affect the current client. @@ -46,7 +44,7 @@ Ensure that you have `SUPER` privilege when setting SQL mode at `GLOBAL` level, | `NO_AUTO_CREATE_USER` | Prevents `GRANT` from automatically creating new users, except for the specified password (full support)| | `HIGH_NOT_PRECEDENCE` | The precedence of the NOT operator is such that expressions such as `NOT a BETWEEN b AND c` are parsed as `NOT (a BETWEEN b AND c)`. In some older versions of MySQL, this expression is parsed as `(NOT a) BETWEEN b AND c`. (full support) | | `NO_ENGINE_SUBSTITUTION` | Prevents the automatic replacement of storage engines if the required storage engine is disabled or not compiled. (syntax support only)| -| `PAD_CHAR_TO_FULL_LENGTH` | If this mode is enabled, the system does not trim the trailing spaces for `CHAR` types. (full support) | +| `PAD_CHAR_TO_FULL_LENGTH` | If this mode is enabled, the system does not trim the trailing spaces for `CHAR` types. (syntax support only. This mode has been [deprecated in MySQL 8.0](https://dev.mysql.com/doc/refman/8.0/en/sql-mode.html#sqlmode_pad_char_to_full_length).) | | `REAL_AS_FLOAT` | Treats `REAL` as the synonym of `FLOAT`, not the synonym of `DOUBLE` (full support)| | `POSTGRESQL` | Equivalent to `PIPES_AS_CONCAT`, `ANSI_QUOTES`, `IGNORE_SPACE`, `NO_KEY_OPTIONS`, `NO_TABLE_OPTIONS`, `NO_FIELD_OPTIONS` (syntax support only)| | `MSSQL` | Equivalent to `PIPES_AS_CONCAT`, `ANSI_QUOTES`, `IGNORE_SPACE`, `NO_KEY_OPTIONS`, `NO_TABLE_OPTIONS`, `NO_FIELD_OPTIONS` (syntax support only)| diff --git a/sql-plan-management.md b/sql-plan-management.md index e4e2b24b4fc1..b2cf69acd896 100644 --- a/sql-plan-management.md +++ b/sql-plan-management.md @@ -17,10 +17,78 @@ An SQL binding is the basis of SPM. The [Optimizer Hints](/optimizer-hints.md) d {{< copyable "sql" >}} ```sql -CREATE [GLOBAL | SESSION] BINDING FOR SelectStmt USING SelectStmt +CREATE [GLOBAL | SESSION] BINDING FOR BindableStmt USING BindableStmt ``` -This statement binds SQL execution plans at the GLOBAL or SESSION level. The default scope is SESSION. The bound SQL statement is parameterized and stored in the system table. When a SQL query is processed, as long as the parameterized SQL statement and a bound one in the system table are consistent and the system variable `tidb_use_plan_baselines` is set to `on` (the default value is `on`), the corresponding optimizer hint is available. If multiple execution plans are available, the optimizer chooses to bind the plan with the least cost. +This statement binds SQL execution plans at the GLOBAL or SESSION level. Currently, supported bindable SQL statements (BindableStmt) in TiDB include `SELECT`, `DELETE`, `UPDATE`, and `INSERT` / `REPLACE` with `SELECT` subqueries. + +Specifically, two types of these statements cannot be bound to execution plans due to syntax conflicts. See the following examples: + +```sql +-- Type one: Statements that get the Cartesian product by using the `join` keyword and not specifying the associated columns with the `using` keyword. +create global binding for + select * from t t1 join t t2 +using + select * from t t1 join t t2; + +-- Type two: `DELETE` statements that contain the `using` keyword. +create global binding for + delete from t1 using t1 join t2 on t1.a = t2.a +using + delete from t1 using t1 join t2 on t1.a = t2.a; +``` + +You can bypass syntax conflicts by using equivalent statements. For example, you can rewrite the above statements in the following ways: + +```sql +-- First rewrite of type one statements: Add a `using` clause for the `join` keyword. +create global binding for + select * from t t1 join t t2 using (a) +using + select * from t t1 join t t2 using (a); + +-- Second rewrite of type one statements: Delete the `join` keyword. +create global binding for + select * from t t1, t t2 +using + select * from t t1, t t2; + +-- Rewrite of type two statements: Remove the `using` keyword from the `delete` statement. +create global binding for + delete t1 from t1 join t2 on t1.a = t2.a +using + delete t1 from t1 join t2 on t1.a = t2.a; +``` + +> **Note:** +> +> When creating execution plan bindings for `INSERT` / `REPLACE` statements with `SELECT` subqueries, you need to specify the optimizer hints you want to bind in the `SELECT` subquery, not after the `INSERT` / `REPLACE` keyword. Otherwise, the optimizer hints do not take effect as intended. + +Here are two examples: + +```sql +-- The hint takes effect in the following statement. +create global binding for + insert into t1 select * from t2 where a > 1 and b = 1 +using + insert into t1 select /*+ use_index(@sel_1 t2, a) */ * from t2 where a > 1 and b = 1; + +-- The hint cannot take effect in the following statement. +create global binding for + insert into t1 select * from t2 where a > 1 and b = 1 +using + insert /*+ use_index(@sel_1 t2, a) */ into t1 select * from t2 where a > 1 and b = 1; +``` + +If you do not specify the scope when creating an execution plan binding, the default scope is SESSION. The TiDB optimizer normalizes bound SQL statements and stores them in the system table. When processing SQL queries, if a normalized statement matches one of the bound SQL statements in the system table and the system variable `tidb_use_plan_baselines` is set to `on` (the default value is `on`), TiDB then uses the corresponding optimizer hint for this statement. If there are multiple matchable execution plans, the optimizer chooses the least costly one to bind. + +`Normalization` is a process that converts a constant in an SQL statement to a variable parameter and explicitly specifies the database for tables referenced in the query, with standardized processing on the spaces and line breaks in the SQL statement. See the following example: + +```sql +select * from t where a > 1 +-- Normalized: +select * from test . t where a > ? +``` When a SQL statement has bound execution plans in both GLOBAL and SESSION scopes, because the optimizer ignores the bound execution plan in the GLOBAL scope when it encounters the SESSION binding, the bound execution plan of this statement in the SESSION scope shields the execution plan in the GLOBAL scope. @@ -31,7 +99,7 @@ For example: create global binding for select * from t1, t2 where t1.id = t2.id using - select /*+ sm_join(t1, t2) */ * from t1, t2 where t1.id = t2.id; + select /*+ merge_join(t1, t2) */ * from t1, t2 where t1.id = t2.id; -- The execution plan of this SQL statement uses the `sort merge join` specified in the GLOBAL binding. explain select * from t1, t2 where t1.id = t2.id; @@ -48,42 +116,34 @@ explain select * from t1, t2 where t1.id = t2.id; When the first `select` statement is being executed, the optimizer adds the `sm_join(t1, t2)` hint to the statement through the binding in the GLOBAL scope. The top node of the execution plan in the `explain` result is MergeJoin. When the second `select` statement is being executed, the optimizer uses the binding in the SESSION scope instead of the binding in the GLOBAL scope and adds the `hash_join(t1, t2)` hint to the statement. The top node of the execution plan in the `explain` result is HashJoin. -`Parameterization` is a process that converts a constant in an SQL statement to a variable parameter, with standardized processing on the spaces and line breaks in the SQL statement, for example, +Each standardized SQL statement can have only one binding created using `CREATE BINDING` at a time. When multiple bindings are created for the same standardized SQL statement, the last created binding is retained, and all previous bindings (created and evolved) are marked as deleted. But session bindings and global bindings can coexist and are not affected by this logic. -{{< copyable "sql" >}} +In addition, when you create a binding, TiDB requires that the session is in a database context, which means that a database is specified when the client is connected or `use ${database}` is executed. -```sql -select * from t where a > 1 --- parameterized: -select * from t where a > ？ -``` +The original SQL statement and the bound statement must have the same text after normalization and hint removal, or the binding will fail. Take the following examples: -Each standardized SQL statement can have only one binding created using `CREATE BINDING` at a time. When multiple bindings are created for the same standardized SQL statement, the last created binding is retained, and all previous bindings (created and evolved) are marked as deleted. But session bindings and global bindings can coexist and are not affected by this logic. +- This binding can be created successfully because the texts before and after parameterization and hint removal are the same: `select * from test . t where a > ?` -In addition, when you create a binding, TiDB requires that the session is in a database context, which means that a database is specified when the client is connected or `use ${database}` is executed. + ```sql + CREATE BINDING FOR SELECT * FROM t WHERE a > 1 USING SELECT * FROM t use index (idx) WHERE a > 2 + ``` + +- This binding will fail because the original SQL statement is processed as `select * from test . t where a > ?`, while the bound SQL statement is processed differently as `select * from test . t where b > ?`. + + ```sql + CREATE BINDING FOR SELECT * FROM t WHERE a > 1 USING SELECT * FROM t use index(idx) WHERE b > 2 + ``` > **Note:** > -> The text must be the same before and after parameterization and hint removal for both the original SQL statement and the bound statement, or the binding will fail. Take the following examples: -> -> - This binding can be created successfully because the texts before and after parameterization and hint removal are the same: `select * from t where a > ?` -> -> ```sql -> CREATE BINDING FOR SELECT * FROM t WHERE a > 1 USING SELECT * FROM t use index (idx) WHERE a > 2 -> ``` -> -> - This binding will fail because the original SQL statement is processed as `select * from t where a > ?`, while the bound SQL statement is processed differently as `select * from t where b > ?`. -> -> ```sql -> CREATE BINDING FOR SELECT * FROM t WHERE a > 1 USING SELECT * FROM t use index(idx) WHERE b > 2 -> ``` +> For `PREPARE` / `EXECUTE` statements and for queries executed with binary protocols, you need to create execution plan bindings for the real query statements, not for the `PREPARE` / `EXECUTE` statements. ### Remove binding {{< copyable "sql" >}} ```sql -DROP [GLOBAL | SESSION] BINDING FOR SelectStmt +DROP [GLOBAL | SESSION] BINDING FOR BindableStmt; ``` This statement removes a specified execution plan binding at the GLOBAL or SESSION level. The default scope is SESSION. @@ -132,7 +192,15 @@ To enable baseline capturing, set `tidb_capture_plan_baselines` to `on`. The def > > Because the automatic binding creation function relies on [Statement Summary](/statement-summary-tables.md), make sure to enable Statement Summary before using automatic binding. -After automatic binding creation is enabled, the historical SQL statements in the Statement Summary are traversed every `bind-info-lease` (the default value is `3s`), and baseline capturing is automatically created for SQL statements that appear at least twice. +After automatic binding creation is enabled, the historical SQL statements in the Statement Summary are traversed every `bind-info-lease` (the default value is `3s`), and a binding is automatically created for SQL statements that appear at least twice. For these SQL statements, TiDB automatically binds the execution plan recorded in Statement Summary. + +However, TiDB does not automatically capture bindings for the following types of SQL statements: + +- `EXPLAIN` and `EXPLAIN ANALYZE` statements. +- SQL statements executed internally in TiDB, such as `SELECT` queries used for automatically loading statistical information. +- SQL statements that are bound to a manually created execution plan. + +For `PREPARE` / `EXECUTE` statements and for queries executed with binary protocols, TiDB automatically captures bindings for the real query statements, not for the `PREPARE` / `EXECUTE` statements. > **Note:** > @@ -140,7 +208,7 @@ After automatic binding creation is enabled, the historical SQL statements in th ## Baseline evolution -Baseline evolution is an important feature of SPM introduced in TiDB v4.0.0-rc. +Baseline evolution is an important feature of SPM introduced in TiDB v4.0. As data updates, the previously bound execution plan might no longer be optimal. The baseline evolution feature can automatically optimize the bound execution plan. @@ -160,9 +228,9 @@ The default value of `tidb_evolve_plan_baselines` is `off`. > **Note:** > -> The global variable does not take effect in the current session. It only takes effect in a newly created session. To enable automatic binding evolution in the current session, change the keyword `global` to `session`. +> The feature baseline evolution is not generally available for now. It is **NOT RECOMMENDED** to use it in the production environment. -After the automatic binding evolution feature is enabled, if the optimal execution plan selected by the optimizer is not among the binding execution plans, the optimizer marks the plan as an execution plan that waits for verification. At every `bind-info-lease` (the default value is `3s`) interval, an execution plan to be verified is selected and compared with the binding execution plan that has the least cost in terms of the actual execution time. If the plan to be verified has shorter execution time, this plan is marked as a usable binding. The following example describes the process above. +After the automatic binding evolution feature is enabled, if the optimal execution plan selected by the optimizer is not among the binding execution plans, the optimizer marks the plan as an execution plan that waits for verification. At every `bind-info-lease` (the default value is `3s`) interval, an execution plan to be verified is selected and compared with the binding execution plan that has the least cost in terms of the actual execution time. If the plan to be verified has shorter execution time (the current criterion for the comparison is that the execution time of the plan to be verified is no longer than 2/3 that of the binding execution plan), this plan is marked as a usable binding. The following example describes the process above. Assume that table `t` is defined as follows: @@ -196,7 +264,7 @@ The binding evolution can address this kind of issues. When the optimizer recogn To reduce the impact that the automatic evolution has on clusters, use the following configurations: -- Set `tidb_evolve_plan_task_max_time` to limit the maximum execution time of each execution plan. The default value is `600s`. +- Set `tidb_evolve_plan_task_max_time` to limit the maximum execution time of each execution plan. The default value is `600s`. In the actual verification process, the maximum execution time is also limited to no more than twice the time of the verified execution plan. - Set `tidb_evolve_plan_task_start_time` (`00:00 +0000` by default) and `tidb_evolve_plan_task_end_time` (`23:59 +0000` by default) to limit the time window. ### Notes diff --git a/sql-prepare-plan-cache.md b/sql-prepare-plan-cache.md index 14f9e157dace..74620243f67b 100644 --- a/sql-prepare-plan-cache.md +++ b/sql-prepare-plan-cache.md @@ -5,7 +5,7 @@ summary: Learn about SQL Prepare Execution Plan Cache in TiDB. # SQL Prepare Execution Plan Cache -In the current development release of TiDB, the execution plan of prepared statements is cached by default. This includes both forms of prepared statements: +TiDB supports execution plan caching for `Prepare` and `Execute` queries. This includes both forms of prepared statements: - Using the `COM_STMT_PREPARE` and `COM_STMT_EXECUTE` protocol features. - Using the SQL statements `PREPARE` and `EXECUTE`. @@ -25,32 +25,26 @@ In the current version of TiDB, when the `Prepare` statement meets any of the fo - The window frame definition of the `Window` function contains `?`; - Partition tables are involved in the query. -The LRU linked list is designed as a session-level cache because `Prepare` / -`Execute` cannot be executed across sessions. Each element of the LRU list is a -key-value pair. The value is the execution plan, and the key is composed of the -following parts: +The LRU linked list is designed as a session-level cache because `Prepare` / `Execute` cannot be executed across sessions. Each element of the LRU list is a key-value pair. The value is the execution plan, and the key is composed of the following parts: - The name of the database where `Execute` is executed; -- The identifier of the `Prepare` statement, that is, the name after the `PREPARE` - keyword; +- The identifier of the `Prepare` statement, that is, the name after the `PREPARE` keyword; - The current schema version, which is updated after every successfully executed DDL statement; - The SQL mode when executing `Execute`; - The current time zone, which is the value of the `time_zone` system variable. -Any change in the above information (e.g. switching databases, renaming `Prepare` statement, executing DDL statements, or modifying the value of SQL mode / `time_zone`), or the LRU cache elimination mechanism causes the execution plan cache miss when executing. +Any change in the above information (for example, switching databases, renaming `Prepare` statement, executing DDL statements, or modifying the value of SQL mode / `time_zone`), or the LRU cache elimination mechanism causes the execution plan cache miss when executing. After the execution plan cache is obtained from the cache, TiDB first checks whether the execution plan is still valid. If the current `Execute` statement is executed in an explicit transaction, and the referenced table is modified in the transaction pre-order statement, the cached execution plan accessing this table does not contain the `UnionScan` operator, then it cannot be executed. -After the validation test is passed, the scan range of the execution plan is adjusted according to the current parameter values, and then used -to perform data querying. +After the validation test is passed, the scan range of the execution plan is adjusted according to the current parameter values, and then used to perform data querying. -There are two points worth noting about execution plan caching and query -performance: +There are two points worth noting about execution plan caching and query performance: - Considering that the parameters of `Execute` are different, the execution plan cache prohibits some aggressive query optimization methods that are closely related to specific parameter values to ensure adaptability. This causes that the query plan may not be optimal for certain parameter values. For example, the filter condition of the query is `where a > ? And a < ?`, the parameters of the first `Execute` statement are `2` and `1` respectively. Considering that these two parameters maybe be `1` and `2` in the next execution time, the optimizer does not generate the optimal `TableDual` execution plan that is specific to current parameter values; - If cache invalidation and elimination are not considered, an execution plan cache is applied to various parameter values, which in theory also result in non-optimal execution plans for certain values. For example, if the filter condition is `where a < ?` and the parameter value used for the first execution is `1`, then the optimizer generates the optimal `IndexScan` execution plan and puts it into the cache. In the subsequent executions, if the value becomes `10000`, the `TableScan` plan might be the better one. But due to the execution plan cache, the previously generated `IndexScan` is used for execution. Therefore, the execution plan cache is more suitable for application scenarios where the query is simple (the ratio of compilation is high) and the execution plan is relatively fixed. -The default capacity for the plan cache is `100` statements. You can configure this by modifying [`prepare-plan-cache`](/tidb-configuration-file.md#prepared-plan-cache) in the TiDB configuration file. +Currently, the execution plan cache is disabled by default. You can enable this feature by enabling the [`prepare-plan-cache`](/tidb-configuration-file.md#prepared-plan-cache) in the TiDB configuration file. > **Note：** > diff --git a/sql-statements/sql-statement-add-column.md b/sql-statements/sql-statement-add-column.md index c8f85e4012fb..5471ef172893 100644 --- a/sql-statements/sql-statement-add-column.md +++ b/sql-statements/sql-statement-add-column.md @@ -10,21 +10,39 @@ The `ALTER TABLE.. ADD COLUMN` statement adds a column to an existing table. Thi ## Synopsis -**AlterTableStmt:** - -![AlterTableStmt](/media/sqlgram/AlterTableStmt.png) - -**AlterTableSpec:** - -![AlterTableSpec](/media/sqlgram/AlterTableSpec.png) - -**ColumnDef:** - -![ColumnDef](/media/sqlgram/ColumnDef.png) - -**ColumnPosition:** - -![ColumnPosition](/media/sqlgram/ColumnPosition.png) +```ebnf+diagram +AlterTableStmt ::= + 'ALTER' IgnoreOptional 'TABLE' TableName ( AlterTableSpecListOpt AlterTablePartitionOpt | 'ANALYZE' 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? AnalyzeOptionListOpt ) + +AlterTableSpec ::= + TableOptionList +| 'SET' 'TIFLASH' 'REPLICA' LengthNum LocationLabelList +| 'CONVERT' 'TO' CharsetKw ( CharsetName | 'DEFAULT' ) OptCollate +| 'ADD' ( ColumnKeywordOpt IfNotExists ( ColumnDef ColumnPosition | '(' TableElementList ')' ) | Constraint | 'PARTITION' IfNotExists NoWriteToBinLogAliasOpt ( PartitionDefinitionListOpt | 'PARTITIONS' NUM ) ) +| ( ( 'CHECK' | 'TRUNCATE' ) 'PARTITION' | ( 'OPTIMIZE' | 'REPAIR' | 'REBUILD' ) 'PARTITION' NoWriteToBinLogAliasOpt ) AllOrPartitionNameList +| 'COALESCE' 'PARTITION' NoWriteToBinLogAliasOpt NUM +| 'DROP' ( ColumnKeywordOpt IfExists ColumnName RestrictOrCascadeOpt | 'PRIMARY' 'KEY' | 'PARTITION' IfExists PartitionNameList | ( KeyOrIndex IfExists | 'CHECK' ) Identifier | 'FOREIGN' 'KEY' IfExists Symbol ) +| 'EXCHANGE' 'PARTITION' Identifier 'WITH' 'TABLE' TableName WithValidationOpt +| ( 'IMPORT' | 'DISCARD' ) ( 'PARTITION' AllOrPartitionNameList )? 'TABLESPACE' +| 'REORGANIZE' 'PARTITION' NoWriteToBinLogAliasOpt ReorganizePartitionRuleOpt +| 'ORDER' 'BY' AlterOrderItem ( ',' AlterOrderItem )* +| ( 'DISABLE' | 'ENABLE' ) 'KEYS' +| ( 'MODIFY' ColumnKeywordOpt IfExists | 'CHANGE' ColumnKeywordOpt IfExists ColumnName ) ColumnDef ColumnPosition +| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier IndexInvisible ) +| 'RENAME' ( ( 'COLUMN' | KeyOrIndex ) Identifier 'TO' Identifier | ( 'TO' | '='? | 'AS' ) TableName ) +| LockClause +| AlgorithmClause +| 'FORCE' +| ( 'WITH' | 'WITHOUT' ) 'VALIDATION' +| 'SECONDARY_LOAD' +| 'SECONDARY_UNLOAD' + +ColumnDef ::= + ColumnName ( Type | 'SERIAL' ) ColumnOptionListOpt + +ColumnPosition ::= + ( 'FIRST' | 'AFTER' ColumnName )? +``` ## Examples diff --git a/sql-statements/sql-statement-add-index.md b/sql-statements/sql-statement-add-index.md index 42af2fe15aac..d55a068c4fe5 100644 --- a/sql-statements/sql-statement-add-index.md +++ b/sql-statements/sql-statement-add-index.md @@ -10,61 +10,80 @@ The `ALTER TABLE.. ADD INDEX` statement adds an index to an existing table. This ## Synopsis -**AlterTableStmt:** - -![AlterTableStmt](/media/sqlgram/AlterTableStmt.png) - -**AlterTableSpec:** - -![AlterTableSpec](/media/sqlgram/AlterTableSpec.png) - -**Constraint:** - -![Constraint](/media/sqlgram/Constraint.png) - -**ConstraintKeywordOpt:** - -![ConstraintKeywordOpt](/media/sqlgram/ConstraintKeywordOpt.png) - -**ConstraintElem:** - -![ConstraintElem](/media/sqlgram/ConstraintElem.png) - -**IndexNameAndTypeOpt:** - -![IndexNameAndTypeOpt](/media/sqlgram/IndexNameAndTypeOpt.png) - -**IndexPartSpecificationList:** - -![IndexPartSpecificationList](/media/sqlgram/IndexPartSpecificationList.png) - -**IndexPartSpecification:** - -![IndexPartSpecification](/media/sqlgram/IndexPartSpecification.png) - -**IndexOptionList:** - -![IndexOptionList](/media/sqlgram/IndexOptionList.png) - -**IndexOption:** - -![IndexOption](/media/sqlgram/IndexOption.png) - -**KeyOrIndex:** - -![KeyOrIndex](/media/sqlgram/KeyOrIndex.png) - -**IndexKeyTypeOpt:** - -![IndexKeyTypeOpt](/media/sqlgram/IndexKeyTypeOpt.png) - -**IndexInvisible:** - -![IndexInvisible](/media/sqlgram/IndexInvisible.png) - -**IndexTypeName:** - -![IndexTypeName](/media/sqlgram/IndexTypeName.png) +```ebnf+diagram +AlterTableStmt ::= + 'ALTER' IgnoreOptional 'TABLE' TableName ( AlterTableSpecListOpt AlterTablePartitionOpt | 'ANALYZE' 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? AnalyzeOptionListOpt ) + +AlterTableSpec ::= + TableOptionList +| 'SET' 'TIFLASH' 'REPLICA' LengthNum LocationLabelList +| 'CONVERT' 'TO' CharsetKw ( CharsetName | 'DEFAULT' ) OptCollate +| 'ADD' ( ColumnKeywordOpt IfNotExists ( ColumnDef ColumnPosition | '(' TableElementList ')' ) | Constraint | 'PARTITION' IfNotExists NoWriteToBinLogAliasOpt ( PartitionDefinitionListOpt | 'PARTITIONS' NUM ) ) +| ( ( 'CHECK' | 'TRUNCATE' ) 'PARTITION' | ( 'OPTIMIZE' | 'REPAIR' | 'REBUILD' ) 'PARTITION' NoWriteToBinLogAliasOpt ) AllOrPartitionNameList +| 'COALESCE' 'PARTITION' NoWriteToBinLogAliasOpt NUM +| 'DROP' ( ColumnKeywordOpt IfExists ColumnName RestrictOrCascadeOpt | 'PRIMARY' 'KEY' | 'PARTITION' IfExists PartitionNameList | ( KeyOrIndex IfExists | 'CHECK' ) Identifier | 'FOREIGN' 'KEY' IfExists Symbol ) +| 'EXCHANGE' 'PARTITION' Identifier 'WITH' 'TABLE' TableName WithValidationOpt +| ( 'IMPORT' | 'DISCARD' ) ( 'PARTITION' AllOrPartitionNameList )? 'TABLESPACE' +| 'REORGANIZE' 'PARTITION' NoWriteToBinLogAliasOpt ReorganizePartitionRuleOpt +| 'ORDER' 'BY' AlterOrderItem ( ',' AlterOrderItem )* +| ( 'DISABLE' | 'ENABLE' ) 'KEYS' +| ( 'MODIFY' ColumnKeywordOpt IfExists | 'CHANGE' ColumnKeywordOpt IfExists ColumnName ) ColumnDef ColumnPosition +| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier IndexInvisible ) +| 'RENAME' ( ( 'COLUMN' | KeyOrIndex ) Identifier 'TO' Identifier | ( 'TO' | '='? | 'AS' ) TableName ) +| LockClause +| AlgorithmClause +| 'FORCE' +| ( 'WITH' | 'WITHOUT' ) 'VALIDATION' +| 'SECONDARY_LOAD' +| 'SECONDARY_UNLOAD' + +Constraint ::= + ConstraintKeywordOpt ConstraintElem + +ConstraintKeywordOpt ::= + ( 'CONSTRAINT' Symbol? )? + +ConstraintElem ::= + ( ( 'PRIMARY' 'KEY' | KeyOrIndex IfNotExists | 'UNIQUE' KeyOrIndexOpt ) IndexNameAndTypeOpt | 'FULLTEXT' KeyOrIndexOpt IndexName ) '(' IndexPartSpecificationList ')' IndexOptionList +| 'FOREIGN' 'KEY' IfNotExists IndexName '(' IndexPartSpecificationList ')' ReferDef +| 'CHECK' '(' Expression ')' EnforcedOrNotOpt + +IndexNameAndTypeOpt ::= + IndexName ( 'USING' IndexTypeName )? +| Identifier 'TYPE' IndexTypeName + +IndexPartSpecificationList ::= + IndexPartSpecification ( ',' IndexPartSpecification )* + +IndexPartSpecification ::= + ( ColumnName OptFieldLen | '(' Expression ')' ) Order + +IndexOptionList ::= + IndexOption* + +IndexOption ::= + 'KEY_BLOCK_SIZE' '='? LengthNum +| IndexType +| 'WITH' 'PARSER' Identifier +| 'COMMENT' stringLit +| IndexInvisible + +KeyOrIndex ::= + 'KEY' +| 'INDEX' + +IndexKeyTypeOpt ::= + ( 'UNIQUE' | 'SPATIAL' | 'FULLTEXT' )? + +IndexInvisible ::= + 'VISIBLE' +| 'INVISIBLE' + +IndexTypeName ::= + 'BTREE' +| 'HASH' +| 'RTREE' +``` ## Examples @@ -102,10 +121,9 @@ mysql> EXPLAIN SELECT * FROM t1 WHERE c1 = 3; ## MySQL compatibility * `FULLTEXT`, `HASH` and `SPATIAL` indexes are not supported. -* `VISIBLE/INVISIBLE` index is not supported (currently only the master branch actually supports this feature). * Descending indexes are not supported (similar to MySQL 5.7). * Adding multiple indexes at the same time is currently not supported. -* Adding the primary key constraint to a table is not supported by default. You can enable the feature by setting the `alter-primary-key` configuration item to `true`. For details, see [alter-primary-key](/tidb-configuration-file.md#alter-primary-key). +* Adding the primary key of the `CLUSTERED` type to a table is not supported. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). ## See also diff --git a/sql-statements/sql-statement-admin-cancel-ddl.md b/sql-statements/sql-statement-admin-cancel-ddl.md index 2cdc472fe8c2..ee7c9c421807 100644 --- a/sql-statements/sql-statement-admin-cancel-ddl.md +++ b/sql-statements/sql-statement-admin-cancel-ddl.md @@ -10,13 +10,13 @@ The `ADMIN CANCEL DDL` statement allows you to cancel a running DDL job. The `jo ## Synopsis -**AdminStmt:** +```ebnf+diagram +AdminStmt ::= + 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) -![AdminStmt](/media/sqlgram/AdminStmt.png) - -**NumList:** - -![NumList](/media/sqlgram/NumList.png) +NumList ::= + Int64Num ( ',' Int64Num )* +``` ## Examples diff --git a/sql-statements/sql-statement-admin-check-table-index.md b/sql-statements/sql-statement-admin-check-table-index.md index 6666d9b34838..a52fc061af51 100644 --- a/sql-statements/sql-statement-admin-check-table-index.md +++ b/sql-statements/sql-statement-admin-check-table-index.md @@ -10,13 +10,13 @@ The `ADMIN CHECK [TABLE|INDEX]` statement checks for data consistency of tables ## Synopsis -**AdminStmt:** +```ebnf+diagram +AdminStmt ::= + 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) -![AdminStmt](/media/sqlgram/AdminStmt.png) - -**TableNameList:** - -![TableNameList](/media/sqlgram/TableNameList.png) +TableNameList ::= + TableName ( ',' TableName )* +``` ## Examples diff --git a/sql-statements/sql-statement-admin-checksum-table.md b/sql-statements/sql-statement-admin-checksum-table.md index 3ecb7c6e19b1..80d1ff05028d 100644 --- a/sql-statements/sql-statement-admin-checksum-table.md +++ b/sql-statements/sql-statement-admin-checksum-table.md @@ -10,13 +10,13 @@ The `ADMIN CHECKSUM TABLE` statement calculates a CRC64 checksum for the data an ## Synopsis -**AdminStmt:** +```ebnf+diagram +AdminStmt ::= + 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) -![AdminStmt](/media/sqlgram/AdminStmt.png) - -**TableNameList:** - -![TableNameList](/media/sqlgram/TableNameList.png) +TableNameList ::= + TableName ( ',' TableName )* +``` ## Examples diff --git a/sql-statements/sql-statement-admin-show-ddl.md b/sql-statements/sql-statement-admin-show-ddl.md index 894c23e75c8c..42c48b9522fc 100644 --- a/sql-statements/sql-statement-admin-show-ddl.md +++ b/sql-statements/sql-statement-admin-show-ddl.md @@ -9,17 +9,16 @@ The `ADMIN SHOW DDL [JOBS|QUERIES]` statement shows information about running an ## Synopsis -**AdminStmt:** +```ebnf+diagram +AdminStmt ::= + 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) -![AdminStmt](/media/sqlgram/AdminStmt.png) +NumList ::= + Int64Num ( ',' Int64Num )* -**NumList:** - -![NumList](/media/sqlgram/NumList.png) - -**WhereClauseOptional:** - -![WhereClauseOptional](/media/sqlgram/WhereClauseOptional.png) +WhereClauseOptional ::= + WhereClause? +``` ## Examples diff --git a/sql-statements/sql-statement-admin.md b/sql-statements/sql-statement-admin.md index db865dbc9141..8bdf6836adc9 100644 --- a/sql-statements/sql-statement-admin.md +++ b/sql-statements/sql-statement-admin.md @@ -117,9 +117,10 @@ For details, refer to [admin show slow statement](/identify-slow-queries.md#admi ## Synopsis -**AdminStmt:** - -![AdminStmt](/media/sqlgram/AdminStmt.png) +```ebnf+diagram +AdminStmt ::= + 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) +``` ## Examples @@ -193,7 +194,7 @@ admin show ddl jobs 5 where state!='synced' and db_name='test'; * `JOB_TYPE`: the type of the DDL operations. * `SCHEMA_STATE`: the current state of the schema. If the `JOB_TYPE` is `add index`, it is the state of the index; if the `JOB_TYPE` is `add column`, it is the state of the column; if the `JOB_TYPE` is `create table`, it is the state of the table. The common states include: * `none`: it indicates not existing. When the `drop` or `create` operation fails and rolls back, it usually becomes the `none` state. - * `delete only`, `write only`, `delete reorganization`, `write reorganization`: these four states are intermediate states. For details, see the paper [Online, Asynchronous Schema Change in F1](https://static.googleusercontent.com/media/research.google.com/zh-CN//pubs/archive/41376.pdf). These states are not visible in common operations, because the conversion from the intermediate states is so quick. You can see the `write reorganization` state only in `add index` operations, which means that the index data is being added. + * `delete only`, `write only`, `delete reorganization`, `write reorganization`: these four states are intermediate states. These states are not visible in common operations, because the conversion from the intermediate states is so quick. You can see the `write reorganization` state only in `add index` operations, which means that the index data is being added. * `public`: it indicates existing and usable. When operations like `create table` and `add index/column` are finished, it usually becomes the `public` state, which means that the created table/column/index can be normally read and written now. * `SCHEMA_ID`: the ID of the database on which the DDL operations are performed. * `TABLE_ID`: the ID of the table on which the DDL operations are performed. diff --git a/sql-statements/sql-statement-alter-database.md b/sql-statements/sql-statement-alter-database.md index 832f7666f044..9420182e2fd0 100644 --- a/sql-statements/sql-statement-alter-database.md +++ b/sql-statements/sql-statement-alter-database.md @@ -10,13 +10,13 @@ aliases: ['/docs/dev/sql-statements/sql-statement-alter-database/','/docs/dev/re ## Synopsis -**AlterDatabaseStmt:** +```ebnf+diagram +AlterDatabaseStmt ::= + 'ALTER' 'DATABASE' DBName? DatabaseOptionList -![AlterDatabaseStmt](/media/sqlgram/AlterDatabaseStmt.png) - -**DatabaseOption:** - -![DatabaseOption](/media/sqlgram/DatabaseOption.png) +DatabaseOption ::= + DefaultKwdOpt ( CharsetKw '='? CharsetName | 'COLLATE' '='? CollationName | 'ENCRYPTION' '='? EncryptionOpt ) +``` ## Examples diff --git a/sql-statements/sql-statement-alter-index.md b/sql-statements/sql-statement-alter-index.md index 71a4ae8e84bd..bff21e3ff4fc 100644 --- a/sql-statements/sql-statement-alter-index.md +++ b/sql-statements/sql-statement-alter-index.md @@ -10,17 +10,37 @@ The `ALTER INDEX` statement is used to modify the visibility of the index to `Vi ## Synopsis -**AlterTableStmt:** - -![AlterTableStmt](/media/sqlgram/AlterTableStmt.png) - -**AlterTableSpec:** - -![AlterTableSpec](/media/sqlgram/AlterTableSpec.png) - -**IndexInvisible:** - -![IndexInvisible](/media/sqlgram/IndexInvisible.png) +```ebnf+diagram +AlterTableStmt ::= + 'ALTER' IgnoreOptional 'TABLE' TableName ( AlterTableSpecListOpt AlterTablePartitionOpt | 'ANALYZE' 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? AnalyzeOptionListOpt ) + +AlterTableSpec ::= + TableOptionList +| 'SET' 'TIFLASH' 'REPLICA' LengthNum LocationLabelList +| 'CONVERT' 'TO' CharsetKw ( CharsetName | 'DEFAULT' ) OptCollate +| 'ADD' ( ColumnKeywordOpt IfNotExists ( ColumnDef ColumnPosition | '(' TableElementList ')' ) | Constraint | 'PARTITION' IfNotExists NoWriteToBinLogAliasOpt ( PartitionDefinitionListOpt | 'PARTITIONS' NUM ) ) +| ( ( 'CHECK' | 'TRUNCATE' ) 'PARTITION' | ( 'OPTIMIZE' | 'REPAIR' | 'REBUILD' ) 'PARTITION' NoWriteToBinLogAliasOpt ) AllOrPartitionNameList +| 'COALESCE' 'PARTITION' NoWriteToBinLogAliasOpt NUM +| 'DROP' ( ColumnKeywordOpt IfExists ColumnName RestrictOrCascadeOpt | 'PRIMARY' 'KEY' | 'PARTITION' IfExists PartitionNameList | ( KeyOrIndex IfExists | 'CHECK' ) Identifier | 'FOREIGN' 'KEY' IfExists Symbol ) +| 'EXCHANGE' 'PARTITION' Identifier 'WITH' 'TABLE' TableName WithValidationOpt +| ( 'IMPORT' | 'DISCARD' ) ( 'PARTITION' AllOrPartitionNameList )? 'TABLESPACE' +| 'REORGANIZE' 'PARTITION' NoWriteToBinLogAliasOpt ReorganizePartitionRuleOpt +| 'ORDER' 'BY' AlterOrderItem ( ',' AlterOrderItem )* +| ( 'DISABLE' | 'ENABLE' ) 'KEYS' +| ( 'MODIFY' ColumnKeywordOpt IfExists | 'CHANGE' ColumnKeywordOpt IfExists ColumnName ) ColumnDef ColumnPosition +| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier IndexInvisible ) +| 'RENAME' ( ( 'COLUMN' | KeyOrIndex ) Identifier 'TO' Identifier | ( 'TO' | '='? | 'AS' ) TableName ) +| LockClause +| AlgorithmClause +| 'FORCE' +| ( 'WITH' | 'WITHOUT' ) 'VALIDATION' +| 'SECONDARY_LOAD' +| 'SECONDARY_UNLOAD' + +IndexInvisible ::= + 'VISIBLE' +| 'INVISIBLE' +``` ## Examples diff --git a/sql-statements/sql-statement-alter-instance.md b/sql-statements/sql-statement-alter-instance.md index d57d61437535..a71c46ef53b7 100644 --- a/sql-statements/sql-statement-alter-instance.md +++ b/sql-statements/sql-statement-alter-instance.md @@ -20,11 +20,13 @@ When an error occurs during reloading, by default, this error message is returne **AlterInstanceStmt:** -![AlterInstanceStmt](/media/sqlgram/AlterInstanceStmt.png) +```ebnf+diagram +AlterInstanceStmt ::= + 'ALTER' 'INSTANCE' InstanceOption -**InstanceOption:** - -![InstanceOption](/media/sqlgram/InstanceOption.png) +InstanceOption ::= + 'RELOAD' 'TLS' ('NO' 'ROLLBACK' 'ON' 'ERROR')? +``` ## Example diff --git a/sql-statements/sql-statement-alter-table.md b/sql-statements/sql-statement-alter-table.md index c7f4de79384b..0389f85fd451 100644 --- a/sql-statements/sql-statement-alter-table.md +++ b/sql-statements/sql-statement-alter-table.md @@ -13,17 +13,36 @@ This statement modifies an existing table to conform to a new table structure. T ## Synopsis -**AlterTableStmt:** - -![AlterTableStmt](/media/sqlgram/AlterTableStmt.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) - -**AlterTableSpec:** - -![AlterTableSpec](/media/sqlgram/AlterTableSpec.png) +```ebnf+diagram +AlterTableStmt ::= + 'ALTER' IgnoreOptional 'TABLE' TableName ( AlterTableSpecListOpt AlterTablePartitionOpt | 'ANALYZE' 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? AnalyzeOptionListOpt ) + +TableName ::= + Identifier ('.' Identifier)? + +AlterTableSpec ::= + TableOptionList +| 'SET' 'TIFLASH' 'REPLICA' LengthNum LocationLabelList +| 'CONVERT' 'TO' CharsetKw ( CharsetName | 'DEFAULT' ) OptCollate +| 'ADD' ( ColumnKeywordOpt IfNotExists ( ColumnDef ColumnPosition | '(' TableElementList ')' ) | Constraint | 'PARTITION' IfNotExists NoWriteToBinLogAliasOpt ( PartitionDefinitionListOpt | 'PARTITIONS' NUM ) ) +| ( ( 'CHECK' | 'TRUNCATE' ) 'PARTITION' | ( 'OPTIMIZE' | 'REPAIR' | 'REBUILD' ) 'PARTITION' NoWriteToBinLogAliasOpt ) AllOrPartitionNameList +| 'COALESCE' 'PARTITION' NoWriteToBinLogAliasOpt NUM +| 'DROP' ( ColumnKeywordOpt IfExists ColumnName RestrictOrCascadeOpt | 'PRIMARY' 'KEY' | 'PARTITION' IfExists PartitionNameList | ( KeyOrIndex IfExists | 'CHECK' ) Identifier | 'FOREIGN' 'KEY' IfExists Symbol ) +| 'EXCHANGE' 'PARTITION' Identifier 'WITH' 'TABLE' TableName WithValidationOpt +| ( 'IMPORT' | 'DISCARD' ) ( 'PARTITION' AllOrPartitionNameList )? 'TABLESPACE' +| 'REORGANIZE' 'PARTITION' NoWriteToBinLogAliasOpt ReorganizePartitionRuleOpt +| 'ORDER' 'BY' AlterOrderItem ( ',' AlterOrderItem )* +| ( 'DISABLE' | 'ENABLE' ) 'KEYS' +| ( 'MODIFY' ColumnKeywordOpt IfExists | 'CHANGE' ColumnKeywordOpt IfExists ColumnName ) ColumnDef ColumnPosition +| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier IndexInvisible ) +| 'RENAME' ( ( 'COLUMN' | KeyOrIndex ) Identifier 'TO' Identifier | ( 'TO' | '='? | 'AS' ) TableName ) +| LockClause +| AlgorithmClause +| 'FORCE' +| ( 'WITH' | 'WITHOUT' ) 'VALIDATION' +| 'SECONDARY_LOAD' +| 'SECONDARY_UNLOAD' +``` ## Examples @@ -131,9 +150,15 @@ Query OK, 0 rows affected, 1 warning (0.25 sec) The following major restrictions apply to `ALTER TABLE` in TiDB: -* Multiple operations cannot be completed in a single `ALTER TABLE` statement. +* Making multiple changes in a single `ALTER TABLE` statement is currently not supported. + +* Changes of the [Reorg-Data](/sql-statements/sql-statement-modify-column.md#reorg-data-change) types on primary key columns are not supported. + +* Changes of column types on partitioned tables are not supported. + +* Changes of column types on generated columns are not supported. -* Lossy changes such as changing from `BIGINT` to `INT` are currently not supported. +* Changes of some data types (for example, some TIME, Bit, Set, Enum, and JSON types) are not supported due to the compatibility issues of the `CAST` function's behavior between TiDB and MySQL. * Spatial data types are not supported. diff --git a/sql-statements/sql-statement-alter-user.md b/sql-statements/sql-statement-alter-user.md index 0d6e4c313417..293680864cc3 100644 --- a/sql-statements/sql-statement-alter-user.md +++ b/sql-statements/sql-statement-alter-user.md @@ -10,25 +10,22 @@ This statement changes an existing user inside the TiDB privilege system. In the ## Synopsis -**AlterUserStmt:** +```ebnf+diagram +AlterUserStmt ::= + 'ALTER' 'USER' IfExists (UserSpecList RequireClauseOpt ConnectionOptions PasswordOrLockOptions | 'USER' '(' ')' 'IDENTIFIED' 'BY' AuthString) -![AlterUserStmt](/media/sqlgram/AlterUserStmt.png) +UserSpecList ::= + UserSpec ( ',' UserSpec )* -**UserSpecList:** +UserSpec ::= + Username AuthOption -![UserSpecList](/media/sqlgram/UserSpecList.png) +Username ::= + StringName ('@' StringName | singleAtIdentifier)? | 'CURRENT_USER' OptionalBraces -**UserSpec:** - -![UserSpec](/media/sqlgram/UserSpec.png) - -**Username:** - -![Username](/media/sqlgram/Username.png) - -**AuthOption:** - -![AuthOption](/media/sqlgram/AuthOption.png) +AuthOption ::= + ( 'IDENTIFIED' ( 'BY' ( AuthString | 'PASSWORD' HashString ) | 'WITH' StringName ( 'BY' AuthString | 'AS' HashString )? ) )? +``` ## Examples diff --git a/sql-statements/sql-statement-analyze-table.md b/sql-statements/sql-statement-analyze-table.md index 6e6e21b6100f..e95b2164f915 100644 --- a/sql-statements/sql-statement-analyze-table.md +++ b/sql-statements/sql-statement-analyze-table.md @@ -14,17 +14,16 @@ Currently, TiDB collects statistical information in two ways: full collection (i ## Synopsis -**AnalyzeTableStmt:** +```ebnf+diagram +AnalyzeTableStmt ::= + 'ANALYZE' ( 'TABLE' ( TableNameList | TableName ( 'INDEX' IndexNameList | 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? ) ) | 'INCREMENTAL' 'TABLE' TableName ( 'PARTITION' PartitionNameList )? 'INDEX' IndexNameList ) AnalyzeOptionListOpt -![AnalyzeTableStmt](/media/sqlgram/AnalyzeTableStmt.png) +TableNameList ::= + TableName (',' TableName)* -**TableNameList:** - -![TableNameList](/media/sqlgram/TableNameList.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) +TableName ::= + Identifier ( '.' Identifier )? +``` ## Examples diff --git a/sql-statements/sql-statement-backup.md b/sql-statements/sql-statement-backup.md index 374c957a90bf..394553ee97b0 100644 --- a/sql-statements/sql-statement-backup.md +++ b/sql-statements/sql-statement-backup.md @@ -10,13 +10,13 @@ This statement is used to perform a distributed backup of the TiDB cluster. The `BACKUP` statement uses the same engine as the [BR tool](/br/backup-and-restore-use-cases.md) does, except that the backup process is driven by TiDB itself rather than a separate BR tool. All benefits and warnings of BR also apply in this statement. -Executing `BACKUP` requires `SUPER` privilege. Additionally, both the TiDB node executing the backup and all TiKV nodes in the cluster must have read or write permission to the destination. +Executing `BACKUP` requires either the `BACKUP_ADMIN` or `SUPER` privilege. Additionally, both the TiDB node executing the backup and all TiKV nodes in the cluster must have read or write permission to the destination. The `BACKUP` statement is blocked until the entire backup task is finished, failed, or canceled. A long-lasting connection should be prepared for executing `BACKUP`. The task can be canceled using the [`KILL TIDB QUERY`](/sql-statements/sql-statement-kill.md) statement. Only one `BACKUP` and [`RESTORE`](/sql-statements/sql-statement-restore.md) task can be executed at a time. If a `BACKUP` or `RESTORE` statement is already being executed on the same TiDB server, the new `BACKUP` execution will wait until all previous tasks are finished. -`BACKUP` can only be used with "tikv" storage engine. Using `BACKUP` with the "mocktikv" engine will fail. +`BACKUP` can only be used with "tikv" storage engine. Using `BACKUP` with the "unistore" engine will fail. ## Synopsis @@ -98,7 +98,7 @@ BACKUP DATABASE * TO 'local:///mnt/backup/full/'; Note that the system tables (`mysql.*`, `INFORMATION_SCHEMA.*`, `PERFORMANCE_SCHEMA.*`, …) will not be included into the backup. -### Remote destinations +### External storages BR supports backing up data to S3 or GCS: @@ -108,7 +108,7 @@ BR supports backing up data to S3 or GCS: BACKUP DATABASE `test` TO 's3://example-bucket-2020/backup-05/?region=us-west-2&access-key={YOUR_ACCESS_KEY}&secret-access-key={YOUR_SECRET_KEY}'; ``` -The URL syntax is further explained in [BR storages](/br/backup-and-restore-storages.md). +The URL syntax is further explained in [External Storages](/br/backup-and-restore-storages.md). When running on cloud environment where credentials should not be distributed, set the `SEND_CREDENTIALS_TO_TIKV` option to `FALSE`: diff --git a/sql-statements/sql-statement-begin.md b/sql-statements/sql-statement-begin.md index f40aa5baaaeb..6c7fa5f1fb29 100644 --- a/sql-statements/sql-statement-begin.md +++ b/sql-statements/sql-statement-begin.md @@ -12,9 +12,11 @@ In the absence of a `BEGIN` statement, every statement will by default autocommi ## Synopsis -**BeginTransactionStmt:** - -![BeginTransactionStmt](/media/sqlgram/BeginTransactionStmt.png) +```ebnf+diagram +BeginTransactionStmt ::= + 'BEGIN' ( 'PESSIMISTIC' | 'OPTIMISTIC' )? +| 'START' 'TRANSACTION' ( 'READ' ( 'WRITE' | 'ONLY' ( 'WITH' 'TIMESTAMP' 'BOUND' TimestampBound )? ) | 'WITH' 'CONSISTENT' 'SNAPSHOT' )? +``` ## Examples diff --git a/sql-statements/sql-statement-change-column.md b/sql-statements/sql-statement-change-column.md index a6094e961140..2a62cd0d63d3 100644 --- a/sql-statements/sql-statement-change-column.md +++ b/sql-statements/sql-statement-change-column.md @@ -8,60 +8,155 @@ aliases: ['/docs/dev/sql-statements/sql-statement-change-column/','/docs/dev/ref The `ALTER TABLE.. CHANGE COLUMN` statement changes a column on an existing table. The change can include both renaming the column, and changing the data type to a compatible type. -## Synopsis - -**AlterTableStmt:** - -![AlterTableStmt](/media/sqlgram/AlterTableStmt.png) - -**AlterTableSpec:** +Since v5.1.0, TiDB has supported changing the Reorg data type, including but not limited to: -![AlterTableSpec](/media/sqlgram/AlterTableSpec.png) +- Changing `VARCHAR` to `BIGINT` +- Modifying the `DECIMAL` precision +- Compressing the length of `VARCHAR(10)` to `VARCHAR(5)` -**ColumnName:** +## Synopsis -![ColumnName](/media/sqlgram/ColumnName.png) +```ebnf+diagram +AlterTableStmt ::= + 'ALTER' IgnoreOptional 'TABLE' TableName ( AlterTableSpecListOpt AlterTablePartitionOpt | 'ANALYZE' 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? AnalyzeOptionListOpt ) + +AlterTableSpec ::= + TableOptionList +| 'SET' 'TIFLASH' 'REPLICA' LengthNum LocationLabelList +| 'CONVERT' 'TO' CharsetKw ( CharsetName | 'DEFAULT' ) OptCollate +| 'ADD' ( ColumnKeywordOpt IfNotExists ( ColumnDef ColumnPosition | '(' TableElementList ')' ) | Constraint | 'PARTITION' IfNotExists NoWriteToBinLogAliasOpt ( PartitionDefinitionListOpt | 'PARTITIONS' NUM ) ) +| ( ( 'CHECK' | 'TRUNCATE' ) 'PARTITION' | ( 'OPTIMIZE' | 'REPAIR' | 'REBUILD' ) 'PARTITION' NoWriteToBinLogAliasOpt ) AllOrPartitionNameList +| 'COALESCE' 'PARTITION' NoWriteToBinLogAliasOpt NUM +| 'DROP' ( ColumnKeywordOpt IfExists ColumnName RestrictOrCascadeOpt | 'PRIMARY' 'KEY' | 'PARTITION' IfExists PartitionNameList | ( KeyOrIndex IfExists | 'CHECK' ) Identifier | 'FOREIGN' 'KEY' IfExists Symbol ) +| 'EXCHANGE' 'PARTITION' Identifier 'WITH' 'TABLE' TableName WithValidationOpt +| ( 'IMPORT' | 'DISCARD' ) ( 'PARTITION' AllOrPartitionNameList )? 'TABLESPACE' +| 'REORGANIZE' 'PARTITION' NoWriteToBinLogAliasOpt ReorganizePartitionRuleOpt +| 'ORDER' 'BY' AlterOrderItem ( ',' AlterOrderItem )* +| ( 'DISABLE' | 'ENABLE' ) 'KEYS' +| ( 'MODIFY' ColumnKeywordOpt IfExists | 'CHANGE' ColumnKeywordOpt IfExists ColumnName ) ColumnDef ColumnPosition +| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier IndexInvisible ) +| 'RENAME' ( ( 'COLUMN' | KeyOrIndex ) Identifier 'TO' Identifier | ( 'TO' | '='? | 'AS' ) TableName ) +| LockClause +| AlgorithmClause +| 'FORCE' +| ( 'WITH' | 'WITHOUT' ) 'VALIDATION' +| 'SECONDARY_LOAD' +| 'SECONDARY_UNLOAD' + +ColumnName ::= + Identifier ( '.' Identifier ( '.' Identifier )? )? + +ColumnDef ::= + ColumnName ( Type | 'SERIAL' ) ColumnOptionListOpt + +ColumnPosition ::= + ( 'FIRST' | 'AFTER' ColumnName )? +``` -**ColumnDef:** +## Examples -![ColumnDef](/media/sqlgram/ColumnDef.png) +{{< copyable "sql" >}} -**ColumnPosition:** +```sql +CREATE TABLE t1 (id int not null primary key AUTO_INCREMENT, col1 INT); +``` -![ColumnPosition](/media/sqlgram/ColumnPosition.png) +``` +Query OK, 0 rows affected (0.11 sec) +``` -## Examples +{{< copyable "sql" >}} ```sql -mysql> CREATE TABLE t1 (id int not null primary key AUTO_INCREMENT, col1 INT); -Query OK, 0 rows affected (0.11 sec) +INSERT INTO t1 (col1) VALUES (1),(2),(3),(4),(5); +``` -mysql> INSERT INTO t1 (col1) VALUES (1),(2),(3),(4),(5); +``` Query OK, 5 rows affected (0.02 sec) Records: 5 Duplicates: 0 Warnings: 0 +``` + +{{< copyable "sql" >}} + +```sql +ALTER TABLE t1 CHANGE col1 col2 INT; +``` -mysql> -mysql> ALTER TABLE t1 CHANGE col1 col2 INT; +``` Query OK, 0 rows affected (0.09 sec) +``` + +{{< copyable "sql" >}} -mysql> ALTER TABLE t1 CHANGE col2 col3 BIGINT, ALGORITHM=INSTANT; +```sql +ALTER TABLE t1 CHANGE col2 col3 BIGINT, ALGORITHM=INSTANT; +``` + +``` Query OK, 0 rows affected (0.08 sec) +``` + +{{< copyable "sql" >}} + +```sql +ALTER TABLE t1 CHANGE col3 col4 BIGINT, CHANGE id id2 INT NOT NULL; +``` -mysql> -mysql> ALTER TABLE t1 CHANGE col3 col3 INT; -ERROR 1105 (HY000): unsupported modify column length 11 is less than origin 20 -mysql> ALTER TABLE t1 CHANGE col3 col3 BLOB; -ERROR 1105 (HY000): unsupported modify column type 252 not match origin 8 -mysql> ALTER TABLE t1 CHANGE col3 col4 BIGINT, CHANGE id id2 INT NOT NULL; +``` ERROR 1105 (HY000): can't run multi schema change ``` +{{< copyable "sql" >}} + +```sql +CREATE TABLE t (a int primary key); +ALTER TABLE t CHANGE COLUMN a a VARCHAR(10); +``` + +``` +ERROR 8200 (HY000): Unsupported modify column: column has primary key flag +``` + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t (c1 INT, c2 INT, c3 INT) partition by range columns(c1) ( partition p0 values less than (10), partition p1 values less than (maxvalue)); +ALTER TABLE t CHANGE COLUMN c1 c1 DATETIME; +``` + +``` +ERROR 8200 (HY000): Unsupported modify column: table is partition table +``` + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t (a INT, b INT as (a+1)); +ALTER TABLE t CHANGE COLUMN b b VARCHAR(10); +``` + +``` +ERROR 8200 (HY000): Unsupported modify column: column is generated +``` + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t (a DECIMAL(13, 7)); +ALTER TABLE t CHANGE COLUMN a a DATETIME; +``` + +``` +ERROR 8200 (HY000): Unsupported modify column: change from original type decimal(13,7) to datetime is currently unsupported yet +``` + ## MySQL compatibility -* Making multiple changes in a single `ALTER TABLE` statement is not currently supported. -* Only certain types of data type changes are supported. For example, an `INTEGER` to `BIGINT` is supported, but the reverse is not possible. Changing from an integer to a string format or blob is not supported. -* Modifying precision of the `DECIMAL` type is not supported. -* Changing the `UNSIGNED` attribute is not supported. +* Making multiple changes in a single `ALTER TABLE` statement is currently not supported. +* Changes of [Reorg-Data](/sql-statements/sql-statement-modify-column.md#reorg-data-change) types on primary key columns are not supported. +* Changes of column types on partitioned tables are not supported. +* Changes of column types on generated columns are not supported. +* Changes of some data types (for example, some TIME, Bit, Set, Enum, and JSON types) are not supported due to the compatibility issues of the `CAST` function's behavior between TiDB and MySQL. ## See also diff --git a/sql-statements/sql-statement-commit.md b/sql-statements/sql-statement-commit.md index f9541ec82c3f..85d47841a5fc 100644 --- a/sql-statements/sql-statement-commit.md +++ b/sql-statements/sql-statement-commit.md @@ -12,13 +12,14 @@ In the absence of a `BEGIN` or `START TRANSACTION` statement, the default behavi ## Synopsis -**CommitStmt:** +```ebnf+diagram +CommitStmt ::= + 'COMMIT' CompletionTypeWithinTransaction? -![CommitStmt](/media/sqlgram/CommitStmt.png) - -**CompletionTypeWithinTransaction:** - -![CompletionTypeWithinTransaction](/media/sqlgram/CompletionTypeWithinTransaction.png) +CompletionTypeWithinTransaction ::= + 'AND' ( 'CHAIN' ( 'NO' 'RELEASE' )? | 'NO' 'CHAIN' ( 'NO'? 'RELEASE' )? ) +| 'NO'? 'RELEASE' +``` ## Examples diff --git a/sql-statements/sql-statement-create-binding.md b/sql-statements/sql-statement-create-binding.md index ceb896552c66..c265daea2b59 100644 --- a/sql-statements/sql-statement-create-binding.md +++ b/sql-statements/sql-statement-create-binding.md @@ -14,17 +14,16 @@ The bound SQL statement is parameterized and stored in the system table. When a ## Synopsis -**CreateBindingStmt:** +```ebnf+diagram +CreateBindingStmt ::= + 'CREATE' GlobalScope 'BINDING' 'FOR' BindableStmt 'USING' BindableStmt -![CreateBindingStmt](/media/sqlgram/CreateBindingStmt.png) +GlobalScope ::= + ( 'GLOBAL' | 'SESSION' )? -**GlobalScope:** - -![GlobalScope](/media/sqlgram/GlobalScope.png) - -**SelectStmt** - -![SelectStmt](/media/sqlgram/SelectStmt.png) +BindableStmt ::= + ( SelectStmt | UpdateStmt | InsertIntoStmt | ReplaceIntoStmt | DeleteStmt ) +``` **** @@ -142,4 +141,4 @@ This statement is a TiDB extension to MySQL syntax. * [SHOW [GLOBAL|SESSION] BINDINGS](/sql-statements/sql-statement-show-bindings.md) * [ANALYZE TABLE](/sql-statements/sql-statement-analyze-table.md) * [Optimizer Hints](/optimizer-hints.md) -* [SQL Plan Management](/sql-plan-management.md) \ No newline at end of file +* [SQL Plan Management](/sql-plan-management.md) diff --git a/sql-statements/sql-statement-create-database.md b/sql-statements/sql-statement-create-database.md index e8605f08605e..06acd08633d9 100644 --- a/sql-statements/sql-statement-create-database.md +++ b/sql-statements/sql-statement-create-database.md @@ -10,21 +10,19 @@ This statement creates a new database in TiDB. The MySQL terminology for 'databa ## Synopsis -**CreateDatabaseStmt:** +```ebnf+diagram +CreateDatabaseStmt ::= + 'CREATE' 'DATABASE' IfNotExists DBName DatabaseOptionListOpt -![CreateDatabaseStmt](/media/sqlgram/CreateDatabaseStmt.png) +IfNotExists ::= + ( 'IF' 'NOT' 'EXISTS' )? -**IfNotExists:** +DBName ::= + Identifier -![IfNotExists](/media/sqlgram/IfNotExists.png) - -**DBName:** - -![DBName](/media/sqlgram/DBName.png) - -**DatabaseOptionListOpt:** - -![DatabaseOptionListOpt](/media/sqlgram/DatabaseOptionListOpt.png) +DatabaseOptionListOpt ::= + DatabaseOptionList? +``` ## Syntax diff --git a/sql-statements/sql-statement-create-index.md b/sql-statements/sql-statement-create-index.md index b54b628cde16..f59aba3a17dd 100644 --- a/sql-statements/sql-statement-create-index.md +++ b/sql-statements/sql-statement-create-index.md @@ -10,65 +10,58 @@ This statement adds a new index to an existing table. It is an alternative synta ## Synopsis -**CreateIndexStmt:** +```ebnf+diagram +CreateIndexStmt ::= + 'CREATE' IndexKeyTypeOpt 'INDEX' IfNotExists Identifier IndexTypeOpt 'ON' TableName '(' IndexPartSpecificationList ')' IndexOptionList IndexLockAndAlgorithmOpt -![CreateIndexStmt](/media/sqlgram/CreateIndexStmt.png) +IndexKeyTypeOpt ::= + ( 'UNIQUE' | 'SPATIAL' | 'FULLTEXT' )? -**IndexKeyTypeOpt:** +IfNotExists ::= + ( 'IF' 'NOT' 'EXISTS' )? -![IndexKeyTypeOpt](/media/sqlgram/IndexKeyTypeOpt.png) +IndexTypeOpt ::= + IndexType? -**IfNotExists:** +IndexPartSpecificationList ::= + IndexPartSpecification ( ',' IndexPartSpecification )* -![IfNotExists](/media/sqlgram/IfNotExists.png) +IndexOptionList ::= + IndexOption* -**IndexTypeOpt:** +IndexLockAndAlgorithmOpt ::= + ( LockClause AlgorithmClause? | AlgorithmClause LockClause? )? -![IndexTypeOpt](/media/sqlgram/IndexTypeOpt.png) +IndexType ::= + ( 'USING' | 'TYPE' ) IndexTypeName -**IndexPartSpecificationList:** +IndexPartSpecification ::= + ( ColumnName OptFieldLen | '(' Expression ')' ) Order -![IndexPartSpecificationList](/media/sqlgram/IndexPartSpecificationList.png) +IndexOption ::= + 'KEY_BLOCK_SIZE' '='? LengthNum +| IndexType +| 'WITH' 'PARSER' Identifier +| 'COMMENT' stringLit +| IndexInvisible -**IndexOptionList:** +IndexTypeName ::= + 'BTREE' +| 'HASH' +| 'RTREE' -![IndexOptionList](/media/sqlgram/IndexOptionList.png) +ColumnName ::= + Identifier ( '.' Identifier ( '.' Identifier )? )? -**IndexLockAndAlgorithmOpt:** +OptFieldLen ::= + FieldLen? -![IndexLockAndAlgorithmOpt](/media/sqlgram/IndexLockAndAlgorithmOpt.png) +IndexNameList ::= + ( Identifier | 'PRIMARY' )? ( ',' ( Identifier | 'PRIMARY' ) )* -**IndexType:** - -![IndexType](/media/sqlgram/IndexType.png) - -**IndexPartSpecification:** - -![IndexPartSpecification](/media/sqlgram/IndexPartSpecification.png) - -**IndexOption:** - -![IndexOption](/media/sqlgram/IndexOption.png) - -**IndexTypeName:** - -![IndexTypeName](/media/sqlgram/IndexTypeName.png) - -**ColumnName:** - -![ColumnName](/media/sqlgram/ColumnName.png) - -**OptFieldLen:** - -![OptFieldLen](/media/sqlgram/OptFieldLen.png) - -**IndexNameList:** - -![IndexNameList](/media/sqlgram/IndexNameList.png) - -**KeyOrIndex:** - -![KeyOrIndex](/media/sqlgram/KeyOrIndex.png) +KeyOrIndex ::= + 'Key' | 'Index' +``` ## Examples @@ -168,7 +161,7 @@ The global variables associated with the `CREATE INDEX` statement are `tidb_ddl_ * `FULLTEXT`, `HASH` and `SPATIAL` indexes are not supported. * Descending indexes are not supported (similar to MySQL 5.7). -* Adding the primary key constraint to a table is not supported by default. You can enable the feature by setting the `alter-primary-key` configuration item to `true`. For details, see [alter-primary-key](/tidb-configuration-file.md#alter-primary-key). +* Adding the primary key of the `CLUSTERED` type to a table is not supported. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). ## See also diff --git a/sql-statements/sql-statement-create-role.md b/sql-statements/sql-statement-create-role.md index 0bcae965ddcf..4f8f5899fefd 100644 --- a/sql-statements/sql-statement-create-role.md +++ b/sql-statements/sql-statement-create-role.md @@ -9,17 +9,16 @@ This statement creates a new role, which can be assigned to users as part of rol ## Synopsis -**CreateRoleStmt:** +```ebnf+diagram +CreateRoleStmt ::= + 'CREATE' 'ROLE' IfNotExists RoleSpec (',' RoleSpec)* -![CreateRoleStmt](/media/sqlgram/CreateRoleStmt.png) +IfNotExists ::= + ('IF' 'NOT' 'EXISTS')? -**IfNotExists:** - -![IfNotExists](/media/sqlgram/IfNotExists.png) - -**RoleSpec:** - -![RoleSpec](/media/sqlgram/RoleSpec.png) +RoleSpec ::= + Rolename +``` ## Examples diff --git a/sql-statements/sql-statement-create-sequence.md b/sql-statements/sql-statement-create-sequence.md index 99cd667416fc..5d9e08794271 100644 --- a/sql-statements/sql-statement-create-sequence.md +++ b/sql-statements/sql-statement-create-sequence.md @@ -10,29 +10,31 @@ The `CREATE SEQUENCE` statement creates sequence objects in TiDB. The sequence i ## Synopsis -**CreateSequenceStmt:** - -![CreateSequenceStmt](/media/sqlgram/CreateSequenceStmt.png) - -**IfNotExists:** - -![IfNotExists](/media/sqlgram/IfNotExists.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) - -**CreateSequenceOptionListOpt:** - -![CreateSequenceOptionListOpt](/media/sqlgram/CreateSequenceOptionListOpt.png) - -**SequenceOptionList:** - -![SequenceOptionList](/media/sqlgram/SequenceOptionList.png) - -**SequenceOption:** - -![SequenceOption](/media/sqlgram/SequenceOption.png) +```ebnf+diagram +CreateSequenceStmt ::= + 'CREATE' 'SEQUENCE' IfNotExists TableName CreateSequenceOptionListOpt CreateTableOptionListOpt + +IfNotExists ::= + ('IF' 'NOT' 'EXISTS')? + +TableName ::= + Identifier ('.' Identifier)? + +CreateSequenceOptionListOpt ::= + SequenceOption* + +SequenceOptionList ::= + SequenceOption + +SequenceOption ::= + ( 'INCREMENT' ( '='? | 'BY' ) | 'START' ( '='? | 'WITH' ) | ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' ) '='? ) SignedNum +| 'NOMINVALUE' +| 'NO' ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' | 'CYCLE' ) +| 'NOMAXVALUE' +| 'NOCACHE' +| 'CYCLE' +| 'NOCYCLE' +``` ## Syntax diff --git a/sql-statements/sql-statement-create-table-like.md b/sql-statements/sql-statement-create-table-like.md index afad9c46ecb5..2d5f65c5bd71 100644 --- a/sql-statements/sql-statement-create-table-like.md +++ b/sql-statements/sql-statement-create-table-like.md @@ -10,13 +10,14 @@ This statement copies the definition of an existing table, without copying any d ## Synopsis -**CreateTableStmt:** +```ebnf+diagram +CreateTableLikeStmt ::= + 'CREATE' OptTemporary 'TABLE' IfNotExists TableName LikeTableWithOrWithoutParen -![CreateTableLikeStmt](/media/sqlgram/CreateTableLikeStmt.png) - -**LikeTableWithOrWithoutParen:** - -![LikeTableWithOrWithoutParen](/media/sqlgram/LikeTableWithOrWithoutParen.png) +LikeTableWithOrWithoutParen ::= + 'LIKE' TableName +| '(' 'LIKE' TableName ')' +``` ## Examples diff --git a/sql-statements/sql-statement-create-table.md b/sql-statements/sql-statement-create-table.md index 1c394b7c3be1..55add15b8416 100644 --- a/sql-statements/sql-statement-create-table.md +++ b/sql-statements/sql-statement-create-table.md @@ -10,65 +10,77 @@ This statement creates a new table in the currently selected database. It behave ## Synopsis -**CreateTableStmt:** - -![CreateTableStmt](/media/sqlgram/CreateTableStmt.png) - -**IfNotExists:** - -![IfNotExists](/media/sqlgram/IfNotExists.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) - -**TableElementListOpt:** - -![TableElementListOpt](/media/sqlgram/TableElementListOpt.png) - -**TableElementList:** - -![TableElementList](/media/sqlgram/TableElementList.png) - -**TableElement:** - -![TableElement](/media/sqlgram/TableElement.png) - -**ColumnDef:** - -![ColumnDef](/media/sqlgram/ColumnDef.png) - -**ColumnOptionListOpt:** - -![ColumnOptionListOpt](/media/sqlgram/ColumnOptionListOpt.png) - -**ColumnOptionList:** - -![ColumnOptionList](/media/sqlgram/ColumnOptionList.png) - -**ColumnOption:** - -![ColumnOption](/media/sqlgram/ColumnOption.png) - -**CreateTableOptionListOpt:** - -![CreateTableOptionListOpt](/media/sqlgram/CreateTableOptionListOpt.png) - -**PartitionOpt:** - -![PartitionOpt](/media/sqlgram/PartitionOpt.png) - -**DuplicateOpt:** - -![DuplicateOpt](/media/sqlgram/DuplicateOpt.png) - -**TableOptionList:** - -![TableOptionList](/media/sqlgram/TableOptionList.png) - -**TableOption:** - -![TableOption](/media/sqlgram/TableOption.png) +```ebnf+diagram +CreateTableStmt ::= + 'CREATE' OptTemporary 'TABLE' IfNotExists TableName ( TableElementListOpt CreateTableOptionListOpt PartitionOpt DuplicateOpt AsOpt CreateTableSelectOpt | LikeTableWithOrWithoutParen ) + +IfNotExists ::= + ('IF' 'NOT' 'EXISTS')? + +TableName ::= + Identifier ('.' Identifier)? + +TableElementListOpt ::= + ( '(' TableElementList ')' )? + +TableElementList ::= + TableElement ( ',' TableElement )* + +TableElement ::= + ColumnDef +| Constraint + +ColumnDef ::= + ColumnName ( Type | 'SERIAL' ) ColumnOptionListOpt + +ColumnOptionListOpt ::= + ColumnOption* + +ColumnOptionList ::= + ColumnOption* + +ColumnOption ::= + 'NOT'? 'NULL' +| 'AUTO_INCREMENT' +| PrimaryOpt 'KEY' +| 'UNIQUE' 'KEY'? +| 'DEFAULT' DefaultValueExpr +| 'SERIAL' 'DEFAULT' 'VALUE' +| 'ON' 'UPDATE' NowSymOptionFraction +| 'COMMENT' stringLit +| ConstraintKeywordOpt 'CHECK' '(' Expression ')' EnforcedOrNotOrNotNullOpt +| GeneratedAlways 'AS' '(' Expression ')' VirtualOrStored +| ReferDef +| 'COLLATE' CollationName +| 'COLUMN_FORMAT' ColumnFormat +| 'STORAGE' StorageMedia +| 'AUTO_RANDOM' OptFieldLen + +CreateTableOptionListOpt ::= + TableOptionList? + +PartitionOpt ::= + ( 'PARTITION' 'BY' PartitionMethod PartitionNumOpt SubPartitionOpt PartitionDefinitionListOpt )? + +DuplicateOpt ::= + ( 'IGNORE' | 'REPLACE' )? + +TableOptionList ::= + TableOption ( ','? TableOption )* + +TableOption ::= + PartDefOption +| DefaultKwdOpt ( CharsetKw EqOpt CharsetName | 'COLLATE' EqOpt CollationName ) +| ( 'AUTO_INCREMENT' | 'AUTO_ID_CACHE' | 'AUTO_RANDOM_BASE' | 'AVG_ROW_LENGTH' | 'CHECKSUM' | 'TABLE_CHECKSUM' | 'KEY_BLOCK_SIZE' | 'DELAY_KEY_WRITE' | 'SHARD_ROW_ID_BITS' | 'PRE_SPLIT_REGIONS' ) EqOpt LengthNum +| ( 'CONNECTION' | 'PASSWORD' | 'COMPRESSION' ) EqOpt stringLit +| RowFormat +| ( 'STATS_PERSISTENT' | 'PACK_KEYS' ) EqOpt StatsPersistentVal +| ( 'STATS_AUTO_RECALC' | 'STATS_SAMPLE_PAGES' ) EqOpt ( LengthNum | 'DEFAULT' ) +| 'STORAGE' ( 'MEMORY' | 'DISK' ) +| 'SECONDARY_ENGINE' EqOpt ( 'NULL' | StringName ) +| 'UNION' EqOpt '(' TableNameListOpt ')' +| 'ENCRYPTION' EqOpt EncryptionOpt +``` The following *table_options* are supported. Other options such as `AVG_ROW_LENGTH`, `CHECKSUM`, `COMPRESSION`, `CONNECTION`, `DELAY_KEY_WRITE`, `ENGINE`, `KEY_BLOCK_SIZE`, `MAX_ROWS`, `MIN_ROWS`, `ROW_FORMAT` and `STATS_PERSISTENT` are parsed but ignored. @@ -170,7 +182,7 @@ mysql> DESC t1; ## MySQL compatibility -* TiDB does not support temporary tables, but it ignores the `CREATE TEMPORARY TABLE` syntax. +* TiDB does not support temporary tables. The `CREATE TEMPORARY TABLE` syntax will raise an error if [`tidb_enable_noop_functions = 0`](/system-variables.md#tidb_enable_noop_functions-new-in-v40); TiDB simply ignores the `TEMPORARY` keyword if `tidb_enable_noop_functions = 1`. * All of the data types except spatial types are supported. * `FULLTEXT`, `HASH` and `SPATIAL` indexes are not supported. * For compatibility, the `index_col_name` attribute supports the length option with a maximum length limit of 3072 bytes by default. The length limit can be changed through the `max-index-length` configuration option. For details, see [TiDB configuration file](/tidb-configuration-file.md#max-index-length). diff --git a/sql-statements/sql-statement-create-user.md b/sql-statements/sql-statement-create-user.md index 5b751b015bc8..465177d6dc56 100644 --- a/sql-statements/sql-statement-create-user.md +++ b/sql-statements/sql-statement-create-user.md @@ -10,29 +10,26 @@ This statement creates a new user, specified with a password. In the MySQL privi ## Synopsis -**CreateUserStmt:** +```ebnf+diagram +CreateUserStmt ::= + 'CREATE' 'USER' IfNotExists UserSpecList RequireClauseOpt ConnectionOptions PasswordOrLockOptions -![CreateUserStmt](/media/sqlgram/CreateUserStmt.png) +IfNotExists ::= + ('IF' 'NOT' 'EXISTS')? -**IfNotExists:** +UserSpecList ::= + UserSpec ( ',' UserSpec )* -![IfNotExists](/media/sqlgram/IfNotExists.png) +UserSpec ::= + Username AuthOption -**UserSpecList:** +AuthOption ::= + ( 'IDENTIFIED' ( 'BY' ( AuthString | 'PASSWORD' HashString ) | 'WITH' StringName ( 'BY' AuthString | 'AS' HashString )? ) )? -![UserSpecList](/media/sqlgram/UserSpecList.png) - -**UserSpec:** - -![UserSpec](/media/sqlgram/UserSpec.png) - -**AuthOption:** - -![AuthOption](/media/sqlgram/AuthOption.png) - -**StringName:** - -![StringName](/media/sqlgram/StringName.png) +StringName ::= + stringLit +| Identifier +``` ## Examples @@ -55,7 +52,7 @@ Create a user who is enforced to log in using TLS connection. ```sql CREATE USER 'newuser3'@'%' REQUIRE SSL IDENTIFIED BY 'newuserpassword'; Query OK, 1 row affected (0.02 sec) -``` +``` Create a user who is required to use X.509 certificate at login. diff --git a/sql-statements/sql-statement-create-view.md b/sql-statements/sql-statement-create-view.md index 2db28bc7a539..6a4b907678d4 100644 --- a/sql-statements/sql-statement-create-view.md +++ b/sql-statements/sql-statement-create-view.md @@ -10,37 +10,30 @@ The `CREATE VIEW` statement saves a `SELECT` statement as a queryable object, si ## Synopsis -**CreateViewStmt:** +```ebnf+diagram +CreateViewStmt ::= + 'CREATE' OrReplace ViewAlgorithm ViewDefiner ViewSQLSecurity 'VIEW' ViewName ViewFieldList 'AS' CreateViewSelectOpt ViewCheckOption -![CreateViewStmt](/media/sqlgram/CreateViewStmt.png) +OrReplace ::= + ( 'OR' 'REPLACE' )? -**OrReplace:** +ViewAlgorithm ::= + ( 'ALGORITHM' '=' ( 'UNDEFINED' | 'MERGE' | 'TEMPTABLE' ) )? -![OrReplace](/media/sqlgram/OrReplace.png) +ViewDefiner ::= + ( 'DEFINER' '=' Username )? -**ViewAlgorithm:** +ViewSQLSecurity ::= + ( 'SQL' 'SECURITY' ( 'DEFINER' | 'INVOKER' ) )? -![ViewAlgorithm](/media/sqlgram/ViewAlgorithm.png) +ViewName ::= TableName -**ViewDefiner:** +ViewFieldList ::= + ( '(' Identifier ( ',' Identifier )* ')' )? -![ViewDefiner](/media/sqlgram/ViewDefiner.png) - -**ViewSQLSecurity:** - -![ViewSQLSecurity](/media/sqlgram/ViewSQLSecurity.png) - -**ViewName:** - -![ViewName](/media/sqlgram/ViewName.png) - -**ViewFieldList:** - -![ViewFieldList](/media/sqlgram/ViewFieldList.png) - -**ViewCheckOption:** - -![ViewCheckOption](/media/sqlgram/ViewCheckOption.png) +ViewCheckOption ::= + ( 'WITH' ( 'CASCADED' | 'LOCAL' ) 'CHECK' 'OPTION' )? +``` ## Examples diff --git a/sql-statements/sql-statement-deallocate.md b/sql-statements/sql-statement-deallocate.md index 1de3d602d2fc..7681a85bb1b3 100644 --- a/sql-statements/sql-statement-deallocate.md +++ b/sql-statements/sql-statement-deallocate.md @@ -10,17 +10,20 @@ The `DEALLOCATE` statement provides an SQL interface to server-side prepared sta ## Synopsis -**DeallocateStmt:** - -![DeallocateStmt](/media/sqlgram/DeallocateStmt.png) - -**DeallocateSym:** - -![DeallocateSym](/media/sqlgram/DeallocateSym.png) - -**Identifier:** - -![Identifier](/media/sqlgram/Identifier.png) +```ebnf+diagram +DeallocateStmt ::= + DeallocateSym 'PREPARE' Identifier + +DeallocateSym ::= + 'DEALLOCATE' +| 'DROP' + +Identifier ::= + identifier +| UnReservedKeyword +| NotKeywordToken +| TiDBKeyword +``` ## Examples diff --git a/sql-statements/sql-statement-delete.md b/sql-statements/sql-statement-delete.md index 640751e3ae43..3fc54b1779f4 100644 --- a/sql-statements/sql-statement-delete.md +++ b/sql-statements/sql-statement-delete.md @@ -10,9 +10,10 @@ The `DELETE` statement removes rows from a specified table. ## Synopsis -**DeleteFromStmt:** - -![DeleteFromStmt](/media/sqlgram/DeleteFromStmt.png) +```ebnf+diagram +DeleteFromStmt ::= + 'DELETE' TableOptimizerHints PriorityOpt QuickOptional IgnoreOptional ( 'FROM' ( TableName TableAsNameOpt IndexHintListOpt WhereClauseOptional OrderByOptional LimitClause | TableAliasRefList 'USING' TableRefs WhereClauseOptional ) | TableAliasRefList 'FROM' TableRefs WhereClauseOptional ) +``` ## Examples diff --git a/sql-statements/sql-statement-do.md b/sql-statements/sql-statement-do.md index 785eafb01b37..6babd12a9474 100644 --- a/sql-statements/sql-statement-do.md +++ b/sql-statements/sql-statement-do.md @@ -10,23 +10,23 @@ aliases: ['/docs/dev/sql-statements/sql-statement-do/','/docs/dev/reference/sql/ > **Note:** > -> `DO` only executes expressions. It cannot be used in all cases where `SELECT` can be used. For example, `DO id FROM t1` is invalid because it references a table. +> `DO` only executes expressions. It cannot be used in all cases where `SELECT` can be used. For example, `DO id FROM t1` is invalid because it references a table. In MySQL, a common use case is to execute stored procedure or trigger. Since TiDB does not provide stored procedure or trigger, this function has a limited use. ## Synopsis -**DoStmt:** +```ebnf+diagram +DoStmt ::= 'DO' ExpressionList -![DoStmt](/media/sqlgram/DoStmt.png) +ExpressionList ::= + Expression ( ',' Expression )* -**ExpressionList:** - -![ExpressionList](/media/sqlgram/ExpressionList.png) - -**Expression:** - -![Expression](/media/sqlgram/Expression.png) +Expression ::= + ( singleAtIdentifier assignmentEq | 'NOT' | Expression ( logOr | 'XOR' | logAnd ) ) Expression +| 'MATCH' '(' ColumnNameList ')' 'AGAINST' '(' BitExpr FulltextSearchModifierOpt ')' +| PredicateExpr ( IsOrNotOp 'NULL' | CompareOp ( ( singleAtIdentifier assignmentEq )? PredicateExpr | AnyOrAll SubSelect ) )* ( IsOrNotOp ( trueKwd | falseKwd | 'UNKNOWN' ) )? +``` ## Examples diff --git a/sql-statements/sql-statement-drop-binding.md b/sql-statements/sql-statement-drop-binding.md index 84aa83a30610..22e031cb7a0d 100644 --- a/sql-statements/sql-statement-drop-binding.md +++ b/sql-statements/sql-statement-drop-binding.md @@ -12,17 +12,16 @@ A `BINDING` can be on either a `GLOBAL` or `SESSION` basis. The default is `SESS ## Synopsis -**DropBindingStmt:** +```ebnf+diagram +DropBindingStmt ::= + 'DROP' GlobalScope 'BINDING' 'FOR' BindableStmt ( 'USING' BindableStmt )? -![DropBindingStmt](/media/sqlgram/DropBindingStmt.png) +GlobalScope ::= + ( 'GLOBAL' | 'SESSION' )? -**GlobalScope:** - -![GlobalScope](/media/sqlgram/GlobalScope.png) - -**SelectStmt** - -![SelectStmt](/media/sqlgram/SelectStmt.png) +BindableStmt ::= + ( SelectStmt | UpdateStmt | InsertIntoStmt | ReplaceIntoStmt | DeleteStmt ) +``` ## Syntax description @@ -141,4 +140,4 @@ This statement is a TiDB extension to MySQL syntax. * [SHOW [GLOBAL|SESSION] BINDINGS](/sql-statements/sql-statement-show-bindings.md) * [ANALYZE TABLE](/sql-statements/sql-statement-analyze-table.md) * [Optimizer Hints](/optimizer-hints.md) -* [SQL Plan Management](/sql-plan-management.md) \ No newline at end of file +* [SQL Plan Management](/sql-plan-management.md) diff --git a/sql-statements/sql-statement-drop-column.md b/sql-statements/sql-statement-drop-column.md index cf330979e17d..a6e5dcbe1ce1 100644 --- a/sql-statements/sql-statement-drop-column.md +++ b/sql-statements/sql-statement-drop-column.md @@ -10,17 +10,36 @@ This statement drops a column from a specified table. `DROP COLUMN` is online in ## Synopsis -**AlterTableStmt:** - -![AlterTableStmt](/media/sqlgram/AlterTableStmt.png) - -**AlterTableSpec:** - -![AlterTableSpec](/media/sqlgram/AlterTableSpec.png) - -**ColumnName:** - -![ColumnName](/media/sqlgram/ColumnName.png) +```ebnf+diagram +AlterTableStmt ::= + 'ALTER' IgnoreOptional 'TABLE' TableName ( AlterTableSpecListOpt AlterTablePartitionOpt | 'ANALYZE' 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? AnalyzeOptionListOpt ) + +AlterTableSpec ::= + TableOptionList +| 'SET' 'TIFLASH' 'REPLICA' LengthNum LocationLabelList +| 'CONVERT' 'TO' CharsetKw ( CharsetName | 'DEFAULT' ) OptCollate +| 'ADD' ( ColumnKeywordOpt IfNotExists ( ColumnDef ColumnPosition | '(' TableElementList ')' ) | Constraint | 'PARTITION' IfNotExists NoWriteToBinLogAliasOpt ( PartitionDefinitionListOpt | 'PARTITIONS' NUM ) ) +| ( ( 'CHECK' | 'TRUNCATE' ) 'PARTITION' | ( 'OPTIMIZE' | 'REPAIR' | 'REBUILD' ) 'PARTITION' NoWriteToBinLogAliasOpt ) AllOrPartitionNameList +| 'COALESCE' 'PARTITION' NoWriteToBinLogAliasOpt NUM +| 'DROP' ( ColumnKeywordOpt IfExists ColumnName RestrictOrCascadeOpt | 'PRIMARY' 'KEY' | 'PARTITION' IfExists PartitionNameList | ( KeyOrIndex IfExists | 'CHECK' ) Identifier | 'FOREIGN' 'KEY' IfExists Symbol ) +| 'EXCHANGE' 'PARTITION' Identifier 'WITH' 'TABLE' TableName WithValidationOpt +| ( 'IMPORT' | 'DISCARD' ) ( 'PARTITION' AllOrPartitionNameList )? 'TABLESPACE' +| 'REORGANIZE' 'PARTITION' NoWriteToBinLogAliasOpt ReorganizePartitionRuleOpt +| 'ORDER' 'BY' AlterOrderItem ( ',' AlterOrderItem )* +| ( 'DISABLE' | 'ENABLE' ) 'KEYS' +| ( 'MODIFY' ColumnKeywordOpt IfExists | 'CHANGE' ColumnKeywordOpt IfExists ColumnName ) ColumnDef ColumnPosition +| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier IndexInvisible ) +| 'RENAME' ( ( 'COLUMN' | KeyOrIndex ) Identifier 'TO' Identifier | ( 'TO' | '='? | 'AS' ) TableName ) +| LockClause +| AlgorithmClause +| 'FORCE' +| ( 'WITH' | 'WITHOUT' ) 'VALIDATION' +| 'SECONDARY_LOAD' +| 'SECONDARY_UNLOAD' + +ColumnName ::= + Identifier ( '.' Identifier ( '.' Identifier )? )? +``` ## Examples diff --git a/sql-statements/sql-statement-drop-database.md b/sql-statements/sql-statement-drop-database.md index 70919dc5d115..b8a822f3ea39 100644 --- a/sql-statements/sql-statement-drop-database.md +++ b/sql-statements/sql-statement-drop-database.md @@ -10,13 +10,12 @@ The `DROP DATABASE` statement permanently removes a specified database schema, a ## Synopsis -**DropDatabaseStmt:** +```ebnf+diagram +DropDatabaseStmt ::= + 'DROP' 'DATABASE' IfExists DBName -![DropDatabaseStmt](/media/sqlgram/DropDatabaseStmt.png) - -**IfExists:** - -![IfExists](/media/sqlgram/IfExists.png) +IfExists ::= ( 'IF' 'EXISTS' )? +``` ## Examples diff --git a/sql-statements/sql-statement-drop-index.md b/sql-statements/sql-statement-drop-index.md index 226421b30d66..702ced4eb24f 100644 --- a/sql-statements/sql-statement-drop-index.md +++ b/sql-statements/sql-statement-drop-index.md @@ -10,29 +10,25 @@ This statement removes an index from a specified table, marking space as free in ## Synopsis -**AlterTableDropIndexStmt:** +```ebnf+diagram +AlterTableDropIndexStmt ::= + 'ALTER' IgnoreOptional 'TABLE' AlterTableDropIndexSpec -![AlterTableDropIndexStmt](/media/sqlgram/AlterTableDropIndexStmt.png) +IgnoreOptional ::= + 'IGNORE'? -**IgnoreOptional:** +TableName ::= + Identifier ('.' Identifier)? -![IgnoreOptional](/media/sqlgram/IgnoreOptional.png) +AlterTableDropIndexSpec ::= + 'DROP' ( KeyOrIndex | 'FOREIGN' 'KEY' ) IfExists Identifier -**TableName:** +KeyOrIndex ::= + 'KEY' +| 'INDEX' -![TableName](/media/sqlgram/TableName.png) - -**AlterTableDropIndexSpec:** - -![AlterTableDropIndexSpec](/media/sqlgram/AlterTableDropIndexSpec.png) - -**KeyOrIndex:** - -![KeyOrIndex](/media/sqlgram/KeyOrIndex.png) - -**IfExists:** - -![IfExists](/media/sqlgram/IfExists.png) +IfExists ::= ( 'IF' 'EXISTS' )? +``` ## Examples @@ -72,7 +68,7 @@ Query OK, 0 rows affected (0.30 sec) ## MySQL compatibility -* Removing the primary key constraint from a column is not supported by default. You can enable the feature by setting the `alter-primary-key` configuration item to `true`. For details, see [alter-primary-key](/tidb-configuration-file.md#alter-primary-key). +* Dropping the primary key of the `CLUSTERED` type is not supported. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). ## See also diff --git a/sql-statements/sql-statement-drop-role.md b/sql-statements/sql-statement-drop-role.md index 6e36daa360e5..e93dc3789435 100644 --- a/sql-statements/sql-statement-drop-role.md +++ b/sql-statements/sql-statement-drop-role.md @@ -9,13 +9,13 @@ This statement removes a role, that was previously created with `CREATE ROLE`. ## Synopsis -**DropRoleStmt:** +```ebnf+diagram +DropRoleStmt ::= + 'DROP' 'ROLE' ( 'IF' 'EXISTS' )? RolenameList -![DropRoleStmt](/media/sqlgram/DropRoleStmt.png) - -**RolenameList:** - -![RolenameList](/media/sqlgram/RolenameList.png) +RolenameList ::= + Rolename ( ',' Rolename )* +``` ## Examples diff --git a/sql-statements/sql-statement-drop-sequence.md b/sql-statements/sql-statement-drop-sequence.md index 248a9753d171..ca346c168859 100644 --- a/sql-statements/sql-statement-drop-sequence.md +++ b/sql-statements/sql-statement-drop-sequence.md @@ -10,21 +10,18 @@ The `DROP SEQUENCE` statement drops the sequence object in TiDB. ## Synopsis -**DropSequenceStmt:** +```ebnf+diagram +DropSequenceStmt ::= + 'DROP' 'SEQUENCE' IfExists TableNameList -![DropSequenceStmt](/media/sqlgram/DropSequenceStmt.png) +IfExists ::= ( 'IF' 'EXISTS' )? -**IfExists:** +TableNameList ::= + TableName ( ',' TableName )* -![IfExists](/media/sqlgram/IfExists.png) - -**TableNameList:** - -![TableNameList](/media/sqlgram/TableNameList.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) +TableName ::= + Identifier ('.' Identifier)? +``` ## Examples diff --git a/sql-statements/sql-statement-drop-stats.md b/sql-statements/sql-statement-drop-stats.md index 7cf0b2c2f444..e0d2f4e668a6 100644 --- a/sql-statements/sql-statement-drop-stats.md +++ b/sql-statements/sql-statement-drop-stats.md @@ -10,13 +10,13 @@ The `DROP STATS` statement is used to delete the statistics of the selected tabl ## Synopsis -**DropStatsStmt:** +```ebnf+diagram +DropStatsStmt ::= + 'DROP' 'STATS' TableName -![DropTableStmt](/media/sqlgram/DropStatsStmt.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) +TableName ::= + Identifier ('.' Identifier)? +``` ## Examples diff --git a/sql-statements/sql-statement-drop-table.md b/sql-statements/sql-statement-drop-table.md index 32f78b497f17..b52af1f3ff25 100644 --- a/sql-statements/sql-statement-drop-table.md +++ b/sql-statements/sql-statement-drop-table.md @@ -10,17 +10,17 @@ This statement drops a table from the currently selected database. An error is r ## Synopsis -**DropTableStmt:** +```ebnf+diagram +DropTableStmt ::= + 'DROP' OptTemporary TableOrTables IfExists TableNameList RestrictOrCascadeOpt -![DropTableStmt](/media/sqlgram/DropTableStmt.png) +TableOrTables ::= + 'TABLE' +| 'TABLES' -**TableOrTables:** - -![TableOrTables](/media/sqlgram/TableOrTables.png) - -**TableNameList:** - -![TableNameList](/media/sqlgram/TableNameList.png) +TableNameList ::= + TableName ( ',' TableName )* +``` ## Examples diff --git a/sql-statements/sql-statement-drop-user.md b/sql-statements/sql-statement-drop-user.md index 88daf9f5d9eb..dd9b4ae9c024 100644 --- a/sql-statements/sql-statement-drop-user.md +++ b/sql-statements/sql-statement-drop-user.md @@ -6,18 +6,17 @@ aliases: ['/docs/dev/sql-statements/sql-statement-drop-user/','/docs/dev/referen # DROP USER -This statement removes a user from the TiDB system database. The optional keyword `IF EXISTS` can be used to silence an error if the user does not exist. -This statement requires the `CREATE USER` privilege. +This statement removes a user from the TiDB system database. The optional keyword `IF EXISTS` can be used to silence an error if the user does not exist. This statement requires the `CREATE USER` privilege. ## Synopsis -**DropUserStmt:** +```ebnf+diagram +DropUserStmt ::= + 'DROP' 'USER' ( 'IF' 'EXISTS' )? UsernameList -![DropUserStmt](/media/sqlgram/DropUserStmt.png) - -**Username:** - -![Username](/media/sqlgram/Username.png) +Username ::= + StringName ('@' StringName | singleAtIdentifier)? | 'CURRENT_USER' OptionalBraces +``` ## Examples diff --git a/sql-statements/sql-statement-drop-view.md b/sql-statements/sql-statement-drop-view.md index 501a35430fb4..968507262923 100644 --- a/sql-statements/sql-statement-drop-view.md +++ b/sql-statements/sql-statement-drop-view.md @@ -10,17 +10,16 @@ This statement drops an view object from the currently selected database. It doe ## Synopsis -**DropViewStmt:** +```ebnf+diagram +DropViewStmt ::= + 'DROP' 'VIEW' ( 'IF' 'EXISTS' )? TableNameList RestrictOrCascadeOpt -![DropViewStmt](/media/sqlgram/DropViewStmt.png) +TableNameList ::= + TableName ( ',' TableName )* -**TableNameList:** - -![TableNameList](/media/sqlgram/TableNameList.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) +TableName ::= + Identifier ('.' Identifier)? +``` ## Examples diff --git a/sql-statements/sql-statement-execute.md b/sql-statements/sql-statement-execute.md index 07da45701aad..8c76b17055ed 100644 --- a/sql-statements/sql-statement-execute.md +++ b/sql-statements/sql-statement-execute.md @@ -10,9 +10,10 @@ The `EXECUTE` statement provides an SQL interface to server-side prepared statem ## Synopsis -**ExecuteStmt:** - -![ExecuteStmt](/media/sqlgram/ExecuteStmt.png) +```ebnf+diagram +ExecuteStmt ::= + 'EXECUTE' Identifier ( 'USING' UserVariable ( ',' UserVariable )* )? +``` ## Examples diff --git a/sql-statements/sql-statement-explain-analyze.md b/sql-statements/sql-statement-explain-analyze.md index 2f7bb90ff25d..ae67470fd32e 100644 --- a/sql-statements/sql-statement-explain-analyze.md +++ b/sql-statements/sql-statement-explain-analyze.md @@ -14,17 +14,23 @@ The `EXPLAIN ANALYZE` statement works similar to `EXPLAIN`, with the major diffe ## Synopsis -**ExplainSym:** - -![ExplainSym](/media/sqlgram/ExplainSym.png) - -**ExplainStmt:** - -![ExplainStmt](/media/sqlgram/ExplainStmt.png) - -**ExplainableStmt:** - -![ExplainableStmt](/media/sqlgram/ExplainableStmt.png) +```ebnf+diagram +ExplainSym ::= + 'EXPLAIN' +| 'DESCRIBE' +| 'DESC' + +ExplainStmt ::= + ExplainSym ( TableName ColumnName? | 'ANALYZE'? ExplainableStmt | 'FOR' 'CONNECTION' NUM | 'FORMAT' '=' ( stringLit | ExplainFormatType ) ( 'FOR' 'CONNECTION' NUM | ExplainableStmt ) ) + +ExplainableStmt ::= + SelectStmt +| DeleteFromStmt +| UpdateStmt +| InsertIntoStmt +| ReplaceIntoStmt +| UnionStmt +``` ## EXPLAIN ANALYZE output format diff --git a/sql-statements/sql-statement-explain.md b/sql-statements/sql-statement-explain.md index 385e46a57bc8..23738f042e52 100644 --- a/sql-statements/sql-statement-explain.md +++ b/sql-statements/sql-statement-explain.md @@ -14,20 +14,30 @@ TiDB supports the `EXPLAIN [options] FOR CONNECTION connection_id` statement. Ho ## Synopsis -**ExplainSym:** - -![ExplainSym](/media/sqlgram/ExplainSym.png) - -**ExplainStmt:** - -![ExplainStmt](/media/sqlgram/ExplainStmt.png) - -**ExplainableStmt:** - -![ExplainableStmt](/media/sqlgram/ExplainableStmt.png) +```ebnf+diagram +ExplainSym ::= + 'EXPLAIN' +| 'DESCRIBE' +| 'DESC' + +ExplainStmt ::= + ExplainSym ( TableName ColumnName? | 'ANALYZE'? ExplainableStmt | 'FOR' 'CONNECTION' NUM | 'FORMAT' '=' ( stringLit | ExplainFormatType ) ( 'FOR' 'CONNECTION' NUM | ExplainableStmt ) ) + +ExplainableStmt ::= + SelectStmt +| DeleteFromStmt +| UpdateStmt +| InsertIntoStmt +| ReplaceIntoStmt +| UnionStmt +``` ## `EXPLAIN` output format +> **Note:** +> +> When you use the MySQL client to connect to TiDB, to read the output result in a clearer way without line wrapping, you can use the `pager less -S` command. Then, after the `EXPLAIN` result is output, you can press the right arrow → button on your keyboard to horizontally scroll through the output. + Currently, `EXPLAIN` in TiDB outputs 5 columns: `id`, `estRows`, `task`, `access object`, `operator info`. Each operator in the execution plan is described by these attributes, with each row in the `EXPLAIN` output describing an operator. The description of each attribute is as follows: | Attribute name | Description | @@ -239,7 +249,7 @@ If the `dot` program is not installed on your computer, copy the result to [this ## `EXPLAIN FOR CONNECTION` -`EXPLAIN FOR CONNECTION` is used to get the execution plan of the last executed query in a connection. The output format is the same as that of `EXPLAIN`. However, the implementation of `EXPLAIN FOR CONNECTION` in TiDB is different from that in MySQL. Their differences (apart from the output format) are listed as follows: +`EXPLAIN FOR CONNECTION` is used to get the execution plan of the currently executed SQL query or the last executed SQL query in a connection. The output format is the same as that of `EXPLAIN`. However, the implementation of `EXPLAIN FOR CONNECTION` in TiDB is different from that in MySQL. Their differences (apart from the output format) are listed as follows: - MySQL returns the query plan that is **being executing**, while TiDB returns the **last executed** query plan. - MySQL requires the login user to be the same as the connection being queried, or the login user has the **`PROCESS`** privilege; while TiDB requires the login user to be the same as the connection being queried, or the login user has the **`SUPER`** privilege. diff --git a/sql-statements/sql-statement-flashback-table.md b/sql-statements/sql-statement-flashback-table.md index 814540f68b49..90f0d1b89044 100644 --- a/sql-statements/sql-statement-flashback-table.md +++ b/sql-statements/sql-statement-flashback-table.md @@ -8,14 +8,16 @@ aliases: ['/docs/dev/sql-statements/sql-statement-flashback-table/','/docs/dev/r The `FLASHBACK TABLE` syntax is introduced since TiDB 4.0. You can use the `FLASHBACK TABLE` statement to restore the tables and data dropped by the `DROP` or `TRUNCATE` operation within the Garbage Collection (GC) lifetime. -Use the following command to query the TiDB cluster's `tikv_gc_safe_point` and `tikv_gc_life_time`. As long as the table is dropped by `DROP` or `TRUNCATE` statements after the `tikv_gc_safe_point` time, you can restore the table using the `FLASHBACK TABLE` statement. +The system variable [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) (default: `10m0s`) defines the retention time of earlier versions of rows. The current `safePoint` of where garabage collection has been performed up to can be obtained with the following query: {{< copyable "sql" >}} ```sql -select * from mysql.tidb where variable_name in ('tikv_gc_safe_point','tikv_gc_life_time'); +SELECT * FROM mysql.tidb WHERE variable_name = 'tikv_gc_safe_point'; ``` +As long as the table is dropped by `DROP` or `TRUNCATE` statements after the `tikv_gc_safe_point` time, you can restore the table using the `FLASHBACK TABLE` statement. + ## Syntax {{< copyable "sql" >}} @@ -26,17 +28,16 @@ FLASHBACK TABLE table_name [TO other_table_name] ## Synopsis -**FlashbackTableStmt:** - -![FlashbackTableStmt](/media/sqlgram/FlashbackTableStmt.png) - -**TableName:** +```ebnf+diagram +FlashbackTableStmt ::= + 'FLASHBACK' 'TABLE' TableName FlashbackToNewName -![TableName](/media/sqlgram/TableName.png) +TableName ::= + Identifier ( '.' Identifier )? -**FlashbackToNewName:** - -![FlashbackToNewName](/media/sqlgram/FlashbackToNewName.png) +FlashbackToNewName ::= + ( 'TO' Identifier )? +``` ## Notes diff --git a/sql-statements/sql-statement-flush-privileges.md b/sql-statements/sql-statement-flush-privileges.md index 09d3672e1df4..6bad67d7acd4 100644 --- a/sql-statements/sql-statement-flush-privileges.md +++ b/sql-statements/sql-statement-flush-privileges.md @@ -6,22 +6,25 @@ aliases: ['/docs/dev/sql-statements/sql-statement-flush-privileges/','/docs/dev/ # FLUSH PRIVILEGES -This statement triggers TiDB to reload the in-memory copy of privileges from the privilege tables. You should execute `FLUSH PRIVILEGES` after making manual edits to tables such as `mysql.user`. Executing this statement is not required after using privilege statements such as `GRANT` or `REVOKE`. -Executing this statement requires the `RELOAD` privilege. +This statement triggers TiDB to reload the in-memory copy of privileges from the privilege tables. You should execute `FLUSH PRIVILEGES` after making manual edits to tables such as `mysql.user`. Executing this statement is not required after using privilege statements such as `GRANT` or `REVOKE`. Executing this statement requires the `RELOAD` privilege. ## Synopsis -**FlushStmt:** +```ebnf+diagram +FlushStmt ::= + 'FLUSH' NoWriteToBinLogAliasOpt FlushOption -![FlushStmt](/media/sqlgram/FlushStmt.png) +NoWriteToBinLogAliasOpt ::= + ( 'NO_WRITE_TO_BINLOG' | 'LOCAL' )? -**NoWriteToBinLogAliasOpt:** - -![NoWriteToBinLogAliasOpt](/media/sqlgram/NoWriteToBinLogAliasOpt.png) - -**FlushOption:** - -![FlushOption](/media/sqlgram/FlushOption.png) +FlushOption ::= + 'PRIVILEGES' +| 'STATUS' +| 'TIDB' 'PLUGINS' PluginNameList +| 'HOSTS' +| LogTypeOpt 'LOGS' +| TableOrTables TableNameListOpt WithReadLockOpt +``` ## Examples diff --git a/sql-statements/sql-statement-flush-status.md b/sql-statements/sql-statement-flush-status.md index 6d7a13fe9657..a55a69bcec5c 100644 --- a/sql-statements/sql-statement-flush-status.md +++ b/sql-statements/sql-statement-flush-status.md @@ -10,17 +10,21 @@ This statement is included for compatibility with MySQL. It has no effect on TiD ## Synopsis -**FlushStmt:** - -![FlushStmt](/media/sqlgram/FlushStmt.png) - -**NoWriteToBinLogAliasOpt:** - -![NoWriteToBinLogAliasOpt](/media/sqlgram/NoWriteToBinLogAliasOpt.png) - -**FlushOption:** - -![FlushOption](/media/sqlgram/FlushOption.png) +```ebnf+diagram +FlushStmt ::= + 'FLUSH' NoWriteToBinLogAliasOpt FlushOption + +NoWriteToBinLogAliasOpt ::= + ( 'NO_WRITE_TO_BINLOG' | 'LOCAL' )? + +FlushOption ::= + 'PRIVILEGES' +| 'STATUS' +| 'TIDB' 'PLUGINS' PluginNameList +| 'HOSTS' +| LogTypeOpt 'LOGS' +| TableOrTables TableNameListOpt WithReadLockOpt +``` ## Examples diff --git a/sql-statements/sql-statement-flush-tables.md b/sql-statements/sql-statement-flush-tables.md index 0adef6e86da8..4ce04037238f 100644 --- a/sql-statements/sql-statement-flush-tables.md +++ b/sql-statements/sql-statement-flush-tables.md @@ -10,33 +10,34 @@ This statement is included for compatibility with MySQL. It has no effective usa ## Synopsis -**FlushStmt:** +```ebnf+diagram +FlushStmt ::= + 'FLUSH' NoWriteToBinLogAliasOpt FlushOption -![FlushStmt](/media/sqlgram/FlushStmt.png) +NoWriteToBinLogAliasOpt ::= + ( 'NO_WRITE_TO_BINLOG' | 'LOCAL' )? -**NoWriteToBinLogAliasOpt:** +FlushOption ::= + 'PRIVILEGES' +| 'STATUS' +| 'TIDB' 'PLUGINS' PluginNameList +| 'HOSTS' +| LogTypeOpt 'LOGS' +| TableOrTables TableNameListOpt WithReadLockOpt -![NoWriteToBinLogAliasOpt](/media/sqlgram/NoWriteToBinLogAliasOpt.png) +LogTypeOpt ::= + ( 'BINARY' | 'ENGINE' | 'ERROR' | 'GENERAL' | 'SLOW' )? -**FlushOption:** +TableOrTables ::= + 'TABLE' +| 'TABLES' -![FlushOption](/media/sqlgram/FlushOption.png) +TableNameListOpt ::= + TableNameList? -**LogTypeOpt:** - -![LogTypeOpt](/media/sqlgram/LogTypeOpt.png) - -**TableOrTables:** - -![TableOrTables](/media/sqlgram/TableOrTables.png) - -**TableNameListOpt:** - -![TableNameListOpt](/media/sqlgram/TableNameListOpt.png) - -**WithReadLockOpt:** - -![WithReadLockOpt](/media/sqlgram/WithReadLockOpt.png) +WithReadLockOpt ::= + ( 'WITH' 'READ' 'LOCK' )? +``` ## Examples diff --git a/sql-statements/sql-statement-grant-privileges.md b/sql-statements/sql-statement-grant-privileges.md index 9936dd7ee06c..98ad080f6eef 100644 --- a/sql-statements/sql-statement-grant-privileges.md +++ b/sql-statements/sql-statement-grant-privileges.md @@ -6,38 +6,56 @@ aliases: ['/docs/dev/sql-statements/sql-statement-grant-privileges/','/docs/dev/ # `GRANT ` -This statement allocates privileges to a pre-existing user in TiDB. The privilege system in TiDB follows MySQL, where credentials are assigned based on a database/table pattern. -Executing this statement requires the `GRANT OPTION` privilege and all privileges you allocate. +This statement allocates privileges to a pre-existing user in TiDB. The privilege system in TiDB follows MySQL, where credentials are assigned based on a database/table pattern. Executing this statement requires the `GRANT OPTION` privilege and all privileges you allocate. ## Synopsis -**GrantStmt:** - -![GrantStmt](/media/sqlgram/GrantStmt.png) - -**PrivElemList:** - -![PrivElemList](/media/sqlgram/PrivElemList.png) - -**PrivElem:** - -![PrivElem](/media/sqlgram/PrivElem.png) - -**PrivType:** - -![PrivType](/media/sqlgram/PrivType.png) - -**ObjectType:** - -![ObjectType](/media/sqlgram/ObjectType.png) - -**PrivLevel:** - -![PrivLevel](/media/sqlgram/PrivLevel.png) - -**UserSpecList:** - -![UserSpecList](/media/sqlgram/UserSpecList.png) +```ebnf+diagram +GrantStmt ::= + 'GRANT' PrivElemList 'ON' ObjectType PrivLevel 'TO' UserSpecList RequireClauseOpt WithGrantOptionOpt + +PrivElemList ::= + PrivElem ( ',' PrivElem )* + +PrivElem ::= + PrivType ( '(' ColumnNameList ')' )? + +PrivType ::= + 'ALL' 'PRIVILEGES'? +| 'ALTER' 'ROUTINE'? +| 'CREATE' ( 'USER' | 'TEMPORARY' 'TABLES' | 'VIEW' | 'ROLE' | 'ROUTINE' )? +| 'TRIGGER' +| 'DELETE' +| 'DROP' 'ROLE'? +| 'PROCESS' +| 'EXECUTE' +| 'INDEX' +| 'INSERT' +| 'SELECT' +| 'SUPER' +| 'SHOW' ( 'DATABASES' | 'VIEW' ) +| 'UPDATE' +| 'GRANT' 'OPTION' +| 'REFERENCES' +| 'REPLICATION' ( 'SLAVE' | 'CLIENT' ) +| 'USAGE' +| 'RELOAD' +| 'FILE' +| 'CONFIG' +| 'LOCK' 'TABLES' +| 'EVENT' +| 'SHUTDOWN' + +ObjectType ::= + 'TABLE'? + +PrivLevel ::= + '*' ( '.' '*' )? +| Identifier ( '.' ( '*' | Identifier ) )? + +UserSpecList ::= + UserSpec ( ',' UserSpec )* +``` ## Examples diff --git a/sql-statements/sql-statement-grant-role.md b/sql-statements/sql-statement-grant-role.md index 779c202b347b..a2cac8ae67dc 100644 --- a/sql-statements/sql-statement-grant-role.md +++ b/sql-statements/sql-statement-grant-role.md @@ -9,17 +9,16 @@ Assigns a previously created role to an existing user. The user can use then use ## Synopsis -**GrantRoleStmt:** +```ebnf+diagram +GrantRoleStmt ::= + 'GRANT' RolenameList 'TO' UsernameList -![GrantRoleStmt](/media/sqlgram/GrantRoleStmt.png) +RolenameList ::= + Rolename ( ',' Rolename )* -**RolenameList:** - -![RolenameList](/media/sqlgram/RolenameList.png) - -**UsernameList:** - -![UsernameList](/media/sqlgram/UsernameList.png) +UsernameList ::= + Username ( ',' Username )* +``` ## Examples diff --git a/sql-statements/sql-statement-insert.md b/sql-statements/sql-statement-insert.md index 16ffc66ab01e..7cc8a30e417f 100644 --- a/sql-statements/sql-statement-insert.md +++ b/sql-statements/sql-statement-insert.md @@ -10,41 +10,37 @@ This statement inserts new rows into a table. ## Synopsis -**InsertIntoStmt:** +```ebnf+diagram +InsertIntoStmt ::= + 'INSERT' TableOptimizerHints PriorityOpt IgnoreOptional IntoOpt TableName PartitionNameListOpt InsertValues OnDuplicateKeyUpdate -![InsertIntoStmt](/media/sqlgram/InsertIntoStmt.png) +TableOptimizerHints ::= + hintComment? -**TableOptimizerHints** +PriorityOpt ::= + ( 'LOW_PRIORITY' | 'HIGH_PRIORITY' | 'DELAYED' )? -![TableOptimizerHints](/media/sqlgram/TableOptimizerHints.png) +IgnoreOptional ::= + 'IGNORE'? -**PriorityOpt:** +IntoOpt ::= 'INTO'? -![PriorityOpt](/media/sqlgram/PriorityOpt.png) +TableName ::= + Identifier ( '.' Identifier )? -**IgnoreOptional:** +PartitionNameListOpt ::= + ( 'PARTITION' '(' Identifier ( ',' Identifier )* ')' )? -![IgnoreOptional](/media/sqlgram/IgnoreOptional.png) +InsertValues ::= + '(' ( ColumnNameListOpt ')' ( ValueSym ValuesList | SelectStmt | '(' SelectStmt ')' | UnionStmt ) | SelectStmt ')' ) +| ValueSym ValuesList +| SelectStmt +| UnionStmt +| 'SET' ColumnSetValue? ( ',' ColumnSetValue )* -**IntoOpt:** - -![IntoOpt](/media/sqlgram/IntoOpt.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) - -**PartitionNameListOpt:** - -![PartitionNameListOpt](/media/sqlgram/PartitionNameListOpt.png) - -**InsertValues:** - -![InsertValues](/media/sqlgram/InsertValues.png) - -**OnDuplicateKeyUpdate:** - -![OnDuplicateKeyUpdate](/media/sqlgram/OnDuplicateKeyUpdate.png) +OnDuplicateKeyUpdate ::= + ( 'ON' 'DUPLICATE' 'KEY' 'UPDATE' AssignmentList )? +``` ## Examples diff --git a/sql-statements/sql-statement-kill.md b/sql-statements/sql-statement-kill.md index 2d870945dacc..2d986bc64db1 100644 --- a/sql-statements/sql-statement-kill.md +++ b/sql-statements/sql-statement-kill.md @@ -10,13 +10,11 @@ The statement `KILL TIDB` is used to terminate connections in TiDB. ## Synopsis -**KillStmt:** +```ebnf+diagram +KillStmt ::= KillOrKillTiDB ( 'CONNECTION' | 'QUERY' )? NUM -![KillStmt](/media/sqlgram/KillStmt.png) - -**KillOrKillTiDB:** - -![KillOrKillTiDB](/media/sqlgram/KillOrKillTiDB.png) +KillOrKillTiDB ::= 'KILL' 'TIDB'? +``` ## Examples @@ -30,7 +28,7 @@ mysql> SHOW PROCESSLIST; +------+------+-----------+------+---------+------+-------+------------------+ 2 rows in set (0.00 sec) -mysql> KILL TIDB 2; +KILL TIDB 2; Query OK, 0 rows affected (0.00 sec) ``` @@ -42,3 +40,4 @@ Query OK, 0 rows affected (0.00 sec) ## See also * [SHOW \[FULL\] PROCESSLIST](/sql-statements/sql-statement-show-processlist.md) +* [CLUSTER_PROCESSLIST](/information-schema/information-schema-processlist.md#cluster_processlist) diff --git a/sql-statements/sql-statement-load-data.md b/sql-statements/sql-statement-load-data.md index 73659d1218f8..4e249a26414e 100644 --- a/sql-statements/sql-statement-load-data.md +++ b/sql-statements/sql-statement-load-data.md @@ -10,9 +10,10 @@ The `LOAD DATA` statement batch loads data into a TiDB table. ## Synopsis -**LoadDataStmt:** - -![LoadDataStmt](/media/sqlgram/LoadDataStmt.png) +```ebnf+diagram +LoadDataStmt ::= + 'LOAD' 'DATA' LocalOpt 'INFILE' stringLit DuplicateOpt 'INTO' 'TABLE' TableName CharsetOpt Fields Lines IgnoreLines ColumnNameOrUserVarListOptWithBrackets LoadDataSetSpecOpt +``` ## Parameters @@ -60,17 +61,17 @@ In addition, TiDB currently only supports parsing the syntax of the `DuplicateOp ```sql CREATE TABLE trips ( - -> trip_id bigint NOT NULL PRIMARY KEY AUTO_INCREMENT, - -> duration integer not null, - -> start_date datetime, - -> end_date datetime, - -> start_station_number integer, - -> start_station varchar(255), - -> end_station_number integer, - -> end_station varchar(255), - -> bike_number varchar(255), - -> member_type varchar(255) - -> ); + trip_id bigint NOT NULL PRIMARY KEY AUTO_INCREMENT, + duration integer not null, + start_date datetime, + end_date datetime, + start_station_number integer, + start_station varchar(255), + end_station_number integer, + end_station varchar(255), + bike_number varchar(255), + member_type varchar(255) + ); ``` ``` @@ -104,7 +105,7 @@ In the above example, `x'2c'` is the hexadecimal representation of the `,` chara ## MySQL compatibility -This statement is understood to be fully compatible with MySQL. Any compatibility differences should be [reported via an issue](https://github.com/pingcap/tidb/issues/new/choose) on GitHub. +This statement is understood to be fully compatible with MySQL except for the `LOAD DATA...REPLACE INTO` syntax [#24515](https://github.com/pingcap/tidb/issues/24515). Any other compatibility differences should be [reported via an issue](https://github.com/pingcap/tidb/issues/new/choose) on GitHub. > **Note:** > diff --git a/sql-statements/sql-statement-load-stats.md b/sql-statements/sql-statement-load-stats.md index 4977dcfbc87e..a202e6d16a3c 100644 --- a/sql-statements/sql-statement-load-stats.md +++ b/sql-statements/sql-statement-load-stats.md @@ -10,9 +10,10 @@ The `LOAD STATS` statement is used to load the statistics into TiDB. ## Synopsis -**LoadStatsStmt:** - -![LoadStatsStmt](/media/sqlgram/LoadStatsStmt.png) +```ebnf+diagram +LoadStatsStmt ::= + 'LOAD' 'STATS' stringLit +``` ## Examples diff --git a/sql-statements/sql-statement-modify-column.md b/sql-statements/sql-statement-modify-column.md index 2d5f72107d7d..03fe83aee369 100644 --- a/sql-statements/sql-statement-modify-column.md +++ b/sql-statements/sql-statement-modify-column.md @@ -8,42 +8,92 @@ aliases: ['/docs/dev/sql-statements/sql-statement-modify-column/','/docs/dev/ref The `ALTER TABLE.. MODIFY COLUMN` statement modifies a column on an existing table. The modification can include changing the data type and attributes. To rename at the same time, use the [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) statement instead. -## Synopsis - -**AlterTableStmt:** - -![AlterTableStmt](/media/sqlgram/AlterTableStmt.png) +Since v5.1.0, TiDB has supported changes of data types for Reorg data, including but not limited to: -**AlterTableSpec:** +- Changing `VARCHAR` to `BIGINT` +- Modifying the `DECIMAL` precision +- Compressing the length of `VARCHAR(10)` to `VARCHAR(5)` -![AlterTableSpec](/media/sqlgram/AlterTableSpec.png) +## Synopsis -**ColumnKeywordOpt:** +```ebnf+diagram +AlterTableStmt ::= + 'ALTER' IgnoreOptional 'TABLE' TableName ( AlterTableSpecListOpt AlterTablePartitionOpt | 'ANALYZE' 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? AnalyzeOptionListOpt ) + +AlterTableSpec ::= + TableOptionList +| 'SET' 'TIFLASH' 'REPLICA' LengthNum LocationLabelList +| 'CONVERT' 'TO' CharsetKw ( CharsetName | 'DEFAULT' ) OptCollate +| 'ADD' ( ColumnKeywordOpt IfNotExists ( ColumnDef ColumnPosition | '(' TableElementList ')' ) | Constraint | 'PARTITION' IfNotExists NoWriteToBinLogAliasOpt ( PartitionDefinitionListOpt | 'PARTITIONS' NUM ) ) +| ( ( 'CHECK' | 'TRUNCATE' ) 'PARTITION' | ( 'OPTIMIZE' | 'REPAIR' | 'REBUILD' ) 'PARTITION' NoWriteToBinLogAliasOpt ) AllOrPartitionNameList +| 'COALESCE' 'PARTITION' NoWriteToBinLogAliasOpt NUM +| 'DROP' ( ColumnKeywordOpt IfExists ColumnName RestrictOrCascadeOpt | 'PRIMARY' 'KEY' | 'PARTITION' IfExists PartitionNameList | ( KeyOrIndex IfExists | 'CHECK' ) Identifier | 'FOREIGN' 'KEY' IfExists Symbol ) +| 'EXCHANGE' 'PARTITION' Identifier 'WITH' 'TABLE' TableName WithValidationOpt +| ( 'IMPORT' | 'DISCARD' ) ( 'PARTITION' AllOrPartitionNameList )? 'TABLESPACE' +| 'REORGANIZE' 'PARTITION' NoWriteToBinLogAliasOpt ReorganizePartitionRuleOpt +| 'ORDER' 'BY' AlterOrderItem ( ',' AlterOrderItem )* +| ( 'DISABLE' | 'ENABLE' ) 'KEYS' +| ( 'MODIFY' ColumnKeywordOpt IfExists | 'CHANGE' ColumnKeywordOpt IfExists ColumnName ) ColumnDef ColumnPosition +| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier IndexInvisible ) +| 'RENAME' ( ( 'COLUMN' | KeyOrIndex ) Identifier 'TO' Identifier | ( 'TO' | '='? | 'AS' ) TableName ) +| LockClause +| AlgorithmClause +| 'FORCE' +| ( 'WITH' | 'WITHOUT' ) 'VALIDATION' +| 'SECONDARY_LOAD' +| 'SECONDARY_UNLOAD' + +ColumnKeywordOpt ::= 'COLUMN'? + +ColumnDef ::= + ColumnName ( Type | 'SERIAL' ) ColumnOptionListOpt + +ColumnPosition ::= + ( 'FIRST' | 'AFTER' ColumnName )? +``` -![ColumnKeywordOpt](/media/sqlgram/ColumnKeywordOpt.png) +## Examples -**ColumnDef:** +### Meta-Only Change -![ColumnDef](/media/sqlgram/ColumnDef.png) +{{< copyable "sql" >}} -**ColumnPosition:** +```sql +CREATE TABLE t1 (id int not null primary key AUTO_INCREMENT, col1 INT); +``` -![ColumnPosition](/media/sqlgram/ColumnPosition.png) +``` +Query OK, 0 rows affected (0.11 sec) +``` -## Examples +{{< copyable "sql" >}} ```sql -mysql> CREATE TABLE t1 (id int not null primary key AUTO_INCREMENT, col1 INT); -Query OK, 0 rows affected (0.11 sec) +INSERT INTO t1 (col1) VALUES (1),(2),(3),(4),(5); +``` -mysql> INSERT INTO t1 (col1) VALUES (1),(2),(3),(4),(5); +``` Query OK, 5 rows affected (0.02 sec) Records: 5 Duplicates: 0 Warnings: 0 +``` + +{{< copyable "sql" >}} -mysql> ALTER TABLE t1 MODIFY col1 BIGINT; +```sql +ALTER TABLE t1 MODIFY col1 BIGINT; +``` + +``` Query OK, 0 rows affected (0.09 sec) +``` -mysql> SHOW CREATE TABLE t1\G +{{< copyable "sql" >}} + +```sql +SHOW CREATE TABLE t1\G; +``` + +```sql *************************** 1. row *************************** Table: t1 Create Table: CREATE TABLE `t1` ( @@ -54,29 +104,116 @@ Create Table: CREATE TABLE `t1` ( 1 row in set (0.00 sec) ``` +### Reorg-Data Change + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t1 (id int not null primary key AUTO_INCREMENT, col1 INT); +``` + +``` +Query OK, 0 rows affected (0.11 sec) +``` + +{{< copyable "sql" >}} + +```sql +INSERT INTO t1 (col1) VALUES (12345),(67890); +``` + +``` +Query OK, 2 rows affected (0.00 sec) +Records: 2 Duplicates: 0 Warnings: 0 +``` + +{{< copyable "sql" >}} + +```sql +ALTER TABLE t1 MODIFY col1 VARCHAR(5); +``` + +``` +Query OK, 0 rows affected (2.52 sec) +``` + +{{< copyable "sql" >}} + +```sql +SHOW CREATE TABLE t1\G; +``` + +```sql +*************************** 1. row *************************** + Table: t1 +CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `col1` varchar(5) DEFAULT NULL, + PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=30001 +1 row in set (0.00 sec) +``` + +> **Note:** +> +> - TiDB returns an error when the changed data type conflicts with an existing data row. In the above example, TiDB returns the following error: +> +> ``` +> alter table t1 modify column col1 varchar(4); +> ERROR 1406 (22001): Data Too Long, field len 4, data len 5 +> ``` +> +> - Due to the compatibility with the Async Commit feature, the DDL statement waits for a period of time (about 2.5s) before starting to process into Reorg Data. +> +> ``` +> Query OK, 0 rows affected (2.52 sec) +> ``` + ## MySQL compatibility -* Does not support modifying multiple columns in a single `ALTER TABLE` statement. For example: +* Does not support modifying multiple columns using a single `ALTER TABLE` statement. For example: ```sql ALTER TABLE t1 MODIFY col1 BIGINT, MODIFY id BIGINT NOT NULL; ERROR 1105 (HY000): Unsupported multi schema change ``` -* Does not support changes of lossy data types and some other types (including changes from integer to string or to `BLOB`). For example: +* Does not support modifying the Reorg-Data types on the primary key columns but supports modifying the Meta-Only types. For example: + + ```sql + CREATE TABLE t (a int primary key); + ALTER TABLE t MODIFY COLUMN a VARCHAR(10); + ERROR 8200 (HY000): Unsupported modify column: column has primary key flag + ``` + + ```sql + CREATE TABLE t (a int primary key); + ALTER TABLE t MODIFY COLUMN a bigint; + Query OK, 0 rows affected (0.01 sec) + ``` + +* Does not support modifying the column types on generated columns. For example: + + ```sql + CREATE TABLE t (a INT, b INT as (a+1)); + ALTER TABLE t MODIFY COLUMN b VARCHAR(10); + ERROR 8200 (HY000): Unsupported modify column: column is generated + ``` + +* Does not support modifying the column types on the partitioned tables. For example: ```sql - CREATE TABLE t1 (col1 BIGINT); - ALTER TABLE t1 MODIFY col1 INT; - ERROR 8200 (HY000): Unsupported modify column length 11 is less than origin 20 + CREATE TABLE t (c1 INT, c2 INT, c3 INT) partition by range columns(c1) ( partition p0 values less than (10), partition p1 values less than (maxvalue)); + ALTER TABLE t MODIFY COLUMN c1 DATETIME; + ERROR 8200 (HY000): Unsupported modify column: table is partition table ``` -* Does not support modifying the precision of the `DECIMAL` data type. For example: +* Does not support modifying some data types (for example, some TIME types, Bit, Set, Enum, JSON) are not supported due to some compatibility issues of the `cast` function's behavior between TiDB and MySQL. ```sql - CREATE TABLE t (a DECIMAL(5, 3)); - ALTER TABLE t MODIFY COLUMN a DECIMAL(6, 3); - ERROR 8200 (HY000): Unsupported modify column: can't change decimal column precision + CREATE TABLE t (a DECIMAL(13, 7)); + ALTER TABLE t MODIFY COLUMN a DATETIME; + ERROR 8200 (HY000): Unsupported modify column: change from original type decimal(13,7) to datetime is currently unsupported yet ``` ## See also diff --git a/sql-statements/sql-statement-prepare.md b/sql-statements/sql-statement-prepare.md index 7095a5a237ba..85ce7fffe0b2 100644 --- a/sql-statements/sql-statement-prepare.md +++ b/sql-statements/sql-statement-prepare.md @@ -10,13 +10,14 @@ The `PREPARE` statement provides an SQL interface to server-side prepared statem ## Synopsis -**PreparedStmt:** +```ebnf+diagram +PreparedStmt ::= + 'PREPARE' Identifier 'FROM' PrepareSQL -![PreparedStmt](/media/sqlgram/PreparedStmt.png) - -**PrepareSQL:** - -![PrepareSQL](/media/sqlgram/PrepareSQL.png) +PrepareSQL ::= + stringLit +| UserVariable +``` ## Examples diff --git a/sql-statements/sql-statement-recover-table.md b/sql-statements/sql-statement-recover-table.md index d2611b86918c..3069a7d4152e 100644 --- a/sql-statements/sql-statement-recover-table.md +++ b/sql-statements/sql-statement-recover-table.md @@ -24,21 +24,17 @@ RECOVER TABLE BY JOB ddl_job_id ## Synopsis -**RecoverTableStmt:** +```ebnf+diagram +RecoverTableStmt ::= + 'RECOVER' 'TABLE' ( 'BY' 'JOB' Int64Num | TableName Int64Num? ) -![RecoverTableStmt](/media/sqlgram/RecoverTableStmt.png) +TableName ::= + Identifier ( '.' Identifier )? -**TableName:** +Int64Num ::= NUM -![TableName](/media/sqlgram/TableName.png) - -**Int64Num:** - -![Int64Num](/media/sqlgram/Int64Num.png) - -**NUM:** - -![NUM](/media/sqlgram/NUM.png) +NUM ::= intLit +``` > **Note:** > @@ -48,7 +44,7 @@ RECOVER TABLE BY JOB ddl_job_id > > + `RECOVER TABLE` is supported in the Binlog version 3.0.1, so you can use `RECOVER TABLE` in the following three situations: > -> - Binglog version is 3.0.1 or later. +> - Binlog version is 3.0.1 or later. > - TiDB 3.0 is used both in the upstream cluster and the downstream cluster. > - The GC life time of the secondary cluster must be longer than that of the primary cluster. However, as latency occurs during data replication between upstream and downstream databases, data recovery might fail in the downstream. diff --git a/sql-statements/sql-statement-rename-index.md b/sql-statements/sql-statement-rename-index.md index f58b92dee348..52d3803e982f 100644 --- a/sql-statements/sql-statement-rename-index.md +++ b/sql-statements/sql-statement-rename-index.md @@ -10,13 +10,14 @@ The statement `ALTER TABLE .. RENAME INDEX` renames an existing index to a new n ## Synopsis -**AlterTableStmt:** +```ebnf+diagram +AlterTableStmt ::= + 'ALTER' IgnoreOptional 'TABLE' TableName ( AlterTableSpecListOpt AlterTablePartitionOpt | 'ANALYZE' 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList )? AnalyzeOptionListOpt ) -![AlterTableStmt](/media/sqlgram/AlterTableStmt.png) - -**KeyOrIndex:** - -![KeyOrIndex](/media/sqlgram/KeyOrIndex.png) +KeyOrIndex ::= + 'KEY' +| 'INDEX' +``` ## Examples diff --git a/sql-statements/sql-statement-rename-table.md b/sql-statements/sql-statement-rename-table.md index 9bce0b13b4fd..857d8ad5ba3a 100644 --- a/sql-statements/sql-statement-rename-table.md +++ b/sql-statements/sql-statement-rename-table.md @@ -10,13 +10,13 @@ This statement renames an existing table to a new name. ## Synopsis -**RenameTableStmt:** +```ebnf+diagram +RenameTableStmt ::= + 'RENAME' 'TABLE' TableToTable ( ',' TableToTable )* -![RenameTableStmt](/media/sqlgram/RenameTableStmt.png) - -**TableToTable:** - -![TableToTable](/media/sqlgram/TableToTable.png) +TableToTable ::= + TableName 'TO' TableName +``` ## Examples diff --git a/sql-statements/sql-statement-replace.md b/sql-statements/sql-statement-replace.md index a561d85c059d..f95340a83905 100644 --- a/sql-statements/sql-statement-replace.md +++ b/sql-statements/sql-statement-replace.md @@ -10,29 +10,28 @@ The `REPLACE` statement is semantically a combined `DELETE`+`INSERT` statement. ## Synopsis -**ReplaceIntoStmt:** +```ebnf+diagram +ReplaceIntoStmt ::= + 'REPLACE' PriorityOpt IntoOpt TableName PartitionNameListOpt InsertValues -![ReplaceIntoStmt](/media/sqlgram/ReplaceIntoStmt.png) +PriorityOpt ::= + ( 'LOW_PRIORITY' | 'HIGH_PRIORITY' | 'DELAYED' )? -**PriorityOpt:** +IntoOpt ::= 'INTO'? -![PriorityOpt](/media/sqlgram/PriorityOpt.png) +TableName ::= + Identifier ( '.' Identifier )? -**IntoOpt:** +PartitionNameListOpt ::= + ( 'PARTITION' '(' Identifier ( ',' Identifier )* ')' )? -![IntoOpt](/media/sqlgram/IntoOpt.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) - -**PartitionNameListOpt:** - -![PartitionNameListOpt](/media/sqlgram/PartitionNameListOpt.png) - -**InsertValues:** - -![InsertValues](/media/sqlgram/InsertValues.png) +InsertValues ::= + '(' ( ColumnNameListOpt ')' ( ValueSym ValuesList | SelectStmt | '(' SelectStmt ')' | UnionStmt ) | SelectStmt ')' ) +| ValueSym ValuesList +| SelectStmt +| UnionStmt +| 'SET' ColumnSetValue? ( ',' ColumnSetValue )* +``` ## Examples diff --git a/sql-statements/sql-statement-restore.md b/sql-statements/sql-statement-restore.md index bcb98e6b14c2..76156f9018c9 100644 --- a/sql-statements/sql-statement-restore.md +++ b/sql-statements/sql-statement-restore.md @@ -14,13 +14,13 @@ The `RESTORE` statement uses the same engine as the [BR tool](/br/backup-and-res * When a full restore is being performed, the tables being restored should not already exist, because existing data might be overridden and causes inconsistency between the data and indices. * When an incremental restore is being performed, the tables should be at the exact same state as the `LAST_BACKUP` timestamp when the backup is created. -Running `RESTORE` requires `SUPER` privilege. Additionally, both the TiDB node executing the restore and all TiKV nodes in the cluster must have read permission from the destination. +Running `RESTORE` requires either the `RESTORE_ADMIN` or `SUPER` privilege. Additionally, both the TiDB node executing the restore and all TiKV nodes in the cluster must have read permission from the destination. The `RESTORE` statement is blocking, and will finish only after the entire restore task is finished, failed, or canceled. A long-lasting connection should be prepared for running `RESTORE`. The task can be canceled using the [`KILL TIDB QUERY`](/sql-statements/sql-statement-kill.md) statement. Only one `BACKUP` and `RESTORE` task can be executed at a time. If a `BACKUP` or `RESTORE` task is already running on the same TiDB server, the new `RESTORE` execution will wait until all previous tasks are done. -`RESTORE` can only be used with "tikv" storage engine. Using `RESTORE` with the "mocktikv" engine will fail. +`RESTORE` can only be used with "tikv" storage engine. Using `RESTORE` with the "unistore" engine will fail. ## Synopsis @@ -89,7 +89,7 @@ RESTORE DATABASE `test` FROM 'local:///mnt/backup/2020/04/'; RESTORE TABLE `test`.`sbtest01`, `test`.`sbtest02` FROM 'local:///mnt/backup/2020/04/'; ``` -### Remote destinations +### External storages BR supports restoring data from S3 or GCS: @@ -99,7 +99,7 @@ BR supports restoring data from S3 or GCS: RESTORE DATABASE * FROM 's3://example-bucket-2020/backup-05/?region=us-west-2'; ``` -The URL syntax is further explained in [BR storages](/br/backup-and-restore-storages.md). +The URL syntax is further explained in [External Storages](/br/backup-and-restore-storages.md). When running on cloud environment where credentials should not be distributed, set the `SEND_CREDENTIALS_TO_TIKV` option to `FALSE`: diff --git a/sql-statements/sql-statement-revoke-privileges.md b/sql-statements/sql-statement-revoke-privileges.md index 7e6381fca9cc..4bd176a5aad4 100644 --- a/sql-statements/sql-statement-revoke-privileges.md +++ b/sql-statements/sql-statement-revoke-privileges.md @@ -6,38 +6,56 @@ aliases: ['/docs/dev/sql-statements/sql-statement-revoke-privileges/','/docs/dev # `REVOKE ` -This statement removes privileges from an existing user. -Executing this statement requires the `GRANT OPTION` privilege and all privileges you revoke. +This statement removes privileges from an existing user. Executing this statement requires the `GRANT OPTION` privilege and all privileges you revoke. ## Synopsis -**GrantStmt:** - -![GrantStmt](/media/sqlgram/GrantStmt.png) - -**PrivElemList:** - -![PrivElemList](/media/sqlgram/PrivElemList.png) - -**PrivElem:** - -![PrivElem](/media/sqlgram/PrivElem.png) - -**PrivType:** - -![PrivType](/media/sqlgram/PrivType.png) - -**ObjectType:** - -![ObjectType](/media/sqlgram/ObjectType.png) - -**PrivLevel:** - -![PrivLevel](/media/sqlgram/PrivLevel.png) - -**UserSpecList:** - -![UserSpecList](/media/sqlgram/UserSpecList.png) +```ebnf+diagram +GrantStmt ::= + 'GRANT' PrivElemList 'ON' ObjectType PrivLevel 'TO' UserSpecList RequireClauseOpt WithGrantOptionOpt + +PrivElemList ::= + PrivElem ( ',' PrivElem )* + +PrivElem ::= + PrivType ( '(' ColumnNameList ')' )? + +PrivType ::= + 'ALL' 'PRIVILEGES'? +| 'ALTER' 'ROUTINE'? +| 'CREATE' ( 'USER' | 'TEMPORARY' 'TABLES' | 'VIEW' | 'ROLE' | 'ROUTINE' )? +| 'TRIGGER' +| 'DELETE' +| 'DROP' 'ROLE'? +| 'PROCESS' +| 'EXECUTE' +| 'INDEX' +| 'INSERT' +| 'SELECT' +| 'SUPER' +| 'SHOW' ( 'DATABASES' | 'VIEW' ) +| 'UPDATE' +| 'GRANT' 'OPTION' +| 'REFERENCES' +| 'REPLICATION' ( 'SLAVE' | 'CLIENT' ) +| 'USAGE' +| 'RELOAD' +| 'FILE' +| 'CONFIG' +| 'LOCK' 'TABLES' +| 'EVENT' +| 'SHUTDOWN' + +ObjectType ::= + 'TABLE'? + +PrivLevel ::= + '*' ( '.' '*' )? +| Identifier ( '.' ( '*' | Identifier ) )? + +UserSpecList ::= + UserSpec ( ',' UserSpec )* +``` ## Examples diff --git a/sql-statements/sql-statement-revoke-role.md b/sql-statements/sql-statement-revoke-role.md index f0c8627675b7..75878257422e 100644 --- a/sql-statements/sql-statement-revoke-role.md +++ b/sql-statements/sql-statement-revoke-role.md @@ -9,17 +9,16 @@ This statement removes a previously assigned role from a specified user (or list ## Synopsis -**RevokeRoleStmt:** +```ebnf+diagram +RevokeRoleStmt ::= + 'REVOKE' RolenameList 'FROM' UsernameList -![RevokeRoleStmt](/media/sqlgram/RevokeRoleStmt.png) +RolenameList ::= + Rolename ( ',' Rolename )* -**RolenameList:** - -![RolenameList](/media/sqlgram/RolenameList.png) - -**UsernameList:** - -![UsernameList](/media/sqlgram/UsernameList.png) +UsernameList ::= + Username ( ',' Username )* +``` ## Examples diff --git a/sql-statements/sql-statement-rollback.md b/sql-statements/sql-statement-rollback.md index 71243588e084..642ec1f6b14c 100644 --- a/sql-statements/sql-statement-rollback.md +++ b/sql-statements/sql-statement-rollback.md @@ -10,13 +10,14 @@ This statement reverts all changes in the current transaction inside of TIDB. I ## Synopsis -**RollbackStmt:** +```ebnf+diagram +RollbackStmt ::= + 'ROLLBACK' CompletionTypeWithinTransaction? -![RollbackStmt](/media/sqlgram/RollbackStmt.png) - -**CompletionTypeWithinTransaction:** - -![CompletionTypeWithinTransaction](/media/sqlgram/CompletionTypeWithinTransaction.png) +CompletionTypeWithinTransaction ::= + 'AND' ( 'CHAIN' ( 'NO' 'RELEASE' )? | 'NO' 'CHAIN' ( 'NO'? 'RELEASE' )? ) +| 'NO'? 'RELEASE' +``` ## Examples diff --git a/sql-statements/sql-statement-select.md b/sql-statements/sql-statement-select.md index 40a387400fe7..fc2811a18b60 100644 --- a/sql-statements/sql-statement-select.md +++ b/sql-statements/sql-statement-select.md @@ -32,7 +32,13 @@ The `SELECT` statement is used to read data from TiDB. **TableRefsClause:** -![TableRefsClause](/media/sqlgram/TableRefsClause.png) +```ebnf+diagram +TableRefsClause ::= + TableRef AsOfClause? ( ',' TableRef AsOfClause? )* + +AsOfClause ::= + 'AS' 'OF' 'TIMESTAMP' Expression +``` **WhereClauseOptional:** diff --git a/sql-statements/sql-statement-set-role.md b/sql-statements/sql-statement-set-role.md index f74f8ea5d126..0711182781cc 100644 --- a/sql-statements/sql-statement-set-role.md +++ b/sql-statements/sql-statement-set-role.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/sql-statements/sql-statement-set-role/'] # SET ROLE -The `SET ROLE` statement is used to enable roles in the current session. After enabling roles, users can use the privileges of the role(s). +The `SET ROLE` statement is used to enable roles in the current session. After enabling roles, users can use the privileges of the role(s). ## Synopsis @@ -24,16 +24,15 @@ The `SET ROLE` statement is used to enable roles in the current session. After e ## Examples -Create a user `'u1'@'%'` and three roles: `'r1'@'%'`, `'r2'@'%'` and `'r3'@'%'`. -Grant these roles to `'u1'@'%'` and set `'r1'@'%'` as the default role of `'u1'@'%'`. +Create a user `'u1'@'%'` and three roles: `'r1'@'%'`, `'r2'@'%'` and `'r3'@'%'`. Grant these roles to `'u1'@'%'` and set `'r1'@'%'` as the default role of `'u1'@'%'`. {{< copyable "sql" >}} ```sql CREATE USER 'u1'@'%'; CREATE ROLE 'r1', 'r2', 'r3'; -GRANT 'r1', 'r2', 'r3' TO 'u1'@'%'; -SET DEFAULT ROLE 'r1' TO 'u1'@'%'; +GRANT 'r1', 'r2', 'r3' TO 'u1'@'%'; +SET DEFAULT ROLE 'r1' TO 'u1'@'%'; ``` Log in as `'u1'@'%'` and execute the following `SET ROLE` statement to enable all roles. diff --git a/sql-statements/sql-statement-set-transaction.md b/sql-statements/sql-statement-set-transaction.md index b01acd98fe02..56c6354d533b 100644 --- a/sql-statements/sql-statement-set-transaction.md +++ b/sql-statements/sql-statement-set-transaction.md @@ -10,17 +10,22 @@ The `SET TRANSACTION` statement can be used to change the current isolation leve ## Synopsis -**SetStmt:** +```ebnf+diagram +SetStmt ::= + 'SET' ( VariableAssignmentList | + 'PASSWORD' ('FOR' Username)? '=' PasswordOpt | + ( 'GLOBAL'| 'SESSION' )? 'TRANSACTION' TransactionChars | + 'CONFIG' ( Identifier | stringLit) ConfigItemName EqOrAssignmentEq SetExpr ) -![SetStmt](/media/sqlgram/SetStmt.png) +TransactionChars ::= + ( 'ISOLATION' 'LEVEL' IsolationLevel | 'READ' 'WRITE' | 'READ' 'ONLY' AsOfClause? ) -**TransactionChar:** +IsolationLevel ::= + ( 'REPEATABLE' 'READ' | 'READ' ( 'COMMITTED' | 'UNCOMMITTED' ) | 'SERIALIZABLE' ) -![TransactionChar](/media/sqlgram/TransactionChar.png) - -**IsolationLevel:** - -![IsolationLevel](/media/sqlgram/IsolationLevel.png) +AsOfClause ::= + ( 'AS' 'OF' 'TIMESTAMP' Expression) +``` ## Examples diff --git a/sql-statements/sql-statement-set-variable.md b/sql-statements/sql-statement-set-variable.md index 60c642a7a110..6fb9326754b5 100644 --- a/sql-statements/sql-statement-set-variable.md +++ b/sql-statements/sql-statement-set-variable.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/sql-statements/sql-statement-set-variable/','/docs/dev/refe # `SET [GLOBAL|SESSION] ` -The statement `SET [GLOBAL|SESSION]` modifies one of TiDB's built in variables, of either `SESSION` or `GLOBAL` scope. +The statement `SET [GLOBAL|SESSION]` modifies one of TiDB's built in variables, of either `SESSION` or `GLOBAL` scope. > **Note:** > @@ -44,8 +44,7 @@ mysql> SHOW SESSION VARIABLES LIKE 'sql_mode'; 1 row in set (0.00 sec) ``` -Update the value of `sql_mode` globally. -If you check the value of `SQL_mode` after the update, you can see that the value of `SESSION` level has not been updated: +Update the value of `sql_mode` globally. If you check the value of `SQL_mode` after the update, you can see that the value of `SESSION` level has not been updated: ```sql mysql> SET GLOBAL sql_mode = 'STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER'; diff --git a/sql-statements/sql-statement-show-analyze-status.md b/sql-statements/sql-statement-show-analyze-status.md index 4d31e91e7a8a..874a70595ea0 100644 --- a/sql-statements/sql-statement-show-analyze-status.md +++ b/sql-statements/sql-statement-show-analyze-status.md @@ -1,6 +1,6 @@ --- title: SHOW ANALYZE STATUS -summary: An overview of the usage of SHOW ANALYZE STATUS for the TiDB database。 +summary: An overview of the usage of SHOW ANALYZE STATUS for the TiDB database. aliases: ['/docs/dev/sql-statements/sql-statement-show-analyze-status/'] --- diff --git a/sql-statements/sql-statement-show-columns-from.md b/sql-statements/sql-statement-show-columns-from.md index 1fa2c7988ac2..f94a90618f9f 100644 --- a/sql-statements/sql-statement-show-columns-from.md +++ b/sql-statements/sql-statement-show-columns-from.md @@ -6,9 +6,13 @@ aliases: ['/docs/dev/sql-statements/sql-statement-show-columns-from/','/docs/dev # SHOW [FULL] COLUMNS FROM -This statement describes the columns of a table or view in a useful tabular format. The optional keyword `FULL` displays the privileges the current user has to that column, and the `comment` from the table definition. +The statement `SHOW [FULL] COLUMNS FROM ` describes the columns of a table or view in a useful tabular format. The optional keyword `FULL` displays the privileges the current user has to that column, and the `comment` from the table definition. -The statements `SHOW [FULL] FIELDS FROM`, `DESC
`, `DESCRIBE
` and `EXPLAIN
` are aliases of this statement. +The statements `SHOW [FULL] FIELDS FROM `, `DESC `, `DESCRIBE `, and `EXPLAIN ` are aliases of this statement. + +> **Note:** +> +> `DESC TABLE `, `DESCRIBE TABLE `, and `EXPLAIN TABLE ` are not equivalent to the above statements. They are aliases of [`DESC SELECT * FROM `](/sql-statements/sql-statement-explain.md). ## Synopsis diff --git a/sql-statements/sql-statement-show-config.md b/sql-statements/sql-statement-show-config.md index 06a4ff631ff7..71adf0b3351c 100644 --- a/sql-statements/sql-statement-show-config.md +++ b/sql-statements/sql-statement-show-config.md @@ -37,7 +37,6 @@ SHOW CONFIG; | Type | Instance | Name | Value | +------+----------------+-------------------------------------------------+---------------------------------------------------------------------+ | tidb | 127.0.0.1:4000 | advertise-address | 127.0.0.1 | -| tidb | 127.0.0.1:4000 | alter-primary-key | false | | tidb | 127.0.0.1:4000 | binlog.binlog-socket | | | tidb | 127.0.0.1:4000 | binlog.enable | false | ... @@ -74,7 +73,6 @@ SHOW CONFIG LIKE 'tidb'; | Type | Instance | Name | Value | +------+----------------+-------------------------------------------------+---------------------------------------------------------------------+ | tidb | 127.0.0.1:4000 | advertise-address | 127.0.0.1 | -| tidb | 127.0.0.1:4000 | alter-primary-key | false | | tidb | 127.0.0.1:4000 | binlog.binlog-socket | | | tidb | 127.0.0.1:4000 | binlog.enable | false | ... diff --git a/sql-statements/sql-statement-show-histograms.md b/sql-statements/sql-statement-show-histograms.md index ee3ee952f669..333a791532c6 100644 --- a/sql-statements/sql-statement-show-histograms.md +++ b/sql-statements/sql-statement-show-histograms.md @@ -1,6 +1,6 @@ --- title: SHOW STATS_HISTOGRAMS -summary: An overview of the usage of SHOW HISTOGRAMS for TiDB database。 +summary: An overview of the usage of SHOW HISTOGRAMS for TiDB database. aliases: ['/docs/dev/sql-statements/sql-statement-show-histograms/'] --- diff --git a/sql-statements/sql-statement-show-table-regions.md b/sql-statements/sql-statement-show-table-regions.md index 2c76d141a999..02482a0eecaf 100644 --- a/sql-statements/sql-statement-show-table-regions.md +++ b/sql-statements/sql-statement-show-table-regions.md @@ -151,7 +151,7 @@ In the above example: * Table t corresponds to six Regions. In these Regions, `102`, `106`, `110`, `114`, and `3` store the row data and `98` stores the index data. * For `START_KEY` and `END_KEY` of Region `102`, `t_43` indicates the table prefix and ID. `_r` is the prefix of the record data in table t. `_i` is the prefix of the index data. -* In Region `102`, `START_KEY` and `END_KEY` mean that record data in the range of `[-inf, 20000)` is stored. In similar way, the ranges of data storage in Regions (`103`, `109`, `113`, `2`) can also be calculated. +* In Region `102`, `START_KEY` and `END_KEY` mean that record data in the range of `[-inf, 20000)` is stored. In similar way, the ranges of data storage in Regions (`106`, `110`, `114`, `3`) can also be calculated. * Region `98` stores the index data. The start key of table t's index data is `t_43_i`, which is in the range of Region `98`. To check the Region that corresponds to table t in store 1, use the `WHERE` clause: diff --git a/sql-statements/sql-statement-show-variables.md b/sql-statements/sql-statement-show-variables.md index df3237ed7bb9..6a8120d4d8f6 100644 --- a/sql-statements/sql-statement-show-variables.md +++ b/sql-statements/sql-statement-show-variables.md @@ -130,7 +130,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'tidb%'; | tidb_stmt_summary_history_size | 24 | | tidb_stmt_summary_internal_query | 0 | | tidb_stmt_summary_max_sql_length | 4096 | -| tidb_stmt_summary_max_stmt_count | 200 | +| tidb_stmt_summary_max_stmt_count | 3000 | | tidb_stmt_summary_refresh_interval | 1800 | | tidb_store_limit | 0 | | tidb_txn_mode | | diff --git a/sql-statements/sql-statement-split-region.md b/sql-statements/sql-statement-split-region.md index 1af059c5af02..28accb9dfa76 100644 --- a/sql-statements/sql-statement-split-region.md +++ b/sql-statements/sql-statement-split-region.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/sql-statements/sql-statement-split-region/','/docs/dev/refe # Split Region -For each new table created in TiDB, one Region is segmented by default to store the data of this table. This default behavior is controlled by `split-table` in the configuration file. When the data in this Region exceeds the default Region size limit, the Region starts to split into two. +For each new table created in TiDB, one [Region](/tidb-storage.md#region) is segmented by default to store the data of this table. This default behavior is controlled by `split-table` in the TiDB configuration file. When the data in this Region exceeds the default Region size limit, the Region starts to split into two. In the above case, because there is only one Region at the beginning, all write requests occur on the TiKV where the Region is located. If there are a large number of writes for the newly created table, hotspots are caused. @@ -46,21 +46,25 @@ To solve the hotspot problem in the above scenario, TiDB introduces the pre-spli There are two types of Split Region syntax: -{{< copyable "sql" >}} +- The syntax of even split: -```sql -SPLIT TABLE table_name [INDEX index_name] BETWEEN (lower_value) AND (upper_value) REGIONS region_num -``` + {{< copyable "sql" >}} -`BETWEEN lower_value AND upper_value REGIONS region_num` defines the upper boundary, the lower boundary, and the Region amount. Then the current region will be evenly spilt into the number of regions (as specified in `region_num`) between the upper and lower boundaries. + ```sql + SPLIT TABLE table_name [INDEX index_name] BETWEEN (lower_value) AND (upper_value) REGIONS region_num + ``` -{{< copyable "sql" >}} + `BETWEEN lower_value AND upper_value REGIONS region_num` defines the upper boundary, the lower boundary, and the Region amount. Then the current region will be evenly spilt into the number of regions (as specified in `region_num`) between the upper and lower boundaries. -```sql -SPLIT TABLE table_name [INDEX index_name] BY (value_list) [, (value_list)] ... -``` +- The syntax of uneven split: -`BY value_list…` specifies a series of points manually, based on which the current Region is spilt. It is suitable for scenarios with unevenly distributed data. + {{< copyable "sql" >}} + + ```sql + SPLIT TABLE table_name [INDEX index_name] BY (value_list) [, (value_list)] ... + ``` + + `BY value_list…` specifies a series of points manually, based on which the current Region is spilt. It is suitable for scenarios with unevenly distributed data. The following example shows the result of the `SPLIT` statement: @@ -100,7 +104,7 @@ Row data in the same table have the same `table_id`, but each row has its unique #### Even Split -Because `row_id` is an integer, the value of the key to be split can be calculated according to the specified `lower_value`, `upper_value`, and `region_num`. TiDB first calculates the step value (`step = (upper_value - lower_value)/num`). Then split will be done evenly per each "step" between `lower_value` and `upper_value` to generate the number of Regions as specified by `num`. +Because `row_id` is an integer, the value of the key to be split can be calculated according to the specified `lower_value`, `upper_value`, and `region_num`. TiDB first calculates the step value (`step = (upper_value - lower_value)/region_num`). Then split will be done evenly per each "step" between `lower_value` and `upper_value` to generate the number of Regions as specified by `region_num`. For example, if you want 16 evenly split Regions split from key range`minInt64`~`maxInt64` for table t, you can use this statement: @@ -263,9 +267,9 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or SPLIT [PARTITION] TABLE table_name [PARTITION (partition_name_list...)] [INDEX index_name] BY (value_list) [, (value_list)] ... ``` -#### Examples +#### Examples of Split Regions for partitioned tables -1. Create a partitioned table `t`. +1. Create a partitioned table `t`. Suppose that you want to create a Hash table divided into two partitions. The example statement is as follows: {{< copyable "sql" >}} @@ -290,7 +294,7 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or +-----------+-----------+---------+-----------+-----------------+------------------+------------+---------------+------------+----------------------+------------------+ ``` -2. Use the `SPLIT` syntax to split a Region for each partition. In the following example, four Regions are split in the range of `[0,10000]`. +2. Use the `SPLIT` syntax to split a Region for each partition. Suppose that you want to split the data in the `[0,10000]` range of each partition into four Regions. The example statement is as follows: {{< copyable "sql" >}} @@ -298,6 +302,12 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or split partition table t between (0) and (10000) regions 4; ``` + In the above statement, `0` and `10000` respectively represent the `row_id` of the upper and lower boundaries corresponding to the hotspot data you want to scatter. + + > **Note:** + > + > This example only applies to scenarios where hotspot data is evenly distributed. If the hotspot data is unevenly distributed in a specified data range, refer to the syntax of uneven split in [Split Regions for partitioned tables](#split-regions-for-partitioned-tables). + 3. Use the `SHOW TABLE REGIONS` syntax to view the Regions of this table again. You can see that this table now has ten Regions, each partition with five Regions, four of which are the row data and one is the index data. {{< copyable "sql" >}} @@ -323,7 +333,7 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or +-----------+---------------+---------------+-----------+-----------------+------------------+------------+---------------+------------+----------------------+------------------+ ``` -4. You can also split Regions for the index of each partition. For example, you can split the `[1000,10000]` range of the `idx` index into two Regions: +4. You can also split Regions for the index of each partition. For example, you can split the `[1000,10000]` range of the `idx` index into two Regions. The example statement is as follows: {{< copyable "sql" >}} @@ -331,11 +341,11 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or split partition table t index idx between (1000) and (10000) regions 2; ``` -#### Split Region for a single partition +#### Examples of Split Region for a single partition -You can specify the partition to be split. See the following usage example: +You can specify the partition to be split. -1. Create a partitioned table: +1. Create a partitioned table. Suppose that you want to create a Range partitioned table split into three partitions. The example statement is as follows: {{< copyable "sql" >}} @@ -346,7 +356,7 @@ You can specify the partition to be split. See the following usage example: partition p3 values less than (MAXVALUE) ); ``` -2. Split two Regions in the `[0,10000]` range of the `p1` partition: +2. Suppose that you want to split the data in the `[0,10000]` range of the `p1` partition into two Regions. The example statement is as follows: {{< copyable "sql" >}} @@ -354,7 +364,7 @@ You can specify the partition to be split. See the following usage example: split partition table t partition (p1) between (0) and (10000) regions 2; ``` -3. Split two Regions in the `[10000,20000]` range of the `p2` partition: +3. Suppose that you want to split the data in the `[10000,20000]` range of the `p2` partition into two Regions. The example statement is as follows: {{< copyable "sql" >}} @@ -362,7 +372,7 @@ You can specify the partition to be split. See the following usage example: split partition table t partition (p2) between (10000) and (20000) regions 2; ``` -4. Use the `SHOW TABLE REGIONS` syntax to view the Regions of this table: +4. You can use the `SHOW TABLE REGIONS` syntax to view the Regions of this table: {{< copyable "sql" >}} @@ -382,7 +392,7 @@ You can specify the partition to be split. See the following usage example: +-----------+----------------+----------------+-----------+-----------------+------------------+------------+---------------+------------+----------------------+------------------+ ``` -5. Split two Regions in the `[0,20000]` range of the `idx` index of `p1` and `p2` partitions: +5. Suppose that you want to split the `[0,20000]` range of the `idx` index of the `p1` and `p2` partitions into two Regions. The example statement is as follows: {{< copyable "sql" >}} @@ -400,7 +410,7 @@ To have evenly split Regions when a table is created, it is recommended you use The `tidb_scatter_region` global variable affects the behavior of `PRE_SPLIT_REGIONS`. This variable controls whether to wait for Regions to be pre-split and scattered before returning results after the table creation. If there are intensive writes after creating the table, you need to set the value of this variable to `1`, then TiDB will not return the results to the client until all the Regions are split and scattered. Otherwise, TiDB writes the data before the scattering is completed, which will have a significant impact on write performance. -### Example +### Examples of pre_split_regions {{< copyable "sql" >}} @@ -419,6 +429,10 @@ region3: [ 2<<61 , 3<<61 ) region4: [ 3<<61 , +inf ) ``` +## Notes + +The Region split by the Split Region statement is controlled by the [Region merge](/best-practices/pd-scheduling-best-practices.md#region-merge) scheduler in PD. To avoid PD re-merging the newly split Region soon after, you need to [dynamically modify](/pd-control.md) configuration items related to the Region merge feature. + ## MySQL compatibility This statement is a TiDB extension to MySQL syntax. diff --git a/sql-statements/sql-statement-start-transaction.md b/sql-statements/sql-statement-start-transaction.md index 89f33f7e8d49..b943d8f2c69d 100644 --- a/sql-statements/sql-statement-start-transaction.md +++ b/sql-statements/sql-statement-start-transaction.md @@ -14,7 +14,14 @@ In the absence of a `START TRANSACTION` statement, every statement will by defau **BeginTransactionStmt:** -![BeginTransactionStmt](/media/sqlgram/BeginTransactionStmt.png) +```ebnf+diagram +BeginTransactionStmt ::= + 'BEGIN' ( 'PESSIMISTIC' | 'OPTIMISTIC' )? +| 'START' 'TRANSACTION' ( 'READ' ( 'WRITE' | 'ONLY' ( ( 'WITH' 'TIMESTAMP' 'BOUND' TimestampBound )? | AsOfClause ) ) | 'WITH' 'CONSISTENT' 'SNAPSHOT' | 'WITH' 'CAUSAL' 'CONSISTENCY' 'ONLY' )? + +AsOfClause ::= + ( 'AS' 'OF' 'TIMESTAMP' Expression) +``` ## Examples @@ -43,3 +50,4 @@ Query OK, 0 rows affected (0.01 sec) * [COMMIT](/sql-statements/sql-statement-commit.md) * [ROLLBACK](/sql-statements/sql-statement-rollback.md) * [BEGIN](/sql-statements/sql-statement-begin.md) +* [START TRANSACTION WITH CAUSAL CONSISTENCY ONLY](/transaction-overview.md#causal-consistency) diff --git a/sql-statements/sql-statement-table.md b/sql-statements/sql-statement-table.md new file mode 100644 index 000000000000..2bdba64f50f3 --- /dev/null +++ b/sql-statements/sql-statement-table.md @@ -0,0 +1,96 @@ +--- +title: TABLE | TiDB SQL Statement Reference +summary: An overview of the usage of TABLE for the TiDB database. +--- + +# TABLE + +The `TABLE` statement can be used instead of `SELECT * FROM` when no aggregation or complex filtering is needed. + +## Synopsis + +```ebnf+diagram +TableStmt ::= + "TABLE" Table ( "ORDER BY" Column )? ( "LIMIT" NUM )? +``` + +## Examples + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t1(id INT PRIMARY KEY); +``` + +```sql +Query OK, 0 rows affected (0.31 sec) +``` + +{{< copyable "sql" >}} + +```sql +INSERT INTO t1 VALUES (1),(2),(3); +``` + +```sql +Query OK, 3 rows affected (0.06 sec) +Records: 3 Duplicates: 0 Warnings: 0 +``` + +{{< copyable "sql" >}} + +```sql +TABLE t1; +``` + +```sql ++----+ +| id | ++----+ +| 1 | +| 2 | +| 3 | ++----+ +3 rows in set (0.01 sec) +``` + +{{< copyable "sql" >}} + +```sql +TABLE t1 ORDER BY id DESC; +``` + +```sql ++----+ +| id | ++----+ +| 3 | +| 2 | +| 1 | ++----+ +3 rows in set (0.01 sec) +``` + +{{< copyable "sql" >}} + +```sql +TABLE t1 LIMIT 1; +``` + +```sql ++----+ +| id | ++----+ +| 1 | ++----+ +1 row in set (0.01 sec) +``` + +## MySQL compatibility + +The `TABLE` statement was introduced in MySQL 8.0.19. + +## See also + +- [SELECT](/sql-statements/sql-statement-select.md) +- [TABLE statements in MySQL](https://dev.mysql.com/doc/refman/8.0/en/table.html) diff --git a/sql-statements/sql-statement-with.md b/sql-statements/sql-statement-with.md new file mode 100644 index 000000000000..e1cd8c98af35 --- /dev/null +++ b/sql-statements/sql-statement-with.md @@ -0,0 +1,95 @@ +--- +title: WITH | TiDB SQL Statement Reference +summary: An overview of the usage of WITH (Common Table Expression) for the TiDB database. +--- + +# WITH + +A Common Table Expression (CTE) is a temporary result set that can be referred multiple times within a SQL statement to improve the statement's readability and execution efficiency. You can apply the `WITH` statement to use Common Table Expressions. + +## Synopsis + +**WithClause:** + +```ebnf+diagram +WithClause ::= + "WITH" WithList +| "WITH" recursive WithList +``` + +**WithList:** + +```ebnf+diagram +WithList ::= + WithList ',' CommonTableExpr +| CommonTableExpr +``` + +**CommonTableExpr:** + +```ebnf+diagram +CommonTableExpr ::= + Identifier IdentListWithParenOpt "AS" SubSelect +``` + +**IdentListWithParenOpt:** + +```ebnf+diagram +IdentListWithParenOpt ::= +| '(' IdentList ')' +``` + +## Examples + +Non-recursive CTE: + +{{< copyable "sql" >}} + +```sql +WITH CTE AS (SELECT 1, 2) SELECT * FROM cte t1, cte t2; +``` + +``` ++---+---+---+---+ +| 1 | 2 | 1 | 2 | ++---+---+---+---+ +| 1 | 2 | 1 | 2 | ++---+---+---+---+ +1 row in set (0.00 sec) +``` + +Recursive CTE: + +{{< copyable "sql" >}} + +```sql +WITH RECURSIVE cte(a) AS (SELECT 1 UNION SELECT a+1 FROM cte WHERE a < 5) SELECT * FROM cte; +``` + +``` ++---+ +| a | ++---+ +| 1 | +| 2 | +| 3 | +| 4 | +| 5 | ++---+ +5 rows in set (0.00 sec) +``` + +## MySQL compatibility + +* In strict mode, when the data length recursively calculated exceeds the data length of the seed part, TiDB returns a warning while MySQL returns an error. In non-strict mode, the behavior of TiDB is consistent with that of MySQL. +* The data type for recursive CTE is determined by the seed part. The data type of the seed part is not completely consistent with MySQL in some cases (such as functions). +* In the case of multiple `UNION` / `UNION ALL` operators, MySQL does not allow `UNION` to be followed by `UNION ALL`, but TiDB does. +* If there is a problem with the definition of a CTE, TiDB will report an error, while MySQL will not if the CTE is not referred. + +## See also + +* [SELECT](/sql-statements/sql-statement-select.md) +* [INSERT](/sql-statements/sql-statement-insert.md) +* [DELETE](/sql-statements/sql-statement-delete.md) +* [UPDATE](/sql-statements/sql-statement-update.md) +* [REPLACE](/sql-statements/sql-statement-replace.md) diff --git a/stale-read.md b/stale-read.md new file mode 100644 index 000000000000..ddccc76b31d0 --- /dev/null +++ b/stale-read.md @@ -0,0 +1,23 @@ +--- +title: Usage Scenarios of Stale Read +summary: Learn about Stale Read and its usage scenarios. +--- + +# Usage Scenarios of Stale Read + +This document describes the usage scenarios of Stale Read. Stale Read is a mechanism that TiDB applies to read historical versions of data stored in TiDB. Using this mechanism, you can read the corresponding historical data of a specific point in time or within a specified time range, and thus save the latency brought by data replication between storage nodes. + +When you are using Steal Read, TiDB will randomly select a replica for data reading, which means that all replicas are available for data reading. If your application cannot tolerate reading non-real-time data, do not use Stale Read; otherwise, the data read from the replica might not be the latest data written into TiDB. + +## Scenario examples + ++ Scenario one: If a transaction only involves read operations and is tolerant of data staleness to some extent, you can use Stale Read to get historical data. Using Stale Read, TiDB makes the query requests sent to any replica at the expense of some real-time performance, and thus increases the throughput of query executions. Especially in some scenarios where small tables are queried, if strongly consistent reads are used, leader might be concentrated on a certain storage node, causing the query pressure to be concentrated on that node as well. Therefore, that node might become a bottleneck for the whole query. Stale Read, however, can improve the overall query throughput and significantly improve the query performance. + ++ Scenario two: In some scenarios of geo-distributed deployment, if strongly consistent follower reads are used, to make sure that the data read from the Followers is consistent with that stored in the Leader, TiDB requests `Readindex` from different data centers for verification, which increases the access latency for the whole query process. With Stale Read, TiDB accesses the replica in the current data center to read the corresponding data at the expense of some real-time performance, which avoids network latency brought by cross-center connection and reduces the access latency for the entire query. For more information, see [Local Read under Three Data Centers Deployment](/best-practices/three-dc-local-read.md). + +## Usages + +In TiDB, you can specify either an exact point in time or a time range when performing stale reads: + +- Specifying an exact point in time (recommended): If you need TiDB to read data that follows the global transaction consistency from a specific point in time without damaging the isolation level, you can specify the corresponding timestamp of that point in time in the query statement. For detailed usage, see [`AS OF TIMESTAMP` Clause](/as-of-timestamp.md#syntax). +- Specifying a time range: If you need TiDB to read data as new as possible within a time range without damaging the isolation level, you can specify the time range in the query statement. Then, TiDB selects a suitable timestamp within the specified time range to read the corresponding data. "Suitable" means there are no transactions that start before this timestamp and have not been committed on the accessed replica, that is, TiDB can perform read operations on the accessed replica and the read operations are not blocked. For detailed usage, refer to the introduction of the [`AS OF TIMESTAMP` clause](/as-of-timestamp.md#syntax) and the [`TIDB_BOUNDED_STALENESS` function](/as-of-timestamp.md#syntax). diff --git a/statement-summary-tables.md b/statement-summary-tables.md index 298287874c90..740869a12425 100644 --- a/statement-summary-tables.md +++ b/statement-summary-tables.md @@ -31,7 +31,6 @@ select * from EMPLOYEE where ID in (4, 5) and SALARY between 3000 and 4000; ``` After normalization, they are both of the following category: -aliases: ['/docs/dev/reference/performance/statement-summary/'] {{< copyable "sql" >}} @@ -103,10 +102,14 @@ The following system variables are used to control the statement summary: - `tidb_enable_stmt_summary`: Determines whether to enable the statement summary feature. `1` represents `enable`, and `0` means `disable`. The feature is enabled by default. The statistics in the system table are cleared if this feature is disabled. The statistics are re-calculated next time this feature is enabled. Tests have shown that enabling this feature has little impact on performance. - `tidb_stmt_summary_refresh_interval`: The interval at which the `statements_summary` table is refreshed. The time unit is second (s). The default value is `1800`. - `tidb_stmt_summary_history_size`: The size of each SQL statement category stored in the `statements_summary_history` table. The default value is `24`. -- `tidb_stmt_summary_max_stmt_count`: Limits the number of SQL statements that can be stored in statement summary tables. The default value is `200`. If the limit is exceeded, those SQL statements that recently remain unused are cleared. +- `tidb_stmt_summary_max_stmt_count`: Limits the number of SQL statements that can be stored in statement summary tables. The default value is `3000`. If the limit is exceeded, those SQL statements that recently remain unused are cleared. - `tidb_stmt_summary_max_sql_length`: Specifies the longest display length of `DIGEST_TEXT` and `QUERY_SAMPLE_TEXT`. The default value is `4096`. - `tidb_stmt_summary_internal_query`: Determines whether to count the TiDB SQL statements. `1` means to count, and `0` means not to count. The default value is `0`. +> **Note:** +> +> When a category of SQL statement needs to be removed because the `tidb_stmt_summary_max_stmt_count` limit is exceeded, TiDB removes the data of that SQL statement category of all time ranges from the `statement summary history` table. Therefore, even if the number of SQL statement categories in a certain time range does not reach the limit, the number of SQL statements stored in the `statement summary history` table is less than the actual number of SQL statements. If this situation occurs, you are recommended to increase the value of `tidb_stmt_summary_max_stmt_count`. + An example of the statement summary configuration is shown as follows: {{< copyable "sql" >}} diff --git a/statistics.md b/statistics.md index b5165d918e9e..aba956eacf9e 100644 --- a/statistics.md +++ b/statistics.md @@ -6,7 +6,25 @@ aliases: ['/docs/dev/statistics/','/docs/dev/reference/performance/statistics/'] # Introduction to Statistics -In TiDB, the statistical information you need to maintain includes the total number of rows in the table, the equal-depth histogram of columns, Count-Min Sketch, the number of `Null`s, the average length, the number of different values, etc. This document briefly introduces the histogram and Count-Min Sketch, and details the collection and maintenance of statistics. +TiDB uses statistics to decide [which index to choose](/choose-index.md). The `tidb_analyze_version` variable controls the statistics collected by TiDB. Currently, two versions of statistics are supported: `tidb_analyze_version = 1` and `tidb_analyze_version = 2`. In versions before v5.1.0, the default value of this variable is `1`. In v5.1.0, the default value of this variable is `2`, which serves as an experimental feature. These two versions include different information in TiDB: + +| Information | Version 1 | Version 2| +| --- | --- | ---| +| The total number of rows in the table | √ | √ | +| Column Count-Min Sketch | √ | × | +| Index Count-Min Sketch | √ | × | +| Column Top-N | √ | √ (Maintenance methods and precision are improved) | +| Index Top-N | √ (Insufficient maintenance precision might cause inaccuracy) | √ (Maintenance methods and precision are improved) | +| Column histogram | √ | √ (The histogram does not include Top-N values.) | +| Index histogram | √ | √ (The histogram buckets record the number of different values in each bucket, and the histogram does not include Top-N values.) | +| The number of `NULL`s in the column | √ | √ | +| The number of `NULL`s in the index | √ | √ | +| The average length of columns | √ | √ | +| The average length of indexes | √ | √ | + +Compared to Version 1, Version 2 statistics avoids the potential inaccuracy caused by hash collision when the data volume is huge. It also maintains the estimate precision in most scenarios. + +This document briefly introduces the histogram, Count-Min Sketch, and Top-N, and details the collection and maintenance of statistics. ## Histogram @@ -27,14 +45,16 @@ A hash collision might occur since Count-Min Sketch is a hash structure. In the - Modify the `WITH NUM TOPN` parameter. TiDB stores the high-frequency (top x) data separately, with the other data stored in Count-Min Sketch. Therefore, to prevent a larger value and a smaller value from being hashed together, you can increase the value of `WITH NUM TOPN`. In TiDB, its default value is 20. The maximum value is 1024. For more information about this parameter, see [Full Collection](#full-collection). - Modify two parameters `WITH NUM CMSKETCH DEPTH` and `WITH NUM CMSKETCH WIDTH`. Both affect the number of hash buckets and the collision probability. You can increase the values of the two parameters appropriately according to the actual scenario to reduce the probability of hash collision, but at the cost of higher memory usage of statistics. In TiDB, the default value of `WITH NUM CMSKETCH DEPTH` is 5, and the default value of `WITH NUM CMSKETCH WIDTH` is 2048. For more information about the two parameters, see [Full Collection](#full-collection). +## Top-N values + +Top-N values are values with the top N occurrences in a column or index. TiDB records the values and occurences of Top-N values. + ## Collect statistics ### Manual collection You can run the `ANALYZE` statement to collect statistics. -#### Full collection - > **Note:** > > The execution time of `ANALYZE TABLE` in TiDB is longer than that in MySQL or InnoDB. In InnoDB, only a small number of pages are sampled, while in TiDB a comprehensive set of statistics is completely rebuilt. Scripts that were written for MySQL may naively expect `ANALYZE TABLE` will be a short-lived operation. @@ -42,6 +62,10 @@ You can run the `ANALYZE` statement to collect statistics. > For quicker analysis, you can set `tidb_enable_fast_analyze` to `1` to enable the Quick Analysis feature. The default value for this parameter is `0`. > > After Quick Analysis is enabled, TiDB randomly samples approximately 10,000 rows of data to build statistics. Therefore, in the case of uneven data distribution or a relatively small amount of data, the accuracy of statistical information is relatively poor. It might lead to poor execution plans, such as choosing the wrong index. If the execution time of the normal `ANALYZE` statement is acceptable, it is recommended to disable the Quick Analysis feature. +> +> `tidb_enable_fast_analyze` is an experimental feature, which currently **does not match exactly** with the statistical information of `tidb_analyze_version=2`. Therefore, you need to set the value of `tidb_analyze_version` to `1` when `tidb_enable_fast_analyze` is enabled. + +#### Full collection You can perform full collection using the following syntax. @@ -82,9 +106,13 @@ You can perform full collection using the following syntax. {{< copyable "sql" >}} ```sql - ANALYZE TABLE TableName PARTITION PartitionNameList [IndexNameList] [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH|SAMPLES]; + ANALYZE TABLE TableName PARTITION PartitionNameList INDEX [IndexNameList] [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH|SAMPLES]; ``` +> **Note:** +> +> To ensure that the statistical information before and after the collection is consistent, when you set `tidb_analyze_version=2`, `ANALYZE TABLE TableName INDEX` will also collect statistics of the whole table instead of the given index. + #### Incremental collection To improve the speed of analysis after full collection, incremental collection could be used to analyze the newly added sections in monotonically non-decreasing columns such as time columns. @@ -126,11 +154,7 @@ Three system variables related to automatic update of statistics are as follows: When the ratio of the number of modified rows to the total number of rows of `tbl` in a table is greater than `tidb_auto_analyze_ratio`, and the current time is between `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time`, TiDB executes the `ANALYZE TABLE tbl` statement in the background to automatically update the statistics of this table. -When the query is executed, TiDB collects feedback with the probability of `feedback-probability` and uses it to update the histogram and Count-Min Sketch. You can modify the value of `feedback-probability` in the configuration file. The default value is `0.05`. You can set the value to `0.0` to disable this feature. - -> **Note:** -> -> If you set the value of `feedback-probability` to `0` in the configuration file, a failure will occur and an error will be reported. To disable `feedback-probability`, you need to set the value to `0.0`. +Before v5.0, when the query is executed, TiDB collects feedback with the probability of `feedback-probability` and uses it to update the histogram and Count-Min Sketch. **In v5.0, this feature is disabled by default, and it is not recommended to enable this feature.** ### Control `ANALYZE` concurrency @@ -168,7 +192,7 @@ Currently, the `SHOW ANALYZE STATUS` statement returns the following 7 columns: | table_name | The table name | | partition_name| The partition name | | job_info | The task information. The element includes index names when index analysis is performed. | -| row_count | The the number of rows that have been analyzed | +| row_count | The number of rows that have been analyzed | | start_time | The time at which the task starts | | state | The state of a task, including `pending`, `running`, `finished`, and `failed` | @@ -238,7 +262,7 @@ SHOW STATS_HISTOGRAMS [ShowLikeOrWhere] This statement returns the number of different values and the number of `NULL` in all the columns. You can use `ShowLikeOrWhere` to filter the information you need. -Currently, the `SHOW STATS_HISTOGRAMS` statement returns the following 8 columns: +Currently, the `SHOW STATS_HISTOGRAMS` statement returns the following 10 columns: | Syntax Element | Description | | :-------- | :------------- | @@ -251,6 +275,7 @@ Currently, the `SHOW STATS_HISTOGRAMS` statement returns the following 8 columns | `distinct_count` | The number of different values | | `null_count` | The number of `NULL` | | `avg_col_size` | The average length of columns | +| correlation | The Pearson correlation coefficient of the column and the integer primary key, which indicates the degree of association between the two columns| ### Buckets of histogram @@ -270,7 +295,7 @@ The diagram is as follows: This statement returns information about all the buckets. You can use `ShowLikeOrWhere` to filter the information you need. -Currently, the `SHOW STATS_BUCKETS` statement returns the following 10 columns: +Currently, the `SHOW STATS_BUCKETS` statement returns the following 11 columns: | Syntax Element | Description | | :-------- | :------------- | @@ -284,6 +309,31 @@ Currently, the `SHOW STATS_BUCKETS` statement returns the following 10 columns: | `repeats` | The occurrence number of the maximum value | | `lower_bound` | The minimum value | | `upper_bound` | The maximum value | +| `ndv` | The number of different values in the bucket. When `tidb_analyze_version` = `1`, `ndv` is always `0`, which has no actual meaning. | + +### Top-N information + +You can use the `SHOW STATS_TOPN` statement to view the Top-N information currently collected by TiDB. + +The syntax is as follows: + +{{< copyable "sql" >}} + +```sql +SHOW STATS_TOPN [ShowLikeOrWhere]; +``` + +Currently, the `SHOW STATS_TOPN` statement returns the following 7 columns: + +| Syntax Element | Description | +| ---- | ----| +| `db_name` | The database name | +| `table_name` | The table name | +| `partition_name` | The partition name | +| `column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | +| `is_index` | Whether it is an index column or not | +| `value` | The value of this column | +| `count` | How many times the value appears | ## Delete statistics diff --git a/storage-engine/titan-overview.md b/storage-engine/titan-overview.md index db4a9fed9835..d1fa7ecac018 100644 --- a/storage-engine/titan-overview.md +++ b/storage-engine/titan-overview.md @@ -27,7 +27,7 @@ Titan is suitable for the scenarios where a huge volume of data is written to th The prerequisites for enabling Titan are as follows: -- The average size of values is large, or the size of all large values accounts for much of the total value size. Currently, the size of a value greater than 1 KB is considered as a large value. In some situations, this number (1 KB) can be 512 B. Note that a single value written to TiKV cannot exceed 6 MB due to the limitation of the TiKV Raft layer. +- The average size of values is large, or the size of all large values accounts for much of the total value size. Currently, the size of a value greater than 1 KB is considered as a large value. In some situations, this number (1 KB) can be 512 B. Note that a single value written to TiKV cannot exceed 8 MB due to the limitation of the TiKV Raft layer. You can adjust the [`raft-entry-max-size`](/tikv-configuration-file.md#raft-entry-max-size) configuration value to relax the limit. - No range query will be performed or you do not need a high performance of range query. Because the data stored in Titan is not well-ordered, its performance of range query is poorer than that of RocksDB, especially for the query of a large range. According PingCAP's internal test, Titan's range query performance is 40% to a few times lower than that of RocksDB. - Sufficient disk space, because Titan reduces write amplification at the cost of disk space. In addition, Titan compresses values one by one, and its compression rate is lower than that of RocksDB. RocksDB compresses blocks one by one. Therefore, Titan consumes more storage space than RocksDB, which is expected and normal. In some situations, Titan's storage consumption can be twice that of RocksDB. diff --git a/sync-diff-inspector/sync-diff-inspector-overview.md b/sync-diff-inspector/sync-diff-inspector-overview.md index a2674c04f05e..bcbecd9df8cb 100644 --- a/sync-diff-inspector/sync-diff-inspector-overview.md +++ b/sync-diff-inspector/sync-diff-inspector-overview.md @@ -142,7 +142,7 @@ fix-sql-file = "fix.sql" # Specifies the column used to divide data into chunks. If you do not configure it, # sync-diff-inspector chooses an appropriate column (primary key, unique key, or a field with index). - index-field = "id" + index-fields = "id" # Specifies the range of the data to be checked # It needs to comply with the syntax of the WHERE clause in SQL. diff --git a/syncer-overview.md b/syncer-overview.md deleted file mode 100644 index ac29cf778790..000000000000 --- a/syncer-overview.md +++ /dev/null @@ -1,575 +0,0 @@ ---- -title: Syncer User Guide -summary: Use Syncer to import data incrementally to TiDB. -aliases: ['/docs/dev/syncer-overview/','/docs/dev/reference/tools/syncer/'] ---- - -# Syncer User Guide - -> **Warning:** -> -> Syncer is no longer maintained. Its features are completely superseded by [TiDB Data Migration](https://pingcap.com/docs-cn/tidb-data-migration/stable/overview/). It is strongly recommended that you use TiDB Data Migration instead. - -## About Syncer - -Syncer is a tool used to import data incrementally. It is a part of the TiDB enterprise toolset. - -It can be [downloaded](/download-ecosystem-tools.md) as part of the Enterprise Tools package. - -## Syncer architecture - -![syncer sharding](/media/syncer_architecture.png) - -## Where to deploy Syncer - -You can deploy Syncer to any of the machines that can connect to MySQL or the TiDB cluster. But it is recommended to deploy Syncer to the TiDB cluster. - -## Use Syncer to import data incrementally - -Before importing data, read [Check before importing data using Syncer](#check-before-replicating-data-using-syncer). - -### 1. Set the position to replicate - -Edit the meta file of Syncer, assuming the meta file is `syncer.meta`: - -```bash -# cat syncer.meta -binlog-name = "mysql-bin.000003" -binlog-pos = 930143241 -binlog-gtid = "2bfabd22-fff7-11e6-97f7-f02fa73bcb01:1-23,61ccbb5d-c82d-11e6-ac2e-487b6bd31bf7:1-4" -``` - -> **Note:** -> -> - The `syncer.meta` file only needs to be configured when it is first used. The position is automatically updated when the new subsequent binlog is replicated. -> - If you use the binlog position to replicate, you only need to configure `binlog-name` and `binlog-pos`; if you use `binlog-gtid` to replicate, you need to configure `binlog-gtid` and set `--enable-gtid` when starting Syncer. - -### 2. Start Syncer - -Description of Syncer command line options: - -``` -Usage of syncer: - -L string - log level: debug, info, warn, error, fatal (default "info") - -V to print Syncer version info (default false) - -b int - the size of batch transactions (default 100) - -c int - the number of batch threads that Syncer processes (default 16) - -config string - to specify the corresponding configuration file when starting Syncer; for example, `--config config.toml` - -enable-ansi-quotes - to enable ANSI_QUOTES sql_mode - -enable-gtid - to start Syncer using the mode; default false; before enabling this option, you need to enable GTID in the upstream MySQL - -flavor string - use flavor for different MySQL source versions; support "mysql", "mariadb" now; if you replicate data from MariaDB, set it to "mariadb" (default "mysql") - -log-file string - to specify the log file directory, such as `--log-file ./syncer.log` - -log-rotate string - to specify the log file rotating cycle, hour/day (default "day") - -max-retry int - to specify the maximum times an SQL statement should be retried. One common cause of statement retries is network interruption (default 100) - -meta string - to specify the meta file of the upstream of Syncer (in the same directory with the configuration file, "syncer.meta" by default) - -persistent-dir string - to specify the persistent file (historical reason: it is not a directory) of Syncer history table schemas; if you set it to a non-empty string, the history table schema is chosen according to the column length when you construct DML statements - -safe-mode - to specify and enable the safe mode to make Syncer reentrant - -server-id int - to specify MySQL slave sever-id (default 101) - -status-addr string - to specify Syncer metrics (default :8271), such as `--status-addr 127:0.0.1:8271` - -timezone string - the time zone used by the target database; it is required to use the IANA time zone identifier such as `Asia/Shanghai` -``` - -The `config.toml` configuration file of Syncer: - -```toml -log-level = "info" -log-file = "syncer.log" -log-rotate = "day" - -server-id = 101 - -# The file path for meta: -meta = "./syncer.meta" -worker-count = 16 -batch = 100 -flavor = "mysql" - -# It can be used by Prometheus to pull Syncer metrics, and is also the pprof address of Syncer. -status-addr = ":8271" - -# If you set its value to true, Syncer stops and exits when it encounters the DDL operation. -stop-on-ddl = false - -# The maximum number of times an SQL statement should be retried. One common cause of statement retries is network interruption. -max-retry = 100 - -# Specify the time zone used by the target database; all timestamp fields in binlog are converted according to the time zone; the local time zone of Syncer is used by default. -# timezone = "Asia/Shanghai" - -# Skip the DDL statement; the format is **prefix exact match**, for example, you need to fill at least `DROP TABLE` in to skip `DROP TABLE ABC`. -# skip-ddls = ["ALTER USER", "CREATE USER"] - -# After Syncer uses `route-rules` to map the upstream schema and table into `target-schema` and `target-table`, -# Syncer matches the mapped `target-schema` and `target-table` with do/ignore rules, -# and the matching sequence is: replicate-do-db --> replicate-do-table --> replicate-ignore-db --> replicate-ignore-table. -# Specify the database name to be replicated. Support regular expressions. Start with '~' to use regular expressions. -# replicate-do-db = ["~^b.*","s1"] - -# Specify the database you want to ignore in replication. Support regular expressions. Start with '~' to use regular expressions. -# replicate-ignore-db = ["~^b.*","s1"] - -# skip-dmls skips the DML binlog events. The type value can be 'insert', 'update' and 'delete'. -# The 'delete' statements that skip-dmls skips in the foo.bar table: -# [[skip-dmls]] -# db-name = "foo" -# tbl-name = "bar" -# type = "delete" -# -# The 'delete' statements that skip-dmls skips in all tables: -# [[skip-dmls]] -# type = "delete" -# -# The 'delete' statements that skip-dmls skips in all foo.* tables: -# [[skip-dmls]] -# db-name = "foo" -# type = "delete" - -# Specify the db.table to be replicated. -# db-name and tbl-name do not support the `db-name ="dbname, dbname2"` format. -# [[replicate-do-table]] -# db-name ="dbname" -# tbl-name = "table-name" - -# [[replicate-do-table]] -# db-name ="dbname1" -# tbl-name = "table-name1" - -# Specify the db.table to be replicated. Support regular expressions. Start with '~' to use regular expressions. -# [[replicate-do-table]] -# db-name ="test" -# tbl-name = "~^a.*" - -# Specify the database table you want to ignore in replication. -# db-name and tbl-name do not support the `db-name ="dbname, dbname2"` format. -# [[replicate-ignore-table]] -# db-name = "your_db" -# tbl-name = "your_table" - -# Specify the database table you want to ignore in replication. Support regular expressions. Start with '~' to use regular expressions. -# [[replicate-ignore-table]] -# db-name ="test" -# tbl-name = "~^a.*" - -# The sharding replicating rules support wildcharacter. -# 1. The asterisk character ("*", also called "star") matches zero or more characters, -# For example, "doc*" matches "doc" and "document" but not "dodo"; -# The asterisk character must be in the end of the wildcard word, -# and there is only one asterisk in one wildcard word. -# 2. The question mark ("?") matches any single character. -# [[route-rules]] -# pattern-schema = "route_*" -# pattern-table = "abc_*" -# target-schema = "route" -# target-table = "abc" - -# [[route-rules]] -# pattern-schema = "route_*" -# pattern-table = "xyz_*" -# target-schema = "route" -# target-table = "xyz" - -[from] -host = "127.0.0.1" -user = "root" -password = "" -port = 3306 - -[to] -host = "127.0.0.1" -user = "root" -password = "" -port = 4000 -``` - -Start Syncer: - -```bash -./bin/syncer -config config.toml - -2016/10/27 15:22:01 binlogsyncer.go:226: [info] begin to sync binlog from position (mysql-bin.000003, 1280) -2016/10/27 15:22:01 binlogsyncer.go:130: [info] register slave for master server 127.0.0.1:3306 -2016/10/27 15:22:01 binlogsyncer.go:552: [info] rotate to (mysql-bin.000003, 1280) -2016/10/27 15:22:01 syncer.go:549: [info] rotate binlog to (mysql-bin.000003, 1280) -``` - -### 3. Insert data into MySQL - -```sql -INSERT INTO t1 VALUES (4, 4), (5, 5); -``` - -### 4. Log in to TiDB and view the data - -```sql -mysql -h127.0.0.1 -P4000 -uroot -p -mysql> select * from t1; -+----+------+ -| id | age | -+----+------+ -| 1 | 1 | -| 2 | 2 | -| 3 | 3 | -| 4 | 4 | -| 5 | 5 | -+----+------+ -``` - -Syncer outputs the current replicated data statistics every 30 seconds: - -```bash -2017/06/08 01:18:51 syncer.go:934: [info] [syncer]total events = 15, total tps = 130, recent tps = 4, -master-binlog = (ON.000001, 11992), master-binlog-gtid=53ea0ed1-9bf8-11e6-8bea-64006a897c73:1-74, -syncer-binlog = (ON.000001, 2504), syncer-binlog-gtid = 53ea0ed1-9bf8-11e6-8bea-64006a897c73:1-17 -2017/06/08 01:19:21 syncer.go:934: [info] [syncer]total events = 15, total tps = 191, recent tps = 2, -master-binlog = (ON.000001, 11992), master-binlog-gtid=53ea0ed1-9bf8-11e6-8bea-64006a897c73:1-74, -syncer-binlog = (ON.000001, 2504), syncer-binlog-gtid = 53ea0ed1-9bf8-11e6-8bea-64006a897c73:1-35 -``` - -The update in MySQL is automatically replicated in TiDB. - -## Description of Syncer configuration - -### Specify the database to be replicated - -This section describes the priority of parameters when you use Syncer to replicate the database. - -- To use the route-rules, see [Support for replicating data from sharded tables](#support-for-replicating-data-from-sharded-tables). -- Priority: replicate-do-db --> replicate-do-table --> replicate-ignore-db --> replicate-ignore-table - -```toml -# Specify the ops database to be replicated. -# Specify to replicate the database starting with ti. -replicate-do-db = ["ops","~^ti.*"] - -# The "china" database includes multiple tables such as guangzhou, shanghai and beijing. You only need to replicate the shanghai and beijing tables. -# Specify to replicate the shanghai table in the "china" database. -[[replicate-do-table]] -db-name ="china" -tbl-name = "shanghai" - -# Specify to replicate the beijing table in the "china" database. -[[replicate-do-table]] -db-name ="china" -tbl-name = "beijing" - -# The "ops" database includes multiple tables such as ops_user, ops_admin, weekly. You only need to replicate the ops_user table. -# Because replicate-do-db has a higher priority than replicate-do-table, it is invalid if you only set to replicate the ops_user table. In fact, the whole "ops" database is replicated. -[[replicate-do-table]] -db-name ="ops" -tbl-name = "ops_user" - -# The "history" database includes multiple tables such as 2017_01 2017_02 ... 2017_12/2016_01 2016_02 ... 2016_12. You only need to replicate the tables of 2017. -[[replicate-do-table]] -db-name ="history" -tbl-name = "~^2017_.*" - -# Ignore the "ops" and "fault" databases in replication -# Ignore the databases starting with "www" in replication -# Because replicate-do-db has a higher priority than replicate-ignore-db, it is invalid to ignore the "ops" database here in replication. -replicate-ignore-db = ["ops","fault","~^www"] - -# The "fault" database includes multiple tables such as faults, user_feedback, ticket. -# Ignore the user_feedback table in replication. -# Because replicate-ignore-db has a higher priority than replicate-ignore-table, it is invalid to only ignore the user_feedback table in replication. In fact, the whole "fault" database is ignored in replication. -[[replicate-ignore-table]] -db-name = "fault" -tbl-name = "user_feedback" - -# The "order" database includes multiple tables such as 2017_01 2017_02 ... 2017_12/2016_01 2016_02 ... 2016_12. You need to ignore the tables of 2016. -[[replicate-ignore-table]] -db-name ="order" -tbl-name = "~^2016_.*" -``` - -### Support for replicating data from sharded tables - -You can use Syncer to import data from sharded tables into one table within one database according to the `route-rules`. But before replicating, you need to check: - -- Whether the sharding rules can be represented using the `route-rules` syntax. -- Whether the sharded tables contain unique increasing primary keys, or whether conflicts exist in the unique indexes or the primary keys after the combination. - -Currently, the support for DDL is still in progress. - -![syncer sharding](/media/syncer-sharding.png) - -#### Usage of replicating data from sharded tables - -1. Start Syncer in all MySQL instances and configure the route-rules. -2. In scenarios using replicate-do-db & replicate-ignore-db and route-rules at the same time, you need to specify the target-schema & target-table content in route-rules. - -```toml -# The scenarios are as follows: -# Database A includes multiple databases such as order_2016 and history_2016. -# Database B includes multiple databases such as order_2017 and history_2017. -# Specify to replicate order_2016 in database A; the data tables are 2016_01 2016_02 ... 2016_12 -# Specify to replicate order_2017 in database B; the data tables are 2017_01 2017_02 ... 2017_12 -# Use order_id as the primary key in the table, and the primary keys among data do not conflict. -# Ignore the history_2016 and history_2017 databases in replication -# The target database is "order" and the target data tables are order_2017 and order_2016. - -# When Syncer finds that the route-rules is enabled after Syncer gets the upstream data, it first combines databases and tables, and then determines do-db & do-table. -# You need to configure the database to be replicated, which is required when you determine the target-schema & target-table. -[[replicate-do-table]] -db-name ="order" -tbl-name = "order_2016" - -[[replicate-do-table]] -db-name ="order" -tbl-name = "order_2017" - -[[route-rules]] -pattern-schema = "order_2016" -pattern-table = "2016_??" -target-schema = "order" -target-table = "order_2016" - -[[route-rules]] -pattern-schema = "order_2017" -pattern-table = "2017_??" -target-schema = "order" -target-table = "order_2017" -``` - -### Check before replicating data using Syncer - -Before replicating data using Syncer, check the following items: - -1. Check the database version. - - Use the `select @@version;` command to check your database version. Currently, Syncer supports the following versions: - - - 5.5 < MySQL version < 8.0 - - MariaDB version >= 10.1.2 - - In earlier versions of MariaDB, the format of some binlog field types is inconsistent with that in MySQL. - - > **Note:** - > - > If there is a source-replica replication structure between the upstream MySQL/MariaDB servers, then choose the following version. - > - > - 5.7.1 < MySQL version < 8.0 - > - MariaDB version >= 10.1.3 - -2. Check the `server-id` of the source database. - - Check the `server-id` using the following command: - - ```sql - mysql> show global variables like 'server_id'; - +---------------+------- - | Variable_name | Value | - +---------------+-------+ - | server_id | 1 | - +---------------+-------+ - 1 row in set (0.01 sec) - ``` - - - If the result is null or 0, Syncer cannot replicate data. - - Syncer `server-id` must be different from the MySQL `server-id`, and must be unique in the MySQL cluster. - -3. Check binlog related parameters. - - 1. Check whether the binlog is enabled in MySQL using the following command: - - ```sql - mysql> show global variables like 'log_bin'; - +--------------------+---------+ - | Variable_name | Value | - +--------------------+---------+ - | log_bin | ON | - +--------------------+---------+ - 1 row in set (0.00 sec) - ``` - - If the result is `log_bin = OFF`, you need to enable the binlog. See the [document about enabling the binlog](https://dev.mysql.com/doc/refman/5.7/en/replication-howto-masterbaseconfig.html). - - 2. The binlog format must be `ROW` and the binary log must be written with `FULL` row images. Check both of these variables: - - ```sql - mysql> select variable_name, variable_value from information_schema.global_variables where variable_name in ('binlog_format','binlog_row_image'); - +------------------+----------------+ - | variable_name | variable_value | - +------------------+----------------+ - | BINLOG_FORMAT | ROW | - | BINLOG_ROW_IMAGE | FULL | - +------------------+----------------+ - 2 rows in set (0.001 sec) - ``` - - - If one of the settings is not correct, you should change the configuration file on disk and restart the MySQL service. - - It's important to persist any configuration changes to disk, so that they're reflected if the MySQL service restarts. - - Because existing connections will keep old values of global variables, you should *not* use the `SET` statement to dynamically change these settings. - -4. Check user privileges. - - 1. Check the user privileges required by Mydumper for full data export. - - - To export the full data using Mydumper, the user must have the privileges of `select` and `reload`. - - You can add the `--no-locks` option when the operation object is RDS, to avoid applying for the `reload` privilege. - - 2. Check the upstream MySQL or MariaDB user privileges required by Syncer for incremental replication. - - The upstream MySQL user must have the following privileges at least: - - ``` - select, replication slave, replication client - ``` - - 3. Check the downstream user privileges required by TiDB. - - | Privileges | Scope | - | :--- | :--- | - | `SELECT` | Tables | - | `INSERT` | Tables | - | `UPDATE` | Tables | - | `DELETE` | Tables | - | `CREATE` | Databases, tables | - | `DROP` | Databases, tables | - | `ALTER` | Tables | - | `INDEX` | Tables | - - Execute the following `GRANT` statement for the databases or tables that you need to replicate: - - ```sql - GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,INDEX ON db.table TO 'your_user'@'your_wildcard_of_host'; - ``` - -5. Check the SQL mode. - - Make sure that the upstream SQL mode is consistent with the downstream SQL mode. Otherwise, the data replication error might occur. - - ```sql - mysql> show variables like '%sql_mode%'; - +---------------+-----------------------------------------------------------------------------------+ - | Variable_name | Value | - +---------------+-----------------------------------------------------------------------------------+ - | sql_mode | ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION | - +---------------+-----------------------------------------------------------------------------------+ - 1 row in set (0.01 sec) - ``` - -6. Check the Character Set. - - TiDB differs from MySQL in [character sets](/character-set-and-collation.md). - -7. Check whether the table to be replicated has a primary key or a unique index. - - If the table does not have a primary key or a unique index, idempotent operations cannot be achieved. In this situation, a full table scan is performed every time an entry of data is updated in the downstream, which might slow down the replication task. Therefore, it is recommended that you add a primary key to every table to be replicated. - -## Syncer monitoring solution - -The `syncer` monitoring solution contains the following components: - -- Prometheus, an open source time series database, used to store the monitoring and performance metrics -- Grafana, an open source project for analyzing and visualizing metrics, used to display the performance metrics -- Alertmanager, combined with Grafana to implement the alerting mechanism - -See the following diagram: - -![syncer_monitor_scheme](/media/syncer-monitor-scheme.png) - -### Configure Syncer monitor and alert - -Syncer provides the metric interface, and requires Prometheus to actively obtain data. Take the following steps to configure Syncer monitor and alert: - -1. To add the Syncer job information to Prometheus, flush the following content to the configuration file of Prometheus. The monitor is enabled when you restart Prometheus. - - ```yaml - - job_name: 'syncer_ops' // name of the job, to distinguish the reported data - static_configs: - - targets: ['10.1.1.4:10086'] // Syncer monitoring address and port; to inform Prometheus of obtaining the monitoring data of Syncer - ``` - -2. To configure Prometheus [alert](https://prometheus.io/docs/alerting/alertmanager/), flush the following content to the `alert.rule` configuration file. The alert is enabled when you restart Prometheus. - - ``` - # syncer - ALERT syncer_status - IF syncer_binlog_file{node='master'} - ON(instance, job) syncer_binlog_file{node='syncer'} > 1 - FOR 1m - LABELS {channels="alerts", env="test-cluster"} - ANNOTATIONS { - summary = "syncer status error", - description="alert: syncer_binlog_file{node='master'} - ON(instance, job) syncer_binlog_file{node='syncer'} > 1 instance: {{ $labels.instance }} values: {{ $value }}", - } - ``` - -#### Configure Grafana - -1. Log in to the Grafana Web interface. - - - The default address is: `http://localhost:3000` - - The default account name: admin - - The password for the default account: admin - -2. Import the configuration file of Grafana dashboard. - - Click the Grafana Logo -> click Dashboards -> click Import -> choose and import the dashboard [configuration file](https://github.com/pingcap/tidb-ansible/blob/master/scripts/syncer.json) -> choose the corresponding data source. - -### Description of Grafana Syncer metrics - -#### title: binlog events - -- metrics: `rate(syncer_binlog_event_count[1m])` -- info: QPS of the binlog event that has been received by Syncer - -#### title: binlog event transform - -- metrics: `histogram_quantile(0.8, sum(rate(syncer_binlog_event_bucket[1m])) by (le))` -- info: the cost of transforming the binlog event to SQL statements by Syncer - -#### title: transaction latency - -- metrics: `histogram_quantile(0.95, sum(rate(syncer_txn_cost_in_second_bucket[1m])) by (le))` -- info: the cost of executing a transaction on TiDB - -#### title: transaction tps - -- metrics: `rate(syncer_txn_cost_in_second_count[1m])` -- info: TPS of executing a transaction on TiDB - -#### title: binlog file gap - -- metrics: `syncer_binlog_file{node="master"} - ON(instance, job) syncer_binlog_file{node="syncer"}` -- info: the number of different binlog files between the upstream and the downstream in the process of replication; the normal value is 0, which indicates real-time replication; a larger value indicates a larger number of binlog files discrepancy - -#### title: binlog skipped events - -- metrics: `rate(syncer_binlog_skipped_events_total[1m])` -- info: the total number of SQL statements that Syncer skips when the upstream replicates binlog files with the downstream; you can configure the format of SQL statements skipped by Syncer using the `skip-ddls` and `skip-dmls` parameters in the `syncer.toml` file. - -#### title: position binlog position - -- metrics: `syncer_binlog_pos{node="syncer"}` and `syncer_binlog_pos{node="master"}` -- info: it works with `file number of binlog position`. `syncer_binlog_pos{node="master"}` indicates the position of latest binlog position fetched from MySQL, and `syncer_binlog_pos{node="syncer"}` indicates the position of the binlog position that Syncer has replicated. - -#### title: file number of binlog position - -- metrics: `syncer_binlog_file{node="syncer"}` and `syncer_binlog_file{node="master"}` -- info: it works with `position of binlog position`. `syncer_binlog_file{node="master"}` indicates the file number of the latest binlog position fetched from MySQL, and `syncer_binlog_file{node="syncer"}` indicates the file number of the binlog position that Syncer has replicated. - -#### title: execution jobs - -- metrics: `sum(rate(syncer_add_jobs_total[1m])) by (queueNo)` -- info: the count of jobs that have been added into the execution queue - -#### title: pending jobs - -- metrics: `sum(rate(syncer_add_jobs_total[1m]) - rate(syncer_finished_jobs_total[1m])) by (queueNo)` -- info: the count of jobs that have been applied into TiDB diff --git a/system-variables.md b/system-variables.md index fa3689f5a8e9..8a3197954132 100644 --- a/system-variables.md +++ b/system-variables.md @@ -10,6 +10,7 @@ TiDB system variables behave similar to MySQL with some differences, in that set - Changes to `GLOBAL` scoped variables **only apply to new connection sessions with TiDB**. Currently active connection sessions are not affected. These changes are persisted and valid after restarts. - Changes to `INSTANCE` scoped variables apply to all active or new connection sessions with the current TiDB instance immediately after the changes are made. Other TiDB instances are not affected. These changes are not persisted and become invalid after TiDB restarts. +- Variables can also have `NONE` scope. These variables are read-only, and are typically used to convey static information that will not change after a TiDB server has started. Variables can be set with the [`SET` statement](/sql-statements/sql-statement-set-variable.md) on a per-session, instance or global basis: @@ -25,34 +26,126 @@ SET GLOBAL tidb_distsql_scan_concurrency = 10; > **Note:** > -> TiDB differs from MySQL in that `GLOBAL` scoped variables **persist** through TiDB server restarts. Changes are also propagated to other TiDB servers every 2 seconds [TiDB #14531](https://github.com/pingcap/tidb/issues/14531). -> Additionally, TiDB presents several MySQL variables from MySQL 5.7 as both readable and settable. This is required for compatibility, since it is common for both applications and connectors to read MySQL variables. For example: JDBC connectors both read and set query cache settings, despite not relying on the behavior. +> Executing `SET GLOBAL` applies immediately on the TiDB server where the statement was issued. A notification is then sent to all TiDB servers to refresh their system variable cache, which will start immediately as a background operation. Because there is a risk that some TiDB servers might miss the notification, the system variable cache is also refreshed automatically every 30 seconds. This helps ensure that all servers are operating with the same configuration. +> +> TiDB differs from MySQL in that `GLOBAL` scoped variables **persist** through TiDB server restarts. Additionally, TiDB presents several MySQL variables as both readable and settable. This is required for compatibility, because it is common for both applications and connectors to read MySQL variables. For example, JDBC connectors both read and set query cache settings, despite not relying on the behavior. ## Variable Reference +### allow_auto_random_explicit_insert New in v4.0.3 + +- Scope: SESSION | GLOBAL +- Default value: `OFF` +- Determines whether to allow explicitly specifying the values of the column with the `AUTO_RANDOM` attribute in the `INSERT` statement. + +### auto_increment_increment + +- Scope: SESSION | GLOBAL +- Default value: `1` +- Range: `[1, 65535]` +- Controls the step size of `AUTO_INCREMENT` values to be allocated to a column. It is often used in combination with `auto_increment_offset`. + +### auto_increment_offset + +- Scope: SESSION | GLOBAL +- Default value: `1` +- Range: `[1, 65535]` +- Controls the initial offset of `AUTO_INCREMENT` values to be allocated to a column. This setting is often used in combination with `auto_increment_increment`. For example: + +```sql +mysql> CREATE TABLE t1 (a int not null primary key auto_increment); +Query OK, 0 rows affected (0.10 sec) + +mysql> set auto_increment_offset=1; +Query OK, 0 rows affected (0.00 sec) + +mysql> set auto_increment_increment=3; +Query OK, 0 rows affected (0.00 sec) + +mysql> INSERT INTO t1 VALUES (),(),(),(); +Query OK, 4 rows affected (0.04 sec) +Records: 4 Duplicates: 0 Warnings: 0 + +mysql> SELECT * FROM t1; ++----+ +| a | ++----+ +| 1 | +| 4 | +| 7 | +| 10 | ++----+ +4 rows in set (0.00 sec) +``` + ### autocommit - Scope: SESSION | GLOBAL -- Default value: ON +- Default value: `ON` - Controls whether statements should automatically commit when not in an explicit transaction. See [Transaction Overview](/transaction-overview.md#autocommit) for more information. -### allow_auto_random_explicit_insert New in v4.0.3 +### character_set_client + +- Scope: SESSION | GLOBAL +- Default value: `utf8mb4` +- The character set for data sent from the client. See [Character Set and Collation](/character-set-and-collation.md) for details on the use of character sets and collations in TiDB. It is recommended to use [`SET NAMES`](/sql-statements/sql-statement-set-names.md) to change the character set when needed. + +### character_set_connection + +- Scope: SESSION | GLOBAL +- Default value: `utf8mb4` +- The character set for string literals that do not have a specified character set. + +### character_set_database + +- Scope: SESSION | GLOBAL +- Default value: `utf8mb4` +- This variable indicates the character set of the default database in use. **It is NOT recommended to set this variable**. When a new default database is selected, the server changes the variable value. + +### character_set_results + +- Scope: SESSION | GLOBAL +- Default value: `utf8mb4` +- The character set that is used when data is sent to the client. + +### character_set_server + +- Scope: SESSION | GLOBAL +- Default value: `utf8mb4` +- The character set used for new schemas when no character set is specified in the `CREATE SCHEMA` statement. + +### cte_max_recursion_depth + +- Scope: SESSION | GLOBAL +- Default value: `1000` +- Range: `[0, 4294967295]` +- Controls the maximum recursion depth in Common Table Expressions. -- Scope: SESSION (since v4.0.5: SESSION | GLOBAL) -- Default value: 0 -- Determines whether to allow explicitly specifying the values of the column with the `AUTO_RANDOM` attribute in the `INSERT` statement. `1` means to allow and `0` means to disallow. +### datadir + +- Scope: NONE +- Default value: /tmp/tidb +- This variable indicates the location where data is stored. This location can be a local path or point to a PD server if the data is stored on TiKV. +- A value in the format of `ip_address:port` indicates the PD server that TiDB connects to on startup. + +### default_authentication_plugin + +- Scope: GLOBAL +- Default value: `mysql_native_password` +- This variable sets the authentication method that the server advertises when the server-client connection is being established. Possible values for this variable are documented in [Authentication plugin status](/security-compatibility-with-mysql.md#authentication-plugin-status). +- Value options: `mysql_native_password` and `caching_sha2_password`. For more details, see [Authentication plugin status](/security-compatibility-with-mysql.md#authentication-plugin-status). ### ddl_slow_threshold - Scope: INSTANCE -- Default value: 300 +- Default value: `300` - DDL operations whose execution time exceeds the threshold value are output to the log. The unit is millisecond. ### foreign_key_checks -- Scope: NONE -- Default value: OFF -- For compatibility, TiDB returns foreign key checks as OFF. +- Scope: SESSION | GLOBAL +- Default value: `OFF` +- For compatibility, TiDB returns foreign key checks as `OFF`. ### hostname @@ -60,39 +153,69 @@ SET GLOBAL tidb_distsql_scan_concurrency = 10; - Default value: (system hostname) - The hostname of the TiDB server as a read-only variable. +### init_connect + +- Scope: GLOBAL +- Default value: "" +- The `init_connect` feature permits a SQL statement to be automatically executed when you first connect to a TiDB server. If you have the `CONNECTION_ADMIN` or `SUPER` privileges, this `init_connect` statement will not be executed. If the `init_connect` statement results in an error, your user connection will be terminated. + ### innodb_lock_wait_timeout - Scope: SESSION | GLOBAL -- Default value: 50 +- Default value: `50` +- Range: `[1, 1073741824]` - The lock wait timeout for pessimistic transactions (default) in seconds. +### interactive_timeout + +- Scope: SESSION | GLOBAL +- Default value: `28800` +- Range: `[1, 31536000]` +- Unit: Seconds +- This variable represents the idle timeout of the interactive user session, which is measured in seconds. Interactive user session refers to the session established by calling [`mysql_real_connect()`](https://dev.mysql.com/doc/c-api/5.7/en/mysql-real-connect.html) API using the `CLIENT_INTERACTIVE` option (for example, MySQL shell client). This variable is fully compatible with MySQL. + +### last_plan_from_binding New in v4.0 + +- Scope: SESSION +- Default value: `OFF` +- This variable is used to show whether the execution plan used in the previous statement was influenced by a [plan binding](/sql-plan-management.md) + ### last_plan_from_cache New in v4.0 - Scope: SESSION -- Default value: 0 +- Default value: `OFF` - This variable is used to show whether the execution plan used in the previous `execute` statement is taken directly from the plan cache. -### last_plan_from_binding +### license -- Scope: SESSION -- Default value: 0 -- This variable is used to show whether the execution plan used in the previous statement was influenced by a [plan binding](/sql-plan-management.md) +- Scope: NONE +- Default value: Apache License 2.0 +- This variable indicates the license of your TiDB server installation. ### max_execution_time - Scope: SESSION | GLOBAL -- Default value: 0 +- Default value: `0` +- Range: `[0, 2147483647]` +- Unit: Milliseconds - The maximum execution time of a statement in milliseconds. The default value is unlimited (zero). > **Note:** > > Unlike in MySQL, the `max_execution_time` system variable currently works on all kinds of statements in TiDB, not only restricted to the `SELECT` statement. The precision of the timeout value is roughly 100ms. This means the statement might not be terminated in accurate milliseconds as you specify. -### interactive_timeout +### port -- Scope: SESSION | GLOBAL -- Default value: 28800 -- This variable represents the idle timeout of the interactive user session, which is measured in seconds. Interactive user session refers to the session established by calling [`mysql_real_connect()`](https://dev.mysql.com/doc/c-api/5.7/en/mysql-real-connect.html) API using the `CLIENT_INTERACTIVE` option (for example, MySQL shell client). This variable is fully compatible with MySQL. +- Scope: NONE +- Default value: `4000` +- Range: `[0, 65535]` +- The port that the `tidb-server` is listening on when speaking the MySQL protocol. + +### socket + +- Scope: NONE +- Default value: "" +- The local unix socket file that the `tidb-server` is listening on when speaking the MySQL protocol. ### sql_mode @@ -100,38 +223,70 @@ SET GLOBAL tidb_distsql_scan_concurrency = 10; - Default value: `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION` - This variable controls a number of MySQL compatibility behaviors. See [SQL Mode](/sql-mode.md) for more information. -### sql_select_limit New in v4.0.2 version +### sql_select_limit New in v4.0.2 - Scope: SESSION | GLOBAL -- Default value: `2^64 - 1` (18446744073709551615) +- Default value: `18446744073709551615` +- Range: `[0, 18446744073709551615]` - The maximum number of rows returned by the `SELECT` statements. -### tidb_allow_batch_cop New in v4.0 version +### system_time_zone + +- Scope: NONE +- Default value: (system dependent) +- This variable shows the system time zone from when TiDB was first bootstrapped. See also [`time_zone`](#time_zone). + +### tidb_allow_batch_cop New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 0 +- Default value: `1` +- Range: `[0, 2]` - This variable is used to control how TiDB sends a coprocessor request to TiFlash. It has the following values: * `0`: Never send requests in batches * `1`: Aggregation and join requests are sent in batches * `2`: All coprocessor requests are sent in batches +### tidb_allow_fallback_to_tikv New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: "" +- This variable is used to specify a list of storage engines that might fall back to TiKV. If the execution of a SQL statement fails due to a failure of the specified storage engine in the list, TiDB retries executing this SQL statement with TiKV. This variable can be set to "" or "tiflash". When this variable is set to "tiflash", if the execution of a SQL statement fails due to a failure of TiFlash, TiDB retries executing this SQL statement with TiKV. + +### tidb_allow_mpp New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `ON` +- Controls whether to use the MPP mode of TiFlash to execute queries. The value options are as follows: + - `0` or `OFF`, which means that the MPP mode will not be used. + - `1` or `ON`, which means that the optimizer determines whether to use the MPP mode based on the cost estimation (by default). + +MPP is a distributed computing framework provided by the TiFlash engine, which allows data exchange between nodes and provides high-performance, high-throughput SQL algorithms. For details about the selection of the MPP mode, refer to [Control whether to select the MPP mode](/tiflash/use-tiflash.md#control-whether-to-select-the-mpp-mode). + ### tidb_allow_remove_auto_inc New in v2.1.18 and v3.0.4 - Scope: SESSION -- Default value: 0 +- Default value: `OFF` - This variable is used to set whether the `AUTO_INCREMENT` property of a column is allowed to be removed by executing `ALTER TABLE MODIFY` or `ALTER TABLE CHANGE` statements. It is not allowed by default. +### tidb_analyze_version New in v5.1.0 + +- Scope: SESSION | GLOBAL +- Value options: `1` and `2` +- Default value: `2` +- Controls how TiDB collects statistics. +- In versions before v5.1.0, the default value of this variable is `1`. In v5.1.0, the default value of this variable is `2`, which serves as an experimental feature. For detailed introduction, see [Introduction to Statistics](/statistics.md). + ### tidb_auto_analyze_end_time - Scope: GLOBAL -- Default value: 23:59 +0000 +- Default value: `23:59 +0000` - This variable is used to restrict the time window that the automatic update of statistics is permitted. For example, to only allow automatic statistics updates between 1AM and 3AM, set `tidb_auto_analyze_start_time='01:00 +0000'` and `tidb_auto_analyze_end_time='03:00 +0000'`. ### tidb_auto_analyze_ratio - Scope: GLOBAL -- Default value: 0.5 +- Default value: `0.5` - This variable is used to set the threshold when TiDB automatically executes [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) in a background thread to update table statistics. For example, a value of 0.5 means that auto-analyze is triggered when greater than 50% of the rows in a table have been modified. Auto-analyze can be restricted to only execute during certain hours of the day by specifying `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time`. > **Note:** @@ -141,50 +296,67 @@ SET GLOBAL tidb_distsql_scan_concurrency = 10; ### tidb_auto_analyze_start_time - Scope: GLOBAL -- Default value: 00:00 +0000 +- Default value: `00:00 +0000` - This variable is used to restrict the time window that the automatic update of statistics is permitted. For example, to only allow automatic statistics updates between 1AM and 3AM, set `tidb_auto_analyze_start_time='01:00 +0000'` and `tidb_auto_analyze_end_time='03:00 +0000'`. ### tidb_backoff_lock_fast - Scope: SESSION | GLOBAL -- Default value: 100 +- Default value: `100` +- Range: `[1, 2147483647]` - This variable is used to set the `backoff` time when the read request meets a lock. ### tidb_backoff_weight - Scope: SESSION | GLOBAL -- Default value: 2 +- Default value: `2` +- Range: `[1, 2147483647]` - This variable is used to increase the weight of the maximum time of TiDB `backoff`, that is, the maximum retry time for sending a retry request when an internal network or other component (TiKV, PD) failure is encountered. This variable can be used to adjust the maximum retry time and the minimum value is 1. For example, the base timeout for TiDB to take TSO from PD is 15 seconds. When `tidb_backoff_weight = 2`, the maximum timeout for taking TSO is: *base time \* 2 = 30 seconds*. In the case of a poor network environment, appropriately increasing the value of this variable can effectively alleviate error reporting to the application end caused by timeout. If the application end wants to receive the error information more quickly, minimize the value of this variable. +### tidb_broadcast_join_threshold_count New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `10240` +- Range: `[0, 9223372036854775807]` +- The unit of the variable is rows. If the objects of the join operation belong to a subquery, the optimizer cannot estimate the size of the subquery result set. In this situation, the size is determined by the number of rows in the result set. If the estimated number of rows in the subquery is less than the value of this variable, the Broadcast Hash Join algorithm is used. Otherwise, the Shuffled Hash Join algorithm is used. + +### tidb_broadcast_join_threshold_size New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `104857600` (100 MiB) +- Range: `[0, 9223372036854775807]` +- Unit: Bytes +- If the table size is less than the value of the variable, the Broadcast Hash Join algorithm is used. Otherwise, the Shuffled Hash Join algorithm is used. + ### tidb_build_stats_concurrency -- Scope: SESSION -- Default value: 4 +- Scope: SESSION | GLOBAL +- Default value: `4` - This variable is used to set the concurrency of executing the `ANALYZE` statement. - When the variable is set to a larger value, the execution performance of other queries is affected. ### tidb_capture_plan_baselines New in v4.0 - Scope: SESSION | GLOBAL -- Default value: off +- Default value: `OFF` - This variable is used to control whether to enable the [baseline capturing](/sql-plan-management.md#baseline-capturing) feature. This feature depends on the statement summary, so you need to enable the statement summary before you use baseline capturing. - After this feature is enabled, the historical SQL statements in the statement summary are traversed periodically, and bindings are automatically created for SQL statements that appear at least twice. ### tidb_check_mb4_value_in_utf8 - Scope: INSTANCE -- Default value: 1, indicating check the validity of UTF-8 data. This default behavior is compatible with MySQL. -- This variable is used to set whether to check the validity of UTF-8 data. -- To upgrade an earlier version (TiDB v2.1.1 or earlier), you may need to disable this option. Otherwise, you can successfully write invalid strings in an earlier version but fail to do this in a later version, because there is no data validity check in the earlier version. For details, see [FAQs After Upgrade](/faq/upgrade-faq.md). +- Default value: `ON` +- This variable is used to enforce that the `utf8` character set only stores values from the [Basic Multilingual Plane (BMP)](https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane). To store characters outside the BMP, it is recommended to use the `utf8mb4` character set. +- You might need to disable this option when upgrading your cluster from an earlier version of TiDB where the `utf8` checking was more relaxed. For details, see [FAQs After Upgrade](/faq/upgrade-faq.md). ### tidb_checksum_table_concurrency - Scope: SESSION -- Default value: 4 +- Default value: `4` - This variable is used to set the scan index concurrency of executing the `ADMIN CHECKSUM TABLE` statement. - When the variable is set to a larger value, the execution performance of other queries is affected. @@ -197,8 +369,8 @@ SET GLOBAL tidb_distsql_scan_concurrency = 10; ### tidb_constraint_check_in_place - Scope: SESSION | GLOBAL -- Default value: 0 -- This setting only applies to optimistic transactions. When this variable is set to zero, checking for duplicate values in UNIQUE indexes is deferred until the transaction commits. This helps improve performance, but might be an unexpected behavior for some applications. See [Constraints](/constraints.md) for details. +- Default value: `OFF` +- This setting only applies to optimistic transactions. When this variable is set to `OFF`, checking for duplicate values in UNIQUE indexes is deferred until the transaction commits. This helps improve performance, but might be an unexpected behavior for some applications. See [Constraints](/constraints.md) for details. - When set to zero and using optimistic transactions: @@ -226,26 +398,28 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_current_ts - Scope: SESSION -- Default value: 0 +- Default value: `0` - This variable is read-only. It is used to obtain the timestamp of the current transaction. ### tidb_ddl_error_count_limit - Scope: GLOBAL -- Default value: 512 +- Default value: `512` +- Range: `[0, 9223372036854775807]` - This variable is used to set the number of retries when the DDL operation fails. When the number of retries exceeds the parameter value, the wrong DDL operation is canceled. ### tidb_ddl_reorg_batch_size - Scope: GLOBAL -- Default value: 256 +- Default value: `256` +- Range: `[32, 10240]` - This variable is used to set the batch size during the `re-organize` phase of the DDL operation. For example, when TiDB executes the `ADD INDEX` operation, the index data needs to backfilled by `tidb_ddl_reorg_worker_cnt` (the number) concurrent workers. Each worker backfills the index data in batches. - If many updating operations such as `UPDATE` and `REPLACE` exist during the `ADD INDEX` operation, a larger batch size indicates a larger probability of transaction conflicts. In this case, you need to adjust the batch size to a smaller value. The minimum value is 32. - If the transaction conflict does not exist, you can set the batch size to a large value. The maximum value is 10240. This can increase the speed of the backfilling data, but the write pressure on TiKV also becomes higher. ### tidb_ddl_reorg_priority -- Scope: SESSION | GLOBAL +- Scope: SESSION - Default value: `PRIORITY_LOW` - This variable is used to set the priority of executing the `ADD INDEX` operation in the `re-organize` phase. - You can set the value of this variable to `PRIORITY_LOW`, `PRIORITY_NORMAL` or `PRIORITY_HIGH`. @@ -253,25 +427,29 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_ddl_reorg_worker_cnt - Scope: GLOBAL -- Default value: 4 +- Default value: `4` +- Range: `[1, 128]` - This variable is used to set the concurrency of the DDL operation in the `re-organize` phase. ### tidb_disable_txn_auto_retry - Scope: SESSION | GLOBAL -- Default: on -- This variable is used to set whether to disable the automatic retry of explicit transactions. The default value of `on` means that transactions will not automatically retry in TiDB and `COMMIT` statements might return errors that need to be handled in the application layer. +- Default value: `ON` +- This variable is used to set whether to disable the automatic retry of explicit optimistic transactions. The default value of `ON` means that transactions will not automatically retry in TiDB and `COMMIT` statements might return errors that need to be handled in the application layer. - Setting the value to `off` means that TiDB will automatically retry transactions, resulting in fewer errors from `COMMIT` statements. Be careful when making this change, because it might result in lost updates. + Setting the value to `OFF` means that TiDB will automatically retry transactions, resulting in fewer errors from `COMMIT` statements. Be careful when making this change, because it might result in lost updates. This variable does not affect automatically committed implicit transactions and internally executed transactions in TiDB. The maximum retry count of these transactions is determined by the value of `tidb_retry_limit`. For more details, see [limits of retry](/optimistic-transaction.md#limits-of-retry). + This variable only applies to optimistic transactions, not to pessimistic transactions. The number of retries for pessimistic transactions is controlled by [`max_retry_count`](/tidb-configuration-file.md#max-retry-count). + ### tidb_distsql_scan_concurrency - Scope: SESSION | GLOBAL -- Default value: 15 +- Default value: `15` +- Range: `[1, 2147483647]` - This variable is used to set the concurrency of the `scan` operation. - Use a bigger value in OLAP scenarios, and a smaller value in OLTP scenarios. - For OLAP scenarios, the maximum value cannot exceed the number of CPU cores of all the TiKV nodes. @@ -280,14 +458,27 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_dml_batch_size - Scope: SESSION | GLOBAL -- Default value: 0 +- Default value: `0` +- Range: `[0, 2147483647]` - When this value is greater than `0`, TiDB will batch commit statements such as `INSERT` or `LOAD DATA` into smaller transactions. This reduces memory usage and helps ensure that the `txn-total-size-limit` is not reached by bulk modifications. - Only the value `0` provides ACID compliance. Setting this to any other value will break the atomicity and isolation guarantees of TiDB. -### `tidb_enable_amend_pessimistic_txn` New in v4.0.7 +### tidb_enable_1pc New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `ON` +- This variable is used to specify whether to enable the one-phase commit feature for transactions that only affect one Region. Compared with the often-used two-phase commit, one-phase commit can greatly reduce the latency of transaction commit and increase the throughput. + +> **Note:** +> +> - The default value of `ON` only applies to new clusters. if your cluster was upgraded from an earlier version of TiDB, the value `OFF` will be used instead. +> - If you have enabled TiDB Binlog, enabling this variable cannot improve the performance. To improve the performance, it is recommended to use [TiCDC](/ticdc/ticdc-overview.md) instead. +> - Enabling this parameter only means that one-phase commit becomes an optional mode of transaction commit. In fact, the most suitable mode of transaction commit is determined by TiDB. + +### tidb_enable_amend_pessimistic_txn New in v4.0.7 - Scope: SESSION | GLOBAL -- Default value: 0 +- Default value: `OFF` - This variable is used to control whether to enable the `AMEND TRANSACTION` feature. If you enable the `AMEND TRANSACTION` feature in a pessimistic transaction, when concurrent DDL operations and SCHEMA VERSION changes exist on tables associated with this transaction, TiDB attempts to amend the transaction. TiDB corrects the transaction commit to make the commit consistent with the latest valid SCHEMA VERSION so that the transaction can be successfully committed without getting the `Information schema is changed` error. This feature is effective on the following concurrent DDL operations: - `ADD COLUMN` or `DROP COLUMN` operations. @@ -298,106 +489,189 @@ Constraint checking is always performed in place for pessimistic transactions (d > > Currently, this feature is incompatible with TiDB Binlog in some scenarios and might cause semantic changes on a transaction. For more usage precautions of this feature, refer to [Incompatibility issues about transaction semantic](https://github.com/pingcap/tidb/issues/21069) and [Incompatibility issues about TiDB Binlog](https://github.com/pingcap/tidb/issues/20996). -### tidb_enable_cascades_planner +### tidb_enable_async_commit New in v5.0 - Scope: SESSION | GLOBAL -- Default value: 0 -- This variable is used to control whether to enable the cascades planner, which is currently considered experimental. +- Default value: `ON` +- This variable controls whether to enable the async commit feature for the second phase of the two-phase transaction commit to perform asynchronously in the background. Enabling this feature can reduce the latency of transaction commit. -### tidb_enable_clustered_index +> **Note:** +> +> - The default value of `ON` only applies to new clusters. if your cluster was upgraded from an earlier version of TiDB, the value `OFF` will be used instead. +> - If you have enabled TiDB Binlog, enabling this variable cannot improve the performance. To improve the performance, it is recommended to use [TiCDC](/ticdc/ticdc-overview.md) instead. +> - Enabling this parameter only means that Async Commit becomes an optional mode of transaction commit. In fact, the most suitable mode of transaction commit is determined by TiDB. -- Scope: SESSION | GLOBAL -- Default value: 0 -- This variable is used to control whether to enable the [clustered index](/clustered-indexes.md) feature. - - This feature is only applicable to newly created tables and does not affect the existing old tables. - - This feature is only applicable to tables whose primary key is the single-column non-integer type or the multi-column type. It does not affect the tables without a primary key or tables with the primary key of the single-column non-integer type. - - You can execute `select tidb_pk_type from information_schema.tables where table_name ='{table_name}'` to check whether the clustered index feature has been enabled on a table. -- After you enable this feature, rows are stored directly on the primary key instead of on the internally allocated `rows_id` to which the extra primary key index is created to point. +### tidb_enable_cascades_planner - This feature impacts performance in the following aspects: - - For each `INSERT` operation, there is one less index key written into each row. - - When you make a query using the primary key as the equivalent condition, one read request can be saved. - - When you make a query using the primary key as the range condition, multiple read requests can be saved. - - When you make a query using the prefix of the multi-column primary key as the equivalent condition or range condition, multiple read requests can be saved. +- Scope: SESSION | GLOBAL +- Default value: `OFF` +- This variable is used to control whether to enable the cascades planner, which is currently considered experimental. ### tidb_enable_chunk_rpc New in v4.0 - Scope: SESSION -- Default value: 1 +- Default value: `ON` - This variable is used to control whether to enable the `Chunk` data encoding format in Coprocessor. +### tidb_enable_clustered_index New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `INT_ONLY` +- Possible values: `OFF`, `ON`, `INT_ONLY` +- This variable is used to control whether to create the primary key as a [clustered index](/clustered-indexes.md) by default. "By default" here means that the statement does not explicitly specify the keyword `CLUSTERED`/`NONCLUSTERED`. Supported values are `OFF`, `ON`, and `INT_ONLY`: + - `OFF` indicates that primary keys are created as non-clustered indexes by default. + - `ON` indicates that primary keys are created as clustered indexes by default. + - `INT_ONLY` indicates that the behavior is controlled by the configuration item `alter-primary-key`. If `alter-primary-key` is set to `true`, all primary keys are created as non-clustered indexes by default. If it is set to `false`, only the primary keys which consist of an integer column are created as clustered indexes. + +### tidb_enable_collect_execution_info + +- Scope: INSTANCE +- Default value: `ON` +- This variable controls whether to record the execution information of each operator in the slow query log. + +### tidb_enable_enhanced_security + +- Scope: NONE +- Default value: `OFF` +- This variable indicates whether the TiDB server you are connected to has the Security Enhanced Mode (SEM) enabled. To change its value, you need to modify the value of `enable-sem` in your TiDB server configuration file and restart the TiDB server. +- SEM is inspired by the design of systems such as [Security-Enhanced Linux](https://en.wikipedia.org/wiki/Security-Enhanced_Linux). It reduces the abilities of users with the MySQL `SUPER` privilege and instead requires `RESTRICTED` fine-grained privileges to be granted as a replacement. These fine-grained privileges include: + - `RESTRICTED_TABLES_ADMIN`: The ability to write data to system tables in the `mysql` schema and to see sensitive columns on `information_schema` tables. + - `RESTRICTED_STATUS_ADMIN`: The ability to see sensitive variables in the command `SHOW STATUS`. + - `RESTRICTED_VARIABLES_ADMIN`: The ability to see and set sensitive variables in `SHOW [GLOBAL] VARIABLES` and `SET`. + - `RESTRICTED_USER_ADMIN`: The ability to prevent other users from making changes or dropping a user account. + ### tidb_enable_fast_analyze - Scope: SESSION | GLOBAL -- Default value: 0, indicating not enabling the statistics fast `Analyze` feature. +- Default value: `OFF` - This variable is used to set whether to enable the statistics `Fast Analyze` feature. - If the statistics `Fast Analyze` feature is enabled, TiDB randomly samples about 10,000 rows of data as statistics. When the data is distributed unevenly or the data size is small, the statistics accuracy is low. This might lead to a non-optimal execution plan, for example, selecting a wrong index. If the execution time of the regular `Analyze` statement is acceptable, it is recommended to disable the `Fast Analyze` feature. ### tidb_enable_index_merge New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 0 +- Default value: `OFF` - This variable is used to control whether to enable the index merge feature. +### tidb_enable_list_partition New in v5.0 + +> **Warning:** +> +> Currently, List partition and List COLUMNS partition are experimental features. It is not recommended that you use it in the production environment. + +- Scope: SESSION +- Default value: `OFF` +- This variable is used to set whether to enable the `LIST (COLUMNS) TABLE PARTITION` feature. + ### tidb_enable_noop_functions New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 0 -- By default, TiDB returns an error when you attempt to use the syntax for functionality that is not yet implemented. When the variable value is set to `1`, TiDB silently ignores such cases of unavailable functionality, which is helpful if you cannot make changes to the SQL code. +- Default value: `OFF` +- By default, TiDB returns an error when you attempt to use the syntax for functionality that is not yet implemented. When the variable value is set to `ON`, TiDB silently ignores such cases of unavailable functionality, which is helpful if you cannot make changes to the SQL code. - Enabling `noop` functions controls the following behaviors: * `get_lock` and `release_lock` functions * `LOCK IN SHARE MODE` syntax * `SQL_CALC_FOUND_ROWS` syntax + * `CREATE TEMPORARY TABLE` syntax + * `DROP TEMPORARY TABLE` syntax + * `START TRANSACTION READ ONLY` and `SET TRANSACTION READ ONLY` syntax + * The `tx_read_only`, `transaction_read_only`, `offline_mode`, `super_read_only` and `read_only` system variables > **Note:** > -> Only the default value of `0` can be considered safe. Setting `tidb_enable_noop_functions=1` might lead to unexpected behaviors in your application, because it permits TiDB to ignore certain syntax without providing an error. +> Only the default value of `OFF` can be considered safe. Setting `tidb_enable_noop_functions=1` might lead to unexpected behaviors in your application, because it permits TiDB to ignore certain syntax without providing an error. + +### tidb_enable_parallel_apply New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `OFF` +- This variable controls whether to enable concurrency for the `Apply` operator. The number of concurrencies is controlled by the `tidb_executor_concurrency` variable. The `Apply` operator processes correlated subqueries and has no concurrency by default, so the execution speed is slow. Setting this variable value to `1` can increase concurrency and speed up execution. Currently, concurrency for `Apply` is disabled by default. + +### tidb_enable_rate_limit_action + +- Scope: SESSION | GLOBAL +- Default value: `ON` +- This variable controls whether to enable the dynamic memory control feature for the operator that reads data. By default, this operator enables the maximum number of threads that [`tidb_disql_scan_concurrency`](/system-variables.md#tidb_distsql_scan_concurrency) allows to read data. When the memory usage of a single SQL statement exceeds [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) each time, the operator that reads data stops one thread. +- When the operator that reads data has only one thread left and the memory usage of a single SQL statement continues to exceed [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query), this SQL statement triggers other memory control behaviors, such as [spilling data to disk](/tidb-configuration-file.md#spilled-file-encryption-method). ### tidb_enable_slow_log - Scope: INSTANCE -- Default value: `1` -- This variable is used to control whether to enable the slow log feature. It is enabled by default. +- Default value: `ON` +- This variable is used to control whether to enable the slow log feature. ### tidb_enable_stmt_summary New in v3.0.4 - Scope: SESSION | GLOBAL -- Default value: 1 (the value of the default configuration file) +- Default value: `ON` - This variable is used to control whether to enable the statement summary feature. If enabled, SQL execution information like time consumption is recorded to the `information_schema.STATEMENTS_SUMMARY` system table to identify and troubleshoot SQL performance issues. -### tidb_enable_table_partition +### tidb_enable_strict_double_type_check New in v5.0 - Scope: SESSION | GLOBAL -- Default value: "on" -- This variable is used to set whether to enable the `TABLE PARTITION` feature. - - `off` indicates disabling the `TABLE PARTITION` feature. In this case, the syntax that creates a partition table can be executed, but the table created is not a partitioned one. - - `on` indicates enabling the `TABLE PARTITION` feature for the supported partition types. Currently, it indicates enabling range partition, hash partition and range column partition with one single column. - - `auto` functions the same way as `on` does. +- Default value: `ON` +- This variable is used to control if tables can be created with invalid definitions of type `DOUBLE`. This setting is intended to provide an upgrade path from earlier versions of TiDB, which were less strict in validating types. +- The default value of `ON` is compatible with MySQL. -- Currently, TiDB only supports range partition and hash partition. +For example, the type `DOUBLE(10)` is now considered invalid because the precision of floating point types is not guaranteed. After changing `tidb_enable_strict_double_type_check` to `OFF`, the table is created: -### tidb_enable_telemetry New in v4.0.2 version +```sql +mysql> CREATE TABLE t1 (id int, c double(10)); +ERROR 1149 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use + +mysql> SET tidb_enable_strict_double_type_check = 'OFF'; +Query OK, 0 rows affected (0.00 sec) + +mysql> CREATE TABLE t1 (id int, c double(10)); +Query OK, 0 rows affected (0.09 sec) +``` + +> **Note:** +> +> This setting only applies to the type `DOUBLE` since MySQL permits precision for `FLOAT` types. This behavior is deprecated starting with MySQL 8.0.17, and it is not recommended to specify precision for either `FLOAT` or `DOUBLE` types. + +### tidb_enable_table_partition + +- Scope: SESSION | GLOBAL +- Default value: `ON` +- Possible values: `OFF`, `ON`, `AUTO` +- This variable is used to set whether to enable the `TABLE PARTITION` feature: + - `ON` indicates enabling Range partitioning, Hash partitioning, and Range column partitioning with one single column. + - `AUTO` functions the same way as `ON` does. + - `OFF` indicates disabling the `TABLE PARTITION` feature. In this case, the syntax that creates a partition table can be executed, but the table created is not a partitioned one. + +### tidb_enable_telemetry New in v4.0.2 - Scope: GLOBAL -- Default value: 1 -- This variable is used to dynamically control whether the telemetry collection in TiDB is enabled. By setting the value to `0`, the telemetry collection is disabled. If the [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402) TiDB configuration item is set to `false` on all TiDB instances, the telemetry collection is always disabled and this system variable will not take effect. See [Telemetry](/telemetry.md) for details. +- Default value: `ON` +- This variable is used to dynamically control whether the telemetry collection in TiDB is enabled. By setting the value to `OFF`, the telemetry collection is disabled. If the [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402) TiDB configuration item is set to `false` on all TiDB instances, the telemetry collection is always disabled and this system variable will not take effect. See [Telemetry](/telemetry.md) for details. ### tidb_enable_vectorized_expression New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 1 +- Default value: `ON` - This variable is used to control whether to enable vectorized execution. ### tidb_enable_window_function - Scope: SESSION | GLOBAL -- Default value: 1, indicating enabling the window function feature. -- This variable is used to control whether to enable the support for window functions. Note that window functions may use reserved keywords. This might cause SQL statements that could be executed normally cannot be parsed after upgrading TiDB. In this case, you can set `tidb_enable_window_function` to `0`. +- Default value: `ON` +- This variable is used to control whether to enable the support for window functions. Note that window functions may use reserved keywords. This might cause SQL statements that could be executed normally cannot be parsed after upgrading TiDB. In this case, you can set `tidb_enable_window_function` to `OFF`. + +### tidb_enforce_mpp New in v5.1 + +- Scope: SESSION +- Default value: `OFF`. To change this default value, modify the [`performance.enforce-mpp`](/tidb-configuration-file.md#enforce-mpp) configuration value. +- Controls whether to ignore the optimizer's cost estimation and to forcibly use TiFlash's MPP mode for query execution. The value options are as follows: + - `0` or `OFF`, which means that the MPP mode is not forcibly used (by default). + - `1` or `ON`, which means that the cost estimation is ignored and the MPP mode is forcibly used. Note that this setting only takes effect when `tidb_allow_mpp=true`. + +MPP is a distributed computing framework provided by the TiFlash engine, which allows data exchange between nodes and provides high-performance, high-throughput SQL algorithms. For details about the selection of the MPP mode, refer to [Control whether to select the MPP mode](/tiflash/use-tiflash.md#control-whether-to-select-the-mpp-mode). ### tidb_evolve_plan_baselines New in v4.0 - Scope: SESSION | GLOBAL -- Default value: off +- Default value: `OFF` - This variable is used to control whether to enable the baseline evolution feature. For detailed introduction or usage , see [Baseline Evolution](/sql-plan-management.md#baseline-evolution). - To reduce the impact of baseline evolution on the cluster, use the following configurations: - Set `tidb_evolve_plan_task_max_time` to limit the maximum execution time of each execution plan. The default value is 600s. @@ -406,25 +680,56 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_evolve_plan_task_end_time New in v4.0 - Scope: GLOBAL -- Default value: 23:59 +0000 +- Default value: `23:59 +0000` - This variable is used to set the end time of baseline evolution in a day. ### tidb_evolve_plan_task_max_time New in v4.0 - Scope: GLOBAL -- Default value: 600 +- Default value: `600` +- Range: `[-1, 9223372036854775807]` - This variable is used to limit the maximum execution time of each execution plan in the baseline evolution feature. The unit is second. ### tidb_evolve_plan_task_start_time New in v4.0 - Scope: GLOBAL -- Default value: 00:00 +0000 +- Default value: `00:00 +0000` - This variable is used to set the start time of baseline evolution in a day. +### tidb_executor_concurrency New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `5` +- Range: `[1, 2147483647]` + +This variable is used to set the concurrency of the following SQL operators (to one value): + +- `index lookup` +- `index lookup join` +- `hash join` +- `hash aggregation` (the `partial` and `final` phases) +- `window` +- `projection` + +`tidb_executor_concurrency` incorporates the following existing system variables as a whole for easier management: + ++ `tidb_index_lookup_concurrency` ++ `tidb_index_lookup_join_concurrency` ++ `tidb_hash_join_concurrency` ++ `tidb_hashagg_partial_concurrency` ++ `tidb_hashagg_final_concurrency` ++ `tidb_projection_concurrency` ++ `tidb_window_concurrency` + +Since v5.0, you can still separately modify the system variables listed above (with a deprecation warning returned) and your modification only affects the corresponding single operators. After that, if you use `tidb_executor_concurrency` to modify the operator concurrency, the separately modified operators will not be affected. If you want to use `tidb_executor_concurrency` to modify the concurrency of all operators, you can set the values of all variables listed above to `-1`. + +For a system upgraded to v5.0 from an earlier version, if you have not modified any value of the variables listed above (which means that the `tidb_hash_join_concurrency` value is `5` and the values of the rest are `4`), the operator concurrency previously managed by these variables will automatically be managed by `tidb_executor_concurrency`. If you have modified any of these variables, the concurrency of the corresponding operators will still be controlled by the modified variables. + ### tidb_expensive_query_time_threshold - Scope: INSTANCE -- Default value: 60 +- Default value: `60` +- Range: `[10, 2147483647]` - This variable is used to set the threshold value that determines whether to print expensive query logs. The unit is second. The difference between expensive query logs and slow query logs is: - Slow logs are printed after the statement is executed. - Expensive query logs print the statements that are being executed, with execution time exceeding the threshold value, and their related information. @@ -432,14 +737,61 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_force_priority - Scope: INSTANCE -- Default value: NO_PRIORITY +- Default value: `NO_PRIORITY` - This variable is used to change the default priority for statements executed on a TiDB server. A use case is to ensure that a particular user that is performing OLAP queries receives lower priority than users performing OLTP queries. - You can set the value of this variable to `NO_PRIORITY`, `LOW_PRIORITY`, `DELAYED` or `HIGH_PRIORITY`. +### tidb_gc_concurrency New in v5.0 + +- Scope: GLOBAL +- Default value: `-1` +- Range: `[1, 128]` +- Specifies the number of threads in the [Resolve Locks](/garbage-collection-overview.md#resolve-locks) step of GC. A value of `-1` means that TiDB will automatically decide the number of garbage collection threads to use. + +### tidb_gc_enable New in v5.0 + +- Scope: GLOBAL +- Default value: `ON` +- Enables garbage collection for TiKV. Disabling garbage collection will reduce system performance, as old versions of rows will no longer be purged. + +### tidb_gc_life_time New in v5.0 + +- Scope: GLOBAL +- Default value: `10m0s` +- Range: `[10m0s, 8760h0m0s]` +- The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point. + +> **Note:** +> +> - In scenarios of frequent updates, a large value (days or even months) for `tidb_gc_life_time` may cause potential issues, such as: +> - Larger storage use +> - A large amount of history data may affect performance to a certain degree, especially for range queries such as `select count(*) from t` +> - If there is any transaction that has been running longer than `tidb_gc_life_time`, during GC, the data since `start_ts` is retained for this transaction to continue execution. For example, if `tidb_gc_life_time` is configured to 10 minutes, among all transactions being executed, the transaction that starts earliest has been running for 15 minutes, GC will retain data of the recent 15 minutes. + +### tidb_gc_run_interval New in v5.0 + +- Scope: GLOBAL +- Default value: `10m0s` +- Range: `[10m0s, 8760h0m0s]` +- Specifies the GC interval, in the format of Go Duration, for example, `"1h30m"`, and `"15m"` + +### tidb_gc_scan_lock_mode New in v5.0 + +> **Warning:** +> +> Currently, Green GC is an experimental feature. It is not recommended that you use it in the production environment. + +- Scope: GLOBAL +- Default value: `LEGACY` +- Possible values: `PHYSICAL`, `LEGACY` + - `LEGACY`: Uses the old way of scanning, that is, disable Green GC. + - `PHYSICAL`: Uses the physical scanning method, that is, enable Green GC. +- This variable specifies the way of scanning locks in the Resolve Locks step of GC. When the variable value is set to `LEGACY`, TiDB scans locks by Regions. When the value `PHYSICAL` is used, it enables each TiKV node to bypass the Raft layer and directly scan data, which can effectively mitigate the impact of GC wakening up all Regions when the [Hibernate Region](/tikv-configuration-file.md#hibernate-regions) feature is enabled, thus improving the execution speed in the Resolve Locks step. + ### tidb_general_log - Scope: INSTANCE -- Default value: 0 +- Default value: `OFF` - This variable is used to set whether to record all SQL statements in the [log](/tidb-configuration-file.md#logfile). This feature is disabled by default. If maintenance personnel needs to trace all SQL statements when locating issues, they can enable this feature. - To see all records of this feature in the log, query the `"GENERAL_LOG"` string. The following information is recorded: - `conn`: The ID of the current session. @@ -454,121 +806,201 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_hash_join_concurrency +> **Warning:** +> +> Since v5.0, this variable is deprecated. Instead, use [`tidb_executor_concurrency`](#tidb_executor_concurrency-new-in-v50) for setting. + - Scope: SESSION | GLOBAL -- Default value: 5 +- Default value: `-1` +- Range: `[1, 2147483647]` - This variable is used to set the concurrency of the `hash join` algorithm. +- A value of `-1` means that the value of `tidb_executor_concurrency` will be used instead. ### tidb_hashagg_final_concurrency +> **Warning:** +> +> Since v5.0, this variable is deprecated. Instead, use [`tidb_executor_concurrency`](#tidb_executor_concurrency-new-in-v50) for setting. + - Scope: SESSION | GLOBAL -- Default value: 4 +- Default value: `-1` +- Range: `[1, 2147483647]` - This variable is used to set the concurrency of executing the concurrent `hash aggregation` algorithm in the `final` phase. - When the parameter of the aggregate function is not distinct, `HashAgg` is run concurrently and respectively in two phases - the `partial` phase and the `final` phase. +- A value of `-1` means that the value of `tidb_executor_concurrency` will be used instead. ### tidb_hashagg_partial_concurrency +> **Warning:** +> +> Since v5.0, this variable is deprecated. Instead, use [`tidb_executor_concurrency`](#tidb_executor_concurrency-new-in-v50) for setting. + - Scope: SESSION | GLOBAL -- Default value: 4 +- Default value: `-1` +- Range: `[1, 2147483647]` - This variable is used to set the concurrency of executing the concurrent `hash aggregation` algorithm in the `partial` phase. - When the parameter of the aggregate function is not distinct, `HashAgg` is run concurrently and respectively in two phases - the `partial` phase and the `final` phase. +- A value of `-1` means that the value of `tidb_executor_concurrency` will be used instead. ### tidb_index_join_batch_size - Scope: SESSION | GLOBAL -- Default value: 25000 +- Default value: `25000` +- Range: `[1, 2147483647]` - This variable is used to set the batch size of the `index lookup join` operation. - Use a bigger value in OLAP scenarios, and a smaller value in OLTP scenarios. ### tidb_index_lookup_concurrency +> **Warning:** +> +> Since v5.0, this variable is deprecated. Instead, use [`tidb_executor_concurrency`](#tidb_executor_concurrency-new-in-v50) for setting. + - Scope: SESSION | GLOBAL -- Default value: 4 +- Default value: `-1` +- Range: `[1, 2147483647]` - This variable is used to set the concurrency of the `index lookup` operation. - Use a bigger value in OLAP scenarios, and a smaller value in OLTP scenarios. +- A value of `-1` means that the value of `tidb_executor_concurrency` will be used instead. ### tidb_index_lookup_join_concurrency +> **Warning:** +> +> Since v5.0, this variable is deprecated. Instead, use [`tidb_executor_concurrency`](#tidb_executor_concurrency-new-in-v50) for setting. + - Scope: SESSION | GLOBAL -- Default value: 4 +- Default value: `-1` +- Range: `[1, 2147483647]` - This variable is used to set the concurrency of the `index lookup join` algorithm. +- A value of `-1` means that the value of `tidb_executor_concurrency` will be used instead. ### tidb_index_lookup_size - Scope: SESSION | GLOBAL -- Default value: 20000 +- Default value: `20000` +- Range: `[1, 2147483647]` - This variable is used to set the batch size of the `index lookup` operation. - Use a bigger value in OLAP scenarios, and a smaller value in OLTP scenarios. ### tidb_index_serial_scan_concurrency - Scope: SESSION | GLOBAL -- Default value: 1 +- Default value: `1` +- Range: `[1, 2147483647]` - This variable is used to set the concurrency of the `serial scan` operation. - Use a bigger value in OLAP scenarios, and a smaller value in OLTP scenarios. ### tidb_init_chunk_size - Scope: SESSION | GLOBAL -- Default value: 32 -- Range: 1 - 32 +- Default value: `32` +- Range: `[1, 32]` - This variable is used to set the number of rows for the initial chunk during the execution process. ### tidb_isolation_read_engines New in v4.0 - Scope: SESSION -- Default value: tikv, tiflash, tidb +- Default value: `tikv,tiflash,tidb` - This variable is used to set the storage engine list that TiDB can use when reading data. ### tidb_low_resolution_tso - Scope: SESSION -- Default value: 0 +- Default value: `OFF` - This variable is used to set whether to enable the low precision TSO feature. After this feature is enabled, new transactions use a timestamp updated every 2 seconds to read data. - The main applicable scenario is to reduce the overhead of acquiring TSO for small read-only transactions when reading old data is acceptable. ### tidb_max_chunk_size - Scope: SESSION | GLOBAL -- Default value: 1024 -- Minimum value: 32 +- Default value: `1024` +- Range: `[32, 2147483647]` - This variable is used to set the maximum number of rows in a chunk during the execution process. Setting to too large of a value may cause cache locality issues. ### tidb_max_delta_schema_count New in v2.1.18 and v3.0.5 - Scope: GLOBAL -- Default value: 1024 +- Default value: `1024` +- Range: `[100, 16384]` - This variable is used to set the maximum number of schema versions (the table IDs modified for corresponding versions) allowed to be cached. The value range is 100 ~ 16384. +### tidb_mem_quota_apply_cache New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `33554432` (32 MiB) +- Range: `[0, 9223372036854775807]` +- Unit: Bytes +- This variable is used to set the memory usage threshold of the local cache in the `Apply` operator. +- The local cache in the `Apply` operator is used to speed up the computation of the `Apply` operator. You can set the variable to `0` to disable the `Apply` cache feature. + ### tidb_mem_quota_query - Scope: SESSION -- Default value: 1 GB +- Default value: `1073741824` (1 GiB) +- Range: `[-1, 9223372036854775807]` +- Unit: Bytes - This variable is used to set the threshold value of memory quota for a query. - If the memory quota of a query during execution exceeds the threshold value, TiDB performs the operation designated by the OOMAction option in the configuration file. The initial value of this variable is configured by [`mem-quota-query`](/tidb-configuration-file.md#mem-quota-query). +### tidb_memory_usage_alarm_ratio + +- Scope: INSTANCE +- Default value: `0.8` +- TiDB triggers an alarm when the percentage of the memory it takes exceeds a certain threshold. For the detailed usage description of this feature, see [`memory-usage-alarm-ratio`](/tidb-configuration-file.md#memory-usage-alarm-ratio-new-in-v409). +- You can set the initial value of this variable by configuring [`memory-usage-alarm-ratio`](/tidb-configuration-file.md#memory-usage-alarm-ratio-new-in-v409). + ### tidb_metric_query_range_duration New in v4.0 - Scope: SESSION -- Default value: 60 +- Default value: `60` +- Range: `[10, 216000]` - This variable is used to set the range duration of the Prometheus statement generated when querying METRIC_SCHEMA. The unit is second. ### tidb_metric_query_step New in v4.0 - Scope: SESSION -- Default value: 60 +- Default value: `60` +- Range: `[10, 216000]` - This variable is used to set the step of the Prometheus statement generated when querying `METRIC_SCHEMA`. The unit is second. +### tidb_multi_statement_mode New in v4.0.11 + +- Scope: SESSION | GLOBAL +- Default value: `OFF` +- Possible values: `OFF`, `ON`, `WARN` +- This variable controls whether to allow multiple queries to be executed in the same `COM_QUERY` call. +- To reduce the impact of SQL injection attacks, TiDB now prevents multiple queries from being executed in the same `COM_QUERY` call by default. This variable is intended to be used as part of an upgrade path from earlier versions of TiDB. The following behaviors apply: + +| Client setting | `tidb_multi_statement_mode` value | Multiple statements permitted? | +| ------------------------- | --------------------------------- | ------------------------------ | +| Multiple Statements = ON | OFF | Yes | +| Multiple Statements = ON | ON | Yes | +| Multiple Statements = ON | WARN | Yes | +| Multiple Statements = OFF | OFF | No | +| Multiple Statements = OFF | ON | Yes | +| Multiple Statements = OFF | WARN | Yes (+warning returned) | + +> **Note:** +> +> Only the default value of `OFF` can be considered safe. Setting `tidb_multi_statement_mode=ON` might be required if your application was specifically designed for an earlier version of TiDB. If your application requires multiple statement support, it is recommended to use the setting provided by your client library instead of the `tidb_multi_statement_mode` option. For example: +> +> * [go-sql-driver](https://github.com/go-sql-driver/mysql#multistatements) (`multiStatements`) +> * [Connector/J](https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-configuration-properties.html) (`allowMultiQueries`) +> * PHP [mysqli](https://dev.mysql.com/doc/apis-php/en/apis-php-mysqli.quickstart.multiple-statement.html) (`mysqli_multi_query`) + ### tidb_opt_agg_push_down - Scope: SESSION -- Default value: 0 -- This variable is used to set whether the optimizer executes the optimization operation of pushing down the aggregate function to the position before Join. -- When the aggregate operation is slow in query, you can set the variable value to 1. +- Default value: `OFF` +- This variable is used to set whether the optimizer executes the optimization operation of pushing down the aggregate function to the position before Join, Projection, and UnionAll. +- When the aggregate operation is slow in query, you can set the variable value to ON. ### tidb_opt_correlation_exp_factor - Scope: SESSION | GLOBAL -- Default value: 1 +- Default value: `1` +- Range: `[0, 2147483647]` - When the method that estimates the number of rows based on column order correlation is not available, the heuristic estimation method is used. This variable is used to control the behavior of the heuristic method. - When the value is 0, the heuristic method is not used. - When the value is greater than 0: @@ -578,13 +1010,13 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_opt_correlation_threshold - Scope: SESSION | GLOBAL -- Default value: 0.9 +- Default value: `0.9` - This variable is used to set the threshold value that determines whether to enable estimating the row count by using column order correlation. If the order correlation between the current column and the `handle` column exceeds the threshold value, this method is enabled. ### tidb_opt_distinct_agg_push_down - Scope: SESSION -- Default value: 0 +- Default value: `OFF` - This variable is used to set whether the optimizer executes the optimization operation of pushing down the aggregate function with `distinct` (such as `select count(distinct a) from t`) to Coprocessor. - When the aggregate function with the `distinct` operation is slow in the query, you can set the variable value to `1`. @@ -619,7 +1051,7 @@ mysql> desc select count(distinct a) from test.t; ### tidb_opt_insubq_to_join_and_agg - Scope: SESSION | GLOBAL -- Default value: 1 +- Default value: `ON` - This variable is used to set whether to enable the optimization rule that converts a subquery to join and aggregation. - For example, after you enable this optimization rule, the subquery is converted as follows: @@ -639,46 +1071,104 @@ mysql> desc select count(distinct a) from test.t; select * from t, t1 where t.a=t1.a ``` +### tidb_opt_prefer_range_scan New in v5.0 + +- Scope: SESSION +- Default value: `OFF` +- After you set the value of this variable to `1`, the optimizer always prefers index scans over full table scans. +- In the following example, before you enable `tidb_opt_prefer_range_scan`, the TiDB optimizer performs a full table scan. After you enable `tidb_opt_prefer_range_scan`, the optimizer selects an index scan. + +```sql +explain select * from t where age=5; ++-------------------------+------------+-----------+---------------+-------------------+ +| id | estRows | task | access object | operator info | ++-------------------------+------------+-----------+---------------+-------------------+ +| TableReader_7 | 1048576.00 | root | | data:Selection_6 | +| └─Selection_6 | 1048576.00 | cop[tikv] | | eq(test.t.age, 5) | +| └─TableFullScan_5 | 1048576.00 | cop[tikv] | table:t | keep order:false | ++-------------------------+------------+-----------+---------------+-------------------+ +3 rows in set (0.00 sec) + +set session tidb_opt_prefer_range_scan = 1; + +explain select * from t where age=5; ++-------------------------------+------------+-----------+-----------------------------+-------------------------------+ +| id | estRows | task | access object | operator info | ++-------------------------------+------------+-----------+-----------------------------+-------------------------------+ +| IndexLookUp_7 | 1048576.00 | root | | | +| ├─IndexRangeScan_5(Build) | 1048576.00 | cop[tikv] | table:t, index:idx_age(age) | range:[5,5], keep order:false | +| └─TableRowIDScan_6(Probe) | 1048576.00 | cop[tikv] | table:t | keep order:false | ++-------------------------------+------------+-----------+-----------------------------+-------------------------------+ +3 rows in set (0.00 sec) +``` + ### tidb_opt_write_row_id - Scope: SESSION -- Default value: 0 -- This variable is used to set whether to allow `insert`, `replace` and `update` statements to operate on the column `_tidb_rowid`. It is not allowed by default. This variable can be used only when importing data with TiDB tools. +- Default value: `OFF` +- This variable is used to control whether to allow `INSERT`, `REPLACE`, and `UPDATE` statements to operate on the `_tidb_rowid` column. This variable can be used only when you import data using TiDB tools. + +### tidb_partition_prune_mode New in v5.1 + +> **Warning:** + +> Currently, the dynamic pruning mode for partitioned tables is an experimental feature. It is not recommended that you use it in the production environment. + +- Scope: SESSION | GLOBAL +- Default value: `static` +- Specifies whether to enable `dynamic` mode for partitioned tables. For details about the dynamic pruning mode, see [Dynamic Pruning Mode for Partitioned Tables](/partitioned-table.md#dynamic-pruning-mode). + +### tidb_pprof_sql_cpu New in v4.0 + +- Scope: INSTANCE +- Default value: `0` +- Range: `[0, 1]` +- This variable is used to control whether to mark the corresponding SQL statement in the profile output to identify and troubleshoot performance issues. ### tidb_projection_concurrency +> **Warning:** +> +> Since v5.0, this variable is deprecated. Instead, use [`tidb_executor_concurrency`](#tidb_executor_concurrency-new-in-v50) for setting. + - Scope: SESSION | GLOBAL -- Default value: 4 +- Default value: `-1` +- Range: `[-1, 2147483647]` - This variable is used to set the concurrency of the `Projection` operator. +- A value of `-1` means that the value of `tidb_executor_concurrency` will be used instead. ### tidb_query_log_max_len - Scope: INSTANCE -- Default value: 4096 (bytes) +- Default value: `4096` (4 KiB) +- Range: `[-1, 9223372036854775807]` +- Unit: Bytes - The maximum length of the SQL statement output. When the output length of a statement is larger than the `tidb_query-log-max-len` value, the statement is truncated to output. Usage example: ```sql -set tidb_query_log_max_len = 20 +SET tidb_query_log_max_len = 20 ``` -### tidb_pprof_sql_cpu New in v4.0 - -- Scope: INSTANCE -- Default value: 0 -- This variable is used to control whether to mark the corresponding SQL statement in the profile output to identify and troubleshoot performance issues. - ### tidb_record_plan_in_slow_log - Scope: INSTANCE -- Default value: `1` +- Default value: `ON` - This variable is used to control whether to include the execution plan of slow queries in the slow log. +### tidb_redact_log + +- Scope: SESSION | GLOBAL +- Default value: `OFF` +- This variable controls whether to hide user information in the SQL statement being recorded into the TiDB log and slow log. +- When you set the variable to `1`, user information is hidden. For example, if the executed SQL statement is `insert into t values (1,2)`, the statement is recorded as `insert into t values (?,?)` in the log. + ### tidb_replica_read New in v4.0 - Scope: SESSION -- Default value: leader +- Default value: `leader` +- Possible values: `leader`, `follower`, `leader-and-follower` - This variable is used to control where TiDB reads data. Here are three options: - leader: Read only from leader node - follower: Read only from follower node @@ -688,13 +1178,15 @@ set tidb_query_log_max_len = 20 ### tidb_retry_limit - Scope: SESSION | GLOBAL -- Default value: 10 -- This variable is used to set the maximum number of the retries. When a transaction encounters retryable errors (such as transaction conflicts, very slow transaction commit, or table schema changes), this transaction is re-executed according to this variable. Note that setting `tidb_retry_limit` to `0` disables the automatic retry. +- Default value: `10` +- Range: `[-1, 9223372036854775807]` +- This variable is used to set the maximum number of the retries for optimistic transactions. When a transaction encounters retryable errors (such as transaction conflicts, very slow transaction commit, or table schema changes), this transaction is re-executed according to this variable. Note that setting `tidb_retry_limit` to `0` disables the automatic retry. This variable only applies to optimistic transactions, not to pessimistic transactions. ### tidb_row_format_version - Scope: GLOBAL - Default value: `2` +- Range: `[1, 2]` - Controls the format version of the newly saved data in the table. In TiDB v4.0, the [new storage row format](https://github.com/pingcap/tidb/blob/master/docs/design/2018-07-19-row-format.md) version `2` is used by default to save new data. - If you upgrade from a TiDB version earlier than 4.0.0 to 4.0.0, the format version is not changed, and TiDB continues to use the old format of version `1` to write data to the table, which means that **only newly created clusters use the new data format by default**. - Note that modifying this variable does not affect the old data that has been saved, but applies the corresponding version format only to the newly written data after modifying this variable. @@ -702,13 +1194,20 @@ set tidb_query_log_max_len = 20 ### tidb_scatter_region - Scope: GLOBAL -- Default value: 0 +- Default value: `OFF` - By default, Regions are split for a new table when it is being created in TiDB. After this variable is enabled, the newly split Regions are scattered immediately during the execution of the `CREATE TABLE` statement. This applies to the scenario where data need to be written in batches right after the tables are created in batches, because the newly split Regions can be scattered in TiKV beforehand and do not have to wait to be scheduled by PD. To ensure the continuous stability of writing data in batches, the `CREATE TABLE` statement returns success only after the Regions are successfully scattered. This makes the statement's execution time multiple times longer than that when you disable this variable. +### tidb_skip_ascii_check New in v5.0 + +- Scope: SESSION | GLOBAL +- Default value: `OFF` +- This variable is used to set whether to skip ASCII validation. +- Validating ASCII characters affects the performance. When you are sure that the input characters are valid ASCII characters, you can set the variable value to `ON`. + ### tidb_skip_isolation_level_check - Scope: SESSION | GLOBAL -- Default value: 0 +- Default value: `OFF` - After this switch is enabled, if an isolation level unsupported by TiDB is assigned to `tx_isolation`, no error is reported. This helps improve compatibility with applications that set (but do not depend on) a different isolation level. ```sql @@ -724,14 +1223,16 @@ Query OK, 0 rows affected, 1 warning (0.00 sec) ### tidb_skip_utf8_check - Scope: SESSION | GLOBAL -- Default value: 0 +- Default value: `OFF` - This variable is used to set whether to skip UTF-8 validation. -- Validating UTF-8 characters affects the performance. When you are sure that the input characters are valid UTF-8 characters, you can set the variable value to 1. +- Validating UTF-8 characters affects the performance. When you are sure that the input characters are valid UTF-8 characters, you can set the variable value to `ON`. ### tidb_slow_log_threshold - Scope: INSTANCE -- Default value: 300ms +- Default value: `300` +- Range: `[-1, 9223372036854775807]` +- Unit: Milliseconds - This variable is used to output the threshold value of the time consumed by the slow log. When the time consumed by a query is larger than this value, this query is considered as a slow log and its log is output to the slow query log. Usage example: @@ -740,19 +1241,6 @@ Usage example: SET tidb_slow_log_threshold = 200; ``` -### tidb_enable_collect_execution_info - -- Scope: INSTANCE -- Default value: 1 -- This variable controls whether to record the execution information of each operator in the slow query log. - -### tidb_log_desensitization - -- Scope: GLOBAL -- Default value: 0 -- This variable controls whether to hide user information in the SQL statement being recorded into the TiDB log and slow log. -- When you set the variable to `1`, user information is hidden. For example, if the executed SQL statement is `insert into t values (1,2)`, the statement is recorded as `insert into t values (?,?)` in the log. - ### tidb_slow_query_file - Scope: SESSION @@ -768,43 +1256,49 @@ SET tidb_slow_log_threshold = 200; ### tidb_stmt_summary_history_size New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 24 (the value of the default configuration file) +- Default value: `24` +- Range: `[0, 255]` - This variable is used to set the history capacity of the statement summary. ### tidb_stmt_summary_internal_query New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 0 (the value of the default configuration file) +- Default value: `OFF` - This variable is used to control whether to include the SQL information of TiDB in the statement summary. ### tidb_stmt_summary_max_sql_length New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 4096 (the value of the default configuration file) +- Default value: `4096` +- Range: `[0, 2147483647]` - This variable is used to control the length of the SQL string in the statement summary. ### tidb_stmt_summary_max_stmt_count New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 200 (the value of the default configuration file) +- Default value: `3000` +- Range: `[1, 32767]` - This variable is used to set the maximum number of statements that the statement summary stores in memory. ### tidb_stmt_summary_refresh_interval New in v4.0 - Scope: SESSION | GLOBAL -- Default value: 1800 (the value of the default configuration file) +- Default value: `1800` +- Range: `[1, 2147483647]` - This variable is used to set the refresh time of the statement summary. The unit is second. ### tidb_store_limit New in v3.0.4 and v4.0 - Scope: INSTANCE | GLOBAL -- Default value: 0 +- Default value: `0` +- Range: `[0, 9223372036854775807]` - This variable is used to limit the maximum number of requests TiDB can send to TiKV at the same time. 0 means no limit. ### tidb_txn_mode - Scope: SESSION | GLOBAL -- Default value: "pessimistic" +- Default value: `pessimistic` +- Possible values: `pessimistic`, `optimistic` - This variable is used to set the transaction mode. TiDB 3.0 supports the pessimistic transactions. Since TiDB 3.0.8, the [pessimistic transaction mode](/pessimistic-transaction.md) is enabled by default. - If you upgrade TiDB from v3.0.7 or earlier versions to v3.0.8 or later versions, the default transaction mode does not change. **Only the newly created clusters use the pessimistic transaction mode by default**. - If this variable is set to "optimistic" or "", TiDB uses the [optimistic transaction mode](/optimistic-transaction.md). @@ -812,38 +1306,49 @@ SET tidb_slow_log_threshold = 200; ### tidb_use_plan_baselines New in v4.0 - Scope: SESSION | GLOBAL -- Default value: on -- This variable is used to control whether to enable the execution plan binding feature. It is enabled by default, and can be disabled by assigning the `off` value. For the use of the execution plan binding, see [Execution Plan Binding](/sql-plan-management.md#create-a-binding). +- Default value: `ON` +- This variable is used to control whether to enable the execution plan binding feature. It is enabled by default, and can be disabled by assigning the `OFF` value. For the use of the execution plan binding, see [Execution Plan Binding](/sql-plan-management.md#create-a-binding). ### tidb_wait_split_region_finish - Scope: SESSION -- Default value: 1, indicating returning the result after all Regions are scattered. -- It usually takes a long time to scatter Regions, which is determined by PD scheduling and TiKV loads. This variable is used to set whether to return the result to the client after all Regions are scattered completely when the `SPLIT REGION` statement is being executed. Value `0` indicates returning the value before finishing scattering all Regions. +- Default value: `ON` +- It usually takes a long time to scatter Regions, which is determined by PD scheduling and TiKV loads. This variable is used to set whether to return the result to the client after all Regions are scattered completely when the `SPLIT REGION` statement is being executed: + - `ON` requires that the `SPLIT REGIONS` statement waits until all Regions are scattered. + - `OFF` permits the `SPLIT REGIONS` statement to return before finishing scattering all Regions. - Note that when scattering Regions, the write and read performances for the Region that is being scattered might be affected. In batch-write or data importing scenarios, it is recommended to import data after Regions scattering is finished. ### tidb_wait_split_region_timeout - Scope: SESSION -- Default value: 300 +- Default value: `300` +- Range: `[1, 2147483647]` - This variable is used to set the timeout for executing the `SPLIT REGION` statement. The unit is second. If a statement is not executed completely within the specified time value, a timeout error is returned. ### tidb_window_concurrency New in v4.0 +> **Warning:** +> +> Since v5.0, this variable is deprecated. Instead, use [`tidb_executor_concurrency`](#tidb_executor_concurrency-new-in-v50) for setting. + - Scope: SESSION | GLOBAL -- Default value: 4 +- Default value: `-1` +- Range: `[1, 2147483647]` - This variable is used to set the concurrency degree of the window operator. +- A value of `-1` means that the value of `tidb_executor_concurrency` will be used instead. ### time_zone - Scope: SESSION | GLOBAL -- Default value: SYSTEM -- This variable sets the system time zone. Values can be specified as either an offset such as '-8:00' or a named zone 'America/Los_Angeles'. +- Default value: `SYSTEM` +- This variable returns the current time zone. Values can be specified as either an offset such as '-8:00' or a named zone 'America/Los_Angeles'. +- The value `SYSTEM` means that the time zone should be the same as the system host, which is available via the [`system_time_zone`](#system_time_zone) variable. ### transaction_isolation - Scope: SESSION | GLOBAL -- Default value: REPEATABLE-READ +- Default value: `REPEATABLE-READ` +- Possible values: `READ-UNCOMMITTED`, `READ-COMMITTED`, `REPEATABLE-READ`, `SERIALIZABLE` - This variable sets the transaction isolation. TiDB advertises `REPEATABLE-READ` for compatibility with MySQL, but the actual isolation level is Snapshot Isolation. See [transaction isolation levels](/transaction-isolation-levels.md) for further details. ### tx_isolation @@ -853,7 +1358,7 @@ This variable is an alias for _transaction_isolation_. ### version - Scope: NONE -- Default value: 5.7.25-TiDB-(tidb version) +- Default value: `5.7.25-TiDB-`(tidb version) - This variable returns the MySQL version, followed by the TiDB version. For example '5.7.25-TiDB-v4.0.0-beta.2-716-g25e003253'. ### version_comment @@ -865,56 +1370,19 @@ This variable is an alias for _transaction_isolation_. ### wait_timeout - Scope: SESSION | GLOBAL -- Default value: 0 -- This variable controls the idle timeout of user sessions in seconds. A zero-value means unlimited. +- Default value: `0` +- Range: `[0, 31536000]` +- Unit: Seconds +- This variable controls the idle timeout of user sessions. A zero-value means unlimited. -### windowing_use_high_precision - -- Scope: SESSION | GLOBAL -- Default value: ON -- This variable controls whether to use the high precision mode when computing the window functions. - -### tidb_opt_prefer_range_scan +### warning_count - Scope: SESSION -- Default value: 0 -- After you set the value of this variable to `1`, the optimizer always prefers index scans over full table scans. -- In the following example, before you enable `tidb_opt_prefer_range_scan`, the TiDB optimizer performs a full table scan . After you enable `tidb_opt_prefer_range_scan`, the optimizer selects an index scan. - -```sql -explain select * from t where age=5; -+-------------------------+------------+-----------+---------------+-------------------+ -| id | estRows | task | access object | operator info | -+-------------------------+------------+-----------+---------------+-------------------+ -| TableReader_7 | 1048576.00 | root | | data:Selection_6 | -| └─Selection_6 | 1048576.00 | cop[tikv] | | eq(test.t.age, 5) | -| └─TableFullScan_5 | 1048576.00 | cop[tikv] | table:t | keep order:false | -+-------------------------+------------+-----------+---------------+-------------------+ -3 rows in set (0.00 sec) +- Default value: `0` +- This read-only variable indicates the number of warnings that occurred in the statement that was previously executed. -set session tidb_opt_prefer_range_scan = 1; - -explain select * from t where age=5; -+-------------------------------+------------+-----------+-----------------------------+-------------------------------+ -| id | estRows | task | access object | operator info | -+-------------------------------+------------+-----------+-----------------------------+-------------------------------+ -| IndexLookUp_7 | 1048576.00 | root | | | -| ├─IndexRangeScan_5(Build) | 1048576.00 | cop[tikv] | table:t, index:idx_age(age) | range:[5,5], keep order:false | -| └─TableRowIDScan_6(Probe) | 1048576.00 | cop[tikv] | table:t | keep order:false | -+-------------------------------+------------+-----------+-----------------------------+-------------------------------+ -3 rows in set (0.00 sec) -``` - -### `tidb_enable_rate_limit_action` +### windowing_use_high_precision - Scope: SESSION | GLOBAL -- Default value: ON -- This variable controls whether to enable the dynamic memory control feature for the operator that reads data. By default, this operator enables the maximum number of threads that [`tidb_disql_scan_concurrency`](/system-variables.md#tidb_distsql_scan_concurrency) allows to read data. When the memory usage of a single SQL statement exceeds [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) each time, the operator that reads data stops one thread. -- When the operator that reads data has only one thread left and the memory usage of a single SQL statement continues to exceed [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query), this SQL statement triggers other memory control behaviors, such as [spilling data to disk](/tidb-configuration-file.md#spilled-file-encryption-method). - -### `tidb_memory_usage_alarm_ratio` - -- Scope: SESSION -- Default value: 0.8 -- TiDB triggers an alarm when the percentage of the memory it takes exceeds a certain threshold. For the detailed usage description of this feature, see [`memory-usage-alarm-ratio`](/tidb-configuration-file.md#memory-usage-alarm-ratio-new-in-v409). -- You can set the initial value of this variable by configuring [`memory-usage-alarm-ratio`](/tidb-configuration-file.md#memory-usage-alarm-ratio-new-in-v409). +- Default value: `ON` +- This variable controls whether to use the high precision mode when computing the window functions. diff --git a/table-filter.md b/table-filter.md index d436df02442d..dd000906a74d 100644 --- a/table-filter.md +++ b/table-filter.md @@ -36,7 +36,7 @@ Table filters can be applied to the tools using multiple `-f` or `--filter` comm # ^~~~~~~~~~~~~~~~~~~~~~~ ``` -* [Lightning](/tidb-lightning/tidb-lightning-overview.md): +* [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md): {{< copyable "shell-regular" >}} @@ -49,7 +49,7 @@ Table filters can be applied to the tools using multiple `-f` or `--filter` comm Table filters in TOML files are specified as [array of strings](https://toml.io/en/v1.0.0-rc.1#section-15). The following lists the example usage in each tool. -* Lightning: +* TiDB Lightning: ```toml [mydumper] @@ -241,7 +241,7 @@ The filtered outcome is as follows: > **Note:** > -> In TiDB tools, the system schemas are always excluded regardless of the table filter settings. The system schemas are: +> In TiDB tools, the system schemas are always excluded in the default configuration. The system schemas are: > > * `INFORMATION_SCHEMA` > * `PERFORMANCE_SCHEMA` diff --git a/telemetry.md b/telemetry.md index 89b5f47709c9..d067ffcd0595 100644 --- a/telemetry.md +++ b/telemetry.md @@ -18,10 +18,13 @@ The following sections describe the shared usage information in detail for each ### TiDB -When the telemetry collection feature is enabled in TiDB, the TiDB cluster collects usage details on a daily basis. These usage details include but are not limited to: +When the telemetry collection feature is enabled in TiDB, the TiDB cluster collects usage details on a 6-hour basis. These usage details include but are not limited to: - A randomly generated telemetry ID. - Deployment characteristics, such as the size of hardware (CPU, memory, disk), TiDB components versions, OS name. +- The status of query requests in the system, such as the number of query requests and the duration. +- Component usage, for example, whether the Async Commit feature is in use or not. +- The usage information of built-in functions, including per-function statistics on how often the function gets called. The arguments for the functions are not included. To view the full content of the usage information shared to PingCAP, execute the following SQL statement: @@ -33,7 +36,7 @@ ADMIN SHOW TELEMETRY; ### TiDB Dashboard -When the telemetry collection feature is enabled in TiDB Dashboard, usage information on the TiDB Dashboard web UI will be shared, including (but not limited to): +When the telemetry collection feature is enabled for TiDB Dashboard, usage information on the TiDB Dashboard web UI will be shared, including (but not limited to): - A randomly generated telemetry ID. - User operation information, such as the name of the TiDB Dashboard web page accessed by the user. @@ -120,25 +123,6 @@ server_configs: -
- Deployment using TiDB Ansible - -Locate the following contents in the configuration file `tidb-ansible/conf/tidb.yml`: - -```yaml -# enable-telemetry: true -``` - -And change this content as follow: - -```yaml -enable-telemetry: false -``` - -See [Deploy TiDB Using TiDB Ansible](/online-deployment-using-ansible.md) for details. - -
-
Deployment in Kubernetes via TiDB Operator @@ -154,7 +138,7 @@ See [Deploy TiDB Operator in Kubernetes](https://docs.pingcap.com/tidb-in-kubern ### Disable TiDB telemetry for deployed TiDB clusters -In existing TiDB clusters, you can also modify the system variable [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402-version) to dynamically disable the TiDB telemetry collection: +In existing TiDB clusters, you can also modify the system variable [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) to dynamically disable the TiDB telemetry collection: {{< copyable "sql" >}} @@ -229,29 +213,6 @@ server_configs:
-
- Deployment using TiDB Ansible - -Locate the following content in the `tidb-ansible/conf/pd.yml` configuration file: - -```yaml -dashboard: - ... - # enable-telemetry: true -``` - -And change the content as follows: - -```yaml -dashboard: - ... - enable-telemetry: false -``` - -See [Deploy TiDB Using TiDB Ansible](/online-deployment-using-ansible.md) for details. - -
-
Deployment in Kubernetes via TiDB Operator diff --git a/test-deployment-from-binary-tarball.md b/test-deployment-from-binary-tarball.md deleted file mode 100644 index cf9c64bd5543..000000000000 --- a/test-deployment-from-binary-tarball.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -title: Testing Deployment from Binary Tarball -summary: Use the binary to deploy a TiDB cluster. -aliases: ['/docs/dev/test-deployment-from-binary-tarball/','/docs/dev/how-to/deploy/from-tarball/testing-environment/'] ---- - -# Testing Deployment from Binary Tarball - -This guide provides installation instructions for all TiDB components across multiple nodes for testing purposes. It does not match the recommended usage for production systems. - -See also [local deployment](/deploy-tidb-from-binary.md) and [production environment](/production-deployment-from-binary-tarball.md) deployment. - -## Prepare - -Before you start, see [TiDB architecture](/tidb-architecture.md) and [Software and Hardware Recommendations](/hardware-and-software-requirements.md). Make sure the following requirements are satisfied: - -### Operating system - -For the operating system, it is recommended to use RHEL/CentOS 7.3 or higher. The following additional requirements are recommended: - -| Configuration | Description | -| :-- | :-------------------- | -| Supported Platform | RHEL/CentOS 7.3+ ([more details](/hardware-and-software-requirements.md)) | -| File System | ext4 is recommended | -| Swap Space | Should be disabled | -| Disk Block Size | Set the system disk `Block` size to `4096` | - -### Network and firewall - -| Configuration | Description | -| :-- | :------------------- | -| Firewall/Port | Check whether the ports required by TiDB are accessible between the nodes | - -### Operating system parameters - -| Configuration | Description | -| :-- | :-------------------------- | -| Nice Limits | For system users, set the default value of `nice` in TiDB to `0` | -| min_free_kbytes | The setting for `vm.min_free_kbytes` in `sysctl.conf` needs to be high enough | -| User Open Files Limit | For database administrators, set the number of TiDB open files to `1000000` | -| System Open File Limits | Set the number of system open files to `1000000` | -| User Process Limits | For TiDB users, set the `nproc` value to `4096` in `limits.conf` | -| Address Space Limits | For TiDB users, set the space to `unlimited` in `limits.conf` | -| File Size Limits | For TiDB users, set the `fsize` value to `unlimited` in `limits.conf` | -| Disk Readahead | Set the value of the `readahead` data disk to `4096` at a minimum | -| NTP service | Configure the NTP time synchronization service for each node | -| SELinux | Turn off the SELinux service for each node | -| CPU Frequency Scaling | It is recommended to turn on CPU overclocking | -| Transparent Hugepages | For Red Hat 7+ and CentOS 7+ systems, it is required to set the Transparent Hugepages to `always` | -| I/O Scheduler | Set the I/O Scheduler of data disks to the `deadline` mode | -| vm.swappiness | Set `vm.swappiness = 0` in `sysctl.conf` | -| net.core.somaxconn | Set `net.core.somaxconn = 32768` in `sysctl.conf` | -| net.ipv4.tcp_syncookies | Set `net.ipv4.tcp_syncookies = 0` in `sysctl.conf` | - -### Database running user settings - -| Configuration | Description | -| :-- | :---------------------------- | -| LANG environment | Set `LANG = en_US.UTF8` | -| TZ time zone | Set the TZ time zone of all nodes to the same value | - -## TiDB components and default ports - -Before you deploy a TiDB cluster, see the [required components](#tidb-database-components-required) and [optional components](#tidb-database-components-optional). - -### TiDB database components (required) - -See the following table for the default ports for the TiDB components: - -| Component | Default Port | Protocol | Description | -| :-- | :-- | :-- | :----------- | -| ssh | 22 | TCP | the sshd service | -| TiDB| 4000 | TCP | the communication port for the application and DBA tools | -| TiDB| 10080 | TCP | the communication port to report TiDB status | -| TiKV| 20160 | TCP | the TiKV communication port | -| PD | 2379 | TCP | the communication port between TiDB and PD | -| PD | 2380 | TCP | the inter-node communication port within the PD cluster | - -### TiDB database components (optional) - -See the following table for the default ports for the optional TiDB components: - -| Component | Default Port | Protocol | Description | -| :-- | :-- | :-- | :------------------------ | -| Prometheus | 9090| TCP | the communication port for the Prometheus service | -| Pushgateway | 9091 | TCP | the aggregation and report port for TiDB, TiKV, and PD monitor | -| Node_exporter| 9100| TCP | the communication port to report the system information of every TiDB cluster node | -| Grafana | 3000 | TCP | the port for the external Web monitoring service and client (Browser) access | -| alertmanager | 9093 | TCP | the port for the alert service | - -## Create a database running user account - -1. Log in to the machine using the `root` user account and create a database running user account (`tidb`) using the following command: - - ```bash - # useradd tidb -m - ``` - -2. Switch the user from `root` to `tidb` by using the following command. You can use this `tidb` user account to deploy your TiDB cluster. - - ```bash - # su - tidb - ``` - -## Download the official binary package - -``` -# Download the package. -$ wget https://download.pingcap.org/tidb-latest-linux-amd64.tar.gz -$ wget https://download.pingcap.org/tidb-latest-linux-amd64.sha256 - -# Check the file integrity. If the result is OK, the file is correct. -$ sha256sum -c tidb-latest-linux-amd64.sha256 - -# Extract the package. -$ tar -xzf tidb-latest-linux-amd64.tar.gz -$ cd tidb-latest-linux-amd64 -``` - -## Multiple nodes cluster deployment for test - -If you want to test TiDB but have a limited number of nodes, you can use one PD instance to test the entire cluster. - -Assuming that you have four nodes, you can deploy 1 PD instance, 3 TiKV instances, and 1 TiDB instance. See the following table for details: - -| Name | Host IP | Services | -| :-- | :-- | :------------------- | -| Node1 | 192.168.199.113 | PD1, TiDB | -| Node2 | 192.168.199.114 | TiKV1 | -| Node3 | 192.168.199.115 | TiKV2 | -| Node4 | 192.168.199.116 | TiKV3 | - -Follow the steps below to start PD, TiKV and TiDB: - -1. Start PD on Node1. - - ```bash - $ ./bin/pd-server --name=pd1 \ - --data-dir=pd \ - --client-urls="http://192.168.199.113:2379" \ - --peer-urls="http://192.168.199.113:2380" \ - --initial-cluster="pd1=http://192.168.199.113:2380" \ - --log-file=pd.log & - ``` - -2. Start TiKV on Node2, Node3 and Node4. - - ```bash - $ ./bin/tikv-server --pd="192.168.199.113:2379" \ - --addr="192.168.199.114:20160" \ - --data-dir=tikv \ - --log-file=tikv.log & - - $ ./bin/tikv-server --pd="192.168.199.113:2379" \ - --addr="192.168.199.115:20160" \ - --data-dir=tikv \ - --log-file=tikv.log & - - $ ./bin/tikv-server --pd="192.168.199.113:2379" \ - --addr="192.168.199.116:20160" \ - --data-dir=tikv \ - --log-file=tikv.log & - ``` - -3. Start TiDB on Node1. - - ```bash - $ ./bin/tidb-server --store=tikv \ - --path="192.168.199.113:2379" \ - --log-file=tidb.log - ``` - -4. Use the MySQL client to connect to TiDB. - - ```sh - $ mysql -h 192.168.199.113 -P 4000 -u root -D test - ``` diff --git a/test-deployment-using-docker.md b/test-deployment-using-docker.md deleted file mode 100644 index 8e467c04d604..000000000000 --- a/test-deployment-using-docker.md +++ /dev/null @@ -1,213 +0,0 @@ ---- -title: Deploy TiDB Using Docker -summary: Use Docker to manually deploy a multi-node TiDB cluster on multiple machines. -aliases: ['/docs/dev/test-deployment-using-docker/','/docs/dev/how-to/deploy/orchestrated/docker/'] ---- - -# Deploy TiDB Using Docker - -> **Warning:** -> -> The Docker deployment method provided in this document is no longer maintained. If you want to test TiDB, it is recommended to refer to [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md) for deployment. For **production environment**, **do not use** Docker Compose for deployment, but [deploy TiDB with TiUP](/production-deployment-using-tiup.md) or [TiDB Operator in Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable/deploy-tidb-operator). - -This document shows you how to manually deploy a multi-node TiDB cluster on multiple machines using Docker. - -To learn more, see [TiDB architecture](/tidb-architecture.md) and [Software and Hardware Recommendations](/hardware-and-software-requirements.md). - -## Preparation - -Before you start, make sure that you have: - -+ Installed the latest version of [Docker](https://www.docker.com/products/docker) -+ Pulled the latest images of TiDB, TiKV and PD from [Docker Hub](https://hub.docker.com/u/pingcap). If not, pull the images using the following commands: - - ```bash - docker pull pingcap/tidb:latest - docker pull pingcap/tikv:latest - docker pull pingcap/pd:latest - ``` - -## Multi nodes deployment - -Assume we have 6 machines with the following details: - -| Host Name | IP | Services | Data Path | -| --------- | ------------- | ---------- | --------- | -| **host1** | 192.168.1.101 | PD1 & TiDB | /data | -| **host2** | 192.168.1.102 | PD2 | /data | -| **host3** | 192.168.1.103 | PD3 | /data | -| **host4** | 192.168.1.104 | TiKV1 | /data | -| **host5** | 192.168.1.105 | TiKV2 | /data | -| **host6** | 192.168.1.106 | TiKV3 | /data | - -### 1. Start PD - -Start PD1 on the **host1** - -```bash -docker run -d --name pd1 \ - -p 2379:2379 \ - -p 2380:2380 \ - -v /etc/localtime:/etc/localtime:ro \ - -v /data:/data \ - pingcap/pd:latest \ - --name="pd1" \ - --data-dir="/data/pd1" \ - --client-urls="http://0.0.0.0:2379" \ - --advertise-client-urls="http://192.168.1.101:2379" \ - --peer-urls="http://0.0.0.0:2380" \ - --advertise-peer-urls="http://192.168.1.101:2380" \ - --initial-cluster="pd1=http://192.168.1.101:2380,pd2=http://192.168.1.102:2380,pd3=http://192.168.1.103:2380" -``` - -Start PD2 on the **host2** - -```bash -docker run -d --name pd2 \ - -p 2379:2379 \ - -p 2380:2380 \ - -v /etc/localtime:/etc/localtime:ro \ - -v /data:/data \ - pingcap/pd:latest \ - --name="pd2" \ - --data-dir="/data/pd2" \ - --client-urls="http://0.0.0.0:2379" \ - --advertise-client-urls="http://192.168.1.102:2379" \ - --peer-urls="http://0.0.0.0:2380" \ - --advertise-peer-urls="http://192.168.1.102:2380" \ - --initial-cluster="pd1=http://192.168.1.101:2380,pd2=http://192.168.1.102:2380,pd3=http://192.168.1.103:2380" -``` - -Start PD3 on the **host3** - -```bash -docker run -d --name pd3 \ - -p 2379:2379 \ - -p 2380:2380 \ - -v /etc/localtime:/etc/localtime:ro \ - -v /data:/data \ - pingcap/pd:latest \ - --name="pd3" \ - --data-dir="/data/pd3" \ - --client-urls="http://0.0.0.0:2379" \ - --advertise-client-urls="http://192.168.1.103:2379" \ - --peer-urls="http://0.0.0.0:2380" \ - --advertise-peer-urls="http://192.168.1.103:2380" \ - --initial-cluster="pd1=http://192.168.1.101:2380,pd2=http://192.168.1.102:2380,pd3=http://192.168.1.103:2380" -``` - -### 2. Start TiKV - -Start TiKV1 on the **host4** - -```bash -docker run -d --name tikv1 \ - -p 20160:20160 \ - -v /etc/localtime:/etc/localtime:ro \ - -v /data:/data \ - pingcap/tikv:latest \ - --addr="0.0.0.0:20160" \ - --advertise-addr="192.168.1.104:20160" \ - --data-dir="/data/tikv1" \ - --pd="192.168.1.101:2379,192.168.1.102:2379,192.168.1.103:2379" -``` - -Start TiKV2 on the **host5** - -```bash -docker run -d --name tikv2 \ - -p 20160:20160 \ - -v /etc/localtime:/etc/localtime:ro \ - -v /data:/data \ - pingcap/tikv:latest \ - --addr="0.0.0.0:20160" \ - --advertise-addr="192.168.1.105:20160" \ - --data-dir="/data/tikv2" \ - --pd="192.168.1.101:2379,192.168.1.102:2379,192.168.1.103:2379" -``` - -Start TiKV3 on the **host6** - -```bash -docker run -d --name tikv3 \ - -p 20160:20160 \ - -v /etc/localtime:/etc/localtime:ro \ - -v /data:/data \ - pingcap/tikv:latest \ - --addr="0.0.0.0:20160" \ - --advertise-addr="192.168.1.106:20160" \ - --data-dir="/data/tikv3" \ - --pd="192.168.1.101:2379,192.168.1.102:2379,192.168.1.103:2379" -``` - -### 3. Start TiDB - -Start TiDB on the **host1** - -```bash -docker run -d --name tidb \ - -p 4000:4000 \ - -p 10080:10080 \ - -v /etc/localtime:/etc/localtime:ro \ - pingcap/tidb:latest \ - --store=tikv \ - --path="192.168.1.101:2379,192.168.1.102:2379,192.168.1.103:2379" -``` - -### 4. Use the MySQL client to connect to TiDB - -Install the [MySQL client](http://dev.mysql.com/downloads/mysql/) on **host1** and run: - -```bash -$ mysql -h 127.0.0.1 -P 4000 -u root -D test -mysql> show databases; -+--------------------+ -| Database | -+--------------------+ -| INFORMATION_SCHEMA | -| PERFORMANCE_SCHEMA | -| mysql | -| test | -+--------------------+ -4 rows in set (0.00 sec) -``` - -### How to customize the configuration file - -The TiKV and PD can be started with a specified configuration file, which includes some advanced parameters, for the performance tuning. - -Assume that the path to configuration file of PD and TiKV on the host is `/path/to/config/pd.toml` and `/path/to/config/tikv.toml` - -You can start TiKV and PD as follows: - -```bash -docker run -d --name tikv1 \ - -p 20160:20160 \ - -v /etc/localtime:/etc/localtime:ro \ - -v /data:/data \ - -v /path/to/config/tikv.toml:/tikv.toml:ro \ - pingcap/tikv:latest \ - --addr="0.0.0.0:20160" \ - --advertise-addr="192.168.1.104:20160" \ - --data-dir="/data/tikv1" \ - --pd="192.168.1.101:2379,192.168.1.102:2379,192.168.1.103:2379" \ - --config="/tikv.toml" -``` - -```bash -docker run -d --name pd1 \ - -p 2379:2379 \ - -p 2380:2380 \ - -v /etc/localtime:/etc/localtime:ro \ - -v /data:/data \ - -v /path/to/config/pd.toml:/pd.toml:ro \ - pingcap/pd:latest \ - --name="pd1" \ - --data-dir="/data/pd1" \ - --client-urls="http://0.0.0.0:2379" \ - --advertise-client-urls="http://192.168.1.101:2379" \ - --peer-urls="http://0.0.0.0:2380" \ - --advertise-peer-urls="http://192.168.1.101:2380" \ - --initial-cluster="pd1=http://192.168.1.101:2380,pd2=http://192.168.1.102:2380,pd3=http://192.168.1.103:2380" \ - --config="/pd.toml" -``` diff --git a/three-data-centers-in-two-cities-deployment.md b/three-data-centers-in-two-cities-deployment.md index 847fb9b35643..620496c7491b 100644 --- a/three-data-centers-in-two-cities-deployment.md +++ b/three-data-centers-in-two-cities-deployment.md @@ -50,8 +50,8 @@ The configuration of the three DCs in two cities (Seattle and San Francisco) dep - From the illustration above, you can see that Seattle has two DCs: IDC1 and IDC2. IDC1 has three sets of racks: RAC1, RAC2, and RAC3. IDC2 has two racks: RAC4 and RAC5. The IDC3 DC in San Francisco has the RAC6 rack. - From the RAC1 rack illustrated above, TiDB and PD services are deployed on the same server. Each of the two TiKV servers are deployed with two TiKV instances (tikv-server). This is similar to RAC2, RAC4, RAC5, and RAC6. -- The TiDB server, the control machine, and the monitoring server are on RAC3. The TiDB server is deployed for regular maintenance and backup. TiDB Ansible, Prometheus, Grafana, and the restore tools are deployed on the control machine and monitoring machine. -- Another backup server can be added to deploy Mydumper and Drainer. Drainer saves binlog data to a specified location by outputting files, to achieve incremental backup. +- The TiDB server, the control machine, and the monitoring server are on RAC3. The TiDB server is deployed for regular maintenance and backup. Prometheus, Grafana, and the restore tools are deployed on the control machine and monitoring machine. +- Another backup server can be added to deploy Drainer. Drainer saves binlog data to a specified location by outputting files, to achieve incremental backup. ## Configuration diff --git a/ticdc-deployment-topology.md b/ticdc-deployment-topology.md index c126ad538266..761cbad6a468 100644 --- a/ticdc-deployment-topology.md +++ b/ticdc-deployment-topology.md @@ -10,7 +10,7 @@ aliases: ['/docs/dev/ticdc-deployment-topology/'] > > TiCDC is a feature for general availability (GA) since v4.0.6. You can use it in the production environment. -This document describes the deployment topology of TiCDC based on the minimal cluster topology. +This document describes the deployment topology of [TiCDC](/ticdc/ticdc-overview.md) based on the minimal cluster topology. TiCDC is a tool for replicating the incremental data of TiDB, introduced in TiDB 4.0. It supports multiple downstream platforms, such as TiDB, MySQL, and MQ. Compared with TiDB Binlog, TiCDC has lower latency and native high availability. @@ -29,6 +29,8 @@ TiCDC is a tool for replicating the incremental data of TiDB, introduced in TiDB - [The simple template for the TiCDC topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-cdc.yaml) - [The complex template for the TiCDC topology](https://github.com/pingcap/docs/blob/master/config-templates/complex-cdc.yaml) +For detailed descriptions of the configuration items in the above TiDB cluster topology file, see [Topology Configuration File for Deploying TiDB Using TiUP](/tiup/tiup-cluster-topology-reference.md). + > **Note:** > > - You do not need to manually create the `tidb` user in the configuration file. The TiUP cluster component automatically creates the `tidb` user on the target machines. You can customize the user, or keep the user consistent with the control machine. diff --git a/ticdc/deploy-ticdc.md b/ticdc/deploy-ticdc.md index dd289e7e54e9..6c08f53951a4 100644 --- a/ticdc/deploy-ticdc.md +++ b/ticdc/deploy-ticdc.md @@ -24,7 +24,7 @@ For more information, see [Software and Hardware Recommendations](/hardware-and- ## Deploy a new TiDB cluster that includes TiCDC using TiUP -When you deploy a new TiDB cluster using TiUP, you can also deploy TiCDC at the same time. You only need to add the `cdc_servers` section in the initialization configuration file that TiUP uses to start the TiDB cluster. For detailed operations, see [Edit the initialization configuration file](/production-deployment-using-tiup.md#step-3-edit-the-initialization-configuration-file). +When you deploy a new TiDB cluster using TiUP, you can also deploy TiCDC at the same time. You only need to add the `cdc_servers` section in the initialization configuration file that TiUP uses to start the TiDB cluster. For detailed operations, see [Edit the initialization configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file). For detailed configurable fields, see [Configure `cdc_servers` using TiUP](/tiup/tiup-cluster-topology-reference.md#cdc_servers). ## Add TiCDC to an existing TiDB cluster using TiUP @@ -46,6 +46,8 @@ cdc server --pd=http://10.0.10.25:2379 --log-file=ticdc_2.log --addr=0.0.0.0:830 cdc server --pd=http://10.0.10.25:2379 --log-file=ticdc_3.log --addr=0.0.0.0:8303 --advertise-addr=127.0.0.1:8303 ``` +## Description of TiCDC `cdc server` command-line parameters + The following are descriptions of options available in the `cdc server` command: - `gc-ttl`: The TTL (Time To Live) of the service level `GC safepoint` in PD set by TiCDC, in seconds. The default value is `86400`, which means 24 hours. @@ -58,3 +60,5 @@ The following are descriptions of options available in the `cdc server` command: - `ca`: The path of the CA certificate file used by TiCDC, in the PEM format (optional). - `cert`: The path of the certificate file used by TiCDC, in the PEM format (optional). - `key`: The path of the certificate key file used by TiCDC, in the PEM format (optional). +- `config`: The address of the configuration file that TiCDC uses (optional). This option is supported since TiCDC v5.0.0. This option can be used in the TiCDC deployment since TiUP v1.4.0. +- `sort-dir`: Specifies the temporary file directory of the sorting engine. The default value of this configuration item is `/tmp/cdc_sort`. When Unified Sorter is enabled, if this directory on the server is not writable or the available space is insufficient, you need to manually specify a directory in `sort-dir`. Make sure that TiCDC can read and write data in the `sort-dir` path. diff --git a/ticdc/manage-ticdc.md b/ticdc/manage-ticdc.md index 9b602e70eca0..b529daa446f6 100644 --- a/ticdc/manage-ticdc.md +++ b/ticdc/manage-ticdc.md @@ -6,25 +6,54 @@ aliases: ['/docs/dev/ticdc/manage-ticdc/','/docs/dev/reference/tools/ticdc/manag # Manage TiCDC Cluster and Replication Tasks -This document describes how to manage the TiCDC cluster and replication tasks using the command line tool `cdc cli` and the HTTP interface. +This document describes how to upgrade TiCDC cluster and modify the configuration of TiCDC cluster using TiUP, and how to manage the TiCDC cluster and replication tasks using the command-line tool `cdc cli` and the HTTP interface. ## Upgrade TiCDC using TiUP -This section introduces how to upgrade the TiCDC cluster using TiUP. In the following example, assume that you need to upgrade TiCDC and the entire TiDB cluster to v4.0.6. +This section introduces how to upgrade the TiCDC cluster using TiUP. In the following example, assume that you need to upgrade TiCDC and the entire TiDB cluster to v5.1.0. {{< copyable "shell-regular" >}} ```shell tiup update --self && \ tiup update --all && \ -tiup cluster upgrade v4.0.6 +tiup cluster upgrade v5.1.0 ``` ### Notes for upgrade -* The `changefeed` configuration has changed in TiCDC v4.0.2. See [Compatibility notes for the configuration file](/production-deployment-using-tiup.md#step-3-edit-the-initialization-configuration-file) for details. +* The `changefeed` configuration has changed in TiCDC v4.0.2. See [Compatibility notes for the configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file) for details. * If you encounter any issues, see [Upgrade TiDB using TiUP - FAQ](/upgrade-tidb-using-tiup.md#faq). +## Modify TiCDC configuration using TiUP + +This section introduces how to modify the configuration of TiCDC cluster using the [`tiup cluster edit-config`](/tiup/tiup-component-cluster-edit-config.md) command of TiUP. The following example changes the value of `gc-ttl` from the default `86400` to `3600`, namely, one hour. + +First, execute the following command. You need to replace `` with your actual cluster name. + +{{< copyable "shell-regular" >}} + +```shell +tiup cluster edit-config +``` + +Then, enter the vi editor page and modify the `cdc` configuraion under [`server-configs`](/tiup/tiup-cluster-topology-reference.md#server_configs). The configuration is shown below: + +```shell + server_configs: + tidb: {} + tikv: {} + pd: {} + tiflash: {} + tiflash-learner: {} + pump: {} + drainer: {} + cdc: + gc-ttl: 3600 +``` + +After the modification, execute the `tiup cluster relaod -R cdc` command to reload the configuration. + ## Use TLS For details about using encrypted data transmission (TLS), see [Enable TLS Between TiDB Components](/enable-tls-between-components.md). @@ -36,6 +65,10 @@ This section introduces how to use `cdc cli` to manage a TiCDC cluster and data - `cli` commands are executed directly using the `cdc` binary; - PD listens on `10.0.10.25` and the port is `2379`. +> **Note:** +> +> The IP address and port that PD listens on correspond to the `advertise-client-urls` parameter specified during the `pd-server` startup. Multiple `pd-server`s have multiple `advertise-client-urls` parameters and you can specify one or multiple parameters. For example, `--pd=http://10.0.10.25:2379` or `--pd=http://10.0.10.25:2379,http://10.0.10.26:2379,http://10.0.10.27:2379`. + If you deploy TiCDC using TiUP, replace `cdc cli` in the following commands with `tiup ctl cdc`. ### Manage TiCDC service progress (`capture`) @@ -69,6 +102,29 @@ If you deploy TiCDC using TiUP, replace `cdc cli` in the following commands with ### Manage replication tasks (`changefeed`) +#### State transfer of replication tasks + +The state of a replication task represents the running status of the replication task. During the running of TiCDC, replication tasks might fail with errors, be manually paused, resumed, or reach the specified `TargetTs`. These behaviors can lead to the change of the replication task state. This section describes the states of TiCDC replication tasks and the transfer relationships between states. + +![TiCDC state transfer](/media/ticdc-state-transfer.png) + +The states in the above state transfer diagram are described as follows: + +- `Normal`: The replication task runs normally and the checkpoint-ts proceeds normally. +- `Stopped`: The replication task is stopped, because the user manually pauses the changefeed. The changefeed in this state blocks GC operations. +- `Error`: The replication task returns an error. The replication cannot continue due to some recoverable errors. The changefeed in this state keeps trying to resume until the state transfers to `Normal`. The changefeed in this state blocks GC operations. +- `Finished`: The replication task is finished and has reached the preset `TargetTs`. The changefeed in this state does not block GC operations. +- `Failed`: The replication task fails. Due to some unrecoverable errors, the replication task cannot resume and cannot be recovered. The changefeed in this state does not block GC operations. + +The numbers in the above state transfer diagram are described as follows. + +- ① Execute the `changefeed pause` command +- ② Execute the `changefeed resume` command to resume the replication task +- ③ Recoverable errors occur during the `changefeed` operation +- ④ Execute the `changefeed resume` command to resume the replication task +- ⑤ Recoverable errors occur during the `changefeed` operation +- ⑥ The replication task has reached the preset `TargetTs`, and the replication is automatically stopped. + #### Create a replication task Execute the following commands to create a replication task: @@ -76,17 +132,17 @@ Execute the following commands to create a replication task: {{< copyable "shell-regular" >}} ```shell -cdc cli changefeed create --pd=http://10.0.10.25:2379 --sink-uri="mysql://root:123456@127.0.0.1:3306/" --changefeed-id="simple-replication-task" +cdc cli changefeed create --pd=http://10.0.10.25:2379 --sink-uri="mysql://root:123456@127.0.0.1:3306/" --changefeed-id="simple-replication-task" --sort-engine="unified" ``` ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"sink-uri":"mysql://root:123456@127.0.0.1:3306/","opts":{},"create-time":"2020-03-12T22:04:08.103600025+08:00","start-ts":415241823337054209,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":".","config":{"case-sensitive":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"default"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null} +Info: {"sink-uri":"mysql://root:123456@127.0.0.1:3306/","opts":{},"create-time":"2020-03-12T22:04:08.103600025+08:00","start-ts":415241823337054209,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":".","config":{"case-sensitive":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"default"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null} ``` - `--changefeed-id`: The ID of the replication task. The format must match the `^[a-zA-Z0-9]+(\-[a-zA-Z0-9]+)*$` regular expression. If this ID is not specified, TiCDC automatically generates a UUID (the version 4 format) as the ID. -- `--sink-uri`: The downstream address of the replication task. Configure `--sink-uri` according to the following format. Currently, the scheme supports `mysql`/`tidb`/`kafka`/`pulsar`. +- `--sink-uri`: The downstream address of the replication task. Configure `--sink-uri` according to the following format. Currently, the scheme supports `mysql`/`tidb`/`kafka`/`pulsar`/`s3`/`local`. {{< copyable "" >}} @@ -98,10 +154,14 @@ Info: {"sink-uri":"mysql://root:123456@127.0.0.1:3306/","opts":{},"create-time": - `--start-ts`: Specifies the starting TSO of the `changefeed`. From this TSO, the TiCDC cluster starts pulling data. The default value is the current time. - `--target-ts`: Specifies the ending TSO of the `changefeed`. To this TSO, the TiCDC cluster stops pulling data. The default value is empty, which means that TiCDC does not automatically stop pulling data. -- `--sort-engine`: Specifies the sorting engine for the `changefeed`. Because TiDB and TiKV adopt distributed architectures, TiCDC must sort the data changes before writing them to the sink. This option supports `memory`/`unified`/`file`. - - `memory`: Sorts data changes in memory. It is recommended to use `memory` in a production environment. - - `unified`: An experimental feature introduced in v4.0.9. When `unified` is used, TiCDC prefers data sorting in memory. If the memory is insufficient, TiCDC automatically uses the disk to store the temporary data. It is **NOT** recommended to use it in a production environment unless `memory` cannot be used due to insufficient memory. - - `file`: Entirely uses the disk to store the temporary data. This feature is **deprecated**. It is not recommended to use it. +- `--sort-engine`: Specifies the sorting engine for the `changefeed`. Because TiDB and TiKV adopt distributed architectures, TiCDC must sort the data changes before writing them to the sink. This option supports `unified` (by default)/`memory`/`file`. + + - `unified`: When `unified` is used, TiCDC prefers data sorting in memory. If the memory is insufficient, TiCDC automatically uses the disk to store the temporary data. This is the default value of `--sort-engine`. + - `memory`: Sorts data changes in memory. It is **NOT recommended** to use this sorting engine, because OOM is easily triggered when you replicate a large amount of data. + - `file`: Entirely uses the disk to store the temporary data. This feature is **deprecated**. It is **NOT recommended** to use it in **any** situation. + +- `--sort-dir`: Specifies the temporary file directory of the sorting engine. It is **NOT recommended** to use this option in the command `cdc cli changefeed create`. You are recommended to use this option [in the command `cdc server` to set the temporary file directory](/ticdc/deploy-ticdc.md#description-of-ticdc-cdc-server-command-line-parameters). The default value of this option is `/tmp/cdc_sort`. When the unified sorter is enabled, if the default directory `/tmp/cdc_sort` on the sever is not writable or there is not enough space, you need to manually specify a directory in `sort-dir`. If the directory specified in `sort-dir` is not writable, `changefeed` stops automatically. + - `--config`: Specifies the configuration file of the `changefeed`. #### Configure sink URI with `mysql`/`tidb` @@ -146,7 +206,7 @@ The following are descriptions of parameters and parameter values that can be co | `127.0.0.1` | The IP address of the downstream Kafka services | | `9092` | The port for the downstream Kafka | | `cdc-test` | The name of the Kafka topic | -| `kafka-version` | The version of the downstream Kafka (optional, `2.4.0` by default. Currently, the earlist supported Kafka version is `0.11.0.2` and the latest one is `2.6.0`.) | +| `kafka-version` | The version of the downstream Kafka (optional, `2.4.0` by default. Currently, the earliest supported Kafka version is `0.11.0.2` and the latest one is `2.7.0`. This value needs to be consistent with the actual version of the downstream Kafka.) | | `kafka-client-id` | Specifies the Kafka client ID of the replication task (optional, `TiCDC_sarama_producer_replication ID` by default) | | `partition-num` | The number of the downstream Kafka partitions (Optional. The value must be **no greater than** the actual number of partitions. If you do not configure this parameter, the partition number is obtained automatically.) | | `max-message-bytes` | The maximum size of data that is sent to Kafka broker each time (optional, `64MB` by default) | @@ -200,8 +260,8 @@ The following are descriptions of parameters that can be configured for the sink | `tlsAllowInsecureConnection` | Determines whether to allow unencrypted connection after TLS is enabled (optional) | | `tlsValidateHostname` | Determines whether to verify the host name of the certificate from the downstream Pulsar (optional) | | `maxConnectionsPerBroker` | The maximum number of connections allowed to a single downstream Pulsar broker, which is optional and defaults to 1 | -| `auth.tls` | Uses the TLS mode to verify the downstream Pulsar (optional). For example, `"{"tlsCertFile":"/path/to/cert", "tlsKeyFile":"/path/to/key"}"`. | -| `auth.token` | Uses the token mode to verify the downstream Pulsar (optional). For example, `"{"token":"secret-token"}"` or `"{"file":"path/to/secret-token-file"}"`. | +| `auth.tls` | Uses the TLS mode to verify the downstream Pulsar (optional). For example, `auth=tls&auth.tlsCertFile=/path/to/cert&auth.tlsKeyFile=/path/to/key`. | +| `auth.token` | Uses the token mode to verify the downstream Pulsar (optional). For example, `auth=token&auth.token=secret-token` or `auth=token&auth.file=path/to/secret-token-file`. | | `name` | The name of Pulsar producer in TiCDC (optional) | | `maxPendingMessages` | Sets the maximum size of the pending message queue, which is optional and defaults to 1000. For example, pending for the confirmation message from Pulsar. | | `disableBatching` | Disables automatically sending messages in batches (optional) | @@ -212,6 +272,26 @@ The following are descriptions of parameters that can be configured for the sink For more parameters of Pulsar, see [pulsar-client-go ClientOptions](https://godoc.org/github.com/apache/pulsar-client-go/pulsar#ClientOptions) and [pulsar-client-go ProducerOptions](https://godoc.org/github.com/apache/pulsar-client-go/pulsar#ProducerOptions). +#### Configure sink URI with cdclog + +The `cdclog` files (files written by TiCDC on the local filesystem or on the Amazon S3-compatible storage) can be used together with Backup & Restore (BR) to provide point-in-time (PITR) recovery. See [Point in Time recovery (experimental feature)](/br/use-br-command-line-tool.md#point-in-time-recovery-experimental-feature) for details. + +The following command creates a changefeed that will write cdclog files locally to the `/data/cdc/log` directory. + +{{< copyable "shell-regular" >}} + +```shell +cdc cli changefeed create --pd=http://10.0.10.25:2379 --sink-uri="local:///data/cdclog" --config changefeed.toml +``` + +The following command creates a changefeed that will write cdclog files to an external S3 storage in the `logbucket` bucket with a subdirectory of `test`. The endpoint is set in the URI, which is needed if you are using an S3-compatible storage other than Amazon S3. + +{{< copyable "shell-regular" >}} + +```shell +cdc cli changefeed create --pd=http://10.0.10.25:2379 --sink-uri="s3://logbucket/test?endpoint=http://$S3_ENDPOINT/" --config changefeed.toml +``` + #### Use the task configuration file For more replication configuration (for example, specify replicating a single table), see [Task configuration file](#task-configuration-file). @@ -251,7 +331,8 @@ cdc cli changefeed list --pd=http://10.0.10.25:2379 - `checkpoint` indicates that TiCDC has already replicated data before this time point to the downstream. - `state` indicates the state of the replication task. - `normal`: The replication task runs normally. - - `stopped`: The replication task is stopped (manually paused or stopped by an error). + - `stopped`: The replication task is stopped (manually paused). + - `error`: The replication task is stopped (by an error). - `removed`: The replication task is removed. Tasks of this state are displayed only when you have specified the `--all` option. To see these tasks when this option is not specified, execute the `changefeed query` command. - `finished`: The replication task is finished (data is replicated to the `target-ts`). Tasks of this state are displayed only when you have specified the `--all` option. To see these tasks when this option is not specified, execute the `changefeed query` command. @@ -296,7 +377,7 @@ cdc cli changefeed query --pd=http://10.0.10.25:2379 --changefeed-id=simple-repl "start-ts": 419036036249681921, "target-ts": 0, "admin-job-type": 0, - "sort-engine": "memory", + "sort-engine": "unified", "sort-dir": ".", "config": { "case-sensitive": true, @@ -410,16 +491,6 @@ In the above command: - `--changefeed-id=uuid` represents the ID of the `changefeed` that corresponds to the replication task you want to remove. -After the replication task is removed, the state information of the task will be retained for 24 hours, mainly used for recording the replication checkpoint. Within this 24 hours, you cannot create a replication task of the same name. - -If you want to completely remove the task information, you can specify the `--force` or `-f` argument in the command. Then all information of the `changefeed` will be removed, and you can immediately create a `changefeed` of the same name. - -{{< copyable "shell-regular" >}} - -```shell -cdc cli changefeed remove --pd=http://10.0.10.25:2379 --changefeed-id simple-replication-task --force -``` - ### Update task configuration Starting from v4.0.4, TiCDC supports modifying the configuration of the replication task (not dynamically). To modify the `changefeed` configuration, pause the task, modify the configuration, and then resume the task. @@ -427,9 +498,9 @@ Starting from v4.0.4, TiCDC supports modifying the configuration of the replicat {{< copyable "shell-regular" >}} ```shell -cdc cli changefeed pause -c test-cf -cdc cli changefeed update -c test-cf --sink-uri="mysql://127.0.0.1:3306/?max-txn-row=20&worker-number=8" --config=changefeed.toml -cdc cli changefeed resume -c test-cf +cdc cli changefeed pause -c test-cf --pd=http://10.0.10.25:2379 +cdc cli changefeed update -c test-cf --pd=http://10.0.10.25:2379 --sink-uri="mysql://127.0.0.1:3306/?max-txn-row=20&worker-number=8" --config=changefeed.toml +cdc cli changefeed resume -c test-cf --pd=http://10.0.10.25:2379 ``` Currently, you can modify the following configuration items: @@ -593,8 +664,8 @@ This section introduces the configuration of a replication task. # This configuration item affects configurations related to filter and sink. case-sensitive = true -# Specifies whether to output the old value. New in v4.0.5. -enable-old-value = false +# Specifies whether to output the old value. New in v4.0.5. Since v5.0, the default value is `true`. +enable-old-value = true [filter] # Ignores the transaction of specified start_ts. @@ -741,7 +812,9 @@ To create a cyclic replication task, take the following steps: + The name of the table with cyclic replication enabled must match the `^[a-zA-Z0-9_]+$` regular expression. + Before creating the cyclic replication task, the tables for the task must be created. + After enabling the cyclic replication, you cannot create a table that will be replicated by the cyclic replication task. ++ To avoid causing errors, do not execute DDL statements such as `ADD COLUMN`/`DROP COLUMN` when data is written into multiple clusters at the same time. + To perform online DDL operations, ensure the following requirements are met: + - The application is compatible with the table schema before and after executing the DDL operations. - The TiCDC components of multiple clusters form a one-way DDL replication chain, which is not cyclic. For example, in the example above, only the TiCDC component of cluster C disables `sync-ddl`. - DDL operations must be performed on the cluster that is the starting point of the one-way DDL replication chain, such as cluster A in the example above. @@ -777,3 +850,29 @@ force-replicate = true > **Warning:** > > For tables without a valid index, operations such as `INSERT` and `REPLACE` are not reentrant, so there is a risk of data redundancy. TiCDC guarantees that data is distributed only at least once during the replication process. Therefore, enabling this feature to replicate tables without a valid index will definitely cause data redundancy. If you do not accept data redundancy, it is recommended to add an effective index, such as adding a primary key column with the `AUTO RANDOM` attribute. + +## Unified Sorter + +Unified sorter is the sorting engine in TiCDC. It can mitigate OOM problems caused by the following scenarios: + ++ The data replication task in TiCDC is paused for a long time, during which a large amount of incremental data is accumulated and needs to be replicated. ++ The data replication task is started from an early timestamp so it becomes necessary to replicate a large amount of incremental data. + +For the changefeeds created using `cdc cli` after v4.0.13, Unified Sorter is enabled by default; for the changefeeds that have existed before v4.0.13, the previous configuration is used. + +To check whether or not the Unified Sorter feature is enabled on a changefeed, you can execute the following example command (assuming the IP address of the PD instance is `http://10.0.10.25:2379`): + +{{< copyable "shell-regular" >}} + +```shell +cdc cli --pd="http://10.0.10.25:2379" changefeed query --changefeed-id=simple-replication-task | grep 'sort-engine' +``` + +In the output of the above command, if the value of `sort-engine` is "unified", it means that Unified Sorter is enabled on the changefeed. + +> **Note:** +> +> + If your servers use mechanical hard drives or other storage devices that have high latency or limited bandwidth, use the unified sorter with caution. +> + The total free capacity of hard drives must be greater than or equal to 128G. If you need to replicate a large amount of historical data, make sure that the free capacity on each node is greater than or equal to the size of the incremental data that needs to be replicated. +> + Unified sorter is enabled by default. If your servers do not match the above requirements and you want to disable the unified sorter, you need to manually set `sort-engine` to `memory` for the changefeed. +> + To enable Unified Sorter on an existing changefeed, see the methods provided in [How do I handle the OOM that occurs after TiCDC is restarted after a task interruption?](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-the-oom-that-occurs-after-ticdc-is-restarted-after-a-task-interruption). diff --git a/ticdc/monitor-ticdc.md b/ticdc/monitor-ticdc.md new file mode 100644 index 000000000000..8a0c758049b7 --- /dev/null +++ b/ticdc/monitor-ticdc.md @@ -0,0 +1,117 @@ +--- +title: Key Monitoring Metrics of TiCDC +summary: Learn some key metrics displayed on the Grafana TiCDC dashboard. +--- + +# Key Monitoring Metrics of TiCDC + +If you use TiUP to deploy the TiDB cluster, you can see a sub-dashboard for TiCDC in the monitoring system which is deployed at the same time. You can get an overview of TiCDC's current status from the TiCDC dashboard, where the key metrics are displayed. This document provides a detailed description of these key metrics. + +The metric description in this document is based on the following replication task example, which replicates data to MySQL using the default configuration. + +```shell +cdc cli changefeed create --pd=http://10.0.10.25:2379 --sink-uri="mysql://root:123456@127.0.0.1:3306/" --changefeed-id="simple-replication-task" +``` + +The TiCDC dashboard contains four monitoring panels. See the following screenshot: + +![TiCDC Dashboard - Overview](/media/ticdc/ticdc-dashboard-overview.png) + +The description of each panel is as follows: + +- [**Server**](#server): The summary information of TiKV nodes and TiCDC nodes in the TiDB cluster +- [**Changefeed**](#changefeed): The detailed information of TiCDC replication tasks +- [**Events**](#events): The detail information about the data flow within the TiCDC cluster +- [**TiKV**](#tikv): TiKV information related to TiCDC + +## Server + +The following is an example of the **Server** panel: + +![TiCDC Dashboard - Server metrics](/media/ticdc/ticdc-dashboard-server.png) + +The description of each metric in the **Server** panel is as follows: + +- Uptime: The time for which TiKV nodes and TiCDC nodes have been running +- Goroutine count: The number of goroutines of a TiCDC node +- Open FD count: The number of file handles opened by TiCDC nodes +- Ownership: The current status of nodes in the TiCDC cluster +- Ownership history: The ownership history of the TiCDC cluster +- CPU usage: The CPU usage of TiCDC nodes +- Memory usage: The memory usage of TiCDC nodes + +## Changefeed + +The following is an example of the **Changefeed** panel: + +![TiCDC Dashboard - Changefeed metrics 1](/media/ticdc/ticdc-dashboard-changefeed-1.png) +![TiCDC Dashboard - Changefeed metrics 2](/media/ticdc/ticdc-dashboard-changefeed-2.png) +![TiCDC Dashboard - Changefeed metrics 3](/media/ticdc/ticdc-dashboard-changefeed-3.png) + +The description of each metric in the **Changefeed** panel is as follows: + +- Changefeed table count: The number of tables that each TiCDC node needs to replicate in the replication task +- Processor resolved ts: The timestamps that have been resolved in the TiCDC cluster +- Table resolved ts: The replication progress of each table in the replication task +- Changefeed checkpoint: The progress of replicating data to the downstream. Normally, the green bars are connected to the yellow line +- PD etcd requests/s: The number of requests that a TiCDC node sends to PD per second +- Exit error count: The number of errors that interrupt the replication task per minute +- Changefeed checkpoint lag: The progress lag of data replication (the unit is second) between the upstream and the downstream +- Changefeed resolved ts lag: The progress lag of data replication (the unit is second) between the upstream and TiCDC nodes +- Flush sink duration: The histogram of the time spent by TiCDC asynchronously flushing data to the downstream +- Flush sink duration percentile: The time (P95, P99, and P999) spent by TiCDC asynchronously flushing data to the downstream within one second +- Sink write duration: The histogram of the time spent by TiCDC writing a transaction change to the downstream +- Sink write duration percentile: The time (P95, P99, and P999) spent by TiCDC writing a transaction change to the downstream within one second +- MySQL sink conflict detect duration: The histogram of the time spent on detecting MySQL sink conflicts +- MySQL sink conflict detect duration percentile: The time (P95, P99, and P999) spent on detecting MySQL sink conflicts within one second +- MySQL sink worker load: The workload of MySQL sink workers of TiCDC nodes + +## Events + +The following is an example of the **Events** panel: + +![TiCDC Dashboard - Events metrics 2](/media/ticdc/ticdc-dashboard-events-1.png) +![TiCDC Dashboard - Events metrics 2](/media/ticdc/ticdc-dashboard-events-2.png) +![TiCDC Dashboard - Events metrics 2](/media/ticdc/ticdc-dashboard-events-3.png) + +The description of each metric in the **Events** panel is as follows: + +- Eventfeed count: The number of Eventfeed RPC requests of TiCDC nodes +- Event size percentile: The event size (P95, P99, and P999) which TiCDC receives from TiKV within one second +- Eventfeed error/m: The number of errors reported by Eventfeed RPC requests of TiCDC nodes per minute +- KV client receive events/s: The number of events that the KV client module of TiCDC nodes receives from TiKV per second +- Puller receive events/s: The number of events that the Puller module of TiCDC nodes receives from the KV client per second +- Puller output events/s: The number of events that the Puller module of TiCDC nodes sends to the Sorter module per second +- Sink flush rows/s: The number of events that TiCDC nodes write to the downstream per second +- Puller buffer size: The number of events that TiCDC nodes cache in the Puller module +- Entry sorter buffer size: The number of events that TiCDC nodes cache in the Sorter module +- Processor/Mounter buffer size: The number of events that TiCDC nodes cache in the Processor module and the Mounter module +- Sink row buffer size: The number of events that TiCDC nodes cache in the Sink module +- Entry sorter sort duration: The histogram of the time spent by TiCDC nodes sorting events +- Entry sorter sort duration percentile: The time (P95, P99, and P999) spent by TiCDC sorting events within one second +- Entry sorter merge duration: The histogram of the time spent by TiCDC nodes merging sorted events +- Entry sorter merge duration percentile: The time (P95, P99, and P999) spent by TiCDC merging sorted events within one second +- Mounter unmarshal duration: The histogram of the time spent by TiCDC nodes unmarshaling events +- Mounter unmarshal duration percentile: The time (P95, P99, and P999) spent by TiCDC unmarshaling events within one second +- KV client dispatch events/s: The number of events that the KV client module dispatches among the TiCDC nodes +- KV client batch resolved size: The batch size of resolved timestamp messages that TiKV sends to TiCDC + +## TiKV + +The following is an example of the **TiKV** panel: + +![TiCDC Dashboard - TiKV metrics 1](/media/ticdc/ticdc-dashboard-tikv-1.png) +![TiCDC Dashboard - TiKV metrics 2](/media/ticdc/ticdc-dashboard-tikv-2.png) + +The description of each metric in the **TiKV** panel is as follows: + +- CDC endpoint CPU: The CPU usage of the CDC endpoint threads on TiKV nodes +- CDC worker CPU: The CPU usage of the CDC worker threads on TiKV nodes +- Min resolved ts: The minimum resolved timestamp on TiKV nodes +- Min resolved region: The Region ID of the minimum resolved timestamp on TiKV nodes +- Resolved ts lag duration percentile: The lag between the minimum resolved timestamp on TiKV nodes and the current time +- Initial scan duration: The histogram of the time spent on incremental scan when TiKV nodes connect to TiCDC nodes +- Initial scan duration percentile: The time (P95, P99, and P999) spent on the incremental scan of TiKV nodes within one second +- Memory without block cache: The memory usage of TiKV nodes excluding the RocksDB block cache +- CDC pending bytes in memory: The memory usage of CDC module on TiKV nodes +- Captured region count: The number of event-capturing Regions on TiKV nodes diff --git a/ticdc/ticdc-glossary.md b/ticdc/ticdc-glossary.md new file mode 100644 index 000000000000..b4debae89e9b --- /dev/null +++ b/ticdc/ticdc-glossary.md @@ -0,0 +1,36 @@ +--- +title: TiCDC Glossary +summary: Learn the terms about TiCDC and their definitions. +--- + +# TiCDC Glossary + +This glossary provides TiCDC-related terms and definitions. These terms appears in TiCDC logs, monitoring metrics, configurations, and documents. + +For TiDB-related terms and definitions, refer to [TiDB glossary](/glossary.md). + +## C + +### Capture + +A single TiCDC instance on which the replication task of the cluster runs. Multiple captures form a TiCDC cluster. + +### Changed data + +The data to be written to TiCDC from the upstream TiDB cluster, including the DML-caused data changes and the DDL-caused table schema changes. + +### Changefeed + +An incremental replication task in TiCDC, which outputs the data change logs of several tables in a TiDB cluster to the designated downstream. + +## O + +### Owner + +A [capture](#capture) of a special role that manages the TiCDC cluster and schedules replication tasks of the cluster. An owner is elected by captures and there is at most one owner at any time. + +## P + +### Processor + +TiCDC replication tasks allocate data tables on TiCDC instances, and the processor refers to the replication processing unit of these tables. Processor tasks include pulling, sorting, restoring, and distributing changed data. diff --git a/ticdc/ticdc-open-protocol.md b/ticdc/ticdc-open-protocol.md index 77e644f2eb6b..22c5d9e6aa53 100644 --- a/ticdc/ticdc-open-protocol.md +++ b/ticdc/ticdc-open-protocol.md @@ -147,8 +147,8 @@ This section introduces the formats of Row Changed Event, DDL Event, and Resolve | :---------- | :----- | :--------------------- | | Column Name | String | The column name. | | Column Type | Number | The column type. For details, see [Column Type Code](#column-type-code). | - | Where Handle | Bool | Determines whether this column can be the filter condition of the `Where` clause. When this column is unique on the table, `Where Handle` is `true`. | - | Flag (**experimental**) | Number | The bit flags of columns. For details, see [Bit flags of columns](#bit-flags-of-columns). | + | Where Handle | Boolean | Determines whether this column can be the filter condition of the `Where` clause. When this column is unique on the table, `Where Handle` is `true`. | + | Flag | Number | The bit flags of columns. For details, see [Bit flags of columns](#bit-flags-of-columns). | | Column Value | Any | The Column value. | ### DDL Event @@ -283,7 +283,7 @@ Currently, TiCDC does not provide the standard parsing library for TiCDC Open Pr | Type | Code | Output Example | Description | | :-------------------- | :--- | :------ | :-- | -| TINYINT/BOOL | 1 | {"t":1,"v":1} | | +| TINYINT/BOOLEAN | 1 | {"t":1,"v":1} | | | SMALLINT | 2 | {"t":2,"v":1} | | | INT | 3 | {"t":3,"v":123} | | | FLOAT | 4 | {"t":4,"v":153.123} | | @@ -385,6 +385,5 @@ If the value of a column is `46`, the column is a composite index column, a prim > **Note:** > -> + This feature is still experimental. Do **NOT** use it in the production environment. > + `BinaryFlag` is meaningful only when the column type is BLOB/TEXT (including TINYBLOB/TINYTEXT and BINARY/CHAR). When the upstream column is the BLOB type, the `BinaryFlag` value is set to `1`. When the upstream column is the TEXT type, the `BinaryFlag` value is set to `0`. > + To replicate a table from the upstream, TiCDC selects a [valid index](/ticdc/ticdc-overview.md#restrictions) as the Handle index. The `HandleKeyFlag` value of the Handle index column is set to `1`. diff --git a/ticdc/ticdc-overview.md b/ticdc/ticdc-overview.md index 8adf040c3ce7..da926141a934 100644 --- a/ticdc/ticdc-overview.md +++ b/ticdc/ticdc-overview.md @@ -43,6 +43,8 @@ Currently, the TiCDC sink component supports replicating data to the following d - Databases compatible with MySQL protocol. The sink component provides the final consistency support. - Kafka based on the TiCDC Open Protocol. The sink component ensures the row-level order, final consistency or strict transactional consistency. +- `cdclog` (experimental): Files written on the local filesystem or on the Amazon S3-compatible storage. +- Apache Pulsar (experimental) ### Ensure replication order and consistency @@ -58,9 +60,10 @@ Currently, the TiCDC sink component supports replicating data to the following d - MySQL sink - - TiCDC does not split in-table transactions. This is to **ensure** the transactional consistency within a single table. However, TiCDC does **not ensure** consistency with the transactional order in the upstream table. If the primary keys between transactions have intersections, TiCDC still **ensures** consistency with the transactional order in the upstream table. - - TiCDC splits cross-table transactions in the unit of tables. TiCDC does **not ensure** that cross-table transactions are always consistent. - - TiCDC **ensures** that the order of single-row updates are consistent with that in the upstream. + - TiCDC does not split single-table transactions and **ensures** the atomicity of single-table transactions. + - TiCDC does **not ensure** that the execution order of downstream transactions is the same as that of upstream transactions. + - TiCDC splits cross-table transactions in the unit of table and does **not ensure** the atomicity of cross-table transactions. + - TiCDC **ensures** that the order of single-row updates is consistent with that in the upstream. - Kafka sink @@ -84,8 +87,29 @@ Since v4.0.8, TiCDC supports replicating tables **without a valid index** by mod Currently, The following scenarios are not supported: - The TiKV cluster that uses RawKV alone. -- The [DDL operation `CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) and the [SEQUENCE function](/sql-statements/sql-statement-create-sequence.md#sequence-function) in TiDB v4.0. When the upstream TiDB uses `SEQUENCE`, TiCDC ignores `SEQUENCE` DDL operations/functions performed upstream. However, DML operations using `SEQUENCE` functions can be correctly replicated. -- The [TiKV Hibernate Region](https://github.com/tikv/tikv/blob/master/docs/reference/configuration/raftstore-config.md#hibernate-region). TiCDC prevents the Region from entering the hibernated state. +- The [DDL operation `CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) and the [SEQUENCE function](/sql-statements/sql-statement-create-sequence.md#sequence-function) in TiDB. When the upstream TiDB uses `SEQUENCE`, TiCDC ignores `SEQUENCE` DDL operations/functions performed upstream. However, DML operations using `SEQUENCE` functions can be correctly replicated. + +TiCDC only provides partial support for scenarios of large transactions in the upstream. For details, refer to [FAQ: Does TiCDC support replicating large transactions? Is there any risk?](/ticdc/troubleshoot-ticdc.md#does-ticdc-support-replicating-large-transactions-is-there-any-risk). + +## Notice for compatibility issues + +### Incompatibility issue caused by using the TiCDC v5.0.0-rc `cdc cli` tool to operate a v4.0.x cluster + +When using the `cdc cli` tool of TiCDC v5.0.0-rc to operate a v4.0.x TiCDC cluster, you might encounter the following abnormal situations: + +- If the TiCDC cluster is v4.0.8 or an earlier version, using the v5.0.0-rc `cdc cli` tool to create a replication task might cause cluster anomalies and get the replication task stuck. + +- If the TiCDC cluster is v4.0.9 or a later version, using the v5.0.0-rc `cdc cli` tool to create a replication task will cause the old value and unified sorter features to be unexpectedly enabled by default. + +Solutions: Use the `cdc` executable file corresponding to the TiCDC cluster version to perform the following operations: + +1. Delete the changefeed created using the v5.0.0-rc `cdc cli` tool. For example, run the `tiup cdc:v4.0.9 cli changefeed remove -c xxxx --pd=xxxxx --force` command. +2. If the replication task is stuck, restart the TiCDC cluster. For example, run the `tiup cluster restart -R cdc` command. +3. Re-create the changefeed. For example, run the `tiup cdc:v4.0.9 cli changefeed create --sink-uri=xxxx --pd=xxx` command. + +> **Note:** +> +> The above issue exists only when `cdc cli` is v5.0.0-rc. Other v5.0.x `cdc cli` tool can be compatible with v4.0.x clusters. ## Install and deploy TiCDC diff --git a/ticdc/troubleshoot-ticdc.md b/ticdc/troubleshoot-ticdc.md index 393864eb7c0f..40921ce072a4 100644 --- a/ticdc/troubleshoot-ticdc.md +++ b/ticdc/troubleshoot-ticdc.md @@ -8,6 +8,10 @@ aliases: ['/docs/dev/ticdc/troubleshoot-ticdc/'] This document introduces the common issues and errors that you might encounter when using TiCDC, and the corresponding maintenance and troubleshooting methods. +> **Note:** +> +> In this document, the PD address specified in `cdc cli` commands is `--pd=http://10.0.10.25:2379`. When you use the command, replace the address with your actual PD address. + ## How do I choose `start-ts` when creating a task in TiCDC? The `start-ts` of a replication task corresponds to a Timestamp Oracle (TSO) in the upstream TiDB cluster. TiCDC requests data from this TSO in a replication task. Therefore, the `start-ts` of the replication task must meet the following requirements: @@ -58,7 +62,7 @@ If the replication task is interrupted for a long time and a large volume of new {{< copyable "shell-regular" >}} ```shell -cdc cli changefeed update -c [changefeed-id] --sort-engine="unified" --sort-dir="/data/cdc/sort" +cdc cli changefeed update -c [changefeed-id] --sort-engine="unified" --sort-dir="/data/cdc/sort" --pd=http://10.0.10.25:2379 ``` > **Note:** @@ -67,6 +71,10 @@ cdc cli changefeed update -c [changefeed-id] --sort-engine="unified" --sort-dir= > + TiCDC (the 4.0 version) does not support dynamically modifying the sorting engine yet. Make sure that the changefeed has stopped before modifying the sorter settings. > + Currently, the unified sorter is an experimental feature. When the number of tables is too large (>=100), the unified sorter might cause performance issues and affect replication throughput. Therefore, it is not recommended to use it in a production environment. Before you enable the unified sorter, make sure that the machine of each TiCDC node has enough disk capacity. If the total size of unprocessed data changes might exceed 1 TB, it is not recommend to use TiCDC for replication. +## What is the complete behavior of TiCDC garbage collection (GC) safepoint? + +If a replication task starts after the TiCDC service starts, the TiCDC owner updates the PD service GC safepoint with the smallest value of `checkpoint-ts` among all replication tasks. The service GC safepoint ensures that TiCDC does not delete data generated at that time and after that time. If the replication task is interrupted, the `checkpoint-ts` of this task does not change and PD's corresponding service GC safepoint is not updated either. The Time-To-Live (TTL) that TiCDC sets for a service GC safepoint is 24 hours, which means that the GC mechanism does not delete any data if the TiCDC service can be recovered within 24 hours after it is interrupted. + ## How do I handle the `Error 1298: Unknown or incorrect time zone: 'UTC'` error when creating the replication task or replicating data to MySQL? This error is returned when the downstream MySQL does not load the time zone. You can load the time zone by running [`mysql_tzinfo_to_sql`](https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html). After loading the time zone, you can create tasks and replicate data normally. @@ -77,6 +85,8 @@ This error is returned when the downstream MySQL does not load the time zone. Yo mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql -p ``` +If the output of the command above is similar to the following one, the import is successful: + ``` Enter password: Warning: Unable to load '/usr/share/zoneinfo/iso3166.tab' as time zone. Skipping it. @@ -85,7 +95,7 @@ Warning: Unable to load '/usr/share/zoneinfo/zone.tab' as time zone. Skipping it Warning: Unable to load '/usr/share/zoneinfo/zone1970.tab' as time zone. Skipping it. ``` -If you use MySQL in a special public cloud environment, such Alibaba Cloud RDS, and if you do not have the permission to modify MySQL, you need to specify the time zone using the `--tz` parameter: +If the downstream is a special MySQL environment (a public cloud RDS or some MySQL derivative versions) and importing the time zone using the above method fails, you need to specify the MySQL time zone of the downstream using the `time-zone` parameter in `sink-uri`. You can first query the time zone used by MySQL: 1. Query the time zone used by MySQL: @@ -109,27 +119,41 @@ If you use MySQL in a special public cloud environment, such Alibaba Cloud RDS, {{< copyable "shell-regular" >}} ```shell - cdc cli changefeed create --sink-uri="mysql://root@127.0.0.1:3306/" --tz=Asia/Shanghai + cdc cli changefeed create --sink-uri="mysql://root@127.0.0.1:3306/?time-zone=CST" --pd=http://10.0.10.25:2379 ``` > **Note:** > - > In MySQL, CST refers to the China Standard Time (UTC+08:00). Usually you cannot use `CST` directly in your system, but use `Asia/Shanghai` instead. + > CST might be an abbreviation for the following four different time zones: + > + > + Central Standard Time (USA) UT-6:00 + > + Central Standard Time (Australia) UT+9:30 + > + China Standard Time UT+8:00 + > + Cuba Standard Time UT-4:00 + > + > In China, CST usually stands for China Standard Time. -Be cautious when you set the time zone of the TiCDC server, because the time zone will be used for the conversion of time type. It is recommended that you use the same time zone in the upstream and downstream databases, and specify the time zone using `--tz` when you start the TiCDC server. +## How to understand the relationship between the TiCDC time zone and the time zones of the upstream/downstream databases? -The TiCDC server chooses its time zone in the following priority: +||Upstream time zone| TiCDC time zone|Downstream time zone| +| :-: | :-: | :-: | :-: | +| Configuration method | See [Time Zone Support](/configure-time-zone.md) | Configured using the `--tz` parameter when you start the TiCDC server | Configured using the `time-zone` parameter in `sink-uri` | +| Description | The time zone of the upstream TiDB, which affects DML operations of the timestamp type and DDL operations related to timestamp type columns.| TiCDC assumes that the upstream TiDB's time zone is the same as the TiCDC time zone configuration, and performs related operations on the timestamp column.| The downstream MySQL processes the timestamp in the DML and DDL operations according to the downstream time zone setting.| -1. TiCDC first uses the time zone specified by `--tz`. -2. When `--tz` is not available, TiCDC tries to read the time zone set by the `TZ` environment variable. -3. When the `TZ` environment variable is not available, TiCDC uses the default time zone of the machine. + > **Note:** + > + > Be careful when you set the time zone of the TiCDC server, because this time zone is used for converting the time type. Keep the upstream time zone, TiCDC time zone, and the downstream time zone consistent. The TiCDC server chooses its time zone in the following priority: + > + > - TiCDC first uses the time zone specified using `--tz`. + > - When `--tz` is not available, TiCDC tries to read the time zone set using the `TZ` environment variable. + > - When the `TZ` environment variable is not available, TiCDC uses the default time zone of the machine. ## What is the default behavior of TiCDC if I create a replication task without specifying the configuration file in `--config`? If you use the `cdc cli changefeed create` command without specifying the `-config` parameter, TiCDC creates the replication task in the following default behaviors: * Replicates all tables except system tables -* Disables the Old Value feature +* Enables the Old Value feature * Skips replicating tables that do not contain [valid indexes](/ticdc/ticdc-overview.md#restrictions) ## How do I handle the incompatibility issue of configuration files caused by TiCDC upgrade? @@ -149,7 +173,7 @@ cdc cli changefeed create --pd=http://10.0.10.25:2379 --sink-uri="kafka://127.0. > **Note:** > > * This feature is introduced in TiCDC 4.0.2. -> * TiCDC currently supports outputting data changes in the Canal format only to Kafka. +> * TiCDC currently supports outputting data changes in the Canal format only to MQ sinks such as Kafka and Pulsar. For more information, refer to [Create a replication task](/ticdc/manage-ticdc.md#create-a-replication-task). @@ -222,7 +246,7 @@ For more information, refer to [Replication task configuration file](/ticdc/mana ## When TiCDC replicates data to Kafka, can I control the maximum size of a single message in TiDB? -No. Currently TiCDC sets the maximum size of batch messages to 512 MB, and that of a single message to 4 MB. +Yes. You can set the `max-message-bytes` parameter to control the maximum size of data sent to the Kafka broker each time (optional, `64MB` by default). You can also set `max-batch-size` to specify the maximum number of change records in each Kafka message. Currently, the setting only takes effect when Kafka's protocol is `default` (optional, `4096` by default). ## When TiCDC replicates data to Kafka, does a message contain multiple types of data changes? @@ -257,6 +281,10 @@ In TiCDC Open Protocol, the type code `6` represents `null`. For more information, refer to [TiCDC Open Protocol column type code](/ticdc/ticdc-open-protocol.md#column-type-code). +## The `start-ts` timestamp of the TiCDC task is quite different from the current time. During the execution of this task, replication is interrupted and an error `[CDC:ErrBufferReachLimit]` occurs + +Since v4.0.9, you can try to enable the unified sorter feature in your replication task, or use the BR tool for an incremental backup and restore, and then start the TiCDC replication task from a new time. + ## How can I tell if a Row Changed Event of TiCDC Open Protocol is an `INSERT` event or an `UPDATE` event? If the Old Value feature is not enabled, you cannot tell whether a Row Changed Event of TiCDC Open Protocol is an `INSERT` event or an `UPDATE` event. If the feature is enabled, you can determine the event type by the fields it contains: @@ -266,3 +294,123 @@ If the Old Value feature is not enabled, you cannot tell whether a Row Changed E * `DELETE` event only contains the `"d"` field For more information, refer to [Open protocol Row Changed Event format](/ticdc/ticdc-open-protocol.md#row-changed-event). + +## How much PD storage does TiCDC use? + +TiCDC uses etcd in PD to store and regularly update the metadata. Because the time interval between the MVCC of etcd and PD's default compaction is one hour, the amount of PD storage that TiCDC uses is proportional to the amount of metadata versions generated within this hour. However, in v4.0.5, v4.0.6, and v4.0.7, TiCDC has a problem of frequent writing, so if there are 1000 tables created or scheduled in an hour, it then takes up all the etcd storage and returns the `etcdserver: mvcc: database space exceeded` error. You need to clean up the etcd storage after getting this error. See [etcd maintaince space-quota](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) for details. It is recommended to upgrade your cluster to v4.0.9 or later versions. + +## Does TiCDC support replicating large transactions? Is there any risk? + +TiCDC provides partial support for large transactions (more than 5 GB in size). Depending on different scenarios, the following risks might exist: + +- When TiCDC's internal processing capacity is insufficient, the replication task error `ErrBufferReachLimit` might occur. +- When TiCDC's internal processing capacity is insufficient or the throughput capacity of TiCDC's downstream is insufficient, out of memory (OOM) might occur. + +If you encounter an error above, it is recommended to use BR to restore the incremental data of large transactions. The detailed operations are as follows: + +1. Record the `checkpoint-ts` of the changefeed that is terminated due to large transactions, use this TSO as the `--lastbackupts` of the BR incremental backup, and execute [incremental data backup](/br/use-br-command-line-tool.md#back-up-incremental-data). +2. After backing up the incremental data, you can find a log record similar to `["Full backup Failed summary : total backup ranges: 0, total success: 0, total failed: 0"] [BackupTS=421758868510212097]` in the BR log output. Record the `BackupTS` in this log. +3. [Restore the incremental data](/br/use-br-command-line-tool.md#restore-incremental-data). +4. Create a new changefeed and start the replication task from `BackupTS`. +5. Delete the old changefeed. + +## When the downstream of a changefeed is a database similar to MySQL and TiCDC executes a time-consuming DDL statement, all other changefeeds are blocked. How should I handle the issue? + +1. Pause the execution of the changefeed that contains the time-consuming DDL statement. Then you can see that other changefeeds are no longer blocked. +2. Search for the `apply job` field in the TiCDC log and confirm the `start-ts` of the time-consuming DDL statement. +3. Manually execute the DDL statement in the downstream. After the execution finishes, go on performing the following operations. +4. Modify the changefeed configuration and add the above `start-ts` to the `ignore-txn-start-ts` configuration item. +5. Resume the paused changefeed. + +## After I upgrade the TiCDC cluster to v4.0.8, the `[CDC:ErrKafkaInvalidConfig]Canal requires old value to be enabled` error is reported when I execute a changefeed + +Since v4.0.8, if the `canal` or `maxwell` protocol is used for output in a changefeed, TiCDC enables the old value feature automatically. However, if you have upgraded TiCDC from an earlier version to v4.0.8 or later, when the changefeed uses the `canal` or `maxwell` protocol and the old value feature is disabled, this error is reported. To fix the error, take the following steps: + +1. Set the value of `enable-old-value` in the changefeed configuration file to `true`. +2. Execute `cdc cli changefeed pause` to pause the replication task. + + {{< copyable "shell-regular" >}} + + ```shell + cdc cli changefeed pause -c test-cf --pd=http://10.0.10.25:2379 + ``` + +3. Execute `cdc cli changefeed update` to update the original changefeed configuration. + + {{< copyable "shell-regular" >}} + + ```shell + cdc cli changefeed update -c test-cf --pd=http://10.0.10.25:2379 --sink-uri="mysql://127.0.0.1:3306/?max-txn-row=20&worker-number=8" --config=changefeed.toml + ``` + +4. Execute `cdc cli changfeed resume` to resume the replication task. + + {{< copyable "shell-regular" >}} + + ```shell + cdc cli changefeed resume -c test-cf --pd=http://10.0.10.25:2379 + ``` + +## The `[tikv:9006]GC life time is shorter than transaction duration, transaction starts at xx, GC safe point is yy` error is reported when I use TiCDC to create a changefeed + +Solution: You need to execute the `pd-ctl service-gc-safepoint --pd ` command to query the current GC safepoint and service GC safepoint. If the GC safepoint is smaller than the `start-ts` of the TiCDC replication task (changefeed), you can directly add the `--disable-gc-check` option to the `cdc cli create changefeed` command to create a changefeed. + +If the result of `pd-ctl service-gc-safepoint --pd ` does not have `gc_worker service_id`: + +- If your PD version is v4.0.8 or earlier, refer to [PD issue #3128](https://github.com/tikv/pd/issues/3128) for details. +- If your PD is upgraded from v4.0.8 or an earlier version to a later version, refer to [PD issue #3366](https://github.com/tikv/pd/issues/3366) for details. +- For other situations, report the execution result of the above command to our [Slack channel](https://slack.tidb.io/invite?team=tidb-community&channel=sig-migrate&ref=pingcap-community). + +## `enable-old-value` is set to `true` when I create a TiCDC replication task, but `INSERT`/`UPDATE` statements from the upstream become `REPLACE INTO` after being replicated to the downstream + +When a changefeed is created in TiCDC, the `safe-mode` setting defaults to `true`, which generates the `REPLACE INTO` statement to execute for the upstream `INSERT`/`UPDATE` statements. + +Currently, users cannot modify the `safe-mode` setting, so this issue currently has no solution. + +## When I use TiCDC to replicate messages to Kafka, Kafka returns the `Message was too large` error + +For TiCDC v4.0.8 or earlier versions, you cannot effectively control the size of the message output to Kafka only by configuring the `max-message-bytes` setting for Kafka in the Sink URI. To control the message size, you also need to increase the limit on the bytes of messages to be received by Kafka. To add such a limit, add the following configuration to the Kafka server configuration. + +``` +# The maximum byte number of a message that the broker receives +message.max.bytes=2147483648 +# The maximum byte number of a message that the broker copies +replica.fetch.max.bytes=2147483648 +# The maximum message byte number that the consumer side reads +fetch.message.max.bytes=2147483648 +``` + +## How can I find out whether a DDL statement fails to execute in downstream during TiCDC replication? How to resume the replication? + +If a DDL statement fails to execute, the replication task (changefeed) automatically stops. The checkpoint-ts is the DDL statement's finish-ts minus one. If you want TiCDC to retry executing this statement in the downstream, use `cdc cli changefeed resume` to resume the replication task. For example: + +{{< copyable "shell-regular" >}} + +```shell +cdc cli changefeed resume -c test-cf --pd=http://10.0.10.25:2379 +``` + +If you want to skip this DDL statement that goes wrong, set the start-ts of the changefeed to the checkpoint-ts (the timestamp at which the DDL statement goes wrong) plus one. For example, if the checkpoint-ts at which the DDL statement goes wrong is `415241823337054209`, execute the following commands to skip this DDL statement: + +{{< copyable "shell-regular" >}} + +```shell +cdc cli changefeed update -c test-cf --pd=http://10.0.10.25:2379 --start-ts 415241823337054210 +cdc cli changefeed resume -c test-cf --pd=http://10.0.10.25:2379 +``` + +## When the sink of the replication downstream is TiDB or MySQL, what permissions do users of the downstream database need? + +When the sink is TiDB or MySQL, the users of the downstream database need the following permissions: + +- `Select` +- `Index` +- `Insert` +- `Update` +- `Delete` +- `Create` +- `Drop` +- `Alter` +- `Create View` + +If you need to replicate `recover table` to the downstream TiDB, the `Super` permission is required. diff --git a/tidb-architecture.md b/tidb-architecture.md index d6741a73b5d8..d2a98ef684d6 100644 --- a/tidb-architecture.md +++ b/tidb-architecture.md @@ -16,7 +16,7 @@ Compared with the traditional standalone databases, TiDB has the following advan As a distributed database, TiDB is designed to consist of multiple components. Theses components communicate with each other and form a complete TiDB system. The architecture is as follows: -![TiDB Architecture](/media/tidb-architecture.png) +![TiDB Architecture](/media/tidb-architecture-v3.1.png) ## TiDB server diff --git a/tidb-binlog-deployment-topology.md b/tidb-binlog-deployment-topology.md index 606800241583..3f991426c7b2 100644 --- a/tidb-binlog-deployment-topology.md +++ b/tidb-binlog-deployment-topology.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/tidb-binlog-deployment-topology/'] # TiDB Binlog Deployment Topology -This document describes the deployment topology of TiDB Binlog based on the minimal TiDB topology. +This document describes the deployment topology of [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) based on the minimal TiDB topology. TiDB Binlog is the widely used component for replicating incremental data. It provides near real-time backup and replication. @@ -26,6 +26,8 @@ TiDB Binlog is the widely used component for replicating incremental data. It pr - [The simple template for the TiDB Binlog topology (with `file` as the downstream type)](https://github.com/pingcap/docs/blob/master/config-templates/simple-file-binlog.yaml) - [The complex template for the TiDB Binlog topology](https://github.com/pingcap/docs/blob/master/config-templates/complex-tidb-binlog.yaml) +For detailed descriptions of the configuration items in the above TiDB cluster topology file, see [Topology Configuration File for Deploying TiDB Using TiUP](/tiup/tiup-cluster-topology-reference.md). + ### Key parameters The key parameters in the topology configuration templates are as follows: diff --git a/tidb-binlog/bidirectional-replication-between-tidb-clusters.md b/tidb-binlog/bidirectional-replication-between-tidb-clusters.md index 618ccd9033f7..16b770fd0921 100644 --- a/tidb-binlog/bidirectional-replication-between-tidb-clusters.md +++ b/tidb-binlog/bidirectional-replication-between-tidb-clusters.md @@ -4,7 +4,7 @@ summary: Learn how to perform the bidirectional replication between TiDB cluster aliases: ['/docs/dev/tidb-binlog/bidirectional-replication-between-tidb-clusters/','/docs/dev/reference/tidb-binlog/bidirectional-replication/'] --- -# Bidirectional Replication Between TiDB Clusters +# Bidirectional Replication between TiDB Clusters This document describes the bidirectional replication between two TiDB clusters, how the replication works, how to enable it, and how to replicate DDL operations. diff --git a/tidb-binlog/binlog-control.md b/tidb-binlog/binlog-control.md index 43e8f12db977..867ecfa03b13 100644 --- a/tidb-binlog/binlog-control.md +++ b/tidb-binlog/binlog-control.md @@ -22,6 +22,10 @@ The following are its usage scenarios: ## Download `binlogctl` +> **Note:** +> +> It is recommended that the version of the Control tool you use is consistent with the version of the cluster. + {{< copyable "shell-regular" >}} ```bash diff --git a/tidb-binlog/deploy-tidb-binlog.md b/tidb-binlog/deploy-tidb-binlog.md index b5dc150007bf..75d805a69cf2 100644 --- a/tidb-binlog/deploy-tidb-binlog.md +++ b/tidb-binlog/deploy-tidb-binlog.md @@ -6,12 +6,7 @@ aliases: ['/docs/dev/tidb-binlog/deploy-tidb-binlog/','/docs/dev/reference/tidb- # TiDB Binlog Cluster Deployment -This document describes two methods of deploying TiDB Binlog: - -- [Deploy TiDB Binlog using TiDB Ansible](#deploy-tidb-binlog-using-tidb-ansible) -- [Deploy TiDB Binlog using a Binary package](#deploy-tidb-binlog-using-a-binary-package) - -It is recommended to deploy TiDB Binlog using TiDB Ansible. If you just want to do a simple testing, you can deploy TiDB Binlog using a Binary package. +This document describes how to [deploy TiDB Binlog using a Binary package](#deploy-tidb-binlog-using-a-binary-package). ## Hardware requirements @@ -24,243 +19,9 @@ In environments of development, testing and production, the requirements on serv | Pump | 3 | 8 core+ | SSD, 200 GB+ | 16G | | Drainer | 1 | 8 core+ | SAS, 100 GB+ (If binlogs are output as local files, the disk size depends on how long these files are retained.) | 16G | -## Deploy TiDB Binlog using TiDB Ansible - -### Step 1: Download TiDB Ansible - -1. Use the TiDB user account to log in to the control machine and go to the `/home/tidb` directory. The information about the branch of TiDB Ansible and the corresponding TiDB version is as follows. If you have questions regarding which version to use, email to [info@pingcap.com](mailto:info@pingcap.com) for more information or [file an issue](https://github.com/pingcap/tidb-ansible/issues/new). - - | tidb-ansible branch | TiDB version | Note | - | :------------------- | :------------ | :---- | - | master | master version | This version includes the latest features with a daily update. | - -2. Use the following command to download the corresponding branch of TiDB Ansible from the [TiDB Ansible project](https://github.com/pingcap/tidb-ansible) on GitHub. The default folder name is `tidb-ansible`. - - - Download the master version: - - {{< copyable "shell-regular" >}} - - ```bash - git clone https://github.com/pingcap/tidb-ansible.git - ``` - -### Step 2: Deploy Pump - -1. Modify the `tidb-ansible/inventory.ini` file. - - 1. Set `enable_binlog = True` to start `binlog` of the TiDB cluster. - - ```ini - ## binlog trigger - enable_binlog = True - ``` - - 2. Add the target machine IPs for `pump_servers`. - - ```ini - ## Binlog Part - [pump_servers] - 172.16.10.72 - 172.16.10.73 - 172.16.10.74 - ``` - - Pump retains the data of the latest 7 days by default. You can modify the value of the `gc` variable in the `tidb-ansible/conf/pump.yml` file (or `tidb-ansible/conf/pump-cluster.yml` in TiDB 3.0.0~3.0.2) and remove the related comments: - - {{< copyable "" >}} - - ```yaml - global: - # an integer value to control the expiry date of the binlog data, which indicates for how long (in days) the binlog data would be stored - # must be bigger than 0 - # gc: 7 - ``` - - Make sure the space of the deployment directory is sufficient for storing Binlog. For more details, see [Configure the deployment directory](/online-deployment-using-ansible.md#configure-the-deployment-directory). You can also set a separate deployment directory for Pump. - - ```ini - ## Binlog Part - [pump_servers] - pump1 ansible_host=172.16.10.72 deploy_dir=/data1/pump - pump2 ansible_host=172.16.10.73 deploy_dir=/data2/pump - pump3 ansible_host=172.16.10.74 deploy_dir=/data3/pump - ``` - -2. Deploy and start the TiDB cluster containing Pump. - - After configuring the `inventory.ini` file, you can choose one method from below to deploy the TiDB cluster. - - **Method #1**: Add Pump on the existing TiDB cluster. - - 1. Deploy `pump_servers` and `node_exporters`. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook deploy.yml --tags=pump -l ${pump1_ip},${pump2_ip},[${alias1_name},${alias2_name}] - ``` - - > **Note:** - > - > Do not add a space after the commas in the above command. Otherwise, an error is reported. - - 2. Start `pump_servers`. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook start.yml --tags=pump - ``` - - 3. Update and restart `tidb_servers`. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook rolling_update.yml --tags=tidb - ``` - - 4. Update the monitoring data. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook rolling_update_monitor.yml --tags=prometheus - ``` - - **Method #2**: Deploy a TiDB cluster containing Pump from scratch. - - For how to use TiDB Ansible to deploy the TiDB cluster, see [Deploy TiDB Using TiDB Ansible](/online-deployment-using-ansible.md). - -3. Check the Pump status. +## Deploy TiDB Binlog using TiUP - Use `binlogctl` to check the Pump status. Change the `pd-urls` parameter to the PD address of the cluster. If `State` is `online`, Pump is started successfully. - - {{< copyable "shell-regular" >}} - - ```bash - cd /home/tidb/tidb-ansible && - resources/bin/binlogctl -pd-urls=http://172.16.10.72:2379 -cmd pumps - ``` - - ``` - INFO[0000] pump: {NodeID: ip-172-16-10-72:8250, Addr: 172.16.10.72:8250, State: online, MaxCommitTS: 403051525690884099, UpdateTime: 2018-12-25 14:23:37 +0800 CST} - INFO[0000] pump: {NodeID: ip-172-16-10-73:8250, Addr: 172.16.10.73:8250, State: online, MaxCommitTS: 403051525703991299, UpdateTime: 2018-12-25 14:23:36 +0800 CST} - INFO[0000] pump: {NodeID: ip-172-16-10-74:8250, Addr: 172.16.10.74:8250, State: online, MaxCommitTS: 403051525717360643, UpdateTime: 2018-12-25 14:23:35 +0800 CST} - ``` - -### Step 3: Deploy Drainer - -1. Obtain the value of `initial_commit_ts`. - - When Drainer starts for the first time, the timestamp information `initial_commit_ts` is required. - - - If the replication is started from the latest time point, you just need to set `initial_commit_ts` to `-1`. - - - If the downstream database is MySQL or TiDB, to ensure data integrity, you need to perform full data backup and recovery. In this case, the value of `initial_commit_ts` must be the timestamp information of the full backup. - - If you use mydumper to perform full data backup, you can get the timestamp by referring to the `Pos` field in the metadata file from the export directory. An example of the metadata file is as follows: - - ``` - Started dump at: 2019-12-30 13:25:41 - SHOW MASTER STATUS: - Log: tidb-binlog - Pos: 413580274257362947 - GTID: - Finished dump at: 2019-12-30 13:25:41 - ``` - -2. Modify the `tidb-ansible/inventory.ini` file. - - Add the target machine IPs for `drainer_servers`. Set `initial_commit_ts` to the value you have obtained, which is only used for the initial start of Drainer. - - - Assume that the downstream is MySQL with the alias `drainer_mysql`: - - ```ini - [drainer_servers] - drainer_mysql ansible_host=172.16.10.71 initial_commit_ts="402899541671542785" - ``` - - - Assume that the downstream is `file` with the alias `drainer_file`: - - ```ini - [drainer_servers] - drainer_file ansible_host=172.16.10.71 initial_commit_ts="402899541671542785" - ``` - -3. Modify the configuration file. - - - Assume that the downstream is MySQL: - - {{< copyable "shell-regular" >}} - - ```bash - cd /home/tidb/tidb-ansible/conf && - cp drainer-cluster.toml drainer_mysql_drainer.toml && - vi drainer_mysql_drainer.toml - ``` - - > **Note:** - > - > Name the configuration file as `alias_drainer.toml`. Otherwise, the customized configuration file cannot be found during the deployment process. - - Set `db-type` to `mysql` and configure the downstream MySQL information: - - {{< copyable "" >}} - - ```toml - # downstream storage, equal to --dest-db-type - # Valid values are "mysql", "file", "tidb", and "kafka". - db-type = "mysql" - - # the downstream MySQL protocol database - [syncer.to] - host = "172.16.10.72" - user = "root" - password = "123456" - port = 3306 - ``` - - - Assume that the downstream is incremental backup data: - - {{< copyable "shell-regular" >}} - - ```bash - cd /home/tidb/tidb-ansible/conf && - cp drainer-cluster.toml drainer_file_drainer.toml && - vi drainer_file_drainer.toml - ``` - - Set `db-type` to `file`. - - {{< copyable "" >}} - - ```toml - # downstream storage, equal to --dest-db-type - # Valid values are "mysql", "file", "tidb", and "kafka". - db-type = "file" - - # Uncomment this if you want to use "file" as "db-type". - [syncer.to] - # default data directory: "{{ deploy_dir }}/data.drainer" - dir = "data.drainer" - ``` - -4. Deploy Drainer. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook deploy_drainer.yml - ``` - -5. Start Drainer. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook start_drainer.yml - ``` +It is recommended to deploy TiDB Binlog using TiUP. To do that, when deploying TiDB using TiUP, you need to add the node information of `drainer` and `pump` of TiDB Binlog in [TiDB Binlog Deployment Topology](/tidb-binlog-deployment-topology.md). For detailed deployment information, refer to [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). ## Deploy TiDB Binlog using a Binary package @@ -574,9 +335,9 @@ The following part shows how to use Pump and Drainer based on the nodes above. [syncer.to] host = "192.168.0.13" user = "root" + # If you do not want to set a cleartext `password` in the configuration file, you can create `encrypted_password` using `./binlogctl -cmd encrypt -text string`. + # When you have created an `encrypted_password` that is not empty, the `password` above will be ignored, because `encrypted_password` and `password` cannot take effect at the same time. password = "" - # `encrypted_password` is encrypted using `./binlogctl -cmd encrypt -text string`. - # When `encrypted_password` is not empty, the `password` above will be ignored. encrypted_password = "" port = 3306 @@ -620,7 +381,7 @@ The following part shows how to use Pump and Drainer based on the nodes above. > **Note:** > - > If the downstream is MySQL/TiDB, to guarantee the data integrity, you need to obtain the `initial-commit-ts` value and make a full backup of the data and restore the data before the initial start of Drainer. For details, see [Deploy Drainer](#step-3-deploy-drainer). + > If the downstream is MySQL/TiDB, to guarantee the data integrity, you need to obtain the `initial-commit-ts` value and make a full backup of the data and restore the data before the initial start of Drainer. When Drainer is started for the first time, use the `initial-commit-ts` parameter. diff --git a/tidb-binlog/get-started-with-tidb-binlog.md b/tidb-binlog/get-started-with-tidb-binlog.md index 5c25bb8b7f85..c91ddf21aa60 100644 --- a/tidb-binlog/get-started-with-tidb-binlog.md +++ b/tidb-binlog/get-started-with-tidb-binlog.md @@ -42,8 +42,6 @@ We're using MariaDB Server in this case instead of MySQL Server because RHEL/Cen sudo yum install -y mariadb-server ``` -Even if you've already started a TiDB cluster, it will be easier to follow along with this tutorial where we will set up a new, simple cluster. We will install from a tarball, using a simplified form of the [Local Deployment](/deploy-tidb-from-binary.md) guide. You may also wish to refer to [Testing Deployment from Binary Tarball](/test-deployment-from-binary-tarball.md) for best practices of establishing a real testing deployment, but that goes beyond the scope of this tutorial. - ```bash curl -L https://download.pingcap.org/tidb-latest-linux-amd64.tar.gz | tar xzf - cd tidb-latest-linux-amd64 diff --git a/tidb-binlog/handle-tidb-binlog-errors.md b/tidb-binlog/handle-tidb-binlog-errors.md index ba53db3dbb62..5ba106db4941 100644 --- a/tidb-binlog/handle-tidb-binlog-errors.md +++ b/tidb-binlog/handle-tidb-binlog-errors.md @@ -33,3 +33,16 @@ Solution: Clean up the disk space and then restart Pump. Cause: When Pump is started, it notifies all Drainer nodes that are in the `online` state. If it fails to notify Drainer, this error log is printed. Solution: Use the [binlogctl tool](/tidb-binlog/binlog-control.md) to check whether each Drainer node is normal or not. This is to ensure that all Drainer nodes that are in the `online` state are working normally. If the state of a Drainer node is not consistent with its actual working status, use the binlogctl tool to change its state and then restart Pump. + +## Data loss occurs during the TiDB Binlog replication + +You need to confirm that TiDB Binlog is enabled on all TiDB instances and runs normally. If the cluster version is later than v3.0, use the `curl {TiDB_IP}:{STATUS_PORT}/info/all` command to confirm the TiDB Binlog status on all TiDB instances. + +## When the upstream transaction is large, Pump reports an error `rpc error: code = ResourceExhausted desc = trying to send message larger than max (2191430008 vs. 2147483647)` + +This error occurs because the gRPC message sent by TiDB to Pump exceeds the size limit. You can adjust the maximum size of a gRPC message that Pump allows by specifying `max-message-size` when starting Pump. + +## Is there any cleaning mechanism for the incremental data of the file format output by Drainer? Will the data be deleted? + +- In Drainer v3.0.x, there is no cleaning mechanism for incremental data of the file format. +- In the v4.0.x version, there is a time-based data cleaning mechanism. For details, refer to [Drainer's `retention-time` configuration item](https://github.com/pingcap/tidb-binlog/blob/v4.0.9/cmd/drainer/drainer.toml#L153). diff --git a/tidb-binlog/monitor-tidb-binlog-cluster.md b/tidb-binlog/monitor-tidb-binlog-cluster.md index 0efd10ab0fdd..0c2525f3c1e0 100644 --- a/tidb-binlog/monitor-tidb-binlog-cluster.md +++ b/tidb-binlog/monitor-tidb-binlog-cluster.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/tidb-binlog/monitor-tidb-binlog-cluster/','/docs/dev/refere # TiDB Binlog Monitoring -After you have deployed TiDB Binlog using TiDB Ansible successfully, you can go to the Grafana Web (default address: , default account: admin, password: admin) to check the state of Pump and Drainer. +After you have deployed TiDB Binlog successfully, you can go to the Grafana Web (default address: , default account: admin, password: admin) to check the state of Pump and Drainer. ## Monitoring metrics diff --git a/tidb-binlog/tidb-binlog-configuration-file.md b/tidb-binlog/tidb-binlog-configuration-file.md index bdc710c2c437..dfd047f0efb6 100644 --- a/tidb-binlog/tidb-binlog-configuration-file.md +++ b/tidb-binlog/tidb-binlog-configuration-file.md @@ -173,8 +173,12 @@ This section introduces the configuration items of Drainer. For the example of a ### initial-commit-ts -* Specifies from which commit timestamp the replication task starts. This configuration is only applicable to the Drainer node that starts replication for the first time. If a checkpoint already exists downstream, the replication will be performed according to the time recorded in the checkpoint. -* Default value: `-1`. Drainer will get a new timestamp from PD as the starting time. +* Specifies from which commit timestamp of the transaction the replication process starts. This configuration is applicable only to the Drainer node that is in the replication process for the first time. If a checkpoint already exists in the downstream, the replication will be performed according to the time recorded in the checkpoint. +* commit ts (commit timestamp) is a specific point in time for [transaction](/transaction-overview.md#transactions) commits in TiDB. It is a globally unique and increasing timestamp from PD as the unique ID of the current transaction. You can get the `initial-commit-ts` configuration in the following typical ways: + - If BR is used, you can get `initial-commit-ts` from the backup TS recorded in the metadata backed up by BR (backupmeta). + - If Dumpling is used, you can get `initial-commit-ts` from the Pos recorded in the metadata backed up by Dumpling (metadata), + - If PD Control is used, `initial-commit-ts` is in the output of the `tso` command. +* Default value: `-1`. Drainer will get a new timestamp from PD as the starting time, which means that the replication process starts from the current time. ### synced-check-time diff --git a/tidb-binlog/tidb-binlog-faq.md b/tidb-binlog/tidb-binlog-faq.md index e66ca4731f02..596e2169e08e 100644 --- a/tidb-binlog/tidb-binlog-faq.md +++ b/tidb-binlog/tidb-binlog-faq.md @@ -128,7 +128,7 @@ If the data in the downstream is not affected, you can redeploy Drainer on the n ## How to redeploy Drainer when enabling `ignore-error` in Primary-Secondary replication triggers a critical error? -If a critical error is trigged when TiDB fails to write binlog after enabling `ignore-error`, TiDB stops writing binlog and binlog data loss occurs. To resume replication, perform the following steps: +If a critical error is triggered when TiDB fails to write binlog after enabling `ignore-error`, TiDB stops writing binlog and binlog data loss occurs. To resume replication, perform the following steps: 1. Stop the Drainer instance. @@ -244,7 +244,29 @@ To solve the problem, follow these steps: 3. Check `drainer.log`. Search for the failed DDL operation and find the `commit-ts` of this operation. For example: ``` - [2020/05/21 09:51:58.019 +08:00] [INFO] [syncer.go:398] ["add ddl item to syncer, you can add this commit ts to `ignore-txn-commit-ts` to skip this ddl if needed"] [sql="ALTER TABLE `test` ADD INDEX (`index1`)"] ["commit ts"=416815754209656834]。 + [2020/05/21 09:51:58.019 +08:00] [INFO] [syncer.go:398] ["add ddl item to syncer, you can add this commit ts to `ignore-txn-commit-ts` to skip this ddl if needed"] [sql="ALTER TABLE `test` ADD INDEX (`index1`)"] ["commit ts"=416815754209656834]. ``` 4. Modify the `drainer.toml` configuration file. Add the `commit-ts` in the `ignore-txn-commit-ts` item and restart the Drainer node. + +## TiDB fails to write to binlog and gets stuck, and `listener stopped, waiting for manual stop` appears in the log + +In TiDB v3.0.12 and earlier versions, the binlog write failure causes TiDB to report the fatal error. TiDB does not automatically exit but only stops the service, which seems like getting stuck. You can see the `listener stopped, waiting for manual stop` error in the log. + +You need to determine the specific causes of the binlog write failure. If the failure occurs because binlog is slowly written into the downstream, you can consider scaling out Pump or increasing the timeout time for writing binlog. + +Since v3.0.13, the error-reporting logic is optimized. The binlog write failure causes transaction execution to fail and TiDB Binlog will return an error but will not get TiDB stuck. + +## TiDB writes duplicate binlogs to Pump + +This issue does not affect the downstream and replication logic. + +When the binlog write fails or becomes timeout, TiDB retries writing binlog to the next available Pump node until the write succeeds. Therefore, if the binlog write to a Pump node is slow and causes TiDB timeout (default 15s), then TiDB determines that the write fails and tries to write to the next Pump node. If binlog is actually successfully written to the timeout-causing Pump node, the same binlog is written to multiple Pump nodes. When Drainer processes the binlog, it automatically de-duplicates binlogs with the same TSO, so this duplicate write does not affect the downstream and replication logic. + +## Reparo is interrupted during the full and incremental restore process. Can I use the last TSO in the log to resume replication? + +Yes. Reparo does not automatically enable the safe-mode when you start it. You need to perform the following steps manually: + +1. After Reparo is interrupted, record the last TSO in the log as `checkpoint-tso`. +2. Modify the Reparo configuration file, set the configuration item `start-tso` to `checkpoint-tso + 1`, set `stop-tso` to `checkpoint-tso + 80,000,000,000` (approximately five minutes after the `checkpoint-tso`), and set `safe-mode` to `true`. Start Reparo, and Reparo replicates data to `stop-tso` and then stops automatically. +3. After Reparo stops automatically, set `start-tso` to `checkpoint tso + 80,000,000,001`, set `stop-tso` to `0`, and set `safe-mode` to `false`. Start Reparo to resume replication. diff --git a/tidb-binlog/tidb-binlog-overview.md b/tidb-binlog/tidb-binlog-overview.md index ebcedd41acf5..1818a2bee269 100644 --- a/tidb-binlog/tidb-binlog-overview.md +++ b/tidb-binlog/tidb-binlog-overview.md @@ -15,6 +15,10 @@ TiDB Binlog has the following features: * **Data replication:** replicate the data in the TiDB cluster to other databases * **Real-time backup and restoration:** back up the data in the TiDB cluster and restore the TiDB cluster when the cluster fails +> **Note:** +> +> TiDB Binlog is not compatible with some features introduced in TiDB v5.0 and they cannot be used together. For details, see [Notes](#notes). It is recommended to use [TiCDC](/ticdc/ticdc-overview.md) instead of TiDB Binlog. + ## TiDB Binlog architecture The TiDB Binlog architecture is as follows: @@ -50,7 +54,20 @@ The TiDB Binlog cluster is composed of Pump and Drainer. ## Notes -* You need to use TiDB v2.0.8-binlog, v2.1.0-rc.5 or a later version. Older versions of TiDB cluster are not compatible with the cluster version of TiDB Binlog. +* In v5.1, the incompatibility between the clustered index feature introduced in v5.0 and TiDB Binlog has been resolved. After you upgrade TiDB Binlog and TiDB Server to v5.1 and enable TiDB Binlog, TiDB will support creating tables with clustered indexes; data insertion, deletion, and update on the created tables with clustered indexes will be replicated to the downstream via TiDB Binlog. When you use TiDB Binlog to replicate the tables with clustered indexes, pay attention to the following: + + - If you have upgraded the cluster to v5.1 from v5.0 by manually controlling the upgrade sequence, make sure that TiDB binlog is upgraded to v5.1 before upgrading the TiDB server to v5.1. + - It is recommended to configure the system variable [`tidb_enable_clustered_index`](/system-variables.md#tidb_enable_clustered_index-new-in-v50) to a same value to ensure that the structure of TiDB clustered index tables between the upstream and downstream is consistent. + +* TiDB Binlog is incompatible with the following features introduced in TiDB v5.0 and they cannot be used together. + + - [TiDB Clustered Index](/clustered-indexes.md#limitations): After TiDB Binlog is enabled, TiDB does not allow creating clustered indexes with non-single integer columns as primary keys; data insertion, deletion, and update of the created clustered index tables will not be replicated downstream via TiDB Binlog. If you need to replicate tables with clustered indexes, upgrade your cluster to v5.1 or use [TiCDC](/ticdc/ticdc-overview.md) instead. + - TiDB system variable [tidb_enable_async_commit](/system-variables.md#tidb_enable_async_commit-new-in-v50): After TiDB Binlog is enabled, performance cannot be improved by enabling this option. It is recommended to use [TiCDC](/ticdc/ticdc-overview.md) instead of TiDB Binlog. + - TiDB system variable [tidb_enable_1pc](/system-variables.md#tidb_enable_1pc-new-in-v50): After TiDB Binlog is enabled, performance cannot be improved by enabling this option. It is recommended to use [TiCDC](/ticdc/ticdc-overview.md) instead of TiDB Binlog. + +* TiDB Binlog is incompatible with the following feature introduced in TiDB v4.0.7 and they cannot be used together: + + - TiDB system variable [tidb_enable_amend_pessimistic_txn](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407): The two features have compatibility issues. Using them together might cause the issue that TiDB Binlog replicates data inconsistently. * Drainer supports replicating binlogs to MySQL, TiDB, Kafka or local files. If you need to replicate binlogs to other Drainer unsuppored destinations, you can set Drainer to replicate the binlog to Kafka and read the data in Kafka for customized processing according to binlog consumer protocol. See [Binlog Consumer Client User Guide](/tidb-binlog/binlog-consumer-client.md). diff --git a/tidb-binlog/upgrade-tidb-binlog.md b/tidb-binlog/upgrade-tidb-binlog.md index c1190e1f7065..9e200e85703e 100644 --- a/tidb-binlog/upgrade-tidb-binlog.md +++ b/tidb-binlog/upgrade-tidb-binlog.md @@ -6,26 +6,7 @@ aliases: ['/docs/dev/tidb-binlog/upgrade-tidb-binlog/','/docs/dev/reference/tidb # Upgrade TiDB Binlog -This document introduces how to upgrade TiDB Binlog that is deployed with TiDB Ansible and deployed manually to the latest [cluster](/tidb-binlog/tidb-binlog-overview.md) version. There is also a section on how to upgrade TiDB Binlog from an earlier incompatible version (Kafka/Local version) to the latest version. - -## Upgrade TiDB Binlog deployed with TiDB Ansible - -Follow the steps in this section if you deploy TiDB Binlog with [TiDB Ansible Playbook](https://github.com/pingcap/tidb-ansible). - -### Upgrade Pump - -First, upgrade the Pump component: - -1. Copy the new version of the binary `pump` file into the`({ resources_dir })/bin` directory. -2. Execute the `ansible-playbook rolling_update.yml --tags=pump` command to perform a rolling update for Pump. - -### Upgrade Drainer - -Second, upgrade the Drainer component: - -1. Copy the new version of the binary `drainer` file into the`({ resources_dir })/bin` directory. -2. Execute the `ansible-playbook stop_drainer.yml --tags=drainer` command. -3. Execute the `ansible-playbook start_drainer.yml --tags=drainer` command. +This document introduces how to upgrade TiDB Binlog that is deployed manually to the latest [cluster](/tidb-binlog/tidb-binlog-overview.md) version. There is also a section on how to upgrade TiDB Binlog from an earlier incompatible version (Kafka/Local version) to the latest version. ## Upgrade TiDB Binlog deployed manually @@ -86,4 +67,4 @@ If you want to resume replication from the original checkpoint, perform the foll If the return value of `Synced` is True, it means Drainer has replicated the data in the old version of Pump to the downstream completely. 6. Start the new version of Drainer. -7. Close the Pump and Drainer of the old versions and the dependent Kafka and Zookeeper. +7. Close the Pump and Drainer of the old versions and the dependent Kafka and ZooKeeper. diff --git a/tidb-computing.md b/tidb-computing.md index 516621b39c82..faa3b7a77c20 100644 --- a/tidb-computing.md +++ b/tidb-computing.md @@ -69,14 +69,14 @@ Also note that in the above encoding schemes, no matter table data or index data This section shows a simple example for you to understand the Key-Value mapping relationship of TiDB. Suppose the following table exists in TiDB. ```sql -CREATE TABLE User { +CREATE TABLE User ( ID int, Name varchar(20), Role varchar(20), Age int, PRIMARY KEY (ID), KEY idxAge (Age) -}; +); ``` Suppose there are 3 rows of data in the table. @@ -111,7 +111,7 @@ Each database and table in TiDB has metadata that indicates its definition and v Each database or table is assigned a unique ID. As the unique identifier, when table data is encoded to Key-Value, this ID is encoded in the Key with the `m_` prefix. This constructs a key-value pair with the serialized metadata stored in it. -In addition, TiDB also uses a dedicated (Key, Value) key-value pair to store the latest version number of structure information of all tables. This key-value pair is global, and its version number is increased by `1` each time the state of the DDL operation changes. TiDB stores this key-value pair persistently in the PD server with the key of `/tidb/ddl/global_schema_version`, and Value is the version number value of the `int64` type. Inspired by Google F1's Online Schema change algorithm, TiDB keeps a background thread that constantly checks whether the version number of the table structure information stored in the PD server changes. This thread also ensures that the changes of version can be obtained within a certain period of time. +In addition, TiDB also uses a dedicated (Key, Value) key-value pair to store the latest version number of structure information of all tables. This key-value pair is global, and its version number is increased by `1` each time the state of the DDL operation changes. TiDB stores this key-value pair persistently in the PD server with the key of `/tidb/ddl/global_schema_version`, and Value is the version number value of the `int64` type. Meanwhile, because TiDB applies schema changes online, it keeps a background thread that constantly checks whether the version number of the table structure information stored in the PD server changes. This thread also ensures that the changes of version can be obtained within a certain period of time. ## SQL layer overview diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index 8ce730152e31..49fb30aa8798 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -20,7 +20,11 @@ The TiDB configuration file supports more options than command-line parameters. ### `token-limit` + The number of sessions that can execute requests concurrently. ++ Type: Integer + Default value: `1000` ++ Minimum value: `1` ++ Maximum Value (64-bit platforms): `18446744073709551615` ++ Maximum Value (32-bit platforms): `4294967295` ### `mem-quota-query` @@ -88,12 +92,16 @@ The TiDB configuration file supports more options than command-line parameters. - Determines whether to treat the `utf8` character set in old tables as `utf8mb4`. - Default value: `true` -### `alter-primary-key` +### `alter-primary-key` (Deprecated) - Determines whether to add or remove the primary key constraint to or from a column. - Default value: `false` - With this default setting, adding or removing the primary key constraint is not supported. You can enable this feature by setting `alter-primary-key` to `true`. However, if a table already exists before the switch is on, and the data type of its primary key column is an integer, dropping the primary key from the column is not possible even if you set this configuration item to `true`. +> **Note:** +> +> This configuration item has been deprecated, and currently takes effect only when the value of `@tidb_enable_clustered_index` is `INT_ONLY`. If you need to add or remove the primary key, use the `NONCLUSTERED` keyword instead when creating the table. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). + ### `server-version` + Modifies the version string returned by TiDB in the following situations: @@ -133,11 +141,41 @@ The TiDB configuration file supports more options than command-line parameters. - Unit: byte - Currently, the valid value range is `[3072, 3072*4]`. MySQL and TiDB (version < v3.0.11) do not have this configuration item, but both limit the length of the newly created index. This limit in MySQL is `3072`. In TiDB (version =< 3.0.7), this limit is `3072*4`. In TiDB (3.0.7 < version < 3.0.11), this limit is `3072`. This configuration is added to be compatible with MySQL and earlier versions of TiDB. +### `table-column-count-limit` New in v5.0 + +- Sets the limit on the number of columns in a single table. +- Default value: `1017` +- Currently, the valid value range is `[1017, 4096]`. + +### `index-limit` New in v5.0 + +- Sets the limit on the number of indexes in a single table. +- Default value: `64` +- Currently, the valid value range is `[64, 512]`. + ### `enable-telemetry` New in v4.0.2 - Enables or disables the telemetry collection in TiDB. - Default value: `true` -- When this configuration is set to `false` on all TiDB instances, the telemetry collection in TiDB is disabled and the [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402-version) system variable does not take effect. See [Telemetry](/telemetry.md) for details. +- When this configuration is set to `false` on all TiDB instances, the telemetry collection in TiDB is disabled and the [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) system variable does not take effect. See [Telemetry](/telemetry.md) for details. + +### `enable-tcp4-only` New in v5.0 + +- Enables or disables listening on TCP4 only. +- Default value: `false` +- Enabling this option is useful when TiDB is used with LVS for load balancing because the [real client IP from the TCP header](https://github.com/alibaba/LVS/tree/master/kernel/net/toa) can be correctly parsed by the "tcp4" protocol. + +### `enable-enum-length-limit` New in v5.0 + ++ Determines whether to limit the maximum length of a single `ENUM` element and a single `SET` element. ++ Default value: `true` ++ When this configuration value is `true`, the maximum length of a single `ENUM` element and a single `SET` element is 255 characters, which is compatible with [MySQL 8.0](https://dev.mysql.com/doc/refman/8.0/en/string-type-syntax.html). When this configuration value is `false`, there is no limit on the length of a single element, which is compatible with TiDB (earlier than v5.0). + +#### `graceful-wait-before-shutdown` New in v5.0 + +- Specifies the number of seconds that TiDB waits when you shut down the server, which allows the clients to disconnect. +- Default value: `0` +- When TiDB is waiting for shutdown (in the grace period), the HTTP status will indicate a failure, which allows the load balancers to reroute traffic. ## Log @@ -236,6 +274,12 @@ Configuration items related to log files. Configuration items related to security. +### `enable-sem` + +- Enables the Security Enhanced Mode (SEM). +- Default value: `false` +- The status of SEM is available via the system variable [`tidb_enable_enhanced_security`](/system-variables.md#tidb_enable_enhanced_security). + ### `ssl-ca` - The file path of the trusted CA certificate in the PEM format. @@ -304,12 +348,32 @@ Configuration items related to performance. + When TiDB detects that the memory usage of the tidb-server instance exceeds the threshold, it considers that there might be a risk of OOM. Therefore, it records ten SQL statements with the highest memory usage, ten SQL statements with the longest running time, and the heap profile among all SQL statements currently being executed to the directory [`tmp-storage-path/record`](/tidb-configuration-file.md#tmp-storage-path) and outputs a log containing the keyword `tidb-server has the risk of OOM`. + The value of this configuration item is the initial value of the system variable [`tidb_memory_usage_alarm_ratio`](/system-variables.md#tidb_memory_usage_alarm_ratio). +### `max-txn-ttl` + +- The longest time that a single transaction can hold locks. If this time is exceeded, the locks of a transaction might be cleared by other transactions so that this transaction cannot be successfully committed. +- Default value: `3600000` +- Unit: Millisecond +- The transaction that holds locks longer than this time can only be committed or rolled back. The commit might not be successful. + +### `committer-concurrency` + ++ The number of goroutines for requests related to executing commit in the commit phase of the single transaction. ++ Default value: `16` ++ If the transaction to commit is too large, the waiting time for the flow control queue when the transaction is committed might be too long. In this situation, you can increase the configuration value to speed up the commit. + ### `stmt-count-limit` - The maximum number of statements allowed in a single TiDB transaction. - Default value: `5000` - If a transaction does not roll back or commit after the number of statements exceeds `stmt-count-limit`, TiDB returns the `statement count 5001 exceeds the transaction limitation, autocommit = false` error. This configuration takes effect **only** in the retriable optimistic transaction. If you use the pessimistic transaction or have disabled the transaction retry, the number of statements in a transaction is not limited by this configuration. +### `txn-entry-size-limit` New in v5.0 + +- The size limit of a single row of data in TiDB. +- Default value: `6291456` (in bytes) +- The size limit of a single key-value record in a transaction. If the size limit is exceeded, TiDB returns the `entry too large` error. The maximum value of this configuration item does not exceed `125829120` (120 MB). +- Note that TiKV has a similar limit. If the data size of a single write request exceeds [`raft-entry-max-size`](/tikv-configuration-file.md#raft-entry-max-size), which is 8 MB by default, TiKV refuses to process this request. When a table has a row of large size, you need to modify both configurations at the same time. + ### `txn-total-size-limit` - The size limit of a single transaction in TiDB. @@ -321,6 +385,11 @@ Configuration items related to performance. - Determines whether to enable `keepalive` in the TCP layer. - Default value: `true` +### `tcp-no-delay` + +- Determines whether to enable TCP_NODELAY at the TCP layer. After it is enabled, TiDB disables the Nagle algorithm in the TCP/IP protocol and allows sending small data packets to reduce network latency. This is suitable for latency-sensitive applications with a small transmission volume of data. +- Default value: `true` + ### `cross-join` - Default value: `true` @@ -336,7 +405,7 @@ Configuration items related to performance. - At intervals of `stats-lease`, TiDB checks for column statistics that need to be loaded to the memory. - At intervals of `200 * stats-lease`, TiDB writes the feedback cached in the memory to the system table. - At intervals of `5 * stats-lease`, TiDB reads the feedback in the system table, and updates the statistics cached in the memory. -- When `stats-lease` is set to 0, TiDB periodically reads the feedback in the system table, and updates the statistics cached in the memory every three seconds. But TiDB no longer automatically modifies the following statistics-related system tables: +- When `stats-lease` is set to 0s, TiDB periodically reads the feedback in the system table, and updates the statistics cached in the memory every three seconds. But TiDB no longer automatically modifies the following statistics-related system tables: - `mysql.stats_meta`: TiDB no longer automatically records the number of table rows that are modified by the transaction and updates it to this system table. - `mysql.stats_histograms`/`mysql.stats_buckets` and `mysql.stats_top_n`: TiDB no longer automatically analyzes and proactively updates statistics. - `mysql.stats_feedback`: TiDB no longer updates the statistics of the tables and indexes according to a part of statistics returned by the queried data. @@ -349,8 +418,8 @@ Configuration items related to performance. ### `feedback-probability` - The probability that TiDB collects the feedback statistics of each query. -- Default value: `0.05` -- TiDB collects the feedback of each query at the probability of `feedback-probability`, to update statistics. +- Default value: `0` +- This feature is disabled by default, and it is not recommended to enable this feature. If it is enabled, TiDB collects the feedback of each query at the probability of `feedback-probability`, to update statistics. ### `query-feedback-limit` @@ -381,20 +450,30 @@ Configuration items related to performance. + Default value: `20971520` + When `nested-loop-join-cache-capacity` is set to `0`, nested loop join cache is disabled by default. When the LRU size is larger than the value of `nested-loop-join-cache-capacity`, the elements in the LRU are removed. +### `enforce-mpp` + ++ Determines whether to ignore the optimizer's cost estimation and to forcibly use TiFlash's MPP mode for query execution. ++ Default value: `false` ++ This configuration item controls the initial value of [`tidb_enforce_mpp`](/system-variables.md#tidb_enforce_mpp-new-in-v51). For example, when this configuration item is set to `true`, the default value of `tidb_enforce_mpp` is `ON`. + ## prepared-plan-cache The Plan Cache configuration of the `PREPARE` statement. +> **Warning:** +> +> This is still an experimental feature. It is **NOT** recommended that you use it in the production environment. + ### `enabled` - Determines whether to enable Plan Cache of the `PREPARE` statement. -- Default value: `true` +- Default value: `false` ### `capacity` - The number of cached statements. - Default value: `100` -- The type is `uint`. Values less than `0` are converted to large integers. +- The type is `UINT`. Values less than `0` are converted to large integers. ### `memory-guard-ratio` @@ -407,7 +486,7 @@ The Plan Cache configuration of the `PREPARE` statement. ### `grpc-connection-count` - The maximum number of connections established with each TiKV. -- Default value: `16` +- Default value: `4` ### `grpc-keepalive-time` @@ -427,13 +506,6 @@ The Plan Cache configuration of the `PREPARE` statement. - Default value: `41s` - It is required to set this value larger than twice of the Raft election timeout. -### `max-txn-ttl` - -- The longest time that a single transaction can hold locks. If this time is exceeded, the locks of a transaction might be cleared by other transactions so that this transaction cannot be successfully committed. -- Default value: `600000` -- Unit: Millisecond -- The transaction that holds locks longer than this time can only be committed or rolled back. The commit might not be successful. - ### `max-batch-size` - The maximum number of RPC packets sent in batch. If the value is not `0`, the `BatchCommands` API is used to send requests to TiKV, and the RPC latency can be reduced in the case of high concurrency. It is recommended that you do not modify this value. @@ -456,79 +528,18 @@ The Plan Cache configuration of the `PREPARE` statement. - The threshold of the TiKV load. If the TiKV load exceeds this threshold, more `batch` packets are collected to relieve the pressure of TiKV. It is valid only when the value of `tikv-client.max-batch-size` is greater than `0`. It is recommended not to modify this value. - Default value: `200` -### `enable-one-pc` - -- Specifies whether to use the one-phase commit feature for transactions that involve only one Region. Compared with the two-phase commit, the one-phase commit feature can greatly reduce the commit latency and increase throughput. -- Default value: `false` - -> **Warning:** -> -> This is still an experimental feature. It is **NOT** recommended that you use it in a production environment. Currently, the following issues are found, and be aware of them if you need to use this feature: -> -> - This feature is incompatible with [TiCDC](/ticdc/ticdc-overview.md) and might cause TiCDC to run abnormally. -> -> - This feature is incompatible with [Follower Read](/follower-read.md) and [TiFlash](/tiflash/tiflash-overview.md), and snapshot isolation cannot be guaranteed. -> -> - External consistency cannot be guaranteed. -> -> - If the transaction commit is interrupted abnormally by the machine crash when the DDL operation is executed, the data format might be incorrect. - -## tikv-client.async-commit - -### `enable` - -- Enables or disables the async commit feature for the second phase of the two-phase transaction commit to perform asynchronously in the background. Enabling this feature can reduce the latency of transaction commit. This feature is not compatible with [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) and does not take effect when Binlog is enabled. -- Default value: `false` - -### `keys-limit` - -- Specifies the upper limit of the number of keys in an async commit transaction. The async commit feature is **NOT** suitable for transactions that are too large. Transactions that exceed this limit will use the two-phase commit. -- Default value: `256` - -### `total-key-size-limit` - -- Specifies the upper limit of the total size of keys in an async commit transaction. The async commit feature is **NOT** suitable for transactions in which the involved key ranges are too long. Transactions that exceed this limit will use the two-phase commit. -- Default value: `4096` -- Unit: byte - -> **Warning:** -> -> This is still an experimental feature. It is **NOT** recommended that you use it in a production environment. Currently, the following issues are found, and be aware of them if you need to use this feature: -> -> - This feature is incompatible with [Follower Read](/follower-read.md) and [TiFlash](/tiflash/tiflash-overview.md), and snapshot isolation cannot be guaranteed. -> -> - External consistency cannot be guaranteed. -> -> - If the transaction commit is interrupted abnormally by the machine crash when the DDL operation is executed, the data format might be incorrect. - ## tikv-client.copr-cache New in v4.0.0 This section introduces configuration items related to the Coprocessor Cache feature. -### `enable` - -- Determines whether to enable [Coprocessor Cache](/coprocessor-cache.md). -- Default value: `false` (which means that Coprocessor Cache is disabled by default) - ### `capacity-mb` -- The total size of the cached data. When the cache space is full, old cache entries are evicted. +- The total size of the cached data. When the cache space is full, old cache entries are evicted. When the value is `0.0`, the Coprocessor Cache feature is disabled. - Default value: `1000.0` - Unit: MB +- Type: Float -### `admission-max-result-mb` - -- Specifies the largest single push-down calculation result set that can be cached. If the result set of a single push-down calculation returned on the Coprocessor is larger than the result set specified by this parameter, the result set is cached. Increasing this value means that more types of push-down requests are cached, but also cause the cache space to be occupied more easily. Note that the size of each push-down calculation result set is generally smaller than the size of the Region. Therefore, it is meaningless to set this value far beyond the size of a Region. -- Default value: `10.0` -- Unit: MB - -### `admission-min-process-ms` - -- Specifies the minimum calculation time for a single push-down calculation result set that can be cached. If the calculation time of a single push-down calculation on the Coprocessor is less than the time specified by this parameter, the result set is not cached. Requests that are processed quickly do not need to be cached, and only the requests that take a long time to process need to be cached, which makes the cache less likely to be evicted. -- Default value: `5` -- Unit: ms - -### txn-local-latches +## txn-local-latches Configuration related to the transaction latch. It is recommended to enable it when many local transaction conflicts occur. @@ -610,6 +621,13 @@ For pessimistic transaction usage, refer to [TiDB Pessimistic Transaction Mode]( - The maximum number of retries of each statement in pessimistic transactions. If the number of retries exceeds this limit, an error occurs. - Default value: `256` +### deadlock-history-capacity + ++ The maximum number of deadlock events that can be recorded in the [`INFORMATION_SCHEMA.DEADLOCKS`](/information-schema/information-schema-deadlocks.md) table of a single TiDB server. If this table is in full volume and an additional deadlock event occurs, the earliest record in the table will be removed to make place for the newest error. ++ Default value: `10` ++ Minimum value: `0` ++ Maximum value: `10000` + ## experimental The `experimental` section, introduced in v3.1.0, describes configurations related to the experimental features of TiDB. @@ -617,4 +635,4 @@ The `experimental` section, introduced in v3.1.0, describes configurations relat ### `allow-expression-index` New in v4.0.0 - Determines whether to create the expression index. -- Default value: false \ No newline at end of file +- Default value: `false` diff --git a/tidb-control.md b/tidb-control.md index 8d7d48cd65e8..706f4ff98d02 100644 --- a/tidb-control.md +++ b/tidb-control.md @@ -12,6 +12,10 @@ TiDB Control is a command-line tool of TiDB, usually used to obtain the status i You can get TiDB Control by installing it using TiUP or by compiling it from source code. +> **Note:** +> +> It is recommended that the version of the Control tool you use is consistent with the version of the cluster. + ### Install TiDB Control using TiUP After installing TiUP, you can use `tiup ctl tidb` command to get and execute TiDB Control. diff --git a/tidb-lightning/deploy-tidb-lightning.md b/tidb-lightning/deploy-tidb-lightning.md index 9e9765382f00..b5c6f9a3f966 100644 --- a/tidb-lightning/deploy-tidb-lightning.md +++ b/tidb-lightning/deploy-tidb-lightning.md @@ -6,15 +6,14 @@ aliases: ['/docs/dev/tidb-lightning/deploy-tidb-lightning/','/docs/dev/reference # TiDB Lightning Deployment -This document describes the hardware requirements of TiDB Lightning using the Local-backend, and how to deploy it using TiDB Ansible or manually. +This document describes the hardware requirements of TiDB Lightning using the Local-backend, and how to deploy it manually. -If you do not want the TiDB services to be impacted, read [TiDB Lightning TiDB-backend](/tidb-lightning/tidb-lightning-backends.md#tidb-lightning-tidb-backend) for the changes to the deployment steps. +If Local-backend is used for data import, during the import process, **the cluster cannot provide services**. If you do not want the TiDB services to be impacted, perform the data import according to [TiDB Lightning TiDB-backend](/tidb-lightning/tidb-lightning-backends.md#tidb-lightning-tidb-backend). ## Notes Before starting TiDB Lightning, note that: -- During the import process, the cluster cannot provide normal services. - If `tidb-lightning` crashes, the cluster is left in "import mode". Forgetting to switch back to "normal mode" can lead to a high amount of uncompacted data on the TiKV cluster, and cause abnormally high CPU usage and stall. You can manually switch the cluster back to "normal mode" via the `tidb-lightning-ctl` tool: ```sh @@ -49,8 +48,7 @@ Before starting TiDB Lightning, note that: > > - `tidb-lightning` is a CPU intensive program. In an environment with mixed components, the resources allocated to `tidb-lightning` must be limited. Otherwise, other components might not be able to run. It is recommended to set the `region-concurrency` to 75% of CPU logical cores. For instance, if the CPU has 32 logical cores, you can set the `region-concurrency` to 24. -Additionally, the target TiKV cluster should have enough space to absorb the new data. -Besides [the standard requirements](/hardware-and-software-requirements.md), the total free space of the target TiKV cluster should be larger than **Size of data source × [Number of replicas](/faq/deploy-and-maintain-faq.md#is-the-number-of-replicas-in-each-region-configurable-if-yes-how-to-configure-it) × 2**. +Additionally, the target TiKV cluster should have enough space to absorb the new data. Besides [the standard requirements](/hardware-and-software-requirements.md), the total free space of the target TiKV cluster should be larger than **Size of data source × [Number of replicas](/faq/deploy-and-maintain-faq.md#is-the-number-of-replicas-in-each-region-configurable-if-yes-how-to-configure-it) × 2**. With the default replica count of 3, this means the total free space should be at least 6 times the size of data source. @@ -73,82 +71,23 @@ If the data source consists of CSV files, see [CSV support](/tidb-lightning/migr ## Deploy TiDB Lightning -This section describes two deployment methods of TiDB Lightning: - -- [Deploy TiDB Lightning using TiDB Ansible](#deploy-tidb-lightning-using-tidb-ansible) -- [Deploy TiDB Lightning manually](#deploy-tidb-lightning-manually) - -### Deploy TiDB Lightning using TiDB Ansible - -You can deploy TiDB Lightning using TiDB Ansible together with the [deployment of the TiDB cluster itself using TiDB Ansible](/online-deployment-using-ansible.md). - -1. Edit `inventory.ini` to configure an IP address for `tidb-lightning`. - - ```ini - ... - - [lightning_server] - 192.168.20.10 - - ... - ``` - -2. Configure `tidb-lightning` by editing the settings under `group_vars/*.yml`. - - * `group_vars/lightning_server.yml` - - ```yaml - --- - dummy: - - # The listening port for metrics gathering. Should be open to the monitoring servers. - tidb_lightning_pprof_port: 8289 - - # The file path that tidb-lightning reads the data source (Dumpling SQL dump or CSV) from. - data_source_dir: "{{ deploy_dir }}/mydumper" - ``` - -3. Deploy the cluster. - - ```sh - ansible-playbook bootstrap.yml && - ansible-playbook deploy.yml - ``` - -4. Mount the data source to the path specified in the `data_source_dir` setting. - -5. Log in to the `tidb-lightning` server and edit the `conf/tidb-lighting.toml` file as follows: - - ``` - [tikv-importer] - # Uses the Local-backend. - backend = "local" - # Sets the directory for temporarily storing the sorted key-value pairs. - # The target directory must be empty. - "sorted-kv-dir" = "/mnt/ssd/sorted-kv-dir" - - [tidb] - # An address of pd-server. - pd-addr = "172.16.31.4:2379 - ``` - -6. Log in to the `tidb-lightning` server, and manually run the following command to start Lightning and import the data into the TiDB cluster. - - ```sh - scripts/start_lightning.sh - ``` +This section describes how to [deploy TiDB Lightning manually](#deploy-tidb-lightning-manually). ### Deploy TiDB Lightning manually #### Step 1: Deploy a TiDB cluster -Before importing data, you need to have a deployed TiDB cluster, with the cluster version 2.0.9 or above. It is highly recommended to use the latest version. +Before importing data, you need to have a deployed TiDB cluster. It is highly recommended to use the latest stable version. You can find deployment instructions in [TiDB Quick Start Guide](/quick-start-with-tidb.md). #### Step 2: Download the TiDB Lightning installation package -Refer to the [TiDB enterprise tools download page](/download-ecosystem-tools.md#tidb-lightning) to download the TiDB Lightning package (choose the same version as that of the TiDB cluster). +Refer to the [TiDB enterprise tools download page](/download-ecosystem-tools.md#tidb-lightning) to download the TiDB Lightning package. + +> **Note:** +> +> TiDB Lightning is compatible with TiDB clusters of earlier versions. It is recommended that you download the latest stable version of the TiDB Lightning installation package. #### Step 3: Start `tidb-lightning` @@ -191,8 +130,7 @@ Refer to the [TiDB enterprise tools download page](/download-ecosystem-tools.md# pd-addr = "172.16.31.4:2379" ``` - The above only shows the essential settings. - See the [Configuration](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-global) section for the full list of settings. + The above only shows the essential settings. See the [Configuration](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-global) section for the full list of settings. 4. Run `tidb-lightning`. diff --git a/tidb-lightning/migrate-from-csv-using-tidb-lightning.md b/tidb-lightning/migrate-from-csv-using-tidb-lightning.md index 2ad8487b2263..a44feeff3c04 100644 --- a/tidb-lightning/migrate-from-csv-using-tidb-lightning.md +++ b/tidb-lightning/migrate-from-csv-using-tidb-lightning.md @@ -6,34 +6,26 @@ aliases: ['/docs/dev/tidb-lightning/migrate-from-csv-using-tidb-lightning/','/do # TiDB Lightning CSV Support -TiDB Lightning supports reading CSV (comma-separated values) data source, as well as other -delimited format such as TSV (tab-separated values). +TiDB Lightning supports reading CSV (comma-separated values) data source, as well as other delimited format such as TSV (tab-separated values). ## File name -A CSV file representing a whole table must be named as `db_name.table_name.csv`. This will be -restored as a table `table_name` inside the database `db_name`. +A CSV file representing a whole table must be named as `db_name.table_name.csv`. This will be restored as a table `table_name` inside the database `db_name`. -If a table spans multiple CSV files, they should be named like `db_name.table_name.003.csv`. -The number part do not need to be continuous, but must be increasing and zero-padded. +If a table spans multiple CSV files, they should be named like `db_name.table_name.003.csv`. The number part do not need to be continuous, but must be increasing and zero-padded. The file extension must be `*.csv`, even if the content is not separated by commas. ## Schema -CSV files are schema-less. To import them into TiDB, a table schema must be provided. This could be -done either by: +CSV files are schema-less. To import them into TiDB, a table schema must be provided. This could be done either by: -* Providing a file named `db_name.table_name-schema.sql` containing the `CREATE TABLE` DDL - statement, and also a file named `db_name-schema-create.sql` containing the `CREATE DATABASE` - DDL statement. -* Creating the empty tables directly in TiDB in the first place, and then setting - `[mydumper] no-schema = true` in `tidb-lightning.toml`. +* Providing a file named `db_name.table_name-schema.sql` containing the `CREATE TABLE` DDL statement, and also a file named `db_name-schema-create.sql` containing the `CREATE DATABASE` DDL statement. +* Creating the empty tables directly in TiDB in the first place, and then setting `[mydumper] no-schema = true` in `tidb-lightning.toml`. ## Configuration -The CSV format can be configured in `tidb-lightning.toml` under the `[mydumper.csv]` section. -Most settings have a corresponding option in the MySQL [`LOAD DATA`] statement. +The CSV format can be configured in `tidb-lightning.toml` under the `[mydumper.csv]` section. Most settings have a corresponding option in the MySQL [`LOAD DATA`] statement. ```toml [mydumper.csv] @@ -47,7 +39,7 @@ header = true # Whether the CSV contains any NULL value. # If `not-null` is true, all columns from CSV cannot be NULL. not-null = false -# When `not-null` is false (i.e. CSV can contain NULL), +# When `not-null` is false (that is, CSV can contain NULL), # fields equal to this value will be treated as NULL. null = '\N' # Whether to interpret backslash escapes inside fields. @@ -85,14 +77,12 @@ trim-last-separator = false ### `header` - Whether *all* CSV files contain a header row. -- If `header` is true, the first row will be used as the -*column names*. If `header` is false, the first row is not special and treated as an ordinary data row. +- If `header` is true, the first row will be used as the *column names*. If `header` is false, the first row is not special and treated as an ordinary data row. ### `not-null` and `null` - The `not-null` setting controls whether all fields are non-nullable. -- If `not-null` is false, the -string specified by `null` will be transformed to the SQL NULL instead of a concrete value. +- If `not-null` is false, the string specified by `null` will be transformed to the SQL NULL instead of a concrete value. - Quoting will not affect whether a field is null. For example, with the CSV file: @@ -102,15 +92,12 @@ string specified by `null` will be transformed to the SQL NULL instead of a conc \N,"\N", ``` - In the default settings (`not-null = false; null = '\N'`), the columns `A` and `B` are both -converted to NULL after importing to TiDB. The column `C` is simply the empty string `''` but not -NULL. + In the default settings (`not-null = false; null = '\N'`), the columns `A` and `B` are both converted to NULL after importing to TiDB. The column `C` is simply the empty string `''` but not NULL. ### `backslash-escape` - Whether to interpret backslash escapes inside fields. -- If `backslash-escape` is true, the following sequences are -recognized and transformed: +- If `backslash-escape` is true, the following sequences are recognized and transformed: | Sequence | Converted to | |----------|--------------------------| @@ -144,38 +131,28 @@ recognized and transformed: TiDB Lightning does not support every option supported by the `LOAD DATA` statement. Some examples: -* The line terminator must only be CR (`\r`), LF (`\n`) or CRLF (`\r\n`), i.e. `LINES TERMINATED BY` - is not customizable. +* The line terminator must only be CR (`\r`), LF (`\n`) or CRLF (`\r\n`), which means `LINES TERMINATED BY` is not customizable. * There cannot be line prefixes (`LINES STARTING BY`). * The header cannot be simply skipped (`IGNORE n LINES`), it must be valid column names if present. * Delimiters and separators can only be a single ASCII character. ## Strict format -Lightning works the best when the input files have uniform size around 256 MB. When the input is a -single huge CSV file, Lightning can only use one thread to process it, which slows down import speed -a lot. +Lightning works the best when the input files have uniform size around 256 MB. When the input is a single huge CSV file, Lightning can only use one thread to process it, which slows down import speed a lot. -This can be fixed by splitting the CSV into multiple files first. For the generic CSV format, there -is no way to quickly identify when a row starts and ends without reading the whole file. Therefore, -Lightning by default does *not* automatically split a CSV file. However, if you are certain that the -CSV input adheres to certain restrictions, you can enable the `strict-format` setting to allow -Lightning to split the file into multiple 256 MB-sized chunks for parallel processing. +This can be fixed by splitting the CSV into multiple files first. For the generic CSV format, there is no way to quickly identify when a row starts and ends without reading the whole file. Therefore, Lightning by default does *not* automatically split a CSV file. However, if you are certain that the CSV input adheres to certain restrictions, you can enable the `strict-format` setting to allow Lightning to split the file into multiple 256 MB-sized chunks for parallel processing. ```toml [mydumper] strict-format = true ``` -Currently, a strict CSV file means every field occupies only a single line. In the other words, one -of the following must be true: +Currently, a strict CSV file means every field occupies only a single line. In the other words, one of the following must be true: * Delimiter is empty, or * Every field does not contain CR (`\r`) or LF (`\n`). -If a CSV file is not strict, but `strict-format` was wrongly set to `true`, a field spanning -multiple lines may be cut in half into two chunks, causing parse failure, or even worse, quietly -importing corrupted data. +If a CSV file is not strict, but `strict-format` was wrongly set to `true`, a field spanning multiple lines may be cut in half into two chunks, causing parse failure, or even worse, quietly importing corrupted data. ## Common configurations diff --git a/tidb-lightning/monitor-tidb-lightning.md b/tidb-lightning/monitor-tidb-lightning.md index b989eb390fbc..4df902b9cf27 100644 --- a/tidb-lightning/monitor-tidb-lightning.md +++ b/tidb-lightning/monitor-tidb-lightning.md @@ -10,8 +10,7 @@ aliases: ['/docs/dev/tidb-lightning/monitor-tidb-lightning/','/docs/dev/referenc ## Monitor configuration -- If TiDB Lightning is installed using TiDB Ansible, simply add the servers to the `[monitored_servers]` section in the `inventory.ini` file. Then the Prometheus server can collect their metrics. -- If TiDB Lightning is manually installed, follow the instructions below. +If TiDB Lightning is manually installed, follow the instructions below. The metrics of `tidb-lightning` can be gathered directly by Prometheus as long as it is discovered. You can set the metrics port in `tidb-lightning.toml`: @@ -47,8 +46,7 @@ scrape_configs: [Grafana](https://grafana.com/) is a web interface to visualize Prometheus metrics as dashboards. -If TiDB Lightning is installed using TiDB Ansible, its dashboard is already installed. -Otherwise, the dashboard JSON can be imported from . +When you [deploy a TiDB cluster using TiUP](/production-deployment-using-tiup.md) and have added Grafana and Prometheus in the topology configuration, a set of [Grafana + Prometheus monitoring platform](/tidb-monitoring-framework.md) is deployed simultaneously. In this situation, you must first import [the JSON file of the dashboard](https://raw.githubusercontent.com/pingcap/tidb-ansible/master/scripts/lightning.json). ### Row 1: Speed @@ -56,7 +54,7 @@ Otherwise, the dashboard JSON can be imported from = v4.0.0 | All | All | +| TiDB services impacted | Yes | Yes | No | ## How to choose the backend modes @@ -46,6 +47,10 @@ To deploy TiDB Lightning in the Local-backend mode, see [TiDB Lightning Deployme ## TiDB Lightning TiDB-backend +> **Note:** +> +> Since TiDB v4.0, PingCAP no longer maintains the [Loader](https://docs.pingcap.com/tidb/v4.0/loader-overview) tool. Since v5.0, the Loader documentation is no longer available. Loader's functionality has been completely replaced by the TiDB-backend of TiDB Lightning, so it is highly recommended to switch to TiDB Lightning. + ### Deployment for TiDB-backend When using the TiDB-backend, deploying `tikv-importer` is not necessary. Compared with the [standard deployment procedure](/tidb-lightning/deploy-tidb-lightning.md), the TiDB-backend deployment has the following two differences: @@ -61,37 +66,6 @@ The speed of TiDB Lightning using TiDB-backend is limited by the SQL processing * An SSD large enough to store the entire data source, preferring higher read speed * 1 Gigabit network card -#### Deploy TiDB Lightning using TiDB Ansible - -1. The `[importer_server]` section in `inventory.ini` can be left blank. - - ```ini - ... - - [importer_server] - # keep empty - - [lightning_server] - 192.168.20.10 - - ... - ``` - -2. The `tikv_importer_port` setting in `group_vars/all.yml` is ignored, and the file `group_vars/importer_server.yml` does not need to be changed. But you need to edit `conf/tidb-lightning.yml` and change the `backend` setting to `tidb`. - - ```yaml - ... - tikv_importer: - backend: "tidb" # <-- change this - ... - ``` - -3. Bootstrap and deploy the cluster as usual. - -4. Mount the data source for TiDB Lightning as usual. - -5. Start `tidb-lightning` as usual. - #### Manual deployment You do not need to download and configure `tikv-importer`. You can download TiDB Lightning from [here](/download-ecosystem-tools.md#tidb-lightning). @@ -123,7 +97,7 @@ on-duplicate = "replace" # or "error" or "ignore" ### Migrating from Loader to TiDB Lightning TiDB-backend -TiDB Lightning using the TiDB-backend can completely replace functions of [Loader](/loader-overview.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/tidb-lightning/tidb-lightning-configuration.md). +If you need to import data into a TiDB cluster, TiDB Lightning using the TiDB-backend can completely replace the functionalities of [Loader](https://docs.pingcap.com/tidb/v4.0/loader-overview). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/tidb-lightning/tidb-lightning-configuration.md).
@@ -132,8 +106,10 @@ TiDB Lightning using the TiDB-backend can completely replace functions of [Loade ```toml -# logging +# log level log-level = "info" + +# The directory to which the log is output log-file = "loader.log" # Prometheus @@ -147,8 +123,10 @@ pool-size = 16 ```toml [lightning] -# logging +# log level level = "info" + +# The directory to which the log is output. If this directory is not specified, it defaults to the directory where the command is executed. file = "tidb-lightning.log" # Prometheus @@ -236,7 +214,9 @@ password = "" # TiDB connection parameters host = "127.0.0.1" port = 4000 -status-port = 10080 # <- this is required + +# In the TiDB-backend mode, this parameter is optional. +# status-port = 10080 user = "root" password = "" @@ -273,10 +253,7 @@ password = "" ### Deployment for Importer-backend mode -This section describes two deployment methods of TiDB Lightning in the Importer-backend mode: - -- [Deploy TiDB Lightning using TiDB Ansible](#deploy-tidb-lightning-using-tidb-ansible) -- [Deploy TiDB Lightning manually](#deploy-tidb-lightning-manually) +This section describes how to [deploy TiDB Lightning manually](#deploy-tidb-lightning-manually) in the Importer-backend mode: #### Hardware requirements @@ -302,81 +279,6 @@ To achieve the best performance, it is recommended to use the following hardware If you have sufficient machines, you can deploy multiple `tidb lightning` + `tikv importer` servers, with each working on a distinct set of tables, to import the data in parallel. -#### Deploy TiDB Lightning using TiDB Ansible - -You can deploy TiDB Lightning using TiDB Ansible together with the [deployment of the TiDB cluster itself using TiDB Ansible](/online-deployment-using-ansible.md). - -1. Edit `inventory.ini` to add the addresses of the `tidb-lightning` and `tikv-importer` servers. - - ```ini - ... - - [importer_server] - 192.168.20.9 - - [lightning_server] - 192.168.20.10 - - ... - ``` - -2. Configure these tools by editing the settings under `group_vars/*.yml`. - - * `group_vars/all.yml` - - ```yaml - ... - # The listening port of tikv-importer. Should be open to the tidb-lightning server. - tikv_importer_port: 8287 - ... - ``` - - * `group_vars/lightning_server.yml` - - ```yaml - --- - dummy: - - # The listening port for metrics gathering. Should be open to the monitoring servers. - tidb_lightning_pprof_port: 8289 - - # The file path that tidb-lightning reads the data source (Mydumper SQL dump or CSV) from. - data_source_dir: "{{ deploy_dir }}/mydumper" - ``` - - * `group_vars/importer_server.yml` - - ```yaml - --- - dummy: - - # The file path to store engine files. Should reside on a partition with a large capacity. - import_dir: "{{ deploy_dir }}/data.import" - ``` - -3. Deploy the cluster. - - ```sh - ansible-playbook bootstrap.yml && - ansible-playbook deploy.yml - ``` - -4. Mount the data source to the path specified in the `data_source_dir` setting. - -5. Log in to the `tikv-importer` server, and manually run the following command to start Importer. - - ```sh - scripts/start_importer.sh - ``` - -6. Log in to the `tidb-lightning` server, and manually run the following command to start Lightning and import the data into the TiDB cluster. - - ```sh - scripts/start_lightning.sh - ``` - -7. After completion, run `scripts/stop_importer.sh` on the `tikv-importer` server to stop Importer. - #### Deploy TiDB Lightning manually ##### Step 1: Deploy a TiDB cluster @@ -461,8 +363,7 @@ Refer to the [TiDB enterprise tools download page](/download-ecosystem-tools.md# status-port = 10080 ``` - The above only shows the essential settings. - See the [Configuration](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-global) section for the full list of settings. + The above only shows the essential settings. See the [Configuration](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-global) section for the full list of settings. 4. Run `tidb-lightning`. If you directly run the command in the command-line, the process might exit because of the SIGHUP signal received. Instead, it's preferable to run a bash script that contains the `nohup` command: diff --git a/tidb-lightning/tidb-lightning-checkpoints.md b/tidb-lightning/tidb-lightning-checkpoints.md index 284ffb23f644..ef6e8e7583a7 100644 --- a/tidb-lightning/tidb-lightning-checkpoints.md +++ b/tidb-lightning/tidb-lightning-checkpoints.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/tidb-lightning/tidb-lightning-checkpoints/','/docs/dev/refe # TiDB Lightning Checkpoints -Importing a large database usually takes hours or days, and if such long running processes spuriously crashes, it can be very time-wasting to redo the previously completed tasks. To solve this, Lightning uses *checkpoints* to store the import progress, so that `tidb-lightning` continues importing from where it lefts off after restarting. +Importing a large database usually takes hours or days, and if such long running processes spuriously crashes, it can be very time-wasting to redo the previously completed tasks. To solve this, TiDB Lightning uses *checkpoints* to store the import progress, so that `tidb-lightning` continues importing from where it lefts off after restarting. This document describes how to enable, configure, store, and control *checkpoints*. @@ -15,8 +15,8 @@ This document describes how to enable, configure, store, and control *checkpoint ```toml [checkpoint] # Whether to enable checkpoints. -# While importing data, Lightning records which tables have been imported, so -# even if Lightning or some other component crashes, you can start from a known +# While importing data, TiDB Lightning records which tables have been imported, so +# even if TiDB Lightning or some other component crashes, you can start from a known # good state instead of redoing everything. enable = true @@ -58,7 +58,7 @@ While using the target database as the checkpoints storage, Lightning is importi ## Checkpoints control -If `tidb-lightning` exits abnormally due to unrecoverable errors (e.g. data corruption), it refuses to reuse the checkpoints until the errors are resolved. This is to prevent worsening the situation. The checkpoint errors can be resolved using the `tidb-lightning-ctl` program. +If `tidb-lightning` exits abnormally due to unrecoverable errors (for example, data corruption), it refuses to reuse the checkpoints until the errors are resolved. This is to prevent worsening the situation. The checkpoint errors can be resolved using the `tidb-lightning-ctl` program. ### `--checkpoint-error-destroy` diff --git a/tidb-lightning/tidb-lightning-configuration.md b/tidb-lightning/tidb-lightning-configuration.md index 242c08fe732e..47975da4d7a1 100644 --- a/tidb-lightning/tidb-lightning-configuration.md +++ b/tidb-lightning/tidb-lightning-configuration.md @@ -75,7 +75,7 @@ io-concurrency = 5 [checkpoint] # Whether to enable checkpoints. # While importing data, TiDB Lightning records which tables have been imported, so -# even if Lightning or another component crashes, you can start from a known +# even if TiDB Lightning or another component crashes, you can start from a known # good state instead of redoing everything. enable = true # The schema name (database name) to store the checkpoints. @@ -85,7 +85,7 @@ schema = "tidb_lightning_checkpoint" # - mysql: store into a remote MySQL-compatible database driver = "file" # The data source name (DSN) indicating the location of the checkpoint storage. -# For the "file" driver, the DSN is a path. If the path is not specified, Lightning would +# For the "file" driver, the DSN is a path. If the path is not specified, TiDB Lightning would # default to "/tmp/CHECKPOINT_SCHEMA.pb". # For the "mysql" driver, the DSN is a URL in the form of "USER:PASS@tcp(HOST:PORT)/". # If the URL is not specified, the TiDB server from the [tidb] section is used to @@ -132,7 +132,7 @@ read-block-size = 65536 # Byte (default = 64 KB) # The engine file needs to be imported sequentially. Due to parallel processing, # multiple data engines will be imported at nearly the same time, and this -# creates a queue and wastes resources. Therefore, Lightning slightly +# creates a queue and wastes resources. Therefore, TiDB Lightning slightly # increases the size of the first few batches to properly distribute # resources. The scale up factor is controlled by this parameter, which # expresses the ratio of duration between the "import" and "write" steps @@ -143,7 +143,7 @@ read-block-size = 65536 # Byte (default = 64 KB) # This value should be in the range (0 <= batch-import-ratio < 1). batch-import-ratio = 0.75 -# Local source data directory. +# Local source data directory or the URL of the external storage. data-source-dir = "/data/my_database" # If no-schema is set to true, tidb-lightning assumes that the table skeletons # already exist on the target TiDB cluster, and will not execute the `CREATE @@ -151,10 +151,9 @@ data-source-dir = "/data/my_database" no-schema = false # The character set of the schema files, containing CREATE TABLE statements; # only supports one of: -# - utf8mb4: the schema files must be encoded as UTF-8, otherwise Lightning -# will emit errors -# - gb18030: the schema files must be encoded as GB-18030, otherwise -# Lightning will emit errors +# - utf8mb4: the schema files must be encoded as UTF-8; otherwise, an error is reported. +# - gb18030: the schema files must be encoded as GB-18030; otherwise, +# an error is reported # - auto: (default) automatically detects whether the schema is UTF-8 or # GB-18030. An error is reported if the encoding is neither. # - binary: do not try to decode the schema files @@ -165,14 +164,14 @@ character-set = "auto" # Assumes the input data are "strict" to speed up processing. # Implications of strict-format = true are: # * in CSV, every value cannot contain literal new lines (U+000A and U+000D, or \r and \n) even -# when quoted, i.e. new lines are strictly used to separate rows. -# Strict format allows Lightning to quickly locate split positions of a large file for parallel +# when quoted, which means new lines are strictly used to separate rows. +# Strict format allows TiDB Lightning to quickly locate split positions of a large file for parallel # processing. However, if the input data is not strict, it may split a valid data in half and # corrupt the result. # The default value is false for safety over speed. strict-format = false -# If strict-format is true, Lightning will split large CSV files into multiple chunks to process in +# If strict-format is true, TiDB Lightning will split large CSV files into multiple chunks to process in # parallel. max-region-size is the maximum size of each chunk after splitting. # max-region-size = 268_435_456 # Byte (default = 256 MB) @@ -191,7 +190,7 @@ header = true # Whether the CSV contains any NULL value. # If `not-null` is true, all columns from CSV cannot be NULL. not-null = false -# When `not-null` is false (i.e. CSV can contain NULL), +# When `not-null` is false (that is, CSV can contain NULL), # fields equal to this value will be treated as NULL. null = '\N' # Whether to interpret backslash escapes inside fields. @@ -266,7 +265,7 @@ analyze = true # Configures the background periodic actions. # Supported units: h (hour), m (minute), s (second). [cron] -# Duration between which Lightning automatically refreshes the import mode +# Duration between which TiDB Lightning automatically refreshes the import mode # status. Should be shorter than the corresponding TiKV setting. switch-mode = "5m" # Duration between which an import progress is printed to the log. @@ -360,7 +359,7 @@ min-available-ratio = 0.05 |:----|:----|:----| | --config *file* | Reads global configuration from *file*. If not specified, the default configuration would be used. | | | -V | Prints program version | | -| -d *directory* | Directory of the data dump to read from | `mydumper.data-source-dir` | +| -d *directory* | Directory or [external storage URL](/br/backup-and-restore-storages.md) of the data dump to read from | `mydumper.data-source-dir` | | -L *level* | Log level: debug, info, warn, error, fatal (default = info) | `lightning.log-level` | | -f *rule* | [Table filter rules](/table-filter.md) (can be specified multiple times) | `mydumper.filter` | | --backend *backend* | [Delivery backend](/tidb-lightning/tidb-lightning-backends.md) (`importer`, `local`, or `tidb`) | `tikv-importer.backend` | @@ -381,7 +380,7 @@ min-available-ratio = 0.05 | --ca *file* | CA certificate path for TLS connection | `security.ca-path` | | --cert *file* | Certificate path for TLS connection | `security.cert-path` | | --key *file* | Private key path for TLS connection | `security.key-path` | -| --server-mode | Start Lightning in server mode | `lightning.server-mode` | +| --server-mode | Start TiDB Lightning in server mode | `lightning.server-mode` | If a command line parameter and the corresponding setting in the configuration file are both provided, the command line parameter will be used. For example, running `./tidb-lightning -L debug --config cfg.toml` would always set the log level to "debug" regardless of the content of `cfg.toml`. diff --git a/tidb-lightning/tidb-lightning-faq.md b/tidb-lightning/tidb-lightning-faq.md index d7951e6a85e1..1033900cf127 100644 --- a/tidb-lightning/tidb-lightning-faq.md +++ b/tidb-lightning/tidb-lightning-faq.md @@ -1,16 +1,12 @@ --- title: TiDB Lightning FAQs summary: Learn about the frequently asked questions (FAQs) and answers about TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-faq/','/docs/dev/faq/tidb-lightning/'] +aliases: ['/docs/dev/tidb-lightning/tidb-lightning-faq/','/docs/dev/faq/tidb-lightning/','/docs/dev/troubleshoot-tidb-lightning/','/docs/dev/how-to/troubleshoot/tidb-lightning/','/docs/dev/tidb-lightning/tidb-lightning-misuse-handling/','/docs/dev/reference/tools/error-case-handling/lightning-misuse-handling/','/tidb/dev/tidb-lightning-misuse-handling','/tidb/dev/troubleshoot-tidb-lightning'] --- # TiDB Lightning FAQs ->**Note:** -> -> See also [TiDB Lightning Troubleshooting](/troubleshoot-tidb-lightning.md) if you have encountered errors. - -## What is the minimum TiDB/TiKV/PD cluster version supported by Lightning? +## What is the minimum TiDB/TiKV/PD cluster version supported by TiDB Lightning? The version of TiDB Lightning should be the same as the cluster. If you use the Local-backend mode, the earliest available version is 4.0.0. If you use the Importer-backend mode or the TiDB-backend mode, the earliest available version is 2.0.9, but it is recommended to use the 3.0 stable version. @@ -43,7 +39,7 @@ If only one table has an error encountered, the rest will still be processed nor ## How to properly restart TiDB Lightning? -Depending on the status of `tikv-importer`, the basic sequence of restarting TiDB Lightning is like this: +If you are using Importer-backend, depending on the status of `tikv-importer`, the basic sequence of restarting TiDB Lightning is like this: If `tikv-importer` is still running: @@ -60,9 +56,11 @@ If `tikv-importer` needs to be restarted: 4. Start `tikv-importer`. 5. Start `tidb-lightning` *and wait until the program fails with CHECKSUM error, if any*. * Restarting `tikv-importer` would destroy all engine files still being written, but `tidb-lightning` did not know about it. As of v3.0 the simplest way is to let `tidb-lightning` go on and retry. -6. [Destroy the failed tables and checkpoints](/troubleshoot-tidb-lightning.md#checkpoint-for--has-invalid-status-error-code) +6. [Destroy the failed tables and checkpoints](#checkpoint-for--has-invalid-status-error-code) 7. Start `tidb-lightning` again. +If you are using Local-backend or TiDB-backend, the operations are the same as those of using Importer-backend when the `tikv-importer` is still running. + ## How to ensure the integrity of the imported data? TiDB Lightning by default performs checksum on the local data source and the imported tables. If there is checksum mismatch, the process would be aborted. These checksum information can be read from the log. @@ -82,14 +80,13 @@ ADMIN CHECKSUM TABLE `schema`.`table`; 1 row in set (0.01 sec) ``` -## What kind of data source format is supported by Lightning? +## What kind of data source format is supported by TiDB Lightning? TiDB Lightning only supports the SQL dump generated by [Dumpling](/dumpling-overview.md) or [CSV files](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) stored in the local file system. ## Could TiDB Lightning skip creating schema and tables? -Yes. If you have already created the tables in the target database, you could set `no-schema = true` in the `[mydumper]` section in `tidb-lightning.toml`. This makes TiDB Lightning skip the -`CREATE TABLE` invocations and fetch the metadata directly from the target database. TiDB Lightning will exit with error if a table is actually missing. +Yes. If you have already created the tables in the target database, you could set `no-schema = true` in the `[mydumper]` section in `tidb-lightning.toml`. This makes TiDB Lightning skip the `CREATE TABLE` invocations and fetch the metadata directly from the target database. TiDB Lightning will exit with error if a table is actually missing. ## Can the Strict SQL Mode be disabled to allow importing invalid data? @@ -110,16 +107,12 @@ Yes, as long as every `tidb-lightning` instance operates on different tables. To stop the `tikv-importer` process, you can choose the corresponding operation according to your deployment method. -- For deployment using TiDB Ansible: run `scripts/stop_importer.sh` on the Importer server. - - For manual deployment: if `tikv-importer` is running in foreground, press Ctrl+C to exit. Otherwise, obtain the process ID using the `ps aux | grep tikv-importer` command and then terminate the process using the `kill «pid»` command. ## How to stop the `tidb-lightning` process? To stop the `tidb-lightning` process, you can choose the corresponding operation according to your deployment method. -- For deployment using TiDB Ansible: run `scripts/stop_lightning.sh` on the Lightning server. - - For manual deployment: if `tidb-lightning` is running in foreground, press Ctrl+C to exit. Otherwise, obtain the process ID using the `ps aux | grep tidb-lighting` command and then terminate the process using the `kill -2 «pid»` command. ## Why the `tidb-lightning` process suddenly quits while running in background? @@ -132,6 +125,8 @@ It is potentially caused by starting `tidb-lightning` incorrectly, which causes It is not recommended to directly use `nohup` in the command line to start `tidb-lightning`. You can [start `tidb-lightning`](/tidb-lightning/deploy-tidb-lightning.md#step-3-start-tidb-lightning) by executing a script. +In addition, if the last log of TiDB Lightning shows that the error is "Context canceled", you need to search for the first "ERROR" level log. This "ERROR" level log is usually followed by "got signal to exit", which indicates that TiDB Lightning received an interrupt signal and then exited. + ## Why my TiDB cluster is using lots of CPU resources and running very slowly after using TiDB Lightning? If `tidb-lightning` abnormally exited, the cluster might be stuck in the "import mode", which is not suitable for production. The current mode can be retrieved using the following command: @@ -171,7 +166,7 @@ With the default settings of 3 replicas, the space requirement of the target TiK ## Can TiKV Importer be restarted while TiDB Lightning is running? -No. Importer stores some information of engines in memory. If `tikv-importer` is restarted, `tidb-lightning` will be stopped due to lost connection. At this point, you need to [destroy the failed checkpoints](/tidb-lightning/tidb-lightning-checkpoints.md#--checkpoint-error-destroy) as those Importer-specific information is lost. You can restart Lightning afterwards. +No. TiKV Importer stores some information of engines in memory. If `tikv-importer` is restarted, `tidb-lightning` will be stopped due to lost connection. At this point, you need to [destroy the failed checkpoints](/tidb-lightning/tidb-lightning-checkpoints.md#--checkpoint-error-destroy) as those TiKV Importer-specific information is lost. You can restart TiDB Lightning afterwards. See also [How to properly restart TiDB Lightning?](#how-to-properly-restart-tidb-lightning) for the correct sequence. @@ -187,6 +182,169 @@ See also [How to properly restart TiDB Lightning?](#how-to-properly-restart-tidb If, for some reason, you cannot run this command, try manually deleting the file `/tmp/tidb_lightning_checkpoint.pb`. -2. Delete the entire "import" directory on the machine hosting `tikv-importer`. +2. If you are using Local-backend, delete the `sorted-kv-dir` directory in the configuration. If you are using Importer-backend, delete the entire `import` directory on the machine hosting `tikv-importer`. 3. Delete all tables and databases created on the TiDB cluster, if needed. + +## Why does TiDB Lightning report the `could not find first pair, this shouldn't happen` error? + +This error occurs possibly because the number of files opened by TiDB Lightning exceeds the system limit when TiDB Lightning reads the sorted local files. In the Linux system, you can use the `ulimit -n` command to confirm whether the value of this system limit is too small. It is recommended that you adjust this value to `1000000` (`ulimit -n 1000000`) during the import. + +## Import speed is too slow + +Normally it takes TiDB Lightning 2 minutes per thread to import a 256 MB data file. If the speed is much slower than this, there is an error. You can check the time taken for each data file from the log mentioning `restore chunk … takes`. This can also be observed from metrics on Grafana. + +There are several reasons why TiDB Lightning becomes slow: + +**Cause 1**: `region-concurrency` is set too high, which causes thread contention and reduces performance. + +1. The setting can be found from the start of the log by searching `region-concurrency`. +2. If TiDB Lightning shares the same machine with other services (for example, TiKV Importer), `region-concurrency` must be **manually** set to 75% of the total number of CPU cores. +3. If there is a quota on CPU (for example, limited by Kubernetes settings), TiDB Lightning may not be able to read this out. In this case, `region-concurrency` must also be **manually** reduced. + +**Cause 2**: The table schema is too complex. + +Every additional index introduces a new KV pair for each row. If there are N indices, the actual size to be imported would be approximately (N+1) times the size of the Dumpling output. If the indices are negligible, you may first remove them from the schema, and add them back using `CREATE INDEX` after the import is complete. + +**Cause 3**: Each file is too large. + +TiDB Lightning works the best when the data source is broken down into multiple files of size around 256 MB so that the data can be processed in parallel. If each file is too large, TiDB Lightning might not respond. + +If the data source is CSV, and all CSV files have no fields containing newline control characters (U+000A and U+000D), you can turn on "strict format" to let TiDB Lightning automatically split the large files. + +```toml +[mydumper] +strict-format = true +``` + +**Cause 4**: TiDB Lightning is too old. + +Try the latest version! Maybe there is new speed improvement. + +## `checksum failed: checksum mismatched remote vs local` + +**Cause**: The checksum of a table in the local data source and the remote imported database differ. This error has several deeper reasons: + +1. The table might already have data before. These old data can affect the final checksum. + +2. If the remote checksum is 0, which means nothing is imported, it is possible that the cluster is too hot and fails to take in any data. + +3. If the data is mechanically generated, ensure it respects the constrains of the table: + + * `AUTO_INCREMENT` columns need to be positive, and do not contain the value "0". + * The UNIQUE and PRIMARY KEYs must have no duplicated entries. + +4. If TiDB Lightning has failed before and was not properly restarted, a checksum mismatch may happen due to data being out-of-sync. + +**Solutions**: + +1. Delete the corrupted data using `tidb-lightning-ctl`, and restart TiDB Lightning to import the affected tables again. + + {{< copyable "shell-regular" >}} + + ```sh + tidb-lightning-ctl --config conf/tidb-lightning.toml --checkpoint-error-destroy=all + ``` + +2. Consider using an external database to store the checkpoints (change `[checkpoint] dsn`) to reduce the target database's load. + +3. If TiDB Lightning was improperly restarted, see also the "[How to properly restart TiDB Lightning](#how-to-properly-restart-tidb-lightning)" section in the FAQ. + +## `Checkpoint for … has invalid status:` (error code) + +**Cause**: [Checkpoint](/tidb-lightning/tidb-lightning-checkpoints.md) is enabled, and TiDB Lightning or TiKV Importer has previously abnormally exited. To prevent accidental data corruption, TiDB Lightning will not start until the error is addressed. + +The error code is an integer smaller than 25, with possible values of 0, 3, 6, 9, 12, 14, 15, 17, 18, 20, and 21. The integer indicates the step where the unexpected exit occurs in the import process. The larger the integer is, the later step the exit occurs at. + +**Solutions**: + +If the error was caused by invalid data source, delete the imported data using `tidb-lightning-ctl` and start Lightning again. + +```sh +tidb-lightning-ctl --config conf/tidb-lightning.toml --checkpoint-error-destroy=all +``` + +See the [Checkpoints control](/tidb-lightning/tidb-lightning-checkpoints.md#checkpoints-control) section for other options. + +## `ResourceTemporarilyUnavailable("Too many open engines …: …")` + +**Cause**: The number of concurrent engine files exceeds the limit specified by `tikv-importer`. This could be caused by misconfiguration. Additionally, if `tidb-lightning` exited abnormally, an engine file might be left at a dangling open state, which could cause this error as well. + +**Solutions**: + +1. Increase the value of `max-open-engines` setting in `tikv-importer.toml`. This value is typically dictated by the available memory. This could be calculated by using: + + Max Memory Usage ≈ `max-open-engines` × `write-buffer-size` × `max-write-buffer-number` + +2. Decrease the value of `table-concurrency` + `index-concurrency` so it is less than `max-open-engines`. + +3. Restart `tikv-importer` to forcefully remove all engine files (default to `./data.import/`). This also removes all partially imported tables, which requires TiDB Lightning to clear the outdated checkpoints. + + ```sh + tidb-lightning-ctl --config conf/tidb-lightning.toml --checkpoint-error-destroy=all + ``` + +## `cannot guess encoding for input file, please convert to UTF-8 manually` + +**Cause**: TiDB Lightning only recognizes the UTF-8 and GB-18030 encodings for the table schemas. This error is emitted if the file isn't in any of these encodings. It is also possible that the file has mixed encoding, such as containing a string in UTF-8 and another string in GB-18030, due to historical `ALTER TABLE` executions. + +**Solutions**: + +1. Fix the schema so that the file is entirely in either UTF-8 or GB-18030. + +2. Manually `CREATE` the affected tables in the target database, and then set `[mydumper] no-schema = true` to skip automatic table creation. + +3. Set `[mydumper] character-set = "binary"` to skip the check. Note that this might introduce mojibake into the target database. + +## `[sql2kv] sql encode error = [types:1292]invalid time format: '{1970 1 1 …}'` + +**Cause**: A table contains a column with the `timestamp` type, but the time value itself does not exist. This is either because of DST changes or the time value has exceeded the supported range (Jan 1, 1970 to Jan 19, 2038). + +**Solutions**: + +1. Ensure TiDB Lightning and the source database are using the same time zone. + + When executing TiDB Lightning directly, the time zone can be forced using the `$TZ` environment variable. + + ```sh + # Manual deployment, and force Asia/Shanghai. + TZ='Asia/Shanghai' bin/tidb-lightning -config tidb-lightning.toml + ``` + +2. When exporting data using Mydumper, make sure to include the `--skip-tz-utc` flag. + +3. Ensure the entire cluster is using the same and latest version of `tzdata` (version 2018i or above). + + On CentOS, run `yum info tzdata` to check the installed version and whether there is an update. Run `yum upgrade tzdata` to upgrade the package. + +## `[Error 8025: entry too large, the max entry size is 6291456]` + +**Cause**: A single row of key-value pairs generated by TiDB Lightning exceeds the limit set by TiDB. + +**Solution**: + +Currently, the limitation of TiDB cannot be bypassed. You can only ignore this table to ensure the successful import of other tables. + +## Encounter `rpc error: code = Unimplemented ...` when TiDB Lightning switches the mode + +**Cause**: Some node(s) in the cluster does not support `switch-mode`. For example, if the TiFlash version is earlier than `v4.0.0-rc.2`, [`switch-mode` is not supported](https://github.com/pingcap/tidb-lightning/issues/273). + +**Solutions**: + +- If there are TiFlash nodes in the cluster, you can update the cluster to `v4.0.0-rc.2` or higher versions. +- Temporarily disable TiFlash if you do not want to upgrade the cluster. + +## `tidb lightning encountered error: TiDB version too old, expected '>=4.0.0', found '3.0.18'` + +TiDB Lightning Local-backend only supports importing data to TiDB clusters of v4.0.0 and later versions. If you try to use Local-backend to import data to a v2.x or v3.x cluster, the above error is reported. At this time, you can modify the configuration to use Importer-backend or TiDB-backend for data import. + +Some `nightly` versions might be similar to v4.0.0-beta.2. These `nightly` versions of TiDB Lightning actually support Local-backend. If you encounter this error when using a `nightly` version, you can skip the version check by setting the configuration `check-requirements = false`. Before setting this parameter, make sure that the configuration of TiDB Lightning supports the corresponding version; otherwise, the import might fail. + +## `restore table test.district failed: unknown columns in header [...]` + +This error occurs usually because the CSV data file does not contain a header (the first row is not column names but data). Therefore, you need to add the following configuration to the TiDB Lightning configuration file: + +``` +[mydumper.csv] +header = false +``` diff --git a/tidb-lightning/tidb-lightning-glossary.md b/tidb-lightning/tidb-lightning-glossary.md index f5bdd7b75343..80c423c24551 100644 --- a/tidb-lightning/tidb-lightning-glossary.md +++ b/tidb-lightning/tidb-lightning-glossary.md @@ -14,7 +14,7 @@ This page explains the special terms used in TiDB Lightning's logs, monitoring, ### Analyze -An operation to rebuild the [statistics](/statistics.md) information of a TiDB table, i.e. running the [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) statement. +An operation to rebuild the [statistics](/statistics.md) information of a TiDB table, which is running the [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) statement. Because TiDB Lightning imports data without going through TiDB, the statistics information is not automatically updated. Therefore, TiDB Lightning explicitly analyzes every table after importing. This step can be omitted by setting the `post-restore.analyze` configuration to `false`. @@ -52,13 +52,13 @@ In TiDB Lightning, the checksum of a table is a set of 3 numbers calculated from TiDB Lightning [validates the imported data](/tidb-lightning/tidb-lightning-faq.md#how-to-ensure-the-integrity-of-the-imported-data) by comparing the [local](/tidb-lightning/tidb-lightning-glossary.md#local-checksum) and [remote checksums](/tidb-lightning/tidb-lightning-glossary.md#remote-checksum) of every table. The program would stop if any pair does not match. You can skip this check by setting the `post-restore.checksum` configuration to `false`. -See also the [Troubleshooting guide](/troubleshoot-tidb-lightning.md#checksum-failed-checksum-mismatched-remote-vs-local) for how to properly handle checksum mismatch. +See also the [FAQs](/tidb-lightning/tidb-lightning-faq.md#checksum-failed-checksum-mismatched-remote-vs-local) for how to properly handle checksum mismatch. ### Chunk A continuous range of source data, normally equivalent to a single file in the data source. -When a file is too large, Lightning may split a file into multiple chunks. +When a file is too large, TiDB Lightning might split a file into multiple chunks. ### Compaction diff --git a/tidb-lightning/tidb-lightning-overview.md b/tidb-lightning/tidb-lightning-overview.md index 89c7600083ce..94a0c0676c5d 100644 --- a/tidb-lightning/tidb-lightning-overview.md +++ b/tidb-lightning/tidb-lightning-overview.md @@ -8,11 +8,16 @@ aliases: ['/docs/dev/tidb-lightning/tidb-lightning-overview/','/docs/dev/referen [TiDB Lightning](https://github.com/pingcap/tidb-lightning) is a tool used for fast full import of large amounts of data into a TiDB cluster. You can download TiDB Lightning from [here](/download-ecosystem-tools.md#tidb-lightning). -Currently, TiDB Lightning supports reading SQL dump exported via Dumpling or CSV data source. You can use it in the following two scenarios: +Currently, TiDB Lightning can mainly be used in the following two scenarios: - Importing **large amounts** of **new** data **quickly** - Restore all backup data +Currently, TiDB Lightning supports: + +- The data source of the [Dumpling](/dumpling-overview.md), CSV or [Amazon Aurora Parquet](/migrate-from-aurora-using-lightning.md) exported formats. +- Reading data from a local disk or from the Amazon S3 storage. For details, see [External Storages](/br/backup-and-restore-storages.md). + ## TiDB Lightning architecture ![Architecture of TiDB Lightning tool set](/media/tidb-lightning-architecture.png) @@ -39,4 +44,10 @@ The complete import process is as follows: If the target cluster of data import is v3.x or earlier versions, you need to use the Importer-backend to import data. In this mode, `tidb-lightning` sends the parsed KV pairs to `tikv-importer` via gRPC and `tikv-importer` imports the data. -TiDB Lightning also supports using TiDB-backend for data import. In this mode, `tidb-lightning` transforms data into `INSERT` SQL statements and directly executes them on the target cluster, which is similar to Loader's operations. See [TiDB Lightning Backends](/tidb-lightning/tidb-lightning-backends.md) for details. +TiDB Lightning also supports using TiDB-backend for data import. In this mode, `tidb-lightning` transforms data into `INSERT` SQL statements and directly executes them on the target cluster. See [TiDB Lightning Backends](/tidb-lightning/tidb-lightning-backends.md) for details. + +## Restrictions + +If you use TiDB Lighting together with TiFlash: + +No matter a table has TiFlash replica(s) or not, you can import data to that table using TiDB Lightning. Note that this might slow the TiDB Lightning procedure, which depends on the NIC bandwidth on the lightning host, the CPU and disk load of the TiFlash node, and the number of TiFlash replicas. diff --git a/tidb-lightning/tidb-lightning-web-interface.md b/tidb-lightning/tidb-lightning-web-interface.md index 4b4cf9574850..bc0f5765a0c3 100644 --- a/tidb-lightning/tidb-lightning-web-interface.md +++ b/tidb-lightning/tidb-lightning-web-interface.md @@ -55,11 +55,9 @@ Click the **+** button on the title bar to submit a task. ![Submit task dialog](/media/lightning-web-submit.png) -Tasks are TOML files described as [task configurations](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task). -One could also open a local TOML file by clicking **UPLOAD**. +Tasks are TOML files described as [task configurations](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task). One could also open a local TOML file by clicking **UPLOAD**. -Click **SUBMIT** to run the task. -If a task is already running, the new task will be queued and executed after the current task succeeds. +Click **SUBMIT** to run the task. If a task is already running, the new task will be queued and executed after the current task succeeds. ## Table progress @@ -77,8 +75,6 @@ Click the **ⓘ** button on the title bar to manage the current and queued tasks ![Task management page](/media/lightning-web-queue.png) -Each task is labeled by the time it was submitted. -Clicking the task would show the configuration formatted as JSON. +Each task is labeled by the time it was submitted. Clicking the task would show the configuration formatted as JSON. -Manage tasks by clicking the **⋮** button next to a task. -You can stop a task immediately, or reorder queued tasks. +Manage tasks by clicking the **⋮** button next to a task. You can stop a task immediately, or reorder queued tasks. diff --git a/tidb-limitations.md b/tidb-limitations.md index 9b9d64a90094..86e4cae006e9 100644 --- a/tidb-limitations.md +++ b/tidb-limitations.md @@ -36,19 +36,22 @@ This document describes the common usage limitations of TiDB, including the maxi ## Limitations on a single table -| Type | Upper limit | +| Type | Upper limit (default value) | |:----------|:----------| -| Columns | 512 | -| Indexes | 64 | +| Columns | Defaults to 1071 and can be adjusted up to 4096 | +| Indexes | Defaults to 64 and can be adjusted up to 512 | | Rows | unlimited | | Size | unlimited | | Partitions | 1024 | +* The upper limit of `Columns` can be modified via [`table-column-count-limit`](/tidb-configuration-file.md#table-column-count-limit-new-in-v50). +* The upper limit of `Indexes` can be modified via [`index-limit`](/tidb-configuration-file.md#index-limit-new-in-v50). + ## Limitation on a single row | Type | Upper limit | |:----------|:----------| -| Size | 6 MB | +| Size | 6 MB by default. You can adjust the size limit via the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration item. | ## Limitation on a single column diff --git a/tidb-monitoring-api.md b/tidb-monitoring-api.md index 5054d582d4bc..f5e6313b60b2 100644 --- a/tidb-monitoring-api.md +++ b/tidb-monitoring-api.md @@ -75,7 +75,6 @@ curl http://127.0.0.1:2379/pd/api/v1/stores The metrics interface monitors the state and performance of the entire TiDB cluster. -- If you use TiDB Ansible to deploy the TiDB cluster, the monitoring system (Prometheus and Grafana) is deployed at the same time. - If you use other deployment ways, [deploy Prometheus and Grafana](/deploy-monitoring-services.md) before using this interface. After Prometheus and Grafana are successfully deployed, [configure Grafana](/deploy-monitoring-services.md#configure-grafana). diff --git a/tidb-monitoring-framework.md b/tidb-monitoring-framework.md index d9ff2b646512..30a2cbe8639d 100644 --- a/tidb-monitoring-framework.md +++ b/tidb-monitoring-framework.md @@ -46,6 +46,7 @@ Grafana is an open source project for analyzing and visualizing metrics. TiDB us - {TiDB_Cluster_name}-TiKV-Details: Detailed monitoring metrics related to the TiKV server. - {TiDB_Cluster_name}-TiKV-Summary: Monitoring overview related to the TiKV server. - {TiDB_Cluster_name}-TiKV-Trouble-Shooting: Monitoring metrics related to the TiKV error diagnostics. +- {TiDB_Cluster_name}-TiCDC：Detailed monitoring metrics related to TiCDC. Each group has multiple panel labels of monitoring metrics, and each panel contains detailed information of multiple monitoring metrics. For example, the **Overview** monitoring group has five panel labels, and each labels corresponds to a monitoring panel. See the following UI: diff --git a/tidb-scheduling.md b/tidb-scheduling.md index 84e9cae9f222..a011bdbb3030 100644 --- a/tidb-scheduling.md +++ b/tidb-scheduling.md @@ -5,7 +5,7 @@ summary: Introduces the PD scheduling component in a TiDB cluster. # TiDB Scheduling -PD works as the manager in a TiDB cluster, and it also schedules Regions in the cluster. This article introduces the design and core concepts of the PD scheduling component. +The Placement Driver ([PD](https://github.com/tikv/pd)) works as the manager in a TiDB cluster, and it also schedules Regions in the cluster. This article introduces the design and core concepts of the PD scheduling component. ## Scheduling situations @@ -18,11 +18,11 @@ Now consider about the following situations: * When a new TiKV store is added, data can be rebalanced to it; * When a TiKV store fails, PD needs to consider: * Recovery time of the failed store. - * If it's short (e.g. the service is restarted), whether scheduling is necessary or not. - * If it's long (e.g. disk fault, data is lost, etc.), how to do scheduling. + * If it's short (for example, the service is restarted), whether scheduling is necessary or not. + * If it's long (for example, disk fault, data is lost, etc.), how to do scheduling. * Replicas of all Regions. * If the number of replicas is not enough for some Regions, PD needs to complete them. - * If the number of replicas is more than expected (e.g. the failed store re-joins into the cluster after recovery), PD needs to delete them. + * If the number of replicas is more than expected (for example, the failed store re-joins into the cluster after recovery), PD needs to delete them. * Read/Write operations are performed on leaders, which can not be distributed only on a few individual stores; * Not all Regions are hot, so loads of all TiKV stores need to be balanced; * When Regions are in balancing, data transferring utilizes much network/disk traffic and CPU time, which can influence online services. @@ -67,7 +67,7 @@ Scheduling is based on information collection. In short, the PD scheduling compo - State information reported by each TiKV peer: - Each TiKV peer sends heartbeats to PD periodically. PD not only checks whether the store is alive, but also collects [`StoreState`](https://github.com/pingcap/kvproto/blob/release-3.1/proto/pdpb.proto#L421) in the heartbeat message. `StoreState` includes: + Each TiKV peer sends heartbeats to PD periodically. PD not only checks whether the store is alive, but also collects [`StoreState`](https://github.com/pingcap/kvproto/blob/master/proto/pdpb.proto#L473) in the heartbeat message. `StoreState` includes: * Total disk space * Available disk space @@ -79,7 +79,7 @@ Scheduling is based on information collection. In short, the PD scheduling compo - Information reported by Region leaders: - Each Region leader sends heartbeats to PD periodically to report [`RegionState`](https://github.com/pingcap/kvproto/blob/release-3.1/proto/pdpb.proto#L271), including: + Each Region leader sends heartbeats to PD periodically to report [`RegionState`](https://github.com/pingcap/kvproto/blob/master/proto/pdpb.proto#L312), including: * Position of the leader itself * Positions of other replicas diff --git a/tidb-storage.md b/tidb-storage.md index 1908bad7bc0f..65bf03689008 100644 --- a/tidb-storage.md +++ b/tidb-storage.md @@ -1,11 +1,11 @@ --- -title: Storage of TiDB database -summary: Understand the storage layer of a TiDB database +title: TiDB Storage +summary: Understand the storage layer of a TiDB database. --- -# Storage of TiDB database +# TiDB Storage -This document introduces some design ideas and key concepts of TiKV. +This document introduces some design ideas and key concepts of [TiKV](https://github.com/tikv/tikv). ![storage-architecture](/media/tidb-storage-architecture.png) diff --git a/tidb-troubleshooting-map.md b/tidb-troubleshooting-map.md index b5ad72139f14..c50760a7a819 100644 --- a/tidb-troubleshooting-map.md +++ b/tidb-troubleshooting-map.md @@ -129,7 +129,7 @@ Refer to [5 PD issues](#5-pd-issues). > **Note:** > - > The default threshold for a single SQL memory usage is `32GB` (in bytes, scope:`SESSION`). You can set this parameter by configuring `tidb_mem_quota_query`. You can also modify the `mem-quota-query` item (in bytes) in the configuration file by hot loading the configuration items. + > The default threshold for a single SQL memory usage is `1GB` (in bytes, scope:`SESSION`). You can set this parameter by configuring `tidb_mem_quota_query`. You can also modify the `mem-quota-query` item (in bytes) in the configuration file by hot loading the configuration items. - 3.2.3 Mitigate OOM issues @@ -163,7 +163,7 @@ Refer to [5 PD issues](#5-pd-issues). - For v3.0 and later versions, use the `SQL Bind` feature to bind the execution plan. - - Update the statistics. If you are roughly sure that the problem is caused by the statistics, [dump the statistics](/statistics.md#export-statistics). If the cause is outdated statistics, such as the `modify count/row count` in `show stats_meta` is greater than a certain value (e.g. 0.3), or the table has an index of time column, you can try recovering by using `analyze table`. If `auto analyze` is configured, check whether the `tidb_auto_analyze_ratio` system variable is too large (e.g. > 0.3), and whether the current time is between `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time`. + - Update the statistics. If you are roughly sure that the problem is caused by the statistics, [dump the statistics](/statistics.md#export-statistics). If the cause is outdated statistics, such as the `modify count/row count` in `show stats_meta` is greater than a certain value (for example, 0.3), or the table has an index of time column, you can try recovering by using `analyze table`. If `auto analyze` is configured, check whether the `tidb_auto_analyze_ratio` system variable is too large (for example, greater than 0.3), and whether the current time is between `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time`. - For other situations, [report a bug](https://github.com/pingcap/tidb/issues/new?labels=type%2Fbug&template=bug-report.md). @@ -436,7 +436,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - Solution: Use the binlogctl tool to check whether each Drainer node is normal or not. This is to ensure that all Drainer nodes in the `online` state are working normally. If the state of a Drainer node is not consistent with its actual working status, use the binlogctl tool to change its state and then restart Pump. See the case [fail-to-notify-all-living-drainer](/tidb-binlog/handle-tidb-binlog-errors.md#fail-to-notify-all-living-drainer-is-returned-when-pump-is-started). -- 6.1.9 Draienr reports the `gen update sqls failed: table xxx: row data is corruption []` error. +- 6.1.9 Drainer reports the `gen update sqls failed: table xxx: row data is corruption []` error. - Trigger: The upstream performs DML operations on this table while performing `DROP COLUMN` DDL. This issue has been fixed in v3.0.6. See [case-820](https://github.com/pingcap/tidb-map/blob/master/maps/diagnose-case-study/case820.md) in Chinese. @@ -505,10 +505,10 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - `region-concurrency` is set too high, which causes thread contention and reduces performance. Three ways to troubleshoot: - The setting can be found from the start of the log by searching `region-concurrency`. - - If TiDB Lightning shares a server with other services (e.g. Importer), you must manually set `region-concurrency` to 75% of the total number of CPU cores on that server. + - If TiDB Lightning shares a server with other services (for example, Importer), you must manually set `region-concurrency` to 75% of the total number of CPU cores on that server. - If there is a quota on CPU (for example, limited by Kubernetes settings), TiDB Lightning might not be able to read this out. In this case, `region-concurrency` must also be manually reduced. - - Every additional index introduces a new KV pair for each row. If there are N indices, the actual size to be imported would be approximately (N+1) times the size of the Mydumper output. If the indices are negligible, you may first remove them from the schema, and add them back via `CREATE INDEX` after the import is complete. + - Every additional index introduces a new KV pair for each row. If there are N indices, the actual size to be imported would be approximately (N+1) times the size of the [Mydumper](https://docs.pingcap.com/tidb/v4.0/mydumper-overview) output. If the indices are negligible, you may first remove them from the schema, and add them back via `CREATE INDEX` after the import is complete. - The version of TiDB Lightning is old. Try the latest version, which might improve the import speed. - 6.3.3 `checksum failed: checksum mismatched remote vs local`. @@ -517,35 +517,35 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - Cause 2: If the checksum of the target database is 0, which means nothing is imported, it is possible that the cluster is too hot and fails to take in any data. - - Cause 3: If the data source is generated by the machine and not backed up by Mydumper, ensure it respects the constrains of the table. For example: + - Cause 3: If the data source is generated by the machine and not backed up by [Mydumper](https://docs.pingcap.com/tidb/v4.0/mydumper-overview), ensure it respects the constrains of the table. For example: - `AUTO_INCREMENT` columns need to be positive, and do not contain the value “0”. - UNIQUE and PRIMARY KEYs must not have duplicate entries. - - Solution: See [Troubleshooting Solution](/troubleshoot-tidb-lightning.md#checksum-failed-checksum-mismatched-remote-vs-local). + - Solution: See [Troubleshooting Solution](/tidb-lightning/tidb-lightning-faq.md#checksum-failed-checksum-mismatched-remote-vs-local). - 6.3.4 `Checkpoint for … has invalid status:(error code)` - - Cause: Checkpoint is enabled, and Lightning/Importer has previously abnormally exited. To prevent accidental data corruption, Lightning will not start until the error is addressed. The error code is an integer less than 25, with possible values as `0, 3, 6, 9, 12, 14, 15, 17, 18, 20 and 21`. The integer indicates the step where the unexpected exit occurs in the import process. The larger the integer is, the later the exit occurs. + - Cause: Checkpoint is enabled, and Lightning/Importer has previously abnormally exited. To prevent accidental data corruption, TiDB Lightning will not start until the error is addressed. The error code is an integer less than 25, with possible values as `0, 3, 6, 9, 12, 14, 15, 17, 18, 20 and 21`. The integer indicates the step where the unexpected exit occurs in the import process. The larger the integer is, the later the exit occurs. - - Solution: See [Troubleshooting Solution](/troubleshoot-tidb-lightning.md#checkpoint-for--has-invalid-status-error-code). + - Solution: See [Troubleshooting Solution](/tidb-lightning/tidb-lightning-faq.md#checkpoint-for--has-invalid-status-error-code). - 6.3.5 `ResourceTemporarilyUnavailable("Too many open engines …: 8")` - Cause: The number of concurrent engine files exceeds the limit specified by tikv-importer. This could be caused by misconfiguration. In addition, even when the configuration is correct, if tidb-lightning has exited abnormally before, an engine file might be left at a dangling open state, which could cause this error as well. - - Solution: See [Troubleshooting Solution](/troubleshoot-tidb-lightning.md#resourcetemporarilyunavailabletoo-many-open-engines--). + - Solution: See [Troubleshooting Solution](/tidb-lightning/tidb-lightning-faq.md#resourcetemporarilyunavailabletoo-many-open-engines--). - 6.3.6 `cannot guess encoding for input file, please convert to UTF-8 manually` - Cause: TiDB Lightning only supports the UTF-8 and GB-18030 encodings. This error means the file is not in any of these encodings. It is also possible that the file has mixed encoding, such as containing a string in UTF-8 and another string in GB-18030, due to historical ALTER TABLE executions. - - Solution: See [Troubleshooting Solution](/troubleshoot-tidb-lightning.md#cannot-guess-encoding-for-input-file-please-convert-to-utf-8-manually). + - Solution: See [Troubleshooting Solution](/tidb-lightning/tidb-lightning-faq.md#cannot-guess-encoding-for-input-file-please-convert-to-utf-8-manually). - 6.3.7 `[sql2kv] sql encode error = [types:1292]invalid time format: '{1970 1 1 0 45 0 0}'` - Cause: A timestamp type entry has a time value that does not exist. This is either because of DST changes or because the time value has exceeded the supported range (from Jan 1, 1970 to Jan 19, 2038). - - Solution: See [Troubleshooting Solution](/troubleshoot-tidb-lightning.md#sql2kv-sql-encode-error--types1292invalid-time-format-1970-1-1-). + - Solution: See [Troubleshooting Solution](/tidb-lightning/tidb-lightning-faq.md#sql2kv-sql-encode-error--types1292invalid-time-format-1970-1-1-). ## 7. Common log analysis @@ -555,7 +555,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** The transaction duration exceeds the GC lifetime (10 minutes by default). - You can increase the GC lifetime by modifying the `mysql.tidb` table. Generally, it is not recommended to modify this parameter, because changing it might cause many old versions to pile up if this transaction has a large number of `update` and `delete` statements. + You can increase the GC lifetime by modifying the [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) system variable. Generally, it is not recommended to modify this parameter, because changing it might cause many old versions to pile up if this transaction has a large number of `UPDATE` and `DELETE` statements. - 7.1.2 `txn takes too much time`. @@ -582,12 +582,10 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - 7.1.5 `distsql.go` reports `inconsistent index`. - The data index seems to be inconsistent. Run the `admin check table ` command on the table where the reported index is. If the check fails, close GC by running the following command, and [report a bug](https://github.com/pingcap/tidb/issues/new?labels=type%2Fbug&template=bug-report.md): + The data index seems to be inconsistent. Run the `admin check table ` command on the table where the reported index is. If the check fails, disable garbage collection by running the following command, and [report a bug](https://github.com/pingcap/tidb/issues/new?labels=type%2Fbug&template=bug-report.md): ```sql - begin; - update mysql.tidb set variable_value='72h' where variable_name='tikv_gc_life_time'; - commit; + SET GLOBAL tidb_gc_enable = 0; ``` ### 7.2 TiKV diff --git a/tiflash-deployment-topology.md b/tiflash-deployment-topology.md index 5160667d0513..39196a252e96 100644 --- a/tiflash-deployment-topology.md +++ b/tiflash-deployment-topology.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/tiflash-deployment-topology/'] # TiFlash Deployment Topology -This document describes the deployment topology of TiFlash based on the minimal TiDB topology. +This document describes the deployment topology of [TiFlash](/tiflash/tiflash-overview.md) based on the minimal TiDB topology. TiFlash is a columnar storage engine, and gradually becomes the standard cluster topology. It is suitable for real-time HTAP applications. @@ -25,6 +25,8 @@ TiFlash is a columnar storage engine, and gradually becomes the standard cluster - [The simple template for the TiFlash topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-tiflash.yaml) - [The complex template for the TiFlash topology](https://github.com/pingcap/docs/blob/master/config-templates/complex-tiflash.yaml) +For detailed descriptions of the configuration items in the above TiDB cluster topology file, see [Topology Configuration File for Deploying TiDB Using TiUP](/tiup/tiup-cluster-topology-reference.md). + ### Key parameters - To enable the [Placement Rules](/configure-placement-rules.md) feature of PD, set the value of `replication.enable-placement-rules` in the configuration template to `true`. diff --git a/tiflash/maintain-tiflash.md b/tiflash/maintain-tiflash.md index b008ebefc1ef..d53b05868f15 100644 --- a/tiflash/maintain-tiflash.md +++ b/tiflash/maintain-tiflash.md @@ -32,18 +32,6 @@ There are two ways to check the TiFlash version: ## TiFlash critical logs -Starting from TiDB 4.0.5, TiFlash's log format has been refined for compatibility with TiDB. Therefore, there are two versions of log messages. - -Before TiDB 4.0.5: - -| Log Information | Log Description | -|---------------|-------------------| -| [ 23 ] `` KVStore: Start to persist [region 47, applied: term 6 index 10] | Data starts to be replicated (the number in the square brackets at the start of the log refers to the thread ID | -| [ 30 ] `` CoprocessorHandler: grpc::Status DB::CoprocessorHandler::execute() | Handling DAG request, that is, TiFlash starts to handle a Coprocessor request | -| [ 30 ] `` CoprocessorHandler: grpc::Status DB::CoprocessorHandler::execute() | Handling DAG request done, that is, TiFlash finishes handling a Coprocessor request | - -After TiDB 4.0.5: - | Log Information | Log Description | |---------------|-------------------| | [INFO] [``] ["KVStore: Start to persist [region 47, applied: term 6 index 10]"] [thread_id=23] | Data starts to be replicated (the number in the square brackets at the start of the log refers to the thread ID | diff --git a/tiflash/monitor-tiflash.md b/tiflash/monitor-tiflash.md index f5c900614fc6..90d28c38238e 100644 --- a/tiflash/monitor-tiflash.md +++ b/tiflash/monitor-tiflash.md @@ -8,7 +8,7 @@ aliases: ['/docs/dev/tiflash/monitor-tiflash/','/docs/dev/reference/tiflash/moni This document describes the monitoring items of TiFlash. -If you use TiUP or TiDB Ansible to deploy the TiDB cluster, the monitoring system (Prometheus & Grafana) is deployed at the same time. For more information, see [Overview of the Monitoring Framework](/tidb-monitoring-framework.md). +If you use TiUP to deploy the TiDB cluster, the monitoring system (Prometheus & Grafana) is deployed at the same time. For more information, see [Overview of the Monitoring Framework](/tidb-monitoring-framework.md). The Grafana dashboard is divided into a series of sub dashboards which include Overview, PD, TiDB, TiKV, Node\_exporter, and so on. A lot of metrics are there to help you diagnose. diff --git a/tiflash/tiflash-command-line-flags.md b/tiflash/tiflash-command-line-flags.md index 0b28d518ac35..94e71aeda715 100644 --- a/tiflash/tiflash-command-line-flags.md +++ b/tiflash/tiflash-command-line-flags.md @@ -4,7 +4,7 @@ summary: Learn the command-line startup flags of TiFlash. aliases: ['/docs/dev/tiflash/tiflash-command-line-flags/'] --- -# TiFlash Command-line Flags +# TiFlash Command-Line Flags This document introduces the command-line flags that you can use when you launch TiFlash. diff --git a/tiflash/tiflash-configuration.md b/tiflash/tiflash-configuration.md index fecfcf6b1ac2..db418a6a8a82 100644 --- a/tiflash/tiflash-configuration.md +++ b/tiflash/tiflash-configuration.md @@ -18,16 +18,18 @@ You can adjust the PD scheduling parameters using [pd-ctl](/pd-control.md). Note > > The value of this parameter should be less than that of `region-schedule-limit`. Otherwise, the normal Region scheduling among TiKV nodes is affected. -- [`store-balance-rate`](/pd-configuration-file.md#store-balance-rate): limits the rate at which Regions of each TiKV/TiFlash store are scheduled. Note that this parameter takes effect only when the stores have newly joined the cluster. If you want to change the setting for existing stores, use the following command. +- `store-balance-rate`: limits the rate at which Regions of each TiKV/TiFlash store are scheduled. Note that this parameter takes effect only when the stores have newly joined the cluster. If you want to change the setting for existing stores, use the following command. > **Note:** > > Since v4.0.2, the `store-balance-rate` parameter has been deprecated and changes have been made to the `store limit` command. See [store-limit](/configure-store-limit.md) for details. - - Execute the `pd-ctl -u store limit ` command to set the scheduling rate of a specified store. (To get `store_id`, you can execute the `pd-ctl -u store` command. + - Execute the `pd-ctl -u store limit ` command to set the scheduling rate of a specified store. (To get `store_id`, you can execute the `pd-ctl -u store` command. - If you do not set the scheduling rate for Regions of a specified store, this store inherits the setting of `store-balance-rate`. - You can execute the `pd-ctl -u store limit` command to view the current setting value of `store-balance-rate`. +- [`replication.location-labels`](/pd-configuration-file.md#location-labels): indicates the topological relationship of TiKV instances. The order of the keys indicates the layering relationship of different labels. If TiFlash is enabled, you need to use [`pd-ctl config placement-rules`](/pd-control.md#config-show--set-option-value--placement-rules) to set the default value. For details, see [geo-distributed-deployment-topology](/geo-distributed-deployment-topology.md). + ## TiFlash configuration parameters This section introduces the configuration parameters of TiFlash. @@ -65,7 +67,12 @@ delta_index_cache_size = 0 ## Storage paths settings take effect starting from v4.0.9 [storage] - ## [Experimental] Introduced in v5.0. Limits the total write rate of background tasks in bytes per second. By default 0, means no limit. It is not recommended to use this experimental feature in a production environment. + ## [Experimental] New in v5.0. This item limits the total write rate of background tasks in bytes per second. It is not recommended to use this experimental feature in a production environment. + ## The unit is bytes. Currently, the setting such as "10GB" is not supported. + ## The default value is 0, which means no limit. + ## This parameter is used to control the usage of machine disk bandwidth by background tasks mainly for the scenario where TiFlash is deployed on the AWS EBS (gp2/gp3) disk. + ## This parameter can be used to improve the stability of the TiFlash query performance. The recommended configuration in this scenario is 50% of the disk bandwidth. + ## It is not recommended to modify this configuration in other scenarios. bg_task_io_rate_limit = 0 [storage.main] @@ -112,6 +119,7 @@ delta_index_cache_size = 0 data-dir = The data storage path of proxy. config = The proxy configuration file path. log-file = The proxy log path. + log-level = The proxy log level. "info" is used by default. status-addr = The listening address from which the proxy metrics | status information is pulled. advertise-status-addr = The external access address of status-addr. If it is left empty, status-addr is used by default. @@ -136,15 +144,40 @@ delta_index_cache_size = 0 ## of DeltaTree Storage Engine uses logical split. ## Using the logical split can reduce the write amplification, and improve the write speed. ## However, these are at the cost of disk space waste. - dt_enable_logical_split = true + dt_enable_logical_split = true ## The memory usage limit for the generated intermediate data when a single ## coprocessor query is executed. The default value is 0, which means no limit. - max_memory_usage = 0 + max_memory_usage = 0 ## The memory usage limit for the generated intermediate data when all queries ## are executed. The default value is 0 (in bytes), which means no limit. - max_memory_usage_for_all_queries = 0 + max_memory_usage_for_all_queries = 0 + + ## New in v5.0. This item specifies the maximum number of cop requests that TiFlash Coprocessor executes at the same time. If the number of requests exceeds the specified value, the exceeded requests will queue. If the configuration value is set to 0 or not set, the default value is used, which is twice the number of physical cores. + cop_pool_size = 0 + ## New in v5.0. This item specifies the maximum number of batch requests that TiFlash Coprocessor executes at the same time. If the number of requests exceeds the specified value, the exceeded requests will queue. If the configuration value is set to 0 or not set, the default value is used, which is twice the number of physical cores. + batch_cop_pool_size = 0 + +## Security settings take effect starting from v4.0.5. +[security] + ## New in v5.0. This configuration item enables or disables log redaction. If the configuration value + ## is set to `true`, all user data in the log will be replaced by `?`. + ## Note that you also need to set `security.redact-info-log` for tiflash-learner's logging in tiflash-learner.toml. + # redact_info_log = false + + ## Path of the file that contains a list of trusted SSL CAs. If set, the following settings + ## `cert_path` and `key_path` are also needed. + # ca_path = "/path/to/ca.pem" + ## Path of the file that contains X509 certificate in PEM format. + # cert_path = "/path/to/tiflash-server.pem" + ## Path of the file that contains X509 key in PEM format. + # key_path = "/path/to/tiflash-server-key.pem" + + ## New in v5.0. This configuration item enables or disables log redaction. If the configuration value + ## is set to `true`, all user data in the log will be replaced by `?`. + ## Note that you also need to set `security.redact-info-log` for tiflash-learner's logging in tiflash-learner.toml. + # redact_info_log = false ``` ### Configure the `tiflash-learner.toml` file @@ -154,22 +187,27 @@ delta_index_cache_size = 0 engine-addr = The external access address of the TiFlash coprocessor service. [raftstore] ## Specifies the number of threads that handle snapshots. - ## The default number is 2. + ## The default number is 2. ## If you set it to 0, the multi-thread optimization is disabled. - snap-handle-pool-size = 2 + snap-handle-pool-size = 2 ## Specifies the shortest interval at which Raft store persists WAL. ## You can properly increase the latency to reduce IOPS usage. ## The default value is "4ms". ## If you set it to 0ms, the optimization is disabled. store-batch-retry-recv-timeout = "4ms" +[security] + ## New in v5.0. This configuration item enables or disables log redaction. + ## If the configuration value is set to true, + ## all user data in the log will be replaced by ?. The default value is false. + redact-info-log = false ``` In addition to the items above, other parameters are the same with those of TiKV. Note that the configuration items in `tiflash.toml [flash.proxy]` will override the overlapping parameters in `tiflash-learner.toml`; The `label` whose key is `engine` is reserved and cannot be configured manually. ### Multi-disk deployment -TiFlash supports multi-disk deployment. If there are multiple disks in your TiFlash node, you can make full use of those disks by configuring the parameters described in the following sections. For TiFlash's configuration template to be used for TiUP, see [The complex template for the TiFlash topology](https://github.com/pingcap/docs/blob/master/config-templates/complex-tiflash.yaml). +TiFlash supports multi-disk deployment. If there are multiple disks in your TiFlash node, you can make full use of those disks by configuring the parameters described in the following sections. For TiFlash's configuration template to be used for TiUP, see [The complex template for the TiFlash topology](https://github.com/pingcap/docs/blob/master/config-templates/complex-tiflash.yaml). #### Multi-disk deployment with TiDB version earlier than v4.0.9 diff --git a/tiflash/tune-tiflash-performance.md b/tiflash/tune-tiflash-performance.md index c57d736819ba..9bbe13d0dcc5 100644 --- a/tiflash/tune-tiflash-performance.md +++ b/tiflash/tune-tiflash-performance.md @@ -24,7 +24,7 @@ If you want to save machine resources and have no requirement on isolation, you 2. Enable the super batch feature: - You can use the [`tidb_allow_batch_cop`](/system-variables.md#tidb_allow_batch_cop-new-in-v40-version) variable to set whether to merge Region requests when reading from TiFlash. + You can use the [`tidb_allow_batch_cop`](/system-variables.md#tidb_allow_batch_cop-new-in-v40) variable to set whether to merge Region requests when reading from TiFlash. When the number of Regions involved in the query is relatively large, try to set this variable to `1` (effective for coprocessor requests with `aggregation` operators that are pushed down to TiFlash), or set this variable to `2` (effective for all coprocessor requests that are pushed down to TiFlash). @@ -53,3 +53,13 @@ If you want to save machine resources and have no requirement on isolation, you ```sql set @@tidb_opt_distinct_agg_push_down = 1; ``` + +5. If the `JOIN` operator does not choose the MPP mode, you can modify the value of `tidb_opt_network_factor` to make the`JOIN` operator choose the MPP mode: + + The variable `tidb_opt_network_factor` is used to set the ratio of network overhead that the optimizer takes into account when calculating the cost. The smaller the variable value is, the smaller the estimated cost for a large amount of network transmissions is, and the more TiDB inclined to choose the MPP operator. + + {{< copyable "sql" >}} + + ```sql + set @@tidb_opt_network_factor = 0.001; + ``` diff --git a/tiflash/use-tiflash.md b/tiflash/use-tiflash.md index f679c21e68ce..d7adc9cf3853 100644 --- a/tiflash/use-tiflash.md +++ b/tiflash/use-tiflash.md @@ -54,10 +54,14 @@ ALTER TABLE `tpch50`.`lineitem` SET TIFLASH REPLICA 0 CREATE TABLE table_name like t ``` -* For the current version, if you create the TiFlash replica before using TiDB Lightning to import the data, the data import will fail. You must import data to the table before creating the TiFlash replica for the table. +* For versions earlier than v4.0.6, if you create the TiFlash replica before using TiDB Lightning to import the data, the data import will fail. You must import data to the table before creating the TiFlash replica for the table. + +* If TiDB and TiDB Lightning are both v4.0.6 or later, no matter a table has TiFlash replica(s) or not, you can import data to that table using TiDB Lightning. Note that this might slow the TiDB Lightning procedure, which depends on the NIC bandwidth on the lightning host, the CPU and disk load of the TiFlash node, and the number of TiFlash replicas. * It is recommended that you do not replicate more than 1,000 tables because this lowers the PD scheduling performance. This limit will be removed in later versions. +* In v5.1 and later versions, setting the replicas for the system tables is no longer supported. Before upgrading the cluster, you need to clear the replicas of the relevant system tables. Otherwise, you cannot modify the replica settings of the system tables after you upgrade the cluster to a later version. + ## Check the replication progress You can check the status of the TiFlash replicas of a specific table using the following statement. The table is specified using the `WHERE` clause. If you remove the `WHERE` clause, you will check the replica status of all tables. @@ -189,7 +193,7 @@ In the above three ways of reading TiFlash replicas, engine isolation specifies > **Note:** > -> Before v4.0.3, the behavior of reading from TiFlash replica in a non-read-only SQL statement (for example, `INSERT INTO ... SELECT`, `SELECT ... FOR UPDATE`, `UPDATE ...`, `DELETE ...`) is undefined. In v4.0.3 and later versions, internally TiDB ignores the TiFlash replica for a non-read-only SQL statement to guarantee the data correctness. That is, for [smart selection](#smart-selection), TiDB automatically chooses the non-TiFlash replica; for [engine isolation](#engine-isolation) that specifies TiFlash replica **only**, TiDB reports an error; and for [manual hint](#manual-hint), TiDB ignores the hint. +> Before v4.0.3, the behavior of reading from TiFlash replica in a non-read-only SQL statement (for example, `INSERT INTO ... SELECT`, `SELECT ... FOR UPDATE`, `UPDATE ...`, `DELETE ...`) is undefined. In v4.0.3 and later versions, internally TiDB ignores the TiFlash replica for a non-read-only SQL statement to guarantee the data correctness. That is, for [smart selection](#smart-selection), TiDB automatically selects the non-TiFlash replica; for [engine isolation](#engine-isolation) that specifies TiFlash replica **only**, TiDB reports an error; and for [manual hint](#manual-hint), TiDB ignores the hint. ## Use TiSpark to read TiFlash replicas @@ -215,30 +219,176 @@ You can configure this parameter in one of the following ways: ## Supported push-down calculations -> **Note:** -> -> Before v4.0.2, TiDB does not support the new framework for collations, so in those previous versions, if you enable the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations), none of the expressions can be pushed down. This restriction is removed in v4.0.2 and later versions. +TiFlash supports the push-down of the following operators: + +* TableScan: Reads data from tables. +* Selection: Filters data. +* HashAgg: Performs data aggregation based on the [Hash Aggregation](/explain-aggregation.md#hash-aggregation) algorithm. +* StreamAgg: Performs data aggregation based on the [Stream Aggregation](/explain-aggregation.md#stream-aggregation) algorithm. SteamAgg only supports the aggregation without the `GROUP BY` condition. +* TopN: Performs the TopN calculation. +* Limit: Performs the limit calculation. +* Project: Performs the projection calculation. +* HashJoin (Equi Join): Performs the join calculation based on the [Hash Join](/explain-joins.md#hash-join) algorithm, but with the following conditions: + * The operator can be pushed down only in the [MPP mode](#use-the-mpp-mode). + * The push-down of `Full Outer Join` is not supported. +* HashJoin (Non-Equi Join): Performs the Cartesian Join algorithm, but with the following conditions: + * The operator can be pushed down only in the [MPP mode](#use-the-mpp-mode). + * Cartesian Join is supported only in Broadcast Join. + +In TiDB, operators are organized in a tree structure. For an operator to be pushed down to TiFlash, all of the following prerequisites must be met: + ++ All of its child operators can be pushed down to TiFlash. ++ If an operator contains expressions (most of the operators contain expressions), all expressions of the operator can be pushed down to TiFlash. + +Currently, TiFlash supports the following push-down expressions: + +* Mathematical functions: `+, -, /, *, >=, <=, =, !=, <, >, round(int), round(double), abs, floor(int), ceil(int), ceiling(int)` +* Logical functions: `and, or, not, case when, if, ifnull, isnull, in` +* Bitwise operations: `bitand, bitor, bigneg, bitxor` +* String functions: `substr, char_length, replace, concat, concat_ws, left, right` +* Date functions: `date_format, timestampdiff, from_unixtime, unix_timestamp(int), unix_timestamp(decimal), str_to_date(date), str_to_date(datetime), date_add(string, int), date_add(datetime, int), date_sub(datetime, int), date_sub(string, int), datediff, year, month, day, extract(datetime)` +* JSON function: `json_length` +* Conversion functions: `cast(int as double), cast(int as decimal), cast(int as string), cast(int as time), cast(double as int), cast(double as decimal), cast(double as string), cast(double as time), cast(string as int), cast(string as double), cast(string as decimal), cast(string as time), cast(decimal as int), cast(decimal as string), cast(decimal as time), cast(time as int), cast(time as decimal), cast(time as string)` +* Aggregate functions: `min, max, sum, count, avg, approx_count_distinct` -TiFlash supports predicate, aggregate push-down calculations, and table joins. Push-down calculations can help TiDB perform distributed acceleration. Currently, `Full Outer Join` and `DISTINCT COUNT` are not the supported calculation types, which will be optimized in later versions. +Among them, the push-down of `cast` and `date_add` is not enabled by default. To enable it, refer to [Blocklist of Optimization Rules and Expression Pushdown](/blocklist-control-plan.md). -You can enable the push-down of `join` using the following session variable (`Full Outer Join` is currently not supported): +In addition, expressions that contain the Time/Bit/Set/Enum/Geometry type cannot be pushed down to TiFlash. -``` -set @@session.tidb_opt_broadcast_join=1 +If a query encounters unsupported push-down calculations, TiDB needs to complete the remaining calculations, which might greatly affect the TiFlash acceleration effect. The currently unsupported operators and expressions might be supported in future versions. + +## Use the MPP mode + +TiFlash supports using the MPP mode to execute queries, which introduces cross-node data exchange (data shuffle process) into the computation. TiDB automatically determines whether to select the MPP mode using the optimizer's cost estimation. You can change the selection strategy by modifying the values of [`tidb_allow_mpp`](/system-variables.md#tidb_allow_mpp-new-in-v50) and [`tidb_enforce_mpp`](/system-variables.md#tidb_enforce_mpp-new-in-v51). + +### Control whether to select the MPP mode + +The `tidb_allow_mpp` variable controls whether TiDB can select the MPP mode to execute queries. The `tidb_enforce_mpp` variable controls whether the optimizer's cost estimation is ignored and the MPP mode of TiFlash is forcibly used to execute queries. + +The results corresponding to all values of these two variables are as follows: + +| | tidb_allow_mpp=off | tidb_allow_mpp=on (by default) | +| ---------------------- | -------------------- | -------------------------------- | +| tidb_enforce_mpp=off (by default) | The MPP mode is not used. | The optimizer selects the MPP mode based on cost estimation. (by default)| +| tidb_enforce_mpp=on | The MPP mode is not used. | TiDB ignores the cost estimation and selects the MPP mode. | + +For example, if you do not want to use the MPP mode, you can execute the following statements: + +{{< copyable "sql" >}} + +```sql +set @@session.tidb_allow_mpp=1; +set @@session.tidb_enforce_mpp=0; ``` -Currently, TiFlash supports pushing down a limited number of expressions, including: +If you want TiDB's cost-based optimizer to automatically decide whether to use the MPP mode (by default), you can execute the following statements: + +{{< copyable "sql" >}} +```sql +set @@session.tidb_allow_mpp=1; +set @@session.tidb_enforce_mpp=0; ``` -+, -, /, *, >=, <=, =, !=, <, >, ifnull, isnull, bitor, in, mod, bitand, or, and, like, not, -case when, month, substr, timestampdiff, date_format, from_unixtime, json_length, if, bitneg, bitxor, round without fraction, cast(int as decimal), date_add(datetime, int), date_add(datetime, string) + +If you want TiDB to ignore the optimizer's cost estimation and to forcibly select the MPP mode, you can execute the following statements: + +{{< copyable "sql" >}} + +```sql +set @@session.tidb_allow_mpp=1; +set @@session.tidb_enforce_mpp=1; ``` -Among them, the push-down of `cast` and `date_add` is not enabled by default. To enable it, refer to [Blocklist of Optimization Rules and Expression Pushdown](/blocklist-control-plan.md). +The initial value of the `tidb_enforce_mpp` session variable is equal to the [`enforce-mpp`](/tidb-configuration-file.md#enforce-mpp) configuration value of this tidb-server instance (which is `false` by default). If multiple tidb-server instances in a TiDB cluster only perform analytical queries and you want to make sure that the MPP mode is used on these instances, you can change their [`enforce-mpp`](/tidb-configuration-file.md#enforce-mpp) configuration values to `true`. -TiFlash does not support push-down calculations in the following situations: +> **Note:** +> +> When `tidb_enforce_mpp=1` takes effect, the TiDB optimizer will ignore the cost estimation to choose the MPP mode. However, if other factors block the MPP mode, TiDB will not select the MPP mode. These factors include the absence of TiFlash replica, unfinished replication of TiFlash replicas, and statements containing operators or functions that are not supported by the MPP mode. +> +> If TiDB optimizer cannot select the MPP mode due to reasons other than cost estimation, when you use the `EXPLAIN` statement to check out the execution plan, a warning is returned to explain the reason. For example: +> +> {{< copyable "sql" >}} +> +> ```sql +> set @@session.tidb_enforce_mpp=1; +> create table t(a int); +> explain select count(*) from t; +> show warnings; +> ``` +> +> ``` +> +---------+------+-----------------------------------------------------------------------------+ +> | Level | Code | Message | +> +---------+------+-----------------------------------------------------------------------------+ +> | Warning | 1105 | MPP mode may be blocked because there aren't tiflash replicas of table `t`. | +> +---------+------+-----------------------------------------------------------------------------+ +> ``` + +### Algorithm support for the MPP mode + +The MPP mode supports these physical algorithms: Broadcast Hash Join, Shuffled Hash Join, Shuffled Hash Aggregation, Union All, TopN, and Limit. The optimizer automatically determines which algorithm to be used in a query. To check the specific query execution plan, you can execute the `EXPLAIN` statement. If the result of the `EXPLAIN` statement shows ExchangeSender and ExchangeReceiver operators, it indicates that the MPP mode has taken effect. + +The following statement takes the table structure in the TPC-H test set as an example: -- Expressions that contain `Duration` cannot be pushed down. -- If an aggregate function or a `WHERE` clause contains expressions that are not included in the list above, the aggregate or related predicate filtering cannot be pushed down. +```sql +explain select count(*) from customer c join nation n on c.c_nationkey=n.n_nationkey; ++------------------------------------------+------------+-------------------+---------------+----------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------------------+------------+-------------------+---------------+----------------------------------------------------------------------------+ +| HashAgg_23 | 1.00 | root | | funcs:count(Column#16)->Column#15 | +| └─TableReader_25 | 1.00 | root | | data:ExchangeSender_24 | +| └─ExchangeSender_24 | 1.00 | batchCop[tiflash] | | ExchangeType: PassThrough | +| └─HashAgg_12 | 1.00 | batchCop[tiflash] | | funcs:count(1)->Column#16 | +| └─HashJoin_17 | 3000000.00 | batchCop[tiflash] | | inner join, equal:[eq(tpch.nation.n_nationkey, tpch.customer.c_nationkey)] | +| ├─ExchangeReceiver_21(Build) | 25.00 | batchCop[tiflash] | | | +| │ └─ExchangeSender_20 | 25.00 | batchCop[tiflash] | | ExchangeType: Broadcast | +| │ └─TableFullScan_18 | 25.00 | batchCop[tiflash] | table:n | keep order:false | +| └─TableFullScan_22(Probe) | 3000000.00 | batchCop[tiflash] | table:c | keep order:false | ++------------------------------------------+------------+-------------------+---------------+----------------------------------------------------------------------------+ +9 rows in set (0.00 sec) +``` -If a query encounters unsupported push-down calculations, TiDB needs to complete the remaining calculations, which might greatly affect the TiFlash acceleration effect. +In the example execution plan, the `ExchangeReceiver` and `ExchangeSender` operators are included. The execution plan indicates that after the `nation` table is read, the `ExchangeSender` operator broadcasts the table to each node, the `HashJoin` and `HashAgg` operations are performed on the `nation` table and the `customer` table, and then the results are returned to TiDB. + +TiFlash provides the following two global/session variables to control whether to use Broadcast Hash Join: + +- [`tidb_broadcast_join_threshold_size`](/system-variables.md#tidb_broadcast_join_threshold_count-new-in-v50): The unit of the value is bytes. If the table size (in the unit of bytes) is less than the value of the variable, the Broadcast Hash Join algorithm is used. Otherwise, the Shuffled Hash Join algorithm is used. +- [`tidb_broadcast_join_threshold_count`](/system-variables.md#tidb_broadcast_join_threshold_count-new-in-v50): The unit of the value is rows. If the objects of the join operation belong to a subquery, the optimizer cannot estimate the size of the subquery result set, so the size is determined by the number of rows in the result set. If the estimated number of rows in the subquery is less than the value of this variable, the Broadcast Hash Join algorithm is used. Otherwise, the Shuffled Hash Join algorithm is used. + +## Notes + +Currently, TiFlash does not support some features. These features might be incompatible with the native TiDB: + +* In the TiFlash computation layer: + * Checking overflowed numerical values is not supported. For example, adding two maximum values of the `BIGINT` type `9223372036854775807 + 9223372036854775807`. The expected behavior of this calculation in TiDB is to return the `ERROR 1690 (22003): BIGINT value is out of range` error. However, if this calculation is performed in TiFlash, an overflow value of `-2` is returned without any error. + * The window function is not supported. + * Reading data from TiKV is not supported. + * Currently, the `sum` function in TiFlash does not support the string-type argument. But TiDB cannot identify whether any string-type argument has been passed into the `sum` function during the compiling. Therefore, when you execute statements similar to `select sum(string_col) from t`, TiFlash returns the `[FLASH:Coprocessor:Unimplemented] CastStringAsReal is not supported.` error. To avoid such an error in this case, you need to modify this SQL statement to `select sum(cast(string_col as double)) from t`. + * Currently, TiFlash's decimal division calculation is incompatible with that of TiDB. For example, when dividing decimal, TiFlash performs the calculation always using the type inferred from the compiling. However, TiDB performs this calculation using a type that is more precise than that inferred from the compiling. Therefore, some SQL statements involving the decimal division return different execution results when executed in TiDB + TiKV and in TiDB + TiFlash. For example: + + ```sql + mysql> create table t (a decimal(3,0), b decimal(10, 0)); + Query OK, 0 rows affected (0.07 sec) + mysql> insert into t values (43, 1044774912); + Query OK, 1 row affected (0.03 sec) + mysql> alter table t set tiflash replica 1; + Query OK, 0 rows affected (0.07 sec) + mysql> set session tidb_isolation_read_engines='tikv'; + Query OK, 0 rows affected (0.00 sec) + mysql> select a/b, a/b + 0.0000000000001 from t where a/b; + +--------+-----------------------+ + | a/b | a/b + 0.0000000000001 | + +--------+-----------------------+ + | 0.0000 | 0.0000000410001 | + +--------+-----------------------+ + 1 row in set (0.00 sec) + mysql> set session tidb_isolation_read_engines='tiflash'; + Query OK, 0 rows affected (0.00 sec) + mysql> select a/b, a/b + 0.0000000000001 from t where a/b; + Empty set (0.01 sec) + ``` + + In the example above, `a/b`'s inferred type from the compiling is `Decimal(7,4)` both in TiDB and in TiFlash. Constrained by `Decimal(7,4)`, `a/b`'s returned type should be `0.0000`. In TiDB, `a/b`'s runtime precision is higher than `Decimal(7,4)`, so the original table data is not filtered by the `where a/b` condition. However, in TiFlash, the calculation of `a/b` uses `Decimal(7,4)` as the result type, so the original table data is filtered by the `where a/b` condition. + +* The MPP mode in TiFlash does not support the following features: + * If the [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) configuration item's value is `true`, the MPP mode does not support the string-type join key or the string column type in the `group by` aggregation. For these two query types, the MPP mode is not selected by default. diff --git a/tikv-configuration-file.md b/tikv-configuration-file.md index ece8796bf26e..ad34ce379ffd 100644 --- a/tikv-configuration-file.md +++ b/tikv-configuration-file.md @@ -12,63 +12,38 @@ The TiKV configuration file supports more options than command-line parameters. This document only describes the parameters that are not included in command-line parameters. For more details, see [command-line parameter](/command-line-flags-for-tikv-configuration.md). -### `status-thread-pool-size` +## Global configuration -+ The number of worker threads for the `HTTP` API service -+ Default value: `1` -+ Minimum value: `1` - -### `server.concurrent-send-snap-limit` - -+ The maximum number of snapshots that can be sent at the same time -+ Default value: `32` -+ Minimum value: `1` - -### `server.concurrent-recv-snap-limit` +### abort-on-panic -+ The maximum number of snapshots that can be received at the same time -+ Default value: `32` -+ Minimum value: `1` ++ Sets whether to call `abort()` to exit the process when TiKV panics. This option affects whether TiKV allows the system to generate core dump files. -### `server.end-point-recursion-limit` + + If the value of this configuration item is `false`, when TiKV panics, it calls `exit()` to exit the process. + + If the value of this configuration item is `true`, when TiKV panics, TiKV calls `abort()` to exit the process. At this time, TiKV allows the system to generate core dump files when exiting. To generate the core dump file, you also need to perform the system configuration related to core dump (for example, setting the size limit of the core dump file via `ulimit -c` command, and configure the core dump path. Different operating systems have different related configurations). To avoid the core dump files occupying too much disk space and causing insufficient TiKV disk space, it is recommended to set the core dump generation path to a disk partition different to that of TiKV data. -+ The maximum number of recursive layers allowed when TiKV decodes the Coprocessor DAG expression -+ Default value: `1000` -+ Minimum value: `1` ++ Default value: `false` -### `server.end-point-request-max-handle-duration` +### server -+ The longest duration allowed for a TiDB's push down request to TiKV for processing tasks -+ Default value: `"60s"` -+ Minimum value: `"1s"` ++ Configuration items related to the server -### `server.end-point-slow-log-threshold` +## `status-thread-pool-size` -+ The time threshold for a TiDB's push down request to print slow log -+ Default value: `"1s"` -+ Minimum value: `0` - -### `server.snap-max-write-bytes-per-sec` - -+ The maximum allowable disk bandwidth for processing snapshots -+ Default value: `"1000MB"` -+ Unit: KB|MB|GB -+ Minimum value: `"1KB"` - -## gRPC - -Configuration items related to gRPC. ++ The number of worker threads for the `HTTP` API service ++ Default value: `1` ++ Minimum value: `1` ### `grpc-compression-type` + The compression algorithm for gRPC messages + Optional values: `"none"`, `"deflate"`, `"gzip"` + Default value: `"none"` ++ Note: When the value is `gzip`, TiDB Dashboard will have a display error because it might not complete the corresponding compression algorithm in some cases. If you adjust the value back to the default `none`, TiDB Dashboard will display normally. ### `grpc-concurrency` + The number of gRPC worker threads -+ Default value: `4` ++ Default value: `5` + Minimum value: `1` ### `grpc-concurrent-stream` @@ -79,59 +54,94 @@ Configuration items related to gRPC. ### `grpc-memory-pool-quota` -+ Limit the memory size that can be used by gRPC -+ Default: `"32G"` ++ Limits the memory size that can be used by gRPC ++ Default: No limit + Limit the memory in case OOM is observed. Note that limit the usage can lead to potential stall -### `server.grpc-raft-conn-num` +### `grpc-raft-conn-num` + The maximum number of links among TiKV nodes for Raft communication -+ Default: `10` ++ Default: `1` + Minimum value: `1` -### `server.grpc-stream-initial-window-size` +### `grpc-stream-initial-window-size` + The window size of the gRPC stream + Default: 2MB + Unit: KB|MB|GB + Minimum value: `"1KB"` -### `server.grpc-keepalive-time` +### `grpc-keepalive-time` + The time interval at which that gRPC sends `keepalive` Ping messages + Default: `"10s"` + Minimum value: `"1s"` -### `server.grpc-keepalive-timeout` +### `grpc-keepalive-timeout` + Disables the timeout for gRPC streams + Default: `"3s"` + Minimum value: `"1s"` -## readpool.unified +### `concurrent-send-snap-limit` + ++ The maximum number of snapshots sent at the same time ++ Default value: `32` ++ Minimum value: `1` + +### `concurrent-recv-snap-limit` + ++ The maximum number of snapshots received at the same time ++ Default value: `32` ++ Minimum value: `1` + +### `end-point-recursion-limit` + ++ The maximum number of recursive levels allowed when TiKV decodes the Coprocessor DAG expression ++ Default value: `1000` ++ Minimum value: `1` + +### `end-point-request-max-handle-duration` + ++ The longest duration allowed for a TiDB's push down request to TiKV for processing tasks ++ Default value: `"60s"` ++ Minimum value: `"1s"` + +### `snap-max-write-bytes-per-sec` + ++ The maximum allowable disk bandwidth when processing snapshots ++ Default value: `"100MB"` ++ Unit: KB|MB|GB ++ Minimum value: `"1KB"` + +### `end-point-slow-log-threshold` -> **Warning:** -> -> Unified read pool is still an experimental feature. It is **NOT** recommended that you use it in the production environment. ++ The time threshold for a TiDB's push down request to print slow log ++ Default value: `"1s"` ++ Minimum value: `0` + +## readpool.unified Configuration items related to the single thread pool serving read requests. This thread pool supersedes the original storage thread pool and coprocessor thread pool since the 4.0 version. ### `min-thread-count` -+ The minimal working thread count of the unified read pool. ++ The minimal working thread count of the unified read pool + Default value: `1` ### `max-thread-count` -+ The maximum working thread count of the unified read pool. ++ The maximum working thread count of the unified read pool + Default value: `MAX(4, CPU * 0.8)` ### `stack-size` -+ The stack size of the threads in the unified thread pool. ++ The stack size of the threads in the unified thread pool ++ Type: Integer + Unit + Default value: `"10MB"` + Unit: KB|MB|GB + Minimum value: `"2MB"` ++ Maximum value: The number of Kbytes output in the result of the `ulimit -sH` command executed in the system. ### `max-tasks-per-worker` @@ -146,24 +156,24 @@ Configuration items related to storage thread pool. ### `use-unified-pool` + Determines whether to use the unified thread pool (configured in [`readpool.unified`](#readpoolunified)) for storage requests. If the value of this parameter is `false`, a separate thread pool is used, which is configured through the rest parameters in this section (`readpool.storage`). -+ Default value: `false` ++ Default value: If this section (`readpool.storage`) has no other configurations, the default value is `true`. Otherwise, for the backward compatibility, the default value is `false`. Change the configuration in [`readpool.unified`](#readpoolunified) as needed before enabling this option. ### `high-concurrency` + The allowable number of concurrent threads that handle high-priority `read` requests -+ Default value: `4` ++ When `8` ≤ `cpu num` ≤ `16`, the default value is `cpu_num * 0.5`; when `cpu num` is greater than `8`, the default value is `4`; when `cpu num` is greater than `16`, the default value is `8`. + Minimum value: `1` ### `normal-concurrency` + The allowable number of concurrent threads that handle normal-priority `read` requests -+ Default value: `4` ++ When `8` ≤ `cpu num` ≤ `16`, the default value is `cpu_num * 0.5`; when `cpu num` is greater than `8`, the default value is `4`; when `cpu num` is greater than `16`, the default value is `8`. + Minimum value: `1` ### `low-concurrency` + The allowable number of concurrent threads that handle low-priority `read` requests -+ Default value: `4` ++ When `8` ≤ `cpu num` ≤ `16`, the default value is `cpu_num * 0.5`; when `cpu num` is greater than `8`, the default value is `4`; when `cpu num` is greater than `16`, the default value is `8`. + Minimum value: `1` ### `max-tasks-per-worker-high` @@ -187,9 +197,11 @@ Configuration items related to storage thread pool. ### `stack-size` + The stack size of threads in the Storage read thread pool ++ Type: Integer + Unit + Default value: `"10MB"` + Unit: KB|MB|GB + Minimum value: `"2MB"` ++ Maximum value: The number of Kbytes output in the result of the `ulimit -sH` command executed in the system. ## `readpool.coprocessor` @@ -239,9 +251,11 @@ Configuration items related to the Coprocessor thread pool. ### `stack-size` + The stack size of the thread in the Coprocessor thread pool ++ Type: Integer + Unit + Default value: `"10MB"` + Unit: KB|MB|GB + Minimum value: `"2MB"` ++ Maximum value: The number of Kbytes output in the result of the `ulimit -sH` command executed in the system. ## storage @@ -250,12 +264,12 @@ Configuration items related to storage ### `scheduler-concurrency` + A built-in memory lock mechanism to prevent simultaneous operations on a key. Each key has a hash in a different slot. -+ Default value: `2048000` ++ Default value: `524288` + Minimum value: `1` ### `scheduler-worker-pool-size` -+ The number of `scheduler` threads, mainly used for checking transaction consistency before data writing ++ The number of `scheduler` threads, mainly used for checking transaction consistency before data writing. If the number of CPU cores is greater than or equal to `16`, the default value is `8`; otherwise, the default value is `4`. + Default value: `4` + Minimum value: `1` @@ -268,9 +282,21 @@ Configuration items related to storage ### `reserve-space` + The size of the temporary file that preoccupies the extra space when TiKV is started. The name of temporary file is `space_placeholder_file`, located in the `storage.data-dir` directory. When TiKV runs out of disk space and cannot be started normally, you can delete this file as an emergency intervention and set `reserve-space` to `"0MB"`. -+ Default value: `"2GB"` ++ Default value: `"5GB"` + Unite: MB|GB +### `enable-ttl` + ++ TTL is short for "Time to live". If this item is enabled, TiKV automatically deletes data that reaches its TTL. To set the value of TTL, you need to specify it in the requests when writing data via the client. If the TTL is not specified, it means that TiKV does not automatically delete the corresponding data. ++ Note: The TTL feature is only available for the RawKV interface for now. You can only configure this feature when creating a new cluster because TTL uses different data formats in the storage layer. If you modify this item on an existing cluster, TiKV reports errors when it starts. ++ Default value: `false` + +### `ttl-check-poll-interval` + ++ The interval of checking data to reclaim physical spaces. If data reaches its TTL, TiKV forcibly reclaims its physical space during the check. ++ Default value: `"12h"` ++ Minimum value: `"0s"` + ## storage.block-cache Configuration items related to the sharing of block cache among multiple RocksDB Column Families (CF). When these configuration items are enabled, block cache separately configured for each column family is disabled. @@ -286,6 +312,21 @@ Configuration items related to the sharing of block cache among multiple RocksDB + Default value: 45% of the size of total system memory + Unit: KB|MB|GB +## storage.io-rate-limit + +Configuration items related to the I/O rate limiter. + +### `max-bytes-per-sec` + ++ Limits the maximum I/O bytes that a server can write to or read from the disk (determined by the `mode` configuration item below) in one second. When this limit is reached, TiKV prefers throttling background operations over foreground ones. The value of this configuration item should be set to the disk's optimal I/O bandwidth, for example, the maximum I/O bandwidth specified by your cloud disk vendor. When this configuration value is set to zero, disk I/O operations are not limited. ++ Default value: `"0MB"` + +### `mode` + ++ Determines which types of I/O operations are counted and restrained below the `max-bytes-per-sec` threshold. Currently, only the write-only mode is supported. ++ Optional value: `"write-only"` ++ Default value: `"write-only"` + ## raftstore Configuration items related to Raftstore @@ -330,7 +371,7 @@ Configuration items related to Raftstore + Default value: `0` + Minimum value: `0` -### `raft-max-size-per-message` +### `raft-max-size-per-msg` + The soft limit on the size of a single message packet + Default value: `"1MB"` @@ -386,10 +427,10 @@ Configuration items related to Raftstore + Default value: `"3s"` + Minimum value: `0` -### `raftstore.hibernate-regions` (**Experimental**) +### `hibernate-regions` + Enables or disables Hibernate Region. When this option is enabled, a Region idle for a long time is automatically set as hibernated. This reduces the extra overhead caused by heartbeat messages between the Raft leader and the followers for idle Regions. You can use `raftstore.peer-stale-state-check-interval` to modify the heartbeat interval between the leader and the followers of hibernated Regions. -+ Default value: true ++ Default value: `true` in v5.0.2 and later versions; `false` in versions before v5.0.2 ### `raftstore.peer-stale-state-check-interval` @@ -414,12 +455,6 @@ Configuration items related to Raftstore + Default value: `"5m"` + Minimum value: `0` -### `clean-stale-peer-delay` - -+ Delays the time in deleting expired replica data -+ Default value: `"10m"` -+ Minimum value: `0` - ### `region-compact-check-step` + The number of Regions checked at one time for each round of manual compaction @@ -454,7 +489,7 @@ Configuration items related to Raftstore ### `snap-mgr-gc-tick-interval` + The time interval at which the recycle of expired snapshot files is triggered. `0` means that this feature is disabled. -+ Default value: `"5s"` ++ Default value: `"1m"` + Minimum value: `0` ### `snap-gc-timeout` @@ -492,8 +527,8 @@ Configuration items related to Raftstore ### `max-peer-down-duration` + The longest inactive duration allowed for a peer. A peer with timeout is marked as `down`, and PD tries to delete it later. -+ Default value: `"5m"` -+ Minimum value: `0` ++ Default value: `"10m"` ++ Minimum value: When Hibernate Region is enabled, the minimum value is `peer-stale-check-interval * 2`; when Hibernate Region is disabled, the minimum value is `0`. ### `max-leader-missing-duration` @@ -516,7 +551,7 @@ Configuration items related to Raftstore ### `leader-transfer-max-log-lag` + The maximum number of missing logs allowed for the transferee during a Raft leader transfer -+ Default value: `10` ++ Default value: `128` + Minimum value: `10` ### `snap-apply-batch-size` @@ -552,7 +587,7 @@ Configuration items related to Raftstore ### `merge-check-tick-interval` + The time interval at which TiKV checks whether a Region needs merge -+ Default value: `"10s"` ++ Default value: `"2s"` + Minimum value: greater than `0` ### `use-delete-range` @@ -575,7 +610,7 @@ Configuration items related to Raftstore ### `apply-max-batch-size` + The maximum number of requests for data flushing in one batch -+ Default value: `1024` ++ Default value: `256` + Minimum value: greater than `0` ### `apply-pool-size` @@ -587,7 +622,7 @@ Configuration items related to Raftstore ### `store-max-batch-size` + The maximum number of requests processed in one batch -+ Default value: `1024` ++ If `hibernate-regions` is enabled, the default value is `256`. If `hibernate-regions` is disabled, the default value is `1024`. + Minimum value: greater than `0` ### `store-pool-size` @@ -609,7 +644,7 @@ Configuration items related to Coprocessor ### `split-region-on-table` + Determines whether to split Region by table. It is recommended for you to use the feature only in TiDB mode. -+ Default value: `true` ++ Default value: `false` ### `batch-split-limit` @@ -647,7 +682,7 @@ Configuration items related to RocksDB + The number of background threads in RocksDB + Default value: `8` -+ Minimum value: `1` ++ Minimum value: `2` ### `max-background-flushes` @@ -713,8 +748,8 @@ Configuration items related to RocksDB ### `stats-dump-period` -+ Enables or disables Pipelined Write -+ Default value: `true` ++ The interval at which statistics are output to the log. ++ Default value: `10m` ### `compaction-readahead-size` @@ -750,9 +785,9 @@ Configuration items related to RocksDB + Minimum value: `1` + Maximum value: `3` -### `rate-limiter-auto-tuned` +### `rate-limiter-auto-tuned` New in v5.0 -+ Determines whether to automatically optimize the configuration of the RocksDB's compaction rate limiter ++ Determines whether to automatically optimize the configuration of the RocksDB's compaction rate limiter based on recent workload. When this configuration is enabled, compaction pending bytes will be slightly higher than usual. + Default value: `true` ### `enable-pipelined-write` @@ -819,24 +854,27 @@ Configuration items related to Titan ### `max-background-gc` + The maximum number of GC threads in Titan -+ Default value: `1` ++ Default value: `4` + Minimum value: `1` -## rocksdb.defaultcf +## rocksdb.defaultcf | rocksdb.writecf | rocksdb.lockcf -Configuration items related to `rocksdb.defaultcf` +Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rocksdb.lockcf`. ### `block-size` + The default size of a RocksDB block -+ Default value: `"64KB"` ++ Default value for `defaultcf` and `writecf`: `"64KB"` ++ Default value for `lockcf`: `"16KB"` + Minimum value: `"1KB"` + Unit: KB|MB|GB ### `block-cache-size` + The cache size of a RocksDB block -+ Default value: `Total machine memory * 25%` ++ Default value for `defaultcf`: `Total machine memory * 25%` ++ Default value for `writecf`: `Total machine memory * 15%` ++ Default value for `lockcf`: `Total machine memory * 2%` + Minimum value: `0` + Unit: KB|MB|GB @@ -852,7 +890,7 @@ Configuration items related to `rocksdb.defaultcf` ### `pin-l0-filter-and-index-blocks` -+ Determines whether to pin the index and filter at L0 ++ Determines whether to pin the index and filter blocks of the level 0 SST files in memory. + Default value: `true` ### `use-bloom-filter` @@ -863,12 +901,14 @@ Configuration items related to `rocksdb.defaultcf` ### `optimize-filters-for-hits` + Determines whether to optimize the hit ratio of filters -+ Default value: `true` ++ Default value for `defaultcf`: `true` ++ Default value for `writecf` and `lockcf`: `false` -### `whole_key_filtering` +### `whole-key-filtering` + Determines whether to put the entire key to bloom filter -+ Default value: `true` ++ Default value for `defaultcf` and `lockcf`: `true` ++ Default value for `writecf`: `false` ### `bloom-filter-bits-per-key` @@ -892,7 +932,8 @@ Configuration items related to `rocksdb.defaultcf` + The default compression algorithm for each level + Optional values: ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] -+ Default value: `No` for the first two levels, and `lz4` for the next five levels ++ Default value for `defaultcf` and `writecf`: ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] ++ Default value for `lockcf`: ["no", "no", "no", "no", "no", "no", "no"] ### `bottommost-level-compression` @@ -904,7 +945,8 @@ Configuration items related to `rocksdb.defaultcf` ### `write-buffer-size` + Memtable size -+ Default value: `"128MB"` ++ Default value for `defaultcf` and `writecf`: `"128MB"` ++ Default value for `lockcf`: `"32MB"` + Minimum value: `0` + Unit: KB|MB|GB @@ -923,7 +965,8 @@ Configuration items related to `rocksdb.defaultcf` ### `max-bytes-for-level-base` + The maximum number of bytes at base level (L1). Generally, it is set to 4 times the size of a memtable. -+ Default value: `"512MB"` ++ Default value for `defaultcf` and `writecf`: `"512MB"` ++ Default value for `lockcf`: `"128MB"` + Minimum value: `0` + Unit: KB|MB|GB @@ -937,7 +980,8 @@ Configuration items related to `rocksdb.defaultcf` ### `level0-file-num-compaction-trigger` + The maximum number of files at L0 that trigger compaction -+ Default value: `4` ++ Default value for `defaultcf` and `writecf`: `4` ++ Default value for `lockcf`: `1` + Minimum value: `0` ### `level0-slowdown-writes-trigger` @@ -962,8 +1006,9 @@ Configuration items related to `rocksdb.defaultcf` ### `compaction-pri` + The priority type of compaction -+ Optional values: `3` (`MinOverlappingRatio`), `0` (`ByCompensatedSize`), `1` (`OldestLargestSeqFirst`), `2` (`OldestSmallestSeqFirst`) -+ Default value: `3` ++ Optional values: `0` (`ByCompensatedSize`), `1` (`OldestLargestSeqFirst`), `2` (`OldestSmallestSeqFirst`), `3` (`MinOverlappingRatio`) ++ Default value for `defaultcf` and `writecf`: `3` ++ Default value for `lockcf`: `0` ### `dynamic-level-bytes` @@ -980,7 +1025,7 @@ Configuration items related to `rocksdb.defaultcf` + The default amplification multiple for each layer + Default value: `10` -### `rocksdb.defaultcf.compaction-style` +### `compaction-style` + Compaction method + Optional values: `"level"`, `"universal"` @@ -994,7 +1039,7 @@ Configuration items related to `rocksdb.defaultcf` ### `soft-pending-compaction-bytes-limit` + The soft limit on the pending compaction bytes -+ Default value: `"64GB"` ++ Default value: `"192GB"` + Unit: KB|MB|GB ### `hard-pending-compaction-bytes-limit` @@ -1006,7 +1051,8 @@ Configuration items related to `rocksdb.defaultcf` ### `enable-compaction-guard` + Enables or disables the compaction guard, which is an optimization to split SST files at TiKV Region boundaries. This optimization can help reduce compaction I/O and allows TiKV to use larger SST file size (thus less SST files overall) and at the time efficiently clean up stale data when migrating Regions. -+ Default value: `true` ++ Default value for `defaultcf` and `writecf`: `true` ++ Default value for `lockcf`: `false` ### `compaction-guard-min-output-file-size` @@ -1020,9 +1066,9 @@ Configuration items related to `rocksdb.defaultcf` + Default value: `"128MB"` + Unit: KB|MB|GB -## `rocksdb.defaultcf.titan` +## rocksdb.defaultcf.titan -Configuration items related to `rocksdb.defaultcf.titan` +Configuration items related to `rocksdb.defaultcf.titan`. ### `min-blob-size` @@ -1098,72 +1144,15 @@ Configuration items related to `rocksdb.defaultcf.titan` + Determines whether to use the merge operator to write back blob indexes for Titan GC. When `gc-merge-rewrite` is enabled, it reduces the effect of Titan GC on the writes in the foreground. + Default value: `false` -## rocksdb.writecf - -Configuration items related to `rocksdb.writecf` - -### `block-cache-size` - -+ Block cache size -+ Default value: `Total machine memory * 15%` -+ Unit: MB|GB - -### `optimize-filters-for-hits` - -+ Determines whether to optimize the hit ratio of the filter -+ Default value: `false` - -### `whole-key-filtering` - -+ Determines whether to put the entire key to bloom filter -+ Default value: `false` - -### `enable-compaction-guard` - -+ Enables or disables the compaction guard, which is an optimization to split SST files at TiKV Region boundaries. This optimization can help reduce compaction I/O and allows TiKV to use larger SST file size (thus less SST files overall) and at the time efficiently clean up stale data when migrating Regions. -+ Default value: `true` - -### `compaction-guard-min-output-file-size` - -+ The minimum SST file size when the compaction guard is enabled. This configuration prevents SST files from being too small when the compaction guard is enabled. -+ Default value: `"8MB"` -+ Unit: KB|MB|GB - -### `compaction-guard-max-output-file-size` - -+ The maximum SST file size when the compaction guard is enabled. The configuration prevents SST files from being too large when the compaction guard is enabled. This configuration overrides `target-file-size-base` for the same column family. -+ Default value: `"128MB"` -+ Unit: KB|MB|GB - -## rocksdb.lockcf - -Configuration items related to `rocksdb.lockcf` - -### `block-cache-size` - -+ Block cache size -+ Default value: `Total machine memory * 2%` -+ Unit: MB|GB - -### `optimize-filters-for-hits` - -+ Determines whether to optimize the hit ratio of the filter -+ Default value: `false` - -### `level0-file-num-compaction-trigger` - -+ The number of files at L0 required to trigger compaction -+ Default value: `1` - -## `raftdb` +## raftdb Configuration items related to `raftdb` ### `max-background-jobs` + The number of background threads in RocksDB -+ Default value: `2` -+ Minimum value: `1` ++ Default value: `4` ++ Minimum value: `2` ### `max-sub-compactions` @@ -1195,7 +1184,7 @@ Configuration items related to security + The path of the PEM file that contains the X509 key + Default value: "" -### `redact-info-log` +### `redact-info-log` New in v4.0.8 + This configuration item enables or disables log redaction. If the configuration value is set to `true`, all user data in the log will be replaced by `?`. + Default value: `false` @@ -1248,10 +1237,10 @@ Configuration items related to TiDB Lightning import and BR restore. ## gc -### `enable-compaction-filter` +### `enable-compaction-filter` New in v5.0 + Controls whether to enable the GC in Compaction Filter feature -+ Default value: `true` ++ Default value: `false` ## backup @@ -1263,6 +1252,60 @@ Configuration items related to BR backup. + Default value: `MIN(CPU * 0.75, 32)`. + Minimum value: `1` +## cdc + +Configuration items related to TiCDC. + +### `min-ts-interval` + ++ The interval at which Resolved TS is calculated and forwarded. ++ Default value: `"1s"` + +### `old-value-cache-memory-quota` + ++ The upper limit of memory usage by TiCDC old values. ++ Default value: `512MB` + +### `sink-memory-quota` + ++ The upper limit of memory usage by TiCDC data change events. ++ Default value: `512MB` + +### `incremental-scan-speed-limit` + ++ The maximum speed at which historical data is incrementally scanned. ++ Default value: `"128MB"`, which means 128 MB per second. + +### `incremental-scan-threads` + ++ The number of threads for the task of incrementally scanning historical data. ++ Default value: `4`, which means 4 threads. + +### `incremental-scan-concurrency` + ++ The maximum number of concurrent executions for the tasks of incrementally scanning historical data. ++ Default value: `6`, which means 6 tasks can be concurrent executed at most. ++ Note: The value of `incremental-scan-concurrency` must be greater than or equal to that of `incremental-scan-threads`; otherwise, TiKV will report an error at startup. + +## resolved-ts + +Configuration items related to maintaining the Resolved TS to serve Stale Read requests. + +### `enable` + ++ Determines whether to maintain the Resolved TS for all Regions. ++ Default value: `true` + +### `advance-ts-interval` + ++ The interval at which Resolved TS is calculated and forwarded. ++ Default value: `"1s"` + +### `scan-lock-pool-size` + ++ The number of threads that TiKV uses to scan the MVCC (multi-version concurrency control) lock data when initializing the Resolved TS. ++ Default value: `2`, which means 2 threads. + ## pessimistic-txn For pessimistic transaction usage, refer to [TiDB Pessimistic Transaction Mode](/pessimistic-transaction.md). @@ -1280,6 +1323,5 @@ For pessimistic transaction usage, refer to [TiDB Pessimistic Transaction Mode]( ### `pipelined` -This configuration item enables the pipelined process of adding the pessimistic lock. With this feature enabled, after detecting that data can be locked, TiKV immediately notifies TiDB to execute the subsequent requests and write the pessimistic lock asynchronously, which reduces most of the latency and significantly improves the performance of pessimistic transactions. But there is a still low probability that the asynchronous write of the pessimistic lock fails, which might cause the failure of pessimistic transaction commits. - -The default value of `pipelined` is `false`. +- This configuration item enables the pipelined process of adding the pessimistic lock. With this feature enabled, after detecting that data can be locked, TiKV immediately notifies TiDB to execute the subsequent requests and write the pessimistic lock asynchronously, which reduces most of the latency and significantly improves the performance of pessimistic transactions. But there is a still low probability that the asynchronous write of the pessimistic lock fails, which might cause the failure of pessimistic transaction commits. +- Default value: `true` diff --git a/tikv-control.md b/tikv-control.md index ef5560e5bca8..fd70acaf1b16 100644 --- a/tikv-control.md +++ b/tikv-control.md @@ -8,11 +8,14 @@ aliases: ['/docs/dev/tikv-control/','/docs/dev/reference/tools/tikv-control/'] TiKV Control (`tikv-ctl`) is a command line tool of TiKV, used to manage the cluster. Its installation directory is as follows: -* If the cluster is deployed using TiDB Ansible, `tikv-ctl` directory is in the `resources/bin` subdirectory under the `ansible` directory. * If the cluster is deployed using TiUP, `tikv-ctl` directory is in the in `~/.tiup/components/ctl/{VERSION}/` directory. ## Use TiKV Control in TiUP +> **Note:** +> +> It is recommended that the version of the Control tool you use is consistent with the version of the cluster. + `tikv-ctl` is also integrated in the `tiup` command. Execute the following command to call the `tikv-ctl` tool: {{< copyable "shell-regular" >}} @@ -186,8 +189,7 @@ In this command, the key is also the escaped form of raw key. The `raw-scan` command scans directly from the RocksDB. Note that to scan data keys you need to add a `'z'` prefix to keys. -Use `--from` and `--to` options to specify the range to scan (unbounded by default). Use `--limit` to limit at most how -many keys to print out (30 by default). Use `--cf` to specify which cf to scan (can be `default`, `write` or `lock`). +Use `--from` and `--to` options to specify the range to scan (unbounded by default). Use `--limit` to limit at most how many keys to print out (30 by default). Use `--cf` to specify which cf to scan (can be `default`, `write` or `lock`). ```bash $ ./tikv-ctl --db /var/lib/tikv/db/ raw-scan --from 'zt' --limit 2 --cf default @@ -223,11 +225,15 @@ The properties can be used to check whether the Region is healthy or not. If not ### Compact data of each TiKV manually -Use the `compact` command to manually compact data of each TiKV. If you specify the `--from` and `--to` options, then their flags are also in the form of escaped raw key. +Use the `compact` command to manually compact data of each TiKV. If you specify the `--from` and `--to` options, then their flags are also in the form of escaped raw key. -- Use the `--host` option to specify the TiKV that you need to compact. -- Use the `-d` option to specify the RocksDB that you need to compact. The optional values are `kv` and `raft`. -- Use the `--threads` option allows you to specify the concurrency that you compact and its default value is 8. Generally, a higher concurrency comes with a faster compact speed, which might yet affect the service. You need to choose an appropriate concurrency based on the scenario. +- Use the `--host` option to specify the TiKV that needs to perform compaction. +- Use the `-d` option to specify the RocksDB that performs compaction. The optional values are `kv` and `raft`. +- Use the `--threads` option allows you to specify the concurrency for the TiKV compaction and its default value is `8`. Generally, a higher concurrency comes with a faster compaction speed, which might yet affect the service. You need to choose an appropriate concurrency count based on your scenario. +- Use the `--bottommost` option to include or exclude the bottommost files when TiKV performs compaction. The value options are `default`, `skip`, and `force`. The default value is `default`. + - `default` means that the bottommost files are included only when the Compaction Filter feature is enabled. + - `skip` means that the bottommost files are excluded when TiKV performs compaction. + - `force` means that the bottommost files are always included when TiKV performs compaction. ```bash $ tikv-ctl --db /path/to/tikv/db compact -d kv @@ -377,6 +383,28 @@ tikv-ctl --host ip:port modify-tikv-config -n raftstore.sync-log -v false success ``` +When the compaction rate limit causes accumulated compaction pending bytes, disable the `rate-limiter-auto-tuned` mode or set a higher limit for the compaction flow: + +{{< copyable "shell-regular" >}} + +```shell +tikv-ctl --host ip:port modify-tikv-config -n rocksdb.rate-limiter-auto-tuned -v false +``` + +``` +success +``` + +{{< copyable "shell-regular" >}} + +```shell +tikv-ctl --host ip:port modify-tikv-config -n rocksdb.rate-bytes-per-sec -v "1GB" +``` + +``` +success +``` + ### Force Region to recover the service from failure of multiple replicas Use the `unsafe-recover remove-fail-stores` command to remove the failed machines from the peer list of Regions. Then after you restart TiKV, these Regions can continue to provide services using the other healthy replicas. This command is usually used in circumstances where multiple TiKV stores are damaged or deleted. @@ -391,7 +419,8 @@ success! > **Note:** > > - This command only supports the local mode. It prints `success!` when successfully run. -> - You must run this command for all stores where specified Regions' peers are located. If `-r` is not set, all Regions are involved, and you need to run this command for all stores. +> - You must run this command for all stores where specified Regions' peers are located. +> - If the `--all-regions` option is used, usually you need to run this command on all the remaining healthy stores in the cluster. You need to ensure that the healthy stores stop providing services before recovering the damaged stores. Otherwise, the inconsistent peer lists in Region replicas will cause errors when you execute `split-region` or `remove-peer`. This further causes inconsistency between other metadata, and finally, the Regions will become unavailable. ### Recover from MVCC data corruption @@ -413,8 +442,7 @@ success! ### Ldb Command -The `ldb` command line tool offers multiple data access and database administration commands. Some examples are listed below. -For more information, refer to the help message displayed when running `tikv-ctl ldb` or check the documents from RocksDB. +The `ldb` command line tool offers multiple data access and database administration commands. Some examples are listed below. For more information, refer to the help message displayed when running `tikv-ctl ldb` or check the documents from RocksDB. Examples of data access sequence: @@ -447,7 +475,7 @@ data-dir = "/path/to/tikv/data" ``` The `--path` option can be used to specify an absolute or relative path to the data file of interest. The command might give empty output if the data file is not encrypted. If `--path` is not provided, encryption info for all data files will be printed. - + ```bash $ tikv-ctl --config=./conf.toml encryption-meta dump-file --path=/path/to/tikv/data/db/CURRENT /path/to/tikv/data/db/CURRENT: key_id: 9291156302549018620 iv: E3C2FDBF63FC03BFC28F265D7E78283F method: Aes128Ctr diff --git a/tikv-overview.md b/tikv-overview.md index df672d12aed8..16662510c2dd 100644 --- a/tikv-overview.md +++ b/tikv-overview.md @@ -15,8 +15,7 @@ TiKV implements the multi-raft-group replica mechanism based on the design of Go ### Region and RocksDB -There is a RocksDB database within each Store and it stores data into the local disk. All the Region data are stored in the same RocksDB instance in each Store. All the logs used for the Raft consensus algorithm is stored in another RocksDB instance in each Store. This is because the performance of sequential I/O is -better than random I/O. With different RocksDB instances storing raft logs and Region data, TiKV combines all the data write operations of raft logs and TiKV Regions into one I/O operation to improve the performance. +There is a RocksDB database within each Store and it stores data into the local disk. All the Region data are stored in the same RocksDB instance in each Store. All the logs used for the Raft consensus algorithm is stored in another RocksDB instance in each Store. This is because the performance of sequential I/O is better than random I/O. With different RocksDB instances storing raft logs and Region data, TiKV combines all the data write operations of raft logs and TiKV Regions into one I/O operation to improve the performance. ### Region and Raft Consensus Algorithm diff --git a/tispark-deployment-topology.md b/tispark-deployment-topology.md index 0f0489be2729..53da7b9dc48d 100644 --- a/tispark-deployment-topology.md +++ b/tispark-deployment-topology.md @@ -13,7 +13,7 @@ This document introduces the TiSpark deployment topology and how to deploy TiSpa TiSpark is a component built for running Apache Spark on top of TiDB/TiKV to answer complex OLAP queries. It brings benefits of both the Spark platform and the distributed TiKV cluster to TiDB and makes TiDB a one-stop solution for both online transactions and analytics. -For more information about TiSpark, see [TiSpark User Guide](/tispark-overview.md). +For more information about the TiSpark architecture and how to use it, see [TiSpark User Guide](/tispark-overview.md). ## Topology information @@ -30,6 +30,8 @@ For more information about TiSpark, see [TiSpark User Guide](/tispark-overview.m - [Simple TiSpark topology template](https://github.com/pingcap/docs/blob/master/config-templates/simple-tispark.yaml) - [Complex TiSpark topology template](https://github.com/pingcap/docs/blob/master/config-templates/complex-tispark.yaml) +For detailed descriptions of the configuration items in the above TiDB cluster topology file, see [Topology Configuration File for Deploying TiDB Using TiUP](/tiup/tiup-cluster-topology-reference.md). + > **Note:** > > - You do not need to manually create the `tidb` user in the configuration file. The TiUP cluster component automatically creates the `tidb` user on the target machines. You can customize the user, or keep the user consistent with the control machine. diff --git a/tispark-overview.md b/tispark-overview.md index 7668bd8eece6..8f6c36c3f69e 100644 --- a/tispark-overview.md +++ b/tispark-overview.md @@ -66,6 +66,13 @@ spark.tispark.pd.addresses $your_pd_servers spark.sql.extensions org.apache.spark.sql.TiExtensions ``` +Add the following configuration in the `CDH` spark version: + +``` +spark.tispark.pd.addresses=$your_pd_servers +spark.sql.extensions=org.apache.spark.sql.TiExtensions +``` + `your_pd_servers` are comma-separated PD addresses, with each in the format of `$your_pd_address:$port`. For example, when you have multiple PD servers on `10.16.20.1,10.16.20.2,10.16.20.3` with the port 2379, put it as `10.16.20.1:2379,10.16.20.2:2379,10.16.20.3:2379`. @@ -90,8 +97,6 @@ spark-shell --jars $TISPARK_FOLDER/tispark-${name_with_version}.jar If you do not have a Spark cluster, we recommend using the standalone mode. To use the Spark Standalone model, you can simply place a compiled version of Spark on each node of the cluster. If you encounter problems, see its [official website](https://spark.apache.org/docs/latest/spark-standalone.html). And you are welcome to [file an issue](https://github.com/pingcap/tispark/issues/new) on our GitHub. -If you are using TiDB Ansible to deploy a TiDB cluster, you can also use TiDB Ansible to deploy a Spark standalone cluster, and TiSpark is also deployed at the same time. - #### Download and install You can download [Apache Spark](https://spark.apache.org/downloads.html) @@ -166,8 +171,7 @@ spark-sql> select count(*) from lineitem; Time taken: 0.673 seconds, Fetched 1 row(s) ``` -For JDBC connection with Thrift Server, you can try it with various JDBC supported tools including SQuirreLSQL and hive-beeline. -For example, to use it with beeline: +For JDBC connection with Thrift Server, you can try it with various JDBC supported tools including SQuirreLSQL and hive-beeline. For example, to use it with beeline: ```sh ./beeline @@ -202,9 +206,49 @@ tisparkDF.write.saveAsTable("hive_table") // save table to hive spark.sql("select * from hive_table a, tispark_table b where a.col1 = b.col1").show // join table across Hive and Tispark ``` +## Batch write DataFrames into TiDB using TiSpark + +Starting from v2.3, TiSpark natively supports batch writing DataFrames into TiDB clusters. This writing mode is implemented through the two-phase commit protocol of TiKV. + +Compared with the writing through Spark + JDBC, the TiSpark batch writing has the following advantages: + +| Aspects to compare | TiSpark batch writes | Spark + JDBC writes| +| ------- | --------------- | --------------- | +| Atomicity | The DataFrames either are all written successfully or all fail to write. | If the Spark task fails and exits during the writing process, a part of the data might be written successfully. | +| Isolation | During the writing process, the data being written is invisible to other transactions. | During the writing process, some successfully written data is visible to other transactions. | +| Error recovery | If the batch write fails, you only need to re-run Spark. | An application is required to achieve idempotence. For example, if the batch write fails, you need to clean up the part of the successfully written data and re-run Spark. You need to set `spark.task.maxFailures=1` to prevent data duplication caused by task retry. | +| Speed | Data is directly written into TiKV, which is faster. | Data is written to TiKV through TiDB, which affects the speed. | + +The following example shows how to batch write data using TiSpark via the scala API: + +```scala +// select data to write +val df = spark.sql("select * from tpch.ORDERS") + +// write data to tidb +df.write. + format("tidb"). + option("tidb.addr", "127.0.0.1"). + option("tidb.port", "4000"). + option("tidb.user", "root"). + option("tidb.password", ""). + option("database", "tpch"). + option("table", "target_orders"). + mode("append"). + save() +``` + +If the amount of data to write is large and the writing time exceeds ten minutes, you need to ensure that the GC time is longer than the writing time. + +```sql +update mysql.tidb set VARIABLE_VALUE="6h" where VARIABLE_NAME="tikv_gc_life_time"; +``` + +Refer to [this document](https://github.com/pingcap/tispark/blob/master/docs/datasource_api_userguide.md) for details. + ## Load Spark Dataframe into TiDB using JDBC -TiSpark does not provide a direct way of loading data into your TiDB cluster, but you can load data using JDBC like this: +In addition to using TiSpark to batch write DataFrames into the TiDB cluster, you can also use Spark's native JDBC support for the data writing: ```scala import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions diff --git a/tiup/tiup-bench.md b/tiup/tiup-bench.md index c7b3c3803880..296b0a078551 100644 --- a/tiup/tiup-bench.md +++ b/tiup/tiup-bench.md @@ -15,7 +15,7 @@ tiup bench ``` ``` -Starting component `bench`: /Users/joshua/.tiup/components/bench/v0.0.1/bench +Starting component `bench`: /home/tidb/.tiup/components/bench/v1.5.0/bench Benchmark database with different workloads Usage: @@ -35,8 +35,8 @@ Flags: -H, --host string Database host (default "127.0.0.1") --ignore-error Ignore error when running workload --interval duration Output interval time (default 10s) - --isolation int Isolation Level 0: Default, 1: ReadUncommitted, - 2: ReadCommitted, 3: WriteCommitted, 4: RepeatableRead, + --isolation int Isolation Level 0: Default, 1: ReadUncommitted, + 2: ReadCommitted, 3: WriteCommitted, 4: RepeatableRead, 5: Snapshot, 6: Serializable, 7: Linerizable --max-procs int runtime.GOMAXPROCS -p, --password string Database password @@ -75,6 +75,8 @@ Flags: 1. Create 4 warehouses using 4 partitions via hash: + {{< copyable "shell-regular" >}} + ```shell tiup bench tpcc --warehouses 4 --parts 4 prepare ``` @@ -163,7 +165,7 @@ Flags: ```shell tiup bench tpch --sf=1 --check=true run ``` - + - If you do not check the result, run this command: {{< copyable "shell-regular" >}} @@ -171,7 +173,7 @@ Flags: ```shell tiup bench tpch --sf=1 run ``` - + 3. Clean up data: {{< copyable "shell-regular" >}} diff --git a/tiup/tiup-cluster-topology-reference.md b/tiup/tiup-cluster-topology-reference.md new file mode 100644 index 000000000000..795dbe68d1e1 --- /dev/null +++ b/tiup/tiup-cluster-topology-reference.md @@ -0,0 +1,791 @@ +--- +title: Topology Configuration File for TiDB Deployment Using TiUP +--- + +# Topology Configuration File for TiDB Deployment Using TiUP + +To deploy or scale TiDB using TiUP, you need to provide a topology file ([sample](https://github.com/pingcap/tiup/blob/master/embed/templates/examples/topology.example.yaml)) to describe the cluster topology. + +Similarly, to modify the cluster topology, you need to modify the topology file. The difference is that, after the cluster is deployed, you can only modify a part of the fields in the topology file. This document introduces each section of the topology file and each field in each section. + +## File structure + +A topology configuration file for TiDB deployment using TiUP might contain the following sections: + +- [global](#global): The cluster's global configuration. Some of the configuration items use the default values and you can configure them separately in each instance. +- [monitored](#monitored): Configuration for monitoring services, namely, the blackbox_exporter and the `node_exporter`. On each machine, a `node_exporter` and a `blackbox_exporter` are deployed. +- [server_configs](#server_configs): Components' global configuration. You can configure each component separately. If an instance has a configuration item with the same name, the instance's configuration item will take effect. +- [pd_servers](#pd_servers): The configuration of the PD instance. This configuration specifies the machines to which the PD component is deployed. +- [tidb_servers](#tidb_servers): The configuration of the TiDB instance. This configuration specifies the machines to which the TiDB component is deployed. +- [tikv_servers](#tikv_servers): The configuration of the TiKV instance. This configuration specifies the machines to which the TiKV component is deployed. +- [tiflash_servers](#tiflash_servers): The configuration of the TiFlash instance. This configuration specifies the machines to which the TiFlash component is deployed. +- [pump_servers](#pump_servers): The configuration of the Pump instance. This configuration specifies the machines to which the Pump component is deployed. +- [drainer_servers](#drainer_servers): The configuration of the Drainer instance. This configuration specifies the machines to which the Drainer component is deployed. +- [cdc_servers](#cdc_servers): The configuration of the TiCDC instance. This configuration specifies the machines to which the TiCDC component is deployed. +- [tispark_masters](#tispark_masters): The configuration of the TiSpark master instance. This configuration specifies the machines to which the TiSpark master component is deployed. Only one node of TiSpark master can be deployed. +- [tispark_workers](#tispark_workers): The configuration of the TiSpark worker instance. This configuration specifies the machines to which the TiSpark worker component is deployed. +- [monitoring_servers](#monitoring_servers): Specifies the machines to which Prometheus is deployed. TiUP supports deploying multiple Prometheus instances but only the first instance is used. +- [grafana_servers](#grafana_servers): The configuration of the Grafana instance. This configuration specifies the machines to which Grafana is deployed. +- [alertmanager_servers](#alertmanager_servers): The configuration of the Alertmanager instance. This configuration specifies the machines to which Alertmanager is deployed. + +### `global` + +The `global` section corresponds to the cluster's global configuration and has the following fields: + +- `user`: The user used to start the deployed cluster. The default value is `"tidb"`. If the user specified in the `` field does not exist on the target machine, this user is automatically created. + +- `group`: The user group to which a user belongs. It is specified when the user is created. The value defaults to that of the `` field. If the specified group does not exist, it is automatically created. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. The default value is `22`. + +- `enable_tls`: Specifies whether to enable TLS for the cluster. After TLS is enabled, the generated TLS certificate must be used for connections between components or between the client and the component. **Once it is enabled, it cannot be disabled**. The default value is `false`. + +- `deploy_dir`: The deployment directory of each component. The default value is `"deployed"`. Its application rules are as follows: + + - If the absolute path of `deploy_dir` is configured at the instance level, the actual deployment directory is `deploy_dir` configured for the instance. + + - For each instance, if you do not configure `deploy_dir`, its default value is the relative path `-`. + + - If `global.deploy_dir` is an absolute path, the component is deployed to the `/` directory. + + - If `global.deploy_dir` is a relative path, the component is deployed to the `/home///` directory. + +- `data_dir`: The data directory. Default value: `"data"`. Its application rules are as follows: + + - If the absolute path of `data_dir` is configured at the instance level, the actual deployment directory is `data_dir` configured for the instance. + + - For each instance, if you do not configure `data_dir`, its default value is ``. + + - If `data_dir` is a relative path, the component data is placed in `/`. For the calculation rules of ``, see the application rules of the `deploy_dir` field. + +- `log_dir`: The data directory. Default value: `"log"`. Its application rules are as follows: + + - If the absolute path `log_dir` is configured at the instance level, the actual log directory is the `log_dir` configured for the instance. + + - For each instance, if you not configure `log_dir`, its default value is ``. + + - If `log_dir` is a relative path, the component log is placed in `/`. For the calculation rules of ``, see the application rules of the `deploy_dir` field. + +- `os`: The operating system of the target machine. The field controls which operating system to adapt to for the components pushed to the target machine. The default value is "linux". + +- `arch`: The CPU architecture of the target machine. The field controls which platform to adapt to for the binary packages pushed to the target machine. The supported values are "amd64" and "arm64". The default value is "amd64". + +- `resource_control`: Runtime resource control. All configurations in this field are written into the service file of systemd. There is no limit by default. The resources that can be controlled are as follows: + + - `memory_limit`: Limits the maximum runtime memory. For example, "2G" means that the maximum memory of 2 GB can be used. + + - `cpu_quota`: Limits the maximum CPU usage at runtime. For example, "200%". + + - `io_read_bandwidth_max`: Limits the maximum I/O bandwidth for disk reads. For example, `"/dev/disk/by-path/pci-0000:00:1f.2-scsi-0:0:0:0 100M"`. + + - `io_write_bandwidth_max`: Limits maximum I/O bandwidth for disk writes. For example, `/dev/disk/by-path/pci-0000:00:1f.2-scsi-0:0:0:0 100M`. + + - `limit_core`: Controls the size of core dump. + +A `global` configuration example is as follows: + +```yaml +global: + user: "tidb" + resource_control: + memory_limit: "2G" +``` + +In the above configuration, the `tidb` user is used to start the cluster. At the same time, each component is restricted to a maximum of 2 GB of memory when it is running. + +### `monitored` + +`monitored` is used to configure the monitoring service on the target machine: [`node_exporter`](https://github.com/prometheus/node_exporter) and [`blackbox_exporter`](https://github.com/prometheus/blackbox_exporter). The following fields are included: + +- `node_exporter_port`: The service port of `node_exporter`. The default value is `9100`. + +- `blackbox_exporter_port`: The service port of `blackbox_exporter`. The default value is `9115`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`: Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +A `monitored` configuration example is as follows: + +```yaml +monitored: + node_exporter_port: 9100 + blackbox_exporter_port: 9115 +``` + +The above configuration specifies that `node_exporter` uses the `9100` port and `blackbox_exporter` uses the `9115` port. + +### `server_configs` + +`server_configs` is used to configure services and to generate configuration files for each component. Similar to the `global` section, the configuration of this section can be overwritten by the configurations with the same names in an instance. `server_configs` mainly includes the following fields: + +- `tidb`: TiDB service-related configuration. For the complete configuration, see [TiDB configuration file](/tidb-configuration-file.md). + +- `tikv`: TiKV service-related configuration. For the complete configuration, see [TiKV configuration file](/tikv-configuration-file.md). + +- `pd`: PD service-related configuration. For the complete configuration, see [PD configuration file](/pd-configuration-file.md). + +- `tiflash`: TiFlash service-related configuration. For the complete configuration, see [TiFlash configuration file](/tiflash/tiflash-configuration.md). + +- `tiflash_learner`: Each TiFlash node has a special built-in TiKV. This configuration item is used to configure this special TiKV. It is generally not recommended to modify the content under this configuration item. + +- `pump`: Pump service-related configuration. For the complete configuration, see [TiDB Binlog configuration file](/tidb-binlog/tidb-binlog-configuration-file.md#pump). + +- `drainer`: Drainer service-related configuration. For the complete configuration, see [TiDB Binlog configuration file](/tidb-binlog/tidb-binlog-configuration-file.md#drainer). + +- `cdc`: TiCDC service-related configuration. For the complete configuration, see [Deploy TiCDC](/ticdc/deploy-ticdc.md). + +A `server_configs` configuration example is as follows: + +```yaml +server_configs: + tidb: + run-ddl: true + lease: "45s" + split-table: true + token-limit: 1000 + tikv: + log-level: "info" + readpool.unified.min-thread-count: 1 +``` + +The above configuration specifies the global configuration of TiDB and TiKV. + +### `pd_servers` + +`pd_servers` specifies the machines to which PD services are deployed. It also specifies the service configuration on each machine. `pd_servers` is an array, and each element of the array contains the following fields: + +- `host`: Specifies the machine to which the PD services are deployed. The field value is an IP address and is mandatory. + +- `listen_host`: When the machine has multiple IP addresses, `listen_host` specifies the listening IP address of the service. The default value is `0.0.0.0`. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `name`: Specifies the name of the PD instance. Different instances must have unique names; otherwise, instances cannot be deployed. + +- `client_port`: Specifies the port that PD uses to connect to the client. The default value is `2379`. + +- `peer_port`: Specifies the port for communication between PDs. The default value is `2380`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`: Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config`: The configuration rule of this field is the same as the `pd` configuration rule in `server_configs`. If this field is configured, the field content is merged with the `pd` content in `server_configs` (if the two fields overlap, the content of this field takes effect). Then, a configuration file is generated and sent to the machine specified in `host`. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `listen_host` +- `name` +- `client_port` +- `peer_port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +A `pd_servers` configuration example is as follows: + +```yaml +pd_servers: + - host: 10.0.1.11 + config: + schedule.max-merge-region-size: 20 + schedule.max-merge-region-keys: 200000 + - host: 10.0.1.12 +``` + +The above configuration specifies that PD will be deployed on `10.0.1.11` and `10.0.1.12`, and makes specific configurations for the PD of `10.0.1.11`. + +### `tidb_servers` + +`tidb_servers` specifies the machines to which TiDB services are deployed. It also specifies the service configuration on each machine. `tidb_servers` is an array, and each element of the array contains the following fields: + +- `host`: Specifies the machine to which the TiDB services are deployed. The field value is an IP address and is mandatory. + +- `listen_host`: When the machine has multiple IP addresses, `listen_host` specifies the listening IP address of the service. The default value is `0.0.0.0`. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: The listening port of TiDB services, which is used to provide connection to the MySQL client. The default value is `4000`. + +- `status_port`: The listening port of the TiDB status service, which is used to view the status of the TiDB services from the external via HTTP requests. The default value is `10080`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config`: The configuration rule of this field is the same as the `tidb` configuration rule in `server_configs`. If this field is configured, the field content is merged with the `tidb` content in `server_configs` (if the two fields overlap, the content of this field takes effect). Then, a configuration file is generated and sent to the machine specified in `host`. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `listen_host` +- `port` +- `status_port` +- `deploy_dir` +- `log_dir` +- `arch` +- `os` + +A `tidb_servers` configuration example is as follows: + +```yaml +tidb_servers: + - host: 10.0.1.14 + config: + log.level: warn + log.slow-query-file: tidb-slow-overwrited.log + - host: 10.0.1.15 +``` + +### `tikv_servers` + +`tikv_servers` specifies the machines to which TiKV services are deployed. It also specifies the service configuration on each machine. `tikv_servers` is an array, and each element of the array contains the following fields: + +- `host`: Specifies the machine to which the TiKV services are deployed. The field value is an IP address and is mandatory. + +- `listen_host`: When the machine has multiple IP addresses, `listen_host` specifies the listening IP address of the service. The default value is `0.0.0.0`. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: The listening port of the TiKV services. The default value is `20160`. + +- `status_port`: The listening port of the TiKV status service. The default value is `20180`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`: Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config`: The configuration rule of this field is the same as the `tikv` configuration rule in `server_configs`. If this field is configured, the field content is merged with the `tikv` content in `server_configs` (if the two fields overlap, the content of this field takes effect). Then, a configuration file is generated and sent to the machine specified in `host`. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `listen_host` +- `port` +- `status_port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +A `tikv_servers` configuration example is as follows: + +```yaml +tikv_servers: + - host: 10.0.1.14 + config: + server.labels: { zone: "zone1", host: "host1" } + - host: 10.0.1.15 + config: + server.labels: { zone: "zone1", host: "host2" } +``` + +### `tiflash_servers` + +`tiflash_servers` specifies the machines to which TiFlash services are deployed. It also specifies the service configuration on each machine. This section is an array, and each element of the array contains the following fields: + +- `host`: Specifies the machine to which the TiFlash services are deployed. The field value is an IP address and is mandatory. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `tcp_port`: The port of the TiFlash TCP service. The default value is `9000`. + +- `http_port`: The port of the TiFlash HTTP service. The default value is `8123`. + +- `flash_service_port`: The port via which TiFlash provides services. TiDB reads data from TiFlash via this port. The default value is `3930`. + +- `metrics_port`: TiFlash's status port, which is used to output metric data. The default value is `8234`. + +- `flash_proxy_port`: The port of the built-in TiKV. The default value is `20170`. + +- `flash_proxy_status_port`: The status port of the built-in TiKV. The default value is `20292`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`: Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. TiFlash supports multiple `data_dir` directories separated by commas. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `tmp_path`: The storage path of TiFlash temporary files. The default value is [`path` or the first directory of `storage.latest.dir`] + "/tmp". + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config`: The configuration rule of this field is the same as the `tiflash` configuration rule in `server_configs`. If this field is configured, the field content is merged with the `tiflash` content in `server_configs` (if the two fields overlap, the content of this field takes effect). Then, a configuration file is generated and sent to the machine specified in `host`. + +- `learner_config`: Each TiFlash node has a special built-in TiKV. This configuration item is used to configure this special TiKV. It is generally not recommended to modify the content under this configuration item. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +After the deployment, for the fields above, you can only add directories to `data_dir`; for the fields below, you cannot modified these fields: + +- `host` +- `tcp_port` +- `http_port` +- `flash_service_port` +- `flash_proxy_port` +- `flash_proxy_status_port` +- `metrics_port` +- `deploy_dir` +- `log_dir` +- `tmp_path` +- `arch` +- `os` + +A `tiflash_servers` configuration example is as follows: + +```yaml +tiflash_servers: + - host: 10.0.1.21 + - host: 10.0.1.22 +``` + +### `pump_servers` + +`pump_servers` specifies the machines to which the Pump services of TiDB Binlog are deployed. It also specifies the service configuration on each machine. `pump_servers` is an array, and each element of the array contains the following fields: + +- `host`: Specifies the machine to which the Pump services are deployed. The field value is an IP address and is mandatory. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: The listening port of the Pump services. The default value is `8250`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`: Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config`: The configuration rule of this field is the same as the `pump` configuration rule in `server_configs`. If this field is configured, the field content is merged with the `pump` content in `server_configs` (if the two fields overlap, the content of this field takes effect). Then, a configuration file is generated and sent to the machine specified in `host`. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +A `pump_servers` configuration example is as follows: + +```yaml +pump_servers: + - host: 10.0.1.21 + config: + gc: 7 + - host: 10.0.1.22 +``` + +### `drainer_servers` + +`drainer_servers` specifies the machines to which the Drainer services of TiDB Binlog are deployed. It also specifies the service configuration on each machine. `drainer_servers` is an array. Each array element contains the following fields: + +- `host`: Specifies the machine to which the Drainer services are deployed. The field value is an IP address and is mandatory. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: The listening port of Drainer services. The default value is `8249`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`: Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `commit_ts`: When Drainer starts, it reads the checkpoint. If Drainer cannot read the checkpoint, it uses this field as the replication time point for the initial startup. This field defaults to `-1` (Drainer always gets the latest timestamp from the PD as the commit_ts). + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config`: The configuration rule of this field is the same as the `drainer` configuration rule in `server_configs`. If this field is configured, the field content is merged with the `drainer` content in `server_configs` (if the two fields overlap, the content of this field takes effect). Then, a configuration file is generated and sent to the machine specified in `host`. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `commit_ts` +- `arch` +- `os` + +A `drainer_servers` configuration example is as follows: + +```yaml +drainer_servers: + - host: 10.0.1.21 + config: + syncer.db-type: "mysql" + syncer.to.host: "127.0.0.1" + syncer.to.user: "root" + syncer.to.password: "" + syncer.to.port: 3306 + syncer.ignore-table: + - db-name: test + tbl-name: log + - db-name: test + tbl-name: audit +``` + +### `cdc_servers` + +`cdc_servers` specifies the machines to which the TiCDC services are deployed. It also specifies the service configuration on each machine. `cdc_servers` is an array. Each array element contains the following fields: + +- `host`: Specifies the machine to which the TiCDC services are deployed. The field value is an IP address and is mandatory. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: The listening port of the TiCDC services. The default value is `8300`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`：Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `gc-ttl`: The Time To Live (TTL) duration of the service level GC safepoint set by TiCDC in PD, in seconds. The default value is `86400`, which is 24 hours. + +- `tz`: The time zone that the TiCDC services use. TiCDC uses this time zone when internally converting time data types such as timestamp and when replicating data to the downstream. The default value is the local time zone where the process runs. + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config`: The field content is merged with the `cdc` content in `server_configs` (if the two fields overlap, the content of this field takes effect). Then, a configuration file is generated and sent to the machine specified in `host`. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `gc-ttl` +- `tz` +- `arch` +- `os` + +A `cdc_servers` configuration example is as follows: + +```yaml +cdc_servers: + - host: 10.0.1.20 + gc-ttl: 86400 + - host: 10.0.1.21 +``` + +### `tispark_masters` + +`tispark_masters` specifies the machines to which the master node of TiSpark is deployed. It also specifies the service configuration on each machine. `tispark_masters` is an array. Each array element contains the following fields: + +- `host`: Specifies the machine to which the TiSpark master is deployed. The field value is an IP address and is mandatory. + +- `listen_host`: When the machine has multiple IP addresses, `listen_host` specifies the listening IP address of the service. The default value is `0.0.0.0`. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: Spark's listening port, used for communication before the node. The default value is `7077`. + +- `web_port`: Spark's web port, which provides web services and the task status. The default value is `8080`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `java_home`: Specifies the path of the JRE environment to be used. This parameter corresponds to the `JAVA_HOME` system environment variable. + +- `spark_config`: Configures to configure the TiSpark services. Then, a configuration file is generated and sent to the machine specified in `host`. + +- `spark_env`: Configures the environment variables when Spark starts. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `listen_host` +- `port` +- `web_port` +- `deploy_dir` +- `arch` +- `os` + +A `tispark_masters` configuration example is as follows: + +```yaml +tispark_masters: + - host: 10.0.1.21 + spark_config: + spark.driver.memory: "2g" + spark.eventLog.enabled: "False" + spark.tispark.grpc.framesize: 2147483647 + spark.tispark.grpc.timeout_in_sec: 100 + spark.tispark.meta.reload_period_in_sec: 60 + spark.tispark.request.command.priority: "Low" + spark.tispark.table.scan_concurrency: 256 + spark_env: + SPARK_EXECUTOR_CORES: 5 + SPARK_EXECUTOR_MEMORY: "10g" + SPARK_WORKER_CORES: 5 + SPARK_WORKER_MEMORY: "10g" + - host: 10.0.1.22 +``` + +### `tispark_workers` + +`tispark_workers` specifies the machines to which the worker nodes of TiSpark are deployed. It also specifies the service configuration on each machine. `tispark_workers` is an array. Each array element contains the following fields: + +- `host`: Specifies the machine to which the TiSpark workers are deployed. The field value is an IP address and is mandatory. + +- `listen_host`: When the machine has multiple IP addresses, `listen_host` specifies the listening IP address of the service. The default value is `0.0.0.0`. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: Spark's listening port, used for communication before the node. The default value is `7077`. + +- `web_port`: Spark's web port, which provides web services and the task status. The default value is `8080`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `java_home`: Specifies the path in which the JRE environment to be used is located. This parameter corresponds to the `JAVA_HOME` system environment variable. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `listen_host` +- `port` +- `web_port` +- `deploy_dir` +- `arch` +- `os` + +A `tispark_workers` configuration example is as follows: + +```yaml +tispark_workers: + - host: 10.0.1.22 + - host: 10.0.1.23 +``` + +### `monitoring_servers` + +`monitoring_servers` specifies the machines to which the Prometheus services are deployed. It also specifies the service configuration on each machine. `monitoring_servers` is an array. Each array element contains the following fields: + +- `host`: Specifies the machine to which the monitoring services are deployed. The field value is an IP address and is mandatory. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: The listening port of the Prometheus services. The default value is `9090`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`: Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `storage_retention`: The retention time of the Prometheus monitoring data. The default value is `"15d"`. + +- `rule_dir`: Specifies a local directory that should contain complete `*.rules.yml` files. These files are transferred to the target machine during the initialization phase of the cluster configuration as the rules for Prometheus. +- `remote_config`: Supports writing Prometheus data to the remote, or reading data from the remote. This field has two configurations: + - `remote_write`: See the Prometheus document [``](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). + - `remote_read`: See the Prometheus document [``](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read). +- `external_alertmanagers`: If the `external_alertmanagers` field is configured, Prometheus alerts the configuration behavior to the Alertmanager that is outside the cluster. This field is an array, each element of which is an external Alertmanager and consists of the `host` and `web_port` fields. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +A `monitoring_servers` configuration example is as follows: + +```yaml +monitoring_servers: + - host: 10.0.1.11 + rule_dir: /local/rule/dir + remote_config: + remote_write: + - queue_config: + batch_send_deadline: 5m + capacity: 100000 + max_samples_per_send: 10000 + max_shards: 300 + url: http://127.0.0.1:8003/write + remote_read: + - url: http://127.0.0.1:8003/read + external_alertmanagers: + - host: 10.1.1.1 + web_port: 9093 + - host: 10.1.1.2 + web_port: 9094 +``` + +### `grafana_servers` + +`grafana_servers` specifies the machines to which the Grafana services are deployed. It also specifies the service configuration on each machine. `grafana_servers` is an array. Each array element contains the following fields: + +- `host`: Specifies the machine to which the Grafana services are deployed. The field value is an IP address and is mandatory. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: The listening port of the Grafana services. The default value is `3000`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `username`: The user name on the Grafana login interface. + +- `password`: The password corresponding to Grafana. + +- `dashboard_dir`: Specifies a local directory that should contain complete `dashboard(*.json)` files. These files are transferred to the target machine during the initialization phase of the cluster configuration as the dashboards for Grafana. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +> **Note:** +> +> If the `dashboard_dir` field of `grafana_servers` is configured, after executing the `tiup cluster rename` command to rename the cluster, you need to perform the following operations: +> +> 1. For the `*.json` files in the local dashboards directory, update the value of the `datasource` field to the new cluster name (because `datasource` is named after the cluster name). +> 2. Execute the `tiup cluster reload -R grafana` command. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `port` +- `deploy_dir` +- `arch` +- `os` + +A `grafana_servers` configuration example is as follows: + +```yaml +grafana_servers: + - host: 10.0.1.11 + dashboard_dir: /local/dashboard/dir +``` + +### `alertmanager_servers` + +`alertmanager_servers` specifies the machines to which the Alertmanager services are deployed. It also specifies the service configuration on each machine. `alertmanager_servers` is an array. Each array element contains the following fields: + +- `host`: Specifies the machine to which the Alertmanager services are deployed. The field value is an IP address and is mandatory. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `web_port`: Specifies the port used that Alertmanager uses to provide web services. The default value is `9093`. + +- `cluster_port`: Specifies the communication port between one Alertmanger and other Alertmanager. The default value is `9094`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data_dir`: Specifies the data directory. If it is not specified or specified as a relative directory, the directory is generated according to the `data_dir` directory configured in `global`. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config_file`: Specifies a local file that is transferred to the target machine during the initialization phase of the cluster configuration as the configuration of Alertmanager. + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `web_port` +- `cluster_port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +A `alertmanager_servers` configuration example is as follows: + +```yaml +alertmanager_servers: + - host: 10.0.1.11 + config_file: /local/config/file + - host: 10.0.1.12 + config_file: /local/config/file +``` diff --git a/tiup/tiup-cluster.md b/tiup/tiup-cluster.md index 2965ffc1a03b..e5c685d32d24 100644 --- a/tiup/tiup-cluster.md +++ b/tiup/tiup-cluster.md @@ -8,7 +8,7 @@ aliases: ['/docs/dev/tiup/tiup-cluster/','/docs/dev/reference/tools/tiup/cluster This document focuses on how to use the TiUP cluster component. For the complete steps of online deployment, refer to [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). -Similar to [the TiUP playground component](/tiup/tiup-playground.md) used for local deployment, the TiUP cluster component quickly deploys TiDB for production environment. Compared with playground, the cluster component provides more powerful cluster management features, including upgrading, scaling, and even operation and auditing. +Similar to [the TiUP playground component](/tiup/tiup-playground.md) used for a local test deployment, the TiUP cluster component quickly deploys TiDB for production environment. Compared with playground, the cluster component provides more powerful production cluster management features, including upgrading, scaling, and even operation and auditing. For the help information of the cluster component, run the following command: @@ -17,9 +17,7 @@ tiup cluster ``` ``` -The component `cluster` is not installed; downloading from repository. -download https://tiup-mirrors.pingcap.com/cluster-v0.4.9-darwin-amd64.tar.gz 15.32 MiB / 15.34 MiB 99.90% 10.04 MiB p/s -Starting component `cluster`: /Users/joshua/.tiup/components/cluster/v0.4.9/cluster +Starting component `cluster`: /home/tidb/.tiup/components/cluster/v1.3.0/cluster Deploy a TiDB cluster for production Usage: @@ -41,7 +39,7 @@ Available Commands: display Display information of a TiDB cluster list List all clusters audit Show audit log of cluster operation - import Import an exist TiDB cluster from TiDB-Ansible + import Import an exist TiDB cluster from TiDB Ansible edit-config Edit TiDB cluster config reload Reload a TiDB cluster's config and restart if needed patch Replace the remote package with a specified package and restart the service @@ -65,7 +63,7 @@ tiup cluster deploy [flags] This command requires you to provide the cluster name, the TiDB cluster version, and a topology file of the cluster. -To write a topology file, refer to [the example](https://github.com/pingcap/tiup/blob/master/examples/topology.example.yaml). The following file is an example of the simplest topology: +To write a topology file, refer to [the example](https://github.com/pingcap/tiup/blob/master/embed/templates/examples/topology.example.yaml). The following file is an example of the simplest topology: > **Note:** > @@ -165,7 +163,7 @@ tiup cluster list ``` ``` -Starting /root/.tiup/components/cluster/v0.4.5/cluster list +Starting /root/.tiup/components/cluster/v1.3.0/cluster list Name User Version Path PrivateKey ---- ---- ------- ---- ---------- prod-cluster tidb v3.0.12 /root/.tiup/storage/cluster/clusters/prod-cluster /root/.tiup/storage/cluster/clusters/prod-cluster/ssh/id_rsa @@ -194,7 +192,7 @@ tiup cluster display prod-cluster ``` ``` -Starting /root/.tiup/components/cluster/v0.4.5/cluster display prod-cluster +Starting /root/.tiup/components/cluster/v1.3.0/cluster display prod-cluster TiDB Cluster: prod-cluster TiDB Version: v3.0.12 ID Role Host Ports Status Data Dir Deploy Dir @@ -222,7 +220,7 @@ For the PD component, `|L` or `|UI` might be appended to `Up` or `Down`. `|L` in > > This section describes only the syntax of the scale-in command. For detailed steps of online scaling, refer to [Scale the TiDB Cluster Using TiUP](/scale-tidb-using-tiup.md). -Scaling in a cluster means making some node(s) offline. This operation removes the specific node(s) from the cluster and deletes the remaining data files. +Scaling in a cluster means making some node(s) offline. This operation removes the specific node(s) from the cluster and deletes the remaining files. Because the offline process of the TiKV and TiDB Binlog components is asynchronous (which requires removing the node through API), and the process takes a long time (which requires continuous observation on whether the node is successfully taken offline), special treatment is given to the TiKV and TiDB Binlog components. @@ -265,7 +263,7 @@ tiup cluster display prod-cluster ``` ``` -Starting /root/.tiup/components/cluster/v0.4.5/cluster display prod-cluster +Starting /root/.tiup/components/cluster/v1.3.0/cluster display prod-cluster TiDB Cluster: prod-cluster TiDB Version: v3.0.12 ID Role Host Ports Status Data Dir Deploy Dir @@ -375,12 +373,12 @@ Global Flags: -y, --yes Skip all confirmations and assumes 'yes' ``` -For example, the following command upgrades the cluster to v4.0.0-rc: +For example, the following command upgrades the cluster to v5.0.0: {{< copyable "shell-regular" >}} ```bash -tiup cluster upgrade tidb-test v4.0.0-rc +tiup cluster upgrade tidb-test v5.0.0 ``` ## Update configuration @@ -429,7 +427,7 @@ The content and format requirements for files under the specified path are as fo - The folder specified in the `dashboard_dir` field of `grafana_servers` must contain full `*.json` files. - The folder specified in the `rule_dir` field of `monitoring_servers` must contain full `*.rules.yml` files. -- For the format of files specified in the `config_file` field of `alertmanager_servers`, refer to [the Alertmanager configuration template](https://github.com/pingcap/tiup/blob/master/templates/config/alertmanager.yml). +- For the format of files specified in the `config_file` field of `alertmanager_servers`, refer to [the Alertmanager configuration template](https://github.com/pingcap/tiup/blob/master/embed/templates/config/alertmanager.yml). When you execute `tiup reload`, TiUP first deletes all old configuration files in the target machine and then uploads the corresponding configuration from the control machine to the corresponding configuration directory of the target machine. Therefore, if you want to modify a particular configuration file, make sure that all configuration files (including the unmodified ones) are in the same directory. For example, to modify Grafana's `tidb.json` file, you need to first copy all the `*.json` files from Grafana's `dashboards` directory to your local directory. Otherwise, other JSON files will be missing from the target machine. @@ -559,14 +557,14 @@ tiup cluster audit ``` ``` -Starting component `cluster`: /Users/joshua/.tiup/components/cluster/v0.6.0/cluster audit +Starting component `cluster`: /home/tidb/.tiup/components/cluster/v1.3.0/cluster audit ID Time Command -- ---- ------- -4BLhr0 2020-04-29T13:25:09+08:00 /Users/joshua/.tiup/components/cluster/v0.6.0/cluster deploy test v4.0.0-rc /tmp/topology.yaml -4BKWjF 2020-04-28T23:36:57+08:00 /Users/joshua/.tiup/components/cluster/v0.6.0/cluster deploy test v4.0.0-rc /tmp/topology.yaml -4BKVwH 2020-04-28T23:02:08+08:00 /Users/joshua/.tiup/components/cluster/v0.6.0/cluster deploy test v4.0.0-rc /tmp/topology.yaml -4BKKH1 2020-04-28T16:39:04+08:00 /Users/joshua/.tiup/components/cluster/v0.4.9/cluster destroy test -4BKKDx 2020-04-28T16:36:57+08:00 /Users/joshua/.tiup/components/cluster/v0.4.9/cluster deploy test v4.0.0-rc /tmp/topology.yaml +4BLhr0 2020-04-29T13:25:09+08:00 /home/tidb/.tiup/components/cluster/v1.3.0/cluster deploy test v5.0.0-rc /tmp/topology.yaml +4BKWjF 2020-04-28T23:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.3.0/cluster deploy test v5.0.0-rc /tmp/topology.yaml +4BKVwH 2020-04-28T23:02:08+08:00 /home/tidb/.tiup/components/cluster/v1.3.0/cluster deploy test v5.0.0-rc /tmp/topology.yaml +4BKKH1 2020-04-28T16:39:04+08:00 /home/tidb/.tiup/components/cluster/v1.3.0/cluster destroy test +4BKKDx 2020-04-28T16:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.3.0/cluster deploy test v5.0.0-rc /tmp/topology.yaml ``` The first column is `audit-id`. To view the execution log of a certain command, pass the `audit-id` of a command as the flag as follows: @@ -590,7 +588,7 @@ Flags: -h, --help help for exec -N, --node strings Only exec on host with specified nodes -R, --role strings Only exec on host with specified roles - --sudo use root permissions (default false) + --sudo use root permissions (default false) Global Flags: --ssh-timeout int Timeout in seconds to connect host via SSH, ignored for operations that don't need an SSH connection. (default 5) @@ -690,7 +688,7 @@ You can add `--native-ssh` in all cluster operation commands above to use the sy To avoid adding such a flag in every command, you can use the `TIUP_NATIVE_SSH` system variable to specify whether to use the local SSH client: -```sh +```shell export TIUP_NATIVE_SSH=true # or export TIUP_NATIVE_SSH=1 @@ -709,11 +707,11 @@ If you specify this environment variable and `--native-ssh` at the same time, `- The TiUP data is stored in the `.tiup` directory in the user's home directory. To migrate the control machine, you can take the following steps to copy the `.tiup` directory to the corresponding target machine: 1. Execute `tar czvf tiup.tar.gz .tiup` in the home directory of the original machine. -2. Copy `tip.tar.gz` to the home directory of the target machine. +2. Copy `tiup.tar.gz` to the home directory of the target machine. 3. Execute `tar xzvf tiup.tar.gz` in the home directory of the target machine. 4. Add the `.tiup` directory to the `PATH` environment variable. - If you use `bash` and you are a `tidb` user, you can add `export PATH=/home/tidb/.tiup/bin:$PATH` in `~/.bashr` and execute `source ~/.bashrc`. Then make corresponding adjustments according to the shell and the user you use. + If you use `bash` and you are a `tidb` user, you can add `export PATH=/home/tidb/.tiup/bin:$PATH` in `~/.bashrc` and execute `source ~/.bashrc`. Then make corresponding adjustments according to the shell and the user you use. > **Note:** > diff --git a/tiup/tiup-command-clean.md b/tiup/tiup-command-clean.md new file mode 100644 index 000000000000..c1495e26fa7b --- /dev/null +++ b/tiup/tiup-command-clean.md @@ -0,0 +1,31 @@ +--- +title: tiup clean +--- + +# tiup clean + +The `tiup clean` command is used to clear the data generated during component operation. + +## Syntax + +```shell +tiup clean [name] [flags] +``` + +The value of `[name]` is the `Name` field output by the [`status` command](/tiup/tiup-command-status.md). If `[name]` is omitted, you must add the `--all` option in the `tiup clean` command. + +## Option + +### --all + +- Clears all operation records +- Data type: Boolean +- Default: false + +## Output + +``` +Clean instance of `%s`, directory: %s +``` + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-completion.md b/tiup/tiup-command-completion.md new file mode 100644 index 000000000000..ba14ca86faef --- /dev/null +++ b/tiup/tiup-command-completion.md @@ -0,0 +1,45 @@ +--- +title: tiup completion +--- + +# tiup completion + +To reduce user costs, TiUP provides the `tiup completion` command to generate a configuration file for automatic command-line completion. Currently, TiUP supports completing `bash` and `zsh` commands. + +If you want to complete `bash` commands, you need to install `bash-completion` first. See the following instructions: + +- On macOS: If your bash version is earlier than 4.1, run `brew install bash-completion`; otherwise, run `brew install bash-completion@2`. +- On Linux: Use a package manager to install `bash-completion`. For example, run `yum install bash-completion` or `apt install bash-completion`. + +## Syntax + +```shell +tiup completion +``` + +`` is used to set the type of shell you use. Currently, `bash` and `zsh` are supported. + +## Usage + +### bash + +Write the `tiup completion bash` command into a file and source the file in `.bash_profile`. See the following example: + +```shell +tiup completion bash > ~/.tiup.completion.bash + +printf " +# tiup shell completion +source '$HOME/.tiup.completion.bash' +" >> $HOME/.bash_profile + +source $HOME/.bash_profile +``` + +### zsh + +```shell +tiup completion zsh > "${fpath[1]}/_tiup" +``` + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-env.md b/tiup/tiup-command-env.md new file mode 100644 index 000000000000..0219eb878f0c --- /dev/null +++ b/tiup/tiup-command-env.md @@ -0,0 +1,28 @@ +--- +title: tiup env +--- + +# tiup env + +TiUP provides users with flexible and customized interfaces, some of which are implemented using environment variables. The `tiup env` command is used to query the user-defined environment variables that TiUP supports and their values. + +## Syntax + +```shell +tiup env [name1...N] +``` + +`[name1...N]` is used to view the specified environment variables. If it is not specified, all supported environment variables are viewed by default. + +## Option + +None + +## Output + +- If `[name1...N]` is not specified, a list of "{key}"="{value}" is output. +- If `[name1...N]` is specified, the "{value}" list is output in order. + +In the above output, if `value` is empty, it means that the value of the environment variable is not set. In this case, TiUP uses the default value. + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-help.md b/tiup/tiup-command-help.md new file mode 100644 index 000000000000..f398f9e120dd --- /dev/null +++ b/tiup/tiup-command-help.md @@ -0,0 +1,25 @@ +--- +title: tiup help +--- + +# tiup help + +The TiUP command-line interface provides users with a wealth of help information. You can view it via the `help` command or the `--help` option. + +## Syntax + +```shell +tiup help [command] +``` + +`[command]` is used to specify the help information of which command that users need to view. If it is not specified, the help information of TiUP is viewed. + +## Option + +None + +## Output + +The help information of `[command]` or TiUP. + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-install.md b/tiup/tiup-command-install.md new file mode 100644 index 000000000000..5244441c5451 --- /dev/null +++ b/tiup/tiup-command-install.md @@ -0,0 +1,27 @@ +--- +title: tiup install +--- + +# tiup install + +The `tiup install` command is used for component installation. It downloads the component package of a specified version from the mirror repository and decompresses it in the local TiUP data directory for later use. In addition, when TiUP needs to run a component that does not exist in the mirror repository, it tries to download the component first and then runs it automatically. If the component does not exist in the repository, an error is reported. + +## Syntax + +```shell +tiup install [:version] [component2...N] [flags] +``` + +`` and `` represent component names, and `[version]` represents an optional version number. If `version` is not added, the latest stable version of the specified component is installed. `[component2...N]` means that you can specify multiple components or multiple versions of the same component at the same time. + +## Option + +None + +## Output + +- Normally outputs the download information of the component. +- If the component does not exist, the `The component "%s" not found` error is reported. +- If the version does not exist, the `version %s not supported by component %s` error is reported. + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-list.md b/tiup/tiup-command-list.md new file mode 100644 index 000000000000..9f9a0b54ed8d --- /dev/null +++ b/tiup/tiup-command-list.md @@ -0,0 +1,46 @@ +--- +title: tiup list +--- + +# tiup list + +The command `tiup list` is used to get the list of available components of a mirror. + +## Syntax + +```shell +tiup list [component] [flags] +``` + +`[component]` is an optional parameter used to specify a certain component. If `[component]` is set, TiUP lists all versions of the specified component; if not, TiUP lists all components. + +## Options + +### --all + +- Displays all components. By default, TiUP does not show hidden components. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --installed + +- Only displays components and versions that have been installed. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --verbose + +- Displays installed component versions in the components list. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Outputs + +- If `[component]` is not set: + - If `--verbose` is specified: TiUP outputs a component information list consisting of `Name` (component name), `Installed` (installed versions), `Owner` (component owner), and `Description` (component description). + - If `--verbose` is not specified: TiUP outputs a component information list consisting of `Name` (component name), `Owner` (component owner), and `Description` (component description). +- If `[component]` is set: + - If the specified component exists: TiUP outputs a version information list of the specified component, consisting of `Version` (version number), `Installed` (installation status), `Release` (release date), and `Platforms` (supported platforms). + - If the specified component does not exist: TiUP reports the error `failed to fetch component: unknown component`. + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-mirror-clone.md b/tiup/tiup-command-mirror-clone.md new file mode 100644 index 000000000000..1ba0725aafd2 --- /dev/null +++ b/tiup/tiup-command-mirror-clone.md @@ -0,0 +1,50 @@ +--- +title: tiup mirror clone +--- + +# tiup mirror clone + +The command `tiup mirror clone` is used to clone an existing mirror or clone some of its components to create a new mirror. The new mirror has the same components as the old one, but uses a different signature key. + +## Syntax + +```shell +tiup mirror clone [global version] [flags] +``` + +- `` is used to set the local path to the cloned mirror. If the path does not exist, TiUP automatically creates one. +- If `[global version]` is specified, TiUP tries to clone all components of the specified version. If some components do not have the specified version, then TiUP clones its latest version. + +## Options + +### -f, --full + +- Whether to clone the whole mirror. If this option is set, other options becomes ignored and TiUP completely clones all components of all versions from the targeted mirror. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### -a, --arch + +- Only clones components that can run on the specified platform. +- Data type: `STRING` +- Default: "amd64,arm64" + +### -o, --os + +- Only clones components that can run on the specified operating system. +- Data type: `STRING` +- Default: "linux,darwin" + +### --prefix + +- Whether to only match the prefix of versions. By default, TiUP downloads a component version when it is strictly matched. If this option is set, TiUP also downloads component versions of which prefixes are matched. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --{component} + +- Specifies the version list of the component to be cloned. Fill component names in `{component}`. You can run [`tiup list --all`](/tiup/tiup-command-list.md) to view available component names. +- Data type: Strings +- Default: Null + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-genkey.md b/tiup/tiup-command-mirror-genkey.md new file mode 100644 index 000000000000..18739fbe308f --- /dev/null +++ b/tiup/tiup-command-mirror-genkey.md @@ -0,0 +1,55 @@ +--- +title: tiup mirror genkey +--- + +# tiup mirror genkey + +TiUP [mirror](/tiup/tiup-mirror-reference.md), according its definition, has three roles of users: + +- Mirror administrators: They have the permission to modify `root.json`, `index.json`, `snapshot.json`, and `timestamp.json`. +- Component owners: They have the permission to modify the corresponding component. +- Normal users: They can download and use the components. + + Because TiUP requires the signature of the corresponding owner/administrator to modify a file, owners/administrators must have his or her own private key. The command `tiup mirror genkey` is used to generate a private key. + +> **Warning:** +> +> **DO NOT** transmit private keys over the Internet. + +## Syntax + +```shell +tiup mirror genkey [flags] +``` + +## Options + +### -n, --name + +- Specifies the name of the key, which also determines the name of the final generated file. The path of the generated private key file is `${TIUP_HOME}/keys/{name}.json`. `TIUP_HOME` refers to the home directory of TiUP, which is `$HOME/.tiup` by default. `name` refers to the private key name that `-n/--name` specifies. +- Data type: `STRING` +- Default: "private" + +### -p, --public + +- Shows the corresponding public key of the private key specified in the option `-n/--name`. +- TiUP does not create a new private key when `-p/--public` is specified. If the private key specified in `-n/--name` does not exist, TiUP returns an error. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --save + +- Saves the information of the public key as a file in the current directory. The file name is `{hash-prefix}-public.json`. `hash-prefix` is the first 16 bits of the key ID. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Outputs + +- If `-p/--public` is not specified: + - If the private key specified in `-n/--name` exists: TiUP outputs `Key already exists, skipped`. + - If the private key specified in `-n/--name` does not exist: TiUP outputs `private key have been write to ${TIUP_HOME}/keys/{name}.json`. +- If `-p/--public` is specified: + - If the private key specified in `-n/--name` does not exist: TiUP reports the error `Error: open ${TIUP_HOME}/keys/{name}.json: no such file or directory`. + - If the private key specified in `-n/--name` exists: TiUP outputs the content of the corresponding public key. + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-grant.md b/tiup/tiup-command-mirror-grant.md new file mode 100644 index 000000000000..8f3ad9a95931 --- /dev/null +++ b/tiup/tiup-command-mirror-grant.md @@ -0,0 +1,44 @@ +--- +title: tiup mirror grant +--- + +# tiup mirror grant + +The `tiup mirror grant` command is used to introduce a component owner to the current mirror. + +Component owners can use their keys to publish new components or to modify components they previously published. Before adding a new component owner, the component owner to be added needs to send his or her own public key to the mirror administrator. + +> **Note:** +> +> This command is only supported when the current mirror is a local mirror. + +## Syntax + +```shell +tiup mirror grant [flags] +``` + +`` stands for the component owner's ID, which must be unique in the whole mirror. It is recommended to use an ID that matches the regular expression `^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,38}$`. + +## Options + +### -k, --key + +- Specifies the key of the introduced component owner. This key can either be public or private. If it is a private key, TiUP converts it to the corresponding public key before storing it in the mirror. +- A key can be used by only one component owner. +- Data type: `STRING` +- Default: "${TIUP_HOME}/keys/private.json" + +### -n, --name + +- Specifies the name of the component owner. The name is displayed on the `Owner` field of the component list. If `-n/--name` is not specified, `` is used as the component owner's name. +- Data type: `STRING` +- Default: `` + +### Outputs + +- If the command is executed successfully, there is no output. +- If the component owner's ID is duplicated, TiUP reports the error `Error: owner %s exists`. +- If the key is used by another component owner, TiUP reports the error `Error: key %s exists`. + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-init.md b/tiup/tiup-command-mirror-init.md new file mode 100644 index 000000000000..7d5f48088a57 --- /dev/null +++ b/tiup/tiup-command-mirror-init.md @@ -0,0 +1,46 @@ +--- +title: tiup mirror init +--- + +# tiup mirror init + +The command `tiup mirror init` is used to initialize an empty mirror. The initialized mirror does not contain any components or component owners. The command only generates the following files for the initialized mirror: + +``` ++ # Mirror's root directory +|-- root.json # Mirror's root certificate +|-- 1.index.json # Component/user index +|-- snapshot.json # Mirror's latest snapshot +|-- timestamp.json # Mirror's latest timestamp +|--+ keys # Mirror's private key (can be moved to other locations) + |-- {hash1..hashN}-root.json # Private key of the root certificate + |-- {hash}-index.json # Private key of the indexes + |-- {hash}-snapshot.json # Private key of the snapshots + |-- {hash}-timestamp.json # Private key of the timestamps +``` + +For the specific usage and content format of the above files, refer to [TiUP Mirror Reference Guide](/tiup/tiup-mirror-reference.md). + +## Syntax + +```shell +tiup mirror init [flags] +``` + +`` is used to specify a local directory where TiUP generates and stores mirror files. The local directory can be a relative path. If the specified directory already exists, it must be empty; if it does not exist, TiUP creates it automatically. + +## Options + +### -k, --key-dir + +- Specifies the directory where TiUP generates private key files. If the specified directory does not exist, TiUP automatically creates it. +- Data type: `STRING` +- If this option is not specified in the command, TiUP generates private key files in `{path}/keys` by default. + +### Outputs + +- If the command is executed successfully, there is no output. +- If the specified `` is not empty, TiUP reports the error `Error: the target path '%s' is not an empty directory`. +- If the specified `` is not a directory, TiUP reports the error `Error: fdopendir: not a directory`. + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-merge.md b/tiup/tiup-command-mirror-merge.md new file mode 100644 index 000000000000..ca16e2225c57 --- /dev/null +++ b/tiup/tiup-command-mirror-merge.md @@ -0,0 +1,32 @@ +--- +title: tiup mirror merge +--- + +# tiup mirror merge + +The `tiup mirror merge` command is used to merge one or more mirrors to the current mirror. + +To execute this command, the following conditions must be met: + +- The owner IDs of all components of the target mirror exist in the current mirror. +- The `${TIUP_HOME}/keys` directory of the user who executes this command contains all the private keys corresponding to the above owner IDs in the current mirror (you can use the command [`tiup mirror set`](/tiup/tiup-command-mirror-set.md) to switch the current mirror to the mirror that is currently authorized to modify). + +## Syntax + +```shell +tiup mirror merge [mirror-dir-N] [flags] +``` + +- ``: the first mirror to be merged into the current mirror +- `[mirror-dir-N]`: the Nth mirror to be merged into the current mirror + +## Option + +None + +## Outputs + +- If the command is executed successfully, there is no output. +- If the current mirror does not have a component owner of the target mirror, or if `${TIUP_HOME}/keys` does not have the owner's private key, TiUP reports the `Error: missing owner keys for owner %s on component %s` error. + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-modify.md b/tiup/tiup-command-mirror-modify.md new file mode 100644 index 000000000000..d9553f439801 --- /dev/null +++ b/tiup/tiup-command-mirror-modify.md @@ -0,0 +1,64 @@ +--- +title: tiup mirror modify +--- + +# tiup mirror modify + +The `tiup mirror modify` command is used to modify published components. Only valid component owners can modify the components that they have published on their own. For the method to publish a component, refer to the [`publish` command](/tiup/tiup-command-mirror-publish.md). + +## Syntax + +```shell +tiup mirror modify [:version] [flags] +``` + +Each parameter is explained as follows: + +- ``: the component name +- `[version]`: the component version to modify. If it is not specified, the entire component is modified. + +## Options + +### -k, --key + +- Specifies the component owner's private key used for signing the component information (`{component}.json`). +- Data type: `STRING` +- If this option is not specified in the command, `"${TIUP_HOME}/keys/private.json"` is used by default to sign the component information. + +### --yank + +Marks a specified component or version as unavailable. + +- After the component is marked as unavailable, you can neither see it in the result list of `tiup list` nor install the new version of the component. +- After a component version is marked as unavailable, you can neither see it in the result list of `tiup list ` nor install this version. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --hide + +- Specifies whether to hide the component. If a component is hidden, you cannot see it in the result list of `tiup list`. To see the hidden component, you can use `tiup list --all`. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +> **Note:** +> +> This option can only be applied to the component, not to the component version. + +### --standalone + +- Controls whether the component can run standalone. This option is currently **NOT available**. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +> **Note:** +> +> This option can only be applied to the component, not to the component version. + +## Outputs + +- If the command is executed successfully, there is no output. +- If the component owner is not authorized to modify the target component: + - If the mirror is a remote mirror, TiUP reports the error `Error: The server refused, make sure you have access to this component`. + - If the mirror is a local mirror, TiUP reports the error `Error: the signature is not correct`. + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-publish.md b/tiup/tiup-command-mirror-publish.md new file mode 100644 index 000000000000..50312cf430ea --- /dev/null +++ b/tiup/tiup-command-mirror-publish.md @@ -0,0 +1,85 @@ +--- +title: tiup mirror publish +--- + +# tiup mirror publish + +The command `tiup mirror publish` is used to publish a new component or a new version of an existing component. Only component owner that has the access to the target component can publish it. To add a new component owner, see the usage of the [`grant` command](/tiup/tiup-command-mirror-grant.md). + +## Syntax + +```shell +tiup mirror publish [flags] +``` + +The meaning of each parameter is as follows: + +- ``: The name of the components, such as `tidb`. It is recommended to use a string that matches the regular expression `^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,38}$`. +- ``: The version of the component to be published. The version number needs to follow the requirements of [Semantic Versioning](https://semver.org/). +- ``: The local directory of the `.tar.gz` package. You need to put dependencies and the executable file of the component in this package. TiUP uploads this package to the mirror. +- ``: The location of the component's executable file in ``. + +## Options + +### -k, --key + +- Specifies the component owner's private key. The client uses the private key to sign `{component}.json` files. +- Data type: `STRING` +- Default: "${TIUP_HOME}/keys/private.json" + +### --arch + +- Specifies the platform on which the binary files in `` can run. For a single `` package, you can only choose the platform from the following options: + + - `amd64`: Indicates that the files run on AMD64 machines. + - `arm64`: Indicates that the files run on ARM64 machines. + - `any`: Indicates that the files, such as scripts, run on both AMD64 and ARM64 machines. + +- Data type: `STRING` +- Default: "${GOARCH}" + +> **Note:** +> +> If `--arch` is set to `any`, then `--os` must be set to `any` as well. + +### --os + +- Specifies the operating system on which the binary files in `` can run. For a single `` package, you can only choose the operating system from the following options: + + - `linux`: Indicates that the files run on the Linux operating system. + - `darwin`: Indicates that the files run on the Darwin operating system. + - `any`: Indicates that the files, such as scripts, run on both the Linux and Darwin operating systems. + +- Data type: `STRING` +- Default: "${GOOS}" + +> **Note:** +> +> If `--os` is set to `any`, then `--arch` must be set to `any` as well. + +### --desc + +- Specifies the description of the component. +- Data type: `String` +- Default: NULL + +### --hide + +- Specifies whether the component is hidden. If it is a hidden component, it can be seen in the result list of `tiup list -all`, but not in that of `tiup list`. +- Data type: `STRING` +- Default: NULL + +### --standalone + +- Controls whether the component can run standalone. This option is currently **NOT available**. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Outputs + +- If the command is executed successfully, there is no output. +- If the component owner is not authorized to modify the target component: + - If the mirror is a remote mirror, TiUP reports the error `Error: The server refused, make sure you have access to this component`. + - If the mirror is a local mirror, TiUP reports the error `Error: the signature is not correct`. + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-rotate.md b/tiup/tiup-command-mirror-rotate.md new file mode 100644 index 000000000000..9359f84abac6 --- /dev/null +++ b/tiup/tiup-command-mirror-rotate.md @@ -0,0 +1,62 @@ +--- +title: tiup mirror rotate +--- + +# tiup mirror rotate + +`root.json` is an important file in a TiUP mirror. It stores the public keys needed for the entire system and is the basis of the chain of trust in TiUP. It mainly contains the following parts: + +- Signatures of mirror administrators. For the official mirror, there are five signatures. For an initialized mirror, there are three signatures by default. +- The public keys used to verify the following files: + - root.json + - index.json + - snapshot.json + - timestamp.json +- Expiration date of `root.json`. For the official mirror, the expiration date is one year later than the creation date of `root.json`. + +For detailed description of TiUP mirror, see [TiUP Mirror Reference](/tiup/tiup-mirror-reference.md). + +You need to update `root.json` in the following cases: + +- Replace the key of the mirror. +- Update the expiration date of certificate files. + +After the content of `root.json` is updated, the file must be re-signed by all administrators; otherwise, the client rejects the file. The update process is as follows: + +1. The user (client) updates the content of `root.json`. +2. All administrators sign the new `root.json` file. +3. tiup-server updates `snapshot.json` to record the version of the new `root.json` file. +4. tiup-server signs the new `snapshot.json` file. +5. tiup-server updates `timestamp.json` to record the hash value of the new `snapshot.json` file. +6. tiup-server signs the new `timestamp.json` file. + +TiUP uses the command `tiup mirror rotate` to automate the above process. + +> **Note:** +> +> + For TiUP versions earlier than v1.5.0, running this command does not returns a correct new `root.json` file. See [#983](https://github.com/pingcap/tiup/issues/983). +> + Before using this command, make sure that all TiUP clients are upgraded to v1.5.0 or a later version. + +## Syntax + +```shell +tiup mirror rotate [flags] +``` + +After executing this command, TiUP starts an editor for the user to modify the file content to the target value, such as changing the value of the `expires` field to a later date. Then, TiUP changes the `version` field from `N` to `N+1` and saves the file. After the file is saved, TiUP starts a temporary HTTP server and waits for all mirror administrators to sign the file. + +For how mirror administrators sign files, refer to the [`sign` command](/tiup/tiup-command-mirror-sign.md). + +## Options + +### --addr + +- Specifies the listening address of the temporary server. You need to make sure that the address is accessible to other mirror administrators so that they can use the [`sign` command](/tiup/tiup-command-mirror-sign.md) to sign the file. +- Data type: `STRING` +- If this option is not specified in the command, TiUP listens on `0.0.0.0:8080` by default. + +## Outputs + +The current signature status of each mirror administrator. + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-set.md b/tiup/tiup-command-mirror-set.md new file mode 100644 index 000000000000..2a8a9ba9a79f --- /dev/null +++ b/tiup/tiup-command-mirror-set.md @@ -0,0 +1,49 @@ +--- +title: tiup mirror set +--- + +# tiup mirror set + +The `tiup mirror set` command is used to switch the current mirror and supports two forms of mirrors: local file system and remote network address. + +The address of the official mirror is `https://tiup-mirrors.pingcap.com`. + +## Syntax + +```shell +tiup mirror set [flags] +``` + +`` is the mirror address, which has two forms: + +- Network address: starts with `http` or `https`. For example, `http://172.16.5.5:8080`, `https://tiup-mirrors.pingcap.com`. +- Local file path: the absolute path of the mirror directory. For example, `/path/to/local-tiup-mirror`. + +## Option + +### -r, --root + +This option specifies the root certificate. + +As the most critical part of mirror security, the root certificate of each mirror is different from one another. When you use the network mirror, it might suffer from man-in-the-middle attacks. To avoid such attacks, it is recommended to manually download the root certificate of the root network mirror to the local: + +``` +wget /root.json -O /path/to/local/root.json +``` + +Perform a manual check to ensure that the root certificate is correct, and then switch the mirror by manually specifying the root certificate: + +``` +tiup mirror set -r /path/to/local/root.json +``` + +In the steps above, if the mirror is attacked before the `wget` command, you can find that the root certificate is incorrect. If the mirror is attacked after the `wget` command, TiUP will find that the mirror does not match the root certificate. + +- Data type: `String` +- Default: `{mirror-dir}/root.json` + +## Output + +None + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror-sign.md b/tiup/tiup-command-mirror-sign.md new file mode 100644 index 000000000000..7c5f45657fbb --- /dev/null +++ b/tiup/tiup-command-mirror-sign.md @@ -0,0 +1,49 @@ +--- +title: tiup mirror sign +--- + +# tiup mirror sign + +The `tiup mirror sign` command is used to sign the metadata files (*.json）defined in TiUP [mirror](/tiup/tiup-mirror-reference.md). These metadata files might be stored on the local file system or remotely stored using the HTTP protocol to provide a signature entry. + +## Syntax + +```shell +tiup mirror sign [flags] +``` + +`` is the address of the file to be signed, which has two forms: + +- Network address, which starts with HTTP or HTTPS, such as `http://172.16.5.5:8080/rotate/root.json` +- Local file path, which is a relative path or an absolute path + +If it is a network address, this address must provide the following features: + +- Supports the access via `http get` that returns the complete content of the signed file (including the `signatures` field). +- Supports the access via `http post`. The client adds the signature to the `signatures` field of the content that is returned by `http get` and posts to this network address. + +## Options + +### -k, --key + +- Specifies the location of the private key used for signing the `{component}.json` file. +- Data type: `STRING` +- - If this option is not specified in the command, `"${TIUP_HOME}/keys/private.json"` is used by default. + +### --timeout + +- Specifies the access timeout time for signing through the network. The unit is in seconds. +- Data type: `INT` +- Default: 10 + +> **Note:** +> +> This option is valid only when `` is a network address. + +## Output + +- If the command is executed successfully, there is no output. +- If the file has been signed by the specified key, TiUP reports the error `Error: this manifest file has already been signed by specified key`. +- If the file is not a valid manifest, TiUP reports the error `Error: unmarshal manifest: %s`. + +[<< Back to the previous page - TiUP Mirror command list](/tiup/tiup-command-mirror.md#command-list) diff --git a/tiup/tiup-command-mirror.md b/tiup/tiup-command-mirror.md new file mode 100644 index 000000000000..a534d5bd1dbb --- /dev/null +++ b/tiup/tiup-command-mirror.md @@ -0,0 +1,39 @@ +--- +title: tiup mirror +--- + +# tiup mirror + +In TiUP, [mirror](/tiup/tiup-mirror-reference.md) is an important concept. TiUP currently supports two forms of mirroring: + +- Local mirror: the TiUP client and the mirror are on the same machine, and the client accesses the mirror through the file system. +- Remote mirror: the TiUP client and the mirror are not on the same machine, and the client accesses the mirror through network. + +The `tiup mirror` command is used to manage mirrors and provides ways to create mirrors, distribute components, and manage keys. + +## Syntax + +```shell +tiup mirror [flags] +``` + +`` stands for sub-commands. For the list of supported sub-commands, refer to the [command list](#command-list) below. + +## Option + +None + +## Command list + +- [genkey](/tiup/tiup-command-mirror-genkey.md): generates the private key file +- [sign](/tiup/tiup-command-mirror-sign.md): signs a specific file using a private key file +- [init](/tiup/tiup-command-mirror-init.md): initiates an empty mirror +- [set](/tiup/tiup-command-mirror-set.md): sets the current mirror +- [grant](/tiup/tiup-command-mirror-grant.md): grants a new component owner for the current mirror +- [publish](/tiup/tiup-command-mirror-publish.md): publishes new components to the current mirror +- [modify](/tiup/tiup-command-mirror-modify.md): modifies the attributes of the components in the current mirror +- [rotate](/tiup/tiup-command-mirror-rotate.md): updates the root certificate in the current mirror +- [clone](/tiup/tiup-command-mirror-clone.md): clones a new mirror from an existing one +- [merge](/tiup/tiup-command-mirror-merge.md): merges mirrors + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-status.md b/tiup/tiup-command-status.md new file mode 100644 index 000000000000..a43febf01a6c --- /dev/null +++ b/tiup/tiup-command-status.md @@ -0,0 +1,39 @@ +--- +title: tiup status +--- + +# tiup status + +The `tiup status` command is used to view the operation information of the components after you run the components using the `tiup [flags] [args...]` command. + +> **Note:** +> +> You can only check the information of the following components: +> +> - Components that are still in operation +> - Components that run through the tag specified by `tiup -T/--tag` + +## Syntax + +```shell +tiup status [flags] +``` + +## Option + +None + +## Output + +A table consisting of the following fields: + +- `Name`: The tag name specified by `-T/--tag`. If not specified, it is a random string. +- `Component`: The operating components. +- `PID`: The corresponding process ID of the operating components. +- `Status`: The status of the operating components. +- `Created Time`: The starting time of the components. +- `Directory`: The data directory of the components. +- `Binary`: The binary file path of the components. +- `Args`: The starting arguments of the operating components. + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-telemetry.md b/tiup/tiup-command-telemetry.md new file mode 100644 index 000000000000..f8412ad0bac1 --- /dev/null +++ b/tiup/tiup-command-telemetry.md @@ -0,0 +1,56 @@ +--- +title: tiup telemetry +--- + +# tiup telemetry + +TiDB, TiUP, and TiDB Dashboard collect usage information by default and share the information with PingCAP to improve the product. For example, through this usage information, PingCAP learns about common TiDB cluster operations and thereby determines the priority of new features. + +When TiUP telemetry is enabled, usage information is shared with PingCAP when TiUP commands are executed, including (but not limited to): + +- Randomly generated telemetry identifiers. +- The execution status of the TiUP command, such as whether the command execution is successful and the duration of command execution. +- Situations using TiUP for deployment, such as target machine hardware information, component version number, and modified deployment configuration name. + +The information below is not shared: + +- The accurate name of the cluster +- The cluster topology +- The cluster configuration file + +TiUP uses the `tiup telemetry` command to control telemetry. + +> **Note:** +> +> This feature is enabled by default. + +## Syntax + +```shell +tiup telemetry +``` + +`` stands for sub-commands. For the list of supported sub-commands, refer to the commands section below. + +## Commands + +### status + +The `tiup telemetry status` command is used to view the current telemetry settings and output the following information: + +- `status`: specifies enabling or disabling the telemetry `(enable|disable)`. +- `uuid`: specifies the randomly generated telemetry identifiers. + +### reset + +The `tiup telemetry reset` command is used to reset the current telemetry identifier and replaces it with a new random identifier. + +### enable + +The `tiup telemetry enable` command is used to enable the telemetry. + +### disable + +The `tiup telemetry disable` command is used to disable the telemetry. + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-uninstall.md b/tiup/tiup-command-uninstall.md new file mode 100644 index 000000000000..ad71761195d7 --- /dev/null +++ b/tiup/tiup-command-uninstall.md @@ -0,0 +1,38 @@ +--- +title: tiup uninstall +--- + +# tiup uninstall + +The `tiup uninstall` command is used to uninstall the installed components. + +## Syntax + +```shell +tiup uninstall : [component2...N] [flags] +``` + +- ``: the name of the component to uninstall. +- ``: the version to uninstall. If this field is omitted, all installed versions of the component are uninstalled. For security reasons, the `--all` option must be added when `` is omitted, indicating that all versions of a component need to be uninstalled. +- `[component2...N]`: multiple components or versions to uninstall. + +## Options + +### --all + +- Uninstalls all installed versions of the specified component(s). You must use this option when `` is omitted. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --self + +- Uninstalls TiUP itself. When this option is used, all data downloaded from the mirror is deleted, but the data generated by TiUP and its components is retained. The data is stored in the directory specified by the `TIUP_HOME` environment variable. If `TIUP_HOME` is not set, the default value is `~/.tiup/`. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Outputs + +- If the command exits without any error, `Uninstalled component "%s" successfully!` is output. +- If neither `` nor `--all` is specified, the `Use "tiup uninstall tidbx --all" if you want to remove all versions.` error is reported. + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-command-update.md b/tiup/tiup-command-update.md new file mode 100644 index 000000000000..9cbc2a3bfab3 --- /dev/null +++ b/tiup/tiup-command-update.md @@ -0,0 +1,52 @@ +--- +title: tiup update +--- + +# tiup update + +The `tiup update` command is used to update the installed components or TiUP itself. + +## Syntax + +```shell +tiup update [component1][:version] [component2..N] [flags] +``` + +- `[component1]`: the name of the component to update. +- `[version]`: the version to update. If this field is omitted, it means updating to the latest stable version of the component. +- `[component2...N]`: specifies updating multiple components or versions. If no component is specified, which means `[component1][:version] [component2..N]` is empty, you need to use the `--all` or the `--self` option together. + +The update operation does not delete the old version. You can still specify using the old version during execution. + +## Options + +### --all + +- If no component is specified, this option must be specified. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --force + +- If the specified version of the component is already installed, the update operation is skipped by default. Specifying this option will have the installed version forcibly updated. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --nightly + +- Updates the specified components to the nightly version. The `tiup update` command with this option is equivalent to the `tiup update :nightly` command. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --self + +- Updates TiUP itself. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Outputs + +- If the update is successful, `Updated successfully!` is output. +- If target version does not exist, the `Error: version %s not supported by component %s` error is reported. + +[<< Back to the previous page - TiUP Reference command list](/tiup/tiup-reference.md#command-list) diff --git a/tiup/tiup-component-cluster-audit.md b/tiup/tiup-component-cluster-audit.md new file mode 100644 index 000000000000..abefeb0fbc03 --- /dev/null +++ b/tiup/tiup-component-cluster-audit.md @@ -0,0 +1,34 @@ +--- +title: tiup cluster audit +--- + +# tiup cluster audit + +The `tiup cluster audit` command is used to view commands executed on all clusters in the history and the execution log of each command. + +## Syntax + +```shell +tiup cluster audit [audit-id] [flags] +``` + +- If you do not fill in the `[audit-id]`, the table of operation records is output in reverse chronological order. The first column is the `audit-id`. +- If you fill in the `[audit-id]`, it means checking the execution log of the specified `audit-id`. + +## Option + +### -h, --help + +- Prints the help information. +- Data type: `Boolean` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Outputs + +- If `[audit-id]` is specified, the corresponding execution log is output. +- If `[audit-id]` is not specified, a table with the following fields is output: + - ID: the `audit-id` corresponding to the record + - Time: the execution time of the command corresponding to the record + - Command：the command corresponding to the record + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-check.md b/tiup/tiup-component-cluster-check.md new file mode 100644 index 000000000000..8088ac88c5db --- /dev/null +++ b/tiup/tiup-component-cluster-check.md @@ -0,0 +1,238 @@ +--- +title: tiup cluster check +--- + +# tiup cluster check + +For a formal production environment, before the environment goes live, you need to perform a series of checks to ensure the clusters are in their best performance. To simplify the manual check steps, TiUP Cluster provides the `check` command to check whether the hardware and software environments of the target machines of a specified cluster meet the requirements to work normally. + +## List of check items + +### Operating system version + +Check the operating system distribution and version of the deployed machines. Currently, only CentOS 7 is supported for deployment. More system versions may be supported in later releases for compatibility improvement. + +### CPU EPOLLEXCLUSIVE + +Check whether the CPU of the target machine supports EPOLLEXCLUSIVE. + +### numactl + +Check whether numactl is installed on the target machine. If tied cores are configured on the target machine, you must install numactl. + +### System time + +Check whether the system time of the target machine is synchronized. Compare the system time of the target machine with that of the central control machine, and report an error if the deviation exceeds a certain threshold (500ms). + +### Time synchronization service + +Check whether the time synchronization service is configured on the target machine. Namely, check whether ntpd is running. + +### Swap partitioning + +Check whether swap partitioning is enabled on the target machine. It is recommended to disable swap partitioning. + +### Kernel parameters + +Check the values of the following kernel parameters: + +- `net.ipv4.tcp_tw_recycle`: 0 +- `net.ipv4.tcp_syncookies`: 0 +- `net.core.somaxconn`: 32768 +- `vm.swappiness`: 0 +- `vm.overcommit_memory`: 0 or 1 +- `fs.file-max`: 1000000 + +### Transparent Huge Pages (THP) + +Check whether THP is enabled on the target machine. It is recommended to disable THP. + +### System limits + +Check the limit values in the `/etc/security/limits.conf` file: + +``` + soft nofile 1000000 + hard nofile 1000000 + soft stack 10240 +``` + +`` is the user who deploys and runs the TiDB cluster, and the last column is the minimum value required for the system. + +### SELinux + +Check whether SELinux is enabled. It is recommended to disable SELinux. + +### Firewall + +Check whether the FirewallD service is enabled. It is recommended to either disable the FirewallD service or add permission rules for each service in the TiDB cluster. + +### irqbalance + +Check whether the irqbalance service is enabled. It is recommended to enable the irqbalance service. + +### Disk mount options + +Check the mount options for ext4 partitions. Make sure the mount options include the nodelalloc option and the noatime option. + +### Port usage + +Check if the ports defined in the topology (including the auto-completion default ports) are already used by the processes on the target machine. + +> **Note:** +> +> The port usage check assumes that a cluster is not started yet. If a cluster is already deployed and started, the port usage check on the cluster fails because the ports must be in use in this case. + +### CPU core number + +Check the CPU information of the target machine. For a production cluster, it is recommended that the number of the CPU logical core is greater than or equal to 16. + +> **Note:** +> +> CPU core number is not checked by default. To enable the check, you need to add the `-enable-cpu` option to the command. + +### Memory size + +Check the memory size of the target machine. For a production cluster, it is recommended that the total memory capacity is greater than or equal to 32GB. + +> **Note:** +> +> Memory size is not checked by default. To enable the check, you need to add the `-enable-mem` option to the command. + +### Fio disk performance test + +Use flexible I/O tester (fio) to test the performance of the disk where `data_dir` is located, including the following three test items: + +- fio_randread_write_latency +- fio_randread_write +- fio_randread + +> **Note:** +> +> The fio disk performance test is not performed by default. To perform the test, you need to add the `-enable-disk` option to the command. + +## Syntax + +```shell +tiup cluster check [flags] +``` + +- If a cluster is not deployed yet, you need to pass the [topology.yml](/tiup/tiup-cluster-topology-reference.md) file that is used to deploy the cluster. According to the content in this file, tiup-cluster connects to the corresponding machine to perform the check. +- If a cluster is already deployed, you can use the `` as the check object. + +> **Note:** +> +> If `` is used for the check, you need to add the `--cluster` option in the command. + +## Options + +### --apply + +- Attempts to automatically repair the failed check items. Currently, tiup-cluster only attempts to repair the following check items: + - SELinux + - firewall + - irqbalance + - kernel parameters + - System limits + - THP (Transparent Huge Pages) +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### --cluster + +- Indicates that the check is for the deployed clusters. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +> **Note:** +> +> tiup-cluster supports checking both un-deployed clusters and deployed clusters with the following command format: +> +> ```shell +> tiup cluster check [flags] +> ``` +> +> If the `tiup cluster check ` command is used, you must add the `--cluster` option: `tiup cluster check --cluster`. + +### -N, --node + +- Specifies the nodes to be checked. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup cluster display`](/tiup/tiup-component-cluster-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all nodes are checked by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are checked. + +### -R, --role + +- Specifies the roles to be checked. The value of this option is a comma-separated list of node roles. You can get the roles of nodes from the second column of the cluster status table returned by the [`tiup cluster display`](/tiup/tiup-component-cluster-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all roles are checked by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are checked. + +### --enable-cpu + +- Enables the check of CPU core number. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### --enable-disk + +- Enables the fio disk performance test. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### --enable-mem + +- Enables the memory size check. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### --u, --user + +- Specifies the user name to connect to the target machine. The specified user needs to have the password-free sudo root privileges on the target machine. +- Data type: `STRING` +- If this option is not specified in the command, the user who executes the command is used as the default value. + +> **Note:** +> +> This option is valid only if the `-cluster` option is false. Otherwise, the value of this option is fixed to the username specified in the topology file for the cluster deployment. + +### -i, --identity_file + +- Specifies the key file to connect to the target machine. +- Data type: `STRING` +- The option is enabled by default with `~/.ssh/id_rsa` (the default value) passed in. + +> **Note:** +> +> This option is valid only if the `--cluster` option is false. Otherwise, the value of this option is fixed to `${TIUP_HOME}/storage/cluster/clusters//ssh/id_rsa`. + +### -p, --password + +- Logs in with a password when connecting to the target machine. + - If the `--cluster` option is added for a cluster, the password is the password of the user specified in the topology file when the cluster was deployed. + - If the `--cluster` option is not added for a cluster, the password is the password of the user specified in the `-u/--user` option. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### -h, --help + +- Prints the help information of the related commands. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +A table containing the following fields: + +- `Node`: the target node +- `Check`: the check item +- `Result`: the check result (Pass, Warn, or Fail) +- `Message`: the result description + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-clean.md b/tiup/tiup-component-cluster-clean.md new file mode 100644 index 000000000000..47eec3b1f325 --- /dev/null +++ b/tiup/tiup-component-cluster-clean.md @@ -0,0 +1,66 @@ +--- +title: tiup cluster clean +--- + +# tiup cluster clean + +In the test environment, sometimes you might need to reset the cluster back to the state as it has been just deployed, which means deleting all data. You can do that easily using the `tiup cluster clean` command. After running it, the cluster is stopped and then cluster data is deleted. After restarting the cluster manually, you will get a clean cluster. + +> **Warning:** +> +> This command will first stop the cluster even if you choose only to clean up logs. Therefore, do not use it in a production environment. + +## Syntax + +```shell +tiup cluster clean [flags] +``` + +`` is the cluster to clean. + +## Options + +### --all + +- Cleans data and the log at the same time. It is equivalent to specifying `--data` and `--log` at the same time. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. +- If it is not specified, you must specify at lease one of the following options: + - --data: Cleans data + - --log: Cleans the log + +### --data + +- Cleans data. If neither of it nor `--all` is specified, data will not be cleaned. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --log + +- Cleans the log. If neither of it nor `--all` is specified, the log will not be cleaned. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --ignore-node + +- Specifies the node that does not need cleaning. To specify multiple nodes, you can use this option multiple times. For example, `--ignore-node --ignore-node `. +- Data type: `StringArray` +- Default: empty + +### --ignore-role + +- Specifies the role that does not need cleaning. To specify multiple roles, you can use this option multiple times. For example, `--ignore-role --ignore-role `. +- Data type: `StringArray` +- Default: empty + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Output + +The execution logs of tiup-cluster. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-deploy.md b/tiup/tiup-component-cluster-deploy.md new file mode 100644 index 000000000000..19eb1bd01c34 --- /dev/null +++ b/tiup/tiup-component-cluster-deploy.md @@ -0,0 +1,69 @@ +--- +title: tiup cluster deploy +--- + +# tiup cluster deploy + +The `tiup cluster deploy` command is used to deploy a new cluster. + +## Syntax + +```shell +tiup cluster deploy [flags] +``` + +- ``: the name of the new cluster, which cannot be the same as the existing cluster names. +- ``: the version number of the TiDB cluster to deploy, such as `v5.1.0`. +- ``: the prepared [topology file](/tiup/tiup-cluster-topology-reference.md). + +## Options + +### -u, --user + +- Specifies the user name used to connect to the target machine. This user must have the secret-free sudo root permission on the target machine. +- Data type: `STRING` +- Default: the current user who executes the command. + +### -i, --identity_file + +- Specifies the key file used to connect to the target machine. +- Data type: `STRING` +- If this option is not specified in the command, the `~/.ssh/id_rsa` file is used to connect to the target machine by default. + +### -p, --password + +- Specifies the password used to connect to the target machine. Do not use this option with `-i/--identity_file` at the same time. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --ignore-config-check + +- This option is used to skip the configuration check. After the binary files of components are deployed, the configurations of TiDB, TiKV, and PD components are checked using ` --config-check `. `` is the path of the deployed binary file. `` is the configuration file generated based on the user configuration. +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. +- Default: false + +### --no-labels + +- This option is used to skip the label check. +- When two or more TiKV nodes are deployed on the same physical machine, a risk exists: PD cannot learn the cluster topology, so PD might schedule multiple replicas of a Region to different TiKV nodes on one physical machine, which makes this physical machine a single point. To avoid this risk, you can use labels to tell PD not to schedule the same Region to the same machine. See [Schedule Replicas by Topology Labels](/schedule-replicas-by-topology-labels.md) for label configuration. +- For the test environment, this risk might matter and you can use `--no-labels` to skip the check. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --skip-create-user + +- During the cluster deployment, tiup-cluster checks whether the specified user name in the topology file exists or not. If not, it creates one. To skip this check, you can use the `--skip-create-user` option. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Output + +The deployment log. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-destroy.md b/tiup/tiup-component-cluster-destroy.md new file mode 100644 index 000000000000..cb69edf8d641 --- /dev/null +++ b/tiup/tiup-component-cluster-destroy.md @@ -0,0 +1,51 @@ +--- +title: tiup cluster destroy +--- + +# tiup cluster destroy + +After an application goes offline, if you want to release the machines occupied by the cluster for use by other applications, you need to clean up the data on the cluster and the deployed binary files. To destroy the cluster, the `tiup cluster destroy` command performs the following operations: + +- Stops the cluster. +- For each service, delete its log directory, deployment directory, and data directory. +- If the parent directory of the data directory or deployment directory of each service is created by tiup-cluster, also delete the parent directory. + +## Syntax + +```shell +tiup cluster destroy [flags] +``` + +``: the name of the cluster to destroy. + +## Options + +### --force + +- In some cases, some nodes in the cluster have been down, making it impossible to connect to the node through SSH for operation. At this time, you can use the `--force` option to ignore these errors. +- Data type: `Boolean` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### --retain-node-data + +- Specifies the nodes that need to retain data. If you need to specify more than one node, use this option multiple times: `--retain-node-data --retain-node-data `. +- Data type: `StringArray` +- Default: empty + +### --retain-role-data + +- Specifies the role that needs to retain data. If you need to specify more than one role, use this option multiple times: `--retain-role-data --retain-role-data `. +- Data type: `StringArray` +- Default: empty + +### -h, --help + +- Prints the help information. +- Data type: `Boolean` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Output + +The execution log of the tiup-cluster. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-disable.md b/tiup/tiup-component-cluster-disable.md new file mode 100644 index 000000000000..765fe5423a7e --- /dev/null +++ b/tiup/tiup-component-cluster-disable.md @@ -0,0 +1,49 @@ +--- +title: tiup cluster disable +--- + +# tiup cluster disable + +After restarting the machine on which the cluster service is located, the cluster service will be automatically enabled. To disable the auto-enabling of cluster service, you can use the `tiup cluster disable` command. This command executes `systemctl disable ` on the specified node to disable the auto-enabling of the service. + +## Syntax + +```shell +tiup cluster disable [flags] +``` + +``: the cluster whose service auto-enabling is to be disabled. + +## Options + +### -N, --node + +- Specifies the nodes whose service auto-enabling is to be disabled. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup cluster display`](/tiup/tiup-component-cluster-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, the auto-enabling of all nodes is disabled by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, the auto-enabling of services that match both the specifications of `-N, --node` and `-R, --role` is disabled. + +### -R, --role + +- Specifies the roles whose service auto-enabling is to be disabled. The value of this option is a comma-separated list of node roles. You can get the roles of nodes from the second column of the cluster status table returned by the [`tiup cluster display`](/tiup/tiup-component-cluster-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, the auto-enabling of all roles is disabled by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, the auto-enabling of services that match both the specifications of `-N, --node` and `-R, --role` is disabled. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Output + +The execution log of the tiup-cluster. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-display.md b/tiup/tiup-component-cluster-display.md new file mode 100644 index 000000000000..d8e0c81f2204 --- /dev/null +++ b/tiup/tiup-component-cluster-display.md @@ -0,0 +1,67 @@ +--- +title: tiup cluster display +--- + +# tiup cluster display + +If you want to see the operation status of each component in the cluster, it is obviously inefficient to log in to each machine one by one. Therefore, tiup-cluster provides the `tiup cluster display` command to efficiently complete this task. + +## Syntax + +```shell +tiup cluster display [flags] +``` + +``: the name of the cluster to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-cluster-list.md) command. + +## Options + +### --dashboard + +- By default, all node information of the entire cluster is displayed. With the `--dashboard` option, only dashboard information is displayed. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### -N, --node + +- Specifies the node to display. If this option is not specified, all nodes are displayed. The value of this option is a comma-separated list of node IDs. If you are not sure about the ID of a node, you can skip this option in the command to show the IDs and status of all nodes in the output. +- Data type: `STRINGS` +- If this option is not specified in the command, all nodes are checked by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are checked. + +### -R, --role + +- Specifies the role to display. If it is not specified, all roles are displayed. The value of this option is a comma-separated list of node roles. If you are not sure about the role deployed on a node, you can skip this option in the command to show the roles and status of all nodes in the output. +- Data type: `STRINGS` +- If this option is not specified in the command, all roles are displayed by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are displayed. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Outputs + +- The cluster name +- The cluster version +- SSH Client Type +- Dashboard address +- The table with the following fields: + - ID: the node ID, composed of `IP:PORT` + - Role: the service role deployed on this node (such as TiDB, TiKV) + - Host: the IP of the machine corresponding to the node + - Ports: the port number occupied by the service + - OS/Arch: the operating system and the machine architecture of this node + - Status: the current status of the node service + - Data Dir: the data directory of the service. `-` means no data directory. + - Deploy Dir: the deployment directory of the service + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-edit-config.md b/tiup/tiup-component-cluster-edit-config.md new file mode 100644 index 000000000000..ef7b6780396a --- /dev/null +++ b/tiup/tiup-component-cluster-edit-config.md @@ -0,0 +1,35 @@ +--- +title: tiup cluster edit-config +--- + +# tiup cluster edit-config + +If you need to modify the cluster configuration after the cluster is deployed, you can use the `tiup cluster edit-config` command that starts an editor for you to modify the [topology file](/tiup/tiup-cluster-topology-reference.md) of a cluster. This editor is specified in the `$EDITOR` environment variable by default. If the `$EDITOR` environment variable does not exist, the `vi` editor is used. + +> **Note:** +> +> + When you modify the configuration, you cannot add or delete machines. For how to add machines, see [Scale out a cluster](/tiup/tiup-component-cluster-scale-out.md). For how to delete machines, see [Scale in a cluster](/tiup/tiup-component-cluster-scale-in.md). +> + After you execute the `tiup cluster edit-config` command, the configuration is modified only on the control machine. Then you need to execute the `tiup cluster reload` command to reload the configuration. + +## Syntax + +```shell +tiup cluster edit-config [flags] +``` + +`` is the cluster to operate on. + +## Option + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Output + +- If the command is successfully executed, there is no output. +- If you have mistakenly modified the fields that cannot be modified, when you save the file, an error will be reported, reminding you to edit the file again. For the fields that cannot be modified, see the [topology file](/tiup/tiup-cluster-topology-reference.md). + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-enable.md b/tiup/tiup-component-cluster-enable.md new file mode 100644 index 000000000000..094a0c5ab46c --- /dev/null +++ b/tiup/tiup-component-cluster-enable.md @@ -0,0 +1,53 @@ +--- +title: tiup cluster enable +--- + +# tiup cluster enable + +The `tiup cluster enable` command is used to set the auto-enabling of the cluster service after a machine is restarted. This command enables the auto-enabling of the service by executing `systemctl enable ` at the specified node. + +> **Note:** +> +> When all clusters are shut down and restarted, the order of service startup is determined by the node's operating system startup order. When the restart order is incorrect, in some cases, the restarted cluster still cannot provide services. For example, if TiKV is started first but PD is not started, systemd gives up if TiKV is restarted multiple times while PD is not found). + +## Syntax + +```shell +tiup cluster enable [flags] +``` + +``: the cluster whose service auto-enabling is to be enabled. + +## Options + +### -N, --node + +- Specifies the nodes whose service auto-enabling is to be enabled. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup cluster display`](/tiup/tiup-component-cluster-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, the auto-enabling of all nodes is enabled by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, the auto-enabling of services that match both the specifications of `-N, --node` and `-R, --role` is enabled. + +### -R, --role + +- Specifies the roles whose service auto-enabling is to be enabled. The value of this option is a comma-separated list of node roles. You can get the roles of nodes from the second column of the cluster status table returned by the [`tiup cluster display`](/tiup/tiup-component-cluster-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, the auto-enabling of all roles is enabled by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, the auto-enabling of services that match both the specifications of `-N, --node` and `-R, --role` is enabled. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +The execution log of the tiup-cluster. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-help.md b/tiup/tiup-component-cluster-help.md new file mode 100644 index 000000000000..8c0e1f1f0eda --- /dev/null +++ b/tiup/tiup-component-cluster-help.md @@ -0,0 +1,27 @@ +--- +title: tiup cluster help +--- + +# tiup cluster help + +tiup-cluster provides a wealth of help information for users in the command line interface. You can obtain it via the `help` command or the `--help` option. `tiup cluster help ` is basically equivalent to `tiup cluster --help`. + +## Syntax + +```shell +tiup cluster help [command] [flags] +``` + +`[command]` is used to specify the help information of which command that users need to view. If it is not specified, the help information of tiup-cluster is viewed. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The help information of the `[command]` or tiup-cluster. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-import.md b/tiup/tiup-component-cluster-import.md new file mode 100644 index 000000000000..d9cf8054aacf --- /dev/null +++ b/tiup/tiup-component-cluster-import.md @@ -0,0 +1,69 @@ +--- +title: tiup cluster import +--- + +# tiup cluster import + +Before TiDB v4.0, TiDB clusters were mainly deployed using TiDB Ansible. For TiDB v4.0 and later releases, TiUP Cluster provides the `import` command to transfer the clusters to the tiup-cluster component for management. + +> **Note:** +> +> + After importing the TiDB Ansible configuration to TiUP for management, **DO NOT** use TiDB Ansible for cluster operations anymore. Otherwise, conflicts might be caused due to inconsistent meta information. +> + If the clusters deployed using TiDB Ansible are in any of the following situations, do not use the `import` command. +> + Clusters with TLS encryption enabled +> + Pure KV clusters (clusters without TiDB instances) +> + Clusters with Kafka enabled +> + Clusters with Spark enabled +> + Clusters with TiDB Lightning/TiKV Importer enabled +> + Clusters still using the old `push` mode to collect monitoring metrics (if you keep the default mode `pull` unchanged, using the `import` command is supported) +> + Clusters in which the non-default ports (the ports configured in the `group_vars` directory are compatible) are separately configured in the `inventory.ini` configuration file using `node_exporter_port` / `blackbox_exporter_port` + +## Syntax + +```shell +tiup cluster import [flags] +``` + +## Options + +### -d, --dir + +- Specifies the directory where TiDB Ansible is located. +- Data type: `STRING` +- The option is enabled by default with the current directory (the default value) passed in. + +### --ansible-config + +- Specifies the path of the Ansible configuration file. +- Data type: `STRING` +- The option is enabled by default with `. /ansible.cfg` (the default value) passed in. + +### --inventory + +- Specifies the name of the Ansible inventory file. +- Data type: `STRING` +- The option is enabled by default with `inventory.ini` (the default value) passed in. + +### --no-backup + +- Controls whether to disable the backup of files in the directory where TiDB Ansible is located. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. After a successful import, everything in the directory specified by the `-dir` option is backed up to the `${TIUP_HOME}/.tiup/storage/cluster/clusters/{cluster-name}/ansible-backup` directory. If there are multiple inventory files (when multiple clusters are deployed) in this directory, it is recommended to enable this option. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### --rename + +- Renames the imported cluster. +- Data type: `STRING` +- Default: NULL. If this option is not specified in the command, the cluster_name specified in inventory is used as the cluster name. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +Shows the logs of the import process. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-list.md b/tiup/tiup-component-cluster-list.md new file mode 100644 index 000000000000..8e694bb08802 --- /dev/null +++ b/tiup/tiup-component-cluster-list.md @@ -0,0 +1,37 @@ +--- +title: tiup cluster list +--- + +# tiup cluster list + +tiup-cluster supports deploying multiple clusters using the same control machine. The `tiup cluster list` command outputs all clusters deployed by the currently logged-in user using this control machine. + +> **Note:** +> +> The deployed cluster data is stored in the `~/.tiup/storage/cluster/clusters/` directory by default, so on the same control machine, the currently logged-in user cannot view the clusters deployed by other users. + +## Syntax + +```shell +tiup cluster list [flags] +``` + +## Options + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +## Outputs + +Outputs the table with the following fields: + +- Name: the cluster name +- User: the deployment user +- Version: the cluster version +- Path: the path of the cluster deployment data on the control machine +- PrivateKey: the path of the private key that is used to connect the cluster + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-patch.md b/tiup/tiup-component-cluster-patch.md new file mode 100644 index 000000000000..87d3b31c1d9b --- /dev/null +++ b/tiup/tiup-component-cluster-patch.md @@ -0,0 +1,91 @@ +--- +title: tiup cluster patch +--- + +# tiup cluster patch + +If you need to dynamically replace the binaries of a service while the cluster is running (namely, keep the cluster available during the replacement process), you can use the `tiup cluster patch` command. After the command is executed, TiUP does the following things: + +- Uploads the binary package for replacement to the target machine. +- If the target service is a storage service such as TiKV, TiFlash, or TiDB Binlog, TiUP first takes the related nodes offline via the API. +- Stops the target service. +- Unpacks the binary package and replace the service. +- Starts the target service. + +## Syntax + +```shell +tiup cluster patch [flags] +``` + +- ``: The name of the cluster to be operated. +- ``: The path to the binary package used for replacement. + +### Preparation + +You need to pack the binary package required for this command in advance according to the following steps: + +- Determine the name `${component}` of the component to be replaced (tidb, tikv, pd...), the `${version}` of the component (v4.0.0, v4.0.1 ...), and the operating system `${os}` (`linux`) and platform `${arch}` on which the component runs. +- Download the current component package using the command `wget https://tiup-mirrors.pingcap.com/${component}-${version}-${os}-${arch}.tar.gz -O /tmp/${component}-${version}-${os}-${arch}.tar.gz`. +- Run `mkdir -p /tmp/package && cd /tmp/package` to create a temporary directory to pack files. +- Run `tar xf /tmp/${component}-${version}-${os}-${arch}.tar.gz` to unpack the original binary package. +- Run `find .` to view the file structure in the temporary package directory. +- Copy the binary files or configuration files to the corresponding locations in the temporary directory. +- Run `tar czf /tmp/${component}-hotfix-${os}-${arch}.tar.gz *` to pack the files in the temporary directory. +- Finally, you can use `/tmp/${component}-hotfix-${os}-${arch}.tar.gz` as the `` in the `tiup cluster patch` command. + +## Options + +### --overwrite + +- After you patch a certain component (such as TiDB or TiKV), when the tiup cluster scales out the component, TiUP uses the original component version by default. To use the version that you patch when the cluster scales out in the future, you need to specified the option `--overwrite` in the command. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### --transfer-timeout + +- When restarting the PD or TiKV service, TiKV/PD first transfers the leader of the node to be restarted to another node. Because the transfer process takes some time, you can use the option `--transfer-timeout` to set the maximum waiting time (in seconds). After the timeout, TiUP directly restarts the service. +- Data type: `UINT` +- If this option is not specified, TiUP directly restarts the service after waiting for `300` seconds. + +> **Note:** +> +> If TiUP directly restarts the service after the timeout, the service performance might jitter. + +### -N, --node + +- Specifies nodes to be replaced. The value of this option is a comma-separated list of node IDs. You can get the node ID from the first column of the [cluster status table](/tiup/tiup-component-cluster-display.md) returned by the `tiup cluster display` command. +- Data type: `STRINGS` +- If this option is not specified, TiUP does not select any nodes to replace by default. + +> **Note:** +> +> If the option `-R, --role` is specified at the same time, TiUP then replaces service nodes that match both the requirements of `-N, --node` and `-R, --role`. + +### -R, --role + +- Specifies the roles to be replaced. The value of this option is a comma-separated list of the roles of the nodes. You can get the role deployed on a node from the second column of the [cluster status table](/tiup/tiup-component-cluster-display.md) returned by the `tiup cluster display` command. +- Data type: `STRINGS` +- If this option is not specified, TiUP does not select any roles to replace by default. + +> **Note:** +> +> If the option `-N, --node` is specified at the same time, TiUP then replaces service nodes that match both the requirements of `-N, --node` and `-R, --role`. + +### --offline + +- Declares that the current cluster is not running. When the option is specified, TiUP does not evict the service leader to another node or restart the service, but only replaces the binary files of cluster components. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Outputs + +The execution log of the tiup-cluster. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-prune.md b/tiup/tiup-component-cluster-prune.md new file mode 100644 index 000000000000..194033878181 --- /dev/null +++ b/tiup/tiup-component-cluster-prune.md @@ -0,0 +1,27 @@ +--- +title: tiup cluster prune +--- + +# tiup cluster prune + +When [scaling in the cluster](/tiup/tiup-component-cluster-scale-in.md), for some components, TiUP does not immediately stop their services or delete their data. You need to wait for the data scheduling to complete and then manually execute the `tiup cluster prune` command to clean up. + +## Syntax + +```shell +tiup cluster prune [flags] +``` + +## Option + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The log of the cleanup process. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-reload.md b/tiup/tiup-component-cluster-reload.md new file mode 100644 index 000000000000..9372e9ff36b2 --- /dev/null +++ b/tiup/tiup-component-cluster-reload.md @@ -0,0 +1,85 @@ +--- +title: tiup cluster reload +--- + +# tiup cluster reload + +After [modifying the cluster configuration](/tiup/tiup-component-cluster-edit-config.md), the cluster needs to be reloaded using the `tiup cluster reload` command for the configuration to take effect. This command publishes the configuration of the control machine to the remote machine where the service is running, and follows the upgrade process to restart the services in order according to the upgrade process. The cluster is still available during the restart process. + +## Syntax + +```shell +tiup cluster reload [flags] +``` + +``: the cluster name to operate on. + +## Options + +### --force + +- Ignores errors in the reloading process and forces reload. +- Data type: `BOOLEAN` +- Default: false + +### --transfer-timeout + +- When restarting PD or TiKV, the leader of the restarted node is migrated to other nodes first, and the migration process takes some time. You can set the maximum wait time (in seconds) by setting `-transfer-timeout`. After the timeout, the service can be restarted directly without waiting. +- Data type: `UINT` +- Default: 300 + +> **Note:** +> +> In the case of skipping the waiting and restarting directly, the service performance might jitter. + +### --ignore-config-check + +- After the binary files of components are deployed, the configurations of TiDB, TiKV, and PD components are checked using ` --config-check `. `` is the path of the deployed binary file. `` is the configuration file generated based on the user configuration. If you want to skip this check, you can use this option. +- Data type: `BOOLEAN` +- Default: false + +### -N, --node + +- Specifies the nodes to be restarted. If not specified, all nodes are restarted. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup cluster display`](/tiup/tiup-component-cluster-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all nodes are selected by default. + +> **Note:** +> +> + If the `-R, --role` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are restarted. +> + If the `--skip-restart` option is specified, the `-N, --node` option is invalid. + +### -R, --role + +- Specifies the roles to be restarted. If not specified, all roles are restarted. The value of this option is a comma-separated list of node roles. The role is the second column of the [cluster status](/tiup/tiup-component-cluster-display.md) table. +- Data type: `STRINGS` +- If this option is not specified in the command, all roles are selected by default. + +> **Note:** +> +> 1. If the `-N, --node` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are restarted. +> 2. If the `--skip-restart` option is specified, the `-R, --role` option is invalid. + +### --skip-restart + +The `tiup cluster reload` command performs two operations: + +- Refreshes all node configurations +- Restarts the specified node + +After you specify the `--skip-restart` option, it only refreshes the configuration without restarting any nodes, so that the refreshed configuration is not applied and does not take effect until the next restart of the corresponding service. + +- Data type: `BOOLEAN` +- Default: false + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The execution log of the tiup-cluster. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-rename.md b/tiup/tiup-component-cluster-rename.md new file mode 100644 index 000000000000..79e7e3a4c0f3 --- /dev/null +++ b/tiup/tiup-component-cluster-rename.md @@ -0,0 +1,37 @@ +--- +title: tiup cluster rename +--- + +# tiup cluster rename + +The cluster name is specified when [the cluster is deployed](/tiup/tiup-component-cluster-deploy.md). If you want to change the cluster name after the cluster is deployed, you can use the command `tiup cluster rename`. + +> **Note:** +> +> If the `dashboard_dir` field of `grafana_servers` is configured for the TiUP cluster, after you execute the command `tiup cluster rename` to rename the cluster, the following additional steps are required: +> +> + For the `*.json` files in the local dashboards directory, update the `datasource` field of each file to the new cluster name, because the value of `datasource` must be the name of the cluster. +> + Execute the command `tiup cluster reload -R grafana`. + +## Syntax + +```shell +tiup cluster rename [flags] +``` + +- ``: The old cluster name. +- ``: The new cluster name. + +## Options + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Outputs + +The execution log of the tiup-cluster. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-replay.md b/tiup/tiup-component-cluster-replay.md new file mode 100644 index 000000000000..cfa134c6794e --- /dev/null +++ b/tiup/tiup-component-cluster-replay.md @@ -0,0 +1,27 @@ +--- +title: tiup cluster replay +--- + +# tiup cluster replay + +When you perform a cluster operation such as upgrade or restart, the operation might fail due to cluster environment issues. If you re-perform the operation, you need to perform all the steps from the very beginning. If the cluster is large, re-performing these steps will take a long time. In this case, you can use the `tiup cluster replay` command to retry the failed commands and skip the successfully performed steps. + +## Syntax + +```shell +tiup cluster replay [flags] +``` + +- ``: the `audit-id` of the command to be retried. You can view the historical commands and their `audit-id`s using the [`tiup cluster audit`](/tiup/tiup-component-cluster-audit.md) command. + +## Option + +### -h, --help + +Prints the help information. + +## Output + +The output of the command corresponding to ``. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-restart.md b/tiup/tiup-component-cluster-restart.md new file mode 100644 index 000000000000..5bd9a994d320 --- /dev/null +++ b/tiup/tiup-component-cluster-restart.md @@ -0,0 +1,53 @@ +--- +title: tiup cluster restart +--- + +# tiup cluster restart + +The command `tiup cluster restart` is used to restart all or some of the services of the specified cluster. + +> **Note:** +> +> During the restart process, the related services are unavailable for a period of time. + +## Syntax + +```shell +tiup cluster restart [flags] +``` + +``: the name of the cluster to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-cluster-list.md) command. + +## Options + +### -N, --node + +- Specifies the nodes to be restarted. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the [cluster status table](/tiup/tiup-component-cluster-display.md) returned by the `tiup cluster display` command. +- Data type: `STRING` +- If this option is not specified, TiUP restarts all nodes by default. + +> **Note:** +> +> If the option `-R, --role` is specified at the same time, TiUP restarts service nodes that match both the requirements of `-N, --node` and `-R, --role`. + +### -R, --role + +- Specified the roles of nodes to be restarted. The value of this option is a comma-separated list of the roles of the nodes. You can get the roles of the nodes from the second column of the [cluster status table](/tiup/tiup-component-cluster-display.md) returned by the `tiup cluster display` command. +- Data type: `STRING` +- If this option is not specified, TiUP restarts nodes of all roles by default. + +> **Note:** +> +> If the option `-N, --node` is specified at the same time, TiUP restarts service nodes that match both the requirements of `-N, --node` and `-R, --role`. + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Outputs + +The log of the service restart process. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-scale-in.md b/tiup/tiup-component-cluster-scale-in.md new file mode 100644 index 000000000000..d60ca8205329 --- /dev/null +++ b/tiup/tiup-component-cluster-scale-in.md @@ -0,0 +1,72 @@ +--- +title: tiup cluster scale-in +--- + +# tiup cluster scale-in + +The `tiup cluster scale-in` command is used to scale in the cluster, which takes the services of the specified nodes offline, removes the specified nodes from the cluster, and deletes the remaining files from those nodes. + +Because the TiKV, TiFlash, and TiDB Binlog components are taken offline asynchronously (which requires TiUP to remove the node through API first) and the stopping process takes a long time (which requires TiUP to continuously check whether the node is successfully taken offline), the TiKV, TiFlash, and TiDB Binlog components are handled particularly as follows: + +- For TiKV, TiFlash and, TiDB Binlog components: + + 1. TiUP Cluster takes the node offline through API and directly exits without waiting for the process to be completed. + 2. To check the status of the nodes being scaled in, you need to execute the `tiup cluster display` command and wait for the status to become `Tombstone`. + 3. To clean up the nodes in the `Tombstone` status, you need to execute the `tiup cluster prune` command. The `tiup cluster prune` command performs the following operations: + + - Stops the services of the nodes that have been taken offline. + - Cleans up the data files of the nodes that have been taken offline. + - Updates the cluster topology and removes the nodes that have been taken offline. + +For other components: + +- When taking the PD components offline, TiUP Cluster quickly deletes the specified nodes from the cluster through API, stops the service of the specified PD nodes, and then deletes the related data files from the nodes. +- When taking other components down, TiUP Cluster directly stops the node services and deletes the related data files from the specified nodes. + +## Syntax + +```shell +tiup cluster scale-in [flags] +``` + +`` is the name of the cluster to scale in. If you forget the cluster name, you can check it using the [`tiup cluster list`](/tiup/tiup-component-cluster-list.md) command. + +## Options + +### -N, --node + +- Specifies the nodes to take down. Multiple nodes are separated by commas. +- Data type: `STRING` +- There is no default value. This option is mandatory and the value must be not null. + +### --force + +- Controls whether to forcibly remove the specified nodes from the cluster. Sometimes, the host of the node to take offline might be down, which makes it impossible to connect to the node via SSH for operations, so you can forcibly remove the node from the cluster using the `-force` option. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +> **Note:** +> +> Because the forced removal of a TiKV node does not wait for data to be scheduled, removing more than one serving TiKV node is at the risk of data loss. + +### --transfer-timeout + +- When a PD or TiKV node is to be removed, the Region leader on the node will be transferred to another node first. Because the transferring process takes some time, you can set the maximum waiting time (in seconds) by configuring `--transfer-timeout`. After the timeout, the `tiup cluster scale-in` command skips waiting and starts the scaling-in directly. +- Data type: `UINT` +- The option is enabled by default with `300` seconds (the default value) passed in. + +> **Note:** +> +> If a PD or TiKV node is taken offline directly without waiting for the leader transfer to be completed, the service performance might jitter. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +Shows the logs of the scaling-in process. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-scale-out.md b/tiup/tiup-component-cluster-scale-out.md new file mode 100644 index 000000000000..fba6627d5355 --- /dev/null +++ b/tiup/tiup-component-cluster-scale-out.md @@ -0,0 +1,65 @@ +--- +title: tiup cluster scale-out +--- + +# tiup cluster scale-out + +The `tiup cluster scale-out` command is used for scaling out the cluster. The internal logic of scaling out the cluster is similar to the cluster deployment. The tiup-cluster component first establishes an SSH connection to the new node, creates the necessary directories on the target node, then performs the deployment and starts the service. + +When PD is scaled out, new PD nodes are added to the cluster by the join operation, and the configuration of the services associated with PD is updated; other services are directly started and added to the cluster. + +## Syntax + +```shell +tiup cluster scale-out [flags] +``` + +``: the name of the cluster to operate on. If you forget the cluster name, you can check it with the [`cluster list`](/tiup/tiup-component-dm-list.md) command. + +``: the prepared [topology file](/tiup/tiup-dm-topology-reference.md). This topology file should only contain the new nodes that are to be added to the current cluster. + +## Options + +### -u, --user + +- Specifies the user name used to connect to the target machine. This user must have the secret-free sudo root permission on the target machine. +- Data type: `STRING` +- Default: the current user who executes the command. + +### -i, --identity_file + +- Specifies the key file used to connect to the target machine. +- Data type: `STRING` +- If this option is not specified in the command, the `~/.ssh/id_rsa` file is used to connect to the target machine by default. + +### -p, --password + +- Specifies the password used to connect to the target machine. Do not use this option and `-i/--identity_file` at the same time. +- Data type: `BOOLEAN` +- Default: false + +### --no-labels + +- This option is used to skip the label check. +- When two or more TiKV nodes are deployed on the same physical machine, a risk exists: PD does not know the cluster topology, so it might schedule multiple replicas of a Region to different TiKV nodes on one physical machine, which makes this physical machine a single point of failure. To avoid this risk, you can use labels to tell PD not to schedule the same Region to the same machine. See [Schedule Replicas by Topology Labels](/schedule-replicas-by-topology-labels.md) for label configuration. +- For the test environment, this risk might not matter, and you can use `--no-labels` to skip the check. +- Data type: `BOOLEAN` +- Default: false + +### --skip-create-user + +- During the cluster deployment, tiup-cluster checks whether the specified user name in the topology file exists or not. If not, it creates one. To skip this check, you can use the `--skip-create-user` option. +- Data type: `BOOLEAN` +- Default: false + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The log of scaling out. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-start.md b/tiup/tiup-component-cluster-start.md new file mode 100644 index 000000000000..6ee0a10279f6 --- /dev/null +++ b/tiup/tiup-component-cluster-start.md @@ -0,0 +1,49 @@ +--- +title: tiup cluster start +--- + +# tiup cluster start + +The `tiup cluster start` command is used to start all services or some services of the specified cluster. + +## Syntax + +```shell +tiup cluster start [flags] +``` + +`` is the name of the cluster to operate on. If you forget the cluster name, you can check it using the [`tiup cluster list`](/tiup/tiup-component-cluster-list.md) command. + +## Options + +### -N, --node + +- Specifies the nodes to be started. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the [cluster status table](/tiup/tiup-component-cluster-display.md) returned by the `tiup cluster display` command. +- Data type: `STRINGS` +- If this option is not specified in the command, all nodes are started by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are started. + +### -R, --role + +- Specifies the roles of nodes to be started. The value of this option is a comma-separated list of the roles of the nodes. You can get the roles of the nodes from the second column of the [cluster status table](/tiup/tiup-component-cluster-display.md) returned by the `tiup cluster display` command. +- Data type: `STRINGS` +- If this option is not specified in the command, all roles are started by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are started. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +The log of starting the service. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-stop.md b/tiup/tiup-component-cluster-stop.md new file mode 100644 index 000000000000..f8192e8ef02a --- /dev/null +++ b/tiup/tiup-component-cluster-stop.md @@ -0,0 +1,53 @@ +--- +title: tiup cluster stop +--- + +# tiup cluster stop + +The `tiup cluster stop` command is used to stop all services or some services of the specified cluster. + +> **Note:** +> +> If the core services of a cluster are stopped, the cluster cannot provide services anymore. + +## Syntax + +```shell +tiup cluster stop [flags] +``` + +`` is the name of the cluster to operate on. If you forget the cluster name, you can check it using the [`tiup cluster list`](/tiup/tiup-component-cluster-list.md) command. + +## Options + +### -N, --node + +- Specifies the nodes to be stopped. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the [cluster status table](/tiup/tiup-component-cluster-display.md) returned by the `tiup cluster display` command. +- Data type: `STRINGS` +- If this option is not specified in the command, the command stops all the nodes by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are stopped. + +### -R, --role + +- Specifies the roles of nodes to be stopped. The value of this option is a comma-separated list of the roles of the nodes. You can get the roles of the nodes from the second column of the [cluster status table](/tiup/tiup-component-cluster-display.md) returned by the `tiup cluster display` command. +- Data type: `STRINGS` +- If this option is not specified in the command, the command stops all the roles by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are stopped. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +The log of stopping the service. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-template.md b/tiup/tiup-component-cluster-template.md new file mode 100644 index 000000000000..628002ccaee5 --- /dev/null +++ b/tiup/tiup-component-cluster-template.md @@ -0,0 +1,44 @@ +--- +title: tiup cluster template +--- + +# tiup cluster template + +Before deploying the cluster, you need to prepare a [topology file](/tiup/tiup-cluster-topology-reference.md) of the cluster. TiUP has a built-in topology file template, and you can modify this template to create the final topology file. To output the built-in template content, you can use the `tiup cluster template` command. + +## Syntax + +```shell +tiup cluster template [flags] +``` + +If this option is not specified, the output default template contains the following instances: + +- 3 PD instances +- 3 TiKV instances +- 1 TiDB instance +- 1 Prometheus instance +- 1 Grafana instance +- 1 Alertmanager instance + +## Options + +### --full + +- Outputs a detailed topology template that is commented with configurable parameters. To enable this option, add it to the command. +- If this option is not specified, the simple topology template is output by default. + +### --multi-dc + +- Outputs the topology template of multiple data centers. To enable this option, add it to the command. +- If this option is not specified, the topology template of a single data center is output by default. + +### -h, --help + +Prints the help information. + +## Output + +Outputs the topology template according to the specified options, which can be redirected to the topology file for deployment. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster-upgrade.md b/tiup/tiup-component-cluster-upgrade.md new file mode 100644 index 000000000000..02df5f7ea6fb --- /dev/null +++ b/tiup/tiup-component-cluster-upgrade.md @@ -0,0 +1,62 @@ +--- +title: tiup cluster upgrade +--- + +# tiup cluster upgrade + +The `tiup cluster upgrade` command is used to upgrade the specified cluster to a specific version. + +## Syntax + +```shell +tiup cluster upgrade [flags] +``` + +- ``: the cluster name to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-cluster-list.md) command. +- ``: the target version to upgrade to. Currently, it is only allowed to upgrade to a version higher than the current cluster, that is, no downgrade is allowed. It is also not allowed to upgrade to the nightly version. + +## Options + +### --force + +- To upgrade the cluster, you need to ensure that the cluster is currently started. In some cases, you might want to upgrade when the cluster is not started. At this time, you can use `--force` to ignore the error during the upgrade, forcibly replace the binary file and start the cluster. +- Data type: `BOOLEAN` +- Default: false + +> **Note:** +> +> Forcing an upgrade of the cluster that is providing services might result in service unavailability. Unstarted clusters are started automatically after a successful upgrade. + +### --transfer-timeout + +- When upgrading PD or TiKV, the leader of the upgraded node is migrated to other nodes first. The migration process takes some time, and you can set the maximum wait time (in seconds) by the `-transfer-timeout` option. After the timeout, the wait is skipped and the service is upgraded directly. +- Data type: `uint` +- Default: 300 + +> **Note:** +> +> If the wait is skipped and the service is upgraded directly, the service performance might jitter. + +### --ignore-config-check + +- After the binary is updated, a configuration check is performed on the TiDB, TiKV and PD components using ` --config-check `. `` is the path to the newly deployed binary and `` is the configuration file generated based on the user configuration. To skip this check, you can use the `--ignore-config-check` option. +- Data type: `BOOLEAN` +- Default: false + +### --offline + +- Declares that the current cluster is not running. When this option is specified, TiUP does not evict the service leader to another node or restart the service, but only replaces the binary files of the cluster components. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The log of the upgrading progress. + +[<< Back to the previous page - TiUP Cluster command list](/tiup/tiup-component-cluster.md#command-list) diff --git a/tiup/tiup-component-cluster.md b/tiup/tiup-component-cluster.md new file mode 100644 index 000000000000..a1ace2810dee --- /dev/null +++ b/tiup/tiup-component-cluster.md @@ -0,0 +1,87 @@ +--- +title: TiUP Cluster +--- + +# TiUP Cluster + +TiUP Cluster is a cluster management component of TiUP written in Golang. You can use the TiUP Cluster component to perform daily operations and maintenance, including deploying, starting, shutting down, destroying, elastic scaling, upgrading TiDB clusters, and managing TiDB cluster parameters. + +## Syntax + +```shell +tiup cluster [command] [flags] +``` + +`[command]` is the name of the command. For the supported commands, refer to the [command list](#command-list) below. + +## Options + +### --ssh + +- Specifies the SSH client to connect to the remote end (the machine where the TiDB service is deployed) for the command execution. +- Data type: `STRING` +- Supported values: + + - `builtin`: uses the easyssh client built in tiup-cluster as the SSH client. + - `system`: uses the default SSH client of the current operating system. + - `none`: The SSH client is not used. The deployment is only for the current machine. + +- If this option is not specified in the command, `builtin` is used as the default value. + +### --ssh-timeout + +- Specifies the SSH connection timeout in seconds. +- Data type: `UINT` +- If this option is not specified in the command, the default timeout is `5` seconds. + +### --wait-timeout + +- Specifies the maximum waiting time (in seconds) for each step in the operation process. The operation process consists of many steps, such as specifying systemctl to start or stop services, and waiting for ports to be online or offline. Each step may take several seconds. If the execution time of a step exceeds the specified timeout, the step exits with an error. +- Data type: `UINT` +- If this option is not specified in the command, the maximum waiting time for each step is `120` seconds. + +### -y, --yes + +- Skips the secondary confirmation of all risky operations. Using this option is not recommended, unless you are using a script to call TiUP. +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### -v, --version + +- Prints the current version of TiUP Cluster. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### -h, --help + +- Prints the help information of the related commands. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Command list + +- [import](/tiup/tiup-component-cluster-import.md): imports a cluster deployed by Ansible +- [template](/tiup/tiup-component-cluster-template.md): outputs the topology template +- [check](/tiup/tiup-component-cluster-check.md): checks a cluster before and after the deployment +- [deploy](/tiup/tiup-component-cluster-deploy.md): deploys a cluster based on a specified topology +- [list](/tiup/tiup-component-cluster-list.md): queries the list of deployed clusters +- [display](/tiup/tiup-component-cluster-display.md): displays the status of a specified cluster +- [start](/tiup/tiup-component-cluster-start.md): starts a specified cluster +- [stop](/tiup/tiup-component-cluster-stop.md): stops a specified cluster +- [restart](/tiup/tiup-component-cluster-restart.md): restarts a specified cluster +- [scale-in](/tiup/tiup-component-cluster-scale-in.md): scales in a specified cluster +- [scale-out](/tiup/tiup-component-cluster-scale-out.md): scales out a specified cluster +- [upgrade](/tiup/tiup-component-cluster-upgrade.md): upgrades a specified cluster +- [prune](/tiup/tiup-component-cluster-prune.md): cleans up the instances in the Tombstone status for a specified cluster +- [edit-config](/tiup/tiup-component-cluster-edit-config.md): modifies the configuration of a specified cluster +- [reload](/tiup/tiup-component-cluster-reload.md): reloads the configuration of a specified cluster +- [patch](/tiup/tiup-component-cluster-patch.md): replaces a service in a deployed cluster +- [rename](/tiup/tiup-component-cluster-rename.md): renames a cluster +- [clean](/tiup/tiup-component-cluster-clean.md): deletes data from the specified cluster +- [destroy](/tiup/tiup-component-cluster-destroy.md): destroys a specified cluster +- [audit](/tiup/tiup-component-cluster-audit.md): queries the operation audit log of a specified cluster +- [replay](/tiup/tiup-component-cluster-replay.md): retries the specified command +- [enable](/tiup/tiup-component-cluster-enable.md): enables the auto-enabling of the cluster service after a machine is restarted. +- [disable](/tiup/tiup-component-cluster-disable.md): disables the auto-enabling of the cluster service after a machine is restarted. +- [help](/tiup/tiup-component-cluster-help.md): prints the help information + +[<< Back to the previous page - TiUP Reference component list](/tiup/tiup-reference.md#component-list) diff --git a/tiup/tiup-component-dm-audit.md b/tiup/tiup-component-dm-audit.md new file mode 100644 index 000000000000..f30f7d957b47 --- /dev/null +++ b/tiup/tiup-component-dm-audit.md @@ -0,0 +1,34 @@ +--- +title: tiup dm audit +--- + +# tiup dm audit + +The `tiup dm audit` command is used to view historical commands executed on all clusters and the execution log of each command. + +## Syntax + +```shell +tiup dm audit [audit-id] [flags] +``` + +- If you do not fill in the `[audit-id]`, the table of operation records is output in reverse chronological order. The first column is the `audit-id`. +- If you fill in the `[audit-id]`, the execution log of the specified `audit-id` is checked. + +## Option + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +- If `[audit-id]` is specified, the corresponding execution log is output. +- If `[audit-id]` is not specified, a table with the following fields is output: + - ID: the `audit-id` corresponding to this record + - Time: the execution time of the command corresponding to the record + - Command: the command corresponding to the record + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-deploy.md b/tiup/tiup-component-dm-deploy.md new file mode 100644 index 000000000000..3ef4b0f3b5ca --- /dev/null +++ b/tiup/tiup-component-dm-deploy.md @@ -0,0 +1,49 @@ +--- +title: tiup dm deploy +--- + +# tiup dm deploy + +The `tiup dm deploy` command is used to deploy a new cluster. + +## Syntax + +```shell +tiup dm deploy [flags] +``` + +- ``: the name of the new cluster, which cannot be the same as the existing cluster names. +- ``: the version number of the DM cluster to be deployed, such as `v2.0.0`. +- ``: the prepared [topology file](/tiup/tiup-dm-topology-reference.md). + +## Options + +### -u, --user + +- Specifies the user name used to connect to the target machine. This user must have the secret-free sudo root permission on the target machine. +- Data type: `STRING` +- Default: the current user who executes the command. + +### -i, --identity_file + +- Specifies the key file used to connect to the target machine. +- Data type: `STRING` +- Default: `~/.ssh/id_rsa` + +### -p, --password + +- Specifies the password used to connect to the target machine. Do not use this option and `-i/--identity_file` at the same time. +- Data type: `BOOLEAN` +- Default: false + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The deployment log. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-destroy.md b/tiup/tiup-component-dm-destroy.md new file mode 100644 index 000000000000..0c7f04d4097a --- /dev/null +++ b/tiup/tiup-component-dm-destroy.md @@ -0,0 +1,33 @@ +--- +title: tiup dm destroy +--- + +# tiup dm destroy + +After an application goes offline, if you want to release the machines occupied by the cluster for use by other applications, you need to clean up the data on the cluster and the deployed binary files. To destroy the cluster, the `tiup dm destroy` command performs the following operations: + +- Stops the cluster. +- For each service, delete its log directory, deployment directory, and data directory. +- If the parent directory of the data directory or deployment directory of each service is created by `tiup-dm`, also delete the parent directory. + +## Syntax + +```shell +tiup dm destroy [flags] +``` + +``: the name of the cluster to be destroyed. + +## Option + +### -h, --help + +- Prints the help information. +- Data type: `Boolean` +- Default: false + +## Output + +The execution log of the tiup-dm. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-disable.md b/tiup/tiup-component-dm-disable.md new file mode 100644 index 000000000000..fca46d3ae5c4 --- /dev/null +++ b/tiup/tiup-component-dm-disable.md @@ -0,0 +1,47 @@ +--- +title: tiup dm disable +--- + +# tiup dm disable + +After restarting the machine on which the cluster service is located, the cluster service will be automatically enabled. To disable the auto-enabling of cluster service, you can use the `tiup dm disable` command. This command executes `systemctl disable ` on the specified node to disable the auto-enabling of the service. + +## Syntax + +```shell +tiup dm disable [flags] +``` + +`` is the cluster whose service auto-enabling is to be disabled. + +## Options + +### -N, --node + +- Specifies the nodes whose service auto-enabling is to be disabled. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, the auto-enabling of all nodes is disabled by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, the auto-enabling of services that match both the specifications of `-N, --node` and `-R, --role` is disabled. + +### -R, --role + +- Specifies the roles whose service auto-enabling is to be disabled. The value of this option is a comma-separated list of node roles. You can get the roles of nodes from the second column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, the auto-enabling of all roles is disabled by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, the auto-enabling of services that match both the specifications of `-N, --node` and `-R, --role` is disabled. + +### -h, --help + +Prints the help information. + +## Output + +The execution log of the tiup-dm. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-display.md b/tiup/tiup-component-dm-display.md new file mode 100644 index 000000000000..4d20cfef25b5 --- /dev/null +++ b/tiup/tiup-component-dm-display.md @@ -0,0 +1,60 @@ +--- +title: tiup dm display +--- + +# tiup dm display + +If you want to check the operational status of each component in a DM cluster, it is inefficient to log in to each machine one by one. Therefore, tiup-dm provides the `tiup dm display` command to do this job efficiently. + +## Syntax + +```shell +tiup dm display [flags] +``` + +`` is the name of the cluster to be operated. If you forget the cluster name, you can check it using the [`tiup dm list`](/tiup/tiup-component-dm-list.md) command. + +## Options + +### -N, --node + +- Specifies the IDs of the nodes to query, splitting by commas for multiple nodes. If you are not sure about the ID of a node, you can skip this option in the command to show the IDs and status of all nodes in the output. +- Data type: `STRING` +- This option is enabled by default with `[]` (which means all nodes) passed in. + +> **Note:** +> +> If `-R, --role` is also specified, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are queried. + +### -R, --role + +- Specifies the roles to query, splitting by commas for multiple roles. If you are not sure about the role deployed on a node, you can skip this option in the command to show the roles and status of all nodes in the output. +- Data type: `STRING` +- This option is enabled by default with `[]` (which means all roles) passed in. + +> **Note:** +> +> If `-N, --node` is also specified, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are queried. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +- Cluster name +- Cluster version +- SSH client type +- A table containing the following fields: + - `ID`: the node ID, consisting of IP:PORT. + - `Role`: the service role deployed on the node (for example, TiDB or TiKV). + - `Host`: the IP address of the machine corresponding to the node. + - `Ports`: the port number used by the service. + - `OS/Arch`: the operating system and machine architecture of the node. + - `Status`: the current status of the services on the node. + - `Data Dir`: the data directory of the service. `-` means that there is no data directory. + - `Deploy Dir`: the deployment directory of the service. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-edit-config.md b/tiup/tiup-component-dm-edit-config.md new file mode 100644 index 000000000000..b2cd02bc55f4 --- /dev/null +++ b/tiup/tiup-component-dm-edit-config.md @@ -0,0 +1,35 @@ +--- +title: tiup dm edit-config +--- + +# tiup dm edit-config + +If you need to modify the cluster service configuration after the cluster is deployed, you can use the `tiup dm edit-config` command that starts an editor for you to modify the [topology file](/tiup/tiup-dm-topology-reference.md). of the specified cluster. This editor is specified in the `$EDITOR` environment variable by default. If the `$EDITOR` environment variable does not exist, the `vi` editor is used. + +> **Note:** +> +> + When you modify the configuration, you cannot add or delete machines. For how to add machines, see [Scale out a cluster](/tiup/tiup-component-dm-scale-out.md). For how to delete machines, see [Scale in a cluster](/tiup/tiup-component-dm-scale-in.md). +> + After you execute the `tiup dm edit-config` command, the configuration is modified only on the control machine. Then you need to execute the `tiup dm relaod` command to reload the configuration. + +## Syntax + +```shell +tiup dm edit-config [flags] +``` + +``: the cluster to operate on. + +## Option + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +- Normally, no output. +- If you have mistakenly modified the fields that cannot be modified, when you save the file, an error is reported, reminding you to edit the file again. For the fields that cannot be modified, see [the topology file](/tiup/tiup-dm-topology-reference.md). + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-enable.md b/tiup/tiup-component-dm-enable.md new file mode 100644 index 000000000000..f16000128a84 --- /dev/null +++ b/tiup/tiup-component-dm-enable.md @@ -0,0 +1,47 @@ +--- +title: tiup dm enable +--- + +# tiup dm enable + +The `tiup dm enable` command is used to set the auto-enabling of the cluster service after a machine is restarted. This command enables the auto-enabling of the service by executing `systemctl enable ` at the specified node. + +## Syntax + +```shell +tiup dm enable [flags] +``` + +`` is the cluster whose service auto-enabling is to be enabled. + +## Options + +### -N, --node + +- Specifies the nodes whose service auto-enabling is to be enabled. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, the auto-enabling of all nodes is enabled by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, the auto-enabling of services that match both the specifications of `-N, --node` and `-R, --role` is enabled. + +### -R, --role + +- Specifies the roles whose service auto-enabling is to be enabled. The value of this option is a comma-separated list of node roles. You can get the roles of nodes from the second column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, the auto-enabling of all roles is enabled by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, the auto-enabling of services that match both the specifications of `-N, --node` and `-R, --role` is enabled. + +### -h, --help + +Prints the help information. + +## Output + +the execution log of tiup-dm. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-help.md b/tiup/tiup-component-dm-help.md new file mode 100644 index 000000000000..035530c224aa --- /dev/null +++ b/tiup/tiup-component-dm-help.md @@ -0,0 +1,27 @@ +--- +title: tiup dm help +--- + +# tiup dm help + +tiup-dm command-line interface provides users with a wealth of help information. You can view it via the `help` command or the `--help` option. Basically, `tiup dm help ` is equivalent to `tiup dm --help`. + +## Syntax + +```shell +tiup dm help [command] [flags] +``` + +`[command]` is used to specify the help information of which command that users need to view. If it is not specified, the help information of `tiup-dm` is viewed. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The help information of `[command]` or `tiup-dm`. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-import.md b/tiup/tiup-component-dm-import.md new file mode 100644 index 000000000000..b84c1de36a19 --- /dev/null +++ b/tiup/tiup-component-dm-import.md @@ -0,0 +1,62 @@ +--- +title: tiup dm import +--- + +# tiup dm import + +In DM v1.0, the cluster is basically deployed using TiDB Ansible. TiUP DM provides the `import` command to import v1.0 clusters and redeploy the clusters in DM v2.0. + +> **Note:** +> +> - The command does not support importing DM Portal components from DM v1.0 clusters. +> - Before importing the cluster, stop running the original cluster first. +> - For data migration tasks that need to be upgraded to v2.0, do not execute `stop-task` on these tasks. +> - The command only supports importing to DM v2.0.0-rc.2 and later versions. +> - The `import` command is used to import a DM v1.0 cluster to a new DM v2.0 cluster. If you need to import data migration tasks to an existing v2.0 cluster, refer to [Manually Upgrade TiDB Data Migration from v1.0.x to v2.0.x](https://docs.pingcap.com/tidb-data-migration/stable/manually-upgrade-dm-1.0-to-2.0) +> - The deployment directories of some components might be different from those in the original cluster. You can check it with the `display` command. +> - Before importing the cluster, run `tiup update --self && tiup update dm` to upgrade TiUP DM components to the latest version. +> - After the cluster is imported, there is only one DM-master node in the cluster. You can refer to [the `scale out` command](/tiup/tiup-component-dm-scale-out.md) to scale out the DM-master node. + +## Syntax + +```shell +tiup dm import [flags] +``` + +## Options + +### -v, --cluster-version + +- Specifies the version number for redeploying. You must use a version later than v2.0.0-rc.2 (including v2.0.0-rc.2). +- Data type: `STRING` +- This option is **required** to execute the command. + +### -d, --dir + +- Specifies the directory of TiDB Ansible. +- Data type: `STRING` +- If this option is not specified in the command, the current directory is the default directory. + +### --inventory + +- Specifies the name of the Ansible inventory file. +- Data type: `STRING` +- If this option is not specified in the command, the default file name is `"inventory.ini"`. + +### --rename + +- Renames the imported cluster. +- Data type: `STRING` +- If this option is not specified in the command, the default cluster name is the `cluster_name` specified in the inventory file. + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Outputs + +The log of the importing process. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-list.md b/tiup/tiup-component-dm-list.md new file mode 100644 index 000000000000..c7719d5999b5 --- /dev/null +++ b/tiup/tiup-component-dm-list.md @@ -0,0 +1,37 @@ +--- +title: tiup dm list +--- + +# tiup dm list + +`tiup-dm` supports deploying multiple clusters using the same control machine. You can use the `tiup dm list` command to check which clusters are deployed using the control machine by the currently logged-in user. + +> **Note:** +> +> By default, the data of the deployed clusters is stored in the `~/.tiup/storage/dm/clusters/` directory. The currently logged-in user cannot view the clusters deployed by other users on the same control machine. + +## Syntax + +```shell +tiup dm list [flags] +``` + +## Options + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +A table consisting of the following fields: + +- `Name`: the cluster name. +- `User`: the user who deployed the cluster. +- `Version`: the cluster version. +- `Path`: the path of the cluster deployment data on the control machine. +- `PrivateKey`: the path of the private key to the cluster. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-patch.md b/tiup/tiup-component-dm-patch.md new file mode 100644 index 000000000000..03acd41f5a35 --- /dev/null +++ b/tiup/tiup-component-dm-patch.md @@ -0,0 +1,79 @@ +--- +title: tiup dm patch +--- + +# tiup dm patch + +If you need to dynamically replace the binaries of a service while the cluster is running (that is, to keep the cluster available during the replacement), you can use the `tiup dm patch` command. The command does the following: + +- Uploads the binary package for replacement to the target machine. +- Takes the related nodes offline using the API. +- Stops the target service. +- Unpacks the binary package and replaces the service. +- Starts the target service. + +## Syntax + +```shell +tiup dm patch [flags] +``` + +- ``: The name of the cluster to be operated. +- ``: The path to the binary package used for replacement. + +### Preparation + +You need to pack the binary package required for this command in advance according to the following steps: + +- Determine the name `${component}` of the component to be replaced (dm-master, dm-worker ...), the `${version}` of the component (v2.0.0, v2.0.1 ...), and the operating system `${os}` and platform `${arch}` on which the component runs. +- Download the current component package using the command `wget https://tiup-mirrors.pingcap.com/${component}-${version}-${os}-${arch}.tar.gz -O /tmp/${component}-${version}-${os}-${arch}.tar.gz`. +- Run `mkdir -p /tmp/package && cd /tmp/package` to create a temporary directory to pack files. +- Run `tar xf /tmp/${component}-${version}-${os}-${arch}.tar.gz` to unpack the original binary package. +- Run `find .` to view the file structure in the temporary package directory. +- Copy the binary files or configuration files to the corresponding locations in the temporary directory. +- Run `tar czf /tmp/${component}-hotfix-${os}-${arch}.tar.gz *` to pack the files in the temporary directory. +- Finally, you can use `/tmp/${component}-hotfix-${os}-${arch}.tar.gz` as the value of `` in the `tiup dm patch` command. + +## Options + +### --overwrite + +- After you patch a certain component (such as dm-worker), when the tiup-dm scales out the component, tiup-dm uses the original component version by default. To use the version that you patch when the cluster scales out in the future, you need to specify the option `--overwrite` in the command. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### -N, --node + +- Specifies the nodes to be replaced. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the `[tiup dm display](/tiup/tiup-component-dm-display.md)` command. +- Data type: `STRING` +- If this option is not specified, TiUP selects all nodes to replace by default. + +> **Note:** +> +> If the option `-R, --role` is specified at the same time, TiUP then replaces service nodes that match both the requirements of `-N, --node` and `-R, --role`. + +### -R, --role + +- Specifies the roles to be replaced. The value of this option is a comma-separated list of the roles of the nodes. You can get the roles of the nodes from the second column of the cluster status table returned by the `[tiup dm display](/tiup/tiup-component-dm-display.md)` command. +- Data type: `STRING` +- If this option is not specified, TiUP selects all roles to replace by default. + +> **Note:** +> +> If the option `-N, --node` is specified at the same time, TiUP then replaces service nodes that match both the requirements of `-N, --node` and `-R, --role`. + +### --offline + +- Declares that the current cluster is offline. When this option is specified, TiUP DM only replaces the binary files of the cluster components in place without restarting the service. + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Outputs + +The execution log of tiup-dm. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-prune.md b/tiup/tiup-component-dm-prune.md new file mode 100644 index 000000000000..66fef792543d --- /dev/null +++ b/tiup/tiup-component-dm-prune.md @@ -0,0 +1,27 @@ +--- +title: tiup dm prune +--- + +# tiup dm prune + +When you scale in the cluster(/tiup/tiup-component-dm-scale-in.md), a small amount of metadata in etcd is not cleaned up, which usually causes no problem. If you need to clean up the metadata, you can manually execute the `tiup dm prune` command. + +## Syntax + +```shell +tiup dm prune [flags] +``` + +## Option + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The log of the cleanup process. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-reload.md b/tiup/tiup-component-dm-reload.md new file mode 100644 index 000000000000..7fda1d818692 --- /dev/null +++ b/tiup/tiup-component-dm-reload.md @@ -0,0 +1,63 @@ +--- +title: tiup dm reload +--- + +# tiup dm reload + +After [modifying the cluster configuration](/tiup/tiup-component-dm-edit-config.md), the cluster needs to be reloaded using the `tiup dm reload` command for the configuration to take effect. This command publishes the configuration of the control machine to the remote machine where the service is running and restarts the service in order according to the upgrade process. The cluster remains available during the restart process. + +## Syntax + +```shell +tiup dm reload [flags] +``` + +``: the name of the cluster to operate on. + +## Options + +### -N, --node + +- Specifies the nodes to be restarted. If not specified, all nodes are restarted. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all nodes are selected by default. + +> **Note:** +> +> + If the `-R, --role` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are restarted. +> + If the `--skip-restart` option is specified, the `-N, --node` option is invalid. + +### -R, --role + +- Specifies the roles to be restarted. If not specified, all roles are restarted. The value of this option is a comma-separated list of node roles. You can get the roles of nodes from the second column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all roles are selected by default. + +> **Note:** +> +> + If the `-N, --node` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are restarted. +> + If the `--skip-restart` option is specified, the `-R, --role` option is invalid. + +### --skip-restart + +The `tiup dm reload` command performs two operations: + +- Refreshes all node configurations +- Restarts the specified node + +After you specify the `--skip-restart` option, it only refreshes the configuration without restarting any nodes, so that the refreshed configuration is not applied and does not take effect until the next restart of the corresponding service. + +- Data type: `BOOLEAN` +- Default: false + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The execution log of the tiup-dm. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-replay.md b/tiup/tiup-component-dm-replay.md new file mode 100644 index 000000000000..7a15f900f413 --- /dev/null +++ b/tiup/tiup-component-dm-replay.md @@ -0,0 +1,27 @@ +--- +title: tiup dm replay +--- + +# tiup dm replay + +When you perform a cluster operation such as upgrade or restart, the operation might fail due to cluster environment issues. If you re-perform the operation, you need to perform all the steps from the very beginning. If the cluster is large, re-performing these steps will take a long time. In this case, you can use the `tiup dm replay` command to retry the failed commands and skip the successfully performed steps. + +## Syntax + +```shell +tiup dm replay [flags] +``` + +- ``: the `audit-id` of the command to be retried. You can view the historical commands and their `audit-id`s using the [`tiup dm audit`](/tiup/tiup-component-dm-audit.md) command. + +## Option + +### -h, --help + +Prints the help information. + +## Output + +The output of the command corresponding to ``. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-restart.md b/tiup/tiup-component-dm-restart.md new file mode 100644 index 000000000000..6910fae1455a --- /dev/null +++ b/tiup/tiup-component-dm-restart.md @@ -0,0 +1,53 @@ +--- +title: tiup dm restart +--- + +# tiup dm restart + +The command `tiup dm restart` is used to restart all or some of the services of the specified cluster. + +> **Note:** +> +> During the restart process, the related services are unavailable for a period of time. + +## Syntax + +```shell +tiup dm restart [flags] +``` + +``: the name of the cluster to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-cluster-list.md) command. + +## Options + +### -N, --node + +- Specifies the nodes to be restarted. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the `[tiup dm display](/tiup/tiup-component-dm-display.md)` command. +- Data type: `STRING` +- If this option is not specified, TiUP restarts all nodes by default. + +> **Note:** +> +> If the option `-R, --role` is specified at the same time, TiUP restarts service nodes that match both the requirements of `-N, --node` and `-R, --role`. + +### -R, --role + +- Specifies the roles of nodes to be restarted. The value of this option is a comma-separated list of the roles of the nodes. You can get the roles of the nodes from the second column of the cluster status table returned by the `[tiup dm display](/tiup/tiup-component-dm-display.md)` command. +- Data type: `STRING` +- If this option is not specified, TiUP restarts nodes of all roles by default. + +> **Note:** +> +> If the option `-N, --node` is specified at the same time, TiUP restarts service nodes that match both the requirements of `-N, --node` and `-R, --role`. + +### -h, --help + +- Prints help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Outputs + +The log of the service restart process. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-scale-in.md b/tiup/tiup-component-dm-scale-in.md new file mode 100644 index 000000000000..af3b3c1a022f --- /dev/null +++ b/tiup/tiup-component-dm-scale-in.md @@ -0,0 +1,41 @@ +--- +title: tiup dm scale-in +--- + +# tiup dm scale-in + +The `tiup dm scale-in` command is used to scale in the cluster. Scaling in the cluster means taking the service offline, which eventually removes the specified node from the cluster and deletes the remaining related files. + +## Syntax + +```shell +tiup dm scale-in [flags] +``` + +``: the name of the cluster to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-dm-list.md) command. + +## Options + +### -N, --node + +- Specifies the nodes to be scaled in. If you need to scale in multiple nodes, split them by commas. +- Data type: `STRINGS` +- Default: no. This option is mandatory and the value must be not null. + +### --force + +- In some cases, some scale-in nodes in the cluster have been down, making it impossible to connect to the node through SSH for operation. At this time, you can use the `--force` option to remove these nodes from the cluster. +- Data type: `BOOLEAN` +- Default: false. If this option is not specified in the command, the specified nodes are not forcibly removed. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The log of scaling in. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-scale-out.md b/tiup/tiup-component-dm-scale-out.md new file mode 100644 index 000000000000..da77f96e9481 --- /dev/null +++ b/tiup/tiup-component-dm-scale-out.md @@ -0,0 +1,49 @@ +--- +title: tiup dm scale-out +--- + +# tiup dm scale-out + +The `tiup dm scale-out` command is used for scaling out the cluster. The internal logic of scaling out the cluster is similar to the cluster deployment. The `tiup-dm` components first establish an SSH connection to the new node, create the necessary directories on the target node, then perform the deployment and start the service. + +## Syntax + +```shell +tiup dm scale-out [flags] +``` + +``: the name of the cluster to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-dm-list.md) command. + +``: the prepared [topology file](/tiup/tiup-dm-topology-reference.md). This topology file should only contain the new nodes that are to be added to the current cluster. + +## Options + +### -u, --user + +- Specifies the user name used to connect to the target machine. This user must have the secret-free sudo root permission on the target machine. +- Data type: `STRING` +- Default: the current user who executes the command. + +### -i, --identity_file + +- Specifies the key file used to connect to the target machine. +- Data type: `STRING` +- If this option is not specified in the command, the `~/.ssh/id_rsa` file is used to connect to the target machine by default. + +### -p, --password + +- Specifies the password used to connect to the target machine. Do not use this option and `-i/--identity_file` at the same time. +- Data type: `BOOLEAN` +- Default: false + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The log of scaling out. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-start.md b/tiup/tiup-component-dm-start.md new file mode 100644 index 000000000000..4a5c9ff1acf2 --- /dev/null +++ b/tiup/tiup-component-dm-start.md @@ -0,0 +1,49 @@ +--- +title: tiup dm start +--- + +# tiup dm start + +The `tiup dm start` command is used to start all or part of the services of the specified cluster. + +## Syntax + +```shell +tiup dm start [flags] +``` + +``: the name of the cluster to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-dm-list.md) command. + +## Options + +### -N, --node + +- Specifies the nodes to be started. If not specified, all nodes are started. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all nodes are started. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are started. + +### -R, --role + +- Specifies the roles to be started. If not specified, all roles are started. The value of this option is a comma-separated list of node roles. You can get the roles of nodes from the second column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all roles are started. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are started. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The log of starting the service. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-stop.md b/tiup/tiup-component-dm-stop.md new file mode 100644 index 000000000000..72cd009d4e73 --- /dev/null +++ b/tiup/tiup-component-dm-stop.md @@ -0,0 +1,53 @@ +--- +title: tiup dm stop +--- + +# tiup dm stop + +The `tiup dm stop` command is used to stop all or part of the services of the specified cluster. + +> **Note:** +> +> The cluster cannot provide services after the core service is stopped. + +## Syntax + +```shell +tiup dm stop [flags] +``` + +``: the name of the cluster to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-dm-list.md) command. + +## Options + +### -N, --node + +- Specifies the nodes to be stopped. If not specified, all nodes are stopped. The value of this option is a comma-separated list of node IDs. You can get the node IDs from the first column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all nodes are selected by default. + +> **Note:** +> +> If the `-R, --role` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are stopped. + +### -R, --role + +- Specifies the roles to be stopped. If not specified, all roles are stopped. The value of this option is a comma-separated list of node roles. You can get the roles of nodes from the second column of the cluster status table returned by the [`tiup dm display`](/tiup/tiup-component-dm-display.md) command. +- Data type: `STRINGS` +- If this option is not specified in the command, all roles are selected by default. + +> **Note:** +> +> If the `-N, --node` option is specified at the same time, only the service nodes that match both the specifications of `-N, --node` and `-R, --role` are stopped. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: false + +## Output + +The log of stopping the service. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-template.md b/tiup/tiup-component-dm-template.md new file mode 100644 index 000000000000..eedd9c492fd9 --- /dev/null +++ b/tiup/tiup-component-dm-template.md @@ -0,0 +1,38 @@ +--- +title: tiup dm template +--- + +# tiup dm template + +Before deploying the cluster, you need to prepare a [topology file](/tiup/tiup-dm-topology-reference.md) of the cluster. TiUP has a built-in topology file template, and you can modify this template to create the final topology file. To output the built-in template content, you can use the `tiup dm template` command. + +## Syntax + +```shell +tiup dm template [flags] +``` + +If this option is not specified, the output default template contains the following instances: + +- 3 DM-master instances +- 3 DM-worker instances +- 1 Prometheus instance +- 1 Grafana instance +- 1 Alertmanager instance + +## Options + +### --full + +- Outputs a detailed topology template that is commented with configurable parameters. To enable this option, add it to the command. +- If this option is not specified, the simple topology template is output by default. + +### -h, --help + +Prints the help information. + +## Output + +Outputs the topology template according to the specified options, which can be redirected to the topology file for deployment. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm-upgrade.md b/tiup/tiup-component-dm-upgrade.md new file mode 100644 index 000000000000..87b8b12be595 --- /dev/null +++ b/tiup/tiup-component-dm-upgrade.md @@ -0,0 +1,34 @@ +--- +title: tiup dm upgrade +--- + +# tiup dm upgrade + +The `tiup dm upgrade` command is used to upgrade a specified cluster to a specific version. + +## Syntax + +```shell +tiup dm upgrade [flags] +``` + +- `` is the name of the cluster to be operated on. If you forget the cluster name, you can check it using the [`tiup dm list`](/tiup/tiup-component-dm-list.md) command. +- `` is the target version to be upgraded to. Currently, only upgrading to a later version is allowed, and upgrading to an earlier version is not allowed, which means the downgrade is not allowed. Upgrading to a nightly version is not allowed either. + +## Options + +### --offline + +- Declares that the current cluster is offline. When this option is specified, TiUP DM only replaces the binary files of the cluster components in place without restarting the service. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Output + +Log of the service upgrade process. + +[<< Back to the previous page - TiUP DM command list](/tiup/tiup-component-dm.md#command-list) diff --git a/tiup/tiup-component-dm.md b/tiup/tiup-component-dm.md new file mode 100644 index 000000000000..f162b08c76df --- /dev/null +++ b/tiup/tiup-component-dm.md @@ -0,0 +1,85 @@ +--- +title: TiUP DM +--- + +# TiUP DM + +Similar to [TiUP Cluster](/tiup/tiup-component-cluster.md) which is used to manage TiDB clusters, TiUP DM is used to manage DM clusters. You can use the TiUP DM component to perform daily operations and maintenance tasks of DM clusters, including deploying, starting, stopping, destroying, elastic scaling, upgrading DM clusters, and managing the configuration parameters of DM clusters. + +## Syntax + +```shell +tiup dm [command] [flags] +``` + +`[command]` is used to pass the name of the command. See the [Command list](#command-list) for supported commands. + +## Options + +### --ssh + +- Specifies the SSH client to connect to the remote end (the machine where the TiDB service is deployed) for the command execution. +- Data type: `STRING` +- Support values: + + - `builtin`: Uses the built-in easyssh client of tiup-cluster as the SSH client. + - `system`: Uses the default SSH client of the current operating system. + - `none`: No SSH client is used. The deployment is only for the current machine. + +- If this option is not specified in the command, `builtin` is used as the default value. + +### --ssh-timeout + +- Specifies the SSH connection timeout in seconds. +- Data type: `UINT` +- If this option is not specified in the command, the default timeout is `5` seconds. + +### --wait-timeout + +- Specifies the maximum waiting time (in seconds) for each step in the operation process. The operation process consists of many steps, such as specifying systemctl to start or stop services, and waiting for ports to be online or offline. Each step may take several seconds. If the execution time of a step exceeds the specified timeout, the step exits with an error. +- Data type: `UINT` +- If this option is not specified in the command, the maximum waiting time for each steps is `120` seconds. + +### -y, --yes + +- Skips the secondary confirmation of all risky operations. It is not recommended to use this option unless you use a script to call TiUP. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### -v, --version + +- Prints the current version of TiUP DM. +- Data type:`BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +### -h, --help + +- Prints help information about the specified command. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + +## Command list + +- [import](/tiup/tiup-component-dm-import.md): Imports a DM v1.0 cluster deployed by DM-Ansible. +- [template](/tiup/tiup-component-dm-template.md): Outputs the topology template. +- [deploy](/tiup/tiup-component-dm-deploy.md): Deploys a cluster based on a specified topology. +- [list](/tiup/tiup-component-dm-list.md): Queries the list of deployed clusters. +- [display](/tiup/tiup-component-dm-display.md): Displays the status of a specified cluster. +- [start](/tiup/tiup-component-dm-start.md): Starts a specified cluster. +- [stop](/tiup/tiup-component-dm-stop.md): Stops a specified cluster. +- [restart](/tiup/tiup-component-dm-restart.md): Restarts a specified cluster. +- [scale-in](/tiup/tiup-component-dm-scale-in.md): Scales in a specified cluster. +- [scale-out](/tiup/tiup-component-dm-scale-out.md): Scales out a specified cluster. +- [upgrade](/tiup/tiup-component-dm-upgrade.md): Upgrades a specified cluster. +- [prune](/tiup/tiup-component-dm-prune.md): Cleans up instances in the Tombstone status for a specified cluster. +- [edit-config](/tiup/tiup-component-dm-edit-config.md): Modifies the configuration of a specified cluster. +- [reload](/tiup/tiup-component-dm-reload.md): Reloads the configuration of a specified cluster. +- [patch](/tiup/tiup-component-dm-patch.md): Replaces a specified service in a deployed cluster. +- [destroy](/tiup/tiup-component-dm-destroy.md): Destroys a specified cluster. +- [audit](/tiup/tiup-component-dm-audit.md): Queries the operation audit log of a specified cluster. +- [replay](/tiup/tiup-component-dm-replay.md): Replays the specified commands +- [enable](/tiup/tiup-component-dm-enable.md): Enables the auto-enabling of the cluster service after a machine is restarted. +- [disable](/tiup/tiup-component-dm-disable.md): Disables the auto-enabling of the cluster service after a machine is restarted. +- [help](/tiup/tiup-component-dm-help.md): Prints help information. + +[<< Back to the previous page - TiUP Reference component list](/tiup/tiup-reference.md#component-list) diff --git a/tiup/tiup-component-management.md b/tiup/tiup-component-management.md index a3abb56204e3..342e757e295c 100644 --- a/tiup/tiup-component-management.md +++ b/tiup/tiup-component-management.md @@ -70,12 +70,12 @@ Example 2: Use TiUP to install the nightly version of TiDB. tiup install tidb:nightly ``` -Example 3: Use TiUP to install TiKV v3.0.6. +Example 3: Use TiUP to install TiKV v5.1.0. {{< copyable "shell-regular" >}} ```shell -tiup install tikv:v3.0.6 +tiup install tikv:v5.1.0 ``` ## Upgrade components @@ -95,7 +95,7 @@ Example 1: Upgrade all components to the latest versions. tiup update --all ``` -Example: Upgrade all components to the nightly version. +Example 2: Upgrade all components to the nightly version. {{< copyable "shell-regular" >}} @@ -128,12 +128,12 @@ Before the component is started, TiUP creates a directory for it, and then puts If you want to start the same component multiple times and reuse the previous working directory, you can use `--tag` to specify the same name when the component is started. After the tag is specified, the working directory will *not be automatically deleted* when the instance is terminated, which makes it convenient to reuse the working directory. -Example 1: Operate TiDB v3.0.8. +Example 1: Operate TiDB v5.1.0. {{< copyable "shell-regular" >}} ```shell -tiup tidb:v3.0.8 +tiup tidb:v5.1.0 ``` Example 2: Specify the tag with which TiKV operates. @@ -219,12 +219,12 @@ The following flags are supported in this command: - If the version is ignored, adding `--all` means to uninstall all versions of this component. - If the version and the component are both ignored, adding `--all` means to uninstall all components of all versions. -Example 1: Uninstall TiDB v3.0.8. +Example 1: Uninstall TiDB v5.0.0. {{< copyable "shell-regular" >}} ```shell -tiup uninstall tidb:v3.0.8 +tiup uninstall tidb:v5.0.0 ``` Example 2: Uninstall TiKV of all versions. diff --git a/tiup/tiup-dm-topology-reference.md b/tiup/tiup-dm-topology-reference.md new file mode 100644 index 000000000000..a135e4583acd --- /dev/null +++ b/tiup/tiup-dm-topology-reference.md @@ -0,0 +1,312 @@ +--- +title: Topology Configuration File for DM Cluster Deployment Using TiUP +--- + +# Topology Configuration File for DM Cluster Deployment Using TiUP + +To deploy or scale a TiDB Data Migration (DM) cluster, you need to provide a topology file ([sample](https://github.com/pingcap/tiup/blob/master/embed/templates/examples/dm/topology.example.yaml)) to describe the cluster topology. + +Similarly, to modify the cluster topology, you need to modify the topology file. The difference is that, after the cluster is deployed, you can only modify a part of the fields in the topology file. This document introduces each section of the topology file and each field in each section. + +## File structure + +A topology configuration file for DM cluster deployment using TiUP might contain the following sections: + +- [global](#global): the cluster's global configuration. Some of the configuration items use the default values of the cluster, and you can configure them separately in each instance. +- [server_configs](#server_configs): the components' global configuration. You can configure each component separately. If an instance has a configuration item with the same key, the instance's configuration item will take effect. +- [master_servers](#master_servers): the configuration of the DM-master instance. The configuration specifies the machines to which the master service of the DM component is deployed. +- [worker_servers](#worker_servers): the configuration of the DM-worker instance. The configuration specifies the machines to which the worker service of the DM component is deployed. +- [monitoring_servers](#monitoring_servers): specifies the machines to which the Prometheus instances are deployed. TiUP supports deploying multiple Prometheus instances but only the first instance is used. +- [grafana_servers](#grafana_servers): the configuration of the Grafana instances. The configuration specifies the machines to which the Grafana instances are deployed. +- [alertmanager_servers](#alertmanager_servers): the configuration of the Alertemanager instances. The configuration specifies the machines to which the Alertmanager instances are deployed. + +### `global` + +The `global` section corresponds to the cluster's global configuration and has the following fields: + +- `user`: the user to start the deployed cluster. The default value is "tidb". If the user specified in the `` field does not exist on the target machine, TiUP will automatically try to create the user. +- `group`: the user group to which a user belongs when the user is automatically created. The default value is the same as the `` field. If the specified group does not exist, it will be created automatically. +- `ssh_port`: the SSH port to connect to the target machine for operations. The default value is "22". +- `deploy_dir`: the deployment directory for each component. The default value is "deploy". The construction rules are as follows: + - If the absolute path `deploy_dir` is configured at the instance level, the actual deployment directory is the `deploy_dir` configured for the instance. + - For each instance, if you do not configure `deploy_dir`, the default value is the relative path `-`. + - If `global.deploy_dir` is set to an absolute path, the component is deployed to the `/` directory. + - If `global.deploy_dir` is set to a relative path, the component is deployed to the `/home///` directory. +- `data_dir`: the data directory. The default value is "data". The construction rules are as follows. + - If the absolute path `data_dir` is configured at the instance level, the actual data directory is the `data_dir` configured for the instance. + - For each instance, if `data_dir` is not configured, the default value is ``. + - If `data_dir` is set to a relative path, the component data is stored in `/`. For the construction rules of ``, see the construction rules of the `deploy_dir` field. +- `log_dir`: the data directory. The default value is "log". The construction rules are as follows. + - If the absolute path of `log_dir` is configured at the instance level, the actual log directory is the `log_dir` configured for the instance. + - For each instance, if `log_dir` is not configured by the user, the default value is ``. + - If `log_dir` is a relative path, the component logs will be stored in `/`. For the construction rules of ``, see the construction rules of the `deploy_dir` field. +- `os`: the operating system of the target machine. The field controls which operating system to adapt to for the components pushed to the target machine. The default value is "linux". +- `arch`: the CPU architecture of the target machine. The field controls which platform to adapt to for the binary packages pushed to the target machine. The supported values are "amd64" and "arm64". The default value is "amd64". +- `resource_control`: runtime resource control. All configurations in this field are written to the service file of systemd. There is no limit by default. The resources that can be controlled are as follows: + - `memory_limit`: limits the maximum memory at runtime. For example, "2G" means that the maximum memory of 2 GB can be used. + - `cpu_quota`: limits the maximum CPU usage at runtime. For example, "200%". + - `io_read_bandwidth_max`: limits the maximum I/O bandwidth for disk reads. For example, `"/dev/disk/by-path/pci-0000:00:1f.2-scsi-0:0:0:0:0 100M"`. + - `io_write_bandwidth_max`: limits the maximum I/O bandwidth for disk writes. For example, `"/dev/disk/by-path/pci-0000:00:1f.2-scsi-0:0:0:0:0 100M"`. + - `limit_core`: controls the size of core dump. + +A `global` configuration example: + +```yaml +global: + user: "tidb" + resource_control: + memory_limit: "2G" +``` + +In the example, the configuration specifies that the `tidb` user is used to start the cluster, and that each component is limited to a maximum of 2 GB of memory when it is running. + +### `server_configs` + +`server_configs` is used to configure services and to generate configuration files for each component. Similar to the `global` section, the configurations in the `server_configs` section can be overwritten by the configurations with the same keys in an instance. `server_configs` mainly contains the following fields: + +- `master`: the configuration related to the DM-master service. For all the supported configuration items, see [DM-master Configuration File](https://docs.pingcap.com/tidb-data-migration/stable/dm-master-configuration-file). +- `worker`: the configuration related to the DM-worker service, For all the supported configuration items, see [DM-worker Configuration File](https://docs.pingcap.com/tidb-data-migration/stable/dm-worker-configuration-file). + +A `server_configs` configuration example is as follows: + +```yaml +server_configs: + master: + log-level: info + rpc-timeout: "30s" + rpc-rate-limit: 10.0 + rpc-rate-burst: 40 + worker: + log-level: info +``` + +## `master_servers` + +`master_servers` specifies the machines to which the master node of the DM component is deployed. You can also specify the service configuration on each machine. `master_servers` is an array. Each array element contains the following fields: + +- `host`: specifies the machine to deploy to. The field value is an IP address and is mandatory. +- `ssh_port`: specifies the SSH port to connect to the target machine for operations. If the field is not specified, the `ssh_port` in the `global` section is used. +- `name`: specifies the name of the DM-master instance. The name must be unique for different instances. Otherwise, the cluster cannot be deployed. +- `port`: specifies the port on which DM-master provides services. The default value is "8261". +- `peer_port`: specifies the port for communication between DM-masters. The default value is "8291". +- `deploy_dir`: specifies the deployment directory. If the field is not specified, or specified as a relative directory, the deployment directory is generated according to the `deploy_dir` configuration in the `global` section. +- `data_dir`: specifies the data directory. If the field is not specified, or specified as a relative directory, the data directory is generated according to the `data_dir` configuration in the `global` section. +- `log_dir`: specifies the log directory. If the field is not specified, or specified as a relative directory, the log directory is generated according to the `log_dir` configuration in the `global` section. +- `numa_node`: allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is a string type. The field value is the ID of the NUMA node, such as "0,1". +- `config`: the configuration rules of this field are the same as that of `master` in the `server_configs` section. If `config` is specified, the configuration of `config` will be merged with the configuration of `master` in `server_configs` (if the two fields overlap, the configuration of this field takes effect), and then the configuration file is generated and distributed to the machine specified in the `host` field. +- `os`: the operating system of the machine specified in the `host` field. If the field is not specified, the default value is the `os` value configured in the `global` section. +- `arch`: the architecture of the machine specified in the `host` field. If the field is not specified, the default value is the `arch` value configured in the `global` section. +- `resource_control`: resource control on this service. If this field is specified, the configuration of this field will be merged with the configuration of `resource_control` in the `global` section (if the two fields overlap, the configuration of this field takes effect), and then the configuration file of systemd is generated and distributed to the machine specified in the `host` field. The configuration rules of this field are the same as that of `resource_control` in the `global` section. +- `v1_source_path`: when upgrading from v1.0.x, you can specify the directory where the configuration file of the V1 source is located in this field. + +In the `master_servers` section, the following fields cannot be modified after the deployment is completed: + +- `host` +- `name` +- `port` +- `peer_port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` +- `v1_source_path` + +A `master_servers` configuration example is as follows: + +```yaml +master_servers: + - host: 10.0.1.11 + name: master1 + ssh_port: 22 + port: 8261 + peer_port: 8291 + deploy_dir: "/dm-deploy/dm-master-8261" + data_dir: "/dm-data/dm-master-8261" + log_dir: "/dm-deploy/dm-master-8261/log" + numa_node: "0,1" + # The following configs are used to overwrite the `server_configs.master` values. + config: + log-level: info + rpc-timeout: "30s" + rpc-rate-limit: 10.0 + rpc-rate-burst: 40 + - host: 10.0.1.18 + name: master2 + - host: 10.0.1.19 + name: master3 +``` + +## `worker_servers` + +`worker_servers` specifies the machines to which the master node of the DM component is deployed. You can also specify the service configuration on each machine. `worker_servers` is an array. Each array element contains the following fields: + +- `host`: specifies the machine to deploy to. The field value is an IP address and is mandatory. +- `ssh_port`: specifies the SSH port to connect to the target machine for operations. If the field is not specified, the `ssh_port` in the `global` section is used. +- `name`: specifies the name of the DM-worker instance. The name must be unique for different instances. Otherwise, the cluster cannot be deployed. +- `port`: specifies the port on which DM-worker provides services. The default value is "8262". +- `deploy_dir`: specifies the deployment directory. If the field is not specified, or specified as a relative directory, the deployment directory is generated according to the `deploy_dir` configuration in the `global` section. +- `data_dir`: specifies the data directory. If the field is not specified, or specified as a relative directory, the data directory is generated according to the `data_dir` configuration in the `global` section. +- `log_dir`: specifies the log directory. If the field is not specified, or specified as a relative directory, the log directory is generated according to the `log_dir` configuration in the `global` section. +- `numa_node`: allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is a string type. The field value is the ID of the NUMA node, such as "0,1". +- `config`: the configuration rules of this field are the same as that of `worker` in the `server_configs` section. If `config` is specified, the configuration of `config` will be merged with the configuration of `worker` in `server_configs` (if the two fields overlap, the configuration of this field takes effect), and then the configuration file is generated and distributed to the machine specified in the `host` field. +- `os`: the operating system of the machine specified in the `host` field. If the field is not specified, the default value is the `os` value configured in the `global` section. +- `arch`: the architecture of the machine specified in the `host` field. If the field is not specified, the default value is the `arch` value configured in the `global` section. +- `resource_control`: resource control on this service. If this field is specified, the configuration of this field will be merged with the configuration of `resource_control` in the `global` section (if the two fields overlap, the configuration of this field takes effect), and then the configuration file of systemd is generated and distributed to the machine specified in the `host` field. The configuration rules of this field are the same as that of `resource_control` in the `global` section. + +In the `worker_servers` section, the following fields cannot be modified after the deployment is completed: + +- `host` +- `name` +- `port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +A `worker_servers` configuration example is as follows: + +```yaml +worker_servers: + - host: 10.0.1.12 + ssh_port: 22 + port: 8262 + deploy_dir: "/dm-deploy/dm-worker-8262" + log_dir: "/dm-deploy/dm-worker-8262/log" + numa_node: "0,1" + # config is used to overwrite the `server_configs.worker` values + config: + log-level: info + - host: 10.0.1.19 +``` + +### `monitoring_servers` + +`monitoring_servers` specifies the machines to which the Prometheus service is deployed. You can also specify the service configuration on the machine. `monitoring_servers` is an array. Each array element contains the following fields: + +- `host`: specifies the machine to deploy to. The field value is an IP address and is mandatory. +- `ssh_port`: specifies the SSH port to connect to the target machine for operations. If the field is not specified, the `ssh_port` in the `global` section is used. +- `port`: specifies the port on which Prometheus provides services. The default value is "9090". +- `deploy_dir`: specifies the deployment directory. If the field is not specified, or specified as a relative directory, the deployment directory is generated according to the `deploy_dir` configuration in the `global` section. +- `data_dir`: specifies the data directory. If the field is not specified, or specified as a relative directory, the data directory is generated according to the `data_dir` configuration in the `global` section. +- `log_dir`: specifies the log directory. If the field is not specified, or specified as a relative directory, the log directory is generated according to the `log_dir` configuration in the `global` section. +- `numa_node`: allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is a string type. The field value is the ID of the NUMA node, such as "0,1" +- `storage_retention`: specifies the retention time of the Prometheus monitoring data. The default value is "15d". +- `rule_dir`: specifies a local directory where the complete `*.rules.yml` files are located. The files in the specified directory will be sent to the target machine as the Prometheus rules during the initialization phase of the cluster configuration. +- `remote_config`: Supports writing Prometheus data to the remote, or reading data from the remote. This field has two configurations: + - `remote_write`: See the Prometheus document [``](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). + - `remote_read`: See the Prometheus document [``](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read). +- `external_alertmanagers`: If the `external_alertmanagers` field is configured, Prometheus alerts the configuration behavior to the Alertmanager that is outside the cluster. This field is an array, each element of which is an external Alertmanager and consists of the `host` and `web_port` fields. +- `os`: the operating system of the machine specified in the `host` field. If the field is not specified, the default value is the `os` value configured in the `global` section. +- `arch`: the architecture of the machine specified in the `host` field. If the field is not specified, the default value is the `arch` value configured in the `global` section. +- `resource_control`: resource control on this service. If this field is specified, the configuration of this field will be merged with the configuration of `resource_control` in the `global` section (if the two fields overlap, the configuration of this field takes effect), and then the configuration file of systemd is generated and distributed to the machine specified in the `host` field. The configuration rules of this field are the same as that of `resource_control` in the `global` section. + +In the `monitoring_servers` section, the following fields cannot be modified after the deployment is completed: + +- `host` +- `port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +A `monitoring_servers` configuration example is as follows: + +```yaml +monitoring_servers: + - host: 10.0.1.11 + rule_dir: /local/rule/dir + remote_config: + remote_write: + - queue_config: + batch_send_deadline: 5m + capacity: 100000 + max_samples_per_send: 10000 + max_shards: 300 + url: http://127.0.0.1:8003/write + remote_read: + - url: http://127.0.0.1:8003/read\ + external_alertmanagers: + - host: 10.1.1.1 + web_port: 9093 + - host: 10.1.1.2 + web_port: 9094 +``` + +### `grafana_servers` + +`grafana_servers` specifies the machines to which the Grafana service is deployed. You can also specify the service configuration on the machine. `grafana_servers` is an array. Each array element contains the following fields: + +- `host`: specifies the machine to deploy to. The field value is an IP address and is mandatory. +- `ssh_port`: specifies the SSH port to connect to the target machine for operations. If the field is not specified, the `ssh_port` in the `global` section is used. +- `port`: specifies the port on which Grafana provides services. The default value is "3000". +- `deploy_dir`: specifies the deployment directory. If the field is not specified, or specified as a relative directory, the deployment directory is generated according to the `deploy_dir` configuration in the `global` section. +- `os`: the operating system of the machine specified in the `host` field. If the field is not specified, the default value is the `os` value configured in the `global` section. +- `arch`: the architecture of the machine specified in the `host` field. If the field is not specified, the default value is the `arch` value configured in the `global` section. +- `username`: specifies the username of the Grafana login screen. +- `password`: specifies the corresponding password of Grafana. +- `dashboard_dir`: specifies a local directory where the complete `dashboard(*.json)` files are located. The files in the specified directory will be sent to the target machine as Grafana dashboards during the initialization phase of the cluster configuration. +- `resource_control`: resource control on this service. If this field is specified, the configuration of this field will be merged with the configuration of `resource_control` in the `global` section (if the two fields overlap, the configuration of this field takes effect), and then the configuration file of systemd is generated and distributed to the machine specified in the `host` field. The configuration rules of this field are the same as that of `resource_control` in the `global` section. + +> **Note:** +> +> If the `dashboard_dir` field of `grafana_servers` is configured, after executing the `tiup cluster rename` command to rename the cluster, you need to perform the following operations: +> +> 1. In the local `dashboards` directory, update the value of the `datasource` field to the new cluster name (the `datasource` is named after the cluster name). +> 2. Execute the `tiup cluster reload -R grafana` command. + +In `grafana_servers`, the following fields cannot be modified after the deployment is completed: + +- `host` +- `port` +- `deploy_dir` +- `arch` +- `os` + +A `grafana_servers` configuration example is as follows: + +```yaml +grafana_servers: + - host: 10.0.1.11 + dashboard_dir: /local/dashboard/dir +``` + +### `alertmanager_servers` + +`alertmanager_servers` specifies the machines to which the Alertmanager service is deployed. You can also specify the service configuration on each machine. `alertmanager_servers` is an array. Each array element contains the following fields: + +- `host`: specifies the machine to deploy to. The field value is an IP address and is mandatory. +- `ssh_port`: specifies the SSH port to connect to the target machine for operations. If the field is not specified, the `ssh_port` in the `global` section is used. +- `web_port`: specify the port on which Alertmanager provides web services. The default value is "9093". +- `cluster_port`: Specify the communication port between one Alertmanger and other Alertmanager. The default value is "9094". +- `deploy_dir`: specifies the deployment directory. If the field is not specified, or specified as a relative directory, the deployment directory is generated according to the `deploy_dir` configuration in the `global` section. +- `data_dir`: specifies the data directory. If the field is not specified, or specified as a relative directory, the data directory is generated according to the `data_dir` configuration in the `global` section. +- `log_dir`: specifies the log directory. If the field is not specified, or specified as a relative directory, the log directory is generated according to the `log_dir` configuration in the `global` section. +- `numa_node`: allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is a string type. The field value is the ID of the NUMA node, such as "0,1" +- `config_file`: specifies a local file. The specified file will be sent to the target machine as the configuration for Alertmanager during the initialization phase of the cluster configuration. +- `os`: the operating system of the machine specified in the `host` field. If the field is not specified, the default value is the `os` value configured in the `global` section. +- `arch`: the architecture of the machine specified in the `host` field. If the field is not specified, the default value is the `arch` value configured in the `global` section. +- `resource_control`: resource control on this service. If this field is specified, the configuration of this field will be merged with the configuration of `resource_control` in the `global` section (if the two fields overlap, the configuration of this field takes effect), and then the configuration file of systemd is generated and distributed to the machine specified in the `host` field. The configuration rules of this field are the same as that of `resource_control` in the `global` section. + +In `alertmanager_servers`, the following fields cannot be modified after the deployment is completed: + +- `host` +- `web_port` +- `cluster_port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +An `alertmanager_servers` configuration example is as follows: + +```yaml +alertmanager_servers: + - host: 10.0.1.11 + config_file: /local/config/file + - host: 10.0.1.12 + config_file: /local/config/file +``` diff --git a/tiup/tiup-documentation-guide.md b/tiup/tiup-documentation-guide.md index c86e0c28fd20..629648a6d528 100644 --- a/tiup/tiup-documentation-guide.md +++ b/tiup/tiup-documentation-guide.md @@ -1,26 +1,20 @@ --- -title: TiUP Documentation Guide +title: TiUP Documentation Map summary: Guide you through TiUP documentation with links and introductions. aliases: ['/docs/dev/tiup/tiup-documentation-guide/'] --- -# TiUP Documentation Guide +# TiUP Documentation Map -## TiUP user guide +## User guide -- [TiUP Overview](/tiup/tiup-overview.md): Gives an overall introduction to TiUP, for example, how to install and use TiUP +- [TiUP Overview](/tiup/tiup-overview.md): Gives an overall introduction to TiUP, for example, how to install and use TiUP, and the related terminologies. - [TiUP Terminology and Concepts](/tiup/tiup-terminology-and-concepts.md): Explains the terms that you might bump into when using TiUP, and help you understand the key concepts of TiUP - [TiUP Component Management](/tiup/tiup-component-management.md): Introduces all TiUP commands in detail, and how to use TiUP to download, update and delete components - [TiUP FAQ](/tiup/tiup-faq.md): Introduces common issues when you use TiUP, including FAQs of the third-party components of TiUP - [TiUP Troubleshooting Guide](/tiup/tiup-troubleshooting-guide.md): Introduces the troubleshooting methods and solutions if you encounter issues when using TiUP - -## TiUP component user guide - -- [tiup-playground](/tiup/tiup-playground.md): Introduces how to use the TiUP playground component to quickly build a local TiDB cluster for testing -- [tiup-cluster](/tiup/tiup-cluster.md): Introduces the usage and command-line flags of the TiUP cluster component, which is used to deploy and maintain a TiDB cluster for production -- [tiup-mirror](/tiup/tiup-mirror.md): Introduces how to use the TiUP mirror component to customize a local offline mirror -- [tiup-bench](/tiup/tiup-bench.md): Introduces how to use the TiUP bench component for common stress testings, such as TPCC/TPCH +- [TiUP Reference Guide](/tiup/tiup-reference.md): Introduces detailed references, including commands, components, and mirrors. ## TiUP resources -- [TiUP Issues](https://github.com/pingcap/tiup/issues): Lists TiUP Github issues \ No newline at end of file +- [TiUP Issues](https://github.com/pingcap/tiup/issues): Lists TiUP GitHub issues diff --git a/tiup/tiup-mirror-reference.md b/tiup/tiup-mirror-reference.md new file mode 100644 index 000000000000..4eb30cd2fcb1 --- /dev/null +++ b/tiup/tiup-mirror-reference.md @@ -0,0 +1,311 @@ +--- +title: TiUP Mirror Reference Guide +summary: Learn the general information of TiUP mirrors. +--- + +# TiUP Mirror Reference Guide + +TiUP mirrors are TiUP's component warehouse, which stores components and their metadata. TiUP mirrors take the following two forms: + ++ Directory on the local disk: serves the local TiUP client, which is called a local mirror in this document. ++ HTTP mirror started based on the remote disk directory: serves the remote TiUP client, which is called a remote mirror in this document. + +## Create and update mirror + +You can create a TiUP mirror using one of the following two methods: + ++ Execute `tiup mirror init` to create a mirror from scratch. ++ Execute `tiup mirror clone` to clone from an existing mirror. + +After the mirror is created, you can add components to or delete components from the mirror using the `tiup mirror` commands. TiUP updates a mirror by adding files and assigning a new version number to it, rather than deleting any files from the mirror. + +## Mirror structure + +A typical mirror structure is as follows: + +``` ++ # Mirror's root directory +|-- root.json # Mirror's root certificate +|-- {2..N}.root.json # Mirror's root certificate +|-- {1..N}.index.json # Component/user index +|-- {1..N}.{component}.json # Component metadata +|-- {component}-{version}-{os}-{arch}.tar.gz # Component binary package +|-- snapshot.json # Mirror's latest snapshot +|-- timestamp.json # Mirror's latest timestamp +|--+ commits # Mirror's update log (deletable) + |--+ commit-{ts1..tsN} + |-- {N}.root.json + |-- {N}.{component}.json + |-- {N}.index.json + |-- {component}-{version}-{os}-{arch}.tar.gz + |-- snapshot.json + |-- timestamp.json +|--+ keys # Mirror's private key (can be moved to other locations) + |-- {hash1..hashN}-root.json # Private key of the root certificate + |-- {hash}-index.json # Private key of the indexes + |-- {hash}-snapshot.json # Private key of the snapshots + |-- {hash}-timestamp.json # Private key of the timestamps +``` + +> **Note:** +> +> + The `commits` directory stores the logs generated in the process of mirror update and is used to roll back the mirror. You can delete the old log directories regularly when the disk space is insufficient. +> + The private key stored in the `keys` directory is sensitive. It is recommended to keep it separately. + +### Root directory + +In a TiUP mirror, the root certificate is used to store the public key of other metadata files. Each time any metadata file (`*.json`) is obtained, TiUP client needs to find the corresponding public key in the installed `root.json` based on the metadata file type (root, index, snapshot, timestamp). Then TiUP client uses the public key to verify whether the signature is valid. + +The root certificate's format is as follows: + +``` +{ + "signatures": [ # Each metadata file has some signatures which are signed by several private keys corresponding to the file. + { + "keyid": "{id-of-root-key-1}", # The ID of the first private key that participates in the signature. This ID is obtained by hashing the content of the public key that corresponds to the private key. + "sig": "{signature-by-root-key-1}" # The signed part of this file by this private key. + }, + ... + { + "keyid": "{id-of-root-key-N}", # The ID of the Nth private key that participates in the signature. + "sig": "{signature-by-root-key-N}" # The signed part of this file by this private key. + } + ], + "signed": { # The signed part. + "_type": "root", # The type of this file. root.json's type is root. + "expires": "{expiration-date-of-this-file}", # The expiration time of the file. If the file expires, the client rejects the file. + "roles": { # Records the keys used to sign each metadata file. + "{role:index,root,snapshot,timestamp}": { # Each involved metadata file includes index, root, snapshot, and timestamp. + "keys": { # Only the key's signature recorded in `keys` is valid. + "{id-of-the-key-1}": { # The ID of the first key used to sign {role}. + "keytype": "rsa", # The key's type. Currently, the key type is fixed as rsa. + "keyval": { # The key's payload. + "public": "{public-key-content}" # The public key's content. + }, + "scheme": "rsassa-pss-sha256" # Currently, the scheme is fixed as rsassa-pss-sha256. + }, + "{id-of-the-key-N}": { # The ID of the Nth key used to sign {role}. + "keytype": "rsa", + "keyval": { + "public": "{public-key-content}" + }, + "scheme": "rsassa-pss-sha256" + } + }, + "threshold": {N}, # Indicates that the metadata file needs at least N key signatures. + "url": "/{role}.json" # The address from which the file can be obtained. For index files, prefix it with the version number (for example, /{N}.index.json). + } + }, + "spec_version": "0.1.0", # The specified version followed by this file. If the file structure is changed in the future, the version number needs to be upgraded. The current version number is 0.1.0. + "version": {N} # The version number of this file. You need to create a new {N+1}.root.json every time you update the file, and set its version to N + 1. + } +} +``` + +### Index + +The index file records all the components in the mirror and the owner information of the components. + +The index file's format is as follows: + +``` +{ + "signatures": [ # The file's signature. + { + "keyid": "{id-of-index-key-1}", # The ID of the first private key that participates in the signature. + "sig": "{signature-by-index-key-1}", # The signed part of this file by this private key. + }, + ... + { + "keyid": "{id-of-root-key-N}", # The ID of the Nth private key that participates in the signature. + "sig": "{signature-by-root-key-N}" # The signed part of this file by this private key. + } + ], + "signed": { + "_type": "index", # The file type. + "components": { # The component list. + "{component1}": { # The name of the first component. + "hidden": {bool}, # Whether it is a hidden component. + "owner": "{owner-id}", # The component owner's ID. + "standalone": {bool}, # Whether it is a standalone component. + "url": "/{component}.json", # The address from which the component can be obtained. You need to prefix it with the version number (for example, /{N}.{component}.json). + "yanked": {bool} # Indicates whether the component is marked as deleted. + }, + ... + "{componentN}": { # The name of the Nth component. + ... + }, + }, + "default_components": ["{component1}".."{componentN}"], # The default component that a mirror must contain. Currently, this field defaults to empty (disabled). + "expires": "{expiration-date-of-this-file}", # The expiration time of the file. If the file expires, the client rejects the file. + "owners": { + "{owner1}": { # The ID of the first owner. + "keys": { # Only the key's signature recorded in `keys` is valid. + "{id-of-the-key-1}": { # The first key of the owner. + "keytype": "rsa", # The key's type. Currently, the key type is fixed as rsa. + "keyval": { # The key's payload. + "public": "{public-key-content}" # The public key's content. + }, + "scheme": "rsassa-pss-sha256" # Currently, the scheme is fixed as rsassa-pss-sha256. + }, + ... + "{id-of-the-key-N}": { # The Nth key of the owner. + ... + } + }, + "name": "{owner-name}", # The name of the owner. + "threshod": {N} # Indicates that the components owned by the owner must have at least N valid signatures. + }, + ... + "{ownerN}": { # The ID of the Nth owner. + ... + } + } + "spec_version": "0.1.0", # The specified version followed by this file. If the file structure is changed in the future, the version number needs to be upgraded. The current version number is 0.1.0. + "version": {N} # The version number of this file. You need to create a new {N+1}.index.json every time you update the file, and set its version to N + 1. + } +} +``` + +### Component + +The component's metadata file records information of the component-specific platform and the version. + +The component metadata file's format is as follows: + +``` +{ + "signatures": [ # The file's signature. + { + "keyid": "{id-of-index-key-1}", # The ID of the first private key that participates in the signature. + "sig": "{signature-by-index-key-1}", # The signed part of this file by this private key. + }, + ... + { + "keyid": "{id-of-root-key-N}", # The ID of the Nth private key that participates in the signature. + "sig": "{signature-by-root-key-N}" # The signed part of this file by this private key. + } + ], + "signed": { + "_type": "component", # The file type. + "description": "{description-of-the-component}", # The description of the component. + "expires": "{expiration-date-of-this-file}", # The expiration time of the file. If the file expires, the client rejects the file. + "id": "{component-id}", # The globally unique ID of the component. + "nightly": "{nightly-cursor}", # The nightly cursor, and the value is the latest nightly version number (for example, v5.0.0-nightly-20201209). + "platforms": { # The component's supported platforms (such as darwin/amd64, linux/arm64). + "{platform-pair-1}": { + "{version-1}": { # The semantic version number (for example, v1.0.0). + "dependencies": null, # Specifies the dependency relationship between components. The field is not used yet and is fixed as null. + "entry": "{entry}", # The relative path of the entry binary file in the tar package. + "hashs": { # The checksum of the tar package. sha256 and sha512 are used. + "sha256": "{sum-of-sha256}", + "sha512": "{sum-of-sha512}", + }, + "length": {length-of-tar}, # The length of the tar package. + "released": "{release-time}", # The release date of the version. + "url": "{url-of-tar}", # The download address of the tar package. + "yanked": {bool} # Indicates whether this version is disabled. + } + }, + ... + "{platform-pair-N}": { + ... + } + }, + "spec_version": "0.1.0", # The specified version followed by this file. If the file structure is changed in the future, the version number needs to be upgraded. The current version number is 0.1.0. + "version": {N} # The version number of this file. You need to create a new {N+1}.{component}.json every time you update the file, and set its version to N + 1. +} +``` + +### Snapshot + +The snapshot file records the version number of each metadata file: + +The snapshot file's structure is as follows: + +``` +{ + "signatures": [ # The file's signature. + { + "keyid": "{id-of-index-key-1}", # The ID of the first private key that participates in the signature. + "sig": "{signature-by-index-key-1}", # The signed part of this file by this private key. + }, + ... + { + "keyid": "{id-of-root-key-N}", # The ID of the Nth private key that participates in the signature. + "sig": "{signature-by-root-key-N}" # The signed part of this file by this private key. + } + ], + "signed": { + "_type": "snapshot", # The file type. + "expires": "{expiration-date-of-this-file}", # The expiration time of the file. If the file expires, the client rejects the file. + "meta": { # Other metadata files' information. + "/root.json": { + "length": {length-of-json-file}, # The length of root.json + "version": {version-of-json-file} # The version of root.json + }, + "/index.json": { + "length": {length-of-json-file}, + "version": {version-of-json-file} + }, + "/{component-1}.json": { + "length": {length-of-json-file}, + "version": {version-of-json-file} + }, + ... + "/{component-N}.json": { + ... + } + }, + "spec_version": "0.1.0", # The specified version followed by this file. If the file structure is changed in the future, the version number needs to be upgraded. The current version number is 0.1.0. + "version": 0 # The version number of this file, which is fixed as 0. + } +``` + +### Timestamp + +The timestamp file records the checksum of the current snapshot. + +The timestamp file's format is as follows: + +``` +{ + "signatures": [ # The file's signature. + { + "keyid": "{id-of-index-key-1}", # The ID of the first private key that participates in the signature. + "sig": "{signature-by-index-key-1}", # The signed part of this file by this private key. + }, + ... + { + "keyid": "{id-of-root-key-N}", # The ID of the Nth private key that participates in the signature. + "sig": "{signature-by-root-key-N}" # The signed part of this file by this private key. + } + ], + "signed": { + "_type": "timestamp", # The file type. + "expires": "{expiration-date-of-this-file}", # The expiration time of the file. If the file expires, the client rejects the file. + "meta": { # The information of snapshot.json. + "/snapshot.json": { + "hashes": { + "sha256": "{sum-of-sha256}" # snapshot.json's sha256. + }, + "length": {length-of-json-file} # The length of snapshot.json. + } + }, + "spec_version": "0.1.0", # The specified version followed by this file. If the file structure is changed in the future, the version number needs to be upgraded. The current version number is 0.1.0. + "version": {N} # The version number of this file. You need to overwrite timestamp.json every time you update the file, and set its version to N + 1. +``` + +## Client workflow + +The client uses the following logic to ensure that the files downloaded from the mirror are safe: + ++ A `root.json` file is included with the binary when the client is installed. ++ The running client performs the following tasks based on the existing `root.json`: + 1. Obtain the version from `root.json` and mark it as `N`. + 2. Request `{N+1}.root.json` from the mirror. If the request is successful, use the public key recorded in `root.json` to verify whether the file is valid. + 3. Request `timestamp.json` from the mirror and use the public key recorded in `root.json` to verify whether the file is valid. + 4. Check whether the checksum of `snapshot.json` recorded in `timestamp.json` matches the checksum of the local `snapshot.json`. If the two do not match, request the latest `snapshot.json` from the mirror and use the public key recorded in `root.json` to verify whether the file is valid. + 5. Obtain the version number `N` of the `index.json` file from `snapshot.json` and request `{N}.index.json` from the mirror. Then use the public key recorded in `root.json` to verify whether the file is valid. + 6. For components such as `tidb.json` and `tikv.json`, the client obtains the version numbers `N` of the components from `snapshot.json` and requests `{N}.{component}.json` from the mirror. Then the client uses the public key recorded in `index.json` to verify whether the file is valid. + 7. For component's tar files, the client obtains the URLs and checksums of the files from `{component}.json` and request the URLs for the tar packages. Then the client verifies whether the checksum is correct. diff --git a/tiup/tiup-mirror.md b/tiup/tiup-mirror.md index e21d7f889d61..efa07e9092cb 100644 --- a/tiup/tiup-mirror.md +++ b/tiup/tiup-mirror.md @@ -55,7 +55,7 @@ The `tiup mirror clone` command provides many optional flags (might provide more - Determines whether to use prefix matching to match the version when cloning - If the `--prefix` flag is specified, the version number is matched by prefix for the clone. For example, if you specify `--prefix` as "v4.0.0", then "v4.0.0-rc.1", "v4.0.0-rc.2", and "v4.0.0" are matched. + If the `--prefix` flag is specified, the version number is matched by prefix for the clone. For example, if you specify `--prefix` as "v5.0.0", then "v5.0.0-rc", and "v5.0.0" are matched. - Determines whether to use the full clone @@ -77,14 +77,14 @@ The `tiup mirror clone` command provides many optional flags (might provide more If you want to clone only one version (not all versions) of a component, use `--=` to specify this version. For example: - - Execute the `tiup mirror clone --tidb v4.0.0` command to clone the v4.0.0 version of the TiDB component. - - Execute the `tiup mirror clone --tidb v4.0.0 --tikv all` command to clone the v4.0.0 version of the TiDB component and all versions of the TiKV component. - - Execute the `tiup mirror clone v4.0.0` command to clone the v4.0.0 version of all components in a cluster. + - Execute the `tiup mirror clone --tidb v5.1.0` command to clone the v5.1.0 version of the TiDB component. + - Execute the `tiup mirror clone --tidb v5.1.0 --tikv all` command to clone the v5.1.0 version of the TiDB component and all versions of the TiKV component. + - Execute the `tiup mirror clone v5.1.0` command to clone the v5.1.0 version of all components in a cluster. ## Usage examples This section introduces the usage examples of the `mirror` command. -### Deploy a TiDB Cluster offline using TiUP +The repository that is cloned using `tiup mirror clone` can be shared among hosts either by sharing the files via SCP, NFS or by making the repository available over the HTTP or HTTPS protocol. Use `tiup mirror set ` to specify the location of the repository. -Refer to [Deploy a TiDB Cluster Offline Using TiUP](/production-offline-deployment-using-tiup.md#option-2-manually-pack-an-offline-component-package-using-tiup-mirror-clone) to install the TiUP offline mirror, deploy a TiDB cluster, and start it. +Refer to [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md#method-2-deploy-tiup-offline) to install the TiUP offline mirror, deploy a TiDB cluster, and start it. diff --git a/tiup/tiup-overview.md b/tiup/tiup-overview.md index ab94554fbdde..2af990945f6c 100644 --- a/tiup/tiup-overview.md +++ b/tiup/tiup-overview.md @@ -113,7 +113,6 @@ The output is long but you can focus on only two parts: - client: used to connect to a TiDB cluster in a local machine - cluster: used to deploy a TiDB cluster for production environments - bench: used to stress test the database - - doc: used to open online document > **Note:** > diff --git a/tiup/tiup-playground.md b/tiup/tiup-playground.md index 2291797b7bc5..f907980aec6b 100644 --- a/tiup/tiup-playground.md +++ b/tiup/tiup-playground.md @@ -20,9 +20,9 @@ If you directly execute the `tiup playground` command, TiUP uses the locally ins This command actually performs the following operations: -- Because this command does not specify the version of the playground component, TiUP first checks the latest version of the installed playground component. Assume that the latest version is v0.0.6, then this command works the same as `tiup playground:v0.0.6`. +- Because this command does not specify the version of the playground component, TiUP first checks the latest version of the installed playground component. Assume that the latest version is v1.5.0, then this command works the same as `tiup playground:v1.5.0`. - If you have not used TiUP playground to install the TiDB, TiKV, and PD components, the playground component installs the latest stable version of these components, and then start these instances. -- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v4.0.0-rc, then this command works the same as `tiup playground:v0.0.6 v4.0.0-rc`. +- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v5.0.0, then this command works the same as `tiup playground:v1.5.0 v5.1.0`. - Because this command does not specify the number of each component, TiUP playground, by default, starts a smallest cluster that consists of one TiDB instance, one TiKV instance, and one PD instance. - After starting each TiDB component, TiUP playground reminds you that the cluster is successfully started and provides you some useful information, such as how to connect to the TiDB cluster through the MySQL client and how to access the [TiDB Dashboard](/dashboard/dashboard-intro.md). @@ -64,7 +64,7 @@ Flags: tiup playground nightly ``` -In the command above, `nightly` is the version number of the cluster. Similarly, you can replace `nightly` with `v4.0.0-rc`, and the command is `tiup playground v4.0.0-rc`. +In the command above, `nightly` is the version number of the cluster. Similarly, you can replace `nightly` with `v5.1.0`, and the command is `tiup playground v5.1.0`. ### Start a cluster with monitor diff --git a/tiup/tiup-reference.md b/tiup/tiup-reference.md new file mode 100644 index 000000000000..dcca0f5dbc30 --- /dev/null +++ b/tiup/tiup-reference.md @@ -0,0 +1,88 @@ +--- +title: TiUP Reference +--- + +# TiUP Reference + +TiUP serves as the package manager of the TiDB ecosystem. It manages components in the TiDB ecosystem, such as TiDB, PD, and TiKV. + +## Syntax + +```shell +tiup [flags] [args...] # Executes a command +# or +tiup [flags] [args...] # Runs a component +``` + +You can use the `help` command to get the information of a specific command. The summary of each command shows its parameters and their usage. Mandatory parameters are shown in angle brackets, and optional parameters are shown in square brackets. + +`` represents the command name. For the list of supported commands, see the [Command list](#command-list) below. `` represents the component name. For the list of supported components, see the [Component list](#component-list) below. + +## Options + +### -B, --binary + +- If you enable this option, the specified binary file path is printed. + + - Executing `tiup -B/--binary ` will have the path of the latest stable installed `` component printed. If `` is not installed, an error is returned. + - Executing `tiup -B/--binary :` will have the path of the installed `` component's `` printed. If this `` is not printed, an error is returned. + +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +> **Note:** +> +> This option can only be used in commands of the `tiup [flags] [args...]` format. + +### --binpath + +> **Note:** +> +> This option can only be used in commands of the `tiup [flags] [args...]` format. + +- Specifies the path of the component to be executed. When a component is executed, if you do not want to use the binary file in the TiUP mirror, you can add this option to specify using the binary file in a custom path. +- Data type: `STRING` + +### --skip-version-check + +> **Note:** +> +> This option is deprecated since v1.3.0. + +- Skips the validity check for version numbers. By default, the specified version number can only be the semantic version. +- Data type: `BOOLEAN` +- This option is disabled by default and its default value is `false`. To enable this option, you can add this option to the command, and pass the `true` value or do not pass any value. + +### -T, --tag + +- Specifies a tag for the component to be started. Some components need to use disk storage during the execution, and TiUP allocates a temporary storage directory for this execution. If you want TiUP to allocate a fixed directory, you can use `-T/--tag` to specify the name of the directory, so that the same batch of files can be read and written in multiple executions with the same tag. +- Data type: `STRING` + +### -v, --version + +Prints the TiUP version. + +### --help + +Prints the help information. + +## Command list + +TiUP has multiple commands, and these commands have multiple sub-commands. For the specific commands and their detailed descriptions, click the corresponding links in the list below: + +- [install](/tiup/tiup-command-install.md): Installs a component. +- [list](/tiup/tiup-command-list.md): Shows the component list. +- [uninstall](/tiup/tiup-command-uninstall.md): Uninstalls a component. +- [update](/tiup/tiup-command-update.md): Updates the installed component. +- [status](/tiup/tiup-command-status.md): Shows the running status of a component. +- [clean](/tiup/tiup-command-clean.md): Cleans the data directory of a component. +- [mirror](/tiup/tiup-command-mirror.md): Manages the mirror. +- [telemetry](/tiup/tiup-command-telemetry.md): Enables or disables the telemetry. +- [completion](/tiup/tiup-command-completion.md): Completes the TiUP command. +- [env](/tiup/tiup-command-env.md): Shows the TiUP-related environment variables. +- [help](/tiup/tiup-command-help.md): Shows the help information of a command or component. + +## Component list + +- [cluster](/tiup/tiup-component-cluster.md): Manages the TiDB cluster in a production environment. +- [dm](/tiup/tiup-component-dm.md): Manages the TiDB Data Migration (DM) cluster in a production environment. diff --git a/transaction-overview.md b/transaction-overview.md index 5ff958b3bdbf..4b4b1ae151cc 100644 --- a/transaction-overview.md +++ b/transaction-overview.md @@ -12,6 +12,10 @@ This document introduces commonly used transaction-related statements, explicit The common variables include [`autocommit`](#autocommit), [`tidb_disable_txn_auto_retry`](/system-variables.md#tidb_disable_txn_auto_retry), [`tidb_retry_limit`](/system-variables.md#tidb_retry_limit), and [`tidb_txn_mode`](/system-variables.md#tidb_txn_mode). +> **Note:** +> +> The [`tidb_disable_txn_auto_retry`](/system-variables.md#tidb_disable_txn_auto_retry) and [`tidb_retry_limit`](/system-variables.md#tidb_retry_limit) variables only apply to optimistic transactions, not to pessimistic transactions. + ## Common statements ### Starting a transaction @@ -38,6 +42,12 @@ START TRANSACTION; START TRANSACTION WITH CONSISTENT SNAPSHOT; ``` +{{< copyable "sql" >}} + +```sql +START TRANSACTION WITH CAUSAL CONSISTENCY ONLY; +``` + If the current session is in the process of a transaction when one of these statements is executed, TiDB automatically commits the current transaction before starting a new transaction. > **Note:** @@ -294,3 +304,73 @@ TiDB previously limited the total number of key-value pairs for a single transac > Usually, TiDB Binlog is enabled to replicate data to the downstream. In some scenarios, message middleware such as Kafka is used to consume binlogs that are replicated to the downstream. > > Taking Kafka as an example, the upper limit of Kafka's single message processing capability is 1 GB. Therefore, when `txn-total-size-limit` is set to more than 1 GB, it might happen that the transaction is successfully executed in TiDB, but the downstream Kafka reports an error. To avoid this situation, you need to decide the actual value of `txn-total-size-limit` according to the limit of the end consumer. For example, if Kafka is used downstream, `txn-total-size-limit` must not exceed 1 GB. + +## Causal consistency + +> **Note:** +> +> Transactions with causal consistency take effect only when the async commit and one-phase commit features are enabled. For details of the two features, see [`tidb_enable_async_commit`](/system-variables.md#tidb_enable_async_commit-new-in-v50) and [`tidb_enable_1pc`](/system-variables.md#tidb_enable_1pc-new-in-v50). + +TiDB supports enabling causal consistency for transactions. Transactions with causal consistency, when committed, do not need to get timestamp from PD and have lower commit latency. The syntax to enable causal consistency is as follows: + +{{< copyable "sql" >}} + +```sql +START TRANSACTION WITH CAUSAL CONSISTENCY ONLY; +``` + +By default, TiDB guarantees linear consistency. In the case of linear consistency, if transaction 2 is committed after transaction 1 is committed, logically, transaction 2 should occur after transaction 1. Causal consistency is weaker than linear consistency. In the case of causal consistency, the commit order and occurrence order of two transactions can be guaranteed consistent only when the data locked or written by transaction 1 and transaction 2 have an intersection, which means that the two transactions have a causal relationship known to the database. Currently, TiDB does not support passing in external causal relationship. + +Two transactions with causal consistency enabled have the following characteristics: + ++ [Transactions with potential causal relationship have the consistent logical order and physical commit order](#transactions-with-potential-causal-relationship-have-the-consistent-logical-order-and-physical-commit-order) ++ [Transactions with no causal relationship do not guarantee consistent logical order and physical commit order](#transactions-with-no-causal-relationship-do-not-guarantee-consistent-logical-order-and-physical-commit-order) ++ [Reads without lock do not create causal relationship](#reads-without-lock-do-not-create-causal-relationship) + +### Transactions with potential causal relationship have the consistent logical order and physical commit order + +Assume that both transaction 1 and transaction 2 adopt causal consistency and have the following statements executed: + +| Transaction 1 | Transaction 2 | +|-------|-------| +| START TRANSACTION WITH CAUSAL CONSISTENCY ONLY | START TRANSACTION WITH CAUSAL CONSISTENCY ONLY | +| x = SELECT v FROM t WHERE id = 1 FOR UPDATE | | +| UPDATE t set v = $(x + 1) WHERE id = 2 | | +| COMMIT | | +| | UPDATE t SET v = 2 WHERE id = 1 | +| | COMMIT | + +In the example above, transaction 1 locks the `id = 1` record and transaction 2 modifies the `id = 1` record. Therefore, transaction 1 and transaction 2 have a potential causal relationship. Even with the causal consistency enabled, as long as transaction 2 is committed after transaction 1 is successfully committed, logically, transaction 2 must occur after transaction 1. Therefore, it is impossible that a transaction reads transaction 2's modification on the `id = 1` record without reading transaction 1's modification on the `id = 2` record. + +### Transactions with no causal relationship do not guarantee consistent logical order and physical commit order + +Assume that the initial values of `id = 1` and `id = 2` are both `0`. Assume that both transaction 1 and transaction 2 adopt causal consistency and have the following statements executed: + +| Transaction 1 | Transaction 2 | Transaction 3 | +|-------|-------|-------| +| START TRANSACTION WITH CAUSAL CONSISTENCY ONLY | START TRANSACTION WITH CAUSAL CONSISTENCY ONLY | | +| UPDATE t set v = 3 WHERE id = 2 | | | +| | UPDATE t SET v = 2 WHERE id = 1 | | +| | | BEGIN | +| COMMIT | | | +| | COMMIT | | +| | | SELECT v FROM t WHERE id IN (1, 2) | + +In the example above, transaction 1 does not read the `id = 1` record, so transaction 1 and transaction 2 have no causal relationship known to the database. With causal consistency enabled for the transactions, even if transaction 2 is committed after transaction 1 is committed in terms of physical time order, TiDB does not guarantee that transaction 2 logically occurs after transaction 1. + +If transaction 3 begins before transaction 1 is committed, and if transaction 3 reads the `id = 1` and `id = 2` records after transaction 2 is committed, transaction 3 might read the value of `id = 1` to be `2` but the value of `id = 2` to be `0`. + +### Reads without lock do not create causal relationship + +Assume that both transaction 1 and transaction 2 adopt causal consistency and have the following statements executed: + +| Transaction 1 | Transaction 2 | +|-------|-------| +| START TRANSACTION WITH CAUSAL CONSISTENCY ONLY | START TRANSACTION WITH CAUSAL CONSISTENCY ONLY | +| | UPDATE t SET v = 2 WHERE id = 1 | +| SELECT v FROM t WHERE id = 1 | | +| UPDATE t set v = 3 WHERE id = 2 | | +| | COMMIT | +| COMMIT | | + +In the example above, reads without lock do not create causal relationship. Transaction 1 and transaction 2 have created write skew. In this case, it would have been unreasonable if the two transactions still had causal relationship. Therefore, the two transactions with causal consistency enabled have no definite logical order. diff --git a/troubleshoot-high-disk-io.md b/troubleshoot-high-disk-io.md index 6274abd26169..c462f8a05b51 100644 --- a/troubleshoot-high-disk-io.md +++ b/troubleshoot-high-disk-io.md @@ -13,7 +13,7 @@ If TiDB's response slows down after you have troubleshot the CPU bottleneck and ### Locate I/O issues from monitor -The quickest way to locate I/O issues is to view the overall I/O status from the monitor, such as the Grafana dashboard which is deployed by default by TiDB Ansible and TiUP. The dashboard panels related to I/O include **Overview**, **Node_exporter**, and **Disk-Performance**. +The quickest way to locate I/O issues is to view the overall I/O status from the monitor, such as the Grafana dashboard which is deployed by default by TiUP. The dashboard panels related to I/O include **Overview**, **Node_exporter**, and **Disk-Performance**. #### The first type of monitoring panels @@ -78,7 +78,7 @@ In addition, some other panel metrics might help you determine whether the bottl ### I/O issues found in alerts -The cluster deployment tools (TiDB Ansible and TiUP) deploy the cluster with alert components by default that have built-in alert items and thresholds. The following alert items are related to I/O: +The cluster deployment tool (TiUP) deploys the cluster with alert components by default that have built-in alert items and thresholds. The following alert items are related to I/O: - TiKV_write_stall - TiKV_raft_log_lag diff --git a/troubleshoot-hot-spot-issues.md b/troubleshoot-hot-spot-issues.md index d9ecf2e6dd56..97f32d9d79d9 100644 --- a/troubleshoot-hot-spot-issues.md +++ b/troubleshoot-hot-spot-issues.md @@ -106,7 +106,7 @@ ALTER TABLE：ALTER TABLE t SHARD_ROW_ID_BITS = 4; The value of `SHARD_ROW_ID_BITS` can be dynamically modified. The modified value only takes effect for newly written data. -When TiDB's `alter-primary-key` parameter is set to false, the table's integer primary key is used as the RowID. At this time, the `SHARD_ROW_ID_BITS` option can not be used because it changes the RowID generation rules. If the `alter-primary-key` parameter is set to true, TiDB no longer uses the integer primary key as the RowID when creating a table, and the table with the integer primary key can also use the `SHARD_ROW_ID_BITS` feature. +For the table with a primary key of the `CLUSTERED` type, TiDB uses the primary key of the table as the RowID. At this time, the `SHARD_ROW_ID_BITS` option cannot be used because it changes the RowID generation rules. For the table with the primary key of the `NONCLUSTERED` type, TiDB uses an automatically allocated 64-bit integer as the RowID. In this case, you can use the `SHARD_ROW_ID_BITS` feature. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). The following two load diagrams shows the case where two tables without primary keys use `SHARD_ROW_ID_BITS` to scatter hotspots. The first diagram shows the situation before scattering hotspots, while the second one shows the situation after scattering hotspots. @@ -170,7 +170,7 @@ For more details, see [AUTO_RANDOM](/auto-random.md). ## Optimization of small table hotspots -From v4.0, TiDB introduces the Coprocessor Cache feature to support pushing down computing result caches. After this feature is enabled, it caches the computing results that will be pushed down to TiKV. This feature works well for read hotspots of small tables. +The Coprocessor Cache feature of TiDB supports pushing down computing result caches. After this feature is enabled, TiDB caches the computing results that will be pushed down to TiKV. This feature works well for read hotspots of small tables. For more details, see [Coprocessor Cache](/coprocessor-cache.md). diff --git a/troubleshoot-lock-conflicts.md b/troubleshoot-lock-conflicts.md index a9a57b21be9e..16646e062855 100644 --- a/troubleshoot-lock-conflicts.md +++ b/troubleshoot-lock-conflicts.md @@ -204,8 +204,8 @@ Solutions: * If the above error occurs frequently, it is recommended to adjust the application logic. #### TTL manager has timed out - -The transaction execution time can not exceed the GC time limit. In addition, the TTL time of pessimistic transactions has an upper limit, whose default value is 10 minutes. Therefore, a pessimistic transaction executed for more than 10 minutes will fail to commit. This timeout threshold is controlled by the TiDB parameter [performance.max-txn-ttl](https://github.com/pingcap/tidb/blob/master/config/config.toml.example). + +The transaction execution time can not exceed the GC time limit. In addition, the TTL time of pessimistic transactions has an upper limit, whose default value is 1 hour. Therefore, a pessimistic transaction executed for more than 1 hour will fail to commit. This timeout threshold is controlled by the TiDB parameter [performance.max-txn-ttl](https://github.com/pingcap/tidb/blob/master/config/config.toml.example). When the execution time of a pessimistic transaction exceeds the TTL time, the following error message occurs in the TiDB log: @@ -215,7 +215,7 @@ TTL manager has timed out, pessimistic locks may expire, please commit or rollba Solutions: -* First, confirm whether the application logic can be optimized. For example, large transactions may trigger TiDB's transaction size limit, which can be split into multiple small transactions. +* First, confirm whether the application logic can be optimized. For example, large transactions may trigger TiDB's transaction size limit, which can be split into multiple small transactions. * Also, you can adjust the related parameters properly to meet the application transaction logic. #### Deadlock found when trying to get lock @@ -230,4 +230,148 @@ When a pessimistic transaction has a deadlock, one of the transactions must be t Solutions: -* The application needs to adjust transaction request logic when there too many deadlocks. +* If it is difficult to confirm the cause of the deadlock, for v5.1 and later versions, you are recommended to try to query the `INFORMATION_SCHEMA.DEADLOCKS` or `INFORMATION_SCHEMA.CLUSTER_DEADLOCKS` system table to get the information of deadlock waiting chain. For details, see the [Deadlock errors](#deadlock-errors) section and the [`DEADLOCKS` table](/information-schema/information-schema-deadlocks.md) document. +* If the deadlock occurs frequently, you need to adjust the transaction query logic in your application to reduce such occurrences. + +### Use Lock View to troubleshoot issues related to pessimistic locks + +Since v5.1, TiDB supports the Lock View feature. This feature has several system tables built in `information_schema` that provide more information about the pessimistic lock conflicts and pessimistic lock waitings. For the detailed introduction of these tables, see the following documents: + +* [`TIDB_TRX` and `CLUSTER_TIDB_TRX`](/information-schema/information-schema-tidb-trx.md): Provides information of all running transactions on the current TiDB node or in the entire cluster, including whether the transaction is in the lock-waiting state, the lock-waiting time, and the digests of statements that have been executed in the transaction. +* [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md): Provides the pessimistic lock-waiting information in TiKV, including the `start_ts` of the blocking and blocked transaction, the digest of the blocked SQL statement, and the key on which the waiting occurs. +* [`DEADLOCKS` and `CLUSTER_DEADLOCKS`](/information-schema/information-schema-deadlocks.md): Provides the information of several deadlock events that have recently occurred on the current TiDB node or in the entire cluster, including the waiting relationship among transactions in the deadlock loops, the digest of the statement currently being executed in the transaction, and the key on which the waiting occurs. + +> **Warning:** +> +> Currently, this is an experimental feature. The definition and behavior of the table structure might have major changes in future releases. + +The following sections show the examples of troubleshooting some issues using these tables. + +#### Deadlock errors + +To get the information of the recent deadlock errors, you can query the `DEADLOCKS` or `CLUSTER_DEADLOCKS` table. For example: + +{{< copyable "sql" >}} + +```sql +select * from information_schema.deadlocks; +``` + +```sql ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +| DEADLOCK_ID | OCCUR_TIME | RETRYABLE | TRY_LOCK_TRX_ID | CURRENT_SQL_DIGEST | KEY | TRX_HOLDING_LOCK | ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +| 1 | 2021-06-04 08:22:38.765699 | 0 | 425405959304904707 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000002 | 425405959304904708 | +| 1 | 2021-06-04 08:22:38.765699 | 0 | 425405959304904708 | 22230766411edb40f27a68dadefc63c6c6970d5827f1e5e22fc97be2c4d8350d | 7480000000000000385F728000000000000001 | 425405959304904707 | ++-------------+----------------------------+-----------+--------------------+------------------------------------------------------------------+----------------------------------------+--------------------+ +``` + +The query result above shows the waiting relationship among multiple transactions in the deadlock error, the digest of the SQL statement currently being executed in each transaction, and the key on which the conflict occurs. + +You can get the text of the normalized SQL statement corresponding to the digest of the SQL statements executed recently from the `STATEMENTS_SUMMARY` or `STATEMENTS_SUMMARY_HISTORY` table. For details, see [`STATEMENTS_SUMMARY` and `STATEMENTS_SUMMARY_HISTORY` tables](/statement-summary-tables.md). You can also join the obtained results directly with the `DEADLOCKS` table. Note that the `STATEMENTS_SUMMARY` table might not contain the information of all SQL statements, so left join is used in the following example: + +{{< copyable "sql" >}} + +```sql +select l.deadlock_id, l.occur_time, l.try_lock_trx_id, l.trx_holding_lock, s.digest_text from information_schema.deadlocks as l left join information_schema.statements_summary as s on l.current_sql_digest = s.digest; +``` + +```sql ++-------------+----------------------------+--------------------+--------------------+-----------------------------------------+ +| deadlock_id | occur_time | try_lock_trx_id | trx_holding_lock | digest_text | ++-------------+----------------------------+--------------------+--------------------+-----------------------------------------+ +| 1 | 2021-06-04 08:22:38.765699 | 425405959304904707 | 425405959304904708 | update `t` set `v` = ? where `id` = ? ; | +| 1 | 2021-06-04 08:22:38.765699 | 425405959304904708 | 425405959304904707 | update `t` set `v` = ? where `id` = ? ; | ++-------------+----------------------------+--------------------+--------------------+-----------------------------------------+ +``` + +#### A few hot keys cause queueing locks + +The `DATA_LOCK_WAITS` system table provides the lock-waiting status on the TiKV nodes. When you query this table, TiDB automatically obtains the real-time lock-waiting information from all TiKV nodes. If a few hot keys are frequently locked and block many transactions, you can query the `DATA_LOCK_WAITS` table and aggregate the results by key to try to find the keys on which issues frequently occur: + +{{< copyable "sql" >}} + +```sql +select `key`, count(*) as `count` from information_schema.data_lock_waits group by `key` order by `count` desc; +``` + +```sql ++----------------------------------------+-------+ +| key | count | ++----------------------------------------+-------+ +| 7480000000000000415f728000000000000001 | 2 | +| 7480000000000000415f728000000000000002 | 1 | ++----------------------------------------+-------+ +``` + +To avoid contingency, you might need to make multiple queries. + +If you know the key that frequently has issues occurred, you can try to get the information of the transaction that tries to lock the key from the `TIDB_TRX` or `CLUSTER_TIDB_TRX` table. + +Note that the information displayed in the `TIDB_TRX` and `CLUSTER_TIDB_TRX` tables is also the information of the transactions that are running at the time the query is performed. These tables do not display the information of the completed transactions. If there is a large number of concurrent transactions, the result set of the query might also be large. You can use the `limit` clause or the `where` clause to filter out transactions with a long lock-waiting time. Note that when you join multiple tables in Lock View, the data in different tables might not be obtained at the same time, so the information in different tables might not be consistent. + +{{< copyable "sql" >}} + +```sql +select trx.* from information_schema.data_lock_waits as l left join information_schema.tidb_trx as trx on l.trx_id = trx.id where l.key = "7480000000000000415f728000000000000001"\G +``` + +```sql +*************************** 1. row *************************** + ID: 425496938634543111 + START_TIME: 2021-06-08 08:46:48.341000 +CURRENT_SQL_DIGEST: a4e28cc182bdd18288e2a34180499b9404cd0ba07e3cc34b6b3be7b7c2de7fe9 + STATE: LockWaiting +WAITING_START_TIME: 2021-06-08 08:46:48.388024 + MEM_BUFFER_KEYS: 1 + MEM_BUFFER_BYTES: 19 + SESSION_ID: 87 + USER: root + DB: test + ALL_SQL_DIGESTS: [0fdc781f19da1c6078c9de7eadef8a307889c001e05f107847bee4cfc8f3cdf3, a4e28cc182bdd18288e2a34180499b9404cd0 +ba07e3cc34b6b3be7b7c2de7fe9, a4e28cc182bdd18288e2a34180499b9404cd0ba07e3cc34b6b3be7b7c2de7fe9] +*************************** 2. row *************************** + ID: 425496940994101249 + START_TIME: 2021-06-08 08:46:57.342000 +CURRENT_SQL_DIGEST: a4e28cc182bdd18288e2a34180499b9404cd0ba07e3cc34b6b3be7b7c2de7fe9 + STATE: LockWaiting +WAITING_START_TIME: 2021-06-08 08:46:57.590060 + MEM_BUFFER_KEYS: 0 + MEM_BUFFER_BYTES: 0 + SESSION_ID: 85 + USER: root + DB: test + ALL_SQL_DIGESTS: [0fdc781f19da1c6078c9de7eadef8a307889c001e05f107847bee4cfc8f3cdf3, a4e28cc182bdd18288e2a34180499b9404cd0 +ba07e3cc34b6b3be7b7c2de7fe9] +2 rows in set (0.00 sec) +``` + +#### A transaction is blocked for a long time + +If a transaction is known to be blocked by another transaction (or multiple transactions) and the `start_ts` (transaction ID) of the current transaction is known, you can use the following method to obtain the information of the blocking transaction. Note that when you join multiple tables in Lock View, the data in different tables might not be obtained at the same time, so the information in different tables might not be consistent. + +{{< copyable "sql" >}} + +```sql +select l.key, trx.* from information_schema.data_lock_waits as l join information_schema.tidb_trx as trx on l.current_holding_trx_id = trx.id where l.trx_id = 425497223886536705\G +``` + +```sql +*************************** 1. row *************************** + key: 7480000000000000475f728000000000000002 + ID: 425497219115778059 + START_TIME: 2021-06-08 09:04:38.292000 +CURRENT_SQL_DIGEST: a4e28cc182bdd18288e2a34180499b9404cd0ba07e3cc34b6b3be7b7c2de7fe9 + STATE: LockWaiting +WAITING_START_TIME: 2021-06-08 09:04:38.336264 + MEM_BUFFER_KEYS: 1 + MEM_BUFFER_BYTES: 19 + SESSION_ID: 97 + USER: root + DB: test + ALL_SQL_DIGESTS: [0fdc781f19da1c6078c9de7eadef8a307889c001e05f107847bee4cfc8f3cdf3, a4e28cc182bdd18288e2a34180499b9404cd0 +ba07e3cc34b6b3be7b7c2de7fe9, a4e28cc182bdd18288e2a34180499b9404cd0ba07e3cc34b6b3be7b7c2de7fe9] +1 row in set (0.01 sec) +``` + +If the `start_ts` of the current transaction is unknown, you can try to find it out from the information in the `TIDB_TRX` / `CLUSTER_TIDB_TRX` table or in the [`PROCESSLIST` / `CLUSTER_PROCESSLIST`](/information-schema/information-schema-processlist.md) table. diff --git a/troubleshoot-tidb-cluster.md b/troubleshoot-tidb-cluster.md index 7ec240503757..8e24f9560e71 100644 --- a/troubleshoot-tidb-cluster.md +++ b/troubleshoot-tidb-cluster.md @@ -29,10 +29,8 @@ For other information, see [Frequently Asked Questions (FAQ)](/faq/tidb-faq.md). 3. If the data is cleared and the services are re-deployed, make sure that: - - All the data in `tikv-server` and `pd-server` are cleared. - The specific data is stored in `tikv-server` and the metadata is stored in `pd-server`. If only one of the two servers is cleared, the data will be inconsistent. - - After the data in `pd-server` and `tikv-server` are cleared and the `pd-server` and `tikv-server` are restarted, the `tidb-server` must be restarted too. - The cluster ID is randomly allocated when the `pd-server` is initialized. So when the cluster is re-deployed, the cluster ID changes and you need to restart the `tidb-server` to get the new cluster ID. + - All the data in `tikv-server` and `pd-server` are cleared. The specific data is stored in `tikv-server` and the metadata is stored in `pd-server`. If only one of the two servers is cleared, the data will be inconsistent. + - After the data in `pd-server` and `tikv-server` are cleared and the `pd-server` and `tikv-server` are restarted, the `tidb-server` must be restarted too. The cluster ID is randomly allocated when the `pd-server` is initialized. So when the cluster is re-deployed, the cluster ID changes and you need to restart the `tidb-server` to get the new cluster ID. ## Cannot start `tidb-server` @@ -74,8 +72,11 @@ See the following for the situations when the `tikv-server` cannot be started: See the following for the situations when the `pd-server` cannot be started: - Error in the startup parameters. + See the [PD configuration and options](/command-line-flags-for-pd-configuration.md). + - The port is occupied. + Use the `lsof -i:port` command to show all the networking related to a given port and make sure the port to start the `pd-server` is not occupied. ## The TiDB/TiKV/PD process aborts unexpectedly diff --git a/troubleshoot-tidb-lightning.md b/troubleshoot-tidb-lightning.md deleted file mode 100644 index 5a8857fbe261..000000000000 --- a/troubleshoot-tidb-lightning.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -title: TiDB Lightning Troubleshooting -summary: Learn about common errors and solutions of TiDB Lightning. -aliases: ['/docs/dev/troubleshoot-tidb-lightning/','/docs/dev/how-to/troubleshoot/tidb-lightning/','/docs/dev/tidb-lightning/tidb-lightning-misuse-handling/','/docs/dev/reference/tools/error-case-handling/lightning-misuse-handling/','/tidb/dev/tidb-lightning-misuse-handling'] ---- - -# TiDB Lightning Troubleshooting - -When TiDB Lightning encounters an unrecoverable error, it exits with nonzero exit code and leaves the reason in the log file. Errors are typically printed at the end of the log. You can also search for the string `[error]` to look for non-fatal errors. - -This document summarizes some commonly encountered errors in the `tidb-lightning` log file and their solutions. - -## Import speed is too slow - -Normally it takes TiDB Lightning 2 minutes per thread to import a 256 MB data file. It is an error if the speed is much slower than this. The time taken for each data file can be checked from the log mentioning `restore chunk … takes`. This can also be observed from metrics on Grafana. - -There are several reasons why TiDB Lightning becomes slow: - -**Cause 1**: `region-concurrency` is too high, which causes thread contention and reduces performance. - -1. The setting can be found from the start of the log by searching `region-concurrency`. -2. If Lightning shares the same machine with other services (e.g. Importer), `region-concurrency` must be **manually** set to 75% of the total number of CPU cores -3. If there is a quota on CPU (e.g. limited by K8s settings), Lightning may not be able to read this out. In this case, `region-concurrency` must also be **manually** reduced. - -**Cause 2**: The table is too complex. - -Every additional index will introduce a new KV pair for each row. If there are N indices, the actual size to be imported would be approximately (N+1) times the size of the Dumpling output. If the indices are negligible, you may first remove them from the schema, and add them back via `CREATE INDEX` after import is complete. - -**Cause 3**: Each file is too large. - -TiDB Lightning works the best when the data source are broken down into multiple files of size around 256 MB, so that the data can be processed in parallel. Lightning seems unresponsive if each file is too large. - -If the data source is CSV, and all CSV files have no fields containing literal new line characters (U+000A and U+000D), you can turn on "strict format" to let Lightning automatically split the large files. - -```toml -[mydumper] -strict-format = true -``` - -**Cause 4**: TiDB Lightning is too old. - -Try the latest version! Maybe there is new speed improvement. - -## checksum failed: checksum mismatched remote vs local - -**Cause**: The checksum of a table in the local data source and the remote imported database differ. This error has several deeper reasons: - -1. The table might already have data before. These old data can affect the final checksum. - -2. If the remote checksum is 0, which means nothing is imported, it is possible that the cluster is too hot and fails to take in any data. - -3. If the data is mechanically generated, ensure it respects the constrains of the table: - - * `AUTO_INCREMENT` columns need to be positive, and do not contain the value "0". - * The UNIQUE and PRIMARY KEYs must have no duplicated entries. - -4. If TiDB Lightning has failed before and was not properly restarted, a checksum mismatch may happen due to data being out-of-sync. - -**Solutions**: - -1. Delete the corrupted data with via `tidb-lightning-ctl`, and restart Lightning to import the affected tables again. - - ```sh - tidb-lightning-ctl --config conf/tidb-lightning.toml --checkpoint-error-destroy=all - ``` - -2. Consider using an external database to store the checkpoints (change `[checkpoint] dsn`) to reduce the target database's load. - -3. If TiDB Lightning was improperly restarted, see also the "[How to properly restart TiDB Lightning](/tidb-lightning/tidb-lightning-faq.md#how-to-properly-restart-tidb-lightning)" section in the FAQ. - -## Checkpoint for … has invalid status: (error code) - -**Cause**: [Checkpoint](/tidb-lightning/tidb-lightning-checkpoints.md) is enabled, and Lightning or Importer has previously abnormally exited. To prevent accidental data corruption, Lightning will not start until the error is addressed. - -The error code is an integer less than 25, with possible values of 0, 3, 6, 9, 12, 14, 15, 17, 18, 20 and 21. The integer indicates the step where the unexpected exit occurs in the import process. The larger the integer is, the later step where the exit occurs. - -**Solutions**: - -If the error was caused by invalid data source, delete the imported data using `tidb-lightning-ctl` and start Lightning again. - -```sh -tidb-lightning-ctl --config conf/tidb-lightning.toml --checkpoint-error-destroy=all -``` - -See the [Checkpoints control](/tidb-lightning/tidb-lightning-checkpoints.md#checkpoints-control) section for other options. - -## ResourceTemporarilyUnavailable("Too many open engines …: …") - -**Cause**: The number of concurrent engine files exceeds the limit specified by `tikv-importer`. This could be caused by misconfiguration. Additionally, if `tidb-lightning` exited abnormally, an engine file might be left at a dangling open state, which could cause this error as well. - -**Solutions**: - -1. Increase the value of `max-open-engines` setting in `tikv-importer.toml`. This value is typically dictated by the available memory. This could be calculated by using: - - Max Memory Usage ≈ `max-open-engines` × `write-buffer-size` × `max-write-buffer-number` - -2. Decrease the value of `table-concurrency` + `index-concurrency` so it is less than `max-open-engines`. - -3. Restart `tikv-importer` to forcefully remove all engine files (default to `./data.import/`). This also removes all partially imported tables, which requires Lightning to clear the outdated checkpoints. - - ```sh - tidb-lightning-ctl --config conf/tidb-lightning.toml --checkpoint-error-destroy=all - ``` - -## cannot guess encoding for input file, please convert to UTF-8 manually - -**Cause**: TiDB Lightning only recognizes the UTF-8 and GB-18030 encodings for the table schemas. This error is emitted if the file isn't in any of these encodings. It is also possible that the file has mixed encoding, such as containing a string in UTF-8 and another string in GB-18030, due to historical `ALTER TABLE` executions. - -**Solutions**: - -1. Fix the schema so that the file is entirely in either UTF-8 or GB-18030. - -2. Manually `CREATE` the affected tables in the target database, and then set `[mydumper] no-schema = true` to skip automatic table creation. - -3. Set `[mydumper] character-set = "binary"` to skip the check. Note that this might introduce mojibake into the target database. - -## [sql2kv] sql encode error = [types:1292]invalid time format: '{1970 1 1 …}' - -**Cause**: A table contains a column with the `timestamp` type, but the time value itself does not exist. This is either because of DST changes or the time value has exceeded the supported range (Jan 1, 1970 to Jan 19, 2038). - -**Solutions**: - -1. Ensure Lightning and the source database are using the same time zone. When deploying via TiDB Ansible, the timezone is defined in `inventory.ini`. - - ```ini - # inventory.ini - [all:vars] - timezone = Asia/Shanghai - ``` - - When executing Lightning directly, the time zone can be forced using the `$TZ` environment variable. - - ```sh - # Manual deployment, and force Asia/Shanghai. - TZ='Asia/Shanghai' bin/tidb-lightning -config tidb-lightning.toml - ``` - -2. When exporting data using Mydumper, make sure to include the `--skip-tz-utc` flag. - -3. Ensure the entire cluster is using the same and latest version of `tzdata` (version 2018i or above). - - On CentOS, run `yum info tzdata` to check the installed version and whether there is an update. Run `yum upgrade tzdata` to upgrade the package. - -## `[Error 8025: entry too large, the max entry size is 6291456]` - -**Cause**: A single row of key-value pairs generated by TiDB Lightning exceeds the limit set by TiDB. - -**Solution**: - -Currently, the limitation of TiDB cannot be bypassed. You can only ignore this table to ensure the successful import of other tables. - -## Encounter `rpc error: code = Unimplemented ...` when TiDB Lightning switches the mode - -**Cause**: Some node(s) in the cluster does not support `switch-mode`. For example, if the TiFlash version is earlier than `v4.0.0-rc.2`, [`switch-mode` is not supported](https://github.com/pingcap/tidb-lightning/issues/273). - -**Solutions**: - -- If there are TiFlash nodes in the cluster, you can update the cluster to `v4.0.0-rc.2` or higher versions. -- Temporarily disable TiFlash if you do not want to upgrade the cluster. diff --git a/tune-tikv-memory-performance.md b/tune-tikv-memory-performance.md index c8e2a8f3bcb7..8da49518bd93 100644 --- a/tune-tikv-memory-performance.md +++ b/tune-tikv-memory-performance.md @@ -8,8 +8,7 @@ aliases: ['/docs/dev/tune-tikv-performance/','/docs/dev/reference/performance/tu This document describes how to tune the TiKV parameters for optimal performance. -TiKV uses RocksDB for persistent storage at the bottom level of the TiKV architecture. Therefore, many of the performance parameters are related to RocksDB. -TiKV uses two RocksDB instances: the default RocksDB instance stores KV data, the Raft RocksDB instance (RaftDB) stores Raft logs. +TiKV uses RocksDB for persistent storage at the bottom level of the TiKV architecture. Therefore, many of the performance parameters are related to RocksDB. TiKV uses two RocksDB instances: the default RocksDB instance stores KV data, the Raft RocksDB instance (RaftDB) stores Raft logs. TiKV implements `Column Families` (CF) from RocksDB. @@ -250,8 +249,7 @@ target-file-size-base = "32MB" ## TiKV memory usage -Besides `block cache` and `write buffer` which occupy the system memory, the system memory is occupied in the -following scenarios: +Besides `block cache` and `write buffer` which occupy the system memory, the system memory is occupied in the following scenarios: + Some of the memory is reserved as the system's page cache. diff --git a/tune-tikv-thread-performance.md b/tune-tikv-thread-performance.md index 465db019e990..b6ddfa986854 100644 --- a/tune-tikv-thread-performance.md +++ b/tune-tikv-thread-performance.md @@ -10,7 +10,7 @@ This document introduces TiKV internal thread pools and how to tune their perfor ## Thread pool introduction -In TiKV 4.0, the TiKV thread pool is mainly composed of gRPC, Scheduler, UnifyReadPool, Raftstore, Apply, RocksDB, and some scheduled tasks and detection components that do not consume much CPU. This document mainly introduces a few CPU-intensive thread pools that affect the performance of read and write requests. +The TiKV thread pool is mainly composed of gRPC, Scheduler, UnifyReadPool, Raftstore, Apply, RocksDB, and some scheduled tasks and detection components that do not consume much CPU. This document mainly introduces a few CPU-intensive thread pools that affect the performance of read and write requests. * The gRPC thread pool: it handles all network requests and forwards requests of different task types to different thread pools. @@ -22,7 +22,7 @@ In TiKV 4.0, the TiKV thread pool is mainly composed of gRPC, Scheduler, UnifyRe * The RocksDB thread pool: it is a thread pool for RocksDB to compact and flush tasks. For RocksDB's architecture and `Compact` operation, refer to [RocksDB: A Persistent Key-Value Store for Flash and RAM Storage](https://github.com/facebook/rocksdb). -* The UnifyReadPool thread pool: it is a new feature introduced in TiKV 4.0. It is a combination of the previous Coprocessor thread pool and Storage Read Pool. All read requests such as kv get, kv batch get, raw kv get, and coprocessor are executed in this thread pool. +* The UnifyReadPool thread pool: it is a combination of the Coprocessor thread pool and Storage Read Pool. All read requests such as kv get, kv batch get, raw kv get, and coprocessor are executed in this thread pool. ## TiKV read-only requests @@ -31,7 +31,7 @@ TiKV's read requests are divided into the following types: - Simple queries that specify a certain row or several rows, running in the Storage Read Pool. - Complex aggregate calculation and range queries, running in the Coprocessor Read Pool. -Starting from version 4.0, the above types of read requests can be configured to use the same thread pool, which reduces the number of threads and user costs. It is disabled by default (Point queries and Coprocessor requests use different thread pools by default). To enable the unified thread pool, set the `readpool.storage.use-unified-pool` configuration item to `true`. +Starting from TiKV v5.0, all read requests use the unified thread pool for queries by default. If your TiKV cluster is upgraded from TiKV v4.0 and the `use-unified-pool` configuration of `readpool.storage` was set to `false` before the upgrade, all read requests continue using different thread pools after the upgrade. In this scenario, to make all read requests use the unified thread pool for queries, you can set the value of `readpool.storage.use-unified-pool` to `true`. ## Performance tuning for TiKV thread pools diff --git a/upgrade-tidb-using-ansible.md b/upgrade-tidb-using-ansible.md deleted file mode 100644 index 1d5b78ce60d0..000000000000 --- a/upgrade-tidb-using-ansible.md +++ /dev/null @@ -1,230 +0,0 @@ ---- -title: Upgrade TiDB Using TiDB Ansible -summary: Learn how to upgrade TiDB using TiDB Ansible. -aliases: ['/docs/dev/upgrade-tidb-using-ansible/','/docs/dev/how-to/upgrade/from-previous-version/','/docs/dev/how-to/upgrade/rolling-updates-with-ansible/'] ---- - -# Upgrade TiDB Using TiDB Ansible - -This document is targeted for users who want to upgrade from TiDB 2.0, 2.1, 3.0, or 3.1 versions to the latest development version (`latest`), or from an earlier development version to the latest development version. The latest development version is compatible with [TiDB Binlog of the cluster version](/tidb-binlog/tidb-binlog-overview.md). - -> **Warning:** -> -> The latest development version of TiDB is not a stable version. Do not use it in the production environment. - -## Upgrade caveat - -- Rolling back to 2.1.x or earlier versions after upgrading is not supported. -- Before upgrading to `latest` from 2.0.6 or earlier versions, check if there are any running DDL operations, especially time-consuming ones like `Add Index`. If there are any, wait for the DDL operations to finish before you upgrade. -- Parallel DDL is supported in TiDB 2.1 and later versions. Therefore, for clusters with a TiDB version earlier than 2.0.1, rolling update to `latest` is not supported. To upgrade, you can choose either of the following two options: - - - Stop the cluster and upgrade to `latest` directly. - - Roll update to 2.0.1 or later 2.0.x versions, and then roll update to the `latest` version. - -> **Note:** -> -> Do not execute any DDL statements during the upgrading process, otherwise the undefined behavior error might occur. - -## Step 1: Install Ansible and dependencies on the control machine - -> **Note:** -> -> If you have installed Ansible and its dependencies, you can skip this step. - -The latest development version of TiDB Ansible depends on Ansible 2.4.2 ~ 2.7.11 (`2.4.2 ≦ ansible ≦ 2.7.11`, Ansible 2.7.11 recommended) and the Python modules of `jinja2 ≧ 2.9.6` and `jmespath ≧ 0.9.0`. - -To make it easy to manage dependencies, use `pip` to install Ansible and its dependencies. For details, see [Install Ansible and its dependencies on the control machine](/online-deployment-using-ansible.md#step-4-install-tidb-ansible-and-its-dependencies-on-the-control-machine). For offline environment, see [Install Ansible and its dependencies offline on the control machine](/offline-deployment-using-ansible.md#step-3-install-tidb-ansible-and-its-dependencies-offline-on-the-control-machine). - -After the installation is finished, you can view the version information using the following command: - -{{< copyable "shell-regular" >}} - -```bash -ansible --version -``` - -``` -ansible 2.7.11 -``` - -{{< copyable "shell-regular" >}} - -```bash -pip show jinja2 -``` - -``` -Name: Jinja2 -Version: 2.10 -``` - -{{< copyable "shell-regular" >}} - -```bash -pip show jmespath -``` - -``` -Name: jmespath -Version: 0.9.0 -``` - -> **Note:** -> -> - You must install Ansible and its dependencies following the above procedures. -> - Make sure that the Jinja2 version is correct, otherwise an error occurs when you start Grafana. -> - Make sure that the jmespath version is correct, otherwise an error occurs when you perform a rolling update to TiKV. - -## Step 2: Download TiDB Ansible to the control machine - -1. Log in to the control machine using the `tidb` user account and enter the `/home/tidb` directory. - -2. Back up the `tidb-ansible` folders of TiDB 2.0, 2.1, 3.0, or an earlier `latest` version using the following command: - - {{< copyable "shell-regular" >}} - - ```bash - mv tidb-ansible tidb-ansible-bak - ``` - -3. Download the tidb-ansible with the tag corresponding to the `latest` version of TiDB. For more details, See [Download TiDB-Ansible to the control machine](/online-deployment-using-ansible.md#step-3-download-tidb-ansible-to-the-control-machine). The default folder name is `tidb-ansible`. - - {{< copyable "shell-regular" >}} - - ```bash - git clone https://github.com/pingcap/tidb-ansible.git - ``` - -## Step 3: Edit the `inventory.ini` file and the configuration file - -Log in to the control machine using the `tidb` user account and enter the `/home/tidb/tidb-ansible` directory. - -### Edit the `inventory.ini` file - -Edit the `inventory.ini` file. For IP information, see the `/home/tidb/tidb-ansible-bak/inventory.ini` backup file. - -> **Note:** -> -> Pay special attention to the following variables configuration. For variable meaning, see [Description of other variables](/online-deployment-using-ansible.md#edit-other-variables-optional). - -1. Make sure that `ansible_user` is the normal user. For unified privilege management, remote installation using the root user is no longer supported. The default configuration uses the `tidb` user as the SSH remote user and the program running user. - - ``` - ## Connection - # ssh via normal user - ansible_user = tidb - ``` - - You can refer to [How to configure SSH mutual trust and sudo rules on the control machine](/online-deployment-using-ansible.md#step-5-configure-the-ssh-mutual-trust-and-sudo-rules-on-the-control-machine) to automatically configure the mutual trust among hosts. - -2. Keep the `process_supervision` variable consistent with that in the previous version. It is recommended to use `systemd` by default. - - ``` - # process supervision, [systemd, supervise] - process_supervision = systemd - ``` - - If you need to modify this variable, see [How to modify the supervision method of a process from `supervise` to `systemd`](/online-deployment-using-ansible.md#how-to-modify-the-supervision-method-of-a-process-from-supervise-to-systemd). Before you upgrade, first use the `/home/tidb/tidb-ansible-bak/` backup branch to modify the supervision method of a process. - -3. Add the `cpu_architecture` parameter according to the CPU architecture. The default value is `amd64`. - -### Edit the configuration file of TiDB cluster components - -If you have previously customized the configuration file of TiDB cluster components, refer to the backup file to modify the corresponding configuration file in the `/home/tidb/tidb-ansible/conf` directory. - -**Note the following parameter changes:** - -- In the TiKV configuration, `end-point-concurrency` is changed to three parameters: `high-concurrency`, `normal-concurrency` and `low-concurrency`. - - ```yaml - readpool: - coprocessor: - # Notice: if CPU_NUM > 8, default thread pool size for coprocessors - # will be set to CPU_NUM * 0.8. - # high-concurrency: 8 - # normal-concurrency: 8 - # low-concurrency: 8 - ``` - - > **Note:** - > - > For the cluster topology of multiple TiKV instances (processes) on a single machine, you need to modify the three parameters above. - - Recommended configuration: the number of TiKV instances \* the parameter value = the number of CPU cores \* 0.8. - -- In the TiKV configuration, the `block-cache-size` parameter of different CFs is changed to `block-cache`. - - ``` - storage: - block-cache: - capacity: "1GB" - ``` - - > **Note:** - > - > For the cluster topology of multiple TiKV instances (processes) on a single machine, you need to modify the `capacity` parameter. - - Recommended configuration: `capacity` = MEM_TOTAL \* 0.5 / the number of TiKV instances. If the `capacity` parameter is configured reasonably in the current version, it needs no modification. - -- In the TiKV configuration, you need to configure the `tikv_status_port` port for the multiple instances on a single machine scenario. If the current version has the above configuration, it needs no modification. -- Before you configure `tikv_status_port`, check whether a port conflict exists. - - ``` - [tikv_servers] - TiKV1-1 ansible_host=172.16.10.4 deploy_dir=/data1/deploy tikv_port=20171 tikv_status_port=20181 labels="host=tikv1" - TiKV1-2 ansible_host=172.16.10.4 deploy_dir=/data2/deploy tikv_port=20172 tikv_status_port=20182 labels="host=tikv1" - TiKV2-1 ansible_host=172.16.10.5 deploy_dir=/data1/deploy tikv_port=20171 tikv_status_port=20181 labels="host=tikv2" - TiKV2-2 ansible_host=172.16.10.5 deploy_dir=/data2/deploy tikv_port=20172 tikv_status_port=20182 labels="host=tikv2" - TiKV3-1 ansible_host=172.16.10.6 deploy_dir=/data1/deploy tikv_port=20171 tikv_status_port=20181 labels="host=tikv3" - TiKV3-2 ansible_host=172.16.10.6 deploy_dir=/data2/deploy tikv_port=20172 tikv_status_port=20182 labels="host=tikv3" - ``` - -## Step 4: Download TiDB latest binary to the control machine - -Make sure that `tidb_version = latest` in the `tidb-ansible/inventory.ini` file, and then run the following command to download TiDB latest binary to the control machine: - -{{< copyable "shell-regular" >}} - -```bash -ansible-playbook local_prepare.yml -``` - -## Step 5: Perform a rolling update to TiDB cluster components - -- If the `process_supervision` variable uses the default `systemd` parameter, perform a rolling update to the TiDB cluster using the following command corresponding to your current TiDB cluster version. - - - When the TiDB cluster version < 3.0.0, use `excessive_rolling_update.yml`. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook excessive_rolling_update.yml - ``` - - - When the TiDB cluster version ≧ 3.0.0, use `rolling_update.yml` for both rolling updates and daily rolling restarts. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook rolling_update.yml - ``` - -- If the `process_supervision` variable uses the `supervise` parameter, perform a rolling update to the TiDB cluster using `rolling_update.yml`, no matter what version the current TiDB cluster is. - - {{< copyable "shell-regular" >}} - - ```bash - ansible-playbook rolling_update.yml - ``` - -## Step 6: Perform a rolling update to TiDB monitoring components - -{{< copyable "shell-regular" >}} - -```bash -ansible-playbook rolling_update_monitor.yml -``` - -> **Note:** -> -> By default, TiDB (starting from v4.0.2) periodically shares usage details with PingCAP to help understand how to improve the product. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). diff --git a/upgrade-tidb-using-tiup-offline.md b/upgrade-tidb-using-tiup-offline.md deleted file mode 100644 index 07374ea8859b..000000000000 --- a/upgrade-tidb-using-tiup-offline.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: Upgrade TiDB Using TiUP Offline Mirror -summary: Learn how to upgrade TiDB using the TiUP offline mirror. -aliases: ['/docs/dev/upgrade-tidb-using-tiup-offline/'] ---- - -# Upgrade TiDB Using TiUP Offline Mirror - -This document describes how to upgrade TiDB using TiUP offline mirror. The upgrade steps are as follows. - -## Update TiUP offline mirror - -1. To update the local TiUP offline mirror, refer to Step 1 and Step 2 in [Deploy a TiDB Cluster Offline Using TiUP](/production-offline-deployment-using-tiup.md) to download and deploy the new version of the TiUP offline mirror. - - After you execute `local_install.sh`, TiUP completes the overwrite and upgrade. - -2. The offline mirror is successfully updated. After the overwrite, if an error is reported when TiUP is running, it might be that `manifest` is not updated. To fix this, you can execute `rm -rf ~/.tiup/manifests` before using TiUP. - -## Upgrade TiDB cluster - -After the local mirror is updated, refer to [Upgrade TiDB Using TiUP](/upgrade-tidb-using-tiup.md) to upgrade the TiDB cluster. - -> **Note:** -> -> By default, TiUP and TiDB (starting from v4.0.2) share usage details with PingCAP to help understand how to improve the product. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). diff --git a/upgrade-tidb-using-tiup.md b/upgrade-tidb-using-tiup.md index 3dc45cd204ad..0c49c9d5b49c 100644 --- a/upgrade-tidb-using-tiup.md +++ b/upgrade-tidb-using-tiup.md @@ -1,166 +1,139 @@ --- title: Upgrade TiDB Using TiUP summary: Learn how to upgrade TiDB using TiUP. -aliases: ['/docs/dev/upgrade-tidb-using-tiup/','/docs/dev/how-to/upgrade/using-tiup/'] +aliases: ['/docs/dev/upgrade-tidb-using-tiup/','/docs/dev/how-to/upgrade/using-tiup/','/tidb/dev/upgrade-tidb-using-tiup-offline','/docs/dev/upgrade-tidb-using-tiup-offline/'] --- # Upgrade TiDB Using TiUP -This document is targeted for users who want to upgrade from TiDB 3.0 or 3.1 versions to TiDB 4.0 versions, or from TiDB 4.0 to a later version. +This document is targeted for the following upgrade paths: -If you have deployed the TiDB cluster using TiDB Ansible, you can use TiUP to import the TiDB Ansible configuration and perform the upgrade. +- Upgrade from TiDB 4.0 versions to TiDB 5.1 versions. +- Upgrade from TiDB 5.0 versions to TiDB 5.1 versions. + +> **Note:** +> +> If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v5.1 or its patch versions is not supported. You need to upgrade your cluster first to v4.0 and then to v5.1. ## Upgrade caveat -- After the upgrade, rolling back to 3.0 or earlier versions is not supported. -- To update versions earlier than 3.0 to 4.0, first update this version to 3.0 using TiDB Ansible, and use TiUP to import the TiDB Ansible configuration and update the 3.0 version to 4.0. -- After the TiDB Ansible configuration is imported into and managed by TiUP, you can no longer operate on the cluster using TiDB Ansible. Otherwise, conflicts might occur because of the inconsistent metadata. -- Currently, you cannot import the TiDB Ansible configuration if the cluster deployed using TiDB Ansible meets one of the following situations: - - The TLS encryption is enabled for the cluster. - - This is a pure key-value cluster (cluster with no TiDB instance). - - `Kafka` is enabled for the cluster. - - `Spark` is enabled for the cluster. - - `Lightning` / `Importer` is enabled for the cluster. - - You still use the `'push'` method to collect monitoring metrics (since v3.0, `pull` is the default mode, which is supported if you have not modified this mode). - - In the `inventory.ini` configuration file, the `node_exporter` or `blackbox_exporter` item of the machine is set to non-default ports through `node_exporter_port` or `blackbox_exporter_port`, which is compatible if you have unified the configuration in the `group_vars` directory. +- TiDB currently does not support version downgrade or rolling back to an earlier version after the upgrade. +- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v5.1 or its patch versions according to this document. +- To update versions earlier than 3.0 to 5.1: + 1. Update this version to 3.0 using [TiDB Ansible](https://docs.pingcap.com/tidb/v3.0/upgrade-tidb-using-ansible). + 2. Use TiUP (`tiup cluster`) to import the TiDB Ansible configuration. + 3. Update the 3.0 version to 4.0 according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). + 4. Upgrade the cluster to v5.1 according to this document. - Support upgrading the versions of TiDB Binlog, TiCDC, TiFlash, and other components. -- Before you upgrade from v2.0.6 or earlier to v4.0.0 or later, you must make sure that no DDL operations are running in the cluster, especially the `Add Index` operation that is time-consuming. Perform the upgrade after all DDL operations are completed. -- Starting from v2.1, TiDB enables parallel DDL. Therefore, clusters **older than v2.0.1** cannot be upgraded to v4.0.0 or later via a direct rolling upgrade. Instead, you can choose one of the following solutions: - - Upgrade directly from TiDB v2.0.1 or earlier to v4.0.0 or later in planned downtime - - Perform a rolling upgrade from the current version to v2.0.1 or a later 2.0 version, then perform another rolling upgrade to v4.0.0 or later +- For detailed compatibility changes of different versions, see the [Release Notes](/releases/release-notes.md) of each version. Modify your cluster configuration according to the "Compatibility Changes" section of the corresponding release notes. > **Note:** > > Do not execute any DDL request during the upgrade, otherwise an undefined behavior issue might occur. -## Install TiUP on the control machine +## Preparations -1. Execute the following command on the control machine to install TiUP: +This section introduces the preparation works needed before upgrading your TiDB cluster, including upgrading TiUP and the TiUP Cluster component. - {{< copyable "shell-regular" >}} - - ```shell - curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh - ``` +### Step 1: Upgrade TiUP or TiUP offline mirror -2. Redeclare the global environment variables: +Before upgrading your TiDB cluster, you first need to upgrade TiUP or TiUP mirror. - {{< copyable "shell-regular" >}} +#### Upgrade TiUP and TiUP Cluster - ```shell - source .bash_profile - ``` +> **Note:** +> +> If the control machine of the cluster to upgrade cannot access `https://tiup-mirrors.pingcap.com`, skip this section and see [Upgrade TiUP offline mirror](#upgrade-tiup-offline-mirror). -3. Check whether TiUP is installed: +1. Upgrade the TiUP version. It is recommended that the TiUP version is `1.5.0` or later. {{< copyable "shell-regular" >}} ```shell - which tiup + tiup update --self + tiup --version ``` -4. Install the TiUP cluster tool: +2. Upgrade the TiUP Cluster version. It is recommended that the TiUP Cluster version is `1.5.0` or later. {{< copyable "shell-regular" >}} ```shell - tiup cluster + tiup update cluster + tiup cluster --version ``` -If you have installed TiUP before, execute the following command to update TiUP to the latest version: +#### Upgrade TiUP offline mirror > **Note:** > -> If the result of `tiup --version` shows that your TiUP version is earlier than v1.0.0, run `tiup update --self` first to update the TiUP version before running the following command. +> If the cluster to upgrade was deployed not using the offline method, skip this step. + +Refer to [Deploy a TiDB Cluster Using TiUP - Deploy TiUP offline](/production-deployment-using-tiup.md#method-2-deploy-tiup-offline) to download the TiUP mirror of the new version and upload it to the control machine. After executing `local_install.sh`, TiUP will complete the overwrite upgrade. {{< copyable "shell-regular" >}} ```shell -tiup update cluster +tar xzvf tidb-community-server-${version}-linux-amd64.tar.gz +sh tidb-community-server-${version}-linux-amd64/local_install.sh +source /home/tidb/.bash_profile ``` -## Import TiDB Ansible and the `inventory.ini` configuration to TiUP +After the overwrite upgrade, execute the following command to upgrade the TiUP Cluster component. -> **Note:** -> -> + Currently, TiUP only supports the `systemd` supervision method of a process. If you have previously selected the `supervise` method when deploying TiDB with TiDB Ansible, you need to modify the supervision method from `supervise` to `systemd` according to [Deploy TiDB Using TiDB Ansible](/online-deployment-using-ansible.md#how-to-modify-the-supervision-method-of-a-process-from-supervise-to-systemd). -> + If the original cluster is deployed using TiUP, you can skip this step. -> + Currently, the `inventory.ini` configuration file is identified by default. If your configuration file uses another name, specify this name. -> + Ensure that the state of the current cluster is consistent with the topology in `inventory.ini`; that components of the cluster are operating normally. Otherwise, the cluster metadata becomes abnormal after the import. -> + If multiple different `inventory.ini` files and TiDB clusters are managed in one TiDB Ansible directory, when importing one of the clusters into TiUP, you need to specify `--no-backup` to avoid moving the Ansible directory to the TiUP management directory. +{{< copyable "shell-regular" >}} -### Import the TiDB Ansible cluster to TiUP +```shell +tiup update cluster +``` -1. Execute the following command to import the TiDB Ansible cluster into TiUP (for example, in the `/home/tidb/tidb-ansible` path). +Now, the offline mirror has been upgraded successfully. If an error occurs during TiUP operation after the overwriting, it might be that the `manifest` is not updated. You can try `rm -rf ~/.tiup/manifests/*` before running TiUP again. - {{< copyable "shell-regular" >}} +### Step 2: Edit TiUP topology configuration file - ```shell - tiup cluster import -d /home/tidb/tidb-ansible - ``` +> **Note:** +> +> Skip this step if one of the following situations applies: +> +> + You have not modified the configuration parameters of the original cluster. Or you have modified the configuration parameters using `tiup cluster` but no more modification is needed. +> + After the upgrade, you want to use v5.1's default parameter values for the unmodified configuration items. -2. After executing the above import command, if the `Inventory` information of the cluster is parsed successfully, the following prompt appears: +1. Enter the `vi` editing mode to edit the topology file: {{< copyable "shell-regular" >}} ```shell - tiup cluster import -d /home/tidb/tidb-ansible/ - ``` - - ``` - Found inventory file /home/tidb/tidb-ansible/inventory.ini, parsing... - Found cluster "ansible-cluster" (v3.0.12), deployed with user tidb. - Prepared to import TiDB v3.0.12 cluster ansible-cluster. - Do you want to continue? [y/N]: + tiup cluster edit-config ``` -3. After checking that the parsed cluster name and the version are correct, enter `y` to continue the import process. - - + If an error occurs when parsing the `Inventory` information, the import process is stopped, which does not have any impact on the original Ansible deployment method. Then you need to adjust and retry the process according to the error prompt. - - + If the original cluster name in Ansible is the same with any existing cluster name in TiUP, a warning message is returned with a new cluster name. Therefore, **do not repeatedly import the same cluster**, which results in multiple names for the same cluster in TiUP. +2. Refer to the format of [topology](https://github.com/pingcap/tiup/blob/release-1.4/embed/templates/examples/topology.example.yaml) configuration template and fill the parameters you want to modify in the `server_configs` section of the topology file. -After the import is complete, you can check the current cluster status by executing the `tiup cluster display ` command to verify the import result. Because the `display` command is used to query the real-time status of each node, it might take a little time to execute the command. - -### Edit TiUP topology configuration file +3. After the modification, enter : + w + q to save the change and exit the editing mode. Enter Y to confirm the change. > **Note:** > -> You can skip this step for the following situations: +> Before you upgrade the cluster to v5.1, make sure that the parameters you have modified in v4.0 are compatible in v5.1. For details, see [TiKV Configuration File](/tikv-configuration-file.md). +> +> The following three TiKV parameters are obsolete in TiDB v5.1. If the following parameters have been configured in your original cluster, you need to delete these parameters through `edit-config`: > -> - The configuration parameters in the original cluster have not been modified. -> - You want to use the default parameters of `4.0` after the upgrade. +> - pessimistic-txn.enabled +> - server.request-batch-enable-cross-command +> - server.request-batch-wait-duration -1. Enter `~/.tiup/storage/cluster/clusters/{cluster_name}/config`, the backup directory of TiDB Ansible, and confirm the modified parameters in the configuration template. +### Step 3: Check the health status of the current cluster -2. Enter the `vi` editing mode of the topology file: +To avoid the undefined behaviors or other issues during the upgrade, it is recommended to check the health status of Regions of the current cluster before the upgrade. To do that, you can use the `check` sub-command. - {{< copyable "shell-regular" >}} - - ```shell - tiup cluster edit-config - ``` +{{< copyable "shell-regular" >}} -3. See the configuration template format of [topology](https://github.com/pingcap/tiup/blob/master/examples/topology.example.yaml) and fill in the modified parameters of the original cluster in the `server_configs` section of the topology file. +```shell +tiup cluster check --cluster +``` - Even if the label has been configured for the cluster, you also need to fill in the label in the configuration according to the format in the template. In later versions, the label will be automatically imported. +After the command is executed, the "Region status" check result will be output. - After the modification is completed, execute the `wq` command to save the change and exit the editing mode. Enter `Y` to confirm the change. - -> **Note:** -> -> Before upgrading to v4.0, confirm that the parameters modified in v3.0 are compatible in v4.0. See [configuration template](/tikv-configuration-file.md) for details. -> -> If the TiUP version <= v1.0.8, TiUP might not correctly obtain the data directory of TiFlash, and you need to check whether `data_dir` and `path` configured in TiFlash is consistent. If not, configure `data_dir` of TiFlash to the same value as `path` by taking the following steps: -> -> 1. Execute `tiup cluster edit-config ` to modify the configuration file. -> 2. Modify the corresponding `data_dir` value of TiFlash: -> -> ```yaml -> tiflash_servers: -> - host: 10.0.1.14 -> data_dir: /data/tiflash-11315 # Modify it to the `path` value of the TiFlash configuration file -> ``` ++ If the result is "All Regions are healthy", all Regions in the current cluster are healthy and you can continue the upgrade. ++ If the result is "Regions are not fully healthy: m miss-peer, n pending-peer" with the "Please fix unhealthy regions before other operations." prompt, some Regions in the current cluster are abnormal. You need to troubleshoot the anomalies until the check result becomes "All Regions are healthy". Then you can continue the upgrade. ## Perform a rolling upgrade to the TiDB cluster @@ -168,25 +141,61 @@ This section describes how to perform a rolling upgrade to the TiDB cluster and ### Upgrade the TiDB cluster to a specified version +You can upgrade your cluster in one of the two ways: online upgrade and offline upgrade. + +By default, TiUP Cluster upgrades the TiDB cluster using the online method, which means that the TiDB cluster can still provide services during the upgrade process. With the online method, the leaders are migrated one by one on each node before the upgrade and restart. Therefore, for a large-scale cluster, it takes a long time to complete the entire upgrade operation. + +If your application has a maintenance window for the database to be stopped for maintenance, you can use the offline upgrade method to quickly perform the upgrade operation. + +#### Online upgrade + {{< copyable "shell-regular" >}} ```shell tiup cluster upgrade ``` -For example, if you want to upgrade the cluster to v4.0.0: +For example, if you want to upgrade the cluster to v5.1.0: {{< copyable "shell-regular" >}} ```shell -tiup cluster upgrade v4.0.0 +tiup cluster upgrade v5.1.0 ``` -Performing a rolling upgrade to the cluster will upgrade all components one by one. During the upgrade of TiKV, all leaders in a TiKV instance are evicted before stopping the instance. The default timeout time is 5 minutes. The instance is directly stopped after this timeout time. +> **Note:** +> +> + Performing a rolling upgrade to the cluster will upgrade all components one by one. During the upgrade of TiKV, all leaders in a TiKV instance are evicted before stopping the instance. The default timeout time is 5 minutes (300 seconds). The instance is directly stopped after this timeout time. +> +> + To perform the upgrade immediately without evicting the leader, specify `--force` in the command above. This method causes performance jitter but not data loss. +> +> + To keep a stable performance, make sure that all leaders in a TiKV instance are evicted before stopping the instance. You can set `--transfer-timeout` to a larger value, for example, `--transfer-timeout 3600` (unit: second). + +#### Offline upgrade -To perform the upgrade immediately without evicting the leader, specify `--force` in the command above. This method causes performance jitter but not data loss. +1. Before the offline upgrade, you first need to stop the entire cluster. -To keep a stable performance, make sure that all leaders in a TiKV instance are evicted before stopping the instance. You can set `--transfer-timeout` to a super large value, for example, `--transfer-timeout 100000000` (unit: second). + {{< copyable "shell-regular" >}} + + ```shell + tiup cluster stop + ``` + +2. Use the `upgrade` command with the `--offline` option to perform the offline upgrade. + + {{< copyable "shell-regular" >}} + + ```shell + tiup cluster upgrade --offline + ``` + +3. After the upgrade, the cluster will not be automatically restarted. You need to use the `start` command to restart it. + + {{< copyable "shell-regular" >}} + + ```shell + tiup cluster start + ``` ### Verify the cluster version @@ -199,14 +208,14 @@ tiup cluster display ``` ``` -Starting /home/tidblk/.tiup/components/cluster/v1.0.0/cluster display -TiDB Cluster: -TiDB Version: v4.0.0 +Cluster type: tidb +Cluster name: +Cluster version: v5.1.0 ``` > **Note:** > -> By default, TiUP and TiDB (starting from v4.0.2) share usage details with PingCAP to help understand how to improve the product. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). +> By default, TiUP and TiDB share usage details with PingCAP to help understand how to improve the product. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). ## FAQ @@ -214,7 +223,25 @@ This section describes common problems encountered when updating the TiDB cluste ### If an error occurs and the upgrade is interrupted, how to resume the upgrade after fixing this error? -Re-execute the `tiup cluster upgrade` command to resume the upgrade. The upgrade operation restarts the nodes that have been previously upgraded. In subsequent 4.0 versions, TiDB will support resuming the upgrade from the interrupted point. +Re-execute the `tiup cluster upgrade` command to resume the upgrade. The upgrade operation restarts the nodes that have been previously upgraded. If you do not want the upgraded nodes to be restarted, use the `replay` sub-command to retry the operation: + +1. Execute `tiup cluster audit` to see the operation records: + + {{< copyable "shell-regular" >}} + + ```shell + tiup cluster audit + ``` + + Find the failed upgrade operation record and keep the ID of this operation record. The ID is the `` value in the next step. + +2. Execute `tiup cluster replay ` to retry the corresponding operation: + + {{< copyable "shell-regular" >}} + + ```shell + tiup cluster replay + ``` ### The evict leader has waited too long during the upgrade. How to skip this step for a quick upgrade? @@ -223,29 +250,20 @@ You can specify `--force`. Then the processes of transferring PD leader and evic {{< copyable "shell-regular" >}} ```shell -tiup cluster upgrade v4.0.0 --force +tiup cluster upgrade --force ``` ### How to update the version of tools such as pd-ctl after upgrading the TiDB cluster? -Currently, TiUP does not update and manage the version of tools. If you need the tool package of the latest version, directly download the TiDB package and replace `{version}` with the corresponding version such as `v4.0.0`. Here is the download address: +You can upgrade the tool version by using TiUP to install the `ctl` component of the corresponding version: -{{< copyable "" >}} +{{< copyable "shell-regular" >}} +```shell +tiup install ctl:v5.1.0 ``` -https://download.pingcap.org/tidb-{version}-linux-amd64.tar.gz -``` - -### Failure to upgrade the TiFlash component during the cluster upgrade - -Before v4.0.0-rc.2, TiFlash might have some incompatibility issues. This might cause problems when you upgrade a cluster that includes the TiFlash component to v4.0.0-rc.2 or a later version. If so, [contact the R&D support](mailto:support@pingcap.com). -## TiDB 4.0 compatibility changes +## TiDB 5.1 compatibility changes -- If you set the value of the `oom-action` parameter to `cancel`, when the query statement triggers the OOM threshold, the statement is killed. In v4.0, in addition to `select`, DML statements such as `insert`/`update`/`delete` might also be killed. -- TiDB v4.0 supports the length check for table names. The length limit is 64 characters. If you rename a table after the upgrade and the new name exceeds this length limit, an error is reported. v3.0 and earlier versions do not have this error reporting. -- TiDB v4.0 supports the length check for partition names of the partitioned tables. The length limit is 64 characters. After the upgrade, if you create or alter a partitioned table with a partition name that exceeds the length limit, an error is expected to occur in 4.0 versions, but not in 3.0 and earlier versions. -- In v4.0, the format of the `explain` execution plan is improved. Pay attention to any automatic analysis program that is customized for `explain`. -- TiDB v4.0 supports [Read Committed isolation level](/transaction-isolation-levels.md#read-committed-isolation-level). After upgrading to v4.0, setting the isolation level to `READ-COMMITTED` in a pessimistic transaction takes effect. In 3.0 and earlier versions, the setting does not take effect. -- In v4.0, executing `alter reorganize partition` returns an error. In earlier versions, no error is reported because only the syntax is supported and the statement is not taking any effect. -- In v4.0, creating `linear hash partition` or `subpartition` tables does not take effect and they are converted to regular tables. In earlier versions, they are converted to regular partitioned tables. +- See TiDB 5.1 Release Notes for the compatibility changes. +- Try to avoid creating a new clustered index table when you apply rolling updates to the clusters using TiDB-Binlog. diff --git a/user-defined-variables.md b/user-defined-variables.md index 0152254a783e..43bd94f8b7bd 100644 --- a/user-defined-variables.md +++ b/user-defined-variables.md @@ -86,7 +86,7 @@ SELECT @a1, @a2, @a3, @a4 := @a1+@a2+@a3; +------+------+------+--------------------+ ``` -Before the variable `@a4` is modified or the connection is closed, its value is always `7`。 +Before the variable `@a4` is modified or the connection is closed, its value is always `7`. If a hexadecimal literal or binary literal is used when setting the user-defined variable, TiDB will treat it as a binary string. If you want to set it to a number, you can manually add the `CAST` conversion, or use the numeric operator in the expression: @@ -179,4 +179,8 @@ SELECT @col FROM t; +------+ ``` +## MySQL compatibility + +Except for `SELECT ... INTO `, the syntax supported in MySQL and TiDB is identical. + For more information, see [User-Defined Variables in MySQL](https://dev.mysql.com/doc/refman/5.7/en/user-variables.html). diff --git a/whats-new-in-tidb-4.0.md b/whats-new-in-tidb-4.0.md index 191866170571..f69740970193 100644 --- a/whats-new-in-tidb-4.0.md +++ b/whats-new-in-tidb-4.0.md @@ -52,7 +52,7 @@ TiUP is a new package manager tool introduced in v4.0 that is used to manage all - Add the `FLASHBACK` statement to support recovering the truncated tables. See [`Flashback Table`](/sql-statements/sql-statement-flashback-table.md) for details. - Support writing the intermediate results of Join and Sort to the local disk when you make queries, which avoids the Out of Memory (OOM) issue because the queries occupy excessive memory. This also improves system stability. - Optimize the output of `EXPLAIN` and `EXPLAIN ANALYZE`. More information is shown in the result, which improves troubleshooting efficiency. See [Explain Analyze](/sql-statements/sql-statement-explain-analyze.md) and [Explain](/sql-statements/sql-statement-explain.md) for details. -- Support using the Index Merge feature to access tables. When you make a query on a single table, the TiDB optimizer automatically reads multiple index data according to the query condition and makes a union of the result, which improves the performance of querying on a single table. See [Index Merge](/explain-overview.md#indexmerge-example) for details. +- Support using the Index Merge feature to access tables. When you make a query on a single table, the TiDB optimizer automatically reads multiple index data according to the query condition and makes a union of the result, which improves the performance of querying on a single table. See [Index Merge](/explain-index-merge.md) for details. - Support `AUTO_RANDOM` keys as an extended syntax for the TiDB columnar attribute. `AUTO_RANDOM` is designed to address the hotspot issue caused by the auto-increment column and provides a low-cost migration solution from MySQL for users who work with auto-increment columns. See [`AUTO_RANDOM` Key](/auto-random.md) for details. - Add system tables that provide information of cluster topology, configuration, logs, hardware, operating systems, and slow queries, which helps DBAs to quickly learn, analyze system metrics. See [Information Schema](/information-schema/information-schema.md) and [SQL Diagnosis](/information-schema/information-schema-sql-diagnostics.md) for details.
LoaderTiDB Lightning