diff --git a/.gitignore b/.gitignore index 72fc0c9b645a89..0a8837eb3cc738 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,9 @@ be/output be/build output -docs/build +docs/.temp +docs/.vuepress/dist +docs/node_modules gensrc/build fe/target thirdparty/src diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000000..cf4b04f9bf141d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +language: node_js +# nodejs版本 +node_js: + - '8' + +# Travis-CI Caching +cache: + directories: + - docs/node_modules + + +# S: Build Lifecycle +install: + - cd docs && npm install + +before_script: + - export PR=https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/$TRAVIS_PULL_REQUEST + - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo `curl -s $PR | jq -r .head.ref`; fi) + - echo $BRANCH + - sed -i 's/base:.*,/base:\"\/'$BRANCH'\/\",/g' .vuepress/config.js + - sed -i 's/docsBranch:.*,/docsBranch:\"'$BRANCH'\",/g' .vuepress/config.js + - rm -rf site-repo + +script: + - npm run build + +after_script: + - if [ "$TRAVIS_EVENT_TYPE" != "push" ]; then exit 0; fi + - git config user.name "${GIT_NAME}" + - git config user.email "${GIT_EMAIL}" + - git clone https://${SITE_REPO} site-repo + - cd site-repo + - mkdir -p ${BRANCH} && rm -rf ${BRANCH}/* + - cp -r ../.vuepress/dist/* ./${BRANCH}/ + - git checkout ${SITE_BRANCH} + - git add . + - git commit -am "Auto Build" + - git push --force --quiet "https://${SITE_PUSH_TOKEN}@${SITE_REPO}" ${SITE_BRANCH}:${SITE_BRANCH} + +branches: + only: + - master + - /^branch-.*$/ diff --git a/LICENSE.txt b/LICENSE.txt index 571e94b6c87c4c..1dabf41b6576db 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -630,3 +630,27 @@ be/src/util/condition_variable* : BSD-style license Copyright (c) 2011 The Chromium Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. + +-------------------------------------------------------------------------------- + +docs/.vuepress/* The MIT License (MIT) + +Copyright (c) 2018-present, Yuxi (Evan) You + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/docs/.markdownlint.yml b/docs/.markdownlint.yml new file mode 100644 index 00000000000000..f01d39144fb1fb --- /dev/null +++ b/docs/.markdownlint.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "default": true, + "MD013": false, +} diff --git a/docs/.markdownlintignore b/docs/.markdownlintignore new file mode 100644 index 00000000000000..1aea57f9f29788 --- /dev/null +++ b/docs/.markdownlintignore @@ -0,0 +1,2 @@ +node_modules +.vuepress diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js new file mode 100644 index 00000000000000..d8f76a99f66014 --- /dev/null +++ b/docs/.vuepress/config.js @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function convertSidebar(list, path) { + if (list.length > 0) { + list.forEach((element, i) => { + if (element.children) { + convertSidebar(element.children, path + element.directoryPath) + delete element.directoryPath + } else { + list[i] = path + element + } + }); + } + return list +} + +module.exports = { + base: '', + locales: { + '/en/': { + lang: 'en', + title: 'Apache Doris', + description: 'Apache Doris' + }, + '/zh-CN/': { + lang: 'zh-CN', + title: 'Apache Doris', + description: 'Apache Doris' + } + }, + head: [ + ['meta', { name: 'theme-color', content: '#3eaf7c' }], + ['meta', { name: 'apple-mobile-web-app-capable', content: 'yes' }], + ['meta', { name: 'apple-mobile-web-app-status-bar-style', content: 'black' }], + ['meta', { name: 'msapplication-TileColor', content: '#000000' }] + ], + title: 'Apache Doris', + description: 'Apache Doris', + themeConfig: { + title: 'Doris', + logo: '/images/doris-logo-only.png', + search: true, + smoothScroll: true, + searchMaxSuggestions: 10, + nextLinks: true, + prevLinks: true, + repo: 'apache/incubator-doris', + repoLabel: 'GitHub', + lastUpdated: 'Last Updated', + editLinks: true, + docsDir: 'docs', + docsBranch: '', + locales: { + '/en/': { + selectText: 'Languages', + label: 'English', + ariaLabel: 'Languages', + editLinkText: 'Edit this page on GitHub', + algolia: {}, + nav: [ + { + text: 'Home', link: '/en/' + }, + { + text: 'Docs', link: '/en/installing/compilation' + }, + { + text: 'Download', link: '/en/downloads/downloads' + }, + { + text: 'Apache', link: 'https://www.apache.org/', target: '_blank' + } + ], + sidebar: convertSidebar(require('./sidebar/en.js'), '/en/') + }, + '/zh-CN/': { + selectText: '选择语言', + label: '简体中文', + editLinkText: '在 GitHub 上编辑此页', + nav: [ + { + text: '主页', link: '/zh-CN/' + }, + { + text: '文档', link: '/zh-CN/installing/compilation' + }, + { + text: '下载', link: '/zh-CN/downloads/downloads' + }, + { + text: 'Apache', link: 'https://www.apache.org/', target: '_blank' + } + ], + algolia: {}, + sidebar: { + '/zh-CN/': convertSidebar(require('./sidebar/zh-CN.js'), '/zh-CN/') + } + } + } + }, + plugins: [ + 'reading-progress', 'plugin-back-to-top', 'plugin-medium-zoom' + ] +}; diff --git a/docs/.vuepress/public/favicon.ico b/docs/.vuepress/public/favicon.ico new file mode 100644 index 00000000000000..c79ff458a207a9 Binary files /dev/null and b/docs/.vuepress/public/favicon.ico differ diff --git a/docs/resources/images/apache-asf-compressed.png b/docs/.vuepress/public/images/apache-asf-compressed.png similarity index 100% rename from docs/resources/images/apache-asf-compressed.png rename to docs/.vuepress/public/images/apache-asf-compressed.png diff --git a/docs/resources/images/apache-incubator-logo.png b/docs/.vuepress/public/images/apache-incubator-logo.png similarity index 100% rename from docs/resources/images/apache-incubator-logo.png rename to docs/.vuepress/public/images/apache-incubator-logo.png diff --git a/docs/resources/images/apache-incubator.png b/docs/.vuepress/public/images/apache-incubator.png similarity index 100% rename from docs/resources/images/apache-incubator.png rename to docs/.vuepress/public/images/apache-incubator.png diff --git a/docs/resources/images/apache_incubator_logo.png b/docs/.vuepress/public/images/apache_incubator_logo.png similarity index 100% rename from docs/resources/images/apache_incubator_logo.png rename to docs/.vuepress/public/images/apache_incubator_logo.png diff --git a/docs/resources/images/architecture.png b/docs/.vuepress/public/images/architecture.png similarity index 100% rename from docs/resources/images/architecture.png rename to docs/.vuepress/public/images/architecture.png diff --git a/docs/resources/images/asf_logo_wide_small.png b/docs/.vuepress/public/images/asf_logo_wide_small.png similarity index 100% rename from docs/resources/images/asf_logo_wide_small.png rename to docs/.vuepress/public/images/asf_logo_wide_small.png diff --git a/docs/resources/images/backend_state.png b/docs/.vuepress/public/images/backend_state.png similarity index 100% rename from docs/resources/images/backend_state.png rename to docs/.vuepress/public/images/backend_state.png diff --git a/docs/resources/images/cluster_link_and_migrate_db.png b/docs/.vuepress/public/images/cluster_link_and_migrate_db.png similarity index 100% rename from docs/resources/images/cluster_link_and_migrate_db.png rename to docs/.vuepress/public/images/cluster_link_and_migrate_db.png diff --git a/docs/resources/images/cluster_namaspace.png b/docs/.vuepress/public/images/cluster_namaspace.png similarity index 100% rename from docs/resources/images/cluster_namaspace.png rename to docs/.vuepress/public/images/cluster_namaspace.png diff --git a/docs/resources/images/cpu-flame-demo.svg b/docs/.vuepress/public/images/cpu-flame-demo.svg similarity index 100% rename from docs/resources/images/cpu-flame-demo.svg rename to docs/.vuepress/public/images/cpu-flame-demo.svg diff --git a/docs/resources/images/cpu-pprof-demo.png b/docs/.vuepress/public/images/cpu-pprof-demo.png similarity index 100% rename from docs/resources/images/cpu-pprof-demo.png rename to docs/.vuepress/public/images/cpu-pprof-demo.png diff --git a/docs/resources/images/cpu-pprof-demo.svg b/docs/.vuepress/public/images/cpu-pprof-demo.svg similarity index 100% rename from docs/resources/images/cpu-pprof-demo.svg rename to docs/.vuepress/public/images/cpu-pprof-demo.svg diff --git a/docs/resources/images/create-pr.png b/docs/.vuepress/public/images/create-pr.png similarity index 100% rename from docs/resources/images/create-pr.png rename to docs/.vuepress/public/images/create-pr.png diff --git a/docs/resources/images/create-pr2.png b/docs/.vuepress/public/images/create-pr2.png similarity index 100% rename from docs/resources/images/create-pr2.png rename to docs/.vuepress/public/images/create-pr2.png diff --git a/docs/resources/images/create-pr3.png b/docs/.vuepress/public/images/create-pr3.png similarity index 100% rename from docs/resources/images/create-pr3.png rename to docs/.vuepress/public/images/create-pr3.png diff --git a/docs/resources/images/dashboard_navibar.png b/docs/.vuepress/public/images/dashboard_navibar.png similarity index 100% rename from docs/resources/images/dashboard_navibar.png rename to docs/.vuepress/public/images/dashboard_navibar.png diff --git a/docs/resources/images/dashboard_overview.png b/docs/.vuepress/public/images/dashboard_overview.png similarity index 100% rename from docs/resources/images/dashboard_overview.png rename to docs/.vuepress/public/images/dashboard_overview.png diff --git a/docs/resources/images/dashboard_panel.png b/docs/.vuepress/public/images/dashboard_panel.png similarity index 100% rename from docs/resources/images/dashboard_panel.png rename to docs/.vuepress/public/images/dashboard_panel.png diff --git a/docs/resources/images/dashboard_row.png b/docs/.vuepress/public/images/dashboard_row.png similarity index 100% rename from docs/resources/images/dashboard_row.png rename to docs/.vuepress/public/images/dashboard_row.png diff --git a/docs/resources/images/doris-logo-1.png b/docs/.vuepress/public/images/doris-logo-1.png similarity index 100% rename from docs/resources/images/doris-logo-1.png rename to docs/.vuepress/public/images/doris-logo-1.png diff --git a/docs/resources/images/doris-logo-2.png b/docs/.vuepress/public/images/doris-logo-2.png similarity index 100% rename from docs/resources/images/doris-logo-2.png rename to docs/.vuepress/public/images/doris-logo-2.png diff --git a/docs/.vuepress/public/images/doris-logo-only.png b/docs/.vuepress/public/images/doris-logo-only.png new file mode 100644 index 00000000000000..1c0fa86c1cb493 Binary files /dev/null and b/docs/.vuepress/public/images/doris-logo-only.png differ diff --git a/docs/.vuepress/public/images/doris-logo.png b/docs/.vuepress/public/images/doris-logo.png new file mode 100644 index 00000000000000..b5660e641dcdc9 Binary files /dev/null and b/docs/.vuepress/public/images/doris-logo.png differ diff --git a/docs/resources/images/egg-logo.png b/docs/.vuepress/public/images/egg-logo.png similarity index 100% rename from docs/resources/images/egg-logo.png rename to docs/.vuepress/public/images/egg-logo.png diff --git a/docs/resources/images/egg-logo2.png b/docs/.vuepress/public/images/egg-logo2.png similarity index 100% rename from docs/resources/images/egg-logo2.png rename to docs/.vuepress/public/images/egg-logo2.png diff --git a/docs/resources/images/export_plan_tree_1.png b/docs/.vuepress/public/images/export_plan_tree_1.png similarity index 100% rename from docs/resources/images/export_plan_tree_1.png rename to docs/.vuepress/public/images/export_plan_tree_1.png diff --git a/docs/resources/images/export_plan_tree_2.png b/docs/.vuepress/public/images/export_plan_tree_2.png similarity index 100% rename from docs/resources/images/export_plan_tree_2.png rename to docs/.vuepress/public/images/export_plan_tree_2.png diff --git a/docs/resources/images/export_status_change.png b/docs/.vuepress/public/images/export_status_change.png similarity index 100% rename from docs/resources/images/export_status_change.png rename to docs/.vuepress/public/images/export_status_change.png diff --git a/docs/resources/images/fe_page_index.png b/docs/.vuepress/public/images/fe_page_index.png similarity index 100% rename from docs/resources/images/fe_page_index.png rename to docs/.vuepress/public/images/fe_page_index.png diff --git a/docs/resources/images/fe_page_logs.png b/docs/.vuepress/public/images/fe_page_logs.png similarity index 100% rename from docs/resources/images/fe_page_logs.png rename to docs/.vuepress/public/images/fe_page_logs.png diff --git a/docs/resources/images/fe_page_queries.png b/docs/.vuepress/public/images/fe_page_queries.png similarity index 100% rename from docs/resources/images/fe_page_queries.png rename to docs/.vuepress/public/images/fe_page_queries.png diff --git a/docs/resources/images/fe_page_sessions.png b/docs/.vuepress/public/images/fe_page_sessions.png similarity index 100% rename from docs/resources/images/fe_page_sessions.png rename to docs/.vuepress/public/images/fe_page_sessions.png diff --git a/docs/resources/images/fe_page_system.png b/docs/.vuepress/public/images/fe_page_system.png similarity index 100% rename from docs/resources/images/fe_page_system.png rename to docs/.vuepress/public/images/fe_page_system.png diff --git a/docs/resources/images/fe_page_system_access.png b/docs/.vuepress/public/images/fe_page_system_access.png similarity index 100% rename from docs/resources/images/fe_page_system_access.png rename to docs/.vuepress/public/images/fe_page_system_access.png diff --git a/docs/resources/images/fe_page_system_backends.png b/docs/.vuepress/public/images/fe_page_system_backends.png similarity index 100% rename from docs/resources/images/fe_page_system_backends.png rename to docs/.vuepress/public/images/fe_page_system_backends.png diff --git a/docs/resources/images/fe_page_system_brokers.png b/docs/.vuepress/public/images/fe_page_system_brokers.png similarity index 100% rename from docs/resources/images/fe_page_system_brokers.png rename to docs/.vuepress/public/images/fe_page_system_brokers.png diff --git a/docs/resources/images/fe_page_system_dbs.png b/docs/.vuepress/public/images/fe_page_system_dbs.png similarity index 100% rename from docs/resources/images/fe_page_system_dbs.png rename to docs/.vuepress/public/images/fe_page_system_dbs.png diff --git a/docs/resources/images/fe_page_system_error_hub.png b/docs/.vuepress/public/images/fe_page_system_error_hub.png similarity index 100% rename from docs/resources/images/fe_page_system_error_hub.png rename to docs/.vuepress/public/images/fe_page_system_error_hub.png diff --git a/docs/resources/images/fe_page_system_frontends.png b/docs/.vuepress/public/images/fe_page_system_frontends.png similarity index 100% rename from docs/resources/images/fe_page_system_frontends.png rename to docs/.vuepress/public/images/fe_page_system_frontends.png diff --git a/docs/resources/images/fe_page_system_jobs.png b/docs/.vuepress/public/images/fe_page_system_jobs.png similarity index 100% rename from docs/resources/images/fe_page_system_jobs.png rename to docs/.vuepress/public/images/fe_page_system_jobs.png diff --git a/docs/resources/images/fe_page_system_statistic.png b/docs/.vuepress/public/images/fe_page_system_statistic.png similarity index 100% rename from docs/resources/images/fe_page_system_statistic.png rename to docs/.vuepress/public/images/fe_page_system_statistic.png diff --git a/docs/resources/images/fe_page_system_tasks.png b/docs/.vuepress/public/images/fe_page_system_tasks.png similarity index 100% rename from docs/resources/images/fe_page_system_tasks.png rename to docs/.vuepress/public/images/fe_page_system_tasks.png diff --git a/docs/resources/images/fork-repo.png b/docs/.vuepress/public/images/fork-repo.png similarity index 100% rename from docs/resources/images/fork-repo.png rename to docs/.vuepress/public/images/fork-repo.png diff --git a/docs/resources/images/graduation-timeline.png b/docs/.vuepress/public/images/graduation-timeline.png similarity index 100% rename from docs/resources/images/graduation-timeline.png rename to docs/.vuepress/public/images/graduation-timeline.png diff --git a/docs/resources/images/howtoincubateaproject-thumb.png b/docs/.vuepress/public/images/howtoincubateaproject-thumb.png similarity index 100% rename from docs/resources/images/howtoincubateaproject-thumb.png rename to docs/.vuepress/public/images/howtoincubateaproject-thumb.png diff --git a/docs/resources/images/howtoincubateaproject.png b/docs/.vuepress/public/images/howtoincubateaproject.png similarity index 100% rename from docs/resources/images/howtoincubateaproject.png rename to docs/.vuepress/public/images/howtoincubateaproject.png diff --git a/docs/resources/images/incbuator_feather_egg_logo_crop.png b/docs/.vuepress/public/images/incbuator_feather_egg_logo_crop.png similarity index 100% rename from docs/resources/images/incbuator_feather_egg_logo_crop.png rename to docs/.vuepress/public/images/incbuator_feather_egg_logo_crop.png diff --git a/docs/resources/images/incubation-process.png b/docs/.vuepress/public/images/incubation-process.png similarity index 100% rename from docs/resources/images/incubation-process.png rename to docs/.vuepress/public/images/incubation-process.png diff --git a/docs/resources/images/incubator_ring_logo.png b/docs/.vuepress/public/images/incubator_ring_logo.png similarity index 100% rename from docs/resources/images/incubator_ring_logo.png rename to docs/.vuepress/public/images/incubator_ring_logo.png diff --git a/docs/resources/images/log_replication.jpg b/docs/.vuepress/public/images/log_replication.jpg similarity index 100% rename from docs/resources/images/log_replication.jpg rename to docs/.vuepress/public/images/log_replication.jpg diff --git a/docs/resources/images/login-gitter1.png b/docs/.vuepress/public/images/login-gitter1.png similarity index 100% rename from docs/resources/images/login-gitter1.png rename to docs/.vuepress/public/images/login-gitter1.png diff --git a/docs/resources/images/login-gitter2.PNG b/docs/.vuepress/public/images/login-gitter2.PNG similarity index 100% rename from docs/resources/images/login-gitter2.PNG rename to docs/.vuepress/public/images/login-gitter2.PNG diff --git a/docs/resources/images/metadata_contents.png b/docs/.vuepress/public/images/metadata_contents.png similarity index 100% rename from docs/resources/images/metadata_contents.png rename to docs/.vuepress/public/images/metadata_contents.png diff --git a/docs/resources/images/metadata_stream.png b/docs/.vuepress/public/images/metadata_stream.png similarity index 100% rename from docs/resources/images/metadata_stream.png rename to docs/.vuepress/public/images/metadata_stream.png diff --git a/docs/resources/images/monitor_arch.png b/docs/.vuepress/public/images/monitor_arch.png similarity index 100% rename from docs/resources/images/monitor_arch.png rename to docs/.vuepress/public/images/monitor_arch.png diff --git a/docs/resources/images/multi_tenant_arch.png b/docs/.vuepress/public/images/multi_tenant_arch.png similarity index 100% rename from docs/resources/images/multi_tenant_arch.png rename to docs/.vuepress/public/images/multi_tenant_arch.png diff --git a/docs/resources/images/new-pr.png b/docs/.vuepress/public/images/new-pr.png similarity index 100% rename from docs/resources/images/new-pr.png rename to docs/.vuepress/public/images/new-pr.png diff --git a/docs/resources/images/palo_architecture.jpg b/docs/.vuepress/public/images/palo_architecture.jpg similarity index 100% rename from docs/resources/images/palo_architecture.jpg rename to docs/.vuepress/public/images/palo_architecture.jpg diff --git a/docs/resources/images/palo_meta.png b/docs/.vuepress/public/images/palo_meta.png similarity index 100% rename from docs/resources/images/palo_meta.png rename to docs/.vuepress/public/images/palo_meta.png diff --git a/docs/resources/images/perf-report-demo.png b/docs/.vuepress/public/images/perf-report-demo.png similarity index 100% rename from docs/resources/images/perf-report-demo.png rename to docs/.vuepress/public/images/perf-report-demo.png diff --git a/docs/resources/images/replica_recover.png b/docs/.vuepress/public/images/replica_recover.png similarity index 100% rename from docs/resources/images/replica_recover.png rename to docs/.vuepress/public/images/replica_recover.png diff --git a/docs/resources/images/segment_v2.png b/docs/.vuepress/public/images/segment_v2.png similarity index 100% rename from docs/resources/images/segment_v2.png rename to docs/.vuepress/public/images/segment_v2.png diff --git a/docs/resources/images/spark_doris_connector.jpg b/docs/.vuepress/public/images/spark_doris_connector.jpg similarity index 100% rename from docs/resources/images/spark_doris_connector.jpg rename to docs/.vuepress/public/images/spark_doris_connector.jpg diff --git a/docs/resources/images/subscribe-mail-list-step1.png b/docs/.vuepress/public/images/subscribe-mail-list-step1.png similarity index 100% rename from docs/resources/images/subscribe-mail-list-step1.png rename to docs/.vuepress/public/images/subscribe-mail-list-step1.png diff --git a/docs/resources/images/subscribe-mail-list-step2.png b/docs/.vuepress/public/images/subscribe-mail-list-step2.png similarity index 100% rename from docs/resources/images/subscribe-mail-list-step2.png rename to docs/.vuepress/public/images/subscribe-mail-list-step2.png diff --git a/docs/resources/images/subscribe-mail-list-step3.png b/docs/.vuepress/public/images/subscribe-mail-list-step3.png similarity index 100% rename from docs/resources/images/subscribe-mail-list-step3.png rename to docs/.vuepress/public/images/subscribe-mail-list-step3.png diff --git a/docs/resources/images/subscribe-mail-list-step4.png b/docs/.vuepress/public/images/subscribe-mail-list-step4.png similarity index 100% rename from docs/resources/images/subscribe-mail-list-step4.png rename to docs/.vuepress/public/images/subscribe-mail-list-step4.png diff --git a/docs/resources/images/user_authority.png b/docs/.vuepress/public/images/user_authority.png similarity index 100% rename from docs/resources/images/user_authority.png rename to docs/.vuepress/public/images/user_authority.png diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js new file mode 100644 index 00000000000000..5c24f34952eaaa --- /dev/null +++ b/docs/.vuepress/sidebar/en.js @@ -0,0 +1,442 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module.exports = [ + { + title: "Downloads", + directoryPath: "downloads/", + children: ["downloads"], + sidebarDepth: 1, + }, + { + title: "Compilation and Deployment", + directoryPath: "installing/", + children: ["compilation", "install-deploy", "upgrade"], + }, + { + title: "Getting Started", + directoryPath: "getting-started/", + children: [ + "basic-usage", + "advance-usage", + "best-practice", + "data-partition", + "data-model-rollup", + "hit-the-rollup", + ], + }, + { + title: "Administrator Guide", + directoryPath: "administrator-guide/", + children: [ + { + title: "Load Data", + directoryPath: "load-data/", + children: [ + "load-manual", + "broker-load-manual", + "stream-load-manual", + "routine-load-manual", + "insert-into-manual", + "delete-manual", + ], + sidebarDepth: 2, + }, + { + title: "Schema Change", + directoryPath: "alter-table/", + children: [ + "alter-table-bitmap-index", + "alter-table-rollup", + "alter-table-schema-change", + "alter-table-temp-partition", + ], + sidebarDepth: 2, + }, + { + title: "HTTP API", + directoryPath: "http-actions/", + children: [ + "cancel-label", + "compaction-action", + "fe-get-log-file", + "get-label-state", + "restore-tablet", + ], + sidebarDepth: 1, + }, + { + title: "Maintainence Operation", + directoryPath: "operation/", + children: [ + "metadata-operation", + "monitor-alert", + "multi-tenant", + "tablet-meta-tool", + "tablet-repair-and-balance", + ], + sidebarDepth: 2, + }, + { + title: "Configuration", + directoryPath: "config/", + children: ["fe_config"], + sidebarDepth: 1, + }, + "backup-restore", + "broker", + "colocation-join", + "dynamic-partition", + "export_manual", + "privilege", + "small-file-mgr", + "sql-mode", + "time-zone", + "variables", + ], + sidebarDepth: 1, + }, + { + title: "Extending Ability", + directoryPath: "extending-doris/", + children: [ + "audit-plugin", + "doris-on-es", + "plugin-development-manual", + "user-defined-function", + ], + }, + { + title: "Design Documents", + directoryPath: "internal/", + children: [ + "doris_storage_optimization", + "grouping_sets_design", + "metadata-design", + ], + }, + { + title: "SQL Manual", + directoryPath: "sql-reference/", + children: [ + { + title: "SQL Functions", + directoryPath: "sql-functions/", + children: [ + { + title: "Date Time Functions", + directoryPath: "date-time-functions/", + children: [ + "curdate", + "current_timestamp", + "date_add", + "date_format", + "date_sub", + "datediff", + "day", + "dayname", + "dayofmonth", + "dayofweek", + "dayofyear", + "from_days", + "from_unixtime", + "hour", + "minute", + "month", + "monthname", + "now", + "second", + "str_to_date", + "timediff", + "timestampadd", + "timestampdiff", + "to_days", + "unix_timestamp", + "utc_timestamp", + "workofyear", + "year", + ], + }, + { + title: "Sptial Functions", + directoryPath: "spatial-functions/", + children: [ + "st_astext", + "st_circle", + "st_contains", + "st_distance_sphere", + "st_geometryfromtext", + "st_linefromtext", + "st_point", + "st_polygon", + "st_x", + "st_y", + ], + }, + { + title: "String Functions", + directoryPath: "string-functions/", + children: [ + "ascii", + "concat", + "concat_ws", + "ends_with", + "find_in_set", + "get_json_double", + "get_json_int", + "get_json_string", + "group_concat", + "instr", + "lcase", + "left", + "length", + "locate", + "lower", + "lpad", + "ltrim", + "money_format", + "null_or_empty", + "regexp_extract", + "regexp_replace", + "repeat", + "right", + "split_part", + "starts_with", + "strleft", + "strright", + ], + }, + { + title: "Aggregate Functions", + directoryPath: "aggregate-functions/", + children: [ + "avg", + "bitmap", + "count", + "hll_union_agg", + "max", + "min", + "ndv", + "percentile_approx", + "stddev", + "stddev_samp", + "sum", + "var_samp", + "variance", + ], + }, + { + title: "bitmap functions", + directoryPath: "bitmap-functions/", + children: [ + "bitmap_and", + "bitmap_contains", + "bitmap_empty", + "bitmap_from_string", + "bitmap_has_any", + "bitmap_hash", + "bitmap_or", + "bitmap_to_string", + "to_bitmap", + ], + }, + { + title: "Hash Functions", + directoryPath: "hash-functions/", + children: ["murmur_hash3_32"], + }, + "cast", + ], + }, + { + title: "DDL Statements", + directoryPath: "sql-statements/", + children: [ + { + title: "Account Management", + directoryPath: "Account Management/", + children: [ + "CREATE ROLE", + "CREATE USER", + "DROP ROLE", + "DROP USER", + "GRANT", + "REVOKE", + "SET PASSWORD", + "SET PROPERTY", + "SHOW GRANTS", + "SHOW ROLES", + ], + }, + { + title: "Administration", + directoryPath: "Administration/", + children: [ + "ADMIN CANCEL REPAIR", + "ADMIN CHECK TABLET", + "ADMIN REPAIR", + "ADMIN SET CONFIG", + "ADMIN SET REPLICA STATUS", + "ADMIN SHOW CONFIG", + "ADMIN SHOW REPLICA DISTRIBUTION", + "ADMIN SHOW REPLICA STATUS", + "ALTER CLUSTER", + "ALTER SYSTEM", + "CANCEL DECOMMISSION", + "CREATE CLUSTER", + "CREATE FILE", + "DROP CLUSTER", + "DROP FILE", + "ENTER", + "INSTALL PLUGIN", + "LINK DATABASE", + "MIGRATE DATABASE", + "SHOW BACKENDS", + "SHOW BROKER", + "SHOW FILE", + "SHOW FRONTENDS", + "SHOW FULL COLUMNS", + "SHOW INDEX", + "SHOW MIGRATIONS", + "SHOW PLUGINS", + "SHOW TABLE STATUS", + "UNINTALL PLUGIN", + ], + }, + { + title: "Data Definition", + directoryPath: "Data Definition/", + children: [ + "ALTER DATABASE", + "ALTER TABLE", + "ALTER VIEW", + "BACKUP", + "CANCEL ALTER", + "CANCEL BACKUP", + "CANCEL RESTORE", + "Colocate Join", + "CREATE DATABASE", + "CREATE INDEX", + "CREATE MATERIALIZED VIEW", + "CREATE REPOSITORY", + "CREATE TABLE", + "CREATE VIEW", + "create-function", + "DROP DATABASE", + "DROP INDEX", + "DROP MATERIALIZED VIEW", + "DROP REPOSITORY", + "DROP TABLE", + "DROP VIEW", + "drop-function", + "HLL", + "RECOVER", + "RESTORE", + "show-functions", + "TRUNCATE TABLE", + ], + }, + { + title: "Data Manipulation", + directoryPath: "Data Manipulation/", + children: [ + "BROKER LOAD", + "CANCEL DELETE", + "CANCEL LABEL", + "CANCEL LOAD", + "DELETE", + "EXPORT", + "GET LABEL STATE", + "GROUP BY", + "insert", + "LOAD", + "MINI LOAD", + "MULTI LOAD", + "PAUSE ROUTINE LOAD", + "RESTORE TABLET", + "RESUME ROUTINE LOAD", + "ROUTINE LOAD", + "SHOW ALTER", + "SHOW BACKUP", + "SHOW DATA", + "SHOW DATABASES", + "SHOW DELETE", + "SHOW DYNAMIC PARTITION TABLES", + "SHOW EXPORT", + "SHOW LOAD", + "SHOW PARTITIONS", + "SHOW PROPERTY", + "SHOW REPOSITORIES", + "SHOW RESTORE", + "SHOW ROUTINE LOAD TASK", + "SHOW ROUTINE LOAD", + "SHOW SNAPSHOT", + "SHOW TABLES", + "SHOW TABLET", + "SHOW TRANSACTION", + "STOP ROUTINE LOAD", + "STREAM LOAD", + ], + }, + { + title: "Data Types", + directoryPath: "Data Types/", + children: [ + "BIGINT", + "BOOLEAN", + "CHAR", + "DATE", + "DATETIME", + "DECIMAL", + "DOUBLE", + "FLOAT", + "HLL(HyperLogLog)", + "INT", + "SMALLINT", + "TINYINT", + "VARCHAR", + ], + }, + { + title: "Utility", + directoryPath: "Utility/", + children: ["util_stmt"], + }, + ], + }, + ], + }, + { + title: "Developer Guide", + directoryPath: "developer-guide/", + children: ["debug-tool", "format-code"], + }, + { + title: "Apache Commnity", + directoryPath: "community/", + children: [ + "gitter", + "how-to-contribute", + "members", + "pull-request", + "release-process", + "subscribe-mail-list", + "verify-apache-release", + ], + }, +] diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js new file mode 100644 index 00000000000000..23c0088b14d421 --- /dev/null +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -0,0 +1,450 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module.exports = [ + { + title: "下载", + directoryPath: "downloads/", + children: ["downloads"], + sidebarDepth: 1, + }, + { + title: "编译与部署", + directoryPath: "installing/", + children: ["compilation", "install-deploy", "upgrade"], + }, + { + title: "开始使用", + directoryPath: "getting-started/", + children: [ + "basic-usage", + "advance-usage", + "best-practice", + "data-partition", + "data-model-rollup", + "hit-the-rollup", + ], + }, + { + title: "操作手册", + directoryPath: "administrator-guide/", + children: [ + { + title: "数据导入", + directoryPath: "load-data/", + children: [ + "load-manual", + "broker-load-manual", + "stream-load-manual", + "routine-load-manual", + "insert-into-manual", + "delete-manual", + ], + sidebarDepth: 2, + }, + { + title: "表结构变更", + directoryPath: "alter-table/", + children: [ + "alter-table-bitmap-index", + "alter-table-rollup", + "alter-table-schema-change", + "alter-table-temp-partition", + ], + sidebarDepth: 2, + }, + { + title: "物化视图", + directoryPath: "materialized-view/", + children: [], + sidebarDepth: 2, + }, + { + title: "HTTP API", + directoryPath: "http-actions/", + children: [ + "cancel-label", + "compaction-action", + "fe-get-log-file", + "get-label-state", + "restore-tablet", + ], + sidebarDepth: 1, + }, + { + title: "运维操作", + directoryPath: "operation/", + children: [ + "disk-capacity", + "metadata-operation", + "monitor-alert", + "multi-tenant", + "tablet-meta-tool", + "tablet-repair-and-balance", + "tablet-restore-tool", + ], + sidebarDepth: 2, + }, + { + title: "配置文件", + directoryPath: "config/", + children: ["fe_config"], + sidebarDepth: 1, + }, + "backup-restore", + "broker", + "colocation-join", + "dynamic-partition", + "export-manual", + "privilege", + "segment-v2-usage", + "small-file-mgr", + "sql-mode", + "time-zone", + "variables", + ], + sidebarDepth: 1, + }, + { + title: "扩展功能", + directoryPath: "extending-doris/", + children: [ + "audit-plugin", + "doris-on-es", + "plugin-development-manual", + "user-defined-function", + ], + }, + { + title: "设计文档", + directoryPath: "internal/", + children: [ + "doris_storage_optimization", + "grouping_sets_design", + "metadata-design", + "spark_load", + ], + }, + { + title: "SQL 手册", + directoryPath: "sql-reference/", + children: [ + { + title: "SQL 函数", + directoryPath: "sql-functions/", + children: [ + { + title: "日期函数", + directoryPath: "date-time-functions/", + children: [ + "convert_tz", + "curdate", + "current_timestamp", + "curtime", + "date_add", + "date_format", + "date_sub", + "datediff", + "day", + "dayname", + "dayofmonth", + "dayofweek", + "dayofyear", + "from_days", + "from_unixtime", + "hour", + "minute", + "month", + "monthname", + "now", + "second", + "str_to_date", + "timediff", + "timestampadd", + "timestampdiff", + "to_days", + "unix_timestamp", + "utc_timestamp", + "workofyear", + "year", + ], + }, + { + title: "地理位置函数", + directoryPath: "spatial-functions/", + children: [ + "st_astext", + "st_circle", + "st_contains", + "st_distance_sphere", + "st_geometryfromtext", + "st_linefromtext", + "st_point", + "st_polygon", + "st_x", + "st_y", + ], + }, + { + title: "字符串函数", + directoryPath: "string-functions/", + children: [ + "ascii", + "concat", + "concat_ws", + "ends_with", + "find_in_set", + "get_json_double", + "get_json_int", + "get_json_string", + "group_concat", + "instr", + "lcase", + "left", + "length", + "locate", + "lower", + "lpad", + "ltrim", + "money_format", + "null_or_empty", + "regexp_extract", + "regexp_replace", + "repeat", + "right", + "split_part", + "starts_with", + "strleft", + "strright", + ], + }, + { + title: "聚合函数", + directoryPath: "aggregate-functions/", + children: [ + "avg", + "bitmap", + "count", + "hll_union_agg", + "max", + "min", + "ndv", + "percentile_approx", + "stddev", + "stddev_samp", + "sum", + "var_samp", + "variance", + ], + }, + { + title: "bitmap函数", + directoryPath: "bitmap-functions/", + children: [ + "bitmap_and", + "bitmap_contains", + "bitmap_empty", + "bitmap_from_string", + "bitmap_has_any", + "bitmap_hash", + "bitmap_or", + "bitmap_to_string", + "to_bitmap", + ], + }, + { + title: "Hash函数", + directoryPath: "hash-functions/", + children: ["murmur_hash3_32"], + }, + "cast", + ], + }, + { + title: "语法帮助", + directoryPath: "sql-statements/", + children: [ + { + title: "用户账户管理", + directoryPath: "Account Management/", + children: [ + "CREATE ROLE", + "CREATE USER", + "DROP ROLE", + "DROP USER", + "GRANT", + "REVOKE", + "SET PASSWORD", + "SET PROPERTY", + "SHOW GRANTS", + "SHOW ROLES", + ], + }, + { + title: "集群管理", + directoryPath: "Administration/", + children: [ + "ADMIN CANCEL REPAIR", + "ADMIN CHECK TABLET", + "ADMIN REPAIR", + "ADMIN SET CONFIG", + "ADMIN SET REPLICA STATUS", + "ADMIN SHOW CONFIG", + "ADMIN SHOW REPLICA DISTRIBUTION", + "ADMIN SHOW REPLICA STATUS", + "ALTER CLUSTER", + "ALTER SYSTEM", + "CANCEL DECOMMISSION", + "CREATE CLUSTER", + "CREATE FILE", + "DROP CLUSTER", + "DROP FILE", + "ENTER", + "INSTALL PLUGIN", + "LINK DATABASE", + "MIGRATE DATABASE", + "SHOW BACKENDS", + "SHOW BROKER", + "SHOW FILE", + "SHOW FRONTENDS", + "SHOW FULL COLUMNS", + "SHOW INDEX", + "SHOW MIGRATIONS", + "SHOW PLUGINS", + "SHOW TABLE STATUS", + "UNINSTALL PLUGIN", + ], + }, + { + title: "DDL", + directoryPath: "Data Definition/", + children: [ + "ALTER DATABASE", + "ALTER TABLE", + "ALTER VIEW", + "BACKUP", + "CANCEL ALTER", + "CANCEL BACKUP", + "CANCEL RESTORE", + "CREATE DATABASE", + "CREATE INDEX", + "CREATE MATERIALIZED VIEW", + "CREATE REPOSITORY", + "CREATE TABLE", + "CREATE VIEW", + "create-function", + "DROP DATABASE", + "DROP INDEX", + "DROP MATERIALIZED VIEW", + "DROP REPOSITORY", + "DROP TABLE", + "DROP VIEW", + "drop-function", + "HLL", + "RECOVER", + "RESTORE", + "show-functions", + "TRUNCATE TABLE", + ], + }, + { + title: "DML", + directoryPath: "Data Manipulation/", + children: [ + "BROKER LOAD", + "CANCEL LOAD", + "DELETE", + "EXPORT", + "GROUP BY", + "insert", + "LOAD", + "MINI LOAD", + "MULTI LOAD", + "PAUSE ROUTINE LOAD", + "RESUME ROUTINE LOAD", + "ROUTINE LOAD", + "SHOW ALTER", + "SHOW BACKUP", + "SHOW DATA", + "SHOW DATABASES", + "SHOW DELETE", + "SHOW DYNAMIC PARTITION TABLES", + "SHOW EXPORT", + "SHOW LOAD", + "SHOW PARTITIONS", + "SHOW PROPERTY", + "SHOW REPOSITORIES", + "SHOW RESTORE", + "SHOW ROUTINE LOAD", + "SHOW ROUTINE LOAD TASK", + "SHOW SNAPSHOT", + "SHOW TABLES", + "SHOW TABLET", + "SHOW TRANSACTION", + "STOP ROUTINE LOAD", + "STREAM LOAD", + ], + }, + { + title: "数据类型", + directoryPath: "Data Types/", + children: [ + "BIGINT", + "BOOLEAN", + "CHAR", + "DATE", + "DATETIME", + "DECIMAL", + "DOUBLE", + "FLOAT", + "HLL", + "INT", + "LARGEINT", + "SMALLINT", + "TINYINT", + "VARCHAR", + ], + }, + { + title: "辅助命令", + directoryPath: "Utility/", + children: ["DESCRIBE"], + }, + ], + }, + ], + }, + { + title: "开发者手册", + directoryPath: "developer-guide/", + children: ["debug-tool", "format-code"], + }, + { + title: "Apache 社区", + directoryPath: "community/", + children: [ + "gitter", + "how-to-contribute", + "members", + "pull-request", + "release-process", + "subscribe-mail-list", + "verify-apache-release", + ], + }, +]; diff --git a/docs/.vuepress/theme/components/Footer.vue b/docs/.vuepress/theme/components/Footer.vue new file mode 100644 index 00000000000000..607e5b1845cf88 --- /dev/null +++ b/docs/.vuepress/theme/components/Footer.vue @@ -0,0 +1,53 @@ + + + + + \ No newline at end of file diff --git a/docs/.vuepress/theme/components/Home.vue b/docs/.vuepress/theme/components/Home.vue new file mode 100644 index 00000000000000..32a391e56b464e --- /dev/null +++ b/docs/.vuepress/theme/components/Home.vue @@ -0,0 +1,38 @@ + + + + \ No newline at end of file diff --git a/docs/.vuepress/theme/components/NavLinks.vue b/docs/.vuepress/theme/components/NavLinks.vue new file mode 100644 index 00000000000000..66f1023fe48eeb --- /dev/null +++ b/docs/.vuepress/theme/components/NavLinks.vue @@ -0,0 +1,56 @@ + + + + diff --git a/docs/.vuepress/theme/index.js b/docs/.vuepress/theme/index.js new file mode 100644 index 00000000000000..d882e00623fa2b --- /dev/null +++ b/docs/.vuepress/theme/index.js @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module.exports = { + extend: "@vuepress/theme-default" +}; diff --git a/docs/.vuepress/theme/layouts/Layout.vue b/docs/.vuepress/theme/layouts/Layout.vue new file mode 100644 index 00000000000000..34b0b570c80cc3 --- /dev/null +++ b/docs/.vuepress/theme/layouts/Layout.vue @@ -0,0 +1,38 @@ + + + + \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 6d971114bc8806..00000000000000 --- a/docs/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# This file build all docs -# - -BUILD_DIR = ${CURDIR}/build -HELP_DIR = ${BUILD_DIR}/contents - -all: zip_help -.PHONY: all - -${BUILD_DIR}: - mkdir -p ${BUILD_DIR} -${HELP_DIR}: - mkdir -p ${HELP_DIR} - -# build help zip -HELP_OUTPUT = ${BUILD_DIR}/help-resource.zip -${HELP_OUTPUT}: documentation/cn/sql-reference ${BUILD_DIR} ${HELP_DIR} - cp -r $ + +# Doris Document + +[Vuepress](https://github.com/vuejs/vuepress.git) is used as our document site generator, configurations are in `./docs/.vuepress` folder. + +## Getting Started + +Download and install [nodejs](http://nodejs.cn/download/) + +```bash +npm config set registry https://registry.npm.taobao.org // Only if you are in Mainland China. +cd docs && npm install +npm run dev +``` + +Open your browser and navigate to `localhost:8080/en/` or `localhost:8080/zh-CN/`. + +## Docs' Directories + +```bash + . + ├─ docs/ + │ ├─ .vuepress + │ │ ├─ dist // Built site files. + │ │ ├─ public // Assets + │ │ ├─ sidebar // Side bar configurations. + │ │ │ ├─ en.js + │ │ │ └─ zh-CN.js + │ ├─ theme // Global styles and customizations. + │ └─ config.js // Vuepress configurations. + ├─ zh-CN/ + │ ├─ xxxx.md + │ └─ README.md // Will be rendered as entry page. + └─ en/ + ├─ one.md + └─ README.md // Will be rendered as entry page. +``` + +## Start Writing + +1. Write markdown files in multi languages and put them in separated folders `./en/` and `./zh-CN/`. **But they should be with the same name.** + + ```bash + . + ├─ en/ + │ ├─ one.md + │ └─ two.md + └─ zh-CN/ + │ ├─ one.md + │ └─ two.md + ``` + +2. Frontmatters like below should always be on the top of each file: + + ```markdown + --- + { + "title": "Backup and Recovery", // sidebar title + "language": "en" // writing language + } + --- + ``` + +3. Assets are in `.vuepress/public/`. + + Assuming that there exists a png `.vuepress/public/images/image_x.png`, then it can be used like: + + ```markdown + ![alter text](/images/image_x.png) + ``` + +4. Remember to update the sidebar configurations in `.vuepress/sidebar/` after adding a new file or a folder. + + Assuming that the directories are: + + ```bash + . + ├─ en/ + │ ├─ subfolder + │ │ ├─ one.md + │ │ └─ two.md + │ └─ three.md + └─ zh-CN/ + ├─ subfolder + │ ├─ one.md + │ └─ two.md + └─ three.md + ``` + + Then the sidebar configurations would be like: + + ```javascript + // .vuepress/sidebar/en.js` + module.exports = [ + { + title: "subfolder name", + directoryPath: "subfolder/", + children: ["one", "two"] + }, + "three" + ] + ``` + + ```javascript + // .vuepress/sidebar/zh-CN.js + module.exports = [ + { + title: "文件夹名称", + directoryPath: "subfolder/", + children: ["one", "two"] + }, + "three" + ] + ``` + +5. Run `npm run lint` before starting a PR. + + Surely that there will be lots of error logs if the mardown files are not following the rules, and these logs will all be printed in the console: + +```shell + +en/administrator-guide/alter-table/alter-table-bitmap-index.md:92 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: " ```"] +en/administrator-guide/alter-table/alter-table-rollup.md:45 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-rollup.md:77 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-rollup.md:178 MD046/code-block-style Code block style [Expected: fenced; Actual: indented] +en/administrator-guide/alter-table/alter-table-schema-change.md:50 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:82 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:127 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:144 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:153 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:199 MD046/code-block-style Code block style [Expected: fenced; Actual: indented] +en/administrator-guide/backup-restore.md:45:1 MD029/ol-prefix Ordered list item prefix [Expected: 1; Actual: 2; Style: 1/1/1] +en/administrator-guide/backup-restore.md:57:1 MD029/ol-prefix Ordered list item prefix [Expected: 1; Actual: 2; Style: 1/1/1] +en/administrator-guide/backup-restore.md:61:1 MD029/ol-prefix Ordered list item prefix [Expected: 1; Actual: 3; Style: 1/1/1] +npm ERR! code ELIFECYCLE +npm ERR! errno 1 +npm ERR! docs@ lint: `markdownlint '**/*.md' -f` +npm ERR! Exit status 1 +npm ERR! +npm ERR! Failed at the docs@ lint script. + +``` + +## Deployment + +Just start a PR, and all things will be done automatically. + +## What Travis Does + +Once a PR accepted, travis ci will be triggered to build and deploy the whole website within its own branch. Here is what `.travis.yml` does: + +1. Prepare nodejs and vuepress enviorment. + +2. Use current branch's name as the relative url path in `.vuepress/config.js`(which is the `base` property). + +3. Build the documents into a website all by vuepress. + +4. Fetch asf-site repo to local directory, and copy `.vupress/dist/` into `{BRANCH}/`. + +5. Push the new site to asf-site repo with `GitHub Token`(which is preset in Travis console as a variable used in .travis.yml). + +## asf-site repository + +Finally the asf-site repository will be like: + +```bash +. +├─ master/ +│ ├─ en/ +│ │ ├─ subfolder +│ │ │ ├─ one.md +│ │ └─ three.md +│ └─ zh-CN/ +│ ├─ subfolder +│ │ ├─ one.md +│ └─ three.md +├─ incubating-0.11/ +│ ├─ en/ +│ │ ├─ subfolder +│ │ │ ├─ one.md +│ │ └─ three.md +│ └─ zh-CN/ +│ ├─ subfolder +│ │ ├─ one.md +│ └─ three.md +├─ index.html // user entry, and auto redirected to master folder +└─ versions.json // all versions that can be seleted on the website are defined here +``` + +And the `versions.json` is like: + +```json +{ + "en": [ + { + "text": "Versions", // dropdown label + "items": [ + { + "text": "master", // dropdown-item label + "link": "/../master/en/installing/compilation.html", // entry page for this version + "target": "_blank" + }, + { + "text": "branch-0.11", + "link": "/../branch-0.11/en/installing/compilation.html", + "target": "_blank" + } + ] + } + ], + "zh-CN": [ + { + "text": "版本", + "items": [ + { + "text": "master", + "link": "/../master/zh-CN/installing/compilation.html", + "target": "_blank" + }, + { + "text": "branch-0.11", + "link": "/../branch-0.11/zh-CN/installing/compilation.html", + "target": "_blank" + } + ] + } + ] +} +``` diff --git a/docs/documentation/cn/administrator-guide/alter-table/alter-table-bitmap-index.md b/docs/documentation/cn/administrator-guide/alter-table/alter-table-bitmap-index.md deleted file mode 100644 index a2a1ef0579250a..00000000000000 --- a/docs/documentation/cn/administrator-guide/alter-table/alter-table-bitmap-index.md +++ /dev/null @@ -1,79 +0,0 @@ - - -# Bitmap 索引 -用户可以通过创建bitmap index 加速查询 -本文档主要介绍如何创建 index 作业，以及创建 index 的一些注意事项和常见问题。 - -## 名词解释 -* bitmap index：位图索引，是一种快速数据结构，能够加快查询速度 - -## 原理介绍 -创建和删除本质上是一个 schema change 的作业，具体细节可以参照 [Schema Change](alter-table-schema-change)。 - -## 语法 -index 创建和修改相关语法有两种形式，一种集成与 alter table 语句中，另一种是使用单独的 -create/drop index 语法 -1. 创建索引 - - 创建索引的的语法可以参见 [CREATE INDEX](../../sql-reference/sql-statements/Data%20Definition/CREATE%20INDEX.html) - 或 [ALTER TABLE](../../sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE.html) 中bitmap 索引相关的操作, - 也可以通过在创建表时指定bitmap 索引，参见[CREATE TABLE](../../sql-reference/sql-statements/Data%20Definition/CREATE%20TABLE.html) - -2. 查看索引 - - 参照[SHOW INDEX](../../sql-reference/sql-statements/Administration/SHOW%20INDEX.html) - -3. 删除索引 - - 参照[DROP INDEX](../../sql-reference/sql-statements/Data%20Definition/DROP%20INDEX.html) - 或者 [ALTER TABLE](../../sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE.html) 中bitmap 索引相关的操作 - -## 创建作业 -参照 schema change 文档 [Scheam Change](alter-table-schema-change.html) - -## 查看作业 -参照 schema change 文档 [Scheam Change](alter-table-schema-change.html) - -## 取消作业 -参照 schema change 文档 [Scheam Change](alter-table-schema-change.html) - -## 注意事项 -* 目前索引仅支持 bitmap 类型的索引。 -* bitmap 索引仅在单列上创建。 -* bitmap 索引能够应用在 `Duplicate` 数据模型的所有列和 `Aggregate`, `Uniq` 模型的key列上。 -* bitmap 索引支持的数据类型如下: - * `TINYINT` - * `SMALLINT` - * `INT` - * `UNSIGNEDINT` - * `BIGINT` - * `CHAR` - * `VARCHAE` - * `DATE` - * `DATETIME` - * `LARGEINT` - * `DECIMAL` - * `BOOL` - -* bitmap索引仅在 segmentV2 下生效，需要在be的配置文件中增加如下配置 - - ``` - default_rowset_type=BETA - ``` diff --git a/docs/documentation/cn/administrator-guide/alter-table/alter-table-rollup.md b/docs/documentation/cn/administrator-guide/alter-table/alter-table-rollup.md deleted file mode 100644 index ef377d8e2717f6..00000000000000 --- a/docs/documentation/cn/administrator-guide/alter-table/alter-table-rollup.md +++ /dev/null @@ -1,187 +0,0 @@ - - -# Rollup - -用户可以通过创建上卷表（Rollup）加速查询。关于 Rollup 的概念和使用方式可以参阅 [数据模型、ROLLUP 及前缀索引](../../getting-started/data-model-rollup.md) 和 [Rollup 与查询](../../getting-started/hit-the-rollup.md) 两篇文档。 - -本文档主要介绍如何创建 Rollup 作业，以及创建 Rollup 的一些注意事项和常见问题。 - -## 名词解释 - -* Base Table：基表。每一个表被创建时，都对应一个基表。基表存储了这个表的完整的数据。Rollup 通常基于基表中的数据创建（也可以通过其他 Rollup 创建）。 -* Index：物化索引。Rollup 或 Base Table 都被称为物化索引。 -* Transaction：事务。每一个导入任务都是一个事务，每个事务有一个唯一递增的 Transaction ID。 - -## 原理介绍 - -创建 Rollup 的基本过程，是通过 Base 表的数据，生成一份新的包含指定列的 Rollup 的数据。其中主要需要进行两部分数据转换，一是已存在的历史数据的转换，二是在 Rollup 执行过程中，新到达的导入数据的转换。 - -``` -+----------+ -| Load Job | -+----+-----+ - | - | Load job generates both base and rollup index data - | - | +------------------+ +---------------+ - | | Base Index | | Base Index | - +------> New Incoming Data| | History Data | - | +------------------+ +------+--------+ - | | - | | Convert history data - | | - | +------------------+ +------v--------+ - | | Rollup Index | | Rollup Index | - +------> New Incoming Data| | History Data | - +------------------+ +---------------+ -``` - -在开始转换历史数据之前，Doris 会获取一个最新的 Transaction ID。并等待这个 Transaction ID 之前的所有导入事务完成。这个 Transaction ID 成为分水岭。意思是，Doris 保证在分水岭之后的所有导入任务，都会同时为 Rollup Index 生成数据。这样当历史数据转换完成后，可以保证 Rollup 和 Base 表的数据是齐平的。 - -## 创建作业 - -创建 Rollup 的具体语法可以查看帮助 `HELP ALTER TABLE` 中 Rollup 部分的说明。 - -Rollup 的创建是一个异步过程，作业提交成功后，用户需要通过 `SHOW ALTER TABLE ROLLUP` 命令来查看作业进度。 - -## 查看作业 - -`SHOW ALTER TABLE ROLLUP` 可以查看当前正在执行或已经完成的 Rollup 作业。举例如下： - -``` - JobId: 20037 - TableName: tbl1 - CreateTime: 2019-08-06 15:38:49 - FinishedTime: N/A - BaseIndexName: tbl1 -RollupIndexName: r1 - RollupId: 20038 - TransactionId: 10034 - State: PENDING - Msg: - Progress: N/A - Timeout: 86400 -``` - -* JobId：每个 Rollup 作业的唯一 ID。 -* TableName：Rollup 对应的基表的表名。 -* CreateTime：作业创建时间。 -* FinishedTime：作业结束时间。如未结束，则显示 "N/A"。 -* BaseIndexName：Rollup 对应的源 Index 的名称。 -* RollupIndexName：Rollup 的名称。 -* RollupId：Rollup 的唯一 ID。 -* TransactionId：转换历史数据的分水岭 transaction ID。 -* State：作业所在阶段。 - * PENDING：作业在队列中等待被调度。 - * WAITING_TXN：等待分水岭 transaction ID 之前的导入任务完成。 - * RUNNING：历史数据转换中。 - * FINISHED：作业成功。 - * CANCELLED：作业失败。 -* Msg：如果作业失败，这里会显示失败信息。 -* Progress：作业进度。只有在 RUNNING 状态才会显示进度。进度是以 M/N 的形式显示。其中 N 为 Rollup 的总副本数。M 为已完成历史数据转换的副本数。 -* Timeout：作业超时时间。单位秒。 - -## 取消作业 - -在作业状态不为 FINISHED 或 CANCELLED 的情况下，可以通过以下命令取消 Rollup 作业： - -`CANCEL ALTER TABLE ROLLUP FROM tbl_name;` - -## 注意事项 - -* 一张表在同一时间只能有一个 Rollup 作业在运行。且一个作业中只能创建一个 Rollup。 - -* Rollup 操作不阻塞导入和查询操作。 - -* 如果 DELETE 操作，where 条件中的某个 Key 列在某个 Rollup 中不存在，则不允许该 DELETE。 - - 如果某个 Key 列在某一 Rollup 中不存在，则 DELETE 操作无法对该 Rollup 进行数据删除，从而无法保证 Rollup 表和 Base 表的数据一致性。 - -* Rollup 的列必须存在于 Base 表中。 - - Rollup 的列永远是 Base 表列的子集。不能出现 Base 表中不存在的列。 - -* 如果 Rollup 中包含 REPLACE 聚合类型的列，则该 Rollup 必须包含所有 Key 列。 - - 假设 Base 表结构如下： - - ```(k1 INT, k2 INT, v1 INT REPLACE, v2 INT SUM)``` - - 如果需要创建的 Rollup 包含 `v1` 列，则必须包含 `k1`, `k2` 列。否则系统无法决定 `v1` 列在 Rollup 中的取值。 - - 注意，Unique 数据模型表中的所有 Value 列都是 REPLACE 聚合类型。 - -* DUPLICATE 数据模型表的 Rollup，可以指定 Rollup 的 DUPLICATE KEY。 - - DUPLICATE 数据模型表中的 DUPLICATE KEY 其实就是排序列。Rollup 可以指定自己的排序列，但排序列必须是 Rollup 列顺序的前缀。如果不指定，则系统会检查 Rollup 是否包含了 Base 表的所有排序列，如果没有包含，则会报错。举例： - - Base 表结构：`(k1 INT, k2 INT, k3 INT) DUPLICATE KEY(k1, k2)` - - 则 Rollup 可以为：`(k2 INT, k1 INT) DUPLICATE KEY(k2)` - -* Rollup 不需要包含 Base 表的分区列或分桶列。 - -## 常见问题 - -* 一个表可以创建多少 Rollup - - 一个表能够创建的 Rollup 个数理论上没有限制，但是过多的 Rollup 会影响导入性能。因为导入时，会同时给所有 Rollup 产生数据。同时 Rollup 会占用物理存储空间。通常一个表的 Rollup 数量在 10 个以内比较合适。 - -* Rollup 创建的速度 - - 目前 Rollup 创建速度按照最差效率估计约为 10MB/s。保守起见，用户可以根据这个速率来设置作业的超时时间。 - -* 提交作业报错 `Table xxx is not stable. ...` - - Rollup 只有在表数据完整且非均衡状态下才可以开始。如果表的某些数据分片副本不完整，或者某些副本正在进行均衡操作，则提交会被拒绝。 - - 数据分片副本是否完整，可以通过以下命令查看： - - ```ADMIN SHOW REPLICA STATUS FROM tbl WHERE STATUS != "OK";``` - - 如果有返回结果，则说明有副本有问题。通常系统会自动修复这些问题，用户也可以通过以下命令优先修复这个表： - - ```ADMIN REPAIR TABLE tbl1;``` - - 用户可以通过以下命令查看是否有正在运行的均衡任务： - - ```SHOW PROC "/cluster_balance/pending_tablets";``` - - 可以等待均衡任务完成，或者通过以下命令临时禁止均衡操作： - - ```ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");``` - -## 相关配置 - -### FE 配置 - -* `alter_table_timeout_second`：作业默认超时时间，86400 秒。 - -### BE 配置 - -* `alter_tablet_worker_count`：在 BE 端用于执行历史数据转换的线程数。默认为 3。如果希望加快 Rollup 作业的速度，可以适当调大这个参数后重启 BE。但过多的转换线程可能会导致 IO 压力增加，影响其他操作。该线程和 Schema Change 作业共用。 - - - - - - - - diff --git a/docs/documentation/cn/administrator-guide/alter-table/alter-table-schema-change.md b/docs/documentation/cn/administrator-guide/alter-table/alter-table-schema-change.md deleted file mode 100644 index b4c681c54009fb..00000000000000 --- a/docs/documentation/cn/administrator-guide/alter-table/alter-table-schema-change.md +++ /dev/null @@ -1,242 +0,0 @@ - - -# Schema Change - -用户可以通过 Schema Change 操作来修改已存在表的 Schema。目前 Doris 支持以下几种修改: - -* 增加、删除列 -* 修改列类型 -* 调整列顺序 -* 增加、修改 Bloom Filter -* 增加、删除 bitmap index - -本文档主要介绍如何创建 Schema Change 作业，以及进行 Schema Change 的一些注意事项和常见问题。 - -## 名词解释 - -* Base Table：基表。每一个表被创建时，都对应一个基表。 -* Rollup：基于基表或者其他 Rollup 创建出来的上卷表。 -* Index：物化索引。Rollup 或 Base Table 都被称为物化索引。 -* Transaction：事务。每一个导入任务都是一个事务，每个事务有一个唯一递增的 Transaction ID。 - -## 原理介绍 - -执行 Schema Change 的基本过程，是通过原 Index 的数据，生成一份新 Schema 的 Index 的数据。其中主要需要进行两部分数据转换，一是已存在的历史数据的转换，二是在 Schema Change 执行过程中，新到达的导入数据的转换。 - -``` -+----------+ -| Load Job | -+----+-----+ - | - | Load job generates both origin and new index data - | - | +------------------+ +---------------+ - | | Origin Index | | Origin Index | - +------> New Incoming Data| | History Data | - | +------------------+ +------+--------+ - | | - | | Convert history data - | | - | +------------------+ +------v--------+ - | | New Index | | New Index | - +------> New Incoming Data| | History Data | - +------------------+ +---------------+ -``` - -在开始转换历史数据之前，Doris 会获取一个最新的 Transaction ID。并等待这个 Transaction ID 之前的所有导入事务完成。这个 Transaction ID 成为分水岭。意思是，Doris 保证在分水岭之后的所有导入任务，都会同时为原 Index 和新 Index 生成数据。这样当历史数据转换完成后，可以保证新的 Index 中的数据是完整的。 - -## 创建作业 - -创建 Schema Change 的具体语法可以查看帮助 `HELP ALTER TABLE` 中 Schema Change 部分的说明。 - -Schema Change 的创建是一个异步过程，作业提交成功后，用户需要通过 `SHOW ALTER TABLE COLUMN` 命令来查看作业进度。 - -## 查看作业 - -`SHOW ALTER TABLE COLUMN` 可以查看当前正在执行或已经完成的 Schema Change 作业。当一次 Schema Change 作业涉及到多个 Index 时，该命令会显示多行，每行对应一个 Index。举例如下： - -``` - JobId: 20021 - TableName: tbl1 - CreateTime: 2019-08-05 23:03:13 - FinishTime: 2019-08-05 23:03:42 - IndexName: tbl1 - IndexId: 20022 -OriginIndexId: 20017 -SchemaVersion: 2:792557838 -TransactionId: 10023 - State: FINISHED - Msg: - Progress: N/A - Timeout: 86400 -``` - -* JobId：每个 Schema Change 作业的唯一 ID。 -* TableName：Schema Change 对应的基表的表名。 -* CreateTime：作业创建时间。 -* FinishedTime：作业结束时间。如未结束，则显示 "N/A"。 -* IndexName： 本次修改所涉及的某一个 Index 的名称。 -* IndexId：新的 Index 的唯一 ID。 -* OriginIndexId：旧的 Index 的唯一 ID。 -* SchemaVersion：以 M:N 的格式展示。其中 M 表示本次 Schema Change 变更的版本，N 表示对应的 Hash 值。每次 Schema Change，版本都会递增。 -* TransactionId：转换历史数据的分水岭 transaction ID。 -* State：作业所在阶段。 - * PENDING：作业在队列中等待被调度。 - * WAITING_TXN：等待分水岭 transaction ID 之前的导入任务完成。 - * RUNNING：历史数据转换中。 - * FINISHED：作业成功。 - * CANCELLED：作业失败。 -* Msg：如果作业失败，这里会显示失败信息。 -* Progress：作业进度。只有在 RUNNING 状态才会显示进度。进度是以 M/N 的形式显示。其中 N 为 Schema Change 涉及的总副本数。M 为已完成历史数据转换的副本数。 -* Timeout：作业超时时间。单位秒。 - -## 取消作业 - -在作业状态不为 FINISHED 或 CANCELLED 的情况下，可以通过以下命令取消 Schema Change 作业： - -`CANCEL ALTER TABLE COLUMN FROM tbl_name;` - -## 最佳实践 - -Schema Change 可以在一个作业中，对多个 Index 进行不同的修改。举例如下： - -源 Schema： - -``` -+-----------+-------+------+------+------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+-----------+-------+------+------+------+---------+-------+ -| tbl1 | k1 | INT | No | true | N/A | | -| | k2 | INT | No | true | N/A | | -| | k3 | INT | No | true | N/A | | -| | | | | | | | -| rollup2 | k2 | INT | No | true | N/A | | -| | | | | | | | -| rollup1 | k1 | INT | No | true | N/A | | -| | k2 | INT | No | true | N/A | | -+-----------+-------+------+------+------+---------+-------+ -``` - -可以通过以下命令给 rollup1 和 rollup2 都加入一列 k4，并且再给 rollup2 加入一列 k5： - -``` -ALTER TABLE tbl1 -ADD COLUMN k4 INT default "1" to rollup1, -ADD COLUMN k4 INT default "1" to rollup2, -ADD COLUMN k5 INT default "1" to rollup2; -``` - -完成后，Schema 变为： - -``` -+-----------+-------+------+------+------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+-----------+-------+------+------+------+---------+-------+ -| tbl1 | k1 | INT | No | true | N/A | | -| | k2 | INT | No | true | N/A | | -| | k3 | INT | No | true | N/A | | -| | k4 | INT | No | true | 1 | | -| | k5 | INT | No | true | 1 | | -| | | | | | | | -| rollup2 | k2 | INT | No | true | N/A | | -| | k4 | INT | No | true | 1 | | -| | k5 | INT | No | true | 1 | | -| | | | | | | | -| rollup1 | k1 | INT | No | true | N/A | | -| | k2 | INT | No | true | N/A | | -| | k4 | INT | No | true | 1 | | -+-----------+-------+------+------+------+---------+-------+ -``` - -可以看到，Base 表 tbl1 也自动加入了 k4, k5 列。即给任意 rollup 增加的列，都会自动加入到 Base 表中。 - -同时，不允许向 Rollup 中加入 Base 表已经存在的列。如果用户需要这样做，可以重新建立一个包含新增列的 Rollup，之后再删除原 Rollup。 - -## 注意事项 - -* 一张表在同一时间只能有一个 Schema Change 作业在运行。 - -* Schema Change 操作不阻塞导入和查询操作。 - -* 分区列和分桶列不能修改。 - -* 如果 Schema 中有 REPLACE 方式聚合的 value 列，则不允许删除 Key 列。 - - 如果删除 Key 列，Doris 无法决定 REPLACE 列的取值。 - - Unique 数据模型表的所有非 Key 列都是 REPLACE 聚合方式。 - -* 在新增聚合类型为 SUM 或者 REPLACE 的 value 列时，该列的默认值对历史数据没有含义。 - - 因为历史数据已经失去明细信息，所以默认值的取值并不能实际反映聚合后的取值。 - -* 当修改列类型时，除 Type 以外的字段都需要按原列上的信息补全。 - - 如修改列 `k1 INT SUM NULL DEFAULT "1"` 类型为 BIGINT，则需执行命令如下： - - ```ALTER TABLE tbl1 MODIFY COLUMN `k1` BIGINT SUM NULL DEFAULT "1";``` - - 注意，除新的列类型外，如聚合方式，Nullable 属性，以及默认值都要按照原信息补全。 - -* 不支持修改列名称、聚合类型、Nullable 属性、默认值以及列注释。 - -## 常见问题 - -* Schema Change 的执行速度 - - 目前 Schema Change 执行速度按照最差效率估计约为 10MB/s。保守起见，用户可以根据这个速率来设置作业的超时时间。 - -* 提交作业报错 `Table xxx is not stable. ...` - - Schema Change 只有在表数据完整且非均衡状态下才可以开始。如果表的某些数据分片副本不完整，或者某些副本正在进行均衡操作，则提交会被拒绝。 - - 数据分片副本是否完整，可以通过以下命令查看： - - ```ADMIN SHOW REPLICA STATUS FROM tbl WHERE STATUS != "OK";``` - - 如果有返回结果，则说明有副本有问题。通常系统会自动修复这些问题，用户也可以通过以下命令优先修复这个表： - - ```ADMIN REPAIR TABLE tbl1;``` - - 用户可以通过以下命令查看是否有正在运行的均衡任务： - - ```SHOW PROC "/cluster_balance/pending_tablets";``` - - 可以等待均衡任务完成，或者通过以下命令临时禁止均衡操作： - - ```ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");``` - -## 相关配置 - -### FE 配置 - -* `alter_table_timeout_second`：作业默认超时时间，86400 秒。 - -### BE 配置 - -* `alter_tablet_worker_count`：在 BE 端用于执行历史数据转换的线程数。默认为 3。如果希望加快 Schema Change 作业的速度，可以适当调大这个参数后重启 BE。但过多的转换线程可能会导致 IO 压力增加，影响其他操作。该线程和 Rollup 作业共用。 - - - - - - - - diff --git a/docs/documentation/cn/administrator-guide/alter-table/alter-table-temp-partition.md b/docs/documentation/cn/administrator-guide/alter-table/alter-table-temp-partition.md deleted file mode 100644 index 0911ee598278f9..00000000000000 --- a/docs/documentation/cn/administrator-guide/alter-table/alter-table-temp-partition.md +++ /dev/null @@ -1,244 +0,0 @@ - - -# 临时分区 - -在 0.12 版本中，Doris 支持了临时分区功能。 - -临时分区是归属于某一分区表的。只有分区表可以创建临时分区。 - -## 规则 - -* 临时分区的分区列和正式分区相同，且不可修改。 -* 一张表所有临时分区之间的分区范围不可重叠，但临时分区的范围和正式分区范围可以重叠。 -* 临时分区的分区名称不能和正式分区以及其他临时分区重复。 - -## 支持的操作 - -临时分区支持添加、删除、替换操作。 - -### 添加临时分区 - -可以通过 `ALTER TABLE ADD TEMPORARY PARTITION` 语句对一个表添加临时分区： - -``` -ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN("2020-02-01"); - -ALTER TABLE tbl2 ADD TEMPORARY PARTITION tp1 VALUES [("2020-01-01"), ("2020-02-01")); - -ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN("2020-02-01") -("in_memory" = "true", "replication_num" = "1") -DISTRIBUTED BY HASH(k1) BUCKETS 5; -``` - -通过 `HELP ALTER TABLE;` 查看更多帮助和示例。 - -添加操作的一些说明： - -* 临时分区的添加和正式分区的添加操作相似。临时分区的分区范围独立于正式分区。 -* 临时分区可以独立指定一些属性。包括分桶数、副本数、是否是内存表、存储介质等信息。 - -### 删除临时分区 - -可以通过 `ALTER TABLE DROP TEMPORARY PARTITION` 语句删除一个表的临时分区： - -``` -ALTER TABLE tbl1 DROP TEMPORARY PARTITION tp1; -``` - -通过 `HELP ALTER TABLE;` 查看更多帮助和示例。 - -删除操作的一些说明： - -* 删除临时分区，不影响正式分区的数据。 - -### 替换分区 - -可以通过 `ALTER TABLE REPLACE PARTITION` 语句将一个表的正式分区替换为临时分区。 - -``` -ALTER TABLE tbl1 REPLACE PARTITION (p1) WITH TEMPORARY PARTITION (tp1); - -ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1, tp2, tp3); - -ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1, tp2) -PROPERTIES ( - "strict_range" = "false", - "use_temp_partition_name" = "true" -); -``` - -通过 `HELP ALTER TABLE;` 查看更多帮助和示例。 - -替换操作有两个特殊的可选参数： - -1. `strict_range` - - 默认为 true。当该参数为 true 时，表示要被替换的所有正式分区的范围并集需要和替换的临时分区的范围并集完全相同。当置为 false 时，只需要保证替换后，新的正式分区间的范围不重叠即可。下面举例说明： - - * 示例1 - - 待替换的分区 p1, p2, p3 的范围 (=> 并集)： - - ``` - [10, 20), [20, 30), [40, 50) => [10, 30), [40, 50) - ``` - - 替换分区 tp1, tp2 的范围(=> 并集)： - - ``` - [10, 30), [40, 45), [45, 50) => [10, 30), [40, 50) - ``` - - 范围并集相同，则可以使用 tp1 和 tp2 替换 p1, p2, p3。 - - * 示例2 - - 待替换的分区 p1 的范围 (=> 并集)： - - ``` - [10, 50) => [10, 50) - ``` - - 替换分区 tp1, tp2 的范围(=> 并集)： - - ``` - [10, 30), [40, 50) => [10, 30), [40, 50) - ``` - - 范围并集不相同，如果 `strict_range` 为 true，则不可以使用 tp1 和 tp2 替换 p1。如果为 false，且替换后的两个分区范围 `[10, 30), [40, 50)` 和其他正式分区不重叠，则可以替换。 - -2. `use_temp_partition_name` - - 默认为 false。当该参数为 false，并且待替换的分区和替换分区的个数相同时，则替换后的正式分区名称维持不变。如果为 true，则替换后，正式分区的名称为替换分区的名称。下面举例说明： - - * 示例1 - - ``` - ALTER TABLE tbl1 REPLACE PARTITION (p1) WITH TEMPORARY PARTITION (tp1); - ``` - - `use_temp_partition_name` 默认为 false，则在替换后，分区的名称依然为 p1，但是相关的数据和属性都替换为 tp1 的。 - - 如果 `use_temp_partition_name` 默认为 true，则在替换后，分区的名称为 tp1。p1 分区不再存在。 - - * 示例2 - - ``` - ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1); - ``` - - `use_temp_partition_name` 默认为 false，但因为待替换分区的个数和替换分区的个数不同，则该参数无效。替换后，分区名称为 tp1，p1 和 p2 不再存在。 - -替换操作的一些说明： - -* 分区替换成功后，被替换的分区将被删除且不可恢复。 - -## 临时分区的导入和查询 - -用户可以将数据导入到临时分区，也可以指定临时分区进行查询。 - -1. 导入临时分区 - - 根据导入方式的不同，指定导入临时分区的语法稍有差别。这里通过示例进行简单说明 - - ``` - INSERT INTO tbl TEMPORARY PARTITION(tp1, tp2, ...) SELECT .... - ``` - - ``` - curl --location-trusted -u root: -H "label:123" -H "temporary_partitions: tp1, tp2, ..." -T testData http://host:port/api/testDb/testTbl/_stream_load - ``` - - ``` - LOAD LABEL example_db.label1 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - TEMPORARY PARTITION (tp1, tp2, ...) - ... - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - ``` - - ``` - CREATE ROUTINE LOAD example_db.test1 ON example_tbl - COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), - TEMPORARY PARTITIONS(tp1, tp2, ...), - WHERE k1 > 100 - PROPERTIES - (...) - FROM KAFKA - (...); - ``` - -2. 查询临时分区 - - ``` - SELECT ... FROM - tbl1 TEMPORARY PARTITION(tp1, tp2, ...) - JOIN - tbl2 TEMPORARY PARTITION(tp1, tp2, ...) - ON ... - WHERE ...; - ``` - -## 和其他操作的关系 - -### DROP - -* 使用 Drop 操作直接删除数据库或表后，可以通过 Recover 命令恢复数据库或表（限定时间内），但临时分区不会被恢复。 -* 使用 Alter 命令删除正式分区后，可以通过 Recover 命令恢复分区（限定时间内）。操作正式分区和临时分区无关。 -* 使用 Alter 命令删除临时分区后，无法通过 Recover 命令恢复临时分区。 - -### TRUNCATE - -* 使用 Truncate 命令清空表，表的临时分区会被删除，且不可恢复。 -* 使用 Truncate 命令清空正式分区时，不影响临时分区。 -* 不可使用 Truncate 命令清空临时分区。 - -### ALTER - -* 当表存在临时分区时，无法使用 Alter 命令对表进行 Schema Change、Rollup 等变更操作。 -* 当表在进行变更操作时，无法对表添加临时分区。 - - -## 最佳实践 - -1. 原子的覆盖写操作 - - 某些情况下，用户希望能够重写某一分区的数据，但如果采用先删除再导入的方式进行，在中间会有一段时间无法查看数据。这是，用户可以先创建一个对应的临时分区，将新的数据导入到临时分区后，通过替换操作，原子的替换原有分区，以达到目的。 - -2. 修改分桶数 - - 某些情况下，用户在创建分区时使用了不合适的分桶数。则用户可以先创建一个对应分区范围的临时分区，并指定新的分桶数。然后通过 `INSERT INTO` 命令将正式分区的数据导入到临时分区中，通过替换操作，原子的替换原有分区，以达到目的。 - -3. 合并或分割分区 - - 某些情况下，用户希望对分区的范围进行修改，比如合并两个分区，或将一个大分区分割成多个小分区。则用户可以先建立对应合并或分割后范围的临时分区，然后通过 `INSERT INTO` 命令将正式分区的数据导入到临时分区中，通过替换操作，原子的替换原有分区，以达到目的。 - - - - - - - - - - diff --git a/docs/documentation/cn/administrator-guide/alter-table/index.rst b/docs/documentation/cn/administrator-guide/alter-table/index.rst deleted file mode 100644 index 208741231d8ef0..00000000000000 --- a/docs/documentation/cn/administrator-guide/alter-table/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============= -表结构变更 -============= - -.. toctree:: - :maxdepth: 2 - :glob: - - * diff --git a/docs/documentation/cn/administrator-guide/backup-restore.md b/docs/documentation/cn/administrator-guide/backup-restore.md deleted file mode 100644 index ff5c262957eeb9..00000000000000 --- a/docs/documentation/cn/administrator-guide/backup-restore.md +++ /dev/null @@ -1,179 +0,0 @@ - - -# 备份与恢复 - -Doris 支持将当前数据以文件的形式，通过 broker 备份到远端存储系统中。之后可以通过 恢复 命令，从远端存储系统中将数据恢复到任意 Doris 集群。通过这个功能，Doris 可以支持将数据定期的进行快照备份。也可以通过这个功能，在不同集群间进行数据迁移。 - -该功能需要 Doris 版本 0.8.2+ - -使用该功能，需要部署对应远端存储的 broker。如 BOS、HDFS 等。可以通过 `SHOW BROKER;` 查看当前部署的 broker。 - -## 简要原理说明 - -### 备份（Backup） - -备份操作是将指定表或分区的数据，直接以 Doris 存储的文件的形式，上传到远端仓库中进行存储。当用户提交 Backup 请求后，系统内部会做如下操作： - -1. 快照及快照上传 - - 快照阶段会对指定的表或分区数据文件进行快照。之后，备份都是对快照进行操作。在快照之后，对表进行的更改、导入等操作都不再影响备份的结果。快照只是对当前数据文件产生一个硬链，耗时很少。快照完成后，会开始对这些快照文件进行逐一上传。快照上传由各个 Backend 并发完成。 - -2. 元数据准备及上传 - - 数据文件快照上传完成后，Frontend 会首先将对应元数据写成本地文件，然后通过 broker 将本地元数据文件上传到远端仓库。完成最终备份作业。 - -### 恢复（Restore） - -恢复操作需要指定一个远端仓库中已存在的备份，然后将这个备份的内容恢复到本地集群中。当用户提交 Restore 请求后，系统内部会做如下操作： - -1. 在本地创建对应的元数据 - - 这一步首先会在本地集群中，创建恢复对应的表分区等结构。创建完成后，该表可见，但是不可访问。 - -2. 本地snapshot - - 这一步是将上一步创建的表做一个快照。这其实是一个空快照（因为刚创建的表是没有数据的），其目的主要是在 Backend 上产生对应的快照目录，用于之后接收从远端仓库下载的快照文件。 - -3. 下载快照 - - 远端仓库中的快照文件，会被下载到对应的上一步生成的快照目录中。这一步由各个 Backend 并发完成。 - -4. 生效快照 - - 快照下载完成后，我们要将各个快照映射为当前本地表的元数据。然后重新加载这些快照，使之生效，完成最终的恢复作业。 - -## 最佳实践 - -### 备份 - -当前我们支持最小分区（Partition）粒度的全量备份（增量备份有可能在未来版本支持）。如果需要对数据进行定期备份，首先需要在建表时，合理的规划表的分区及分桶，比如按时间进行分区。然后在之后的运行过程中，按照分区粒度进行定期的数据备份。 - -### 数据迁移 - -用户可以先将数据备份到远端仓库，再通过远端仓库将数据恢复到另一个集群，完成数据迁移。因为数据备份是通过快照的形式完成的，所以，在备份作业的快照阶段之后的新的导入数据，是不会备份的。因此，在快照完成后，到恢复作业完成这期间，在原集群上导入的数据，都需要在新集群上同样导入一遍。 - -建议在迁移完成后，对新旧两个集群并行导入一段时间。完成数据和业务正确性校验后，再将业务迁移到新的集群。 - -## 重点说明 - -1. 备份恢复相关的操作目前只允许拥有 ADMIN 权限的用户执行。 -2. 一个 Database 内，只允许有一个正在执行的备份或恢复作业。 -3. 备份和恢复都支持最小分区（Partition）级别的操作，当表的数据量很大时，建议按分区分别执行，以降低失败重试的代价。 -4. 因为备份恢复操作，操作的都是实际的数据文件。所以当一个表的分片过多，或者一个分片有过多的小版本时，可能即使总数据量很小，依然需要备份或恢复很长时间。用户可以通过 `SHOW PARTITIONS FROM table_name;` 和 `SHOW TABLET FROM table_name;` 来查看各个分区的分片数量，以及各个分片的文件版本数量，来预估作业执行时间。文件数量对作业执行的时间影响非常大，所以建议在建表时，合理规划分区分桶，以避免过多的分片。 -5. 当通过 `SHOW BACKUP` 或者 `SHOW RESTORE` 命令查看作业状态时。有可能会在 `TaskErrMsg` 一列中看到错误信息。但只要 `State` 列不为 - `CANCELLED`，则说明作业依然在继续。这些 Task 有可能会重试成功。当然，有些 Task 错误，也会直接导致作业失败。 -6. 如果恢复作业是一次覆盖操作（指定恢复数据到已经存在的表或分区中），那么从恢复作业的 `COMMIT` 阶段开始，当前集群上被覆盖的数据有可能不能再被还原。此时如果恢复作业失败或被取消，有可能造成之前的数据已损坏且无法访问。这种情况下，只能通过再次执行恢复操作，并等待作业完成。因此，我们建议，如无必要，尽量不要使用覆盖的方式恢复数据，除非确认当前数据已不再使用。 - -## 相关命令 - -和备份恢复功能相关的命令如下。以下命令，都可以通过 mysql-client 连接 Doris 后，使用 `help cmd;` 的方式查看详细帮助。 - -1. CREATE REPOSITORY - - 创建一个远端仓库路径，用于备份或恢复。该命令需要借助 Broker 进程访问远端存储，不同的 Broker 需要提供不同的参数，具体请参阅 [Broker文档](broker.md) - -1. BACKUP - - 执行一次备份操作。 - -3. SHOW BACKUP - - 查看最近一次 backup 作业的执行情况，包括： - - * JobId：本次备份作业的 id。 - * SnapshotName：用户指定的本次备份作业的名称（Label）。 - * DbName：备份作业对应的 Database。 - * State：备份作业当前所在阶段： - * PENDING：作业初始状态。 - * SNAPSHOTING：正在进行快照操作。 - * UPLOAD_SNAPSHOT：快照结束，准备上传。 - * UPLOADING：正在上传快照。 - * SAVE_META：正在本地生成元数据文件。 - * UPLOAD_INFO：上传元数据文件和本次备份作业的信息。 - * FINISHED：备份完成。 - * CANCELLED：备份失败或被取消。 - * BackupObjs：本次备份涉及的表和分区的清单。 - * CreateTime：作业创建时间。 - * SnapshotFinishedTime：快照完成时间。 - * UploadFinishedTime：快照上传完成时间。 - * FinishedTime：本次作业完成时间。 - * UnfinishedTasks：在 `SNAPSHOTTING`，`UPLOADING` 等阶段，会有多个子任务在同时进行，这里展示的当前阶段，未完成的子任务的 task id。 - * TaskErrMsg：如果有子任务执行出错，这里会显示对应子任务的错误信息。 - * Status：用于记录在整个作业过程中，可能出现的一些状态信息。 - * Timeout：作业的超时时间，单位是秒。 - -4. SHOW SNAPSHOT - - 查看远端仓库中已存在的备份。 - - * Snapshot：备份时指定的该备份的名称（Label）。 - * Timestamp：备份的时间戳。 - * Status：该备份是否正常。 - - 如果在 `SHOW SNAPSHOT` 后指定了 where 子句，则可以显示更详细的备份信息。 - - * Database：备份时对应的 Database。 - * Details：展示了该备份完整的数据目录结构。 - -5. RESTORE - - 执行一次恢复操作。 - -6. SHOW RESTORE - - 查看最近一次 restore 作业的执行情况，包括： - - * JobId：本次恢复作业的 id。 - * Label：用户指定的仓库中备份的名称（Label）。 - * Timestamp：用户指定的仓库中备份的时间戳。 - * DbName：恢复作业对应的 Database。 - * State：恢复作业当前所在阶段： - * PENDING：作业初始状态。 - * SNAPSHOTING：正在进行本地新建表的快照操作。 - * DOWNLOAD：正在发送下载快照任务。 - * DOWNLOADING：快照正在下载。 - * COMMIT：准备生效已下载的快照。 - * COMMITTING：正在生效已下载的快照。 - * FINISHED：恢复完成。 - * CANCELLED：恢复失败或被取消。 - * AllowLoad：恢复期间是否允许导入。 - * ReplicationNum：恢复指定的副本数。 - * RestoreObjs：本次恢复涉及的表和分区的清单。 - * CreateTime：作业创建时间。 - * MetaPreparedTime：本地元数据生成完成时间。 - * SnapshotFinishedTime：本地快照完成时间。 - * DownloadFinishedTime：远端快照下载完成时间。 - * FinishedTime：本次作业完成时间。 - * UnfinishedTasks：在 `SNAPSHOTTING`，`DOWNLOADING`, `COMMITTING` 等阶段，会有多个子任务在同时进行，这里展示的当前阶段，未完成的子任务的 task id。 - * TaskErrMsg：如果有子任务执行出错，这里会显示对应子任务的错误信息。 - * Status：用于记录在整个作业过程中，可能出现的一些状态信息。 - * Timeout：作业的超时时间，单位是秒。 - -7. CANCEL BACKUP - - 取消当前正在执行的备份作业。 - -8. CANCEL RESTORE - - 取消当前正在执行的恢复作业。 - -9. DROP REPOSITORY - - 删除已创建的远端仓库。删除仓库，仅仅是删除该仓库在 Doris 中的映射，不会删除实际的仓库数据。 \ No newline at end of file diff --git a/docs/documentation/cn/administrator-guide/broker.md b/docs/documentation/cn/administrator-guide/broker.md deleted file mode 100644 index 0fdcff070689ed..00000000000000 --- a/docs/documentation/cn/administrator-guide/broker.md +++ /dev/null @@ -1,280 +0,0 @@ - - -# Broker - -Broker 是 Doris 集群中一种可选进程，主要用于支持 Doris 读写远端存储上的文件和目录，如 HDFS、BOS 和 AFS 等。 - -Broker 通过提供一个 RPC 服务端口来提供服务，是一个无状态的 Java 进程，负责为远端存储的读写操作封装一些类 POSIX 的文件操作，如 open，pread，pwrite 等等。除此之外，Broker 不记录任何其他信息，所以包括远端存储的连接信息、文件信息、权限信息等等，都需要通过参数在 RPC 调用中传递给 Broker 进程，才能使得 Broker 能够正确读写文件。 - -Broker 仅作为一个数据通路，并不参与任何计算，因此仅需占用较少的内存。通常一个 Doris 系统中会部署一个或多个 Broker 进程。并且相同类型的 Broker 会组成一个组，并设定一个 **名称（Broker name）**。 - -Broker 在 Doris 系统架构中的位置如下： - -``` -+----+ +----+ -| FE | | BE | -+-^--+ +--^-+ - | | - | | -+-v---------v-+ -| Broker | -+------^------+ - | - | -+------v------+ -|HDFS/BOS/AFS | -+-------------+ -``` - -本文档主要介绍 Broker 在访问不同远端存储时需要的参数，如连接信息、权限认证信息等等。 - -## 支持的存储系统 - -不同的 Broker 类型支持不同的存储系统。 - -1. 社区版 HDFS - - * 支持简单认证访问 - * 支持通过 kerberos 认证访问 - * 支持 HDFS HA 模式访问 - -2. 百度 HDFS/AFS（开源版本不支持） - - * 支持通过 ugi 简单认证访问 - -3. 百度对象存储 BOS（开源版本不支持） - - * 支持通过 AK/SK 认证访问 - -## 需要 Broker 的操作 - -1. Broker Load - - Broker Load 功能通过 Broker 进程读取远端存储上的文件数据并导入到 Doris 中。示例如下： - - ``` - LOAD LABEL example_db.label6 - ( - DATA INFILE("bos://my_bucket/input/file") - INTO TABLE `my_table` - ) - WITH BROKER "broker_name" - ( - "bos_endpoint" = "http://bj.bcebos.com", - "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "bos_secret_accesskey" = "yyyyyyyyyyyyyyyyyyyy" - ) - ``` - - 其中 `WITH BROKER` 以及之后的 Property Map 用于提供 Broker 相关信息。 - -2. 数据导出（Export） - - Export 功能通过 Broker 进程，将 Doris 中存储的数据以文本的格式导出到远端存储的文件中。示例如下： - - ``` - EXPORT TABLE testTbl - TO "hdfs://hdfs_host:port/a/b/c" - WITH BROKER "broker_name" - ( - "username" = "xxx", - "password" = "yyy" - ); - ``` - - 其中 `WITH BROKER` 以及之后的 Property Map 用于提供 Broker 相关信息。 - -3. 创建用于备份恢复的仓库（Create Repository） - - 当用户需要使用备份恢复功能时，需要先通过 `CREATE REPOSITORY` 命令创建一个 “仓库”，仓库元信息中记录了所使用的 Broker 以及相关信息。之后的备份恢复操作，会通过 Broker 将数据备份到这个仓库，或从这个仓库读取数据恢复到 Doris 中。示例如下： - - ``` - CREATE REPOSITORY `bos_repo` - WITH BROKER `broker_name` - ON LOCATION "bos://doris_backup" - PROPERTIES - ( - "bos_endpoint" = "http://gz.bcebos.com", - "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", - "bos_secret_accesskey" = "70999999999999de274d59eaa980a" - ); - ``` - - 其中 `WITH BROKER` 以及之后的 Property Map 用于提供 Broker 相关信息。 - - -## Broker 信息 - -Broker 的信息包括 **名称（Broker name）** 和 **认证信息** 两部分。通常的语法格式如下： - -``` -WITH BROKER "broker_name" -( - "username" = "xxx", - "password" = "yyy", - "other_prop" = "prop_value", - ... -); -``` - -### 名称 - -通常用户需要通过操作命令中的 `WITH BROKER "broker_name"` 子句来指定一个已经存在的 Broker Name。Broker Name 是用户在通过 `ALTER SYSTEM ADD BROKER` 命令添加 Broker 进程时指定的一个名称。一个名称通常对应一个或多个 Broker 进程。Doris 会根据名称选择可用的 Broker 进程。用户可以通过 `SHOW BROKER` 命令查看当前集群中已经存在的 Broker。 - -**注：Broker Name 只是一个用户自定义名称，不代表 Broker 的类型。** - -### 认证信息 - -不同的 Broker 类型，以及不同的访问方式需要提供不同的认证信息。认证信息通常在 `WITH BROKER "broker_name"` 之后的 Property Map 中以 Key-Value 的方式提供。 - -#### 社区版 HDFS - -1. 简单认证 - - 简单认证即 Hadoop 配置 `hadoop.security.authentication` 为 `simple`。 - - 使用系统用户访问 HDFS。或者在 Broker 启动的环境变量中添加：```HADOOP_USER_NAME```。 - - ``` - ( - "username" = "user", - "password" = "" - ); - ``` - - 密码置空即可。 - -2. Kerberos 认证 - - 该认证方式需提供以下信息： - - * `hadoop.security.authentication`：指定认证方式为 kerberos。 - * `kerberos_principal`：指定 kerberos 的 principal。 - * `kerberos_keytab`：指定 kerberos 的 keytab 文件路径。该文件必须为 Broker 进程所在服务器上的文件的绝对路径。并且可以被 Broker 进程访问。 - * `kerberos_keytab_content`：指定 kerberos 中 keytab 文件内容经过 base64 编码之后的内容。这个跟 `kerberos_keytab` 配置二选一即可。 - - 示例如下： - - ``` - ( - "hadoop.security.authentication" = "kerberos", - "kerberos_principal" = "doris@YOUR.COM", - "kerberos_keytab" = "/home/doris/my.keytab" - ) - ``` - ``` - ( - "hadoop.security.authentication" = "kerberos", - "kerberos_principal" = "doris@YOUR.COM", - "kerberos_keytab_content" = "ASDOWHDLAWIDJHWLDKSALDJSDIWALD" - ) - ``` - 如果采用Kerberos认证方式，则部署Broker进程的时候需要[krb5.conf](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html)文件， - krb5.conf文件包含Kerberos的配置信息，通常，您应该将krb5.conf文件安装在目录/etc中。您可以通过设置环境变量KRB5_CONFIG覆盖默认位置。 - krb5.conf文件的内容示例如下： - ``` - [libdefaults] - default_realm = DORIS.HADOOP - default_tkt_enctypes = des3-hmac-sha1 des-cbc-crc - default_tgs_enctypes = des3-hmac-sha1 des-cbc-crc - dns_lookup_kdc = true - dns_lookup_realm = false - - [realms] - DORIS.HADOOP = { - kdc = kerberos-doris.hadoop.service:7005 - } - ``` - -3. HDFS HA 模式 - - 这个配置用于访问以 HA 模式部署的 HDFS 集群。 - - * `dfs.nameservices`：指定 hdfs 服务的名字，自定义，如："dfs.nameservices" = "my_ha"。 - * `dfs.ha.namenodes.xxx`：自定义 namenode 的名字,多个名字以逗号分隔。其中 xxx 为 `dfs.nameservices` 中自定义的名字，如： "dfs.ha.namenodes.my_ha" = "my_nn"。 - * `dfs.namenode.rpc-address.xxx.nn`：指定 namenode 的rpc地址信息。其中 nn 表示 `dfs.ha.namenodes.xxx` 中配置的 namenode 的名字，如："dfs.namenode.rpc-address.my_ha.my_nn" = "host:port"。 - * `dfs.client.failover.proxy.provider`：指定 client 连接 namenode 的 provider，默认为：org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider。 - - 示例如下： - - ``` - ( - "dfs.nameservices" = "my_ha", - "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", - "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", - "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", - "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" - ) - ``` - - HA 模式可以和前面两种认证方式组合，进行集群访问。如通过简单认证访问 HA HDFS： - - ``` - ( - "username"="user", - "password"="passwd", - "dfs.nameservices" = "my_ha", - "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", - "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", - "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", - "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" - ) - ``` - 关于HDFS集群的配置可以写入hdfs-site.xml文件中，用户使用Broker进程读取HDFS集群的信息时，只需要填写集群的文件路径名和认证信息即可。 - -#### 百度对象存储 BOS - -**（开源版本不支持）** - -1. 通过 AK/SK 访问 - - * AK/SK：Access Key 和 Secret Key。在百度云安全认证中心可以查看用户的 AK/SK。 - * Region Endpoint：BOS 所在地区的 Endpoint： - - * 华北-北京：http://bj.bcebos.com - * 华北-保定：http://bd.bcebos.com - * 华南-广州：http://gz.bcebos.com - * 华东-苏州：http://sz.bcebos.com - - - 示例如下： - - ``` - ( - "bos_endpoint" = "http://bj.bcebos.com", - "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "bos_secret_accesskey" = "yyyyyyyyyyyyyyyyyyyyyyyyyy" - ) - ``` - -#### 百度 HDFS/AFS - -**（开源版本不支持）** - -百度 AFS 和 HDFS 仅支持使用 ugi 的简单认证访问。示例如下： - -``` -( - "username" = "user", - "password" = "passwd" -); -``` - -其中 user 和 passwd 为 Hadoop 的 UGI 配置。 diff --git a/docs/documentation/cn/administrator-guide/colocation-join.md b/docs/documentation/cn/administrator-guide/colocation-join.md deleted file mode 100644 index 463f6b5b80c3dc..00000000000000 --- a/docs/documentation/cn/administrator-guide/colocation-join.md +++ /dev/null @@ -1,441 +0,0 @@ - - -# Colocation Join - -Colocation Join 是在 Doris 0.9 版本中引入的新功能。旨在为某些 Join 查询提供本地性优化，来减少数据在节点间的传输耗时，加速查询。 - -最初的设计、实现和效果可以参阅 [ISSUE 245](https://github.com/apache/incubator-doris/issues/245)。 - -Colocation Join 功能经过一次改版，设计和使用方式和最初设计稍有不同。本文档主要介绍 Colocation Join 的原理、实现、使用方式和注意事项。 - -## 名词解释 - -* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 -* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 -* Colocation Group（CG）：一个 CG 中会包含一张及以上的 Table。在同一个 Group 内的 Table 有着相同的 Colocation Group Schema，并且有着相同的数据分片分布。 -* Colocation Group Schema（CGS）：用于描述一个 CG 中的 Table，和 Colocation 相关的通用 Schema 信息。包括分桶列类型，分桶数以及副本数等。 - -## 原理 - -Colocation Join 功能，是将一组拥有相同 CGS 的 Table 组成一个 CG。并保证这些 Table 对应的数据分片会落在同一个 BE 节点上。使得当 CG 内的表进行分桶列上的 Join 操作时，可以通过直接进行本地数据 Join，减少数据在节点间的传输耗时。 - -一个表的数据，最终会根据分桶列值 Hash、对桶数取模的后落在某一个分桶内。假设一个 Table 的分桶数为 8，则共有 `[0, 1, 2, 3, 4, 5, 6, 7]` 8 个分桶（Bucket），我们称这样一个序列为一个 `BucketsSequence`。每个 Bucket 内会有一个或多个数据分片（Tablet）。当表为单分区表时，一个 Bucket 内仅有一个 Tablet。如果是多分区表，则会有多个。 - -为了使得 Table 能够有相同的数据分布，同一 CG 内的 Table 必须保证以下属性相同： - -1. 分桶列和分桶数 - - 分桶列，即在建表语句中 `DISTRIBUTED BY HASH(col1, col2, ...)` 中指定的列。分桶列决定了一张表的数据通过哪些列的值进行 Hash 划分到不同的 Tablet 中。同一 CG 内的 Table 必须保证分桶列的类型和数量完全一致，并且桶数一致，才能保证多张表的数据分片能够一一对应的进行分布控制。 - -2. 副本数 - - 同一个 CG 内所有表的所有分区（Partition）的副本数必须一致。如果不一致，可能出现某一个 Tablet 的某一个副本，在同一个 BE 上没有其他的表分片的副本对应。 - -同一个 CG 内的表，分区的个数、范围以及分区列的类型不要求一致。 - -在固定了分桶列和分桶数后，同一个 CG 内的表会拥有相同的 BucketsSequnce。而副本数决定了每个分桶内的 Tablet 的多个副本，存放在哪些 BE 上。假设 BucketsSequnce 为 `[0, 1, 2, 3, 4, 5, 6, 7]`，BE 节点有 `[A, B, C, D]` 4个。则一个可能的数据分布如下： - -``` -+---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ -| 0 | | 1 | | 2 | | 3 | | 4 | | 5 | | 6 | | 7 | -+---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ -| A | | B | | C | | D | | A | | B | | C | | D | -| | | | | | | | | | | | | | | | -| B | | C | | D | | A | | B | | C | | D | | A | -| | | | | | | | | | | | | | | | -| C | | D | | A | | B | | C | | D | | A | | B | -+---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ -``` - -CG 内所有表的数据都会按照上面的规则进行统一分布，这样就保证了，分桶列值相同的数据都在同一个 BE 节点上，可以进行本地数据 Join。 - -## 使用方式 - -### 建表 - -建表时，可以在 `PROPERTIES` 中指定属性 `"colocate_with" = "group_name"`，表示这个表是一个 Colocation Join 表，并且归属于一个指定的 Colocation Group。 - -示例： - -``` -CREATE TABLE tbl (k1 int, v1 int sum) -DISTRIBUTED BY HASH(k1) -BUCKETS 8 -PROPERTIES( - "colocate_with" = "group1" -); -``` - -如果指定的 Group 不存在，则 Doris 会自动创建一个只包含当前这张表的 Group。如果 Group 已存在，则 Doris 会检查当前表是否满足 Colocation Group Schema。如果满足，则会创建该表，并将该表加入 Group。同时，表会根据已存在的 Group 中的数据分布规则创建分片和副本。 -Group 归属于一个 Database，Group 的名字在一个 Database 内唯一。在内部存储是 Group 的全名为 `dbId_groupName`，但用户只感知 groupName。 - -### 删表 - -当 Group 中最后一张表彻底删除后（彻底删除是指从回收站中删除。通常，一张表通过 `DROP TABLE` 命令删除后，会在回收站默认停留一天的时间后，再删除），该 Group 也会被自动删除。 - -### 查看 Group - -以下命令可以查看集群内已存在的 Group 信息。 - -``` -SHOW PROC '/colocation_group'; - -+-------------+--------------+--------------+------------+----------------+----------+----------+ -| GroupId | GroupName | TableIds | BucketsNum | ReplicationNum | DistCols | IsStable | -+-------------+--------------+--------------+------------+----------------+----------+----------+ -| 10005.10008 | 10005_group1 | 10007, 10040 | 10 | 3 | int(11) | true | -+-------------+--------------+--------------+------------+----------------+----------+----------+ -``` - -* GroupId： 一个 Group 的全集群唯一标识，前半部分为 db id，后半部分为 group id。 -* GroupName： Group 的全名。 -* TabletIds： 该 Group 包含的 Table 的 id 列表。 -* BucketsNum： 分桶数。 -* ReplicationNum： 副本数。 -* DistCols： Distribution columns，即分桶列类型。 -* IsStable： 该 Group 是否稳定（稳定的定义，见 `Colocation 副本均衡和修复` 一节）。 - -通过以下命令可以进一步查看一个 Group 的数据分布情况： - -``` -SHOW PROC '/colocation_group/10005.10008'; - -+-------------+---------------------+ -| BucketIndex | BackendIds | -+-------------+---------------------+ -| 0 | 10004, 10002, 10001 | -| 1 | 10003, 10002, 10004 | -| 2 | 10002, 10004, 10001 | -| 3 | 10003, 10002, 10004 | -| 4 | 10002, 10004, 10003 | -| 5 | 10003, 10002, 10001 | -| 6 | 10003, 10004, 10001 | -| 7 | 10003, 10004, 10002 | -+-------------+---------------------+ -``` - -* BucketIndex： 分桶序列的下标。 -* BackendIds： 分桶中数据分片所在的 BE 节点 id 列表。 - -> 以上命令需要 AMDIN 权限。暂不支持普通用户查看。 - -### 修改表 Colocate Group 属性 - -可以对一个已经创建的表，修改其 Colocation Group 属性。示例： - -`ALTER TABLE tbl SET ("colocate_with" = "group2");` - -* 如果该表之前没有指定过 Group，则该命令检查 Schema，并将该表加入到该 Group（Group 不存在则会创建）。 -* 如果该表之前有指定其他 Group，则该命令会先将该表从原有 Group 中移除，并加入新 Group（Group 不存在则会创建）。 - -也可以通过以下命令，删除一个表的 Colocation 属性： - -`ALTER TABLE tbl SET ("colocate_with" = "");` - -### 其他相关操作 - -当对一个具有 Colocation 属性的表进行增加分区（ADD PARTITION）、修改副本数时，Doris 会检查修改是否会违反 Colocation Group Schema，如果违反则会拒绝。 - -## Colocation 副本均衡和修复 - -Colocation 表的副本分布需要遵循 Group 中指定的分布，所以在副本修复和均衡方面和普通分片有所区别。 - -Group 自身有一个 Stable 属性，当 Stable 为 true 时，表示当前 Group 内的表的所有分片没有正在进行变动，Colocation 特性可以正常使用。当 Stable 为 false 时（Unstable），表示当前 Group 内有部分表的分片正在做修复或迁移，此时，相关表的 Colocation Join 将退化为普通 Join。 - -### 副本修复 - -副本只能存储在指定的 BE 节点上。所以当某个 BE 不可用时（宕机、Decommission 等），需要寻找一个新的 BE 进行替换。Doris 会优先寻找负载最低的 BE 进行替换。替换后，该 Bucket 内的所有在旧 BE 上的数据分片都要做修复。迁移过程中，Group 被标记为 Unstable。 - -### 副本均衡 - -Doris 会尽力将 Colocation 表的分片均匀分布在所有 BE 节点上。对于普通表的副本均衡，是以单副本为粒度的，即单独为每一个副本寻找负载较低的 BE 节点即可。而 Colocation 表的均衡是 Bucket 级别的，即一个 Bucket 内的所有副本都会一起迁移。我们采用一个简单的均衡算法，即在不考虑副本实际大小，而只根据副本数量，将 BucketsSequnce 均匀的分布在所有 BE 上。具体算法可以参阅 `ColocateTableBalancer.java` 中的代码注释。 - -> 注1：当前的 Colocation 副本均衡和修复算法，对于异构部署的 Doris 集群效果可能不佳。所谓异构部署，即 BE 节点的磁盘容量、数量、磁盘类型（SSD 和 HDD）不一致。在异构部署情况下，可能出现小容量的 BE 节点和大容量的 BE 节点存储了相同的副本数量。 -> -> 注2：当一个 Group 处于 Unstable 状态时，其中的表的 Join 将退化为普通 Join。此时可能会极大降低集群的查询性能。如果不希望系统自动均衡，可以设置 FE 的配置项 `disable_colocate_balance` 来禁止自动均衡。然后在合适的时间打开即可。（具体参阅 `高级操作` 一节） - -## 查询 - -对 Colocation 表的查询方式和普通表一样，用户无需感知 Colocation 属性。如果 Colocation 表所在的 Group 处于 Unstable 状态，将自动退化为普通 Join。 - -举例说明： - -表1： - -``` -CREATE TABLE `tbl1` ( - `k1` date NOT NULL COMMENT "", - `k2` int(11) NOT NULL COMMENT "", - `v1` int(11) SUM NOT NULL COMMENT "" -) ENGINE=OLAP -AGGREGATE KEY(`k1`, `k2`) -PARTITION BY RANGE(`k1`) -( - PARTITION p1 VALUES LESS THAN ('2019-05-31'), - PARTITION p2 VALUES LESS THAN ('2019-06-30') -) -DISTRIBUTED BY HASH(`k2`) BUCKETS 8 -PROPERTIES ( - "colocate_with" = "group1" -); -``` - -表2： - -``` -CREATE TABLE `tbl2` ( - `k1` datetime NOT NULL COMMENT "", - `k2` int(11) NOT NULL COMMENT "", - `v1` double SUM NOT NULL COMMENT "" -) ENGINE=OLAP -AGGREGATE KEY(`k1`, `k2`) -DISTRIBUTED BY HASH(`k2`) BUCKETS 8 -PROPERTIES ( - "colocate_with" = "group1" -); -``` - -查看查询计划： - -``` -DESC SELECT * FROM tbl1 INNER JOIN tbl2 ON (tbl1.k2 = tbl2.k2); - -+----------------------------------------------------+ -| Explain String | -+----------------------------------------------------+ -| PLAN FRAGMENT 0 | -| OUTPUT EXPRS:`tbl1`.`k1` | | -| PARTITION: RANDOM | -| | -| RESULT SINK | -| | -| 2:HASH JOIN | -| | join op: INNER JOIN | -| | hash predicates: | -| | colocate: true | -| | `tbl1`.`k2` = `tbl2`.`k2` | -| | tuple ids: 0 1 | -| | | -| |----1:OlapScanNode | -| | TABLE: tbl2 | -| | PREAGGREGATION: OFF. Reason: null | -| | partitions=0/1 | -| | rollup: null | -| | buckets=0/0 | -| | cardinality=-1 | -| | avgRowSize=0.0 | -| | numNodes=0 | -| | tuple ids: 1 | -| | | -| 0:OlapScanNode | -| TABLE: tbl1 | -| PREAGGREGATION: OFF. Reason: No AggregateInfo | -| partitions=0/2 | -| rollup: null | -| buckets=0/0 | -| cardinality=-1 | -| avgRowSize=0.0 | -| numNodes=0 | -| tuple ids: 0 | -+----------------------------------------------------+ -``` -如果 Colocation Join 生效，则 Hash Join 节点会显示 `colocate: true`。 - -如果没有生效，则查询计划如下： - -``` -+----------------------------------------------------+ -| Explain String | -+----------------------------------------------------+ -| PLAN FRAGMENT 0 | -| OUTPUT EXPRS:`tbl1`.`k1` | | -| PARTITION: RANDOM | -| | -| RESULT SINK | -| | -| 2:HASH JOIN | -| | join op: INNER JOIN (BROADCAST) | -| | hash predicates: | -| | colocate: false, reason: group is not stable | -| | `tbl1`.`k2` = `tbl2`.`k2` | -| | tuple ids: 0 1 | -| | | -| |----3:EXCHANGE | -| | tuple ids: 1 | -| | | -| 0:OlapScanNode | -| TABLE: tbl1 | -| PREAGGREGATION: OFF. Reason: No AggregateInfo | -| partitions=0/2 | -| rollup: null | -| buckets=0/0 | -| cardinality=-1 | -| avgRowSize=0.0 | -| numNodes=0 | -| tuple ids: 0 | -| | -| PLAN FRAGMENT 1 | -| OUTPUT EXPRS: | -| PARTITION: RANDOM | -| | -| STREAM DATA SINK | -| EXCHANGE ID: 03 | -| UNPARTITIONED | -| | -| 1:OlapScanNode | -| TABLE: tbl2 | -| PREAGGREGATION: OFF. Reason: null | -| partitions=0/1 | -| rollup: null | -| buckets=0/0 | -| cardinality=-1 | -| avgRowSize=0.0 | -| numNodes=0 | -| tuple ids: 1 | -+----------------------------------------------------+ -``` - -HASH JOIN 节点会显示对应原因：`colocate: false, reason: group is not stable`。同时会有一个 EXCHANGE 节点生成。 - - -## 高级操作 - -### FE 配置项 - -* disable\_colocate\_relocate - - 是否关闭 Doris 的自动 Colocation 副本修复。默认为 false，即不关闭。该参数只影响 Colocation 表的副本修复，不影响普通表。 - -* disable\_colocate\_balance - - 是否关闭 Doris 的自动 Colocation 副本均衡。默认为 false，即不关闭。该参数只影响 Colocation 表的副本均衡，不影响普通表。 - -以上参数可以动态修改，设置方式请参阅 `HELP ADMIN SHOW CONFIG;` 和 `HELP ADMIN SET CONFIG;`。 - -* disable\_colocate\_join - - 是否关闭 Colocation Join 功能。在 0.10 及之前的版本，默认为 true，即关闭。在之后的某个版本中将默认为 false，即开启。 - -* use\_new\_tablet\_scheduler - - 在 0.10 及之前的版本中，新的副本调度逻辑与 Colocation Join 功能不兼容，所以在 0.10 及之前版本，如果 `disable_colocate_join = false`，则需设置 `use_new_tablet_scheduler = false`，即关闭新的副本调度器。之后的版本中，`use_new_tablet_scheduler` 将衡为 true。 - -### HTTP Restful API - -Doris 提供了几个和 Colocation Join 有关的 HTTP Restful API，用于查看和修改 Colocation Group。 - -该 API 实现在 FE 端，使用 `fe_host:fe_http_port` 进行访问。需要 ADMIN 权限。 - -1. 查看集群的全部 Colocation 信息 - - ``` - GET /api/colocate - - 返回以 Json 格式表示内部 Colocation 信息。 - - { - "colocate_meta": { - "groupName2Id": { - "g1": { - "dbId": 10005, - "grpId": 10008 - } - }, - "group2Tables": {}, - "table2Group": { - "10007": { - "dbId": 10005, - "grpId": 10008 - }, - "10040": { - "dbId": 10005, - "grpId": 10008 - } - }, - "group2Schema": { - "10005.10008": { - "groupId": { - "dbId": 10005, - "grpId": 10008 - }, - "distributionColTypes": [{ - "type": "INT", - "len": -1, - "isAssignedStrLenInColDefinition": false, - "precision": 0, - "scale": 0 - }], - "bucketsNum": 10, - "replicationNum": 2 - } - }, - "group2BackendsPerBucketSeq": { - "10005.10008": [ - [10004, 10002], - [10003, 10002], - [10002, 10004], - [10003, 10002], - [10002, 10004], - [10003, 10002], - [10003, 10004], - [10003, 10004], - [10003, 10004], - [10002, 10004] - ] - }, - "unstableGroups": [] - }, - "status": "OK" - } - ``` - -2. 将 Group 标记为 Stable 或 Unstable - - * 标记为 Stable - - ``` - POST /api/colocate/group_stable?db_id=10005&group_id=10008 - - 返回：200 - ``` - - * 标记为 Unstable - - ``` - DELETE /api/colocate/group_stable?db_id=10005&group_id=10008 - - 返回：200 - ``` - -3. 设置 Group 的数据分布 - - 该接口可以强制设置某一 Group 的数分布。 - - ``` - POST /api/colocate/bucketseq?db_id=10005&group_id= 10008 - - Body: - [[10004,10002],[10003,10002],[10002,10004],[10003,10002],[10002,10004],[10003,10002],[10003,10004],[10003,10004],[10003,10004],[10002,10004]] - - 返回 200 - ``` - 其中 Body 是以嵌套数组表示的 BucketsSequence 以及每个 Bucket 中分片分布所在 BE 的 id。 - - 注意，使用该命令，可能需要将 FE 的配置 `disable_colocate_relocate` 和 `disable_colocate_balance` 设为 true。即关闭系统自动的 Colocation 副本修复和均衡。否则可能在修改后，会被系统自动重置。 \ No newline at end of file diff --git a/docs/documentation/cn/administrator-guide/config/fe_config.md b/docs/documentation/cn/administrator-guide/config/fe_config.md deleted file mode 100644 index 241dbe1b9c5c84..00000000000000 --- a/docs/documentation/cn/administrator-guide/config/fe_config.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# 基本配置 - -## brpc_max_body_size - - 这个配置主要用来修改 brpc 的参数 max_body_size ，默认配置是 64M。一般发生在 multi distinct + 无 group by + 超过1T 数据量的情况下。尤其如果发现查询卡死，且 BE 出现类似 body_size is too large 的字样。 - - 由于这是一个 brpc 的配置，用户也可以在运行中直接修改该参数。通过访问 http://host:brpc_port/flags 修改。 - -## max_running_txn_num_per_db - - 这个配置主要是用来控制同一个 db 的并发导入个数的，默认配置是100。当导入的并发执行的个数超过这个配置的值的时候，同步执行的导入就会失败比如 stream load。异步执行的导入就会一直处在 pending 状态比如 broker load。 - - 一般来说不推荐更改这个并发数。如果当前导入并发超过这个值，则需要先检查是否单个导入任务过慢，或者小文件太多没有合并后导入的问题。 - - 报错信息比如：current running txns on db xxx is xx, larger than limit xx。就属于这类问题。 diff --git a/docs/documentation/cn/administrator-guide/config/index.rst b/docs/documentation/cn/administrator-guide/config/index.rst deleted file mode 100644 index 15374d5cd34b9b..00000000000000 --- a/docs/documentation/cn/administrator-guide/config/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============= -配置文件 -============= - -.. toctree:: - :maxdepth: 1 - :glob: - - * diff --git a/docs/documentation/cn/administrator-guide/dynamic-partition.md b/docs/documentation/cn/administrator-guide/dynamic-partition.md deleted file mode 100644 index bf5d4f410307ab..00000000000000 --- a/docs/documentation/cn/administrator-guide/dynamic-partition.md +++ /dev/null @@ -1,201 +0,0 @@ - - -# 动态分区 - -动态分区是在 Doris 0.12 版本中引入的新功能。旨在对表级别的分区实现生命周期管理(TTL)，减少用户的使用负担。 - -最初的设计、实现和效果可以参阅 [ISSUE 2262](https://github.com/apache/incubator-doris/issues/2262)。 - -目前实现了动态添加分区及动态删除分区的功能。 - -## 名词解释 - -* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 -* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 - -## 原理 - -在某些使用场景下，用户会将表按照天进行分区划分，每天定时执行例行任务，这时需要使用方手动管理分区，否则可能由于使用方没有创建分区导致数据导入失败，这给使用方带来了额外的维护成本。 - -在实现方式上, FE会启动一个后台线程，根据fe.conf中`dynamic_partition_enable` 及 `dynamic_partition_check_interval_seconds`参数决定该线程是否启动以及该线程的调度频率。每次调度时，会在注册表中读取动态分区表的属性，并根据动态分区属性动态添加及删除分区。 - -建表时，在properties中指定dynamic_partition属性，FE首先对动态分区属性进行解析，校验输入参数的合法性，然后将对应的属性持久化到FE的元数据中，并将该表注册到动态分区列表中，后台线程会根据配置参数定期对动态分区列表进行扫描，读取表的动态分区属性，执行添加分区及删除分区的任务，每次的调度信息会保留在FE的内存中（重启后则丢失），可以通过`SHOW DYNAMIC PARTITION TABLES`查看调度任务是否成功，如果存在分区创建或删除失败，会将失败信息输出。 - -## 使用方式 - -### 动态分区属性参数说明: - -`dynamic_partition.enable`: 是否开启动态分区特性，可指定为 `TRUE` 或 `FALSE`。如果不填写，默认为 `TRUE`。 - - -`dynamic_partition.time_unit`: 动态分区调度的单位，可指定为 `DAY` `WEEK` `MONTH`，当指定为 `DAY` 时，动态创建的分区名后缀格式为`yyyyMMdd`，例如`20200325`。当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。 - -`dynamic_partition.start`: 动态分区的开始时间, 以当天为基准，超过该时间范围的分区将会被删除。如果不填写，则默认为`Integer.MIN_VALUE` 即 `-2147483648`。 - - -`dynamic_partition.end`: 动态分区的结束时间, 以当天为基准，会提前创建N个单位的分区范围。 - -`dynamic_partition.prefix`: 动态创建的分区名前缀。 - -`dynamic_partition.buckets`: 动态创建的分区所对应的分桶数量。 - -### 建表 - -建表时，可以在 `PROPERTIES` 中指定以下`dynamic_partition`属性，表示这个表是一个动态分区表。 - -示例： - -``` -CREATE TABLE example_db.dynamic_partition -( -k1 DATE, -k2 INT, -k3 SMALLINT, -v1 VARCHAR(2048), -v2 DATETIME DEFAULT "2014-02-04 15:36:00" -) -ENGINE=olap -DUPLICATE KEY(k1, k2, k3) -PARTITION BY RANGE (k1) -( -PARTITION p20200321 VALUES LESS THAN ("2020-03-22"), -PARTITION p20200322 VALUES LESS THAN ("2020-03-23"), -PARTITION p20200323 VALUES LESS THAN ("2020-03-24"), -PARTITION p20200324 VALUES LESS THAN ("2020-03-25") -) -DISTRIBUTED BY HASH(k2) BUCKETS 32 -PROPERTIES( -"storage_medium" = "SSD", -"dynamic_partition.enable" = "true", -"dynamic_partition.time_unit" = "DAY", -"dynamic_partition.start" = "-3", -"dynamic_partition.end" = "3", -"dynamic_partition.prefix" = "p", -"dynamic_partition.buckets" = "32" - ); -``` -创建一张动态分区表，指定开启动态分区特性，以当天为2020-03-25为例，在每次调度时，会删除分区上界小于 `2020-03-22` 的分区，为了避免删除非动态创建的分区，动态删除分区只会删除分区名符合动态创建分区规则的分区，例如分区名为a1, 则即使分区范围在待删除的分区范围内，也不会被删除。同时在调度时会提前创建今天以及以后3天（总共4天）的分区(若分区已存在则会忽略)，分区名根据指定前缀分别为`p20200325` `p20200326` `p20200327` `p20200328`,每个分区的分桶数量为32。同时会删除 `p20200321` 的分区，最终的分区范围如下: -``` -[types: [DATE]; keys: [2020-03-22]; ‥types: [DATE]; keys: [2020-03-23]; ) -[types: [DATE]; keys: [2020-03-23]; ‥types: [DATE]; keys: [2020-03-24]; ) -[types: [DATE]; keys: [2020-03-24]; ‥types: [DATE]; keys: [2020-03-25]; ) -[types: [DATE]; keys: [2020-03-25]; ‥types: [DATE]; keys: [2020-03-26]; ) -[types: [DATE]; keys: [2020-03-26]; ‥types: [DATE]; keys: [2020-03-27]; ) -[types: [DATE]; keys: [2020-03-27]; ‥types: [DATE]; keys: [2020-03-28]; ) -[types: [DATE]; keys: [2020-03-28]; ‥types: [DATE]; keys: [2020-03-29]; ) -``` - -### 开启动态分区功能 -1. 首先需要在fe.conf中设置`dynamic_partition_enable=true`，可以在集群启动时通过修改配置文件指定，或者通过MySQL连接后使用命令行 `ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true")`修改，也可以在运行时通过http接口动态修改,修改方法查看高级操作部分 - -2. 如果需要对0.12版本之前的表添加动态分区属性，则需要通过以下命令修改表的属性 -``` -ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32"); -``` - -### 停止动态分区功能 - -如果需要对集群中所有动态分区表停止动态分区功能，则需要在fe.conf中设置`dynamic_partition_enable=false` - -如果需要对指定表停止动态分区功能，则可以通过以下命令修改表的属性 -``` -ALTER TABLE dynamic_partition SET ("dynamic_partition.enable" = "false") -``` - -### 修改动态分区属性 - -通过如下命令可以修改动态分区的属性 -``` -ALTER TABLE dynamic_partition SET ("key" = "value") -``` - -### 查看动态分区表调度情况 - -通过以下命令可以进一步查看动态分区表的调度情况： - -``` -SHOW DYNAMIC PARTITION TABLES; - -+-------------------+--------+----------+-------+------+--------+---------+---------------------+---------------------+--------+------------------------+----------------------+ -| TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | -+-------------------+--------+----------+-------+------+--------+---------+---------------------+---------------------+--------+------------------------+----------------------+ -| dynamic_partition | true | DAY | -3 | 3 | p | 32 | 2020-03-12 17:25:47 | 2020-03-12 17:25:52 | NORMAL | N/A | N/A | -+-------------------+--------+----------+-------+------+--------+---------+---------------------+---------------------+--------+------------------------+----------------------+ -1 row in set (0.00 sec) - -``` - -* LastUpdateTime: 最后一次修改动态分区属性的时间 -* LastSchedulerTime: 最后一次执行动态分区调度的时间 -* State: 最后一次执行动态分区调度的状态 -* LastCreatePartitionMsg: 最后一次执行动态添加分区调度的错误信息 -* LastDropPartitionMsg: 最后一次执行动态删除分区调度的错误信息 - -## 高级操作 - -### FE 配置项 - -* dynamic\_partition\_enable - - 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。 - -* dynamic\_partition\_check\_interval\_seconds - - 动态分区线程的执行频率，默认为3600(1个小时)，即每1个小时进行一次调度 - -### HTTP Restful API - -Doris 提供了修改动态分区配置参数的 HTTP Restful API，用于运行时修改动态分区配置参数。 - -该 API 实现在 FE 端，使用 `fe_host:fe_http_port` 进行访问。需要 ADMIN 权限。 - -1. 将 dynamic_partition_enable 设置为 true 或 false - - * 标记为 true - - ``` - GET /api/_set_config?dynamic_partition_enable=true - - 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true - - 返回：200 - ``` - - * 标记为 false - - ``` - GET /api/_set_config?dynamic_partition_enable=false - - 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=false - - 返回：200 - ``` - -2. 设置 dynamic partition 的调度频率 - - * 设置调度时间为12小时调度一次 - - ``` - GET /api/_set_config?dynamic_partition_check_interval_seconds=432000 - - 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000 - - 返回：200 - ``` diff --git a/docs/documentation/cn/administrator-guide/http-actions/cancel-label.md b/docs/documentation/cn/administrator-guide/http-actions/cancel-label.md deleted file mode 100644 index 29a904689a3eef..00000000000000 --- a/docs/documentation/cn/administrator-guide/http-actions/cancel-label.md +++ /dev/null @@ -1,52 +0,0 @@ - - -# CANCEL LABEL -## description - NAME: - cancel_label: cancel a transaction with label - - SYNOPSIS - curl -u user:passwd -XPOST http://host:port/api/{db}/{label}/_cancel - - DESCRIPTION - 该命令用于cancel一个指定Label对应的事务，事务在Prepare阶段能够被成功cancel - - RETURN VALUES - 执行完成后，会以Json格式返回这次导入的相关内容。当前包括一下字段 - Status: 是否成功cancel - Success: 成功cancel事务 - 其他: cancel失败 - Message: 具体的失败信息 - - ERRORS - -## example - - 1. cancel testDb, testLabel的作业 - curl -u root -XPOST http://host:port/api/testDb/testLabel/_cancel - -## keyword - CANCEL，LABEL - - - - - - diff --git a/docs/documentation/cn/administrator-guide/http-actions/compaction-action.md b/docs/documentation/cn/administrator-guide/http-actions/compaction-action.md deleted file mode 100644 index 41ce72f3d4eb23..00000000000000 --- a/docs/documentation/cn/administrator-guide/http-actions/compaction-action.md +++ /dev/null @@ -1,79 +0,0 @@ - - -# Compaction Action - -该 API 用于查看某个 BE 节点总体的 compaction 状态，或者指定 tablet 的 compaction 状态。也可以用于手动触发 Compaction。 - -## 查看 Compaction 状态 - -### 节点整体 compaction 状态 - -(TODO) - -### 指定 tablet 的 compaction 状态 - -``` -curl -X GET http://be_host:webserver_port/api/compaction/show?tablet_id=xxxx\&schema_hash=yyyy -``` - -若 tablet 不存在，返回 JSON 格式的错误： - -``` -{ - "status": "Fail", - "msg": "Tablet not found" -} -``` - -若 tablet 存在，则返回 JSON 格式的结果: - -``` -{ - "cumulative point": 50, - "last cumulative failure time": "2019-12-16 18:13:43.224", - "last base failure time": "2019-12-16 18:13:23.320", - "last cumu success time": "2019-12-16 18:12:15.110", - "last base success time": "2019-12-16 18:11:50.780", - "rowsets": [ - "[0-48] 10 DATA OVERLAPPING", - "[49-49] 2 DATA OVERLAPPING", - "[50-50] 0 DELETE NONOVERLAPPING", - "[51-51] 5 DATA OVERLAPPING" - ] -} -``` - -结果说明： - -* cumulative point：base 和 cumulative compaction 的版本分界线。在 point（不含）之前的版本由 base compaction 处理。point（含）之后的版本由 cumulative compaction 处理。 -* last cumulative failure time：上一次尝试 cumulative compaction 失败的时间。默认 10min 后才会再次尝试对该 tablet 做 cumulative compaction。 -* last base failure time：上一次尝试 base compaction 失败的时间。默认 10min 后才会再次尝试对该 tablet 做 base compaction。 -* rowsets：该 tablet 当前的 rowset 集合。如 [0-48] 表示 0-48 版本。第二位数字表示该版本中 segment 的数量。`DELETE` 表示 delete 版本。`DATA` 表示数据版本。`OVERLAPPING` 和 `NONOVERLAPPING` 表示segment数据是否重叠。 - -### 示例 - -``` -curl -X GET http://192.168.10.24:8040/api/compaction/show?tablet_id=10015\&schema_hash=1294206575 -``` - -## 手动触发 Compaction - -(TODO) - diff --git a/docs/documentation/cn/administrator-guide/http-actions/fe-get-log-file.md b/docs/documentation/cn/administrator-guide/http-actions/fe-get-log-file.md deleted file mode 100644 index f2479d9927ae0f..00000000000000 --- a/docs/documentation/cn/administrator-guide/http-actions/fe-get-log-file.md +++ /dev/null @@ -1,73 +0,0 @@ - - -# get\_log\_file - -用户可以通过该 HTTP 接口获取 FE 的日志文件。 - -## 日志类型 - -支持获取以下类型的 FE 日志： - -1. fe.audit.log（审计日志） - - 审计日志记录了对应 FE 节点的所有请求语句已经请求的信息。审计日志的文件命名规则如下： - - ``` - fe.audit.log # 当前的最新日志 - fe.audit.log.20190603.1 # 对应日期的审计日志，当对应日期的日志大小超过 1GB 后，会生成序号后缀。序号越小的日志，内容越新。 - fe.audit.log.20190603.2 - fe.audit.log.20190602.1 - ... - ``` - -## 接口示例 - -1. 获取对应类型的日志文件列表 - - 示例： - - `curl -v -X HEAD -uuser:passwd http://fe_host:http_port/api/get_log_file?type=fe.audit.log` - - 返回结果： - - ``` - HTTP/1.1 200 OK - file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934} - content-type: text/html - connection: keep-alive - ``` - - 在返回的 header 中，`file_infos` 字段以 json 格式展示文件列表以及对应文件大小（单位字节） - -2. 下载日志文件 - - 示例： - - ``` - curl -X GET -uuser:passwd http://fe_host:http_port/api/get_log_file?type=fe.audit.log\&file=fe.audit.log.20190528.1 - ``` - - 返回结果： - - 以文件的形式下载指定的文件。 - -## 接口说明 - -该接口需要 admin 权限。 diff --git a/docs/documentation/cn/administrator-guide/http-actions/get-label-state.md b/docs/documentation/cn/administrator-guide/http-actions/get-label-state.md deleted file mode 100644 index 7d6d0b7eee8fcf..00000000000000 --- a/docs/documentation/cn/administrator-guide/http-actions/get-label-state.md +++ /dev/null @@ -1,52 +0,0 @@ - - -# GET LABEL STATE -## description - NAME: - get_label_state: get label's state - - SYNOPSIS - curl -u user:passwd http://host:port/api/{db}/{label}/_state - - DESCRIPTION - 该命令用于查看一个Label对应的事务状态 - - RETURN VALUES - 执行完毕后，会以Json格式返回这次导入的相关内容。当前包括一下字段 - Label：本次导入的 label，如果没有指定，则为一个 uuid。 - Status：此命令是否成功执行，Success表示成功执行 - Message： 具体的执行信息 - State: 只有在Status为Success时才有意义 - UNKNOWN: 没有找到对应的Label - PREPARE: 对应的事务已经prepare，但尚未提交 - COMMITTED: 事务已经提交，不能被cancel - VISIBLE: 事务提交，并且数据可见，不能被cancel - ABORTED: 事务已经被ROLLBACK，导入已经失败。 - - ERRORS - -## example - - 1. 获得testDb, testLabel的状态 - curl -u root http://host:port/api/testDb/testLabel/_state - -## keyword - GET, LABEL, STATE - diff --git a/docs/documentation/cn/administrator-guide/http-actions/index.rst b/docs/documentation/cn/administrator-guide/http-actions/index.rst deleted file mode 100644 index 268e99b5a8fd78..00000000000000 --- a/docs/documentation/cn/administrator-guide/http-actions/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============= -HTTP API -============= - -.. toctree:: - :maxdepth: 1 - :glob: - - * diff --git a/docs/documentation/cn/administrator-guide/http-actions/restore-tablet.md b/docs/documentation/cn/administrator-guide/http-actions/restore-tablet.md deleted file mode 100644 index a6eff81f0a6e5c..00000000000000 --- a/docs/documentation/cn/administrator-guide/http-actions/restore-tablet.md +++ /dev/null @@ -1,36 +0,0 @@ - - -# RESTORE TABLET -## description - - 该功能用于恢复trash目录中被误删的tablet数据。 - - 说明：这个功能暂时只在be服务中提供一个http接口。如果要使用， - 需要向要进行数据恢复的那台be机器的http端口发送restore tablet api请求。api格式如下： - METHOD: POST - URI: http://be_host:be_http_port/api/restore_tablet?tablet_id=xxx&schema_hash=xxx - -## example - - curl -X POST "http://hostname:8088/api/restore_tablet?tablet_id=123456\&schema_hash=1111111" - -## keyword - - RESTORE,TABLET,RESTORE,TABLET diff --git a/docs/documentation/cn/administrator-guide/index.rst b/docs/documentation/cn/administrator-guide/index.rst deleted file mode 100644 index 21c1bffc6012f5..00000000000000 --- a/docs/documentation/cn/administrator-guide/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -============= -操作手册 -============= - -.. toctree:: - :hidden: - - load-data/index - alter-table/index - materialized-view/index - http-actions/index - operation/index - config/index - -.. toctree:: - :maxdepth: 1 - :glob: - - * - diff --git a/docs/documentation/cn/administrator-guide/load-data/broker-load-manual.md b/docs/documentation/cn/administrator-guide/load-data/broker-load-manual.md deleted file mode 100644 index 158674766004e0..00000000000000 --- a/docs/documentation/cn/administrator-guide/load-data/broker-load-manual.md +++ /dev/null @@ -1,510 +0,0 @@ - - -# Broker Load - -Broker load 是一个异步的导入方式，支持的数据源取决于 Broker 进程支持的数据源。 - -用户需要通过 MySQL 协议 创建 Broker load 导入，并通过查看导入命令检查导入结果。 - -## 适用场景 - -* 源数据在 Broker 可以访问的存储系统中，如 HDFS。 -* 数据量在 几十到百GB 级别。 - -## 名词解释 - -1. Frontend（FE）：Doris 系统的元数据和调度节点。在导入流程中主要负责导入 plan 生成和导入任务的调度工作。 -2. Backend（BE）：Doris 系统的计算和存储节点。在导入流程中主要负责数据的 ETL 和存储。 -3. Broker：Broker 为一个独立的无状态进程。封装了文件系统接口，提供 Doris 读取远端存储系统中文件的能力。 -4. Plan：导入执行计划，BE 会执行导入执行计划将数据导入到 Doris 系统中。 - -## 基本原理 - -用户在提交导入任务后，FE 会生成对应的 Plan 并根据目前 BE 的个数和文件的大小，将 Plan 分给 多个 BE 执行，每个 BE 执行一部分导入数据。 - -BE 在执行的过程中会从 Broker 拉取数据，在对数据 transform 之后将数据导入系统。所有 BE 均完成导入，由 FE 最终决定导入是否成功。 - -``` - + - | 1. user create broker load - v - +----+----+ - | | - | FE | - | | - +----+----+ - | - | 2. BE etl and load the data - +--------------------------+ - | | | -+---v---+ +--v----+ +---v---+ -| | | | | | -| BE | | BE | | BE | -| | | | | | -+---+-^-+ +---+-^-+ +--+-^--+ - | | | | | | - | | | | | | 3. pull data from broker -+---v-+-+ +---v-+-+ +--v-+--+ -| | | | | | -|Broker | |Broker | |Broker | -| | | | | | -+---+-^-+ +---+-^-+ +---+-^-+ - | | | | | | -+---v-+-----------v-+----------v-+-+ -| HDFS/BOS/AFS cluster | -| | -+----------------------------------+ - -``` - -## 基本操作 - -### 创建导入 - -Broker load 创建导入语句 - -语法： - -``` -LOAD LABEL db_name.label_name -(data_desc, ...) -WITH BROKER broker_name broker_properties -[PROPERTIES (key1=value1, ... )] - -* data_desc: - - DATA INFILE ('file_path', ...) - [NEGATIVE] - INTO TABLE tbl_name - [PARTITION (p1, p2)] - [COLUMNS TERMINATED BY separator ] - [(col1, ...)] - [SET (k1=f1(xx), k2=f2(xx))] - [WHERE predicate] - -* broker_properties: - - (key1=value1, ...) -``` -示例： - -``` -LOAD LABEL db1.label1 -( - DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file1") - INTO TABLE tbl1 - COLUMNS TERMINATED BY "," - (tmp_c1,tmp_c2) - SET - ( - id=tmp_c2, - name=tmp_c1 - ), - DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file2") - INTO TABLE tbl2 - COLUMNS TERMINATED BY "," - (col1, col2) - where col1 > 1 -) -WITH BROKER 'broker' -( - "username"="user", - "password"="pass" -) -PROPERTIES -( - "timeout" = "3600" -); - -``` - -创建导入的详细语法执行 ```HELP BROKER LOAD``` 查看语法帮助。这里主要介绍 Broker load 的创建导入语法中参数意义和注意事项。 - -#### Label - -导入任务的标识。每个导入任务，都有一个在单 database 内部唯一的 Label。Label 是用户在导入命令中自定义的名称。通过这个 Label，用户可以查看对应导入任务的执行情况。 - -Label 的另一个作用，是防止用户重复导入相同的数据。**强烈推荐用户同一批次数据使用相同的label。这样同一批次数据的重复请求只会被接受一次，保证了 At-Most-Once 语义** - -当 Label 对应的导入作业状态为 CANCELLED 时，可以再次使用该 Label 提交导入作业。 - -#### 数据描述类参数 - -数据描述类参数主要指的是 Broker load 创建导入语句中的属于 ```data_desc``` 部分的参数。每组 ```data_desc ``` 主要表述了本次导入涉及到的数据源地址，ETL 函数，目标表及分区等信息。 - -下面主要对数据描述类的部分参数详细解释： - -+ 多表导入 - - Broker load 支持一次导入任务涉及多张表，每个 Broker load 导入任务可在多个 ``` data_desc ``` 声明多张表来实现多表导入。每个单独的 ```data_desc``` 还可以指定属于该表的数据源地址。Broker load 保证了单次导入的多张表之间原子性成功或失败。 - -+ negative - - ```data_desc```中还可以设置数据取反导入。这个功能主要用于，当数据表中聚合列的类型都为 SUM 类型时。如果希望撤销某一批导入的数据。则可以通过 `negative` 参数导入同一批数据。Doris 会自动为这一批数据在聚合列上数据取反，以达到消除同一批数据的功能。 - -+ partition - - 在 ```data_desc``` 中可以指定待导入表的 partition 信息，如果待导入数据不属于指定的 partition 则不会被导入。同时，不在指定 Partition 的数据会被认为是错误数据。 - -+ set column mapping - - 在 ```data_desc``` 中的 SET 语句负责设置列函数变换，这里的列函数变换支持所有查询的等值表达式变换。如果原始数据的列和表中的列不一一对应，就需要用到这个属性。 - -+ where predicate - - 在 ```data_desc``` 中的 WHERE 语句中负责过滤已经完成 transform 的数据，被 filter 的数据不会进入容忍率的统计中。如果多个 data_desc 中声明了同一张表的多个条件的话，则会 merge 同一张表的多个条件，merge 策略是 AND 。 - -#### 导入作业参数 - -导入作业参数主要指的是 Broker load 创建导入语句中的属于 ```opt_properties```部分的参数。导入作业参数是作用于整个导入作业的。 - -下面主要对导入作业参数的部分参数详细解释： - -+ timeout - - 导入作业的超时时间(以秒为单位)，用户可以在 ```opt_properties``` 中自行设置每个导入的超时时间。导入任务在设定的 timeout 时间内未完成则会被系统取消，变成 CANCELLED。Broker load 的默认导入超时时间为4小时。 - - 通常情况下，用户不需要手动设置导入任务的超时时间。当在默认超时时间内无法完成导入时，可以手动设置任务的超时时间。 - - > 推荐超时时间 - > - > 总文件大小（MB） / 用户 Doris 集群最慢导入速度(MB/s) > timeout > （（总文件大小(MB) * 待导入的表及相关 Roll up 表的个数） / (10 * 导入并发数） ） - - > 导入并发数见文档最后的导入系统配置说明，公式中的 10 为目前的导入限速 10MB/s。 - - > 例如一个 1G 的待导入数据，待导入表包含3个 Rollup 表，当前的导入并发数为 3。则 timeout 的 最小值为 ```(1 * 1024 * 3 ) / (10 * 3) = 102 秒``` - - 由于每个 Doris 集群的机器环境不同且集群并发的查询任务也不同，所以用户 Doris 集群的最慢导入速度需要用户自己根据历史的导入任务速度进行推测。 - -+ max\_filter\_ratio - - 导入任务的最大容忍率，默认为0容忍，取值范围是0~1。当导入的错误率超过该值，则导入失败。 - - 如果用户希望忽略错误的行，可以通过设置这个参数大于 0，来保证导入可以成功。 - - 计算公式为： - - ``` max_filter_ratio = (dpp.abnorm.ALL / (dpp.abnorm.ALL + dpp.norm.ALL ) ) ``` - - ```dpp.abnorm.ALL``` 表示数据质量不合格的行数。如类型不匹配，列数不匹配，长度不匹配等等。 - - ```dpp.norm.ALL``` 指的是导入过程中正确数据的条数。可以通过 ```SHOW LOAD``` 命令查询导入任务的正确数据量。 - - 原始文件的行数 = `dpp.abnorm.ALL + dpp.norm.ALL` - -+ exec\_mem\_limit - - 导入内存限制。默认是 2GB。单位为字节。 - -+ strict\_mode - - Broker load 导入可以开启 strict mode 模式。开启方式为 ```properties ("strict_mode" = "true")``` 。默认的 strict mode 为关闭。 - - strict mode 模式的意思是：对于导入过程中的列类型转换进行严格过滤。严格过滤的策略如下： - - 1. 对于列类型转换来说，如果 strict mode 为true，则错误的数据将被 filter。这里的错误数据是指：原始数据并不为空值，在参与列类型转换后结果为空值的这一类数据。 - - 2. 对于导入的某列由函数变换生成时，strict mode 对其不产生影响。 - - 3. 对于导入的某列类型包含范围限制的，如果原始数据能正常通过类型转换，但无法通过范围限制的，strict mode 对其也不产生影响。例如：如果类型是 decimal(1,0), 原始数据为 10，则属于可以通过类型转换但不在列声明的范围内。这种数据 strict 对其不产生影响。 - -#### strict mode 与 source data 的导入关系 - -这里以列类型为 TinyInt 来举例 - ->注：当表中的列允许导入空值时 - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|---------| -|空值 | \N | N/A | true or false | NULL| -|not null | aaa or 2000 | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 | 1 | true or false | correct data| - -这里以列类型为 Decimal(1,0) 举例 - ->注：当表中的列允许导入空值时 - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|--------| -|空值 | \N | N/A | true or false | NULL| -|not null | aaa | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 or 10 | 1 | true or false | correct data| - -> 注意：10 虽然是一个超过范围的值，但是因为其类型符合 decimal的要求，所以 strict mode对其不产生影响。10 最后会在其他 ETL 处理流程中被过滤。但不会被 strict mode 过滤。 - -#### Broker 参数 - -Broker Load 需要借助 Broker 进程访问远端存储，不同的 Broker 需要提供不同的参数，具体请参阅 [Broker文档](../broker.md) - -### 查看导入 - -Broker load 导入方式由于是异步的，所以用户必须将创建导入的 Label 记录，并且在**查看导入命令中使用 Label 来查看导入结果**。查看导入命令在所有导入方式中是通用的，具体语法可执行 ```HELP SHOW LOAD``` 查看。 - -示例： - -``` -mysql> show load order by createtime desc limit 1\G -*************************** 1. row *************************** - JobId: 76391 - Label: label1 - State: FINISHED - Progress: ETL:N/A; LOAD:100% - Type: BROKER - EtlInfo: unselected.rows=4; dpp.abnorm.ALL=15; dpp.norm.ALL=28133376 - TaskInfo: cluster:N/A; timeout(s):10800; max_filter_ratio:5.0E-5 - ErrorMsg: N/A - CreateTime: 2019-07-27 11:46:42 - EtlStartTime: 2019-07-27 11:46:44 - EtlFinishTime: 2019-07-27 11:46:44 - LoadStartTime: 2019-07-27 11:46:44 -LoadFinishTime: 2019-07-27 11:50:16 - URL: http://192.168.1.1:8040/api/_load_error_log?file=__shard_4/error_log_insert_stmt_4bb00753932c491a-a6da6e2725415317_4bb00753932c491a_a6da6e2725415317 - JobDetails: {"Unfinished backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"ScannedRows":2390016,"TaskNumber":1,"All backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"FileNumber":1,"FileSize":1073741824} -``` - -下面主要介绍了查看导入命令返回结果集中参数意义： - -+ JobId - - 导入任务的唯一ID，每个导入任务的 JobId 都不同，由系统自动生成。与 Label 不同的是，JobId永远不会相同，而 Label 则可以在导入任务失败后被复用。 - -+ Label - - 导入任务的标识。 - -+ State - - 导入任务当前所处的阶段。在 Broker load 导入过程中主要会出现 PENDING 和 LOADING 这两个导入中的状态。如果 Broker load 处于 PENDING 状态，则说明当前导入任务正在等待被执行；LOADING 状态则表示正在执行中。 - - 导入任务的最终阶段有两个：CANCELLED 和 FINISHED，当 Load job 处于这两个阶段时，导入完成。其中 CANCELLED 为导入失败，FINISHED 为导入成功。 - -+ Progress - - 导入任务的进度描述。分为两种进度：ETL 和 LOAD，对应了导入流程的两个阶段 ETL 和 LOADING。目前 Broker load 由于只有 LOADING 阶段，所以 ETL 则会永远显示为 `N/A` - - LOAD 的进度范围为：0~100%。 - - ```LOAD 进度 = 当前完成导入的表个数 / 本次导入任务设计的总表个数 * 100%``` - - **如果所有导入表均完成导入，此时 LOAD 的进度为 99%** 导入进入到最后生效阶段，整个导入完成后，LOAD 的进度才会改为 100%。 - - 导入进度并不是线性的。所以如果一段时间内进度没有变化，并不代表导入没有在执行。 - -+ Type - - 导入任务的类型。Broker load 的 type 取值只有 BROKER。 - -+ EtlInfo - - 主要显示了导入的数据量指标 ```unselected.rows``` , ```dpp.norm.ALL``` 和 ```dpp.abnorm.ALL```。用户可以根据第一个数值判断 where 条件过滤了多少行，后两个指标验证当前导入任务的错误率是否超过 ```max_filter_ratio```。 - - 三个指标之和就是原始数据量的总行数。 - -+ TaskInfo - - 主要显示了当前导入任务参数，也就是创建 Broker load 导入任务时用户指定的导入任务参数，包括：`cluster`，`timeout` 和`max_filter_ratio`。 - -+ ErrorMsg - - 在导入任务状态为CANCELLED，会显示失败的原因，显示分两部分：type 和 msg，如果导入任务成功则显示 ```N/A```。 - - type的取值意义： - - ``` - USER_CANCEL: 用户取消的任务 - ETL_RUN_FAIL：在ETL阶段失败的导入任务 - ETL_QUALITY_UNSATISFIED：数据质量不合格，也就是错误数据率超过了 max_filter_ratio - LOAD_RUN_FAIL：在LOADING阶段失败的导入任务 - TIMEOUT：导入任务没在超时时间内完成 - UNKNOWN：未知的导入错误 - ``` - -+ CreateTime/EtlStartTime/EtlFinishTime/LoadStartTime/LoadFinishTime - - 这几个值分别代表导入创建的时间，ETL阶段开始的时间，ETL阶段完成的时间，Loading阶段开始的时间和整个导入任务完成的时间。 - - Broker load 导入由于没有 ETL 阶段，所以其 EtlStartTime, EtlFinishTime, LoadStartTime 被设置为同一个值。 - - 导入任务长时间停留在 CreateTime，而 LoadStartTime 为 N/A 则说明目前导入任务堆积严重。用户可减少导入提交的频率。 - - ``` - LoadFinishTime - CreateTime = 整个导入任务所消耗时间 - LoadFinishTime - LoadStartTime = 整个 Broker load 导入任务执行时间 = 整个导入任务所消耗时间 - 导入任务等待的时间 - ``` - -+ URL - - 导入任务的错误数据样例，访问 URL 地址既可获取本次导入的错误数据样例。当本次导入不存在错误数据时，URL 字段则为 N/A。 - -+ JobDetails - - 显示一些作业的详细运行状态。包括导入文件的个数、总大小（字节）、子任务个数、已处理的原始行数，运行子任务的 BE 节点 Id，未完成的 BE 节点 Id。 - - ``` - {"Unfinished backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"ScannedRows":2390016,"TaskNumber":1,"All backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"FileNumber":1,"FileSize":1073741824} - ``` - - 其中已处理的原始行数，每 5 秒更新一次。该行数仅用于展示当前的进度，不代表最终实际的处理行数。实际处理行数以 EtlInfo 中显示的为准。 - -### 取消导入 - -当 Broker load 作业状态不为 CANCELLED 或 FINISHED 时，可以被用户手动取消。取消时需要指定待取消导入任务的 Label 。取消导入命令语法可执行 ```HELP CANCEL LOAD```查看。 - -## 相关系统配置 - -### FE 配置 - -下面几个配置属于 Broker load 的系统级别配置，也就是作用于所有 Broker load 导入任务的配置。主要通过修改 ``` fe.conf```来调整配置值。 - -+ min\_bytes\_per\_broker\_scanner/max\_bytes\_per\_broker\_scanner/max\_broker\_concurrency - - 前两个配置限制了单个 BE 处理的数据量的最小和最大值。第三个配置限制了一个作业的最大的导入并发数。最小处理的数据量，最大并发数，源文件的大小和当前集群 BE 的个数 **共同决定了本次导入的并发数**。 - - ``` - 本次导入并发数 = Math.min(源文件大小/最小处理量，最大并发数，当前BE节点个数) - 本次导入单个BE的处理量 = 源文件大小/本次导入的并发数 - ``` - - 通常一个导入作业支持的最大数据量为 `max_bytes_per_broker_scanner * BE 节点数`。如果需要导入更大数据量，则需要适当调整 `max_bytes_per_broker_scanner` 参数的大小。 - - 默认配置： - - ``` - 参数名：min_bytes_per_broker_scanner， 默认 64MB，单位bytes。 - 参数名：max_broker_concurrency， 默认 10。 - 参数名：max_bytes_per_broker_scanner，默认 3G，单位bytes。 - ``` - -## 最佳实践 - -### 应用场景 - -使用 Broker load 最适合的场景就是原始数据在文件系统（HDFS，BOS，AFS）中的场景。其次，由于 Broker load 是单次导入中唯一的一种异步导入的方式，所以如果用户在导入大文件中，需要使用异步接入，也可以考虑使用 Broker load。 - -### 数据量 - -这里仅讨论单个 BE 的情况，如果用户集群有多个 BE 则下面标题中的数据量应该乘以 BE 个数来计算。比如：如果用户有3个 BE，则 3G 以下（包含）则应该乘以 3，也就是 9G 以下（包含）。 - -+ 3G 以下（包含） - - 用户可以直接提交 Broker load 创建导入请求。 - -+ 3G 以上 - - 由于单个导入 BE 最大的处理量为 3G，超过 3G 的待导入文件就需要通过调整 Broker load 的导入参数来实现大文件的导入。 - - 1. 根据当前 BE 的个数和原始文件的大小修改单个 BE 的最大扫描量和最大并发数。 - - ``` - 修改 fe.conf 中配置 - - max_broker_concurrency = BE 个数 - 当前导入任务单个 BE 处理的数据量 = 原始文件大小 / max_broker_concurrency - max_bytes_per_broker_scanner >= 当前导入任务单个 BE 处理的数据量 - - 比如一个 100G 的文件，集群的 BE 个数为 10 个 - max_broker_concurrency = 10 - max_bytes_per_broker_scanner >= 10G = 100G / 10 - - ``` - - 修改后，所有的 BE 会并发的处理导入任务，每个 BE 处理原始文件的一部分。 - - *注意：上述两个 FE 中的配置均为系统配置，也就是说其修改是作用于所有的 Broker load的任务的。* - - 2. 在创建导入的时候自定义当前导入任务的 timeout 时间 - - ``` - 当前导入任务单个 BE 处理的数据量 / 用户 Doris 集群最慢导入速度(MB/s) >= 当前导入任务的 timeout 时间 >= 当前导入任务单个 BE 处理的数据量 / 10M/s - - 比如一个 100G 的文件，集群的 BE 个数为 10个 - timeout >= 1000s = 10G / 10M/s - - ``` - - 3. 当用户发现第二步计算出的 timeout 时间超过系统默认的导入最大超时时间 4小时 - - 这时候不推荐用户将导入最大超时时间直接改大来解决问题。单个导入时间如果超过默认的导入最大超时时间4小时，最好是通过切分待导入文件并且分多次导入来解决问题。主要原因是：单次导入超过4小时的话，导入失败后重试的时间成本很高。 - - 可以通过如下公式计算出 Doris 集群期望最大导入文件数据量： - - ``` - 期望最大导入文件数据量 = 14400s * 10M/s * BE 个数 - 比如：集群的 BE 个数为 10个 - 期望最大导入文件数据量 = 14400s * 10M/s * 10 = 1440000M ≈ 1440G - - 注意：一般用户的环境可能达不到 10M/s 的速度，所以建议超过 500G 的文件都进行文件切分，再导入。 - - ``` - -### 完整例子 - -数据情况：用户数据在 HDFS 中，文件地址为 hdfs://abc.com:8888/store_sales, hdfs 的认证用户名为 root, 密码为 password, 数据量大小约为 30G，希望导入到数据库 bj_sales 的表 store_sales 中。 - -集群情况：集群的 BE 个数约为 3 个，Broker 名称均为 broker。 - -+ step1: 经过上述方法的计算，本次导入的单个 BE 导入量为 10G，则需要先修改 FE 的配置，将单个 BE 导入最大量修改为： - - ``` - max_bytes_per_broker_scanner = 10737418240 - - ``` - -+ step2: 经计算，本次导入的时间大约为 1000s，并未超过默认超时时间，可不配置导入自定义超时时间。 - -+ step3：创建导入语句 - - ``` - LOAD LABEL bj_sales.store_sales_broker_load_01 - ( - DATA INFILE("hdfs://abc.com:8888/store_sales") - INTO TABLE store_sales - ) - WITH BROKER 'broker' - ("username"="root", "password"="password"); - ``` - -## 常见问题 - -* 导入报错：`Scan bytes per broker scanner exceed limit:xxx` - - 请参照文档中最佳实践部分，修改 FE 配置项 `max_bytes_per_broker_scanner` 和 `max_broker_concurrency` - -* 导入报错：`failed to send batch` 或 `TabletWriter add batch with unknown id` - - 请参照 [导入手册](./load-manual.md) 中 **通用系统配置** 中 **BE 配置**，适当修改 `query_timeout` 和 `streaming_load_rpc_max_alive_time_sec`。 - -* 导入报错：`LOAD_RUN_FAIL; msg:Invalid Column Name:xxx` - - 如果是PARQUET或者ORC格式的数据,需要再文件头的列名与doris表中的列名一致，如 : - ``` - (tmp_c1,tmp_c2) - SET - ( - id=tmp_c2, - name=tmp_c1 - ) - ``` - 代表获取在parquet或orc中以(tmp_c1, tmp_c2)为列名的列，映射到doris表中的(id, name)列。如果没有设置set, 则以column中的列作为映射。 - - 注：如果使用某些hive版本直接生成的orc文件，orc文件中的表头并非hive meta数据，而是（_col0, _col1, _col2, ...）, 可能导致Invalid Column Name错误，那么则需要使用set进行映射 - diff --git a/docs/documentation/cn/administrator-guide/load-data/delete-manual.md b/docs/documentation/cn/administrator-guide/load-data/delete-manual.md deleted file mode 100644 index cc5404ec328716..00000000000000 --- a/docs/documentation/cn/administrator-guide/load-data/delete-manual.md +++ /dev/null @@ -1,176 +0,0 @@ - - -# Delete - -Delete不同于其他导入方式，它是一个同步过程。和Insert into相似，所有的Delete操作在Doris中是一个独立的导入作业，一般Delete语句需要指定表和分区以及删除的条件来筛选要删除的数据，并将会同时删除base表和rollup表的数据。 - -## 语法 - -主要的Delete语法如下： - -``` -DELETE FROM table_name [PARTITION partition_name] -WHERE -column_name1 op value[ AND column_name2 op value ...]; -``` - -示例1： - -``` -DELETE FROM my_table PARTITION p1 WHERE k1 = 3; -``` - -示例2: - -``` -DELETE FROM my_table PARTITION p1 WHERE k1 < 3 AND k2 = "abc"; -``` - -下面介绍删除语句中使用到的参数： - -* PARTITION - - Delete语句的目标分区，若未指定，则此表必须为单分区表，否则无法delete - -* WHERE - - Delete语句的条件语句，所有删除语句都必须指定WHERE语句 - -说明： - -1. `Where`语句中的op的类型可包括`=,>,<,>=,<=,!=`，目前暂时不支持 where key in (value1, value2, value3) 的方式选定范围，后续将加上此功能。 -2. `Where`语句中的列只能是`key`列 -3. 当选定的`key`列不存在某个rollup表内时，无法进行delete -4. 条件语句中各个条件只能是`and`关系，如希望达成`or`可将条件分别写入两个delete语句中 -5. 如果指定表为RANGE分区表，则必须指定 `PARTITION`。如果是单分区表，可以不指定。 -6. 不同于Insert into命令，delete不能手动指定`label`，有关label的概念可以查看[Insert Into文档] (./insert-into-manual.md) - -## 返回结果 - -Delete命令是一个SQL命令，返回结果是同步的，分为以下几种： - -1. 执行成功 - - 如果Delete顺利执行完成并可见，将返回下列结果，`Query OK`表示成功 - - ``` - mysql> delete from test_tbl PARTITION p1 where k1 = 1; - Query OK, 0 rows affected (0.04 sec) - {'label':'delete_e7830c72-eb14-4cb9-bbb6-eebd4511d251', 'status':'VISIBLE', 'txnId':'4005'} - ``` - -2. 提交成功，但未可见 - - Doris的事务提交分为两步：提交和发布版本，只有完成了发布版本步骤，结果才对用户是可见的。若已经提交成功了，那么就可以认为最终一定会发布成功，Doris会尝试在提交完后等待发布一段时间，如果超时后即使发布版本还未完成也会优先返回给用户，提示用户提交已经完成。若如果Delete已经提交并执行，但是仍未发布版本和可见，将返回下列结果 - - ``` - mysql> delete from test_tbl PARTITION p1 where k1 = 1; - Query OK, 0 rows affected (0.04 sec) - {'label':'delete_e7830c72-eb14-4cb9-bbb6-eebd4511d251', 'status':'VISIBLE', 'txnId':'4005', 'err':'delete job is committed but may be taking effect later' } - ``` - - 结果会同时返回一个json字符串： - - `affected rows`表示此次删除影响的行，由于Doris的删除目前是逻辑删除，因此对于这个值是恒为0。 - - `label`为自动生成的 label，是该导入作业的标识。每个导入作业，都有一个在单 database 内部唯一的 Label。 - - `status`表示数据删除是否可见，如果可见，显示`VISIBLE`，如果不可见,显示`COMMITTED`。 - - `txnId`为这个Delete job对应的事务id - - `err`字段会显示一些本次删除的详细信息 - -3. 提交失败，事务取消 - - 如果Delete语句没有提交成功，将会被Doris自动中止，返回下列结果 - - ``` - mysql> delete from test_tbl partition p1 where k1 > 80; - ERROR 1064 (HY000): errCode = 2, detailMessage = {错误原因} - ``` - - 示例： - - 比如说一个超时的删除，将会返回timeout时间和未完成的`(tablet=replica)` - - ``` - mysql> delete from test_tbl partition p1 where k1 > 80; - ERROR 1064 (HY000): errCode = 2, detailMessage = failed to delete replicas from job: 4005, Unfinished replicas:10000=60000, 10001=60000, 10002=60000 - ``` - - **综上，对于Delete操作返回结果的正确处理逻辑为：** - - 1. 如果返回结果为`ERROR 1064 (HY000)`，则表示删除失败 - - 2. 如果返回结果为`Query OK`，则表示删除执行成功 - - 1. 如果`status`为`COMMITTED`，表示数据仍不可见，用户可以稍等一段时间再用`show delete`命令查看结果 - 2. 如果`status`为`VISIBLE`，表示数据删除成功。 - -## 可配置项 - -### FE配置 - -**TIMEOUT配置** - -总体来说，Doris的删除作业的超时时间限制在30秒到5分钟时间内，具体时间可通过下面配置项调整 - -* tablet\_delete\_timeout\_second - - delete自身的超时时间是可受指定分区下tablet的数量弹性改变的，此项配置为平均一个tablet所贡献的timeout时间，默认值为2。 - - 假设此次删除所指定分区下有5个tablet，那么可提供给delete的timeout时间为10秒，由于低于最低超时时间30秒，因此最终超时时间为30秒。 - -* load\_straggler\_wait\_second - - 如果用户预估的数据量确实比较大，使得5分钟的上限不足时，用户可以通过此项调整timeout上限，默认值为300。 - - **TIMEOUT的具体计算规则为(秒)** - - `TIMEOUT = MIN(load_straggler_wait_second, MAX(30, tablet_delete_timeout_second * tablet_num))` - -* query_timeout - - 因为delete本身是一个SQL命令，因此删除语句也会受session限制，timeout还受Session中的`query_timeout`值影响，可以通过`SET query_timeout = xxx`来增加超时时间，单位是秒。 - -## 查看历史记录 - -1. 用户可以通过show delete语句查看历史上已执行完成的删除记录 - - 语法 - - ``` - SHOW DELETE [FROM db_name] - ``` - - 示例 - - ``` - mysql> show delete from test_db; - +-----------+---------------+---------------------+-----------------+----------+ - | TableName | PartitionName | CreateTime | DeleteCondition | State | - +-----------+---------------+---------------------+-----------------+----------+ - | empty_tbl | p3 | 2020-04-15 23:09:35 | k1 EQ "1" | FINISHED | - | test_tbl | p4 | 2020-04-15 23:09:53 | k1 GT "80" | FINISHED | - +-----------+---------------+---------------------+-----------------+----------+ - 2 rows in set (0.00 sec) - ``` - diff --git a/docs/documentation/cn/administrator-guide/load-data/index.rst b/docs/documentation/cn/administrator-guide/load-data/index.rst deleted file mode 100644 index b5aad9ff8591d4..00000000000000 --- a/docs/documentation/cn/administrator-guide/load-data/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -============= -数据导入 -============= - -.. toctree:: - :maxdepth: 2 - - load-manual.md - broker-load-manual.md - stream-load-manual.md - routine-load-manual.md - insert-into-manual.md - delete-manual.md diff --git a/docs/documentation/cn/administrator-guide/load-data/insert-into-manual.md b/docs/documentation/cn/administrator-guide/load-data/insert-into-manual.md deleted file mode 100644 index c00f1355c33bd7..00000000000000 --- a/docs/documentation/cn/administrator-guide/load-data/insert-into-manual.md +++ /dev/null @@ -1,282 +0,0 @@ - - -# Insert Into - -Insert Into 语句的使用方式和 MySQL 等数据库中 Insert Into 语句的使用方式类似。但在 Doris 中，所有的数据写入都是一个独立的导入作业。所以这里将 Insert Into 也作为一种导入方式介绍。 - -主要的 Insert Into 命令包含以下两种； - -* INSERT INTO tbl SELECT ... -* INSERT INTO tbl (col1, col2, ...) VALUES (1, 2, ...), (1,3, ...); - -其中第二种命令仅用于 Demo，不要使用在测试或生产环境中。 - -## 基本操作 - -### 创建导入 - -Insert Into 命令需要通过 MySQL 协议提交，创建导入请求会同步返回导入结果。 - -语法： - -``` -INSERT INTO table_name [WITH LABEL label] [partition_info] [col_list] [query_stmt] [VALUES]; -``` - -示例： - -``` -INSERT INTO tbl2 WITH LABEL label1 SELECT * FROM tbl3; -INSERT INTO tbl1 VALUES ("qweasdzxcqweasdzxc"), ("a"); -``` - -**注意** - -当需要使用 `CTE(Common Table Expressions)` 作为 insert 操作中的查询部分时，必须指定 `WITH LABEL` 和 column list 部分。示例 - -``` -INSERT INTO tbl1 WITH LABEL label1 -WITH cte1 AS (SELECT * FROM tbl1), cte2 AS (SELECT * FROM tbl2) -SELECT k1 FROM cte1 JOIN cte2 WHERE cte1.k1 = 1; - - -INSERT INTO tbl1 (k1) -WITH cte1 AS (SELECT * FROM tbl1), cte2 AS (SELECT * FROM tbl2) -SELECT k1 FROM cte1 JOIN cte2 WHERE cte1.k1 = 1; -``` - -下面主要介绍创建导入语句中使用到的参数： - -+ partition\_info - - 导入表的目标分区，如果指定目标分区，则只会导入符合目标分区的数据。如果没有指定，则默认值为这张表的所有分区。 - -+ col\_list - - 导入表的目标列，可以以任意的顺序存在。如果没有指定目标列，那么默认值是这张表的所有列。如果待表中的某个列没有存在目标列中，那么这个列需要有默认值，否则 Insert Into 就会执行失败。 - - 如果查询语句的结果列类型与目标列的类型不一致，那么会调用隐式类型转化，如果不能够进行转化，那么 Insert Into 语句会报语法解析错误。 - -+ query\_stmt - - 通过一个查询语句，将查询语句的结果导入到 Doris 系统中的其他表。查询语句支持任意 Doris 支持的 SQL 查询语法。 - -+ VALUES - - 用户可以通过 VALUES 语法插入一条或者多条数据。 - - *注意：VALUES 方式仅适用于导入几条数据作为导入 DEMO 的情况，完全不适用于任何测试和生产环境。Doris 系统本身也不适合单条数据导入的场景。建议使用 INSERT INTO SELECT 的方式进行批量导入。* - -* WITH LABEL - - INSERT 操作作为一个导入任务，也可以指定一个 label。如果不指定，则系统会自动指定一个 UUID 作为 label。 - - 该功能需要 0.11+ 版本。 - - *注意：建议指定 Label 而不是由系统自动分配。如果由系统自动分配，但在 Insert Into 语句执行过程中，因网络错误导致连接断开等，则无法得知 Insert Into 是否成功。而如果指定 Label，则可以再次通过 Label 查看任务结果。* - -### 导入结果 - -Insert Into 本身就是一个 SQL 命令，其返回结果会根据执行结果的不同，分为以下几种： - -1. 结果集为空 - - 如果 insert 对应 select 语句的结果集为空，则返回如下： - - ``` - mysql> insert into tbl1 select * from empty_tbl; - Query OK, 0 rows affected (0.02 sec) - ``` - - `Query OK` 表示执行成功。`0 rows affected` 表示没有数据被导入。 - -2. 结果集不为空 - - 在结果集不为空的情况下。返回结果分为如下几种情况： - - 1. Insert 执行成功并可见： - - ``` - mysql> insert into tbl1 select * from tbl2; - Query OK, 4 rows affected (0.38 sec) - {'label':'insert_8510c568-9eda-4173-9e36-6adc7d35291c', 'status':'visible', 'txnId':'4005'} - - mysql> insert into tbl1 with label my_label1 select * from tbl2; - Query OK, 4 rows affected (0.38 sec) - {'label':'my_label1', 'status':'visible', 'txnId':'4005'} - - mysql> insert into tbl1 select * from tbl2; - Query OK, 2 rows affected, 2 warnings (0.31 sec) - {'label':'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status':'visible', 'txnId':'4005'} - - mysql> insert into tbl1 select * from tbl2; - Query OK, 2 rows affected, 2 warnings (0.31 sec) - {'label':'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status':'committed', 'txnId':'4005'} - ``` - - `Query OK` 表示执行成功。`4 rows affected` 表示总共有4行数据被导入。`2 warnings` 表示被过滤的行数。 - - 同时会返回一个 json 串： - - ``` - {'label':'my_label1', 'status':'visible', 'txnId':'4005'} - {'label':'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status':'committed', 'txnId':'4005'} - {'label':'my_label1', 'status':'visible', 'txnId':'4005', 'err':'some other error'} - ``` - - `label` 为用户指定的 label 或自动生成的 label。Label 是该 Insert Into 导入作业的标识。每个导入作业，都有一个在单 database 内部唯一的 Label。 - - `status` 表示导入数据是否可见。如果可见，显示 `visible`，如果不可见，显示 `committed`。 - - `txnId` 为这个 insert 对应的导入事务的 id。 - - `err` 字段会显示一些其他非预期错误。 - - 当需要查看被过滤的行时，用户可以通过如下语句 - - ``` - show load where label="xxx"; - ``` - - 返回结果中的 URL 可以用于查询错误的数据，具体见后面 **查看错误行** 小结。 - - **数据不可见是一个临时状态，这批数据最终是一定可见的** - - 可以通过如下语句查看这批数据的可见状态： - - ``` - show transaction where id=4005; - ``` - - 返回结果中的 `TransactionStatus` 列如果为 `visible`，则表述数据可见。 - - 2. Insert 执行失败 - - 执行失败表示没有任何数据被成功导入，并返回如下： - - ``` - mysql> insert into tbl1 select * from tbl2 where k1 = "a"; - ERROR 1064 (HY000): all partitions have no load data. url: http://10.74.167.16:8042/api/_load_error_log?file=__shard_2/error_log_insert_stmt_ba8bb9e158e4879-ae8de8507c0bf8a2_ba8bb9e158e4879_ae8de8507c0bf8a2 - ``` - - 其中 `ERROR 1064 (HY000): all partitions have no load data` 显示失败原因。后面的 url 可以用于查询错误的数据，具体见后面 **查看错误行** 小结。 - - -**综上，对于 insert 操作返回结果的正确处理逻辑应为：** - -1. 如果返回结果为 `ERROR 1064 (HY000)`，则表示导入失败。 -2. 如果返回结果为 `Query OK`，则表示执行成功。 - 1. 如果 `rows affected` 为 0，表示结果集为空，没有数据被导入。 - 2. 如果 `rows affected` 大于 0： - 1. 如果 `status` 为 `committed`，表示数据还不可见。需要通过 `show transaction` 语句查看状态直到 `visible` - 2. 如果 `status` 为 `visible`，表示数据导入成功。 - 3. 如果 `warnings` 大于 0，表示有数据被过滤，可以通过 `show load` 语句获取 url 查看被过滤的行。 - -## 相关系统配置 - -### FE 配置 - -+ timeout - - 导入任务的超时时间(以秒为单位)，导入任务在设定的 timeout 时间内未完成则会被系统取消，变成 CANCELLED。 - - 目前 Insert Into 并不支持自定义导入的 timeout 时间，所有 Insert Into 导入的超时时间是统一的，默认的 timeout 时间为1小时。如果导入的源文件无法再规定时间内完成导入，则需要调整 FE 的参数```insert_load_default_timeout_second```。 - - 同时 Insert Into 语句收到 Session 变量 `query_timeout` 的限制。可以通过 `SET query_timeout = xxx;` 来增加超时时间，单位是秒。 - -### Session 变量 - -+ enable\_insert\_strict - - Insert Into 导入本身不能控制导入可容忍的错误率。用户只能通过 `enable_insert_strict` 这个 Session 参数用来控制。 - - 当该参数设置为 false 时，表示至少有一条数据被正确导入，则返回成功。如果有失败数据，则还会返回一个 Label。 - - 当该参数设置为 true 时，表示如果有一条数据错误，则导入失败。 - - 默认为 false。可通过 `SET enable_insert_strict = true;` 来设置。 - -+ query\_timeout - - Insert Into 本身也是一个 SQL 命令，因此 Insert Into 语句也受到 Session 变量 `query_timeout` 的限制。可以通过 `SET query_timeout = xxx;` 来增加超时时间，单位是秒。 - -## 最佳实践 - -### 应用场景 -1. 用户希望仅导入几条假数据，验证一下 Doris 系统的功能。此时适合使用 INSERT INTO VALUS 的语法。 -2. 用户希望将已经在 Doris 表中的数据进行 ETL 转换并导入到一个新的 Doris 表中，此时适合使用 INSERT INTO SELECT 语法。 -3. 用户可以创建一种外部表，如 MySQL 外部表映射一张 MySQL 系统中的表。或者创建 Broker 外部表来映射 HDFS 上的数据文件。然后通过 INSERT INTO SELECT 语法将外部表中的数据导入到 Doris 表中存储。 - -### 数据量 -Insert Into 对数据量没有限制，大数据量导入也可以支持。但 Insert Into 有默认的超时时间，用户预估的导入数据量过大，就需要修改系统的 Insert Into 导入超时时间。 - -``` -导入数据量 = 36G 约≤ 3600s * 10M/s -其中 10M/s 是最大导入限速，用户需要根据当前集群情况计算出平均的导入速度来替换公式中的 10M/s -``` - -### 完整例子 - -用户有一张表 store\_sales 在数据库 sales 中，用户又创建了一张表叫 bj\_store\_sales 也在数据库 sales 中，用户希望将 store\_sales 中销售记录在 bj 的数据导入到这张新建的表 bj\_store\_sales 中。导入的数据量约为：10G。 - -``` -store_sales schema： -(id, total, user_id, sale_timestamp, region) - -bj_store_sales schema: -(id, total, user_id, sale_timestamp) - -``` - -集群情况：用户当前集群的平均导入速度约为 5M/s - -+ Step1: 判断是否要修改 Insert Into 的默认超时时间 - - ``` - 计算导入的大概时间 - 10G / 5M/s = 2000s - - 修改 FE 配置 - insert_load_default_timeout_second = 2000 - ``` - -+ Step2：创建导入任务 - - 由于用户是希望将一张表中的数据做 ETL 并导入到目标表中，所以应该使用 Insert into query\_stmt 方式导入。 - - ``` - INSERT INTO bj_store_sales WITH LABEL `label` SELECT id, total, user_id, sale_timestamp FROM store_sales where region = "bj"; - ``` - -## 常见问题 - -* 查看错误行 - - 由于 Insert Into 无法控制错误率，只能通过 `enable_insert_strict` 设置为完全容忍错误数据或完全忽略错误数据。因此如果 `enable_insert_strict` 设为 true，则 Insert Into 可能会失败。而如果 `enable_insert_strict` 设为 false，则可能出现仅导入了部分合格数据的情况。 - - 当返回结果中提供了 url 字段时，可以通过以下命令查看错误行： - - ```SHOW LOAD WARNINGS ON "url";``` - - 示例： - - ```SHOW LOAD WARNINGS ON "http://ip:port/api/_load_error_log?file=__shard_13/error_log_insert_stmt_d2cac0a0a16d482d-9041c949a4b71605_d2cac0a0a16d482d_9041c949a4b71605";``` - - 错误的原因通常如：源数据列长度超过目的数据列长度、列类型不匹配、分区不匹配、列顺序不匹配等等。 diff --git a/docs/documentation/cn/administrator-guide/load-data/load-manual.md b/docs/documentation/cn/administrator-guide/load-data/load-manual.md deleted file mode 100644 index b261687ad517d5..00000000000000 --- a/docs/documentation/cn/administrator-guide/load-data/load-manual.md +++ /dev/null @@ -1,210 +0,0 @@ - - -# 导入总览 - -导入（Load）功能就是将用户的原始数据导入到 Doris 中。导入成功后，用户即可通过 Mysql 客户端查询数据。 - -Doris 支持多种导入方式。建议先完整阅读本文档，再根据所选择的导入方式，查看各自导入方式的详细文档。 - -## 基本概念 - -1. Frontend（FE）：Doris 系统的元数据和调度节点。在导入流程中主要负责导入规划生成和导入任务的调度工作。 -2. Backend（BE）：Doris 系统的计算和存储节点。在导入流程中主要负责数据的 ETL 和存储。 -3. Broker：Broker 为一个独立的无状态进程。封装了文件系统接口，提供 Doris 读取远端存储系统中文件的能力。 -4. 导入作业（Load job）：导入作业读取用户提交的源数据，转换或清洗后，将数据导入到 Doris 系统中。导入完成后，数据即可被用户查询到。 -5. Label：所有导入作业都有一个 Label。Label 在一个数据库内唯一，可由用户指定或系统自动生成，用于标识一个导入作业。相同的 Label 仅可用于一个成功的导入作业。 -6. MySQL 协议/HTTP 协议：Doris 提供两种访问协议接口。 MySQL 协议和 HTTP 协议。部分导入方式使用 MySQL 协议接口提交作业，部分导入方式使用 HTTP 协议接口提交作业。 - -## 导入方式 - -为适配不同的数据导入需求，Doris 系统提供了5种不同的导入方式。每种导入方式支持不同的数据源，存在不同的使用方式（异步，同步）。 - -所有导入方式都支持 csv 数据格式。其中 Broker load 还支持 parquet 和 orc 数据格式。 - -每个导入方式的说明请参阅单个导入方式的操作手册。 - -* Broker load - - 通过 Broker 进程访问并读取外部数据源（如 HDFS）导入到 Doris。用户通过 Mysql 协议提交导入作业后，异步执行。通过 `SHOW LOAD` 命令查看导入结果。 - -* Stream load - - 用户通过 HTTP 协议提交请求并携带原始数据创建导入。主要用于快速将本地文件或数据流中的数据导入到 Doris。导入命令同步返回导入结果。 - -* Insert - - 类似 MySQL 中的 Insert 语句，Doris 提供 `INSERT INTO tbl SELECT ...;` 的方式从 Doris 的表中读取数据并导入到另一张表。或者通过 `INSERT INTO tbl VALUES(...);` 插入单条数据。 - -* Multi load - - 用户通过 HTTP 协议提交多个导入作业。Multi Load 可以保证多个导入作业的原子生效。 - -* Routine load - - 用户通过 MySQL 协议提交例行导入作业，生成一个常驻线程，不间断的从数据源（如 Kafka）中读取数据并导入到 Doris 中。 - -## 基本原理 - -### 导入执行流程 - -``` -+---------+ +---------+ +----------+ +-----------+ -| | | | | | | | -| PENDING +----->+ ETL +----->+ LOADING +----->+ FINISHED | -| | | | | | | | -+---------+ +---+-----+ +----+-----+ +-----------+ - | | | - | | | - | | | - | | | +-----------+ - | | | | | - +---------------+-----------------+------------> CANCELLED | - | | - +-----------+ - -``` - -如上图，一个导入作业主要经过上面4个阶段。 - -+ PENDING（非必须）: 该阶段只有 Broker Load 才有。Broker Load 被用户提交后会短暂停留在这个阶段，直到被 FE 中的 Scheduler 调度。 其中 Scheduler 的调度间隔为5秒。 - -+ ETL（非必须）： 该阶段在版本 0.10.0(包含) 之前存在，主要是用于将原始数据按照用户声明的方式进行变换，并且过滤不满足条件的原始数据。在 0.10.0 后的版本，ETL 阶段不再存在，其中数据 transform 的工作被合并到 LOADING 阶段。 - -+ LOADING： 该阶段在版本 0.10.0（包含）之前主要用于将变换后的数据推到对应的 BE 存储中。在 0.10.0 后的版本，该阶段先对数据进行清洗和变换，然后将数据发送到 BE 存储中。当所有导入数据均完成导入后，进入等待生效过程，此时 Load job 依旧是 LOADING。 - -+ FINISHED： 在 Load job 涉及的所有数据均生效后，Load job 的状态变成 FINISHED。FINISHED 后导入的数据均可查询。 - -+ CANCELLED: 在作业 FINISH 的之前，作业都可能被取消并进入 CANCELLED 状态。如用户手动取消，或导入出现错误等。CANCELLED 也是 Load Job 的最终状态，不可被再次执行。 - -上述阶段，除了 PENDING 到 LOADING 阶段是 Scheduler 轮训调度的，其他阶段之前的转移都是回调机制实现。 - -### Label 和 原子性 - -Doris 对所有导入方式提供原子性保证。既保证同一个导入作业内的数据，原子生效。不会出现仅导入部分数据的情况。 - -同时，每一个导入作业都有一个由用户指定或者系统自动生成的 Label。Label 在一个 Database 内唯一。当一个 Label 对应的导入作业成功后，不可再重复使用该 Label 提交导入作业。如果 Label 对应的导入作业失败，则可以重复使用。 - -用户可以通过 Label 机制，来保证 Label 对应的数据最多被导入一次，即At-Most-Once 语义。 - -## 同步和异步 - -Doris 目前的导入方式分为两类，同步和异步。如果是外部程序接入 Doris 的导入功能，需要判断使用导入方式是哪类再确定接入逻辑。 - -### 同步 - -同步导入方式即用户创建导入任务，Doris 同步执行导入，执行完成后返回用户导入结果。用户可直接根据创建导入任务命令返回的结果同步判断导入是否成功。 - -同步类型的导入方式有: **Stream load**，**Insert**。 - -操作步骤： - -1. 用户（外部系统）创建导入任务。 -2. Doris 返回导入结果。 -3. 用户（外部系统）判断导入结果，如果失败可以再次提交导入任务。 - -*注意：如果用户使用的导入方式是同步返回的，且导入的数据量过大，则创建导入请求可能会花很长时间才能返回结果。* - -### 异步 -异步导入方式即用户创建导入任务后，Doris 直接返回创建成功。**创建成功不代表数据已经导入**。导入任务会被异步执行，用户在创建成功后，需要通过轮询的方式发送查看命令查看导入作业的状态。如果创建失败，则可以根据失败信息，判断是否需要再次创建。 - -异步类型的导入方式有：**Broker load**，**Multi load**。 - -操作步骤： - -1. 用户（外部系统）创建导入任务。 -2. Doris 返回导入创建结果。 -3. 用户（外部系统）判断导入创建结果，成功则进入4，失败回到重试创建导入，回到1。 -4. 用户（外部系统）轮询查看导入任务，直到状态变为 FINISHED 或 CANCELLED。 - -### 注意事项 -无论是异步还是同步的导入类型，都不应该在 Doris 返回导入失败或导入创建失败后，无休止的重试。**外部系统在有限次数重试并失败后，保留失败信息，大部分多次重试均失败问题都是使用方法问题或数据本身问题。** - -## 内存限制 - -用户可以通过设置参数来限制单个导入的内存使用，以防止导入占用过多的内存而导致系统OOM。 -不同导入方式限制内存的方式略有不同，可以参阅各自的导入手册查看。 - -一个导入作业通常会分布在多个 Backend 上执行，导入内存限制的是一个导入作业，在单个 Backend 上的内存使用，而不是在整个集群的内存使用。 - -同时，每个 Backend 会设置可用于导入的内存的总体上限。具体配置参阅下面的通用系统配置小节。这个配置限制了所有在该 Backend 上运行的导入任务的总体内存使用上限。 - -较小的内存限制可能会影响导入效率，因为导入流程可能会因为内存达到上限而频繁的将内存中的数据写回磁盘。而过大的内存限制可能导致当导入并发较高时，系统OOM。所以，需要根据需求，合理的设置导入的内存限制。 - -## 最佳实践 - -用户在接入 Doris 导入时，一般会采用程序接入的方式，这样可以保证数据被定期的导入到 Doris 中。下面主要说明了程序接入 Doris 的最佳实践。 - -1. 选择合适的导入方式：根据数据源所在位置选择导入方式。例如：如果原始数据存放在 HDFS 上，则使用 Broker load 导入。 -2. 确定导入方式的协议：如果选择了 Broker load 导入方式，则外部系统需要能使用 MySQL 协议定期提交和查看导入作业。 -3. 确定导入方式的类型：导入方式为同步或异步。比如 Broker load 为异步导入方式，则外部系统在提交创建导入后，必须调用查看导入命令，根据查看导入命令的结果来判断导入是否成功。 -4. 制定 Label 生成策略：Label 生成策略需满足，每一批次数据唯一且固定的原则。这样 Doris 就可以保证 At-Most-Once。 -5. 程序自身保证 At-Least-Once：外部系统需要保证自身的 At-Least-Once，这样就可以保证导入流程的 Exactly-Once。 - -## 通用系统配置 - -下面主要解释了几个所有导入方式均通用的系统级别的配置。 - -### FE 配置 - -以下配置属于 FE 的系统配置，可以通过修改 FE 的配置文件 ```fe.conf``` 来修改配置。 - -+ max\_load\_timeout\_second 和 min\_load\_timeout\_second - - 这两个配置含义为：最大的导入超时时间，最小的导入超时时间，以秒为单位。默认的最大超时时间为3天, 默认的最小超时时间为1秒。用户自定义的导入超时时间不可超过这个范围。该参数通用于所有的导入方式。 - -+ desired\_max\_waiting\_jobs - - 在等待队列中的导入任务个数最大值，默认为100。当在 FE 中处于 PENDING 状态（也就是等待执行的）导入个数超过该值，新的导入请求则会被拒绝。 - - 此配置仅对异步执行的导入有效，当异步执行的导入等待个数超过默认值，则后续的创建导入请求会被拒绝。 - -+ max\_running\_txn\_num\_per\_db - - 这个配置的含义是说，每个 Database 中正在运行的导入最大个数（不区分导入类型，统一计数）。当当前 Database 正在运行的导入个数超过最大值时，后续的导入不会被执行。如果是同步导入作业，则导入会被拒绝。如果是异步导入作业。则作业会在队列中等待。 - -### BE 配置 - -以下配置属于 BE 的系统配置，可以通过修改 BE 的配置文件 ```be.conf``` 来修改配置。 - -+ push\_write\_mbytes\_per\_sec - - BE 上单个 Tablet 的写入速度限制。默认是 10，即 10MB/s。通常 BE 对单个 Tablet 的最大写入速度，根据 Schema 以及系统的不同，大约在 10-30MB/s 之间。可以适当调整这个参数来控制导入速度。 - -+ write\_buffer\_size - - 导入数据在 BE 上会先写入一个 memtable，memtable 达到阈值后才会写回磁盘。默认大小是 100MB。过小的阈值可能导致 BE 上存在大量的小文件。可以适当提高这个阈值减少文件数量。但过大的阈值可能导致 RPC 超时，见下面的配置说明。 - -+ tablet\_writer\_rpc\_timeout\_sec - - 导入过程中，发送一个 Batch（1024行）的 RPC 超时时间。默认 600 秒。因为该 RPC 可能涉及多个 memtable 的写盘操作，所以可能会因为写盘导致 RPC 超时，可以适当调整这个超时时间来减少超时错误（如 `send batch fail` 错误）。同时，如果调大 `write_buffer_size` 配置，也需要适当调大这个参数。 - -+ streaming\_load\_rpc\_max\_alive\_time\_sec - - 在导入过程中，Doris 会为每一个 Tablet 开启一个 Writer，用于接收数据并写入。这个参数指定了 Writer 的等待超时时间。如果在这个时间内，Writer 没有收到任何数据，则 Writer 会被自动销毁。当系统处理速度较慢时，Writer 可能长时间接收不到下一批数据，导致导入报错：`TabletWriter add batch with unknown id`。此时可适当增大这个配置。默认为 600 秒。 - -* load\_process\_max\_memory\_limit\_bytes 和 load\_process\_max\_memory\_limit\_percent - - 这两个参数，限制了单个 Backend 上，可用于导入任务的内存上限。分别是最大内存和最大内存百分比。`load_process_max_memory_limit_percent` 默认为 80，表示对 Backend 总内存限制的百分比（总内存限制 `mem_limit` 默认为 80%，表示对物理内存的百分比）。即假设物理内存为 M，则默认导入内存限制为 M * 80% * 80%。 - - `load_process_max_memory_limit_bytes` 默认为 100GB。系统会在两个参数中取较小者，作为最终的 Backend 导入内存使用上限。 - -+ label\_keep\_max\_second - - 设置导入任务记录保留时间。已经完成的（ FINISHED or CANCELLED ）导入任务记录会保留在 Doris 系统中一段时间，时间由此参数决定。参数默认值时间为3天。该参数通用与所有类型的导入任务。 diff --git a/docs/documentation/cn/administrator-guide/load-data/routine-load-manual.md b/docs/documentation/cn/administrator-guide/load-data/routine-load-manual.md deleted file mode 100644 index 941191e1d0d7f6..00000000000000 --- a/docs/documentation/cn/administrator-guide/load-data/routine-load-manual.md +++ /dev/null @@ -1,292 +0,0 @@ - - -# Routine Load - -例行导入（Routine Load）功能为用户提供了一种自动从指定数据源进行数据导入的功能。 - -本文档主要介绍该功能的实现原理、使用方式以及最佳实践。 - -## 名词解释 - -* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 -* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 -* RoutineLoadJob：用户提交的一个例行导入作业。 -* JobScheduler：例行导入作业调度器，用于调度和拆分一个 RoutineLoadJob 为多个 Task。 -* Task：RoutineLoadJob 被 JobScheduler 根据规则拆分的子任务。 -* TaskScheduler：任务调度器。用于调度 Task 的执行。 - -## 原理 - -``` - +---------+ - | Client | - +----+----+ - | -+-----------------------------+ -| FE | | -| +-----------v------------+ | -| | | | -| | Routine Load Job | | -| | | | -| +---+--------+--------+--+ | -| | | | | -| +---v--+ +---v--+ +---v--+ | -| | task | | task | | task | | -| +--+---+ +---+--+ +---+--+ | -| | | | | -+-----------------------------+ - | | | - v v v - +---+--+ +--+---+ ++-----+ - | BE | | BE | | BE | - +------+ +------+ +------+ - -``` - -如上图，Client 向 FE 提交一个例行导入作业。 - -FE 通过 JobScheduler 将一个导入作业拆分成若干个 Task。每个 Task 负责导入指定的一部分数据。Task 被 TaskScheduler 分配到指定的 BE 上执行。 - -在 BE 上，一个 Task 被视为一个普通的导入任务，通过 Stream Load 的导入机制进行导入。导入完成后，向 FE 汇报。 - -FE 中的 JobScheduler 根据汇报结果，继续生成后续新的 Task，或者对失败的 Task 进行重试。 - -整个例行导入作业通过不断的产生新的 Task，来完成数据不间断的导入。 - -## Kafka 例行导入 - -当前我们仅支持从 Kafka 系统进行例行导入。该部分会详细介绍 Kafka 例行导入使用方式和最佳实践。 - -### 使用限制 - -1. 支持无认证的 Kafka 访问，以及通过 SSL 方式认证的 Kafka 集群。 -2. 支持的消息格式为 csv 文本格式。每一个 message 为一行，且行尾**不包含**换行符。 -3. 仅支持 Kafka 0.10.0.0(含) 以上版本。 - -### 创建例行导入任务 - -创建例行导入任务的的详细语法可以连接到 Doris 后，执行 `HELP ROUTINE LOAD;` 查看语法帮助。这里主要详细介绍，创建作业时的注意事项。 - -* columns_mapping - - `columns_mapping` 主要用于指定表结构和 message 中的列映射关系，以及一些列的转换。如果不指定，Doris 会默认 message 中的列和表结构的列按顺序一一对应。虽然在正常情况下，如果源数据正好一一对应，则不指定也可以进行正常的数据导入。但是我们依然强烈建议用户**显式的指定列映射关系**。这样当表结构发生变化（比如增加一个 nullable 的列），或者源文件发生变化（比如增加了一列）时，导入任务依然可以继续进行。否则，当发生上述变动后，因为列映射关系不再一一对应，导入将报错。 - - 在 `columns_mapping` 中我们同样可以使用一些内置函数进行列的转换。但需要注意函数参数对应的实际列类型。举例说明： - - 假设用户需要导入只包含 `k1` 一列的表，列类型为 `int`。并且需要将源文件中的 null 值转换为 0。该功能可以通过 `ifnull` 函数实现。正确是的使用方式如下： - - `COLUMNS (xx, k1=ifnull(xx, "3"))` - - 注意这里我们使用 `"3"` 而不是 `3`，虽然 `k1` 的类型为 `int`。因为对于导入任务来说，源数据中的列类型都为 `varchar`，所以这里 `xx` 虚拟列的类型也为 `varchar`。所以我们需要使用 `"3"` 来进行对应的匹配，否则 `ifnull` 函数无法找到参数为 `(varchar, int)` 的函数签名，将出现错误。 - - 再举例，假设用户需要导入只包含 `k1` 一列的表，列类型为 `int`。并且需要将源文件中的对应列进行处理：将负数转换为正数，而将正数乘以 100。这个功能可以通过 `case when` 函数实现，正确写法应如下： - - `COLUMNS (xx, case when xx < 0 than cast(-xx as varchar) else cast((xx + '100') as varchar) end)` - - 注意这里我们需要将 `case when` 中所有的参数都最终转换为 varchar，才能得到期望的结果。 - -* where_predicates - - `where_predicates` 中的的列的类型，已经是实际的列类型了，所以无需向 `columns_mapping` 那样强制的转换为 varchar 类型。按照实际的列类型书写即可。 - -* desired\_concurrent\_number - - `desired_concurrent_number` 用于指定一个例行作业期望的并发度。即一个作业，最多有多少 task 同时在执行。对于 Kafka 导入而言，当前的实际并发度计算如下： - - ``` - Min(partition num, desired_concurrent_number, alive_backend_num, Config.max_routine_load_task_concurrrent_num) - ``` - - 其中 `Config.max_routine_load_task_concurrrent_num` 是系统的一个默认的最大并发数限制。这是一个 FE 配置，可以通过改配置调整。默认为 5。 - - 其中 partition num 指订阅的 Kafka topic 的 partition 数量。`alive_backend_num` 是当前正常的 BE 节点数。 - -* max\_batch\_interval/max\_batch\_rows/max\_batch\_size - - 这三个参数用于控制单个任务的执行时间。其中任意一个阈值达到，则任务结束。其中 `max_batch_rows` 用于记录从 Kafka 中读取到的数据行数。`max_batch_size` 用于记录从 Kafka 中读取到的数据量，单位是字节。目前一个任务的消费速率大约为 5-10MB/s。 - - 那么假设一行数据 500B，用户希望每 100MB 或 10 秒为一个 task。100MB 的预期处理时间是 10-20 秒，对应的行数约为 200000 行。则一个合理的配置为： - - ``` - "max_batch_interval" = "10", - "max_batch_rows" = "200000", - "max_batch_size" = "104857600" - ``` - - 以上示例中的参数也是这些配置的默认参数。 - -* max\_error\_number - - `max_error_number` 用于控制错误率。在错误率过高的时候，作业会自动暂停。因为整个作业是面向数据流的，且由于数据流的无边界性，我们无法像其他导入任务一样，通过一个错误比例来计算错误率。因此这里提供了一种新的计算方式，来计算数据流中的错误比例。 - - 我们设定了一个采样窗口。窗口的大小为 `max_batch_rows * 10`。在一个采样窗口内，如果错误行数超过 `max_error_number`，则作业被暂停。如果没有超过，则下一个窗口重新开始计算错误行数。 - - 我们假设 `max_batch_rows` 为 200000，则窗口大小为 2000000。设 `max_error_number` 为 20000，即用户预期每 2000000 行的错误行为 20000。即错误率为 1%。但是因为不是每批次任务正好消费 200000 行，所以窗口的实际范围是 [2000000, 2200000]，即有 10% 的统计误差。 - - 错误行不包括通过 where 条件过滤掉的行。但是包括没有对应的 Doris 表中的分区的行。 - -* data\_source\_properties - - `data_source_properties` 中可以指定消费具体的 Kakfa partition。如果不指定，则默认消费所订阅的 topic 的所有 partition。 - - 注意，当显式的指定了 partition，则导入作业不会再动态的检测 Kafka partition 的变化。如果没有指定，则会根据 kafka partition 的变化，动态调整需要消费的 partition。 - -* strict\_mode - - Routine load 导入可以开启 strict mode 模式。开启方式为在 job\_properties 中增加 ```"strict_mode" = "true"``` 。默认的 strict mode 为关闭。 - - strict mode 模式的意思是：对于导入过程中的列类型转换进行严格过滤。严格过滤的策略如下： - - 1. 对于列类型转换来说，如果 strict mode 为true，则错误的数据将被 filter。这里的错误数据是指：原始数据并不为空值，在参与列类型转换后结果为空值的这一类数据。 - - 2. 对于导入的某列由函数变换生成时，strict mode 对其不产生影响。 - - 3. 对于导入的某列类型包含范围限制的，如果原始数据能正常通过类型转换，但无法通过范围限制的，strict mode 对其也不产生影响。例如：如果类型是 decimal(1,0), 原始数据为 10，则属于可以通过类型转换但不在列声明的范围内。这种数据 strict 对其不产生影响。 - -#### strict mode 与 source data 的导入关系 - -这里以列类型为 TinyInt 来举例 - ->注：当表中的列允许导入空值时 - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|---------| -|空值 | \N | N/A | true or false | NULL| -|not null | aaa or 2000 | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 | 1 | true or false | correct data| - -这里以列类型为 Decimal(1,0) 举例 - ->注：当表中的列允许导入空值时 - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|--------| -|空值 | \N | N/A | true or false | NULL| -|not null | aaa | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 or 10 | 1 | true or false | correct data| - -> 注意：10 虽然是一个超过范围的值，但是因为其类型符合 decimal的要求，所以 strict mode对其不产生影响。10 最后会在其他 ETL 处理流程中被过滤。但不会被 strict mode 过滤。 - -#### 访问 SSL 认证的 Kafka 集群 - -访问 SSL 认证的 Kafka 集群需要用户提供用于认证 Kafka Broker 公钥的证书文件（ca.pem）。如果 Kafka 集群同时开启了客户端认证，则还需提供客户端的公钥（client.pem）、密钥文件（client.key），以及密钥密码。这里所需的文件需要先通过 `CREAE FILE` 命令上传到 Doris 中，**并且 catalog 名称为 `kafka`**。`CREATE FILE` 命令的具体帮助可以参见 `HELP CREATE FILE;`。这里给出示例： - -1. 上传文件 - - ``` - CREATE FILE "ca.pem" PROPERTIES("url" = "https://example_url/kafka-key/ca.pem", "catalog" = "kafka"); - CREATE FILE "client.key" PROPERTIES("url" = "https://example_urlkafka-key/client.key", "catalog" = "kafka"); - CREATE FILE "client.pem" PROPERTIES("url" = "https://example_url/kafka-key/client.pem", "catalog" = "kafka"); - ``` - -2. 创建例行导入作业 - - ``` - CREATE ROUTINE LOAD db1.job1 on tbl1 - PROPERTIES - ( - "desired_concurrent_number"="1" - ) - FROM KAFKA - ( - "kafka_broker_list"= "broker1:9091,broker2:9091", - "kafka_topic" = "my_topic", - "property.security.protocol" = "ssl", - "property.ssl.ca.location" = "FILE:ca.pem", - "property.ssl.certificate.location" = "FILE:client.pem", - "property.ssl.key.location" = "FILE:client.key", - "property.ssl.key.password" = "abcdefg" - ); - ``` - -> Doris 通过 Kafka 的 C++ API `librdkafka` 来访问 Kafka 集群。`librdkafka` 所支持的参数可以参阅 -> -> `https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md` - - -### 查看导入作业状态 - -查看**作业**状态的具体命令和示例可以通过 `HELP SHOW ROUTINE LOAD;` 命令查看。 - -查看**任务**运行状态的具体命令和示例可以通过 `HELP SHOW ROUTINE LOAD TASK;` 命令查看。 - -只能查看当前正在运行中的任务，已结束和未开始的任务无法查看。 - -### 作业控制 - -用户可以通过 `STOP/PAUSE/RESUME` 三个命令来控制作业的停止，暂停和重启。可以通过 `HELP STOP ROUTINE LOAD;`, `HELP PAUSE ROUTINE LOAD;` 以及 `HELP RESUME ROUTINE LOAD;` 三个命令查看帮助和示例。 - -## 其他说明 - -1. 例行导入作业和 ALTER TABLE 操作的关系 - - * 例行导入不会阻塞 SCHEMA CHANGE 和 ROLLUP 操作。但是注意如果 SCHEMA CHANGE 完成后，列映射关系无法匹配，则会导致作业的错误数据激增，最终导致作业暂停。建议通过在例行导入作业中显式指定列映射关系，以及通过增加 Nullable 列或带 Default 值的列来减少这类问题。 - * 删除表的 Partition 可能会导致导入数据无法找到对应的 Partition，作业进入暂停。 - -2. 例行导入作业和其他导入作业的关系（LOAD, DELETE, INSERT） - - * 例行导入和其他 LOAD 作业以及 INSERT 操作没有冲突。 - * 当执行 DELETE 操作时，对应表分区不能有任何正在执行的导入任务。所以在执行 DELETE 操作前，可能需要先暂停例行导入作业，并等待已下发的 task 全部完成后，才可以执行 DELETE。 - -3. 例行导入作业和 DROP DATABASE/TABLE 操作的关系 - - 当例行导入对应的 database 或 table 被删除后，作业会自动 CANCEL。 - -4. kafka 类型的例行导入作业和 kafka topic 的关系 - - 当用户在创建例行导入声明的 `kafka_topic` 在kafka集群中不存在时。 - - * 如果用户 kafka 集群的 broker 设置了 `auto.create.topics.enable = true`，则 `kafka_topic` 会先被自动创建，自动创建的 partition 个数是由**用户方的kafka集群**中的 broker 配置 `num.partitions` 决定的。例行作业会正常的不断读取该 topic 的数据。 - * 如果用户 kafka 集群的 broker 设置了 `auto.create.topics.enable = false`, 则 topic 不会被自动创建，例行作业会在没有读取任何数据之前就被暂停，状态为 `PAUSED`。 - - 所以，如果用户希望当 kafka topic 不存在的时候，被例行作业自动创建的话，只需要将**用户方的kafka集群**中的 broker 设置 `auto.create.topics.enable = true` 即可。 - -## 相关参数 - -一些系统配置参数会影响例行导入的使用。 - -1. max\_routine\_load\_task\_concurrent\_num - - FE 配置项，默认为 5，可以运行时修改。该参数限制了一个例行导入作业最大的子任务并发数。建议维持默认值。设置过大，可能导致同时并发的任务数过多，占用集群资源。 - -2. max\_routine_load\_task\_num\_per\_be - - FE 配置项，默认为5，可以运行时修改。该参数限制了每个 BE 节点最多并发执行的子任务个数。建议维持默认值。如果设置过大，可能导致并发任务数过多，占用集群资源。 - -3. max\_routine\_load\_job\_num - - FE 配置项，默认为100，可以运行时修改。该参数限制的例行导入作业的总数，包括 NEED_SCHEDULED, RUNNING, PAUSE 这些状态。超过后，不能在提交新的作业。 - -4. max\_consumer\_num\_per\_group - - BE 配置项，默认为 3。该参数表示一个子任务中最多生成几个 consumer 进行数据消费。对于 Kafka 数据源，一个 consumer 可能消费一个或多个 kafka partition。假设一个任务需要消费 6 个 kafka partition，则会生成 3 个 consumer，每个 consumer 消费 2 个 partition。如果只有 2 个 partition，则只会生成 2 个 consumer，每个 consumer 消费 1 个 partition。 - -5. push\_write\_mbytes\_per\_sec - - BE 配置项。默认为 10，即 10MB/s。该参数为导入通用参数，不限于例行导入作业。该参数限制了导入数据写入磁盘的速度。对于 SSD 等高性能存储设备，可以适当增加这个限速。 - -6. max\_tolerable\_backend\_down\_num - FE 配置项，默认值是0。在满足某些条件下，Doris可PAUSED的任务重新调度，即变成RUNNING。该参数为0代表只有所有BE节点是alive状态才允许重新调度。 - -7. period\_of\_auto\_resume\_min - FE 配置项，默认是5分钟。Doris重新调度，只会在5分钟这个周期内，最多尝试3次. 如果3次都失败则锁定当前任务，后续不在进行调度。但可通过人为干预，进行手动恢复。 - diff --git a/docs/documentation/cn/administrator-guide/load-data/stream-load-manual.md b/docs/documentation/cn/administrator-guide/load-data/stream-load-manual.md deleted file mode 100644 index 3010236763c712..00000000000000 --- a/docs/documentation/cn/administrator-guide/load-data/stream-load-manual.md +++ /dev/null @@ -1,330 +0,0 @@ - - -# Stream load - -Stream load 是一个同步的导入方式，用户通过发送 HTTP 协议发送请求将本地文件或数据流导入到 Doris 中。Stream load 同步执行导入并返回导入结果。用户可直接通过请求的返回体判断本次导入是否成功。 - -Stream load 主要适用于导入本地文件，或通过程序导入数据流中的数据。 - -## 基本原理 - -下图展示了 Stream load 的主要流程，省略了一些导入细节。 - -``` - ^ + - | | - | | 1A. User submit load to FE - | | - | +--v-----------+ - | | FE | -5. Return result to user | +--+-----------+ - | | - | | 2. Redirect to BE - | | - | +--v-----------+ - +---+Coordinator BE| 1B. User submit load to BE - +-+-----+----+-+ - | | | - +-----+ | +-----+ - | | | 3. Distrbute data - | | | - +-v-+ +-v-+ +-v-+ - |BE | |BE | |BE | - +---+ +---+ +---+ -``` - -Stream load 中，Doris 会选定一个节点作为 Coordinator 节点。该节点负责接数据并分发数据到其他数据节点。 - -用户通过 HTTP 协议提交导入命令。如果提交到 FE，则 FE 会通过 HTTP redirect 指令将请求转发给某一个 BE。用户也可以直接提交导入命令给某一指定 BE。 - -导入的最终结果由 Coordinator BE 返回给用户。 - -## 基本操作 -### 创建导入 - -Stream load 通过 HTTP 协议提交和传输数据。这里通过 `curl` 命令展示如何提交导入。 - -用户也可以通过其他 HTTP client 进行操作。 - -``` -curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_host:http_port/api/{db}/{table}/_stream_load - -Header 中支持属性见下面的 ‘导入任务参数’ 说明 -格式为: -H "key1:value1" -``` - -示例： - -``` -curl --location-trusted -u root -T date -H "label:123" http://abc.com:8030/api/test/date/_stream_load -``` -创建导入的详细语法帮助执行 ```HELP STREAM LOAD``` 查看, 下面主要介绍创建 Stream load 的部分参数意义。 - -#### 签名参数 - -+ user/passwd - - Stream load 由于创建导入的协议使用的是 HTTP 协议，通过 Basic access authentication 进行签名。Doris 系统会根据签名验证用户身份和导入权限。 - -#### 导入任务参数 - -Stream load 由于使用的是 HTTP 协议，所以所有导入任务有关的参数均设置在 Header 中。下面主要介绍了 Stream load 导入任务参数的部分参数意义。 - -+ label - - 导入任务的标识。每个导入任务，都有一个在单 database 内部唯一的 label。label 是用户在导入命令中自定义的名称。通过这个 label，用户可以查看对应导入任务的执行情况。 - - label 的另一个作用，是防止用户重复导入相同的数据。**强烈推荐用户同一批次数据使用相同的 label。这样同一批次数据的重复请求只会被接受一次，保证了 At-Most-Once** - - 当 label 对应的导入作业状态为 CANCELLED 时，该 label 可以再次被使用。 - -+ max\_filter\_ratio - - 导入任务的最大容忍率，默认为0容忍，取值范围是0~1。当导入的错误率超过该值，则导入失败。 - - 如果用户希望忽略错误的行，可以通过设置这个参数大于 0，来保证导入可以成功。 - - 计算公式为： - - ``` (dpp.abnorm.ALL / (dpp.abnorm.ALL + dpp.norm.ALL ) ) > max_filter_ratio ``` - - ```dpp.abnorm.ALL``` 表示数据质量不合格的行数。如类型不匹配，列数不匹配，长度不匹配等等。 - - ```dpp.norm.ALL``` 指的是导入过程中正确数据的条数。可以通过 ```SHOW LOAD``` 命令查询导入任务的正确数据量。 - - 原始文件的行数 = `dpp.abnorm.ALL + dpp.norm.ALL` - -+ where - - 导入任务指定的过滤条件。Stream load 支持对原始数据指定 where 语句进行过滤。被过滤的数据将不会被导入，也不会参与 filter ratio 的计算，但会被计入```num_rows_unselected```。 - -+ partition - - 待导入表的 Partition 信息，如果待导入数据不属于指定的 Partition 则不会被导入。这些数据将计入 ```dpp.abnorm.ALL ``` - -+ columns - - 待导入数据的函数变换配置，目前 Stream load 支持的函数变换方法包含列的顺序变化以及表达式变换，其中表达式变换的方法与查询语句的一致。 - - ``` - 列顺序变换例子：原始数据有两列，目前表也有两列（c1,c2）但是原始文件的第一列对应的是目标表的c2列, 而原始文件的第二列对应的是目标表的c1列，则写法如下： - columns: c2,c1 - - 表达式变换例子：原始文件有两列，目标表也有两列（c1,c2）但是原始文件的两列均需要经过函数变换才能对应目标表的两列，则写法如下： - columns: tmp_c1, tmp_c2, c1 = year(tmp_c1), c2 = month(tmp_c2) - 其中 tmp_*是一个占位符，代表的是原始文件中的两个原始列。 - ``` - -+ exec\_mem\_limit - - 导入内存限制。默认为 2GB，单位为字节。 - -+ strict\_mode - - Stream load 导入可以开启 strict mode 模式。开启方式为在 HEADER 中声明 ```strict_mode=true``` 。默认的 strict mode 为关闭。 - - strict mode 模式的意思是：对于导入过程中的列类型转换进行严格过滤。严格过滤的策略如下： - - 1. 对于列类型转换来说，如果 strict mode 为true，则错误的数据将被 filter。这里的错误数据是指：原始数据并不为空值，在参与列类型转换后结果为空值的这一类数据。 - - 2. 对于导入的某列由函数变换生成时，strict mode 对其不产生影响。 - - 3. 对于导入的某列类型包含范围限制的，如果原始数据能正常通过类型转换，但无法通过范围限制的，strict mode 对其也不产生影响。例如：如果类型是 decimal(1,0), 原始数据为 10，则属于可以通过类型转换但不在列声明的范围内。这种数据 strict 对其不产生影响。 - -#### strict mode 与 source data 的导入关系 - -这里以列类型为 TinyInt 来举例 - ->注：当表中的列允许导入空值时 - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|---------| -|空值 | \N | N/A | true or false | NULL| -|not null | aaa or 2000 | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 | 1 | true or false | correct data| - -这里以列类型为 Decimal(1,0) 举例 - ->注：当表中的列允许导入空值时 - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|--------| -|空值 | \N | N/A | true or false | NULL| -|not null | aaa | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 or 10 | 1 | true or false | correct data| - -> 注意：10 虽然是一个超过范围的值，但是因为其类型符合 decimal的要求，所以 strict mode对其不产生影响。10 最后会在其他 ETL 处理流程中被过滤。但不会被 strict mode 过滤。 - - -### 返回结果 - -由于 Stream load 是一种同步的导入方式，所以导入的结果会通过创建导入的返回值直接返回给用户。 - -示例： - -``` -{ - "TxnId": 1003, - "Label": "b6f3bc78-0d2c-45d9-9e4c-faa0a0149bee", - "Status": "Success", - "ExistingJobStatus": "FINISHED", // optional - "Message": "OK", - "NumberTotalRows": 1000000, - "NumberLoadedRows": 1000000, - "NumberFilteredRows": 1, - "NumberUnselectedRows": 0, - "LoadBytes": 40888898, - "LoadTimeMs": 2144, - "ErrorURL": "http://192.168.1.1:8042/api/_load_error_log?file=__shard_0/error_log_insert_stmt_db18266d4d9b4ee5-abb00ddd64bdf005_db18266d4d9b4ee5_abb00ddd64bdf005" -} -``` - -下面主要解释了 Stream load 导入结果参数： - -+ TxnId：导入的事务ID。用户可不感知。 - -+ Label：导入 Label。由用户指定或系统自动生成。 - -+ Status：导入完成状态。 - - "Success"：表示导入成功。 - - "Publish Timeout"：该状态也表示导入已经完成，只是数据可能会延迟可见，无需重试。 - - "Label Already Exists"：Label 重复，需更换 Label。 - - "Fail"：导入失败。 - -+ ExistingJobStatus：已存在的 Label 对应的导入作业的状态。 - - 这个字段只有在当 Status 为 "Label Already Exists" 是才会显示。用户可以通过这个状态，知晓已存在 Label 对应的导入作业的状态。"RUNNING" 表示作业还在执行，"FINISHED" 表示作业成功。 - -+ Message：导入错误信息。 - -+ NumberTotalRows：导入总处理的行数。 - -+ NumberLoadedRows：成功导入的行数。 - -+ NumberFilteredRows：数据质量不合格的行数。 - -+ NumberUnselectedRows：被 where 条件过滤的行数。 - -+ LoadBytes：导入的字节数。 - -+ LoadTimeMs：导入完成时间。单位毫秒。 - -+ ErrorURL：如果有数据质量问题，通过访问这个 URL 查看具体错误行。 - -> 注意：由于 Stream load 是同步的导入方式，所以并不会在 Doris 系统中记录导入信息，用户无法异步的通过查看导入命令看到 Stream load。使用时需监听创建导入请求的返回值获取导入结果。 - -### 取消导入 - -用户无法手动取消 Stream load，Stream load 在超时或者导入错误后会被系统自动取消。 - -## 相关系统配置 - -### FE 配置 - -+ stream\_load\_default\_timeout\_second - - 导入任务的超时时间(以秒为单位)，导入任务在设定的 timeout 时间内未完成则会被系统取消，变成 CANCELLED。 - - 默认的 timeout 时间为 600 秒。如果导入的源文件无法在规定时间内完成导入，用户可以在 stream load 请求中设置单独的超时时间。 - - 或者调整 FE 的参数```stream_load_default_timeout_second``` 来设置全局的默认超时时间。 - -### BE 配置 - -+ streaming\_load\_max\_mb - - Stream load 的最大导入大小，默认为 10G，单位是 MB。如果用户的原始文件超过这个值，则需要调整 BE 的参数 ```streaming_load_max_mb```。 - -## 最佳实践 - -### 应用场景 - -使用 Stream load 的最合适场景就是原始文件在内存中，或者在磁盘中。其次，由于 Stream load 是一种同步的导入方式，所以用户如果希望用同步方式获取导入结果，也可以使用这种导入。 - -### 数据量 - -由于 Stream load 的原理是由 BE 发起的导入并分发数据，建议的导入数据量在 1G 到 10G 之间。由于默认的最大 Stream load 导入数据量为 10G，所以如果要导入超过 10G 的文件需要修改 BE 的配置 ```streaming_load_max_mb``` - -``` -比如：待导入文件大小为15G -修改 BE 配置 streaming_load_max_mb 为 16000 即可。 -``` - -Stream load 的默认超时为 300秒，按照 Doris 目前最大的导入限速来看，约超过 3G 的文件就需要修改导入任务默认超时时间了。 - -``` -导入任务超时时间 = 导入数据量 / 10M/s （具体的平均导入速度需要用户根据自己的集群情况计算） -例如：导入一个 10G 的文件 -timeout = 1000s 等于 10G / 10M/s -``` - -### 完整例子 -数据情况： 数据在发送导入请求端的本地磁盘路径 /home/store_sales 中，导入的数据量约为 15G，希望导入到数据库 bj_sales 的表 store_sales 中。 - -集群情况：Stream load 的并发数不受集群大小影响。 - -+ step1: 导入文件大小是否超过默认的最大导入大小10G - - ``` - 修改 BE conf - streaming_load_max_mb = 16000 - ``` -+ step2: 计算大概的导入时间是否超过默认 timeout 值 - - ``` - 导入时间 ≈ 15000 / 10 = 1500s - 超过了默认的 timeout 时间，需要修改 FE 的配置 - stream_load_default_timeout_second = 1500 - ``` - -+ step3：创建导入任务 - - ``` - curl --location-trusted -u user:password -T /home/store_sales -H "label:abc" http://abc.com:8000/api/bj_sales/store_sales/_stream_load - ``` - -## 常见问题 - -* Label Already Exists - - Stream load 的 Label 重复排查步骤如下： - - 1. 是否和其他导入方式已经存在的导入 Label 冲突： - - 由于 Doris 系统中导入的 Label 不区分导入方式，所以存在其他导入方式使用了相同 Label 的问题。 - - 通过 ```SHOW LOAD WHERE LABEL = “xxx”```，其中 xxx 为重复的 Label 字符串，查看是否已经存在一个 FINISHED 导入的 Label 和用户申请创建的 Label 相同。 - - 2. 是否 Stream load 同一个作业被重复提交了 - - 由于 Stream load 是 HTTP 协议提交创建导入任务，一般各个语言的 HTTP Client 均会自带请求重试逻辑。Doris 系统在接受到第一个请求后，已经开始操作 Stream load，但是由于没有及时返回给 Client 端结果， Client 端会发生再次重试创建请求的情况。这时候 Doris 系统由于已经在操作第一个请求，所以第二个请求已经就会被报 Label Already Exists 的情况。 - - 排查上述可能的方法：使用 Label 搜索 FE Master 的日志，看是否存在同一个 Label 出现了两次 ```redirect load action to destination= ``` 的情况。如果有就说明，请求被 Client 端重复提交了。 - - 建议用户根据当前请求的数据量，计算出大致导入的时间，并根据导入超时时间改大 Client 端的请求超时时间，避免请求被 Client 端多次提交。 - - - diff --git a/docs/documentation/cn/administrator-guide/materialized-view/index.rst b/docs/documentation/cn/administrator-guide/materialized-view/index.rst deleted file mode 100644 index 280d7820a1fd28..00000000000000 --- a/docs/documentation/cn/administrator-guide/materialized-view/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============= -物化视图 -============= - -.. toctree:: - :maxdepth: 2 - :glob: - - * diff --git a/docs/documentation/cn/administrator-guide/operation/index.rst b/docs/documentation/cn/administrator-guide/operation/index.rst deleted file mode 100644 index 978ac3d83f8503..00000000000000 --- a/docs/documentation/cn/administrator-guide/operation/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============= -运维操作 -============= - -.. toctree:: - :maxdepth: 2 - :glob: - - * diff --git a/docs/documentation/cn/administrator-guide/operation/metadata-operation.md b/docs/documentation/cn/administrator-guide/operation/metadata-operation.md deleted file mode 100644 index 29d59d25352d7b..00000000000000 --- a/docs/documentation/cn/administrator-guide/operation/metadata-operation.md +++ /dev/null @@ -1,335 +0,0 @@ - - -# 元数据运维 - -本文档主要介绍在实际生产环境中，如何对 Doris 的元数据进行管理。包括 FE 节点建议的部署方式、一些常用的操作方法、以及常见错误的解决方法。 - -在阅读本文当前，请先阅读 [Doris 元数据设计文档](../../internal/metadata-design.md) 了解 Doris 元数据的工作原理。 - -## 重要提示 - -* 当前元数据的设计是无法向后兼容的。即如果新版本有新增的元数据结构变动（可以查看 FE 代码中的 `FeMetaVersion.java` 文件中是否有新增的 VERSION），那么在升级到新版本后，通常是无法再回滚到旧版本的。所以，在升级 FE 之前，请务必按照 [升级文档](../../installing/upgrade.md) 中的操作，测试元数据兼容性。 - -## 元数据目录结构 - -我们假设在 fe.conf 中指定的 `meta_dir` 的路径为 `/path/to/palo-meta`。那么一个正常运行中的 Doris 集群，元数据的目录结构应该如下： - -``` -/path/to/palo-meta/ - |-- bdb/ - | |-- 00000000.jdb - | |-- je.config.csv - | |-- je.info.0 - | |-- je.info.0.lck - | |-- je.lck - | `-- je.stat.csv - `-- image/ - |-- ROLE - |-- VERSION - `-- image.xxxx -``` - -1. bdb 目录 - - 我们将 [bdbje](https://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html) 作为一个分布式的 kv 系统，存放元数据的 journal。这个 bdb 目录相当于 bdbje 的 “数据目录”。 - - 其中 `.jdb` 后缀的是 bdbje 的数据文件。这些数据文件会随着元数据 journal 的不断增多而越来越多。当 Doris 定期做完 image 后，旧的日志就会被删除。所以正常情况下，这些数据文件的总大小从几 MB 到几 GB 不等（取决于使用 Doris 的方式，如导入频率等）。当数据文件的总大小大于 10GB，则可能需要怀疑是否是因为 image 没有成功，或者分发 image 失败导致的历史 journal 一直无法删除。 - - `je.info.0` 是 bdbje 的运行日志。这个日志中的时间是 UTC+0 时区的。我们可能在后面的某个版本中修复这个问题。通过这个日志，也可以查看一些 bdbje 的运行情况。 - -2. image 目录 - - image 目录用于存放 Doris 定期生成的元数据镜像文件。通常情况下，你会看到有一个 `image.xxxxx` 的镜像文件。其中 `xxxxx` 是一个数字。这个数字表示该镜像包含 `xxxxx` 号之前的所有元数据 journal。而这个文件的生成时间（通过 `ls -al` 查看即可）通常就是镜像的生成时间。 - - 你也可能会看到一个 `image.ckpt` 文件。这是一个正在生成的元数据镜像。通过 `du -sh` 命令应该可以看到这个文件大小在不断变大，说明镜像内容正在写入这个文件。当镜像写完后，会自动重名为一个新的 `image.xxxxx` 并替换旧的 image 文件。 - - 只有角色为 Master 的 FE 才会主动定期生成 image 文件。每次生成完后，都会推送给其他非 Master 角色的 FE。当确认其他所有 FE 都收到这个 image 后，Master FE 会删除 bdbje 中旧的元数据 journal。所以，如果 image 生成失败，或者 image 推送给其他 FE 失败时，都会导致 bdbje 中的数据不断累积。 - - `ROLE` 文件记录了 FE 的类型（FOLLOWER 或 OBSERVER），是一个文本文件。 - - `VERSION` 文件记录了这个 Doris 集群的 cluster id，以及用于各个节点之间访问认证的 token，也是一个文本文件。 - - `ROLE` 文件和 `VERSION` 文件只可能同时存在，或同时不存在（如第一次启动时）。 - -## 基本操作 - -### 启动单节点 FE - -单节点 FE 是最基本的一种部署方式。一个完整的 Doris 集群，至少需要一个 FE 节点。当只有一个 FE 节点时，这个节点的类型为 Follower，角色为 Master。 - -1. 第一次启动 - - 1. 假设在 fe.conf 中指定的 `meta_dir` 的路径为 `/path/to/palo-meta`。 - 2. 确保 `/path/to/palo-meta` 已存在，权限正确，且目录为空。 - 3. 直接通过 `sh bin/start_fe.sh` 即可启动。 - 4. 启动后，你应该可以在 fe.log 中看到如下日志： - - * Palo FE starting... - * image does not exist: /path/to/palo-meta/image/image.0 - * transfer from INIT to UNKNOWN - * transfer from UNKNOWN to MASTER - * the very first time to open bdb, dbname is 1 - * start fencing, epoch number is 1 - * finish replay in xxx msec - * QE service start - * thrift server started - - 以上日志不一定严格按照这个顺序，但基本类似。 - - 5. 单节点 FE 的第一次启动通常不会遇到问题。如果你没有看到以上日志，一般来说是没有仔细按照文档步骤操作，请仔细阅读相关 wiki。 - -2. 重启 - - 1. 直接使用 `sh bin/start_fe.sh` 可以重新启动已经停止的 FE 节点。 - 2. 重启后，你应该可以在 fe.log 中看到如下日志： - - * Palo FE starting... - * finished to get cluster id: xxxx, role: FOLLOWER and node name: xxxx - * 如果重启前还没有 image 产生，则会看到： - * image does not exist: /path/to/palo-meta/image/image.0 - - * 如果重启前有 image 产生，则会看到： - * start load image from /path/to/palo-meta/image/image.xxx. is ckpt: false - * finished load image in xxx ms - - * transfer from INIT to UNKNOWN - * replayed journal id is xxxx, replay to journal id is yyyy - * transfer from UNKNOWN to MASTER - * finish replay in xxx msec - * master finish replay journal, can write now. - * begin to generate new image: image.xxxx - * start save image to /path/to/palo-meta/image/image.ckpt. is ckpt: true - * finished save image /path/to/palo-meta/image/image.ckpt in xxx ms. checksum is xxxx - * push image.xxx to other nodes. totally xx nodes, push successed xx nodes - * QE service start - * thrift server started - - 以上日志不一定严格按照这个顺序，但基本类似。 - -3. 常见问题 - - 对于单节点 FE 的部署，启停通常不会遇到什么问题。如果有问题，请先参照相关 wiki，仔细核对你的操作步骤。 - -### 添加 FE - -添加 FE 流程在 [部署和升级文档](https://github.com/apache/incubator-doris/wiki/Doris-Deploy-%26-Upgrade) 有详细介绍，不再赘述。这里主要说明一些注意事项，以及常见问题。 - -1. 注意事项 - - * 在添加新的 FE 之前，一定先确保当前的 Master FE 运行正常（连接是否正常，JVM 是否正常，image 生成是否正常，bdbje 数据目录是否过大等等） - * 第一次启动新的 FE，一定确保添加了 `-helper` 参数指向 Master FE。再次启动时可不用添加 `-helper`。（如果指定了 `-helper`，FE 会直接询问 helper 节点自己的角色，如果没有指定，FE会尝试从 `palo-meta/image/` 目录下的 `ROLE` 和 `VERSION` 文件中获取信息）。 - * 第一次启动新的 FE，一定确保这个 FE 的 `meta_dir` 已经创建、权限正确且为空。 - * 启动新的 FE，和执行 `ALTER SYSTEM ADD FOLLOWER/OBSERVER` 语句在元数据添加 FE，这两个操作的顺序没有先后要求。如果先启动了新的 FE，而没有执行语句，则新的 FE 日志中会一直滚动 `current node is not added to the group. please add it first.` 字样。当执行语句后，则会进入正常流程。 - * 请确保前一个 FE 添加成功后，再添加下一个 FE。 - * 建议直接连接到 MASTER FE 执行 `ALTER SYSTEM ADD FOLLOWER/OBSERVER` 语句。 - -2. 常见问题 - - 1. this node is DETACHED - - 当第一次启动一个待添加的 FE 时，如果 Master FE 上的 palo-meta/bdb 中的数据很大，则可能在待添加的 FE 日志中看到 `this node is DETACHED.` 字样。这时，bdbje 正在复制数据，你可以看到待添加的 FE 的 `bdb/` 目录正在变大。这个过程通常会在数分钟不等（取决于 bdbje 中的数据量）。之后，fe.log 中可能会有一些 bdbje 相关的错误堆栈信息。如果最终日志中显示 `QE service start` 和 `thrift server started`，则通常表示启动成功。可以通过 mysql-client 连接这个 FE 尝试操作。如果没有出现这些字样，则可能是 bdbje 复制日志超时等问题。这时，直接再次重启这个 FE，通常即可解决问题。 - - 2. 各种原因导致添加失败 - - * 如果添加的是 OBSERVER，因为 OBSERVER 类型的 FE 不参与元数据的多数写，理论上可以随意启停。因此，对于添加 OBSERVER 失败的情况。可以直接杀死 OBSERVER FE 的进程，清空 OBSERVER 的元数据目录后，重新进行一遍添加流程。 - - * 如果添加的是 FOLLOWER，因为 FOLLOWER 是参与元数据多数写的。所以有可能FOLLOWER 已经加入 bdbje 选举组内。如果这时只有两个 FOLLOWER 节点（包括 MASTER），那么停掉一个 FE，可能导致另一个 FE 也因无法进行多数写而退出。此时，我们应该先通过 `ALTER SYSTEM DROP FOLLOWER` 命令，从元数据中删除新添加的 FOLLOWER 节点，然后再杀死 FOLLOWER 进程，清空元数据，重新进行一遍添加流程。 - - -### 删除 FE - -通过 `ALTER SYSTEM DROP FOLLOWER/OBSERVER` 命令即可删除对应类型的 FE。以下有几点注意事项： - -* 对于 OBSERVER 类型的 FE，直接 DROP 即可，无风险。 - -* 对于 FOLLOWER 类型的 FE。首先，应保证在有奇数个 FOLLOWER 的情况下（3个或以上），开始删除操作。 - - 1. 如果删除非 MASTER 角色的 FE，建议连接到 MASTER FE，执行 DROP 命令，再杀死进程即可。 - 2. 如果要删除 MASTER FE，先确认有奇数个 FOLLOWER FE 并且运行正常。然后先杀死 MASTER FE 的进程。这时会有某一个 FE 被选举为 MASTER。在确认剩下的 FE 运行正常后，连接到新的 MASTER FE，执行 DROP 命令删除之前老的 MASTER FE 即可。 - -## 高级操作 - -### 故障恢复 - -FE 有可能因为某些原因出现无法启动 bdbje、FE 之间无法同步等问题。现象包括无法进行元数据写操作、没有 MASTER 等等。这时，我们需要手动操作来恢复 FE。手动恢复 FE 的大致原理，是先通过当前 `meta_dir` 中的元数据，启动一个新的 MASTER，然后再逐台添加其他 FE。请严格按照如下步骤操作： - -1. 首先，停止所有 FE 进程，同时停止一切业务访问。保证在元数据恢复期间，不会因为外部访问导致其他不可预期的问题。 - -2. 确认哪个 FE 节点的元数据是最新： - - * 首先，**务必先备份所有 FE 的 `meta_dir` 目录。** - * 通常情况下，Master FE 的元数据是最新的。可以查看 `meta_dir/image` 目录下，image.xxxx 文件的后缀，数字越大，则表示元数据越新。 - * 通常，通过比较所有 FOLLOWER FE 的 image 文件，找出最新的元数据即可。 - * 之后，我们要使用这个拥有最新元数据的 FE 节点，进行恢复。 - * 如果使用 OBSERVER 节点的元数据进行恢复会比较麻烦，建议尽量选择 FOLLOWER 节点。 - -3. 以下操作都在由第2步中选择出来的 FE 节点上进行。 - - 1. 如果该节点是一个 OBSERVER，先将 `meta_dir/image/ROLE` 文件中的 `role=OBSERVER` 改为 `role=FOLLOWER`。（从 OBSERVER 节点恢复会比较麻烦，先按这里的步骤操作，后面会有单独说明） - 2. 在 fe.conf 中添加配置：`metadata_failure_recovery=true`。 - 3. 执行 `sh bin/start_fe.sh` 启动这个 FE。 - 4. 如果正常，这个 FE 会以 MASTER 的角色启动，类似于前面 `启动单节点 FE` 一节中的描述。在 fe.log 应该会看到 `transfer from XXXX to MASTER` 等字样。 - 5. 启动完成后，先连接到这个 FE，执行一些查询导入，检查是否能够正常访问。如果不正常，有可能是操作有误，建议仔细阅读以上步骤，用之前备份的元数据再试一次。如果还是不行，问题可能就比较严重了。 - 6. 如果成功，通过 `show frontends;` 命令，应该可以看到之前所添加的所有 FE，并且当前 FE 是 master。 - 7. 将 fe.conf 中的 `metadata_failure_recovery=true` 配置项删除，或者设置为 `false`，然后重启这个 FE（**重要**）。 - - - > 如果你是从一个 OBSERVER 节点的元数据进行恢复的，那么完成如上步骤后，通过 `show frontends;` 语句你会发现，当前这个 FE 的角色为 OBSERVER，但是 `IsMaster` 显示为 `true`。这是因为，这里看到的 “OBSERVER” 是记录在 Doris 的元数据中的，而是否是 master，是记录在 bdbje 的元数据中的。因为我们是从一个 OBSERVER 节点恢复的，所以这里出现了不一致。请按如下步骤修复这个问题（这个问题我们会在之后的某个版本修复）： - - > 1. 先把除了这个 “OBSERVER” 以外的所有 FE 节点 DROP 掉。 - > 2. 通过 `ADD FOLLOWER` 命令，添加一个新的 FOLLOWER FE，假设在 hostA 上。 - > 3. 在 hostA 上启动一个全新的 FE，通过 `-helper` 的方式加入集群。 - > 4. 启动成功后，通过 `show frontends;` 语句，你应该能看到两个 FE，一个是之前的 OBSERVER，一个是新添加的 FOLLOWER，并且 OBSERVER 是 master。 - > 5. 确认这个新的 FOLLOWER 是可以正常工作之后，用这个新的 FOLLOWER 的元数据，重新执行一遍故障恢复操作。 - > 6. 以上这些步骤的目的，其实就是人为的制造出一个 FOLLOWER 节点的元数据，然后用这个元数据，重新开始故障恢复。这样就避免了从 OBSERVER 恢复元数据所遇到的不一致的问题。 - - > `metadata_failure_recovery=true` 的含义是，清空 "bdbje" 的元数据。这样 bdbje 就不会再联系之前的其他 FE 了，而作为一个独立的 FE 启动。这个参数只有在恢复启动时才需要设置为 true。恢复完成后，一定要设置为 false，否则一旦重启，bdbje 的元数据又会被清空，导致其他 FE 无法正常工作。 - -4. 第3步执行成功后，我们再通过 `ALTER SYSTEM DROP FOLLOWER/OBSERVER` 命令，将之前的其他的 FE 从元数据删除后，按加入新 FE 的方式，重新把这些 FE 添加一遍。 - -5. 如果以上操作正常，则恢复完毕。 - -### FE 类型变更 - -如果你需要将当前已有的 FOLLOWER/OBSERVER 类型的 FE，变更为 OBSERVER/FOLLOWER 类型，请先按照前面所述的方式删除 FE，再添加对应类型的 FE 即可 - -### FE 迁移 - -如果你需要将一个 FE 从当前节点迁移到另一个节点，分以下几种情况。 - -1. 非 MASTER 节点的 FOLLOWER，或者 OBSERVER 迁移 - - 直接添加新的 FOLLOWER/OBSERVER 成功后，删除旧的 FOLLOWER/OBSERVER 即可。 - -2. 单节点 MASTER 迁移 - - 当只有一个 FE 时，参考 `故障恢复` 一节。将 FE 的 palo-meta 目录拷贝到新节点上，按照 `故障恢复` 一节中，步骤3的方式启动新的 MASTER - -3. 一组 FOLLOWER 从一组节点迁移到另一组新的节点 - - 在新的节点上部署 FE，通过添加 FOLLOWER 的方式先加入新节点。再逐台 DROP 掉旧节点即可。在逐台 DROP 的过程中，MASTER 会自动选择在新的 FOLLOWER 节点上。 - -### 更换 FE 端口 - -FE 目前有以下几个端口 - -* edit_log_port：bdbje 的通信端口 -* http_port：http 端口，也用于推送 image -* rpc_port：FE 的 thrift server port -* query_port：Mysql 连接端口 - -1. edit_log_port - - 如果需要更换这个端口，则需要参照 `故障恢复` 一节中的操作，进行恢复。因为该端口已经被持久化到 bdbje 自己的元数据中（同时也记录在 Doris 自己的元数据中），需要通过设置 `metadata_failure_recovery=true` 来清空 bdbje 的元数据。 - -2. http_port - - 所有 FE 的 http_port 必须保持一致。所以如果要修改这个端口，则所有 FE 都需要修改并重启。修改这个端口，在多 FOLLOWER 部署的情况下会比较复杂（涉及到鸡生蛋蛋生鸡的问题...），所以不建议有这种操作。如果必须，直接按照 `故障恢复` 一节中的操作吧。 - -3. rpc_port - - 修改配置后，直接重启 FE 即可。Master FE 会通过心跳将新的端口告知 BE。只有 Master FE 的这个端口会被使用。但仍然建议所有 FE 的端口保持一致。 - -4. query_port - - 修改配置后，直接重启 FE 即可。这个只影响到 mysql 的连接目标。 - - -### 从 FE 内存中恢复元数据 - -在某些极端情况下，磁盘上 image 文件可能会损坏，但是内存中的元数据是完好的，此时我们可以先从内存中 dump 出元数据，再替换掉磁盘上的 image 文件，来恢复元数据，整个**不停查询服务**的操作步骤如下： -1. 集群停止所有 Load,Create,Alter 操作 -2. 执行以下命令，从 Master FE 内存中 dump 出元数据：(下面称为 image_mem) -``` -curl -u $root_user:$password http://$master_hostname:8030/dump -``` -3. 用 image_mem 文件替换掉 OBSERVER FE 节点上`meta_dir/image`目录下的 image 文件，重启 OBSERVER FE 节点， -验证 image_mem 文件的完整性和正确性（可以在 FE Web 页面查看 DB 和 Table 的元数据是否正常，查看fe.log 是否有异常，是否在正常 replayed journal） -4. 依次用 image_mem 文件替换掉 FOLLOWER FE 节点上`meta_dir/image`目录下的 image 文件，重启 FOLLOWER FE 节点， -确认元数据和查询服务都正常 -5. 用 image_mem 文件替换掉 Master FE 节点上`meta_dir/image`目录下的 image 文件，重启 Master FE 节点， -确认 FE Master 切换正常， Master FE 节点可以通过 checkpoint 正常生成新的 image 文件 -6. 集群恢复所有 Load,Create,Alter 操作 - -**注意：如果 Image 文件很大，整个操作过程耗时可能会很长，所以在此期间，要确保 Master FE 不会通过 checkpoint 生成新的 image 文件。 -当观察到 Master FE 节点上 `meta_dir/image`目录下的 `image.ckpt` 文件快和 `image.xxx` 文件一样大时，可以直接删除掉`image.ckpt` 文件。** - -## 最佳实践 - -FE 的部署推荐，在 [安装与部署文档](../../installing/install-deploy.md) 中有介绍，这里再做一些补充。 - -* **如果你并不十分了解 FE 元数据的运行逻辑，或者没有足够 FE 元数据的运维经验，我们强烈建议在实际使用中，只部署一个 FOLLOWER 类型的 FE 作为 MASTER，其余 FE 都是 OBSERVER，这样可以减少很多复杂的运维问题！** 不用过于担心 MASTER 单点故障导致无法进行元数据写操作。首先，如果你配置合理，FE 作为 java 进程很难挂掉。其次，如果 MASTER 磁盘损坏（概率非常低），我们也可以用 OBSERVER 上的元数据，通过 `故障恢复` 的方式手动恢复。 - -* FE 进程的 JVM 一定要保证足够的内存。我们**强烈建议** FE 的 JVM 内存至少在 10GB 以上，推荐 32GB 至 64GB。并且部署监控来监控 JVM 的内存使用情况。因为如果FE出现OOM，可能导致元数据写入失败，造成一些**无法恢复**的故障！ - -* FE 所在节点要有足够的磁盘空间，以防止元数据过大导致磁盘空间不足。同时 FE 日志也会占用十几G 的磁盘空间。 - -## 其他常见问题 - -1. fe.log 中一直滚动 `meta out of date. current time: xxx, synchronized time: xxx, has log: xxx, fe type: xxx` - - 这个通常是因为 FE 无法选举出 Master。比如配置了 3 个 FOLLOWER，但是只启动了一个 FOLLOWER，则这个 FOLLOWER 会出现这个问题。通常，只要把剩余的 FOLLOWER 启动起来就可以了。如果启动起来后，仍然没有解决问题，那么可能需要按照 `故障恢复` 一节中的方式，手动进行恢复。 - -2. `Clock delta: xxxx ms. between Feeder: xxxx and this Replica exceeds max permissible delta: xxxx ms.` - - bdbje 要求各个节点之间的时钟误差不能超过一定阈值。如果超过，节点会异常退出。我们默认设置的阈值为 5000 ms，由 FE 的参数 `max_bdbje_clock_delta_ms` 控制，可以酌情修改。但我们建议使用 ntp 等时钟同步方式保证 Doris 集群各主机的时钟同步。 - - -3. `image/` 目录下的镜像文件很久没有更新 - - Master FE 会默认每 50000 条元数据 journal，生成一个镜像文件。在一个频繁使用的集群中，通常每隔半天到几天的时间，就会生成一个新的 image 文件。如果你发现 image 文件已经很久没有更新了（比如超过一个星期），则可以顺序的按照如下方法，查看具体原因： - - 1. 在 Master FE 的 fe.log 中搜索 `memory is not enough to do checkpoint. Committed memroy xxxx Bytes, used memory xxxx Bytes.` 字样。如果找到，则说明当前 FE 的 JVM 内存不足以用于生成镜像（通常我们需要预留一半的 FE 内存用于 image 的生成）。那么需要增加 JVM 的内存并重启 FE 后，再观察。每次 Master FE 重启后，都会直接生成一个新的 image。也可用这种重启方式，主动地生成新的 image。注意，如果是多 FOLLOWER 部署，那么当你重启当前 Master FE 后，另一个 FOLLOWER FE 会变成 MASTER，则后续的 image 生成会由新的 Master 负责。因此，你可能需要修改所有 FOLLOWER FE 的 JVM 内存配置。 - - 2. 在 Master FE 的 fe.log 中搜索 `begin to generate new image: image.xxxx`。如果找到，则说明开始生成 image 了。检查这个线程的后续日志，如果出现 `checkpoint finished save image.xxxx`，则说明 image 写入成功。如果出现 `Exception when generate new image file`，则生成失败，需要查看具体的错误信息。 - - -4. `bdb/` 目录的大小非常大，达到几个G或更多 - - 如果在排除无法生成新的 image 的错误后，bdb 目录在一段时间内依然很大。则可能是因为 Master FE 推送 image 不成功。可以在 Master FE 的 fe.log 中搜索 `push image.xxxx to other nodes. totally xx nodes, push successed yy nodes`。如果 yy 比 xx 小，则说明有的 FE 没有被推送成功。可以在 fe.log 中查看到具体的错误 `Exception when pushing image file. url = xxx`。 - - 同时，你也可以在 FE 的配置文件中添加配置：`edit_log_roll_num=xxxx`。该参数设定了每多少条元数据 journal，做一次 image。默认是 50000。可以适当改小这个数字，使得 image 更加频繁，从而加速删除旧的 journal。 - -5. FOLLOWER FE 接连挂掉 - - 因为 Doris 的元数据采用多数写策略，即一条元数据 journal 必须至少写入多数个 FOLLOWER FE 后（比如 3 个 FOLLOWER，必须写成功 2 个），才算成功。而如果写入失败，FE 进程会主动退出。那么假设有 A、B、C 三个 FOLLOWER，C 先挂掉，然后 B 再挂掉，那么 A 也会跟着挂掉。所以如 `最佳实践` 一节中所述，如果你没有丰富的元数据运维经验，不建议部署多 FOLLOWER。 - -6. fe.log 中出现 `get exception when try to close previously opened bdb database. ignore it` - - 如果后面有 `ignore it` 字样，通常无需处理。如果你有兴趣，可以在 `BDBEnvironment.java` 搜索这个错误，查看相关注释说明。 - -7. 从 `show frontends;` 看，某个 FE 的 `Join` 列为 `true`，但是实际该 FE 不正常 - - 通过 `show frontends;` 查看到的 `Join` 信息。该列如果为 `true`，仅表示这个 FE **曾经加入过** 集群。并不能表示当前仍然正常的存在于集群中。如果为 `false`，则表示这个 FE **从未加入过** 集群。 - -8. 关于 FE 的配置 `master_sync_policy`, `replica_sync_policy` 和 `txn_rollback_limit` - - `master_sync_policy` 用于指定当 Leader FE 写元数据日志时，是否调用 fsync(), `replica_sync_policy` 用于指定当 FE HA 部署时，其他 Follower FE 在同步元数据时，是否调用 fsync()。在早期的 Doris 版本中，这两个参数默认是 `WRITE_NO_SYNC`，即都不调用 fsync()。在最新版本的 Doris 中，默认已修改为 `SYNC`，即都调用 fsync()。调用 fsync() 会显著降低元数据写盘的效率。在某些环境下，IOPS 可能降至几百，延迟增加到2-3ms（但对于 Doris 元数据操作依然够用）。因此我们建议以下配置： - - 1. 对于单 Follower FE 部署，`master_sync_policy` 设置为 `SYNC`，防止 FE 系统宕机导致元数据丢失。 - 2. 对于多 Follower FE 部署，可以将 `master_sync_policy` 和 `replica_sync_policy` 设为 `WRITE_NO_SYNC`，因为我们认为多个系统同时宕机的概率非常低。 - - 如果在单 Follower FE 部署中，`master_sync_policy` 设置为 `WRITE_NO_SYNC`，则可能出现 FE 系统宕机导致元数据丢失。这时如果有其他 Observer FE 尝试重启时，可能会报错： - - ``` - Node xxx must rollback xx total commits(numPassedDurableCommits of which were durable) to the earliest point indicated by transaction xxxx in order to rejoin the replication group, but the transaction rollback limit of xxx prohibits this. - ``` - - 意思有部分已经持久化的事务需要回滚，但条数超过上限。这里我们的默认上限是 100，可以通过设置 `txn_rollback_limit` 改变。该操作仅用于尝试正常启动 FE，但已丢失的元数据无法恢复。 diff --git a/docs/documentation/cn/administrator-guide/operation/monitor-alert.md b/docs/documentation/cn/administrator-guide/operation/monitor-alert.md deleted file mode 100644 index 5a97000e0caae8..00000000000000 --- a/docs/documentation/cn/administrator-guide/operation/monitor-alert.md +++ /dev/null @@ -1,302 +0,0 @@ - - -# 监控和报警 - -本文档主要介绍 Doris 的监控项及如何采集、展示监控项。以及如何配置报警（TODO） - -[Dashborad 模板点击下载](https://grafana.com/dashboards/9734/revisions) - -> 注：0.9.0（不含）之前的版本请使用 revision 1。0.9.x 版本请使用 revision 2。0.10.x 版本请使用 revision 3。 - -Dashboard 模板会不定期更新。更新模板的方式见最后一小节。 - -欢迎提供更优的 dashboard。 - -## 组件 - -Doris 使用 [Prometheus](https://prometheus.io/) 和 [Grafana](https://grafana.com/) 进项监控项的采集和展示。 - -![](../../../../resources/images/dashboard_overview.png) - -1. Prometheus - - Prometheus 是一款开源的系统监控和报警套件。它可以通过 Pull 或 Push 采集被监控系统的监控项，存入自身的时序数据库中。并且通过丰富的多维数据查询语言，满足用户的不同数据展示需求。 - -2. Grafana - - Grafana 是一款开源的数据分析和展示平台。支持包括 Prometheus 在内的多个主流时序数据库源。通过对应的数据库查询语句，从数据源中获取展现数据。通过灵活可配置的 Dashboard，快速的将这些数据以图表的形式展示给用户。 - -> 注: 本文档仅提供一种使用 Prometheus 和 Grafana 进行 Doris 监控数据采集和展示的方式。原则上不开发、维护这些组件。更多关于这些组件的详细介绍，请移步对应官方文档进行查阅。 - -## 监控数据 - -Doris 的监控数据通过 Frontend 和 Backend 的 http 接口向外暴露。监控数据以 Key-Value 的文本形式对外展现。每个 Key 还可能有不同的 Label 加以区分。当用户搭建好 Doris 后，可以在浏览器，通过以下接口访问到节点的监控数据： - -* Frontend: `fe_host:fe_http_port/metrics` -* Backend: `be_host:be_web_server_port/metrics` -* Broker: 暂不提供 - -用户将看到如下监控项结果（示例为 FE 部分监控项）： - - ``` - # HELP jvm_heap_size_bytes jvm heap stat - # TYPE jvm_heap_size_bytes gauge - jvm_heap_size_bytes{type="max"} 41661235200 - jvm_heap_size_bytes{type="committed"} 19785285632 - jvm_heap_size_bytes{type="used"} 10113221064 - # HELP jvm_non_heap_size_bytes jvm non heap stat - # TYPE jvm_non_heap_size_bytes gauge - jvm_non_heap_size_bytes{type="committed"} 105295872 - jvm_non_heap_size_bytes{type="used"} 103184784 - # HELP jvm_young_size_bytes jvm young mem pool stat - # TYPE jvm_young_size_bytes gauge - jvm_young_size_bytes{type="used"} 6505306808 - jvm_young_size_bytes{type="peak_used"} 10308026368 - jvm_young_size_bytes{type="max"} 10308026368 - # HELP jvm_old_size_bytes jvm old mem pool stat - # TYPE jvm_old_size_bytes gauge - jvm_old_size_bytes{type="used"} 3522435544 - jvm_old_size_bytes{type="peak_used"} 6561017832 - jvm_old_size_bytes{type="max"} 30064771072 - # HELP jvm_direct_buffer_pool_size_bytes jvm direct buffer pool stat - # TYPE jvm_direct_buffer_pool_size_bytes gauge - jvm_direct_buffer_pool_size_bytes{type="count"} 91 - jvm_direct_buffer_pool_size_bytes{type="used"} 226135222 - jvm_direct_buffer_pool_size_bytes{type="capacity"} 226135221 - # HELP jvm_young_gc jvm young gc stat - # TYPE jvm_young_gc gauge - jvm_young_gc{type="count"} 2186 - jvm_young_gc{type="time"} 93650 - # HELP jvm_old_gc jvm old gc stat - # TYPE jvm_old_gc gauge - jvm_old_gc{type="count"} 21 - jvm_old_gc{type="time"} 58268 - # HELP jvm_thread jvm thread stat - # TYPE jvm_thread gauge - jvm_thread{type="count"} 767 - jvm_thread{type="peak_count"} 831 - ... - ``` - -这是一个以 [Promethus 格式](https://prometheus.io/docs/practices/naming/) 呈现的监控数据。我们以其中一个监控项为例进行说明： - -``` -# HELP jvm_heap_size_bytes jvm heap stat -# TYPE jvm_heap_size_bytes gauge -jvm_heap_size_bytes{type="max"} 41661235200 -jvm_heap_size_bytes{type="committed"} 19785285632 -jvm_heap_size_bytes{type="used"} 10113221064 -``` - -1. "#" 开头的行为注释行。其中 HELP 为该监控项的描述说明；TYPE 表示该监控项的数据类型，示例中为 Gauge，即标量数据。还有 Counter、Histogram 等数据类型。具体可见 [Prometheus 官方文档](https://prometheus.io/docs/practices/instrumentation/#counter-vs.-gauge,-summary-vs.-histogram) 。 -2. `jvm_heap_size_bytes` 即监控项的名称（Key）；`type="max"` 即为一个名为 `type` 的 Label，值为 `max`。一个监控项可以有多个 Label。 -3. 最后的数字，如 `41661235200`，即为监控数值。 - -## 监控架构 - -整个监控架构如下图所示： - -![](../../../../resources/images/monitor_arch.png) - -1. 黄色部分为 Prometheus 相关组件。Prometheus Server 为 Prometheus 的主进程，目前 Prometheus 通过 Pull 的方式访问 Doris 节点的监控接口，然后将时序数据存入时序数据库 TSDB 中（TSDB 包含在 Prometheus 进程中，无需单独部署）。Prometheus 也支持通过搭建 [Push Gateway](https://github.com/prometheus/pushgateway) 的方式，允许被监控系统将监控数据通过 Push 的方式推到 Push Gateway, 再由 Prometheus Server 通过 Pull 的方式从 Push Gateway 中获取数据。 -2. [Alert Manager](https://github.com/prometheus/alertmanager) 为 Prometheus 报警组件，需单独部署（暂不提供方案，可参照官方文档自行搭建）。通过 Alert Manager，用户可以配置报警策略，接收邮件、短信等报警。 -3. 绿色部分为 Grafana 相关组件。Grafana Server 为 Grafana 的主进程。启动后，用户可以通过 Web 页面对 Grafana 进行配置，包括数据源的设置、用户设置、Dashboard 绘制等。这里也是最终用户查看监控数据的地方。 - - -## 开始搭建 - -请在完成 Doris 的部署后，开始搭建监控系统。 - -### Prometheus - -1. 在 [Prometheus 官网](https://prometheus.io/download/) 下载最新版本的 Prometheus。这里我们以 2.3.2-linux-amd64 版本为例。 -2. 在准备运行监控服务的机器上，解压下载后的 tar 文件。 -3. 打开配置文件 promethues.yml。这里我们提供一个示例配置并加以说明（配置文件为 yml 格式，一定注意统一的缩进和空格）： - - 这里我们使用最简单的静态文件的方式进行监控配置。Prometheus 支持多种 [服务发现](https://prometheus.io/docs/prometheus/latest/configuration/configuration/) 方式，可以动态的感知节点的加入和删除。 - - ``` - # my global config - global: - scrape_interval: 15s # 全局的采集间隔，默认是 1m，这里设置为 15s - evaluation_interval: 15s # 全局的规则触发间隔，默认是 1m，这里设置 15s - - # Alertmanager configuration - alerting: - alertmanagers: - - static_configs: - - targets: - # - alertmanager:9093 - - # A scrape configuration containing exactly one endpoint to scrape: - # Here it's Prometheus itself. - scrape_configs: - # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'PALO_CLUSTER' # 每一个 Doris 集群，我们称为一个 job。这里可以给 job 取一个名字，作为 Doris 集群在监控系统中的名字。 - metrics_path: '/metrics' # 这里指定获取监控项的 restful api。配合下面的 targets 中的 host:port，Prometheus 最终会通过 host:port/metrics_path 来采集监控项。 - static_configs: # 这里开始分别配置 FE 和 BE 的目标地址。所有的 FE 和 BE 都分别写入各自的 group 中。 - - targets: ['fe_host1:8030', 'fe_host2:8030', 'fe_host3:8030'] - labels: - group: fe # 这里配置了 fe 的 group，该 group 中包含了 3 个 Frontends - - - targets: ['be_host1:8040', 'be_host2:8040', 'be_host3:8040'] - labels: - group: be # 这里配置了 be 的 group，该 group 中包含了 3 个 Backends - - - job_name: 'PALO_CLUSTER_2' # 我们可以在一个 Prometheus 中监控多个 Doris 集群，这里开始另一个 Doris 集群的配置。配置同上，以下略。 - metrics_path: '/metrics' - static_configs: - - targets: ['fe_host1:8030', 'fe_host2:8030', 'fe_host3:8030'] - labels: - group: fe - - - targets: ['be_host1:8040', 'be_host2:8040', 'be_host3:8040'] - labels: - group: be - - ``` - -4. 启动 Promethues - - 通过以下命令启动 Promethues： - - `nohup ./prometheus --web.listen-address="0.0.0.0:8181" &` - - 该命令将后台运行 Prometheus，并指定其 web 端口为 8181。启动后，即开始采集数据，并将数据存放在 data 目录中。 - -5. 停止 Promethues - - 目前没有发现正式的进程停止方式，直接 kill -9 即可。当然也可以将 Prometheus 设为一种 service，以 service 的方式启停。 - -6. 访问 Prometheus - - Prometheus 可以通过 web 页面进行简单的访问。通过浏览器打开 8181 端口，即可访问 Prometheus 的页面。点击导航栏中，`Status` -> `Targets`，可以看到所有分组 Job 的监控主机节点。正常情况下，所有节点都应为 `UP`，表示数据采集正常。点击某一个 `Endpoint`，即可看到当前的监控数值。如果节点状态不为 UP，可以先访问 Doris 的 metrics 接口（见前文）检查是否可以访问，或查询 Prometheus 相关文档尝试解决。 - -7. 至此，一个简单的 Prometheus 已经搭建、配置完毕。更多高级使用方式，请参阅 [官方文档](https://prometheus.io/docs/introduction/overview/) - -### Grafana - -1. 在 [Grafana 官网](https://grafana.com/grafana/download) 下载最新版本的 Grafana。这里我们以 5.2.1.linux-amd64 版本为例。 - -2. 在准备运行监控服务的机器上，解压下载后的 tar 文件。 - -3. 打开配置文件 conf/defaults.ini。这里我们仅列举需要改动的配置项，其余配置可使用默认。 - - ``` - # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) - data = data - - # Directory where grafana can store logs - logs = data/log - - # Protocol (http, https, socket) - protocal = http - - # The ip address to bind to, empty will bind to all interfaces - http_addr = - - # The http port to use - http_port = 8182 - ``` - -4. 启动 Grafana - - 通过以下命令启动 Grafana - - `nohup ./bin/grafana-server &` - - 该命令将后台运行 Grafana，访问端口为上面配置的 8182 - -5. 停止 Grafana - - 目前没有发现正式的进程停止方式，直接 kill -9 即可。当然也可以将 Grafana 设为一种 service，以 service 的方式启停。 - -6. 访问 Grafana - - 通过浏览器，打开 8182 端口，可以开始访问 Grafana 页面。默认用户名密码为 admin。 - -7. 配置 Grafana - - 初次登陆，需要根据提示设置数据源（data source）。我们这里的数据源，即上一步配置的 Prometheus。 - - 数据源配置的 Setting 页面说明如下： - - 1. Name: 数据源的名称，自定义，比如 doris_monitor_data_source - 2. Type: 选择 Prometheus - 3. URL: 填写 Prometheus 的 web 地址，如 http://host:8181 - 4. Access: 这里我们选择 Server 方式，即通过 Grafana 进程所在服务器，访问 Prometheus。 - 5. 其余选项默认即可。 - 6. 点击最下方 `Save & Test`，如果显示 `Data source is working`，即表示数据源可用。 - 7. 确认数据源可用后，点击左边导航栏的 + 号，开始添加 Dashboard。这里我们已经准备好了 Doris 的 Dashboard 模板（本文档开头）。下载完成后，点击上方的 `New dashboard`->`Import dashboard`->`Upload .json File`，将下载的 json 文件导入。 - 8. 导入后，可以命名 Dashboard，默认是 `Doris Overview`。同时，需要选择数据源，这里选择之前创建的 `doris_monitor_data_source` - 9. 点击 `Import`，即完成导入。之后，可以看到 Doris 的 Dashboard 展示。 - -8. 至此，一个简单的 Grafana 已经搭建、配置完毕。更多高级使用方式，请参阅 [官方文档](http://docs.grafana.org/) - - -## Dashboard 说明 - -这里我们简要介绍 Doris Dashboard。Dashboard 的内容可能会随版本升级，不断变化，本文档不保证是最新的 Dashboard 说明。 - -1. 顶栏 - - ![](../../../../resources/images/dashboard_navibar.png) - - * 左上角为 Dashboard 名称。 - * 右上角显示当前监控时间范围，可以下拉选择不同的时间范围，还可以指定定时刷新页面间隔。 - * cluster\_name: 即 Prometheus 配置文件中的各个 job\_name，代表一个 Doris 集群。选择不同的 cluster，下方的图表将展示对应集群的监控信息。 - * fe_master: 对应集群的 Master Frontend 节点。 - * fe_instance: 对应集群的所有 Frontend 节点。选择不同的 Frontend，下方的图表将展示对应 Frontend 的监控信息。 - * be_instance: 对应集群的所有 Backend 节点。选择不同的 Backend，下方的图表将展示对应 Backend 的监控信息。 - * interval: 有些图表展示了速率相关的监控项，这里可选择以多大间隔进行采样计算速率（注：15s 间隔可能导致一些图表无法显示）。 - -2. Row - - ![](../../../../resources/images/dashboard_row.png) - - Grafana 中，Row 的概念，即一组图表的集合。如上图中的 Overview、Cluster Overview 即两个不同的 Row。可以通过点击 Row，对 Row 进行折叠。当前 Dashboard 有如下 Rows（持续更新中）： - - 1. Overview: 所有 Doris 集群的汇总展示。 - 2. Cluster Overview: 选定集群的汇总展示。 - 3. Query Statistic: 选定集群的查询相关监控。 - 4. FE JVM: 选定 Frontend 的 JVM 监控。 - 5. BE: 选定集群的 Backends 的汇总展示。 - 6. BE Task: 选定集群的 Backends 任务信息的展示。 - -3. 图表 - - ![](../../../../resources/images/dashboard_panel.png) - - 一个典型的图标分为以下几部分： - - 1. 鼠标悬停左上角的 i 图标，可以查看该图表的说明。 - 2. 点击下方的图例，可以单独查看某一监控项。再次点击，则显示所有。 - 3. 在图表中拖拽可以选定时间范围。 - 4. 标题的 [] 中显示选定的集群名称。 - 5. 一些数值对应左边的Y轴，一些对应右边的，可以通过图例末尾的 `-right` 区分。 - 6. 点击图表名称->`Edit`，可以对图表进行编辑。 - -## Dashboard 更新 - -1. 点击 Grafana 左边栏的 `+`，点击 `Dashboard`。 -2. 点击左上角的 `New dashboard`，在点击右侧出现的 `Import dashboard`。 -3. 点击 `Upload .json File`，选择最新的模板文件。 -4. 选择数据源 -5. 点击 `Import(Overwrite)`，完成模板更新。 diff --git a/docs/documentation/cn/administrator-guide/operation/multi-tenant.md b/docs/documentation/cn/administrator-guide/operation/multi-tenant.md deleted file mode 100644 index 83379d4083d81b..00000000000000 --- a/docs/documentation/cn/administrator-guide/operation/multi-tenant.md +++ /dev/null @@ -1,232 +0,0 @@ - - -# 多租户(Experimental) - -该功能为实验性质，暂不建议在生产环境使用。 - -## 背景 -Doris 作为一款 PB 级别的在线报表与多维分析数据库，对外通过开放云提供云端的数据库服务，并且对于每个云上的客户都单独部署了一套物理集群。对内，一套物理集群部署了多个业务，对于隔离性要求比较高的业务单独搭建了集群。针对以上存在几点问题： - -- 部署多套物理集群维护代价大（升级、功能上线、bug修复）。 -- 一个用户的查询或者查询引起的bug经常会影响其他用户。 -- 实际生产环境单机只能部署一个BE进程。而多个BE可以更好的解决胖节点问题。并且对于join、聚合操作可以提供更高的并发度。 - -综合以上三点，Doris需要新的多租户方案，既能做到较好的资源隔离和故障隔离，同时也能减少维护的代价，满足公有云和私有云的需求。 - -## 设计原则 - -- 使用简单 -- 开发代价小 -- 方便现有集群的迁移 - -## 名词解释 - -- FE: Frontend，即 Doris 中用于元数据管理即查询规划的模块。 -- BE: Backend，即 Doris 中用于存储和查询数据的模块。 -- Master: FE 的一种角色。一个Doris集群只有一个Master，其他的FE为Observer或者Follower。 -- instance：一个 BE 进程即是一个 instance。 -- host：单个物理机 -- cluster：即一个集群，由多个instance组成。 -- 租户：一个cluster属于一个租户。cluster和租户之间是一对一关系。 -- database：一个用户创建的数据库 - -## 主要思路 - -- 一个host上部署多个BE的instance，在进程级别做资源隔离。 -- 多个instance形成一个cluster，一个cluster分配给一个业务独立的的租户。 -- FE增加cluster这一级并负责cluster的管理。 -- CPU,IO,内存等资源隔离采用cgroup。 - -## 设计方案 - -为了能够达到隔离的目的，引入了**虚拟cluster**的概念。 - -1. cluster表示一个虚拟的集群，由多个BE的instance组成。多个cluster共享FE。 -2. 一个host上可以启动多个instance。cluster创建时，选取任意指定数量的instance，组成一个cluster。 -3. 创建cluster的同时，会创建一个名为superuser的账户，隶属于该cluster。superuser可以对cluster进行管理、创建数据库、分配权限等。 -4. Doris启动后，会创建一个默认的cluster：default_cluster。如果用户不希望使用多cluster的功能，则会提供这个默认的cluster，并隐藏多cluster的其他操作细节。 - -具体架构如下图： -![](../../../../resources/images/multi_tenant_arch.png) - -## SQL 接口 - -- 登录 - - 默认集群登录名： user_name@default_cluster 或者 user_name - - 自定义集群登录名：user_name@cluster_name - - `mysqlclient -h host -P port -u user_name@cluster_name -p password` - -- 添加、删除、下线（decommission）以及取消下线BE - - `ALTER SYSTEM ADD BACKEND "host:port"` - `ALTER SYSTEM DROP BACKEND "host:port"` - `ALTER SYSTEM DECOMMISSION BACKEND "host:port"` - `CANCEL DECOMMISSION BACKEND "host:port"` - - 强烈建议使用 DECOMMISSION 而不是 DROP 来删除 BACKEND。DECOMMISSION 操作会首先将需要下线节点上的数据拷贝到集群内其他instance上。之后，才会真正下线。 - -- 创建集群，并指定superuser账户的密码 - - `CREATE CLUSTER cluster_name PROPERTIES ("instance_num" = "10") identified by "password"` - -- 进入一个集群 - - `ENTER cluster_name` - -- 集群扩容、缩容 - - `ALTER CLUSTER cluster_name PROPERTIES ("instance_num" = "10")` - - 当指定的实例个数多于cluster现有be的个数，则为扩容，如果少于则为缩容。 - -- 链接、迁移db - - `LINK DATABASE src_cluster_name.db_name dest_cluster_name.db_name` - - 软链一个cluster的db到另外一个cluster的db ，对于需要临时访问其他cluster的db却不需要进行实际数据迁移的用户可以采用这种方式。 - - `MIGRATE DATABASE src_cluster_name.db_name dest_cluster_name.db_name` - - 如果需要对db进行跨cluster的迁移，在链接之后，执行migrate对数据进行实际的迁移。 - - 迁移不影响当前两个db的查询、导入等操作，这是一个异步的操作，可以通过`SHOW MIGRATIONS`查看迁移的进度。 - -- 删除集群 - - `DROP CLUSTER cluster_name` - - 删除集群，要求先手动删除的集群内所有database。 - -- 其他 - - `SHOW CLUSTERS` - - 展示系统内已经创建的集群。只有root用户有该权限。 - - `SHOW BACKENDS` - - 查看集群内的BE instance。 - - `SHOW MIGRATIONS` - - 展示当前正在进行的db迁移任务。执行完db的迁移后可以通过此命令查看迁移的进度。 - -## 详细设计 - -1. 命名空间隔离 - - 为了引入多租户，需要对系统内的cluster之间的命名空间进行隔离。 - - Doris现有的元数据采用的是image + journal 的方式(元数据的设计见相关文档)。Doris会把涉及元数据的操作的记录为一个 journal （操作日志），然后定时的按照**图1**的方式写成image，加载的时候按照写入的顺序读即可。但是这样就带来一个问题已经写入的格式不容易修改，比如记录数据分布的元数据格式为：database+table+tablet+replica 嵌套，如果按照以往的方式要做cluster之间的命名空间隔离，则需要在database上增加一层cluster，内部元数据的层级变为:cluster+database+table+tablet+replica，如**图2**所示。但加一层带来的问题有： - - - 增加一层带来的元数据改动，不兼容，需要按照图2的方式cluster＋db＋table＋tablet＋replica层级写,这样就改变了以往的元数据组织方式，老版本的升级会比较麻烦，比较理想的方式是按照图3在现有元数据的格式下顺序写入cluster的元数据。 - - - 代码里所有用到db、user等，都需要加一层cluster，一工作量大改动的地方多，层级深，多数代码都获取db，现有功能几乎都要改一遍，并且需要在db的锁的基础上嵌套一层cluster的锁。 - - ![](../../../../resources/images/palo_meta.png) - - 综上这里采用了一种通过给db、user名加前缀的方式去隔离内部因为cluster之间db、user名字冲突的问题。 - - 如下，所有的sql输入涉及db名、user名的，都需要根据自己所在的cluster来拼写db、user的全名。 - - ![](../../../../resources/images/cluster_namaspace.png) - - 采用这种方式以上两个问题不再有。元数据的组织方式也比较简单。即采用**图3**每个cluster记录下属于自己cluster的db、user，以及节点即可。 - -2. BE 节点管理 - - 每个cluster都有属于自己的一组instance，可以通过`SHOW BACKENDS`查看，为了区分出instance属于哪个cluster以及使用情况，BE引入了多个状态： - - - free：当一个BE节点被加入系统内，此时be不属于任何cluster的时候处于空闲状态 - - using：当创建集群、或者扩容被选取到一个cluster内则处于使用中。 - - cluster decommission：如果执行缩容量，则正在执行缩容的be处于此状态。结束后，be状态变为free。 - - system decommission：be正在下线中。下线完成后，该be将会被永久删除。 - - 只有root用户可以通过`SHOW PROC "/backends"`中cluster这一项查看集群内所有be的是否被使用。为空则为空闲，否则为使用中。`SHOW BACKENDS`只能看到所在cluster的节点。以下是be节点状态变化的示意图。 - - ![](../../../../resources/images/backend_state.png) - -3. 创建集群 - - 只有root用户可以创建一个cluster，并指定任意数量的BE instance。 - - 支持在相同机器上选取多个instance。选择instance的大致原则是：尽可能选取不同机器上的be并且使所有机器上使用的be数尽可能均匀。 - - 对于使用来讲，每一个user、db都属于一个cluster（root除外）。为了创建user、db，首先需要进入一个cluster。在创建cluster的时候系统会默认生成这个cluster的管理员，即superuser账户。superuser具有在所属cluster内创建db、user，以及查看be节点数的权限。所有的非root用户登录必须指定一个cluster，即`user_name@cluster_name`。 - - 只有root用户可以通过`SHOW CLUSTER`查看系统内所有的cluster，并且可以通过@不同的集群名来进入不同的cluster。对于除了root之外的用户cluster都是不可见的。 - - 为了兼容老版本Doris内置了一个名字叫做default_cluster的集群，这个名字在创建集群的时候不能使用。 - - ![](../../../../resources/images/user_authority.png) - -4. 集群扩容 - - 集群扩容的流程同创建集群。会优先选取不在集群之外的host上的BE instance。选取的原则同创建集群。 - -5. 集群缩容、CLUSTER DECOMMISSION - - 用户可以通过设置 cluster 的 instance num 来进行集群缩容。 - - 集群的缩容会优先在BE instance 数量最多的 host 上选取 instance 进行下线。 - - 用户也可以直接使用 `ALTER CLUSTER DECOMMISSION BACKEND` 来指定BE，进行集群缩容。 - -![](../../../../resources/images/replica_recover.png) - -6. 建表 - - 为了保证高可用，每个分片的副本必需在不同的机器上。所以建表时，选择副本所在be的策略为在每个host上随机选取一个be。然后从这些be中随机选取所需副本数量的be。总体上做到每个机器上分片分布均匀。 - - 因此，假如需要创建一个3副本的分片，即使cluster包含3个或以上的instance，但是只有2个或以下的host，依然不能创建该分片。 - -7. 负载均衡 - - 负载均衡的粒度为cluster级别，cluster之间不做负载均衡。但是在计算负载是在host一级进行的，而一个host上可能存在多个不同cluster的BE instance。 cluster内，会通过每个host上所有分片数目、存储使用率计算负载，然后把负载高的机器上的分片往负载低的机器上拷贝（详见负载均衡相关文档）。 - -8. LINK DATABASE（软链） - - 多个集群之间可以通过软链的方式访问彼此的数据。链接的级别为不同cluster的db。 - - 通过在一个cluster内，添加需要访问的其他cluster的db的信息，来访问其他cluster中的db。 - - 当查询链接的db时，所使用的计算以及存储资源为源db所在cluster的资源。 - - 被软链的db不能在源cluster中删除。只有链接的db被删除后，才可以删除源db。而删除链接db，不会删除源db。 - -9. MIGRATE DATABASE - - db可以在cluster之间进行物理迁移。 - - 要迁移db，必须先链接db。执行迁移后数据会迁移到链接的db所在的cluster，并且执行迁移后源db被删除，链接断开。 - - 数据的迁移，复用了负载均衡以及副本恢复中，复制数据的流程（详见负载均衡相关文档）。具体实现上，在执行`MIRAGTE`命令后，Doris会在元数据中，将源db的所有副本所属的cluster，修改为目的cluster。 - - Doris会定期检查集群内机器之间是否均衡、副本是否齐全、是否有多余的副本。db的迁移即借用了这个流程，在检查副本齐全的时候同时检查副本所在的be是否属于该cluster，如果不属于，则记入要恢复的副本。并且副本多余要删除的时候会优先删除cluster外的副本，然后再按照现有的策略选择：宕机的be的副本->clone的副本->版本落后的副本->负载高的host上的副本，直到副本没有多余。 - -![](../../../../resources/images/cluster_link_and_migrate_db.png) - -10. BE的进程隔离 - -  为了实现be进程之间实际cpu、io以及内存的隔离，需要依赖于be的部署。部署的时候需要在外围配置cgroup，把要部署的be的进程都写入cgroup。如果要实现io的物理隔离各be配置的数据存放路径需要在不同磁盘上，这里不做过多的介绍。 - diff --git a/docs/documentation/cn/administrator-guide/operation/tablet-meta-tool.md b/docs/documentation/cn/administrator-guide/operation/tablet-meta-tool.md deleted file mode 100644 index 7cbe002a1b5b26..00000000000000 --- a/docs/documentation/cn/administrator-guide/operation/tablet-meta-tool.md +++ /dev/null @@ -1,107 +0,0 @@ - - -# Tablet 元数据管理工具 - -## 背景 - -在最新版本的代码中，我们在 BE 端引入了 RocksDB，用于存储 tablet 的元信息，以解决之前通过 header 文件的方式存储元信息，带来的各种功能和性能方面的问题。当前每一个数据目录（root\_path），都会有一个对应的 RocksDB 实例，其中以 key-value 的方式，存放对应 root\_path 上的所有 tablet 的元数据。 - -为了方便进行这些元数据的维护，我们提供了在线的 http 接口方式和离线的 meta\_tool 工具以完成相关的管理操作。 - -其中 http 接口仅用于在线的查看 tablet 的元数据，可以在 BE 进程运行的状态下使用。 - -而 meta\_tool 工具则仅用于离线的各类元数据管理操作，必须先停止BE进程后，才可使用。 - -meta\_tool 工具存放在 BE 的 lib/ 目录下。 - -## 操作 - -### 查看 Tablet Meta - -查看 Tablet Meta 信息可以分为在线方法和离线方法 - -#### 在线 - -访问 BE 的 http 接口，获取对应的 Tablet Meta 信息： - -api： - -`http://{host}:{port}/api/meta/header/{tablet_id}/{schema_hash}` - - -> host: BE 的 hostname -> -> port: BE 的 http 端口 -> -> tablet_id: tablet id -> -> schema_hash: tablet 的 schema hash - -举例： - -`http://be_host:8040/api/meta/header/14156/2458238340` - -最终查询成功的话，会将 Tablet Meta 以 json 形式返回。 - -#### 离线 - -基于 meta\_tool 工具获取某个盘上的 Tablet Meta。 - -命令： - -``` -./lib/meta_tool --root_path=/path/to/root_path --operation=get_meta --tablet_id=xxx --schema_hash=xxx -``` - -> root_path: 在 be.conf 中配置的对应的 root_path 路径。 - -结果也是按照 json 的格式展现 Tablet Meta。 - -### 加载 header - -加载 header 的功能是为了完成实现 tablet 人工迁移而提供的。该功能是基于 json 格式的 Tablet Meta 实现的，所以如果涉及 shard 字段、version 信息的更改，可以直接在 Tablet Meta 的 json 内容中更改。然后使用以下的命令进行加载。 - -命令： - -``` -./lib/meta_tool --operation=load_meta --root_path=/path/to/root_path --json_header_path=path -``` - -### 删除 header - -为了实现从某个 be 的某个盘中删除某个 tablet 的功能。 - -命令: - -``` -./lib/meta_tool --operation=delete_meta --root_path=/path/to/root_path --tablet_id=xxx --schema_hash=xxx -``` - -### 展示 pb 格式的 TabletMeta - -这个命令是为了查看旧的基于文件的管理的PB格式的 Tablet Meta，以 json 的格式展示 Tablet Meta。 - -命令： - -``` -./lib/meta_tool --operation=show_meta --root_path=/path/to/root_path --pb_header_path=path -``` - - diff --git a/docs/documentation/cn/administrator-guide/operation/tablet-repair-and-balance.md b/docs/documentation/cn/administrator-guide/operation/tablet-repair-and-balance.md deleted file mode 100644 index 17f2a8c45a322c..00000000000000 --- a/docs/documentation/cn/administrator-guide/operation/tablet-repair-and-balance.md +++ /dev/null @@ -1,666 +0,0 @@ - - -# 数据副本管理 - -从 0.9.0 版本开始，Doris 引入了优化后的副本管理策略，同时支持了更为丰富的副本状态查看工具。本文档主要介绍 Doris 数据副本均衡、修复方面的调度策略，以及副本管理的运维方法。帮助用户更方便的掌握和管理集群中的副本状态。 - -> Colocation 属性的表的副本修复和均衡可以参阅 `docs/documentation/cn/administrator-guide/colocation-join.md` - -## 名词解释 - -1. Tablet：Doris 表的逻辑分片，一个表有多个分片。 -2. Replica：分片的副本，默认一个分片有3个副本。 -3. Healthy Replica：健康副本，副本所在 Backend 存活，且副本的版本完整。 -4. TabletChecker（TC）：是一个常驻的后台线程，用于定期扫描所有的 Tablet，检查这些 Tablet 的状态，并根据检查结果，决定是否将 tablet 发送给 TabletScheduler。 -5. TabletScheduler（TS）：是一个常驻的后台线程，用于处理由 TabletChecker 发来的需要修复的 Tablet。同时也会进行集群副本均衡的工作。 -6. TabletSchedCtx（TSC）：是一个 tablet 的封装。当 TC 选择一个 tablet 后，会将其封装为一个 TSC，发送给 TS。 -7. Storage Medium：存储介质。Doris 支持对分区粒度指定不同的存储介质，包括 SSD 和 HDD。副本调度策略也是针对不同的存储介质分别调度的。 - -``` - - +--------+ +-----------+ - | Meta | | Backends | - +---^----+ +------^----+ - | | | 3. Send clone tasks - 1. Check tablets | | | - +--------v------+ +-----------------+ - | TabletChecker +--------> TabletScheduler | - +---------------+ +-----------------+ - 2. Waiting to be scheduled - - -``` - -上图是一个简化的工作流程。 - - -## 副本状态 - -一个 Tablet 的多个副本，可能因为某些情况导致状态不一致。Doris 会尝试自动修复这些状态不一致的副本，让集群尽快从错误状态中恢复。 - -**一个 Replica 的健康状态有以下几种：** - -1. BAD - - 即副本损坏。包括但不限于磁盘故障、BUG等引起的副本不可恢复的损毁状态。 - -2. VERSION\_MISSING - - 版本缺失。Doris 中每一批次导入都对应一个数据版本。而一个副本的数据由多个连续的版本组成。而由于导入错误、延迟等原因，可能导致某些副本的数据版本不完整。 - -3. HEALTHY - - 健康副本。即数据正常的副本，并且副本所在的 BE 节点状态正常（心跳正常且不处于下线过程中） - -**一个 Tablet 的健康状态由其所有副本的状态决定，有以下几种：** - -1. REPLICA\_MISSING - - 副本缺失。即存活副本数小于期望副本数。 - -2. VERSION\_INCOMPLETE - - 存活副本数大于等于期望副本数，但其中健康副本数小于期望副本数。 - -3. REPLICA\_RELOCATING - - 拥有等于 replication num 的版本完整的存活副本数，但是部分副本所在的 BE 节点处于 unavailable 状态（比如 decommission 中） - -4. REPLICA\_MISSING\_IN\_CLUSTER - - 当使用多 cluster 方式时，健康副本数大于等于期望副本数，但在对应 cluster 内的副本数小于期望副本数。 - -5. REDUNDANT - - 副本冗余。健康副本都在对应 cluster 内，但数量大于期望副本数。或者有多余的 unavailable 副本。 - -6. FORCE\_REDUNDANT - - 这是一个特殊状态。只会出现在当期望副本数大于等于可用节点数时，并且 Tablet 处于副本缺失状态时出现。这种情况下，需要先删除一个副本，以保证有可用节点用于创建新副本。 - -7. COLOCATE\_MISMATCH - - 针对 Colocation 属性的表的分片状态。表示分片副本与 Colocation Group 的指定的分布不一致。 - -8. COLOCATE\_REDUNDANT - - 针对 Colocation 属性的表的分片状态。表示 Colocation 表的分片副本冗余。 - -9. HEALTHY - - 健康分片，即条件[1-8]都不满足。 - -## 副本修复 - -TabletChecker 作为常驻的后台进程，会定期检查所有分片的状态。对于非健康状态的分片，将会交给 TabletScheduler 进行调度和修复。修复的实际操作，都由 BE 上的 clone 任务完成。FE 只负责生成这些 clone 任务。 - -> 注1：副本修复的主要思想是先通过创建或补齐使得分片的副本数达到期望值，然后再删除多余的副本。 -> -> 注2：一个 clone 任务就是完成从一个指定远端 BE 拷贝指定数据到指定目的端 BE 的过程。 - -针对不同的状态，我们采用不同的修复方式： - -1. REPLICA\_MISSING/REPLICA\_RELOCATING - - 选择一个低负载的，可用的 BE 节点作为目的端。选择一个健康副本作为源端。clone 任务会从源端拷贝一个完整的副本到目的端。对于副本补齐，我们会直接选择一个可用的 BE 节点，而不考虑存储介质。 - -2. VERSION\_INCOMPLETE - - 选择一个相对完整的副本作为目的端。选择一个健康副本作为源端。clone 任务会从源端尝试拷贝缺失的版本到目的端的副本。 - -3. REPLICA\_MISSING\_IN\_CLUSTER - - 这种状态处理方式和 REPLICA\_MISSING 相同。 - -4. REDUNDANT - - 通常经过副本修复后，分片会有冗余的副本。我们选择一个冗余副本将其删除。冗余副本的选择遵从以下优先级： - 1. 副本所在 BE 已经下线 - 2. 副本已损坏 - 3. 副本所在 BE 失联或在下线中 - 4. 副本处于 CLONE 状态（该状态是 clone 任务执行过程中的一个中间状态） - 5. 副本有版本缺失 - 6. 副本所在 cluster 不正确 - 7. 副本所在 BE 节点负载高 - -5. FORCE\_REDUNDANT - - 不同于 REDUNDANT，因为此时虽然 Tablet 有副本缺失，但是因为已经没有额外的可用节点用于创建新的副本了。所以此时必须先删除一个副本，以腾出一个可用节点用于创建新的副本。 - 删除副本的顺序同 REDUNDANT。 - -6. COLOCATE\_MISMATCH - - 从 Colocation Group 中指定的副本分布 BE 节点中选择一个作为目的节点进行副本补齐。 - -7. COLOCATE\_REDUNDANT - - 删除一个非 Colocation Group 中指定的副本分布 BE 节点上的副本。 - -Doris 在选择副本节点时，不会将同一个 Tablet 的副本部署在同一个 host 的不同 BE 上。保证了即使同一个 host 上的所有 BE 都挂掉，也不会造成全部副本丢失。 - -### 调度优先级 - -TabletScheduler 里等待被调度的分片会根据状态不同，赋予不同的优先级。优先级高的分片将会被优先调度。目前有以下几种优先级。 - -1. VERY\_HIGH - - * REDUNDANT。对于有副本冗余的分片，我们优先处理。虽然逻辑上来讲，副本冗余的紧急程度最低，但是因为这种情况处理起来最快且可以快速释放资源（比如磁盘空间等），所以我们优先处理。 - * FORCE\_REDUNDANT。同上。 - -2. HIGH - - * REPLICA\_MISSING 且多数副本缺失（比如3副本丢失了2个） - * VERSION\_INCOMPLETE 且多数副本的版本缺失 - * COLOCATE\_MISMATCH 我们希望 Colocation 表相关的分片能够尽快修复完成。 - * COLOCATE\_REDUNDANT - -3. NORMAL - - * REPLICA\_MISSING 但多数存活（比如3副本丢失了1个） - * VERSION\_INCOMPLETE 但多数副本的版本完整 - * REPLICA\_RELOCATING 且多数副本需要 relocate（比如3副本有2个） - -4. LOW - - * REPLICA\_MISSING\_IN\_CLUSTER - * REPLICA\_RELOCATING 但多数副本 stable - -### 手动优先级 - -系统会自动判断调度优先级。但是有些时候，用户希望某些表或分区的分片能够更快的被修复。因此我们提供一个命令，用户可以指定某个表或分区的分片被优先修复： - -`ADMIN REPAIR TABLE tbl [PARTITION (p1, p2, ...)];` - -这个命令，告诉 TC，在扫描 Tablet 时，对需要优先修复的表或分区中的有问题的 Tablet，给予 VERY\_HIGH 的优先级。 - -> 注：这个命令只是一个 hint，并不能保证一定能修复成功，并且优先级也会随 TS 的调度而发生变化。并且当 Master FE 切换或重启后，这些信息都会丢失。 - -可以通过以下命令取消优先级： - -`ADMIN CANCEL REPAIR TABLE tbl [PARTITION (p1, p2, ...)];` - -### 优先级调度 - -优先级保证了损坏严重的分片能够优先被修复，提高系统可用性。但是如果高优先级的修复任务一直失败，则会导致低优先级的任务一直得不到调度。因此，我们会根据任务的运行状态，动态的调整任务的优先级，保证所有任务都有机会被调度到。 - -* 连续5次调度失败（如无法获取资源，无法找到合适的源端或目的端等），则优先级会被下调。 -* 持续 30 分钟未被调度，则上调优先级。 -* 同一 tablet 任务的优先级至少间隔 5 分钟才会被调整一次。 - -同时为了保证初始优先级的权重，我们规定，初始优先级为 VERY\_HIGH 的，最低被下调到 NORMAL。而初始优先级为 LOW 的，最多被上调为 HIGH。这里的优先级调整，也会调整用户手动设置的优先级。 - -## 副本均衡 - -Doris 会自动进行集群内的副本均衡。均衡的主要思想，是对某些分片，先在低负载的节点上创建一个副本，然后再删除这些分片在高负载节点上的副本。同时，因为不同存储介质的存在，在同一个集群内的不同 BE 节点上，可能存在一种或两种存储介质。我们要求存储介质为 A 的分片在均衡后，尽量依然存储在存储介质 A 中。所以我们根据存储介质，对集群的 BE 节点进行划分。然后针对不同的存储介质的 BE 节点集合，进行负载均衡调度。 - -同样，副本均衡会保证不会将同一个 Tablet 的副本部署在同一个 host 的 BE 上。 - -### BE 节点负载 - -我们用 ClusterLoadStatistics（CLS）表示一个 cluster 中各个 Backend 的负载均衡情况。TabletScheduler 根据这个统计值，来触发集群均衡。我们当前通过 **磁盘使用率** 和 **副本数量** 两个指标，为每个BE计算一个 loadScore，作为 BE 的负载分数。分数越高，表示该 BE 的负载越重。 - -磁盘使用率和副本数量各有一个权重系数，分别为 **capacityCoefficient** 和 **replicaNumCoefficient**，其 **和衡为1**。其中 capacityCoefficient 会根据实际磁盘使用率动态调整。当一个 BE 的总体磁盘使用率在 50% 以下，则 capacityCoefficient 值为 0.5，如果磁盘使用率在 75%（可通过 FE 配置项 `capacity_used_percent_high_water` 配置）以上，则值为 1。如果使用率介于 50% ~ 75% 之间，则该权重系数平滑增加，公式为： - -`capacityCoefficient= 2 * 磁盘使用率 - 0.5` - -该权重系数保证当磁盘使用率过高时，该 Backend 的负载分数会更高，以保证尽快降低这个 BE 的负载。 - -TabletScheduler 会每隔 1 分钟更新一次 CLS。 - -### 均衡策略 - -TabletScheduler 在每轮调度时，都会通过 LoadBalancer 来选择一定数目的健康分片作为 balance 的候选分片。在下一次调度时，会尝试根据这些候选分片，进行均衡调度。 - -## 资源控制 - -无论是副本修复还是均衡，都是通过副本在各个 BE 之间拷贝完成的。如果同一台 BE 同一时间执行过多的任务，则会带来不小的 IO 压力。因此，Doris 在调度时控制了每个节点上能够执行的任务数目。最小的资源控制单位是磁盘（即在 be.conf 中指定的一个数据路径）。我们默认为每块磁盘配置两个 slot 用于副本修复。一个 clone 任务会占用源端和目的端各一个 slot。如果 slot 数目为零，则不会再对这块磁盘分配任务。该 slot 个数可以通过 FE 的 `schedule_slot_num_per_path` 参数配置。 - -另外，我们默认为每块磁盘提供 2 个单独的 slot 用于均衡任务。目的是防止高负载的节点因为 slot 被修复任务占用，而无法通过均衡释放空间。 - -## 副本状态查看 - -副本状态查看主要是查看副本的状态，以及副本修复和均衡任务的运行状态。这些状态大部分都**仅存在于** Master FE 节点中。因此，以下命令需直连到 Master FE 执行。 - -### 副本状态 - -1. 全局状态检查 - - 通过 `SHOW PROC '/statistic';` 命令可以查看整个集群的副本状态。 - - ``` - +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ - | DbId | DbName | TableNum | PartitionNum | IndexNum | TabletNum | ReplicaNum | UnhealthyTabletNum | InconsistentTabletNum | - +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ - | 35153636 | default_cluster:DF_Newrisk | 3 | 3 | 3 | 96 | 288 | 0 | 0 | - | 48297972 | default_cluster:PaperData | 0 | 0 | 0 | 0 | 0 | 0 | 0 | - | 5909381 | default_cluster:UM_TEST | 7 | 7 | 10 | 320 | 960 | 1 | 0 | - | Total | 240 | 10 | 10 | 13 | 416 | 1248 | 1 | 0 | - +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ - ``` - - 其中 `UnhealthyTabletNum` 列显示了对应的 Database 中，有多少 Tablet 处于非健康状态。`InconsistentTabletNum` 列显示了对应的 Database 中，有多少 Tablet 处于副本不一致的状态。最后一行 `Total` 行对整个集群进行了统计。正常情况下 `UnhealthyTabletNum` 和 `InconsistentTabletNum` 应为0。如果不为零，可以进一步查看具体有哪些 Tablet。如上图中，UM_TEST 数据库有 1 个 Tablet 状态不健康，则可以使用以下命令查看具体是哪一个 Tablet。 - - `SHOW PROC '/statistic/5909381';` - - 其中 `5909381` 为对应的 DbId。 - - ``` - +------------------+---------------------+ - | UnhealthyTablets | InconsistentTablets | - +------------------+---------------------+ - | [40467980] | [] | - +------------------+---------------------+ - ``` - - 上图会显示具体的不健康的 Tablet ID（40467980）。后面我们会介绍如何查看一个具体的 Tablet 的各个副本的状态。 - -2. 表（分区）级别状态检查 - - 用户可以通过以下命令查看指定表或分区的副本状态，并可以通过 WHERE 语句对状态进行过滤。如查看表 tbl1 中，分区 p1 和 p2 上状态为 NORMAL 的副本： - - `ADMIN SHOW REPLICA STATUS FROM tbl1 PARTITION (p1, p2) WHERE STATUS = "NORMAL";` - - ``` - +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ - | TabletId | ReplicaId | BackendId | Version | LastFailedVersion | LastSuccessVersion | CommittedVersion | SchemaHash | VersionNum | IsBad | State | Status | - +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ - | 29502429 | 29502432 | 10006 | 2 | -1 | 2 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502429 | 36885996 | 10002 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502429 | 48100551 | 10007 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502433 | 29502434 | 10001 | 2 | -1 | 2 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502433 | 44900737 | 10004 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502433 | 48369135 | 10006 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | - +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ - ``` - - 这里会展示所有副本的状态。其中 `IsBad` 列为 `true` 则表示副本已经损坏。而 `Status` 列则会显示另外的其他状态。具体的状态说明，可以通过 `HELP ADMIN SHOW REPLICA STATUS;` 查看帮助。 - - `ADMIN SHOW REPLICA STATUS` 命令主要用于查看副本的健康状态。用户还可以通过以下命令查看指定表中副本的一些额外信息： - - `SHOW TABLET FROM tbl1;` - - ``` - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - | TabletId | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | DataSize | RowCount | State | LstConsistencyCheckTime | CheckVersion | CheckVersionHash | VersionCount | PathHash | - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - | 29502429 | 29502432 | 10006 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -5822326203532286804 | - | 29502429 | 36885996 | 10002 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -1441285706148429853 | - | 29502429 | 48100551 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -4784691547051455525 | - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - ``` - - 上图展示了包括副本大小、行数、版本数量、所在数据路径等一些额外的信息。 - - > 注：这里显示的 `State` 列的内容不代表副本的健康状态，而是副本处于某种任务下的状态，比如 CLONE、SCHEMA\_CHANGE、ROLLUP 等。 - - 此外，用户也可以通过以下命令，查看指定表或分区的副本分布情况，来检查副本分布是否均匀。 - - `ADMIN SHOW REPLICA DISTRIBUTION FROM tbl1;` - - ``` - +-----------+------------+-------+---------+ - | BackendId | ReplicaNum | Graph | Percent | - +-----------+------------+-------+---------+ - | 10000 | 7 | | 7.29 % | - | 10001 | 9 | | 9.38 % | - | 10002 | 7 | | 7.29 % | - | 10003 | 7 | | 7.29 % | - | 10004 | 9 | | 9.38 % | - | 10005 | 11 | > | 11.46 % | - | 10006 | 18 | > | 18.75 % | - | 10007 | 15 | > | 15.62 % | - | 10008 | 13 | > | 13.54 % | - +-----------+------------+-------+---------+ - ``` - - 这里分别展示了表 tbl1 的副本在各个 BE 节点上的个数、百分比，以及一个简单的图形化显示。 - -4. Tablet 级别状态检查 - - 当我们要定位到某个具体的 Tablet 时，可以使用如下命令来查看一个具体的 Tablet 的状态。如查看 ID 为 29502553 的 tablet： - - `SHOW TABLET 29502553;` - - ``` - +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ - | DbName | TableName | PartitionName | IndexName | DbId | TableId | PartitionId | IndexId | IsSync | DetailCmd | - +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ - | default_cluster:test | test | test | test | 29502391 | 29502428 | 29502427 | 29502428 | true | SHOW PROC '/dbs/29502391/29502428/partitions/29502427/29502428/29502553'; | - +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ - ``` - - 上图显示了这个 tablet 所对应的数据库、表、分区、上卷表等信息。用户可以复制 `DetailCmd` 命令中的命令继续执行： - - `SHOW PROC '/dbs/29502391/29502428/partitions/29502427/29502428/29502553';` - - ``` - +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ - | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | SchemaHash | DataSize | RowCount | State | IsBad | VersionCount | PathHash | - +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ - | 43734060 | 10004 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | -8566523878520798656 | - | 29502555 | 10002 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | 1885826196444191611 | - | 39279319 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | 1656508631294397870 | - +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ - ``` - - 上图显示了对应 Tablet 的所有副本情况。这里显示的内容和 `SHOW TABLET FROM tbl1;` 的内容相同。但这里可以清楚的知道，一个具体的 Tablet 的所有副本的状态。 - -### 副本调度任务 - -1. 查看等待被调度的任务 - - `SHOW PROC '/cluster_balance/pending_tablets';` - - ``` - +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ - | TabletId | Type | Status | State | OrigPrio | DynmPrio | SrcBe | SrcPath | DestBe | DestPath | Timeout | Create | LstSched | LstVisit | Finished | Rate | FailedSched | FailedRunning | LstAdjPrio | VisibleVer | VisibleVerHash | CmtVer | CmtVerHash | ErrMsg | - +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ - | 4203036 | REPAIR | REPLICA_MISSING | PENDING | HIGH | LOW | -1 | -1 | -1 | -1 | 0 | 2019-02-21 15:00:20 | 2019-02-24 11:18:41 | 2019-02-24 11:18:41 | N/A | N/A | 2 | 0 | 2019-02-21 15:00:43 | 1 | 0 | 2 | 0 | unable to find source replica | - +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ - ``` - - 各列的具体含义如下： - - * TabletId：等待调度的 Tablet 的 ID。一个调度任务只针对一个 Tablet - * Type：任务类型，可以是 REPAIR（修复） 或 BALANCE（均衡） - * Status：该 Tablet 当前的状态，如 REPLICA\_MISSING（副本缺失） - * State：该调度任务的状态，可能为 PENDING/RUNNING/FINISHED/CANCELLED/TIMEOUT/UNEXPECTED - * OrigPrio：初始的优先级 - * DynmPrio：当前动态调整后的优先级 - * SrcBe：源端 BE 节点的 ID - * SrcPath：源端 BE 节点的路径的 hash 值 - * DestBe：目的端 BE 节点的 ID - * DestPath：目的端 BE 节点的路径的 hash 值 - * Timeout：当任务被调度成功后，这里会显示任务的超时时间，单位秒 - * Create：任务被创建的时间 - * LstSched：上一次任务被调度的时间 - * LstVisit：上一次任务被访问的时间。这里“被访问”指包括被调度，任务执行汇报等和这个任务相关的被处理的时间点 - * Finished：任务结束时间 - * Rate：clone 任务的数据拷贝速率 - * FailedSched：任务调度失败的次数 - * FailedRunning：任务执行失败的次数 - * LstAdjPrio：上一次优先级调整的时间 - * CmtVer/CmtVerHash/VisibleVer/VisibleVerHash：用于执行 clone 任务的 version 信息 - * ErrMsg：任务被调度和运行过程中，出现的错误信息 - -2. 查看正在运行的任务 - - `SHOW PROC '/cluster_balance/running_tablets';` - - 其结果中各列的含义和 `pending_tablets` 相同。 - -3. 查看已结束任务 - - `SHOW PROC '/cluster_balance/history_tablets';` - - 我们默认只保留最近 1000 个完成的任务。其结果中各列的含义和 `pending_tablets` 相同。如果 `State` 列为 `FINISHED`，则说明任务正常完成。如果为其他，则可以根据 `ErrMsg` 列的错误信息查看具体原因。 - -## 集群负载及调度资源查看 - -1. 集群负载 - - 通过以下命令可以查看集群当前的负载情况： - - `SHOW PROC '/cluster_balance/cluster_load_stat';` - - 首先看到的是对不同存储介质的划分： - - ``` - +---------------+ - | StorageMedium | - +---------------+ - | HDD | - | SSD | - +---------------+ - ``` - - 点击某一种存储介质，可以看到包含该存储介质的 BE 节点的均衡状态： - - `SHOW PROC '/cluster_balance/cluster_load_stat/HDD';` - - ``` - +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ - | BeId | Cluster | Available | UsedCapacity | Capacity | UsedPercent | ReplicaNum | CapCoeff | ReplCoeff | Score | Class | - +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ - | 10003 | default_cluster | true | 3477875259079 | 19377459077121 | 17.948 | 493477 | 0.5 | 0.5 | 0.9284678149967587 | MID | - | 10002 | default_cluster | true | 3607326225443 | 19377459077121 | 18.616 | 496928 | 0.5 | 0.5 | 0.948660871419998 | MID | - | 10005 | default_cluster | true | 3523518578241 | 19377459077121 | 18.184 | 545331 | 0.5 | 0.5 | 0.9843539990641831 | MID | - | 10001 | default_cluster | true | 3535547090016 | 19377459077121 | 18.246 | 558067 | 0.5 | 0.5 | 0.9981869446537612 | MID | - | 10006 | default_cluster | true | 3636050364835 | 19377459077121 | 18.764 | 547543 | 0.5 | 0.5 | 1.0011489897614072 | MID | - | 10004 | default_cluster | true | 3506558163744 | 15501967261697 | 22.620 | 468957 | 0.5 | 0.5 | 1.0228319835582569 | MID | - | 10007 | default_cluster | true | 4036460478905 | 19377459077121 | 20.831 | 551645 | 0.5 | 0.5 | 1.057279369420761 | MID | - | 10000 | default_cluster | true | 4369719923760 | 19377459077121 | 22.551 | 547175 | 0.5 | 0.5 | 1.0964036415787461 | MID | - +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ - ``` - - 其中一些列的含义如下： - - * Available：为 true 表示 BE 心跳正常，且没有处于下线中 - * UsedCapacity：字节，BE 上已使用的磁盘空间大小 - * Capacity：字节，BE 上总的磁盘空间大小 - * UsedPercent：百分比，BE 上的磁盘空间使用率 - * ReplicaNum：BE 上副本数量 - * CapCoeff/ReplCoeff：磁盘空间和副本数的权重系数 - * Score：负载分数。分数越高，负载越重 - * Class：根据负载情况分类，LOW/MID/HIGH。均衡调度会将高负载节点上的副本迁往低负载节点 - - 用户可以进一步查看某个 BE 上各个路径的使用率，比如 ID 为 10001 这个 BE： - - `SHOW PROC '/cluster_balance/cluster_load_stat/HDD/10001';` - - ``` - +------------------+------------------+---------------+---------------+---------+--------+----------------------+ - | RootPath | DataUsedCapacity | AvailCapacity | TotalCapacity | UsedPct | State | PathHash | - +------------------+------------------+---------------+---------------+---------+--------+----------------------+ - | /home/disk4/palo | 498.757 GB | 3.033 TB | 3.525 TB | 13.94 % | ONLINE | 4883406271918338267 | - | /home/disk3/palo | 704.200 GB | 2.832 TB | 3.525 TB | 19.65 % | ONLINE | -5467083960906519443 | - | /home/disk1/palo | 512.833 GB | 3.007 TB | 3.525 TB | 14.69 % | ONLINE | -7733211489989964053 | - | /home/disk2/palo | 881.955 GB | 2.656 TB | 3.525 TB | 24.65 % | ONLINE | 4870995507205544622 | - | /home/disk5/palo | 694.992 GB | 2.842 TB | 3.525 TB | 19.36 % | ONLINE | 1916696897889786739 | - +------------------+------------------+---------------+---------------+---------+--------+----------------------+ - ``` - - 这里显示了指定 BE 上，各个数据路径的磁盘使用率情况。 - -2. 调度资源 - - 用户可以通过以下命令，查看当前各个节点的 slot 使用情况： - - `SHOW PROC '/cluster_balance/working_slots';` - - ``` - +----------+----------------------+------------+------------+-------------+----------------------+ - | BeId | PathHash | AvailSlots | TotalSlots | BalanceSlot | AvgRate | - +----------+----------------------+------------+------------+-------------+----------------------+ - | 10000 | 8110346074333016794 | 2 | 2 | 2 | 2.459007474009069E7 | - | 10000 | -5617618290584731137 | 2 | 2 | 2 | 2.4730105014001578E7 | - | 10001 | 4883406271918338267 | 2 | 2 | 2 | 1.6711402709780257E7 | - | 10001 | -5467083960906519443 | 2 | 2 | 2 | 2.7540126380326536E7 | - | 10002 | 9137404661108133814 | 2 | 2 | 2 | 2.417217089806745E7 | - | 10002 | 1885826196444191611 | 2 | 2 | 2 | 1.6327378456676323E7 | - +----------+----------------------+------------+------------+-------------+----------------------+ - ``` - - 这里以数据路径为粒度，展示了当前 slot 的使用情况。其中 `AvgRate` 为历史统计的该路径上 clone 任务的拷贝速率，单位是字节/秒。 - -3. 优先修复查看 - - 以下命令，可以查看通过 `ADMIN REPAIR TABLE` 命令设置的优先修复的表或分区。 - - `SHOW PROC '/cluster_balance/priority_repair'；` - - 其中 `RemainingTimeMs` 表示，这些优先修复的内容，将在这个时间后，被自动移出优先修复队列。以防止优先修复一直失败导致资源被占用。 - -### 调度器统计状态查看 - -我们收集了 TabletChecker 和 TabletScheduler 在运行过程中的一些统计信息，可以通过以下命令查看： - -`SHOW PROC '/cluster_balance/sched_stat'；` - -``` -+---------------------------------------------------+-------------+ -| Item | Value | -+---------------------------------------------------+-------------+ -| num of tablet check round | 12041 | -| cost of tablet check(ms) | 7162342 | -| num of tablet checked in tablet checker | 18793506362 | -| num of unhealthy tablet checked in tablet checker | 7043900 | -| num of tablet being added to tablet scheduler | 1153 | -| num of tablet schedule round | 49538 | -| cost of tablet schedule(ms) | 49822 | -| num of tablet being scheduled | 4356200 | -| num of tablet being scheduled succeeded | 320 | -| num of tablet being scheduled failed | 4355594 | -| num of tablet being scheduled discard | 286 | -| num of tablet priority upgraded | 0 | -| num of tablet priority downgraded | 1096 | -| num of clone task | 230 | -| num of clone task succeeded | 228 | -| num of clone task failed | 2 | -| num of clone task timeout | 2 | -| num of replica missing error | 4354857 | -| num of replica version missing error | 967 | -| num of replica relocating | 0 | -| num of replica redundant error | 90 | -| num of replica missing in cluster error | 0 | -| num of balance scheduled | 0 | -+---------------------------------------------------+-------------+ -``` - -各行含义如下： - -* num of tablet check round：Tablet Checker 检查次数 -* cost of tablet check(ms)：Tablet Checker 检查总耗时 -* num of tablet checked in tablet checker：Tablet Checker 检查过的 tablet 数量 -* num of unhealthy tablet checked in tablet checker：Tablet Checker 检查过的不健康的 tablet 数量 -* num of tablet being added to tablet scheduler：被提交到 Tablet Scheduler 中的 tablet 数量 -* num of tablet schedule round：Tablet Scheduler 运行次数 -* cost of tablet schedule(ms)：Tablet Scheduler 运行总耗时 -* num of tablet being scheduled：被调度的 Tablet 总数量 -* num of tablet being scheduled succeeded：被成功调度的 Tablet 总数量 -* num of tablet being scheduled failed：调度失败的 Tablet 总数量 -* num of tablet being scheduled discard：调度失败且被抛弃的 Tablet 总数量 -* num of tablet priority upgraded：优先级上调次数 -* num of tablet priority downgraded：优先级下调次数 -* num of clone task：生成的 clone 任务数量 -* num of clone task succeeded：clone 任务成功的数量 -* num of clone task failed：clone 任务失败的数量 -* num of clone task timeout：clone 任务超时的数量 -* num of replica missing error：检查的状态为副本缺失的 tablet 的数量 -* num of replica version missing error：检查的状态为版本缺失的 tablet 的数量（该统计值包括了 num of replica relocating 和 num of replica missing in cluster error） -* num of replica relocating：检查的状态为 replica relocating 的 tablet 的数量 -* num of replica redundant error：检查的状态为副本冗余的 tablet 的数量 -* num of replica missing in cluster error：检查的状态为不在对应 cluster 的 tablet 的数量 -* num of balance scheduled：均衡调度的次数 - -> 注：以上状态都只是历史累加值。我们也在 FE 的日志中，定期打印了这些统计信息，其中括号内的数值表示自上次统计信息打印依赖，各个统计值的变化数量。 - -## 相关配置说明 - -### 可调整参数 - -以下可调整参数均为 fe.conf 中可配置参数。 - -* use\_new\_tablet\_scheduler - - * 说明：是否启用新的副本调度方式。新的副本调度方式即本文档介绍的副本调度方式。 - * 默认值：true - * 重要性：高 - -* tablet\_repair\_delay\_factor\_second - - * 说明：对于不同的调度优先级，我们会延迟不同的时间后开始修复。以防止因为例行重启、升级等过程中，产生大量不必要的副本修复任务。此参数为一个基准系数。对于 HIGH 优先级，延迟为 基准系数 * 1；对于 NORMAL 优先级，延迟为 基准系数 * 2；对于 LOW 优先级，延迟为 基准系数 * 3。即优先级越低，延迟等待时间越长。如果用户想尽快修复副本，可以适当降低该参数。 - * 默认值：60秒 - * 重要性：高 - -* schedule\_slot\_num\_per\_path - - * 说明：默认分配给每块磁盘用于副本修复的 slot 数目。该数目表示一块磁盘能同时运行的副本修复任务数。如果想以更快的速度修复副本，可以适当调高这个参数。单数值越高，可能对 IO 影响越大。 - * 默认值：2 - * 重要性：高 - -* balance\_load\_score\_threshold - - * 说明：集群均衡的阈值。默认为 0.1，即 10%。当一个 BE 节点的 load score，不高于或不低于平均 load score 的 10% 时，我们认为这个节点是均衡的。如果想让集群负载更加平均，可以适当调低这个参数。 - * 默认值：0.1 - * 重要性：中 - -* storage\_high\_watermark\_usage\_percent 和 storage\_min\_left\_capacity\_bytes - - * 说明：这两个参数，分别表示一个磁盘的最大空间使用率上限，以及最小的空间剩余下限。当一块磁盘的空间使用率大于上限，或者剩余空间小于下限时，该磁盘将不再作为均衡调度的目的地址。 - * 默认值：0.85 和 1048576000 （1GB） - * 重要性：中 - -* disable\_balance - - * 说明：控制是否关闭均衡功能。当副本处于均衡过程中时，有些功能，如 ALTER TABLE 等将会被禁止。而均衡可能持续很长时间。因此，如果用户希望尽快进行被禁止的操作。可以将该参数设为 true，以关闭均衡调度。 - * 默认值：true - * 重要性：中 - -### 不可调整参数 - -以下参数暂不支持修改，仅作说明。 - -* TabletChecker 调度间隔 - - TabletChecker 每20秒进行一次检查调度。 - -* TabletScheduler 调度间隔 - - TabletScheduler 每5秒进行一次调度 - -* TabletScheduler 每批次调度个数 - - TabletScheduler 每次调度最多 50 个 tablet。 - -* TabletScheduler 最大等待调度和运行中任务数 - - 最大等待调度任务数和运行中任务数为 2000。当超过 2000 后，TabletChecker 将不再产生新的调度任务给 TabletScheduler。 - -* TabletScheduler 最大均衡任务数 - - 最大均衡任务数为 500。当超过 500 后，将不再产生新的均衡任务。 - -* 每块磁盘用于均衡任务的 slot 数目 - - 每块磁盘用于均衡任务的 slot 数目为2。这个 slot 独立于用于副本修复的 slot。 - -* 集群均衡情况更新间隔 - - TabletScheduler 每隔 20 秒会重新计算一次集群的 load score。 - -* Clone 任务的最小和最大超时时间 - - 一个 clone 任务超时时间范围是 3min ~ 2hour。具体超时时间通过 tablet 的大小计算。计算公式为 (tablet size) / (5MB/s)。当一个 clone 任务运行失败 3 次后，该任务将终止。 - -* 动态优先级调整策略 - - 优先级最小调整间隔为 5min。当一个 tablet 调度失败5次后，会调低优先级。当一个 tablet 30min 未被调度时，会调高优先级。 - -## 相关问题 - -* 在某些情况下，默认的副本修复和均衡策略可能会导致网络被打满（多发生在千兆网卡，且每台 BE 的磁盘数量较多的情况下）。此时需要调整一些参数来减少同时进行的均衡和修复任务数。 - -* 目前针对 Colocate Table 的副本的均衡策略无法保证同一个 Tablet 的副本不会分布在同一个 host 的 BE 上。但 Colocate Table 的副本的修复策略会检测到这种分布错误并校正。但可能会出现，校正后，均衡策略再次认为副本不均衡而重新均衡。从而导致在两种状态间不停交替，无法使 Colocate Group 达成稳定。针对这种情况，我们建议在使用 Colocate 属性时，尽量保证集群是同构的，以减小副本分布在同一个 host 上的概率。 - - - - - diff --git a/docs/documentation/cn/administrator-guide/privilege.md b/docs/documentation/cn/administrator-guide/privilege.md deleted file mode 100644 index 8c08ac287db40a..00000000000000 --- a/docs/documentation/cn/administrator-guide/privilege.md +++ /dev/null @@ -1,217 +0,0 @@ - - -# 权限管理 - -Doris 新的权限管理系统参照了 Mysql 的权限管理机制，做到了表级别细粒度的权限控制，基于角色的权限访问控制，并且支持白名单机制。 - -## 名词解释 - -1. 用户标识 user_identity - - 在权限系统中，一个用户被识别为一个 User Identity（用户标识）。用户标识由两部分组成：username 和 userhost。其中 username 为用户名，由英文大小写组成。userhost 表示该用户链接来自的 IP。user_identity 以 username@'userhost' 的方式呈现，表示来自 userhost 的 username。 - - user_identity 的另一种表现方式为 username@['domain']，其中 domain 为域名，可以通过 DNS 或 BNS（百度名字服务）解析为一组 ip。最终表现为一组 username@'userhost'，所以后面我们统一使用 username@'userhost' 来表示。 - -2. 权限 Privilege - - 权限作用的对象是节点、数据库或表。不同的权限代表不同的操作许可。 - -3. 角色 Role - - Doris可以创建自定义命名的角色。角色可以被看做是一组权限的集合。新创建的用户可以被赋予某一角色，则自动被赋予该角色所拥有的权限。后续对角色的权限变更，也会体现在所有属于该角色的用户权限上。 - -4. 用户属性 user_property - - 用户属性直接附属于某一用户，而不是用户标识。即 cmy@'192.%' 和 cmy@['domain'] 都拥有同一组用户属性，该属性属于用户 cmy，而不是 cmy@'192.%' 或 cmy@['domain']。 - - 用户属性包括但不限于： 用户最大连接数、导入集群配置等等。 - -## 支持的操作 - -1. 创建用户：CREATE USER -2. 删除用户：DROP USER -3. 授权：GRANT -4. 撤权：REVOKE -5. 创建角色：CREATE ROLE -6. 删除角色：DROP ROLE -7. 查看当前用户权限：SHOW GRANTS -8. 查看所有用户权限：SHOW ALL GRANTS -9. 查看已创建的角色：SHOW ROLES -10. 查看用户属性：SHOW PROPERTY - -关于以上命令的详细帮助，可以通过 mysql 客户端连接 Doris 后，使用 help + command 获取帮助。如 `HELP CREATE USER`。 - -## 权限类型 - -Doris 目前支持以下几种权限 - -1. Node_priv - - 节点变更权限。包括 FE、BE、BROKER 节点的添加、删除、下线等操作。目前该权限只能授予 Root 用户。 - -2. Grant_priv - - 权限变更权限。允许执行包括授权、撤权、添加/删除/变更 用户/角色 等操作。 - -3. Select_priv - - 对数据库、表的只读权限。 - -4. Load_priv - - 对数据库、表的写权限。包括 Load、Insert、Delete 等。 - -5. Alter_priv - - 对数据库、表的更改权限。包括重命名 库/表、添加/删除/变更 列、添加/删除 分区等操作。 - -6. Create_priv - - 创建数据库、表、视图的权限。 - -7. Drop_priv - - 删除数据库、表、视图的权限。 - -## 权限层级 - -同时，根据权限适用范围的不同，我们将权限分为以下三个层级： - -1. GLOBAL LEVEL：全局权限。即通过 GRANT 语句授予的 `*.*` 上的权限。被授予的权限适用于任意数据库中的任意表。 -2. DATABASE LEVEL：数据库级权限。即通过 GRANT 语句授予的 `db.*` 上的权限。被授予的权限适用于指定数据库中的任意表。 -3. TABLE LEVEL：表级权限。即通过 GRANT 语句授予的 `db.tbl` 上的权限。被授予的权限适用于指定数据库中的指定表。 - - -## ADMIN/GRANT 权限说明 - -ADMIN\_PRIV 和 GRANT\_PRIV 权限同时拥有**授予权限**的权限，较为特殊。这里对和这两个权限相关的操作逐一说明。 - -1. CREATE USER - - * 拥有 ADMIN 权限，或任意层级的 GRANT 权限的用户可以创建新用户。 - -2. DROP USER - - * 只有 ADMIN 权限可以删除用户。 - -3. CREATE/DROP ROLE - - * 只有 ADMIN 权限可以创建角色。 - -4. GRANT/REVOKE - - * 拥有 ADMIN 权限，或者 GLOBAL 层级 GRANT 权限的用户，可以授予或撤销任意用户的权限。 - * 拥有 DATABASE 层级 GRANT 权限的用户，可以授予或撤销任意用户对指定数据库的权限。 - * 拥有 TABLE 层级 GRANT 权限的用户，可以授予或撤销任意用户对指定数据库中指定表的权限。 - -5. SET PASSWORD - - * 拥有 ADMIN 权限，或者 GLOBAL 层级 GRANT 权限的用户，可以设置任意用户的密码。 - * 普通用户可以设置自己对应的 UserIdentity 的密码。自己对应的 UserIdentity 可以通过 `SELECT CURRENT_USER();` 命令查看。 - * 拥有非 GLOBAL 层级 GRANT 权限的用户，不可以设置已存在用户的密码，仅能在创建用户时指定密码。 - - -## 一些说明 - -1. Doris 初始化时，会自动创建如下用户和角色： - - 1. operator 角色：该角色拥有 Node\_priv 和 Admin\_priv，即对Doris的所有权限。后续某个升级版本中，我们可能会将该角色的权限限制为 Node\_priv，即仅授予节点变更权限。以满足某些云上部署需求。 - - 2. admin 角色：该角色拥有 Admin\_priv，即除节点变更以外的所有权限。 - - 3. root@'%'：root 用户，允许从任意节点登陆，角色为 operator。 - - 4. admin@'%'：admin 用户，允许从任意节点登陆，角色为 admin。 - -2. 不支持删除或更改默认创建的角色或用户的权限。 - -3. operator 角色的用户有且只有一个。admin 角色的用户可以创建多个。 - -4. 一些可能产生冲突的操作说明 - - 1. 域名与ip冲突： - - 假设创建了如下用户： - - CREATE USER cmy@['domain']; - - 并且授权： - - GRANT SELECT_PRIV ON \*.\* TO cmy@['domain'] - - 该 domain 被解析为两个 ip：ip1 和 ip2 - - 假设之后，我们对 cmy@'ip1' 进行一次单独授权： - - GRANT ALTER_PRIV ON \*.\* TO cmy@'ip1'; - - 则 cmy@'ip1' 的权限会被修改为 SELECT\_PRIV, ALTER\_PRIV。并且当我们再次变更 cmy@['domain'] 的权限时，cmy@'ip1' 也不会跟随改变。 - - 2. 重复ip冲突： - - 假设创建了如下用户： - - CREATE USER cmy@'%' IDENTIFIED BY "12345"; - - CREATE USER cmy@'192.%' IDENTIFIED BY "abcde"; - - 在优先级上，'192.%' 优先于 '%'，因此，当用户 cmy 从 192.168.1.1 这台机器尝试使用密码 '12345' 登陆 Doris 会被拒绝。 - -5. 忘记密码 - - 如果忘记了密码无法登陆 Doris，可以在 Doris FE 节点所在机器，使用如下命令无密码登陆 Doris： - - `mysql-client -h 127.0.0.1 -P query_port -uroot` - - 登陆后，可以通过 SET PASSWORD 命令重置密码。 - -6. 任何用户都不能重置 root 用户的密码，除了 root 用户自己。 - -7. ADMIN\_PRIV 权限只能在 GLOBAL 层级授予或撤销。 - -8. 拥有 GLOBAL 层级 GRANT_PRIV 其实等同于拥有 ADMIN\_PRIV，因为该层级的 GRANT\_PRIV 有授予任意权限的权限，请谨慎使用。 - -9. `current_user()` 和 `user()` - - 用户可以通过 `SELECT current_user();` 和 `SELECT user();` 分别查看 `current_user` 和 `user`。其中 `current_user` 表示当前用户是以哪种身份通过认证系统的，而 `user` 则是用户当前实际的 `user_identity`。举例说明： - - 假设创建了 `user1@'192.%'` 这个用户，然后以为来自 192.168.10.1 的用户 user1 登陆了系统，则此时的 `current_user` 为 `user1@'192.%'`，而 `user` 为 `user1@'192.168.10.1'`。 - - 所有的权限都是赋予某一个 `current_user` 的，真实用户拥有对应的 `current_user` 的所有权限。 - -## 最佳实践 - -这里举例一些 Doris 权限系统的使用场景。 - -1. 场景一 - - Doris 集群的使用者分为管理员（Admin）、开发工程师（RD）和用户（Client）。其中管理员拥有整个集群的所有权限，主要负责集群的搭建、节点管理等。开发工程师负责业务建模，包括建库建表、数据的导入和修改等。用户访问不同的数据库和表来获取数据。 - - 在这种场景下，可以为管理员赋予 ADMIN 权限或 GRANT 权限。对 RD 赋予对任意或指定数据库表的 CREATE、DROP、ALTER、LOAD、SELECT 权限。对 Client 赋予对任意或指定数据库表 SELECT 权限。同时，也可以通过创建不同的角色，来简化对多个用户的授权操作。 - -2. 场景二 - - 一个集群内有多个业务，每个业务可能使用一个或多个数据。每个业务需要管理自己的用户。在这种场景下。管理员用户可以为每个数据库创建一个拥有 DATABASE 层级 GRANT 权限的用户。该用户仅可以对用户进行指定的数据库的授权。 - -3. 黑名单 - - Doris 本身不支持黑名单，只有白名单功能，但我们可以通过某些方式来模拟黑名单。假设先创建了名为 `user@'192.%'` 的用户，表示允许来自 `192.*` 的用户登录。此时如果想禁止来自 `192.168.10.1` 的用户登录。则可以再创建一个用户 `cmy@'192.168.10.1'` 的用户，并设置一个新的密码。因为 `192.168.10.1` 的优先级高于 `192.%`，所以来自 `192.168.10.1` 将不能再使用旧密码进行登录。 - - diff --git a/docs/documentation/cn/administrator-guide/small-file-mgr.md b/docs/documentation/cn/administrator-guide/small-file-mgr.md deleted file mode 100644 index 568f12b9fec61f..00000000000000 --- a/docs/documentation/cn/administrator-guide/small-file-mgr.md +++ /dev/null @@ -1,97 +0,0 @@ - - -# 文件管理器 - -Doris 中的一些功能需要使用一些用户自定义的文件。比如用于访问外部数据源的公钥、密钥文件、证书文件等等。文件管理器提供这样一个功能，能够让用户预先上传这些文件并保存在 Doris 系统中，然后可以在其他命令中引用或访问。 - -## 名词解释 - -* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 -* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 -* BDBJE：Oracle Berkeley DB Java Edition。FE 中用于持久化元数据的分布式嵌入式数据库。 -* SmallFileMgr：文件管理器。负责创建并维护用户的文件。 - -## 基本概念 - -文件是指用户创建并保存在 Doris 中的文件。 - -一个文件由 `数据库名称（database）`、`分类（catalog）` 和 `文件名（file_name）` 共同定位。同时每个文件也有一个全局唯一的 id（file_id），作为系统内的标识。 - -文件的创建和删除只能由拥有 `admin` 权限的用户进行操作。一个文件隶属于一个数据库。对某一数据库拥有访问权限（查询、导入、修改等等）的用户都可以使用该数据库下创建的文件。 - -## 具体操作 - -文件管理主要有三个命令：`CREATE FILE`，`SHOW FILE` 和 `DROP FILE`，分别为创建、查看和删除文件。这三个命令的具体语法可以通过连接到 Doris 后，执行 `HELP cmd;` 的方式查看帮助。 - -1. CREATE FILE - - 在创建文件的命令中，用户必须提供以下信息： - - * file_name：文件名。用户自定义，在一个 catalog 内唯一即可。 - * catalog：文件所属分类。用户自定义，在一个 database 内唯一即可。 - - > Doris 也有一些特殊的分类名称供特定的命令使用。 - - > 1. kafka - - > 当在例行导入命令中指定数据源为 Kafka，并且需要引用到文件时，Doris 会默认从 catalog 名为 "kafka" 的分类中查找文件。 - - * url：文件的下载地址。目前仅支持无认证的 http 下载地址。该下载地址仅用于在执行创建文件命令时，从这个地址下载文件。当文件成功创建并保存在 Doris 中后，该地址将不再被使用。 - * md5：可选项。文件的 MD5 值。如果用户提供该值，将在文件下载后进行 MD5 值的校验。校验失败则文件创建失败。 - - 文件创建成功后，文件相关的信息将持久化在 Doris 中。用户可以通过 `SHOW FILE` 命令查看已经创建成功的文件。 - -2. SHOW FILE - - 该命令可以查看已经创建成功的文件。具体操作见：`HELP SHOW FILE;` - -3. DROP FILE - - 该命令可以删除一个已经创建的文件。具体操作见：`HELP DROP FILE;` - -## 实现细节 - -### 创建和删除文件 - -当用户执行 `CREATE FILE` 命令后，FE 会从给定的 URL 下载文件。并将文件的内容以 Base64 编码的形式直接保存在 FE 的内存中。同时会将文件内容以及文件相关的元信息持久化在 BDBJE 中。所有被创建的文件，其元信息和文件内容都会常驻于 FE 的内存中。如果 FE 宕机重启，也会从 BDBJE 中加载元信息和文件内容到内存中。当文件被删除时，会直接从 FE 内存中删除相关信息，同时也从 BDBJE 中删除持久化的信息。 - -### 文件的使用 - -如果是 FE 端需要使用创建的文件，则 SmallFileMgr 会直接将 FE 内存中的数据保存为本地文件，存储在指定的目录中，并返回本地的文件路径供使用。 - -如果是 BE 端需要使用创建的文件，BE 会通过 FE 的 http 接口 `/api/get_small_file` 将文件内容下载到 BE 上指定的目录中，供使用。同时，BE 也会在内存中记录当前已经下载过的文件的信息。当 BE 请求一个文件时，会先查看本地文件是否存在并校验。如果校验通过，则直接返回本地文件路径。如果校验失败，则会删除本地文件，重新从 FE 下载。当 BE 重启时，会预先加载本地的文件到内存中。 - -## 使用限制 - -因为文件元信息和内容都存储于 FE 的内存中。所以默认仅支持上传大小在 1MB 以内的文件。并且总文件数量限制为 100 个。可以通过下一小节介绍的配置项进行修改。 - -## 相关配置 - -1. FE 配置 - - * `small_file_dir`：用于存放上传文件的路径，默认为 FE 运行目录的 `small_files/` 目录下。 - * `max_small_file_size_bytes`：单个文件大小限制，单位为字节。默认为 1MB。大于该配置的文件创建将会被拒绝。 - * `max_small_file_number`：一个 Doris 集群支持的总文件数量。默认为 100。当创建的文件数超过这个值后，后续的创建将会被拒绝。 - - > 如果需要上传更多文件或提高单个文件的大小限制，可以通过 `ADMIN SET CONFIG` 命令修改 `max_small_file_size_bytes` 和 `max_small_file_number` 参数。但文件数量和大小的增加，会导致 FE 内存使用量的增加。 - -2. BE 配置 - - * `small_file_dir`：用于存放从 FE 下载的文件的路径，默认为 BE 运行目录的 `lib/small_files/` 目录下。 diff --git a/docs/documentation/cn/administrator-guide/sql-mode.md b/docs/documentation/cn/administrator-guide/sql-mode.md deleted file mode 100644 index 43220bbd94d8d0..00000000000000 --- a/docs/documentation/cn/administrator-guide/sql-mode.md +++ /dev/null @@ -1,69 +0,0 @@ - - -# SQL MODE - -Doris新支持的sql mode参照了 Mysql 的sql mode管理机制，每个客户端都能设置自己的sql mode，拥有Admin权限的数据库管理员可以设置全局sql mode。 - -## sql mode 介绍 - -sql mode使用户能在不同风格的sql语法和数据校验严格度间做切换，使Doris对其他数据库有更好的兼容性。例如在一些数据库里，'||'符号是一个字符串连接符，但在Doris里却是与'or'等价的，这时用户只需要使用sql mode切换到自己想要的风格。每个客户端都能设置sql mode，并在当前对话中有效，只有拥有Admin权限的用户可以设置全局sql mode。 - -## 原理 - -sql mode用一个64位的Long型存储在SessionVariables中，这个地址的每一位都代表一个mode的开启/禁用(1表示开启，0表示禁用)状态，只要知道每一种mode具体是在哪一位，我们就可以通过位运算方便快速的对sql mode进行校验和操作。 - -每一次对sql mode的查询，都会对此Long型进行一次解析，变成用户可读的字符串形式，同理，用户发送给服务器的sql mode字符串，会被解析成能够存储在SessionVariables中的Long型。 - -已被设置好的全局sql mode会被持久化，因此对全局sql mode的操作总是只需一次，即使程序重启后仍可以恢复上一次的全局sql mode。 - -## 操作方式 - -1、设置sql mode - -``` -set global sql_mode = "DEFAULT" -set session sql_mode = "DEFAULT" -``` ->目前Doris的默认sql mode是DEFAULT（但马上会在后续修改中会改变）。 ->设置global sql mode需要Admin权限，并会影响所有在此后连接的客户端。 ->设置session sql mode只会影响当前对话客户端，默认为session方式。 - -2、查询sql mode - -``` -select @@global.sql_mode -select @@session.sql_mode -``` ->除了这种方式，你还可以通过下面方式返回所有session variables来查看当前sql mode - -``` -show global variables -show session variables -``` - -## 已支持mode - -1. `PIPES_AS_CONCAT` - - 在此模式下，'||'符号是一种字符串连接符号（同CONCAT()函数），而不是'OR'符号的同义词。(e.g., `'a'||'b' = 'ab'`, `1||0 = '10'`) - -## 复合mode - -（后续补充） \ No newline at end of file diff --git a/docs/documentation/cn/administrator-guide/time-zone.md b/docs/documentation/cn/administrator-guide/time-zone.md deleted file mode 100644 index dbf6a729c07cb9..00000000000000 --- a/docs/documentation/cn/administrator-guide/time-zone.md +++ /dev/null @@ -1,84 +0,0 @@ - - -# 时区 - -Doris 支持多时区设置 - -## 名词解释 - -* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 -* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 - -## 基本概念 - -Doris 内部存在多个时区相关参数 - -* system_time_zone : - 当服务器启动时，会根据机器设置时区自动设置，设置后不可修改。 - -* time_zone : - 服务器当前时区，区分session级别和global级别 - -## 具体操作 - -1. show variables like '%time_zone%' - - 查看当前时区相关配置 - -2. SET time_zone = 'Asia/Shanghai' - - 该命令可以设置session级别的时区，连接断开后失效 - -3. SET global time_zone = 'Asia/Shanghai' - - 该命令可以设置global级别的时区参数，fe会将参数持久化，连接断开后不失效 - -### 时区的影响 - -时区设置会影响对时区敏感的时间值的显示和存储。 - -包括NOW()或CURTIME()等时间函数显示的值，也包括show load, show backends中的时间值。 - -但不会影响 create table 中时间类型分区列的 less than 值，也不会影响存储为 date/datetime 类型的值的显示。 - -受时区影响的函数： - -* `FROM_UNIXTIME`：给定一个 UTC 时间戳，返回指定时区的日期时间：如 `FROM_UNIXTIME(0)`， 返回 CST 时区：`1970-01-01 08:00:00`。 -* `UNIX_TIMESTAMP`：给定一个指定时区日期时间，返回 UTC 时间戳：如 CST 时区 `UNIX_TIMESTAMP('1970-01-01 08:00:00')`，返回 `0`。 -* `CURTIME`：返回指定时区时间。 -* `NOW`：返指定地时区日期时间。 -* `CONVERT_TZ`：将一个日期时间从一个指定时区转换到另一个指定时区。 - -## 使用限制 - -时区值可以使用几种格式给出，不区分大小写: - -* 表示UTC偏移量的字符串，如'+10:00'或'-6:00' - -* 标准时区格式，如"Asia/Shanghai"、"America/Los_Angeles" - -* 不支持缩写时区格式，如"MET"、"CTT"。因为缩写时区在不同场景下存在歧义，不建议使用。 - -* 为了兼容Doris，支持CST缩写时区，内部会将CST转移为"Asia/Shanghai"的中国标准时区 - -## 时区格式列表 - -[List of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) - diff --git a/docs/documentation/cn/administrator-guide/variables.md b/docs/documentation/cn/administrator-guide/variables.md deleted file mode 100644 index 146b3f19e0ae31..00000000000000 --- a/docs/documentation/cn/administrator-guide/variables.md +++ /dev/null @@ -1,321 +0,0 @@ - - -# 变量 - -本文档主要介绍当前支持的变量（variables）。 - -Doris 中的变量参考 MySQL 中的变量设置。但部分变量仅用于兼容一些 MySQL 客户端协议，并不产生其在 MySQL 数据库中的实际意义。 - -## 变量设置与查看 - -### 查看 - -可以通过 `SHOW VARIABLES [LIKE 'xxx'];` 查看所有或指定的变量。如： - -``` -SHOW VARIABLES; -SHOW VARIABLES LIKE '%time_zone%'; -``` - -### 设置 - -部分变量可以设置全局生效或仅当前会话生效。设置全局生效后，后续新的会话连接中会沿用设置值。而设置仅当前会话生效，则变量仅对当前会话产生作用。 - -仅当前会话生效，通过 `SET var_name=xxx;` 语句来设置。如： - -``` -SET exec_mem_limit = 137438953472; -SET forward_to_master = true; -SET time_zone = "Asia/Shanghai"; -``` - -全局生效，通过 `SET GLOBALE var_name=xxx;` 设置。如： - -``` -SET GLOBAL exec_mem_limit = 137438953472 -``` - -> 注1：只有 ADMIN 用户可以设置变量的全局生效。 -> 注2：全局生效的变量不影响当前会话的变量值，仅影响新的会话中的变量。 - -支持全局生效的变量包括： - -* `time_zone` -* `wait_timeout` -* `sql_mode` -* `is_report_success` -* `query_timeout` -* `exec_mem_limit` -* `batch_size` -* `parallel_fragment_exec_instance_num` -* `parallel_exchange_instance_num` - -同时，变量设置也支持常量表达式。如： - -``` -SET exec_mem_limit = 10 * 1024 * 1024 * 1024; -SET forward_to_master = concat('tr', 'u', 'e'); -``` - -## 支持的变量 - -* `SQL_AUTO_IS_NULL` - - 用于兼容 JDBC 连接池 C3P0。 无实际作用。 - -* `auto_increment_increment` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `autocommit` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `batch_size` - - 用于指定在查询执行过程中，各个节点传输的单个数据包的行数。默认一个数据包的行数为 1024 行，即源端节点每产生 1024 行数据后，打包发给目的节点。 - - 较大的行数，会在扫描大数据量场景下提升查询的吞吐，但可能会在小查询场景下增加查询延迟。同时，也会增加查询的内存开销。建议设置范围 1024 至 4096。 - -* `character_set_client` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `character_set_connection` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `character_set_results` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `character_set_server` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `codegen_level` - - 用于设置 LLVM codegen 的等级。（当前未生效）。 - -* `collation_connection` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `collation_database` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `collation_server` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `disable_colocate_join` - - 控制是否启用 [Colocation Join](./colocation-join.md) 功能。默认为 false，表示启用该功能。true 表示禁用该功能。当该功能被禁用后，查询规划将不会尝试执行 Colocation Join。 - -* `disable_streaming_preaggregations` - - 控制是否开启流式预聚合。默认为 false，即开启。当前不可设置，且默认开启。 - -* `enable_insert_strict` - - 用于设置通过 INSERT 语句进行数据导入时，是否开启 `strict` 模式。默认为 false，即不开启 `strict` 模式。关于该模式的介绍，可以参阅 [这里](./load-data/insert-into-manual.md)。 - -* `enable_spilling` - - 用于设置是否开启大数据量落盘排序。默认为 false，即关闭该功能。当用户未指定 ORDER BY 子句的 LIMIT 条件，同时设置 `enable_spilling` 为 true 时，才会开启落盘排序。该功能启用后，会使用 BE 数据目录下 `doris-scratch/` 目录存放临时的落盘数据，并在查询结束后，清空临时数据。 - - 该功能主要用于使用有限的内存进行大数据量的排序操作。 - - 注意，该功能为实验性质，不保证稳定性，请谨慎开启。 - -* `exec_mem_limit` - - 用于设置单个查询的内存限制。默认为 2GB，单位为字节。 - - 该参数用于限制一个查询计划中，单个查询计划的实例所能使用的内存。一个查询计划可能有多个实例，一个 BE 节点可能执行一个或多个实例。所以该参数并不能准确限制一个查询在整个集群的内存使用，也不能准确限制一个查询在单一 BE 节点上的内存使用。具体需要根据生成的查询计划判断。 - - 通常只有在一些阻塞节点（如排序节点、聚合节点、Join 节点）上才会消耗较多的内存，而其他节点（如扫描节点）中，数据为流式通过，并不会占用较多的内存。 - - 当出现 `Memory Exceed Limit` 错误时，可以尝试指数级增加该参数，如 4G、8G、16G 等。 - -* `forward_to_master` - - 用户设置是否将一些命令转发到 Master FE 节点执行。默认为 false，即不转发。Doris 中存在多个 FE 节点，其中一个为 Master 节点。通常用户可以连接任意 FE 节点进行全功能操作。但部分信息查看指令，只有从 Master FE 节点才能获取详细信息。 - - 如 `SHOW BACKENDS;` 命令，如果不转发到 Master FE 节点，则仅能看到节点是否存活等一些基本信息，而转发到 Master FE 则可以获取包括节点启动时间、最后一次心跳时间等更详细的信息。 - - 当前受该参数影响的命令如下： - - 1. `SHOW FRONTENDS;` - - 转发到 Master 可以查看最后一次心跳信息。 - - 2. `SHOW BACKENDS;` - - 转发到 Master 可以查看启动时间、最后一次心跳信息、磁盘容量信息。 - - 3. `SHOW BROKER;` - - 转发到 Master 可以查看启动时间、最后一次心跳信息。 - - 4. `SHOW TABLET;`/`ADMIN SHOW REPLICA DISTRIBUTION;`/`ADMIN SHOW REPLICA STATUS;` - - 转发到 Master 可以查看 Master FE 元数据中存储的 tablet 信息。正常情况下，不同 FE 元数据中 tablet 信息应该是一致的。当出现问题时，可以通过这个方法比较当前 FE 和 Master FE 元数据的差异。 - - 5. `SHOW PROC;` - - 转发到 Master 可以查看 Master FE 元数据中存储的相关 PROC 的信息。主要用于元数据比对。 - -* `init_connect` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `interactive_timeout` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `is_report_success` - - 用于设置是否需要查看查询的 profile。默认为 false，即不需要 profile。 - - 默认情况下，只有在查询发生错误时，BE 才会发送 profile 给 FE，用于查看错误。正常结束的查询不会发送 profile。发送 profile 会产生一定的网络开销，对高并发查询场景不利。 - 当用户希望对一个查询的 profile 进行分析时，可以将这个变量设为 true 后，发送查询。查询结束后，可以通过在当前连接的 FE 的 web 页面查看到 profile： - - `fe_host:fe_http_port/query` - - 其中会显示最近100条，开启 `is_report_success` 的查询的 profile。 - -* `language` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `license` - - 显示 Doris 的 License。无其他作用。 - -* `load_mem_limit` - - 用于指定导入操作的内存限制。默认为 0，即表示不使用该变量，而采用 `exec_mem_limit` 作为导入操作的内存限制。 - - 这个变量仅用于 INSERT 操作。因为 INSERT 操作设计查询和导入两个部分，如果用户不设置此变量，则查询和导入操作各自的内存限制均为 `exec_mem_limit`。否则，INSERT 的查询部分内存限制为 `exec_mem_limit`，而导入部分限制为 `load_mem_limit`。 - - 其他导入方式，如 BROKER LOAD，STREAM LOAD 的内存限制依然使用 `exec_mem_limit`。 - -* `lower_case_table_names` - - 用于兼容 MySQL 客户端。不可设置。当前 Doris 中的表名默认为大小写敏感。 - -* `max_allowed_packet` - - 用于兼容 JDBC 连接池 C3P0。 无实际作用。 - -* `net_buffer_length` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `net_read_timeout` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `net_write_timeout` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `parallel_exchange_instance_num` - - 用于设置执行计划中，一个上层节点接收下层节点数据所使用的 exchange node 数量。默认为 -1，即表示 exchange node 数量等于下层节点执行实例的个数（默认行为）。当设置大于0，并且小于下层节点执行实例的个数，则 exchange node 数量等于设置值。 - - 在一个分布式的查询执行计划中，上层节点通常有一个或多个 exchange node 用于接收来自下层节点在不同 BE 上的执行实例的数据。通常 exchange node 数量等于下层节点执行实例数量。 - - 在一些聚合查询场景下，如果底层需要扫描的数据量较大，但聚合之后的数据量很小，则可以尝试修改此变量为一个较小的值，可以降低此类查询的资源开销。如在 DUPLICATE KEY 明细模型上进行聚合查询的场景。 - -* `parallel_fragment_exec_instance_num` - - 针对扫描节点，设置其在每个 BE 节点上，执行实例的个数。默认为 1。 - - 一个查询计划通常会产生一组 scan range，即需要扫描的数据范围。这些数据分布在多个 BE 节点上。一个 BE 节点会有一个或多个 scan range。默认情况下，每个 BE 节点的一组 scan range 只由一个执行实例处理。当机器资源比较充裕时，可以将增加该变量，让更多的执行实例同时处理一组 scan range，从而提升查询效率。 - - 而 scan 实例的数量决定了上层其他执行节点，如聚合节点，join 节点的数量。因此相当于增加了整个查询计划执行的并发度。修改该参数会对大查询效率提升有帮助，但较大数值会消耗更多的机器资源，如CPU、内存、磁盘IO。 - -* `query_cache_size` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `query_cache_type` - - 用于兼容 JDBC 连接池 C3P0。 无实际作用。 - -* `query_timeout` - - 用于设置查询超时。该变量会作用于当前连接中所有的查询语句，以及 INSERT 语句。默认为 5 分钟，单位为秒。 - -* `resource_group` - - 暂不使用。 - -* `sql_mode` - - 用于指定 SQL 模式，以适应某些 SQL 方言。关于 SQL 模式，可参阅 [这里](./sql-mode.md)。 - -* `sql_safe_updates` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `sql_select_limit` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `system_time_zone` - - 显示当前系统时区。不可更改。 - -* `time_zone` - - 用于设置当前会话的时区。时区会对某些时间函数的结果产生影响。关于时区，可以参阅 [这里](./time-zone.md)。 - -* `tx_isolation` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `version` - - 用于兼容 MySQL 客户端。无实际作用。 - -* `version_comment` - - 用于显示 Doris 的版本。不可更改。 - -* `wait_timeout` - - 用于设置空闲连接的连接时长。当一个空闲连接在该时长内与 Doris 没有任何交互，则 Doris 会主动断开这个链接。默认为 8 小时，单位为秒。 - -* `default_rowset_type` - - 用于设置计算节点存储引擎默认的存储格式。当前支持的存储格式包括：alpha/beta。 - -* `use_v2_rollup` - - 用于控制查询使用segment v2存储格式的rollup索引获取数据。该变量用于上线segment v2的时候，进行验证使用；其他情况，不建议使用。 - -* `rewrite_count_distinct_to_bitmap_hll` - - 是否将 bitmap 和 hll 类型的 count distinct 查询重写为 bitmap_union_count 和 hll_union_agg 。 - diff --git a/docs/documentation/cn/community/gitter.md b/docs/documentation/cn/community/gitter.md deleted file mode 100644 index db8540cd17e008..00000000000000 --- a/docs/documentation/cn/community/gitter.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# Gitter 使用指南 - -## Gitter 简介 - -Gitter 是一款可支持 Markdown 的针对开发者的即时通讯软件，可无缝关联到 github，可在聊天中关联Github上的PR，可留存讨论的相关历史记录，可查询历史记录，支持中英文。 - -Doris 和很多其他的开源项目一样，可以使用 Gitter 作为技术交流和社区发展的即时通讯媒介，本文介绍如何使用 Gitter 参与到 Doris 的开源开发和社区发展中。 - -## 使用链接登录 - -在浏览器 输入 [https://gitter.im/apache-doris/Lobby](https://gitter.im/apache-doris/Lobby) ，会自动跳转到 Gitter 上 Doris 社区的聊天室界面。 - -点击下方的 `SIGN IN TO START TALKING` 进行登录，可支持两种登录方式，Github 账号或 Twitter 账号，笔者使用 Github 账号进行登录，如下： - -![](../../../resources/images/login-gitter1.png) - -点击红圈处后，输入 Github 账号和密码既可登录进聊天室，从此开始进行技术或社区讨论： - -![](../../../resources/images/login-gitter2.PNG) - -可以像使用微信一样，使用Gitter，并获得比微信更加令开发者和技术人员舒服的功能，如，直接提及某个Activity进行讨论，直接搜索历史聊天记录等。 - -已经加入聊天室的伙伴别忘记点击右上角的五角星进行收藏，这样会让本聊天室更容易被您自己找到。 - -更多 gitter 使用技巧，可以参考： - -[http://www.gitter.net.cn/book/gitter/roomsettings-1.html](http://www.gitter.net.cn/book/gitter/roomsettings-1.html) - -## 安装手机客户端 - -您可以下载 Gitter 的手机客户端，在手机上也随时随地的参与到技术讨论中，下载链接： - -[https://gitter.im/home](https://gitter.im/home) - -## 在 Gitter 上搜索并加入 Doris 社区聊天室 - -已经在使用 Gitter 的伙伴，直接登录 Gitter 后，搜索 `apache-doris` ,找到后可以加入聊天室，其他的功能使用同上一章节，这里不赘述。 diff --git a/docs/documentation/cn/community/how-to-contribute.md b/docs/documentation/cn/community/how-to-contribute.md deleted file mode 100644 index 040810b75399d4..00000000000000 --- a/docs/documentation/cn/community/how-to-contribute.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# 为 Doris 做贡献 - -非常感谢您对 Doris 项目感兴趣，我们非常欢迎您对 Doris 项目的各种建议、意见（包括批评）、评论和贡献。 - -您对 Doris 的各种建议、意见、评论可以直接通过 GitHub 的 [Issues](https://github.com/apache/incubator-doris/issues/new/choose) 提出。 - -参与 Doris 项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表，进一步，有了足够的贡献后，您还可以有机会成为 Aapche 的 Commiter，拥有 Apache 邮箱，并被收录到 [Apache Commiter 列表中](http://people.apache.org/committer-index.html)。 - -任何问题，您都可以联系我们得到及时解答，联系方式包括微信、Gitter（GitHub提供的即时聊天工具）、邮件等等。 - -## 初次接触 - -初次来到 Doris 社区，您可以： - -* 关注 Doris [Github 代码库](https://github.com/apache/incubator-doris) -* 订阅我们的 [邮件列表](./subscribe-mail-list.md)； -* 加入 Doris 微信群(加微信号：morningman-cmy, 备注：加入Doris群) 随时提问； -* 进入 Doris 的 [Gitter](./gitter.md) 聊天室； - -通过以上方式及时了解 Doris 项目的开发动态并为您关注的话题发表意见。 - -## Doris 的代码和文档 - -正如您在 [GitHub] (https://github.com/apache/incubator-doris) 上看到的，Apache Doris (incubating) 的代码库主要包括三部分：Frontend (FE), Backend (BE) 和 Broker (为了支持 HDFS 等外部存储系统上的文件读取)。文档主要是 Doris 网站和 GitHub 上的 wiki，还有运行 Doris 的时候的在线帮助手册。这些组件的详细情况参见下表： - -| 组件名称 | 组件描述 | 相关语言 | -|--------|----------------------------|----------| -| [Frontend daemon (FE)](https://github.com/apache/incubator-doris)| 由“查询协调器”和“元数据管理器”组成 | Java| -| [Backend daemon (BE)](https://github.com/apache/incubator-doris) | 负责存储数据和执行查询片段 | C++| -| [Broker](https://github.com/apache/incubator-doris) | 读取 HDFS 数据到 Doris | Java | -| [Website](https://github.com/apache/incubator-doris-website) | Doris 网站 | Markdown | -| [GitHub Wiki](https://github.com/apache/incubator-doris/wiki) | Doris GitHub Wiki | Markdown | -| Doris 运行时 help 文档 | 运行 Doris 的时候的在线帮助手册 | Markdown | - -## 改进文档 - -文档是您了解 Apache Doris 的最主要的方式，也是我们最需要帮助的地方！ - -浏览文档，可以加深您对 Doris 的了解，也可以帮助您理解 Doris 的功能和技术细节，如果您发现文档有问题，请及时联系我们； - -如果您对改进文档的质量感兴趣，不论是修订一个页面的地址、更正一个链接、以及写一篇更优秀的入门文档，我们都非常欢迎！ - -我们的文档大多数是使用 markdown 格式编写的，您可以直接通过在 [GitHub] (https://github.com/apache/incubator-doris) 中的 `docs/` 中修改并提交文档变更。如果提交代码变更，可以参阅 [Pull Request](./pull-request.md)。 - -## 如果发现了一个 Bug 或问题 - -如果发现了一个 Bug 或问题，您可以直接通过 GitHub 的 [Issues](https://github.com/apache/incubator-doris/issues/new/choose) 提一个新的 Issue，我们会有人定期处理。 - -您也可以通过阅读分析代码自己修复（当然在这之前最好能和我们交流下，或许已经有人在修复同样的问题了），然后提交一个 [Pull Request](./pull-request.md)。 - -## 修改代码和提交PR（Pull Request） - -您可以下载代码，编译安装，部署运行试一试（可以参考[编译文档](../installing/compilation.md)），看看是否与您预想的一样工作。如果有问题，您可以直接联系我们，提 Issue 或者通过阅读和分析源代码自己修复。 - -无论是修复 Bug 还是增加 Feature，我们都非常欢迎。如果您希望给 Doris 提交代码，您需要从 GitHub 上 fork 代码库至您的项目空间下，为您提交的代码创建一个新的分支，添加源项目为upstream，并提交PR。 -提交PR的方式可以参考文档 [Pull Request](./pull-request.md)。 diff --git a/docs/documentation/cn/community/index.rst b/docs/documentation/cn/community/index.rst deleted file mode 100644 index 26ae323869baf2..00000000000000 --- a/docs/documentation/cn/community/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -=========== -Apache 社区 -=========== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/community/members.md b/docs/documentation/cn/community/members.md deleted file mode 100644 index 49536e30467840..00000000000000 --- a/docs/documentation/cn/community/members.md +++ /dev/null @@ -1,61 +0,0 @@ - - -# PMC Members & Committer - -## Mentors (3) - -|Apache ID|Github Username |Public Name| -|--------|-----------|----------| -|wave |dave2wave |Dave Fisher | -|shaofengshi |shaofengshi| Shao Feng Shi | -|ningjiang |WillemJiang |Willem Ning Jiang| - -## PPMC (9) -(the listing below excludes mentors) - -|Apache ID|Github Username |Public Name| -|--------|-----------|----------| -|lingbin| lingbin |Bin Ling | -|lichaoyong |chaoyli |Chaoyong Li | -|zhaoc |imay |Chun Zhao | -|lide |lide-reed, doris-ci |De Li | -|chenhao |chenhao7253886 |Hao Chen | -|morningman |morningman |Mingyu Chen| -|maruyue |maruyue| Ruyue Ma | -|sijie |sijie |Sijie Guo | -|zshao |zshao |Zheng Shao| - -## Committers (13) - -|Apache ID|Github Username |Public Name| -|--------|-----------|----------| -|lingbin| lingbin |Bin Ling | -|lichaoyong |chaoyli |Chaoyong Li | -|zhaoc |imay |Chun Zhao | -|lide |lide-reed, doris-ci |De Li | -|chenhao |chenhao7253886 |Hao Chen | -|morningman |morningman |Mingyu Chen| -|maruyue |maruyue| Ruyue Ma | -|sijie |sijie |Sijie Guo | -|zshao |zshao |Zheng Shao| -|kangkaisen|kangkaisen|Kaisen Kang| -|lingmiao|EmmyMiao87|Ling Miao| -|gaodayue|gaodayue|Dayue Gao| -|liuhangyuan|HangyuanLiu|Hangyuan Liu| diff --git a/docs/documentation/cn/community/pull-request.md b/docs/documentation/cn/community/pull-request.md deleted file mode 100644 index e9f79dca897a46..00000000000000 --- a/docs/documentation/cn/community/pull-request.md +++ /dev/null @@ -1,252 +0,0 @@ - - -# 代码提交指南 - -在 [Github](https://github.com/apache/incubator-doris) 上面可以很方便地提交 [Pull Request (PR)](https://help.github.com/articles/about-pull-requests/)，下面介绍 Doris 项目的 PR 方法。 - -### 1. Fork仓库 - -进入 apache/incubator-doris 的 [github 页面](https://github.com/apache/incubator-doris) ，点击右上角按钮 `Fork` 进行 Fork。 - -![Fork](../../../resources/images/fork-repo.png) - -### 2. 配置git和提交修改 - -#### （1）将代码克隆到本地： - -``` -git clone https://github.com//incubator-doris.git -``` - -注意：请将 \ 替换为您的 github 名字。 - -clone 完成后，origin 会默认指向 github 上的远程 fork 地址。 - -#### （2）将 apache/incubator-doris 添加为本地仓库的远程分支 upstream： - -``` -cd incubator-doris -git remote add upstream https://github.com/apache/incubator-doris.git -``` - -#### （3）检查远程仓库设置： - -``` -git remote -v -origin https://github.com//incubator-doris.git (fetch) -origin https://github.com//incubator-doris.git (push) -upstream https://github.com/apache/incubator-doris.git (fetch) -upstream https://github.com/apache/incubator-doris.git (push) -``` - -#### （4）新建分支以便在分支上做修改： - -``` -git checkout -b -``` - -注意： \ 为您自定义的分支名字。 - -创建完成后可进行代码更改。 - -#### （5）提交代码到远程分支： - -``` -git commit -a -m "" -git push origin -``` - -更多 git 使用方法请访问：[git 使用](https://www.atlassian.com/git/tutorials/setting-up-a-repository)，这里不赘述。 - -### 3. 创建PR - -#### （1）新建 PR -在浏览器切换到自己的 github 页面，切换分支到提交的分支 \ ，点击 `New pull request` 按钮进行创建，如下图所示： - -![new PR](../../../resources/images/new-pr.png) - -#### （2）准备分支 -这时候，会出现 `Create pull request` 按钮，如果没有请检查是否正确选择了分支，也可以点击 “compare across forks” 重新选择 repo 和分支。 - -![create PR](../../../resources/images//create-pr.png) - -#### （3）填写 Commit Message -这里请填写 comment 的总结和详细内容，然后点击 `Create pull request` 进行创建。 - -关于如何写 Commit Message，下面列出了一些 Tips： - -* 请用英文 动词 + 宾语 的形式，动词不用过去式，语句用祈使句； -* 消息主题（Subject）和具体内容（Body）都要写，它们之间要有空行分隔（GitHub PR界面上分别填写即可）; -* 消息主题长度不要超过**50**个字符； -* 消息内容每行不要超过**72**个字符，超过的需要手动换行； -* 消息内容用于解释做了什么、为什么做以及怎么做的； -* 消息主题第一个字母要**大写**，句尾**不要**有句号； -* 消息内容中写明关联的issue(如果有)，例如 #233; - -更详细的内容请参考 - -![create PR](../../../resources/images/create-pr2.png) - -#### （4）完成创建 -创建成功后，您可以看到 Doris 项目需要 review，您可以等待我们 review 和合入，您也可以直接联系我们。 - -![create PR](../../../resources/images/create-pr3.png) - -至此，您的PR创建完成，更多关于 PR 请阅读 [collaborating-with-issues-and-pull-requests](https://help.github.com/categories/collaborating-with-issues-and-pull-requests/) 。 - -### 4. 冲突解决 - -提交PR时的代码冲突一般是由于多人编辑同一个文件引起的，解决冲突主要通过以下步骤即可： - -#### （1）切换至主分支 - -``` -git checkout master -``` - -#### （2）同步远端主分支至本地 - -``` -git pull upstream master -``` - -#### （3）切换回刚才的分支（假设分支名为fix） - -``` -git checkout fix -``` - -#### （4）进行rebase - -``` -git rebase -i master -``` - -此时会弹出修改记录的文件，一般直接保存即可。然后会提示哪些文件出现了冲突，此时可打开冲突文件对冲突部分进行修改，将提示的所有冲突文件的冲突都解决后，执行 - -``` -git add . -git rebase --continue -``` - -依此往复，直至屏幕出现类似 *rebase successful* 字样即可，此时您可以进行往提交PR的分支进行更新： - -``` -git push -f origin fix -``` - -### 5. 一个例子 - -#### （1）对于已经配置好 upstream 的本地分支 fetch 到最新代码 - -``` -$ git branch -* master - -$ git fetch upstream -remote: Counting objects: 195, done. -remote: Compressing objects: 100% (68/68), done. -remote: Total 141 (delta 75), reused 108 (delta 48) -Receiving objects: 100% (141/141), 58.28 KiB, done. -Resolving deltas: 100% (75/75), completed with 43 local objects. -From https://github.com/apache/incubator-doris - 9c36200..0c4edc2 master -> upstream/master -``` - -#### （2）进行rebase - -``` -$ git rebase upstream/master -First, rewinding head to replay your work on top of it... -Fast-forwarded master to upstream/master. -``` - -#### （3）检查看是否有别人提交未同步到自己 repo 的提交 - -``` -$ git status -# On branch master -# Your branch is ahead of 'origin/master' by 8 commits. -# -# Untracked files: -# (use "git add ..." to include in what will be committed) -# -# custom_env.sh -nothing added to commit but untracked files present (use "git add" to track) -``` - -#### （4）合并其他人提交的代码到自己的 repo - -``` -$ git push origin master -Counting objects: 195, done. -Delta compression using up to 32 threads. -Compressing objects: 100% (41/41), done. -Writing objects: 100% (141/141), 56.66 KiB, done. -Total 141 (delta 76), reused 140 (delta 75) -remote: Resolving deltas: 100% (76/76), completed with 44 local objects. -To https://lide-reed:fc35ff925bd8fd6629be3f6412bacee99d4e5f97@github.com/lide-reed/incubator-doris.git - 9c36200..0c4edc2 master -> master -``` - -#### （5）新建分支，准备开发 - -``` -$ git checkout -b my_branch -Switched to a new branch 'my_branch' - -$ git branch - master -* my_branch -``` - -#### （6）代码修改完成后，准备提交 - -``` -$ git add -u -``` - -#### （7）填写 message 并提交到本地的新建分支上 - -``` -$ git commit -m "Fix a typo" -[my_branch 55e0ba2] Fix a typo - 1 files changed, 2 insertions(+), 2 deletions(-) -``` - -#### （8）将分支推到 GitHub 远端自己的 repo 中 - -``` -$ git push origin my_branch -Counting objects: 11, done. -Delta compression using up to 32 threads. -Compressing objects: 100% (6/6), done. -Writing objects: 100% (6/6), 534 bytes, done. -Total 6 (delta 4), reused 0 (delta 0) -remote: Resolving deltas: 100% (4/4), completed with 4 local objects. -remote: -remote: Create a pull request for 'my_branch' on GitHub by visiting: -remote: https://github.com/lide-reed/incubator-doris/pull/new/my_branch -remote: -To https://lide-reed:fc35ff925bd8fd6629be3f6412bacee99d4e5f97@github.com/lide-reed/incubator-doris.git - * [new branch] my_branch -> my_branch -``` - -至此，就可以按照前面的流程进行创建 PR 了。 \ No newline at end of file diff --git a/docs/documentation/cn/community/release-process.md b/docs/documentation/cn/community/release-process.md deleted file mode 100644 index 39db1800cd7630..00000000000000 --- a/docs/documentation/cn/community/release-process.md +++ /dev/null @@ -1,648 +0,0 @@ - - -# Apache Doris 发布流程 - -Apache 的发布必须至少是 IPMC 成员，拥有 apache 邮箱的commiter，这个角色叫做 release manager。 - -发布的大致流程如下： - -1. 环境准备 -2. 发布准备 - 1. 社区发起 DISCUSS 并与社区交流具体发布计划 - 2. 创建分支用于发布 - 3. 清理 issue - 4. 将必要的 Patch 合并到发布的分支 -3. 社区发布投票流程 - 1. 将 tag 打包，签名并上传到[Apache Dev svn 仓库](https://dist.apache.org/repos/dist/dev/incubator/doris) - 2. 在 [Doris 社区](dev@doris.apache.org)发起投票 - 3. 投票通过后，在Doris社区发 Result 邮件 - 4. 在 [Incubator 社区](general@incubator.apache.org) 发起新一轮投票 - 5. 发 Result 邮件到 general@incubator.apache.org -4. 完成工作 - 1. 上传签名的软件包到 [Apache release repo](https://dist.apache.org/repos/dist/release/incubator/doris)，并生成相关链接 - 2. 准备 release note 并发 Announce 邮件到 general@incubator.apache.org - 3. 在 Doris 官网和 github 发布下载链接 - -## 准备环境 - -如果这是你第一次发布，那么你需要在你的环境中准备如下工具 - -1. release signing https://www.apache.org/dev/release-signing.html -2. gpg https://www.apache.org/dev/openpgp.html -3. svn https://www.apache.org/dev/openpgp.html - -### 准备gpg key - -Release manager 在发布前需要先生成自己的签名公钥，并上传到公钥服务器，之后就可以用这个公钥对准备发布的软件包进行签名。 -如果在[KEY](https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS)里已经存在了你的KEY，那么你可以跳过这个步骤了。 - -#### 签名软件 GnuPG 的安装配置 -##### GnuPG - -1991年，程序员 Phil Zimmermann 为了避开政府监视，开发了加密软件PGP。这个软件非常好用，迅速流传开来，成了许多程序员的必备工具。但是，它是商业软件，不能自由使用。所以，自由软件基金会决定，开发一个PGP的替代品，取名为GnuPG。这就是GPG的由来。 - -##### 安装配置 - -CentOS 安装命令： - -``` -yum install gnupg -``` -安装完成后，默认配置文件 gpg.conf 会放在 home 目录下。 - -``` -~/.gnupg/gpg.conf -``` - -如果不存在这个目录或文件，可以直接创建一个空文件。 -编辑gpg.conf, 修改或者增加 keyserver 配置： - -``` -keyserver hkp://keys.gnupg.net -``` - -Apache 签名推荐 SHA512， 可以通过配置 gpg 完成。 -编辑gpg.conf, 增加下面的三行： - -``` -personal-digest-preferences SHA512 -cert-digest-algo SHA512 -default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed -``` -#### 生成新的签名 - -##### 准备签名 - -推荐的生成新签名的设置： - -这里必须通过 SecureCRT 等终端直接登录用户账户，不能通过 su - user 或者 ssh 转，否则密码输入 box 会显示不出来而报错。 - -先看下 gpg 的 version 以及是否支持 SHA512. - -``` -$ gpg --version -gpg (GnuPG) 2.0.22 -libgcrypt 1.5.3 -Copyright (C) 2013 Free Software Foundation, Inc. -License GPLv3+: GNU GPL version 3 or later -This is free software: you are free to change and redistribute it. -There is NO WARRANTY, to the extent permitted by law. - -Home: ~/.gnupg -Supported algorithms: -Pubkey: RSA, ?, ?, ELG, DSA -Cipher: IDEA, 3DES, CAST5, BLOWFISH, AES, AES192, AES256, TWOFISH, - CAMELLIA128, CAMELLIA192, CAMELLIA256 -Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224 -Compression: Uncompressed, ZIP, ZLIB, BZIP2 -``` - -##### 生成新的签名 - -``` -$ gpg --gen-key -gpg (GnuPG) 2.0.22; Copyright (C) 2013 Free Software Foundation, Inc. -This is free software: you are free to change and redistribute it. -There is NO WARRANTY, to the extent permitted by law. - -Please select what kind of key you want: - (1) RSA and RSA (default) - (2) DSA and Elgamal - (3) DSA (sign only) - (4) RSA (sign only) -Your selection? 1 -RSA keys may be between 1024 and 4096 bits long. -What keysize do you want? (2048) 4096 -Requested keysize is 4096 bits -Please specify how long the key should be valid. - 0 = key does not expire - = key expires in n days - w = key expires in n weeks - m = key expires in n months - y = key expires in n years -Key is valid for? (0) -Key does not expire at all -Is this correct? (y/N) y - -GnuPG needs to construct a user ID to identify your key. - -Real name: xxx -Name must be at least 5 characters long -Real name: xxx-yyy -Email address: xxx@apache.org -Comment: xxx's key -You selected this USER-ID: - "xxx-yyy (xxx's key) " - -Change (N)ame, (C)omment, (E)mail or (O)kay/(Q)uit? o -``` - -其中 Real name 需保持和 id.apache.org 中显示的 id 一致。 -Email address 为 apache 的邮箱。 - -##### 查看和输出 - -第一行显示公钥文件名（pubring.gpg），第二行显示公钥特征（4096位，Hash字符串和生成时间），第三行显示"用户ID"，第四行显示私钥特征。 - -``` -$ gpg --list-keys -/home/lide/.gnupg/pubring.gpg ------------------------------ -pub 4096R/33DBF2E0 2018-12-06 -uid xxx-yyy (xxx's key) -sub 4096R/0E8182E6 2018-12-06 -``` - -其中 xxx-yyy 就是用户ID。 - -``` -gpg --armor --output public-key.txt --export [用户ID] -``` - -``` -$ gpg --armor --output public-key.txt --export xxx-yyy -$ cat public-key.txt ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v2.0.22 (GNU/Linux) - -mQINBFwJEQ0BEACwqLluHfjBqD/RWZ4uoYxNYHlIzZvbvxAlwS2mn53BirLIU/G3 -9opMWNplvmK+3+gNlRlFpiZ7EvHsF/YJOAP59HmI2Z... -``` - -#### 上传签名公钥 - -公钥服务器是网络上专门储存用户公钥的服务器。send-keys 参数可以将公钥上传到服务器。 - -``` -gpg --send-keys xxxx - -``` -其中 xxxx 为上一步 `--list-keys` 结果中 pub 后面的字符串，如上为：33DBF2E0 - -也可以通过[网站](https://keys.gnupg.net)上传上述 public-key.txt 的内容： - -上传成功之后，可以通过查询这个[网站](https://keys.gnupg.net)，输入 0x33DBF2E0 查询： - -该网站查询有延迟，可能需要等1个小时。 - -#### 生成 fingerprint 并上传到 apache 用户信息中 - -由于公钥服务器没有检查机制，任何人都可以用你的名义上传公钥，所以没有办法保证服务器上的公钥的可靠性。通常，你可以在网站上公布一个公钥指纹，让其他人核对下载到的公钥是否为真。 - -fingerprint参数生成公钥指纹： - -``` -gpg --fingerprint [用户ID] -``` - -``` -$ gpg --fingerprint xxx-yyy -pub 4096R/33DBF2E0 2018-12-06 - Key fingerprint = 07AA E690 B01D 1A4B 469B 0BEF 5E29 CE39 33DB F2E0 -uid xxx-yyy (xxx's key) -sub 4096R/0E8182E6 2018-12-06 -``` - -将上面的 fingerprint （即 07AA E690 B01D 1A4B 469B 0BEF 5E29 CE39 33DB F2E0）粘贴到自己的用户信息中： - -https://id.apache.org - -OpenPGP Public Key Primary Fingerprint: - -#### 生成 keys - -``` -svn co //dist.apache.org/repos/dist/dev/incubator/doris/ -# edit doris/KEY file -gpg --list-sigs [用户 ID] >> doris/KEYS -gpg --armor --export [用户 ID] >> doris/KEYS -svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Update KEYS" -``` - -## 准备发布 - -### 在社区发起 DISCUSS - -如果觉得已经修复了很多bug，开发了比较重要的 feature，任何 IPMC 成员都可以发起 DISCUSS 讨论发布新版本。 -可以发起一个标题为 [DISCUSS] x.y.z release 的邮件，在社区内部进行讨论，说明已经修复了哪些bug，开发了哪些 features。 -如果 DISCUSS 邮件得到大家支持就可以进行下一步。 - -### 准备分支 - -发布前需要先新建一个分支。例如： - -``` -$ git checkout -b branch-0.9 - -``` - -这个分支要进行比较充分的测试，使得功能可用，bug收敛，重要bug都得到修复。 -这个过程需要等待社区，看看是否有必要的patch需要在这个版本合入，如果有，需要把它 cherry-pick 到发布分支。 - -### 清理issue - -将属于这个版本的所有 issue 都过一遍，关闭已经完成的，如果没法完成的，推迟到更晚的版本。 - -### 合并必要的Patch - -在发布等待过程中，可能会有比较重要的Patch合入，如果社区有人说要有重要的Bug需要合入，那么 Release Manager 需要评估并将重要的Patch合入到发布分支中。 - -## 社区发布投票流程 - -### 打 tag - -当上述分支已经比较稳定后，就可以在此分支上打 tag。 -记得在创建 tag 时，修改 `gensrc/script/gen_build_version.sh` 中的 `build_version` 变量。如 `build_version="0.10.0-release"` - -例如： - -``` -$ git checkout branch-0.9 -$ git tag -a 0.9.0-rc01 -m "0.9.0 release candidate 01" -$ git push origin 0.9.0-rc01 -Counting objects: 1, done. -Writing objects: 100% (1/1), 165 bytes | 0 bytes/s, done. -Total 1 (delta 0), reused 0 (delta 0) -To git@github.com:apache/incubator-doris.git - * [new tag] 0.9.0-rc01 -> 0.9.0-rc01 - -$ git tag -``` - -### 打包、签名上传 - -如下步骤，也需要通过 SecureCRT 等终端直接登录用户账户，不能通过 su - user 或者 ssh 转，否则密码输入 box 会显示不出来而报错。 - -``` -$ git checkout 0.9.0-rc01 - -$ git archive --format=tar 0.9.0-rc01 --prefix=apache-doris-0.9.0-incubating-src/ | gzip > apache-doris-0.9.0-incubating-src.tar.gz - -$ gpg -u xxx@apache.org --armor --output apache-doris-0.9.0-incubating-src.tar.gz.asc --detach-sign apache-doris-0.9.0-incubating-src.tar.gz - -$ gpg --verify apache-doris-0.9.0-incubating-src.tar.gz.asc apache-doris-0.9.0-incubating-src.tar.gz - -$ sha512sum apache-doris-0.9.0-incubating-src.tar.gz > apache-doris-0.9.0-incubating-src.tar.gz.sha512 - -$ sha512sum --check apache-doris-0.9.0-incubating-src.tar.gz.sha512 -``` - -然后将打包的内容上传到svn仓库中，首先下载 svn 库： - -``` -svn co https://dist.apache.org/repos/dist/dev/incubator/doris/ -``` - -将之前得到的全部文件组织成以下svn路径 - -``` -./doris/ -|-- 0.11.0-rc1 -| |-- apache-doris-0.11.0-incubating-src.tar.gz -| |-- apache-doris-0.11.0-incubating-src.tar.gz.asc -| `-- apache-doris-0.11.0-incubating-src.tar.gz.sha512 -`-- KEYS -``` - -上传这些文件 - -``` -svn add 0.11.0-rc1 -svn commit -m "Add 0.11.0-rc1" -``` - -### 发社区投票邮件 - -[VOTE] Release Apache Doris 0.9.0-incubating-rc01 - -``` -Hi all, - -Please review and vote on Apache Doris 0.9.0-incubating-rc01 release. - -The release candidate has been tagged in GitHub as 0.9.0-rc01, available -here: -https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 - -Release Notes are here: -https://github.com/apache/incubator-doris/issues/1891 - -Thanks to everyone who has contributed to this release. - -The artifacts (source, signature and checksum) corresponding to this release -candidate can be found here: -https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc1/ - -This has been signed with PGP key 33DBF2E0, corresponding to -lide@apache.org. -KEYS file is available here: -https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS -It is also listed here: -https://people.apache.org/keys/committer/lide.asc - -To verify and build, you can refer to following wiki: -https://github.com/apache/incubator-doris/wiki/How-to-verify-Apache-Release -https://wiki.apache.org/incubator/IncubatorReleaseChecklist - -The vote will be open for at least 72 hours. -[ ] +1 Approve the release -[ ] +0 No opinion -[ ] -1 Do not release this package because ... - -Best Regards, -xxx - ----- -DISCLAIMER-WIP: -Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), -sponsored by the Apache Incubator. Incubation is required of all newly accepted projects -until a further review indicates that the infrastructure, communications, and decision -making process have stabilized in a manner consistent with other successful ASF projects. -While incubation status is not necessarily a reflection of the completeness or stability -of the code, it does indicate that the project has yet to be fully endorsed by the ASF. - -Some of the incubating project’s releases may not be fully compliant with ASF policy. For -example, releases may have incomplete or un-reviewed licensing conditions. What follows is -a list of known issues the project is currently aware of (note that this list, by definition, -is likely to be incomplete): - - * Releases may have incomplete licensing conditions - -If you are planning to incorporate this work into your product/project, please be aware that -you will need to conduct a thorough licensing review to determine the overall implications of -including this work. For the current status of this project through the Apache Incubator -visit: https://incubator.apache.org/projects/doris.html -``` - -### 投票通过后，发 Result 邮件 - -[Result][VOTE] Release Apache Doris 0.9.0-incubating-rc01 - -``` -Thanks to everyone, and this vote is now closed. - -It has passed with 4 +1 (binding) votes and no 0 or -1 votes. - -Binding: -+1 Zhao Chun -+1 xxx -+1 Li Chaoyong -+1 Mingyu Chen - -Best Regards, -xxx - -``` - -### 发邮件到 general@incubator.apache.org 进行投票 - -[VOTE] Release Apache Doris 0.9.0-incubating-rc01 - -``` -Hi all, - -Please review and vote on Apache Doris 0.9.0-incubating-rc01 release. - -Apache Doris is an MPP-based interactive SQL data warehousing for reporting and analysis. - -The Apache Doris community has voted on and approved this release: -https://lists.apache.org/thread.html/d70f7c8a8ae448bf6680a15914646005c6483564464cfa15f4ddc2fc@%3Cdev.doris.apache.org%3E - -The vote result email thread: -https://lists.apache.org/thread.html/64d229f0ba15d66adc83306bc8d7b7ccd5910ecb7e842718ce6a61da@%3Cdev.doris.apache.org%3E - -The release candidate has been tagged in GitHub as 0.9.0-rc01, available here: -https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 - -There is no CHANGE LOG file because this is the first release of Apache Doris. -Thanks to everyone who has contributed to this release, and there is a simple release notes can be found here: -https://github.com/apache/incubator-doris/issues/406 - -The artifacts (source, signature and checksum) corresponding to this release candidate can be found here: -https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc01/ - -This has been signed with PGP key 33DBF2E0, corresponding to lide@apache.org. -KEYS file is available here: -https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS -It is also listed here: -https://people.apache.org/keys/committer/lide.asc - -The vote will be open for at least 72 hours. -[ ] +1 Approve the release -[ ] +0 No opinion -[ ] -1 Do not release this package because ... - -To verify and build, you can refer to following instruction: - -Firstly, you must be install and start docker service, and then you could build Doris as following steps: - -Step1: Pull the docker image with Doris building environment -$ docker pull apachedoris/doris-dev:build-env -You can check it by listing images, its size is about 3.28GB. - -Step2: Run the Docker image -You can run image directly: -$ docker run -it apachedoris/doris-dev:build-env - -Step3: Download Doris source -Now you should in docker environment, and you can download Doris source package. -(If you have downloaded source and it is not in image, you can map its path to image in Step2.) -$ wget https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc01/apache-doris-0.9.0.rc01-incubating-src.tar.gz - -Step4: Build Doris -Now you can decompress and enter Doris source path and build Doris. -$ tar zxvf apache-doris-0.9.0.rc01-incubating-src.tar.gz -$ cd apache-doris-0.9.0.rc01-incubating-src -$ sh build.sh - -Best Regards, -xxx - ----- -DISCLAIMER-WIP: -Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), -sponsored by the Apache Incubator. Incubation is required of all newly accepted projects -until a further review indicates that the infrastructure, communications, and decision -making process have stabilized in a manner consistent with other successful ASF projects. -While incubation status is not necessarily a reflection of the completeness or stability -of the code, it does indicate that the project has yet to be fully endorsed by the ASF. - -Some of the incubating project’s releases may not be fully compliant with ASF policy. For -example, releases may have incomplete or un-reviewed licensing conditions. What follows is -a list of known issues the project is currently aware of (note that this list, by definition, -is likely to be incomplete): - - * Releases may have incomplete licensing conditions - -If you are planning to incorporate this work into your product/project, please be aware that -you will need to conduct a thorough licensing review to determine the overall implications of -including this work. For the current status of this project through the Apache Incubator -visit: https://incubator.apache.org/projects/doris.html -``` - -邮件的 thread 连接可以在这里找到： - -`https://lists.apache.org/list.html?dev@doris.apache.org` - - -### 发 Result 邮件到 general@incubator.apache.org - -[RESULT][VOTE] Release Apache Doris 0.9.0-incubating-rc01 - - -``` -Hi, - -Thanks to everyone, and the vote for releasing Apache Doris 0.9.0-incubating-rc01 is now closed. - -It has passed with 4 +1 (binding) votes and no 0 or -1 votes. - -Binding: -+1 Willem Jiang -+1 Justin Mclean -+1 ShaoFeng Shi -+1 Makoto Yui - -The vote thread: -https://lists.apache.org/thread.html/da05fdd8d84e35de527f27200b5690d7811a1e97d419d1ea66562130@%3Cgeneral.incubator.apache.org%3E - -Best Regards, -xxx -``` - -## 完成发布流程 - -### 上传 package 到 release - -当正式发布投票成功后，先发[Result]邮件，然后就准备 release package。 -将之前在dev下发布的对应rc文件夹下的源码包、签名文件和hash文件拷贝到另一个目录 0.9.0-incubating，注意文件名字中不要rcxx (可以rename，但不要重新计算签名，hash可以重新计算，结果不会变) - -第一次发布的话 KEYS 文件也需要拷贝过来。然后add到svn release 下。 - -``` - -https://dist.apache.org/repos/dist/release/incubator/doris/0.9.0-incubating/ - -最终能在 apache 官网看到： -http://www.apache.org/dist/incubator/doris/0.9.0-incubating/ - -``` - -### 发 Announce 邮件到 general@incubator.apache.org - -Title: - -``` -[ANNOUNCE] Apache Doris (incubating) 0.9.0 Release -``` - -发送邮件组： - -``` -general@incubator.apache.org -dev@doris.apache.org -``` - -邮件正文： - -``` -Hi All, - -We are pleased to announce the release of Apache Doris 0.9.0-incubating. - -Apache Doris (incubating) is an MPP-based interactive SQL data warehousing for reporting and analysis. - -The release is available at: -http://doris.apache.org/downloads.html - -Thanks to everyone who has contributed to this release, and the release note can be found here: -https://github.com/apache/incubator-doris/releases - -Best Regards, - -On behalf of the Doris team, -xxx - ----- -DISCLAIMER-WIP: -Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), -sponsored by the Apache Incubator. Incubation is required of all newly accepted projects -until a further review indicates that the infrastructure, communications, and decision -making process have stabilized in a manner consistent with other successful ASF projects. -While incubation status is not necessarily a reflection of the completeness or stability -of the code, it does indicate that the project has yet to be fully endorsed by the ASF. - -Some of the incubating project’s releases may not be fully compliant with ASF policy. For -example, releases may have incomplete or un-reviewed licensing conditions. What follows is -a list of known issues the project is currently aware of (note that this list, by definition, -is likely to be incomplete): - - * Releases may have incomplete licensing conditions - -If you are planning to incorporate this work into your product/project, please be aware that -you will need to conduct a thorough licensing review to determine the overall implications of -including this work. For the current status of this project through the Apache Incubator -visit: https://incubator.apache.org/projects/doris.html -``` - -### 在 Doris 官网和 github 发布链接 - -#### 创建下载链接 - -下载链接： -http://www.apache.org/dyn/closer.cgi?filename=incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz&action=download - -wget --trust-server-names "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz" - -原始位置: -https://www.apache.org/dist/incubator/doris/0.9.0-incubating/ - -http://www.apache.org/dyn/closer.cgi/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz - -源码包（source package）: -http://www.apache.org/dyn/closer.cgi/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz - -ASC: -http://archive.apache.org/dist/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.asc - -sha512: -http://archive.apache.org/dist/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.sha512 - -KEYS: -http://archive.apache.org/dist/incubator/doris/KEYS - -refer to: - -#### 准备 release note - -需要修改如下两个地方： - -1、Github 的 release 页面 - -``` -https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 -``` - -2、Doris 官网下载页面 - -``` -http://doris.apache.org/downloads.html -``` - diff --git a/docs/documentation/cn/community/subscribe-mail-list.md b/docs/documentation/cn/community/subscribe-mail-list.md deleted file mode 100644 index 83623163012e2c..00000000000000 --- a/docs/documentation/cn/community/subscribe-mail-list.md +++ /dev/null @@ -1,62 +0,0 @@ - - -# 订阅邮件列表 - -邮件列表（Mail List）是 Apache 社区最被认可的交流方式。一般来说，开源社区的提问与解答、技术讨论、事务决策等都通过邮件列表来承载。邮件列表异步、广播的特性，也非常适合开源社区的沟通交流。那么，如何订阅 Apache Doris (incubating) 的邮件列表呢？主要包括以下五个步骤。 - -## 1. 发送订阅邮件 - -打开自己的邮箱，新建邮件，向`dev-subscribe@doris.apache.org`发送一封邮件（邮件主题和内容任意） - -![step1](../../../resources/images/subscribe-mail-list-step1.png) - -## 2. 接收来自 dev-help@doris.apache.org 的确认邮件 - -执行完第一步之后，您将收到一封来自`dev-help@doris.apache.org`的确认邮件，邮件内容如下图所示。（**如果长时间未能收到，请确认该邮件是否已被拦截，或已经被自动归入“订阅邮件”、“垃圾邮件”、“推广邮件”等文件夹**） - -![step2](../../../resources/images/subscribe-mail-list-step2.png) - -## 3. 回复确认邮件 - -​针对上一步接收到的邮件， - -​**a.直接回复该邮件** - -​***或*** - -**b. 新建一封`收件人`为上一步中的`回复地址`的邮件** - -​均可，内容主题不限 - -![step3](../../../resources/images/subscribe-mail-list-step3.png) - - -## 4. 接收欢迎邮件 - -​完成第三步之后，将会受到一封标题为**WELCOME to dev@doris.apache.org**的欢迎邮件。至此，订阅邮件列表的工作已经完成了，社区的动态都会通过邮件的方式通知您。 - -![step4](../../../resources/images/subscribe-mail-list-step4.png) - - -## 5. 发起邮件讨论（可选） - -​成功订阅邮件列表后，若想发起讨论，直接往`dev@doris.apache.org`发送邮件即可。所有订阅了邮件列表的人都会收到邮件。 -​ -​ \ No newline at end of file diff --git a/docs/documentation/cn/community/verify-apache-release.md b/docs/documentation/cn/community/verify-apache-release.md deleted file mode 100644 index 9f374541370b3c..00000000000000 --- a/docs/documentation/cn/community/verify-apache-release.md +++ /dev/null @@ -1,81 +0,0 @@ - - -# 验证 Apache 发布版本 - -可以按照以下步骤对发布版本进行验证： - -1. [ ] 下载链接是否合法。 -2. [ ] 校验值和 PGP 签名是否合法。 -3. [ ] 是否包含 DISCLAIMER-WIP。 -4. [ ] 代码是否和当前发布版本相匹配。 -5. [ ] LICENSE 和 NOTICE 文件是否正确。 -6. [ ] 所有文件都携带必要的协议说明。 -7. [ ] 在源码包中不包含已经编译好的内容。 -8. [ ] 编译是否能够顺利执行。 - -## 1. 下载源码包、签名文件、校验值文件和 KEYS - -下载所有相关文件, 以 a.b.c-incubating 为示例: - -``` -wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz - -wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz.sha512 - -wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz.asc - -wget https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS -``` - -## 2. 检查签名和校验值 - -推荐使用 GunPG，可以通过以下命令安装： - -``` -CentOS: yum install gnupg -Ubuntu: apt-get install gnupg -``` - -``` -gpg --import KEYS -gpg --verify apache-doris-a.b.c-incubating-src.tar.gz.asc apache-doris-a.b.c-incubating-src.tar.gz -sha512sum --check apache-doris-a.b.c-incubating-src.tar.gz.sha512 -``` - -## 3. 验证源码协议头 - -推荐使用 Apache RAT 验证源码协议，可以从一下连接下载： - -``` -wget http://mirrors.tuna.tsinghua.edu.cn/apache//creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -tar zxvf apache-rat-0.12-bin.tar.gz -``` - -假设源码目录名称为 apache-doris-a.b.c-incubating-src，可以使用以下命令进行验证。 -这个命令会产生一个文件，其中列举了所有非 ASF 协议的文件。 - -``` -/usr/java/jdk/bin/java -jar apache-rat-0.12/apache-rat-0.12.jar -a -d apache-doris-a.b.c-incubating-src -E apache-doris-a.b.c-incubating-src/.rat-excudes -``` - -## 4. 验证编译 - -详细的编译步骤，请参阅 [编译文档](../installing/compilation.html) - diff --git a/docs/documentation/cn/developer-guide/debug-tool.md b/docs/documentation/cn/developer-guide/debug-tool.md deleted file mode 100644 index c761e78a3ec92c..00000000000000 --- a/docs/documentation/cn/developer-guide/debug-tool.md +++ /dev/null @@ -1,266 +0,0 @@ - - -# 调试工具 - -在Doris的使用、开发过程中，经常会遇到需要对Doris进行调试的场景，这里介绍一些常用的调试工具。 - -## 环境准备 - -[pprof](https://github.com/google/pprof): 来自gperftools，用于将gperftools所产生的内容转化成便于人可以阅读的格式，比如pdf, svg, text等. - -[graphviz](http://www.graphviz.org/): 在没有这个库的时候pprof只可以转化为text格式，但这种方式不易查看。那么安装这个库后，pprof可以转化为svg、pdf等格式，对于调用关系则更加清晰明了。 - -[perf](https://perf.wiki.kernel.org/index.php/Main_Page): linux内核自带性能分析工具。[这里](http://www.brendangregg.com/perf.html)有一些perf的使用例子。 - -[FlameGraph](https://github.com/brendangregg/FlameGraph): 可视化工具，用于将perf的输出以火焰图的形式展示出来。 - -## 内存 - -对于内存的调试一般分为两个方面。一个是内存使用的总量是否合理，内存使用量过大一方面可能是由于系统存在内存泄露，另一方面可能是因为程序内存使用不当。其次就是是否存在内存越界、非法访问的问题，比如程序访问一个非法地址的内存，使用了未初始化内存等。对于内存方面的调试我们一般使用如下几种方式来进行问题追踪。 - -### 查看日志 - -当发现内存使用量过大的时候，我们可以先查看be.out日志，看看是否有大内存申请。由于Doris当前使用的TCMalloc管理内存，那么遇到大内存申请时，都会将申请的堆栈打印到be.out文件中，一般的表现形式如下： - -``` -tcmalloc: large alloc 1396277248 bytes == 0x3f3488000 @ 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed -``` - -这个表示在Doris BE在这个堆栈上尝试申请`1396277248 bytes`的内存。我们可以通过`addr2line`命令去把堆栈还原成我们能够看懂的信，具体的例子如下所示。 - -``` -$ addr2line -e lib/palo_be 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed - -/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1335 -/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1357 -/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.cpp:267 -/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.hpp:86 -/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:239 -/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:213 -thread.cpp:? -``` - -### HEAP PROFILE - -有时内存的申请并不是大内存的申请导致，而是通过小内存不断的堆积导致的。那么就没有办法通过查看日志定位到具体的申请信息，那么就需要通过其他方式来获得信息。 - -这个时候我们可以利用TCMalloc的[HEAPPROFILE](https://gperftools.github.io/gperftools/heapprofile.html)的功能。如果设置了HEAPPROFILE功能，那么我们可以获得进程整体的内存申请使用情况。使用方式是在启动Doris BE前设置`HEAPPROFILE`环境变量。比如： - -``` -export HEAPPROFILE=/tmp/doris_be.hprof -./bin/start_be.sh --daemon -``` - -这样，当满足HEAPPROFILE的dump条件时，就会将内存的整体使用情况写到指定路径的文件中。后续我们就可以通过使用`pprof`工具来对输出的内容进行分析。 - -``` -$ pprof --text lib/palo_be /tmp/doris_be.hprof.0012.heap | head -30 - -Using local file lib/palo_be. -Using local file /tmp/doris_be.hprof.0012.heap. -Total: 668.6 MB - 610.6 91.3% 91.3% 610.6 91.3% doris::SystemAllocator::allocate_via_malloc (inline) - 18.1 2.7% 94.0% 18.1 2.7% _objalloc_alloc - 5.6 0.8% 94.9% 63.4 9.5% doris::RowBatch::RowBatch - 5.1 0.8% 95.6% 7.1 1.1% butil::ResourcePool::add_block (inline) - 3.7 0.5% 96.2% 3.7 0.5% butil::iobuf::create_block (inline) - 3.4 0.5% 96.7% 3.4 0.5% butil::FlatMap::init - 3.2 0.5% 97.2% 5.2 0.8% butil::ObjectPool::add_block (inline) - 2.6 0.4% 97.6% 2.6 0.4% __gnu_cxx::new_allocator::allocate (inline) - 2.0 0.3% 97.9% 2.0 0.3% butil::ObjectPool::add_block_group (inline) - 2.0 0.3% 98.2% 2.0 0.3% butil::ResourcePool::add_block_group (inline) - 1.7 0.3% 98.4% 1.7 0.3% doris::SegmentReader::_load_index -``` - -上述文件各个列的内容： - -* 第一列：函数直接申请的内存大小，单位MB -* 第四列：函数以及函数所有调用的函数总共内存大小。 -* 第二列、第五列分别是第一列与第四列的的比例值。 -* 第三列是个第二列的累积值。 - -当然也可以生成调用关系图片，更加方便分析。比如下面的命令就能够生成SVG格式的调用关系图。 - -``` -pprof --svg lib/palo_be /tmp/doris_be.hprof.0012.heap > heap.svg -``` - -**注意：开启这个选项是要影响程序的执行性能的，请慎重对线上的实例开启** - -### pprof remote server - -HEAPPROFILE虽然能够获得全部的内存使用信息，但是也有比较受限的地方。1. 需要重启BE进行。2. 需要一直开启这个命令，导致对整个进程的性能造成影响。 - -对Doris BE也可以使用动态开启、关闭heap profile的方式来对进程进行内存申请分析。Doris内部支持了GPerftools的[远程server调试](https://gperftools.github.io/gperftools/pprof_remote_servers.html)。那么可以通过`pprof`直接对远程运行的Doris BE进行动态的HEAP PROFILE。比如我们可以通过以下命令来查看Doris的内存的使用增量 - -``` -$ pprof --text --seconds=60 http://be_host:be_webport/pprof/heap - -Total: 1296.4 MB - 484.9 37.4% 37.4% 484.9 37.4% doris::StorageByteBuffer::create - 272.2 21.0% 58.4% 273.3 21.1% doris::RowBlock::init - 157.5 12.1% 70.5% 157.5 12.1% doris::RowBatch::RowBatch - 90.7 7.0% 77.5% 90.7 7.0% doris::SystemAllocator::allocate_via_malloc - 66.6 5.1% 82.7% 66.6 5.1% doris::IntegerColumnReader::init - 47.9 3.7% 86.4% 47.9 3.7% __gnu_cxx::new_allocator::allocate - 20.8 1.6% 88.0% 35.4 2.7% doris::SegmentReader::_load_index - 12.7 1.0% 89.0% 12.7 1.0% doris::DecimalColumnReader::init - 12.7 1.0% 89.9% 12.7 1.0% doris::LargeIntColumnReader::init - 12.7 1.0% 90.9% 12.7 1.0% doris::StringColumnDirectReader::init - 12.3 0.9% 91.9% 12.3 0.9% std::__cxx11::basic_string::_M_mutate - 10.4 0.8% 92.7% 10.4 0.8% doris::VectorizedRowBatch::VectorizedRowBatch - 10.0 0.8% 93.4% 10.0 0.8% doris::PlainTextLineReader::PlainTextLineReader -``` - -这个命令的输出与HEAP PROFILE的输出及查看方式一样，这里就不再详细说明。这个命令只有在执行的过程中才会开启统计，相比HEAP PROFILE对于进程性能的影响有限。 - -### LSAN - -[LSAN](https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer)是一个地址检查工具，GCC已经集成。在我们编译代码的时候开启相应的编译选项，就能够开启这个功能。当程序发生可以确定的内存泄露时，会将泄露堆栈打印。Doris BE已经集成了这个工具，只需要在编译的时候使用如下的命令进行编译就能够生成带有内存泄露检测版本的BE二进制 - -``` -BUILD_TYPE=LSAN ./build.sh -``` - -当系统检测到内存泄露的时候，就会在be.out里面输出对应的信息。为了下面的演示，我们故意在代码中插入一段内存泄露代码。我们在`StorageEngine`的`open`函数中插入如下代码 - -``` - char* leak_buf = new char[1024]; - strcpy(leak_buf, "hello world"); - LOG(INFO) << leak_buf; -``` - -我们就在be.out中获得了如下的输出 - -``` -================================================================= -==24732==ERROR: LeakSanitizer: detected memory leaks - -Direct leak of 1024 byte(s) in 1 object(s) allocated from: - #0 0xd10586 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/lsan/lsan_interceptors.cc:164 - #1 0xe333a2 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104 - #2 0xd3cc96 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 - #3 0x7f573b5eebd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) - -SUMMARY: LeakSanitizer: 1024 byte(s) leaked in 1 allocation(s). -``` - -从上述的输出中，我们能看到有1024个字节被泄露了，并且打印出来了内存申请时的堆栈信息。 - -**注意：开启这个选项是要影响程序的执行性能的，请慎重对线上的实例开启** - -**注意：如果开启了LSAN开关的话，tcmalloc就会被自动关闭** - -### ASAN - -除了内存使用不合理、泄露以外。有的时候也会发生内存访问非法地址等错误。这个时候我们可以借助[ASAN](https://github.com/google/sanitizers/wiki/AddressSanitizer)来辅助我们找到问题的原因。与LSAN一样，ASAN也集成在了GCC中。Doris通过如下的方式进行编译就能够开启这个功能 - -``` -BUILD_TYPE=ASAN ./build.sh -``` - -执行编译生成的二进制文件，当检测工具发现有异常访问时，就会立即退出，并将非法访问的堆栈输出在be.out中。对于ASAN的输出与LSAN是一样的分析方法。这里我们也主动注入一个地址访问错误，来展示下具体的内容输出。我们仍然在`StorageEngine`的`open`函数中注入一段非法内存访问，具体的错误代码如下 - -``` - char* invalid_buf = new char[1024]; - for (int i = 0; i < 1025; ++i) { - invalid_buf[i] = i; - } - LOG(INFO) << invalid_buf; -``` - -然后我们就会在be.out中获得如下的输出 - -``` -================================================================= -==23284==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x61900008bf80 at pc 0x00000129f56a bp 0x7fff546eed90 sp 0x7fff546eed88 -WRITE of size 1 at 0x61900008bf80 thread T0 - #0 0x129f569 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 - #1 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 - #2 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) - #3 0xd30794 (/home/ssd0/zc/palo/doris/core/output3/be/lib/palo_be+0xd30794) - -0x61900008bf80 is located 0 bytes to the right of 1024-byte region [0x61900008bb80,0x61900008bf80) -allocated by thread T0 here: - #0 0xdeb040 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/asan/asan_new_delete.cc:82 - #1 0x129f50d in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104 - #2 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 - #3 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) - -SUMMARY: AddressSanitizer: heap-buffer-overflow /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) -``` - -从这段信息中该可以看到在`0x61900008bf80`这个地址我们尝试去写一个字节，但是这个地址是非法的。我们也可以看到 `[0x61900008bb80,0x61900008bf80)`这个地址的申请堆栈。 - -**注意：开启这个选项是要影响程序的执行性能的，请慎重对线上的实例开启** - -**注意：如果开启了ASAN开关的话，tcmalloc就会被自动关闭** - -另外，如果be.out中输出了堆栈信息，但是并没有函数符号，那么这个时候需要我们手动的处理下才能获得可读的堆栈信息。具体的处理方法需要借助一个脚本来解析ASAN的输出。这个时候我们需要使用[asan_symbolize](https://llvm.org/svn/llvm-project/compiler-rt/trunk/lib/asan/scripts/asan_symbolize.py)来帮忙解析下。具体的使用方式如下： - -``` -cat be.out | python asan_symbolize.py | c++filt -``` - -通过上述的命令，我们就能够获得可读的堆栈信息了。 - -## CPU - -当系统的CPU Idle很低的时候，说明系统的CPU已经成为了主要瓶颈，这个时候就需要分析一下当前的CPU使用情况。对于Doris的BE可以有如下两种方式来分析Doris的CPU瓶颈。 - -### pprof - -由于Doris内部已经集成了并兼容了GPerf的REST接口，那么用户可以通过`pprof`工具来分析远程的Doris BE。具体的使用方式如下： - -``` -pprof --svg --seconds=60 http://be_host:be_webport/pprof/profile > be.svg -``` - -这样就能够生成一张BE执行的CPU消耗图。 - -![CPU Pprof](../../../resources/images/cpu-pprof-demo.png) - -### perf + flamegragh - -这个是相当通用的一种CPU分析方式，相比于`pprof`，这种方式必须要求能够登陆到分析对象的物理机上。但是相比于pprof只能定时采点，perf是能够通过不同的事件来完成堆栈信息采集的。具体的的使用方式如下： - -``` -perf record -g -p be_pid -- sleep 60 -``` - -这条命令会统计60秒钟BE的CPU运行情况，并且生成perf.data。对于perf.data的分析，可以通过perf的命令来进行分析 - -``` -perf report -``` - -分析得到如下的图片 - -![Perf Report](../../../resources/images/perf-report-demo.png) - -来对生成的内容进行分析。当然也可以使用flamegragh完成可视化展示。 - -``` -perf script | ./FlameGraph/stackcollapse-perf.pl | ./FlameGraph/flamegraph.pl > be.svg -``` - -这样也会生成一张当时运行的CPU消耗图。 - -![CPU Flame](../../../resources/images/cpu-flame-demo.svg) diff --git a/docs/documentation/cn/developer-guide/format-code.md b/docs/documentation/cn/developer-guide/format-code.md deleted file mode 100644 index bfa3a37c40ff2d..00000000000000 --- a/docs/documentation/cn/developer-guide/format-code.md +++ /dev/null @@ -1,66 +0,0 @@ - - -# 代码格式化 -为了自动格式化代码，推荐使用clang-format进行代码格式化。 - -## 代码风格定制 -Doris的代码风格在Google Style的基础上稍有改动，定制为.clang-format文件，位于Doris根目录。 - -目前，.clang-format配置文件适配clang-format-8.0.1以上的版本。 - -## 环境准备 -需要下载安装clang-format，也可使用IDE或Editor提供的clang-format插件，下面分别介绍。 - -### 下载安装clang-format -Ubuntu: `apt-get install clang-format` - -当前版本为10.0，也可指定旧版本，例如: `apt-get install clang-format-9` - -Centos 7: - -centos yum安装的clang-format版本过老，支持的StyleOption太少，建议源码编译10.0版本。 - -### clang-format插件 -Clion IDE可使用插件"ClangFormat"，`File->Setting->Plugins`搜索下载。但版本无法和 -clang-format程序的版本匹配，从支持的StyleOption上看，应该是低于clang-format-9.0。 - -## 使用方式 - -### 命令行运行 -`clang-format --style=file -i $File$` - -`--sytle=file`就会自动找到.clang-format文件，根据文件Option配置来格式化代码。 - -批量文件clang-format时，需注意过滤不应该格式化的文件。例如，只格式化*.h/*.cpp，并排除某些文件夹： - -`find . -type f -not \( -wholename ./env/* \) -regextype posix-egrep -regex - ".*\.(cpp|h)" | xargs clang-format -i -style=file` - -### 在IDE或Editor中使用clang-format -#### Clion -Clion如果使用插件，点击`Reformat Code`即可。 -#### VS Code -VS Code需安装扩展程序Clang-Format，但需要自行提供clang-format执行程序的位置。 - -``` -"clang-format.executable": "$clang-format path$", -"clang-format.style": "file" -``` -然后，点击`Format Document`即可。 \ No newline at end of file diff --git a/docs/documentation/cn/developer-guide/index.rst b/docs/documentation/cn/developer-guide/index.rst deleted file mode 100644 index ba6bb38a48785c..00000000000000 --- a/docs/documentation/cn/developer-guide/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -开发者手册 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/downloads/downloads.md b/docs/documentation/cn/downloads/downloads.md deleted file mode 100644 index 524e38d4ecf4da..00000000000000 --- a/docs/documentation/cn/downloads/downloads.md +++ /dev/null @@ -1,14 +0,0 @@ -# 下载 - -您可以通过以下连接下载对应版本的 Doris 源码进行编译和部署。 - -| 版本 | 发布日期 | 从镜像网站下载 | -|---|---|---| -| 0.12.0 | 2020-04-24 | [Source](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz.sha512)) | -| 0.11.0 | 2019-11-29 | [Source](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz.sha512)) | -| 0.10.0 | 2019-07-02 | [Source](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz.sha512)) | -| 0.9.0 | 2019-02-18 | [Source](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.sha512)) | - -关于如何校验下载文件，请参阅 [校验下载文件](../community/verify-apache-release.html)。 - -校验完成后，可以参阅 [编译文档](../installing/compilation.html) 以及 [安装与部署文档](../installing/install-deploy.html) 进行 Doris 的编译、安装与部署。 diff --git a/docs/documentation/cn/downloads/index.rst b/docs/documentation/cn/downloads/index.rst deleted file mode 100644 index 0a7331c37c3b93..00000000000000 --- a/docs/documentation/cn/downloads/index.rst +++ /dev/null @@ -1,10 +0,0 @@ -============= -下载 -============= - -.. toctree:: - :maxdepth: 1 - :glob: - - * - diff --git a/docs/documentation/cn/extending-doris/audit-plugin.md b/docs/documentation/cn/extending-doris/audit-plugin.md deleted file mode 100644 index 79bf5c44f5fa35..00000000000000 --- a/docs/documentation/cn/extending-doris/audit-plugin.md +++ /dev/null @@ -1,108 +0,0 @@ - - -# 审计日志插件 - -Doris 的审计日志插件是在 FE 的插件框架基础上开发的。是一个可选插件。用户可以在运行时安装或卸载这个插件。 - -该插件可以将 FE 的审计日志定期的导入到指定 Doris 集群中，以方便用户通过 SQL 对审计日志进行查看和分析。 - -## 编译、配置和部署 - -### 编译 - -在 Doris 代码目录下执行 `sh build_plugin.sh` 后，会在 `fe_plugins/output` 目录下得到 `auditloader.zip` 文件。 - -### 配置 - -解压 `auditloader.zip` 可以看到三个文件： - -``` -auditloader.jar -plugin.properties -plugin.conf -``` - -打开 `plugin.conf` 进行配置。配置项说明参见注释。 - -配置完成后，重新将三个文件打包为 `auditloader.zip`. - -### 部署 - -您可以将这个文件放置在一个 http 下载服务器上，或者拷贝到所有 FE 的指定目录下。这里我们使用后者。 - -### 安装 - -部署完成后，安装插件前，需要创建之前在 `plugin.conf` 中指定的审计数据库和表。其中建表语句如下： - -``` -create table doris_audit_tbl__ -( - query_id varchar(48) comment "Unique query id", - time datetime not null comment "Query start time", - client_ip varchar(32) comment "Client IP", - user varchar(64) comment "User name", - db varchar(96) comment "Database of this query", - state varchar(8) comment "Query result state. EOF, ERR, OK", - query_time bigint comment "Query execution time in millisecond", - scan_bytes bigint comment "Total scan bytes of this query", - scan_rows bigint comment "Total scan rows of this query", - return_rows bigint comment "Returned rows of this query", - stmt_id int comment "An incremental id of statement", - is_query tinyint comment "Is this statemt a query. 1 or 0", - frontend_ip varchar(32) comment "Frontend ip of executing this statement", - stmt varchar(2048) comment "The original statement, trimed if longer than 2048 bytes" -) -partition by range(time) () -distributed by hash(query_id) buckets 1 -properties( - "dynamic_partition.time_unit" = "DAY", - "dynamic_partition.start" = "-30", - "dynamic_partition.end" = "3", - "dynamic_partition.prefix" = "p", - "dynamic_partition.buckets" = "1", - "dynamic_partition.enable" = "true", - "replication_num" = "1" -); -``` - -其中 `dynamic_partition` 属性根据自己的需要，选择审计日志安保留的天数。 - -之后，连接到 Doris 后使用 `INSTALL PLUGIN` 命令完成安装。安装成功后，可以通过 `SHOW PLUGINS` 看到已经安装的插件，并且状态为 `INSTALLED`。 - -完成后，插件会不断的以指定的时间间隔将审计日志插入到这个表中。 - - - - - - - - - - - - - - - - - - - diff --git a/docs/documentation/cn/extending-doris/doris-on-es.md b/docs/documentation/cn/extending-doris/doris-on-es.md deleted file mode 100644 index 7a41c979c0d8dd..00000000000000 --- a/docs/documentation/cn/extending-doris/doris-on-es.md +++ /dev/null @@ -1,224 +0,0 @@ - - -# Doris On ES - -Doris-On-ES将Doris的分布式查询规划能力和ES(Elasticsearch)的全文检索能力相结合，提供更完善的OLAP分析场景解决方案： - - 1. ES中的多index分布式Join查询 - 2. Doris和ES中的表联合查询，更复杂的全文检索过滤 - 3. ES keyword类型字段的聚合查询：适用于index 频繁发生变化、单个分片文档数量千万级以上且该字段基数(cardinality)非常大 - -本文档主要介绍该功能的实现原理、使用方式等。 - -## 名词解释 - -* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 -* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 -* Elasticsearch(ES)：目前最流行的开源分布式搜索引擎。 -* DataNode：ES的数据存储与计算节点。 -* MasterNode：ES的Master节点，管理元数据、节点、数据分布等。 -* scroll：ES内置的数据集游标特性，用来对数据进行流式扫描和过滤。 - - -## 如何使用 - -### 创建外表 - -``` -CREATE EXTERNAL TABLE `es_table` ( - `id` bigint(20) COMMENT "", - `k1` bigint(20) COMMENT "", - `k2` datetime COMMENT "", - `k3` varchar(20) COMMENT "", - `k4` varchar(100) COMMENT "", - `k5` float COMMENT "" -) ENGINE=ELASTICSEARCH -PARTITION BY RANGE(`id`) -() -PROPERTIES ( -"host" = "http://192.168.0.1:8200,http://192.168.0.2:8200", -"user" = "root", -"password" = "root", -"index" = "tindex”, -"type" = "doc" -); -``` - -参数说明： - -参数 | 说明 ----|--- -host | ES集群连接地址，可指定一个或多个，Doris通过这个地址获取到ES版本号、index的shard分布信息 -user | 开启basic认证的ES集群的用户名，需要确保该用户有访问: /\_cluster/state/\_nodes/http等路径权限和对index的读权限 -password | 对应用户的密码信息 -index | Doris中的表对应的ES的index名字，可以是alias -type | 指定index的type，默认是_doc -transport | 内部保留，默认为http - -### 查询 - -#### 基本条件过滤 - -``` -select * from es_table where k1 > 1000 and k3 ='term' or k4 like 'fu*z_' -``` - -#### 扩展的esquery sql语法 -通过`esquery`函数将一些无法用sql表述的ES query如match、geoshape等下推给ES进行过滤处理，`esquery`的第一个列名参数用于关联`index`，第二个参数是ES的基本`Query DSL`的json表述，使用花括号`{}`包含，json的`root key`有且只能有一个，如match、geo_shape、bool等 - -match查询： - -``` -select * from es_table where esquery(k4, '{ - "match": { - "k4": "doris on elasticsearch" - } - }'); -``` -geo相关查询： - -``` -select * from es_table where esquery(k4, '{ - "geo_shape": { - "location": { - "shape": { - "type": "envelope", - "coordinates": [ - [ - 13, - 53 - ], - [ - 14, - 52 - ] - ] - }, - "relation": "within" - } - } - }'); -``` - -bool查询： - -``` -select * from es_table where esquery(k4, ' { - "bool": { - "must": [ - { - "terms": { - "k1": [ - 11, - 12 - ] - } - }, - { - "terms": { - "k2": [ - 100 - ] - } - } - ] - } - }'); -``` - - - -## 原理 - -``` -+----------------------------------------------+ -| | -| Doris +------------------+ | -| | FE +--------------+-------+ -| | | Request Shard Location -| +--+-------------+-+ | | -| ^ ^ | | -| | | | | -| +-------------------+ +------------------+ | | -| | | | | | | | | -| | +----------+----+ | | +--+-----------+ | | | -| | | BE | | | | BE | | | | -| | +---------------+ | | +--------------+ | | | -+----------------------------------------------+ | - | | | | | | | - | | | | | | | - | HTTP SCROLL | | HTTP SCROLL | | -+-----------+---------------------+------------+ | -| | v | | v | | | -| | +------+--------+ | | +------+-------+ | | | -| | | | | | | | | | | -| | | DataNode | | | | DataNode +<-----------+ -| | | | | | | | | | | -| | | +<--------------------------------+ -| | +---------------+ | | |--------------| | | | -| +-------------------+ +------------------+ | | -| Same Physical Node | | -| | | -| +-----------------------+ | | -| | | | | -| | MasterNode +<-----------------+ -| ES | | | -| +-----------------------+ | -+----------------------------------------------+ - - -``` - -1. 创建ES外表后，FE会请求建表指定的主机，获取所有节点的HTTP端口信息以及index的shard分布信息等，如果请求失败会顺序遍历host列表直至成功或完全失败 - -2. 查询时，会根据FE得到的一些节点信息和index的元数据信息，生成查询计划并发给对应的BE节点 - -3. BE节点会根据`就近原则`即优先请求本地部署的ES节点，BE通过`HTTP Scroll`方式流式的从ES index的每个分片中并发的获取数据 - -4. 计算完结果后，返回给client端 - -## Push-Down operations -`Doris On Elasticsearch`一个重要的功能就是过滤条件的下推: 过滤条件下推给ES，这样只有真正满足条件的数据才会被返回，能够显著的提高查询性能和降低Doris和Elasticsearch的CPU、memory、IO利用率 - -下面的操作符(Operators)会被优化成如下下推filters: - -| SQL syntax | ES 5.x+ syntax | -|-------|:---:| -| = | term query| -| in | terms query | -| > , < , >= , ⇐ | range | -| and | bool.filter | -| or | bool.should | -| not | bool.must_not | -| not in | bool.must_not + terms | -| esquery | ES Query DSL | - - -## 其他说明 - -1. ES的版本要求 - - ES主版本大于5，ES在2.x之前和5.x之后数据的扫描方式不同，目前支持5.x之后的 -2. 是否支持X-Pack认证的ES集群 - - 支持所有使用HTTP Basic认证方式的ES集群 -3. 一些查询比请求ES慢很多 - - 是，比如_count相关的query等，ES内部会直接读取满足条件的文档个数相关的元数据，不需要对真实的数据进行过滤 diff --git a/docs/documentation/cn/extending-doris/index.rst b/docs/documentation/cn/extending-doris/index.rst deleted file mode 100644 index 4fda2a01dfaa58..00000000000000 --- a/docs/documentation/cn/extending-doris/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -=========== -扩展功能 -=========== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/extending-doris/plugin-development-manual.md b/docs/documentation/cn/extending-doris/plugin-development-manual.md deleted file mode 100644 index 63e12c67e8aa2d..00000000000000 --- a/docs/documentation/cn/extending-doris/plugin-development-manual.md +++ /dev/null @@ -1,291 +0,0 @@ - - -# 插件开发手册 - -## 介绍 - -Doris 支持动态加载插件。用户可以通过开发自己的插件来扩展Doris的功能。这个手册主要介绍如何开发、编译和部署 Frontend 端的插件。 - -`fe_plugins` 目录是 FE 插件的根模块。这个根模块统一管理插件所需的依赖。添加一个新的插件，相当于在这个根模块添加一个子模块。 - -## 插件组成 - -一个FE的插件可以使一个**zip压缩包**或者是一个**目录**。其内容至少包含两个文件：`plugin.properties` 和 `.jar` 文件。`plugin.properties`用于描述插件信息。 - -文件结构如下： - -``` -# plugin .zip -auditodemo.zip: - -plugin.properties - -auditdemo.jar - -xxx.config - -data/ - -test_data/ - -# plugin local directory -auditodemo/: - -plugin.properties - -auditdemo.jar - -xxx.config - -data/ - -test_data/ -``` - -`plugin.properties` 内容示例: - -``` -### required: -# -# the plugin name -name = audit_plugin_demo -# -# the plugin type -type = AUDIT -# -# simple summary of the plugin -description = just for test -# -# Doris's version, like: 0.11.0 -version = 0.11.0 - -### FE-Plugin optional: -# -# version of java the code is built against -# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4 -java.version = 1.8.31 -# -# the name of the class to load, fully-qualified. -classname = AuditPluginDemo - -### BE-Plugin optional: -# the name of the so to load -soName = example.so -``` - -## 编写一个插件 - -插件的开发环境依赖Doris的开发编译环境。所以请先确保Doris的编译开发环境运行正常。 - -### 创建一个模块 - -我们可以通过以下命令在 `fe_plugins` 目录创建一个子模块用户实现创建和创建工程。其中 `doris-fe-test` 为插件名称。 - -``` -mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false -``` - -这个命令会创建一个新的 maven 工程，并且自动向 `fe_plugins/pom.xml` 中添加一个子模块： - -``` -    ..... -    org.apache -    doris-fe-plugins -    pom -    1.0-SNAPSHOT -     -        auditdemo -        # new plugin module -        doris-fe-test -     -    ..... -``` - -新的工程目录结构如下： - -``` --doris-fe-test/ --pom.xml --src/ - ---- main/java/org/apache/ - ------- App.java # mvn auto generate, ignore - ---- test/java/org/apache -``` - -接下来我们在 `main` 目录下添加一个 `assembly` 目录来存放 `plugin.properties` 和 `zip.xml`。最终的工程目录结构如下： - -``` --doris-fe-test/ --pom.xml --src/ ----- main/ ------- assembly/ --------- plugin.properties --------- zip.xml ------- java/org/apache/ ---------App.java # mvn auto generate, ignore ----- test/java/org/apache -``` - -### 添加 zip.xml - -`zip.xml` 用于描述最终生成的 zip 压缩包中的文件内容。（如 .jar file, plugin.properties 等等） - -``` - -    plugin -     -        zip -     -     -    false -     -         -            target -             -                *.jar -             -            / -         - -         -            src/main/assembly -             -                plugin.properties -             -            / -         -     - -``` - -### 更新 pom.xml - -接下来我们需要更新子模块的 `pom.xml` 文件，添加 doris-fe 依赖： - -``` - - - - org.apache - doris-fe-plugins - 1.0-SNAPSHOT - - 4.0.0 - - auditloader - jar - - - - org.apache - doris-fe - - - - - ... - - - - - - auditloader - - - maven-assembly-plugin - 2.4.1 - - false - - src/main/assembly/zip.xml - - - - - make-assembly - package - - single - - - - - - - - -``` - -### 实现插件 - -之后我们就可以开始愉快的进行插件功能的开发啦。插件需要实现 `Plugin` 接口。具体可以参阅 Doris 自带的 `auditdemo` 插件示例代码。 - -### 编译 - -在编译插件之前，需要先执行 `sh build.sh --fe` 进行 Doris FE 代码的编译，并确保编译成功。 - -之后，执行 `sh build_plugin.sh` 编译所有插件。最终的产出会存放在 `fe_plugins/output` 目录中。 - -或者也可以执行 `sh build_plugin.sh --plugin your_plugin_name` 来仅编译指定的插件。 - -### 另一种开发方式 - -您可以直接通过修改自带的 `auditdemo` 插件示例代码进行开发。 - -## 部署 - -插件可以通过以下三种方式部署。 - -* 将 `.zip` 文件放在 Http 或 Https 服务器上。如：`http://xxx.xxxxxx.com/data/plugin.zip`, Doris 会下载这个文件。 -* 本地 `.zip` 文件。 如：`/home/work/data/plugin.zip`。需要在所有 FE 和 BE 节点部署。 -* 本地目录。如：`/home/work/data/plugin/`。这个相当于 `.zip` 文件解压后的目录。需要在所有 FE 和 BE 节点部署。 - -注意：需保证部署路径在整个插件生命周期内有效。 - -## 安装和卸载插件 - -通过如下命令安装和卸载插件。更多帮助请参阅 `HELP INSTALL PLUGIN;` `HELP IUNNSTALL PLUGIN;` `HELP SHOW PLUGINS;` - -``` -mysql> install plugin from "/home/users/seaven/auditdemo.zip"; -Query OK, 0 rows affected (0.09 sec) - -mysql> mysql> show plugins\G -*************************** 1. row *************************** - Name: auditloader - Type: AUDIT -Description: load audit log to olap load, and user can view the statistic of queries - Version: 0.12.0 -JavaVersion: 1.8.31 - ClassName: AuditLoaderPlugin - SoName: NULL - Sources: /home/cmy/git/doris/core/fe_plugins/output/auditloader.zip - Status: INSTALLED -*************************** 2. row *************************** - Name: AuditLogBuilder - Type: AUDIT -Description: builtin audit logger - Version: 0.12.0 -JavaVersion: 1.8.31 - ClassName: org.apache.doris.qe.AuditLogBuilder - SoName: NULL - Sources: Builtin - Status: INSTALLED -2 rows in set (0.00 sec) - -mysql> uninstall plugin auditloader; -Query OK, 0 rows affected (0.05 sec) - -mysql> show plugins; -Empty set (0.00 sec) -``` diff --git a/docs/documentation/cn/extending-doris/user-defined-function.md b/docs/documentation/cn/extending-doris/user-defined-function.md deleted file mode 100644 index eb489dda317f4c..00000000000000 --- a/docs/documentation/cn/extending-doris/user-defined-function.md +++ /dev/null @@ -1,112 +0,0 @@ - - -# User Define Function - -用户可以通过UDF机制来扩展Doris的能力。通过这篇文档，用户能够创建自己的UDF。 - -## 编写UDF函数 - -在使用UDF之前，用户需要先在Doris的UDF框架下，编写自己的UDF函数。在`be/src/udf_samples/udf_sample.h|cpp`文件中是一个简单的UDF Demo。 - -编写一个UDF函数需要以下几个步骤。 - -### 编写函数 - -创建对应的头文件、CPP文件，在CPP文件中实现你需要的逻辑。CPP文件中的实现函数格式与UDF的对应关系。 - -#### 非可变参数 - -对于非可变参数的UDF，那么两者之间的对应关系很直接。 -比如`INT MyADD(INT, INT)`的UDF就会对应`IntVal AddUdf(FunctionContext* context, const IntVal& arg1, const IntVal& arg2)`。 - -1. `AddUdf`可以为任意的名字，只要创建UDF的时候指定即可。 -2. 实现函数中的第一个参数永远是`FunctionContext*`。实现者可以通过这个结构体获得一些查询相关的内容，以及申请一些需要使用的内存。具体使用的接口可以参考`udf/udf.h`中的定义。 -3. 实现函数中从第二个参数开始需要与UDF的参数一一对应，比如`IntVal`对应`INT`类型。这部分的类型都要使用`const`引用。 -4. 返回参数与UDF的参数的类型要相对应。 - -#### 可变参数 - -对于可变参数，可以参见以下例子，UDF`String md5sum(String, ...)`对应的 -实现函数是`StringVal md5sumUdf(FunctionContext* ctx, int num_args, const StringVal* args)` - -1. `md5sumUdf`这个也是可以任意改变的，创建的时候指定即可。 -2. 第一个参数与非可变参数函数一样，传入的是一个`FunctionContext*`。 -3. 可变参数部分由两部分组成，首先会传入一个整数，说明后面还有几个参数。后面传入的是一个可变参数部分的数组。 - -#### 类型对应关系 - -|UDF Type|Argument Type| -|----|---------| -|TinyInt|TinyIntVal| -|SmallInt|SmallIntVal| -|Int|IntVal| -|BigInt|BigIntVal| -|LargeInt|LargeIntVal| -|Float|FloatVal| -|Double|DoubleVal| -|Date|DateTimeVal| -|Datetime|DateTimeVal| -|Char|StringVal| -|Varchar|StringVal| -|Decimal|DecimalVal| - -## 编译UDF函数 - -### 编译Doris - -在Doris根目录下执行`sh build.sh`就会在`output/udf/`生成对应`headers|libs` - -### 编写CMakeLists.txt - -基于上一步生成的`headers|libs`，用户可以使用`CMakeLists`等工具引入该依赖；在`CMakeLists`中，可以通过向`CMAKE_CXX_FLAGS`添加`-I|L`分别指定`headers|libs`路径；然后使用`add_library`添加动态库。例如，在`be/src/udf_samples/CMakeLists.txt`中，使用`add_library(udfsample SHARED udf_sample.cpp)` `target_link_libraries`(udfsample -static-libstdc++ -static-libgcc)增加了一个`udfsample`动态库。后面需要写上涉及的所有源文件（不包含头文件）。 - -### 执行编译 - -在该目录下创建一个`build`目录并在`build`下执行`cmake ../`生成`Makefile`，并执行`make`就会生成对应动态库。 - -## 创建UDF函数 - -通过上述的步骤后，你可以得到一个动态库。你需要将这个动态库放到一个能够通过HTTP协议访问到的位置。然后执行创建UDF函数在Doris系统内部创建一个UDF，你需要拥有AMDIN权限才能够完成这个操作。 - -``` -CREATE [AGGREGATE] FUNCTION - name ([argtype][,...]) - [RETURNS] rettype - PROPERTIES (["key"="value"][,...]) -``` -说明： - -1. PROPERTIES中`symbol`表示的是，执行入口函数的对应symbol，这个参数是必须设定。你可以通过`nm`命令来获得对应的symbol，比如`nm libudfsample.so | grep AddUdf`获得到的`_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_`就是对应的symbol。 -2. PROPERTIES中`object_file`表示的是从哪里能够下载到对应的动态库，这个参数是必须设定的。 -3. name: 一个function是要归属于某个DB的，name的形式为`dbName`.`funcName`。当`dbName`没有明确指定的时候，就是使用当前session所在的db作为`dbName`。 - -具体使用可以参见 `CREATE FUNCTION` 获取更详细信息。 - -## 使用UDF - -用户使用UDF/UDAF必须拥有对应数据库的 `SELECT` 权限。 - -UDF的使用与普通的函数方式一致，唯一的区别在于，内置函数的作用域是全局的，而UDF的作用域是DB内部。当链接session位于数据内部时，直接使用UDF名字会在当前DB内部查找对应的UDF。否则用户需要显示的指定UDF的数据库名字，例如`dbName`.`funcName`。 - - -## 删除UDF函数 - -当你不再需要UDF函数时，你可以通过下述命令来删除一个UDF函数, 可以参考 `DROP FUNCTION`。 - diff --git a/docs/documentation/cn/getting-started/advance-usage.md b/docs/documentation/cn/getting-started/advance-usage.md deleted file mode 100644 index e02b9de920e140..00000000000000 --- a/docs/documentation/cn/getting-started/advance-usage.md +++ /dev/null @@ -1,265 +0,0 @@ - - -# 高级使用指南 - -这里我们介绍 Doris 的一些高级特性。 - -## 1 表结构变更 - -使用 ALTER TABLE 命令可以修改表的 Schema，包括如下修改： - -* 增加列 -* 删除列 -* 修改列类型 -* 改变列顺序 - -以下举例说明。 - -原表 table1 的 Schema 如下: - -``` -+----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+----------+-------------+------+-------+---------+-------+ -| siteid | int(11) | No | true | 10 | | -| citycode | smallint(6) | No | true | N/A | | -| username | varchar(32) | No | true | | | -| pv | bigint(20) | No | false | 0 | SUM | -+----------+-------------+------+-------+---------+-------+ -``` - -我们新增一列 uv，类型为 BIGINT，聚合类型为 SUM，默认值为 0: - -`ALTER TABLE table1 ADD COLUMN uv BIGINT SUM DEFAULT '0' after pv;` - -提交成功后，可以通过以下命令查看作业进度: - -`SHOW ALTER TABLE COLUMN;` - -当作业状态为 FINISHED，则表示作业完成。新的 Schema 已生效。 - -ALTER TABLE 完成之后, 可以通过 `DESC TABLE` 查看最新的 Schema。 - -``` -mysql> DESC table1; -+----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+----------+-------------+------+-------+---------+-------+ -| siteid | int(11) | No | true | 10 | | -| citycode | smallint(6) | No | true | N/A | | -| username | varchar(32) | No | true | | | -| pv | bigint(20) | No | false | 0 | SUM | -| uv | bigint(20) | No | false | 0 | SUM | -+----------+-------------+------+-------+---------+-------+ -5 rows in set (0.00 sec) -``` - -可以使用以下命令取消当前正在执行的作业: - -`CANCEL ALTER TABLE COLUMN FROM table1` - -更多帮助，可以参阅 `HELP ALTER TABLE`。 - -## 2 Rollup - -Rollup 可以理解为 Table 的一个物化索引结构。**物化** 是因为其数据在物理上独立存储，而 **索引** 的意思是，Rollup可以调整列顺序以增加前缀索引的命中率，也可以减少key列以增加数据的聚合度。 - -以下举例说明。 - -原表table1的Schema如下: - -``` -+----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+----------+-------------+------+-------+---------+-------+ -| siteid | int(11) | No | true | 10 | | -| citycode | smallint(6) | No | true | N/A | | -| username | varchar(32) | No | true | | | -| pv | bigint(20) | No | false | 0 | SUM | -| uv | bigint(20) | No | false | 0 | SUM | -+----------+-------------+------+-------+---------+-------+ -``` - -对于 table1 明细数据是 siteid, citycode, username 三者构成一组 key，从而对 pv 字段进行聚合；如果业务方经常有看城市 pv 总量的需求，可以建立一个只有 citycode, pv 的rollup。 - -`ALTER TABLE table1 ADD ROLLUP rollup_city(citycode, pv);` - -提交成功后，可以通过以下命令查看作业进度： - -`SHOW ALTER TABLE ROLLUP;` - -当作业状态为 FINISHED，则表示作业完成。 - -Rollup 建立完成之后可以使用 `DESC table1 ALL` 查看表的 Rollup 信息。 - -``` -mysql> desc table1 all; -+-------------+----------+-------------+------+-------+--------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+-------------+----------+-------------+------+-------+---------+-------+ -| table1 | siteid | int(11) | No | true | 10 | | -| | citycode | smallint(6) | No | true | N/A | | -| | username | varchar(32) | No | true | | | -| | pv | bigint(20) | No | false | 0 | SUM | -| | uv | bigint(20) | No | false | 0 | SUM | -| | | | | | | | -| rollup_city | citycode | smallint(6) | No | true | N/A | | -| | pv | bigint(20) | No | false | 0 | SUM | -+-------------+----------+-------------+------+-------+---------+-------+ -8 rows in set (0.01 sec) -``` - -可以使用以下命令取消当前正在执行的作业: - -`CANCEL ALTER TABLE ROLLUP FROM table1;` - -Rollup 建立之后，查询不需要指定 Rollup 进行查询。还是指定原有表进行查询即可。程序会自动判断是否应该使用 Rollup。是否命中 Rollup可以通过 `EXPLAIN your_sql;` 命令进行查看。 - -更多帮助，可以参阅 `HELP ALTER TABLE`。 - -## 2 数据表的查询 - -### 2.1 内存限制 - -为了防止用户的一个查询可能因为消耗内存过大。查询进行了内存控制，一个查询任务，在单个 BE 节点上默认使用不超过 2GB 内存。 - -用户在使用时，如果发现报 `Memory limit exceeded` 错误，一般是超过内存限制了。 - -遇到内存超限时，用户应该尽量通过优化自己的 sql 语句来解决。 - -如果确切发现2GB内存不能满足，可以手动设置内存参数。 - -显示查询内存限制: - -``` -mysql> SHOW VARIABLES LIKE "%mem_limit%"; -+---------------+------------+ -| Variable_name | Value | -+---------------+------------+ -| exec_mem_limit| 2147483648 | -+---------------+------------+ -1 row in set (0.00 sec) -``` - -`exec_mem_limit` 的单位是 byte，可以通过 `SET` 命令改变 `exec_mem_limit` 的值。如改为 8GB。 - -`SET exec_mem_limit = 8589934592;` - -``` -mysql> SHOW VARIABLES LIKE "%mem_limit%"; -+---------------+------------+ -| Variable_name | Value | -+---------------+------------+ -| exec_mem_limit| 8589934592 | -+---------------+------------+ -1 row in set (0.00 sec) -``` - -> * 以上该修改为 session 级别，仅在当前连接 session 内有效。断开重连则会变回默认值。 -> * 如果需要修改全局变量，可以这样设置：`SET GLOBAL exec_mem_limit = 8589934592;`。设置完成后，断开 session 重新登录，参数将永久生效。 - -### 2.2 查询超时 - -当前默认查询时间设置为最长为 300 秒，如果一个查询在 300 秒内没有完成，则查询会被 Doris 系统 cancel 掉。用户可以通过这个参数来定制自己应用的超时时间，实现类似 wait(timeout) 的阻塞方式。 - -查看当前超时设置: - -``` -mysql> SHOW VARIABLES LIKE "%query_timeout%"; -+---------------+-------+ -| Variable_name | Value | -+---------------+-------+ -| QUERY_TIMEOUT | 300 | -+---------------+-------+ -1 row in set (0.00 sec) -``` - -修改超时时间到1分钟: - -`SET query_timeout = 60;` - -> * 当前超时的检查间隔为 5 秒，所以小于 5 秒的超时不会太准确。 -> * 以上修改同样为 session 级别。可以通过 `SET GLOBAL` 修改全局有效。 - -### 2.3 Broadcast/Shuffle Join - -系统默认实现 Join 的方式，是将小表进行条件过滤后，将其广播到大表所在的各个节点上，形成一个内存 Hash 表，然后流式读出大表的数据进行Hash Join。但是如果当小表过滤后的数据量无法放入内存的话，此时 Join 将无法完成，通常的报错应该是首先造成内存超限。 - -如果遇到上述情况，建议使用 Shuffle Join 的方式，也被称作 Partitioned Join。即将小表和大表都按照 Join 的 key 进行 Hash，然后进行分布式的 Join。这个对内存的消耗就会分摊到集群的所有计算节点上。 - -使用 Broadcast Join（默认）: - -``` -mysql> select sum(table1.pv) from table1 join table2 where table1.siteid = 2; -+--------------------+ -| sum(`table1`.`pv`) | -+--------------------+ -| 10 | -+--------------------+ -1 row in set (0.20 sec) -``` - -使用 Broadcast Join（显式指定）: - -``` -mysql> select sum(table1.pv) from table1 join [broadcast] table2 where table1.siteid = 2; -+--------------------+ -| sum(`table1`.`pv`) | -+--------------------+ -| 10 | -+--------------------+ -1 row in set (0.20 sec) -``` - -使用 Shuffle Join: - -``` -mysql> select sum(table1.pv) from table1 join [shuffle] table2 where table1.siteid = 2; -+--------------------+ -| sum(`table1`.`pv`) | -+--------------------+ -| 10 | -+--------------------+ -1 row in set (0.15 sec) -``` - -### 2.4 查询重试和高可用 - -当部署多个 FE 节点时，用户可以在多个 FE 之上部署负载均衡层来实现 Doris 的高可用。 - -一下提供一些高可用的方案： - -**第一种** - -自己在应用层代码进行重试和负载均衡。比如发现一个连接挂掉，就自动在其他连接上进行重试。应用层代码重试需要应用自己配置多个doris前端节点地址。 - -**第二种** - -如果使用 mysql jdbc connector 来连接Doris，可以使用 jdbc 的自动重试机制: - -``` -jdbc:mysql://[host:port],[host:port].../[database][?propertyName1][=propertyValue1][&propertyName2][=propertyValue2]... -``` - -**第三种** - -应用可以连接到和应用部署到同一机器上的 MySQL Proxy，通过配置 MySQL Proxy 的 Failover 和 Load Balance 功能来达到目的。 - -`http://dev.mysql.com/doc/refman/5.6/en/mysql-proxy-using.html` diff --git a/docs/documentation/cn/getting-started/basic-usage.md b/docs/documentation/cn/getting-started/basic-usage.md deleted file mode 100644 index df927e7461074a..00000000000000 --- a/docs/documentation/cn/getting-started/basic-usage.md +++ /dev/null @@ -1,374 +0,0 @@ - - -# 基础使用指南 - -Doris 采用 MySQL 协议进行通信，用户可通过 MySQL client 或者 MySQL JDBC连接到 Doris 集群。选择 MySQL client 版本时建议采用5.1 之后的版本，因为 5.1 之前不能支持长度超过 16 个字符的用户名。本文以 MySQL client 为例，通过一个完整的流程向用户展示 Doris 的基本使用方法。 - -## 1 创建用户 - -### 1.1 Root 用户登录与密码修改 - -Doris 内置 root 和 admin 用户，密码默认都为空。启动完 Doris 程序之后，可以通过 root 或 admin 用户连接到 Doris 集群。 -使用下面命令即可登录 Doris： - -``` -mysql -h FE_HOST -P9030 -uroot -``` - -> `fe_host` 是任一 FE 节点的 ip 地址。`9030` 是 fe.conf 中的 query_port 配置。 - -登陆后，可以通过以下命令修改 root 密码 - -``` -SET PASSWORD FOR 'root' = PASSWORD('your_password'); -``` - -### 1.3 创建新用户 - -通过下面的命令创建一个普通用户。 - -``` -CREATE USER 'test' IDENTIFIED BY 'test_passwd'; -``` - -后续登录时就可以通过下列连接命令登录。 - -``` -mysql -h FE_HOST -P9030 -utest -ptest_passwd -``` - -> 新创建的普通用户默认没有任何权限。权限授予可以参考后面的权限授予。 - -## 2 数据表的创建与数据导入 - -### 2.1 创建数据库 - -初始可以通过 root 或 admin 用户创建数据库： - -`CREATE DATABASE example_db;` - -> 所有命令都可以使用 'HELP command;' 查看到详细的语法帮助。如：`HELP CREATE DATABASE;` - -> 如果不清楚命令的全名，可以使用 "help 命令某一字段" 进行模糊查询。如键入'HELP CREATE'，可以匹配到 `CREATE DATABASE`, `CREATE TABLE`, `CREATE USER` 等命令。 - -数据库创建完成之后，可以通过 `SHOW DATABASES;` 查看数据库信息。 - -``` -MySQL> SHOW DATABASES; -+--------------------+ -| Database | -+--------------------+ -| example_db | -| information_schema | -+--------------------+ -2 rows in set (0.00 sec) -``` - -information_schema是为了兼容MySQL协议而存在，实际中信息可能不是很准确，所以关于具体数据库的信息建议通过直接查询相应数据库而获得。 - -### 2.2 账户授权 - -example_db 创建完成之后，可以通过 root/admin 账户将 example_db 读写权限授权给普通账户，如 test。授权之后采用 test 账户登录就可以操作 example_db 数据库了。 - -`GRANT ALL ON example_db TO test;` - -### 2.3 建表 - -使用 `CREATE TABLE` 命令建立一个表(Table)。更多详细参数可以查看: - -`HELP CREATE TABLE;` - -首先切换数据库: - -`USE example_db;` - -Doris支持支持单分区和复合分区两种建表方式。 - -在复合分区中： - -* 第一级称为 Partition，即分区。用户可以指定某一维度列作为分区列（当前只支持整型和时间类型的列），并指定每个分区的取值范围。 - -* 第二级称为 Distribution，即分桶。用户可以指定一个或多个维度列以及桶数对数据进行 HASH 分布。 - -以下场景推荐使用复合分区 - -* 有时间维度或类似带有有序值的维度，可以以这类维度列作为分区列。分区粒度可以根据导入频次、分区数据量等进行评估。 -* 历史数据删除需求：如有删除历史数据的需求（比如仅保留最近N 天的数据）。使用复合分区，可以通过删除历史分区来达到目的。也可以通过在指定分区内发送 DELETE 语句进行数据删除。 -* 解决数据倾斜问题：每个分区可以单独指定分桶数量。如按天分区，当每天的数据量差异很大时，可以通过指定分区的分桶数，合理划分不同分区的数据,分桶列建议选择区分度大的列。 - -用户也可以不使用复合分区，即使用单分区。则数据只做 HASH 分布。 - -下面以聚合模型为例，分别演示两种分区的建表语句。 - -#### 单分区 - -建立一个名字为 table1 的逻辑表。分桶列为 siteid，桶数为 10。 - -这个表的 schema 如下： - -* siteid：类型是INT（4字节）, 默认值为10 -* citycode：类型是SMALLINT（2字节） -* username：类型是VARCHAR, 最大长度为32, 默认值为空字符串 -* pv：类型是BIGINT（8字节）, 默认值是0; 这是一个指标列, Doris内部会对指标列做聚合操作, 这个列的聚合方法是求和（SUM） - -建表语句如下: -``` -CREATE TABLE table1 -( - siteid INT DEFAULT '10', - citycode SMALLINT, - username VARCHAR(32) DEFAULT '', - pv BIGINT SUM DEFAULT '0' -) -AGGREGATE KEY(siteid, citycode, username) -DISTRIBUTED BY HASH(siteid) BUCKETS 10 -PROPERTIES("replication_num" = "1"); -``` - -#### 复合分区 - -建立一个名字为 table2 的逻辑表。 - -这个表的 schema 如下： - -* event_day：类型是DATE，无默认值 -* siteid：类型是INT（4字节）, 默认值为10 -* citycode：类型是SMALLINT（2字节） -* username：类型是VARCHAR, 最大长度为32, 默认值为空字符串 -* pv：类型是BIGINT（8字节）, 默认值是0; 这是一个指标列, Doris 内部会对指标列做聚合操作, 这个列的聚合方法是求和（SUM） - -我们使用 event_day 列作为分区列，建立3个分区: p201706, p201707, p201708 - -* p201706：范围为 [最小值, 2017-07-01) -* p201707：范围为 [2017-07-01, 2017-08-01) -* p201708：范围为 [2017-08-01, 2017-09-01) - -> 注意区间为左闭右开。 - -每个分区使用 siteid 进行哈希分桶，桶数为10 - -建表语句如下: -``` -CREATE TABLE table2 -( - event_day DATE, - siteid INT DEFAULT '10', - citycode SMALLINT, - username VARCHAR(32) DEFAULT '', - pv BIGINT SUM DEFAULT '0' -) -AGGREGATE KEY(event_day, siteid, citycode, username) -PARTITION BY RANGE(event_day) -( - PARTITION p201706 VALUES LESS THAN ('2017-07-01'), - PARTITION p201707 VALUES LESS THAN ('2017-08-01'), - PARTITION p201708 VALUES LESS THAN ('2017-09-01') -) -DISTRIBUTED BY HASH(siteid) BUCKETS 10 -PROPERTIES("replication_num" = "1"); -``` - -表建完之后，可以查看 example_db 中表的信息: - -``` -MySQL> SHOW TABLES; -+----------------------+ -| Tables_in_example_db | -+----------------------+ -| table1 | -| table2 | -+----------------------+ -2 rows in set (0.01 sec) - -MySQL> DESC table1; -+----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+----------+-------------+------+-------+---------+-------+ -| siteid | int(11) | Yes | true | 10 | | -| citycode | smallint(6) | Yes | true | N/A | | -| username | varchar(32) | Yes | true | | | -| pv | bigint(20) | Yes | false | 0 | SUM | -+----------+-------------+------+-------+---------+-------+ -4 rows in set (0.00 sec) - -MySQL> DESC table2; -+-----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+-----------+-------------+------+-------+---------+-------+ -| event_day | date | Yes | true | N/A | | -| siteid | int(11) | Yes | true | 10 | | -| citycode | smallint(6) | Yes | true | N/A | | -| username | varchar(32) | Yes | true | | | -| pv | bigint(20) | Yes | false | 0 | SUM | -+-----------+-------------+------+-------+---------+-------+ -5 rows in set (0.00 sec) -``` - -> 注意事项： -> -> 1. 上述表通过设置 replication_num 建的都是单副本的表，Doris建议用户采用默认的 3 副本设置，以保证高可用。 -> 2. 可以对复合分区表动态的增删分区。详见 `HELP ALTER TABLE` 中 Partition 相关部分。 -> 3. 数据导入可以导入指定的 Partition。详见 `HELP LOAD`。 -> 4. 可以动态修改表的 Schema。 -> 5. 可以对 Table 增加上卷表（Rollup）以提高查询性能，这部分可以参见高级使用指南关于 Rollup 的描述。 -> 6. 表的列的Null属性默认为true，会对查询性能有一定的影响。 - -### 2.4 导入数据 - -Doris 支持多种数据导入方式。具体可以参阅数据导入文档。这里我们使用流式导入和 Broker 导入做示例。 - -#### 流式导入 - -流式导入通过 HTTP 协议向 Doris 传输数据，可以不依赖其他系统或组件直接导入本地数据。详细语法帮助可以参阅 `HELP STREAM LOAD;`。 - -示例1：以 "table1_20170707" 为 Label，使用本地文件 table1_data 导入 table1 表。 - -``` -curl --location-trusted -u test:test -H "label:table1_20170707" -H "column_separator:," -T table1_data http://FE_HOST:8030/api/example_db/table1/_stream_load -``` - -> 1. FE_HOST 是任一 FE 所在节点 IP，8030 为 fe.conf 中的 http_port。 -> 2. 可以使用任一 BE 的 IP，以及 be.conf 中的 webserver_port 进行导入。如：`BE_HOST:8040` - -本地文件 `table1_data` 以 `,` 作为数据之间的分隔，具体内容如下： - -``` -1,1,jim,2 -2,1,grace,2 -3,2,tom,2 -4,3,bush,3 -5,3,helen,3 -``` - -示例2: 以 "table2_20170707" 为 Label，使用本地文件 table2_data 导入 table2 表。 - -``` -curl --location-trusted -u test:test -H "label:table2_20170707" -H "column_separator:|" -T table1_data http://127.0.0.1:8030/api/example_db/table2/_stream_load -``` - -本地文件 `table2_data` 以 `|` 作为数据之间的分隔，具体内容如下： - -``` -2017-07-03|1|1|jim|2 -2017-07-05|2|1|grace|2 -2017-07-12|3|2|tom|2 -2017-07-15|4|3|bush|3 -2017-07-12|5|3|helen|3 -``` - -> 注意事项： -> -> 1. 采用流式导入建议文件大小限制在 10GB 以内，过大的文件会导致失败重试代价变大。 -> 2. 每一批导入数据都需要取一个 Label，Label 最好是一个和一批数据有关的字符串，方便阅读和管理。Doris 基于 Label 保证在一个Database 内，同一批数据只可导入成功一次。失败任务的 Label 可以重用。 -> 3. 流式导入是同步命令。命令返回成功则表示数据已经导入，返回失败表示这批数据没有导入。 - -#### Broker 导入 - -Broker 导入通过部署的 Broker 进程，读取外部存储上的数据进行导入。更多帮助请参阅 `HELP BROKER LOAD;` - -示例：以 "table1_20170708" 为 Label，将 HDFS 上的文件导入 table1 表 - -``` -LOAD LABEL table1_20170708 -( - DATA INFILE("hdfs://your.namenode.host:port/dir/table1_data") - INTO TABLE table1 -) -WITH BROKER hdfs -( - "username"="hdfs_user", - "password"="hdfs_password" -) -PROPERTIES -( - "timeout"="3600", - "max_filter_ratio"="0.1" -); -``` - -Broker 导入是异步命令。以上命令执行成功只表示提交任务成功。导入是否成功需要通过 `SHOW LOAD;` 查看。如： - -`SHOW LOAD WHERE LABLE = "table1_20170708";` - -返回结果中，`State` 字段为 FINISHED 则表示导入成功。 - -关于 `SHOW LOAD` 的更多说明，可以参阅 `HELP SHOW LOAD;` - -异步的导入任务在结束前可以取消： - -`CANCEL LOAD WHERE LABEL = "table1_20170708";` - -## 3 数据的查询 - -### 3.1 简单查询 - -示例: - -``` -MySQL> SELECT * FROM table1 LIMIT 3; -+--------+----------+----------+------+ -| siteid | citycode | username | pv | -+--------+----------+----------+------+ -| 2 | 1 | 'grace' | 2 | -| 5 | 3 | 'helen' | 3 | -| 3 | 2 | 'tom' | 2 | -+--------+----------+----------+------+ -5 rows in set (0.01 sec) - -MySQL> SELECT * FROM table1 ORDER BY citycode; -+--------+----------+----------+------+ -| siteid | citycode | username | pv | -+--------+----------+----------+------+ -| 2 | 1 | 'grace' | 2 | -| 1 | 1 | 'jim' | 2 | -| 3 | 2 | 'tom' | 2 | -| 4 | 3 | 'bush' | 3 | -| 5 | 3 | 'helen' | 3 | -+--------+----------+----------+------+ -5 rows in set (0.01 sec) -``` - -### 3.3 Join 查询 - -示例: - -``` -MySQL> SELECT SUM(table1.pv) FROM table1 JOIN table2 WHERE table1.siteid = table2.siteid; -+--------------------+ -| sum(`table1`.`pv`) | -+--------------------+ -| 12 | -+--------------------+ -1 row in set (0.20 sec) -``` - -### 3.4 子查询 - -示例: - -``` -MySQL> SELECT SUM(pv) FROM table2 WHERE siteid IN (SELECT siteid FROM table1 WHERE siteid > 2); -+-----------+ -| sum(`pv`) | -+-----------+ -| 8 | -+-----------+ -1 row in set (0.13 sec) -``` diff --git a/docs/documentation/cn/getting-started/best-practice.md b/docs/documentation/cn/getting-started/best-practice.md deleted file mode 100644 index 3b58699f6711cb..00000000000000 --- a/docs/documentation/cn/getting-started/best-practice.md +++ /dev/null @@ -1,182 +0,0 @@ - - -# 最佳实践 - -## 1 建表 - -### 1.1 数据模型选择 - -Doris 数据模型上目前分为三类: AGGREGATE KEY, UNIQUE KEY, DUPLICATE KEY。三种模型中数据都是按KEY进行排序。 - -1.1.1 AGGREGATE KEY - - AGGREGATE KEY相同时，新旧记录进行聚合，目前支持的聚合函数有SUM, MIN, MAX, REPLACE。 - - AGGREGATE KEY模型可以提前聚合数据, 适合报表和多维分析业务。 - - ``` - CREATE TABLE site_visit - ( - siteid INT, - city SMALLINT, - username VARCHAR(32), - pv BIGINT SUM DEFAULT '0' - ) - AGGREGATE KEY(siteid, city, username) - DISTRIBUTED BY HASH(siteid) BUCKETS 10; - ``` - -1.1.2. UNIQUE KEY - - UNIQUE KEY 相同时，新记录覆盖旧记录。目前 UNIQUE KEY 实现上和 AGGREGATE KEY 的 REPLACE 聚合方法一样，二者本质上相同。适用于有更新需求的分析业务。 - - ``` - CREATE TABLE sales_order - ( - orderid BIGINT, - status TINYINT, - username VARCHAR(32), - amount BIGINT DEFAULT '0' - ) - UNIQUE KEY(orderid) - DISTRIBUTED BY HASH(orderid) BUCKETS 10; - ``` - -1.1.3. DUPLICATE KEY - - 只指定排序列，相同的行不会合并。适用于数据无需提前聚合的分析业务。 - - ``` - CREATE TABLE session_data - ( - visitorid SMALLINT, - sessionid BIGINT, - visittime DATETIME, - city CHAR(20), - province CHAR(20), - ip varchar(32), - brower CHAR(20), - url VARCHAR(1024) - ) - DUPLICATE KEY(visitorid, sessionid) - DISTRIBUTED BY HASH(sessionid, visitorid) BUCKETS 10; - ``` - -### 1.2 大宽表与 Star Schema - -业务方建表时, 为了和前端业务适配, 往往不对维度信息和指标信息加以区分, 而将 Schema 定义成大宽表。对于 Doris 而言, 这类大宽表往往性能不尽如人意: - -* Schema 中字段数比较多, 聚合模型中可能 key 列比较多, 导入过程中需要排序的列会增加。 -* 维度信息更新会反应到整张表中，而更新的频率直接影响查询的效率。 - -使用过程中，建议用户尽量使用 Star Schema 区分维度表和指标表。频繁更新的维度表也可以放在 MySQL 外部表中。而如果只有少量更新, 可以直接放在 Doris 中。在 Doris 中存储维度表时，可对维度表设置更多的副本，提升 Join 的性能。 - -### 1.3 分区和分桶 - -Doris 支持两级分区存储, 第一层为 RANGE 分区(partition), 第二层为 HASH 分桶(bucket)。 - -1.3.1. RANGE分区(partition) - - RANGE分区用于将数据划分成不同区间, 逻辑上可以理解为将原始表划分成了多个子表。业务上，多数用户会选择采用按时间进行partition, 让时间进行partition有以下好处： - - * 可区分冷热数据 - * 可用上Doris分级存储(SSD + SATA)的功能 - * 按分区删除数据时，更加迅速 - -1.3.2. HASH分桶(bucket) - - 根据hash值将数据划分成不同的 bucket。 - - * 建议采用区分度大的列做分桶, 避免出现数据倾斜 - * 为方便数据恢复, 建议单个 bucket 的 size 不要太大, 保持在 10GB 以内, 所以建表或增加 partition 时请合理考虑 bucket 数目, 其中不同 partition 可指定不同的 buckets 数。 - -### 1.4 稀疏索引和 Bloom Filter - -Doris对数据进行有序存储, 在数据有序的基础上为其建立稀疏索引,索引粒度为 block(1024行)。 - -稀疏索引选取 schema 中固定长度的前缀作为索引内容, 目前 Doris 选取 36 个字节的前缀作为索引。 - -* 建表时建议将查询中常见的过滤字段放在 Schema 的前面, 区分度越大，频次越高的查询字段越往前放。 -* 这其中有一个特殊的地方,就是 varchar 类型的字段。varchar 类型字段只能作为稀疏索引的最后一个字段。索引会在 varchar 处截断, 因此 varchar 如果出现在前面，可能索引的长度可能不足 36 个字节。具体可以参阅 [数据模型、ROLLUP 及前缀索引](./data-model-rollup.md)。 -* 除稀疏索引之外, Doris还提供bloomfilter索引, bloomfilter索引对区分度比较大的列过滤效果明显。 如果考虑到varchar不能放在稀疏索引中, 可以建立bloomfilter索引。 - -### 1.5 物化视图(rollup) - -Rollup 本质上可以理解为原始表(Base Table)的一个物化索引。建立 Rollup 时可只选取 Base Table 中的部分列作为 Schema。Schema 中的字段顺序也可与 Base Table 不同。 - -下列情形可以考虑建立 Rollup： - -1.5.1. Base Table 中数据聚合度不高。 - -这一般是因 Base Table 有区分度比较大的字段而导致。此时可以考虑选取部分列，建立 Rollup。 - -如对于 `site_visit` 表： - -``` -site_visit(siteid, city, username, pv) -``` - -siteid 可能导致数据聚合度不高，如果业务方经常根据城市统计pv需求，可以建立一个只有 city, pv 的 Rollup： - -``` -ALTER TABLE site_visit ADD ROLLUP rollup_city(city, pv); -``` - -1.5.2. Base Table 中的前缀索引无法命中 - -这一般是 Base Table 的建表方式无法覆盖所有的查询模式。此时可以考虑调整列顺序，建立 Rollup。 - -如对于 session_data 表： - -``` -session_data(visitorid, sessionid, visittime, city, province, ip, brower, url) -``` - -如果除了通过 visitorid 分析访问情况外，还有通过 brower, province 分析的情形，可以单独建立 Rollup。 - -``` -ALTER TABLE session_data ADD ROLLUP rollup_brower(brower,province,ip,url) DUPLICATE KEY(brower,province); -``` - -## 2 Schema Change - -Doris中目前进行 Schema Change 的方式有三种：Sorted Schema Change，Direct Schema Change, Linked Schema Change。 - -2.1. Sorted Schema Change - - 改变了列的排序方式，需对数据进行重新排序。例如删除排序列中的一列, 字段重排序。 - - ``` - ALTER TABLE site_visit DROP COLUMN city; - ``` - -2.2. Direct Schema Change: 无需重新排序，但是需要对数据做一次转换。例如修改列的类型，在稀疏索引中加一列等。 - - ``` - ALTER TABLE site_visit MODIFY COLUMN username varchar(64); - ``` - -2.3. Linked Schema Change: 无需转换数据，直接完成。例如加列操作。 - - ``` - ALTER TABLE site_visit ADD COLUMN click bigint SUM default '0'; - ``` - -建表时建议考虑好 Schema，这样在进行 Schema Change 时可以加快速度。 diff --git a/docs/documentation/cn/getting-started/data-model-rollup.md b/docs/documentation/cn/getting-started/data-model-rollup.md deleted file mode 100644 index 733634d93da51d..00000000000000 --- a/docs/documentation/cn/getting-started/data-model-rollup.md +++ /dev/null @@ -1,631 +0,0 @@ - - -# 数据模型、ROLLUP 及前缀索引 - -本文档主要从逻辑层面，描述 Doris 的数据模型、 ROLLUP 以及前缀索引的概念，以帮助用户更好的使用 Doris 应对不同的业务场景。 - -## 基本概念 - -在 Doris 中，数据以表（Table）的形式进行逻辑上的描述。 -一张表包括行（Row）和列（Column）。Row 即用户的一行数据。Column 用于描述一行数据中不同的字段。 - -Column 可以分为两大类：Key 和 Value。从业务角度看，Key 和 Value 可以分别对应维度列和指标列。 - -Doris 的数据模型主要分为3类: - -* Aggregate -* Uniq -* Duplicate - -下面我们分别介绍。 - -## Aggregate 模型 - -我们以实际的例子来说明什么是聚合模型，以及如何正确的使用聚合模型。 - -### 示例1：导入数据聚合 - -假设业务有如下数据表模式： - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -|user\_id|LARGEINT||用户id| -|date|DATE||数据灌入日期| -|city|VARCHAR(20)||用户所在城市| -|age|SMALLINT||用户年龄| -|sex|TINYINT||用户性别| -|last_visit_date|DATETIME|REPLACE|用户最后一次访问时间| -|cost|BIGINT|SUM|用户总消费| -|max\_dwell\_time|INT|MAX|用户最大停留时间| -|min\_dwell\_time|INT|MIN|用户最小停留时间| - -如果转换成建表语句则如下（省略建表语句中的 Partition 和 Distribution 信息） - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `date` DATE NOT NULL COMMENT "数据灌入日期时间", - `city` VARCHAR(20) COMMENT "用户所在城市", - `age` SMALLINT COMMENT "用户年龄", - `sex` TINYINT COMMENT "用户性别", - `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", - `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", - `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", -) -AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) -... /* 省略 Partition 和 Distribution 信息 */ -； -``` - -可以看到，这是一个典型的用户信息和访问行为的事实表。 -在一般星型模型中，用户信息和访问行为一般分别存放在维度表和事实表中。这里我们为了更加方便的解释 Doris 的数据模型，将两部分信息统一存放在一张表中。 - -表中的列按照是否设置了 `AggregationType`，分为 Key (维度列) 和 Value（指标列）。没有设置 `AggregationType` 的，如 `user_id`、`date`、`age` ... 等称为 **Key**，而设置了 `AggregationType` 的称为 **Value**。 - -当我们导入数据时，对于 Key 列相同的行和聚合成一行，而 Value 列会按照设置的 `AggregationType` 进行聚合。 `AggregationType` 目前有以下四种聚合方式： - -1. SUM：求和，多行的 Value 进行累加。 -2. REPLACE：替代，下一批数据中的 Value 会替换之前导入过的行中的 Value。 -3. MAX：保留最大值。 -4. MIN：保留最小值。 - -假设我们有以下导入数据（原始数据）： - -|user\_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -|10000|2017-10-01|北京|20|0|2017-10-01 06:00:00|20|10|10| -|10000|2017-10-01|北京|20|0|2017-10-01 07:00:00|15|2|2| -|10001|2017-10-01|北京|30|1|2017-10-01 17:05:45|2|22|22| -|10002|2017-10-02|上海|20|1|2017-10-02 12:59:12|200|5|5| -|10003|2017-10-02|广州|32|0|2017-10-02 11:20:00|30|11|11| -|10004|2017-10-01|深圳|35|0|2017-10-01 10:00:15|100|3|3| -|10004|2017-10-03|深圳|35|0|2017-10-03 10:20:22|11|6|6| - -我们假设这是一张记录用户访问某商品页面行为的表。我们以第一行数据为例，解释如下： - -|数据|说明| -|---|---| -|10000|用户id，每个用户唯一识别id| -|2017-10-01|数据入库时间，精确到日期| -|北京|用户所在城市| -|20|用户年龄| -|0|性别男（1 代表女性）| -|2017-10-01 06:00:00|用户本次访问该页面的时间，精确到秒| -|20|用户本次访问产生的消费| -|10|用户本次访问，驻留该页面的时间| -|10|用户本次访问，驻留该页面的时间（冗余）| - -那么当这批数据正确导入到 Doris 中后，Doris 中最终存储如下： - -|user\_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -|10000|2017-10-01|北京|20|0|2017-10-01 07:00:00|35|10|2| -|10001|2017-10-01|北京|30|1|2017-10-01 17:05:45|2|22|22| -|10002|2017-10-02|上海|20|1|2017-10-02 12:59:12|200|5|5| -|10003|2017-10-02|广州|32|0|2017-10-02 11:20:00|30|11|11| -|10004|2017-10-01|深圳|35|0|2017-10-01 10:00:15|100|3|3| -|10004|2017-10-03|深圳|35|0|2017-10-03 10:20:22|11|6|6| - -可以看到，用户 10000 只剩下了一行**聚合后**的数据。而其余用户的数据和原始数据保持一致。这里先解释下用户 10000 聚合后的数据： - -前5列没有变化，从第6列 `last_visit_date` 开始： - -* `2017-10-01 07:00:00`：因为 `last_visit_date` 列的聚合方式为 REPLACE，所以 `2017-10-01 07:00:00` 替换了 `2017-10-01 06:00:00` 保存了下来。 - > 注：在同一个导入批次中的数据，对于 REPLACE 这种聚合方式，替换顺序不做保证。如在这个例子中，最终保存下来的，也有可能是 `2017-10-01 06:00:00`。而对于不同导入批次中的数据，可以保证，后一批次的数据会替换前一批次。 - -* `35`：因为 `cost` 列的聚合类型为 SUM，所以由 20 + 15 累加获得 35。 -* `10`：因为 `max_dwell_time` 列的聚合类型为 MAX，所以 10 和 2 取最大值，获得 10。 -* `2`：因为 `min_dwell_time` 列的聚合类型为 MIN，所以 10 和 2 取最小值，获得 2。 - -经过聚合，Doris 中最终只会存储聚合后的数据。换句话说，即明细数据会丢失，用户不能够再查询到聚合前的明细数据了。 - -### 示例2：保留明细数据 - -接示例1，我们将表结构修改如下： - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -|user\_id|LARGEINT||用户id| -|date|DATE||数据灌入日期| -|timestamp|DATETIME||数据灌入时间，精确到秒| -|city|VARCHAR(20)||用户所在城市| -|age|SMALLINT||用户年龄| -|sex|TINYINT||用户性别| -|last\_visit\_date|DATETIME|REPLACE|用户最后一次访问时间| -|cost|BIGINT|SUM|用户总消费| -|max\_dwell\_time|INT|MAX|用户最大停留时间| -|min\_dwell\_time|INT|MIN|用户最小停留时间| - -即增加了一列 `timestamp`，记录精确到秒的数据灌入时间。 - -导入数据如下： - -|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---|---| -|10000|2017-10-01|2017-10-01 08:00:05|北京|20|0|2017-10-01 06:00:00|20|10|10| -|10000|2017-10-01|2017-10-01 09:00:05|北京|20|0|2017-10-01 07:00:00|15|2|2| -|10001|2017-10-01|2017-10-01 18:12:10|北京|30|1|2017-10-01 17:05:45|2|22|22| -|10002|2017-10-02|2017-10-02 13:10:00|上海|20|1|2017-10-02 12:59:12|200|5|5| -|10003|2017-10-02|2017-10-02 13:15:00|广州|32|0|2017-10-02 11:20:00|30|11|11| -|10004|2017-10-01|2017-10-01 12:12:48|深圳|35|0|2017-10-01 10:00:15|100|3|3| -|10004|2017-10-03|2017-10-03 12:38:20|深圳|35|0|2017-10-03 10:20:22|11|6|6| - -那么当这批数据正确导入到 Doris 中后，Doris 中最终存储如下： - -|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---|---| -|10000|2017-10-01|2017-10-01 08:00:05|北京|20|0|2017-10-01 06:00:00|20|10|10| -|10000|2017-10-01|2017-10-01 09:00:05|北京|20|0|2017-10-01 07:00:00|15|2|2| -|10001|2017-10-01|2017-10-01 18:12:10|北京|30|1|2017-10-01 17:05:45|2|22|22| -|10002|2017-10-02|2017-10-02 13:10:00|上海|20|1|2017-10-02 12:59:12|200|5|5| -|10003|2017-10-02|2017-10-02 13:15:00|广州|32|0|2017-10-02 11:20:00|30|11|11| -|10004|2017-10-01|2017-10-01 12:12:48|深圳|35|0|2017-10-01 10:00:15|100|3|3| -|10004|2017-10-03|2017-10-03 12:38:20|深圳|35|0|2017-10-03 10:20:22|11|6|6| - -我们可以看到，存储的数据，和导入数据完全一样，没有发生任何聚合。这是因为，这批数据中，因为加入了 `timestamp` 列，所有行的 Key 都**不完全相同**。也就是说，只要保证导入的数据中，每一行的 Key 都不完全相同，那么即使在聚合模型下，Doris 也可以保存完整的明细数据。 - -### 示例3：导入数据与已有数据聚合 - -接示例1。假设现在表中已有数据如下： - -|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -|10000|2017-10-01|北京|20|0|2017-10-01 07:00:00|35|10|2| -|10001|2017-10-01|北京|30|1|2017-10-01 17:05:45|2|22|22| -|10002|2017-10-02|上海|20|1|2017-10-02 12:59:12|200|5|5| -|10003|2017-10-02|广州|32|0|2017-10-02 11:20:00|30|11|11| -|10004|2017-10-01|深圳|35|0|2017-10-01 10:00:15|100|3|3| -|10004|2017-10-03|深圳|35|0|2017-10-03 10:20:22|11|6|6| - -我们再导入一批新的数据： - -|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -|10004|2017-10-03|深圳|35|0|2017-10-03 11:22:00|44|19|19| -|10005|2017-10-03|长沙|29|1|2017-10-03 18:11:02|3|1|1| - -那么当这批数据正确导入到 Doris 中后，Doris 中最终存储如下： - -|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -|10000|2017-10-01|北京|20|0|2017-10-01 07:00:00|35|10|2| -|10001|2017-10-01|北京|30|1|2017-10-01 17:05:45|2|22|22| -|10002|2017-10-02|上海|20|1|2017-10-02 12:59:12|200|5|5| -|10003|2017-10-02|广州|32|0|2017-10-02 11:20:00|30|11|11| -|10004|2017-10-01|深圳|35|0|2017-10-01 10:00:15|100|3|3| -|10004|2017-10-03|深圳|35|0|2017-10-03 11:22:00|55|19|6| -|10005|2017-10-03|长沙|29|1|2017-10-03 18:11:02|3|1|1| - -可以看到，用户 10004 的已有数据和新导入的数据发生了聚合。同时新增了 10005 用户的数据。 - -数据的聚合，在 Doris 中有如下三个阶段发生： - -1. 每一批次数据导入的 ETL 阶段。该阶段会在每一批次导入的数据内部进行聚合。 -2. 底层 BE 进行数据 Compaction 的阶段。该阶段，BE 会对已导入的不同批次的数据进行进一步的聚合。 -3. 数据查询阶段。在数据查询时，对于查询涉及到的数据，会进行对应的聚合。 - -数据在不同时间，可能聚合的程度不一致。比如一批数据刚导入时，可能还未与之前已存在的数据进行聚合。但是对于用户而言，用户**只能查询到**聚合后的数据。即不同的聚合程度对于用户查询而言是透明的。用户需始终认为数据以**最终的完成的聚合程度**存在，而**不应假设某些聚合还未发生**。（可参阅**聚合模型的局限性**一节获得更多详情。） - -## Uniq 模型 - -在某些多维分析场景下，用户更关注的是如何保证 Key 的唯一性，即如何获得 Primary Key 唯一性约束。因此，我们引入了 Uniq 的数据模型。该模型本质上是聚合模型的一个特例，也是一种简化的表结构表示方式。我们举例说明。 - -|ColumnName|Type|IsKey|Comment| -|---|---|---|---| -|user_id|BIGINT|Yes|用户id| -|username|VARCHAR(50)|Yes|用户昵称| -|city|VARCHAR(20)|No|用户所在城市| -|age|SMALLINT|No|用户年龄| -|sex|TINYINT|No|用户性别| -|phone|LARGEINT|No|用户电话| -|address|VARCHAR(500)|No|用户住址| -|register_time|DATETIME|No|用户注册时间| - -这是一个典型的用户基础信息表。这类数据没有聚合需求，只需保证主键唯一性。（这里的主键为 user_id + username）。那么我们的建表语句如下： - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `username` VARCHAR(50) NOT NULL COMMENT "用户昵称", - `city` VARCHAR(20) COMMENT "用户所在城市", - `age` SMALLINT COMMENT "用户年龄", - `sex` TINYINT COMMENT "用户性别", - `phone` LARGEINT COMMENT "用户电话", - `address` VARCHAR(500) COMMENT "用户地址", - `register_time` DATETIME COMMENT "用户注册时间" -) -UNIQUE KEY(`user_id`, `user_name`) -... /* 省略 Partition 和 Distribution 信息 */ -； -``` - -而这个表结构，完全同等于以下使用聚合模型描述的表结构： - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -|user_id|BIGINT||用户id| -|username|VARCHAR(50)||用户昵称| -|city|VARCHAR(20)|REPLACE|用户所在城市| -|age|SMALLINT|REPLACE|用户年龄| -|sex|TINYINT|REPLACE|用户性别| -|phone|LARGEINT|REPLACE|用户电话| -|address|VARCHAR(500)|REPLACE|用户住址| -|register_time|DATETIME|REPLACE|用户注册时间| - -及建表语句： - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `username` VARCHAR(50) NOT NULL COMMENT "用户昵称", - `city` VARCHAR(20) REPLACE COMMENT "用户所在城市", - `age` SMALLINT REPLACE COMMENT "用户年龄", - `sex` TINYINT REPLACE COMMENT "用户性别", - `phone` LARGEINT REPLACE COMMENT "用户电话", - `address` VARCHAR(500) REPLACE COMMENT "用户地址", - `register_time` DATETIME REPLACE COMMENT "用户注册时间" -) -AGGREGATE KEY(`user_id`, `user_name`) -... /* 省略 Partition 和 Distribution 信息 */ -； -``` - -即 Uniq 模型完全可以用聚合模型中的 REPLACE 方式替代。其内部的实现方式和数据存储方式也完全一样。这里不再继续举例说明。 - -## Duplicate 模型 - -在某些多维分析场景下，数据既没有主键，也没有聚合需求。因此，我们引入 Duplicate 数据模型来满足这类需求。举例说明。 - -|ColumnName|Type|SortKey|Comment| -|---|---|---|---| -|timestamp|DATETIME|Yes|日志时间| -|type|INT|Yes|日志类型| -|error_code|INT|Yes|错误码| -|error_msg|VARCHAR(1024)|No|错误详细信息| -|op_id|BIGINT|No|负责人id| -|op_time|DATETIME|No|处理时间| - -建表语句如下： - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( - `timestamp` DATETIME NOT NULL COMMENT "日志时间", - `type` INT NOT NULL COMMENT "日志类型", - `error_code` INT COMMENT "错误码", - `error_msg` VARCHAR(1024) COMMENT "错误详细信息", - `op_id` BIGINT COMMENT "负责人id", - `op_time` DATETIME COMMENT "处理时间" -) -DUPLICATE KEY(`timestamp`, `type`) -... /* 省略 Partition 和 Distribution 信息 */ -； -``` - -这种数据模型区别于 Aggregate 和 Uniq 模型。数据完全按照导入文件中的数据进行存储，不会有任何聚合。即使两行数据完全相同，也都会保留。 -而在建表语句中指定的 DUPLICATE KEY，只是用来指明底层数据按照那些列进行排序。（更贴切的名称应该为 “Sorted Column”，这里取名 “DUPLICATE KEY” 只是用以明确表示所用的数据模型。关于 “Sorted Column”的更多解释，可以参阅**前缀索引**小节）。在 DUPLICATE KEY 的选择上，我们建议适当的选择前 2-4 列就可以。 - -这种数据模型适用于既没有聚合需求，又没有主键唯一性约束的原始数据的存储。更多使用场景，可参阅**聚合模型的局限性**小节。 - -## ROLLUP - -ROLLUP 在多维分析中是“上卷”的意思，即将数据按某种指定的粒度进行进一步聚合。 - -### 基本概念 - -在 Doris 中，我们将用户通过建表语句创建出来的表成为 Base 表（Base Table）。Base 表中保存着按用户建表语句指定的方式存储的基础数据。 - -在 Base 表之上，我们可以创建任意多个 ROLLUP 表。这些 ROLLUP 的数据是基于 Base 表产生的，并且在物理上是**独立存储**的。 - -ROLLUP 表的基本作用，在于在 Base 表的基础上，获得更粗粒度的聚合数据。 - -下面我们用示例详细说明在不同数据模型中的 ROLLUP 表及其作用。 - -#### Aggregate 和 Uniq 模型中的 ROLLUP - -因为 Uniq 只是 Aggregate 模型的一个特例，所以这里我们不加以区别。 - -1. 示例1：获得每个用户的总消费 - -接**Aggregate 模型**小节的**示例2**，Base 表结构如下： - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -|user_id|LARGEINT||用户id| -|date|DATE||数据灌入日期| -|timestamp|DATETIME||数据灌入时间，精确到秒| -|city|VARCHAR(20)||用户所在城市| -|age|SMALLINT||用户年龄| -|sex|TINYINT||用户性别| -|last_visit_date|DATETIME|REPLACE|用户最后一次访问时间| -|cost|BIGINT|SUM|用户总消费| -|max\_dwell\_time|INT|MAX|用户最大停留时间| -|min\_dwell\_time|INT|MIN|用户最小停留时间| - -存储的数据如下： - -|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---|---| -|10000|2017-10-01|2017-10-01 08:00:05|北京|20|0|2017-10-01 06:00:00|20|10|10| -|10000|2017-10-01|2017-10-01 09:00:05|北京|20|0|2017-10-01 07:00:00|15|2|2| -|10001|2017-10-01|2017-10-01 18:12:10|北京|30|1|2017-10-01 17:05:45|2|22|22| -|10002|2017-10-02|2017-10-02 13:10:00|上海|20|1|2017-10-02 12:59:12|200|5|5| -|10003|2017-10-02|2017-10-02 13:15:00|广州|32|0|2017-10-02 11:20:00|30|11|11| -|10004|2017-10-01|2017-10-01 12:12:48|深圳|35|0|2017-10-01 10:00:15|100|3|3| -|10004|2017-10-03|2017-10-03 12:38:20|深圳|35|0|2017-10-03 10:20:22|11|6|6| - -在此基础上，我们创建一个 ROLLUP： - -|ColumnName| -|---| -|user_id| -|cost| - -该 ROLLUP 只包含两列：user_id 和 cost。则创建完成后，该 ROLLUP 中存储的数据如下： - -|user\_id|cost| -|---|---| -|10000|35| -|10001|2| -|10002|200| -|10003|30| -|10004|111| - -可以看到，ROLLUP 中仅保留了每个 user_id，在 cost 列上的 SUM 的结果。那么当我们进行如下查询时: - -`SELECT user_id, sum(cost) FROM table GROUP BY user_id;` - -Doris 会自动命中这个 ROLLUP 表，从而只需扫描极少的数据量，即可完成这次聚合查询。 - -2. 示例2：获得不同城市，不同年龄段用户的总消费、最长和最短页面驻留时间 - -紧接示例1。我们在 Base 表基础之上，再创建一个 ROLLUP： - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -|city|VARCHAR(20)||用户所在城市| -|age|SMALLINT||用户年龄| -|cost|BIGINT|SUM|用户总消费| -|max\_dwell\_time|INT|MAX|用户最大停留时间| -|min\_dwell\_time|INT|MIN|用户最小停留时间| - -则创建完成后，该 ROLLUP 中存储的数据如下： - -|city|age|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---| -|北京|20|0|30|10|2| -|北京|30|1|2|22|22| -|上海|20|1|200|5|5| -|广州|32|0|30|11|11| -|深圳|35|0|111|6|3| - -当我们进行如下这些查询时: - -* `SELECT city, age, sum(cost), max(max_dwell_time), min(min_dwell_time) FROM table GROUP BY city, age;` -* `SELECT city, sum(cost), max(max_dwell_time), min(min_dwell_time) FROM table GROUP BY city;` -* `SELECT city, age, sum(cost), min(min_dwell_time) FROM table GROUP BY city, age;` - -Doris 会自动命中这个 ROLLUP 表。 - -#### Duplicate 模型中的 ROLLUP - -因为 Duplicate 模型没有聚合的语意。所以该模型中的 ROLLUP，已经失去了“上卷”这一层含义。而仅仅是作为调整列顺序，以命中前缀索引的作用。我们将在接下来的小节中，详细介绍前缀索引，以及如何使用ROLLUP改变前缀索引，以获得更好的查询效率。 - -### 前缀索引与 ROLLUP - -#### 前缀索引 - -不同于传统的数据库设计，Doris 不支持在任意列上创建索引。Doris 这类 MPP 架构的 OLAP 数据库，通常都是通过提高并发，来处理大量数据的。 -本质上，Doris 的数据存储在类似 SSTable（Sorted String Table）的数据结构中。该结构是一种有序的数据结构，可以按照指定的列进行排序存储。在这种数据结构上，以排序列作为条件进行查找，会非常的高效。 - -在 Aggregate、Uniq 和 Duplicate 三种数据模型中。底层的数据存储，是按照各自建表语句中，AGGREGATE KEY、UNIQ KEY 和 DUPLICATE KEY 中指定的列进行排序存储的。 - -而前缀索引，即在排序的基础上，实现的一种根据给定前缀列，快速查询数据的索引方式。 - -我们将一行数据的前 **36 个字节** 作为这行数据的前缀索引。当遇到 VARCHAR 类型时，前缀索引会直接截断。我们举例说明： - -1. 以下表结构的前缀索引为 user_id(8Byte) + age(4Bytes) + message(prefix 24 Bytes)。 - -|ColumnName|Type| -|---|---| -|user_id|BIGINT| -|age|INT| -|message|VARCHAR(100)| -|max\_dwell\_time|DATETIME| -|min\_dwell\_time|DATETIME| - -2. 以下表结构的前缀索引为 user_name(20 Bytes)。即使没有达到 36 个字节，因为遇到 VARCHAR，所以直接截断，不再往后继续。 - -|ColumnName|Type| -|---|---| -|user_name|VARCHAR(20)| -|age|INT| -|message|VARCHAR(100)| -|max\_dwell\_time|DATETIME| -|min\_dwell\_time|DATETIME| - -当我们的查询条件，是**前缀索引的前缀**时，可以极大的加快查询速度。比如在第一个例子中，我们执行如下查询： - -`SELECT * FROM table WHERE user_id=1829239 and age=20；` - -该查询的效率会**远高于**如下查询： - -`SELECT * FROM table WHERE age=20；` - -所以在建表时，**正确的选择列顺序，能够极大地提高查询效率**。 - -#### ROLLUP 调整前缀索引 - -因为建表时已经指定了列顺序，所以一个表只有一种前缀索引。这对于使用其他不能命中前缀索引的列作为条件进行的查询来说，效率上可能无法满足需求。因此，我们可以通过创建 ROLLUP 来人为的调整列顺序。举例说明。 - -Base 表结构如下： - -|ColumnName|Type| -|---|---| -|user\_id|BIGINT| -|age|INT| -|message|VARCHAR(100)| -|max\_dwell\_time|DATETIME| -|min\_dwell\_time|DATETIME| - -我们可以在此基础上创建一个 ROLLUP 表： - -|ColumnName|Type| -|---|---| -|age|INT| -|user\_id|BIGINT| -|message|VARCHAR(100)| -|max\_dwell\_time|DATETIME| -|min\_dwell\_time|DATETIME| - -可以看到，ROLLUP 和 Base 表的列完全一样，只是将 user_id 和 age 的顺序调换了。那么当我们进行如下查询时： - -`SELECT * FROM table where age=20 and massage LIKE "%error%";` - -会优先选择 ROLLUP 表，因为 ROLLUP 的前缀索引匹配度更高。 - -### ROLLUP 的几点说明 - -* ROLLUP 最根本的作用是提高某些查询的查询效率（无论是通过聚合来减少数据量，还是修改列顺序以匹配前缀索引）。因此 ROLLUP 的含义已经超出了 “上卷” 的范围。这也是为什么我们在源代码中，将其命名为 Materized Index（物化索引）的原因。 -* ROLLUP 是附属于 Base 表的，可以看做是 Base 表的一种辅助数据结构。用户可以在 Base 表的基础上，创建或删除 ROLLUP，但是不能在查询中显式的指定查询某 ROLLUP。是否命中 ROLLUP 完全由 Doris 系统自动决定。 -* ROLLUP 的数据是独立物理存储的。因此，创建的 ROLLUP 越多，占用的磁盘空间也就越大。同时对导入速度也会有影响（导入的ETL阶段会自动产生所有 ROLLUP 的数据），但是不会降低查询效率（只会更好）。 -* ROLLUP 的数据更新与 Base 表示完全同步的。用户无需关心这个问题。 -* ROLLUP 中列的聚合方式，与 Base 表完全相同。在创建 ROLLUP 无需指定，也不能修改。 -* 查询能否命中 ROLLUP 的一个必要条件（非充分条件）是，查询所涉及的**所有列**（包括 select list 和 where 中的查询条件列等）都存在于该 ROLLUP 的列中。否则，查询只能命中 Base 表。 -* 某些类型的查询（如 count(*)）在任何条件下，都无法命中 ROLLUP。具体参见接下来的 **聚合模型的局限性** 一节。 -* 可以通过 `EXPLAIN your_sql;` 命令获得查询执行计划，在执行计划中，查看是否命中 ROLLUP。 -* 可以通过 `DESC tbl_name ALL;` 语句显示 Base 表和所有已创建完成的 ROLLUP。 - -在这篇文档中可以查看 [查询如何命中 Rollup](hit-the-rollup) - -## 聚合模型的局限性 - -这里我们针对 Aggregate 模型（包括 Uniq 模型），来介绍下聚合模型的局限性。 - -在聚合模型中，模型对外展现的，是**最终聚合后的**数据。也就是说，任何还未聚合的数据（比如说两个不同导入批次的数据），必须通过某种方式，以保证对外展示的一致性。我们举例说明。 - -假设表结构如下： - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -|user\_id|LARGEINT||用户id| -|date|DATE||数据灌入日期| -|cost|BIGINT|SUM|用户总消费| - -假设存储引擎中有如下两个已经导入完成的批次的数据： - -**batch 1** - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|50| -|10002|2017-11-21|39| - -**batch 2** - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|1| -|10001|2017-11-21|5| -|10003|2017-11-22|22| - -可以看到，用户 10001 分属在两个导入批次中的数据还没有聚合。但是为了保证用户只能查询到如下最终聚合后的数据： - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|51| -|10001|2017-11-21|5| -|10002|2017-11-21|39| -|10003|2017-11-22|22| - -我们在查询引擎中加入了聚合算子，来保证数据对外的一致性。 - -另外，在聚合列（Value）上，执行与聚合类型不一致的聚合类查询时，要注意语意。比如我们在如上示例中执行如下查询： - -`SELECT MIN(cost) FROM table;` - -得到的结果是 5，而不是 1。 - -同时，这种一致性保证，在某些查询中，会极大的降低查询效率。 - -我们以最基本的 count(*) 查询为例： - -`SELECT COUNT(*) FROM table;` - -在其他数据库中，这类查询都会很快的返回结果。因为在实现上，我们可以通过如“导入时对行进行计数，保存count的统计信息”，或者在查询时“仅扫描某一列数据，获得count值”的方式，只需很小的开销，即可获得查询结果。但是在 Doris 的聚合模型中，这种查询的开销**非常大**。 - -我们以刚才的数据为例： - -**batch 1** - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|50| -|10002|2017-11-21|39| - -**batch 2** - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|1| -|10001|2017-11-21|5| -|10003|2017-11-22|22| - -因为最终的聚合结果为： - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|51| -|10001|2017-11-21|5| -|10002|2017-11-21|39| -|10003|2017-11-22|22| - -所以，`select count(*) from table;` 的正确结果应该为 **4**。但如果我们只扫描 `user_id` 这一列，如果加上查询时聚合，最终得到的结果是 **3**（10001, 10002, 10003）。而如果不加查询时聚合，则得到的结果是 **5**（两批次一共5行数据）。可见这两个结果都是不对的。 - -为了得到正确的结果，我们必须同时读取 `user_id` 和 `date` 这两列的数据，**再加上查询时聚合**，才能返回 **4** 这个正确的结果。也就是说，在 count(\*) 查询中，Doris 必须扫描所有的 AGGREGATE KEY 列（这里就是 `user_id` 和 `date`），并且聚合后，才能得到语意正确的结果。当聚合列非常多时，count(\*) 查询需要扫描大量的数据。 - -因此，当业务上有频繁的 count(\*) 查询时，我们建议用户通过增加一个**值恒为 1 的，聚合类型为 SUM 的列来模拟 count(\*)**。如刚才的例子中的表结构，我们修改如下： - -|ColumnName|Type|AggreateType|Comment| -|---|---|---|---| -|user\_id|BIGINT||用户id| -|date|DATE||数据灌入日期| -|cost|BIGINT|SUM|用户总消费| -|count|BIGINT|SUM|用于计算count| - -增加一个 count 列，并且导入数据中，该列值**恒为 1**。则 `select count(*) from table;` 的结果等价于 `select sum(count) from table;`。而后者的查询效率将远高于前者。不过这种方式也有使用限制，就是用户需要自行保证，不会重复导入 AGGREGATE KEY 列都相同的行。否则，`select sum(count) from table;` 只能表述原始导入的行数，而不是 `select count(*) from table;` 的语义。 - -另一种方式，就是 **将如上的 `count` 列的聚合类型改为 REPLACE，且依然值恒为 1**。那么 `select sum(count) from table;` 和 `select count(*) from table;` 的结果将是一致的。并且这种方式，没有导入重复行的限制。 - -### Duplicate 模型 - -Duplicate 模型没有聚合模型的这个局限性。因为该模型不涉及聚合语意，在做 count(*) 查询时，任意选择一列查询，即可得到语意正确的结果。 - -## 数据模型的选择建议 - -因为数据模型在建表时就已经确定，且**无法修改**。所以，选择一个合适的数据模型**非常重要**。 - -1. Aggregate 模型可以通过预聚合，极大地降低聚合查询时所需扫描的数据量和查询的计算量，非常适合有固定模式的报表类查询场景。但是该模型对 count(*) 查询很不友好。同时因为固定了 Value 列上的聚合方式，在进行其他类型的聚合查询时，需要考虑语意正确性。 -2. Uniq 模型针对需要唯一主键约束的场景，可以保证主键唯一性约束。但是无法利用 ROLLUP 等预聚合带来的查询优势（因为本质是 REPLACE，没有 SUM 这种聚合方式）。 -3. Duplicate 适合任意维度的 Ad-hoc 查询。虽然同样无法利用预聚合的特性，但是不受聚合模型的约束，可以发挥列存模型的优势（只读取相关列，而不需要读取所有 Key 列）。 diff --git a/docs/documentation/cn/getting-started/data-partition.md b/docs/documentation/cn/getting-started/data-partition.md deleted file mode 100644 index 5593d5efeb60cd..00000000000000 --- a/docs/documentation/cn/getting-started/data-partition.md +++ /dev/null @@ -1,290 +0,0 @@ - - -# 数据划分 - -本文档主要介绍 Doris 的建表和数据划分，以及建表操作中可能遇到的问题和解决方法。 - -## 基本概念 - -在 Doris 中，数据都以表（Table）的形式进行逻辑上的描述。 - -### Row & Column - -一张表包括行（Row）和列（Column）。Row 即用户的一行数据。Column 用于描述一行数据中不同的字段。 - -Column 可以分为两大类：Key 和 Value。从业务角度看，Key 和 Value 可以分别对应维度列和指标列。从聚合模型的角度来说，Key 列相同的行，会聚合成一行。其中 Value 列的聚合方式由用户在建表时指定。关于更多聚合模型的介绍，可以参阅 [Doris 数据模型](./data-model-rollup.md)。 - -### Tablet & Partition - -在 Doris 的存储引擎中，用户数据被水平划分为若干个数据分片（Tablet，也称作数据分桶）。每个 Tablet 包含若干数据行。各个 Tablet 之间的数据没有交集，并且在物理上是独立存储的。 - -多个 Tablet 在逻辑上归属于不同的分区（Partition）。一个 Tablet 只属于一个 Partition。而一个 Partition 包含若干个 Tablet。因为 Tablet 在物理上是独立存储的，所以可以视为 Partition 在物理上也是独立。Tablet 是数据移动、复制等操作的最小物理存储单元。 - -若干个 Partition 组成一个 Table。Partition 可以视为是逻辑上最小的管理单元。数据的导入与删除，都可以或仅能针对一个 Partition 进行。 - -## 数据划分 - -我们以一个建表操作来说明 Doris 的数据划分。 - -Doris 的建表是一个同步命令，命令返回成功，即表示建表成功。 - -可以通过 `HELP CREATE TABLE;` 查看更多帮助。 - -本小节通过一个例子，来介绍 Doris 的建表方式。 - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `date` DATE NOT NULL COMMENT "数据灌入日期时间", - `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", - `city` VARCHAR(20) COMMENT "用户所在城市", - `age` SMALLINT COMMENT "用户年龄", - `sex` TINYINT COMMENT "用户性别", - `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", - `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", - `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" -) -ENGINE=olap -AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) -PARTITION BY RANGE(`date`) -( - PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), - PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), - PARTITION `p201703` VALUES LESS THAN ("2017-04-01") -) -DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 -PROPERTIES -( - "replication_num" = "3", - "storage_medium" = "SSD", - "storage_cooldown_time" = "2018-01-01 12:00:00" -); - -``` - -### 列定义 - -这里我们只以 AGGREGATE KEY 数据模型为例进行说明。更多数据模型参阅 [Doris 数据模型](./data-model-rollup.md)。 - -列的基本类型，可以通过在 mysql-client 中执行 `HELP CREATE TABLE;` 查看。 - -AGGREGATE KEY 数据模型中，所有没有指定聚合方式（SUM、REPLACE、MAX、MIN）的列视为 Key 列。而其余则为 Value 列。 - -定义列时，可参照如下建议： - -1. Key 列必须在所有 Value 列之前。 -2. 尽量选择整型类型。因为整型类型的计算和查找比较效率远高于字符串。 -3. 对于不同长度的整型类型的选择原则，遵循 **够用即可**。 -4. 对于 VARCHAR 和 STRING 类型的长度，遵循 **够用即可**。 -5. 所有列的总字节长度（包括 Key 和 Value）不能超过 100KB。 - -### 分区与分桶 - -Doris 支持两层的数据划分。第一层是 Partition，仅支持 Range 的划分方式。第二层是 Bucket（Tablet），仅支持 Hash 的划分方式。 - -也可以仅使用一层分区。使用一层分区时，只支持 Bucket 划分。 - -1. Partition - - * Partition 列可以指定一列或多列。分区类必须为 KEY 列。多列分区的使用方式在后面 **多列分区** 小结介绍。 - * 不论分区列是什么类型，在写分区值时，都需要加双引号。 - * 分区列通常为时间列，以方便的管理新旧数据。 - * 分区数量理论上没有上限。 - * 当不使用 Partition 建表时，系统会自动生成一个和表名同名的，全值范围的 Partition。该 Partition 对用户不可见，并且不可删改。 - * Partition 支持通过 `VALUES LESS THAN (...)` 仅指定上界，系统会将前一个分区的上界作为该分区的下界，生成一个左闭右开的区间。通过，也支持通过 `VALUES [...)` 指定同时指定上下界，生成一个左闭右开的区间。 - - * 通过 `VALUES [...)` 同时指定上下界比较容易理解。这里举例说明，当使用 `VALUES LESS THAN (...)` 语句进行分区的增删操作时，分区范围的变化情况： - - * 如上示例，当建表完成后，会自动生成如下3个分区： - - ``` - p201701: [MIN_VALUE, 2017-02-01) - p201702: [2017-02-01, 2017-03-01) - p201703: [2017-03-01, 2017-04-01) - ``` - - * 当我们增加一个分区 p201705 VALUES LESS THAN ("2017-06-01")，分区结果如下： - - ``` - p201701: [MIN_VALUE, 2017-02-01) - p201702: [2017-02-01, 2017-03-01) - p201703: [2017-03-01, 2017-04-01) - p201705: [2017-04-01, 2017-06-01) - ``` - - * 此时我们删除分区 p201703，则分区结果如下： - - ``` - p201701: [MIN_VALUE, 2017-02-01) - p201702: [2017-02-01, 2017-03-01) - p201705: [2017-04-01, 2017-06-01) - ``` - - > 注意到 p201702 和 p201705 的分区范围并没有发生变化，而这两个分区之间，出现了一个空洞：[2017-03-01, 2017-04-01)。即如果导入的数据范围在这个空洞范围内，是无法导入的。 - - * 继续删除分区 p201702，分区结果如下： - - ``` - p201701: [MIN_VALUE, 2017-02-01) - p201705: [2017-04-01, 2017-06-01) - 空洞范围变为：[2017-02-01, 2017-04-01) - ``` - - * 现在增加一个分区 p201702new VALUES LESS THAN ("2017-03-01")，分区结果如下： - - ``` - p201701: [MIN_VALUE, 2017-02-01) - p201702new: [2017-02-01, 2017-03-01) - p201705: [2017-04-01, 2017-06-01) - ``` - - > 可以看到空洞范围缩小为：[2017-03-01, 2017-04-01) - - * 现在删除分区 p201701，并添加分区 p201612 VALUES LESS THAN ("2017-01-01")，分区结果如下： - - ``` - p201612: [MIN_VALUE, 2017-01-01) - p201702new: [2017-02-01, 2017-03-01) - p201705: [2017-04-01, 2017-06-01) - ``` - - > 即出现了一个新的空洞：[2017-01-01, 2017-02-01) - - 综上，分区的删除不会改变已存在分区的范围。删除分区可能出现空洞。通过 `VALUES LESS THAN` 语句增加分区时，分区的下界紧接上一个分区的上界。 - - 不可添加范围重叠的分区。 - -2. Bucket - - * 如果使用了 Partition，则 `DISTRIBUTED ...` 语句描述的是数据在**各个分区内**的划分规则。如果不使用 Partition，则描述的是对整个表的数据的划分规则。 - * 分桶列可以是多列，但必须为 Key 列。分桶列可以和 Partition 列相同或不同。 - * 分桶列的选择，是在 **查询吞吐** 和 **查询并发** 之间的一种权衡： - - 1. 如果选择多个分桶列，则数据分布更均匀。但如果查询条件不包含所有分桶列的等值条件的话，一个查询会扫描所有分桶。这样查询的吞吐会增加，但是单个查询的延迟也会增加。这个方式适合大吞吐低并发的查询场景。 - 2. 如果仅选择一个或少数分桶列，则点查询可以仅查询一个分桶。这种方式适合高并发的点查询场景。 - - * 分桶的数量理论上没有上限。 - -3. 关于 Partition 和 Bucket 的数量和数据量的建议。 - - * 一个表的 Tablet 总数量等于 (Partition num * Bucket num)。 - * 一个表的 Tablet 数量，在不考虑扩容的情况下，推荐略多于整个集群的磁盘数量。 - * 单个 Tablet 的数据量理论上没有上下界，但建议在 1G - 10G 的范围内。如果单个 Tablet 数据量过小，则数据的聚合效果不佳，且元数据管理压力大。如果数据量过大，则不利于副本的迁移、补齐，且会增加 Schema Change 或者 Rollup 操作失败重试的代价（这些操作失败重试的粒度是 Tablet）。 - * 当 Tablet 的数据量原则和数量原则冲突时，建议优先考虑数据量原则。 - * 在建表时，每个分区的 Bucket 数量统一指定。但是在动态增加分区时（`ADD PARTITION`），可以单独指定新分区的 Bucket 数量。可以利用这个功能方便的应对数据缩小或膨胀。 - * 一个 Partition 的 Bucket 数量一旦指定，不可更改。所以在确定 Bucket 数量时，需要预先考虑集群扩容的情况。比如当前只有 3 台 host，每台 host 有 1 块盘。如果 Bucket 的数量只设置为 3 或更小，那么后期即使再增加机器，也不能提高并发度。 - * 举一些例子：假设在有10台BE，每台BE一块磁盘的情况下。如果一个表总大小为 500MB，则可以考虑4-8个分片。5GB：8-16个。50GB：32个。500GB：建议分区，每个分区大小在 50GB 左右，每个分区16-32个分片。5TB：建议分区，每个分区大小在 50GB 左右，每个分区16-32个分片。 - - > 注：表的数据量可以通过 `show data` 命令查看，结果除以副本数，即表的数据量。 - -#### 多列分区 - -Doris 支持指定多列作为分区列，示例如下： - -``` -PARTITION BY RANGE(`date`, `id`) -( - PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), - PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), - PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") -) -``` - -在以上示例中，我们指定 `date`(DATE 类型) 和 `id`(INT 类型) 作为分区列。以上示例最终得到的分区如下： - -``` -* p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) -* p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) -* p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) -``` - -注意，最后一个分区用户缺省只指定了 `date` 列的分区值，所以 `id` 列的分区值会默认填充 `MIN_VALUE`。当用户插入数据时，分区列值会按照顺序依次比较，最终得到对应的分区。举例如下： - -``` -* 数据 --> 分区 -* 2017-01-01, 200 --> p201701_1000 -* 2017-01-01, 2000 --> p201701_1000 -* 2017-02-01, 100 --> p201701_1000 -* 2017-02-01, 2000 --> p201702_2000 -* 2017-02-15, 5000 --> p201702_2000 -* 2017-03-01, 2000 --> p201703_all -* 2017-03-10, 1 --> p201703_all -* 2017-04-01, 1000 --> 无法导入 -* 2017-05-01, 1000 --> 无法导入 -``` - -### PROPERTIES - -在建表语句的最后 PROPERTIES 中，可以指定以下两个参数： - -1. replication_num - - * 每个 Tablet 的副本数量。默认为3，建议保持默认即可。在建表语句中，所有 Partition 中的 Tablet 副本数量统一指定。而在增加新分区时，可以单独指定新分区中 Tablet 的副本数量。 - * 副本数量可以在运行时修改。强烈建议保持奇数。 - * 最大副本数量取决于集群中独立 IP 的数量（注意不是 BE 数量）。Doris 中副本分布的原则是，不允许同一个 Tablet 的副本分布在同一台物理机上，而识别物理机即通过 IP。所以，即使在同一台物理机上部署了 3 个或更多 BE 实例，如果这些 BE 的 IP 相同，则依然只能设置副本数为 1。 - * 对于一些小，并且更新不频繁的维度表，可以考虑设置更多的副本数。这样在 Join 查询时，可以有更大的概率进行本地数据 Join。 - -2. storage_medium & storage\_cooldown\_time - - * BE 的数据存储目录可以显式的指定为 SSD 或者 HDD（通过 .SSD 或者 .HDD 后缀区分）。建表时，可以统一指定所有 Partition 初始存储的介质。注意，后缀作用是显式指定磁盘介质，而不会检查是否与实际介质类型相符。 - * 默认初始存储介质可通过fe的配置文件 `fe.conf` 中指定 `default_storage_medium=xxx`，如果没有指定，则默认为 HDD。如果指定为 SSD，则数据初始存放在 SSD 上。 - * 如果没有指定 storage\_cooldown\_time，则默认 30 天后，数据会从 SSD 自动迁移到 HDD 上。如果指定了 storage\_cooldown\_time，则在到达 storage_cooldown_time 时间后，数据才会迁移。 - * 注意，当指定 storage_medium 时，该参数只是一个“尽力而为”的设置。即使集群内没有设置 SSD 存储介质，也不会报错，而是自动存储在可用的数据目录中。同样，如果 SSD 介质不可访问、空间不足，都可能导致数据初始直接存储在其他可用介质上。而数据到期迁移到 HDD 时，如果 HDD 介质不可访问、空间不足，也可能迁移失败（但是会不断尝试）。 - -### ENGINE - -本示例中，ENGINE 的类型是 olap，即默认的 ENGINE 类型。在 Doris 中，只有这个 ENGINE 类型是由 Doris 负责数据管理和存储的。其他 ENGINE 类型，如 mysql、broker、es 等等，本质上只是对外部其他数据库或系统中的表的映射，以保证 Doris 可以读取这些数据。而 Doris 本身并不创建、管理和存储任何非 olap ENGINE 类型的表和数据。 - -### 其他 - - `IF NOT EXISTS` 表示如果没有创建过该表，则创建。注意这里只判断表名是否存在，而不会判断新建表结构是否与已存在的表结构相同。所以如果存在一个同名但不同构的表，该命令也会返回成功，但并不代表已经创建了新的表和新的结构。 - -## 常见问题 - -### 建表操作常见问题 - -1. 如果在较长的建表语句中出现语法错误，可能会出现语法错误提示不全的现象。这里罗列可能的语法错误供手动纠错： - - * 语法结构错误。请仔细阅读 `HELP CREATE TABLE;`，检查相关语法结构。 - * 保留字。当用户自定义名称遇到保留字时，需要用反引号 `` 引起来。建议所有自定义名称使用这个符号引起来。 - * 中文字符或全角字符。非 utf8 编码的中文字符，或隐藏的全角字符（空格，标点等）会导致语法错误。建议使用带有显示不可见字符的文本编辑器进行检查。 - -2. `Failed to create partition [xxx] . Timeout` - - Doris 建表是按照 Partition 粒度依次创建的。当一个 Partition 创建失败时，可能会报这个错误。即使不使用 Partition，当建表出现问题时，也会报 `Failed to create partition`，因为如前文所述，Doris 会为没有指定 Partition 的表创建一个不可更改的默认的 Partition。 - - 当遇到这个错误是，通常是 BE 在创建数据分片时遇到了问题。可以参照以下步骤排查： - - 1. 在 fe.log 中，查找对应时间点的 `Failed to create partition` 日志。在该日志中，会出现一系列类似 `{10001-10010}` 字样的数字对。数字对的第一个数字表示 Backend ID，第二个数字表示 Tablet ID。如上这个数字对，表示 ID 为 10001 的 Backend 上，创建 ID 为 10010 的 Tablet 失败了。 - 2. 前往对应 Backend 的 be.INFO 日志，查找对应时间段内，tablet id 相关的日志，可以找到错误信息。 - 3. 以下罗列一些常见的 tablet 创建失败错误，包括但不限于： - * BE 没有收到相关 task，此时无法在 be.INFO 中找到 tablet id 相关日志。或者 BE 创建成功，但汇报失败。以上问题，请参阅 [部署与升级文档] 检查 FE 和 BE 的连通性。 - * 预分配内存失败。可能是表中一行的字节长度超过了 100KB。 - * `Too many open files`。打开的文件句柄数超过了 Linux 系统限制。需修改 Linux 系统的句柄数限制。 - - 如果创建数据分片时超时，也可以通过在 fe.conf 中设置 `tablet_create_timeout_second=xxx` 以及 `max_create_table_timeout_second=xxx` 来延长超时时间。其中 `tablet_create_timeout_second` 默认是1秒, `max_create_table_timeout_second` 默认是60秒，总体的超时时间为min(tablet_create_timeout_second * replication_num, max_create_table_timeout_second); - -3. 建表命令长时间不返回结果。 - - Doris 的建表命令是同步命令。该命令的超时时间目前设置的比较简单，即（tablet num * replication num）秒。如果创建较多的数据分片，并且其中有分片创建失败，则可能导致等待较长超时后，才会返回错误。 - - 正常情况下，建表语句会在几秒或十几秒内返回。如果超过一分钟，建议直接取消掉这个操作，前往 FE 或 BE 的日志查看相关错误。 diff --git a/docs/documentation/cn/getting-started/hit-the-rollup.md b/docs/documentation/cn/getting-started/hit-the-rollup.md deleted file mode 100644 index ea9075c083c096..00000000000000 --- a/docs/documentation/cn/getting-started/hit-the-rollup.md +++ /dev/null @@ -1,287 +0,0 @@ - - -# Rollup 与查询 - -在 Doris 里 Rollup 作为一份聚合物化视图，其在查询中可以起到两个作用： - -* 索引 -* 聚合数据（仅用于聚合模型，即aggregate key） - -但是为了命中 Rollup 需要满足一定的条件，并且可以通过执行计划中 ScanNdoe 节点的 PreAggregation 的值来判断是否可以命中 Rollup，以及 Rollup 字段来判断命中的是哪一张 Rollup 表。 - -## 名词解释 - -Base：基表。 - -Rollup：一般指基于 Base 表创建的 Rollup 表，但在一些场景包括 Base 以及 Rollup 表。 - -## 索引 - -前面的查询实践中已经介绍过 Doris 的前缀索引，即 Doris 会把 Base/Rollup 表中的前 36 个字节（有 varchar 类型则可能导致前缀索引不满 36 个字节，varchar 会截断前缀索引，并且最多使用 varchar 的 20 个字节）在底层存储引擎单独生成一份排序的稀疏索引数据(数据也是排序的，用索引定位，然后在数据中做二分查找)，然后在查询的时候会根据查询中的条件来匹配每个 Base/Rollup 的前缀索引，并且选择出匹配前缀索引最长的一个 Base/Rollup。 - -``` - -----> 从左到右匹配 -+----+----+----+----+----+----+ -| c1 | c2 | c3 | c4 | c5 |... | -``` - -如上图，取查询中 where 以及 on 上下推到 ScanNode 的条件，从前缀索引的第一列开始匹配，检查条件中是否有这些列，有则累计匹配的长度，直到匹配不上或者36字节结束（varchar类型的列只能匹配20个字节，并且会匹配不足36个字节截断前缀索引），然后选择出匹配长度最长的一个 Base/Rollup，下面举例说明，创建了一张Base表以及四张rollup： - -``` -+---------------+-------+--------------+------+-------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+---------------+-------+--------------+------+-------+---------+-------+ -| test | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup_index1 | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup_index2 | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k1 | TINYINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup_index3 | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup_index4 | k4 | BIGINT | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -+---------------+-------+--------------+------+-------+---------+-------+ -``` - -三张表的前缀索引分别为 - -``` -Base(k1 ,k2, k3, k4, k5, k6, k7) - -rollup_index1(k9)，rollup_index2(k9) - -rollup_index3(k4, k5, k6, k1, k2, k3, k7) - -rollup_index4(k4, k6, k5, k1, k2, k3, k7) -``` - -能用的上前缀索引的列上的条件需要是 `=` `<` `>` `<=` `>=` `in` `between` 这些并且这些条件是并列的且关系使用 `and` 连接，对于`or`、`!=` 等这些不能命中，然后看以下查询： - - -`SELECT * FROM test WHERE k1 = 1 AND k2 > 3;` - - -有 k1 以及 k2 上的条件，检查只有 Base 的第一列含有条件里的 k1，所以匹配最长的前缀索引即 test，explain一下： - -``` -| 0:OlapScanNode -| TABLE: test -| PREAGGREGATION: OFF. Reason: No AggregateInfo -| PREDICATES: `k1` = 1, `k2` > 3 -| partitions=1/1 -| rollup: test -| buckets=1/10 -| cardinality=-1 -| avgRowSize=0.0 -| numNodes=0 -| tuple ids: 0 -``` - -再看以下查询： - -`SELECT * FROM test WHERE k4 = 1 AND k5 > 3;` - -有 k4 以及 k5 的条件，检查 rollup_index3、rollup_index4 的第一列含有 k4，但是 rollup_index3 的第二列含有k5，所以匹配的前缀索引最长。 - -``` -| 0:OlapScanNode -| TABLE: test -| PREAGGREGATION: OFF. Reason: No AggregateInfo -| PREDICATES: `k4` = 1, `k5` > 3 -| partitions=1/1 -| rollup: rollup_index3 -| buckets=10/10 -| cardinality=-1 -| avgRowSize=0.0 -| numNodes=0 -| tuple ids: 0 -``` - -现在我们尝试匹配含有 varchar 列上的条件，如下： - -`SELECT * FROM test WHERE k9 IN ("xxx", "yyyy") AND k1 = 10;` - -有 k9 以及 k1 两个条件，rollup_index1 以及 rollup_index2 的第一列都含有 k9，按理说这里选择这两个 rollup 都可以命中前缀索引并且效果是一样的随机选择一个即可（因为这里 varchar 刚好20个字节，前缀索引不足36个字节被截断），但是当前策略这里还会继续匹配 k1，因为 rollup_index1 的第二列为 k1，所以选择了 rollup_index1，其实后面的 k1 条件并不会起到加速的作用。(如果对于前缀索引外的条件需要其可以起到加速查询的目的，可以通过建立 Bloom Filter 过滤器加速。一般对于字符串类型建立即可，因为 Doris 针对列存在 Block 级别对于整形、日期已经有 Min/Max 索引) 以下是 explain 的结果。 - -``` -| 0:OlapScanNode -| TABLE: test -| PREAGGREGATION: OFF. Reason: No AggregateInfo -| PREDICATES: `k9` IN ('xxx', 'yyyy'), `k1` = 10 -| partitions=1/1 -| rollup: rollup_index1 -| buckets=1/10 -| cardinality=-1 -| avgRowSize=0.0 -| numNodes=0 -| tuple ids: 0 -``` - -最后看一个多张Rollup都可以命中的查询： - -`SELECT * FROM test WHERE k4 < 1000 AND k5 = 80 AND k6 >= 10000;` - -有 k4,k5,k6 三个条件，rollup_index3 以及 rollup_index4 的前3列分别含有这三列，所以两者匹配的前缀索引长度一致，选取两者都可以，当前默认的策略为选取了比较早创建的一张 rollup，这里为 rollup_index3。 - -``` -| 0:OlapScanNode -| TABLE: test -| PREAGGREGATION: OFF. Reason: No AggregateInfo -| PREDICATES: `k4` < 1000, `k5` = 80, `k6` >= 10000.0 -| partitions=1/1 -| rollup: rollup_index3 -| buckets=10/10 -| cardinality=-1 -| avgRowSize=0.0 -| numNodes=0 -| tuple ids: 0 -``` - -如果稍微修改上面的查询为： - -`SELECT * FROM test WHERE k4 < 1000 AND k5 = 80 OR k6 >= 10000;` - -则这里的查询不能命中前缀索引。（甚至 Doris 存储引擎内的任何 Min/Max,BloomFilter 索引都不能起作用) - -## 聚合数据 - -当然一般的聚合物化视图其聚合数据的功能是必不可少的，这类物化视图对于聚合类查询或报表类查询都有非常大的帮助，要命中聚合物化视图需要下面一些前提： - -1. 查询或者子查询中涉及的所有列都存在一张独立的 Rollup 中。 -2. 如果查询或者子查询中有 Join，则 Join 的类型需要是 Inner join。 - -以下是可以命中Rollup的一些聚合查询的种类， - -| 列类型 查询类型 | Sum | Distinct/Count Distinct | Min | Max | Ndv | -|--------------|-------|-------------------------|-------|-------|-------| -| Key | false | true | true | true | true | -| Value(Sum) | true | false | false | false | false | -|Value(Replace)| false | false | false | false | false | -| Value(Min) | false | false | true | false | false | -| Value(Max) | false | false | false | true | false | - -如果符合上述条件，则针对聚合模型在判断命中 Rollup 的时候会有两个阶段： - -1. 首先通过条件匹配出命中前缀索引索引最长的 Rollup 表，见上述索引策略。 -2. 然后比较 Rollup 的行数，选择最小的一张 Rollup。 - -如下 Base 表以及 Rollup： - -``` -+-------------+-------+--------------+------+-------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+-------------+-------+--------------+------+-------+---------+-------+ -| test_rollup | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup2 | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup1 | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -+-------------+-------+--------------+------+-------+---------+-------+ -``` - -看以下查询： - -`SELECT SUM(k11) FROM test_rollup WHERE k1 = 10 AND k2 > 200 AND k3 in (1,2,3);` - -首先判断查询是否可以命中聚合的 Rollup表，经过查上面的图是可以的，然后条件中含有 k1,k2,k3 三个条件，这三个条件 test_rollup、rollup1、rollup2 的前三列都含有，所以前缀索引长度一致，然后比较行数显然 rollup2 的聚合程度最高行数最少所以选取 rollup2。 - -``` -| 0:OlapScanNode | -| TABLE: test_rollup | -| PREAGGREGATION: ON | -| PREDICATES: `k1` = 10, `k2` > 200, `k3` IN (1, 2, 3) | -| partitions=1/1 | -| rollup: rollup2 | -| buckets=1/10 | -| cardinality=-1 | -| avgRowSize=0.0 | -| numNodes=0 | -| tuple ids: 0 | -``` diff --git a/docs/documentation/cn/getting-started/index.rst b/docs/documentation/cn/getting-started/index.rst deleted file mode 100644 index dcb7cdb916ceb9..00000000000000 --- a/docs/documentation/cn/getting-started/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -============= -开始使用 -============= - -.. toctree:: - - basic-usage.md - advance-usage.md - best-practice.md - data-partition.md - data-model-rollup.md - hit-the-rollup.md diff --git a/docs/documentation/cn/index.rst b/docs/documentation/cn/index.rst deleted file mode 100644 index d94ac3d1a5b77d..00000000000000 --- a/docs/documentation/cn/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -============= -中文 -============= - -欢迎帮助我们改进文档质量。 -你可以通过点击每个页面右上方的 "Edit on Github" 来直接在 Github 编辑文档，然后生成一个 Pull Request。 - -.. toctree:: - - downloads/index - installing/index - getting-started/index - administrator-guide/index - extending-doris/index - internal/index - sql-reference/index - developer-guide/index - community/index diff --git a/docs/documentation/cn/installing/compilation.md b/docs/documentation/cn/installing/compilation.md deleted file mode 100644 index 27535cb34f4b47..00000000000000 --- a/docs/documentation/cn/installing/compilation.md +++ /dev/null @@ -1,101 +0,0 @@ - - -# 编译 - -本文档主要介绍如何通过源码编译 Doris。 - -## 使用 Docker 开发镜像编译（推荐） - -### 使用现成的镜像 - -1. 下载 Docker 镜像 - - `$ docker pull apachedoris/doris-dev:build-env` - - 检查镜像下载完成： - - ``` - $ docker images - REPOSITORY TAG IMAGE ID CREATED SIZE - apachedoris/doris-dev build-env f8bc5d4024e0 21 hours ago 3.28GB - ``` - -注: 针对不同的 Doris 版本，需要下载对应的镜像版本 - -| image version | commit id | release version | -|---|---|---| -| apachedoris/doris-dev:build-env | before [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.8.x, 0.9.x | -| apachedoris/doris-dev:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) | 0.10.x, 0.11.x | -| apachedoris/doris-dev:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) or later | 0.12.x or later - -2. 运行镜像 - - `$ docker run -it apachedoris/doris-dev:build-env` - - 如果你希望编译本地 Doris 源码，则可以挂载路径： - - ``` - $ docker run -it -v /your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/ apachedoris/doris-dev:build-env - ``` - -3. 下载源码 - - 启动镜像后，你应该已经处于容器内。可以通过以下命令下载 Doris 源码（已挂载本地源码目录则不用）： - - ``` - $ wget https://dist.apache.org/repos/dist/dev/incubator/doris/xxx.tar.gz - or - $ git clone https://github.com/apache/incubator-doris.git - ``` - -4. 编译 Doris - - ``` - $ sh build.sh - ``` - - 编译完成后，产出文件在 `output/` 目录中。 - -### 自行编译开发环境镜像 - -你也可以自己创建一个 Doris 开发环境镜像，具体可参阅 `docker/README.md` 文件。 - - -## 直接编译（CentOS/Ubuntu） - -你可以在自己的 linux 环境中直接尝试编译 Doris。 - -1. 系统依赖 - - `GCC 5.3.1+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+` - - 如果使用Ubuntu 16.04 及以上系统 可以执行以下命令来安装依赖 - - `sudo apt-get install build-essential openjdk-8-jdk maven cmake byacc flex automake libtool-bin bison binutils-dev libiberty-dev` - - 安装完成后，自行设置环境变量 `PATH`, `JAVA_HOME` 等。 - -2. 编译 Doris - - ``` - $ sh build.sh - ``` - - 编译完成后，产出文件在 `output/` 目录中。 diff --git a/docs/documentation/cn/installing/index.rst b/docs/documentation/cn/installing/index.rst deleted file mode 100644 index 29e0e2f4a1d0d3..00000000000000 --- a/docs/documentation/cn/installing/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -=========== -编译与部署 -=========== - -.. toctree:: - - compilation.md - install-deploy.md - upgrade.md diff --git a/docs/documentation/cn/installing/install-deploy.md b/docs/documentation/cn/installing/install-deploy.md deleted file mode 100644 index 9207cc48c0be95..00000000000000 --- a/docs/documentation/cn/installing/install-deploy.md +++ /dev/null @@ -1,413 +0,0 @@ - - -# 安装与部署 - -该文档主要介绍了部署 Doris 所需软硬件环境、建议的部署方式、集群扩容缩容，以及集群搭建到运行过程中的常见问题。 -在阅读本文档前，请先根据编译文档编译 Doris。 - -## 软硬件需求 - -### 概述 - -Doris 作为一款开源的 MPP 架构 OLAP 数据库，能够运行在绝大多数主流的商用服务器上。为了能够充分运用 MPP 架构的并发优势，以及 Doris 的高可用特性，我们建议 Doris 的部署遵循以下需求： - -#### Linux 操作系统版本需求 - -| Linux 系统 | 版本 | -|---|---| -| CentOS | 7.1 及以上 | -| Ubuntu | 16.04 及以上 | - -#### 软件需求 - -| 软件 | 版本 | -|---|---| -| Java | 1.8 及以上 | -| GCC | 4.8.2 及以上 | - -#### 开发测试环境 - -| 模块 | CPU | 内存 | 磁盘 | 网络 | 实例数量 | -|---|---|---|---|---|---| -| Frontend | 8核+ | 8GB+ | SSD 或 SATA，10GB+ * | 千兆网卡 | 1 | -| Backend | 8核+ | 16GB+ | SSD 或 SATA，50GB+ * | 千兆网卡 | 1-3 * | - -#### 生产环境 - -| 模块 | CPU | 内存 | 磁盘 | 网络 | 实例数量（最低要求） | -|---|---|---|---|---|---| -| Frontend | 16核+ | 64GB+ | SSD 或 RAID 卡，100GB+ * | 万兆网卡 | 1-5 * | -| Backend | 16核+ | 64GB+ | SSD 或 SATA，100G+ * | 万兆网卡 | 10-100 * | - -> 注1： -> 1. FE 的磁盘空间主要用于存储元数据，包括日志和 image。通常从几百 MB 到几个 GB 不等。 -> 2. BE 的磁盘空间主要用于存放用户数据，总磁盘空间按用户总数据量 * 3（3副本）计算，然后再预留额外 40% 的空间用作后台 compaction 以及一些中间数据的存放。 -> 3. 一台机器上可以部署多个 BE 实例，但是**只能部署一个 FE**。如果需要 3 副本数据，那么至少需要 3 台机器各部署一个 BE 实例（而不是1台机器部署3个BE实例）。**多个FE所在服务器的时钟必须保持一致（允许最多5秒的时钟偏差）** -> 4. 测试环境也可以仅适用一个 BE 进行测试。实际生产环境，BE 实例数量直接决定了整体查询延迟。 -> 5. 所有部署节点关闭 Swap。 - -> 注2：FE 节点的数量 -> 1. FE 角色分为 Follower 和 Observer，（Leader 为 Follower 组中选举出来的一种角色，以下统称 Follower，具体含义见 [元数据设计文档](../internal/metadata-design)）。 -> 2. FE 节点数据至少为1（1 个 Follower）。当部署 1 个 Follower 和 1 个 Observer 时，可以实现读高可用。当部署 3 个 Follower 时，可以实现读写高可用（HA）。 -> 3. Follower 的数量**必须**为奇数，Observer 数量随意。 -> 4. 根据以往经验，当集群可用性要求很高是（比如提供在线业务），可以部署 3 个 Follower 和 1-3 个 Observer。如果是离线业务，建议部署 1 个 Follower 和 1-3 个 Observer。 - -* **通常我们建议 10 ~ 100 台左右的机器，来充分发挥 Doris 的性能（其中 3 台部署 FE（HA），剩余的部署 BE）** -* **当然，Doris的性能与节点数量及配置正相关。在最少4台机器（一台 FE，三台 BE，其中一台 BE 混部一个 Observer FE 提供元数据备份），以及较低配置的情况下，依然可以平稳的运行 Doris。** -* **如果 FE 和 BE 混部，需注意资源竞争问题，并保证元数据目录和数据目录分属不同磁盘。** - -#### Broker 部署 - -Broker 是用于访问外部数据源（如 hdfs）的进程。通常，在每台机器上部署一个 broker 实例即可。 - -#### 网络需求 - -Doris 各个实例直接通过网络进行通讯。以下表格展示了所有需要的端口 - -| 实例名称 | 端口名称 | 默认端口 | 通讯方向 | 说明 | -|---|---|---|---| ---| -| BE | be_port | 9060 | FE --> BE | BE 上 thrift server 的端口，用于接收来自 FE 的请求 | -| BE | webserver_port | 8040 | BE <--> BE | BE 上的 http server 的端口 | -| BE | heartbeat\_service_port | 9050 | FE --> BE | BE 上心跳服务端口（thrift），用于接收来自 FE 的心跳 | -| BE | brpc\_port* | 8060 | FE<-->BE, BE <--> BE | BE 上的 brpc 端口，用于 BE 之间通讯 | -| FE | http_port * | 8030 | FE <--> FE，用户 |FE 上的 http server 端口 | -| FE | rpc_port | 9020 | BE --> FE, FE <--> FE | FE 上的 thrift server 端口 | -| FE | query_port | 9030 | 用户 | FE 上的 mysql server 端口 | -| FE | edit\_log_port | 9010 | FE <--> FE | FE 上的 bdbje 之间通信用的端口 | -| Broker | broker\_ipc_port | 8000 | FE --> Broker, BE --> Broker | Broker 上的 thrift server，用于接收请求 | - -> 注： -> 1. 当部署多个 FE 实例时，要保证 FE 的 http\_port 配置相同。 -> 2. 部署前请确保各个端口在应有方向上的访问权限。 - -#### IP 绑定 - -因为有多网卡的存在，或因为安装过 docker 等环境导致的虚拟网卡的存在，同一个主机可能存在多个不同的 ip。当前 Doris 并不能自动识别可用 IP。所以当遇到部署主机上有多个 IP 时，必须通过 priority\_networks 配置项来强制指定正确的 IP。 - -priority\_networks 是 FE 和 BE 都有的一个配置，配置项需写在 fe.conf 和 be.conf 中。该配置项用于在 FE 或 BE 启动时，告诉进程应该绑定哪个IP。示例如下： - -`priority_networks=10.1.3.0/24` - -这是一种 [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) 的表示方法。FE 或 BE 会根据这个配置项来寻找匹配的IP，作为自己的 localIP。 - -**注意**：当配置完 priority\_networks 并启动 FE 或 BE 后，只是保证了 FE 或 BE 自身的 IP 进行了正确的绑定。而在使用 ADD BACKEND 或 ADD FRONTEND 语句中，也需要指定和 priority\_networks 配置匹配的 IP，否则集群无法建立。举例： - -BE 的配置为：`priority_networks=10.1.3.0/24` - -但是在 ADD BACKEND 时使用的是：`ALTER SYSTEM ADD BACKEND "192.168.0.1:9050";` - -则 FE 和 BE 将无法正常通信。 - -这时，必须 DROP 掉这个添加错误的 BE，重新使用正确的 IP 执行 ADD BACKEND。 - -FE 同理。 - -BROKER 当前没有，也不需要 priority\_networks 这个选项。Broker 的服务默认绑定在 0.0.0.0 上。只需在 ADD BROKER 时，执行正确可访问的 BROKER IP 即可。 - -## 集群部署 - -### 手动部署 - -#### FE 部署 - -* 拷贝 FE 部署文件到指定节点 - - 将源码编译生成的 output 下的 fe 文件夹拷贝到 FE 的节点指定部署路径下。 - -* 配置 FE - - 1. 配置文件为 conf/fe.conf。其中注意：`meta_dir`：元数据存放位置。默认在 fe/doris-meta/ 下。需**手动创建**该目录。 - 2. fe.conf 中 JAVA_OPTS 默认 java 最大堆内存为 4GB，建议生产环境调整至 8G 以上。 - -* 启动FE - - `sh bin/start_fe.sh --daemon` - - FE进程启动进入后台执行。日志默认存放在 fe/log/ 目录下。如启动失败，可以通过查看 fe/log/fe.log 或者 fe/log/fe.out 查看错误信息。 - -* 如需部署多 FE，请参见 "FE 扩容和缩容" 章节 - -#### BE 部署 - -* 拷贝 BE 部署文件到所有要部署 BE 的节点 - - 将源码编译生成的 output 下的 be 文件夹拷贝到 BE 的节点的指定部署路径下。 - -* 修改所有 BE 的配置 - - 修改 be/conf/be.conf。主要是配置 `storage_root_path`：数据存放目录。默认在be/storage下，需要**手动创建**该目录。多个路径之间使用 `;` 分隔（最后一个目录后不要加 `;`）。 - -* 在 FE 中添加所有 BE 节点 - - BE 节点需要先在 FE 中添加，才可加入集群。可以使用 mysql-client 连接到 FE： - - `./mysql-client -h host -P port -uroot` - - 其中 host 为 FE 所在节点 ip；port 为 fe/conf/fe.conf 中的 query_port；默认使用 root 账户，无密码登录。 - - 登录后，执行以下命令来添加每一个 BE： - - `ALTER SYSTEM ADD BACKEND "host:port";` - - 如果使用多租户功能，则执行以下命令添加 BE: - - `ALTER SYSTEM ADD FREE BACKEND "host:port";` - - 其中 host 为 BE 所在节点 ip；port 为 be/conf/be.conf 中的 heartbeat_service_port。 - - 如果不添加 FREE 关键字，BE 默认进入自动生成的 cluster，添加了 FREE 关键字后新的 BE 不属于任何 cluster，这样创建新 cluster 的时候就可以从这些空闲的be中选取，详细见[多租户设计文档](../administrator-guide/operation/multi-tenant.md) - -* 启动 BE - - `sh bin/start_be.sh --daemon` - - BE 进程将启动并进入后台执行。日志默认存放在 be/log/ 目录下。如启动失败，可以通过查看 be/log/be.log 或者 be/log/be.out 查看错误信息。 - -* 查看BE状态 - - 使用 mysql-client 连接到 FE，并执行 `SHOW PROC '/backends';` 查看 BE 运行情况。如一切正常，`isAlive` 列应为 `true`。 - -#### （可选）FS_Broker 部署 - -Broker 以插件的形式，独立于 Doris 部署。如果需要从第三方存储系统导入数据，需要部署相应的 Broker，默认提供了读取 HDFS 和百度云 BOS 的 fs_broker。fs_broker 是无状态的，建议每一个 FE 和 BE 节点都部署一个 Broker。 - -* 拷贝源码 fs_broker 的 output 目录下的相应 Broker 目录到需要部署的所有节点上。建议和 BE 或者 FE 目录保持同级。 - -* 修改相应 Broker 配置 - - 在相应 broker/conf/ 目录下对应的配置文件中，可以修改相应配置。 - - * 启动 Broker - - `sh bin/start_broker.sh --daemon` 启动 Broker。 - -* 添加 Broker - - 要让 Doris 的 FE 和 BE 知道 Broker 在哪些节点上，通过 sql 命令添加 Broker 节点列表。 - - 使用 mysql-client 连接启动的 FE，执行以下命令： - - `ALTER SYSTEM ADD BROKER broker_name "host1:port1","host2:port2",...;` - - 其中 host 为 Broker 所在节点 ip；port 为 Broker 配置文件中的 broker\_ipc\_port。 - -* 查看 Broker 状态 - - 使用 mysql-client 连接任一已启动的 FE，执行以下命令查看 Broker 状态：`SHOW PROC "/brokers";` - -**注：在生产环境中，所有实例都应使用守护进程启动，以保证进程退出后，会被自动拉起，如 [Supervisor](http://supervisord.org/)。如需使用守护进程启动，在 0.9.0 及之前版本中，需要修改各个 start_xx.sh 脚本，去掉最后的 & 符号**。从 0.10.0 版本开始，直接调用 `sh start_xx.sh` 启动即可。也可参考 [这里](https://www.cnblogs.com/lenmom/p/9973401.html) - -## 扩容缩容 - -Doris 可以很方便的扩容和缩容 FE、BE、Broker 实例。 - -### FE 扩容和缩容 - -可以通过将 FE 扩容至 3 个一上节点来实现 FE 的高可用。 - -用户可以通过 mysql 客户端登陆 Master FE。通过: - -`SHOW PROC '/frontends';` - -来查看当前 FE 的节点情况。 - -也可以通过前端页面连接：```http://fe_hostname:fe_http_port/frontend``` 或者 ```http://fe_hostname:fe_http_port/system?path=//frontends``` 来查看 FE 节点的情况。 - -以上方式，都需要 Doris 的 root 用户权限。 - -FE 节点的扩容和缩容过程，不影响当前系统运行。 - -#### 增加 FE 节点 - -FE 分为 Leader，Follower 和 Observer 三种角色。 默认一个集群，只能有一个 Leader，可以有多个 Follower 和 Observer。其中 Leader 和 Follower 组成一个 Paxos 选择组，如果 Leader 宕机，则剩下的 Follower 会自动选出新的 Leader，保证写入高可用。Observer 同步 Leader 的数据，但是不参加选举。如果只部署一个 FE，则 FE 默认就是 Leader。 - -第一个启动的 FE 自动成为 Leader。在此基础上，可以添加若干 Follower 和 Observer。 - -添加 Follower 或 Observer。使用 mysql-client 连接到已启动的 FE，并执行： - -`ALTER SYSTEM ADD FOLLOWER "host:port";` - -或 - -`ALTER SYSTEM ADD OBSERVER "host:port";` - -其中 host 为 Follower 或 Observer 所在节点 ip，port 为其配置文件 fe.conf 中的 edit\_log\_port。 - -配置及启动 Follower 或 Observer。Follower 和 Observer 的配置同 Leader 的配置。第一次启动时，需执行以下命令： - -`./bin/start_fe.sh --helper host:port --daemon` - -其中 host 为 Leader 所在节点 ip, port 为 Leader 的配置文件 fe.conf 中的 edit_log_port。--helper 参数仅在 follower 和 observer 第一次启动时才需要。 - -查看 Follower 或 Observer 运行状态。使用 mysql-client 连接到任一已启动的 FE，并执行：SHOW PROC '/frontends'; 可以查看当前已加入集群的 FE 及其对应角色。 - -> FE 扩容注意事项： -> 1. Follower FE（包括 Leader）的数量必须为奇数，建议最多部署 3 个组成高可用（HA）模式即可。 -> 2. 当 FE 处于高可用部署时（1个 Leader，2个 Follower），我们建议通过增加 Observer FE 来扩展 FE 的读服务能力。当然也可以继续增加 Follower FE，但几乎是不必要的。 -> 3. 通常一个 FE 节点可以应对 10-20 台 BE 节点。建议总的 FE 节点数量在 10 个以下。而通常 3 个即可满足绝大部分需求。 - -#### 删除 FE 节点 - -使用以下命令删除对应的 FE 节点： - -```ALTER SYSTEM DROP FOLLOWER[OBSERVER] "fe_host:edit_log_port";``` - -> FE 缩容注意事项： -> 1. 删除 Follower FE 时，确保最终剩余的 Follower（包括 Leader）节点为奇数。 - -### BE 扩容和缩容 - -用户可以通过 mysql-client 登陆 Leader FE。通过: - -```SHOW PROC '/backends';``` - -来查看当前 BE 的节点情况。 - -也可以通过前端页面连接：```http://fe_hostname:fe_http_port/backend``` 或者 ```http://fe_hostname:fe_http_port/system?path=//backends``` 来查看 BE 节点的情况。 - -以上方式，都需要 Doris 的 root 用户权限。 - -BE 节点的扩容和缩容过程，不影响当前系统运行以及正在执行的任务，并且不会影响当前系统的性能。数据均衡会自动进行。根据集群现有数据量的大小，集群会在几个小时到1天不等的时间内，恢复到负载均衡的状态。集群负载情况，可以参见 [Tablet 负载均衡文档](../administrator-guide/operation/tablet-repair-and-balance.md)。 - -#### 增加 BE 节点 - -BE 节点的增加方式同 **BE 部署** 一节中的方式，通过 `ALTER SYSTEM ADD BACKEND` 命令增加 BE 节点。 - -> BE 扩容注意事项： -> 1. BE 扩容后，Doris 会自动根据负载情况，进行数据均衡，期间不影响使用。 - -#### 删除 BE 节点 - -删除 BE 节点有两种方式：DROP 和 DECOMMISSION - -DROP 语句如下： - -```ALTER SYSTEM DROP BACKEND "be_host:be_heartbeat_service_port";``` - -**注意：DROP BACKEND 会直接删除该 BE，并且其上的数据将不能再恢复！！！所以我们强烈不推荐使用 DROP BACKEND 这种方式删除 BE 节点。当你使用这个语句时，会有对应的防误操作提示。** - -DECOMMISSION 语句如下： - -```ALTER SYSTEM DECOMMISSION BACKEND "be_host:be_heartbeat_service_port";``` - -> DECOMMISSION 命令说明： -> 1. 该命令用于安全删除 BE 节点。命令下发后，Doris 会尝试将该 BE 上的数据向其他 BE 节点迁移，当所有数据都迁移完成后，Doris 会自动删除该节点。 -> 2. 该命令是一个异步操作。执行后，可以通过 ```SHOW PROC '/backends';``` 看到该 BE 节点的 isDecommission 状态为 true。表示该节点正在进行下线。 -> 3. 该命令**不一定执行成功**。比如剩余 BE 存储空间不足以容纳下线 BE 上的数据，或者剩余机器数量不满足最小副本数时，该命令都无法完成，并且 BE 会一直处于 isDecommission 为 true 的状态。 -> 4. DECOMMISSION 的进度，可以通过 ```SHOW PROC '/backends';``` 中的 TabletNum 查看，如果正在进行，TabletNum 将不断减少。 -> 5. 该操作可以通过: -> ```CANCEL DECOMMISSION BACKEND "be_host:be_heartbeat_service_port";``` -> 命令取消。取消后，该 BE 上的数据将维持当前剩余的数据量。后续 Doris 重新进行负载均衡 - -**对于多租户部署环境下，BE 节点的扩容和缩容，请参阅 [多租户设计文档](../administrator-guide/operation/multi-tenant.md)。** - -### Broker 扩容缩容 - -Broker 实例的数量没有硬性要求。通常每台物理机部署一个即可。Broker 的添加和删除可以通过以下命令完成： - -```ALTER SYSTEM ADD BROKER broker_name "broker_host:broker_ipc_port";``` -```ALTER SYSTEM DROP BROKER broker_name "broker_host:broker_ipc_port";``` -```ALTER SYSTEM DROP ALL BROKER broker_name;``` - -Broker 是无状态的进程，可以随意启停。当然，停止后，正在其上运行的作业会失败，重试即可。 - -## 常见问题 - -### 进程相关 - -1. 如何确定 FE 进程启动成功 - - FE 进程启动后，会首先加载元数据，根据 FE 角色的不同，在日志中会看到 ```transfer from UNKNOWN to MASTER/FOLLOWER/OBSERVER```。最终会看到 ```thrift server started``` 日志，并且可以通过 mysql 客户端连接到 FE，则表示 FE 启动成功。 - - 也可以通过如下连接查看是否启动成功： - `http://fe_host:fe_http_port/api/bootstrap` - - 如果返回： - `{"status":"OK","msg":"Success"}` - - 则表示启动成功，其余情况，则可能存在问题。 - - > 注：如果在 fe.log 中查看不到启动失败的信息，也许在 fe.out 中可以看到。 - -2. 如何确定 BE 进程启动成功 - - BE 进程启动后，如果之前有数据，则可能有数分钟不等的数据索引加载时间。 - - 如果是 BE 的第一次启动，或者该 BE 尚未加入任何集群，则 BE 日志会定期滚动 ```waiting to receive first heartbeat from frontend``` 字样。表示 BE 还未通过 FE 的心跳收到 Master 的地址，正在被动等待。这种错误日志，在 FE 中 ADD BACKEND 并发送心跳后，就会消失。如果在接到心跳后，又重复出现 ``````master client, get client from cache failed.host: , port: 0, code: 7`````` 字样，说明 FE 成功连接了 BE，但 BE 无法主动连接 FE。可能需要检查 BE 到 FE 的 rpc_port 的连通性。 - - 如果 BE 已经被加入集群，日志中应该每隔 5 秒滚动来自 FE 的心跳日志：```get heartbeat, host: xx.xx.xx.xx, port: 9020, cluster id: xxxxxx```，表示心跳正常。 - - 其次，日志中应该每隔 10 秒滚动 ```finish report task success. return code: 0``` 的字样，表示 BE 向 FE 的通信正常。 - - 同时，如果有数据查询，应该能看到不停滚动的日志，并且有 ```execute time is xxx``` 日志，表示 BE 启动成功，并且查询正常。 - - 也可以通过如下连接查看是否启动成功： - `http://be_host:be_http_port/api/health` - - 如果返回： - `{"status": "OK","msg": "To Be Added"}` - - 则表示启动成功，其余情况，则可能存在问题。 - - > 注：如果在 be.INFO 中查看不到启动失败的信息，也许在 be.out 中可以看到。 - -3. 搭建系统后，如何确定 FE、BE 连通性正常 - - 首先确认 FE 和 BE 进程都已经单独正常启动，并确认已经通过 `ADD BACKEND` 或者 `ADD FOLLOWER/OBSERVER` 语句添加了所有节点。 - - 如果心跳正常，BE 的日志中会显示 ```get heartbeat, host: xx.xx.xx.xx, port: 9020, cluster id: xxxxxx```。如果心跳失败，在 FE 的日志中会出现 ```backend[10001] got Exception: org.apache.thrift.transport.TTransportException``` 类似的字样，或者其他 thrift 通信异常日志，表示 FE 向 10001 这个 BE 的心跳失败。这里需要检查 FE 向 BE host 的心跳端口的连通性。 - - 如果 BE 向 FE 的通信正常，则 BE 日志中会显示 ```finish report task success. return code: 0``` 的字样。否则会出现 ```master client, get client from cache failed``` 的字样。这种情况下，需要检查 BE 向 FE 的 rpc_port 的连通性。 - -4. Doris 各节点认证机制 - - 除了 Master FE 以外，其余角色节点（Follower FE，Observer FE，Backend），都需要通过 `ALTER SYSTEM ADD` 语句先注册到集群，然后才能加入集群。 - - Master FE 在第一次启动时，会在 doris-meta/image/VERSION 文件中生成一个 cluster_id。 - - FE 在第一次加入集群时，会首先从 Master FE 获取这个文件。之后每次 FE 之间的重新连接（FE 重启），都会校验自身 cluster id 是否与已存在的其它 FE 的 cluster id 相同。如果不同，则该 FE 会自动退出。 - - BE 在第一次接收到 Master FE 的心跳时，会从心跳中获取到 cluster id，并记录到数据目录的 `cluster_id` 文件中。之后的每次心跳都会比对 FE 发来的 cluster id。如果 cluster id 不相等，则 BE 会拒绝响应 FE 的心跳。 - - 心跳中同时会包含 Master FE 的 ip。当 FE 切主时，新的 Master FE 会携带自身的 ip 发送心跳给 BE，BE 会更新自身保存的 Master FE 的 ip。 - - > **priority\_network** - > - > priority\_network 是 FE 和 BE 都有一个配置，其主要目的是在多网卡的情况下，协助 FE 或 BE 识别自身 ip 地址。priority\_network 采用 CIDR 表示法：[RFC 4632](https://tools.ietf.org/html/rfc4632) - > - > 当确认 FE 和 BE 连通性正常后，如果仍然出现建表 Timeout 的情况，并且 FE 的日志中有 `backend does not found. host: xxx.xxx.xxx.xxx` 字样的错误信息。则表示 Doris 自动识别的 IP 地址有问题，需要手动设置 priority\_network 参数。 - > - > 出现这个问题的主要原因是：当用户通过 `ADD BACKEND` 语句添加 BE 后，FE 会识别该语句中指定的是 hostname 还是 IP。如果是 hostname，则 FE 会自动将其转换为 IP 地址并存储到元数据中。当 BE 在汇报任务完成信息时，会携带自己的 IP 地址。而如果 FE 发现 BE 汇报的 IP 地址和元数据中不一致时，就会出现如上错误。 - > - > 这个错误的解决方法：1）分别在 FE 和 BE 设置 **priority\_network** 参数。通常 FE 和 BE 都处于一个网段，所以该参数设置为相同即可。2）在 `ADD BACKEND` 语句中直接填写 BE 正确的 IP 地址而不是 hostname，以避免 FE 获取到错误的 IP 地址。 - -5. BE 进程文件句柄数 - - BE进程文件句柄数，受min_file_descriptor_number/max_file_descriptor_number两个参数控制。 - - 如果不在[min_file_descriptor_number, max_file_descriptor_number]区间内，BE进程启动会出错，可以使用ulimit进行设置。 - - min_file_descriptor_number的默认值为65536。 - - max_file_descriptor_number的默认值为131072. - - 举例而言：ulimit -n 65536; 表示将文件句柄设成65536。 - - 启动BE进程之后，可以通过 cat /proc/$pid/limits 查看进程实际生效的句柄数 diff --git a/docs/documentation/cn/installing/upgrade.md b/docs/documentation/cn/installing/upgrade.md deleted file mode 100644 index 50351c85a84092..00000000000000 --- a/docs/documentation/cn/installing/upgrade.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# 集群升级 - -Doris 可以通过滚动升级的方式，平滑进行升级。建议按照以下步骤进行安全升级。 - -> 注： -> 1. 以下方式均建立在高可用部署的情况下。即数据 3 副本，FE 高可用情况下。 - -## 测试 BE 升级正确性 - -1. 任意选择一个 BE 节点，部署最新的 palo_be 二进制文件。 -2. 重启 BE 节点，通过 BE 日志 be.INFO，查看是否启动成功。 -3. 如果启动失败，可以先排查原因。如果错误不可恢复，可以直接通过 DROP BACKEND 删除该 BE、清理数据后，使用上一个版本的 palo_be 重新启动 BE。然后重新 ADD BACKEND。（**该方法会导致丢失一个数据副本，请务必确保3副本完整的情况下，执行这个操作！！！**） - -## 测试 FE 元数据兼容性 - -0. **重要！!元数据兼容性异常很可能导致数据无法恢复！！** -1. 单独使用新版本部署一个测试用的 FE 进程（比如自己本地的开发机）。 -2. 修改测试用的 FE 的配置文件 fe.conf，将所有端口设置为**与线上不同**。 -3. 在 fe.conf 添加配置：cluster_id=123456 -4. 在 fe.conf 添加配置：metadata\_failure_recovery=true -5. 拷贝线上环境 Master FE 的元数据目录 palo-meta 到测试环境 -6. 将拷贝到测试环境中的 palo-meta/image/VERSION 文件中的 cluster_id 修改为 123456（即与第3步中相同） -7. 在测试环境中，运行 sh bin/start_fe.sh 启动 FE -8. 通过 FE 日志 fe.log 观察是否启动成功。 -9. 如果启动成功，运行 sh bin/stop_fe.sh 停止测试环境的 FE 进程。 -10. **以上 2-6 步的目的是防止测试环境的FE启动后，错误连接到线上环境中。** - -## 升级准备 - -1. 在完成数据正确性验证后，将 BE 和 FE 新版本的二进制文件分发到各自目录下。 -2. 通常小版本升级，BE 只需升级 palo_be；而 FE 只需升级 palo-fe.jar。如果是大版本升级，则可能需要升级其他文件（包括但不限于 bin/ lib/ 等等）如果你不清楚是否需要替换其他文件，建议全部替换。 - -## 滚动升级 - -1. 确认新版本的文件部署完成后。逐台重启 FE 和 BE 实例即可。 -2. 建议逐台重启 BE 后，再逐台重启 FE。因为通常 Doris 保证 FE 到 BE 的向后兼容性，即老版本的 FE 可以访问新版本的 BE。但可能不支持老版本的 BE 访问新版本的 FE。 -3. 建议确认前一个实例启动成功后，在重启下一个实例。实例启动成功的标识，请参阅安装部署文档。 diff --git a/docs/documentation/cn/internal/doris_storage_optimization.md b/docs/documentation/cn/internal/doris_storage_optimization.md deleted file mode 100644 index 95a9db90aa4ee9..00000000000000 --- a/docs/documentation/cn/internal/doris_storage_optimization.md +++ /dev/null @@ -1,225 +0,0 @@ - - -# Doris存储文件格式优化 # - -## 文件格式 ## - -![](../../../resources/images/segment_v2.png) -
图1. doris segment文件格式
- -文件包括： -- 文件开始是8个字节的magic code，用于识别文件格式和版本 -- Data Region：用于存储各个列的数据信息，这里的数据是按需分page加载的 -- Index Region: doris中将各个列的index数据统一存储在Index Region，这里的数据会按照列粒度进行加载，所以跟列的数据信息分开存储 -- Footer信息 - - FileFooterPB:定义文件的元数据信息 - - 4个字节的footer pb内容的checksum - - 4个字节的FileFooterPB消息长度，用于读取FileFooterPB - - 8个字节的MAGIC CODE，之所以在末位存储，是方便不同的场景进行文件类型的识别 - -文件中的数据按照page的方式进行组织，page是编码和压缩的基本单位。现在的page类型包括以下几种: - -### DataPage ### - -DataPage分为两种：nullable和non-nullable的data page。 - -nullable的data page内容包括： -``` - - +----------------+ - | value count | - |----------------| - | first row id | - |----------------| - | bitmap length | - |----------------| - | null bitmap | - |----------------| - | data | - |----------------| - | checksum | - +----------------+ -``` - -non-nullable data page结构如下： - -``` - |----------------| - | value count | - |----------------| - | first row id | - |----------------| - | data | - |----------------| - | checksum | - +----------------+ -``` - -其中各个字段含义如下： - -- value count - - 表示page中的行数 -- first row id - - page中第一行的行号 -- bitmap length - - 表示接下来bitmap的字节数 -- null bitmap - - 表示null信息的bitmap -- data - - 存储经过encoding和compress之后的数据 - - 需要在数据的头部信息中写入：is_compressed - - 各种不同编码的data需要在头部信息写入一些字段信息，以实现数据的解析 - - TODO：添加各种encoding的header信息 -- checksum - - 存储page粒度的校验和，包括page的header和之后的实际数据 - - -### Bloom Filter Pages ### - -针对每个bloom filter列,会在page的粒度相应的生成一个bloom filter的page，保存在bloom filter pages区域 - -### Ordinal Index Page ### - -针对每个列，都会按照page粒度，建立行号的稀疏索引。内容为这个page的起始行的行号到这个block的指针（包括offset和length） - -### Short Key Index page ### - -我们会每隔N行（可配置）生成一个short key的稀疏索引，索引的内容为：short key->行号(ordinal) - -### Column的其他索引 ### - -该格式设计支持后续扩展其他的索引信息，比如bitmap索引，spatial索引等等，只需要将需要的数据写到现有的列数据后面，并且添加对应的元数据字段到FileFooterPB中 - -### 元数据定义 ### -FileFooterPB的定义为： - -``` -message ColumnPB { - optional uint32 column_id = 1; // 这里使用column id，不使用column name是因为计划支持修改列名 - optional string type = 2; // 列类型 - optional string aggregation = 3; // 是否聚合 - optional uint32 length = 4; // 长度 - optional bool is_key = 5; // 是否是主键列 - optional string default_value = 6; // 默认值 - optional uint32 precision = 9 [default = 27]; // 精度 - optional uint32 frac = 10 [default = 9]; - optional bool is_nullable = 11 [default=false]; // 是否有null - optional bool is_bf_column = 15 [default=false]; // 是否有bf词典 - optional bool is_bitmap_column = 16 [default=false]; // 是否有bitmap索引 -} - -// page偏移 -message PagePointerPB { - required uint64 offset; // page在文件中的偏移 - required uint32 length; // page的大小 -} - -message MetadataPairPB { - optional string key = 1; - optional bytes value = 2; -} - -message ColumnMetaPB { - optional ColumnMessage encoding; // 编码方式 - - optional PagePointerPB dict_page // 词典page - repeated PagePointerPB bloom_filter_pages; // bloom filter词典信息 - optional PagePointerPB ordinal_index_page; // 行号索引数据 - optional PagePointerPB page_zone_map_page; // page级别统计信息索引数据 - - optional PagePointerPB bitmap_index_page; // bitmap索引数据 - - optional uint64 data_footprint; // 列中索引的大小 - optional uint64 index_footprint; // 列中数据的大小 - optional uint64 raw_data_footprint; // 原始列数据大小 - - optional CompressKind compress_kind; // 列的压缩方式 - - optional ZoneMapPB column_zone_map; //文件级别的过滤条件 - repeated MetadataPairPB column_meta_datas; -} - -message FileFooterPB { - optional uint32 version = 2 [default = 1]; // 用于版本兼容和升级使用 - repeated ColumnPB schema = 5; // 列Schema - optional uint64 num_values = 4; // 文件中保存的行数 - optional uint64 index_footprint = 7; // 索引大小 - optional uint64 data_footprint = 8; // 数据大小 - optional uint64 raw_data_footprint = 8; // 原始数据大小 - - optional CompressKind compress_kind = 9 [default = COMPRESS_LZO]; // 压缩方式 - repeated ColumnMetaPB column_metas = 10; // 列元数据 - optional PagePointerPB key_index_page; // short key索引page -} - -``` - -## 读写逻辑 ## - -### 写入 ### - -大体的写入流程如下： -1. 写入magic -2. 根据schema信息，生成对应的ColumnWriter，每个ColumnWriter按照不同的类型，获取对应的encoding信息（可配置），根据encoding，生成对应的encoder -3. 调用encoder->add(value)进行数据写入，每个K行，生成一个short key index entry，并且，如果当前的page满足一定条件（大小超过1M或者行数为K），就生成一个新的page，缓存在内存中。 -4. 不断的循环步骤3，直到数据写入完成。将各个列的数据依序刷入文件中 -5. 生成FileFooterPB信息，写入文件中。 - -相关的问题： - -- short key的索引如何生成？ - - 现在还是按照每隔多少行生成一个short key的稀疏索引，保持每隔1024行生成一个short的稀疏索引,具体的内容是：short key -> ordinal - -- ordinal索引里面应该存什么？ - - 存储page的第一个ordinal到page pointer的映射信息 -- 不同encoding类型的page里存什么？ - - 词典压缩 - - plain - - rle - - bshuf - -### 读取 ### - -1. 读取文件的magic，判断文件类型和版本 -2. 读取FileFooterPB，进行checksum校验 -3. 按照需要的列，读取short key索引和对应列的数据ordinal索引信息 -4. 使用start key和end key，通过short key索引定位到要读取的行号，然后通过ordinal索引确定需要读取的row ranges, 同时需要通过统计信息、bitmap索引等过滤需要读取的row ranges -5. 然后按照row ranges通过ordinal索引读取行的数据 - -相关的问题： -1. 如何实现在page内部快速的定位到某一行？ - - page内部是的数据是经过encoding的，无法快速进行行级数据的定位。不同的encoding方式，在内部进行快速的行号定位的方案不一样，需要具体分析： - - 如果是rle编码的，需要通过解析rle的header进行skip，直到到达包含该行的那个rle块之后，再进行反解。 - - binary plain encoding：会在page的中存储offset信息，并且会在page header中指定offset信息的offset，读取的时候会先解析offset信息到数组中，这样子就可以通过各个行的offset数据信息快速的定位block某一行的数据 -2. 如何实现块的高效读取？可以考虑将相邻的块在读取的时候进行merge，一次性读取？ - 这个需要在读取的时候，判断block是否连续，如果连续，就一次性的读取 - -## 编码 ## - -现有的doris存储中，针对string类型的编码，采用plain encoding的方式，效率比较低。经过对比，发现在百度统计的场景下，数据会因为string类型的编码膨胀超过一倍。所以，计划引入基于词典的编码压缩。 - -## 压缩 ## - -实现可扩展的压缩框架，支持多种压缩算法，方便后续添加新的压缩算法，计划引入zstd压缩。 - -## TODO ## -1. 如何实现嵌套类型？如何在嵌套类型中进行行号定位？ -2. 如何优化现在的ScanRange拆分导致的下游bitmap、column statistic统计等进行多次？ diff --git a/docs/documentation/cn/internal/grouping_sets_design.md b/docs/documentation/cn/internal/grouping_sets_design.md deleted file mode 100644 index 9298b7077796d7..00000000000000 --- a/docs/documentation/cn/internal/grouping_sets_design.md +++ /dev/null @@ -1,510 +0,0 @@ - - -# GROUPING SETS 设计文档 - -## 1. GROUPING SETS 相关背景知识 - -### 1.1 GROUPING SETS 子句 - -GROUP BY GROUPING SETS 是对 GROUP BY 子句的扩展，它能够在一个 GROUP BY 子句中一次实现多个集合的分组。其结果等价于将多个相应 GROUP BY 子句进行 UNION 操作。 - -特别地，一个空的子集意味着将所有的行聚集到一个分组。 -GROUP BY 子句是只含有一个元素的 GROUP BY GROUPING SETS 的特例。 - -例如，GROUPING SETS 语句： - -``` -SELECT k1, k2, SUM( k3 ) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k1), (k2), ( ) ); -``` - -其查询结果等价于： - -``` -SELECT k1, k2, SUM( k3 ) FROM t GROUP BY k1, k2 -UNION -SELECT k1, null, SUM( k3 ) FROM t GROUP BY k1 -UNION -SELECT null, k2, SUM( k3 ) FROM t GROUP BY k2 -UNION -SELECT null, null, SUM( k3 ) FROM t -``` - -下面是一个实际数据的例子： - -``` -mysql> SELECT * FROM t; -+------+------+------+ -| k1 | k2 | k3 | -+------+------+------+ -| a | A | 1 | -| a | A | 2 | -| a | B | 1 | -| a | B | 3 | -| b | A | 1 | -| b | A | 4 | -| b | B | 1 | -| b | B | 5 | -+------+------+------+ -8 rows in set (0.01 sec) - -mysql> SELECT k1, k2, SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); -+------+------+-----------+ -| k1 | k2 | sum(`k3`) | -+------+------+-----------+ -| b | B | 6 | -| a | B | 4 | -| a | A | 3 | -| b | A | 5 | -| NULL | B | 10 | -| NULL | A | 8 | -| a | NULL | 7 | -| b | NULL | 11 | -| NULL | NULL | 18 | -+------+------+-----------+ -9 rows in set (0.06 sec) -``` - -### 1.2 ROLLUP 子句 - -ROLLUP 是对 GROUPING SETS 的扩展。 - -``` -SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY ROLLUP(a,b,c) -``` - -这个 ROLLUP 等价于下面的 GROUPING SETS： - -``` -GROUPING SETS ( -(a,b,c), -( a, b ), -( a), -( ) -) -``` - -### 1.3 CUBE 子句 - -CUBE 也是对 GROUPING SETS 的扩展。 - -``` -CUBE ( e1, e2, e3, ... ) -``` - -其含义是 GROUPING SETS 后面列表中的所有子集。 - -例如，CUBE ( a, b, c ) 等价于下面的 GROUPING SETS： - -``` -GROUPING SETS ( -( a, b, c ), -( a, b ), -( a, c ), -( a ), -( b, c ), -( b ), -( c ), -( ) -) -``` - -### 1.4 GROUPING 和 GROUPING_ID 函数 -当我们没有统计某一列时，它的值显示为 NULL，这也可能是列本身就有 NULL 值，这就需要一种方法区分是没有统计还是值本来就是 NULL。为此引入 GROUPING 和 GROUPING_ID 函数。 -GROUPING(column:Column) 函数用于区分分组后的单个列是普通列和聚合列。如果是聚合列，则返回1，反之，则是0. GROUPING() 只能有一个参数列。 - -GROUPING_ID(column1, column2) 则根据指定的column 顺序，否则根据聚合的时候给的集合的元素顺序，计算出一个列列表的 bitmap 值，一个列如果是聚合列为0，否则为1. GROUPING_ID()函数返回位向量的十进制值。 -比如 [0 1 0] ->2 从下列第三个查询可以看到这种对应关系 - -例如，对于下面的表： - -``` -mysql> select * from t; -+------+------+------+ -| k1 | k2 | k3 | -+------+------+------+ -| a | A | 1 | -| a | A | 2 | -| a | B | 1 | -| a | B | 3 | -| b | A | 1 | -| b | A | 4 | -| b | B | 1 | -| b | B | 5 | -+------+------+------+ -``` - -grouping sets 的结果如下： - -``` -mysql> SELECT k1, k2, GROUPING(k1), GROUPING(k2), SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); -+------+------+----------------+----------------+-----------+ -| k1 | k2 | grouping(`k1`) | grouping(`k2`) | sum(`k3`) | -+------+------+----------------+----------------+-----------+ -| a | A | 0 | 0 | 3 | -| a | B | 0 | 0 | 4 | -| a | NULL | 0 | 1 | 7 | -| b | A | 0 | 0 | 5 | -| b | B | 0 | 0 | 6 | -| b | NULL | 0 | 1 | 11 | -| NULL | A | 1 | 0 | 8 | -| NULL | B | 1 | 0 | 10 | -| NULL | NULL | 1 | 1 | 18 | -+------+------+----------------+----------------+-----------+ -9 rows in set (0.02 sec) - -mysql> SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); -+------+------+-------------------------+-----------+ -| k1 | k2 | grouping_id(`k1`, `k2`) | sum(`k3`) | -+------+------+-------------------------+-----------+ -| a | A | 0 | 3 | -| a | B | 0 | 4 | -| a | NULL | 1 | 7 | -| b | A | 0 | 5 | -| b | B | 0 | 6 | -| b | NULL | 1 | 11 | -| NULL | A | 2 | 8 | -| NULL | B | 2 | 10 | -| NULL | NULL | 3 | 18 | -+------+------+-------------------------+-----------+ -9 rows in set (0.02 sec) - -mysql> SELECT k1, k2, grouping(k1), grouping(k2), GROUPING_ID(k1,k2), SUM(k4) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ) order by k1, k2; -+------+------+----------------+----------------+-------------------------+-----------+ -| k1 | k2 | grouping(`k1`) | grouping(`k2`) | grouping_id(`k1`, `k2`) | sum(`k4`) | -+------+------+----------------+----------------+-------------------------+-----------+ -| a | A | 0 | 0 | 0 | 3 | -| a | B | 0 | 0 | 0 | 4 | -| a | NULL | 0 | 1 | 1 | 7 | -| b | A | 0 | 0 | 0 | 5 | -| b | B | 0 | 0 | 0 | 6 | -| b | NULL | 0 | 1 | 1 | 11 | -| NULL | A | 1 | 0 | 2 | 8 | -| NULL | B | 1 | 0 | 2 | 10 | -| NULL | NULL | 1 | 1 | 3 | 18 | -+------+------+----------------+----------------+-------------------------+-----------+ -9 rows in set (0.02 sec) - -``` - -### 1.5 GROUPING SETS 的组合与嵌套 - -首先，一个 GROUP BY 子句本质上是一个 GROUPING SETS 的特例, 例如： - -``` - GROUP BY a -等同于 - GROUP BY GROUPING SETS((a)) -同样地， - GROUP BY a,b,c -等同于 - GROUP BY GROUPING SETS((a,b,c)) -``` - -同样的，CUBE 和 ROLLUP 也可以展开成 GROUPING SETS，因此 GROUP BY, CUBE, ROLLUP, GROUPING SETS 的各种组合和嵌套本质上就是 GROUPING SETS 的组合与嵌套。 - -对于 GROUPING SETS 的嵌套，语义上等价于将嵌套内的语句直接写到外面。（参考：），其中写道： - -``` -The CUBE and ROLLUP constructs can be used either directly in the GROUP BY clause, or nested inside a GROUPING SETS clause. If one GROUPING SETS clause is nested inside another, the effect is the same as if all the elements of the inner clause had been written directly in the outer clause. -``` - -对于多个 GROUPING SETS 的组合列表，很多数据库认为是叉乘（cross product）的关系。 - -例如： - -``` -GROUP BY a, CUBE (b, c), GROUPING SETS ((d), (e)) - -等同于： - -GROUP BY GROUPING SETS ( -(a, b, c, d), (a, b, c, e), -(a, b, d), (a, b, e), -(a, c, d), (a, c, e), -(a, d), (a, e) -) -``` - -对于 GROUPING SETS 的组合与嵌套，各个数据库支持不太一样。例如 snowflake 不支持任何的组合和嵌套。 -（） - -Oracle 既支持组合，也支持嵌套。 -（） - -Presto 支持组合，但不支持嵌套。 -（） - -## 2. 设计目标 - -从语法上支持 GROUPING SETS， ROLLUP 和 CUBE。实现上述所述的1.1, 1.2, 1.3 1.4. - -对于1.6 GROUPING SETS 的组合与嵌套 先不实现。 - -具体语法列出如下： - -### 2.1 GROUPING SETS 语法 - -``` -SELECT ... -FROM ... -[ ... ] -GROUP BY GROUPING SETS ( groupSet [ , groupSet [ , ... ] ] ) -[ ... ] - -groupSet ::= { ( expr [ , expr [ , ... ] ] )} - - -各种表达式，包括列名. - -``` - -### 2.2 ROLLUP 语法 - -``` -SELECT ... -FROM ... -[ ... ] -GROUP BY ROLLUP ( expr [ , expr [ , ... ] ] ) -[ ... ] - - -各种表达式，包括列名. - -``` - -### 2.3 CUBE 语法 - -``` -SELECT ... -FROM ... -[ ... ] -GROUP BY CUBE ( expr [ , expr [ , ... ] ] ) -[ ... ] - - -各种表达式，包括列名. - -``` - -## 3. 实现方案 - -### 3.1 整体思路 - -既然 GROUPING SET 子句逻辑上等价于多个相应 GROUP BY 子句的 UNION，可以通过扩展输入行(此输入行已经是通过下推条件过滤和投影后的), 在此基础上进行一个单一的 GROUP BY 操作来达到目的。 - -关键是怎样扩展输入行呢？下面举例说明： - -例如，对应下面的语句： - -``` -SELECT a, b FROM src GROUP BY a, b GROUPING SETS ((a, b), (a), (b), ()); - -``` - -假定 src 表的数据如下： - -``` -1, 2 -3, 4 - -``` - -根据 GROUPING SETS 子句给出的列表，可以将输入行扩展为下面的 8 行 （GROUPING SETS集合数 * 行数, 同时为每行生成对应的 全列的GROUPING_ID: 和其他grouping 函数的值 - -``` -1, 2 (GROUPING_ID: a, b -> 00->0) -1, null (GUPING_ID: a, null -> 01 -> 1) -null, 2 (GROUPING_ID: null, b -> 10 -> 2) -null, null (GROUPING_ID: null, null -> 11 -> 3) - -3, 4 (GROUPING_ID: a, b -> 00 -> 0) -3, null (GROUPING_ID: a, null -> 01 -> 1) -null, 4 (GROUPING_ID: null, b -> 10 -> 2) -null, null (GROUPING_ID: null, null -> 11 -> 3) - -``` - -然后，将上面的 8 行数据作为输入，对 a, b, GROUPING_ID 进行 GROUP BY 操作即可。 - -### 3.2 具体例子验证说明 - -假设有一个 t 表，包含如下列和数据： - -``` -mysql> select * from t; -+------+------+------+ -| k1 | k2 | k3 | -+------+------+------+ -| a | A | 1 | -| a | A | 2 | -| a | B | 1 | -| a | B | 3 | -| b | A | 1 | -| b | A | 4 | -| b | B | 1 | -| b | B | 5 | -+------+------+------+ -8 rows in set (0.01 sec) - -``` - -对于如下的查询： - -``` -SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ()); - -``` - -首先，对输入行进行扩展，每行数据扩展成 4 行 (GROUPING SETS子句的集合数目)，同时增加 GROUPING_ID() 列 ： - -例如 a, A, 1 扩展后变成下面的 4 行： - -``` -+------+------+------+-------------------------+ -| k1 | k2 | k3 | GROUPING_ID(`k1`, `k2`) | -+------+------+------+-------------------------+ -| a | A | 1 | 0 | -| a | NULL | 1 | 1 | -| NULL | A | 1 | 2 | -| NULL | NULL | 1 | 3 | -+------+------+------+-------------------------+ - -``` - -最终， 全部扩展后的输入行如下（总共 32 行）： - -``` -+------+------+------+-------------------------+ -| k1 | k2 | k3 | GROUPING_ID(`k1`, `k2`) | -+------+------+------+-------------------------+ -| a | A | 1 | 0 | -| a | A | 2 | 0 | -| a | B | 1 | 0 | -| a | B | 3 | 0 | -| b | A | 1 | 0 | -| b | A | 4 | 0 | -| b | B | 1 | 0 | -| b | B | 5 | 0 | -| a | NULL | 1 | 1 | -| a | NULL | 1 | 1 | -| a | NULL | 2 | 1 | -| a | NULL | 3 | 1 | -| b | NULL | 1 | 1 | -| b | NULL | 1 | 1 | -| b | NULL | 4 | 1 | -| b | NULL | 5 | 1 | -| NULL | A | 1 | 2 | -| NULL | A | 1 | 2 | -| NULL | A | 2 | 2 | -| NULL | A | 4 | 2 | -| NULL | B | 1 | 2 | -| NULL | B | 1 | 2 | -| NULL | B | 3 | 2 | -| NULL | B | 5 | 2 | -| NULL | NULL | 1 | 3 | -| NULL | NULL | 1 | 3 | -| NULL | NULL | 1 | 3 | -| NULL | NULL | 1 | 3 | -| NULL | NULL | 2 | 3 | -| NULL | NULL | 3 | 3 | -| NULL | NULL | 4 | 3 | -| NULL | NULL | 5 | 3 | -+------+------+------+-------------------------+ -32 rows in set. - -``` - -现在对k1, k2, GROUPING_ID(`k1`, `k2`) 进行 GROUP BY： - -``` -+------+------+-------------------------+-----------+ -| k1 | k2 | grouping_id(`k1`, `k2`) | sum(`k3`) | -+------+------+-------------------------+-----------+ -| a | A | 0 | 3 | -| a | B | 0 | 4 | -| a | NULL | 1 | 7 | -| b | A | 0 | 5 | -| b | B | 0 | 6 | -| b | NULL | 1 | 11 | -| NULL | A | 2 | 8 | -| NULL | B | 2 | 10 | -| NULL | NULL | 3 | 18 | -+------+------+-------------------------+-----------+ -9 rows in set (0.02 sec) - -``` - -可以看到，其结果与对 GROUPING SETS 子句后每个子集进行 GROUP BY 后再进行 UNION 的结果一致。 - -``` -select k1, k2, sum(k3) from t group by k1, k2 -UNION ALL -select NULL, k2, sum(k3) from t group by k2 -UNION ALL -select k1, NULL, sum(k3) from t group by k1 -UNION ALL -select NULL, NULL, sum(k3) from t; - -+------+------+-----------+ -| k1 | k2 | sum(`k3`) | -+------+------+-----------+ -| b | B | 6 | -| b | A | 5 | -| a | A | 3 | -| a | B | 4 | -| a | NULL | 7 | -| b | NULL | 11 | -| NULL | B | 10 | -| NULL | A | 8 | -| NULL | NULL | 18 | -+------+------+-----------+ -9 rows in set (0.06 sec) - -``` - -### 3.3 FE 规划阶段 - -#### 3.3.1 主要任务 - -1. 引入 GroupByClause 类，封装 Group By 相关信息，替换原有的 groupingExprs. -2. 增加 Grouping Sets, Cube 和 RollUp 的语法支持和语法检查、错误处理和错误信息； -3. 在 SelectStmt 类中增加 GroupByClause 成员； -4. 引入 GroupingFunctionCallExpr 类，封装grouping 和grouping_id 函数调用 -5. 引入 VirtualSlot 类，封装grouping，grouping_id 生成的虚拟列和实际列的对应关系 -6. 增加虚拟列 GROUPING_ID 和其他grouping，grouping_id 函数对应的虚拟列，并将此列加入到原有的 groupingExprs 表达式列表中； -7. 增加一个 PlanNode，考虑更通用的功能，命名为 RepeatNode。对于 GroupingSets 的聚合，在执行计划中插入 RepeatNode。 - -#### 3.3.2 Tuple - -在 GroupByClause 类中为了将 GROUPING_ID 加到 groupingExprs 表达式列表中，需要创建 virtual SlotRef, 相应的，需要对这个 slot 创建一个 tuple, 叫 GROUPING_ID Tuple。 - -对于 RepeatNode 这个执行计划，其输入是子节点的所有 tuple， 输出的 tuple 除了 repeat 子节点的数据外，还需要填写 GROUPING_ID 和其他grouping，grouping_id 对应的虚拟列，因此。 - - -### 3.4 BE 查询执行阶段 - -主要任务： - -1. 通过 RepeatNode 的执行类，增加扩展输入行的逻辑，其功能是在聚合之前将原有数据进行 repeat：对每行增加一列 GROUPING_ID， 然后按照 GroupingSets 中的集合数进行 repeat，并对对应列置为 null。根据grouping list设置新增虚拟列的值 -2. 实现 grouping_id() 和grouping() 函数。 - - - - diff --git a/docs/documentation/cn/internal/index.rst b/docs/documentation/cn/internal/index.rst deleted file mode 100644 index 2d2aee33b2665d..00000000000000 --- a/docs/documentation/cn/internal/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -=========== -设计文档 -=========== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/internal/metadata-design.md b/docs/documentation/cn/internal/metadata-design.md deleted file mode 100644 index c6cf52a9af4fa4..00000000000000 --- a/docs/documentation/cn/internal/metadata-design.md +++ /dev/null @@ -1,119 +0,0 @@ - - -# 元数据设计文档 - -## 名词解释 - -* FE：Frontend，即 Doris 的前端节点。主要负责接收和返回客户端请求、元数据以及集群管理、查询计划生成等工作。 -* BE：Backend，即 Doris 的后端节点。主要负责数据存储与管理、查询计划执行等工作。 -* bdbje：[Oracle Berkeley DB Java Edition](http://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html)。在 Doris 中，我们使用 bdbje 完成元数据操作日志的持久化、FE 高可用等功能。 - -## 整体架构 -![](../../../resources/images/palo_architecture.jpg) - -如上图，Doris 的整体架构分为两层。多个 FE 组成第一层，提供 FE 的横向扩展和高可用。多个 BE 组成第二层，负责数据存储于管理。本文主要介绍 FE 这一层中，元数据的设计与实现方式。 - -1. FE 节点分为 follower 和 observer 两类。各个 FE 之间，通过 bdbje（[BerkeleyDB Java Edition](http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/overview/index-093405.html)）进行 leader 选举，数据同步等工作。 - -2. follower 节点通过选举，其中一个 follower 成为 leader 节点，负责元数据的写入操作。当 leader 节点宕机后，其他 follower 节点会重新选举出一个 leader，保证服务的高可用。 - -3. observer 节点仅从 leader 节点进行元数据同步，不参与选举。可以横向扩展以提供元数据的读服务的扩展性。 - -> 注：follower 和 observer 对应 bdbje 中的概念为 replica 和 observer。下文可能会同时使用两种名称。 - -## 元数据结构 - -Doris 的元数据是全内存的。每个 FE 内存中，都维护一个完整的元数据镜像。在百度内部，一个包含2500张表，100万个分片（300万副本）的集群，元数据在内存中仅占用约 2GB。（当然，查询所使用的中间对象、各种作业信息等内存开销，需要根据实际情况估算。但总体依然维持在一个较低的内存开销范围内。） - -同时，元数据在内存中整体采用树状的层级结构存储，并且通过添加辅助结构，能够快速访问各个层级的元数据信息。 - -下图是 Doris 元信息所存储的内容。 - -![](../../../resources/images/metadata_contents.png) - -如上图，Doris 的元数据主要存储4类数据： - -1. 用户数据信息。包括数据库、表的 Schema、分片信息等。 -2. 各类作业信息。如导入作业，Clone 作业、SchemaChange 作业等。 -3. 用户及权限信息。 -4. 集群及节点信息。 - -## 数据流 - -![](../../../resources/images/metadata_stream.png) - -元数据的数据流具体过程如下： - -1. 只有 leader FE 可以对元数据进行写操作。写操作在修改 leader 的内存后，会序列化为一条log，按照 key-value 的形式写入 bdbje。其中 key 为连续的整型，作为 log id，value 即为序列化后的操作日志。 - -2. 日志写入 bdbje 后，bdbje 会根据策略（写多数/全写），将日志复制到其他 non-leader 的 FE 节点。non-leader FE 节点通过对日志回放，修改自身的元数据内存镜像，完成与 leader 节点的元数据同步。 - -3. leader 节点的日志条数达到阈值后（默认 10w 条），会启动 checkpoint 线程。checkpoint 会读取已有的 image 文件，和其之后的日志，重新在内存中回放出一份新的元数据镜像副本。然后将该副本写入到磁盘，形成一个新的 image。之所以是重新生成一份镜像副本，而不是将已有镜像写成 image，主要是考虑写 image 加读锁期间，会阻塞写操作。所以每次 checkpoint 会占用双倍内存空间。 - -4. image 文件生成后，leader 节点会通知其他 non-leader 节点新的 image 已生成。non-leader 主动通过 http 拉取最新的 image 文件，来更换本地的旧文件。 - -5. bdbje 中的日志，在 image 做完后，会定期删除旧的日志。 - -## 实现细节 - -### 元数据目录 - -1. 元数据目录通过 FE 的配置项 `meta_dir` 指定。 - -2. `bdb/` 目录下为 bdbje 的数据存放目录。 - -3. `image/` 目录下为 image 文件的存放目录。 - - * `image.[logid]` 是最新的 image 文件。后缀 `logid` 表明 image 所包含的最后一条日志的 id。 - * `image.ckpt` 是正在写入的 image 文件，如果写入成功，会重命名为 `image.[logid]`，并替换掉旧的 image 文件。 - * `VERSION` 文件中记录着 `cluster_id`。`cluster_id` 唯一标识一个 Doris 集群。是在 leader 第一次启动时随机生成的一个 32 位整型。也可以通过 fe 配置项 `cluster_id` 来指定一个 cluster id。 - * `ROLE` 文件中记录的 FE 自身的角色。只有 `FOLLOWER` 和 `OBSERVER` 两种。其中 `FOLLOWER` 表示 FE 为一个可选举的节点。（注意：即使是 leader 节点，其角色也为 `FOLLOWER`） - -### 启动流程 - -1. FE 第一次启动，如果启动脚本不加任何参数，则会尝试以 leader 的身份启动。在 FE 启动日志中会最终看到 `transfer from UNKNOWN to MASTER`。 - -2. FE 第一次启动，如果启动脚本中指定了 `-helper` 参数，并且指向了正确的 leader FE 节点，那么该 FE 首先会通过 http 向 leader 节点询问自身的角色（即 ROLE）和 cluster_id。然后拉取最新的 image 文件。读取 image 文件，生成元数据镜像后，启动 bdbje，开始进行 bdbje 日志同步。同步完成后，开始回放 bdbje 中，image 文件之后的日志，完成最终的元数据镜像生成。 - - > 注1：使用 `-helper` 参数启动时，需要首先通过 mysql 命令，通过 leader 来添加该 FE，否则，启动时会报错。 - - > 注2：`-helper` 可以指向任何一个 follower 节点，即使它不是 leader。 - - > 注2：bdbje 在同步日志过程中，fe 日志会显示 `xxx detached`, 此时正在进行日志拉取，属于正常现象。 - -3. FE 非第一次启动，如果启动脚本不加任何参数，则会根据本地存储的 ROLE 信息，来确定自己的身份。同时根据本地 bdbje 中存储的集群信息，获取 leader 的信息。然后读取本地的 image 文件，以及 bdbje 中的日志，完成元数据镜像生成。（如果本地 ROLE 中记录的角色和 bdbje 中记录的不一致，则会报错。） - -4. FE 非第一次启动，且启动脚本中指定了 `-helper` 参数。则和第一次启动的流程一样，也会先去询问 leader 角色。但是会和自身存储的 ROLE 进行比较。如果不一致，则会报错。 - -#### 元数据读写与同步 - -1. 用户可以使用 mysql 连接任意一个 FE 节点进行元数据的读写访问。如果连接的是 non-leader 节点，则该节点会将写操作转发给 leader 节点。leader 写成功后，会返回一个 leader 当前最新的 log id。之后，non-leader 节点会等待自身回放的 log id 大于回传的 log id 后，才将命令成功的消息返回给客户端。这种方式保证了任意 FE 节点的 Read-Your-Write 语义。 - - > 注：一些非写操作，也会转发给 leader 执行。比如 `SHOW LOAD` 操作。因为这些命令通常需要读取一些作业的中间状态，而这些中间状态是不写 bdbje 的，因此 non-leader 节点的内存中，是没有这些中间状态的。（FE 之间的元数据同步完全依赖 bdbje 的日志回放，如果一个元数据修改操作不写 bdbje 日志，则在其他 non-leader 节点中是看不到该操作修改后的结果的。） - -2. leader 节点会启动一个 TimePrinter 线程。该线程会定期向 bdbje 中写入一个当前时间的 key-value 条目。其余 non-leader 节点通过回放这条日志，读取日志中记录的时间，和本地时间进行比较，如果发现和本地时间的落后大于指定的阈值（配置项：`meta_delay_toleration_second`。写入间隔为该配置项的一半），则该节点会处于**不可读**的状态。此机制解决了 non-leader 节点在长时间和 leader 失联后，仍然提供过期的元数据服务的问题。 - -3. 各个 FE 的元数据只保证最终一致性。正常情况下，不一致的窗口期仅为毫秒级。我们保证同一 session 中，元数据访问的单调一致性。但是如果同一 client 连接不同 FE，则可能出现元数据回退的现象。（但对于批量更新系统，该问题影响很小。） - -### 宕机恢复 - -1. leader 节点宕机后，其余 follower 会立即选举出一个新的 leader 节点提供服务。 -2. 当多数 follower 节点宕机时，元数据不可写入。当元数据处于不可写入状态下，如果这时发生写操作请求，目前的处理流程是 **FE 进程直接退出**。后续会优化这个逻辑，在不可写状态下，依然提供读服务。 -3. observer 节点宕机，不会影响任何其他节点的状态。也不会影响元数据在其他节点的读写。 diff --git a/docs/documentation/cn/sql-reference/index.rst b/docs/documentation/cn/sql-reference/index.rst deleted file mode 100644 index c30bebbc01c5f0..00000000000000 --- a/docs/documentation/cn/sql-reference/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -=========== -SQL 手册 -=========== - -.. toctree:: - :hidden: - - sql-functions/index - sql-statements/index diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/avg.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/avg.md deleted file mode 100644 index cc52287fe3f9b3..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/avg.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# AVG -## description -### Syntax - -`AVG([DISTINCT] expr)` - - -用于返回选中字段的平均值 - -可选字段DISTINCT参数可以用来返回去重平均值 - -## example - -``` -mysql> SELECT datetime, AVG(cost_time) FROM log_statis group by datetime; -+---------------------+--------------------+ -| datetime | avg(`cost_time`) | -+---------------------+--------------------+ -| 2019-07-03 21:01:20 | 25.827794561933533 | -+---------------------+--------------------+ - -mysql> SELECT datetime, AVG(distinct cost_time) FROM log_statis group by datetime; -+---------------------+---------------------------+ -| datetime | avg(DISTINCT `cost_time`) | -+---------------------+---------------------------+ -| 2019-07-04 02:23:24 | 20.666666666666668 | -+---------------------+---------------------------+ - -``` -##keyword -AVG diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/bitmap.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/bitmap.md deleted file mode 100644 index 8d428867878302..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/bitmap.md +++ /dev/null @@ -1,139 +0,0 @@ - - -# BITMAP - -## Create table - -建表时需要使用聚合模型，数据类型是 bitmap , 聚合函数是 bitmap_union - -``` -CREATE TABLE `pv_bitmap` ( - `dt` int(11) NULL COMMENT "", - `page` varchar(10) NULL COMMENT "", - `user_id` bitmap BITMAP_UNION NULL COMMENT "" -) ENGINE=OLAP -AGGREGATE KEY(`dt`, `page`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`dt`) BUCKETS 2; -``` -注：当数据量很大时，最好为高频率的 bitmap_union 查询建立对应的 rollup 表 - -``` -ALTER TABLE pv_bitmap ADD ROLLUP pv (page, user_id); -``` - -## Data Load - -`TO_BITMAP(expr)` : 将 0 ~ 18446744073709551615 的 unsigned bigint 转为 bitmap - -`BITMAP_EMPTY()`: 生成空 bitmap 列，用于 insert 或导入的时填充默认值 - -`BITMAP_HASH(expr)`: 将任意类型的列通过 Hash 的方式转为 bitmap - -### Stream Load - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=to_bitmap(user_id)" http://host:8410/api/test/testDb/_stream_load -``` - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=bitmap_hash(user_id)" http://host:8410/api/test/testDb/_stream_load -``` - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=bitmap_empty()" http://host:8410/api/test/testDb/_stream_load -``` - -### Insert Into - -id2 的列类型是 bitmap -``` -insert into bitmap_table1 select id, id2 from bitmap_table2; -``` - -id2 的列类型是 bitmap -``` -INSERT INTO bitmap_table1 (id, id2) VALUES (1001, to_bitmap(1000)), (1001, to_bitmap(2000)); -``` - -id2 的列类型是 bitmap -``` -insert into bitmap_table1 select id, bitmap_union(id2) from bitmap_table2 group by id; -``` - -id2 的列类型是 int -``` -insert into bitmap_table1 select id, to_bitmap(id2) from table; -``` - -id2 的列类型是 String -``` -insert into bitmap_table1 select id, bitmap_hash(id_string) from table; -``` - - -## Data Query -### Syntax - - -`BITMAP_UNION(expr)` : 计算输入 Bitmap 的并集，返回新的bitmap - -`BITMAP_UNION_COUNT(expr)`: 计算输入 Bitmap 的并集，返回其基数，和 BITMAP_COUNT(BITMAP_UNION(expr)) 等价。目前推荐优先使用 BITMAP_UNION_COUNT ，其性能优于 BITMAP_COUNT(BITMAP_UNION(expr)) - -`BITMAP_UNION_INT(expr)` : 计算 TINYINT,SMALLINT 和 INT 类型的列中不同值的个数，返回值和 -COUNT(DISTINCT expr) 相同 - -`INTERSECT_COUNT(bitmap_column_to_count, filter_column, filter_values ...)` : 计算满足 -filter_column 过滤条件的多个 bitmap 的交集的基数值。 -bitmap_column_to_count 是 bitmap 类型的列，filter_column 是变化的维度列，filter_values 是维度取值列表 - - -### Example - -下面的 SQL 以上面的 pv_bitmap table 为例： - -计算 user_id 的去重值： - -``` -select bitmap_union_count(user_id) from pv_bitmap; - -select bitmap_count(bitmap_union(user_id)) from pv_bitmap; -``` - -计算 id 的去重值： - -``` -select bitmap_union_int(id) from pv_bitmap; -``` - -计算 user_id 的 留存: - -``` -select intersect_count(user_id, page, 'meituan') as meituan_uv, -intersect_count(user_id, page, 'waimai') as waimai_uv, -intersect_count(user_id, page, 'meituan', 'waimai') as retention //在 'meituan' 和 'waimai' 两个页面都出现的用户数 -from pv_bitmap -where page in ('meituan', 'waimai'); -``` - - -## keyword - -BITMAP,BITMAP_COUNT,BITMAP_EMPTY,BITMAP_UNION,BITMAP_UNION_INT,TO_BITMAP,BITMAP_UNION_COUNT,INTERSECT_COUNT \ No newline at end of file diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/count.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/count.md deleted file mode 100755 index 0b225c67585233..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/count.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# COUNT -## description -### Syntax - -`COUNT([DISTINCT] expr)` - - -用于返回满足要求的行的数目 - -## example - -``` -MySQL > select count(*) from log_statis group by datetime; -+----------+ -| count(*) | -+----------+ -| 28515903 | -+----------+ - -MySQL > select count(datetime) from log_statis group by datetime; -+-------------------+ -| count(`datetime`) | -+-------------------+ -| 28521682 | -+-------------------+ - -MySQL > select count(distinct datetime) from log_statis group by datetime; -+-------------------------------+ -| count(DISTINCT `datetime`) | -+-------------------------------+ -| 71045 | -+-------------------------------+ -``` -##keyword -COUNT diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md deleted file mode 100644 index 3c44e48195058c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# HLL_UNION_AGG -## description -### Syntax - -`HLL_UNION_AGG(hll)` - - -HLL是基于HyperLogLog算法的工程实现，用于保存HyperLogLog计算过程的中间结果 - -它只能作为表的value列类型、通过聚合来不断的减少数据量，以此来实现加快查询的目的 - -基于它得到的是一个估算结果，误差大概在1%左右，hll列是通过其它列或者导入数据里面的数据生成的 - -导入的时候通过hll_hash函数来指定数据中哪一列用于生成hll列，它常用于替代count distinct，通过结合rollup在业务上用于快速计算uv等 - -## example -``` -MySQL > select HLL_UNION_AGG(uv_set) from test_uv;; -+-------------------------+ -| HLL_UNION_AGG(`uv_set`) | -+-------------------------+ -| 17721 | -+-------------------------+ -``` -##keyword -HLL_UNION_AGG,HLL,UNION,AGG diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/index.rst b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/index.rst deleted file mode 100644 index 10a1dceb92d1d7..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -聚合函数 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/max.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/max.md deleted file mode 100755 index 76868b927f063e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/max.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# MAX -## description -### Syntax - -`MAX(expr)` - - -返回expr表达式的最大值 - -## example -``` -MySQL > select max(scan_rows) from log_statis group by datetime; -+------------------+ -| max(`scan_rows`) | -+------------------+ -| 4671587 | -+------------------+ -``` -##keyword -MAX diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/min.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/min.md deleted file mode 100755 index 112496f98779bd..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/min.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# MIN -## description -### Syntax - -`MIN(expr)` - - -返回expr表达式的最小值 - -## example -``` -MySQL > select min(scan_rows) from log_statis group by datetime; -+------------------+ -| min(`scan_rows`) | -+------------------+ -| 0 | -+------------------+ -``` -##keyword -MIN diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/ndv.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/ndv.md deleted file mode 100644 index d8b7e7fbd54bd1..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/ndv.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# NDV -## description -### Syntax - -`NDV(expr)` - - -返回类似于 COUNT(DISTINCT col) 结果的近似值聚合函数。 - -它比 COUNT 和 DISTINCT 组合的速度更快，并使用固定大小的内存，因此对于高基数的列可以使用更少的内存。 - -## example -``` -MySQL > select ndv(query_id) from log_statis group by datetime; -+-----------------+ -| ndv(`query_id`) | -+-----------------+ -| 17721 | -+-----------------+ -``` -##keyword -NDV diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/percentile_approx.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/percentile_approx.md deleted file mode 100755 index 640354092d1842..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/percentile_approx.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# PERCENTILE_APPROX -## description -### Syntax - -`PERCENTILE_APPROX(expr, DOUBLE p[, DOUBLE compression])` - - -返回第p个百分位点的近似值，p的值介于0到1之间 - -compression参数是可选项，可设置范围是[2048, 10000]，值越大，精度越高，内存消耗越大，计算耗时越长。 -compression参数未指定或设置的值在[2048, 10000]范围外，以10000的默认值运行 - -该函数使用固定大小的内存，因此对于高基数的列可以使用更少的内存，可用于计算tp99等统计值 - -## example -``` -MySQL > select `table`, percentile_approx(cost_time,0.99) from log_statis group by `table`; -+---------------------+---------------------------+ -| table | percentile_approx(`cost_time`, 0.99) | -+----------+--------------------------------------+ -| test | 54.22 | -+----------+--------------------------------------+ - -MySQL > select `table`, percentile_approx(cost_time,0.99, 4096) from log_statis group by `table`; -+---------------------+---------------------------+ -| table | percentile_approx(`cost_time`, 0.99, 4096.0) | -+----------+--------------------------------------+ -| test | 54.21 | -+----------+--------------------------------------+ -##keyword -PERCENTILE_APPROX,PERCENTILE,APPROX diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/stddev.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/stddev.md deleted file mode 100755 index 40e9f7f7c2bf49..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/stddev.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# STDDEV,STDDEV_POP -## description -### Syntax - -`STDDEV(expr)` - - -返回expr表达式的标准差 - -## example -``` -MySQL > select stddev(scan_rows) from log_statis group by datetime; -+---------------------+ -| stddev(`scan_rows`) | -+---------------------+ -| 2.3736656687790934 | -+---------------------+ - -MySQL > select stddev_pop(scan_rows) from log_statis group by datetime; -+-------------------------+ -| stddev_pop(`scan_rows`) | -+-------------------------+ -| 2.3722760595994914 | -+-------------------------+ -``` -##keyword -STDDEV,STDDEV_POP,POP diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/stddev_samp.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/stddev_samp.md deleted file mode 100755 index 6a0fc1c29ec0e8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/stddev_samp.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# STDDEV_SAMP -## description -### Syntax - -`STDDEV_SAMP(expr)` - - -返回expr表达式的样本标准差 - -## example -``` -MySQL > select stddev_samp(scan_rows) from log_statis group by datetime; -+--------------------------+ -| stddev_samp(`scan_rows`) | -+--------------------------+ -| 2.372044195280762 | -+--------------------------+ -``` -##keyword -STDDEV_SAMP,STDDEV,SAMP diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/sum.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/sum.md deleted file mode 100755 index 845f7da3972f86..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/sum.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# SUM -## description -### Syntax - -`SUM(expr)` - - -用于返回选中字段所有值的和 - -## example -``` -MySQL > select sum(scan_rows) from log_statis group by datetime; -+------------------+ -| sum(`scan_rows`) | -+------------------+ -| 8217360135 | -+------------------+ -``` -##keyword -SUM diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/var_samp.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/var_samp.md deleted file mode 100755 index 664352ddd611ce..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/var_samp.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# VAR_SAMP,VARIANCE_SAMP -## description -### Syntax - -`VAR_SAMP(expr)` - - -返回expr表达式的样本方差 - -## example -``` -MySQL > select var_samp(scan_rows) from log_statis group by datetime; -+-----------------------+ -| var_samp(`scan_rows`) | -+-----------------------+ -| 5.6227132145741789 | -+-----------------------+ -##keyword -VAR_SAMP,VARIANCE_SAMP,VAR,SAMP,VARIANCE diff --git a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/variance.md b/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/variance.md deleted file mode 100755 index 7b6f6b0a510765..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/aggregate-functions/variance.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# VARIANCE,VAR_POP,VARIANCE_POP -## description -### Syntax - -`VARIANCE(expr)` - - -返回expr表达式的方差 - -## example -``` -MySQL > select variance(scan_rows) from log_statis group by datetime; -+-----------------------+ -| variance(`scan_rows`) | -+-----------------------+ -| 5.6183332881176211 | -+-----------------------+ - -MySQL > select var_pop(scan_rows) from log_statis group by datetime; -+----------------------+ -| var_pop(`scan_rows`) | -+----------------------+ -| 5.6230744719006163 | -+----------------------+ -##keyword -VARIANCE,VAR_POP,VARIANCE_POP,VAR,POP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_and.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_and.md deleted file mode 100644 index bafe1784889d7c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_and.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# bitmap_and -## description -### Syntax - -`BITMAP BITMAP_AND(BITMAP lhs, BITMAP rhs)` - -计算两个输入bitmap的交集，返回新的bitmap. - -## example - -``` -mysql> select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(2))) cnt; -+------+ -| cnt | -+------+ -| 0 | -+------+ - -mysql> select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(1))) cnt; -+------+ -| cnt | -+------+ -| 1 | -+------+ -``` - -## keyword - - BITMAP_AND,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md deleted file mode 100644 index ed10b6bd17dcba..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# bitmap_contains -## description -### Syntax - -`B00LEAN BITMAP_CONTAINS(BITMAP bitmap, BIGINT input)` - -计算输入值是否在Bitmap列中，返回值是Boolean值. - -## example - -``` -mysql> select bitmap_contains(to_bitmap(1),2) cnt; -+------+ -| cnt | -+------+ -| 0 | -+------+ - -mysql> select bitmap_contains(to_bitmap(1),1) cnt; -+------+ -| cnt | -+------+ -| 1 | -+------+ -``` - -## keyword - - BITMAP_CONTAINS,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md deleted file mode 100644 index 3d329a5ab91bb0..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# bitmap_empty -## description -### Syntax - -`BITMAP BITMAP_EMPTY()` - -返回一个空bitmap。主要用于 insert 或 stream load 时填充默认值。例如 - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,v1,v2=bitmap_empty()" http://host:8410/api/test/testDb/_stream_load -``` - -## example - -``` -mysql> select bitmap_count(bitmap_empty()); -+------------------------------+ -| bitmap_count(bitmap_empty()) | -+------------------------------+ -| 0 | -+------------------------------+ -``` - -## keyword - - BITMAP_EMPTY,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md deleted file mode 100644 index 261da29983dd4e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# bitmap_from_string - -## description -### Syntax - -`BITMAP BITMAP_FROM_STRING(VARCHAR input)` - -将一个字符串转化为一个BITAMP，字符串是由逗号分隔的一组UINT32数字组成. -比如"0, 1, 2"字符串会转化为一个Bitmap，其中的第0, 1, 2位被设置. -当输入字段不合法时，返回NULL - -## example - -``` -mysql> select bitmap_to_string(bitmap_empty()); -+----------------------------------+ -| bitmap_to_string(bitmap_empty()) | -+----------------------------------+ -| | -+----------------------------------+ - -mysql> select bitmap_to_string(bitmap_from_string("0, 1, 2")); -+-------------------------------------------------+ -| bitmap_to_string(bitmap_from_string('0, 1, 2')) | -+-------------------------------------------------+ -| 0,1,2 | -+-------------------------------------------------+ - -mysql> select bitmap_from_string("-1, 0, 1, 2"); -+-----------------------------------+ -| bitmap_from_string('-1, 0, 1, 2') | -+-----------------------------------+ -| NULL | -+-----------------------------------+ -``` - -## keyword - - BITMAP_FROM_STRING,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md deleted file mode 100644 index 74821072557a0b..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# bitmap_has_any -## description -### Syntax - -`B00LEAN BITMAP_HAS_ANY(BITMAP lhs, BITMAP rhs)` - -计算两个Bitmap列是否存在相交元素，返回值是Boolean值. - -## example - -``` -mysql> select bitmap_has_any(to_bitmap(1),to_bitmap(2)) cnt; -+------+ -| cnt | -+------+ -| 0 | -+------+ - -mysql> select bitmap_has_any(to_bitmap(1),to_bitmap(1)) cnt; -+------+ -| cnt | -+------+ -| 1 | -+------+ -``` - -## keyword - - BITMAP_HAS_ANY,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md deleted file mode 100644 index 87efc119411789..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# bitmap_hash -## description -### Syntax - -`BITMAP BITMAP_HASH(expr)` - -对任意类型的输入计算32位的哈希值，返回包含该哈希值的bitmap。主要用于stream load任务将非整型字段导入Doris表的bitmap字段。例如 - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,device_id, device_id=bitmap_hash(device_id)" http://host:8410/api/test/testDb/_stream_load -``` - -## example - -``` -mysql> select bitmap_count(bitmap_hash('hello')); -+------------------------------------+ -| bitmap_count(bitmap_hash('hello')) | -+------------------------------------+ -| 1 | -+------------------------------------+ -``` - -## keyword - - BITMAP_HASH,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_or.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_or.md deleted file mode 100644 index 1c0bca75c2921e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_or.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# bitmap_or -## description -### Syntax - -`BITMAP BITMAP_OR(BITMAP lhs, BITMAP rhs)` - -计算两个输入bitmap的并集，返回新的bitmap. - -## example - -``` -mysql> select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt; -+------+ -| cnt | -+------+ -| 2 | -+------+ - -mysql> select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt; -+------+ -| cnt | -+------+ -| 1 | -+------+ -``` - -## keyword - - BITMAP_OR,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md deleted file mode 100644 index e7a25ebc73b0f9..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md +++ /dev/null @@ -1,62 +0,0 @@ - - -# bitmap_to_string - -## description -### Syntax - -`VARCHAR BITMAP_TO_STRING(BITMAP input)` - -将一个bitmap转化成一个逗号分隔的字符串，字符串中包含所有设置的BIT位。输入是null的话会返回null。 - -## example - -``` -mysql> select bitmap_to_string(null); -+------------------------+ -| bitmap_to_string(NULL) | -+------------------------+ -| NULL | -+------------------------+ - -mysql> select bitmap_to_string(bitmap_empty()); -+----------------------------------+ -| bitmap_to_string(bitmap_empty()) | -+----------------------------------+ -| | -+----------------------------------+ - -mysql> select bitmap_to_string(to_bitmap(1)); -+--------------------------------+ -| bitmap_to_string(to_bitmap(1)) | -+--------------------------------+ -| 1 | -+--------------------------------+ - -mysql> select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))); -+---------------------------------------------------------+ -| bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) | -+---------------------------------------------------------+ -| 1,2 | -+---------------------------------------------------------+ - -``` - -## keyword - - BITMAP_TO_STRING,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/index.rst b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/index.rst deleted file mode 100644 index 0a7ce165b1ffe3..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -bitmap函数 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/to_bitmap.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/to_bitmap.md deleted file mode 100644 index e0236bed0d3ec1..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/to_bitmap.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# to_bitmap -## description -### Syntax - -`BITMAP TO_BITMAP(expr)` - -输入为取值在 0 ~ 18446744073709551615 区间的 unsigned bigint ，输出为包含该元素的bitmap。 -该函数主要用于stream load任务将整型字段导入Doris表的bitmap字段。例如 - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=to_bitmap(user_id)" http://host:8410/api/test/testDb/_stream_load -``` - -## example - -``` -mysql> select bitmap_count(to_bitmap(10)); -+-----------------------------+ -| bitmap_count(to_bitmap(10)) | -+-----------------------------+ -| 1 | -+-----------------------------+ -``` - -## keyword - - TO_BITMAP,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/cast.md b/docs/documentation/cn/sql-reference/sql-functions/cast.md deleted file mode 100644 index dbf2d68ec4c8db..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/cast.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# CAST -## description -### Syntax - -``` -cast (input as type) -``` - -### BIGINT type - -### Syntax(BIGINT) - -``` cast (input as BIGINT) ``` - - -将 input 转成 指定的 type - - -将当前列 input 转换为 BIGINT 类型 - -## example - -1. 转常量，或表中某列 - -``` -mysql> select cast (1 as BIGINT); -+-------------------+ -| CAST(1 AS BIGINT) | -+-------------------+ -| 1 | -+-------------------+ -``` - -2. 转导入的原始数据 - -``` -curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(tmp_k1 as BIGINT)" http://host:port/api/test/bigint/_stream_load -``` - -*注：在导入中，由于原始类型均为String，将值为浮点的原始数据做 cast的时候数据会被转换成 NULL ，比如 12.0 。Doris目前不会对原始数据做截断。* - -如果想强制将这种类型的原始数据 cast to int 的话。请看下面写法： - -``` -curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(cast(tmp_k1 as DOUBLE) as BIGINT)" http://host:port/api/test/bigint/_stream_load - -mysql> select cast(cast ("11.2" as double) as bigint); -+----------------------------------------+ -| CAST(CAST('11.2' AS DOUBLE) AS BIGINT) | -+----------------------------------------+ -| 11 | -+----------------------------------------+ -1 row in set (0.00 sec) -``` -##keyword -CAST diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curdate.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curdate.md deleted file mode 100644 index 74decd99add9cf..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curdate.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# curdate -## description -### Syntax - -`DATE CURDATE()` - -获取当前的日期，以DATE类型返回 - -## Examples - -``` -mysql> SELECT CURDATE(); -+------------+ -| CURDATE() | -+------------+ -| 2019-12-20 | -+------------+ - -mysql> SELECT CURDATE() + 0; -+---------------+ -| CURDATE() + 0 | -+---------------+ -| 20191220 | -+---------------+ -``` - -##keyword - - CURDATE diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/current_timestamp.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/current_timestamp.md deleted file mode 100644 index f07b00711fbfd9..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/current_timestamp.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# current_timestamp -## description -### Syntax - -`DATETIME CURRENT_TIMESTAMP()` - - -获得当前的时间，以Datetime类型返回 - -## example - -``` -mysql> select current_timestamp(); -+---------------------+ -| current_timestamp() | -+---------------------+ -| 2019-05-27 15:59:33 | -+---------------------+ -``` - -## keyword - - CURRENT_TIMESTAMP,CURRENT,TIMESTAMP diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_add.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_add.md deleted file mode 100644 index 2fe7ce5149a8a8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_add.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# date_add -## description -### Syntax - -`INT DATE_ADD(DATETIME date,INTERVAL expr type)` - - -向日期添加指定的时间间隔。 - -date 参数是合法的日期表达式。 - -expr 参数是您希望添加的时间间隔。 - -type 参数可以是下列值：YEAR, MONTH, DAY, HOUR, MINUTE, SECOND - -## example - -``` -mysql> select date_add('2010-11-30 23:59:59', INTERVAL 2 DAY); -+-------------------------------------------------+ -| date_add('2010-11-30 23:59:59', INTERVAL 2 DAY) | -+-------------------------------------------------+ -| 2010-12-02 23:59:59 | -+-------------------------------------------------+ -``` - -## keyword - - DATE_ADD,DATE,ADD diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_format.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_format.md deleted file mode 100644 index 7f8107a12d729d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_format.md +++ /dev/null @@ -1,153 +0,0 @@ - - -# date_format -## description -### Syntax - -`VARCHAR DATE_FORMAT(DATETIME date, VARCHAR format)` - - -将日期类型按照format的类型转化为字符串， -当前支持最大128字节的字符串，如果返回值长度超过128，则返回NULL - -date 参数是合法的日期。format 规定日期/时间的输出格式。 - -可以使用的格式有： - -%a | 缩写星期名 - -%b | 缩写月名 - -%c | 月，数值 - -%D | 带有英文前缀的月中的天 - -%d | 月的天，数值(00-31) - -%e | 月的天，数值(0-31) - -%f | 微秒 - -%H | 小时 (00-23) - -%h | 小时 (01-12) - -%I | 小时 (01-12) - -%i | 分钟，数值(00-59) - -%j | 年的天 (001-366) - -%k | 小时 (0-23) - -%l | 小时 (1-12) - -%M | 月名 - -%m | 月，数值(00-12) - -%p | AM 或 PM - -%r | 时间，12-小时（hh:mm:ss AM 或 PM） - -%S | 秒(00-59) - -%s | 秒(00-59) - -%T | 时间, 24-小时 (hh:mm:ss) - -%U | 周 (00-53) 星期日是一周的第一天 - -%u | 周 (00-53) 星期一是一周的第一天 - -%V | 周 (01-53) 星期日是一周的第一天，与 %X 使用 - -%v | 周 (01-53) 星期一是一周的第一天，与 %x 使用 - -%W | 星期名 - -%w | 周的天 （0=星期日, 6=星期六） - -%X | 年，其中的星期日是周的第一天，4 位，与 %V 使用 - -%x | 年，其中的星期一是周的第一天，4 位，与 %v 使用 - -%Y | 年，4 位 - -%y | 年，2 位 - -%% | 用于表示 % - -## example - -``` -mysql> select date_format('2009-10-04 22:23:00', '%W %M %Y'); -+------------------------------------------------+ -| date_format('2009-10-04 22:23:00', '%W %M %Y') | -+------------------------------------------------+ -| Sunday October 2009 | -+------------------------------------------------+ - -mysql> select date_format('2007-10-04 22:23:00', '%H:%i:%s'); -+------------------------------------------------+ -| date_format('2007-10-04 22:23:00', '%H:%i:%s') | -+------------------------------------------------+ -| 22:23:00 | -+------------------------------------------------+ - -mysql> select date_format('1900-10-04 22:23:00', '%D %y %a %d %m %b %j'); -+------------------------------------------------------------+ -| date_format('1900-10-04 22:23:00', '%D %y %a %d %m %b %j') | -+------------------------------------------------------------+ -| 4th 00 Thu 04 10 Oct 277 | -+------------------------------------------------------------+ - -mysql> select date_format('1997-10-04 22:23:00', '%H %k %I %r %T %S %w'); -+------------------------------------------------------------+ -| date_format('1997-10-04 22:23:00', '%H %k %I %r %T %S %w') | -+------------------------------------------------------------+ -| 22 22 10 10:23:00 PM 22:23:00 00 6 | -+------------------------------------------------------------+ - -mysql> select date_format('1999-01-01 00:00:00', '%X %V'); -+---------------------------------------------+ -| date_format('1999-01-01 00:00:00', '%X %V') | -+---------------------------------------------+ -| 1998 52 | -+---------------------------------------------+ - -mysql> select date_format('2006-06-01', '%d'); -+------------------------------------------+ -| date_format('2006-06-01 00:00:00', '%d') | -+------------------------------------------+ -| 01 | -+------------------------------------------+ - -mysql> select date_format('2006-06-01', '%%%d'); -+--------------------------------------------+ -| date_format('2006-06-01 00:00:00', '%%%d') | -+--------------------------------------------+ -| %01 | -+--------------------------------------------+ -``` - -## keyword - - DATE_FORMAT,DATE,FORMAT diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_sub.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_sub.md deleted file mode 100644 index b7440cd750d8cb..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/date_sub.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# date_sub -## description -### Syntax - -`INT DATE_SUB(DATETIME date,INTERVAL expr type)` - - -从日期减去指定的时间间隔 - -date 参数是合法的日期表达式。 - -expr 参数是您希望添加的时间间隔。 - -type 参数可以是下列值：YEAR, MONTH, DAY, HOUR, MINUTE, SECOND - -## example - -``` -mysql> select date_sub('2010-11-30 23:59:59', INTERVAL 2 DAY); -+-------------------------------------------------+ -| date_sub('2010-11-30 23:59:59', INTERVAL 2 DAY) | -+-------------------------------------------------+ -| 2010-11-28 23:59:59 | -+-------------------------------------------------+ -``` - -##keyword - - DATE_SUB,DATE,SUB diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/datediff.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/datediff.md deleted file mode 100644 index fbd6fd1844b73f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/datediff.md +++ /dev/null @@ -1,51 +0,0 @@ - - -# datediff -## description -### Syntax - -`DATETIME DATEDIFF(DATETIME expr1,DATETIME expr2)` - - -计算expr1 - expr2，结果精确到天。 - -expr1 和 expr2 参数是合法的日期或日期/时间表达式。 - -注释：只有值的日期部分参与计算。 - -## example - -``` -mysql> select datediff(CAST('2007-12-31 23:59:59' AS DATETIME), CAST('2007-12-30' AS DATETIME)); -+-----------------------------------------------------------------------------------+ -| datediff(CAST('2007-12-31 23:59:59' AS DATETIME), CAST('2007-12-30' AS DATETIME)) | -+-----------------------------------------------------------------------------------+ -| 1 | -+-----------------------------------------------------------------------------------+ - -mysql> select datediff(CAST('2010-11-30 23:59:59' AS DATETIME), CAST('2010-12-31' AS DATETIME)); -+-----------------------------------------------------------------------------------+ -| datediff(CAST('2010-11-30 23:59:59' AS DATETIME), CAST('2010-12-31' AS DATETIME)) | -+-----------------------------------------------------------------------------------+ -| -31 | -+-----------------------------------------------------------------------------------+ -``` -##keyword -DATEDIFF diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/day.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/day.md deleted file mode 100644 index f5b33c82ae4484..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/day.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# day -## description -### Syntax - -`INT DAY(DATETIME date)` - - -获得日期中的天信息，返回值范围从1-31。 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select day('1987-01-31'); -+----------------------------+ -| day('1987-01-31 00:00:00') | -+----------------------------+ -| 31 | -+----------------------------+ -``` -##keyword -DAY diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayname.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayname.md deleted file mode 100644 index e503a473e9e5a7..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayname.md +++ /dev/null @@ -1,43 +0,0 @@ - - -# dayname -## description -### Syntax - -`VARCHAR DAYNAME(DATE)` - - -返回日期对应的日期名字 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select dayname('2007-02-03 00:00:00'); -+--------------------------------+ -| dayname('2007-02-03 00:00:00') | -+--------------------------------+ -| Saturday | -+--------------------------------+ -``` - -##keyword - DAYNAME diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofmonth.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofmonth.md deleted file mode 100644 index 212b04cbefce68..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofmonth.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# dayofmonth -## description -### Syntax - -`INT DAYOFMONTH(DATETIME date)` - - -获得日期中的天信息，返回值范围从1-31。 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select dayofmonth('1987-01-31'); -+-----------------------------------+ -| dayofmonth('1987-01-31 00:00:00') | -+-----------------------------------+ -| 31 | -+-----------------------------------+ -``` - -##keyword - - DAYOFMONTH diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofweek.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofweek.md deleted file mode 100644 index ea867dd5f29121..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofweek.md +++ /dev/null @@ -1,51 +0,0 @@ - - -# dayofweek -## description -### Syntax - -`INT dayofweek(DATETIME date)` - - -DAYOFWEEK函数返回日期的工作日索引值，即星期日为1，星期一为2，星期六为7 - -参数为Date或者Datetime类型或者可以cast为Date或者Datetime类型的数字 - -## example - -``` -mysql> select dayofweek('2019-06-25'); -+----------------------------------+ -| dayofweek('2019-06-25 00:00:00') | -+----------------------------------+ -| 3 | -+----------------------------------+ - -mysql> select dayofweek(cast(20190625 as date)); -+-----------------------------------+ -| dayofweek(CAST(20190625 AS DATE)) | -+-----------------------------------+ -| 3 | -+-----------------------------------+ -``` - -##keyword - - DAYOFWEEK diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofyear.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofyear.md deleted file mode 100644 index 66477787f4a28d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/dayofyear.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# dayofyear -## description -### Syntax - -`INT DAYOFYEAR(DATETIME date)` - - -获得日期中对应当年中的哪一天。 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select dayofyear('2007-02-03 00:00:00'); -+----------------------------------+ -| dayofyear('2007-02-03 00:00:00') | -+----------------------------------+ -| 34 | -+----------------------------------+ -``` - -##keyword - - DAYOFYEAR diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/from_days.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/from_days.md deleted file mode 100644 index d88d75b66d8e84..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/from_days.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# from_days -## description -### Syntax - -`DATE FROM_DAYS(INT N)` - - -通过距离0000-01-01日的天数计算出哪一天 - -## example - -``` -mysql> select from_days(730669); -+-------------------+ -| from_days(730669) | -+-------------------+ -| 2000-07-03 | -+-------------------+ -``` - -##keyword - - FROM_DAYS,FROM,DAYS diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/from_unixtime.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/from_unixtime.md deleted file mode 100644 index 397ceaa0c0dfc5..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/from_unixtime.md +++ /dev/null @@ -1,82 +0,0 @@ - - -# from_unixtime -## description -### Syntax - -`DATETIME FROM_UNIXTIME(INT unix_timestamp[, VARCHAR string_format])` - - -将 unix 时间戳转化为对应的 time 格式，返回的格式由 `string_format` 指定 - -默认为 yyyy-MM-dd HH:mm:ss ,也支持date_format中的format格式 - -传入的是整形，返回的是字符串类型 - -目前 `string_format` 支持格式： - - %Y：年。例：2014，1900 - %m：月。例：12，09 - %d：日。例：11，01 - %H：时。例：23，01，12 - %i：分。例：05，11 - %s：秒。例：59，01 - -其余 `string_format` 格式是非法的，返回NULL - -如果给定的时间戳小于 0 或大于 253402271999，则返回 NULL。即时间戳范围是： - -1970-01-01 00:00:00 ~ 9999-12-31 23:59:59 - -## example - -``` -mysql> select from_unixtime(1196440219); -+---------------------------+ -| from_unixtime(1196440219) | -+---------------------------+ -| 2007-12-01 00:30:19 | -+---------------------------+ - -mysql> select from_unixtime(1196440219, 'yyyy-MM-dd HH:mm:ss'); -+--------------------------------------------------+ -| from_unixtime(1196440219, 'yyyy-MM-dd HH:mm:ss') | -+--------------------------------------------------+ -| 2007-12-01 00:30:19 | -+--------------------------------------------------+ - -mysql> select from_unixtime(1196440219, '%Y-%m-%d'); -+-----------------------------------------+ -| from_unixtime(1196440219, '%Y-%m-%d') | -+-----------------------------------------+ -| 2007-12-01 | -+-----------------------------------------+ - -mysql> select from_unixtime(1196440219, '%Y-%m-%d %H:%i:%s'); -+--------------------------------------------------+ -| from_unixtime(1196440219, '%Y-%m-%d %H:%i:%s') | -+--------------------------------------------------+ -| 2007-12-01 00:30:19 | -+--------------------------------------------------+ -``` - -##keyword - - FROM_UNIXTIME,FROM,UNIXTIME diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/hour.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/hour.md deleted file mode 100644 index c935697ca7d23a..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/hour.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# hour -## description -### Syntax - -`INT HOUR(DATETIME date)` - - -获得日期中的小时的信息，返回值范围从0-23。 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select hour('2018-12-31 23:59:59'); -+-----------------------------+ -| hour('2018-12-31 23:59:59') | -+-----------------------------+ -| 23 | -+-----------------------------+ -``` -##keyword -HOUR diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/index.rst b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/index.rst deleted file mode 100644 index 9841d7ea1d124d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -日期函数 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/minute.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/minute.md deleted file mode 100644 index 84c3c9efdf9750..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/minute.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# minute -## description -### Syntax - -`INT MINUTE(DATETIME date)` - - -获得日期中的分钟的信息，返回值范围从0-59。 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select minute('2018-12-31 23:59:59'); -+-----------------------------+ -| minute('2018-12-31 23:59:59') | -+-----------------------------+ -| 59 | -+-----------------------------+ -``` -##keyword -MINUTE diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/month.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/month.md deleted file mode 100644 index 5847d390084be2..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/month.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# month -## description -### Syntax - -`INT MONTH(DATETIME date)` - - -返回时间类型中的月份信息，范围是1, 12 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select month('1987-01-01'); -+-----------------------------+ -| month('1987-01-01 00:00:00') | -+-----------------------------+ -| 1 | -+-----------------------------+ -``` - -##keyword - - MONTH diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/monthname.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/monthname.md deleted file mode 100644 index ce5580eabb69d5..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/monthname.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# monthname -## description -### Syntax - -`VARCHAR MONTHNAME(DATE)` - - -返回日期对应的月份名字 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select monthname('2008-02-03 00:00:00'); -+----------------------------------+ -| monthname('2008-02-03 00:00:00') | -+----------------------------------+ -| February | -+----------------------------------+ -``` - -##keyword - - MONTHNAME diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/now.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/now.md deleted file mode 100644 index 3264ed74a33e4a..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/now.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# now -## description -### Syntax - -`DATETIME NOW()` - - -获得当前的时间，以Datetime类型返回 - -## example - -``` -mysql> select now(); -+---------------------+ -| now() | -+---------------------+ -| 2019-05-27 15:58:25 | -+---------------------+ -``` - -##keyword - - NOW diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/second.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/second.md deleted file mode 100644 index d9226d3403db2c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/second.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# second -## description -### Syntax - -`INT SECOND(DATETIME date)` - - -获得日期中的秒的信息，返回值范围从0-59。 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select second('2018-12-31 23:59:59'); -+-----------------------------+ -| second('2018-12-31 23:59:59') | -+-----------------------------+ -| 59 | -+-----------------------------+ -``` -##keyword -SECOND diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/str_to_date.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/str_to_date.md deleted file mode 100644 index 630e6015c5955b..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/str_to_date.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# str_to_date -## description -### Syntax - -`DATETIME STR_TO_DATE(VARCHAR str, VARCHAR format)` - - -通过format指定的方式将str转化为DATE类型，如果转化结果不对返回NULL - -支持的format格式与date_format一致 - -## example - -``` -mysql> select str_to_date('2014-12-21 12:34:56', '%Y-%m-%d %H:%i:%s'); -+---------------------------------------------------------+ -| str_to_date('2014-12-21 12:34:56', '%Y-%m-%d %H:%i:%s') | -+---------------------------------------------------------+ -| 2014-12-21 12:34:56 | -+---------------------------------------------------------+ - -mysql> select str_to_date('2014-12-21 12:34%3A56', '%Y-%m-%d %H:%i%%3A%s'); -+--------------------------------------------------------------+ -| str_to_date('2014-12-21 12:34%3A56', '%Y-%m-%d %H:%i%%3A%s') | -+--------------------------------------------------------------+ -| 2014-12-21 12:34:56 | -+--------------------------------------------------------------+ - -mysql> select str_to_date('200442 Monday', '%X%V %W'); -+-----------------------------------------+ -| str_to_date('200442 Monday', '%X%V %W') | -+-----------------------------------------+ -| 2004-10-18 | -+-----------------------------------------+ -``` - -##keyword - - STR_TO_DATE,STR,TO,DATE diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timediff.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timediff.md deleted file mode 100644 index 8f43f11c010644..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timediff.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# timediff -## description -### Syntax - -`TIME TIMEDIFF(DATETIME expr1, DATETIME expr2)` - - -TIMEDIFF返回两个DATETIME之间的差值 - -TIMEDIFF函数返回表示为时间值的expr1 - expr2的结果，返回值为TIME类型 - -## example - -``` -mysql> SELECT TIMEDIFF(now(),utc_timestamp()); -+----------------------------------+ -| timediff(now(), utc_timestamp()) | -+----------------------------------+ -| 08:00:00 | -+----------------------------------+ - -mysql> SELECT TIMEDIFF('2019-07-11 16:59:30','2019-07-11 16:59:21'); -+--------------------------------------------------------+ -| timediff('2019-07-11 16:59:30', '2019-07-11 16:59:21') | -+--------------------------------------------------------+ -| 00:00:09 | -+--------------------------------------------------------+ - -mysql> SELECT TIMEDIFF('2019-01-01 00:00:00', NULL); -+---------------------------------------+ -| timediff('2019-01-01 00:00:00', NULL) | -+---------------------------------------+ -| NULL | -+---------------------------------------+ -``` - -##keyword - - TIMEDIFF diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timestampadd.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timestampadd.md deleted file mode 100644 index 34848e6067ef87..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timestampadd.md +++ /dev/null @@ -1,52 +0,0 @@ - - -# timestampadd -## description -### Syntax - -`DATETIME TIMESTAMPADD(unit, interval, DATETIME datetime_expr)` - - -将整数表达式间隔添加到日期或日期时间表达式datetime_expr中。 - -interval的单位由unit参数给出，它应该是下列值之一: - -SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, or YEAR。 - -## example - -``` - -mysql> SELECT TIMESTAMPADD(MINUTE,1,'2019-01-02'); -+------------------------------------------------+ -| timestampadd(MINUTE, 1, '2019-01-02 00:00:00') | -+------------------------------------------------+ -| 2019-01-02 00:01:00 | -+------------------------------------------------+ - -mysql> SELECT TIMESTAMPADD(WEEK,1,'2019-01-02'); -+----------------------------------------------+ -| timestampadd(WEEK, 1, '2019-01-02 00:00:00') | -+----------------------------------------------+ -| 2019-01-09 00:00:00 | -+----------------------------------------------+ -``` -##keyword -TIMESTAMPADD diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timestampdiff.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timestampdiff.md deleted file mode 100644 index 86a3557ef1844f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/timestampdiff.md +++ /dev/null @@ -1,60 +0,0 @@ - - -# timestampdiff -## description -### Syntax - -`INT TIMESTAMPDIFF(unit,DATETIME datetime_expr1, DATETIME datetime_expr2)` - -返回datetime_expr2−datetime_expr1，其中datetime_expr1和datetime_expr2是日期或日期时间表达式。 - -结果(整数)的单位由unit参数给出。interval的单位由unit参数给出，它应该是下列值之一: - -SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, or YEAR。 - -## example - -``` - -MySQL> SELECT TIMESTAMPDIFF(MONTH,'2003-02-01','2003-05-01'); -+--------------------------------------------------------------------+ -| timestampdiff(MONTH, '2003-02-01 00:00:00', '2003-05-01 00:00:00') | -+--------------------------------------------------------------------+ -| 3 | -+--------------------------------------------------------------------+ - -MySQL> SELECT TIMESTAMPDIFF(YEAR,'2002-05-01','2001-01-01'); -+-------------------------------------------------------------------+ -| timestampdiff(YEAR, '2002-05-01 00:00:00', '2001-01-01 00:00:00') | -+-------------------------------------------------------------------+ -| -1 | -+-------------------------------------------------------------------+ - - -MySQL> SELECT TIMESTAMPDIFF(MINUTE,'2003-02-01','2003-05-01 12:05:55'); -+---------------------------------------------------------------------+ -| timestampdiff(MINUTE, '2003-02-01 00:00:00', '2003-05-01 12:05:55') | -+---------------------------------------------------------------------+ -| 128885 | -+---------------------------------------------------------------------+ - -``` -##keyword -TIMESTAMPDIFF diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/to_days.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/to_days.md deleted file mode 100644 index 054e6bc73c4c56..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/to_days.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# to_days -## description -### Syntax - -`INT TO_DAYS(DATETIME date)` - - -返回date距离0000-01-01的天数 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select to_days('2007-10-07'); -+-----------------------+ -| to_days('2007-10-07') | -+-----------------------+ -| 733321 | -+-----------------------+ -``` - -##keyword - - TO_DAYS,TO,DAYS diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/unix_timestamp.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/unix_timestamp.md deleted file mode 100644 index 7e0f061c29480a..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/unix_timestamp.md +++ /dev/null @@ -1,79 +0,0 @@ - - -# unix_timestamp -## description -### Syntax - -`INT UNIX_TIMESTAMP(), UNIX_TIMESTAMP(DATETIME date), UNIX_TIMESTAMP(DATETIME date, STRING fmt),` - -将 Date 或者 Datetime 类型转化为 unix 时间戳。 - -如果没有参数，则是将当前的时间转化为时间戳。 - -参数需要是 Date 或者 Datetime 类型。 - -对于在 1970-01-01 00:00:00 之前或 2038-01-19 03:14:07 之后的时间，该函数将返回 0。 - -Format 的格式请参阅 `date_format` 函数的格式说明。 - -该函数受时区影响。 - -## example - -``` -mysql> select unix_timestamp(); -+------------------+ -| unix_timestamp() | -+------------------+ -| 1558589570 | -+------------------+ - -mysql> select unix_timestamp('2007-11-30 10:30:19'); -+---------------------------------------+ -| unix_timestamp('2007-11-30 10:30:19') | -+---------------------------------------+ -| 1196389819 | -+---------------------------------------+ - -mysql> select unix_timestamp('2007-11-30 10:30-19', '%Y-%m-%d %H:%i-%s'); -+---------------------------------------+ -| unix_timestamp('2007-11-30 10:30-19') | -+---------------------------------------+ -| 1196389819 | -+---------------------------------------+ - -mysql> select unix_timestamp('2007-11-30 10:30%3A19', '%Y-%m-%d %H:%i%%3A%s'); -+---------------------------------------+ -| unix_timestamp('2007-11-30 10:30%3A19') | -+---------------------------------------+ -| 1196389819 | -+---------------------------------------+ - -mysql> select unix_timestamp('1969-01-01 00:00:00'); -+---------------------------------------+ -| unix_timestamp('1969-01-01 00:00:00') | -+---------------------------------------+ -| 0 | -+---------------------------------------+ -``` - -##keyword - - UNIX_TIMESTAMP,UNIX,TIMESTAMP diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/utc_timestamp.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/utc_timestamp.md deleted file mode 100644 index c4a56f3aa79273..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/utc_timestamp.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# utc_timestamp -## description -### Syntax - -`DATETIME UTC_TIMESTAMP()` - - -返回当前UTC日期和时间在 "YYYY-MM-DD HH:MM:SS" 或 - -"YYYYMMDDHHMMSS"格式的一个值 - -根据该函数是否用在字符串或数字语境中 - -## example - -``` -mysql> select utc_timestamp(),utc_timestamp() + 1; -+---------------------+---------------------+ -| utc_timestamp() | utc_timestamp() + 1 | -+---------------------+---------------------+ -| 2019-07-10 12:31:18 | 20190710123119 | -+---------------------+---------------------+ -``` - -##keyword - - UTC_TIMESTAMP,UTC,TIMESTAMP diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/workofyear.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/workofyear.md deleted file mode 100644 index 8f1ca97579402e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/workofyear.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# weekofyear -## description -### Syntax - -`INT WEEKOFYEAR(DATETIME date)` - - - -获得一年中的第几周 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select weekofyear('2008-02-20 00:00:00'); -+-----------------------------------+ -| weekofyear('2008-02-20 00:00:00') | -+-----------------------------------+ -| 8 | -+-----------------------------------+ -``` - -##keyword - - WEEKOFYEAR diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/year.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/year.md deleted file mode 100644 index 62505c32aeb17e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/year.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# year -## description -### Syntax - -`INT YEAR(DATETIME date)` - - -返回date类型的year部分，范围从1000-9999 - -参数为Date或者Datetime类型 - -## example - -``` -mysql> select year('1987-01-01'); -+-----------------------------+ -| year('1987-01-01 00:00:00') | -+-----------------------------+ -| 1987 | -+-----------------------------+ -``` - -##keyword - - YEAR diff --git a/docs/documentation/cn/sql-reference/sql-functions/hash-functions/index.rst b/docs/documentation/cn/sql-reference/sql-functions/hash-functions/index.rst deleted file mode 100644 index b0556ff3b2a2b7..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/hash-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -Hash函数 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md b/docs/documentation/cn/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md deleted file mode 100644 index cf0cc98da797f8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# murmur_hash3_32 - -## description -### Syntax - -`INT MURMUR_HASH3_32(VARCHAR input, ...)` - -返回输入字符串的32位murmur3 hash值 - -## example - -``` -mysql> select murmur_hash3_32(null); -+-----------------------+ -| murmur_hash3_32(NULL) | -+-----------------------+ -| NULL | -+-----------------------+ - -mysql> select murmur_hash3_32("hello"); -+--------------------------+ -| murmur_hash3_32('hello') | -+--------------------------+ -| 1321743225 | -+--------------------------+ - -mysql> select murmur_hash3_32("hello", "world"); -+-----------------------------------+ -| murmur_hash3_32('hello', 'world') | -+-----------------------------------+ -| 984713481 | -+-----------------------------------+ -``` - -## keyword - - MURMUR_HASH3_32,HASH diff --git a/docs/documentation/cn/sql-reference/sql-functions/index.rst b/docs/documentation/cn/sql-reference/sql-functions/index.rst deleted file mode 100644 index 281d1a8bfdc5f1..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -=========== -SQL 函数 -=========== - -.. toctree:: - :glob: - - * - -.. toctree:: - :hidden: - - date-time-functions/index - spatial-functions/index - string-functions/index - aggregate-functions/index - bitmap-functions/index - hash-functions/index diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/index.rst b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/index.rst deleted file mode 100644 index f09712d02b8b01..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -地理位置函数 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_astext.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_astext.md deleted file mode 100644 index 1d4d8cdb9c55f7..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_astext.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_AsText`,`ST_AsWKT` -## description -### Syntax - -`VARCHAR ST_AsText(GEOMETRY geo)` - - -将一个几何图形转化为WKT（Well Known Text）的表示形式 - -## example - -``` -mysql> SELECT ST_AsText(ST_Point(24.7, 56.7)); -+---------------------------------+ -| st_astext(st_point(24.7, 56.7)) | -+---------------------------------+ -| POINT (24.7 56.7) | -+---------------------------------+ -``` -##keyword -ST_ASTEXT,ST_ASWKT,ST,ASTEXT,ASWKT diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_circle.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_circle.md deleted file mode 100644 index f87c13193ab42e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_circle.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# `ST_Circle` -## description -### Syntax - -`GEOMETRY ST_Circle(DOUBLE center_lng, DOUBLE center_lat, DOUBLE radius)` - - -将一个WKT（Well Known Text）转化为地球球面上的一个圆。其中`center_lng`表示的圆心的经度， -`center_lat`表示的是圆心的纬度，`radius`表示的是圆的半径，单位是米,最大支持9999999 - -## example - -``` -mysql> SELECT ST_AsText(ST_Circle(111, 64, 10000)); -+--------------------------------------------+ -| st_astext(st_circle(111.0, 64.0, 10000.0)) | -+--------------------------------------------+ -| CIRCLE ((111 64), 10000) | -+--------------------------------------------+ -``` -##keyword -ST_CIRCLE,ST,CIRCLE diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_contains.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_contains.md deleted file mode 100644 index 93d4863ecd2be2..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_contains.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# `ST_Contains` -## description -### Syntax - -`BOOL ST_Contains(GEOMETRY shape1, GEOMETRY shape2)` - - -判断几何图形shape1是否完全能够包含几何图形shape2 - -## example - -``` -mysql> SELECT ST_Contains(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"), ST_Point(5, 5)); -+----------------------------------------------------------------------------------------+ -| st_contains(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))'), st_point(5.0, 5.0)) | -+----------------------------------------------------------------------------------------+ -| 1 | -+----------------------------------------------------------------------------------------+ - -mysql> SELECT ST_Contains(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"), ST_Point(50, 50)); -+------------------------------------------------------------------------------------------+ -| st_contains(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))'), st_point(50.0, 50.0)) | -+------------------------------------------------------------------------------------------+ -| 0 | -+------------------------------------------------------------------------------------------+ -``` -##keyword -ST_CONTAINS,ST,CONTAINS diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md deleted file mode 100644 index 32c7341daf3f3a..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_Distance_Sphere` -## description -### Syntax - -`DOUBLE ST_Distance_Sphere(DOUBLE x_lng, DOUBLE x_lat, DOUBLE y_lng, DOUBLE x_lat)` - - -计算地球两点之间的球面距离，单位为 米。传入的参数分别为X点的经度，X点的纬度，Y点的经度，Y点的纬度。 - -## example - -``` -mysql> select st_distance_sphere(116.35620117, 39.939093, 116.4274406433, 39.9020987219); -+----------------------------------------------------------------------------+ -| st_distance_sphere(116.35620117, 39.939093, 116.4274406433, 39.9020987219) | -+----------------------------------------------------------------------------+ -| 7336.9135549995917 | -+----------------------------------------------------------------------------+ -``` -##keyword -ST_DISTANCE_SPHERE,ST,DISTANCE,SPHERE diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md deleted file mode 100644 index fe4e0f72d81fba..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_GeometryFromText`,`ST_GeomFromText` -## description -### Syntax - -`GEOMETRY ST_GeometryFromText(VARCHAR wkt)` - - -将一个WKT（Well Known Text）转化为对应的内存的几何形式 - -## example - -``` -mysql> SELECT ST_AsText(ST_GeometryFromText("LINESTRING (1 1, 2 2)")); -+---------------------------------------------------------+ -| st_astext(st_geometryfromtext('LINESTRING (1 1, 2 2)')) | -+---------------------------------------------------------+ -| LINESTRING (1 1, 2 2) | -+---------------------------------------------------------+ -``` -##keyword -ST_GEOMETRYFROMTEXT,ST_GEOMFROMTEXT,ST,GEOMETRYFROMTEXT,GEOMFROMTEXT diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_linefromtext.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_linefromtext.md deleted file mode 100644 index ba7ea26b233b61..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_linefromtext.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_LineFromText`,`ST_LineStringFromText` -## description -### Syntax - -`GEOMETRY ST_LineFromText(VARCHAR wkt)` - - -将一个WKT（Well Known Text）转化为一个Line形式的内存表现形式 - -## example - -``` -mysql> SELECT ST_AsText(ST_LineFromText("LINESTRING (1 1, 2 2)")); -+---------------------------------------------------------+ -| st_astext(st_geometryfromtext('LINESTRING (1 1, 2 2)')) | -+---------------------------------------------------------+ -| LINESTRING (1 1, 2 2) | -+---------------------------------------------------------+ -``` -##keyword -ST_LINEFROMTEXT,ST_LINESTRINGFROMTEXT,ST,LINEFROMTEXT,LINESTRINGFROMTEXT diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_point.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_point.md deleted file mode 100644 index c22150ac5dbb08..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_point.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# `ST_Point` -## description -### Syntax - -`POINT ST_Point(DOUBLE x, DOUBLE y)` - - -通过给定的X坐标值，Y坐标值返回对应的Point。 -当前这个值只是在球面集合上有意义，X/Y对应的是经度/纬度(longitude/latitude);ps:直接select ST_Point()会卡主，慎重！！！ - -## example - -``` -mysql> SELECT ST_AsText(ST_Point(24.7, 56.7)); -+---------------------------------+ -| st_astext(st_point(24.7, 56.7)) | -+---------------------------------+ -| POINT (24.7 56.7) | -+---------------------------------+ -``` -##keyword -ST_POINT,ST,POINT diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_polygon.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_polygon.md deleted file mode 100644 index 591ec2978dacc6..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_polygon.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_Polygon`,`ST_PolyFromText`,`ST_PolygonFromText` -## description -### Syntax - -`GEOMETRY ST_Polygon(VARCHAR wkt)` - - -将一个WKT（Well Known Text）转化为对应的多边形内存形式 - -## example - -``` -mysql> SELECT ST_AsText(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))")); -+------------------------------------------------------------------+ -| st_astext(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))')) | -+------------------------------------------------------------------+ -| POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0)) | -+------------------------------------------------------------------+ -``` -##keyword -ST_POLYGON,ST_POLYFROMTEXT,ST_POLYGONFROMTEXT,ST,POLYGON,POLYFROMTEXT,POLYGONFROMTEXT diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_x.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_x.md deleted file mode 100644 index 784da1aad6a92d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_x.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_X` -## description -### Syntax - -`DOUBLE ST_X(POINT point)` - - -当point是一个合法的POINT类型时，返回对应的X坐标值 - -## example - -``` -mysql> SELECT ST_X(ST_Point(24.7, 56.7)); -+----------------------------+ -| st_x(st_point(24.7, 56.7)) | -+----------------------------+ -| 24.7 | -+----------------------------+ -``` -##keyword -ST_X,ST,X diff --git a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_y.md b/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_y.md deleted file mode 100644 index 19db95328befbb..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/spatial-functions/st_y.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_Y` -## description -### Syntax - -`DOUBLE ST_Y(POINT point)` - - -当point是一个合法的POINT类型时，返回对应的Y坐标值 - -## example - -``` -mysql> SELECT ST_Y(ST_Point(24.7, 56.7)); -+----------------------------+ -| st_y(st_point(24.7, 56.7)) | -+----------------------------+ -| 56.7 | -+----------------------------+ -``` -##keyword -ST_Y,ST,Y diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/ascii.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/ascii.md deleted file mode 100644 index 2e03d83db82e57..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/ascii.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# ascii -## description -### Syntax - -`INT ascii(VARCHAR str)` - - -返回字符串第一个字符对应的 ascii 码 - -## example - -``` -mysql> select ascii('1'); -+------------+ -| ascii('1') | -+------------+ -| 49 | -+------------+ - -mysql> select ascii('234'); -+--------------+ -| ascii('234') | -+--------------+ -| 50 | -+--------------+ -``` -##keyword -ASCII diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/concat.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/concat.md deleted file mode 100644 index 6dba21a30d4216..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/concat.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# concat -## description -### Syntax - -`VARCHAR concat(VARCHAR,...)` - - -将多个字符串连接起来, 如果参数中任意一个值是 NULL，那么返回的结果就是 NULL - -## example - -``` -mysql> select concat("a", "b"); -+------------------+ -| concat('a', 'b') | -+------------------+ -| ab | -+------------------+ - -mysql> select concat("a", "b", "c"); -+-----------------------+ -| concat('a', 'b', 'c') | -+-----------------------+ -| abc | -+-----------------------+ - -mysql> select concat("a", null, "c"); -+------------------------+ -| concat('a', NULL, 'c') | -+------------------------+ -| NULL | -+------------------------+ -``` -##keyword -CONCAT diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/concat_ws.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/concat_ws.md deleted file mode 100644 index ab00a22512552c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/concat_ws.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# concat_ws -## description -### Syntax - -`VARCHAR concat_ws(VARCHAR sep, VARCHAR str,...)` - - -使用第一个参数 sep 作为连接符，将第二个参数以及后续所有参数拼接成一个字符串. -如果分隔符是 NULL，返回 NULL。 -`concat_ws`函数不会跳过空字符串，会跳过 NULL 值 - -## example - -``` -mysql> select concat_ws("or", "d", "is"); -+----------------------------+ -| concat_ws('or', 'd', 'is') | -+----------------------------+ -| doris | -+----------------------------+ - -mysql> select concat_ws(NULL, "d", "is"); -+----------------------------+ -| concat_ws(NULL, 'd', 'is') | -+----------------------------+ -| NULL | -+----------------------------+ - -mysql> select concat_ws("or", "d", NULL,"is"); -+---------------------------------+ -| concat_ws("or", "d", NULL,"is") | -+---------------------------------+ -| doris | -+---------------------------------+ -``` -##keyword -CONCAT_WS,CONCAT,WS diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/ends_with.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/ends_with.md deleted file mode 100644 index c2eecca6844273..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/ends_with.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# ends_with -## description -### Syntax - -`BOOLEAN ENDS_WITH (VARCHAR str, VARCHAR suffix)` - -如果字符串以指定后缀结尾，返回true。否则，返回false。任意参数为NULL，返回NULL。 - -## example - -``` -mysql> select ends_with("Hello doris", "doris"); -+-----------------------------------+ -| ends_with('Hello doris', 'doris') | -+-----------------------------------+ -| 1 | -+-----------------------------------+ - -mysql> select ends_with("Hello doris", "Hello"); -+-----------------------------------+ -| ends_with('Hello doris', 'Hello') | -+-----------------------------------+ -| 0 | -+-----------------------------------+ -``` -##keyword -ENDS_WITH diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/find_in_set.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/find_in_set.md deleted file mode 100644 index c66c779227e783..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/find_in_set.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# find_in_set -## description -### Syntax - -`INT find_in_set(VARCHAR str, VARCHAR strlist)` - - -返回 strlist 中第一次出现 str 的位置（从1开始计数）。strlist 是用逗号分隔的字符串。如果没有找到，返回0。任意参数为 NULL ，返回 NULL。 - -## example - -``` -mysql> select find_in_set("b", "a,b,c"); -+---------------------------+ -| find_in_set('b', 'a,b,c') | -+---------------------------+ -| 2 | -+---------------------------+ -``` -##keyword -FIND_IN_SET,FIND,IN,SET diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_double.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_double.md deleted file mode 100644 index 06a9f2fcf82c01..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_double.md +++ /dev/null @@ -1,67 +0,0 @@ - - -# get_json_double -## description -### Syntax - -`DOUBLE get_json_double(VARCHAR json_str, VARCHAR json_path) - - -解析并获取 json 字符串内指定路径的浮点型内容。 -其中 json_path 必须以 $ 符号作为开头，使用 . 作为路径分割符。如果路径中包含 . ，则可以使用双引号包围。 -使用 [ ] 表示数组下标，从 0 开始。 -path 的内容不能包含 ", [ 和 ]。 -如果 json_string 格式不对，或 json_path 格式不对，或无法找到匹配项，则返回 NULL。 - -## example - -1. 获取 key 为 "k1" 的 value - -``` -mysql> SELECT get_json_double('{"k1":1.3, "k2":"2"}', "$.k1"); -+-------------------------------------------------+ -| get_json_double('{"k1":1.3, "k2":"2"}', '$.k1') | -+-------------------------------------------------+ -| 1.3 | -+-------------------------------------------------+ -``` - -2. 获取 key 为 "my.key" 的数组中第二个元素 - -``` -mysql> SELECT get_json_double('{"k1":"v1", "my.key":[1.1, 2.2, 3.3]}', '$."my.key"[1]'); -+---------------------------------------------------------------------------+ -| get_json_double('{"k1":"v1", "my.key":[1.1, 2.2, 3.3]}', '$."my.key"[1]') | -+---------------------------------------------------------------------------+ -| 2.2 | -+---------------------------------------------------------------------------+ -``` - -3. 获取二级路径为 k1.key -> k2 的数组中，第一个元素 -``` -mysql> SELECT get_json_double('{"k1.key":{"k2":[1.1, 2.2]}}', '$."k1.key".k2[0]'); -+---------------------------------------------------------------------+ -| get_json_double('{"k1.key":{"k2":[1.1, 2.2]}}', '$."k1.key".k2[0]') | -+---------------------------------------------------------------------+ -| 1.1 | -+---------------------------------------------------------------------+ -``` -##keyword -GET_JSON_DOUBLE,GET,JSON,DOUBLE diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_int.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_int.md deleted file mode 100644 index 1d7dde03295f35..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_int.md +++ /dev/null @@ -1,67 +0,0 @@ - - -# get_json_int -## description -### Syntax - -`INT get_json_int(VARCHAR json_str, VARCHAR json_path) - - -解析并获取 json 字符串内指定路径的整型内容。 -其中 json_path 必须以 $ 符号作为开头，使用 . 作为路径分割符。如果路径中包含 . ，则可以使用双引号包围。 -使用 [ ] 表示数组下标，从 0 开始。 -path 的内容不能包含 ", [ 和 ]。 -如果 json_string 格式不对，或 json_path 格式不对，或无法找到匹配项，则返回 NULL。 - -## example - -1. 获取 key 为 "k1" 的 value - -``` -mysql> SELECT get_json_int('{"k1":1, "k2":"2"}', "$.k1"); -+--------------------------------------------+ -| get_json_int('{"k1":1, "k2":"2"}', '$.k1') | -+--------------------------------------------+ -| 1 | -+--------------------------------------------+ -``` - -2. 获取 key 为 "my.key" 的数组中第二个元素 - -``` -mysql> SELECT get_json_int('{"k1":"v1", "my.key":[1, 2, 3]}', '$."my.key"[1]'); -+------------------------------------------------------------------+ -| get_json_int('{"k1":"v1", "my.key":[1, 2, 3]}', '$."my.key"[1]') | -+------------------------------------------------------------------+ -| 2 | -+------------------------------------------------------------------+ -``` - -3. 获取二级路径为 k1.key -> k2 的数组中，第一个元素 -``` -mysql> SELECT get_json_int('{"k1.key":{"k2":[1, 2]}}', '$."k1.key".k2[0]'); -+--------------------------------------------------------------+ -| get_json_int('{"k1.key":{"k2":[1, 2]}}', '$."k1.key".k2[0]') | -+--------------------------------------------------------------+ -| 1 | -+--------------------------------------------------------------+ -``` -##keyword -GET_JSON_INT,GET,JSON,INT diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_string.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_string.md deleted file mode 100644 index cad4ee57fa5d68..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/get_json_string.md +++ /dev/null @@ -1,77 +0,0 @@ - - -# get_json_string -## description -### Syntax - -`VARCHAR get_json_string(VARCHAR json_str, VARCHAR json_path) - - -解析并获取 json 字符串内指定路径的字符串内容。 -其中 json_path 必须以 $ 符号作为开头，使用 . 作为路径分割符。如果路径中包含 . ，则可以使用双引号包围。 -使用 [ ] 表示数组下标，从 0 开始。 -path 的内容不能包含 ", [ 和 ]。 -如果 json_string 格式不对，或 json_path 格式不对，或无法找到匹配项，则返回 NULL。 - -## example - -1. 获取 key 为 "k1" 的 value - -``` -mysql> SELECT get_json_string('{"k1":"v1", "k2":"v2"}', "$.k1"); -+---------------------------------------------------+ -| get_json_string('{"k1":"v1", "k2":"v2"}', '$.k1') | -+---------------------------------------------------+ -| v1 | -+---------------------------------------------------+ -``` - -2. 获取 key 为 "my.key" 的数组中第二个元素 - -``` -mysql> SELECT get_json_string('{"k1":"v1", "my.key":["e1", "e2", "e3"]}', '$."my.key"[1]'); -+------------------------------------------------------------------------------+ -| get_json_string('{"k1":"v1", "my.key":["e1", "e2", "e3"]}', '$."my.key"[1]') | -+------------------------------------------------------------------------------+ -| e2 | -+------------------------------------------------------------------------------+ -``` - -3. 获取二级路径为 k1.key -> k2 的数组中，第一个元素 -``` -mysql> SELECT get_json_string('{"k1.key":{"k2":["v1", "v2"]}}', '$."k1.key".k2[0]'); -+-----------------------------------------------------------------------+ -| get_json_string('{"k1.key":{"k2":["v1", "v2"]}}', '$."k1.key".k2[0]') | -+-----------------------------------------------------------------------+ -| v1 | -+-----------------------------------------------------------------------+ -``` - -4. 获取数组中，key 为 "k1" 的所有 value -``` -mysql> SELECT get_json_string('[{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}]', '$.k1'); -+---------------------------------------------------------------------------------+ -| get_json_string('[{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}]', '$.k1') | -+---------------------------------------------------------------------------------+ -| ["v1","v3","v4"] | -+---------------------------------------------------------------------------------+ -``` -##keyword -GET_JSON_STRING,GET,JSON,STRING diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/group_concat.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/group_concat.md deleted file mode 100644 index 669b866d510ff6..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/group_concat.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# group_concat -## description -### Syntax - -`VARCHAR group_concat(VARCHAR str[, VARCHAR sep])` - - -该函数是类似于 sum() 的聚合函数，group_concat 将结果集中的多行结果连接成一个字符串。第二个参数 sep 为字符串之间的连接符号，该参数可以省略。该函数通常需要和 group by 语句一起使用。 - -## example - -``` -mysql> select value from test; -+-------+ -| value | -+-------+ -| a | -| b | -| c | -+-------+ - -mysql> select group_concat(value) from test; -+-----------------------+ -| group_concat(`value`) | -+-----------------------+ -| a, b, c | -+-----------------------+ - -mysql> select group_concat(value, " ") from test; -+----------------------------+ -| group_concat(`value`, ' ') | -+----------------------------+ -| a b c | -+----------------------------+ -``` -##keyword -GROUP_CONCAT,GROUP,CONCAT diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/index.rst b/docs/documentation/cn/sql-reference/sql-functions/string-functions/index.rst deleted file mode 100644 index 7d372f8a93a082..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -字符串函数 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/instr.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/instr.md deleted file mode 100644 index 53e1d2419d1d36..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/instr.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# instr -## description -### Syntax - -`INT instr(VARCHAR str, VARCHAR substr)` - - -返回 substr 在 str 中第一次出现的位置（从1开始计数）。如果 substr 不在 str 中出现，则返回0。 - -## example - -``` -mysql> select instr("abc", "b"); -+-------------------+ -| instr('abc', 'b') | -+-------------------+ -| 2 | -+-------------------+ - -mysql> select instr("abc", "d"); -+-------------------+ -| instr('abc', 'd') | -+-------------------+ -| 0 | -+-------------------+ -``` -##keyword -INSTR diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/lcase.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/lcase.md deleted file mode 100644 index 47de71fa4d5f7f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/lcase.md +++ /dev/null @@ -1,30 +0,0 @@ - - -# lcase -## description -### Syntax - -`INT lcase(VARCHAR str)` - - -与`lower`一致 - -##keyword -LCASE diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/left.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/left.md deleted file mode 100644 index 5f7f3f815c3b15..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/left.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# left -## description -### Syntax - -`VARCHAR left(VARCHAR str)` - - -它返回具有指定长度的字符串的左边部分 - -## example - -``` -mysql> select left("Hello doris",5); -+------------------------+ -| left('Hello doris', 5) | -+------------------------+ -| Hello | -+------------------------+ -``` -##keyword -LEFT diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/length.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/length.md deleted file mode 100644 index 3de7e6f6345771..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/length.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# length -## description -### Syntax - -`INT length(VARCHAR str)` - - -返回字符串的长度，对于多字节字符，返回的字符数。比如5个两字节宽度字，返回的长度是10。 - -## example - -``` -mysql> select length("abc"); -+---------------+ -| length('abc') | -+---------------+ -| 3 | -+---------------+ - -mysql> select length("中国"); -+------------------+ -| length('中国') | -+------------------+ -| 6 | -+------------------+ -``` -##keyword -LENGTH diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/locate.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/locate.md deleted file mode 100644 index 1ed1aa7904ea48..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/locate.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# locate -## description -### Syntax - -`INT locate(VARCHAR substr, VARCHAR str[, INT pos])` - - -返回 substr 在 str 中出现的位置（从1开始计数）。如果指定第3个参数 pos，则从 str 以 pos 下标开始的字符串处开始查找 substr 出现的位置。如果没有找到，返回0 - -## example - -``` -mysql> SELECT LOCATE('bar', 'foobarbar'); -+----------------------------+ -| locate('bar', 'foobarbar') | -+----------------------------+ -| 4 | -+----------------------------+ - -mysql> SELECT LOCATE('xbar', 'foobar'); -+--------------------------+ -| locate('xbar', 'foobar') | -+--------------------------+ -| 0 | -+--------------------------+ - -mysql> SELECT LOCATE('bar', 'foobarbar', 5); -+-------------------------------+ -| locate('bar', 'foobarbar', 5) | -+-------------------------------+ -| 7 | -+-------------------------------+ -``` -##keyword -LOCATE diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/lower.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/lower.md deleted file mode 100644 index 35aebe992d0952..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/lower.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# lower -## description -### Syntax - -`INT lower(VARCHAR str)` - - -将参数中所有的字符串都转换成小写 - -## example - -``` -mysql> SELECT lower("AbC123"); -+-----------------+ -| lower('AbC123') | -+-----------------+ -| abc123 | -+-----------------+ -``` -##keyword -LOWER diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/lpad.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/lpad.md deleted file mode 100644 index 7025f13725f212..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/lpad.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# lpad -## description -### Syntax - -`VARCHAR lpad(VARCHAR str, INT len, VARCHAR pad)` - - -返回 str 中长度为 len（从首字母开始算起）的字符串。如果 len 大于 str 的长度，则在 str 的前面不断补充 pad 字符，直到该字符串的长度达到 len 为止。如果 len 小于 str 的长度，该函数相当于截断 str 字符串，只返回长度为 len 的字符串。 - -## example - -``` -mysql> SELECT lpad("hi", 5, "xy"); -+---------------------+ -| lpad('hi', 5, 'xy') | -+---------------------+ -| xyxhi | -+---------------------+ - -mysql> SELECT lpad("hi", 1, "xy"); -+---------------------+ -| lpad('hi', 1, 'xy') | -+---------------------+ -| h | -+---------------------+ -``` -##keyword -LPAD diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/ltrim.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/ltrim.md deleted file mode 100644 index b57a1233e6743a..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/ltrim.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# ltrim -## description -### Syntax - -`VARCHAR ltrim(VARCHAR str)` - - -将参数 str 中从开始部分连续出现的空格去掉 - -## example - -``` -mysql> SELECT ltrim(' ab d'); -+------------------+ -| ltrim(' ab d') | -+------------------+ -| ab d | -+------------------+ -``` -##keyword -LTRIM diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/money_format.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/money_format.md deleted file mode 100644 index a26b58f0444f76..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/money_format.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# money_format -## description -### Syntax - -VARCHAR money_format(Number) - - -将数字按照货币格式输出，整数部分每隔3位用逗号分隔，小数部分保留2位 - -## example - -``` -mysql> select money_format(17014116); -+------------------------+ -| money_format(17014116) | -+------------------------+ -| 17,014,116.00 | -+------------------------+ - -mysql> select money_format(1123.456); -+------------------------+ -| money_format(1123.456) | -+------------------------+ -| 1,123.46 | -+------------------------+ - -mysql> select money_format(1123.4); -+----------------------+ -| money_format(1123.4) | -+----------------------+ -| 1,123.40 | -+----------------------+ -``` -##keyword -MONEY_FORMAT,MONEY,FORMAT diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/null_or_empty.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/null_or_empty.md deleted file mode 100644 index b1debd351af2bf..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/null_or_empty.md +++ /dev/null @@ -1,53 +0,0 @@ - - -# null_or_empty -## description -### Syntax - -`BOOLEAN NULL_OR_EMPTY (VARCHAR str)` - -如果字符串为空字符串或者NULL，返回true。否则，返回false。 - -## example - -``` -MySQL [(none)]> select null_or_empty(null); -+---------------------+ -| null_or_empty(NULL) | -+---------------------+ -| 1 | -+---------------------+ - -MySQL [(none)]> select null_or_empty(""); -+-------------------+ -| null_or_empty('') | -+-------------------+ -| 1 | -+-------------------+ - -MySQL [(none)]> select null_or_empty("a"); -+--------------------+ -| null_or_empty('a') | -+--------------------+ -| 0 | -+--------------------+ -``` -##keyword -NULL_OR_EMPTY \ No newline at end of file diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/regexp_extract.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/regexp_extract.md deleted file mode 100644 index cf71a074d30125..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/regexp_extract.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# regexp_extract -## description -### Syntax - -`VARCHAR regexp_extract(VARCHAR str, VARCHAR pattern, int pos)` - - -对字符串 str 进行正则匹配，抽取符合 pattern 的第 pos 个匹配部分。需要 pattern 完全匹配 str 中的某部分，这样才能返回 pattern 部分中需匹配部分。如果没有匹配，返回空字符串。 - -## example - -``` -mysql> SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 1); -+-------------------------------------------------------------+ -| regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 1) | -+-------------------------------------------------------------+ -| b | -+-------------------------------------------------------------+ - -mysql> SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 2); -+-------------------------------------------------------------+ -| regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 2) | -+-------------------------------------------------------------+ -| d | -+-------------------------------------------------------------+ -``` -##keyword -REGEXP_EXTRACT,REGEXP,EXTRACT diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/regexp_replace.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/regexp_replace.md deleted file mode 100644 index 22f0d7e8e6031c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/regexp_replace.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# regexp_replace -## description -### Syntax - -`VARCHAR regexp_replace(VARCHAR str, VARCHAR pattern, VARCHAR repl) - - -对字符串 str 进行正则匹配, 将命中 pattern 的部分使用 repl 来进行替换 - -## example - -``` -mysql> SELECT regexp_replace('a b c', " ", "-"); -+-----------------------------------+ -| regexp_replace('a b c', ' ', '-') | -+-----------------------------------+ -| a-b-c | -+-----------------------------------+ - -mysql> SELECT regexp_replace('a b c','(b)','<\\1>'); -+----------------------------------------+ -| regexp_replace('a b c', '(b)', '<\1>') | -+----------------------------------------+ -| a c | -+----------------------------------------+ -``` -##keyword -REGEXP_REPLACE,REGEXP,REPLACE diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/repeat.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/repeat.md deleted file mode 100644 index 142da95dd27356..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/repeat.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# repeat -## description -### Syntax - -`VARCHAR repeat(VARCHAR str, INT count) - - -将字符串 str 重复 count 次输出，count 小于1时返回空串，str，count 任一为NULL时，返回 NULL - -## example - -``` -mysql> SELECT repeat("a", 3); -+----------------+ -| repeat('a', 3) | -+----------------+ -| aaa | -+----------------+ - -mysql> SELECT repeat("a", -1); -+-----------------+ -| repeat('a', -1) | -+-----------------+ -| | -+-----------------+ -``` -##keyword -REPEAT, diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/right.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/right.md deleted file mode 100644 index 79c7b657420be8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/right.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# right -## description -### Syntax - -`VARCHAR right(VARCHAR str)` - - -它返回具有指定长度的字符串的右边部分 - -## example - -``` -mysql> select right("Hello doris",5); -+-------------------------+ -| right('Hello doris', 5) | -+-------------------------+ -| doris | -+-------------------------+ -``` -##keyword -RIGHT diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/split_part.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/split_part.md deleted file mode 100644 index 71712e5116bb06..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/split_part.md +++ /dev/null @@ -1,62 +0,0 @@ - - -# split_part -## description -### Syntax - -`VARCHAR split_part(VARCHAR content, VARCHAR delimiter, INT field)` - - -根据分割符拆分字符串, 返回指定的分割部分(从一开始计数)。 - -## example - -``` -mysql> select split_part("hello world", " ", 1); -+----------------------------------+ -| split_part('hello world', ' ', 1) | -+----------------------------------+ -| hello | -+----------------------------------+ - - -mysql> select split_part("hello world", " ", 2); -+----------------------------------+ -| split_part('hello world', ' ', 2) | -+----------------------------------+ -| world | -+----------------------------------+ - -mysql> select split_part("2019年7月8号", "月", 1); -+-----------------------------------------+ -| split_part('2019年7月8号', '月', 1) | -+-----------------------------------------+ -| 2019年7 | -+-----------------------------------------+ - -mysql> select split_part("abca", "a", 1); -+----------------------------+ -| split_part('abca', 'a', 1) | -+----------------------------+ -| | -+----------------------------+ -``` -##keyword -SPLIT_PART,SPLIT,PART diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/starts_with.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/starts_with.md deleted file mode 100644 index 5997abd140fbaa..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/starts_with.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# starts_with -## description -### Syntax - -`BOOLEAN STARTS_WITH (VARCHAR str, VARCHAR prefix)` - -如果字符串以指定前缀开头，返回true。否则，返回false。任意参数为NULL，返回NULL。 - -## example - -``` -MySQL [(none)]> select starts_with("hello world","hello"); -+-------------------------------------+ -| starts_with('hello world', 'hello') | -+-------------------------------------+ -| 1 | -+-------------------------------------+ - -MySQL [(none)]> select starts_with("hello world","world"); -+-------------------------------------+ -| starts_with('hello world', 'world') | -+-------------------------------------+ -| 0 | -+-------------------------------------+ -``` -##keyword -STARTS_WITH \ No newline at end of file diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/strleft.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/strleft.md deleted file mode 100644 index 7e27f2d61abb4c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/strleft.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# strleft -## description -### Syntax - -`VARCHAR strleft(VARCHAR str)` - - -它返回具有指定长度的字符串的左边部分 - -## example - -``` -mysql> select strleft("Hello doris",5); -+------------------------+ -| strleft('Hello doris', 5) | -+------------------------+ -| Hello | -+------------------------+ -``` -##keyword -STRLEFT diff --git a/docs/documentation/cn/sql-reference/sql-functions/string-functions/strright.md b/docs/documentation/cn/sql-reference/sql-functions/string-functions/strright.md deleted file mode 100644 index 8ff50f66eb23ac..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-functions/string-functions/strright.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# strright -## description -### Syntax - -`VARCHAR strright(VARCHAR str)` - - -它返回具有指定长度的字符串的右边部分 - -## example - -``` -mysql> select strright("Hello doris",5); -+-------------------------+ -| strright('Hello doris', 5) | -+-------------------------+ -| doris | -+-------------------------+ -``` -##keyword -STRRIGHT diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/CREATE ROLE.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/CREATE ROLE.md deleted file mode 100644 index 4dd50074876e39..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/CREATE ROLE.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# CREATE ROLE -## description - 该语句用户创建一个角色 - - 语法： - CREATE ROLE role1; - - 该语句创建一个无权限的角色，可以后续通过 GRANT 命令赋予该角色权限。 - -## example - - 1. 创建一个角色 - - CREATE ROLE role1; - -## keyword - - CREATE, ROLE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/CREATE USER.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/CREATE USER.md deleted file mode 100644 index 44e3ec40f03d6b..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/CREATE USER.md +++ /dev/null @@ -1,69 +0,0 @@ - - -# CREATE USER -## description - -Syntax: - - CREATE USER user_identity [IDENTIFIED BY 'password'] [DEFAULT ROLE 'role_name'] - - user_identity: - 'user_name'@'host' - -CREATE USER 命令用于创建一个 Doris 用户。在 Doris 中，一个 user_identity 唯一标识一个用户。user_identity 由两部分组成，user_name 和 host，其中 username 为用户名。host 标识用户端连接所在的主机地址。host 部分可以使用 % 进行模糊匹配。如果不指定 host，默认为 '%'，即表示该用户可以从任意 host 连接到 Doris。 - -host 部分也可指定为 domain，语法为：'user_name'@['domain']，即使用中括号包围，则 Doris 会认为这个是一个 domain，并尝试解析其 ip 地址。目前仅支持百度内部的 BNS 解析。 - -如果指定了角色（ROLE），则会自动将该角色所拥有的权限赋予新创建的这个用户。如果不指定，则该用户默认没有任何权限。指定的 ROLE 必须已经存在。 - -## example - -1. 创建一个无密码用户（不指定 host，则等价于 jack@'%'） - - CREATE USER 'jack'; - -2. 创建一个有密码用户，允许从 '172.10.1.10' 登陆 - - CREATE USER jack@'172.10.1.10' IDENTIFIED BY '123456'; - -3. 为了避免传递明文，用例2也可以使用下面的方式来创建 - - CREATE USER jack@'172.10.1.10' IDENTIFIED BY PASSWORD '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9'; - - 后面加密的内容可以通过PASSWORD()获得到,例如： - - SELECT PASSWORD('123456'); - -4. 创建一个允许从 '192.168' 子网登陆的用户，同时指定其角色为 example_role - - CREATE USER 'jack'@'192.168.%' DEFAULT ROLE 'example_role'; - -5. 创建一个允许从域名 'example_domain' 登陆的用户 - - CREATE USER 'jack'@['example_domain'] IDENTIFIED BY '12345'; - -6. 创建一个用户，并指定一个角色 - - CREATE USER 'jack'@'%' IDENTIFIED BY '12345' DEFAULT ROLE 'my_role'; - -## keyword - - CREATE, USER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/DROP ROLE.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/DROP ROLE.md deleted file mode 100644 index 411cf8c14e3361..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/DROP ROLE.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# DROP ROLE -## description - 该语句用户删除一个角色 - - 语法： - DROP ROLE role1; - - 删除一个角色，不会影响之前属于该角色的用户的权限。仅相当于将该角色与用户解耦。用户已经从该角色中获取到的权限，不会改变。 - -## example - - 1. 删除一个角色 - - DROP ROLE role1; - -## keyword - DROP, ROLE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/DROP USER.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/DROP USER.md deleted file mode 100644 index cbdc21274e1a32..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/DROP USER.md +++ /dev/null @@ -1,43 +0,0 @@ - - -# DROP USER -## description - -Syntax: - - DROP USER 'user_identity' - - `user_identity`: - - user@'host' - user@['domain'] - - 删除指定的 user identitiy. - -## example - -1. 删除用户 jack@'192.%' - - DROP USER 'jack'@'192.%' - -## keyword - - DROP, USER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/GRANT.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/GRANT.md deleted file mode 100644 index e8bc853586cb3f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/GRANT.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# GRANT -## description - -GRANT 命令用于赋予指定用户或角色指定的权限。 - -Syntax: - - GRANT privilege_list ON db_name[.tbl_name] TO user_identity [ROLE role_name] - - -privilege_list 是需要赋予的权限列表，以逗号分隔。当前 Doris 支持如下权限： - - NODE_PRIV：集群节点操作权限，包括节点上下线等操作，只有 root 用户有该权限，不可赋予其他用户。 - ADMIN_PRIV：除 NODE_PRIV 以外的所有权限。 - GRANT_PRIV: 操作权限的权限。包括创建删除用户、角色，授权和撤权，设置密码等。 - SELECT_PRIV：对指定的库或表的读取权限 - LOAD_PRIV：对指定的库或表的导入权限 - ALTER_PRIV：对指定的库或表的schema变更权限 - CREATE_PRIV：对指定的库或表的创建权限 - DROP_PRIV：对指定的库或表的删除权限 - - 旧版权限中的 ALL 和 READ_WRITE 会被转换成：SELECT_PRIV,LOAD_PRIV,ALTER_PRIV,CREATE_PRIV,DROP_PRIV； - READ_ONLY 会被转换为 SELECT_PRIV。 - -db_name[.tbl_name] 支持以下三种形式： - - 1. *.* 权限可以应用于所有库及其中所有表 - 2. db.* 权限可以应用于指定库下的所有表 - 3. db.tbl 权限可以应用于指定库下的指定表 - - 这里指定的库或表可以是不存在的库和表。 - -user_identity： - - 这里的 user_identity 语法同 CREATE USER。且必须为使用 CREATE USER 创建过的 user_identity。user_identity 中的host可以是域名，如果是域名的话，权限的生效时间可能会有1分钟左右的延迟。 - - 也可以将权限赋予指定的 ROLE，如果指定的 ROLE 不存在，则会自动创建。 - -## example - - 1. 授予所有库和表的权限给用户 - - GRANT SELECT_PRIV ON *.* TO 'jack'@'%'; - - 2. 授予指定库表的权限给用户 - - GRANT SELECT_PRIV,ALTER_PRIV,LOAD_PRIV ON db1.tbl1 TO 'jack'@'192.8.%'; - - 3. 授予指定库表的权限给角色 - - GRANT LOAD_PRIV ON db1.* TO ROLE 'my_role'; - -## keyword - - GRANT - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/REVOKE.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/REVOKE.md deleted file mode 100644 index 4d387d0df9b263..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/REVOKE.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# REVOKE -## description - - REVOKE 命令用于撤销指定用户或角色指定的权限。 - Syntax： - REVOKE privilege_list ON db_name[.tbl_name] FROM user_identity [ROLE role_name] - - user_identity： - - 这里的 user_identity 语法同 CREATE USER。且必须为使用 CREATE USER 创建过的 user_identity。user_identity 中的host可以是域名，如果是域名的话，权限的撤销时间可能会有1分钟左右的延迟。 - - 也可以撤销指定的 ROLE 的权限，执行的 ROLE 必须存在。 - -## example - - 1. 撤销用户 jack 数据库 testDb 的权限 - - REVOKE SELECT_PRIV ON db1.* FROM 'jack'@'192.%'; - -## keyword - - REVOKE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/SET PASSWORD.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/SET PASSWORD.md deleted file mode 100644 index c030f05e97699e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/SET PASSWORD.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# SET PASSWORD -## description - -Syntax: - - SET PASSWORD [FOR user_identity] = - [PASSWORD('plain password')]|['hashed password'] - - SET PASSWORD 命令可以用于修改一个用户的登录密码。如果 [FOR user_identity] 字段不存在，那么修改当前用户的密码。 - - 注意这里的 user_identity 必须完全匹配在使用 CREATE USER 创建用户时指定的 user_identity，否则会报错用户不存在。如果不指定 user_identity，则当前用户为 'username'@'ip'，这个当前用户，可能无法匹配任何 user_identity。可以通过 SHOW GRANTS 查看当前用户。 - - PASSWORD() 方式输入的是明文密码; 而直接使用字符串，需要传递的是已加密的密码。 - 如果修改其他用户的密码，需要具有管理员权限。 - -## example - -1. 修改当前用户的密码 - - SET PASSWORD = PASSWORD('123456') - SET PASSWORD = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' - -2. 修改指定用户密码 - - SET PASSWORD FOR 'jack'@'192.%' = PASSWORD('123456') - SET PASSWORD FOR 'jack'@['domain'] = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' - -## keyword - SET, PASSWORD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/SET PROPERTY.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/SET PROPERTY.md deleted file mode 100644 index cf0181825e2df6..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/SET PROPERTY.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# SET PROPERTY -## description - - Syntax: - - SET PROPERTY [FOR 'user'] 'key' = 'value' [, 'key' = 'value'] - - 设置用户的属性，包括分配给用户的资源、导入cluster等。这里设置的用户属性，是针对 user 的，而不是 user_identity。即假设通过 CREATE USER 语句创建了两个用户 'jack'@'%' 和 'jack'@'192.%'，则使用 SET PROPERTY 语句，只能针对 jack 这个用户，而不是 'jack'@'%' 或 'jack'@'192.%' - - 导入 cluster 仅适用于百度内部用户。 - - key: - - 超级用户权限: - max_user_connections: 最大连接数。 - resource.cpu_share: cpu资源分配。 - load_cluster.{cluster_name}.priority: 为指定的cluster分配优先级，可以为 HIGH 或 NORMAL - - 普通用户权限： - quota.normal: normal级别的资源分配。 - quota.high: high级别的资源分配。 - quota.low: low级别的资源分配。 - - load_cluster.{cluster_name}.hadoop_palo_path: palo使用的hadoop目录，需要存放etl程序及etl生成的中间数据供palo导入。导入完成后会自动清理中间数据，etl程序自动保留下次使用。 - load_cluster.{cluster_name}.hadoop_configs: hadoop的配置，其中fs.default.name、mapred.job.tracker、hadoop.job.ugi必须填写。 - load_cluster.{cluster_name}.hadoop_http_port: hadoop hdfs name node http端口。其中 hdfs 默认为8070，afs 默认 8010。 - default_load_cluster: 默认的导入cluster。 - -## example - - 1. 修改用户 jack 最大连接数为1000 - SET PROPERTY FOR 'jack' 'max_user_connections' = '1000'; - - 2. 修改用户 jack 的cpu_share为1000 - SET PROPERTY FOR 'jack' 'resource.cpu_share' = '1000'; - - 3. 修改 jack 用户的normal组的权重 - SET PROPERTY FOR 'jack' 'quota.normal' = '400'; - - 4. 为用户 jack 添加导入cluster - SET PROPERTY FOR 'jack' - 'load_cluster.{cluster_name}.hadoop_palo_path' = '/user/palo/palo_path', - 'load_cluster.{cluster_name}.hadoop_configs' = 'fs.default.name=hdfs://dpp.cluster.com:port;mapred.job.tracker=dpp.cluster.com:port;hadoop.job.ugi=user,password;mapred.job.queue.name=job_queue_name_in_hadoop;mapred.job.priority=HIGH;'; - - 5. 删除用户 jack 下的导入cluster。 - SET PROPERTY FOR 'jack' 'load_cluster.{cluster_name}' = ''; - - 6. 修改用户 jack 默认的导入cluster - SET PROPERTY FOR 'jack' 'default_load_cluster' = '{cluster_name}'; - - 7. 修改用户 jack 的集群优先级为 HIGH - SET PROPERTY FOR 'jack' 'load_cluster.{cluster_name}.priority' = 'HIGH'; - -## keyword - SET, PROPERTY - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/SHOW GRANTS.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/SHOW GRANTS.md deleted file mode 100644 index 72487fdb3c8a2d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/SHOW GRANTS.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# SHOW GRANTS -## description - - 该语句用于查看用户权限。 - - 语法： - SHOW [ALL] GRANTS [FOR user_identity]; - - 说明： - 1. SHOW ALL GRANTS 可以查看所有用户的权限。 - 2. 如果指定 user_identity，则查看该指定用户的权限。且该 user_identity 必须为通过 CREATE USER 命令创建的。 - 3. 如果不指定 user_identity，则查看当前用户的权限。 - - -## example - - 1. 查看所有用户权限信息 - - SHOW ALL GRANTS; - - 2. 查看指定 user 的权限 - - SHOW GRANTS FOR jack@'%'; - - 3. 查看当前用户的权限 - - SHOW GRANTS; - -## keyword - - SHOW, GRANTS diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/SHOW ROLES.md b/docs/documentation/cn/sql-reference/sql-statements/Account Management/SHOW ROLES.md deleted file mode 100644 index 245f0abc1cb164..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/SHOW ROLES.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# SHOW ROLES -## description - 该语句用于展示所有已创建的角色信息，包括角色名称，包含的用户以及权限。 - - 语法： - SHOW ROLES; - -## example - - 1. 查看已创建的角色： - - SHOW ROLES; - -## keyword - SHOW,ROLES - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Account Management/index.rst b/docs/documentation/cn/sql-reference/sql-statements/Account Management/index.rst deleted file mode 100644 index bdb0665e65760f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Account Management/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -用户账户管理 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md deleted file mode 100644 index 638c6804efbdc3..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# ADMIN CANCEL REPAIR -## description - - 该语句用于取消以高优先级修复指定表或分区 - - 语法： - - ADMIN CANCEL REPAIR TABLE table_name[ PARTITION (p1,...)]; - - 说明： - - 1. 该语句仅表示系统不再以高优先级修复指定表或分区的分片副本。系统仍会以默认调度方式修复副本。 - -## example - - 1. 取消高优先级修复 - - ADMIN CANCEL REPAIR TABLE tbl PARTITION(p1); - -## keyword - ADMIN,CANCEL,REPAIR - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md deleted file mode 100644 index 35262008a35046..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# ADMIN CHECK TABLET -## description - -该语句用于对一组 tablet 执行指定的检查操作 - -语法： - -``` -ADMIN CHECK TABLE (tablet_id1, tablet_id2, ...) -PROPERTIES("type" = "..."); -``` - -说明： - -1. 必须指定 tablet id 列表以及 PROPERTIES 中的 type 属性。 -2. 目前 type 仅支持： - - * consistency: 对tablet的副本数据一致性进行检查。该命令为异步命令，发送后，Doris 会开始执行对应 tablet 的一致性检查作业。最终的结果，将体现在 `SHOW PROC "/statistic";` 结果中的 InconsistentTabletNum 列。 - -## example - -1. 对指定的一组 tablet 进行副本数据一致性检查 - - ``` - ADMIN CHECK TABLET (10000, 10001) - PROPERTIES("type" = "consistency"); - ``` - -## keyword - - ADMIN,CHECK,TABLET diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN REPAIR.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN REPAIR.md deleted file mode 100644 index f24815e6ec2b93..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN REPAIR.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# ADMIN REPAIR -## description - - 该语句用于尝试优先修复指定的表或分区 - - 语法： - - ADMIN REPAIR TABLE table_name[ PARTITION (p1,...)] - - 说明： - - 1. 该语句仅表示让系统尝试以高优先级修复指定表或分区的分片副本，并不保证能够修复成功。用户可以通过 ADMIN SHOW REPLICA STATUS 命令查看修复情况。 - 2. 默认的 timeout 是 14400 秒(4小时)。超时意味着系统将不再以高优先级修复指定表或分区的分片副本。需要重新使用该命令设置。 - -## example - - 1. 尝试修复指定表 - - ADMIN REPAIR TABLE tbl1; - - 2. 尝试修复指定分区 - - ADMIN REPAIR TABLE tbl1 PARTITION (p1, p2); - -## keyword - ADMIN,REPAIR - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md deleted file mode 100644 index dfc19ebe5f35ba..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# ADMIN SET CONFIG -## description - - 该语句用于设置集群的配置项（当前仅支持设置FE的配置项）。 - 可设置的配置项，可以通过 AMDIN SHOW FRONTEND CONFIG; 命令查看。 - - 语法： - - ADMIN SET FRONTEND CONFIG ("key" = "value"); - -## example - - 1. 设置 'disable_balance' 为 true - - ADMIN SET FRONTEND CONFIG ("disable_balance" = "true"); - -## keyword - ADMIN,SET,CONFIG diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md deleted file mode 100644 index f3e37a48f27d89..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# ADMIN SET REPLICA STATUS -## description - - 该语句用于设置指定副本的状态。 - 该命令目前仅用于手动将某些副本状态设置为 BAD 或 OK，从而使得系统能够自动修复这些副本。 - - 语法： - - ADMIN SET REPLICA STATUS - PROPERTIES ("key" = "value", ...); - - 目前支持如下属性： - "tablet_id"：必需。指定一个 Tablet Id. - "backend_id"：必需。指定 Backend Id. - "status"：必需。指定状态。当前仅支持 "bad" 或 "ok" - - 如果指定的副本不存在，或状态已经是 bad，则会被忽略。 - - 注意： - - 设置为 Bad 状态的副本可能立刻被删除，请谨慎操作。 - -## example - - 1. 设置 tablet 10003 在 BE 10001 上的副本状态为 bad。 - - ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "10003", "backend_id" = "10001", "status" = "bad"); - - 2. 设置 tablet 10003 在 BE 10001 上的副本状态为 ok。 - - ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "10003", "backend_id" = "10001", "status" = "ok"); - -## keyword - - ADMIN,SET,REPLICA,STATUS - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md deleted file mode 100644 index 3af6a0d928386f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# ADMIN SHOW CONFIG -## description - - 该语句用于展示当前集群的配置（当前仅支持展示 FE 的配置项） - - 语法： - - ADMIN SHOW FRONTEND CONFIG; - - 说明： - - 结果中的各列含义如下： - 1. Key： 配置项名称 - 2. Value： 配置项值 - 3. Type： 配置项类型 - 4. IsMutable： 是否可以通过 ADMIN SET CONFIG 命令设置 - 5. MasterOnly： 是否仅适用于 Master FE - 6. Comment： 配置项说明 - -## example - - 1. 查看当前FE节点的配置 - - ADMIN SHOW FRONTEND CONFIG; - -## keyword - ADMIN,SHOW,CONFIG diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md deleted file mode 100644 index 23aa65eb8e1c07..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# ADMIN SHOW REPLICA DISTRIBUTION -## description - - 该语句用于展示一个表或分区副本分布状态 - - 语法： - - ADMIN SHOW REPLICA DISTRIBUTION FROM [db_name.]tbl_name [PARTITION (p1, ...)]; - - 说明： - - 结果中的 Graph 列以图形的形式展示副本分布比例 - -## example - - 1. 查看表的副本分布 - - ADMIN SHOW REPLICA DISTRIBUTION FROM tbl1; - - 2. 查看表的分区的副本分布 - - ADMIN SHOW REPLICA DISTRIBUTION FROM db1.tbl1 PARTITION(p1, p2); - -## keyword - ADMIN,SHOW,REPLICA,DISTRIBUTION - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md deleted file mode 100644 index 80ec9a81129b9a..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# ADMIN SHOW REPLICA STATUS -## description - - 该语句用于展示一个表或分区的副本状态信息 - - 语法： - - ADMIN SHOW REPLICA STATUS FROM [db_name.]tbl_name [PARTITION (p1, ...)] - [where_clause]; - - where_clause: - WHERE STATUS [!]= "replica_status" - - replica_status: - OK: replica 处于健康状态 - DEAD: replica 所在 Backend 不可用 - VERSION_ERROR: replica 数据版本有缺失 - SCHEMA_ERROR: replica 的 schema hash 不正确 - MISSING: replica 不存在 - -## example - - 1. 查看表全部的副本状态 - - ADMIN SHOW REPLICA STATUS FROM db1.tbl1; - - 2. 查看表某个分区状态为 VERSION_ERROR 的副本 - - ADMIN SHOW REPLICA STATUS FROM tbl1 PARTITION (p1, p2) - WHERE STATUS = "VERSION_ERROR"; - - 3. 查看表所有状态不健康的副本 - - ADMIN SHOW REPLICA STATUS FROM tbl1 - WHERE STATUS != "OK"; - -## keyword - ADMIN,SHOW,REPLICA,STATUS - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ALTER CLUSTER.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ALTER CLUSTER.md deleted file mode 100644 index fd763d02bc0a2d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ALTER CLUSTER.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# ALTER CLUSTER -## description - - 该语句用于更新逻辑集群。需要有管理员权限 - - 语法 - - ALTER CLUSTER cluster_name PROPERTIES ("key"="value", ...); - - 1. 缩容，扩容 （根据集群现有的be数目，大则为扩容，小则为缩容), 扩容为同步操作，缩容为异步操作，通过backend的状态可以得知是否缩容完成 - - PROERTIES ("instance_num" = "3") - - instance_num 逻辑集群节点树 - -## example - - 1. 缩容，减少含有3个be的逻辑集群test_cluster的be数为2 - - ALTER CLUSTER test_cluster PROPERTIES ("instance_num"="2"); - - 2. 扩容，增加含有3个be的逻辑集群test_cluster的be数为4 - - ALTER CLUSTER test_cluster PROPERTIES ("instance_num"="4"); - -## keyword - ALTER,CLUSTER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ALTER SYSTEM.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ALTER SYSTEM.md deleted file mode 100644 index 82f85579420d2e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ALTER SYSTEM.md +++ /dev/null @@ -1,113 +0,0 @@ - - -# ALTER SYSTEM -## description - - 该语句用于操作一个系统内的节点。（仅管理员使用！） - 语法： - 1) 增加节点(不使用多租户功能则按照此方法添加) - ALTER SYSTEM ADD BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - 2) 增加空闲节点(即添加不属于任何cluster的BACKEND) - ALTER SYSTEM ADD FREE BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - 3) 增加节点到某个cluster - ALTER SYSTEM ADD BACKEND TO cluster_name "host:heartbeat_port"[,"host:heartbeat_port"...]; - 4) 删除节点 - ALTER SYSTEM DROP BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - 5) 节点下线 - ALTER SYSTEM DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - 6) 增加Broker - ALTER SYSTEM ADD BROKER broker_name "host:port"[,"host:port"...]; - 7) 减少Broker - ALTER SYSTEM DROP BROKER broker_name "host:port"[,"host:port"...]; - 8) 删除所有Broker - ALTER SYSTEM DROP ALL BROKER broker_name - 9) 设置一个 Load error hub，用于集中展示导入时的错误信息 - ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES ("key" = "value"[, ...]); - - 说明： - 1) host 可以是主机名或者ip地址 - 2) heartbeat_port 为该节点的心跳端口 - 3) 增加和删除节点为同步操作。这两种操作不考虑节点上已有的数据，节点直接从元数据中删除，请谨慎使用。 - 4) 节点下线操作用于安全下线节点。该操作为异步操作。如果成功，节点最终会从元数据中删除。如果失败，则不会完成下线。 - 5) 可以手动取消节点下线操作。详见 CANCEL DECOMMISSION - 6) Load error hub: - 当前支持两种类型的 Hub：Mysql 和 Broker。需在 PROPERTIES 中指定 "type" = "mysql" 或 "type" = "broker"。 - 如果需要删除当前的 load error hub，可以将 type 设为 null。 - 1) 当使用 Mysql 类型时，导入时产生的错误信息将会插入到指定的 mysql 库表中，之后可以通过 show load warnings 语句直接查看错误信息。 - - Mysql 类型的 Hub 需指定以下参数： - host：mysql host - port：mysql port - user：mysql user - password：mysql password - database：mysql database - table：mysql table - - 2) 当使用 Broker 类型时，导入时产生的错误信息会形成一个文件，通过 broker，写入到指定的远端存储系统中。须确保已经部署对应的 broker - Broker 类型的 Hub 需指定以下参数： - broker: broker 的名称 - path: 远端存储路径 - other properties: 其他访问远端存储所必须的信息，比如认证信息等。 - -## example - - 1. 增加一个节点 - ALTER SYSTEM ADD BACKEND "host:port"; - - 2. 增加一个空闲节点 - ALTER SYSTEM ADD FREE BACKEND "host:port"; - - 3. 删除两个节点 - ALTER SYSTEM DROP BACKEND "host1:port", "host2:port"; - - 4. 下线两个节点 - ALTER SYSTEM DECOMMISSION BACKEND "host1:port", "host2:port"; - - 5. 增加两个Hdfs Broker - ALTER SYSTEM ADD BROKER hdfs "host1:port", "host2:port"; - - 6. 添加一个 Mysql 类型的 load error hub - ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES - ("type"= "mysql", - "host" = "192.168.1.17" - "port" = "3306", - "user" = "my_name", - "password" = "my_passwd", - "database" = "doris_load", - "table" = "load_errors" - ); - - 7. 添加一个 Broker 类型的 load error hub - ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES - ("type"= "broker", - "name" = "bos", - "path" = "bos://backup-cmy/logs", - "bos_endpoint" = "http://gz.bcebos.com", - "bos_accesskey" = "069fc278xxxxxx24ddb522", - "bos_secret_accesskey"="700adb0c6xxxxxx74d59eaa980a" - ); - - 8. 删除当前的 load error hub - ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES - ("type"= "null"); - -## keyword - ALTER,SYSTEM,BACKEND,BROKER,FREE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md deleted file mode 100644 index bdac66f7af28e8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# CANCEL DECOMMISSION -## description - - 该语句用于撤销一个节点下线操作。（仅管理员使用！） - 语法： - CANCEL DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - -## example - - 1. 取消两个节点的下线操作： - CANCEL DECOMMISSION BACKEND "host1:port", "host2:port"; - -## keyword - CANCEL,DECOMMISSION,BACKEND - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/CREATE CLUSTER.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/CREATE CLUSTER.md deleted file mode 100644 index 4cc61533458bf3..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/CREATE CLUSTER.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# CREATE CLUSTER -## description - - 该语句用于新建逻辑集群 (cluster), 需要管理员权限。如果不使用多租户，直接创建一个名称为default_cluster的cluster。否则创建一个自定义名称的cluster。 - - 语法 - - CREATE CLUSTER [IF NOT EXISTS] cluster_name - - PROPERTIES ("key"="value", ...) - - IDENTIFIED BY 'password' - - 1. PROPERTIES - - 指定逻辑集群的属性 - - PROPERTIES ("instance_num" = "3") - - instance_num 逻辑集群节点树 - - 2. identified by ‘password' 每个逻辑集群含有一个superuser，创建逻辑集群时必须指定其密码 - -## example - - 1. 新建一个含有3个be节点逻辑集群 test_cluster, 并指定其superuser用户密码 - - CREATE CLUSTER test_cluster PROPERTIES("instance_num"="3") IDENTIFIED BY 'test'; - - 2. 新建一个含有3个be节点逻辑集群 default_cluster(不使用多租户), 并指定其superuser用户密码 - - CREATE CLUSTER default_cluster PROPERTIES("instance_num"="3") IDENTIFIED BY 'test'; - -## keyword - CREATE,CLUSTER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/CREATE FILE.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/CREATE FILE.md deleted file mode 100644 index 5441f0acd1211d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/CREATE FILE.md +++ /dev/null @@ -1,70 +0,0 @@ - - -# CREATE FILE -## description - - 该语句用于创建并上传一个文件到 Doris 集群。 - 该功能通常用于管理一些其他命令中需要使用到的文件，如证书、公钥私钥等等。 - - 该命令只用 amdin 权限用户可以执行。 - 某个文件都归属与某一个的 database。对 database 拥有访问权限的用户都可以使用该文件。 - - 单个文件大小限制为 1MB。 - 一个 Doris 集群最多上传 100 个文件。 - - 语法： - - CREATE FILE "file_name" [IN database] - [properties] - - 说明： - file_name: 自定义文件名。 - database: 文件归属于某一个 db，如果没有指定，则使用当前 session 的 db。 - properties 支持以下参数: - - url: 必须。指定一个文件的下载路径。当前仅支持无认证的 http 下载路径。命令执行成功后，文件将被保存在 doris 中，该 url 将不再需要。 - catalog: 必须。对文件的分类名，可以自定义。但在某些命令中，会查找指定 catalog 中的文件。比如例行导入中的，数据源为 kafka 时，会查找 catalog 名为 kafka 下的文件。 - md5: 可选。文件的 md5。如果指定，会在下载文件后进行校验。 - -## example - - 1. 创建文件 ca.pem ，分类为 kafka - - CREATE FILE "ca.pem" - PROPERTIES - ( - "url" = "https://test.bj.bcebos.com/kafka-key/ca.pem", - "catalog" = "kafka" - ); - - 2. 创建文件 client.key，分类为 my_catalog - - CREATE FILE "client.key" - IN my_database - PROPERTIES - ( - "url" = "https://test.bj.bcebos.com/kafka-key/client.key", - "catalog" = "my_catalog", - "md5" = "b5bb901bf10f99205b39a46ac3557dd9" - ); - -## keyword - CREATE,FILE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/DROP CLUSTER.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/DROP CLUSTER.md deleted file mode 100644 index 122080e2b54458..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/DROP CLUSTER.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# DROP CLUSTER -## description - - 该语句用于删除逻辑集群，成功删除逻辑集群需要首先删除集群内的db，需要管理员权限 - - 语法 - - DROP CLUSTER [IF EXISTS] cluster_name - -## example - - 删除逻辑集群 test_cluster - - DROP CLUSTER test_cluster; - -## keyword - DROP,CLUSTER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/DROP FILE.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/DROP FILE.md deleted file mode 100644 index 0157c30d39314a..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/DROP FILE.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# DROP FILE -## description - - 该语句用于删除一个已上传的文件。 - - 语法： - - DROP FILE "file_name" [FROM database] - [properties] - - 说明： - file_name: 文件名。 - database: 文件归属的某一个 db，如果没有指定，则使用当前 session 的 db。 - properties 支持以下参数: - - catalog: 必须。文件所属分类。 - -## example - - 1. 删除文件 ca.pem - - DROP FILE "ca.pem" properties("catalog" = "kafka"); - -## keyword - DROP,FILE diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/ENTER.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/ENTER.md deleted file mode 100644 index 8fdcc362b05141..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/ENTER.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# ENTER -## description - - 该语句用于进入一个逻辑集群, 所有创建用户、创建数据库都需要在一个逻辑集群内执行，创建后并且隶属于这个逻 - - 辑集群，需要管理员权限 - - ENTER cluster_name - -## example - - 1. 进入逻辑集群test_cluster - - ENTER test_cluster; - -## keyword - ENTER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md deleted file mode 100644 index 7233189f1f04c2..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# INSTALL PLUGIN -## description - - 该语句用于安装一个插件。 - - 语法 - - INSTALL PLUGIN FROM [source] - - source 支持三种类型： - - 1. 指向一个 zip 文件的绝对路径。 - 2. 指向一个插件目录的绝对路径。 - 3. 指向一个 http 或 https 协议的 zip 文件下载路径 - -## example - - 1. 安装一个本地 zip 文件插件： - - INSTALL PLUGIN FROM "/home/users/seaven/auditdemo.zip"; - - 2. 安装一个本地目录中的插件： - - INSTALL PLUGIN FROM "/home/users/seaven/auditdemo/"; - - 2. 下载并安装一个插件： - - INSTALL PLUGIN FROM "http://mywebsite.com/plugin.zip"; - -## keyword - INSTALL,PLUGIN diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/LINK DATABASE.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/LINK DATABASE.md deleted file mode 100644 index db1e30a2b1c594..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/LINK DATABASE.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# LINK DATABASE -## description - - （已废弃！！！） - 该语句用户链接一个逻辑集群的数据库到另外一个逻辑集群, 一个数据库只允许同时被链接一次，删除链接的数据库 - - 并不会删除数据，并且被链接的数据库不能被删除, 需要管理员权限 - - 语法 - - LINK DATABASE src_cluster_name.src_db_name des_cluster_name.des_db_name - -## example - - 1. 链接test_clusterA中的test_db到test_clusterB,并命名为link_test_db - - LINK DATABASE test_clusterA.test_db test_clusterB.link_test_db; - - 2. 删除链接的数据库link_test_db - - DROP DATABASE link_test_db; - -## keyword - LINK,DATABASE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md deleted file mode 100644 index ad90c596ddadff..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# MIGRATE DATABASE -## description - - （已废弃！！！） - 该语句用于迁移一个逻辑集群的数据库到另外一个逻辑集群，执行此操作前数据库必须已经处于链接状态, 需要管理 - - 员权限 - - 语法 - - MIGRATE DATABASE src_cluster_name.src_db_name des_cluster_name.des_db_name - -## example - - 1. 迁移test_clusterA中的test_db到test_clusterB - - MIGRATE DATABASE test_clusterA.test_db test_clusterB.link_test_db; - -## keyword - MIGRATE,DATABASE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW BACKENDS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW BACKENDS.md deleted file mode 100644 index 09621988d8b7f0..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW BACKENDS.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# SHOW BACKENDS -## description - 该语句用于查看 cluster 内的 BE 节点 - 语法： - SHOW BACKENDS; - - 说明： - 1. LastStartTime 表示最近一次 BE 启动时间。 - 2. LastHeartbeat 表示最近一次心跳。 - 3. Alive 表示节点是否存活。 - 4. SystemDecommissioned 为 true 表示节点正在安全下线中。 - 5. ClusterDecommissioned 为 true 表示节点正在冲当前cluster中下线。 - 6. TabletNum 表示该节点上分片数量。 - 7. DataUsedCapacity 表示实际用户数据所占用的空间。 - 8. AvailCapacity 表示磁盘的可使用空间。 - 9. TotalCapacity 表示总磁盘空间。TotalCapacity = AvailCapacity + DataUsedCapacity + 其他非用户数据文件占用空间。 - 10. UsedPct 表示磁盘已使用量百分比。 - 11. ErrMsg 用于显示心跳失败时的错误信息。 - -## keyword - SHOW, BACKENDS - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW BROKER.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW BROKER.md deleted file mode 100644 index 9e79cc593b8a31..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW BROKER.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# SHOW BROKER -## description - 该语句用于查看当前存在的 broker - 语法： - SHOW BROKER; - - 说明： - 1. LastStartTime 表示最近一次 BE 启动时间。 - 2. LastHeartbeat 表示最近一次心跳。 - 3. Alive 表示节点是否存活。 - 4. ErrMsg 用于显示心跳失败时的错误信息。 - -## keyword - SHOW, BROKER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FILE.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FILE.md deleted file mode 100644 index 82c41c655c8a61..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FILE.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# SHOW FILE -## description - - 该语句用于展示一个 database 内创建的文件 - - 语法： - - SHOW FILE [FROM database]; - - 说明： - - FileId: 文件ID，全局唯一 - DbName: 所属数据库名称 - Catalog: 自定义分类 - FileName: 文件名 - FileSize: 文件大小，单位字节 - MD5: 文件的 MD5 - -## example - - 1. 查看数据库 my_database 中已上传的文件 - - SHOW FILE FROM my_database; - -## keyword - SHOW,FILE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md deleted file mode 100644 index 9cc2e3135e026e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# SHOW FRONTENDS -## description - 该语句用于查看 FE 节点 - 语法： - SHOW FRONTENDS; - - 说明： - 1. name 表示该 FE 节点在 bdbje 中的名称。 - 2. Join 为 true 表示该节点曾经加入过集群。但不代表当前还在集群内（可能已失联） - 3. Alive 表示节点是否存活。 - 4. ReplayedJournalId 表示该节点当前已经回放的最大元数据日志id。 - 5. LastHeartbeat 是最近一次心跳。 - 6. IsHelper 表示该节点是否是 bdbje 中的 helper 节点。 - 7. ErrMsg 用于显示心跳失败时的错误信息。 - -## keyword - SHOW, FRONTENDS - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md deleted file mode 100644 index eb0c8341208a66..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md +++ /dev/null @@ -1,33 +0,0 @@ - - -# SHOW FULL COLUMNS -## description - 该语句用于指定表的列信息 - 语法： - SHOW FULL COLUMNS FROM tbl; - -## example - 1. 查看指定表的列信息 - - SHOW FULL COLUMNS FROM tbl; - -## keyword - - SHOW,TABLE,STATUS diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW INDEX.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW INDEX.md deleted file mode 100644 index d29a5dfa4fa307..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW INDEX.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# SHOW INDEX - -## description - - 该语句用于展示一个表中索引的相关信息，目前只支持bitmap 索引 - 语法： - SHOW INDEX[ES] FROM [db_name.]table_name [FROM database]; - 或者 - SHOW KEY[S] FROM [db_name.]table_name [FROM database]; - -## example - - 1. 展示指定 table_name 的下索引 - SHOW INDEX FROM example_db.table_name; - -## keyword - - SHOW,INDEX diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md deleted file mode 100644 index 182ba83f3d0431..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md +++ /dev/null @@ -1,31 +0,0 @@ - - -# SHOW MIGRATIONS -## description - - 该语句用于查看数据库迁移的进度 - - 语法 - - SHOW MIGRATIONS - -## keyword - SHOW,MIGRATIONS - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW PLUGINS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW PLUGINS.md deleted file mode 100644 index aceeb9cf8693e8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW PLUGINS.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# SHOW PLUGINS -## description - - 该语句用于展示已安装的插件。 - - 语法 - - SHOW PLUGINS; - - 该命令会展示所有用户安装的和系统内置的插件。 - -## example - - 1. 展示已安装的插件： - - SHOW PLUGINS; - -## keyword - SHOW PLUGINS \ No newline at end of file diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md deleted file mode 100644 index 0b6543cefa6820..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# SHOW TABLE STATUS -## description - 该语句用于查看 Table 的一些信息。 - 语法： - SHOW TABLE STATUS - [FROM db] [LIKE "pattern"] - - 说明： - 1. 该语句主要用于兼容 MySQL 语法，目前仅显示 Comment 等少量信息 - -## example - 1. 查看当前数据库下所有表的信息 - - SHOW TABLE STATUS; - - 2. 查看指定数据库下，名称包含 example 的表的信息 - - SHOW TABLE STATUS FROM db LIKE "%example%"; - -## keyword - - SHOW,TABLE,STATUS diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/index.rst b/docs/documentation/cn/sql-reference/sql-statements/Administration/index.rst deleted file mode 100644 index 7695bcdccce9de..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -集群管理 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER DATABASE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER DATABASE.md deleted file mode 100644 index f6f1c4cee583c1..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER DATABASE.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# ALTER DATABASE -## description - 该语句用于设置指定数据库的属性。（仅管理员使用） - 语法： - 1) 设置数据库数据量配额，单位为B/K/KB/M/MB/G/GB/T/TB/P/PB - ALTER DATABASE db_name SET DATA QUOTA quota; - - 2) 重命名数据库 - ALTER DATABASE db_name RENAME new_db_name; - - 3) 设置数据库的副本数量配额 - ALTER DATABASE db_name SET REPLICA QUOTA quota; - - 说明： - 重命名数据库后，如需要，请使用 REVOKE 和 GRANT 命令修改相应的用户权限。 - 数据库的默认数据量配额为1024GB，默认副本数量配额为1073741824。 - -## example - 1. 设置指定数据库数据量配额 - ALTER DATABASE example_db SET DATA QUOTA 10995116277760; - 上述单位为字节,等价于 - ALTER DATABASE example_db SET DATA QUOTA 10T; - - ALTER DATABASE example_db SET DATA QUOTA 100G; - - ALTER DATABASE example_db SET DATA QUOTA 200M; - - 2. 将数据库 example_db 重命名为 example_db2 - ALTER DATABASE example_db RENAME example_db2; - - 3. 设定指定数据库副本数量配额 - ALTER DATABASE example_db SET REPLICA QUOTA 102400; - -## keyword - ALTER,DATABASE,RENAME - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md deleted file mode 100644 index 0e2ac88f540f4f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md +++ /dev/null @@ -1,352 +0,0 @@ - - -# ALTER TABLE - -## description - - 该语句用于对已有的 table 进行修改。如果没有指定 rollup index，默认操作 base index。 - 该语句分为三种操作类型： schema change 、rollup 、partition - 这三种操作类型不能同时出现在一条 ALTER TABLE 语句中。 - 其中 schema change 和 rollup 是异步操作，任务提交成功则返回。之后可使用 SHOW ALTER 命令查看进度。 - partition 是同步操作，命令返回表示执行完毕。 - - 语法： - ALTER TABLE [database.]table - alter_clause1[, alter_clause2, ...]; - - alter_clause 分为 partition 、rollup、schema change、rename 和index五种。 - - partition 支持如下几种修改方式 - 1. 增加分区 - 语法： - ADD PARTITION [IF NOT EXISTS] partition_name - partition_desc ["key"="value"] - [DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]] - 注意： - 1) partition_desc 支持一下两种写法： - * VALUES LESS THAN [MAXVALUE|("value1", ...)] - * VALUES [("value1", ...), ("value1", ...)) - 1) 分区为左闭右开区间，如果用户仅指定右边界，系统会自动确定左边界 - 2) 如果没有指定分桶方式，则自动使用建表使用的分桶方式 - 3) 如指定分桶方式，只能修改分桶数，不可修改分桶方式或分桶列 - 4) ["key"="value"] 部分可以设置分区的一些属性，具体说明见 CREATE TABLE - - 2. 删除分区 - 语法： - DROP PARTITION [IF EXISTS] partition_name - 注意： - 1) 使用分区方式的表至少要保留一个分区。 - 2) 执行 DROP PARTITION 一段时间内，可以通过 RECOVER 语句恢复被删除的 partition。详见 RECOVER 语句 - - 3. 修改分区属性 - 语法： - MODIFY PARTITION partition_name SET ("key" = "value", ...) - 说明： - 1) 当前支持修改分区的下列属性： - - storage_medium - - storage_cooldown_time - - replication_num - — in_memory - 2) 对于单分区表，partition_name 同表名。 - - rollup 支持如下几种创建方式： - 1. 创建 rollup index - 语法： - ADD ROLLUP rollup_name (column_name1, column_name2, ...) - [FROM from_index_name] - [PROPERTIES ("key"="value", ...)] - - properties: 支持设置超时时间，默认超时时间为1天。 - 例子： - ADD ROLLUP r1(col1,col2) from r0 - 1.2 批量创建 rollup index - 语法： - ADD ROLLUP [rollup_name (column_name1, column_name2, ...) - [FROM from_index_name] - [PROPERTIES ("key"="value", ...)],...] - 例子： - ADD ROLLUP r1(col1,col2) from r0, r2(col3,col4) from r0 - 1.3 注意： - 1) 如果没有指定 from_index_name，则默认从 base index 创建 - 2) rollup 表中的列必须是 from_index 中已有的列 - 3) 在 properties 中，可以指定存储格式。具体请参阅 CREATE TABLE - - 2. 删除 rollup index - 语法： - DROP ROLLUP rollup_name [PROPERTIES ("key"="value", ...)] - 例子： - DROP ROLLUP r1 - 2.1 批量删除 rollup index - 语法：DROP ROLLUP [rollup_name [PROPERTIES ("key"="value", ...)],...] - 例子：DROP ROLLUP r1,r2 - 2.2 注意： - 1) 不能删除 base index - 2) 执行 DROP ROLLUP 一段时间内，可以通过 RECOVER 语句恢复被删除的 rollup index。详见 RECOVER 语句 - - - schema change 支持如下几种修改方式： - 1. 向指定 index 的指定位置添加一列 - 语法： - ADD COLUMN column_name column_type [KEY | agg_type] [DEFAULT "default_value"] - [AFTER column_name|FIRST] - [TO rollup_index_name] - [PROPERTIES ("key"="value", ...)] - 注意： - 1) 聚合模型如果增加 value 列，需要指定 agg_type - 2) 非聚合模型（如 DUPLICATE KEY）如果增加key列，需要指定KEY关键字 - 3) 不能在 rollup index 中增加 base index 中已经存在的列 - 如有需要，可以重新创建一个 rollup index） - - 2. 向指定 index 添加多列 - 语法： - ADD COLUMN (column_name1 column_type [KEY | agg_type] DEFAULT "default_value", ...) - [TO rollup_index_name] - [PROPERTIES ("key"="value", ...)] - 注意： - 1) 聚合模型如果增加 value 列，需要指定agg_type - 2) 非聚合模型如果增加key列，需要指定KEY关键字 - 3) 不能在 rollup index 中增加 base index 中已经存在的列 - （如有需要，可以重新创建一个 rollup index） - - 3. 从指定 index 中删除一列 - 语法： - DROP COLUMN column_name - [FROM rollup_index_name] - 注意： - 1) 不能删除分区列 - 2) 如果是从 base index 中删除列，则如果 rollup index 中包含该列，也会被删除 - - 4. 修改指定 index 的列类型以及列位置 - 语法： - MODIFY COLUMN column_name column_type [KEY | agg_type] [NULL | NOT NULL] [DEFAULT "default_value"] - [AFTER column_name|FIRST] - [FROM rollup_index_name] - [PROPERTIES ("key"="value", ...)] - 注意： - 1) 聚合模型如果修改 value 列，需要指定 agg_type - 2) 非聚合类型如果修改key列，需要指定KEY关键字 - 3) 只能修改列的类型，列的其他属性维持原样（即其他属性需在语句中按照原属性显式的写出，参见 example 8） - 4) 分区列不能做任何修改 - 5) 目前支持以下类型的转换（精度损失由用户保证） - TINYINT/SMALLINT/INT/BIGINT 转换成 TINYINT/SMALLINT/INT/BIGINT/DOUBLE。 - TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL 转换成 VARCHAR - LARGEINT 转换成 DOUBLE - VARCHAR 支持修改最大长度 - VARCHAR 转换成 TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE - VARCHAR 转换成 DATE (目前支持"%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d, "%y/%m/%d"六种格式化格式) - DATETIME 转换成 DATE(仅保留年-月-日信息, 例如: `2019-12-09 21:47:05` <--> `2019-12-09`) - DATE 转换成 DATETIME(时分秒自动补零， 例如: `2019-12-09` <--> `2019-12-09 00:00:00`) - FLOAT 转换成 DOUBLE - INT 转换成 DATE (如果INT类型数据不合法则转换失败，原始数据不变) - 6) 不支持从NULL转为NOT NULL - - 5. 对指定 index 的列进行重新排序 - 语法： - ORDER BY (column_name1, column_name2, ...) - [FROM rollup_index_name] - [PROPERTIES ("key"="value", ...)] - 注意： - 1) index 中的所有列都要写出来 - 2) value 列在 key 列之后 - - 6. 修改table的属性，目前支持修改bloom filter列, colocate_with 属性和dynamic_partition属性，replication_num和default.replication_num属性 - 语法： - PROPERTIES ("key"="value") - 注意： - 也可以合并到上面的schema change操作中来修改，见下面例子 - - - rename 支持对以下名称进行修改： - 1. 修改表名 - 语法： - RENAME new_table_name; - - 2. 修改 rollup index 名称 - 语法： - RENAME ROLLUP old_rollup_name new_rollup_name; - - 3. 修改 partition 名称 - 语法： - RENAME PARTITION old_partition_name new_partition_name; - bitmap index 支持如下几种修改方式 - 1. 创建bitmap 索引 - 语法： - ADD INDEX index_name (column [, ...],) [USING BITMAP] [COMMENT 'balabala']; - 注意： - 1. 目前仅支持bitmap 索引 - 1. BITMAP 索引仅在单列上创建 - 2. 删除索引 - 语法： - DROP INDEX index_name； - -## example - - [table] - 1. 修改表的默认副本数量, 新建分区副本数量默认使用此值 - ATLER TABLE example_db.my_table - SET ("default.replication_num" = "2"); - - 2. 修改单分区表的实际副本数量(只限单分区表) - ALTER TABLE example_db.my_table - SET ("replication_num" = "3"); - - [partition] - 1. 增加分区, 现有分区 [MIN, 2013-01-01)，增加分区 [2013-01-01, 2014-01-01)，使用默认分桶方式 - ALTER TABLE example_db.my_table - ADD PARTITION p1 VALUES LESS THAN ("2014-01-01"); - - 2. 增加分区，使用新的分桶数 - ALTER TABLE example_db.my_table - ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") - DISTRIBUTED BY HASH(k1) BUCKETS 20; - - 3. 增加分区，使用新的副本数 - ALTER TABLE example_db.my_table - ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") - ("replication_num"="1"); - - 4. 修改分区副本数 - ALTER TABLE example_db.my_table - MODIFY PARTITION p1 SET("replication_num"="1"); - - 5. 删除分区 - ALTER TABLE example_db.my_table - DROP PARTITION p1; - - 6. 增加一个指定上下界的分区 - - ALTER TABLE example_db.my_table - ADD PARTITION p1 VALUES [("2014-01-01"), ("2014-02-01")); - - [rollup] - 1. 创建 index: example_rollup_index，基于 base index（k1,k2,k3,v1,v2）。列式存储。 - ALTER TABLE example_db.my_table - ADD ROLLUP example_rollup_index(k1, k3, v1, v2) - PROPERTIES("storage_type"="column"); - - 2. 创建 index: example_rollup_index2，基于 example_rollup_index（k1,k3,v1,v2） - ALTER TABLE example_db.my_table - ADD ROLLUP example_rollup_index2 (k1, v1) - FROM example_rollup_index; - - 3. 创建 index: example_rollup_index3, 基于 base index (k1,k2,k3,v1), 自定义 rollup 超时时间一小时。 - ALTER TABLE example_db.my_table - ADD ROLLUP example_rollup_index(k1, k3, v1) - PROPERTIES("storage_type"="column", "timeout" = "3600"); - - 4. 删除 index: example_rollup_index2 - ALTER TABLE example_db.my_table - DROP ROLLUP example_rollup_index2; - - - - [schema change] - 1. 向 example_rollup_index 的 col1 后添加一个key列 new_col(非聚合模型) - ALTER TABLE example_db.my_table - ADD COLUMN new_col INT KEY DEFAULT "0" AFTER col1 - TO example_rollup_index; - - 2. 向example_rollup_index的col1后添加一个value列new_col(非聚合模型) - ALTER TABLE example_db.my_table - ADD COLUMN new_col INT DEFAULT "0" AFTER col1 - TO example_rollup_index; - - 3. 向example_rollup_index的col1后添加一个key列new_col(聚合模型) - ALTER TABLE example_db.my_table - ADD COLUMN new_col INT DEFAULT "0" AFTER col1 - TO example_rollup_index; - - 4. 向example_rollup_index的col1后添加一个value列new_col SUM聚合类型(聚合模型) - ALTER TABLE example_db.my_table - ADD COLUMN new_col INT SUM DEFAULT "0" AFTER col1 - TO example_rollup_index; - - 5. 向 example_rollup_index 添加多列(聚合模型) - ALTER TABLE example_db.my_table - ADD COLUMN (col1 INT DEFAULT "1", col2 FLOAT SUM DEFAULT "2.3") - TO example_rollup_index; - - 6. 从 example_rollup_index 删除一列 - ALTER TABLE example_db.my_table - DROP COLUMN col2 - FROM example_rollup_index; - - 7. 修改 base index 的 col1 列的类型为 BIGINT，并移动到 col2 列后面 - ALTER TABLE example_db.my_table - MODIFY COLUMN col1 BIGINT DEFAULT "1" AFTER col2; - - 8. 修改 base index 的 val1 列最大长度。原 val1 为 (val1 VARCHAR(32) REPLACE DEFAULT "abc") - ALTER TABLE example_db.my_table - MODIFY COLUMN val1 VARCHAR(64) REPLACE DEFAULT "abc"; - - 9. 重新排序 example_rollup_index 中的列（设原列顺序为：k1,k2,k3,v1,v2） - ALTER TABLE example_db.my_table - ORDER BY (k3,k1,k2,v2,v1) - FROM example_rollup_index; - - 10. 同时执行两种操作 - ALTER TABLE example_db.my_table - ADD COLUMN v2 INT MAX DEFAULT "0" AFTER k2 TO example_rollup_index, - ORDER BY (k3,k1,k2,v2,v1) FROM example_rollup_index; - - 11. 修改表的 bloom filter 列 - ALTER TABLE example_db.my_table SET ("bloom_filter_columns"="k1,k2,k3"); - - 也可以合并到上面的 schema change 操作中（注意多子句的语法有少许区别） - ALTER TABLE example_db.my_table - DROP COLUMN col2 - PROPERTIES ("bloom_filter_columns"="k1,k2,k3"); - - 12. 修改表的Colocate 属性 - - ALTER TABLE example_db.my_table set ("colocate_with" = "t1"); - - 13. 将表的分桶方式由 Random Distribution 改为 Hash Distribution - - ALTER TABLE example_db.my_table set ("distribution_type" = "hash"); - - 14. 修改表的动态分区属性(支持未添加动态分区属性的表添加动态分区属性) - ALTER TABLE example_db.my_table set ("dynamic_partition_enable" = "false"); - - 如果需要在未添加动态分区属性的表中添加动态分区属性，则需要指定所有的动态分区属性 - ALTER TABLE example_db.my_table set ("dynamic_partition.enable" = "true", dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32"); - 15. 修改表的 in_memory 属性 - - ALTER TABLE example_db.my_table set ("in_memory" = "true"); - - - [rename] - 1. 将名为 table1 的表修改为 table2 - ALTER TABLE table1 RENAME table2; - - 2. 将表 example_table 中名为 rollup1 的 rollup index 修改为 rollup2 - ALTER TABLE example_table RENAME ROLLUP rollup1 rollup2; - - 3. 将表 example_table 中名为 p1 的 partition 修改为 p2 - ALTER TABLE example_table RENAME PARTITION p1 p2; - [index] - 1. 在table1 上为siteid 创建bitmap 索引 - ALTER TABLE table1 ADD INDEX index_name (siteid) [USING BITMAP] COMMENT 'balabala'; - 2. 删除table1 上的siteid列的bitmap 索引 - ALTER TABLE table1 DROP INDEX index_name; - -## keyword - - ALTER,TABLE,ROLLUP,COLUMN,PARTITION,RENAME diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER VIEW.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER VIEW.md deleted file mode 100644 index 2baff689990e1c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER VIEW.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# ALTER VIEW -## description - 该语句用于修改一个view的定义 - 语法： - ALTER VIEW - [db_name.]view_name - (column1[ COMMENT "col comment"][, column2, ...]) - AS query_stmt - - 说明： - 1. 视图都是逻辑上的，其中的数据不会存储在物理介质上，在查询时视图将作为语句中的子查询，因此，修改视图的定义等价于修改query_stmt。 - 2. query_stmt 为任意支持的 SQL - -## example - 1、修改example_db上的视图example_view - - ALTER VIEW example_db.example_view - ( - c1 COMMENT "column 1", - c2 COMMENT "column 2", - c3 COMMENT "column 3" - ) - AS SELECT k1, k2, SUM(v1) FROM example_table - GROUP BY k1, k2 - - \ No newline at end of file diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/BACKUP.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/BACKUP.md deleted file mode 100644 index c143069f2e6618..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/BACKUP.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# BACKUP -## description - 该语句用于备份指定数据库下的数据。该命令为异步操作。提交成功后，需通过 SHOW BACKUP 命令查看进度。仅支持备份 OLAP 类型的表。 - 语法： - BACKUP SNAPSHOT [db_name].{snapshot_name} - TO `repository_name` - ON ( - `table_name` [PARTITION (`p1`, ...)], - ... - ) - PROPERTIES ("key"="value", ...); - - 说明： - 1. 同一数据库下只能有一个正在执行的 BACKUP 或 RESTORE 任务。 - 2. ON 子句中标识需要备份的表和分区。如果不指定分区，则默认备份该表的所有分区。 - 3. PROPERTIES 目前支持以下属性： - "type" = "full"：表示这是一次全量更新（默认）。 - "timeout" = "3600"：任务超时时间，默认为一天。单位秒。 - -## example - - 1. 全量备份 example_db 下的表 example_tbl 到仓库 example_repo 中： - BACKUP SNAPSHOT example_db.snapshot_label1 - TO example_repo - ON (example_tbl) - PROPERTIES ("type" = "full"); - - 2. 全量备份 example_db 下，表 example_tbl 的 p1, p2 分区，以及表 example_tbl2 到仓库 example_repo 中： - BACKUP SNAPSHOT example_db.snapshot_label2 - TO example_repo - ON - ( - example_tbl PARTITION (p1,p2), - example_tbl2 - ); - -## keyword - BACKUP - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md deleted file mode 100644 index 98be482a83a36a..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md +++ /dev/null @@ -1,62 +0,0 @@ - - -# CANCEL ALTER -## description - 该语句用于撤销一个 ALTER 操作。 - 1. 撤销 ALTER TABLE COLUMN 操作 - 语法： - CANCEL ALTER TABLE COLUMN - FROM db_name.table_name - - 2. 撤销 ALTER TABLE ROLLUP 操作 - 语法： - CANCEL ALTER TABLE ROLLUP - FROM db_name.table_name - - 3. 根据job id批量撤销rollup操作 - 语法: - CANCEL ALTER TABLE ROLLUP - FROM db_name.table_name (jobid,...) - 注意： - 该命令为异步操作，具体是否执行成功需要使用`show alter table rollup`查看任务状态确认 - 4. 撤销 ALTER CLUSTER 操作 - 语法： - （待实现...） - - -## example - [CANCEL ALTER TABLE COLUMN] - 1. 撤销针对 my_table 的 ALTER COLUMN 操作。 - CANCEL ALTER TABLE COLUMN - FROM example_db.my_table; - - [CANCEL ALTER TABLE ROLLUP] - 1. 撤销 my_table 下的 ADD ROLLUP 操作。 - CANCEL ALTER TABLE ROLLUP - FROM example_db.my_table; - - [CANCEL ALTER TABLE ROLLUP] - 1. 根据job id撤销 my_table 下的 ADD ROLLUP 操作。 - CANCEL ALTER TABLE ROLLUP - FROM example_db.my_table (12801,12802); - -## keyword - CANCEL,ALTER,TABLE,COLUMN,ROLLUP - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md deleted file mode 100644 index 3035f2956f3e0c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# CANCEL BACKUP -## description - 该语句用于取消一个正在进行的 BACKUP 任务。 - 语法： - CANCEL BACKUP FROM db_name; - -## example - 1. 取消 example_db 下的 BACKUP 任务。 - CANCEL BACKUP FROM example_db; - -## keyword - CANCEL, BACKUP - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md deleted file mode 100644 index d7fbc950df4622..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# CANCEL RESTORE -## description - 该语句用于取消一个正在进行的 RESTORE 任务。 - 语法： - CANCEL RESTORE FROM db_name; - - 注意： - 当取消处于 COMMIT 或之后阶段的恢复左右时，可能导致被恢复的表无法访问。此时只能通过再次执行恢复作业进行数据恢复。 - -## example - 1. 取消 example_db 下的 RESTORE 任务。 - CANCEL RESTORE FROM example_db; - -## keyword - CANCEL, RESTORE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md deleted file mode 100644 index 087d4843bf6050..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# CREATE DATABASE -## description - 该语句用于新建数据库（database） - 语法： - CREATE DATABASE [IF NOT EXISTS] db_name; - -## example - 1. 新建数据库 db_test - CREATE DATABASE db_test; - -## keyword - CREATE,DATABASE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE INDEX.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE INDEX.md deleted file mode 100644 index 9767f015c927f6..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE INDEX.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# CREATE INDEX - -## description - - 该语句用于创建索引 - 语法： - CREATE INDEX index_name ON table_name (column [, ...],) [USING BITMAP] [COMMENT'balabala']; - 注意： - 1. 目前只支持bitmap 索引 - 2. BITMAP 索引仅在单列上创建 - -## example - - 1. 在table1 上为siteid 创建bitmap 索引 - CREATE INDEX index_name ON table1 (siteid) USING BITMAP COMMENT 'balabala'; - -## keyword - - CREATE,INDEX diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md deleted file mode 100644 index bd98402d1a0864..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md +++ /dev/null @@ -1,230 +0,0 @@ - - -# CREATE MATERIALIZED VIEW - -## description - - 该语句用于创建物化视图。 - -说明: - 异步语法，调用成功后仅表示创建物化视图的任务提交成功，用户需要先通过 ``` show alter table rollup ``` 来查看物化视图的创建进度。 - 在显示 FINISHED 后既可通过 ``` desc [table_name] all ``` 命令来查看物化视图的 schema 了。 - -语法: - - ``` - - CREATE MATERIALIZED VIEW [MV name] as [query] - [PROPERTIES ("key" = "value")] - - ``` - -1. MV name - - 物化视图的名称，必填项。 - - 相同表的物化视图名称不可重复。 - -2. query - - 用于构建物化视图的查询语句，查询语句的结果既物化视图的数据。目前支持的 query 格式为: - - ``` - - SELECT select_expr[, select_expr ...] - FROM [Base view name] - GROUP BY column_name[, column_name ...] - ORDER BY column_name[, column_name ...] - - 语法和查询语句语法一致。 - - ``` - - select_expr: 物化视图的 schema 中所有的列。 - + 仅支持不带表达式计算的单列，聚合列。 - + 其中聚合函数目前仅支持 SUM, MIN, MAX 三种，且聚合函数的参数只能是不带表达式计算的单列。 - + 至少包含一个单列。 - + 所有涉及到的列，均只能出现一次。 - - base view name: 物化视图的原始表名，必填项。 - + 必须是单表，且非子查询 - - group by: 物化视图的分组列，选填项。 - + 不填则数据不进行分组。 - - order by: 物化视图的排序列，选填项。 - + 排序列的声明顺序必须和 select_expr 中列声明顺序一致。 - + 如果不声明 order by，则根据规则自动补充排序列。 - 如果物化视图是聚合类型，则所有的分组列自动补充为排序列。 - 如果物化视图是非聚合类型，则前 36 个字节自动补充为排序列。如果自动补充的排序个数小于3个，则前三个作为排序列。 - + 如果 query 中包含分组列的话，则排序列必须和分组列一致。 - -3. properties - - 声明物化视图的一些配置，选填项。 - - ``` - - PROPERTIES ("key" = "value", "key" = "value" ...) - - ``` - - 以下几个配置，均可声明在此处： - - short_key: 排序列的个数。 - timeout: 物化视图构建的超时时间。 - -## example - -Base 表结构为 - -``` -mysql> desc duplicate_table; -+-------+--------+------+------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+-------+--------+------+------+---------+-------+ -| k1 | INT | Yes | true | N/A | | -| k2 | INT | Yes | true | N/A | | -| k3 | BIGINT | Yes | true | N/A | | -| k4 | BIGINT | Yes | true | N/A | | -+-------+--------+------+------+---------+-------+ -``` - -1. 创建一个仅包含原始表 （k1, k2）列的物化视图 - - ``` - create materialized view k1_k2 as - select k1, k2 from duplicate_table; - ``` - - 物化视图的 schema 如下图，物化视图仅包含两列 k1, k2 且不带任何聚合 - - ``` - +-----------------+-------+--------+------+------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +-----------------+-------+--------+------+------+---------+-------+ - | k1_k2 | k1 | INT | Yes | true | N/A | | - | | k2 | INT | Yes | true | N/A | | - +-----------------+-------+--------+------+------+---------+-------+ - ``` - -2. 创建一个以 k2 为排序列的物化视图 - - ``` - create materialized view k2_order as - select k2, k1 from duplicate_table order by k2; - ``` - - 物化视图的 schema 如下图，物化视图仅包含两列 k2, k1，其中 k2 列为排序列，不带任何聚合。 - - ``` - +-----------------+-------+--------+------+-------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +-----------------+-------+--------+------+-------+---------+-------+ - | k2_order | k2 | INT | Yes | true | N/A | | - | | k1 | INT | Yes | false | N/A | NONE | - +-----------------+-------+--------+------+-------+---------+-------+ - ``` - -3. 创建一个以 k1, k2 分组，k3 列为 SUM 聚合的物化视图 - - ``` - create materialized view k1_k2_sumk3 as - select k1, k2, sum(k3) from duplicate_table group by k1, k2; - ``` - - 物化视图的 schema 如下图，物化视图包含两列 k1, k2，sum(k3) 其中 k1, k2 为分组列，sum(k3) 为根据 k1, k2 分组后的 k3 列的求和值。 - - 由于物化视图没有声明排序列，且物化视图带聚合数据，系统默认补充分组列 k1, k2 为排序列。 - - ``` - +-----------------+-------+--------+------+-------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +-----------------+-------+--------+------+-------+---------+-------+ - | k1_k2_sumk3 | k1 | INT | Yes | true | N/A | | - | | k2 | INT | Yes | true | N/A | | - | | k3 | BIGINT | Yes | false | N/A | SUM | - +-----------------+-------+--------+------+-------+---------+-------+ - ``` - -4. 创建一个去除重复行的物化视图 - - ``` - create materialized view deduplicate as - select k1, k2, k3, k4 from duplicate_table group by k1, k2, k3, k4; - ``` - - 物化视图 schema 如下图，物化视图包含 k1, k2, k3, k4列，且不存在重复行。 - - ``` - +-----------------+-------+--------+------+-------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +-----------------+-------+--------+------+-------+---------+-------+ - | deduplicate | k1 | INT | Yes | true | N/A | | - | | k2 | INT | Yes | true | N/A | | - | | k3 | BIGINT | Yes | true | N/A | | - | | k4 | BIGINT | Yes | true | N/A | | - +-----------------+-------+--------+------+-------+---------+-------+ - - ``` - -5. 创建一个不声明排序列的非聚合型物化视图 - - all_type_table 的 schema 如下： - - ``` - +-------+--------------+------+-------+---------+-------+ - | Field | Type | Null | Key | Default | Extra | - +-------+--------------+------+-------+---------+-------+ - | k1 | TINYINT | Yes | true | N/A | | - | k2 | SMALLINT | Yes | true | N/A | | - | k3 | INT | Yes | true | N/A | | - | k4 | BIGINT | Yes | true | N/A | | - | k5 | DECIMAL(9,0) | Yes | true | N/A | | - | k6 | DOUBLE | Yes | false | N/A | NONE | - | k7 | VARCHAR(20) | Yes | false | N/A | NONE | - +-------+--------------+------+-------+---------+-------+ - ``` - - 物化视图包含 k3, k4, k5, k6, k7 列，且不声明排序列，则创建语句如下： - - ``` - create materialized view mv_1 as - select k3, k4, k5, k6, k7 from all_type_table; - ``` - - 系统默认补充的排序列为 k3, k4, k5 三列。这三列类型的字节数之和为 4(INT) + 8(BIGINT) + 16(DECIMAL) = 28 < 36。所以补充的是这三列作为排序列。 - 物化视图的 schema 如下，可以看到其中 k3, k4, k5 列的 key 字段为 true，也就是排序列。k6, k7 列的 key 字段为 false，也就是非排序列。 - - ``` - +----------------+-------+--------------+------+-------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +----------------+-------+--------------+------+-------+---------+-------+ - | mv_1 | k3 | INT | Yes | true | N/A | | - | | k4 | BIGINT | Yes | true | N/A | | - | | k5 | DECIMAL(9,0) | Yes | true | N/A | | - | | k6 | DOUBLE | Yes | false | N/A | NONE | - | | k7 | VARCHAR(20) | Yes | false | N/A | NONE | - +----------------+-------+--------------+------+-------+---------+-------+ - ``` - - -## keyword - CREATE, MATERIALIZED, VIEW diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md deleted file mode 100644 index 4cacc5d31cf1ca..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md +++ /dev/null @@ -1,69 +0,0 @@ - - -# CREATE REPOSITORY -## description - 该语句用于创建仓库。仓库用于属于备份或恢复。仅 root 或 superuser 用户可以创建仓库。 - 语法： - CREATE [READ ONLY] REPOSITORY `repo_name` - WITH BROKER `broker_name` - ON LOCATION `repo_location` - PROPERTIES ("key"="value", ...); - - 说明： - 1. 仓库的创建，依赖于已存在的 broker - 2. 如果是只读仓库，则只能在仓库上进行恢复。如果不是，则可以进行备份和恢复操作。 - 3. 根据 broker 的不同类型，PROPERTIES 有所不同，具体见示例。 - -## example - 1. 创建名为 bos_repo 的仓库，依赖 BOS broker "bos_broker"，数据根目录为：bos://palo_backup - CREATE REPOSITORY `bos_repo` - WITH BROKER `bos_broker` - ON LOCATION "bos://palo_backup" - PROPERTIES - ( - "bos_endpoint" = "http://gz.bcebos.com", - "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", - "bos_secret_accesskey"="70999999999999de274d59eaa980a" - ); - - 2. 创建和示例 1 相同的仓库，但属性为只读： - CREATE READ ONLY REPOSITORY `bos_repo` - WITH BROKER `bos_broker` - ON LOCATION "bos://palo_backup" - PROPERTIES - ( - "bos_endpoint" = "http://gz.bcebos.com", - "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", - "bos_secret_accesskey"="70999999999999de274d59eaa980a" - ); - - 3. 创建名为 hdfs_repo 的仓库，依赖 Baidu hdfs broker "hdfs_broker"，数据根目录为：hdfs://hadoop-name-node:54310/path/to/repo/ - CREATE REPOSITORY `hdfs_repo` - WITH BROKER `hdfs_broker` - ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/" - PROPERTIES - ( - "username" = "user", - "password" = "password" - ); - -## keyword - CREATE REPOSITORY - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md deleted file mode 100644 index 80afc1696da130..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ /dev/null @@ -1,605 +0,0 @@ - - -# CREATE TABLE - -## description - -该语句用于创建 table。 -语法： - -``` - CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [database.]table_name - (column_definition1[, column_definition2, ...] - [, index_definition1[, ndex_definition12,]]) - [ENGINE = [olap|mysql|broker]] - [key_desc] - [COMMENT "table comment"]; - [partition_desc] - [distribution_desc] - [rollup_index] - [PROPERTIES ("key"="value", ...)] - [BROKER PROPERTIES ("key"="value", ...)] -``` - -1. column_definition - 语法： - `col_name col_type [agg_type] [NULL | NOT NULL] [DEFAULT "default_value"]` - - 说明： - col_name：列名称 - col_type：列类型 - - ``` - TINYINT（1字节） - 范围：-2^7 + 1 ~ 2^7 - 1 - SMALLINT（2字节） - 范围：-2^15 + 1 ~ 2^15 - 1 - INT（4字节） - 范围：-2^31 + 1 ~ 2^31 - 1 - BIGINT（8字节） - 范围：-2^63 + 1 ~ 2^63 - 1 - LARGEINT（16字节） - 范围：-2^127 + 1 ~ 2^127 - 1 - FLOAT（4字节） - 支持科学计数法 - DOUBLE（12字节） - 支持科学计数法 - DECIMAL[(precision, scale)] (16字节) - 保证精度的小数类型。默认是 DECIMAL(10, 0) - precision: 1 ~ 27 - scale: 0 ~ 9 - 其中整数部分为 1 ~ 18 - 不支持科学计数法 - DATE（3字节） - 范围：0000-01-01 ~ 9999-12-31 - DATETIME（8字节） - 范围：0000-01-01 00:00:00 ~ 9999-12-31 23:59:59 - CHAR[(length)] - 定长字符串。长度范围：1 ~ 255。默认为1 - VARCHAR[(length)] - 变长字符串。长度范围：1 ~ 65533 - HLL (1~16385个字节) - hll列类型，不需要指定长度和默认值、长度根据数据的聚合 - 程度系统内控制，并且HLL列只能通过配套的hll_union_agg、Hll_cardinality、hll_hash进行查询或使用 - BITMAP - bitmap列类型，不需要指定长度和默认值。表示整型的集合，元素最大支持到2^64 - 1 - ``` - - agg_type：聚合类型，如果不指定，则该列为 key 列。否则，该列为 value 列 - * SUM、MAX、MIN、REPLACE - * HLL_UNION(仅用于HLL列，为HLL独有的聚合方式)、 - * BITMAP_UNION(仅用于 BITMAP 列，为 BITMAP 独有的聚合方式)、 - * REPLACE_IF_NOT_NULL：这个聚合类型的含义是当且仅当新导入数据是非NULL值时会发生替换行为，如果新导入的数据是NULL，那么Doris仍然会保留原值。注意：如果用在建表时REPLACE_IF_NOT_NULL列指定了NOT NULL，那么Doris仍然会将其转化NULL，不会向用户报错。用户可以借助这个类型完成部分列导入的功能。 - * 该类型只对聚合模型(key_desc的type为AGGREGATE KEY)有用，其它模型不需要指这个。 - - 是否允许为NULL: 默认不允许为 NULL。NULL 值在导入数据中用 \N 来表示 - - 注意： - BITMAP_UNION聚合类型列在导入时的原始数据类型必须是TINYINT,SMALLINT,INT,BIGINT。 - -2. index_definition - 语法： - `INDEX index_name (col_name[, col_name, ...]) [USING BITMAP] COMMENT 'xxxxxx'` - 说明： - index_name：索引名称 - col_name：列名 - 注意： - 当前仅支持BITMAP索引， BITMAP索引仅支持应用于单列 - -3. ENGINE 类型 - 默认为 olap。可选 mysql, broker - 1) 如果是 mysql，则需要在 properties 提供以下信息： - -``` - PROPERTIES ( - "host" = "mysql_server_host", - "port" = "mysql_server_port", - "user" = "your_user_name", - "password" = "your_password", - "database" = "database_name", - "table" = "table_name" - ) -``` - - 注意： - "table" 条目中的 "table_name" 是 mysql 中的真实表名。 - 而 CREATE TABLE 语句中的 table_name 是该 mysql 表在 Palo 中的名字，可以不同。 - - 在 Palo 创建 mysql 表的目的是可以通过 Palo 访问 mysql 数据库。 - 而 Palo 本身并不维护、存储任何 mysql 数据。 - 1) 如果是 broker，表示表的访问需要通过指定的broker, 需要在 properties 提供以下信息： - ``` - PROPERTIES ( - "broker_name" = "broker_name", - "path" = "file_path1[,file_path2]", - "column_separator" = "value_separator" - "line_delimiter" = "value_delimiter" - ) - ``` - 另外还需要提供Broker需要的Property信息，通过BROKER PROPERTIES来传递，例如HDFS需要传入 - ``` - BROKER PROPERTIES( - "username" = "name", - "password" = "password" - ) - ``` - 这个根据不同的Broker类型，需要传入的内容也不相同 - 注意： - "path" 中如果有多个文件，用逗号[,]分割。如果文件名中包含逗号，那么使用 %2c 来替代。如果文件名中包含 %，使用 %25 代替 - 现在文件内容格式支持CSV，支持GZ，BZ2，LZ4，LZO(LZOP) 压缩格式。 - -1. key_desc - 语法： - `key_type(k1[,k2 ...])` - 说明： - 数据按照指定的key列进行排序，且根据不同的key_type具有不同特性。 - key_type支持一下类型： - AGGREGATE KEY:key列相同的记录，value列按照指定的聚合类型进行聚合， - 适合报表、多维分析等业务场景。 - UNIQUE KEY:key列相同的记录，value列按导入顺序进行覆盖， - 适合按key列进行增删改查的点查询业务。 - DUPLICATE KEY:key列相同的记录，同时存在于Palo中， - 适合存储明细数据或者数据无聚合特性的业务场景。 - 默认为DUPLICATE KEY，key列为列定义中前36个字节, 如果前36个字节的列数小于3，将使用前三列。 - 注意： - 除AGGREGATE KEY外，其他key_type在建表时，value列不需要指定聚合类型。 - -2. partition_desc - partition描述有两种使用方式 - 1) LESS THAN - 语法： - - ``` - PARTITION BY RANGE (k1, k2, ...) - ( - PARTITION partition_name1 VALUES LESS THAN MAXVALUE|("value1", "value2", ...), - PARTITION partition_name2 VALUES LESS THAN MAXVALUE|("value1", "value2", ...) - ... - ) - ``` - - 说明： - 使用指定的 key 列和指定的数值范围进行分区。 - 1) 分区名称仅支持字母开头，字母、数字和下划线组成 - 2) 目前仅支持以下类型的列作为 Range 分区列，且只能指定一个分区列 - TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME - 3) 分区为左闭右开区间，首个分区的左边界为做最小值 - 4) NULL 值只会存放在包含最小值的分区中。当包含最小值的分区被删除后，NULL 值将无法导入。 - 5) 可以指定一列或多列作为分区列。如果分区值缺省，则会默认填充最小值。 - - 注意： - 1) 分区一般用于时间维度的数据管理 - 2) 有数据回溯需求的，可以考虑首个分区为空分区，以便后续增加分区 - - 2）Fixed Range - 语法： - ``` - PARTITION BY RANGE (k1, k2, k3, ...) - ( - PARTITION partition_name1 VALUES [("k1-lower1", "k2-lower1", "k3-lower1",...), ("k1-upper1", "k2-upper1", "k3-upper1", ...)), - PARTITION partition_name2 VALUES [("k1-lower1-2", "k2-lower1-2", ...), ("k1-upper1-2", MAXVALUE, )) - "k3-upper1-2", ... - ) - ``` - 说明： - 1）Fixed Range比LESS THAN相对灵活些，左右区间完全由用户自己确定 - 2）其他与LESS THAN保持同步 - -3. distribution_desc - 1) Hash 分桶 - 语法： - `DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]` - 说明： - 使用指定的 key 列进行哈希分桶。默认分区数为10 - - 建议:建议使用Hash分桶方式 - -4. PROPERTIES - 1) 如果 ENGINE 类型为 olap - 可以在 properties 设置该表数据的初始存储介质、存储到期时间和副本数。 - - ``` - PROPERTIES ( - "storage_medium" = "[SSD|HDD]", - ["storage_cooldown_time" = "yyyy-MM-dd HH:mm:ss"], - ["replication_num" = "3"] - ) - ``` - - storage_medium： 用于指定该分区的初始存储介质，可选择 SSD 或 HDD。默认初始存储介质可通过fe的配置文件 `fe.conf` 中指定 `default_storage_medium=xxx`，如果没有指定，则默认为 HDD。 - storage_cooldown_time： 当设置存储介质为 SSD 时，指定该分区在 SSD 上的存储到期时间。 - 默认存放 30 天。 - 格式为："yyyy-MM-dd HH:mm:ss" - replication_num: 指定分区的副本数。默认为 3 - - 当表为单分区表时，这些属性为表的属性。 - 当表为两级分区时，这些属性为附属于每一个分区。 - 如果希望不同分区有不同属性。可以通过 ADD PARTITION 或 MODIFY PARTITION 进行操作 - - 2 如果 Engine 类型为 olap, 可以指定某列使用 bloom filter 索引 - bloom filter 索引仅适用于查询条件为 in 和 equal 的情况，该列的值越分散效果越好 - 目前只支持以下情况的列:除了 TINYINT FLOAT DOUBLE 类型以外的 key 列及聚合方法为 REPLACE 的 value 列 - -``` - PROPERTIES ( - "bloom_filter_columns"="k1,k2,k3" - ) -``` - - 3) 如果希望使用 Colocate Join 特性，需要在 properties 中指定 - -``` - PROPERTIES ( - "colocate_with"="table1" - ) -``` - - 4) 如果希望使用动态分区特性，需要在properties 中指定 - -``` - PROPERTIES ( - "dynamic_partition.enable" = "true|false", - "dynamic_partition.time_unit" = "DAY|WEEK|MONTH", - "dynamic_partition.start" = "${integer_value}", - "dynamic_partitoin.end" = "${integer_value}", - "dynamic_partition.prefix" = "${string_value}", - "dynamic_partition.buckets" = "${integer_value} -``` - dynamic_partition.enable: 用于指定表级别的动态分区功能是否开启。默认为 true。 - dynamic_partition.time_unit: 用于指定动态添加分区的时间单位，可选择为DAY（天），WEEK(周)，MONTH（月） - dynamic_partition.start: 用于指定向前删除多少个分区。值必须小于0。默认为 Integer.MIN_VALUE。 - dynamic_partition.end: 用于指定提前创建的分区数量。值必须大于0。 - dynamic_partition.prefix: 用于指定创建的分区名前缀，例如分区名前缀为p，则自动创建分区名为p20200108 - dynamic_partition.buckets: 用于指定自动创建的分区分桶数量 - - 5) 建表时可以批量创建多个 Rollup - 语法： - ``` - ROLLUP (rollup_name (column_name1, column_name2, ...) - [FROM from_index_name] - [PROPERTIES ("key"="value", ...)],...) - ``` - - 6) 如果希望使用 内存表 特性，需要在 properties 中指定 - -``` - PROPERTIES ( - "in_memory"="true" - ) -``` - 当 in_memory 属性为 true 时，Doris会尽可能将该表的数据和索引Cache到BE 内存中 -## example - -1. 创建一个 olap 表，使用 HASH 分桶，使用列存，相同key的记录进行聚合 - - ``` - CREATE TABLE example_db.table_hash - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 CHAR(10) REPLACE, - v2 INT SUM - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - COMMENT "my first doris table" - DISTRIBUTED BY HASH(k1) BUCKETS 32 - PROPERTIES ("storage_type"="column"); - ``` - -2. 创建一个 olap 表，使用 Hash 分桶，使用列存，相同key的记录进行覆盖， - 设置初始存储介质和冷却时间 - - ``` - CREATE TABLE example_db.table_hash - ( - k1 BIGINT, - k2 LARGEINT, - v1 VARCHAR(2048) REPLACE, - v2 SMALLINT SUM DEFAULT "10" - ) - ENGINE=olap - UNIQUE KEY(k1, k2) - DISTRIBUTED BY HASH (k1, k2) BUCKETS 32 - PROPERTIES( - "storage_type"="column"， - "storage_medium" = "SSD", - "storage_cooldown_time" = "2015-06-04 00:00:00" - ); - ``` - -3. 创建一个 olap 表，使用 Range 分区，使用Hash分桶，默认使用列存， - 相同key的记录同时存在，设置初始存储介质和冷却时间 - - 1）LESS THAN - - ``` - CREATE TABLE example_db.table_range - ( - k1 DATE, - k2 INT, - k3 SMALLINT, - v1 VARCHAR(2048), - v2 DATETIME DEFAULT "2014-02-04 15:36:00" - ) - ENGINE=olap - DUPLICATE KEY(k1, k2, k3) - PARTITION BY RANGE (k1) - ( - PARTITION p1 VALUES LESS THAN ("2014-01-01"), - PARTITION p2 VALUES LESS THAN ("2014-06-01"), - PARTITION p3 VALUES LESS THAN ("2014-12-01") - ) - DISTRIBUTED BY HASH(k2) BUCKETS 32 - PROPERTIES( - "storage_medium" = "SSD", "storage_cooldown_time" = "2015-06-04 00:00:00" - ); - ``` - - 说明： - 这个语句会将数据划分成如下3个分区： - - ``` - ( { MIN }, {"2014-01-01"} ) - [ {"2014-01-01"}, {"2014-06-01"} ) - [ {"2014-06-01"}, {"2014-12-01"} ) - ``` - - 不在这些分区范围内的数据将视为非法数据被过滤 - - 2) Fixed Range - - ``` - CREATE TABLE table_range - ( - k1 DATE, - k2 INT, - k3 SMALLINT, - v1 VARCHAR(2048), - v2 DATETIME DEFAULT "2014-02-04 15:36:00" - ) - ENGINE=olap - DUPLICATE KEY(k1, k2, k3) - PARTITION BY RANGE (k1, k2, k3) - ( - PARTITION p1 VALUES [("2014-01-01", "10", "200"), ("2014-01-01", "20", "300")), - PARTITION p2 VALUES [("2014-06-01", "100", "200"), ("2014-07-01", "100", "300")) - ) - DISTRIBUTED BY HASH(k2) BUCKETS 32 - PROPERTIES( - "storage_medium" = "SSD" - ); - ``` - -4. 创建一个 mysql 表 - -``` - CREATE TABLE example_db.table_mysql - ( - k1 DATE, - k2 INT, - k3 SMALLINT, - k4 VARCHAR(2048), - k5 DATETIME - ) - ENGINE=mysql - PROPERTIES - ( - "host" = "127.0.0.1", - "port" = "8239", - "user" = "mysql_user", - "password" = "mysql_passwd", - "database" = "mysql_db_test", - "table" = "mysql_table_test" - ) -``` - -5. 创建一个数据文件存储在HDFS上的 broker 外部表, 数据使用 "|" 分割，"\n" 换行 - -``` - CREATE EXTERNAL TABLE example_db.table_broker ( - k1 DATE, - k2 INT, - k3 SMALLINT, - k4 VARCHAR(2048), - k5 DATETIME - ) - ENGINE=broker - PROPERTIES ( - "broker_name" = "hdfs", - "path" = "hdfs://hdfs_host:hdfs_port/data1,hdfs://hdfs_host:hdfs_port/data2,hdfs://hdfs_host:hdfs_port/data3%2c4", - "column_separator" = "|", - "line_delimiter" = "\n" - ) - BROKER PROPERTIES ( - "username" = "hdfs_user", - "password" = "hdfs_password" - ) -``` - -6. 创建一张含有HLL列的表 - -``` - CREATE TABLE example_db.example_table - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 HLL HLL_UNION, - v2 HLL HLL_UNION - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - DISTRIBUTED BY HASH(k1) BUCKETS 32 - PROPERTIES ("storage_type"="column"); -``` - -7. 创建一张含有BITMAP_UNION聚合类型的表（v1和v2列的原始数据类型必须是TINYINT,SMALLINT,INT） - -``` - CREATE TABLE example_db.example_table - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 BITMAP BITMAP_UNION, - v2 BITMAP BITMAP_UNION - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - DISTRIBUTED BY HASH(k1) BUCKETS 32 - PROPERTIES ("storage_type"="column"); -``` - -8. 创建两张支持Colocat Join的表t1 和t2 - -``` - CREATE TABLE `t1` ( - `id` int(11) COMMENT "", - `value` varchar(8) COMMENT "" - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "colocate_with" = "t1" - ); - - CREATE TABLE `t2` ( - `id` int(11) COMMENT "", - `value` varchar(8) COMMENT "" - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "colocate_with" = "t1" - ); -``` - -9. 创建一个数据文件存储在BOS上的 broker 外部表 - -``` - CREATE EXTERNAL TABLE example_db.table_broker ( - k1 DATE - ) - ENGINE=broker - PROPERTIES ( - "broker_name" = "bos", - "path" = "bos://my_bucket/input/file", - ) - BROKER PROPERTIES ( - "bos_endpoint" = "http://bj.bcebos.com", - "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "bos_secret_accesskey"="yyyyyyyyyyyyyyyyyyyy" - ) -``` - -10. 创建一个带有bitmap 索引的表 - -``` - CREATE TABLE example_db.table_hash - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 CHAR(10) REPLACE, - v2 INT SUM, - INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx' - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - COMMENT "my first doris table" - DISTRIBUTED BY HASH(k1) BUCKETS 32 - PROPERTIES ("storage_type"="column"); -``` - -11. 创建一个动态分区表(需要在FE配置中开启动态分区功能)，该表每天提前创建3天的分区，并删除3天前的分区。例如今天为`2020-01-08`，则会创建分区名为`p20200108`, `p20200109`, `p20200110`, `p20200111`的分区. 分区范围分别为: - -``` -[types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) -[types: [DATE]; keys: [2020-01-09]; ‥types: [DATE]; keys: [2020-01-10]; ) -[types: [DATE]; keys: [2020-01-10]; ‥types: [DATE]; keys: [2020-01-11]; ) -[types: [DATE]; keys: [2020-01-11]; ‥types: [DATE]; keys: [2020-01-12]; ) -``` - -``` - CREATE TABLE example_db.dynamic_partition - ( - k1 DATE, - k2 INT, - k3 SMALLINT, - v1 VARCHAR(2048), - v2 DATETIME DEFAULT "2014-02-04 15:36:00" - ) - ENGINE=olap - DUPLICATE KEY(k1, k2, k3) - PARTITION BY RANGE (k1) - ( - PARTITION p1 VALUES LESS THAN ("2014-01-01"), - PARTITION p2 VALUES LESS THAN ("2014-06-01"), - PARTITION p3 VALUES LESS THAN ("2014-12-01") - ) - DISTRIBUTED BY HASH(k2) BUCKETS 32 - PROPERTIES( - "storage_medium" = "SSD", - "dynamic_partition.time_unit" = "DAY", - "dynamic_partition.start" = "-3", - "dynamic_partition.end" = "3", - "dynamic_partition.prefix" = "p", - "dynamic_partition.buckets" = "32" - ); -``` - -12. Create a table with rollup index -``` - CREATE TABLE example_db.rolup_index_table - ( - event_day DATE, - siteid INT DEFAULT '10', - citycode SMALLINT, - username VARCHAR(32) DEFAULT '', - pv BIGINT SUM DEFAULT '0' - ) - AGGREGATE KEY(event_day, siteid, citycode, username) - DISTRIBUTED BY HASH(siteid) BUCKETS 10 - rollup ( - r1(event_day,siteid), - r2(event_day,citycode), - r3(event_day) - ) - PROPERTIES("replication_num" = "3"); - -13. 创建一个内存表 - -``` - CREATE TABLE example_db.table_hash - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 CHAR(10) REPLACE, - v2 INT SUM, - INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx' - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - COMMENT "my first doris table" - DISTRIBUTED BY HASH(k1) BUCKETS 32 - PROPERTIES ("in_memory"="true"); -``` - -## keyword - - CREATE,TABLE diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE VIEW.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE VIEW.md deleted file mode 100644 index fb4bac8aad3ad8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE VIEW.md +++ /dev/null @@ -1,57 +0,0 @@ - - -# CREATE VIEW -## description - 该语句用于创建一个逻辑视图 - 语法： - CREATE VIEW [IF NOT EXISTS] - [db_name.]view_name - (column1[ COMMENT "col comment"][, column2, ...]) - AS query_stmt - - 说明： - 1. 视图为逻辑视图，没有物理存储。所有在视图上的查询相当于在视图对应的子查询上进行。 - 2. query_stmt 为任意支持的 SQL - -## example - 1. 在 example_db 上创建视图 example_view - - CREATE VIEW example_db.example_view (k1, k2, k3, v1) - AS - SELECT c1 as k1, k2, k3, SUM(v1) FROM example_table - WHERE k1 = 20160112 GROUP BY k1,k2,k3; - - 2. 创建一个包含 comment 的 view - - CREATE VIEW example_db.example_view - ( - k1 COMMENT "first key", - k2 COMMENT "second key", - k3 COMMENT "third key", - v1 COMMENT "first value" - ) - COMMENT "my first view" - AS - SELECT c1 as k1, k2, k3, SUM(v1) FROM example_table - WHERE k1 = 20160112 GROUP BY k1,k2,k3; - -## keyword - CREATE,VIEW - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP DATABASE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP DATABASE.md deleted file mode 100644 index f913f3b8322e23..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP DATABASE.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# DROP DATABASE -## description - 该语句用于删除数据库（database） - 语法： - DROP DATABASE [IF EXISTS] db_name; - - 说明： - 执行 DROP DATABASE 一段时间内，可以通过 RECOVER 语句恢复被删除的 database。详见 RECOVER 语句 - -## example - 1. 删除数据库 db_test - DROP DATABASE db_test; - -## keyword - DROP,DATABASE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP INDEX.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP INDEX.md deleted file mode 100644 index 5ed5a78b640d9f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP INDEX.md +++ /dev/null @@ -1,30 +0,0 @@ - - -# DROP INDEX - -## description - - 该语句用于从一个表中删除指定名称的索引，目前仅支持bitmap 索引 - 语法： - DROP INDEX index_name ON [db_name.]table_name; - -## keyword - - DROP,INDEX diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md deleted file mode 100644 index 35078eb09a3624..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md +++ /dev/null @@ -1,101 +0,0 @@ - - -# DROP MATERIALIZED VIEW - -## description - 该语句用于删除物化视图。同步语法 - -语法: - - ``` - DROP MATERIALIZED VIEW [IF EXISTS] mv_name ON table_name - ``` - -1. IF EXISTS - 如果物化视图不存在，不要抛出错误。如果不声明此关键字，物化视图不存在则报错。 - -2. mv_name - 待删除的物化视图的名称。必填项。 - -3. table_name - 待删除的物化视图所属的表名。必填项。 - -## example - -表结构为 - -``` -mysql> desc all_type_table all; -+----------------+-------+----------+------+-------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+----------------+-------+----------+------+-------+---------+-------+ -| all_type_table | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | false | N/A | NONE | -| | k3 | INT | Yes | false | N/A | NONE | -| | k4 | BIGINT | Yes | false | N/A | NONE | -| | k5 | LARGEINT | Yes | false | N/A | NONE | -| | k6 | FLOAT | Yes | false | N/A | NONE | -| | k7 | DOUBLE | Yes | false | N/A | NONE | -| | | | | | | | -| k1_sumk2 | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | false | N/A | SUM | -+----------------+-------+----------+------+-------+---------+-------+ -``` - -1. 删除表 all_type_table 的名为 k1_sumk2 的物化视图 - - ``` - drop materialized view k1_sumk2 on all_type_table; - ``` - 物化视图被删除后的表结构 - - ``` - +----------------+-------+----------+------+-------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+----------------+-------+----------+------+-------+---------+-------+ -| all_type_table | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | false | N/A | NONE | -| | k3 | INT | Yes | false | N/A | NONE | -| | k4 | BIGINT | Yes | false | N/A | NONE | -| | k5 | LARGEINT | Yes | false | N/A | NONE | -| | k6 | FLOAT | Yes | false | N/A | NONE | -| | k7 | DOUBLE | Yes | false | N/A | NONE | -+----------------+-------+----------+------+-------+---------+-------+ - ``` - -2. 删除表 all_type_table 中一个不存在的物化视图 - - ``` - drop materialized view k1_k2 on all_type_table; - ERROR 1064 (HY000): errCode = 2, detailMessage = Materialized view [k1_k2] does not exist in table [all_type_table] - ``` - 删除请求直接报错 - -3. 删除表 all_type_table 中的物化视图 k1_k2，不存在不报错。 - - ``` - drop materialized view if exists k1_k2 on all_type_table; -Query OK, 0 rows affected (0.00 sec) - ``` - - 存在则删除，不存在则不报错。 - -## keyword - DROP, MATERIALIZED, VIEW diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md deleted file mode 100644 index 9bb83ac7980cdf..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# DROP REPOSITORY -## description - 该语句用于删除一个已创建的仓库。仅 root 或 superuser 用户可以删除仓库。 - 语法： - DROP REPOSITORY `repo_name`; - - 说明： - 1. 删除仓库，仅仅是删除该仓库在 Palo 中的映射，不会删除实际的仓库数据。删除后，可以再次通过指定相同的 broker 和 LOCATION 映射到该仓库。 - -## example - 1. 删除名为 bos_repo 的仓库： - DROP REPOSITORY `bos_repo`; - -## keyword - DROP REPOSITORY - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP TABLE.md deleted file mode 100644 index 6452d1907c74dc..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP TABLE.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# DROP TABLE -## description - 该语句用于删除 table 。 - 语法： - DROP TABLE [IF EXISTS] [db_name.]table_name; - - 说明： - 执行 DROP TABLE 一段时间内，可以通过 RECOVER 语句恢复被删除的 table。详见 RECOVER 语句 - -## example - 1. 删除一个 table - DROP TABLE my_table; - - 2. 如果存在，删除指定 database 的 table - DROP TABLE IF EXISTS example_db.my_table; - -## keyword - DROP,TABLE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP VIEW.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP VIEW.md deleted file mode 100644 index 2ce971f8f791f7..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/DROP VIEW.md +++ /dev/null @@ -1,33 +0,0 @@ - - -# DROP VIEW -## description - 该语句用于删除一个逻辑视图 VIEW - 语法： - DROP VIEW [IF EXISTS] - [db_name.]view_name; - -## example - 1. 如果存在，删除 example_db 上的视图 example_view - DROP VIEW IF EXISTS example_db.example_view; - -## keyword - DROP,VIEW - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/HLL.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/HLL.md deleted file mode 100644 index ae1c9afaaf055c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/HLL.md +++ /dev/null @@ -1,102 +0,0 @@ - - -# HLL -## description - HLL是基于HyperLogLog算法的工程实现，用于保存HyperLogLog计算过程的中间结果，它只能作为表的value列类型 - 通过聚合来不断的减少数据量，以此来实现加快查询的目的，基于它到的是一个估算结果，误差大概在1%左右 - hll列是通过其它列或者导入数据里面的数据生成的，导入的时候通过hll_hash函数来指定数据中哪一列用于生成hll列 - 它常用于替代count distinct，通过结合rollup在业务上用于快速计算uv等 - - 相关函数: - - HLL_UNION_AGG(hll) - 此函数为聚合函数，用于计算满足条件的所有数据的基数估算。此函数还可用于分析函数，只支持默认窗口，不支持window从句。 - - HLL_RAW_AGG(hll) - 此函数为聚合函数，用于聚合hll类型字段，并且返回的还是hll类型。 - - HLL_CARDINALITY(hll) - 此函数用于计算单条hll列的基数估算 - - HLL_HASH(column_name) - 生成HLL列类型，用于insert或导入的时候，导入的使用见相关说明 - - EMPTY_HLL() - 生成空HLL列，用于insert或导入的时候补充默认值，导入的使用见相关说明 - -## example - 1. 首先创建一张含有hll列的表 - create table test( - dt date, - id int, - name char(10), - province char(10), - os char(1), - set1 hll hll_union, - set2 hll hll_union) - distributed by hash(id) buckets 32; - - 2. 导入数据，导入的方式见相关help curl - - a. 使用表中的列生成hll列 - curl --location-trusted -uname:password -T data -H "label:load_1" -H "columns:dt, id, name, province, os, set1=hll_hash(id), set2=hll_hash(name)" - http://host/api/test_db/test/_stream_load - b. 使用数据中的某一列生成hll列 - curl --location-trusted -uname:password -T data -H "label:load_1" -H "columns:dt, id, name, province, sex, cuid, os, set1=hll_hash(cuid), set2=hll_hash(os)" - http://host/api/test_db/test/_stream_load - - 3. 聚合数据，常用方式3种：（如果不聚合直接对base表查询，速度可能跟直接使用ndv速度差不多） - - a. 创建一个rollup，让hll列产生聚合， - alter table test add rollup test_rollup(dt, set1); - - b. 创建另外一张专门计算uv的表，然后insert数据） - - create table test_uv( - dt date, - uv_set hll hll_union) - distributed by hash(id) buckets 32; - - insert into test_uv select dt, set1 from test; - - c. 创建另外一张专门计算uv的表，然后insert并通过hll_hash根据test其它非hll列生成hll列 - - create table test_uv( - dt date, - id_set hll hll_union) - distributed by hash(id) buckets 32; - - insert into test_uv select dt, hll_hash(id) from test; - - 4. 查询，hll列不允许直接查询它的原始值，可以通过配套的函数进行查询 - - a. 求总uv - select HLL_UNION_AGG(uv_set) from test_uv; - - b. 求每一天的uv - select dt, HLL_CARDINALITY(uv_set) from test_uv; - - c. 求test表中set1的聚合值 - select dt, HLL_CARDINALITY(uv) from (select dt, HLL_RAW_AGG(set1) as uv from test group by dt) tmp; - select dt, HLL_UNION_AGG(set1) as uv from test group by dt; - -## keyword - HLL - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/RECOVER.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/RECOVER.md deleted file mode 100644 index 1faa15133c3422..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/RECOVER.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# RECOVER -## description - 该语句用于恢复之前删除的 database、table 或者 partition - 语法： - 1) 恢复 database - RECOVER DATABASE db_name; - 2) 恢复 table - RECOVER TABLE [db_name.]table_name; - 3) 恢复 partition - RECOVER PARTITION partition_name FROM [db_name.]table_name; - - 说明： - 1. 该操作仅能恢复之前一段时间内删除的元信息。默认为 1 天。（可通过fe.conf中`catalog_trash_expire_second`参数配置） - 2. 如果删除元信息后新建立了同名同类型的元信息，则之前删除的元信息不能被恢复 - -## example - 1. 恢复名为 example_db 的 database - RECOVER DATABASE example_db; - - 2. 恢复名为 example_tbl 的 table - RECOVER TABLE example_db.example_tbl; - - 3. 恢复表 example_tbl 中名为 p1 的 partition - RECOVER PARTITION p1 FROM example_tbl; - -## keyword - RECOVER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/RESTORE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/RESTORE.md deleted file mode 100644 index 4e6d158c0244da..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/RESTORE.md +++ /dev/null @@ -1,71 +0,0 @@ - - -# RESTORE -## description - 1. RESTORE - 该语句用于将之前通过 BACKUP 命令备份的数据，恢复到指定数据库下。该命令为异步操作。提交成功后，需通过 SHOW RESTORE 命令查看进度。仅支持恢复 OLAP 类型的表。 - 语法： - RESTORE SNAPSHOT [db_name].{snapshot_name} - FROM `repository_name` - ON ( - `table_name` [PARTITION (`p1`, ...)] [AS `tbl_alias`], - ... - ) - PROPERTIES ("key"="value", ...); - - 说明： - 1. 同一数据库下只能有一个正在执行的 BACKUP 或 RESTORE 任务。 - 2. ON 子句中标识需要恢复的表和分区。如果不指定分区，则默认恢复该表的所有分区。所指定的表和分区必须已存在于仓库备份中。 - 3. 可以通过 AS 语句将仓库中备份的表名恢复为新的表。但新表名不能已存在于数据库中。分区名称不能修改。 - 4. 可以将仓库中备份的表恢复替换数据库中已有的同名表，但须保证两张表的表结构完全一致。表结构包括：表名、列、分区、Rollup等等。 - 5. 可以指定恢复表的部分分区，系统会检查分区 Range 是否能够匹配。 - 6. PROPERTIES 目前支持以下属性： - "backup_timestamp" = "2018-05-04-16-45-08"：指定了恢复对应备份的哪个时间版本，必填。该信息可以通过 `SHOW SNAPSHOT ON repo;` 语句获得。 - "replication_num" = "3"：指定恢复的表或分区的副本数。默认为3。若恢复已存在的表或分区，则副本数必须和已存在表或分区的副本数相同。同时，必须有足够的 host 容纳多个副本。 - "timeout" = "3600"：任务超时时间，默认为一天。单位秒。 - "meta_version" = 40：使用指定的 meta_version 来读取之前备份的元数据。注意，该参数作为临时方案，仅用于恢复老版本 Doris 备份的数据。最新版本的备份数据中已经包含 meta version，无需再指定。 - -## example - 1. 从 example_repo 中恢复备份 snapshot_1 中的表 backup_tbl 到数据库 example_db1，时间版本为 "2018-05-04-16-45-08"。恢复为 1 个副本： - RESTORE SNAPSHOT example_db1.`snapshot_1` - FROM `example_repo` - ON ( `backup_tbl` ) - PROPERTIES - ( - "backup_timestamp"="2018-05-04-16-45-08", - "replication_num" = "1" - ); - - 2. 从 example_repo 中恢复备份 snapshot_2 中的表 backup_tbl 的分区 p1,p2，以及表 backup_tbl2 到数据库 example_db1，并重命名为 new_tbl，时间版本为 "2018-05-04-17-11-01"。默认恢复为 3 个副本： - RESTORE SNAPSHOT example_db1.`snapshot_2` - FROM `example_repo` - ON - ( - `backup_tbl` PARTITION (`p1`, `p2`), - `backup_tbl2` AS `new_tbl` - ) - PROPERTIES - ( - "backup_timestamp"="2018-05-04-17-11-01" - ); - -## keyword - RESTORE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md deleted file mode 100644 index 07526300f1beca..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# TRUNCATE TABLE -## description - 该语句用于清空指定表和分区的数据 - 语法： - - TRUNCATE TABLE [db.]tbl[ PARTITION(p1, p2, ...)]; - - 说明： - 1. 该语句清空数据，但保留表或分区。 - 2. 不同于 DELETE，该语句只能整体清空指定的表或分区，不能添加过滤条件。 - 3. 不同于 DELETE，使用该方式清空数据不会对查询性能造成影响。 - 4. 该操作删除的数据不可恢复。 - 5. 使用该命令时，表状态需为 NORMAL，即不允许正在进行 SCHEMA CHANGE 等操作。 - -## example - - 1. 清空 example_db 下的表 tbl - - TRUNCATE TABLE example_db.tbl; - - 2. 清空表 tbl 的 p1 和 p2 分区 - - TRUNCATE TABLE tbl PARTITION(p1, p2); - -## keyword - TRUNCATE,TABLE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/create-function.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/create-function.md deleted file mode 100644 index 67ca3c54866283..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/create-function.md +++ /dev/null @@ -1,107 +0,0 @@ - - -# CREATE FUNCTION -## description -### Syntax - -``` -CREATE [AGGREGATE] FUNCTION function_name - (arg_type [, ...]) - RETURNS ret_type - [INTERMEDIATE inter_type] - [PROPERTIES ("key" = "value" [, ...]) ] -``` - -### Parameters - -> `AGGREGATE`: 如果有此项，表示的是创建的函数是一个聚合函数，否则创建的是一个标量函数。 -> -> `function_name`: 要创建函数的名字, 可以包含数据库的名字。比如：`db1.my_func`。 -> -> `arg_type`: 函数的参数类型，与建表时定义的类型一致。变长参数时可以使用`, ...`来表示，如果是变长类型，那么变长部分参数的类型与最后一个非变长参数类型一致。 -> -> `ret_type`: 函数返回类型。 -> -> `inter_type`: 用于表示聚合函数中间阶段的数据类型。 -> -> `properties`: 用于设定此函数相关属性，能够设置的属性包括 -> -> "object_file": 自定义函数动态库的URL路径，当前只支持 HTTP/HTTPS 协议，此路径需要在函数整个生命周期内保持有效。此选项为必选项 -> -> "symbol": 标量函数的函数签名，用于从动态库里面找到函数入口。此选项对于标量函数是必选项 -> -> "init_fn": 聚合函数的初始化函数签名。对于聚合函数是必选项 -> -> "update_fn": 聚合函数的更新函数签名。对于聚合函数是必选项 -> -> "merge_fn": 聚合函数的合并函数签名。对于聚合函数是必选项 -> -> "serialize_fn": 聚合函数的序列化函数签名。对于聚合函数是可选项，如果没有指定，那么将会使用默认的序列化函数 -> -> "finalize_fn": 聚合函数获取最后结果的函数签名。对于聚合函数是可选项，如果没有指定，将会使用默认的获取结果函数 -> -> "md5": 函数动态链接库的MD5值，用于校验下载的内容是否正确。此选项是可选项 -> -> "prepare_fn": 自定义函数的prepare函数的函数签名，用于从动态库里面找到prepare函数入口。此选项对于自定义函数是可选项 -> -> "close_fn": 自定义函数的close函数的函数签名，用于从动态库里面找到close函数入口。此选项对于自定义函数是可选项 - - -此语句创建一个自定义函数。执行此命令需要用户拥有 `ADMIN` 权限。 - -如果 `function_name` 中包含了数据库名字，那么这个自定义函数会创建在对应的数据库中，否则这个函数将会创建在当前会话所在的数据库。新函数的名字与参数不能够与当前命名空间中已存在的函数相同，否则会创建失败。但是只有名字相同，参数不同是能够创建成功的。 - -## example - -1. 创建一个自定义标量函数 - - ``` - CREATE FUNCTION my_add(INT, INT) RETURNS INT PROPERTIES ( - "symbol" = "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", - "object_file" = "http://host:port/libmyadd.so" - ); - ``` - -2. 创建一个有prepare/close函数的自定义标量函数 - - ``` - CREATE FUNCTION my_add(INT, INT) RETURNS INT PROPERTIES ( - "symbol" = "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", - "prepare_fn" = "_ZN9doris_udf14AddUdf_prepareEPNS_15FunctionContextENS0_18FunctionStateScopeE", - "close_fn" = "_ZN9doris_udf12AddUdf_closeEPNS_15FunctionContextENS0_18FunctionStateScopeE", - "object_file" = "http://host:port/libmyadd.so" - ); - ``` - -2. 创建一个自定义聚合函数 - - ``` - CREATE AGGREGATE FUNCTION my_count (BIGINT) RETURNS BIGINT PROPERTIES ( - "init_fn"="_ZN9doris_udf9CountInitEPNS_15FunctionContextEPNS_9BigIntValE", - "update_fn"="_ZN9doris_udf11CountUpdateEPNS_15FunctionContextERKNS_6IntValEPNS_9BigIntValE", - "merge_fn"="_ZN9doris_udf10CountMergeEPNS_15FunctionContextERKNS_9BigIntValEPS2_", - "finalize_fn"="_ZN9doris_udf13CountFinalizeEPNS_15FunctionContextERKNS_9BigIntValE", - "object_file"="http://host:port/libudasample.so" - ); - ``` - -## keyword - - CREATE,FUNCTION diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/drop-function.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/drop-function.md deleted file mode 100644 index 66b92a3f1c17fe..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/drop-function.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# DROP FUNCTION -## description -### Syntax - -``` -DROP FUNCTION function_name - (arg_type [, ...]) -``` - -### Parameters - -> `function_name`: 要删除函数的名字 -> -> `arg_type`: 要删除函数的参数列表 -> - - -删除一个自定义函数。函数的名字、参数类型完全一致才能够被删除 - -## example - -1. 删除掉一个函数 - -``` -DROP FUNCTION my_add(INT, INT) -``` - -## keyword - - DROP,FUNCTION diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/index.rst b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/index.rst deleted file mode 100644 index 574b4b9045ec57..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -DDL -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/show-functions.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/show-functions.md deleted file mode 100644 index e6d89b1e5c573e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/show-functions.md +++ /dev/null @@ -1,72 +0,0 @@ - - -# SHOW FUNCTIONS -## description -### Syntax - -``` -SHOW [FULL] [BUILTIN] FUNCTIONS [IN|FROM db] [LIKE 'function_pattern'] -``` - -### Parameters - ->`full`:表示显示函数的详细信息 ->`builtin`:表示显示系统提供的函数 ->`db`: 要查询的数据库名字 ->`function_pattern`: 用来过滤函数名称的参数 - - -查看数据库下所有的自定义(系统提供)的函数。如果用户指定了数据库，那么查看对应数据库的，否则直接查询当前会话所在数据库 - -需要对这个数据库拥有 `SHOW` 权限 - -## example - -``` -mysql> show full functions in testDb\G -*************************** 1. row *************************** - Signature: my_add(INT,INT) - Return Type: INT - Function Type: Scalar -Intermediate Type: NULL - Properties: {"symbol":"_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_","object_file":"http://host:port/libudfsample.so","md5":"cfe7a362d10f3aaf6c49974ee0f1f878"} -*************************** 2. row *************************** - Signature: my_count(BIGINT) - Return Type: BIGINT - Function Type: Aggregate -Intermediate Type: NULL - Properties: {"object_file":"http://host:port/libudasample.so","finalize_fn":"_ZN9doris_udf13CountFinalizeEPNS_15FunctionContextERKNS_9BigIntValE","init_fn":"_ZN9doris_udf9CountInitEPNS_15FunctionContextEPNS_9BigIntValE","merge_fn":"_ZN9doris_udf10CountMergeEPNS_15FunctionContextERKNS_9BigIntValEPS2_","md5":"37d185f80f95569e2676da3d5b5b9d2f","update_fn":"_ZN9doris_udf11CountUpdateEPNS_15FunctionContextERKNS_6IntValEPNS_9BigIntValE"} - -2 rows in set (0.00 sec) -mysql> show builtin functions in testDb like 'year%'; -+---------------+ -| Function Name | -+---------------+ -| year | -| years_add | -| years_diff | -| years_sub | -+---------------+ -2 rows in set (0.00 sec) -``` - -## keyword - - SHOW,FUNCTIONS diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md deleted file mode 100644 index 5e96ed9b1f8d51..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md +++ /dev/null @@ -1,428 +0,0 @@ - - -# BROKER LOAD -## description - - Broker load 通过随 Doris 集群一同部署的 broker 进行，访问对应数据源的数据，进行数据导入。 - 可以通过 show broker 命令查看已经部署的 broker。 - 目前支持以下4种数据源： - - 1. Baidu HDFS：百度内部的 hdfs，仅限于百度内部使用。 - 2. Baidu AFS：百度内部的 afs，仅限于百度内部使用。 - 3. Baidu Object Storage(BOS)：百度对象存储。仅限百度内部用户、公有云用户或其他可以访问 BOS 的用户使用。 - 4. Apache HDFS：社区版本 hdfs。 - 5. Amazon S3：Amazon对象存储。 - -语法： - - LOAD LABEL load_label - ( - data_desc1[, data_desc2, ...] - ) - WITH BROKER broker_name - [broker_properties] - [opt_properties]; - - 1. load_label - - 当前导入批次的标签。在一个 database 内唯一。 - 语法： - [database_name.]your_label - - 2. data_desc - - 用于描述一批导入数据。 - 语法： - DATA INFILE - ( - "file_path1"[, file_path2, ...] - ) - [NEGATIVE] - INTO TABLE `table_name` - [PARTITION (p1, p2)] - [COLUMNS TERMINATED BY "column_separator"] - [FORMAT AS "file_type"] - [(column_list)] - [SET (k1 = func(k2))] - [WHERE predicate] - - 说明： - file_path: - - 文件路径，可以指定到一个文件，也可以用 * 通配符指定某个目录下的所有文件。通配符必须匹配到文件，而不能是目录。 - - PARTITION: - - 如果指定此参数，则只会导入指定的分区，导入分区以外的数据会被过滤掉。 - 如果不指定，默认导入table的所有分区。 - - NEGATIVE： - 如果指定此参数，则相当于导入一批“负”数据。用于抵消之前导入的同一批数据。 - 该参数仅适用于存在 value 列，并且 value 列的聚合类型仅为 SUM 的情况。 - - column_separator： - - 用于指定导入文件中的列分隔符。默认为 \t - 如果是不可见字符，则需要加\\x作为前缀，使用十六进制来表示分隔符。 - 如hive文件的分隔符\x01，指定为"\\x01" - - file_type： - - 用于指定导入文件的类型，例如：parquet、orc、csv。默认值通过文件后缀名判断。 - - column_list： - - 用于指定导入文件中的列和 table 中的列的对应关系。 - 当需要跳过导入文件中的某一列时，将该列指定为 table 中不存在的列名即可。 - 语法： - (col_name1, col_name2, ...) - - SET: - - 如果指定此参数，可以将源文件某一列按照函数进行转化，然后将转化后的结果导入到table中。语法为 `column_name` = expression。举几个例子帮助理解。 - 例1: 表中有3个列“c1, c2, c3", 源文件中前两列依次对应(c1,c2)，后两列之和对应c3；那么需要指定 columns (c1,c2,tmp_c3,tmp_c4) SET (c3=tmp_c3+tmp_c4); - 例2: 表中有3个列“year, month, day"三个列，源文件中只有一个时间列，为”2018-06-01 01:02:03“格式。 - 那么可以指定 columns(tmp_time) set (year = year(tmp_time), month=month(tmp_time), day=day(tmp_time)) 完成导入。 - - WHERE: - - 对做完 transform 的数据进行过滤，符合 where 条件的数据才能被导入。WHERE 语句中只可引用表中列名。 - 3. broker_name - - 所使用的 broker 名称，可以通过 show broker 命令查看。 - - 4. broker_properties - - 用于提供通过 broker 访问数据源的信息。不同的 broker，以及不同的访问方式，需要提供的信息不同。 - - 1. Baidu HDFS/AFS - - 访问百度内部的 hdfs/afs 目前仅支持简单认证，需提供： - username：hdfs 用户名 - password：hdfs 密码 - - 2. BOS - - 需提供： - bos_endpoint：BOS 的endpoint - bos_accesskey：公有云用户的 accesskey - bos_secret_accesskey：公有云用户的 secret_accesskey - - 3. Apache HDFS - - 社区版本的 hdfs，支持简单认证、kerberos 认证。以及支持 HA 配置。 - 简单认证： - hadoop.security.authentication = simple (默认) - username：hdfs 用户名 - password：hdfs 密码 - - kerberos 认证： - hadoop.security.authentication = kerberos - kerberos_principal：指定 kerberos 的 principal - kerberos_keytab：指定 kerberos 的 keytab 文件路径。该文件必须为 broker 进程所在服务器上的文件。 - kerberos_keytab_content：指定 kerberos 中 keytab 文件内容经过 base64 编码之后的内容。这个跟 kerberos_keytab 配置二选一就可以。 - - namenode HA： - 通过配置 namenode HA，可以在 namenode 切换时，自动识别到新的 namenode - dfs.nameservices: 指定 hdfs 服务的名字，自定义，如："dfs.nameservices" = "my_ha" - dfs.ha.namenodes.xxx：自定义 namenode 的名字,多个名字以逗号分隔。其中 xxx 为 dfs.nameservices 中自定义的名字，如 "dfs.ha.namenodes.my_ha" = "my_nn" - dfs.namenode.rpc-address.xxx.nn：指定 namenode 的rpc地址信息。其中 nn 表示 dfs.ha.namenodes.xxx 中配置的 namenode 的名字，如："dfs.namenode.rpc-address.my_ha.my_nn" = "host:port" - dfs.client.failover.proxy.provider：指定 client 连接 namenode 的 provider，默认为：org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - 4. Amazon S3 - - 需提供： - fs.s3a.access.key：AmazonS3的access key - fs.s3a.secret.key：AmazonS3的secret key - fs.s3a.endpoint：AmazonS3的endpoint - - 4. opt_properties - - 用于指定一些特殊参数。 - 语法： - [PROPERTIES ("key"="value", ...)] - - 可以指定如下参数： - timeout： 指定导入操作的超时时间。默认超时为4小时。单位秒。 - max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。 - exec_mem_limit： 导入内存限制。默认为 2GB。单位为字节。 - strict mode： 是否对数据进行严格限制。默认为 false。 - timezone: 指定某些受时区影响的函数的时区，如 strftime/alignment_timestamp/from_unixtime 等等，具体请查阅 [时区] 文档。如果不指定，则使用 "Asia/Shanghai" 时区。 - - 5. 导入数据格式样例 - - 整型类（TINYINT/SMALLINT/INT/BIGINT/LARGEINT）：1, 1000, 1234 - 浮点类（FLOAT/DOUBLE/DECIMAL）：1.1, 0.23, .356 - 日期类（DATE/DATETIME）：2017-10-03, 2017-06-13 12:34:03。 - （注：如果是其他日期格式，可以在导入命令中，使用 strftime 或者 time_format 函数进行转换） - 字符串类（CHAR/VARCHAR）："I am a student", "a" - NULL值：\N - -## example - - 1. 从 HDFS 导入一批数据，指定超时时间和过滤比例。使用铭文 my_hdfs_broker 的 broker。简单认证。 - - LOAD LABEL example_db.label1 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - ) - WITH BROKER my_hdfs_broker - ( - "username" = "hdfs_user", - "password" = "hdfs_passwd" - ) - PROPERTIES - ( - "timeout" = "3600", - "max_filter_ratio" = "0.1" - ); - - 其中 hdfs_host 为 namenode 的 host，hdfs_port 为 fs.defaultFS 端口（默认9000） - - 2. 从 AFS 一批数据，包含多个文件。导入不同的 table，指定分隔符，指定列对应关系。 - - LOAD LABEL example_db.label2 - ( - DATA INFILE("afs://afs_host:hdfs_port/user/palo/data/input/file1") - INTO TABLE `my_table_1` - COLUMNS TERMINATED BY "," - (k1, k3, k2, v1, v2), - DATA INFILE("afs://afs_host:hdfs_port/user/palo/data/input/file2") - INTO TABLE `my_table_2` - COLUMNS TERMINATED BY "\t" - (k1, k2, k3, v2, v1) - ) - WITH BROKER my_afs_broker - ( - "username" = "afs_user", - "password" = "afs_passwd" - ) - PROPERTIES - ( - "timeout" = "3600", - "max_filter_ratio" = "0.1" - ); - - - 3. 从 HDFS 导入一批数据，指定hive的默认分隔符\x01，并使用通配符*指定目录下的所有文件。 - 使用简单认证，同时配置 namenode HA - - LOAD LABEL example_db.label3 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/*") - INTO TABLE `my_table` - COLUMNS TERMINATED BY "\\x01" - ) - WITH BROKER my_hdfs_broker - ( - "username" = "hdfs_user", - "password" = "hdfs_passwd", - "dfs.nameservices" = "my_ha", - "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", - "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", - "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", - "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" - ) - - 4. 从 HDFS 导入一批“负”数据。同时使用 kerberos 认证方式。提供 keytab 文件路径。 - - LOAD LABEL example_db.label4 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/old_file) - NEGATIVE - INTO TABLE `my_table` - COLUMNS TERMINATED BY "\t" - ) - WITH BROKER my_hdfs_broker - ( - "hadoop.security.authentication" = "kerberos", - "kerberos_principal"="doris@YOUR.COM", - "kerberos_keytab"="/home/palo/palo.keytab" - ) - - 5. 从 HDFS 导入一批数据，指定分区。同时使用 kerberos 认证方式。提供 base64 编码后的 keytab 文件内容。 - - LOAD LABEL example_db.label5 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (k1, k3, k2, v1, v2) - ) - WITH BROKER my_hdfs_broker - ( - "hadoop.security.authentication"="kerberos", - "kerberos_principal"="doris@YOUR.COM", - "kerberos_keytab_content"="BQIAAABEAAEACUJBSURVLkNPTQAEcGFsbw" - ) - - 6. 从 BOS 导入一批数据，指定分区, 并对导入文件的列做一些转化，如下： - 表结构为： - k1 varchar(20) - k2 int - - 假设数据文件只有一行数据： - - Adele,1,1 - - 数据文件中各列，对应导入语句中指定的各列： - k1,tmp_k2,tmp_k3 - - 转换如下： - - 1) k1: 不变换 - 2) k2：是 tmp_k2 和 tmp_k3 数据之和 - - LOAD LABEL example_db.label6 - ( - DATA INFILE("bos://my_bucket/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (k1, tmp_k2, tmp_k3) - SET ( - k2 = tmp_k2 + tmp_k3 - ) - ) - WITH BROKER my_bos_broker - ( - "bos_endpoint" = "http://bj.bcebos.com", - "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "bos_secret_accesskey"="yyyyyyyyyyyyyyyyyyyy" - ) - - 7. 导入数据到含有HLL列的表，可以是表中的列或者数据里面的列 - - 如果表中有三列分别是（id,v1,v2,v3）。其中v1和v2列是hll列。导入的源文件有3列。则（column_list）中声明第一列为id，第二三列为一个临时命名的k1,k2。 - 在SET中必须给表中的hll列特殊声明 hll_hash。表中的v1列等于原始数据中的hll_hash(k1)列, 表中的v3列在原始数据中并没有对应的值，使用empty_hll补充默认值。 - LOAD LABEL example_db.label7 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (id, k1, k2) - SET ( - v1 = hll_hash(k1), - v2 = hll_hash(k2), - v3 = empty_hll() - ) - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - - LOAD LABEL example_db.label8 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (k1, k2, tmp_k3, tmp_k4, v1, v2) - SET ( - v1 = hll_hash(tmp_k3), - v2 = hll_hash(tmp_k4) - ) - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - - 8. 导入Parquet文件中数据 指定FORMAT 为parquet， 默认是通过文件后缀判断 - - LOAD LABEL example_db.label9 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - FORMAT AS "parquet" - (k1, k2, k3) - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - - 9. 提取文件路径中的分区字段 - - 如果需要，则会根据表中定义的字段类型解析文件路径中的分区字段（partitioned fields），类似Spark中Partition Discovery的功能 - - LOAD LABEL example_db.label10 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/*/*") - INTO TABLE `my_table` - FORMAT AS "csv" - (k1, k2, k3) - COLUMNS FROM PATH AS (city, utc_date) - SET (uniq_id = md5sum(k1, city)) - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - - hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing目录下包括如下文件： - - [hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/utc_date=2019-06-26/0000.csv, hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/utc_date=2019-06-26/0001.csv, ...] - - 则提取文件路径的中的city和utc_date字段 - - 10. 对待导入数据进行过滤，k1 值大于 k2 值的列才能被导入 - - LOAD LABEL example_db.label10 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - where k1 > k2 - ) - - 11. 从 AmazonS3 导入Parquet文件中数据，指定 FORMAT 为parquet，默认是通过文件后缀判断： - - LOAD LABEL example_db.label11 - ( - DATA INFILE("s3a://my_bucket/input/file") - INTO TABLE `my_table` - FORMAT AS "parquet" - (k1, k2, k3) - ) - WITH BROKER my_s3a_broker - ( - "fs.s3a.access.key" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "fs.s3a.secret.key" = "yyyyyyyyyyyyyyyyyyyy", - "fs.s3a.endpoint" = "s3.amazonaws.com" - ) - - 12. 提取文件路径中的时间分区字段，并且时间包含 %3A (在 hdfs 路径中，不允许有 ':'，所有 ':' 会由 %3A 替换) - - 假设有如下文件： - - /user/data/data_time=2020-02-17 00%3A00%3A00/test.txt - /user/data/data_time=2020-02-18 00%3A00%3A00/test.txt - - 表结构为： - data_time DATETIME, - k2 INT, - k3 INT - - LOAD LABEL example_db.label12 - ( - DATA INFILE("hdfs://host:port/user/data/*/test.txt") - INTO TABLE `tbl12` - COLUMNS TERMINATED BY "," - (k2,k3) - COLUMNS FROM PATH AS (data_time) - SET (data_time=str_to_date(data_time, '%Y-%m-%d %H%%3A%i%%3A%s')) - ) - WITH BROKER "hdfs" ("username"="user", "password"="pass"); - -## keyword - - BROKER,LOAD diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md deleted file mode 100644 index f21ebbeab6f11f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# CANCEL LOAD -## description - - 该语句用于撤销指定 load label 的批次的导入作业。 - 这是一个异步操作，任务提交成功则返回。执行后可使用 SHOW LOAD 命令查看进度。 - 语法： - CANCEL LOAD - [FROM db_name] - WHERE LABEL = "load_label"; - -## example - - 1. 撤销数据库 example_db 上， label 为 example_db_test_load_label 的导入作业 - CANCEL LOAD - FROM example_db - WHERE LABEL = "example_db_test_load_label"; - -## keyword - CANCEL,LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/DELETE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/DELETE.md deleted file mode 100644 index a902e8913074f6..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/DELETE.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# DELETE -## description - - 该语句用于按条件删除指定 table（base index） partition 中的数据。 - 该操作会同时删除和此 base index 相关的 rollup index 的数据。 - 语法： - DELETE FROM table_name [PARTITION partition_name] - WHERE - column_name1 op value[ AND column_name2 op value ...]; - - 说明： - 1) op 的可选类型包括：=, >, <, >=, <=, != - 2) 只能指定 key 列上的条件。 - 2) 当选定的 key 列不存在于某个 rollup 中时，无法进行 delete。 - 3) 条件之间只能是“与”的关系。 - 若希望达成“或”的关系，需要将条件分写在两个 DELETE 语句中。 - 4) 如果为RANGE分区表，则必须指定 PARTITION。如果是单分区表，可以不指定。 - - 注意： - 该语句可能会降低执行后一段时间内的查询效率。 - 影响程度取决于语句中指定的删除条件的数量。 - 指定的条件越多，影响越大。 - -## example - - 1. 删除 my_table partition p1 中 k1 列值为 3 的数据行 - DELETE FROM my_table PARTITION p1 - WHERE k1 = 3; - - 2. 删除 my_table partition p1 中 k1 列值大于等于 3 且 k2 列值为 "abc" 的数据行 - DELETE FROM my_table PARTITION p1 - WHERE k1 >= 3 AND k2 = "abc"; - -## keyword - DELETE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/EXPORT.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/EXPORT.md deleted file mode 100644 index 0955f20a7eacdf..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/EXPORT.md +++ /dev/null @@ -1,77 +0,0 @@ - - -# EXPORT -## description - - 该语句用于将指定表的数据导出到指定位置。 - 该功能通过 broker 进程实现。对于不同的目的存储系统，需要部署不同的 broker。可以通过 SHOW BROKER 查看已部署的 broker。 - 这是一个异步操作，任务提交成功则返回。执行后可使用 SHOW EXPORT 命令查看进度。 - - 语法： - EXPORT TABLE table_name - [PARTITION (p1[,p2])] - TO export_path - [opt_properties] - broker; - - 1. table_name - 当前要导出的表的表名，目前支持engine为olap和mysql的表的导出。 - - 2. partition - 可以只导出指定表的某些指定分区 - - 3. export_path - 导出的路径，需为目录。目前不能导出到本地，需要导出到broker。 - - 4. opt_properties - 用于指定一些特殊参数。 - 语法： - [PROPERTIES ("key"="value", ...)] - - 可以指定如下参数： - column_separator: 指定导出的列分隔符，默认为\t。 - line_delimiter: 指定导出的行分隔符，默认为\n。 - exec_mem_limit: 导出在单个 BE 节点的内存使用上限，默认为 2GB，单位为字节。 - timeout：导入作业的超时时间，默认为1天，单位是秒。 - tablet_num_per_task：每个子任务能分配的最大 Tablet 数量。 - - 5. broker - 用于指定导出使用的broker - 语法： - WITH BROKER broker_name ("key"="value"[,...]) - 这里需要指定具体的broker name, 以及所需的broker属性 - - 对于不同存储系统对应的 broker，这里需要输入的参数不同。具体参数可以参阅：`help broker load` 中 broker 所需属性。 - -## example - - 1. 将 testTbl 表中的所有数据导出到 hdfs 上 - EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); - - 2. 将 testTbl 表中的分区p1,p2导出到 hdfs 上 - - EXPORT TABLE testTbl PARTITION (p1,p2) TO "hdfs://hdfs_host:port/a/b/c" WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); - 3. 将 testTbl 表中的所有数据导出到 hdfs 上，以","作为列分隔符 - - EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" PROPERTIES ("column_separator"=",") WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); - -## keyword - EXPORT - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/GROUP BY.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/GROUP BY.md deleted file mode 100644 index e67b9e6ec588fa..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/GROUP BY.md +++ /dev/null @@ -1,163 +0,0 @@ - - -# GROUP BY - -## description - - GROUP BY `GROUPING SETS` ｜ `CUBE` ｜ `ROLLUP` 是对 GROUP BY 子句的扩展，它能够在一个 GROUP BY 子句中实现多个集合的分组的聚合。其结果等价于将多个相应 GROUP BY 子句进行 UNION 操作。 - - GROUP BY 子句是只含有一个元素的 GROUP BY GROUPING SETS 的特例。 - 例如，GROUPING SETS 语句： - - ``` - SELECT a, b, SUM( c ) FROM tab1 GROUP BY GROUPING SETS ( (a, b), (a), (b), ( ) ); - ``` - - 其查询结果等价于： - - ``` - SELECT a, b, SUM( c ) FROM tab1 GROUP BY a, b - UNION - SELECT a, null, SUM( c ) FROM tab1 GROUP BY a - UNION - SELECT null, b, SUM( c ) FROM tab1 GROUP BY b - UNION - SELECT null, null, SUM( c ) FROM tab1 - ``` - - `GROUPING(expr)` 指示一个列是否为聚合列，如果是聚合列为0，否则为1 - - `GROUPING_ID(expr [ , expr [ , ... ] ])` 与GROUPING 类似， GROUPING_ID根据指定的column 顺序，计算出一个列列表的 bitmap 值，每一位为GROUPING的值. GROUPING_ID()函数返回位向量的十进制值。 - -### Syntax - - ``` - SELECT ... - FROM ... - [ ... ] - GROUP BY [ - , ... | - GROUPING SETS [, ...] ( groupSet [ , groupSet [ , ... ] ] ) | - ROLLUP(expr [ , expr [ , ... ] ]) | - expr [ , expr [ , ... ] ] WITH ROLLUP | - CUBE(expr [ , expr [ , ... ] ]) | - expr [ , expr [ , ... ] ] WITH CUBE - ] - [ ... ] - ``` - -### Parameters - - `groupSet` 表示 select list 中的列，别名或者表达式组成的集合 `groupSet ::= { ( expr [ , expr [ , ... ] ] )}` - - `expr` 表示 select list 中的列，别名或者表达式 - -### Note - - doris 支持类似PostgreSQL 语法, 语法实例如下 - - ``` - SELECT a, b, SUM( c ) FROM tab1 GROUP BY GROUPING SETS ( (a, b), (a), (b), ( ) ); - SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY ROLLUP(a,b,c) - SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY CUBE(a,b,c) - ``` - - `ROLLUP(a,b,c)` 等价于如下`GROUPING SETS` 语句 - - ``` - GROUPING SETS ( - (a,b,c), - ( a, b ), - ( a), - ( ) - ) - ``` - - `CUBE ( a, b, c )` 等价于如下`GROUPING SETS` 语句 - - ``` - GROUPING SETS ( - ( a, b, c ), - ( a, b ), - ( a, c ), - ( a ), - ( b, c ), - ( b ), - ( c ), - ( ) - ) - ``` - -## example - - 下面是一个实际数据的例子 - - ``` - > SELECT * FROM t; - +------+------+------+ - | k1 | k2 | k3 | - +------+------+------+ - | a | A | 1 | - | a | A | 2 | - | a | B | 1 | - | a | B | 3 | - | b | A | 1 | - | b | A | 4 | - | b | B | 1 | - | b | B | 5 | - +------+------+------+ - 8 rows in set (0.01 sec) - - > SELECT k1, k2, SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); - +------+------+-----------+ - | k1 | k2 | sum(`k3`) | - +------+------+-----------+ - | b | B | 6 | - | a | B | 4 | - | a | A | 3 | - | b | A | 5 | - | NULL | B | 10 | - | NULL | A | 8 | - | a | NULL | 7 | - | b | NULL | 11 | - | NULL | NULL | 18 | - +------+------+-----------+ - 9 rows in set (0.06 sec) - - > SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ()); - +------+------+---------------+----------------+ - | k1 | k2 | grouping_id(k1,k2) | sum(`k3`) | - +------+------+---------------+----------------+ - | a | A | 0 | 3 | - | a | B | 0 | 4 | - | a | NULL | 1 | 7 | - | b | A | 0 | 5 | - | b | B | 0 | 6 | - | b | NULL | 1 | 11 | - | NULL | A | 2 | 8 | - | NULL | B | 2 | 10 | - | NULL | NULL | 3 | 18 | - +------+------+---------------+----------------+ - 9 rows in set (0.02 sec) - ``` - -## keyword - - GROUP, GROUPING, GROUPING_ID, GROUPING_SETS, GROUPING SETS, CUBE, ROLLUP diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/LOAD.md deleted file mode 100644 index 6134c144824e86..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/LOAD.md +++ /dev/null @@ -1,286 +0,0 @@ - - -# LOAD -## description - - Palo 目前支持以下4种导入方式： - - 1. Hadoop Load：基于 MR 进行 ETL 的导入。 - 2. Broker Load：使用 broker 进行进行数据导入。 - 3. Mini Load：通过 http 协议上传文件进行批量数据导入。 - 4. Stream Load：通过 http 协议进行流式数据导入。 - - 本帮助主要描述第一种导入方式，即 Hadoop Load 相关帮助信息。其余导入方式可以使用以下命令查看帮助： - - !!!该导入方式可能在后续某个版本即不再支持，建议使用其他导入方式进行数据导入。!!! - - 1. help broker load; - 2. help mini load; - 3. help stream load; - - Hadoop Load 仅适用于百度内部环境。公有云、私有云以及开源环境无法使用这种导入方式。 - 该导入方式必须设置用于 ETL 的 Hadoop 计算队列，设置方式可以通过 help set property 命令查看帮助。 - - Stream load 暂时只支持百度内部用户使用。开源社区和公有云用户将在后续版本更新中支持。 - -语法： - - LOAD LABEL load_label - ( - data_desc1[, data_desc2, ...] - ) - [opt_properties]; - - 1. load_label - - 当前导入批次的标签。在一个 database 内唯一。 - 语法： - [database_name.]your_label - - 2. data_desc - - 用于描述一批导入数据。 - 语法： - DATA INFILE - ( - "file_path1"[, file_path2, ...] - ) - [NEGATIVE] - INTO TABLE `table_name` - [PARTITION (p1, p2)] - [COLUMNS TERMINATED BY "column_separator"] - [FORMAT AS "file_type"] - [(column_list)] - [COLUMNS FROM PATH AS (columns_from_path)] - [SET (k1 = func(k2))] - - 说明： - file_path: - - 文件路径，可以指定到一个文件，也可以用 * 通配符指定某个目录下的所有文件。通配符必须匹配到文件，而不能是目录。 - - PARTITION: - - 如果指定此参数，则只会导入指定的分区，导入分区以外的数据会被过滤掉。 - 如果不指定，默认导入table的所有分区。 - - NEGATIVE： - 如果指定此参数，则相当于导入一批“负”数据。用于抵消之前导入的同一批数据。 - 该参数仅适用于存在 value 列，并且 value 列的聚合类型仅为 SUM 的情况。 - - column_separator： - - 用于指定导入文件中的列分隔符。默认为 \t - 如果是不可见字符，则需要加\\x作为前缀，使用十六进制来表示分隔符。 - 如hive文件的分隔符\x01，指定为"\\x01" - - file_type： - - 用于指定导入文件的类型，例如：parquet、orc、csv。默认值通过文件后缀名判断。 - - column_list： - - 用于指定导入文件中的列和 table 中的列的对应关系。 - 当需要跳过导入文件中的某一列时，将该列指定为 table 中不存在的列名即可。 - 语法： - (col_name1, col_name2, ...) - - columns_from_path: - - 用于指定需要从文件路径中解析的字段。 - 语法： - (col_from_path_name1, col_from_path_name2, ...) - - SET: - - 如果指定此参数，可以将源文件某一列按照函数进行转化，然后将转化后的结果导入到table中。 - 目前支持的函数有： - - strftime(fmt, column) 日期转换函数 - fmt: 日期格式，形如%Y%m%d%H%M%S (年月日时分秒) - column: column_list中的列，即输入文件中的列。存储内容应为数字型的时间戳。 - 如果没有column_list，则按照palo表的列顺序默认输入文件的列。 - - time_format(output_fmt, input_fmt, column) 日期格式转化 - output_fmt: 转化后的日期格式，形如%Y%m%d%H%M%S (年月日时分秒) - input_fmt: 转化前column列的日期格式，形如%Y%m%d%H%M%S (年月日时分秒) - column: column_list中的列，即输入文件中的列。存储内容应为input_fmt格式的日期字符串。 - 如果没有column_list，则按照palo表的列顺序默认输入文件的列。 - - alignment_timestamp(precision, column) 将时间戳对齐到指定精度 - precision: year|month|day|hour - column: column_list中的列，即输入文件中的列。存储内容应为数字型的时间戳。 - 如果没有column_list，则按照palo表的列顺序默认输入文件的列。 - 注意：对齐精度为year、month的时候，只支持20050101~20191231范围内的时间戳。 - - default_value(value) 设置某一列导入的默认值 - 不指定则使用建表时列的默认值 - - md5sum(column1, column2, ...) 将指定的导入列的值求md5sum，返回32位16进制字符串 - - replace_value(old_value[, new_value]) 将导入文件中指定的old_value替换为new_value - new_value如不指定则使用建表时列的默认值 - - hll_hash(column) 用于将表或数据里面的某一列转化成HLL列的数据结构 - - 3. opt_properties - - 用于指定一些特殊参数。 - 语法： - [PROPERTIES ("key"="value", ...)] - - 可以指定如下参数： - cluster: 导入所使用的 Hadoop 计算队列。 - timeout： 指定导入操作的超时时间。默认超时为3天。单位秒。 - max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。 - load_delete_flag：指定该导入是否通过导入key列的方式删除数据，仅适用于UNIQUE KEY， - 导入时可不指定value列。默认为false。 - - 5. 导入数据格式样例 - - 整型类（TINYINT/SMALLINT/INT/BIGINT/LARGEINT）：1, 1000, 1234 - 浮点类（FLOAT/DOUBLE/DECIMAL）：1.1, 0.23, .356 - 日期类（DATE/DATETIME）：2017-10-03, 2017-06-13 12:34:03。 - （注：如果是其他日期格式，可以在导入命令中，使用 strftime 或者 time_format 函数进行转换） - 字符串类（CHAR/VARCHAR）："I am a student", "a" - NULL值：\N - -## example - - 1. 导入一批数据，指定超时时间和过滤比例。指定导入队列为 my_cluster。 - - LOAD LABEL example_db.label1 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - ) - PROPERTIES - ( - "cluster" = "my_cluster", - "timeout" = "3600", - "max_filter_ratio" = "0.1" - ); - - 其中 hdfs_host 为 namenode 的 host，hdfs_port 为 fs.defaultFS 端口（默认9000） - - 2. 导入一批数据，包含多个文件。导入不同的 table，指定分隔符，指定列对应关系 - - LOAD LABEL example_db.label2 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file1") - INTO TABLE `my_table_1` - COLUMNS TERMINATED BY "," - (k1, k3, k2, v1, v2), - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file2") - INTO TABLE `my_table_2` - COLUMNS TERMINATED BY "\t" - (k1, k2, k3, v2, v1) - ); - - 3. 导入一批数据，指定hive的默认分隔符\x01，并使用通配符*指定目录下的所有文件 - - LOAD LABEL example_db.label3 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/*") - NEGATIVE - INTO TABLE `my_table` - COLUMNS TERMINATED BY "\\x01" - ); - - 4. 导入一批“负”数据 - - LOAD LABEL example_db.label4 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/old_file) - NEGATIVE - INTO TABLE `my_table` - COLUMNS TERMINATED BY "\t" - ); - - 5. 导入一批数据，指定分区 - - LOAD LABEL example_db.label5 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (k1, k3, k2, v1, v2) - ); - - 6. 导入一批数据，指定分区, 并对导入文件的列做一些转化，如下： - 表结构为： - k1 datetime - k2 date - k3 bigint - k4 varchar(20) - k5 varchar(64) - k6 int - - 假设数据文件只有一行数据，5列，逗号分隔： - - 1537002087,2018-08-09 11:12:13,1537002087,-,1 - - 数据文件中各列，对应导入语句中指定的各列： - tmp_k1, tmp_k2, tmp_k3, k6, v1 - - 转换如下： - - 1) k1：将 tmp_k1 时间戳列转化为 datetime 类型的数据 - 2) k2：将 tmp_k2 datetime 类型的数据转化为 date 的数据 - 3) k3：将 tmp_k3 时间戳列转化为天级别时间戳 - 4) k4：指定导入默认值为1 - 5) k5：将 tmp_k1、tmp_k2、tmp_k3 列计算 md5 值 - 6) k6：将导入文件中的 - 值替换为 10 - - LOAD LABEL example_db.label6 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (tmp_k1, tmp_k2, tmp_k3, k6, v1) - SET ( - k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1), - k2 = time_format("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", tmp_k2), - k3 = alignment_timestamp("day", tmp_k3), - k4 = default_value("1"), - k5 = md5sum(tmp_k1, tmp_k2, tmp_k3), - k6 = replace_value("-", "10") - ) - ); - - 7. 导入数据到含有HLL列的表，可以是表中的列或者数据里面的列 - - LOAD LABEL example_db.label7 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - SET ( - v1 = hll_hash(k1), - v2 = hll_hash(k2) - ) - ); - -## keyword - LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md deleted file mode 100644 index f9d3ccd19ebe54..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md +++ /dev/null @@ -1,132 +0,0 @@ - - -# MINI LOAD -## description - - MINI LOAD 和 STREAM LOAD 的导入实现方式完全一致。在导入功能支持上，MINI LOAD 的功能是 STREAM LOAD 的子集。 - 后续的导入新功能只会在 STREAM LOAD 中支持，MINI LOAD 将不再新增功能。建议改用 STREAM LOAD，具体使用方式请 HELP STREAM LOAD。 - - MINI LOAD 是 通过 http 协议完成的导入方式。用户可以不依赖 Hadoop，也无需通过 Mysql 客户端，即可完成导入。 - 用户通过 http 协议描述导入，数据在接受 http 请求的过程中被流式的导入 Doris , **导入作业完成后** 返回给用户导入的结果。 - - * 注：为兼容旧版本 mini load 使用习惯，用户依旧可以通过 'SHOW LOAD' 命令来查看导入结果。 - - 语法： - 导入： - - curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table}/_load?label=xxx - - 查看导入信息 - - curl -u user:passwd http://host:port/api/{db}/_load_info?label=xxx - - HTTP协议相关说明 - - 权限认证 当前 Doris 使用 http 的 Basic 方式权限认证。所以在导入的时候需要指定用户名密码 - 这种方式是明文传递密码的，暂不支持加密传输。 - - Expect Doris 需要发送过来的 http 请求带有 'Expect' 头部信息，内容为 '100-continue'。 - 为什么呢？因为我们需要将请求进行 redirect，那么必须在传输数据内容之前， - 这样可以避免造成数据的多次传输，从而提高效率。 - - Content-Length Doris 需要在发送请求时带有 'Content-Length' 这个头部信息。如果发送的内容比 - 'Content-Length' 要少，那么 Doris 认为传输出现问题，则提交此次任务失败。 - NOTE: 如果，发送的数据比 'Content-Length' 要多，那么 Doris 只读取 'Content-Length' - 长度的内容，并进行导入 - - - 参数说明： - - user: 用户如果是在default_cluster中的，user即为user_name。否则为user_name@cluster_name。 - - label: 用于指定这一批次导入的 label，用于后期进行作业查询等。 - 这个参数是必须传入的。 - - columns: 用于描述导入文件中对应的列名字。 - 如果不传入，那么认为文件中的列顺序与建表的顺序一致， - 指定的方式为逗号分隔，例如：columns=k1,k2,k3,k4 - - column_separator: 用于指定列与列之间的分隔符，默认的为'\t' - NOTE: 需要进行url编码，譬如 - 需要指定'\t'为分隔符，那么应该传入'column_separator=%09' - 需要指定'\x01'为分隔符，那么应该传入'column_separator=%01' - 需要指定','为分隔符，那么应该传入'column_separator=%2c' - - - max_filter_ratio: 用于指定允许过滤不规范数据的最大比例，默认是0，不允许过滤 - 自定义指定应该如下：'max_filter_ratio=0.2'，含义是允许20%的错误率 - - timeout: 指定 load 作业的超时时间，单位是秒。当load执行时间超过该阈值时，会自动取消。默认超时时间是 600 秒。 - 建议指定 timeout 时间小于 86400 秒。 - - hll: 用于指定数据里面和表里面的HLL列的对应关系，表中的列和数据里面指定的列 - （如果不指定columns，则数据列面的列也可以是表里面的其它非HLL列）通过","分割 - 指定多个hll列使用“:”分割，例如: 'hll1,cuid:hll2,device' - - strict_mode: 指定当前导入是否使用严格模式，默认为 true。严格模式下，非空原始数据在列类型转化后结果为 NULL 的会被过滤。 - 指定方式为 'strict_mode=false' - - NOTE: - 1. 此种导入方式当前是在一台机器上完成导入工作，因而不宜进行数据量较大的导入工作。 - 建议导入数据量不要超过 1 GB - - 2. 当前无法使用 `curl -T "{file1, file2}"` 这样的方式提交多个文件，因为curl是将其拆成多个 - 请求发送的，多个请求不能共用一个label号，所以无法使用 - - 3. mini load 的导入方式和 streaming 完全一致，都是在流式的完成导入后，同步的返回结果给用户。 - 后续查询虽可以查到 mini load 的信息，但不能对其进行操作，查询只为兼容旧的使用方式。 - - 4. 当使用 curl 命令行导入时，需要在 & 前加入 \ 转义，否则参数信息会丢失。 - -## example - - 1. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表（用户是defalut_cluster中的） - curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123 - - 2. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表（用户是test_cluster中的）。超时时间是 3600 秒 - curl --location-trusted -u root@test_cluster:root -T testData http://fe.host:port/api/testDb/testTbl/_load?label=123\&timeout=3600 - - 3. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率（用户是defalut_cluster中的） - curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 - - 4. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率，并且指定文件的列名（用户是defalut_cluster中的） - curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2\&columns=k1,k2,k3 - - 5. 使用streaming方式导入（用户是defalut_cluster中的） - seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_load?label=123 - - 6. 导入含有HLL列的表，可以是表中的列或者数据中的列用于生成HLL列（用户是defalut_cluster中的 - - curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 - \&columns=k1,k2,k3\&hll=hll_column1,k1:hll_column2,k2 - - curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 - \&hll=hll_column1,tmp_k4:hll_column2,tmp_k5\&columns=k1,k2,k3,tmp_k4,tmp_k5 - - 7. 查看提交后的导入情况 - - curl -u root http://host:port/api/testDb/_load_info?label=123 - - 8. 指定非严格模式导入 - curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&strict_mode=false - -## keyword - MINI, LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md deleted file mode 100644 index cc1bb0c2bcca99..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md +++ /dev/null @@ -1,101 +0,0 @@ - - -# MULTI LOAD -## description - - Syntax: - curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_start?label=xxx - curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table1}/_load?label=xxx\&sub_label=yyy - curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table2}/_load?label=xxx\&sub_label=zzz - curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_commit?label=xxx - curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_desc?label=xxx - - 'MULTI LOAD'在'MINI LOAD'的基础上，可以支持用户同时向多个表进行导入，具体的命令如上面所示 - '/api/{db}/_multi_start' 开始一个多表导入任务 - '/api/{db}/{table}/_load' 向一个导入任务添加一个要导入的表，与'MINI LOAD'的主要区别是，需要传入'sub_label'参数 - '/api/{db}/_multi_commit' 提交整个多表导入任务，后台开始进行处理 - '/api/{db}/_multi_abort' 放弃一个多表导入任务 - '/api/{db}/_multi_desc' 可以展示某个多表导入任务已经提交的作业数 - - HTTP协议相关说明 - 权限认证 当前 Doris 使用http的Basic方式权限认证。所以在导入的时候需要指定用户名密码 - 这种方式是明文传递密码的，鉴于我们当前都是内网环境。。。 - - Expect Doris 需要发送过来的http请求，需要有'Expect'头部信息，内容为'100-continue' - 为什么呢？因为我们需要将请求进行redirect，那么必须在传输数据内容之前， - 这样可以避免造成数据的多次传输，从而提高效率。 - - Content-Length Doris 需要在发送请求是带有'Content-Length'这个头部信息。如果发送的内容比 - 'Content-Length'要少，那么Palo认为传输出现问题，则提交此次任务失败。 - NOTE: 如果，发送的数据比'Content-Length'要多，那么 Doris 只读取'Content-Length' - 长度的内容，并进行导入 - - 参数说明： - user: 用户如果是在default_cluster中的，user即为user_name。否则为user_name@cluster_name。 - - label: 用于指定这一批次导入的label号，用于后期进行作业状态查询等。 - 这个参数是必须传入的。 - - sub_label: 用于指定一个多表导入任务内部的子版本号。对于多表导入的load， 这个参数是必须传入的。 - - columns: 用于描述导入文件中对应的列名字。 - 如果不传入，那么认为文件中的列顺序与建表的顺序一致， - 指定的方式为逗号分隔，例如：columns=k1,k2,k3,k4 - - column_separator: 用于指定列与列之间的分隔符，默认的为'\t' - NOTE: 需要进行url编码，譬如需要指定'\t'为分隔符， - 那么应该传入'column_separator=%09' - - max_filter_ratio: 用于指定允许过滤不规范数据的最大比例，默认是0，不允许过滤 - 自定义指定应该如下：'max_filter_ratio=0.2'，含义是允许20%的错误率 - 在'_multi_start'时传入有效果 - - NOTE: - 1. 此种导入方式当前是在一台机器上完成导入工作，因而不宜进行数据量较大的导入工作。 - 建议导入数据量不要超过1GB - - 2. 当前无法使用`curl -T "{file1, file2}"`这样的方式提交多个文件，因为curl是将其拆成多个 - 请求发送的，多个请求不能共用一个label号，所以无法使用 - - 3. 支持类似streaming的方式使用curl来向 Doris 中导入数据，但是，只有等这个streaming结束后 Doris - 才会发生真实的导入行为，这中方式数据量也不能过大。 - -## example - - 1. 将本地文件'testData1'中的数据导入到数据库'testDb'中'testTbl1'的表，并且 - 把'testData2'的数据导入到'testDb'中的表'testTbl2'(用户是defalut_cluster中的) - curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 - curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 - curl --location-trusted -u root -T testData2 http://host:port/api/testDb/testTbl2/_load?label=123\&sub_label=2 - curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_commit?label=123 - - 2. 多表导入中途放弃(用户是defalut_cluster中的) - curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 - curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 - curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_abort?label=123 - - 3. 多表导入查看已经提交多少内容(用户是defalut_cluster中的) - curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 - curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 - curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_desc?label=123 - -## keyword - MULTI, MINI, LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md deleted file mode 100644 index b70a415ec5ba16..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# PAUSE ROUTINE LOAD -## example - -1. 暂停名称为 test1 的例行导入作业。 - - PAUSE ROUTINE LOAD FOR test1; - -## keyword - PAUSE,ROUTINE,LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md deleted file mode 100644 index d7b762212c6783..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# RESUME ROUTINE LOAD -## example - -1. 恢复名称为 test1 的例行导入作业。 - - RESUME ROUTINE LOAD FOR test1; - -## keyword - RESUME,ROUTINE,LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md deleted file mode 100644 index c7a4f7997f3b52..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md +++ /dev/null @@ -1,309 +0,0 @@ - - -# ROUTINE LOAD -## description - - 例行导入（Routine Load）功能，支持用户提交一个常驻的导入任务，通过不断的从指定的数据源读取数据，将数据导入到 Doris 中。 - 目前仅支持通过无认证或者 SSL 认证方式，从 Kakfa 导入文本格式（CSV）的数据。 - -语法： - - CREATE ROUTINE LOAD [db.]job_name ON tbl_name - [load_properties] - [job_properties] - FROM data_source - [data_source_properties] - - 1. [db.]job_name - - 导入作业的名称，在同一个 database 内，相同名称只能有一个 job 在运行。 - - 2. tbl_name - - 指定需要导入的表的名称。 - - 3. load_properties - - 用于描述导入数据。语法： - - [column_separator], - [columns_mapping], - [where_predicates], - [partitions] - - 1. column_separator: - - 指定列分隔符，如： - - COLUMNS TERMINATED BY "," - - 默认为：\t - - 2. columns_mapping: - - 指定源数据中列的映射关系，以及定义衍生列的生成方式。 - - 1. 映射列： - - 按顺序指定，源数据中各个列，对应目的表中的哪些列。对于希望跳过的列，可以指定一个不存在的列名。 - 假设目的表有三列 k1, k2, v1。源数据有4列，其中第1、2、4列分别对应 k2, k1, v1。则书写如下： - - COLUMNS (k2, k1, xxx, v1) - - 其中 xxx 为不存在的一列，用于跳过源数据中的第三列。 - - 2. 衍生列： - - 以 col_name = expr 的形式表示的列，我们称为衍生列。即支持通过 expr 计算得出目的表中对应列的值。 - 衍生列通常排列在映射列之后，虽然这不是强制的规定，但是 Doris 总是先解析映射列，再解析衍生列。 - 接上一个示例，假设目的表还有第4列 v2，v2 由 k1 和 k2 的和产生。则可以书写如下： - - COLUMNS (k2, k1, xxx, v1, v2 = k1 + k2); - - 3. where_predicates - - 用于指定过滤条件，以过滤掉不需要的列。过滤列可以是映射列或衍生列。 - 例如我们只希望导入 k1 大于 100 并且 k2 等于 1000 的列，则书写如下： - - WHERE k1 > 100 and k2 = 1000 - - 4. partitions - - 指定导入目的表的哪些 partition 中。如果不指定，则会自动导入到对应的 partition 中。 - 示例： - - PARTITION(p1, p2, p3) - - 4. job_properties - - 用于指定例行导入作业的通用参数。 - 语法： - - PROPERTIES ( - "key1" = "val1", - "key2" = "val2" - ) - - 目前我们支持以下参数： - - 1. desired_concurrent_number - - 期望的并发度。一个例行导入作业会被分成多个子任务执行。这个参数指定一个作业最多有多少任务可以同时执行。必须大于0。默认为3。 - 这个并发度并不是实际的并发度，实际的并发度，会通过集群的节点数、负载情况，以及数据源的情况综合考虑。 - 例： - - "desired_concurrent_number" = "3" - - 2. max_batch_interval/max_batch_rows/max_batch_size - - 这三个参数分别表示： - 1）每个子任务最大执行时间，单位是秒。范围为 5 到 60。默认为10。 - 2）每个子任务最多读取的行数。必须大于等于200000。默认是200000。 - 3）每个子任务最多读取的字节数。单位是字节，范围是 100MB 到 1GB。默认是 100MB。 - - 这三个参数，用于控制一个子任务的执行时间和处理量。当任意一个达到阈值，则任务结束。 - 例： - - "max_batch_interval" = "20", - "max_batch_rows" = "300000", - "max_batch_size" = "209715200" - - 3. max_error_number - - 采样窗口内，允许的最大错误行数。必须大于等于0。默认是 0，即不允许有错误行。 - 采样窗口为 max_batch_rows * 10。即如果在采样窗口内，错误行数大于 max_error_number，则会导致例行作业被暂停，需要人工介入检查数据质量问题。 - 被 where 条件过滤掉的行不算错误行。 - - 4. strict_mode - - 是否开启严格模式，默认为开启。如果开启后，非空原始数据的列类型变换如果结果为 NULL，则会被过滤。指定方式为 "strict_mode" = "true" - - 5. timezone - - 指定导入作业所使用的时区。默认为使用 Session 的 timezone 参数。该参数会影响所有导入涉及的和时区有关的函数结果。 - - 5. data_source - - 数据源的类型。当前支持： - - KAFKA - - 6. data_source_properties - - 指定数据源相关的信息。 - 语法： - - ( - "key1" = "val1", - "key2" = "val2" - ) - - 1. KAFKA 数据源 - - 1. kafka_broker_list - - Kafka 的 broker 连接信息。格式为 ip:host。多个broker之间以逗号分隔。 - 示例： - - "kafka_broker_list" = "broker1:9092,broker2:9092" - - 2. kafka_topic - - 指定要订阅的 Kafka 的 topic。 - 示例： - - "kafka_topic" = "my_topic" - - 3. kafka_partitions/kafka_offsets - - 指定需要订阅的 kafka partition，以及对应的每个 partition 的起始 offset。 - - offset 可以指定从大于等于 0 的具体 offset，或者： - 1) OFFSET_BEGINNING: 从有数据的位置开始订阅。 - 2) OFFSET_END: 从末尾开始订阅。 - - 如果没有指定，则默认从 OFFSET_END 开始订阅 topic 下的所有 partition。 - 示例： - - "kafka_partitions" = "0,1,2,3", - "kafka_offsets" = "101,0,OFFSET_BEGINNING,OFFSET_END" - - 4. property - - 指定自定义kafka参数。 - 功能等同于kafka shell中 "--property" 参数。 - 当参数的 value 为一个文件时，需要在 value 前加上关键词："FILE:"。 - 关于如何创建文件，请参阅 "HELP CREATE FILE;" - 更多支持的自定义参数，请参阅 librdkafka 的官方 CONFIGURATION 文档中，client 端的配置项。 - - 示例: - "property.client.id" = "12345", - "property.ssl.ca.location" = "FILE:ca.pem" - - 1.使用 SSL 连接 Kafka 时，需要指定以下参数： - - "property.security.protocol" = "ssl", - "property.ssl.ca.location" = "FILE:ca.pem", - "property.ssl.certificate.location" = "FILE:client.pem", - "property.ssl.key.location" = "FILE:client.key", - "property.ssl.key.password" = "abcdefg" - - 其中： - "property.security.protocol" 和 "property.ssl.ca.location" 为必须，用于指明连接方式为 SSL，以及 CA 证书的位置。 - - 如果 Kafka server 端开启了 client 认证，则还需设置： - - "property.ssl.certificate.location" - "property.ssl.key.location" - "property.ssl.key.password" - - 分别用于指定 client 的 public key，private key 以及 private key 的密码。 - - - 2.指定kafka partition的默认起始offset - 如果没有指定kafka_partitions/kafka_offsets,默认消费所有分区,此时可以指定kafka_default_offsets指定起始 offset。默认为 OFFSET_END，即从末尾开始订阅。 - 值为 - 1) OFFSET_BEGINNING: 从有数据的位置开始订阅。 - 2) OFFSET_END: 从末尾开始订阅。 - 示例： - "property.kafka_default_offsets" = "OFFSET_BEGINNING" - - 7. 导入数据格式样例 - - 整型类（TINYINT/SMALLINT/INT/BIGINT/LARGEINT）：1, 1000, 1234 - 浮点类（FLOAT/DOUBLE/DECIMAL）：1.1, 0.23, .356 - 日期类（DATE/DATETIME）：2017-10-03, 2017-06-13 12:34:03。 - 字符串类（CHAR/VARCHAR）（无引号）：I am a student, a - NULL值：\N - -## example - 1. 为 example_db 的 example_tbl 创建一个名为 test1 的 Kafka 例行导入任务。指定列分隔符和 group.id 和 client.id，并且自动默认消费所有分区，且从有数据的位置（OFFSET_BEGINNING）开始订阅 - - CREATE ROUTINE LOAD example_db.test1 ON example_tbl - COLUMNS TERMINATED BY ",", - COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100) - PROPERTIES - ( - "desired_concurrent_number"="3", - "max_batch_interval" = "20", - "max_batch_rows" = "300000", - "max_batch_size" = "209715200", - "strict_mode" = "false" - ) - FROM KAFKA - ( - "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", - "kafka_topic" = "my_topic", - "property.group.id" = "xxx", - "property.client.id" = "xxx", - "property.kafka_default_offsets" = "OFFSET_BEGINNING" - ); - - 2. 为 example_db 的 example_tbl 创建一个名为 test1 的 Kafka 例行导入任务。导入任务为严格模式。 - - CREATE ROUTINE LOAD example_db.test1 ON example_tbl - COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), - WHERE k1 > 100 and k2 like "%doris%" - PROPERTIES - ( - "desired_concurrent_number"="3", - "max_batch_interval" = "20", - "max_batch_rows" = "300000", - "max_batch_size" = "209715200", - "strict_mode" = "false" - ) - FROM KAFKA - ( - "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", - "kafka_topic" = "my_topic", - "kafka_partitions" = "0,1,2,3", - "kafka_offsets" = "101,0,0,200" - ); - - 3. 通过 SSL 认证方式，从 Kafka 集群导入数据。同时设置 client.id 参数。导入任务为非严格模式，时区为 Africa/Abidjan - - CREATE ROUTINE LOAD example_db.test1 ON example_tbl - COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), - WHERE k1 > 100 and k2 like "%doris%" - PROPERTIES - ( - "desired_concurrent_number"="3", - "max_batch_interval" = "20", - "max_batch_rows" = "300000", - "max_batch_size" = "209715200", - "strict_mode" = "false", - "timezone" = "Africa/Abidjan" - ) - FROM KAFKA - ( - "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", - "kafka_topic" = "my_topic", - "property.security.protocol" = "ssl", - "property.ssl.ca.location" = "FILE:ca.pem", - "property.ssl.certificate.location" = "FILE:client.pem", - "property.ssl.key.location" = "FILE:client.key", - "property.ssl.key.password" = "abcdefg", - "property.client.id" = "my_client_id" - ); - -## keyword - - CREATE,ROUTINE,LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md deleted file mode 100644 index e073651b469d37..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# SHOW ALTER -## description - 该语句用于展示当前正在进行的各类修改任务的执行情况 - 语法： - SHOW ALTER [CLUSTER | TABLE [COLUMN | ROLLUP] [FROM db_name]]; - - 说明： - TABLE COLUMN：展示修改列的 ALTER 任务 - 支持语法[WHERE TableName|CreateTime|FinishTime|State] [ORDER BY] [LIMIT] - TABLE ROLLUP：展示创建或删除 ROLLUP index 的任务 - 如果不指定 db_name，使用当前默认 db - CLUSTER: 展示集群操作相关任务情况（仅管理员使用！待实现...） - -## example - 1. 展示默认 db 的所有修改列的任务执行情况 - SHOW ALTER TABLE COLUMN; - - 2. 展示某个表最近一次修改列的任务执行情况 - SHOW ALTER TABLE COLUMN WHERE TableName = "table1" ORDER BY CreateTime DESC LIMIT 1; - - 3. 展示指定 db 的创建或删除 ROLLUP index 的任务执行情况 - SHOW ALTER TABLE ROLLUP FROM example_db; - - 4. 展示集群操作相关任务（仅管理员使用！待实现...） - SHOW ALTER CLUSTER; - -## keyword - SHOW,ALTER - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md deleted file mode 100644 index b10e865706e4bb..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# SHOW BACKUP -## description - 该语句用于查看 BACKUP 任务 - 语法： - SHOW BACKUP [FROM db_name] - - 说明： - 1. Palo 中仅保存最近一次 BACKUP 任务。 - 2. 各列含义如下： - JobId： 唯一作业id - SnapshotName： 备份的名称 - DbName： 所属数据库 - State： 当前阶段 - PENDING： 提交作业后的初始状态 - SNAPSHOTING： 执行快照中 - UPLOAD_SNAPSHOT：快照完成，准备上传 - UPLOADING： 快照上传中 - SAVE_META： 将作业元信息保存为本地文件 - UPLOAD_INFO： 上传作业元信息 - FINISHED： 作业成功 - CANCELLED： 作业失败 - BackupObjs： 备份的表和分区 - CreateTime： 任务提交时间 - SnapshotFinishedTime： 快照完成时间 - UploadFinishedTime： 快照上传完成时间 - FinishedTime： 作业结束时间 - UnfinishedTasks： 在 SNAPSHOTING 和 UPLOADING 阶段会显示还未完成的子任务id - Status： 如果作业失败，显示失败信息 - Timeout： 作业超时时间，单位秒 - -## example - 1. 查看 example_db 下最后一次 BACKUP 任务。 - SHOW BACKUP FROM example_db; - -## keyword - SHOW, BACKUP - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md deleted file mode 100644 index 74f7e2ea14436c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# SHOW DATA -## description - 该语句用于展示数据量和副本数量 - 语法： - SHOW DATA [FROM db_name[.table_name]]; - - 说明： - 1. 如果不指定 FROM 子句，使用展示当前 db 下细分到各个 table 的数据量和副本数量 - 2. 如果指定 FROM 子句，则展示 table 下细分到各个 index 的数据量和副本数量 - 3. 如果想查看各个 Partition 的大小，请参阅 help show partitions - -## example - 1. 展示默认 db 的各个 table 的数据量，副本数量，汇总数据量和汇总副本数量。 - SHOW DATA; - - 2. 展示指定 db 的下指定表的细分数据量和副本数量 - SHOW DATA FROM example_db.table_name; - -## keyword - SHOW,DATA - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md deleted file mode 100644 index 823dd833a63e04..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# SHOW DATABASES -## description - 该语句用于展示当前可见的 db - 语法： - SHOW DATABASES; - -## keyword - SHOW,DATABASES - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md deleted file mode 100644 index 023bed004f0f63..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# SHOW DELETE -## description - 该语句用于展示已执行成功的历史 delete 任务 - 语法： - SHOW DELETE [FROM db_name] - -## example - 1. 展示数据库 database 的所有历史 delete 任务 - SHOW DELETE FROM database; - -## keyword - SHOW,DELETE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md deleted file mode 100644 index f209f55c13cc6d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# SHOW DYNAMIC PARTITION TABLES -## description - 该语句用于展示当前db下所有的动态分区表状态 - 语法： - SHOW DYNAMIC PARTITION TABLES [FROM db_name]; - -## example - 1. 展示数据库 database 的所有动态分区表状态 - SHOW DYNAMIC PARTITION TABLES FROM database; - -## keyword - SHOW,DYNAMIC,PARTITION,TABLES - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md deleted file mode 100644 index f91f1b03d7153e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# SHOW EXPORT -## description - 该语句用于展示指定的导出任务的执行情况 - 语法： - SHOW EXPORT - [FROM db_name] - [ - WHERE - [EXPORT_JOB_ID = your_job_id] - [STATE = ["PENDING"|"EXPORTING"|"FINISHED"|"CANCELLED"]] - ] - [ORDER BY ...] - [LIMIT limit]; - - 说明： - 1) 如果不指定 db_name，使用当前默认db - 2) 如果指定了 STATE，则匹配 EXPORT 状态 - 3) 可以使用 ORDER BY 对任意列组合进行排序 - 4) 如果指定了 LIMIT，则显示 limit 条匹配记录。否则全部显示 - -## example - 1. 展示默认 db 的所有导出任务 - SHOW EXPORT; - - 2. 展示指定 db 的导出任务，按 StartTime 降序排序 - SHOW EXPORT FROM example_db ORDER BY StartTime DESC; - - 3. 展示指定 db 的导出任务，state 为 "exporting", 并按 StartTime 降序排序 - SHOW EXPORT FROM example_db WHERE STATE = "exporting" ORDER BY StartTime DESC; - - 4. 展示指定db，指定job_id的导出任务 - SHOW EXPORT FROM example_db WHERE EXPORT_JOB_ID = job_id; - -## keyword - SHOW,EXPORT - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md deleted file mode 100644 index 7997a3a861d537..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md +++ /dev/null @@ -1,68 +0,0 @@ - - -# SHOW LOAD -## description - 该语句用于展示指定的导入任务的执行情况 - 语法： - SHOW LOAD - [FROM db_name] - [ - WHERE - [LABEL [ = "your_label" | LIKE "label_matcher"]] - [STATE = ["PENDING"|"ETL"|"LOADING"|"FINISHED"|"CANCELLED"|]] - ] - [ORDER BY ...] - [LIMIT limit][OFFSET offset]; - - 说明： - 1) 如果不指定 db_name，使用当前默认db - 2) 如果使用 LABEL LIKE，则会匹配导入任务的 label 包含 label_matcher 的导入任务 - 3) 如果使用 LABEL = ，则精确匹配指定的 label - 4) 如果指定了 STATE，则匹配 LOAD 状态 - 5) 可以使用 ORDER BY 对任意列组合进行排序 - 6) 如果指定了 LIMIT，则显示 limit 条匹配记录。否则全部显示 - 7) 如果指定了 OFFSET，则从偏移量offset开始显示查询结果。默认情况下偏移量为0。 - 8) 如果是使用 broker/mini load，则 URL 列中的连接可以使用以下命令查看： - - SHOW LOAD WARNINGS ON 'url' - -## example - 1. 展示默认 db 的所有导入任务 - SHOW LOAD; - - 2. 展示指定 db 的导入任务，label 中包含字符串 "2014_01_02"，展示最老的10个 - SHOW LOAD FROM example_db WHERE LABEL LIKE "2014_01_02" LIMIT 10; - - 3. 展示指定 db 的导入任务，指定 label 为 "load_example_db_20140102" 并按 LoadStartTime 降序排序 - SHOW LOAD FROM example_db WHERE LABEL = "load_example_db_20140102" ORDER BY LoadStartTime DESC; - - 4. 展示指定 db 的导入任务，指定 label 为 "load_example_db_20140102" ，state 为 "loading", 并按 LoadStartTime 降序排序 - SHOW LOAD FROM example_db WHERE LABEL = "load_example_db_20140102" AND STATE = "loading" ORDER BY LoadStartTime DESC; - - 5. 展示指定 db 的导入任务 并按 LoadStartTime 降序排序,并从偏移量5开始显示10条查询结果 - SHOW LOAD FROM example_db ORDER BY LoadStartTime DESC limit 5,10; - SHOW LOAD FROM example_db ORDER BY LoadStartTime DESC limit 10 offset 5; - - 6. 小批量导入是查看导入状态的命令 - curl --location-trusted -u {user}:{passwd} http://{hostname}:{port}/api/{database}/_load_info?label={labelname} - -## keyword - SHOW,LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md deleted file mode 100644 index 4e2b8552ecd69f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# SHOW PARTITIONS -## description - 该语句用于展示分区信息 - 语法： - SHOW PARTITIONS FROM [db_name.]table_name [WHERE] [ORDER BY] [LIMIT]; - 说明: - 支持PartitionId,PartitionName,State,Buckets,ReplicationNum,LastConsistencyCheckTime等列的过滤 - -## example - 1.展示指定db下指定表的所有分区信息 - SHOW PARTITIONS FROM example_db.table_name; - - 2.展示指定db下指定表的指定分区的信息 - SHOW PARTITIONS FROM example_db.table_name WHERE PartitionName = "p1"; - - 3.展示指定db下指定表的最新分区的信息 - SHOW PARTITIONS FROM example_db.table_name ORDER BY PartitionId DESC LIMIT 1; -## keyword - SHOW,PARTITIONS diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md deleted file mode 100644 index 038e33033de190..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# SHOW PROPERTY -## description - 该语句用于查看用户的属性 - 语法： - SHOW PROPERTY [FOR user] [LIKE key] - -## example - 1. 查看 jack 用户的属性 - SHOW PROPERTY FOR 'jack' - - 2. 查看 jack 用户导入cluster相关属性 - SHOW PROPERTY FOR 'jack' LIKE '%load_cluster%' - -## keyword - SHOW, PROPERTY - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md deleted file mode 100644 index fe1f9e1f8fbd01..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# SHOW REPOSITORIES -## description - 该语句用于查看当前已创建的仓库。 - 语法： - SHOW REPOSITORIES; - - 说明： - 1. 各列含义如下： - RepoId： 唯一的仓库ID - RepoName： 仓库名称 - CreateTime： 第一次创建该仓库的时间 - IsReadOnly： 是否为只读仓库 - Location： 仓库中用于备份数据的根目录 - Broker： 依赖的 Broker - ErrMsg： Palo 会定期检查仓库的连通性，如果出现问题，这里会显示错误信息 - -## example - 1. 查看已创建的仓库： - SHOW REPOSITORIES; - -## keyword - SHOW, REPOSITORY, REPOSITORIES - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md deleted file mode 100644 index 3ce61cae35d4b5..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md +++ /dev/null @@ -1,60 +0,0 @@ - - -# SHOW RESTORE -## description - 该语句用于查看 RESTORE 任务 - 语法： - SHOW RESTORE [FROM db_name] - - 说明： - 1. Palo 中仅保存最近一次 RESTORE 任务。 - 2. 各列含义如下： - JobId： 唯一作业id - Label： 要恢复的备份的名称 - Timestamp： 要恢复的备份的时间版本 - DbName： 所属数据库 - State： 当前阶段 - PENDING： 提交作业后的初始状态 - SNAPSHOTING： 执行快照中 - DOWNLOAD： 快照完成，准备下载仓库中的快照 - DOWNLOADING： 快照下载中 - COMMIT： 快照下载完成，准备生效 - COMMITING： 生效中 - FINISHED： 作业成功 - CANCELLED： 作业失败 - AllowLoad： 恢复时是否允许导入（当前不支持） - ReplicationNum： 指定恢复的副本数 - RestoreJobs： 要恢复的表和分区 - CreateTime： 任务提交时间 - MetaPreparedTime： 元数据准备完成时间 - SnapshotFinishedTime： 快照完成时间 - DownloadFinishedTime： 快照下载完成时间 - FinishedTime： 作业结束时间 - UnfinishedTasks： 在 SNAPSHOTING、DOWNLOADING 和 COMMITING 阶段会显示还未完成的子任务id - Status： 如果作业失败，显示失败信息 - Timeout： 作业超时时间，单位秒 - -## example - 1. 查看 example_db 下最近一次 RESTORE 任务。 - SHOW RESTORE FROM example_db; - -## keyword - SHOW, RESTORE - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md deleted file mode 100644 index 79b1cf711c10b8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# SHOW ROUTINE LOAD TASK -## example - -1. 展示名为 test1 的例行导入任务的子任务信息。 - - SHOW ROUTINE LOAD TASK WHERE JobName = "test1"; - -## keyword - SHOW,ROUTINE,LOAD,TASK diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md deleted file mode 100644 index bb8a1b8a6cb319..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md +++ /dev/null @@ -1,51 +0,0 @@ - - -# SHOW ROUTINE LOAD -## example - -1. 展示名称为 test1 的所有例行导入作业（包括已停止或取消的作业）。结果为一行或多行。 - - SHOW ALL ROUTINE LOAD FOR test1; - -2. 展示名称为 test1 的当前正在运行的例行导入作业 - - SHOW ROUTINE LOAD FOR test1; - -3. 显示 example_db 下，所有的例行导入作业（包括已停止或取消的作业）。结果为一行或多行。 - - use example_db; - SHOW ALL ROUTINE LOAD; - -4. 显示 example_db 下，所有正在运行的例行导入作业 - - use example_db; - SHOW ROUTINE LOAD; - -5. 显示 example_db 下，名称为 test1 的当前正在运行的例行导入作业 - - SHOW ROUTINE LOAD FOR example_db.test1; - -6. 显示 example_db 下，名称为 test1 的所有例行导入作业（包括已停止或取消的作业）。结果为一行或多行。 - - SHOW ALL ROUTINE LOAD FOR example_db.test1; - -## keyword - SHOW,ROUTINE,LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md deleted file mode 100644 index d85eed623f210d..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# SHOW SNAPSHOT -## description - 该语句用于查看仓库中已存在的备份。 - 语法： - SHOW SNAPSHOT ON `repo_name` - [WHERE SNAPSHOT = "snapshot" [AND TIMESTAMP = "backup_timestamp"]]; - - 说明： - 1. 各列含义如下： - Snapshot： 备份的名称 - Timestamp： 对应备份的时间版本 - Status： 如果备份正常，则显示 OK，否则显示错误信息 - - 2. 如果指定了 TIMESTAMP，则会额外显示如下信息： - Database： 备份数据原属的数据库名称 - Details： 以 Json 的形式，展示整个备份的数据目录及文件结构 - -## example - 1. 查看仓库 example_repo 中已有的备份： - SHOW SNAPSHOT ON example_repo; - - 2. 仅查看仓库 example_repo 中名称为 backup1 的备份： - SHOW SNAPSHOT ON example_repo WHERE SNAPSHOT = "backup1"; - - 2. 查看仓库 example_repo 中名称为 backup1 的备份，时间版本为 "2018-05-05-15-34-26" 的详细信息： - SHOW SNAPSHOT ON example_repo - WHERE SNAPSHOT = "backup1" AND TIMESTAMP = "2018-05-05-15-34-26"; - -## keyword - SHOW, SNAPSHOT - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md deleted file mode 100644 index 6c9c2382f64a3f..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# SHOW TABLES -## description - 该语句用于展示当前 db 下所有的 table - 语法： - SHOW TABLES; - -## keyword - SHOW,TABLES - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md deleted file mode 100644 index cb51d9b6cc08f4..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md +++ /dev/null @@ -1,60 +0,0 @@ - - -# SHOW TABLET -## description - 该语句用于显示 tablet 相关的信息（仅管理员使用） - 语法： - SHOW TABLET - [FROM [db_name.]table_name | tablet_id] [partiton(partition_name_1, partition_name_1)] - [where [version=1] [and backendid=10000] [and state="NORMAL|ROLLUP|CLONE|DECOMMISSION"]] - [order by order_column] - [limit [offset,]size] - - 现在show tablet命令支持按照按照以下字段进行过滤：partition, index name, version, backendid, - state，同时支持按照任意字段进行排序，并且提供limit限制返回条数。 - -## example - 1. 显示指定 db 的下指定表所有 tablet 信息 - SHOW TABLET FROM example_db.table_name; - - // 获取partition p1和p2的tablet信息 - SHOW TABLET FROM example_db.table_name partition(p1, p2); - - // 获取10个结果 - SHOW TABLET FROM example_db.table_name limit 10; - - // 从偏移5开始获取10个结果 - SHOW TABLET FROM example_db.table_name limit 5,10; - - // 按照backendid/version/state字段进行过滤 - SHOW TABLET FROM example_db.table_name where backendid=10000 and version=1 and state="NORMAL"; - - // 按照version字段进行排序 - SHOW TABLET FROM example_db.table_name where backendid=10000 order by version; - - // 获取index名字为t1_rollup的tablet相关信息 - SHOW TABLET FROM example_db.table_name where indexname="t1_rollup"; - - 2. 显示指定 tablet id 为 10000 的 tablet 的父层级 id 信息 - SHOW TABLET 10000; - -## keyword - SHOW,TABLET,LIMIT - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md deleted file mode 100644 index 42d45e4d852124..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md +++ /dev/null @@ -1,80 +0,0 @@ - - -# SHOW TRANSACTION -## description - -该语法用于查看指定 transaction id 的事务详情。 - -语法： - -``` -SHOW TRANSACTION -[FROM db_name] -WHERE id = transaction_id; -``` - -返回结果示例： - -``` - TransactionId: 4005 - Label: insert_8d807d5d-bcdd-46eb-be6d-3fa87aa4952d - Coordinator: FE: 10.74.167.16 - TransactionStatus: VISIBLE - LoadJobSourceType: INSERT_STREAMING - PrepareTime: 2020-01-09 14:59:07 - CommitTime: 2020-01-09 14:59:09 - FinishTime: 2020-01-09 14:59:09 - Reason: -ErrorReplicasCount: 0 - ListenerId: -1 - TimeoutMs: 300000 -``` - -* TransactionId：事务id -* Label：导入任务对应的 label -* Coordinator：负责事务协调的节点 -* TransactionStatus：事务状态 - * PREPARE：准备阶段 - * COMMITTED：事务成功，但数据不可见 - * VISIBLE：事务成功且数据可见 - * ABORTED：事务失败 -* LoadJobSourceType：导入任务的类型。 -* PrepareTime：事务开始时间 -* CommitTime：事务提交成功的时间 -* FinishTime：数据可见的时间 -* Reason：错误信息 -* ErrorReplicasCount：有错误的副本数 -* ListenerId：相关的导入作业的id -* TimeoutMs：事务超时时间，单位毫秒 - -## example - -1. 查看 id 为 4005 的事务： - - SHOW TRANSACTION WHERE ID=4005; - -2. 指定 db 中，查看 id 为 4005 的事务： - - SHOW TRANSACTION FROM db WHERE ID=4005; - -## keyword - - SHOW, TRANSACTION - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md deleted file mode 100644 index f71c5cf2388455..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# STOP ROUTINE LOAD -## example - -1. 停止名称为 test1 的例行导入作业。 - - STOP ROUTINE LOAD FOR test1; - -## keyword - STOP,ROUTINE,LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md deleted file mode 100644 index cd392665f645bd..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md +++ /dev/null @@ -1,126 +0,0 @@ - - -# STREAM LOAD -## description - NAME: - stream-load: load data to table in streaming - - SYNOPSIS - curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_host:http_port/api/{db}/{table}/_stream_load - - DESCRIPTION - 该语句用于向指定的 table 导入数据，与普通Load区别是，这种导入方式是同步导入。 - 这种导入方式仍然能够保证一批导入任务的原子性，要么全部数据导入成功，要么全部失败。 - 该操作会同时更新和此 base table 相关的 rollup table 的数据。 - 这是一个同步操作，整个数据导入工作完成后返回给用户导入结果。 - 当前支持HTTP chunked与非chunked上传两种方式，对于非chunked方式，必须要有Content-Length来标示上传内容长度，这样能够保证数据的完整性。 - 另外，用户最好设置Expect Header字段内容100-continue，这样可以在某些出错场景下避免不必要的数据传输。 - - OPTIONS - 用户可以通过HTTP的Header部分来传入导入参数 - - label: 一次导入的标签，相同标签的数据无法多次导入。用户可以通过指定Label的方式来避免一份数据重复导入的问题。 - 当前Palo内部保留30分钟内最近成功的label。 - - column_separator：用于指定导入文件中的列分隔符，默认为\t。如果是不可见字符，则需要加\x作为前缀，使用十六进制来表示分隔符。 - 如hive文件的分隔符\x01，需要指定为-H "column_separator:\x01" - - columns：用于指定导入文件中的列和 table 中的列的对应关系。如果源文件中的列正好对应表中的内容，那么是不需要指定这个字段的内容的。 - 如果源文件与表schema不对应，那么需要这个字段进行一些数据转换。这里有两种形式column，一种是直接对应导入文件中的字段，直接使用字段名表示； - 一种是衍生列，语法为 `column_name` = expression。举几个例子帮助理解。 - 例1: 表中有3个列“c1, c2, c3”，源文件中的三个列一次对应的是"c3,c2,c1"; 那么需要指定-H "columns: c3, c2, c1" - 例2: 表中有3个列“c1, c2, c3", 源文件中前三列依次对应，但是有多余1列；那么需要指定-H "columns: c1, c2, c3, xxx"; - 最后一个列随意指定个名称占位即可 - 例3: 表中有3个列“year, month, day"三个列，源文件中只有一个时间列，为”2018-06-01 01:02:03“格式； - 那么可以指定-H "columns: col, year = year(col), month=month(col), day=day(col)"完成导入 - - where: 用于抽取部分数据。用户如果有需要将不需要的数据过滤掉，那么可以通过设定这个选项来达到。 - 例1: 只导入大于k1列等于20180601的数据，那么可以在导入时候指定-H "where: k1 = 20180601" - - max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。数据不规范不包括通过 where 条件过滤掉的行。 - - partitions: 用于指定这次导入所设计的partition。如果用户能够确定数据对应的partition，推荐指定该项。不满足这些分区的数据将被过滤掉。 - 比如指定导入到p1, p2分区，-H "partitions: p1, p2" - - timeout: 指定导入的超时时间。单位秒。默认是 600 秒。可设置范围为 1 秒 ~ 259200 秒。 - - strict_mode: 用户指定此次导入是否开启严格模式，默认为开启。关闭方式为 -H "strict_mode: false"。 - - timezone: 指定本次导入所使用的时区。默认为东八区。该参数会影响所有导入涉及的和时区有关的函数结果。 - - exec_mem_limit: 导入内存限制。默认为 2GB。单位为字节。 - - RETURN VALUES - 导入完成后，会以Json格式返回这次导入的相关内容。当前包括一下字段 - Status: 导入最后的状态。 - Success：表示导入成功，数据已经可见； - Publish Timeout：表述导入作业已经成功Commit，但是由于某种原因并不能立即可见。用户可以视作已经成功不必重试导入 - Label Already Exists: 表明该Label已经被其他作业占用，可能是导入成功，也可能是正在导入。 - 用户需要通过get label state命令来确定后续的操作 - 其他：此次导入失败，用户可以指定Label重试此次作业 - Message: 导入状态详细的说明。失败时会返回具体的失败原因。 - NumberTotalRows: 从数据流中读取到的总行数 - NumberLoadedRows: 此次导入的数据行数，只有在Success时有效 - NumberFilteredRows: 此次导入过滤掉的行数，即数据质量不合格的行数 - NumberUnselectedRows: 此次导入，通过 where 条件被过滤掉的行数 - LoadBytes: 此次导入的源文件数据量大小 - LoadTimeMs: 此次导入所用的时间 - ErrorURL: 被过滤数据的具体内容，仅保留前1000条 - - ERRORS - 可以通过以下语句查看导入错误详细信息： - - SHOW LOAD WARNINGS ON 'url' - - 其中 url 为 ErrorURL 给出的 url。 - -## example - - 1. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重。指定超时时间为 100 秒 - curl --location-trusted -u root -H "label:123" -H "timeout:100" -T testData http://host:port/api/testDb/testTbl/_stream_load - - 2. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重, 并且只导入k1等于20180601的数据 - curl --location-trusted -u root -H "label:123" -H "where: k1=20180601" -T testData http://host:port/api/testDb/testTbl/_stream_load - - 3. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率（用户是defalut_cluster中的） - curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -T testData http://host:port/api/testDb/testTbl/_stream_load - - 4. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率，并且指定文件的列名（用户是defalut_cluster中的） - curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "columns: k2, k1, v1" -T testData http://host:port/api/testDb/testTbl/_stream_load - - 5. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表中的p1, p2分区, 允许20%的错误率。 - curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "partitions: p1, p2" -T testData http://host:port/api/testDb/testTbl/_stream_load - - 6. 使用streaming方式导入（用户是defalut_cluster中的） - seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_stream_load - - 7. 导入含有HLL列的表，可以是表中的列或者数据中的列用于生成HLL列，也可使用hll_empty补充数据中没有的列 - curl --location-trusted -u root -H "columns: k1, k2, v1=hll_hash(k1), v2=hll_empty()" -T testData http://host:port/api/testDb/testTbl/_stream_load - - 8. 导入数据进行严格模式过滤，并设置时区为 Africa/Abidjan - curl --location-trusted -u root -H "strict_mode: true" -H "timezone: Africa/Abidjan" -T testData http://host:port/api/testDb/testTbl/_stream_load - - 9. 导入含有BITMAP列的表，可以是表中的列或者数据中的列用于生成BITMAP列，也可以使用bitmap_empty填充空的Bitmap - curl --location-trusted -u root -H "columns: k1, k2, v1=to_bitmap(k1), v2=bitmap_empty()" -T testData http://host:port/api/testDb/testTbl/_stream_load - - -## keyword - STREAM,LOAD - diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/index.rst b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/index.rst deleted file mode 100644 index e8298bdc23187c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -DML -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/insert.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/insert.md deleted file mode 100644 index 98588a69b72e43..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/insert.md +++ /dev/null @@ -1,104 +0,0 @@ - - -# INSERT -## description -### Syntax - -``` -INSERT INTO table_name - [ PARTITION (p1, ...) ] - [ WITH LABEL label] - [ (column [, ...]) ] - [ [ hint [, ...] ] ] - { VALUES ( { expression | DEFAULT } [, ...] ) [, ...] | query } -``` - -### Parameters - -> tablet_name: 导入数据的目的表。可以是 `db_name.table_name` 形式 -> -> partitions: 指定待导入的分区，必须是 `table_name` 中存在的分区，多个分区名称用逗号分隔 -> -> label: 为 Insert 任务指定一个 label -> -> column_name: 指定的目的列，必须是 `table_name` 中存在的列 -> -> expression: 需要赋值给某个列的对应表达式 -> -> DEFAULT: 让对应列使用默认值 -> -> query: 一个普通查询，查询的结果会写入到目标中 -> -> hint: 用于指示 `INSERT` 执行行为的一些指示符。`streaming` 和 默认的非 `streaming` 方式均会使用同步方式完成 `INSERT` 语句执行 -> 非 `streaming` 方式在执行完成后会返回一个 label 方便用户通过 `SHOW LOAD` 查询导入的状态 - -### Note - -当前执行 `INSERT` 语句时，对于有不符合目标表格式的数据，默认的行为是过滤，比如字符串超长等。但是对于有要求数据不能够被过滤的业务场景，可以通过设置会话变量 `enable_insert_strict` 为 `true` 来确保当有数据被过滤掉的时候，`INSERT` 不会被执行成功。 - -## example - -`test` 表包含两个列`c1`, `c2`。 - -1. 向`test`表中导入一行数据 - -``` -INSERT INTO test VALUES (1, 2); -INSERT INTO test (c1, c2) VALUES (1, 2); -INSERT INTO test (c1, c2) VALUES (1, DEFAULT); -INSERT INTO test (c1) VALUES (1); -``` - -其中第一条、第二条语句是一样的效果。在不指定目标列时，使用表中的列顺序来作为默认的目标列。 -第三条、第四条语句表达的意思是一样的，使用`c2`列的默认值，来完成数据导入。 - -2. 向`test`表中一次性导入多行数据 - -``` -INSERT INTO test VALUES (1, 2), (3, 2 + 2); -INSERT INTO test (c1, c2) VALUES (1, 2), (3, 2 * 2); -INSERT INTO test (c1) VALUES (1), (3); -INSERT INTO test (c1, c2) VALUES (1, DEFAULT), (3, DEFAULT); -``` - -其中第一条、第二条语句效果一样，向`test`表中一次性导入两条数据 -第三条、第四条语句效果已知，使用`c2`列的默认值向`test`表中导入两条数据 - -3. 向 `test` 表中导入一个查询语句结果 - -``` -INSERT INTO test SELECT * FROM test2; -INSERT INTO test (c1, c2) SELECT * from test2; -``` - -4. 向 `test` 表中导入一个查询语句结果，并指定 partition 和 label - -``` -INSERT INTO test PARTITION(p1, p2) WITH LABEL `label1` SELECT * FROM test2; -INSERT INTO test WITH LABEL `label1` (c1, c2) SELECT * from test2; -``` - -异步的导入其实是，一个同步的导入封装成了异步。填写 streaming 和不填写的**执行效率是一样**的。 - -由于Doris之前的导入方式都是异步导入方式，为了兼容旧有的使用习惯，不加 streaming 的 `INSERT` 语句依旧会返回一个 label，用户需要通过`SHOW LOAD`命令查看此`label`导入作业的状态。 - -## keyword - - INSERT diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/BIGINT.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/BIGINT.md deleted file mode 100644 index 7533d5cbfa349b..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/BIGINT.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# BIGINT -## description - BIGINT - 8字节有符号整数，范围[-9223372036854775808, 9223372036854775807] - -## keyword - - BIGINT diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/BOOLEAN.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/BOOLEAN.md deleted file mode 100644 index b5f46fae0b1ed0..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/BOOLEAN.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# BOOLEAN -## description - BOOL, BOOLEAN - 与TINYINT一样，0代表false，1代表true - -## keyword - - BOOLEAN diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/CHAR.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/CHAR.md deleted file mode 100644 index 0aff8fe145fb08..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/CHAR.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# CHAR -## description - CHAR(M) - 定长字符串，M代表的是定长字符串的长度。M的范围是1-255 - -## keyword - - CHAR diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md deleted file mode 100644 index bcaf2d93509c51..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# DATE -## description - DATE函数 - Syntax: - DATE(expr) - 将输入的类型转化为DATE类型 - DATE类型 - 日期类型，目前的取值范围是['0000-01-01', '9999-12-31'], 默认的打印形式是'YYYY-MM-DD' - -## example - mysql> SELECT DATE('2003-12-31 01:02:03'); - -> '2003-12-31' - -## keyword - - DATE diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md deleted file mode 100644 index 992f00c894329c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# DATETIME -## description - DATETIME - 日期时间类型，取值范围是['0000-01-01 00:00:00', '9999-12-31 23:59:59']. - 打印的形式是'YYYY-MM-DD HH:MM:SS' - -## keyword - - DATETIME diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DECIMAL.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DECIMAL.md deleted file mode 100644 index e71911d443f13b..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DECIMAL.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# DECIMAL -## description - DECIMAL(M[,D]) - 高精度定点数，M代表一共有多少个有效数字(precision)，D代表小数点后最多有多少数字(scale) - M的范围是[1,27], D的范围[1, 9], 另外，M必须要大于等于D的取值。默认的D取值为0 - -## keyword - - DECIMAL diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DOUBLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DOUBLE.md deleted file mode 100644 index 72234da52be6cd..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DOUBLE.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# DOUBLE -## description - DOUBLE - 8字节浮点数 - -## keyword - - DOUBLE diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/FLOAT.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/FLOAT.md deleted file mode 100644 index c87ed69d622715..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/FLOAT.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# FLOAT -## description - FLOAT - 4字节浮点数 - -## keyword - - FLOAT diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/HLL.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/HLL.md deleted file mode 100644 index f18df5fea80341..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/HLL.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# HLL(HyperLogLog) -## description - VARCHAR(M) - 变长字符串，M代表的是变长字符串的长度。M的范围是1-16385 - 用户不需要指定长度和默认值。长度根据数据的聚合程度系统内控制 - 并且HLL列只能通过配套的hll_union_agg、hll_raw_agg、hll_cardinality、hll_hash进行查询或使用 - -## keyword - - HLL,HYPERLOGLOG diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/INT.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/INT.md deleted file mode 100644 index 17b42c06277166..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/INT.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# INT -## description - INT - 4字节有符号整数，范围[-2147483648, 2147483647] - -## keyword - - INT diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/SMALLINT.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/SMALLINT.md deleted file mode 100644 index 59d49432e5a12c..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/SMALLINT.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# SMALLINT -## description - SMALLINT - 2字节有符号整数，范围[-32768, 32767] - -## keyword - - SMALLINT diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/TINYINT.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/TINYINT.md deleted file mode 100644 index a8b079491e5ef0..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/TINYINT.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# TINYINT -## description - TINYINT - 1字节有符号整数，范围[-128, 127] - -## keyword - - TINYINT diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/VARCHAR.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/VARCHAR.md deleted file mode 100644 index b2819f87a36c7e..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/VARCHAR.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# VARCHAR -## description - VARCHAR(M) - 变长字符串，M代表的是变长字符串的长度。M的范围是1-65535 - -## keyword - - VARCHAR diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/index.rst b/docs/documentation/cn/sql-reference/sql-statements/Data Types/index.rst deleted file mode 100644 index 1d5cacebd0cb73..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -数据类型 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-statements/Utility/index.rst b/docs/documentation/cn/sql-reference/sql-statements/Utility/index.rst deleted file mode 100644 index 9abc66d27d76a5..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/Utility/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -辅助命令 -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/cn/sql-reference/sql-statements/index.rst b/docs/documentation/cn/sql-reference/sql-statements/index.rst deleted file mode 100644 index 5bfe1cb21b5ca8..00000000000000 --- a/docs/documentation/cn/sql-reference/sql-statements/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -============ -语法帮助 -============ - -.. toctree:: - :hidden: - - Account Management/index - Administration/index - Data Definition/index - Data Manipulation/index - Data Types/index - Utility/index diff --git a/docs/documentation/en/administrator-guide/alter-table/alter-table-bitmap-index_EN.md b/docs/documentation/en/administrator-guide/alter-table/alter-table-bitmap-index_EN.md deleted file mode 100644 index 3fea165ee860a2..00000000000000 --- a/docs/documentation/en/administrator-guide/alter-table/alter-table-bitmap-index_EN.md +++ /dev/null @@ -1,76 +0,0 @@ - - -# Bitmap Index -Users can speed up queries by creating a bitmap index -This document focuses on how to create an index job, as well as some considerations and frequently asked questions when creating an index. - -## Glossary -* bitmap index：a fast data structure that speeds up queries - -## Basic Principles -Creating and droping index is essentially a schema change job. For details, please refer to -[Schema Change](alter-table-schema-change_EN.html)。 - -## Syntax -There are two forms of index creation and modification related syntax, one is integrated with alter table statement, and the other is using separate -create/drop index syntax -1. Create Index - - Please refer to [CREATE INDEX](../../sql-reference/sql-statements/Data%20Definition/CREATE%20INDEX_EN.html) - or [ALTER TABLE](../../sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE_EN.html), - You can also specify a bitmap index when creating a table，Please refer to [CREATE TABLE](../../sql-reference/sql-statements/Data%20Definition/CREATE%20TABLE_EN.html) - -2. Show Index - - Please refer to [SHOW INDEX](../../sql-reference/sql-statements/Administration/SHOW%20INDEX_EN.html) - -3. Drop Index - - Please refer to [DROP INDEX](../../sql-reference/sql-statements/Data%20Definition/DROP%20INDEX_EN.html) or [ALTER TABLE](../../sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE_EN.html) - -## Create Job -Please refer to [Scheam Change](alter-table-schema-change_EN.html) -## View Job -Please refer to [Scheam Change](alter-table-schema-change_EN.html) - -## Cancel Job -Please refer to [Scheam Change](alter-table-schema-change_EN.html) - -## Notice -* Currently only index of bitmap type is supported. -* The bitmap index is only created on a single column. -* Bitmap indexes can be applied to all columns of the `Duplicate` data model and key columns of the `Aggregate` and `Uniq` models. -* The data types supported by bitmap indexes are as follows: - * `TINYINT` - * `SMALLINT` - * `INT` - * `UNSIGNEDINT` - * `BIGINT` - * `CHAR` - * `VARCHAE` - * `DATE` - * `DATETIME` - * `LARGEINT` - * `DECIMAL` - * `BOOL` -* The bitmap index takes effect only in segmentV2. You need to add the following configuration to the configuration file of be - ``` - default_rowset_type=BETA - ``` diff --git a/docs/documentation/en/administrator-guide/alter-table/alter-table-rollup_EN.md b/docs/documentation/en/administrator-guide/alter-table/alter-table-rollup_EN.md deleted file mode 100644 index 3dc6b522141c88..00000000000000 --- a/docs/documentation/en/administrator-guide/alter-table/alter-table-rollup_EN.md +++ /dev/null @@ -1,181 +0,0 @@ - - -# Rollup - -Users can speed up queries by creating rollup tables. For the concept and usage of Rollup, please refer to [Data - Model, ROLLUP and Prefix Index](../../getting-started/data-model-rollup_EN.md) and - [Rollup and query](../../getting-started/hit-the-rollup_EN.md). - -This document focuses on how to create a Rollup job, as well as some considerations and frequently asked questions about creating a Rollup. - -## Glossary - -* Base Table：When each table is created, it corresponds to a base table. The base table stores the complete data of this table. Rollups are usually created based on the data in the base table (and can also be created from other rollups). -* Index：Materialized index. Rollup or Base Table are both called materialized indexes. -* Transaction：Each import task is a transaction, and each transaction has a unique incrementing Transaction ID. - -## Basic Principles - -The basic process of creating a Rollup is to generate a new Rollup data containing the specified column from the data in the Base table. Among them, two parts of data conversion are needed. One is the conversion of existing historical data, and the other is the conversion of newly arrived imported data during Rollup execution. - -``` -+----------+ -| Load Job | -+----+-----+ - | - | Load job generates both base and rollup index data - | - | +------------------+ +---------------+ - | | Base Index | | Base Index | - +------> New Incoming Data| | History Data | - | +------------------+ +------+--------+ - | | - | | Convert history data - | | - | +------------------+ +------v--------+ - | | Rollup Index | | Rollup Index | - +------> New Incoming Data| | History Data | - +------------------+ +---------------+ -``` - -Before starting the conversion of historical data, Doris will obtain a latest transaction ID. And wait for all import transactions before this Transaction ID to complete. This Transaction ID becomes a watershed. This means that Doris guarantees that all import tasks after the watershed will generate data for the Rollup Index at the same time. In this way, after the historical data conversion is completed, the data of the Rollup and Base tables can be guaranteed to be flush. - -## Create Job - -The specific syntax for creating a Rollup can be found in the description of the Rollup section in the help `HELP ALTER TABLE`. - -The creation of Rollup is an asynchronous process. After the job is submitted successfully, the user needs to use the `SHOW ALTER TABLE ROLLUP` command to view the progress of the job. - -## View Job - -`SHOW ALTER TABLE ROLLUP` You can view rollup jobs that are currently executing or completed. For example: - -``` - JobId: 20037 - TableName: tbl1 - CreateTime: 2019-08-06 15:38:49 - FinishedTime: N/A - BaseIndexName: tbl1 -RollupIndexName: r1 - RollupId: 20038 - TransactionId: 10034 - State: PENDING - Msg: - Progress: N/A - Timeout: 86400 -``` - -* JobId: A unique ID for each Rollup job. -* TableName: The table name of the base table corresponding to Rollup. -* CreateTime: Job creation time. -* FinishedTime: The end time of the job. If it is not finished, "N / A" is displayed. -* BaseIndexName: The name of the source Index corresponding to Rollup. -* RollupIndexName: The name of the Rollup. -* RollupId: The unique ID of the Rollup. -* TransactionId: the watershed transaction ID of the conversion history data. -* State: The phase of the operation. -     * PENDING: The job is waiting in the queue to be scheduled. -     * WAITING_TXN: Wait for the import task before the watershed transaction ID to complete. -     * RUNNING: Historical data conversion. -     * FINISHED: The operation was successful. -     * CANCELLED: The job failed. -* Msg: If the job fails, a failure message is displayed here. -* Progress: operation progress. Progress is displayed only in the RUNNING state. Progress is displayed in M / N. Where N is the total number of copies of Rollup. M is the number of copies of historical data conversion completed. -* Timeout: Job timeout time. Unit of second. - -## Cancel Job - -In the case that the job status is not FINISHED or CANCELLED, you can cancel the Rollup job with the following command: - -`CANCEL ALTER TABLE ROLLUP FROM tbl_name;` - -## Notice - -* A table can have only one Rollup job running at a time. And only one rollup can be created in a job. - -* Rollup operations do not block import and query operations. - -* If a DELETE operation has a Key column in a where condition that does not exist in a Rollup, the DELETE is not allowed. - -    If a Key column does not exist in a Rollup, the DELETE operation cannot delete data from the Rollup, so the data consistency between the Rollup table and the Base table cannot be guaranteed. - -* Rollup columns must exist in the Base table. - -    Rollup columns are always a subset of the Base table columns. Columns that do not exist in the Base table cannot appear. - -* If a rollup contains columns of the REPLACE aggregation type, the rollup must contain all the key columns. - -    Assume the structure of the Base table is as follows: -     -    `` `(k1 INT, k2 INT, v1 INT REPLACE, v2 INT SUM)` `` -     -    If you need to create a Rollup that contains `v1` columns, you must include the` k1`, `k2` columns. Otherwise, the system cannot determine the value of `v1` listed in Rollup. -     -    Note that all Value columns in the Unique data model table are of the REPLACE aggregation type. -     -* Rollup of the DUPLICATE data model table, you can specify the DUPLICATE KEY of the rollup. - -    The DUPLICATE KEY in the DUPLICATE data model table is actually sorted. Rollup can specify its own sort order, but the sort order must be a prefix of the Rollup column order. If not specified, the system will check if the Rollup contains all sort columns of the Base table, and if it does not, it will report an error. For example: -     -    Base table structure: `(k1 INT, k2 INT, k3 INT) DUPLICATE KEY (k1, k2)` -     -    Rollup can be: `(k2 INT, k1 INT) DUPLICATE KEY (k2)` - -* Rollup does not need to include partitioned or bucket columns for the Base table. - -## FAQ - -* How many rollups can a table create - -    There is theoretically no limit to the number of rollups a table can create, but too many rollups can affect import performance. Because when importing, data will be generated for all rollups at the same time. At the same time, Rollup will take up physical storage space. Usually the number of rollups for a table is less than 10. -     -* Rollup creation speed - -    Rollup creation speed is currently estimated at about 10MB / s based on the worst efficiency. To be conservative, users can set the timeout for jobs based on this rate. - -* Submitting job error `Table xxx is not stable. ...` - -    Rollup can start only when the table data is complete and unbalanced. If some data shard copies of the table are incomplete, or if some copies are undergoing an equalization operation, the submission is rejected. -     -    Whether the data shard copy is complete can be checked with the following command: -     -    ```ADMIN SHOW REPLICA STATUS FROM tbl WHERE STATUS! =" OK ";``` -     -    If a result is returned, there is a problem with the copy. These problems are usually fixed automatically by the system. You can also use the following commands to repair this table first: -     -    ```ADMIN REPAIR TABLE tbl1; ``` -     -    You can check if there are running balancing tasks with the following command: -     -    ```SHOW PROC" / cluster_balance / pending_tablets ";``` -     -    You can wait for the balancing task to complete, or temporarily disable the balancing operation with the following command: - - ```ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");``` - -## Configurations - -### FE Configurations - -* `alter_table_timeout_second`：The default timeout for the job is 86400 seconds. - -### BE Configurations - -* `alter_tablet_worker_count`：Number of threads used to perform historical data conversion on the BE side. The default is 3. If you want to speed up the rollup job, you can increase this parameter appropriately and restart the BE. But too many conversion threads can cause increased IO pressure and affect other operations. This thread is shared with the Schema Change job. diff --git a/docs/documentation/en/administrator-guide/alter-table/alter-table-schema-change_EN.md b/docs/documentation/en/administrator-guide/alter-table/alter-table-schema-change_EN.md deleted file mode 100644 index d9f3ced941ceb7..00000000000000 --- a/docs/documentation/en/administrator-guide/alter-table/alter-table-schema-change_EN.md +++ /dev/null @@ -1,224 +0,0 @@ - - -# Scheam Change - -Users can modify the schema of existing tables through the Scheam Change operation. Doris currently supports the following modifications: - -* Add and delete columns -* Modify column type -* Adjust column order -* Add and modify Bloom Filter -* Add and delete bitmap index - -This document mainly describes how to create a Scheam Change job, as well as some considerations and frequently asked questions about Scheam Change. -## Glossary - -* Base Table：When each table is created, it corresponds to a base table. The base table stores the complete data of this table. Rollups are usually created based on the data in the base table (and can also be created from other rollups). -* Index：Materialized index. Rollup or Base Table are both called materialized indexes. -* Transaction：Each import task is a transaction, and each transaction has a unique incrementing Transaction ID. -* Rollup：Roll-up tables based on base tables or other rollups. - -## Basic Principles - -The basic process of executing a Schema Change is to generate a copy of the index data of the new schema from the data of the original index. Among them, two parts of data conversion are required. One is the conversion of existing historical data, and the other is the conversion of newly arrived imported data during the execution of Schema Change. -``` -+----------+ -| Load Job | -+----+-----+ - | - | Load job generates both origin and new index data - | - | +------------------+ +---------------+ - | | Origin Index | | Origin Index | - +------> New Incoming Data| | History Data | - | +------------------+ +------+--------+ - | | - | | Convert history data - | | - | +------------------+ +------v--------+ - | | New Index | | New Index | - +------> New Incoming Data| | History Data | - +------------------+ +---------------+ -``` - -Before starting the conversion of historical data, Doris will obtain a latest transaction ID. And wait for all import transactions before this Transaction ID to complete. This Transaction ID becomes a watershed. This means that Doris guarantees that all import tasks after the watershed will generate data for both the original Index and the new Index. In this way, when the historical data conversion is completed, the data in the new Index can be guaranteed to be complete. -## Create Job - -The specific syntax for creating a Scheam Change can be found in the description of the Scheam Change section in the help `HELP ALTER TABLE`. - -The creation of Scheam Change is an asynchronous process. After the job is submitted successfully, the user needs to view the job progress through the `SHOW ALTER TABLE COLUMN` command. -## View Job - -`SHOW ALTER TABLE COLUMN` You can view the Schema Change jobs that are currently executing or completed. When multiple indexes are involved in a Schema Change job, the command displays multiple lines, each corresponding to an index. For example: - -``` - JobId: 20021 - TableName: tbl1 - CreateTime: 2019-08-05 23:03:13 - FinishTime: 2019-08-05 23:03:42 - IndexName: tbl1 - IndexId: 20022 -OriginIndexId: 20017 -SchemaVersion: 2:792557838 -TransactionId: 10023 - State: FINISHED - Msg: - Progress: N/A - Timeout: 86400 -``` - -* JobId: A unique ID for each Schema Change job. -* TableName: The table name of the base table corresponding to Schema Change. -* CreateTime: Job creation time. -* FinishedTime: The end time of the job. If it is not finished, "N / A" is displayed. -* IndexName: The name of an Index involved in this modification. -* IndexId: The unique ID of the new Index. -* OriginIndexId: The unique ID of the old Index. -* SchemaVersion: Displayed in M: N format. M is the version of this Schema Change, and N is the corresponding hash value. With each Schema Change, the version is incremented. -* TransactionId: the watershed transaction ID of the conversion history data. -* State: The phase of the operation. -    * PENDING: The job is waiting in the queue to be scheduled. -    * WAITING_TXN: Wait for the import task before the watershed transaction ID to complete. -    * RUNNING: Historical data conversion. -    * FINISHED: The operation was successful. -    * CANCELLED: The job failed. -* Msg: If the job fails, a failure message is displayed here. -* Progress: operation progress. Progress is displayed only in the RUNNING state. Progress is displayed in M ​​/ N. Where N is the total number of copies involved in the Schema Change. M is the number of copies of historical data conversion completed. -* Timeout: Job timeout time. Unit of second. - -## Cancel Job - -In the case that the job status is not FINISHED or CANCELLED, you can cancel the Schema Change job with the following command: -`CANCEL ALTER TABLE COLUMN FROM tbl_name;` - -## Best Practice - -Schema Change can make multiple changes to multiple indexes in one job. For example: -Source Schema： - -``` -+-----------+-------+------+------+------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+-----------+-------+------+------+------+---------+-------+ -| tbl1 | k1 | INT | No | true | N/A | | -| | k2 | INT | No | true | N/A | | -| | k3 | INT | No | true | N/A | | -| | | | | | | | -| rollup2 | k2 | INT | No | true | N/A | | -| | | | | | | | -| rollup1 | k1 | INT | No | true | N/A | | -| | k2 | INT | No | true | N/A | | -+-----------+-------+------+------+------+---------+-------+ -``` - -You can add a row k4 to both rollup1 and rollup2 by adding the following k5 to rollup2: -``` -ALTER TABLE tbl1 -ADD COLUMN k4 INT default "1" to rollup1, -ADD COLUMN k4 INT default "1" to rollup2, -ADD COLUMN k5 INT default "1" to rollup2; -``` - -When completion, the Schema becomes: - -``` -+-----------+-------+------+------+------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+-----------+-------+------+------+------+---------+-------+ -| tbl1 | k1 | INT | No | true | N/A | | -| | k2 | INT | No | true | N/A | | -| | k3 | INT | No | true | N/A | | -| | k4 | INT | No | true | 1 | | -| | k5 | INT | No | true | 1 | | -| | | | | | | | -| rollup2 | k2 | INT | No | true | N/A | | -| | k4 | INT | No | true | 1 | | -| | k5 | INT | No | true | 1 | | -| | | | | | | | -| rollup1 | k1 | INT | No | true | N/A | | -| | k2 | INT | No | true | N/A | | -| | k4 | INT | No | true | 1 | | -+-----------+-------+------+------+------+---------+-------+ -``` - -As you can see, the base table tbl1 also automatically added k4, k5 columns. That is, columns added to any rollup are automatically added to the Base table. - -At the same time, columns that already exist in the Base table are not allowed to be added to Rollup. If you need to do this, you can re-create a Rollup with the new columns and then delete the original Rollup. -## Notice - -* Only one Schema Change job can be running on a table at a time. - -* Schema Change operation does not block import and query operations. - -* The partition column and bucket column cannot be modified. - -* If there is a value column aggregated by REPLACE in the schema, the Key column is not allowed to be deleted. - -     If the Key column is deleted, Doris cannot determine the value of the REPLACE column. -     -     All non-Key columns of the Unique data model table are REPLACE aggregated. -     -* When adding a value column whose aggregation type is SUM or REPLACE, the default value of this column has no meaning to historical data. - -     Because the historical data has lost the detailed information, the default value cannot actually reflect the aggregated value. -     -* When modifying the column type, fields other than Type need to be completed according to the information on the original column. - -     If you modify the column `k1 INT SUM NULL DEFAULT" 1 "` as type BIGINT, you need to execute the following command: - - ```ALTER TABLE tbl1 MODIFY COLUMN `k1` BIGINT SUM NULL DEFAULT "1"; ``` - - Note that in addition to the new column types, such as the aggregation mode, Nullable attributes, and default values must be completed according to the original information. - -* Modifying column names, aggregation types, nullable attributes, default values, and column comments is not supported. - -## FAQ - -* the execution speed of Schema Change - - At present, the execution speed of Schema Change is estimated to be about 10MB / s according to the worst efficiency. To be conservative, users can set the timeout for jobs based on this rate. - -* Submit job error `Table xxx is not stable. ...` - - Schema Change can only be started when the table data is complete and unbalanced. If some data shard copies of the table are incomplete, or if some copies are undergoing an equalization operation, the submission is rejected. -      - Whether the data shard copy is complete can be checked with the following command: - ```ADMIN SHOW REPLICA STATUS FROM tbl WHERE STATUS != "OK";``` - - If a result is returned, there is a problem with the copy. These problems are usually fixed automatically by the system. You can also use the following commands to repair this table first: - ```ADMIN REPAIR TABLE tbl1;``` - - You can check if there are running balancing tasks with the following command: - - ```SHOW PROC "/cluster_balance/pending_tablets";``` - - You can wait for the balancing task to complete, or temporarily disable the balancing operation with the following command: - - ```ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");``` - -## Configurations - -### FE Configurations - -* `alter_table_timeout_second`：The default timeout for the job is 86400 seconds. - -### BE Configurations - -* `alter_tablet_worker_count`：Number of threads used to perform historical data conversion on the BE side. The default is 3. If you want to speed up the Schema Change job, you can increase this parameter appropriately and restart the BE. But too many conversion threads can cause increased IO pressure and affect other operations. This thread is shared with the Rollup job. diff --git a/docs/documentation/en/administrator-guide/alter-table/alter-table-temp-partition_EN.md b/docs/documentation/en/administrator-guide/alter-table/alter-table-temp-partition_EN.md deleted file mode 100644 index cddf7cf5a4af09..00000000000000 --- a/docs/documentation/en/administrator-guide/alter-table/alter-table-temp-partition_EN.md +++ /dev/null @@ -1,234 +0,0 @@ - - -# Temporary partition - -Since version 0.12, Doris supports temporary partitioning. - -A temporary partition belongs to a partitioned table. Only partitioned tables can create temporary partitions. - -## Rules - -* The partition columns of the temporary partition is the same as the formal partition and cannot be modified. -* The partition ranges of all temporary partitions of a table cannot overlap, but the ranges of temporary partitions and formal partitions can overlap. -* The partition name of the temporary partition cannot be the same as the formal partitions and other temporary partitions. - -## Supported operations - -The temporary partition supports add, delete, and replace operations. - -### Add temporary partition - -You can add temporary partitions to a table with the `ALTER TABLE ADD TEMPORARY PARTITION` statement: - -``` -ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN ("2020-02-01"); - -ALTER TABLE tbl2 ADD TEMPORARY PARTITION tp1 VALUES [("2020-01-01"), ("2020-02-01")); - -ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN ("2020-02-01") -("in_memory" = "true", "replication_num" = "1") -DISTRIBUTED BY HASH (k1) BUCKETS 5; -``` - -See `HELP ALTER TABLE;` for more help and examples. - -Some instructions for adding operations: - -* Adding a temporary partition is similar to adding a formal partition. The partition range of the temporary partition is independent of the formal partition. -* Temporary partition can independently specify some attributes. Includes information such as the number of buckets, the number of replicas, whether it is a memory table, or the storage medium. - -### Delete temporary partition - -A table's temporary partition can be dropped with the `ALTER TABLE DROP TEMPORARY PARTITION` statement: - -``` -ALTER TABLE tbl1 DROP TEMPORARY PARTITION tp1; -``` - -See `HELP ALTER TABLE;` for more help and examples. - -Some instructions for the delete operation: - -* Deleting the temporary partition will not affect the data of the formal partition. - -### Replace partition - -You can replace formal partitions of a table with temporary partitions with the `ALTER TABLE REPLACE PARTITION` statement. - -``` -ALTER TABLE tbl1 REPLACE PARTITION (p1) WITH TEMPORARY PARTITION (tp1); - -ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1, tp2, tp3); - -ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1, tp2) -PROPERTIES ( -    "strict_range" = "false", -    "use_temp_partition_name" = "true" -); -``` - -See `HELP ALTER TABLE;` for more help and examples. - -The replace operation has two special optional parameters: - -1. `strict_range` - - The default is true. When this parameter is true, the range union of all formal partitions to be replaced needs to be the same as the range union of the temporary partitions to be replaced. When set to false, you only need to ensure that the range between the new formal partitions does not overlap after replacement. Here are some examples: - - * Example 1 - - Range of partitions p1, p2, p3 to be replaced (=> union): - - ``` - (10, 20), [20, 30), [40, 50) => [10, 30), [40, 50) - ``` - - Replace the range of partitions tp1, tp2 (=> union): - - ``` - (10, 30), [40, 45), [45, 50) => [10, 30), [40, 50) - ``` - - The union of ranges is the same, so you can use tp1 and tp2 to replace p1, p2, p3. - - * Example 2 - - Range of partition p1 to be replaced (=> union): - - ``` - (10, 50) => [10, 50) - ``` - - Replace the range of partitions tp1, tp2 (=> union): - - ``` - (10, 30), [40, 50) => [10, 30), [40, 50) - ``` - - The union of ranges is not the same. If `strict_range` is true, you cannot use tp1 and tp2 to replace p1. If false, and the two partition ranges `[10, 30), [40, 50)` and the other formal partitions do not overlap, they can be replaced. - -2. `use_temp_partition_name` - - The default is false. When this parameter is false, and the number of partitions to be replaced is the same as the number of replacement partitions, the name of the formal partition after the replacement remains unchanged. If true, after replacement, the name of the formal partition is the name of the replacement partition. Here are some examples: - - * Example 1 - - ``` - ALTER TABLE tbl1 REPLACE PARTITION (p1) WITH TEMPORARY PARTITION (tp1); - ``` - - `use_temp_partition_name` is false by default. After replacement, the partition name is still p1, but the related data and attributes are replaced with tp1. - - If `use_temp_partition_name` is true by default, the name of the partition is tp1 after replacement. The p1 partition no longer exists. - - * Example 2 - - ``` - ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1); - ``` - - `use_temp_partition_name` is false by default, but this parameter is invalid because the number of partitions to be replaced and the number of replacement partitions are different. After the replacement, the partition name is tp1, and p1 and p2 no longer exist. - -Some instructions for the replacement operation: - -* After the partition is replaced successfully, the replaced partition will be deleted and cannot be recovered. - -## Load and query of temporary partitions - -Users can load data into temporary partitions or specify temporary partitions for querying. - -1. Load temporary partition - - The syntax for specifying a temporary partition is slightly different depending on the load method. Here is a simple illustration through an example: - - ``` - INSERT INTO tbl TEMPORARY PARTITION (tp1, tp2, ...) SELECT .... - ``` - - ``` - curl --location-trusted -u root: -H "label: 123" -H "temporary_partition: tp1, tp2, ..." -T testData http: // host: port / api / testDb / testTbl / _stream_load - ``` - - ``` - LOAD LABEL example_db.label1 - ( - DATA INFILE ("hdfs: // hdfs_host: hdfs_port / user / palo / data / input / file") - INTO TABLE `my_table` - TEMPORARY PARTITION (tp1, tp2, ...) - ... - ) - WITH BROKER hdfs ("username" = "hdfs_user", "password" = "hdfs_password"); - ``` - - ``` - CREATE ROUTINE LOAD example_db.test1 ON example_tbl - COLUMNS (k1, k2, k3, v1, v2, v3 = k1 * 100), - TEMPORARY PARTITIONS (tp1, tp2, ...), - WHERE k1> 100 - PROPERTIES - (...) - FROM KAFKA - (...); - ``` - -2. Query the temporary partition - - ``` - SELECT ... FROM - tbl1 TEMPORARY PARTITION (tp1, tp2, ...) - JOIN - tbl2 TEMPORARY PARTITION (tp1, tp2, ...) - ON ... - WHERE ...; - ``` - -## Relationship to other operations - -### DROP - -* After using the `DROP` operation to directly drop the database or table, you can recover the database or table (within a limited time) through the `RECOVER` command, but the temporary partition will not be recovered. -* After the formal partition is dropped using the `ALTER` command, the partition can be recovered by the `RECOVER` command (within a limited time). Operating a formal partition is not related to a temporary partition. -* After the temporary partition is dropped using the `ALTER` command, the temporary partition cannot be recovered through the `RECOVER` command. - -### TRUNCATE - -* Use the `TRUNCATE` command to empty the table. The temporary partition of the table will be deleted and cannot be recovered. -* When using `TRUNCATE` command to empty the formal partition, it will not affect the temporary partition. -* You cannot use the `TRUNCATE` command to empty the temporary partition. - -### ALTER - -* When the table has a temporary partition, you cannot use the `ALTER` command to perform Schema Change, Rollup, etc. on the table. -* You cannot add temporary partitions to a table while the table is undergoing a alter operation. - - -## Best Practices - -1. Atomic overwrite - - In some cases, the user wants to be able to rewrite the data of a certain partition, but if it is dropped first and then loaded, there will be a period of time when the data cannot be seen. This is, the user can first create a corresponding temporary partition, load new data into the temporary partition, and then replace the original partition atomically through the `REPLACE` operation to achieve the purpose. -     -2. Modify the number of buckets - - In some cases, the user used an inappropriate number of buckets when creating a partition. The user can first create a temporary partition corresponding to the partition range and specify a new number of buckets. Then use the `INSERT INTO` command to load the data of the formal partition into the temporary partition. Through the replacement operation, the original partition is replaced atomically to achieve the purpose. -     -3. Merge or split partitions - - In some cases, users want to modify the range of partitions, such as merging two partitions, or splitting a large partition into multiple smaller partitions. Then the user can first create temporary partitions corresponding to the merged or divided range, and then load the data of the formal partition into the temporary partition through the `INSERT INTO` command. Through the replacement operation, the original partition is replaced atomically to achieve the purpose. \ No newline at end of file diff --git a/docs/documentation/en/administrator-guide/alter-table/index.rst b/docs/documentation/en/administrator-guide/alter-table/index.rst deleted file mode 100644 index f44960b4a3e84e..00000000000000 --- a/docs/documentation/en/administrator-guide/alter-table/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============= -Schema Change -============= - -.. toctree:: - :maxdepth: 2 - :glob: - - * diff --git a/docs/documentation/en/administrator-guide/backup-restore_EN.md b/docs/documentation/en/administrator-guide/backup-restore_EN.md deleted file mode 100644 index 1a563996db8076..00000000000000 --- a/docs/documentation/en/administrator-guide/backup-restore_EN.md +++ /dev/null @@ -1,179 +0,0 @@ - - -# Backup and Recovery - -Doris supports the backup of current data in the form of files to remote storage systems via broker. The data can then be restored from the remote storage system to any Doris cluster by the restore command. With this feature, Doris can support regular snapshot backups of data. It can also be used to migrate data between different clusters. - -This feature requires Doris version 0.8.2+ - -Using this function, brokers corresponding to remote storage need to be deployed. Such as BOS, HDFS, etc. You can view the currently deployed broker through `SHOW BROKER;` - -## Brief Principle Description - -### Backup - -The backup operation is to upload the data of the specified table or partition directly to the remote warehouse in the form of files stored by Doris for storage. When a user submits a Backup request, the following actions will be done within the system: - -1. Snapshot and snapshot upload - - The snapshot phase takes a snapshot of the specified table or partition data file. Later, backups are all snapshots. After the snapshot, changes to tables, imports, and other operations no longer affect the results of the backup. Snapshots only produce a hard link to the current data file, which takes very little time. Once the snapshots are completed, they are uploaded one by one. Snapshot upload is done concurrently by each Backend. - -2. Metadata preparation and upload - - After the data file snapshot is uploaded, Frontend first writes the corresponding metadata to the local file, and then uploads the local metadata file to the remote warehouse through broker. Finish the final backup job. - -### Restore - -Recovery operations need to specify a backup that already exists in a remote repository, and then restore the backup content to the local cluster. When a user submits a Restore request, the following actions will be done within the system: - -1. Create corresponding metadata locally - - This step starts by creating structures such as restoring the corresponding table partitions in the local cluster. When created, the table is visible, but not accessible. - -2. Local snapshot - - This step is to take a snapshot of the table created in the previous step. This is actually an empty snapshot (because the tables just created have no data), and its main purpose is to generate the corresponding snapshot directory on the Backend for receiving the snapshot files downloaded from the remote repository later. - -3. Download snapshots - - The snapshot files in the remote warehouse are downloaded to the corresponding snapshot directory generated in the previous step. This step is done concurrently by each backend. - -4. Effective snapshot - - When the snapshot download is complete, we map each snapshot to the metadata of the current local table. These snapshots are then reloaded to take effect and complete the final recovery operation. - -## Best Practices - -### Backup - -We currently support full backup at the minimum partition granularity (incremental backup may be supported in future versions). If data need to be backed up regularly, first of all, it is necessary to plan the partition and bucket allocation of tables reasonably, such as partitioning according to time. Then in the subsequent run process, periodic data backup is performed according to partition granularity. - -### Data migration - -Users can first backup the data to the remote warehouse, and then restore the data to another cluster through the remote warehouse to complete data migration. Because data backup is done in the form of snapshots, new imported data after the snapshot phase of the backup job will not be backed up. Therefore, after the snapshot is completed, the data imported on the original cluster needs to be imported on the new cluster as well until the recovery job is completed. - -It is suggested that the new and old clusters be imported in parallel for a period of time after the migration is completed. After completing data and business correctness checks, the business is migrated to the new cluster. - -## Highlights - -1. Backup and recovery-related operations are currently only allowed to be performed by users with ADMIN privileges. -2. Within a database, only one backup or recovery job is allowed to be performed. -3. Both backup and recovery support the operation at the minimum partition level. When the table has a large amount of data, it is recommended to perform partition-by-partition to reduce the cost of failed retries. -4. Because backup and recovery operations, the operation is the actual data files. So when there are too many fragments of a table or too many small versions of a fragment, it may take a long time to backup or restore even if the total amount of data is very small. Users can estimate job execution time by `SHOW PARTITIONS FROM table_name;`, and `SHOW TABLET FROM table_name;`, viewing the number of partitions and the number of file versions of each partition. The number of files has a great impact on the execution time of the job, so it is suggested that the partition buckets should be planned reasonably in order to avoid excessive partitioning. -5. When viewing the job status through `SHOW BACKUP` or `SHOW RESTORE`. It is possible to see an error message in the `TaskErrMsg` column. But as long as the `State` column does not -`CANCELLED`, that means the job is still going on. These Tasks may succeed in retrying. Of course, some Task errors can also directly lead to job failure. -6. If the recovery operation is a coverage operation (specifying the recovery data to an existing table or partition), then starting from the `COMMIT` phase of the recovery operation, the data covered on the current cluster may not be restored. At this time, if the recovery operation fails or is cancelled, it may cause the previous data to be damaged and inaccessible. In this case, the recovery operation can only be performed again and wait for the job to complete. Therefore, we recommend that if it is not necessary, try not to use coverage to recover data unless it is confirmed that the current data is no longer in use. - -## Relevant orders - -The commands related to the backup recovery function are as follows. The following commands, you can use `help cmd;'to view detailed help after connecting Doris through mysql-client. - -1. CREATE REPOSITORY - - Create a remote warehouse Path for backup or recovery. - -1. BACKUP - - Perform a backup operation. - -3. SHOW BACKUP - - View the execution of the last backup job, including: - - * JobId: ID of this backup job. - * SnapshotName: User-specified name of this backup job (Label). - * DbName: The database corresponding to the backup job. - * State: The current stage of the backup job: - * PENDING: The initial state of the job. - * SNAPSHOTING: Snapshot operation is in progress. - * UPLOAD_SNAPSHOT: The snapshot is over and ready to upload. - * UPLOADING: Uploading snapshots. - * SAVE_META: Metadata files are being generated locally. - * UPLOAD_INFO: Upload metadata files and information for this backup job. - * FINISHED: The backup is complete. - * CANCELLED: Backup failed or cancelled. - * Backup Objs: List of tables and partitions involved in this backup. - * CreateTime: Job creation time. - * Snapshot Finished Time: Snapshot completion time. - * Upload Finished Time: Snapshot upload completion time. - * FinishedTime: The completion time of this assignment. - * Unfinished Tasks: In the `SNAPSHOTTING', `UPLOADING'and other stages, there will be multiple sub-tasks at the same time, the current stage shown here, the task ID of the unfinished sub-tasks. - * TaskErrMsg: If there is a sub-task execution error, the error message corresponding to the sub-task will be displayed here. - * Status: It is used to record some status information that may appear during the whole operation. - * Timeout: The timeout time of a job in seconds. - -4. SHOW SNAPSHOT - - View the backup that already exists in the remote warehouse. - - * Snapshot: The name of the backup specified at the time of backup (Label). - * Timestamp: Backup timestamp. - * Status: Is the backup normal? - - If the where clause is specified after `SHOW SNAPSHOT', more detailed backup information can be displayed. - - * Database: The database corresponding to backup. - * Details: Shows the complete data directory structure of the backup. - -5. RESTOR - - Perform a recovery operation. - -6. SHOW RESTORE - - View the execution of the last restore job, including: - - * JobId: ID of this resumption job. - * Label: The name of the backup in the user-specified warehouse (Label). - * Timestamp: The timestamp for backup in a user-specified warehouse. - * DbName: Restore the database corresponding to the job. - * State: The current stage of the recovery operation: - * PENDING: The initial state of the job. - * SNAPSHOTING: A snapshot of a new local table is in progress. - * DOWNLOAD: The download snapshot task is being sent. - * DOWNLOADING: Snapshot is downloading. - * COMMIT: Prepare to take effect the downloaded snapshot. - * COMMITTING: The downloaded snapshot is in effect. - * FINISHED: Recovery is complete. - * CANCELLED: Recovery failed or cancelled. - * AllowLoad: Is import allowed during recovery? - * ReplicationNum: Restores the specified number of copies. - * Restore Objs: List of tables and partitions involved in this recovery. - * CreateTime: Job creation time. - * MetaPreparedTime: Completion time of local metadata generation. - * Snapshot Finished Time: Local snapshot completion time. - * Download Finished Time: The download completion time of the remote snapshot. - * FinishedTime: The completion time of this assignment. - * Unfinished Tasks: In the `SNAPSHOTTING`, `DOWNLOADING`, `COMMITTING`, and other stages, there will be multiple sub-tasks at the same time, the current stage shown here, the task ID of the unfinished sub-tasks. - * TaskErrMsg: If there is a sub-task execution error, the error message corresponding to the sub-task will be displayed here. - * Status: It is used to record some status information that may appear during the whole operation. - * Timeout: The timeout time of a job in seconds. - -7. CANCEL BACKUP - - Cancel the backup job currently being performed. - -8. CANCEL RESTORE - - Cancel the recovery job currently being performed. - -9. DROP REPOSITORY - - Delete the created remote warehouse. Delete the warehouse, just delete the mapping of the warehouse in Doris, will not delete the actual warehouse data. diff --git a/docs/documentation/en/administrator-guide/broker_EN.md b/docs/documentation/en/administrator-guide/broker_EN.md deleted file mode 100644 index 4854f0e1571712..00000000000000 --- a/docs/documentation/en/administrator-guide/broker_EN.md +++ /dev/null @@ -1,286 +0,0 @@ - - -# Broker - -Broker is an optional process in the Doris cluster. It is mainly used to support Doris to read and write files or directories on remote storage, such as HDFS, BOS, and AFS. - -Broker provides services through an RPC service port. It is a stateless JVM process that is responsible for encapsulating some POSIX-like file operations for read and write operations on remote storage, such as open, pred, pwrite, and so on. -In addition, the Broker does not record any other information, so the connection information, file information, permission information, and so on stored remotely need to be passed to the Broker process in the RPC call through parameters in order for the Broker to read and write files correctly . - -Broker only acts as a data channel and does not participate in any calculations, so it takes up less memory. Usually one or more Broker processes are deployed in a Doris system. And the same type of Broker will form a group and set a ** Broker name **. - -Broker's position in the Doris system architecture is as follows: - -``` -+----+ +----+ -| FE | | BE | -+-^--+ +--^-+ - | | - | | -+-v---------v-+ -| Broker | -+------^------+ - | - | -+------v------+ -|HDFS/BOS/AFS | -+-------------+ -``` - -This document mainly introduces the parameters that Broker needs when accessing different remote storages, such as connection information, -authorization information, and so on. - -## Supported Storage System - -Different types of brokers support different different storage systems。 - -1. Community HDFS - - * Support simple authentication access - * Support kerberos authentication access - * Support HDFS HA mode access - -2. Baidu HDFS / AFS (not supported by open source version) - - * Support UGI simple authentication access - -3. Baidu Object Storage BOS (not supported by open source version) - - * Support AK / SK authentication access - -## Function provided by Broker - -1. Broker Load - - The Broker Load function reads the file data on the remote storage through the Broker process and imports it into Doris. Examples are as follows: - - ``` - LOAD LABEL example_db.label6 - ( - DATA INFILE("bos://my_bucket/input/file") - INTO TABLE `my_table` - ) - WITH BROKER "broker_name" - ( - "bos_endpoint" = "http://bj.bcebos.com", - "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "bos_secret_accesskey" = "yyyyyyyyyyyyyyyyyyyy" - ) - ``` - - `WITH BROKER` and following Property Map are used to provide Broker's related information. - -2. Export - - The Export function export the data stored in Doris to a file stored in remote storage in text format through Broker process. Examples are as follows: - - ``` - EXPORT TABLE testTbl - TO "hdfs://hdfs_host:port/a/b/c" - WITH BROKER "broker_name" - ( - "username" = "xxx", - "password" = "yyy" - ); - ``` - - `WITH BROKER` and following Property Map are used to provide Broker's related information. - -3. Create Repository - - When users need to use the backup and restore function, they need to first create a "repository" with the `CREATE REPOSITORY` command,and the broker metadata and related information are recorded in the warehouse metadata. - Subsequent backup and restore operations will use Broker to back up data to this warehouse, or read data from this warehouse to restore to Doris. Examples are as follows: - - ``` - CREATE REPOSITORY `bos_repo` - WITH BROKER `broker_name` - ON LOCATION "bos://doris_backup" - PROPERTIES - ( - "bos_endpoint" = "http://gz.bcebos.com", - "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", - "bos_secret_accesskey" = "70999999999999de274d59eaa980a" - ); - ``` - - `WITH BROKER` and following Property Map are used to provide Broker's related information. - - -## Broker Information - -Broker information includes two parts: ** Broker name ** and ** Certification information **. The general syntax is as follows: - -``` -WITH BROKER "broker_name" -( - "username" = "xxx", - "password" = "yyy", - "other_prop" = "prop_value", - ... -); -``` - -### Broker Name - -Usually the user needs to specify an existing Broker Name through the `WITH BROKER" broker_name "` clause in the operation command. -Broker Name is a name that the user specifies when adding a Broker process through the ALTER SYSTEM ADD BROKER command. -A name usually corresponds to one or more broker processes. Doris selects available broker processes based on the name. -You can use the `SHOW BROKER` command to view the Brokers that currently exist in the cluster. - -**Note: Broker Name is just a user-defined name and does not represent the type of Broker.** - -### Certification Information - -Different broker types and different access methods need to provide different authentication information. -Authentication information is usually provided as a Key-Value in the Property Map after `WITH BROKER" broker_name "`. - -#### Community HDFS - -1. Simple Authentication - - Simple authentication means that Hadoop configures `hadoop.security.authentication` to` simple`. - - Use system users to access HDFS. Or add in the environment variable started by Broker：```HADOOP_USER_NAME```。 - - ``` - ( - "username" = "user", - "password" = "" - ); - ``` - - Just leave the password blank. - -2. Kerberos Authentication - - The authentication method needs to provide the following information:： - - * `hadoop.security.authentication`: Specify the authentication method as kerberos. - * `kerberos_principal`： Specify the principal of kerberos. - * `kerberos_keytab`: Specify the path to the keytab file for kerberos. The file must be an absolute path to a file on the server where the broker process is located. And can be accessed by the Broker process. - * `kerberos_keytab_content`: Specify the content of the keytab file in kerberos after base64 encoding. You can choose one of these with `kerberos_keytab` configuration. - - Examples are as follows： - - ``` - ( - "hadoop.security.authentication" = "kerberos", - "kerberos_principal" = "doris@YOUR.COM", - "kerberos_keytab" = "/home/doris/my.keytab" - ) - ``` - ``` - ( - "hadoop.security.authentication" = "kerberos", - "kerberos_principal" = "doris@YOUR.COM", - "kerberos_keytab_content" = "ASDOWHDLAWIDJHWLDKSALDJSDIWALD" - ) - ``` - If Kerberos authentication is used, the [krb5.conf](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html) file is required when deploying the Broker process. - The krb5.conf file contains Kerberos configuration information，Normally, you should install your krb5.conf file in the directory /etc. You can override the default location by setting the environment variable KRB5_CONFIG. - An example of the contents of the krb5.conf file is as follows: - ``` - [libdefaults] - default_realm = DORIS.HADOOP - default_tkt_enctypes = des3-hmac-sha1 des-cbc-crc - default_tgs_enctypes = des3-hmac-sha1 des-cbc-crc - dns_lookup_kdc = true - dns_lookup_realm = false - - [realms] - DORIS.HADOOP = { - kdc = kerberos-doris.hadoop.service:7005 - } - ``` - -3. HDFS HA Mode - - This configuration is used to access HDFS clusters deployed in HA mode. - - * `dfs.nameservices`: Specify the name of the hdfs service, custom, such as "dfs.nameservices" = "my_ha". - * `dfs.ha.namenodes.xxx`: Custom namenode names. Multiple names are separated by commas, where xxx is the custom name in `dfs.nameservices`, such as" dfs.ha.namenodes.my_ha "=" my_nn ". - * `dfs.namenode.rpc-address.xxx.nn`: Specify the rpc address information of namenode, Where nn represents the name of the namenode configured in `dfs.ha.namenodes.xxx`, such as: "dfs.namenode.rpc-address.my_ha.my_nn" = "host:port". - * `dfs.client.failover.proxy.provider`: Specify the provider for the client to connect to the namenode. The default is: org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider. - - Examples are as follows： - - ``` - ( - "dfs.nameservices" = "my_ha", - "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", - "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", - "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", - "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" - ) - ``` - - The HA mode can be combined with the previous two authentication methods for cluster access. If you access HA HDFS with simple authentication: - - ``` - ( - "username"="user", - "password"="passwd", - "dfs.nameservices" = "my_ha", - "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", - "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", - "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", - "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" - ) - ``` - The configuration for accessing the HDFS cluster can be written to the hdfs-site.xml file. When users use the Broker process to read data from the HDFS cluster, they only need to fill in the cluster file path and authentication information. - -#### Baidu Object Storage BOS - -**(Open source version is not supported)** - -1. Access via AK / SK - - * AK/SK: Access Key and Secret Key. You can check the user's AK / SK in Baidu Cloud Security Certification Center. - * Region Endpoint: Endpoint of the BOS region: - - * North China-Beijing: http://bj.bcebos.com - * North China-Baoding: http://bd.bcebos.com - * South China-Guangzhou: http://gz.bcebos.com - * East China-Suzhou: http://sz.bcebos.com - - Examples are as follows： - - ``` - ( - "bos_endpoint" = "http://bj.bcebos.com", - "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "bos_secret_accesskey" = "yyyyyyyyyyyyyyyyyyyyyyyyyy" - ) - ``` - -#### Baidu HDFS/AFS - -**（Open source version is not supported）** - -Baidu AFS and HDFS only support simple authentication access using UGI. Examples are as follows: - -``` -( - "username" = "user", - "password" = "passwd" -); -``` - -User and passwd are UGI configurations for Hadoop. \ No newline at end of file diff --git a/docs/documentation/en/administrator-guide/colocation-join_EN.md b/docs/documentation/en/administrator-guide/colocation-join_EN.md deleted file mode 100644 index 8802dd2aea11b1..00000000000000 --- a/docs/documentation/en/administrator-guide/colocation-join_EN.md +++ /dev/null @@ -1,441 +0,0 @@ - - -# Colocation Join - -Colocation Join is a new feature introduced in Doris 0.9. The purpose of this paper is to provide local optimization for some Join queries to reduce data transmission time between nodes and speed up queries. - -The original design, implementation and effect can be referred to [ISSUE 245] (https://github.com/apache/incubator-doris/issues/245). - -The Colocation Join function has undergone a revision, and its design and use are slightly different from the original design. This document mainly introduces Colocation Join's principle, implementation, usage and precautions. - -## Noun Interpretation - -* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. -* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. -* Colocation Group (CG): A CG contains one or more tables. Tables within the same group have the same Colocation Group Schema and the same data fragmentation distribution. -* Colocation Group Schema (CGS): Used to describe table in a CG and general Schema information related to Colocation. Including bucket column type, bucket number and copy number. - -## Principle - -The Colocation Join function is to make a CG of a set of tables with the same CGS. Ensure that the corresponding data fragments of these tables will fall on the same BE node. When tables in CG perform Join operations on bucket columns, local data Join can be directly performed to reduce data transmission time between nodes. - -The data of a table will eventually fall into a barrel according to the barrel column value Hash and the number of barrels modeled. Assuming that the number of buckets in a table is 8, there are eight buckets `[0, 1, 2, 3, 4, 5, 6, 7] `Buckets'. We call such a sequence a `Buckets Sequence`. Each Bucket has one or more Tablets. When a table is a single partitioned table, there is only one Tablet in a Bucket. If it is a multi-partition table, there will be more than one. - -In order for a table to have the same data distribution, the table in the same CG must ensure the following attributes are the same: - -1. Barrel row and number of barrels - - Bucket column, that is, the column specified in `DISTRIBUTED BY HASH (col1, col2,...)'in the table building statement. Bucket columns determine which column values are used to Hash data from a table into different Tablets. Tables in the same CG must ensure that the type and number of barrel columns are identical, and the number of barrels is identical, so that the data fragmentation of multiple tables can be controlled one by one. - -2. Number of copies - - The number of copies of all partitions of all tables in the same CG must be the same. If inconsistent, there may be a copy of a Tablet, and there is no corresponding copy of other table fragments on the same BE. - -Tables in the same CG do not require consistency in the number, scope, and type of partition columns. - -After fixing the number of bucket columns and buckets, the tables in the same CG will have the same Buckets Sequnce. The number of replicas determines the number of replicas of Tablets in each bucket, which BE they are stored on. Suppose that Buckets Sequnce is `[0, 1, 2, 3, 4, 5, 6, 7] `, and that BE nodes have `[A, B, C, D] `4. A possible distribution of data is as follows: - -``` -+---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ -| 0 | | 1 | | 2 | | 3 | | 4 | | 5 | | 6 | | 7 | -+---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ -| A | | B | | C | | D | | A | | B | | C | | D | -| | | | | | | | | | | | | | | | -| B | | C | | D | | A | | B | | C | | D | | A | -| | | | | | | | | | | | | | | | -| C | | D | | A | | B | | C | | D | | A | | B | -+---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ -``` - -The data of all tables in CG will be uniformly distributed according to the above rules, which ensures that the data with the same barrel column value are on the same BE node, and local data Join can be carried out. - -## Usage - -### Establishment of tables - -When creating a table, you can specify the attribute `"colocate_with"="group_name"` in `PROPERTIES`, which means that the table is a Colocation Join table and belongs to a specified Colocation Group. - -Examples: - -``` -CREATE TABLE tbl (k1 int, v1 int sum) -DISTRIBUTED BY HASH(k1) -BUCKETS 8 -PROPERTIES( - "colocate_with" = "group1" -); -``` - -If the specified group does not exist, Doris automatically creates a group that contains only the current table. If the Group already exists, Doris checks whether the current table satisfies the Colocation Group Schema. If satisfied, the table is created and added to the Group. At the same time, tables create fragments and replicas based on existing data distribution rules in Groups. -Group belongs to a database, and its name is unique in a database. Internal storage is the full name of Group `dbId_groupName`, but users only perceive groupName. - -### Delete table - -When the last table in Group is deleted completely (deleting completely means deleting from the recycle bin). Usually, when a table is deleted by the `DROP TABLE` command, it will be deleted after the default one-day stay in the recycle bin, and the group will be deleted automatically. - -### View Group - -The following command allows you to view the existing Group information in the cluster. - -``` -SHOW PROC '/colocation_group'; - -+-------------+--------------+--------------+------------+----------------+----------+----------+ -| GroupId | GroupName | TableIds | BucketsNum | ReplicationNum | DistCols | IsStable | -+-------------+--------------+--------------+------------+----------------+----------+----------+ -| 10005.10008 | 10005_group1 | 10007, 10040 | 10 | 3 | int(11) | true | -+-------------+--------------+--------------+------------+----------------+----------+----------+ -``` - -* GroupId: The unique identity of a group's entire cluster, with DB ID in the first half and group ID in the second half. -* GroupName: The full name of Group. -* Tablet Ids: The group contains a list of Tables'ID. -* Buckets Num: Number of barrels. -* Replication Num: Number of copies. -* DistCols: Distribution columns, -* IsStable: Is the group stable (for the definition of stability, see section `Collocation replica balancing and repair'). - -You can further view the data distribution of a group by following commands: - -``` -SHOW PROC '/colocation_group/10005.10008'; - -+-------------+---------------------+ -| BucketIndex | BackendIds | -+-------------+---------------------+ -| 0 | 10004, 10002, 10001 | -| 1 | 10003, 10002, 10004 | -| 2 | 10002, 10004, 10001 | -| 3 | 10003, 10002, 10004 | -| 4 | 10002, 10004, 10003 | -| 5 | 10003, 10002, 10001 | -| 6 | 10003, 10004, 10001 | -| 7 | 10003, 10004, 10002 | -+-------------+---------------------+ -``` - -* BucketIndex: Subscript to the bucket sequence. -* Backend Ids: A list of BE node IDs where data fragments are located in buckets. - -> The above commands require AMDIN privileges. Normal user view is not supported at this time. - -### Modify Colocate Group - -You can modify the Colocation Group property of a table that has been created. Examples: - -`ALTER TABLE tbl SET ("colocate_with" = "group2");` - -* If the table has not previously specified a Group, the command checks the Schema and adds the table to the Group (if the Group does not exist, it will be created). -* If other groups are specified before the table, the command first removes the table from the original group and adds a new group (if the group does not exist, it will be created). - -You can also delete the Colocation attribute of a table by following commands: - -`ALTER TABLE tbl SET ("colocate_with" = "");` - -### Other related operations - -When an ADD PARTITION is added to a table with a Colocation attribute and the number of copies is modified, Doris checks whether the modification violates the Colocation Group Schema and rejects it if it does. - -## Colocation Duplicate Balancing and Repair - -Copy distribution of Colocation tables needs to follow the distribution specified in Group, so it is different from common fragmentation in replica repair and balancing. - -Group itself has a Stable attribute, when Stable is true, which indicates that all fragments of the table in the current Group are not changing, and the Colocation feature can be used normally. When Stable is false, it indicates that some tables in Group are being repaired or migrated. At this time, Colocation Join of related tables will degenerate into ordinary Join. - -### Replica Repair - -Copies can only be stored on specified BE nodes. So when a BE is unavailable (downtime, Decommission, etc.), a new BE is needed to replace it. Doris will first look for the BE with the lowest load to replace it. After replacement, all data fragments on the old BE in the Bucket will be repaired. During the migration process, Group is marked Unstable. - -### Duplicate Equilibrium - -Doris will try to distribute the fragments of the Collocation table evenly across all BE nodes. For the replica balancing of common tables, the granularity is single replica, that is to say, it is enough to find BE nodes with lower load for each replica alone. The equilibrium of the Colocation table is at the Bucket level, where all replicas within a Bucket migrate together. We adopt a simple equalization algorithm, which distributes Buckets Sequnce evenly on all BEs, regardless of the actual size of the replicas, but only according to the number of replicas. Specific algorithms can be referred to the code annotations in `ColocateTableBalancer.java`. - -> Note 1: Current Colocation replica balancing and repair algorithms may not work well for heterogeneous deployed Oris clusters. The so-called heterogeneous deployment, that is, the BE node's disk capacity, number, disk type (SSD and HDD) is inconsistent. In the case of heterogeneous deployment, small BE nodes and large BE nodes may store the same number of replicas. -> -> Note 2: When a group is in an Unstable state, the Join of the table in it will degenerate into a normal Join. At this time, the query performance of the cluster may be greatly reduced. If you do not want the system to balance automatically, you can set the FE configuration item `disable_colocate_balance` to prohibit automatic balancing. Then open it at the right time. (See Section `Advanced Operations` for details) - -## Query - -The Colocation table is queried in the same way as ordinary tables, and users do not need to perceive Colocation attributes. If the Group in which the Colocation table is located is in an Unstable state, it will automatically degenerate to a normal Join. - -Examples are given to illustrate: - -Table 1: - -``` -CREATE TABLE `tbl1` ( - `k1` date NOT NULL COMMENT "", - `k2` int(11) NOT NULL COMMENT "", - `v1` int(11) SUM NOT NULL COMMENT "" -) ENGINE=OLAP -AGGREGATE KEY(`k1`, `k2`) -PARTITION BY RANGE(`k1`) -( - PARTITION p1 VALUES LESS THAN ('2019-05-31'), - PARTITION p2 VALUES LESS THAN ('2019-06-30') -) -DISTRIBUTED BY HASH(`k2`) BUCKETS 8 -PROPERTIES ( - "colocate_with" = "group1" -); -``` - -Table 2: - -``` -CREATE TABLE `tbl2` ( - `k1` datetime NOT NULL COMMENT "", - `k2` int(11) NOT NULL COMMENT "", - `v1` double SUM NOT NULL COMMENT "" -) ENGINE=OLAP -AGGREGATE KEY(`k1`, `k2`) -DISTRIBUTED BY HASH(`k2`) BUCKETS 8 -PROPERTIES ( - "colocate_with" = "group1" -); -``` - -View the query plan: - -``` -DESC SELECT * FROM tbl1 INNER JOIN tbl2 ON (tbl1.k2 = tbl2.k2); - -+----------------------------------------------------+ -| Explain String | -+----------------------------------------------------+ -| PLAN FRAGMENT 0 | -| OUTPUT EXPRS:`tbl1`.`k1` | | -| PARTITION: RANDOM | -| | -| RESULT SINK | -| | -| 2:HASH JOIN | -| | join op: INNER JOIN | -| | hash predicates: | -| | colocate: true | -| | `tbl1`.`k2` = `tbl2`.`k2` | -| | tuple ids: 0 1 | -| | | -| |----1:OlapScanNode | -| | TABLE: tbl2 | -| | PREAGGREGATION: OFF. Reason: null | -| | partitions=0/1 | -| | rollup: null | -| | buckets=0/0 | -| | cardinality=-1 | -| | avgRowSize=0.0 | -| | numNodes=0 | -| | tuple ids: 1 | -| | | -| 0:OlapScanNode | -| TABLE: tbl1 | -| PREAGGREGATION: OFF. Reason: No AggregateInfo | -| partitions=0/2 | -| rollup: null | -| buckets=0/0 | -| cardinality=-1 | -| avgRowSize=0.0 | -| numNodes=0 | -| tuple ids: 0 | -+----------------------------------------------------+ -``` - -If Colocation Join works, the Hash Join Node will show `colocate: true`。 - -If not, the query plan is as follows: - -``` -+----------------------------------------------------+ -| Explain String | -+----------------------------------------------------+ -| PLAN FRAGMENT 0 | -| OUTPUT EXPRS:`tbl1`.`k1` | | -| PARTITION: RANDOM | -| | -| RESULT SINK | -| | -| 2:HASH JOIN | -| | join op: INNER JOIN (BROADCAST) | -| | hash predicates: | -| | colocate: false, reason: group is not stable | -| | `tbl1`.`k2` = `tbl2`.`k2` | -| | tuple ids: 0 1 | -| | | -| |----3:EXCHANGE | -| | tuple ids: 1 | -| | | -| 0:OlapScanNode | -| TABLE: tbl1 | -| PREAGGREGATION: OFF. Reason: No AggregateInfo | -| partitions=0/2 | -| rollup: null | -| buckets=0/0 | -| cardinality=-1 | -| avgRowSize=0.0 | -| numNodes=0 | -| tuple ids: 0 | -| | -| PLAN FRAGMENT 1 | -| OUTPUT EXPRS: | -| PARTITION: RANDOM | -| | -| STREAM DATA SINK | -| EXCHANGE ID: 03 | -| UNPARTITIONED | -| | -| 1:OlapScanNode | -| TABLE: tbl2 | -| PREAGGREGATION: OFF. Reason: null | -| partitions=0/1 | -| rollup: null | -| buckets=0/0 | -| cardinality=-1 | -| avgRowSize=0.0 | -| numNodes=0 | -| tuple ids: 1 | -+----------------------------------------------------+ -``` - -The HASH JOIN node displays the corresponding reason: `colocate: false, reason: group is not stable`. At the same time, an EXCHANGE node will be generated. - - -## Advanced Operations - -### FE Configuration Item - -* disable\_colocate\_relocate - -Whether to close Doris's automatic Colocation replica repair. The default is false, i.e. not closed. This parameter only affects the replica repair of the Colocation table, but does not affect the normal table. - -* disable\_colocate\_balance - -Whether to turn off automatic Colocation replica balancing for Doris. The default is false, i.e. not closed. This parameter only affects the replica balance of the Collocation table, but does not affect the common table. - -User can set these configurations at runtime. See `HELP ADMIN SHOW CONFIG;` and `HELP ADMIN SET CONFIG;`. - -* disable\_colocate\_join - -Whether to turn off the Colocation Join function or not. In 0.10 and previous versions, the default is true, that is, closed. In a later version, it will default to false, that is, open. - -* use\_new\_tablet\_scheduler - -In 0.10 and previous versions, the new replica scheduling logic is incompatible with the Colocation Join function, so in 0.10 and previous versions, if `disable_colocate_join = false`, you need to set `use_new_tablet_scheduler = false`, that is, close the new replica scheduler. In later versions, `use_new_tablet_scheduler` will be equal to true. - -###HTTP Restful API - -Doris provides several HTTP Restful APIs related to Colocation Join for viewing and modifying Colocation Group. - -The API is implemented on the FE side and accessed using `fe_host: fe_http_port`. ADMIN privileges are required. - -1. View all Colocation information for the cluster - - ``` - GET /api/colocate - - Return the internal Colocation info in JSON format: - - { - "colocate_meta": { - "groupName2Id": { - "g1": { - "dbId": 10005, - "grpId": 10008 - } - }, - "group2Tables": {}, - "table2Group": { - "10007": { - "dbId": 10005, - "grpId": 10008 - }, - "10040": { - "dbId": 10005, - "grpId": 10008 - } - }, - "group2Schema": { - "10005.10008": { - "groupId": { - "dbId": 10005, - "grpId": 10008 - }, - "distributionColTypes": [{ - "type": "INT", - "len": -1, - "isAssignedStrLenInColDefinition": false, - "precision": 0, - "scale": 0 - }], - "bucketsNum": 10, - "replicationNum": 2 - } - }, - "group2BackendsPerBucketSeq": { - "10005.10008": [ - [10004, 10002], - [10003, 10002], - [10002, 10004], - [10003, 10002], - [10002, 10004], - [10003, 10002], - [10003, 10004], - [10003, 10004], - [10003, 10004], - [10002, 10004] - ] - }, - "unstableGroups": [] - }, - "status": "OK" - } - ``` -2. Mark Group as Stable or Unstable - - * Mark as Stable - - ``` - POST /api/colocate/group_stable?db_id=10005&group_id=10008 - - Returns: 200 - ``` - - * Mark as Unstable - - ``` - DELETE /api/colocate/group_stable?db_id=10005&group_id=10008 - - Returns: 200 - ``` - -3. Setting Data Distribution for Group - - The interface can force the number distribution of a group. - - ``` - POST /api/colocate/bucketseq?db_id=10005&group_id= 10008 - - Body: - [[10004,10002],[10003,10002],[10002,10004],[10003,10002],[10002,10004],[10003,10002],[10003,10004],[10003,10004],[10003,10004],[10002,10004]] - - Returns: 200 - ``` - Body is a Buckets Sequence represented by a nested array and the ID of the BE where the fragments are distributed in each Bucket. - - Note that using this command, you may need to set the FE configuration `disable_colocate_relocate` and `disable_colocate_balance` to true. That is to shut down the system for automatic Colocation replica repair and balancing. Otherwise, it may be automatically reset by the system after modification. diff --git a/docs/documentation/en/administrator-guide/config/fe_config_en.md b/docs/documentation/en/administrator-guide/config/fe_config_en.md deleted file mode 100644 index ab915a0ba81787..00000000000000 --- a/docs/documentation/en/administrator-guide/config/fe_config_en.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# Configuration - -## brpc_max_body_size - - This configuration is mainly used to modify the parameter max_body_size of brpc. The default configuration is 64M. It usually occurs in multi distinct + no group by + exceeds 1t data. In particular, if you find that the query is stuck, and be appears the word "body size is too large" in log. - - Because this is a brpc configuration, users can also directly modify this parameter on-the-fly by visiting ```http://host:brpc_port/flags``` - -## max_running_txn_num_per_db - -   This configuration is mainly used to control the number of concurrent load job in the same db. The default configuration is 100. When the number of concurrent load job exceeds the configured value, the load which is synchronously executed will fail, such as stream load. The load which is asynchronously will always be in a pending state such as broker load. - -   It is generally not recommended to change this property. If the current load concurrency exceeds this value, you need to first check if a single load job is too slow, or if there are too many small files, there is no problem of load after merging those small files. - -   Error information such as: current running txns on db xxx is xx, larger than limit xx. The above info is related by this property. diff --git a/docs/documentation/en/administrator-guide/config/index.rst b/docs/documentation/en/administrator-guide/config/index.rst deleted file mode 100644 index 9681f6000ebb4d..00000000000000 --- a/docs/documentation/en/administrator-guide/config/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============== -Configuration -============== - -.. toctree:: - :maxdepth: 1 - :glob: - - * diff --git a/docs/documentation/en/administrator-guide/dynamic-partition_EN.md b/docs/documentation/en/administrator-guide/dynamic-partition_EN.md deleted file mode 100644 index 3dc9ef12bc452a..00000000000000 --- a/docs/documentation/en/administrator-guide/dynamic-partition_EN.md +++ /dev/null @@ -1,185 +0,0 @@ - - -# Dynamic Partition - -Dynamic partition is a new feature introduced in Doris verion 0.12. It's designed to manage partition's Time-to-Life (TTL), reducing the burden on users. - -The original design, implementation and effect can be referred to [ISSUE 2262](https://github.com/apache/incubator-doris/issues/2262)。 - -Currently, the function of adding partitions dynamically is implemented, and the next version will support removing partitions dynamically. - -## Noun Interpretation - -* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. -* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. - -## Principle - -In some scenarios, the user will create partitions for the table according to the day and perform routine tasks regularly every day. In this case, the user needs to manually manage the partition, otherwise the data import may fail because the partition is forgot to create, which brings additional maintenance costs to the user. - -The design of implementation is that FE will starts a background thread that determines whether or not to start the thread and the scheduling frequency of the thread based on the parameters `dynamic_partition_enable` and `dynamic_partition_check_interval_seconds` in `fe.conf`. - -When create a olap table, the `dynamic_partition` properties will be assigned. FE will parse `dynamic_partition` properties and check the legitimacy of the input parameters firstly, and then persist the properties to FE metadata, register the table to the list of dynamic partition at the same time. Daemon thread will scan the dynamic partition list periodically according to the configuration parameters, -read dynamic partition properties of the table, and doing the task of adding partitions. The scheduling information of each time will be kept in the memory of FE. You can check whether the scheduling task is successful through `SHOW DYNAMIC PARTITION TABLES`. - -## Usage - -### Establishment of tables - -When creating a table, you can specify the attribute `dynamic_partition` in `PROPERTIES`, which means that the table is a dynamic partition table. - -Examples: - -``` -CREATE TABLE example_db.dynamic_partition -( -k1 DATE, -k2 INT, -k3 SMALLINT, -v1 VARCHAR(2048), -v2 DATETIME DEFAULT "2014-02-04 15:36:00" -) -ENGINE=olap -DUPLICATE KEY(k1, k2, k3) -PARTITION BY RANGE (k1) -( -PARTITION p1 VALUES LESS THAN ("2014-01-01"), -PARTITION p2 VALUES LESS THAN ("2014-06-01"), -PARTITION p3 VALUES LESS THAN ("2014-12-01") -) -DISTRIBUTED BY HASH(k2) BUCKETS 32 -PROPERTIES( -"storage_medium" = "SSD", -"dynamic_partition.enable" = "true" -"dynamic_partition.time_unit" = "DAY", -"dynamic_partition.end" = "3", -"dynamic_partition.prefix" = "p", -"dynamic_partition.buckets" = "32" - ); -``` -Create a dynamic partition table, specify enable dynamic partition features, take today is 2020-01-08 for example, at every time of scheduling, will create today and after 3 days in advance of four partitions -(if the partition is existed, the task will be ignored), partition name respectively according to the specified prefix `p20200108` `p20200109` `p20200110` `p20200111`, each partition to 32 the number of points barrels, each partition scope is as follows: -``` -[types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) -[types: [DATE]; keys: [2020-01-09]; ‥types: [DATE]; keys: [2020-01-10]; ) -[types: [DATE]; keys: [2020-01-10]; ‥types: [DATE]; keys: [2020-01-11]; ) -[types: [DATE]; keys: [2020-01-11]; ‥types: [DATE]; keys: [2020-01-12]; ) -``` - -### Enable Dynamic Partition Feature - -1. First of all, `dynamic_partition_enable=true` needs to be set in fe.conf, which can be specified by modifying the configuration file when the cluster starts up, or dynamically modified by HTTP interface at run time - -2. If you need to add dynamic partitioning properties to a table prior to version 0.12, you need to modify the properties of the table with the following command - -``` -ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32"); -``` - -### Disable Dynamic Partition Feature - -If you need to stop dynamic partitioning for all dynamic partitioning tables in the cluster, you need to set 'dynamic_partition_enable=true' in fe.conf - -If you need to stop dynamic partitioning for a specified table, you can modify the properties of the table with the following command - -``` -ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "false") -``` - -### Modify Dynamic Partition Properties - -You can modify the properties of the dynamic partition with the following command - -``` -ALTER TABLE dynamic_partition set("key" = "value") -``` - -### Check Dynamic Partition Table Scheduling Status - -You can further view the scheduling of dynamic partitioned tables by using the following command: - -``` -SHOW DYNAMIC PARTITION TABLES; - -+-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ -| TableName | Enable | TimeUnit | End | Prefix | Buckets | LastUpdateTime | LastSchedulerTime | State | Msg | -+-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ -| dynamic_partition | true | DAY | 3 | p | 32 | 2020-01-08 20:19:09 | 2020-01-08 20:19:34 | NORMAL | N/A | -+-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ -1 row in set (0.00 sec) - -``` - -* LastUpdateTime: The last time of modifying dynamic partition properties -* LastSchedulerTime: The last time of performing dynamic partition scheduling -* State: The state of the last execution of dynamic partition scheduling -* Msg: Error message for the last time dynamic partition scheduling was performed - -## Advanced Operation - -### FE Configuration Item - -* dynamic\_partition\_enable - - Whether to enable Doris's dynamic partition feature. The default value is false, which is off. This parameter only affects the partitioning operation of dynamic partition tables, not normal tables. - -* dynamic\_partition\_check\_interval\_seconds - - The execution frequency of dynamically partitioned threads, by default 3600(1 hour), which means scheduled every 1 hour. - -### HTTP Restful API - -Doris provides an HTTP Restful API for modifying dynamic partition configuration parameters at run time. - -The API is implemented in FE, user can access it by `fe_host:fe_http_port`.The operation needs admin privilege. - -1. Set dynamic_partition_enable to true or false - - * Set to true - - ``` - GET /api/_set_config?dynamic_partition_enable=true - - For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true - - Return Code：200 - ``` - - * Set to false - - ``` - GET /api/_set_config?dynamic_partition_enable=false - - For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=false - - Return Code：200 - ``` - -2. Set the scheduling frequency for dynamic partition - - * Set schedule frequency to 12 hours. - - ``` - GET /api/_set_config?dynamic_partition_check_interval_seconds=432000 - - For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000 - - Return Code：200 - ``` diff --git a/docs/documentation/en/administrator-guide/export_manual_EN.md b/docs/documentation/en/administrator-guide/export_manual_EN.md deleted file mode 100644 index 9a3f094d9cbe00..00000000000000 --- a/docs/documentation/en/administrator-guide/export_manual_EN.md +++ /dev/null @@ -1,184 +0,0 @@ - - -# Data export - -Export is a function provided by Doris to export data. This function can export user-specified table or partition data in text format to remote storage through Broker process, such as HDFS/BOS. - -This document mainly introduces the basic principles, usage, best practices and precautions of Export. - -## Noun Interpretation - -* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. -* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. -* Broker: Doris can manipulate files for remote storage through the Broker process. -* Tablet: Data fragmentation. A table is divided into multiple data fragments. - -## Principle - -After the user submits an Export job. Doris counts all Tablets involved in this job. These tablets are then grouped to generate a special query plan for each group. The query plan reads the data on the included tablet and then writes the data to the specified path of the remote storage through Broker. - -The overall mode of dispatch is as follows: - -``` -+--------+ -| Client | -+---+----+ - | 1. Submit Job - | -+---v--------------------+ -| FE | -| | -| +-------------------+ | -| | ExportPendingTask | | -| +-------------------+ | -| | 2. Generate Tasks -| +--------------------+ | -| | ExportExporingTask | | -| +--------------------+ | -| | -| +-----------+ | +----+ +------+ +---------+ -| | QueryPlan +----------------> BE +--->Broker+---> | -| +-----------+ | +----+ +------+ | Remote | -| +-----------+ | +----+ +------+ | Storage | -| | QueryPlan +----------------> BE +--->Broker+---> | -| +-----------+ | +----+ +------+ +---------+ -+------------------------+ 3. Execute Tasks - -``` - -1. The user submits an Export job to FE. -2. FE's Export scheduler performs an Export job in two stages: - 1. PENDING: FE generates Export Pending Task, sends snapshot command to BE, and takes a snapshot of all Tablets involved. And generate multiple query plans. - 2. EXPORTING: FE generates Export ExporingTask and starts executing the query plan. - -### query plan splitting - -The Export job generates multiple query plans, each of which scans a portion of the Tablet. The number of Tablets scanned by each query plan is specified by the FE configuration parameter `export_tablet_num_per_task`, which defaults to 5. That is, assuming a total of 100 Tablets, 20 query plans will be generated. Users can also specify this number by the job attribute `tablet_num_per_task`, when submitting a job. - -Multiple query plans for a job are executed sequentially. - -### Query Plan Execution - -A query plan scans multiple fragments, organizes read data in rows, batches every 1024 actions, and writes Broker to remote storage. - -The query plan will automatically retry three times if it encounters errors. If a query plan fails three retries, the entire job fails. - -Doris will first create a temporary directory named `doris_export_tmp_12345` (where `12345` is the job id) in the specified remote storage path. The exported data is first written to this temporary directory. Each query plan generates a file with an example file name: - -`export-data-c69fcf2b6db5420f-a96b94c1ff8bccef-1561453713822` - -Among them, `c69fcf2b6db5420f-a96b94c1ff8bccef` is the query ID of the query plan. ` 1561453713822` Timestamp generated for the file. - -When all data is exported, Doris will rename these files to the user-specified path. - -## Use examples - -Export's detailed commands can be passed through `HELP EXPORT;` Examples are as follows: - -``` -EXPORT TABLE db1.tbl1 -PARTITION (p1,p2) -TO "bos://bj-test-cmy/export/" -PROPERTIES -( - "column_separator"=",", - "exec_mem_limit"="2147483648", - "timeout" = "3600" -) -WITH BROKER "hdfs" -( - "username" = "user", - "password" = "passwd", -); -``` - -* `column_separator`: Column separator. The default is `\t`. -* `line_delimiter`: Line separator. The default is `\n`. -* `exec_mem_limit`: Represents the memory usage limitation of a query plan on a single BE in an Export job. Default 2GB. Unit bytes. -* `timeout`: homework timeout. Default 2 hours. Unit seconds. -* `tablet_num_per_task`: The maximum number of fragments allocated per query plan. The default is 5. - -After submitting a job, the job status can be imported by querying the `SHOW EXPORT'command. The results are as follows: - -``` - JobId: 14008 - State: FINISHED - Progress: 100% - TaskInfo: {"partitions":["*"],"exec mem limit":2147483648,"column separator":",","line delimiter":"\n","tablet num":1,"broker":"hdfs","coord num":1,"db":"default_cluster:db1","tbl":"tbl3"} - Path: bos://bj-test-cmy/export/ -CreateTime: 2019-06-25 17:08:24 - StartTime: 2019-06-25 17:08:28 -FinishTime: 2019-06-25 17:08:34 - Timeout: 3600 - ErrorMsg: N/A -``` - - -* JobId: The unique ID of the job -* State: Job status: - * PENDING: Jobs to be Scheduled - * EXPORING: Data Export - * FINISHED: Operation Successful - * CANCELLED: Job Failure -* Progress: Work progress. The schedule is based on the query plan. Assuming a total of 10 query plans have been completed, the progress will be 30%. -* TaskInfo: Job information in Json format: - * db: database name - * tbl: Table name - * partitions: Specify the exported partition. `*` Represents all partitions. - * exec MEM limit: query plan memory usage limit. Unit bytes. - * column separator: The column separator for the exported file. - * line delimiter: The line separator for the exported file. - * tablet num: The total number of tablets involved. - * Broker: The name of the broker used. - * Coord num: Number of query plans. -* Path: Export path on remote storage. -* CreateTime/StartTime/FinishTime: Creation time, start scheduling time and end time of jobs. -* Timeout: Job timeout. The unit is seconds. This time is calculated from CreateTime. -* Error Msg: If there is an error in the job, the cause of the error is shown here. - -## Best Practices - -### Splitting Query Plans - -How many query plans need to be executed for an Export job depends on the total number of Tablets and how many Tablets can be allocated for a query plan at most. Since multiple query plans are executed serially, the execution time of jobs can be reduced if more fragments are processed by one query plan. However, if the query plan fails (e.g., the RPC fails to call Broker, the remote storage jitters, etc.), too many tablets can lead to a higher retry cost of a query plan. Therefore, it is necessary to arrange the number of query plans and the number of fragments to be scanned for each query plan in order to balance the execution time and the success rate of execution. It is generally recommended that the amount of data scanned by a query plan be within 3-5 GB (the size and number of tables in a table can be viewed by `SHOW TABLET FROM tbl_name;`statement. - -### exec\_mem\_limit - -Usually, a query plan for an Export job has only two parts `scan`- `export`, and does not involve computing logic that requires too much memory. So usually the default memory limit of 2GB can satisfy the requirement. But in some scenarios, such as a query plan, too many Tablets need to be scanned on the same BE, or too many data versions of Tablets, may lead to insufficient memory. At this point, larger memory needs to be set through this parameter, such as 4 GB, 8 GB, etc. - -## Notes - -* It is not recommended to export large amounts of data at one time. The maximum amount of exported data recommended by an Export job is tens of GB. Excessive export results in more junk files and higher retry costs. -* If the amount of table data is too large, it is recommended to export it by partition. -* During the operation of the Export job, if FE restarts or cuts the master, the Export job will fail, requiring the user to resubmit. -* If the Export job fails, the `__doris_export_tmp_xxx` temporary directory generated in the remote storage and the generated files will not be deleted, requiring the user to delete them manually. -* If the Export job runs successfully, the `__doris_export_tmp_xxx` directory generated in the remote storage may be retained or cleared according to the file system semantics of the remote storage. For example, in Baidu Object Storage (BOS), after removing the last file in a directory through rename operation, the directory will also be deleted. If the directory is not cleared, the user can clear it manually. -* When the Export runs successfully or fails, the FE reboots or cuts, then some information of the jobs displayed by `SHOW EXPORT` will be lost and can not be viewed. -* Export jobs only export data from Base tables, not Rollup Index. -* Export jobs scan data and occupy IO resources, which may affect the query latency of the system. - -## Relevant configuration - -### FE - -* `expo_checker_interval_second`: Scheduling interval of Export job scheduler, default is 5 seconds. Setting this parameter requires restarting FE. -* `export_running_job_num_limit `: Limit on the number of Export jobs running. If exceeded, the job will wait and be in PENDING state. The default is 5, which can be adjusted at run time. -* `Export_task_default_timeout_second`: Export job default timeout time. The default is 2 hours. It can be adjusted at run time. -* `export_tablet_num_per_task`: The maximum number of fragments that a query plan is responsible for. The default is 5. diff --git a/docs/documentation/en/administrator-guide/http-actions/cancel-label_EN.md b/docs/documentation/en/administrator-guide/http-actions/cancel-label_EN.md deleted file mode 100644 index ba7cab419c5e59..00000000000000 --- a/docs/documentation/en/administrator-guide/http-actions/cancel-label_EN.md +++ /dev/null @@ -1,57 +0,0 @@ - - -# CANCEL LABEL -## description - NAME: - cancel_label: cancel a transaction with label - - SYNOPSIS - curl -u user:passwd -XPOST http://host:port/api/{db}/{label}/_cancel - - DESCRIPTION - - This is to cancel a transaction with specified label. - - RETURN VALUES - - Return a JSON format string: - - Status: - Success: cancel succeed - Others: cancel failed - Message: Error message if cancel failed - - ERRORS - -## example - - 1. Cancel the transaction with label "testLabel" on database "testDb" - - curl -u root -XPOST http://host:port/api/testDb/testLabel/_cancel - -## keyword - - CANCEL，LABEL - - - - - - diff --git a/docs/documentation/en/administrator-guide/http-actions/compaction-action_EN.md b/docs/documentation/en/administrator-guide/http-actions/compaction-action_EN.md deleted file mode 100644 index 3d0215873d3f24..00000000000000 --- a/docs/documentation/en/administrator-guide/http-actions/compaction-action_EN.md +++ /dev/null @@ -1,78 +0,0 @@ - - -# Compaction Action - -This API is used to view the overall compaction status of a BE node or the compaction status of a specified tablet. It can also be used to manually trigger Compaction. - -## View Compaction status - -### The overall compaction status of the node - -(TODO) - -### Specify the compaction status of the tablet - -``` -curl -X GET http://be_host:webserver_port/api/compaction/show?tablet_id=xxxx\&schema_hash=yyyy -``` - -If the tablet does not exist, an error in JSON format is returned: - -``` -{ - "status": "Fail", - "msg": "Tablet not found" -} -``` - -If the tablet exists, the result is returned in JSON format: - -``` -{ - "cumulative point": 50, - "last cumulative failure time": "2019-12-16 18:13:43.224", - "last base failure time": "2019-12-16 18:13:23.320", - "last cumu success time": "2019-12-16 18:12:15.110", - "last base success time": "2019-12-16 18:11:50.780", - "rowsets": [ - "[0-48] 10 DATA OVERLAPPING", - "[49-49] 2 DATA OVERLAPPING", - "[50-50] 0 DELETE NONOVERLAPPING", - "[51-51] 5 DATA OVERLAPPING" - ] -} -``` - -Explanation of results: - -* cumulative point: The version boundary between base and cumulative compaction. Versions before (excluding) points are handled by base compaction. Versions after (inclusive) are handled by cumulative compaction. -* last cumulative failure time: The time when the last cumulative compaction failed. After 10 minutes by default, cumulative compaction is attempted on the this tablet again. -* last base failure time: The time when the last base compaction failed. After 10 minutes by default, base compaction is attempted on the this tablet again. -* rowsets: The current rowsets collection of this tablet. [0-48] means a rowset with version 0-48. The second number is the number of segments in a rowset. The `DELETE` indicates the delete version. `OVERLAPPING` and `NONOVERLAPPING` indicates whether data between segments is overlap. - -### Examples - -``` -curl -X GET http://192.168.10.24:8040/api/compaction/show?tablet_id=10015\&schema_hash=1294206575 -``` - -## Manually trigger Compaction - -(TODO) diff --git a/docs/documentation/en/administrator-guide/http-actions/fe-get-log-file_EN.md b/docs/documentation/en/administrator-guide/http-actions/fe-get-log-file_EN.md deleted file mode 100644 index c51aa1ad9a884b..00000000000000 --- a/docs/documentation/en/administrator-guide/http-actions/fe-get-log-file_EN.md +++ /dev/null @@ -1,67 +0,0 @@ - - -# get\_log\_file - -To get FE log via HTTP - -## Types of FE log - -1. fe.audit.log (Audit log) - - The audit log records the all statements executed. Audit log's name format as follow: - - ``` - fe.audit.log # The latest audit log - fe.audit.log.20190603.1 # The historical audit log. The smaller the sequence number, the newer the log. - fe.audit.log.20190603.2 - fe.audit.log.20190602.1 - ... - ``` - -## Example - -1. Get the list of specified type of logs - - Example - - `curl -v -X HEAD -uuser:passwd http://fe_host:http_port/api/get_log_file?type=fe.audit.log` - - Returns: - - ``` - HTTP/1.1 200 OK - file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934} - content-type: text/html - connection: keep-alive - ``` - - In the header of result, the `file_infos` section saves the file list and file size in JSON format. - -2. Download files - - Example: - - ``` - curl -X GET -uuser:passwd http://fe_host:http_port/api/get_log_file?type=fe.audit.log\&file=fe.audit.log.20190528.1 - ``` - -## Notification - -Need ADMIN priviledge. diff --git a/docs/documentation/en/administrator-guide/http-actions/get-label-state_EN.md b/docs/documentation/en/administrator-guide/http-actions/get-label-state_EN.md deleted file mode 100644 index 49bb6ed782cb27..00000000000000 --- a/docs/documentation/en/administrator-guide/http-actions/get-label-state_EN.md +++ /dev/null @@ -1,52 +0,0 @@ - - -# GET LABEL STATE -## description - NAME: - get_label_state: get label's state - - SYNOPSIS - curl -u user:passwd http://host:port/api/{db}/{label}/_state - - DESCRIPTION - - Check the status of a transaction - - RETURN VALUES - - Return of JSON format string of the status of specified transaction: - Label: The specified label. - Status: Success or not of this request. - Message: Error messages - State: - UNKNOWN/PREPARE/COMMITTED/VISIBLE/ABORTED - - ERRORS - -## example - - 1. Get status of label "testLabel" on database "testDb" - - curl -u root http://host:port/api/testDb/testLabel/_state - -## keyword - - GET, LABEL, STATE - diff --git a/docs/documentation/en/administrator-guide/http-actions/index.rst b/docs/documentation/en/administrator-guide/http-actions/index.rst deleted file mode 100644 index 268e99b5a8fd78..00000000000000 --- a/docs/documentation/en/administrator-guide/http-actions/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============= -HTTP API -============= - -.. toctree:: - :maxdepth: 1 - :glob: - - * diff --git a/docs/documentation/en/administrator-guide/http-actions/restore-tablet_EN.md b/docs/documentation/en/administrator-guide/http-actions/restore-tablet_EN.md deleted file mode 100644 index d08bc1d3f3840d..00000000000000 --- a/docs/documentation/en/administrator-guide/http-actions/restore-tablet_EN.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# RESTORE TABLET -## description - - To restore the tablet data from trash dir on BE - - METHOD: POST - URI: http://be_host:be_http_port/api/restore_tablet?tablet_id=xxx&schema_hash=xxx - -## example - - curl -X POST "http://hostname:8088/api/restore_tablet?tablet_id=123456\&schema_hash=1111111" - -##keyword - - RESTORE,TABLET,RESTORE,TABLET diff --git a/docs/documentation/en/administrator-guide/index.rst b/docs/documentation/en/administrator-guide/index.rst deleted file mode 100644 index bbb7493e07c5f2..00000000000000 --- a/docs/documentation/en/administrator-guide/index.rst +++ /dev/null @@ -1,19 +0,0 @@ -======================== -Administrator Guide -======================== - -.. toctree:: - :hidden: - - load-data/index - alter-table/index - http-actions/index - operation/index - config/index - -.. toctree:: - :maxdepth: 1 - :glob: - - * - diff --git a/docs/documentation/en/administrator-guide/load-data/broker-load-manual_EN.md b/docs/documentation/en/administrator-guide/load-data/broker-load-manual_EN.md deleted file mode 100644 index 311fb458bb490b..00000000000000 --- a/docs/documentation/en/administrator-guide/load-data/broker-load-manual_EN.md +++ /dev/null @@ -1,498 +0,0 @@ - - -# Broker Load - -Broker load is an asynchronous import method, and the data source supported depends on the data source supported by the Broker process. - -Users need to create Broker load imports through MySQL protocol and check the import results by viewing the import commands. - -## Applicable scenarios - -* Source data in Broker accessible storage systems, such as HDFS. -* Data volumes range from tens to hundreds of GB. - -## Noun Interpretation - -1. Frontend (FE): Metadata and scheduling nodes of Doris system. In the import process, it is mainly responsible for the generation of import plan and the scheduling of import tasks. -2. Backend (BE): The computing and storage nodes of Doris system. In the import process, it is mainly responsible for ETL and storage of data. -3. Broker: Broker is an independent stateless process. It encapsulates the file system interface and provides Doris with the ability to read files in the remote storage system. -4. Plan: Import the execution plan, and BE executes the import execution plan to import data into Doris system. - -## Basic Principles - -After the user submits the import task, FE generates the corresponding plan and distributes the plan to several BEs according to the number of BEs and the size of the file. Each BE performs part of the import data. - -BE pulls data from Broker and imports it into the system after transforming the data. All BEs complete the import, and the FE decides whether the import is successful or not. - -``` - + - | 1. user create broker load - v - +----+----+ - | | - | FE | - | | - +----+----+ - | - | 2. BE etl and load the data - +--------------------------+ - | | | -+---v---+ +--v----+ +---v---+ -| | | | | | -| BE | | BE | | BE | -| | | | | | -+---+-^-+ +---+-^-+ +--+-^--+ - | | | | | | - | | | | | | 3. pull data from broker -+---v-+-+ +---v-+-+ +--v-+--+ -| | | | | | -|Broker | |Broker | |Broker | -| | | | | | -+---+-^-+ +---+-^-+ +---+-^-+ - | | | | | | -+---v-+-----------v-+----------v-+-+ -| HDFS/BOS/AFS cluster | -| | -+----------------------------------+ - -``` - -## Basic operations - -### Create a load - -Broker load create a data load job - -Grammar: - -``` -LOAD LABEL db_name.label_name -(data_desc, ...) -WITH BROKER broker_name broker_properties -[PROPERTIES (key1=value1, ... )] - -* data_desc: - - DATA INFILE ('file_path', ...) - [NEGATIVE] - INTO TABLE tbl_name - [PARTITION (p1, p2)] - [COLUMNS TERMINATED BY separator ] - [(col1, ...)] - [SET (k1=f1(xx), k2=f2(xx))] - [WHERE predicate] - -* broker_properties: - - (key1=value1, ...) -``` -Examples: - -``` -LOAD LABEL db1.label1 -( - DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file1") - INTO TABLE tbl1 - COLUMNS TERMINATED BY "," - (tmp_c1,tmp_c2) - SET - ( - id=tmp_c2, - name=tmp_c1) - ), - DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file2") - INTO TABLE tbl2 - COLUMNS TERMINATED BY "," - (col1, col2) - where col1 > 1 -) -WITH BROKER 'broker' -( - "username"="user", - "password"="pass" -) -PROPERTIES -( - "timeout" = "3600" -); - -``` - -Create the imported detailed grammar execution ``HELP BROKER LOAD `` View grammar help. This paper mainly introduces the parametric meaning and points for attention in Broker load's creation import grammar. - -#### Label - -Identity of import task. Each import task has a unique Label within a single database. Label is a user-defined name in the import command. With this Label, users can view the execution of the corresponding import task. - -Another function of Label is to prevent users from repeatedly importing the same data. **It is strongly recommended that users use the same label for the same batch of data. Thus, repeated requests for the same batch of data can only be accepted once, guaranteeing at-Most-One semantics** - -When the corresponding import job status of Label is CANCELLED, it can be used again to submit the import job. - -#### Data Description Class Parameters - -Data description class parameters mainly refer to the parameters belonging to ``data_desc`` in Broker load creating import statements. Each group of ```data_desc``` mainly describes the data source address, ETL function, target table and partition information involved in this import. - -The following is a detailed explanation of some parameters of the data description class: - -+ Multi-table import - - Broker load supports a single import task involving multiple tables, and each Broker load import task can implement multiple tables import by declaring multiple tables in multiple ``data_desc``. Each individual ```data_desc``` can also specify the data source address belonging to the table. Broker load guarantees atomic success or failure between multiple tables imported at a single time. - -+ negative - - ```data_desc``` can also set up data fetching and anti-importing. This function is mainly used when aggregated columns in data tables are of SUM type. If you want to revoke a batch of imported data. The `negative'parameter can be used as a batch of data. Doris automatically retrieves this batch of data on aggregated columns to eliminate the same batch of data. - -+ partition - - In `data_desc`, you can specify the partition information of the table to be imported, but it will not be imported if the data to be imported does not belong to the specified partition. At the same time, data that does not specify a Partition is considered error data. - -+ where predicate - - The where statement in ```data_desc``` is responsible for filtering the data that has been transformed. The unselected rows which is filtered by where predicate will not be calculated in ```max_filter_ratio``` . If there are more then one where predicate of the same table , the multi where predicate will be merged from different ```data_desc``` and the policy is AND. - -#### Import job parameters - -Import job parameters mainly refer to the parameters in Broker load creating import statement that belong to ``opt_properties``. Import operation parameters act on the whole import operation. - -The following is a detailed explanation of some parameters of the import operation parameters: - -+ time out - - The time-out of the import job (in seconds) allows the user to set the time-out of each import by himself in ``opt_properties``. If the import task is not completed within the set timeout time, it will be cancelled by the system and become CANCELLED. The default import timeout for Broker load is 4 hours. - - Usually, the user does not need to manually set the timeout of the import task. When the import cannot be completed within the default timeout time, the task timeout can be set manually. - - > Recommended timeout - > - > Total File Size (MB) / Slowest Import Speed (MB/s) > timeout >((MB) * Number of tables to be imported and related Roll up tables) / (10 * Number of concurrent imports) - - > The concurrency of imports can be seen in the final configuration of the import system in the document. The current import speed limit is 10MB/s in 10 of the formulas. - - > For example, a 1G data to be imported contains three Rollup tables, and the current concurrency of imports is 3. The minimum value of timeout is ```(1 * 1024 * 3) / (10 * 3) = 102 seconds.``` - - Because the machine environment of each Doris cluster is different and the concurrent query tasks of the cluster are different, the slowest import speed of the user Doris cluster requires the user to guess the import task speed according to the history. - -+ max\_filter\_ratio - - The maximum tolerance rate of the import task is 0 by default, and the range of values is 0-1. When the import error rate exceeds this value, the import fails. - - If the user wishes to ignore the wrong row, the import can be successful by setting this parameter greater than 0. - - The calculation formula is as follows: - - ``` (dpp.abnorm.ALL / (dpp.abnorm.ALL + dpp.norm.ALL ) ) > max_filter_ratio ``` - - ``` dpp.abnorm.ALL``` denotes the number of rows whose data quality is not up to standard. Such as type mismatch, column mismatch, length mismatch and so on. - - ``` dpp.norm.ALL ``` refers to the number of correct data in the import process. The correct amount of data for the import task can be queried by the ``SHOW LOAD`` command. - - The number of rows in the original file = `dpp.abnorm.ALL + dpp.norm.ALL` - -* exec\_mem\_limit - - Memory limit. Default is 2GB. Unit is Bytes. - -+ strict\_mode - - Broker load can use `strict mode`. Use ```properties ("strict_mode" = "true")``` to enable `strict mode`, default is false - - The strict mode means that the column type conversion in the import process is strictly filtered. The strategy of strict filtering is as follows: - - 1. For column type conversion, if strict mode is true, the wrong data will be filtered. Error data here refers to the kind of data that the original data is not null and the result is null after participating in column type conversion. - - 2. Strict mode does not affect the imported column when it is generated by a function transformation. - - 3. For a column type imported that contains scope restrictions, strict mode does not affect it if the original data can normally pass type conversion, but can not pass scope restrictions. For example, if the type is decimal (1,0) and the original data is 10, it falls within the scope of type conversion but not column declaration. This data strict has no effect on it. - -#### Import Relation between strict mode source data - -Here's an example of a column type TinyInt - -> Note: When columns in a table allow null values to be imported - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|---------| -|null | \N | N/A | true or false | NULL| -|not null | aaa or 2000 | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 | 1 | true or false | correct data| - -Here's an example of column type Decimal (1,0) - -> Note: When columns in a table allow null values to be imported - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|--------| -|null | \N | N/A | true or false | NULL| -|not null | aaa | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 or 10 | 1 | true or false | correct data| - -> Note: Although 10 is a value beyond the range, strict mode does not affect it because its type meets the requirements of decimal. 10 will eventually be filtered in other ETL processes. But it will not be filtered by strict mode. - -### View load - -Broker load import mode is asynchronous, so the user must create the imported Label record and use Label in the **view Import command to view the import result**. View import commands are common in all import modes. The specific syntax can be `HELP SHOW LOAD`. - -Examples: - -``` -mysql> show load order by createtime desc limit 1\G -*************************** 1. row *************************** - JobId: 76391 - Label: label1 - State: FINISHED - Progress: ETL:N/A; LOAD:100% - Type: BROKER - EtlInfo: dpp.abnorm.ALL=15; dpp.norm.ALL=28133376 - TaskInfo: cluster:N/A; timeout(s):10800; max_filter_ratio:5.0E-5 - ErrorMsg: N/A - CreateTime: 2019-07-27 11:46:42 - EtlStartTime: 2019-07-27 11:46:44 - EtlFinishTime: 2019-07-27 11:46:44 - LoadStartTime: 2019-07-27 11:46:44 -LoadFinishTime: 2019-07-27 11:50:16 - URL: http://192.168.1.1:8040/api/_load_error_log?file=__shard_4/error_log_insert_stmt_4bb00753932c491a-a6da6e2725415317_4bb00753932c491a_a6da6e2725415317 - JobDetails: {"Unfinished backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"ScannedRows":2390016,"TaskNumber":1,"All backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"FileNumber":1,"FileSize":1073741824} -``` - -The following is mainly about the significance of viewing the parameters in the return result set of the import command: - -+ JobId - - The unique ID of the import task is different for each import task, which is automatically generated by the system. Unlike Label, JobId will never be the same, while Label can be reused after the import task fails. - -+ Label - - Identity of import task. - -+ State - - Import the current phase of the task. In the Broker load import process, PENDING and LOADING are the two main import states. If the Broker load is in the PENDING state, it indicates that the current import task is waiting to be executed; the LOADING state indicates that it is executing. - - There are two final stages of the import task: CANCELLED and FINISHED. When Load job is in these two stages, the import is completed. CANCELLED is the import failure, FINISHED is the import success. - -+ Progress - - Import the progress description of the task. There are two kinds of progress: ETL and LOAD, which correspond to the two stages of the import process, ETL and LOADING. At present, Broker load only has the LOADING stage, so ETL will always be displayed as `N/A`. - - The progress range of LOAD is 0-100%. - - ``` LOAD Progress = Number of tables currently completed / Number of tables designed for this import task * 100%``` - - **If all import tables complete the import, then the progress of LOAD is 99%** import enters the final effective stage, and the progress of LOAD will only be changed to 100% after the entire import is completed. - - Import progress is not linear. So if there is no change in progress over a period of time, it does not mean that the import is not being implemented. - -+ Type - - Types of import tasks. The type value of Broker load is only BROKER. -+ Etlinfo - - It mainly shows the imported data quantity indicators `unselected.rows`, `dpp.norm.ALL` and `dpp.abnorm.ALL`. The first value shows the rows which has been filtered by where predicate. Users can verify that the error rate of the current import task exceeds max\_filter\_ratio based on these two indicators. - -+ TaskInfo - - It mainly shows the current import task parameters, that is, the user-specified import task parameters when creating the Broker load import task, including `cluster`, `timeout`, and `max_filter_ratio`. - -+ ErrorMsg - - When the import task status is CANCELLED, the reason for the failure is displayed in two parts: type and msg. If the import task succeeds, the `N/A` is displayed. - - The value meaning of type: - - ``` - USER_CANCEL: User Canceled Tasks - ETL_RUN_FAIL：Import tasks that failed in the ETL phase - ETL_QUALITY_UNSATISFIED：Data quality is not up to standard, that is, the error rate exceedsmax_filter_ratio - LOAD_RUN_FAIL：Import tasks that failed in the LOADING phase - TIMEOUT：Import task not completed in overtime - UNKNOWN：Unknown import error - ``` - -+ CreateTime /EtlStartTime /EtlFinishTime /LoadStartTime /LoadFinishTime - - These values represent the creation time of the import, the beginning time of the ETL phase, the completion time of the ETL phase, the beginning time of the Loading phase and the completion time of the entire import task, respectively. - - Broker load import has no ETL stage, so its EtlStartTime, EtlFinishTime, LoadStartTime are set to the same value. - - Import tasks stay in CreateTime for a long time, while LoadStartTime is N/A, which indicates that import tasks are heavily stacked at present. Users can reduce the frequency of import submissions. - - ``` - LoadFinishTime - CreateTime = Time consumed by the entire import task - LoadFinishTime - LoadStartTime = The entire Broker load import task execution time = the time consumed by the entire import task - the time the import task waits - ``` - -+ URL - - The error data sample of the import task can be obtained by accessing the URL address. When there is no error data in this import, the URL field is N/A. - -+ JobDetails - - Display some details of the running status of the job. Including file number, total file size(Bytes), num of sub tasks, scanned rows, related backend ids and unfinished backend ids. - - ``` - {"Unfinished backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"ScannedRows":2390016,"TaskNumber":1,"All backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"FileNumber":1,"FileSize":1073741824} - ``` - - This info will be updated every 5 seconds. the ScannedRows only for displaying the job progress, not indicate the real numbers. - -### Cancel load - -When the Broker load job status is not CANCELLED or FINISHED, it can be manually cancelled by the user. When canceling, you need to specify a Label for the import task to be cancelled. Canceling Import command syntax can perform `HELP CANCEL LOAD` view. - -## Relevant System Configuration - -### FE configuration - -The following configurations belong to the Broker load system-level configuration, which acts on all Broker load import tasks. Configuration values are adjusted mainly by modifying `fe.conf`. - -+ min\_bytes\_per\_broker\_scanner/max\_bytes\_per\_broker\_scanner/max\_broker\_concurrency - - The first two configurations limit the minimum and maximum amount of data processed by a single BE. The third configuration limits the maximum number of concurrent imports for a job. The minimum amount of data processed, the maximum number of concurrencies, the size of source files and the number of BEs in the current cluster **together determine the concurrency of this import**. - - ``` - The number of concurrent imports = Math. min (source file size / minimum throughput, maximum concurrency, current number of BE nodes) - Processing capacity of this import of a single BE = source file size / concurrency of this import - ``` - - Usually the maximum amount of data supported by an import job is `max_bytes_per_broker_scanner * number of BE nodes`. If you need to import a larger amount of data, you need to adjust the size of the `max_bytes_per_broker_scanner` parameter appropriately. - -Default configuration: - - ``` - Parameter name: min_bytes_per_broker_scanner, default 64MB, unit bytes. - Parameter name: max_broker_concurrency, default 10. - Parameter name: max_bytes_per_broker_scanner, default 3G, unit bytes. - ``` - -## Best Practices - -### Application scenarios - -The most appropriate scenario to use Broker load is the scenario of raw data in a file system (HDFS, BOS, AFS). Secondly, since Broker load is the only way of asynchronous import in a single import, users can also consider using Broker load if they need to use asynchronous access in importing large files. - -### Data volume - -We will only discuss the case of a single BE. If the user cluster has more than one BE, the amount of data in the heading below should be multiplied by the number of BEs. For example, if the user has three BEs, then the number below 3G (including) should be multiplied by 3, that is, under 9G (including). - -+ Below 3G (including) - - Users can submit Broker load to create import requests directly. - -+ Over 3G - - Since the maximum processing capacity of a single imported BE is 3G, the imported files over 3G need to be imported by adjusting the import parameters of Broker load to achieve the import of large files. - - 1. Modify the maximum number of scans and concurrency of a single BE according to the current number of BEs and the size of the original file. - - ``` - Modify the configuration in fe.conf - - max_broker_concurrency = BE number - The amount of data processed by a single BE for the current import task = the original file size / max_broker_concurrency - Max_bytes_per_broker_scanner >= the amount of data processed by a single BE of the current import task - - For example, a 100G file with 10 BEs in the cluster - max_broker_concurrency = 10 - Max================ - - ``` - - After modification, all BEs process import tasks concurrently, and each BE processes part of the original file. - - *Note: The configurations in both FEs are system configurations, that is to say, their modifications work on all Broker load tasks.* - - 2. Customize the timeout time of the current import task when creating the import - - ``` - Current import task single BE processing data volume / user Doris cluster slowest import speed (MB/s) >= current import task timeout time >= current import task single BE processing data volume / 10M/s - - For example, a 100G file with 10 BEs in the cluster - Timeout > 1000s = 10G / 10M /s - - ``` - - 3. When the user finds that the timeout time calculated in the second step exceeds the default maximum time-out time for importing the system by 4 hours. - - At this time, it is not recommended that users directly increase the maximum time-out to solve the problem. If the single import time exceeds the default maximum import timeout of 4 hours, it is better to solve the problem by splitting the file to be imported and importing it several times. The main reason is that if a single import exceeds 4 hours, the time cost of retry after import failure is very high. - - The maximum amount of imported file data expected by the Doris cluster can be calculated by the following formula: - - ``` - Expected maximum imported file data = 14400s * 10M / s * BE number - For example, the BE number of clusters is 10. - Expected maximum imported file data volume = 14400 * 10M / s * 10 = 1440000M 1440G - - Note: The average user's environment may not reach the speed of 10M/s, so it is recommended that more than 500G files be split and imported. - - ``` - -### Complete examples - -Data situation: User data in HDFS, file address is hdfs://abc.com:8888/store_sales, HDFS authentication user name is root, password is password, data size is about 30G, hope to import into database bj_sales table store_sales. - -Cluster situation: The number of BEs in the cluster is about 3, and the Broker name is broker. - -+ Step 1: After the calculation of the above method, the single BE import quantity is 10G, then the configuration of FE needs to be modified first, and the maximum amount of single BE import is changed to: - - ``` - max_bytes_per_broker_scanner = 10737418240 - - ``` - -+ Step 2: Calculated, the import time is about 1000s, which does not exceed the default timeout time. No custom timeout time for import can be configured. - -+ Step 3: Create import statements - - ``` - LOAD LABEL bj_sales.store_sales_broker_load_01 - ( - DATA INFILE("hdfs://abc.com:8888/store_sales") - INTO TABLE store_sales - ) - WITH BROKER 'broker' - ("username"="root", "password"="password"); - ``` - -## Common Questions - -* failed with : `Scan bytes per broker scanner exceed limit:xxx` - - Refer to the Best Practices section of the document to modify the FE configuration items `max_bytes_per_broker_scanner` and `max_broker_concurrency'.` - -* failed with ：`failed to send batch` or `TabletWriter add batch with unknown id` - - Refer to **General System Configuration** in **BE Configuration** in the Import Manual (./load-manual.md), and modify `query_timeout` and `streaming_load_rpc_max_alive_time_sec` appropriately. - -* failed with : `LOAD_RUN_FAIL; msg: Invalid Column Name: xxx` -     -     If it is PARQUET or ORC format data, you need to keep the column names in the file header consistent with the column names in the doris table, such as: -     `` ` -     (tmp_c1, tmp_c2) -     SET -     ( -         id = tmp_c2, -         name = tmp_c1 -     ) -     `` ` -     Represents getting the column with (tmp_c1, tmp_c2) as the column name in parquet or orc, which is mapped to the (id, name) column in the doris table. If set is not set, the column names in the column are used as the mapping relationship. - -     Note: If the orc file directly generated by some hive versions is used, the table header in the orc file is not the column name in the hive meta, but (_col0, _col1, _col2, ...), which may cause the Invalid Column Name error, then You need to use set for mapping. diff --git a/docs/documentation/en/administrator-guide/load-data/delete-manual_EN.md b/docs/documentation/en/administrator-guide/load-data/delete-manual_EN.md deleted file mode 100644 index 04c980e3593641..00000000000000 --- a/docs/documentation/en/administrator-guide/load-data/delete-manual_EN.md +++ /dev/null @@ -1,181 +0,0 @@ - - -# Delete - -Unlike other import methods, delete is a synchronization process. Similar to insert into, all delete operations are an independent import job in Doris. Generally, delete statements need to specify tables, partitions and delete conditions to tell which data to be deleted, and the data on base index and rollup index will be deleted at the same time. - - -## Syntax - -The delete statement's syntax is as follows： - -``` -DELETE FROM table_name [PARTITION partition_name] -WHERE -column_name1 op value[ AND column_name2 op value ...]; -``` - -example 1： - -``` -DELETE FROM my_table PARTITION p1 WHERE k1 = 3; -``` - -example 2: - -``` -DELETE FROM my_table PARTITION p1 WHERE k1 < 3 AND k2 = "abc"; -``` - -The following describes the parameters used in the delete statement: - -* PARTITION - - The target partition of the delete statement. If not specified, the table must be a single partition table, otherwise it cannot be deleted - -* WHERE - - The conditiona of the delete statement. All delete statements must specify a where condition. - -说明： - -1. The type of `OP` in the WHERE condition can only include `=, >, <, > =, < =,!=`. Currently, where key in (value1, Value2, value3) mode is not supported yet, may be added this support later. -2. The column in the WHERE condition can only be the `key` column. -3. Cannot delete when the `key` column does not exist in any rollup table. -4. Each condition in WHERE condition can only be realated by `and`. If you want `or`, you are suggested to write these conditions into two delete statements. -5. If the specified table is a range partitioned table, `PARTITION` must be specified unless the table is a single partition table,. -6. Unlike the insert into command, delete statement cannot specify `label` manually. You can view the concept of `label` in [Insert Into] (./insert-into-manual.md) - -## Delete Result - -The delete command is an SQL command, and the returned results are synchronous. It can be divided into the following types: - -1. Successful visible - - If delete completes successfully and is visible, the following results will be returned, `query OK` indicates success. - - ``` - mysql> delete from test_tbl PARTITION p1 where k1 = 1; - Query OK, 0 rows affected (0.04 sec) - {'label':'delete_e7830c72-eb14-4cb9-bbb6-eebd4511d251', 'status':'VISIBLE', 'txnId':'4005'} - ``` - -2. Submitted successfully, but not visible - - - The transaction submission of Doris is divided into two steps: submission and publish version. Only after the publish version step is completed, the result will be visible to the user. If it has been submitted successfully, then it can be considered that the publish version step will eventually success. Doris will try to wait for publishing for a period of time after submitting. If it has timed out, even if the publishing version has not been completed, it will return to the user in priority and prompt the user that the submission has been completed but not visible. If delete has been committed and executed, but has not been published and visible, the following results will be returned. - - ``` - mysql> delete from test_tbl PARTITION p1 where k1 = 1; - Query OK, 0 rows affected (0.04 sec) - {'label':'delete_e7830c72-eb14-4cb9-bbb6-eebd4511d251', 'status':'VISIBLE', 'txnId':'4005', 'err':'delete job is committed but may be taking effect later' } - ``` - - The result will return a JSON string at the same time: - - `affected rows`: Indicates the row affected by this deletion. Since the deletion of Doris is currently a logical deletion, the value is always 0. - - `label`: The label generated automatically to be the signature of the delete jobs. Each job has a unique label within a single database. - - `status`: Indicates whether the data deletion is visible. If it is visible, `visible` will be displayed. If it is not visible, `committed` will be displayed. - - - `txnId`: The transaction ID corresponding to the delete job - - `err`: Field will display some details of this deletion - -3. Commit failed, transaction cancelled - - If the delete statement is not submitted successfully, it will be automatically aborted by Doris and the following results will be returned - - - ``` - mysql> delete from test_tbl partition p1 where k1 > 80; - ERROR 1064 (HY000): errCode = 2, detailMessage = {错误原因} - ``` - - example： - - A timeout deletion will return the timeout and unfinished replicas displayed as ` (tablet = replica)` - - - ``` - mysql> delete from test_tbl partition p1 where k1 > 80; - ERROR 1064 (HY000): errCode = 2, detailMessage = failed to delete replicas from job: 4005, Unfinished replicas:10000=60000, 10001=60000, 10002=60000 - ``` - - **The correct processing logic for the returned results of the delete operation is as follows:** - - 1. If `Error 1064 (HY000)` is returned, deletion fails - - 2. If the returned result is `Query OK`, the deletion is successful - - 1. If `status` is `committed`, the data deletion is committed and will be eventually invisible. Users can wait for a while and then use the `show delete` command to view the results. - 2. If `status` is `visible`, the data have been deleted successfully. - -## Relevant Configuration - -### FE configuration - -**TIMEOUT configuration** - -In general, Doris's deletion timeout is limited from 30 seconds to 5 minutes. The specific time can be adjusted through the following configuration items - -* tablet\_delete\_timeout\_second - - The timeout of delete itself can be elastically changed by the number of tablets in the specified partition. This configuration represents the average timeout contributed by a tablet. The default value is 2. - - Assuming that there are 5 tablets under the specified partition for this deletion, the timeout time available for the deletion is 10 seconds. Because the minimum timeout is 30 seconds which is higher than former timeout time, the final timeout is 30 seconds. - -* load\_straggler\_wait\_second - - If the user estimates a large amount of data, so that the upper limit of 5 minutes is insufficient, the user can adjust the upper limit of timeout through this item, and the default value is 300. - - **The specific calculation rule of timeout(seconds)** - - `TIMEOUT = MIN(load_straggler_wait_second, MAX(30, tablet_delete_timeout_second * tablet_num))` - -* query_timeout - - Because delete itself is an SQL command, the deletion statement is also limited by the session variables, and the timeout is also affected by the session value `query'timeout`. You can increase the value by `set query'timeout = xxx`. - -## Show delete history - -1. The user can view the deletion completed in history through the show delete statement. - - Syntax - - ``` - SHOW DELETE [FROM db_name] - ``` - - example - - ``` - mysql> show delete from test_db; - +-----------+---------------+---------------------+-----------------+----------+ - | TableName | PartitionName | CreateTime | DeleteCondition | State | - +-----------+---------------+---------------------+-----------------+----------+ - | empty_tbl | p3 | 2020-04-15 23:09:35 | k1 EQ "1" | FINISHED | - | test_tbl | p4 | 2020-04-15 23:09:53 | k1 GT "80" | FINISHED | - +-----------+---------------+---------------------+-----------------+----------+ - 2 rows in set (0.00 sec) - ``` - diff --git a/docs/documentation/en/administrator-guide/load-data/index.rst b/docs/documentation/en/administrator-guide/load-data/index.rst deleted file mode 100644 index eb02840886873e..00000000000000 --- a/docs/documentation/en/administrator-guide/load-data/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -============= -Load Data -============= - -.. toctree:: - :maxdepth: 2 - - load-manual_EN.md - broker-load-manual_EN.md - stream-load-manual_EN.md - routine-load-manual_EN.md - insert-into-manual_EN.md - delete-manual_EN.md diff --git a/docs/documentation/en/administrator-guide/load-data/insert-into-manual_EN.md b/docs/documentation/en/administrator-guide/load-data/insert-into-manual_EN.md deleted file mode 100644 index 71abc96bde1a27..00000000000000 --- a/docs/documentation/en/administrator-guide/load-data/insert-into-manual_EN.md +++ /dev/null @@ -1,268 +0,0 @@ - - -# Insert Into - -The use of Insert Into statements is similar to that of Insert Into statements in databases such as MySQL. But in Doris, all data writing is a separate import job. So Insert Into is also introduced here as an import method. - -The main Insert Into command contains the following two kinds; - -* INSERT INTO tbl SELECT ... -* INSERT INTO tbl (col1, col2, ...) VALUES (1, 2, ...), (1,3, ...); - -The second command is for Demo only, not in a test or production environment. - -## Basic operations - -### Create a Load - -The Insert Into command needs to be submitted through MySQL protocol. Creating an import request returns the import result synchronously. - -Grammar: - -``` -INSERT INTO table_name [WITH LABEL label] [partition_info] [col_list] [query_stmt] [VALUES]; -``` - -Examples: - -``` -INSERT INTO tbl2 WITH LABEL label1 SELECT * FROM tbl3; -INSERT INTO tbl1 VALUES ("qweasdzxcqweasdzxc"), ("a"); -``` - -**Notice** - -When using `CTE(Common Table Expressions)` as the query part of insert operation, the `WITH LABEL` or column list part must be specified. -For example: - -``` -INSERT INTO tbl1 WITH LABEL label1 -WITH cte1 AS (SELECT * FROM tbl1), cte2 AS (SELECT * FROM tbl2) -SELECT k1 FROM cte1 JOIN cte2 WHERE cte1.k1 = 1; - -INSERT INTO tbl1 (k1) -WITH cte1 AS (SELECT * FROM tbl1), cte2 AS (SELECT * FROM tbl2) -SELECT k1 FROM cte1 JOIN cte2 WHERE cte1.k1 = 1; -``` - -The following is a brief introduction to the parameters used in creating import statements: - -+ partition\_info - - Import the target partition of the table. If the target partition is specified, only the data that matches the target partition will be imported. If not specified, the default value is all partitions of the table. - -+ col\_list - - The target column of the import table can exist in any order. If no target column is specified, the default value is all columns in this table. If a column in the table does not exist in the target column, the column needs a default value, otherwise Insert Into will fail. - - If the result column type of the query statement is inconsistent with the type of the target column, an implicit type conversion is invoked. If the conversion is not possible, the Insert Into statement will report a parsing error. - -+ query\_stmt - - Through a query statement, the results of the query statement are imported into other tables in Doris system. Query statements support any SQL query syntax supported by Doris. - -+ VALUES - - Users can insert one or more data through VALUES grammar. - - *Note: VALUES is only suitable for importing several pieces of data as DEMO. It is totally unsuitable for any test and production environment. Doris system itself is not suitable for single data import scenarios. It is recommended to use INSERT INTO SELECT for batch import.* - -* WITH LABEL - - INSERT as a load job, it can also be with a label. If not with a label, Doris will use a UUID as label. - - This feature needs Doris version 0.11+. - - *Note: It is recommended that Label be specified rather than automatically allocated by the system. If the system allocates automatically, but during the execution of the Insert Into statement, the connection is disconnected due to network errors, etc., then it is impossible to know whether Insert Into is successful. If you specify Label, you can view the task results again through Label.* - -### Load results - -Insert Into itself is a SQL command, and the return result is divided into the following types according to the different execution results: - -1. Result set is empty - - If the result set of the insert corresponding SELECT statement is empty, it is returned as follows: - - ``` - mysql> insert into tbl1 select * from empty_tbl; - Query OK, 0 rows affected (0.02 sec) - ``` - - `Query OK` indicates successful execution. `0 rows affected` means that no data was loaded. - -2. The result set is not empty - - In the case where the result set is not empty. The returned results are divided into the following situations: - - 1. Insert is successful and data is visible: - - ``` - mysql> insert into tbl1 select * from tbl2; - Query OK, 4 rows affected (0.38 sec) - {'label': 'insert_8510c568-9eda-4173-9e36-6adc7d35291c', 'status': 'visible', 'txnId': '4005'} - - mysql> insert into tbl1 with label my_label1 select * from tbl2; - Query OK, 4 rows affected (0.38 sec) - {'label': 'my_label1', 'status': 'visible', 'txnId': '4005'} - - mysql> insert into tbl1 select * from tbl2; - Query OK, 2 rows affected, 2 warnings (0.31 sec) - {'label': 'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status': 'visible', 'txnId': '4005'} - - mysql> insert into tbl1 select * from tbl2; - Query OK, 2 rows affected, 2 warnings (0.31 sec) - {'label': 'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status': 'committed', 'txnId': '4005'} - ``` - - `Query OK` indicates successful execution. `4 rows affected` means that a total of 4 rows of data were imported. `2 warnings` indicates the number of lines to be filtered. - - Also returns a json string: - - ``` - {'label': 'my_label1', 'status': 'visible', 'txnId': '4005'} - {'label': 'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status': 'committed', 'txnId': '4005'} - {'label': 'my_label1', 'status': 'visible', 'txnId': '4005', 'err': 'some other error'} - ``` - - `label` is a user-specified label or an automatically generated label. Label is the ID of this Insert Into load job. Each load job has a label that is unique within a single database. - - `status` indicates whether the loaded data is visible. If visible, show `visible`, if not, show` committed`. - - `txnId` is the id of the load transaction corresponding to this insert. - - The `err` field displays some other unexpected errors. - - When user need to view the filtered rows, the user can use the following statement - - ``` - show load where label = "xxx"; - ``` - - The URL in the returned result can be used to query the wrong data. For details, see the following **View Error Lines** Summary. -     - **"Data is not visible" is a temporary status, this batch of data must be visible eventually** - - You can view the visible status of this batch of data with the following statement: - - ``` - show transaction where id = 4005; - ``` - - If the `TransactionStatus` column in the returned result is `visible`, the data is visible. - - 2. Insert fails - - Execution failure indicates that no data was successfully loaded, and returns as follows: - - ``` - mysql> insert into tbl1 select * from tbl2 where k1 = "a"; - ERROR 1064 (HY000): all partitions have no load data. Url: http://10.74.167.16:8042/api/_load_error_log?file=__shard_2/error_log_insert_stmt_ba8bb9e158e4879-ae8de8507c0bf8a2_ba8bb9e158e4879_ae8de850e8de850 - ``` - - Where `ERROR 1064 (HY000): all partitions have no load data` shows the reason for the failure. The latter url can be used to query the wrong data. For details, see the following **View Error Lines** Summary. - -**In summary, the correct processing logic for the results returned by the insert operation should be:** - -1. If the returned result is `ERROR 1064 (HY000)`, it means that the import failed. -2. If the returned result is `Query OK`, it means the execution was successful. - - 1. If `rows affected` is 0, the result set is empty and no data is loaded. - 2. If `rows affected` is greater than 0: - 1. If `status` is` committed`, the data is not yet visible. You need to check the status through the `show transaction` statement until `visible`. - 2. If `status` is` visible`, the data is loaded successfully. - 3. If `warnings` is greater than 0, it means that some data is filtered. You can get the url through the `show load` statement to see the filtered rows. - -## Relevant System Configuration - -### FE configuration - -+ time out - - The timeout time of the import task (in seconds) will be cancelled by the system if the import task is not completed within the set timeout time, and will become CANCELLED. - - At present, Insert Into does not support custom import timeout time. All Insert Into imports have a uniform timeout time. The default timeout time is 1 hour. If the imported source file cannot complete the import within the specified time, the parameter ``insert_load_default_timeout_second`` of FE needs to be adjusted. - - At the same time, the Insert Into statement receives the restriction of the Session variable `query_timeout`. You can increase the timeout time by `SET query_timeout = xxx;` in seconds. - -### Session Variables - -+ enable\_insert\_strict - - The Insert Into import itself cannot control the tolerable error rate of the import. Users can only use the Session parameter `enable_insert_strict`. When this parameter is set to false, it indicates that at least one data has been imported correctly, and then it returns successfully. When this parameter is set to true, the import fails if there is a data error. The default is false. It can be set by `SET enable_insert_strict = true;`. - -+ query u timeout - - Insert Into itself is also an SQL command, so the Insert Into statement is also restricted by the Session variable `query_timeout`. You can increase the timeout time by `SET query_timeout = xxx;` in seconds. - -## Best Practices - -### Application scenarios -1. Users want to import only a few false data to verify the functionality of Doris system. The grammar of INSERT INTO VALUS is suitable at this time. -2. Users want to convert the data already in the Doris table into ETL and import it into a new Doris table, which is suitable for using INSERT INTO SELECT grammar. -3. Users can create an external table, such as MySQL external table mapping a table in MySQL system. Or create Broker external tables to map data files on HDFS. Then the data from the external table is imported into the Doris table for storage through the INSERT INTO SELECT grammar. - -### Data volume -Insert Into has no limitation on the amount of data, and large data imports can also be supported. However, Insert Into has a default timeout time, and the amount of imported data estimated by users is too large, so it is necessary to modify the system's Insert Into import timeout time. - -``` -Import data volume = 36G or less than 3600s*10M/s -Among them, 10M/s is the maximum import speed limit. Users need to calculate the average import speed according to the current cluster situation to replace 10M/s in the formula. -``` - -### Complete examples - -Users have a table store sales in the database sales. Users create a table called bj store sales in the database sales. Users want to import the data recorded in the store sales into the new table bj store sales. The amount of data imported is about 10G. - -``` -large sales scheme -(id, total, user_id, sale_timestamp, region) - -Order large sales schedule: -(id, total, user_id, sale_timestamp) - -``` - -Cluster situation: The average import speed of current user cluster is about 5M/s - -+ Step1: Determine whether you want to modify the default timeout of Insert Into - - ``` - Calculate the approximate time of import - 10G / 5M /s = 2000s - - Modify FE configuration - insert_load_default_timeout_second = 2000 - ``` - -+ Step2: Create Import Tasks - - Since users want to ETL data from a table and import it into the target table, they should use the Insert in query\\stmt mode to import it. - - ``` - INSERT INTO bj_store_sales SELECT id, total, user_id, sale_timestamp FROM store_sales where region = "bj"; - ``` - -## Common Questions - -* View the wrong line - - Because Insert Into can't control the error rate, it can only tolerate or ignore the error data completely by `enable_insert_strict`. So if `enable_insert_strict` is set to true, Insert Into may fail. If `enable_insert_strict` is set to false, then only some qualified data may be imported. However, in either case, Doris is currently unable to provide the ability to view substandard data rows. Therefore, the user cannot view the specific import error through the Insert Into statement. - - The causes of errors are usually: source data column length exceeds destination data column length, column type mismatch, partition mismatch, column order mismatch, etc. When it's still impossible to check for problems. At present, it is only recommended that the SELECT command in the Insert Into statement be run to export the data to a file, and then import the file through Stream load to see the specific errors. diff --git a/docs/documentation/en/administrator-guide/load-data/load-manual_EN.md b/docs/documentation/en/administrator-guide/load-data/load-manual_EN.md deleted file mode 100644 index b75d170e4b7a4a..00000000000000 --- a/docs/documentation/en/administrator-guide/load-data/load-manual_EN.md +++ /dev/null @@ -1,212 +0,0 @@ - - -# Introduction Overview - -The Load function is to import the user's raw data into Doris. After successful import, users can query data through Mysql client. - -Doris supports multiple imports. It is recommended to read this document in full first, and then to view the detailed documents of their respective import modes according to the selected import mode. - -## Basic concepts - -1. Frontend (FE): Metadata and scheduling nodes of Doris system. In the import process, it is mainly responsible for the generation of import planning and the scheduling of import tasks. -2. Backend (BE): The computing and storage nodes of Doris system. In the import process, it is mainly responsible for ETL and storage of data. -3. Broker: Broker is an independent stateless process. It encapsulates the file system interface and provides Doris with the ability to read files in the remote storage system. -4. Load job: The import job reads the source data submitted by the user, transforms or cleans it, and imports the data into the Doris system. After the import is completed, the data can be queried by the user. -5. Label: All import jobs have a Label. Label is unique in a database and can be specified by the user or automatically generated by the system to identify an import job. The same Label can only be used for a successful import job. -6. MySQL Protocol/HTTP Protocol: Doris provides two kinds of access protocol interfaces. MySQL protocol and HTTP protocol. Part of the import mode uses MySQL protocol interface to submit jobs, and part of the import mode uses HTTP protocol interface to submit jobs. - -## Load mode - -To adapt to different data import requirements, Doris system provides five different import methods. Each import mode supports different data sources and has different usage modes (asynchronous, synchronous). - -All import methods support CSV data format. Broker load also supports parquet and orc data format. - -For instructions on each import mode, please refer to the operation manual for a single import mode. - -* Broker load - - Access and read external data sources (such as HDFS) through the Broker process and import them into Doris. The user submits the import job through Mysql protocol and executes it asynchronously. View the import results through the `SHOW LOAD` command. - -* Stream load - - Users submit requests through HTTP protocol and create imports with raw data. It is mainly used to quickly import data from local files or data streams into Doris. The Import command returns the import result synchronously. - -* Insert - - Similar to the Insert statement in MySQL, Doris provides `INSERT INTO tbl SELECT ...;`reading data from Doris's table and importing it into another table. Or by `INSERT INTO tbl VALUES (...);` Insert a single piece of data. - -* Multi load - - Users submit multiple import jobs through HTTP protocol. Multi Load guarantees the atomic validity of multiple import jobs. - -* Routine load - - Users submit routine import jobs through MySQL protocol, generate a resident thread, read and import data from data sources (such as Kafka) uninterruptedly into Doris. - -## Basic Principles - -### Import execution process - - -``` -+---------+ +---------+ +----------+ +-----------+ -| | | | | | | | -| PENDING +----->+ ETL +----->+ LOADING +----->+ FINISHED | -| | | | | | | | -+---------+ +---+-----+ +----+-----+ +-----------+ - | | | - | | | - | | | - | | | +-----------+ - | | | | | - +---------------+-----------------+------------> CANCELLED | - | | - +-----------+ - -``` - -As shown above, an import operation mainly goes through the four stages above. - -+ PENDING (not required): Only Broker Load has this stage. Broker Load is submitted by the user and stays at this stage for a short time until it is scheduled by Scheduler in FE. Scheduler's schedule interval is 5 seconds. - -+ ETL (not required): This stage exists before version 0.10.0 (included), mainly for transforming raw data according to user declaration and filtering raw data that does not meet the requirements. In the version after 0.10.0, the ETL phase no longer exists, and the work of data transformation is merged into the LOADING phase. - -+ LOADING: This stage is mainly used to push the transformed data into the corresponding BE storage before version 0.10.0 (including). In the version after 0.10.0, the data is cleaned and changed first, and then sent to BE storage. When all imported data are imported, the process of waiting for validity enters, and Load job is still LOADING. - -+ FINISHED: After all the data involved in Load Job takes effect, the state of Load Job becomes FINISHED. Data imported after FINISHED can be queried. - -+ CANCELLED: Before job FINISH, jobs may be cancelled and entered the CANCELLED state. For example, the user manually cancels, or imports errors. CANCELLED is also the final state of Load Job and cannot be executed again. - -In the above stage, except for the PENDING to LOADING stage, which is scheduled by Scheduler, the transfer before other stages is implemented by callback mechanism. - -### Label and Atomicity - -Doris provides atomic assurance for all import methods. It ensures that the data in the same import operation is valid for atoms. There will be no case of importing only part of the data. - -At the same time, each import job has a Label designated by the user or automatically generated by the system. Label is unique in a database. When an import job corresponding to a Label is successful enough, the import job cannot be submitted repeatedly using the Label. If the import job corresponding to Label fails, it can be reused. - -Users can use Label mechanism to ensure that the data corresponding to Label can be imported at most once, at the level of At-Most-One semantics. - - -## Synchronization and asynchronization - -Doris's current import methods fall into two categories, synchronous and asynchronous. If an external program accesses Doris's import function, it is necessary to determine which type of import mode is used and then determine the access logic. - -### Synchronization - -Synchronized import means that users create import tasks, Doris executes import synchronously, and returns user import results after execution. Users can directly determine whether the import is successful or not by synchronizing the results returned by creating the import task command. - -The import methods of synchronous type are **Stream load**, **Insert**. - -Operation steps: - -1. Users (external systems) create import tasks. -2. Doris returns the import result. -3. The user (external system) judges the import result and can submit the import task again if it fails. - -*Note: If the user returns the import synchronously and the amount of data imported is too large, it may take a long time to create the import request to return the result.* - -### Asynchronism -Asynchronous import means that after the user creates the import task, Doris directly returns to the successful creation. **Successful creation does not mean that data has been imported into**. The import task will be executed asynchronously. After successful creation, users need to send a polling command to check the status of the import job. If the creation fails, you can judge whether it needs to be created again based on the failure information. - -The ways to import asynchronous types are: **Broker load**, **Multi load**. - -Operation steps: - -1. Users (external systems) create import tasks. -2. Doris returns the import creation result. -3. User (external system) judges the result of import creation, success enters 4, failure returns to retry to create import, return to 1. -4. The user (external system) polls to see the import task until the status changes to FINISHED or CANCELLED. - -### Notes -Neither asynchronous nor synchronous import types should be retried endlessly after Doris returns an import failure or an import creation failure. **After a limited number of retries and failures, the external system retains the failure information. Most of the retries fail because of the problem of using method or data itself.** - -## Memory Limit - -Users can limit the memory usage of a single load by setting parameters to prevent the system from taking up too much memory and causing the system OOM. -Different load methods restrict memory in a slightly different way. You can refer to the respective load manuals for viewing. - -An load job is usually distributed across multiple Backends. The load memory limit is the memory usage of load job on a single Backend, not memory usage across the cluster. - -At the same time, each Backend sets the overall upper limit of the memory available for load. See the General System Configuration section below for specific configuration. This configuration limits the overall memory usage limit for all load tasks running on this Backend. - -Smaller memory limits can affect load efficiency because the load process can frequently write in-memory data back to disk because memory reaches the upper limit. Excessive memory limits can cause system OOM when load concurrency is high. Therefore, you need to properly set the load memory limit according to your needs. - -## Best Practices - -When users access Doris import, they usually use program access mode to ensure that data is imported into Doris regularly. Below is a brief description of the best practices for program access to Doris. - -1. Choose the appropriate import mode: According to the location of the data source, choose the import mode. For example, if raw data is stored on HDFS, import it using Broker load. -2. Protocol for determining the import mode: If Broker load import mode is selected, external systems need to be able to submit and view import jobs regularly using MySQL protocol. -3. Determine the type of import mode: import mode is synchronous or asynchronous. For example, Broker load is an asynchronous import mode. After submitting the creation import, the external system must call the check import command to determine whether the import is successful or not based on the results of the check import command. -4. Label generation strategy: Label generation strategy needs to be satisfied, and each batch of data is unique and fixed. Doris can then guarantee At-Most-Once. -5. The program itself guarantees At-Least-Once: The external system needs to guarantee its own At-Least-Once, so that Exactly-Once of the import process can be guaranteed. - -## General System Configuration - -The following sections explain several system-level configurations that are common to all imports. - -### FE configuration - -The following configuration belongs to the system configuration of FE, which can be modified by modifying the configuration file ``fe.conf``. - -+ max\_load\_timeout\_second and min\_load\_timeout\_second - - The two configurations mean the maximum import timeout time and the minimum import timeout time in seconds. The default maximum timeout time is 3 days and the default minimum timeout time is 1 second. User-defined import timeouts should not exceed this range. This parameter is applicable to all import modes. - -+ desired\_max\_waiting\_jobs - - The maximum number of imported tasks in the waiting queue is 100 by default. New import requests are rejected when the number of imports in the PENDING state (i.e. waiting for execution) in FE exceeds that value. - - This configuration is only valid for asynchronous execution of imports. When the number of import waiting for asynchronous execution exceeds the default value, subsequent creation of import requests will be rejected. - -+ max\_running\_txn\_num\_per\_db - - The implication of this configuration is that the maximum number of imports running in each database (no distinction between import types, uniform counting). When the current database is running more than the maximum number of imports, subsequent imports will not be executed. If the job is imported synchronously, the import will be rejected. If it is an asynchronous import job. The job will wait in the queue. - -### BE configuration - -The following configuration belongs to the BE system configuration, which can be modified by modifying the BE configuration file `be.conf`. - -+ push\_write\_mbytes\_per\_sec - - Writing speed limit for a single Tablet on BE. The default is 10, or 10MB/s. Usually the maximum write speed of BE to a single Tablet is between 10 and 30 MB/s, depending on Schema and the system. This parameter can be adjusted appropriately to control the import speed. - -+ write\_buffer\_size - - The imported data will be written to a memtable on BE, and the memtable will not be written back to disk until it reaches the threshold. The default size is 100MB. Too small threshold may result in a large number of small files on BE. This threshold can be increased appropriately to reduce the number of files. However, excessive thresholds can lead to RPC timeouts, as shown in the configuration instructions below. - -+ tablet\_writer\_rpc\_timeout\_sec - - During the import process, a Batch (1024 rows) RPC timeout is sent. Default 600 seconds. Because the RPC may involve multiple memtable writes, it may cause RPC timeouts, which can be adjusted appropriately to reduce timeout errors (such as `send batch fail`). At the same time, if the `write_buffer_size` configuration is increased, this parameter needs to be adjusted appropriately. - -+ streaming\_load\_rpc\_max\_alive\_time\_sec - - During the import process, Doris opens a Writer for each Tablet to receive and write data. This parameter specifies Writer's waiting timeout time. If Writer does not receive any data at this time, Writer will be destroyed automatically. When the system processing speed is slow, Writer may not receive the next batch of data for a long time, resulting in import error: `Tablet Writer add batch with unknown id`. This configuration can be increased appropriately at this time. The default is 600 seconds. - -+ load\_process\_max\_memory\_limit\_bytes and load\_process\_max\_memory\_limit\_percent - - These two parameters limit the upper memory limit that can be used to load tasks on a single Backend. The maximum memory and maximum memory percentage are respectively. `load_process_max_memory_limit_percent` defaults to 80%, which is 80% of the `mem_limit` configuration. That is, if the physical memory is M, the default load memory limit is M * 80% * 80%. - -     `load_process_max_memory_limit_bytes` defaults to 100GB. The system takes the smaller of the two parameters as the final Backend load memory usage limit. - -+ label\_keep\_max\_second - - The retention time of load job which is FINISHED or CANCELLED. The record of load job will be kept in Doris system for a period of time which is determined by this parameter. The default time of this parameter is 3 days. This parameter is common to all types of load job. diff --git a/docs/documentation/en/administrator-guide/load-data/routine-load-manual_EN.md b/docs/documentation/en/administrator-guide/load-data/routine-load-manual_EN.md deleted file mode 100644 index 14d3cfd5febd76..00000000000000 --- a/docs/documentation/en/administrator-guide/load-data/routine-load-manual_EN.md +++ /dev/null @@ -1,290 +0,0 @@ - - -# Routine Load - -The Routine Load feature provides users with a way to automatically load data from a specified data source. - -This document describes the implementation principles, usage, and best practices of this feature. - -## Glossary - -* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. -* BE: Backend, the backend node of Doris. Responsible for query execution and data storage. -* RoutineLoadJob: A routine load job submitted by the user. -* JobScheduler: A routine load job scheduler for scheduling and dividing a RoutineLoadJob into multiple Tasks. -* Task: RoutineLoadJob is divided by JobScheduler according to the rules. -* TaskScheduler: Task Scheduler. Used to schedule the execution of a Task. - -## Principle - -``` - +---------+ - | Client | - +----+----+ - | -+-----------------------------+ -| FE | | -| +-----------v------------+ | -| | | | -| | Routine Load Job | | -| | | | -| +---+--------+--------+--+ | -| | | | | -| +---v--+ +---v--+ +---v--+ | -| | task | | task | | task | | -| +--+---+ +---+--+ +---+--+ | -| | | | | -+-----------------------------+ - | | | - v v v - +---+--+ +--+---+ ++-----+ - | BE | | BE | | BE | - +------+ +------+ +------+ - -``` - -As shown above, the client submits a routine load job to FE. - -FE splits an load job into several Tasks via JobScheduler. Each Task is responsible for loading a specified portion of the data. The Task is assigned by the TaskScheduler to the specified BE. - -On the BE, a Task is treated as a normal load task and loaded via the Stream Load load mechanism. After the load is complete, report to FE. - -The JobScheduler in the FE continues to generate subsequent new Tasks based on the reported results, or retry the failed Task. - -The entire routine load job completes the uninterrupted load of data by continuously generating new Tasks. - -## Kafka Routine load - -Currently we only support routine load from the Kafka system. This section details Kafka's routine use and best practices. - -### Usage restrictions - -1. Support unauthenticated Kafka access and Kafka clusters certified by SSL. -2. The supported message format is csv text format. Each message is a line, and the end of the line does not contain a ** line break. -3. Only Kafka 0.10.0.0 or above is supported. - -### Create a routine load task - -The detailed syntax for creating a routine load task can be connected to Doris and execute `HELP ROUTINE LOAD;` to see the syntax help. Here is a detailed description of the precautions when creating a job. - -* columns_mapping - - `columns_mapping` is mainly used to specify the column structure of the table structure and message, as well as the conversion of some columns. If not specified, Doris will default to the columns in the message and the columns of the table structure in a one-to-one correspondence. Although under normal circumstances, if the source data is exactly one-to-one, normal data load can be performed without specifying. However, we still strongly recommend that users ** explicitly specify column mappings**. This way, when the table structure changes (such as adding a nullable column), or the source file changes (such as adding a column), the load task can continue. Otherwise, after the above changes occur, the load will report an error because the column mapping relationship is no longer one-to-one. - - In `columns_mapping` we can also use some built-in functions for column conversion. But you need to pay attention to the actual column type corresponding to the function parameters. for example: - - Suppose the user needs to load a table containing only a column of `k1` with a column type of `int`. And you need to convert the null value in the source file to 0. This feature can be implemented with the `ifnull` function. The correct way to use is as follows: - - `COLUMNS (xx, k1=ifnull(xx, "3"))` - - Note that we use `"3"` instead of `3`, although `k1` is of type `int`. Because the column type in the source data is `varchar` for the load task, the `xx` virtual column is also of type `varchar`. So we need to use `"3"` to match the match, otherwise the `ifnull` function can't find the function signature with the parameter `(varchar, int)`, and an error will occur. - - As another example, suppose the user needs to load a table containing only a column of `k1` with a column type of `int`. And you need to process the corresponding column in the source file: convert the negative number to a positive number and the positive number to 100. This function can be implemented with the `case when` function. The correct wording should be as follows: - - `COLUMNS (xx, case when xx < 0 than cast(-xx as varchar) else cast((xx + '100') as varchar) end)` - - Note that we need to convert all the parameters in `case when` to varchar in order to get the desired result. - -* where_predicates - - The type of the column in `where_predicates` is already the actual column type, so there is no need to cast to the varchar type as `columns_mapping`. Write according to the actual column type. - -* desired\_concurrent\_number - - `desired_concurrent_number` is used to specify the degree of concurrency expected for a routine job. That is, a job, at most how many tasks are executing at the same time. For Kafka load, the current actual concurrency is calculated as follows: - - ``` - Min(partition num, desired_concurrent_number, alive_backend_num, Config.max_routine_load_task_concurrrent_num) - ``` - - Where `Config.max_routine_load_task_concurrrent_num` is a default maximum concurrency limit for the system. This is a FE configuration that can be adjusted by changing the configuration. The default is 5. - - Where partition num refers to the number of partitions for the Kafka topic subscribed to. `alive_backend_num` is the current number of normal BE nodes. - -* max\_batch\_interval/max\_batch\_rows/max\_batch\_size - - These three parameters are used to control the execution time of a single task. If any of the thresholds is reached, the task ends. Where `max_batch_rows` is used to record the number of rows of data read from Kafka. `max_batch_size` is used to record the amount of data read from Kafka in bytes. The current consumption rate for a task is approximately 5-10MB/s. - - So assume a row of data 500B, the user wants to be a task every 100MB or 10 seconds. The expected processing time for 100MB is 10-20 seconds, and the corresponding number of rows is about 200000 rows. Then a reasonable configuration is: - - ``` - "max_batch_interval" = "10", - "max_batch_rows" = "200000", - "max_batch_size" = "104857600" - ``` - - The parameters in the above example are also the default parameters for these configurations. - -* max\_error\_number - - `max_error_number` is used to control the error rate. When the error rate is too high, the job will automatically pause. Because the entire job is stream-oriented, and because of the borderless nature of the data stream, we can't calculate the error rate with an error ratio like other load tasks. So here is a new way of calculating to calculate the proportion of errors in the data stream. - - We have set up a sampling window. The size of the window is `max_batch_rows * 10`. Within a sampling window, if the number of error lines exceeds `max_error_number`, the job is suspended. If it is not exceeded, the next window restarts counting the number of error lines. - - We assume that `max_batch_rows` is 200000 and the window size is 2000000. Let `max_error_number` be 20000, that is, the user expects an error behavior of 20000 for every 2000000 lines. That is, the error rate is 1%. But because not every batch of tasks consumes 200000 rows, the actual range of the window is [2000000, 2200000], which is 10% statistical error. - - The error line does not include rows that are filtered out by the where condition. But include rows that do not have a partition in the corresponding Doris table. - -* data\_source\_properties - - The specific Kakfa partition can be specified in `data_source_properties`. If not specified, all partitions of the subscribed topic are consumed by default. - - Note that when partition is explicitly specified, the load job will no longer dynamically detect changes to Kafka partition. If not specified, the partitions that need to be consumed are dynamically adjusted based on changes in the kafka partition. - -* strict\_mode - - Routine load load can turn on strict mode mode. The way to open it is to add ```"strict_mode" = "true"``` to job\_properties. The default strict mode is off. - - The strict mode mode means strict filtering of column type conversions during the load process. The strict filtering strategy is as follows: - - 1. For column type conversion, if strict mode is true, the wrong data will be filtered. The error data here refers to the fact that the original data is not null, and the result is a null value after participating in the column type conversion. - - 2. When a loaded column is generated by a function transformation, strict mode has no effect on it. - - 3. For a column type loaded with a range limit, if the original data can pass the type conversion normally, but cannot pass the range limit, strict mode will not affect it. For example, if the type is decimal(1,0) and the original data is 10, it is eligible for type conversion but not for column declarations. This data strict has no effect on it. - -#### strict mode and load relationship of source data - -Here is an example of a column type of TinyInt. - -> Note: When a column in a table allows a null value to be loaded - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|---------| -|null | \N | N/A | true or false | NULL| -|not null | aaa or 2000 | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 | 1 | true or false | correct data| - -Here the column type is Decimal(1,0) -  -> Note: When a column in a table allows a null value to be loaded - -|source data | source data example | string to int | strict_mode | result| -|------------|---------------------|-----------------|--------------------|--------| -|null | \N | N/A | true or false | NULL| -|not null | aaa | NULL | true | invalid data(filtered)| -|not null | aaa | NULL | false | NULL| -|not null | 1 or 10 | 1 | true or false | correct data| - -> Note: 10 Although it is a value that is out of range, because its type meets the requirements of decimal, strict mode has no effect on it. 10 will eventually be filtered in other ETL processing flows. But it will not be filtered by strict mode. - -#### Accessing SSL-certified Kafka clusters - -Accessing the SSL-certified Kafka cluster requires the user to provide a certificate file (ca.pem) for authenticating the Kafka Broker public key. If the Kafka cluster has both client authentication enabled, you will also need to provide the client's public key (client.pem), key file (client.key), and key password. The files needed here need to be uploaded to Doris via the `CREAE FILE` command, ** and the catalog name is `kafka`**. See `HELP CREATE FILE;` for specific help on the `CREATE FILE` command. Here is an example: - -1. Upload file - - ``` - CREATE FILE "ca.pem" PROPERTIES("url" = "https://example_url/kafka-key/ca.pem", "catalog" = "kafka"); - CREATE FILE "client.key" PROPERTIES("url" = "https://example_urlkafka-key/client.key", "catalog" = "kafka"); - CREATE FILE "client.pem" PROPERTIES("url" = "https://example_url/kafka-key/client.pem", "catalog" = "kafka"); -``` - -2. Create a routine load job - - ``` - CREATE ROUTINE LOAD db1.job1 on tbl1 - PROPERTIES - ( - "desired_concurrent_number"="1" - ) - FROM KAFKA - ( - "kafka_broker_list"= "broker1:9091,broker2:9091", - "kafka_topic" = "my_topic", - "property.security.protocol" = "ssl", - "property.ssl.ca.location" = "FILE:ca.pem", - "property.ssl.certificate.location" = "FILE:client.pem", - "property.ssl.key.location" = "FILE:client.key", - "property.ssl.key.password" = "abcdefg" - ); - ``` - -> Doris accesses Kafka clusters via Kafka's C++ API `librdkafka`. The parameters supported by `librdkafka` can be found. -> -> `https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md` - -### Viewing the status of the load job - -Specific commands and examples for viewing the status of the ** job** can be viewed with the `HELP SHOW ROUTINE LOAD;` command. - -Specific commands and examples for viewing the **Task** status can be viewed with the `HELP SHOW ROUTINE LOAD TASK;` command. - -You can only view tasks that are currently running, and tasks that have ended and are not started cannot be viewed. - -### Job Control - -The user can control the stop, pause and restart of the job by the three commands `STOP/PAUSE/RESUME`. You can view help and examples with the three commands `HELP STOP ROUTINE LOAD;`, `HELP PAUSE ROUTINE LOAD;` and `HELP RESUME ROUTINE LOAD;`. - -## other instructions - -1. The relationship between a routine load job and an ALTER TABLE operation - - * Routine load does not block SCHEMA CHANGE and ROLLUP operations. Note, however, that if the column mappings are not matched after SCHEMA CHANGE is completed, the job's erroneous data will spike and eventually cause the job to pause. It is recommended to reduce this type of problem by explicitly specifying column mappings in routine load jobs and by adding Nullable columns or columns with Default values. - * Deleting a Partition of a table may cause the loaded data to fail to find the corresponding Partition and the job will be paused. - -2. Relationship between routine load jobs and other load jobs (LOAD, DELETE, INSERT) - - * Routine load does not conflict with other LOAD jobs and INSERT operations. - * When performing a DELETE operation, the corresponding table partition cannot have any load tasks being executed. Therefore, before performing the DELETE operation, you may need to pause the routine load job and wait for the delivered task to complete before you can execute DELETE. - -3. Relationship between routine load jobs and DROP DATABASE/TABLE operations - - When the corresponding database or table is deleted, the job will automatically CANCEL. - -4. The relationship between the kafka type routine load job and kafka topic - - When the user creates a routine load declaration, the `kafka_topic` does not exist in the kafka cluster. - - * If the broker of the user kafka cluster has `auto.create.topics.enable = true` set, `kafka_topic` will be automatically created first, and the number of partitions created automatically will be in the kafka cluster** of the user side. The broker is configured with `num.partitions`. The routine job will continue to read the data of the topic continuously. - * If the broker of the user kafka cluster has `auto.create.topics.enable = false` set, topic will not be created automatically, and the routine will be paused before any data is read, with the status `PAUSED`. - - So, if the user wants to be automatically created by the routine when the kafka topic does not exist, just set the broker in the kafka cluster** of the user's side to set auto.create.topics.enable = true` . - -## Related parameters - -Some system configuration parameters can affect the use of routine loads. - -1. max\_routine\_load\_task\_concurrent\_num - - The FE configuration item, which defaults to 5, can be modified at runtime. This parameter limits the maximum number of subtask concurrency for a routine load job. It is recommended to maintain the default value. If the setting is too large, it may cause too many concurrent tasks and occupy cluster resources. - -2. max\_routine_load\_task\_num\_per\_be - - The FE configuration item, which defaults to 5, can be modified at runtime. This parameter limits the number of subtasks that can be executed concurrently by each BE node. It is recommended to maintain the default value. If the setting is too large, it may cause too many concurrent tasks and occupy cluster resources. - -3. max\_routine\_load\_job\_num - - The FE configuration item, which defaults to 100, can be modified at runtime. This parameter limits the total number of routine load jobs, including NEED_SCHEDULED, RUNNING, PAUSE. After the overtime, you cannot submit a new assignment. - -4. max\_consumer\_num\_per\_group - - BE configuration item, the default is 3. This parameter indicates that up to several consumers are generated in a subtask for data consumption. For a Kafka data source, a consumer may consume one or more kafka partitions. Suppose a task needs to consume 6 kafka partitions, it will generate 3 consumers, and each consumer consumes 2 partitions. If there are only 2 partitions, only 2 consumers will be generated, and each consumer will consume 1 partition. - -5. push\_write\_mbytes\_per\_sec - - BE configuration item. The default is 10, which is 10MB/s. This parameter is to load common parameters, not limited to routine load jobs. This parameter limits the speed at which loaded data is written to disk. For high-performance storage devices such as SSDs, this speed limit can be appropriately increased. - -6. max\_tolerable\_backend\_down\_num - FE configuration item, the default is 0. Under certain conditions, Doris can reschedule PAUSED tasks, that becomes RUNNING?This parameter is 0, which means that rescheduling is allowed only when all BE nodes are in alive state. - -7. period\_of\_auto\_resume\_min - FE configuration item, the default is 5 mins. Doris reschedules will only try at most 3 times in the 5 minute period. If all 3 times fail, the current task will be locked, and auto-scheduling will not be performed. However, manual intervention can be performed. diff --git a/docs/documentation/en/administrator-guide/load-data/stream-load-manual_EN.md b/docs/documentation/en/administrator-guide/load-data/stream-load-manual_EN.md deleted file mode 100644 index 0020d5224e23e1..00000000000000 --- a/docs/documentation/en/administrator-guide/load-data/stream-load-manual_EN.md +++ /dev/null @@ -1,286 +0,0 @@ - - -# Stream load - -Stream load is a synchronous way of importing. Users import local files or data streams into Doris by sending HTTP protocol requests. Stream load synchronously executes the import and returns the import result. Users can directly determine whether the import is successful by the return body of the request. - -Stream load is mainly suitable for importing local files or data from data streams through procedures. - -## Basic Principles - -The following figure shows the main flow of Stream load, omitting some import details. - -``` - ^ + - | | - | | 1A. User submit load to FE - | | - | +--v-----------+ - | | FE | -5. Return result to user | +--+-----------+ - | | - | | 2. Redirect to BE - | | - | +--v-----------+ - +---+Coordinator BE| 1B. User submit load to BE - +-+-----+----+-+ - | | | - +-----+ | +-----+ - | | | 3. Distrbute data - | | | - +-v-+ +-v-+ +-v-+ - |BE | |BE | |BE | - +---+ +---+ +---+ -``` - -In Stream load, Doris selects a node as the Coordinator node. This node is responsible for receiving data and distributing data to other data nodes. - -Users submit import commands through HTTP protocol. If submitted to FE, FE forwards the request to a BE via the HTTP redirect instruction. Users can also submit import commands directly to a specified BE. - -The final result of the import is returned to the user by Coordinator BE. - -## Basic operations -### Create a Load - -Stream load submits and transfers data through HTTP protocol. Here, the `curl` command shows how to submit an import. - -Users can also operate through other HTTP clients. - -``` -curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_host:http_port/api/{db}/{table}/_stream_load - -The properties supported in the header are described in "Load Parameters" below -The format is: - H "key1: value1" -``` - -Examples: - -``` -curl --location-trusted -u root -T date -H "label:123" http://abc.com:8030/api/test/date/_stream_load -``` -The detailed syntax for creating imports helps to execute ``HELP STREAM LOAD`` view. The following section focuses on the significance of creating some parameters of Stream load. - -#### Signature parameters - -+ user/passwd - - Stream load uses the HTTP protocol to create the imported protocol and signs it through the Basic Access authentication. The Doris system verifies user identity and import permissions based on signatures. - -#### Load Parameters - -Stream load uses HTTP protocol, so all parameters related to import tasks are set in the header. The significance of some parameters of the import task parameters of Stream load is mainly introduced below. - -+ label - - Identity of import task. Each import task has a unique label inside a single database. Label is a user-defined name in the import command. With this label, users can view the execution of the corresponding import task. - - Another function of label is to prevent users from importing the same data repeatedly. **It is strongly recommended that users use the same label for the same batch of data. This way, repeated requests for the same batch of data will only be accepted once, guaranteeing at-Most-Once** - - When the corresponding import operation state of label is CANCELLED, the label can be used again. - -+ max\_filter\_ratio - - The maximum tolerance rate of the import task is 0 by default, and the range of values is 0-1. When the import error rate exceeds this value, the import fails. - - If the user wishes to ignore the wrong row, the import can be successful by setting this parameter greater than 0. - - The calculation formula is as follows: - - ``` (dpp.abnorm.ALL / (dpp.abnorm.ALL + dpp.norm.ALL ) ) > max_filter_ratio ``` - - ``` dpp.abnorm.ALL``` denotes the number of rows whose data quality is not up to standard. Such as type mismatch, column mismatch, length mismatch and so on. - - ``` dpp.norm.ALL ``` refers to the number of correct data in the import process. The correct amount of data for the import task can be queried by the ``SHOW LOAD` command. - -The number of rows in the original file = `dpp.abnorm.ALL + dpp.norm.ALL` - -+ where - - Import the filter conditions specified by the task. Stream load supports filtering of where statements specified for raw data. The filtered data will not be imported or participated in the calculation of filter ratio, but will be counted as `num_rows_unselected`. - -+ partition - - Partition information for tables to be imported will not be imported if the data to be imported does not belong to the specified Partition. These data will be included in `dpp.abnorm.ALL`. - -+ columns - - The function transformation configuration of data to be imported includes the sequence change of columns and the expression transformation, in which the expression transformation method is consistent with the query statement. - - ``` - Examples of column order transformation: There are two columns of original data, and there are also two columns (c1, c2) in the table at present. But the first column of the original file corresponds to the C2 column of the target table, while the second column of the original file corresponds to the C1 column of the target table, which is written as follows: - columns: c2,c1 - - Example of expression transformation: There are two columns in the original file and two columns in the target table (c1, c2). However, both columns in the original file need to be transformed by functions to correspond to the two columns in the target table. - columns: tmp_c1, tmp_c2, c1 = year(tmp_c1), c2 = mouth(tmp_c2) - Tmp_* is a placeholder, representing two original columns in the original file. - ``` - -+ exec\_mem\_limit - - Memory limit. Default is 2GB. Unit is Bytes - -### Return results - -Since Stream load is a synchronous import method, the result of the import is directly returned to the user by creating the return value of the import. - -Examples: - -``` -{ - "TxnId": 1003, - "Label": "b6f3bc78-0d2c-45d9-9e4c-faa0a0149bee", - "Status": "Success", - "ExistingJobStatus": "FINISHED", // optional - "Message": "OK", - "NumberTotalRows": 1000000, - "NumberLoadedRows": 1000000, - "NumberFilteredRows": 1, - "NumberUnselectedRows": 0, - "LoadBytes": 40888898, - "LoadTimeMs": 2144, - "ErrorURL": "http://192.168.1.1:8042/api/_load_error_log?file=__shard_0/error_log_insert_stmt_db18266d4d9b4ee5-abb00ddd64bdf005_db18266d4d9b4ee5_abb00ddd64bdf005" -} -``` - -The following main explanations are given for the Stream load import result parameters: - -+ TxnId: The imported transaction ID. Users do not perceive. - -+ Label: Import Label. User specified or automatically generated by the system. - -+ Status: Import completion status. - - "Success": Indicates successful import. - - "Publish Timeout": This state also indicates that the import has been completed, except that the data may be delayed and visible without retrying. - - "Label Already Exists"：Label duplicate, need to be replaced Label. - - "Fail": Import failed. - -+ ExistingJobStatus: The state of the load job corresponding to the existing Label. - - This field is displayed only when the status is "Label Already Exists". The user can know the status of the load job corresponding to Label through this state. "RUNNING" means that the job is still executing, and "FINISHED" means that the job is successful. - -+ Message: Import error messages. - -+ NumberTotalRows: Number of rows imported for total processing. - -+ NumberLoadedRows: Number of rows successfully imported. - -+ NumberFilteredRows: Number of rows that do not qualify for data quality. - -+ NumberUnselectedRows: Number of rows filtered by where condition. - -+ LoadBytes: Number of bytes imported. - -+ LoadTimeMs: Import completion time. Unit milliseconds. - -+ ErrorURL: If you have data quality problems, visit this URL to see specific error lines. - -> Note: Since Stream load is a synchronous import mode, import information will not be recorded in Doris system. Users cannot see Stream load asynchronously by looking at import commands. You need to listen for the return value of the create import request to get the import result. - -### Cancel Load - -Users can't cancel Stream load manually. Stream load will be cancelled automatically by the system after a timeout or import error. - -## Relevant System Configuration - -### FE configuration - -+ stream\_load\_default\_timeout\_second - - The timeout time of the import task (in seconds) will be cancelled by the system if the import task is not completed within the set timeout time, and will become CANCELLED. - - At present, Stream load does not support custom import timeout time. All Stream load import timeout time is uniform. The default timeout time is 300 seconds. If the imported source file can no longer complete the import within the specified time, the FE parameter ```stream_load_default_timeout_second``` needs to be adjusted. - -### BE configuration - -+ streaming\_load\_max\_mb - - The maximum import size of Stream load is 10G by default, in MB. If the user's original file exceeds this value, the BE parameter ```streaming_load_max_mb``` needs to be adjusted. - -## Best Practices - -### Application scenarios - -The most appropriate scenario for using Stream load is that the original file is in memory or on disk. Secondly, since Stream load is a synchronous import method, users can also use this import if they want to obtain the import results in a synchronous manner. - -### Data volume - -Since Stream load is based on the BE initiative to import and distribute data, the recommended amount of imported data is between 1G and 10G. Since the default maximum Stream load import data volume is 10G, the configuration of BE ```streaming_load_max_mb``` needs to be modified if files exceeding 10G are to be imported. - -``` -For example, the size of the file to be imported is 15G -Modify the BE configuration streaming_load_max_mb to 16000 -``` - -Stream load default timeout is 300 seconds, according to Doris currently the largest import speed limit, about more than 3G files need to modify the import task default timeout. - -``` -Import Task Timeout = Import Data Volume / 10M / s (Specific Average Import Speed Requires Users to Calculate Based on Their Cluster Conditions) -For example, import a 10G file -Timeout = 1000s -31561;. 20110G / 10M /s -``` - -### Complete examples -Data situation: In the local disk path / home / store_sales of the sending and importing requester, the imported data is about 15G, and it is hoped to be imported into the table store\_sales of the database bj_sales. - -Cluster situation: The concurrency of Stream load is not affected by cluster size. - -+ Step 1: Does the import file size exceed the default maximum import size of 10G - - ``` - BE conf - streaming_load_max_mb = 16000 - ``` -+ Step 2: Calculate whether the approximate import time exceeds the default timeout value - - ``` - Import time 15000/10 = 1500s - Over the default timeout time, you need to modify the FE configuration - stream_load_default_timeout_second = 1500 - ``` - -+ Step 3: Create Import Tasks - - ``` - curl --location-trusted -u user:password -T /home/store_sales -H "label:abc" http://abc.com:8000/api/bj_sales/store_sales/_stream_load - ``` - -## Common Questions - -* Label Already Exists - - The Label repeat checking steps of Stream load are as follows: - - 1. Is there an import Label conflict that already exists with other import methods? - - Because imported Label in Doris system does not distinguish between import methods, there is a problem that other import methods use the same Label. - - Through ``SHOW LOAD WHERE LABEL = "xxx"'``, where XXX is a duplicate Label string, see if there is already a Label imported by FINISHED that is the same as the Label created by the user. - - 2. Are Stream loads submitted repeatedly for the same job? - - Since Stream load is an HTTP protocol submission creation import task, HTTP Clients in various languages usually have their own request retry logic. After receiving the first request, the Doris system has started to operate Stream load, but because the result is not returned to the Client side in time, the Client side will retry to create the request. At this point, the Doris system is already operating on the first request, so the second request will be reported to Label Already Exists. - - To sort out the possible methods mentioned above: Search FE Master's log with Label to see if there are two ``redirect load action to destination = ``redirect load action to destination'cases in the same Label. If so, the request is submitted repeatedly by the Client side. - - It is suggested that the user calculate the approximate import time according to the data quantity of the current request, and change the request time-out time of the Client end according to the import time-out time, so as to avoid the request being submitted by the Client end many times. diff --git a/docs/documentation/en/administrator-guide/operation/index.rst b/docs/documentation/en/administrator-guide/operation/index.rst deleted file mode 100644 index b961f2bc1a0287..00000000000000 --- a/docs/documentation/en/administrator-guide/operation/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -========================== -Maintainence Operation -========================== - -.. toctree:: - :maxdepth: 2 - :glob: - - * diff --git a/docs/documentation/en/administrator-guide/operation/metadata-operation_EN.md b/docs/documentation/en/administrator-guide/operation/metadata-operation_EN.md deleted file mode 100644 index caa626c8707c63..00000000000000 --- a/docs/documentation/en/administrator-guide/operation/metadata-operation_EN.md +++ /dev/null @@ -1,335 +0,0 @@ - - -# Metadata Operations and Maintenance - -This document focuses on how to manage Doris metadata in a real production environment. It includes the proposed deployment of FE nodes, some commonly used operational methods, and common error resolution methods. - -For the time being, read the [Doris metadata design document](../../internal/metadata-design_EN.md) to understand how Doris metadata works. - -## Important tips - -* Current metadata design is not backward compatible. That is, if the new version has a new metadata structure change (you can see whether there is a new VERSION in the `FeMetaVersion. java'file in the FE code), it is usually impossible to roll back to the old version after upgrading to the new version. Therefore, before upgrading FE, be sure to test metadata compatibility according to the operations in the [Upgrade Document](../../installing/upgrade_EN.md). - -## Metadata catalog structure - -Let's assume that the path of `meta_dir` specified in fe.conf is `path/to/palo-meta`. In a normal Doris cluster, the directory structure of metadata should be as follows: - -``` -/path/to/palo-meta/ - |-- bdb/ - | |-- 00000000.jdb - | |-- je.config.csv - | |-- je.info.0 - | |-- je.info.0.lck - | |-- je.lck - | `-- je.stat.csv - `-- image/ - |-- ROLE - |-- VERSION - `-- image.xxxx -``` - -1. bdb - - We use [bdbje] (https://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html) as a distributed kV system to store metadata journal. This BDB directory is equivalent to the "data directory" of bdbje. - - The `.jdb` suffix is the data file of bdbje. These data files will increase with the increasing number of metadata journals. When Doris regularly completes the image, the old log is deleted. So normally, the total size of these data files varies from several MB to several GB (depending on how Doris is used, such as import frequency). When the total size of the data file is larger than 10GB, you may need to wonder whether the image failed or the historical journals that failed to distribute the image could not be deleted. - - ` je.info.0 ` is the running log of bdbje. The time in this log is UTC + 0 time zone. We may fix this in a later version. From this log, you can also see how some bdbje works. - -2. image directory - - The image directory is used to store metadata mirrors generated regularly by Doris. Usually, you will see a `image.xxxxx` mirror file. Where `xxxxx` is a number. This number indicates that the image contains all metadata journal before `xxxx`. And the generation time of this file (viewed through `ls -al`) is usually the generation time of the mirror. - - You may also see a `image.ckpt` file. This is a metadata mirror being generated. The `du -sh` command should show that the file size is increasing, indicating that the mirror content is being written to the file. When the mirror is written, it automatically renames itself to a new `image.xxxxx` and replaces the old image file. - - Only FE with a Master role will actively generate image files on a regular basis. After each generation, FE is pushed to other non-Master roles. When it is confirmed that all other FEs have received this image, Master FE deletes the metadata journal in bdbje. Therefore, if image generation fails or image push fails to other FEs, data in bdbje will accumulate. - - `ROLE` file records the type of FE (FOLLOWER or OBSERVER), which is a text file. - - `VERSION` file records the cluster ID of the Doris cluster and the token used to access authentication between nodes, which is also a text file. - - `ROLE` file and `VERSION` file may only exist at the same time, or they may not exist at the same time (e.g. at the first startup). - -## Basic operations - -### Start single node FE - -Single node FE is the most basic deployment mode. A complete Doris cluster requires at least one FE node. When there is only one FE node, the type of the node is Follower and the role is Master. - -1. First start-up - - 1. Suppose the path of `meta_dir` specified in fe.conf is `path/to/palo-meta`. - 2. Ensure that `path/to/palo-meta` already exists, that the permissions are correct and that the directory is empty. - 3. Start directly through `sh bin/start_fe.sh`. - 4. After booting, you should be able to see the following log in fe.log: - - * Palo FE starting... - * image does not exist: /path/to/palo-meta/image/image.0 - * transfer from INIT to UNKNOWN - * transfer from UNKNOWN to MASTER - * the very first time to open bdb, dbname is 1 - * start fencing, epoch number is 1 - * finish replay in xxx msec - * QE service start - * thrift server started - - The above logs are not necessarily strictly in this order, but they are basically similar. - - 5. The first start-up of a single-node FE usually does not encounter problems. If you haven't seen the above logs, generally speaking, you haven't followed the document steps carefully, please read the relevant wiki carefully. - -2. Restart - - 1. Stopped FE nodes can be restarted by using `sh bin/start_fe.sh`. - 2. After restarting, you should be able to see the following log in fe.log: - - * Palo FE starting... - * finished to get cluster id: xxxx, role: FOLLOWER and node name: xxxx - * If no image has been generated before reboot, you will see: - * image does not exist: /path/to/palo-meta/image/image.0 - - * If an image is generated before the restart, you will see: - * start load image from /path/to/palo-meta/image/image.xxx. is ckpt: false - * finished load image in xxx ms - - * transfer from INIT to UNKNOWN - * replayed journal id is xxxx, replay to journal id is yyyy - * transfer from UNKNOWN to MASTER - * finish replay in xxx msec - * master finish replay journal, can write now. - * begin to generate new image: image.xxxx - * start save image to /path/to/palo-meta/image/image.ckpt. is ckpt: true - * finished save image /path/to/palo-meta/image/image.ckpt in xxx ms. checksum is xxxx - * push image.xxx to other nodes. totally xx nodes, push successed xx nodes - * QE service start - * thrift server started - - The above logs are not necessarily strictly in this order, but they are basically similar. - -3. Common problems - - For the deployment of single-node FE, start-stop usually does not encounter any problems. If you have any questions, please refer to the relevant Wiki and check your operation steps carefully. - -### Add FE - -Adding FE processes is described in detail in the [Deployment and Upgrade Documents] (https://github.com/apache/incubator-doris/wiki/Doris-Deploy-%26-Upgrade) and will not be repeated. Here are some points for attention, as well as common problems. - -1. Notes - - * Before adding a new FE, make sure that the current Master FE runs properly (connection is normal, JVM is normal, image generation is normal, bdbje data directory is too large, etc.) - * The first time you start a new FE, you must make sure that the `-helper` parameter is added to point to Master FE. There is no need to add `-helper` when restarting. (If `-helper` is specified, FE will directly ask the helper node for its role. If not, FE will try to obtain information from `ROLE` and `VERSION` files in the `palo-meta/image/` directory. - * The first time you start a new FE, you must make sure that the `meta_dir` of the FE is created, has correct permissions and is empty. - * Starting a new FE and executing the `ALTER SYSTEM ADD FOLLOWER/OBSERVER` statement adds FE to metadata in a sequence that is not required. If a new FE is started first and no statement is executed, the `current node is not added to the group. Please add it first.` in the new FE log. When the statement is executed, it enters the normal process. - * Make sure that after the previous FE is added successfully, the next FE is added. - * Connect to MASTER FE and execute `ALTER SYSTEM ADD FOLLOWER/OBSERVER` claus。 - -2. Common problems - - 1. this need is DETACHED - - When you first start a FE to be added, if the data in palo-meta/bdb on Master FE is large, you may see the words `this node is DETACHED`. in the FE log to be added. At this point, bdbje is copying data, and you can see that the `bdb/` directory of FE to be added is growing. This process usually takes several minutes (depending on the amount of data in bdbje). Later, there may be some bdbje-related error stack information in fe. log. If `QE service start` and `thrift server start` are displayed in the final log, the start is usually successful. You can try to connect this FE via mysql-client. If these words do not appear, it may be the problem of bdbje replication log timeout. At this point, restarting the FE directly will usually solve the problem. - - 2. Failure to add due to various reasons - - * If OBSERVER is added, because OBSERVER-type FE does not participate in the majority of metadata writing, it can theoretically start and stop at will. Therefore, for the case of adding OBSERVER failure. The process of OBSERVER FE can be killed directly. After clearing the metadata directory of OBSERVER, add the process again. - - * If FOLLOWER is added, because FOLLOWER is mostly written by participating metadata. So it is possible that FOLLOWER has joined the bdbje electoral team. If there are only two FOLLOWER nodes (including MASTER), then stopping one FE may cause another FE to quit because it cannot write most of the time. At this point, we should first delete the newly added FOLLOWER node from the metadata through the `ALTER SYSTEM DROP FOLLOWER` command, then kill the FOLLOWER process, empty the metadata and re-add the process. - - -### Delete FE - -The corresponding type of FE can be deleted by the `ALTER SYSTEM DROP FOLLOWER/OBSERVER` command. The following points should be noted: - -* For OBSERVER type FE, direct DROP is enough, without risk. - -* For FOLLOWER type FE. First, you should make sure that you start deleting an odd number of FOLLOWERs (three or more). - - 1. If the FE of non-MASTER role is deleted, it is recommended to connect to MASTER FE, execute DROP command, and then kill the process. - 2. If you want to delete MASTER FE, first confirm that there are odd FOLLOWER FE and it works properly. Then kill the MASTER FE process first. At this point, a FE will be elected MASTER. After confirming that the remaining FE is working properly, connect to the new MASTER FE and execute the DROP command to delete the old MASTER FE. - -## Advanced Operations - -### Failure recovery - -FE may fail to start bdbje and synchronize between FEs for some reasons. Phenomena include the inability to write metadata, the absence of MASTER, and so on. At this point, we need to manually restore the FE. The general principle of manual recovery of FE is to start a new MASTER through metadata in the current `meta_dir`, and then add other FEs one by one. Please follow the following steps strictly: - -1. First, stop all FE processes and all business access. Make sure that during metadata recovery, external access will not lead to other unexpected problems. - -2. Identify which FE node's metadata is up-to-date: - - * First of all, **be sure to back up all FE's `meta_dir` directories first.** - * Usually, Master FE's metadata is up to date. You can see the suffix of image.xxxx file in the `meta_dir/image` directory. The larger the number, the newer the metadata. - * Usually, by comparing all FOLLOWER FE image files, you can find the latest metadata. - * After that, we use the FE node with the latest metadata to recover. - * If using metadata of OBSERVER node to recover will be more troublesome, it is recommended to choose FOLLOWER node as far as possible. - -3. The following operations are performed on the FE nodes selected in step 2. - - 1. If the node is an OBSERVER, first change the `role=OBSERVER` in the `meta_dir/image/ROLE` file to `role=FOLLOWER`. (Recovery from the OBSERVER node will be more cumbersome, first follow the steps here, followed by a separate description) - 2. Add configuration in fe.conf: `metadata_failure_recovery=true`. - 3. Run `sh bin/start_fe.sh` to start the FE - 4. If normal, the FE will start in the role of MASTER, similar to the description in the previous section `Start a single node FE`. You should see the words `transfer from XXXX to MASTER` in fe.log. - 5. After the start-up is completed, connect to the FE first, and execute some query imports to check whether normal access is possible. If the operation is not normal, it may be wrong. It is recommended to read the above steps carefully and try again with the metadata previously backed up. If not, the problem may be more serious. - 6. If successful, through the `show frontends;` command, you should see all the FEs you added before, and the current FE is master. - 7. Delete the `metadata_failure_recovery=true` configuration item in fe.conf, or set it to `false`, and restart the FE (**Important**). - - - > If you are recovering metadata from an OBSERVER node, after completing the above steps, you will find that the current FE role is OBSERVER, but `IsMaster` appears as `true`. This is because the "OBSERVER" seen here is recorded in Doris's metadata, but whether it is master or not, is recorded in bdbje's metadata. Because we recovered from an OBSERVER node, there was inconsistency. Please take the following steps to fix this problem (we will fix it in a later version): - - > 1. First, all FE nodes except this "OBSERVER" are DROPed out. - > 2. A new FOLLOWER FE is added through the `ADD FOLLOWER` command, assuming that it is on hostA. - > 3. Start a new FE on hostA and join the cluster by `helper`. - > 4. After successful startup, you should see two FEs through the `show frontends;` statement, one is the previous OBSERVER, the other is the newly added FOLLOWER, and the OBSERVER is the master. - > 5. After confirming that the new FOLLOWER is working properly, the new FOLLOWER metadata is used to perform a failure recovery operation again. - > 6. The purpose of the above steps is to manufacture a metadata of FOLLOWER node artificially, and then use this metadata to restart fault recovery. This avoids inconsistencies in recovering metadata from OBSERVER. - - >The meaning of `metadata_failure_recovery = true` is to empty the metadata of `bdbje`. In this way, bdbje will not contact other FEs before, but start as a separate FE. This parameter needs to be set to true only when restoring startup. After recovery, it must be set to false. Otherwise, once restarted, the metadata of bdbje will be emptied again, which will make other FEs unable to work properly. - -4. After the successful execution of step 3, we delete the previous FEs from the metadata by using the `ALTER SYSTEM DROP FOLLOWER/OBSERVER` command and add them again by adding new FEs. - -5. If the above operation is normal, it will be restored. - -### FE type change - -If you need to change the existing FOLLOWER/OBSERVER type FE to OBSERVER/FOLLOWER type, please delete FE in the way described above, and then add the corresponding type FE. - -### FE Migration - -If you need to migrate one FE from the current node to another, there are several scenarios. - -1. FOLLOWER, or OBSERVER migration for non-MASTER nodes - - After adding a new FOLLOWER / OBSERVER directly, delete the old FOLLOWER / OBSERVER. - -2. Single-node MASTER migration - - When there is only one FE, refer to the `Failure Recovery` section. Copy the palo-meta directory of FE to the new node and start the new MASTER in Step 3 of the `Failure Recovery` section - -3. A set of FOLLOWER migrates from one set of nodes to another set of new nodes - - Deploy FE on the new node and add the new node first by adding FOLLOWER. The old nodes can be dropped by DROP one by one. In the process of DROP-by-DROP, MASTER automatically selects the new FOLLOWER node. - -### Replacement of FE port - -FE currently has the following ports - -* Ed_log_port: bdbje's communication port -* http_port: http port, also used to push image -* rpc_port：FE 的 thrift server port -* query_port: Mysql connection port - -1. edit_log_port - - If this port needs to be replaced, it needs to be restored with reference to the operations in the `Failure Recovery` section. Because the port has been persisted into bdbje's own metadata (also recorded in Doris's own metadata), it is necessary to clear bdbje's metadata by setting `metadata_failure_recovery=true`. - -2. http_port - - All FE http_ports must be consistent. So if you want to modify this port, all FEs need to be modified and restarted. Modifying this port will be more complex in the case of multiple FOLLOWER deployments (involving laying eggs and laying hens...), so this operation is not recommended. If necessary, follow the operation in the `Failure Recovery` section directly. - -3. rpc_port - - After modifying the configuration, restart FE directly. Master FE informs BE of the new port through heartbeat. Only this port of Master FE will be used. However, it is still recommended that all FE ports be consistent. - -4. query_port - - After modifying the configuration, restart FE directly. This only affects mysql's connection target. - -### Recover metadata from FE memory -In some extreme cases, the image file on the disk may be damaged, but the metadata in the memory is intact. At this point, we can dump the metadata from the memory and replace the image file on the disk to recover the metadata. the entire non-stop query service operation steps are as follows: - -1. Stop all Load, Create, Alter operations. - -2. Execute the following command to dump metadata from the Master FE memory: (hereafter called image_mem) -``` -curl -u $root_user:$password http://$master_hostname:8030/dump -``` -3. Replace the image file in the `meta_dir/image` directory on the OBSERVER FE node with the image_mem file, restart the OBSERVER FE node, and verify the integrity and correctness of the image_mem file. You can check whether the DB and Table metadata are normal on the FE Web page, whether there is an exception in `fe.log`, whether it is in a normal replayed jour. - -4. Replace the image file in the `meta_dir/image` directory on the FOLLOWER FE node with the image_mem file in turn, restart the FOLLOWER FE node, and confirm that the metadata and query services are normal. - -5. Replace the image file in the `meta_dir/image` directory on the Master FE node with the image_mem file, restart the Master FE node, and then confirm that the FE Master switch is normal and The Master FE node can generate a new image file through checkpoint. - -6. Recover all Load, Create, Alter operations. - -**Note: If the Image file is large, the entire process can take a long time, so during this time, make sure Master FE does not generate a new image file via checkpoint. When the image.ckpt file in the meta_dir/image directory on the Master FE node is observed to be as large as the image.xxx file, the image.ckpt file can be deleted directly.** - - -## Best Practices - -The deployment recommendation of FE is described in the Installation and [Deployment Document](../../installing/install-deploy_EN.md). Here are some supplements. - -* **If you don't know the operation logic of FE metadata very well, or you don't have enough experience in the operation and maintenance of FE metadata, we strongly recommend that only one FOLLOWER-type FE be deployed as MASTER in practice, and the other FEs are OBSERVER, which can reduce many complex operation and maintenance problems.** Don't worry too much about the failure of MASTER single point to write metadata. First, if you configure it properly, FE as a java process is very difficult to hang up. Secondly, if the MASTER disk is damaged (the probability is very low), we can also use the metadata on OBSERVER to recover manually through `fault recovery`. - -* The JVM of the FE process must ensure sufficient memory. We **strongly recommend** that FE's JVM memory should be at least 10GB and 32GB to 64GB. And deploy monitoring to monitor JVM memory usage. Because if OOM occurs in FE, metadata writing may fail, resulting in some failures that **cannot recover**! - -* FE nodes should have enough disk space to prevent the excessive metadata from causing insufficient disk space. At the same time, FE logs also take up more than a dozen gigabytes of disk space. - -## Other common problems - -1. Output `meta out of date. current time: xxx, synchronized time: xxx, has log: xxx, fe type: xxx` in fe.log - - This is usually because the FE cannot elect Master. For example, if three FOLLOWERs are configured, but only one FOLLOWER is started, this FOLLOWER will cause this problem. Usually, just start the remaining FOLLOWER. If the problem has not been solved after the start-up, manual recovery may be required in accordance with the way in the `Failure Recovery` section. - -2. `Clock delta: xxxx ms. between Feeder: xxxx and this Replica exceeds max permissible delta: xxxx ms.` - - Bdbje requires that clock errors between nodes should not exceed a certain threshold. If exceeded, the node will exit abnormally. The default threshold is 5000ms, which is controlled by FE parameter `max_bdbje_clock_delta_ms', and can be modified as appropriate. But we suggest using NTP and other clock synchronization methods to ensure the clock synchronization of Doris cluster hosts. - - -3. Mirror files in the `image/` directory have not been updated for a long time - - Master FE generates a mirror file by default for every 50,000 metadata journal. In a frequently used cluster, a new image file is usually generated every half to several days. If you find that the image file has not been updated for a long time (for example, more than a week), you can see the reasons in sequence as follows: - - 1. Search for `memory is not enough to do checkpoint. Committed memroy XXXX Bytes, used memory XXXX Bytes. ` in the fe.log of Master FE. If found, it indicates that the current FE's JVM memory is insufficient for image generation (usually we need to reserve half of the FE memory for image generation). Then you need to add JVM memory and restart FE before you can observe. Each time Master FE restarts, a new image is generated directly. This restart method can also be used to actively generate new images. Note that if there are multiple FOLLOWER deployments, then when you restart the current Master FE, another FOLLOWER FE will become MASTER, and subsequent image generation will be the responsibility of the new Master. Therefore, you may need to modify the JVM memory configuration of all FOLLOWER FE. - - 2. Search for `begin to generate new image: image.xxxx` in the fe.log of Master FE. If it is found, then the image is generated. Check the subsequent log of this thread, and if `checkpoint finished save image.xxxx` appears, the image is written successfully. If `Exception when generating new image file` occurs, the generation fails and specific error messages need to be viewed. - - -4. The size of the `bdb/` directory is very large, reaching several Gs or more. - - The BDB directory will remain large for some time after eliminating the error that the new image cannot be generated. Maybe it's because Master FE failed to push image. You can search `push image.XXXX to other nodes. totally XX nodes, push successed YY nodes` in the fe. log of Master FE. If YY is smaller than xx, then some FEs are not pushed successfully. You can see the specific error `Exception when pushing image file.url = xxx` in the fe. log. - - At the same time, you can add the configuration in the FE configuration file: `edit_log_roll_num = xxxx`. This parameter sets the number of metadata journals and makes an image once. The default is 50000. This number can be reduced appropriately to make images more frequent, thus speeding up the deletion of old journals. - -5. FOLLOWER FE hangs up one after another - - Because Doris's metadata adopts the majority writing strategy, that is, a metadata journal must be written to at least a number of FOLLOWER FEs (for example, three FOLLOWERs, two must be written successfully) before it can be considered successful. If the write fails, the FE process exits on its own initiative. So suppose there are three FOLLOWERs: A, B and C. C hangs up first, and then B hangs up, then A will hang up. So as described in the `Best Practices `section, if you don't have extensive experience in metadata operations and maintenance, it's not recommended to deploy multiple FOLLOWERs. - -6. fe.log 中出现 `get exception when try to close previously opened bdb database. ignore it` - - If there is the word `ignore it` behind it, there is usually no need to deal with it. If you are interested, you can search for this error in `BDBEnvironment.java`, and see the annotations. - -7. From `show frontends;` Look, the `Join` of a FE is listed as `true`, but actually the FE is abnormal. - - Through `show frontends;` see the `Join` information. If the column is `true`, it only means that the FE **has joined the** cluster. It does not mean that it still exists normally in the cluster. If `false`, it means that the FE **has never joined the** cluster. - -8. Configuration of FE `master_sync_policy`, `replica_sync_policy`, and `txn_rollback_limit.` - - `master_sync_policy` is used to specify whether fsync (), `replica_sync_policy` is called when Leader FE writes metadata log, and `replica_sync_policy` is used to specify whether other Follower FE calls fsync () when FE HA deploys synchronous metadata. In earlier versions of Oris, these two parameters defaulted to `WRITE_NO_SYNC`, i.e., fsync () was not called. In the latest version of Oris, the default has been changed to `SYNC`, that is, fsync () is called. Calling fsync () significantly reduces the efficiency of metadata disk writing. In some environments, IOPS may drop to several hundred and the latency increases to 2-3ms (but it's still enough for Doris metadata manipulation). Therefore, we recommend the following configuration: - - 1. For a single Follower FE deployment, `master_sync_policy` is set to `SYNC`, which prevents the loss of metadata due to the downtime of the FE system. - 2. For multi-Follower FE deployment, we can set `master_sync_policy` and `replica_sync_policy` to `WRITE_NO_SYNC`, because we think that the probability of simultaneous outage of multiple systems is very low. - - If `master_sync_policy` is set to `WRITE_NO_SYNC` in a single Follower FE deployment, then a FE system outage may occur, resulting in loss of metadata. At this point, if other Observer FE attempts to restart, it may report an error: - - ``` - Node xxx must rollback xx total commits(numPassedDurableCommits of which were durable) to the earliest point indicated by transaction xxxx in order to rejoin the replication group, but the transaction rollback limit of xxx prohibits this. - ``` - -This means that some transactions that have been persisted need to be rolled back, but the number of entries exceeds the upper limit. Here our default upper limit is 100, which can be changed by setting `txn_rollback_limit`. This operation is only used to attempt to start FE normally, but lost metadata cannot be recovered. diff --git a/docs/documentation/en/administrator-guide/operation/monitor-alert_EN.md b/docs/documentation/en/administrator-guide/operation/monitor-alert_EN.md deleted file mode 100644 index d8a82a767d325d..00000000000000 --- a/docs/documentation/en/administrator-guide/operation/monitor-alert_EN.md +++ /dev/null @@ -1,302 +0,0 @@ - - -# Monitoring and alarming - -This document mainly introduces Doris's monitoring items and how to collect and display them. And how to configure alarm (TODO) - -[Dashborad template click download](https://grafana.com/dashboards/9734/revisions) - -> Note: Before 0.9.0 (excluding), please use revision 1. For version 0.9.x, use revision 2. For version 0.10.x, use revision 3. - -Dashboard templates are updated from time to time. The way to update the template is shown in the last section. - -Welcome to provide better dashboard. - -## Components - -Doris uses [Prometheus] (https://prometheus.io/) and [Grafana] (https://grafana.com/) to collect and display input monitoring items. - -![](../../../../resources/images/dashboard_overview.png) - -1. Prometheus - - Prometheus is an open source system monitoring and alarm suite. It can collect monitored items by Pull or Push and store them in its own time series database. And through the rich multi-dimensional data query language, to meet the different data display needs of users. - -2. Grafana - - Grafana is an open source data analysis and display platform. Support multiple mainstream temporal database sources including Prometheus. Through the corresponding database query statements, the display data is obtained from the data source. With flexible and configurable dashboard, these data can be quickly presented to users in the form of graphs. - -> Note: This document only provides a way to collect and display Doris monitoring data using Prometheus and Grafana. In principle, these components are not developed or maintained. For more details on these components, please step through the corresponding official documents. - -## Monitoring data - -Doris's monitoring data is exposed through the HTTP interface of Frontend and Backend. Monitoring data is presented in the form of key-value text. Each Key may also be distinguished by different Labels. When the user has built Doris, the monitoring data of the node can be accessed in the browser through the following interfaces: - -* Frontend: `fe_host:fe_http_port/metrics` -* Backend: `be_host:be_web_server_port/metrics` -* Broker: Not available for now - -Users will see the following monitoring item results (for example, FE partial monitoring items): - - ``` - # HELP jvm_heap_size_bytes jvm heap stat - # TYPE jvm_heap_size_bytes gauge - jvm_heap_size_bytes{type="max"} 41661235200 - jvm_heap_size_bytes{type="committed"} 19785285632 - jvm_heap_size_bytes{type="used"} 10113221064 - # HELP jvm_non_heap_size_bytes jvm non heap stat - # TYPE jvm_non_heap_size_bytes gauge - jvm_non_heap_size_bytes{type="committed"} 105295872 - jvm_non_heap_size_bytes{type="used"} 103184784 - # HELP jvm_young_size_bytes jvm young mem pool stat - # TYPE jvm_young_size_bytes gauge - jvm_young_size_bytes{type="used"} 6505306808 - jvm_young_size_bytes{type="peak_used"} 10308026368 - jvm_young_size_bytes{type="max"} 10308026368 - # HELP jvm_old_size_bytes jvm old mem pool stat - # TYPE jvm_old_size_bytes gauge - jvm_old_size_bytes{type="used"} 3522435544 - jvm_old_size_bytes{type="peak_used"} 6561017832 - jvm_old_size_bytes{type="max"} 30064771072 - # HELP jvm_direct_buffer_pool_size_bytes jvm direct buffer pool stat - # TYPE jvm_direct_buffer_pool_size_bytes gauge - jvm_direct_buffer_pool_size_bytes{type="count"} 91 - jvm_direct_buffer_pool_size_bytes{type="used"} 226135222 - jvm_direct_buffer_pool_size_bytes{type="capacity"} 226135221 - # HELP jvm_young_gc jvm young gc stat - # TYPE jvm_young_gc gauge - jvm_young_gc{type="count"} 2186 - jvm_young_gc{type="time"} 93650 - # HELP jvm_old_gc jvm old gc stat - # TYPE jvm_old_gc gauge - jvm_old_gc{type="count"} 21 - jvm_old_gc{type="time"} 58268 - # HELP jvm_thread jvm thread stat - # TYPE jvm_thread gauge - jvm_thread{type="count"} 767 - jvm_thread{type="peak_count"} 831 - ... - ``` - -This is a monitoring data presented in [Promethus Format] (https://prometheus.io/docs/practices/naming/). We take one of these monitoring items as an example to illustrate: - -``` -# HELP jvm_heap_size_bytes jvm heap stat -# TYPE jvm_heap_size_bytes gauge -jvm_heap_size_bytes{type="max"} 41661235200 -jvm_heap_size_bytes{type="committed"} 19785285632 -jvm_heap_size_bytes{type="used"} 10113221064 -``` - -1. Behavior commentary line at the beginning of "#". HELP is the description of the monitored item; TYPE represents the data type of the monitored item, and Gauge is the scalar data in the example. There are also Counter, Histogram and other data types. Specifically, you can see [Prometheus Official Document] (https://prometheus.io/docs/practices/instrumentation/#counter-vs.-gauge,-summary-vs.-histogram). -2. `jvm_heap_size_bytes` is the name of the monitored item (Key); `type= "max"` is a label named `type`, with a value of `max`. A monitoring item can have multiple Labels. -3. The final number, such as `41661235200`, is the monitored value. - -## Monitoring Architecture - -The entire monitoring architecture is shown in the following figure: - -![](../../../../resources/images/monitor_arch.png) - -1. The yellow part is Prometheus related components. Prometheus Server is the main process of Prometheus. At present, Prometheus accesses the monitoring interface of Doris node by Pull, and then stores the time series data in the time series database TSDB (TSDB is included in the Prometheus process, and need not be deployed separately). Prometheus also supports building [Push Gateway] (https://github.com/prometheus/pushgateway) to allow monitored data to be pushed to Push Gateway by Push by monitoring system, and then data from Push Gateway by Prometheus Server through Pull. -2. [Alert Manager] (https://github.com/prometheus/alertmanager) is a Prometheus alarm component, which needs to be deployed separately (no solution is provided yet, but can be built by referring to official documents). Through Alert Manager, users can configure alarm strategy, receive mail, short messages and other alarms. -3. The green part is Grafana related components. Grafana Server is the main process of Grafana. After startup, users can configure Grafana through Web pages, including data source settings, user settings, Dashboard drawing, etc. This is also where end users view monitoring data. - - -## Start building - -Please start building the monitoring system after you have completed the deployment of Doris. - -Prometheus - -1. Download the latest version of Proetheus on the [Prometheus Website] (https://prometheus.io/download/). Here we take version 2.3.2-linux-amd64 as an example. -2. Unzip the downloaded tar file on the machine that is ready to run the monitoring service. -3. Open the configuration file promethues.yml. Here we provide an example configuration and explain it (the configuration file is in YML format, pay attention to uniform indentation and spaces): - - Here we use the simplest way of static files to monitor configuration. Prometheus supports a variety of [service discovery] (https://prometheus.io/docs/prometheus/latest/configuration/configuration/), which can dynamically sense the addition and deletion of nodes. - - ``` - # my global config - global: - scrape_interval: 15s # Global acquisition interval, default 1 m, set to 15s - evaluation_interval: 15s # Global rule trigger interval, default 1 m, set 15s here - - # Alertmanager configuration - alerting: - alertmanagers: - - static_configs: - - targets: - # - alertmanager:9093 - - # A scrape configuration containing exactly one endpoint to scrape: - # Here it's Prometheus itself. - scrape_configs: - # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'PALO_CLUSTER' # Each Doris cluster, we call it a job. Job can be given a name here as the name of Doris cluster in the monitoring system. - metrics_path: '/metrics' # Here you specify the restful API to get the monitors. With host: port in the following targets, Prometheus will eventually collect monitoring items through host: port/metrics_path. - static_configs: # Here we begin to configure the target addresses of FE and BE, respectively. All FE and BE are written into their respective groups. - - targets: ['fe_host1:8030', 'fe_host2:8030', 'fe_host3:8030'] - labels: - group: fe # Here configure the group of fe, which contains three Frontends - - - targets: ['be_host1:8040', 'be_host2:8040', 'be_host3:8040'] - labels: - group: be # Here configure the group of be, which contains three Backends - - - job_name: 'PALO_CLUSTER_2' # We can monitor multiple Doris clusters in a Prometheus, where we begin the configuration of another Doris cluster. Configuration is the same as above, the following is outlined. - metrics_path: '/metrics' - static_configs: - - targets: ['fe_host1:8030', 'fe_host2:8030', 'fe_host3:8030'] - labels: - group: fe - - - targets: ['be_host1:8040', 'be_host2:8040', 'be_host3:8040'] - labels: - group: be - - ``` - -4. start Promethues - - Start Promethues with the following command: - - `nohup ./prometheus --web.listen-address="0.0.0.0:8181" &` - - This command will run Prometheus in the background and specify its Web port as 8181. After startup, data is collected and stored in the data directory. - -5. stop Promethues - - At present, there is no formal way to stop the process, kill - 9 directly. Of course, Prometheus can also be set as a service to start and stop in a service way. - -6. access Prometheus - - Prometheus can be easily accessed through web pages. The page of Prometheus can be accessed by opening port 8181 through browser. Click on the navigation bar, `Status` -> `Targets`, and you can see all the monitoring host nodes of the grouped Jobs. Normally, all nodes should be `UP`, indicating that data acquisition is normal. Click on an `Endpoint` to see the current monitoring value. If the node state is not UP, you can first access Doris's metrics interface (see previous article) to check whether it is accessible, or query Prometheus related documents to try to resolve. - -7. So far, a simple Prometheus has been built and configured. For more advanced usage, see [Official Documents] (https://prometheus.io/docs/introduction/overview/) - -### Grafana - -1. Download the latest version of Grafana on [Grafana's official website] (https://grafana.com/grafana/download). Here we take version 5.2.1.linux-amd64 as an example. - -2. Unzip the downloaded tar file on the machine that is ready to run the monitoring service. - -3. Open the configuration file conf/defaults.ini. Here we only list the configuration items that need to be changed, and the other configurations can be used by default. - - ``` - # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) - data = data - - # Directory where grafana can store logs - logs = data/log - - # Protocol (http, https, socket) - protocal = http - - # The ip address to bind to, empty will bind to all interfaces - http_addr = - - # The http port to use - http_port = 8182 - ``` - -4. start Grafana - - Start Grafana with the following command - - `nohuo ./bin/grafana-server &` - - This command runs Grafana in the background, and the access port is 8182 configured above. - -5. stop Grafana - - At present, there is no formal way to stop the process, kill - 9 directly. Of course, you can also set Grafana as a service to start and stop as a service. - -6. access Grafana - - Through the browser, open port 8182, you can start accessing the Grafana page. The default username password is admin. - -7. Configure Grafana - - For the first landing, you need to set up the data source according to the prompt. Our data source here is Proetheus, which was configured in the previous step. - - The Setting page of the data source configuration is described as follows: - - 1. Name: Name of the data source, customized, such as doris_monitor_data_source - 2. Type: Select Prometheus - 3. URL: Fill in the web address of Prometheus, such as http://host:8181 - 4. Access: Here we choose the Server mode, which is to access Prometheus through the server where the Grafana process is located. - 5. The other options are available by default. - 6. Click `Save & Test` at the bottom. If `Data source is working`, it means that the data source is available. - 7. After confirming that the data source is available, click on the + number in the left navigation bar and start adding Dashboard. Here we have prepared Doris's dashboard template (at the beginning of this document). When the download is complete, click `New dashboard` -> `Import dashboard` -> `Upload.json File` above to import the downloaded JSON file. - 8. After importing, you can name Dashboard by default `Doris Overview`. At the same time, you need to select the data source, where you select the `doris_monitor_data_source` you created earlier. - 9. Click `Import` to complete the import. Later, you can see Doris's dashboard display. - -8. So far, a simple Grafana has been built and configured. For more advanced usage, see [Official Documents] (http://docs.grafana.org/) - - -## Dashboard - -Here we briefly introduce Doris Dashboard. The content of Dashboard may change with the upgrade of version. This document is not guaranteed to be the latest Dashboard description. - -1. Top Bar - - ![](../../../../resources/images/dashboard_navibar.png) - - * The upper left corner is the name of Dashboard. - * The upper right corner shows the current monitoring time range. You can choose different time ranges by dropping down. You can also specify a regular refresh page interval. - * Cluster name: Each job name in the Prometheus configuration file represents a Doris cluster. Select a different cluster, and the chart below shows the monitoring information for the corresponding cluster. - * fe_master: The Master Frontend node corresponding to the cluster. - * fe_instance: All Frontend nodes corresponding to the cluster. Select a different Frontend, and the chart below shows the monitoring information for the Frontend. - * be_instance: All Backend nodes corresponding to the cluster. Select a different Backend, and the chart below shows the monitoring information for the Backend. - * Interval: Some charts show rate-related monitoring items, where you can choose how much interval to sample and calculate the rate (Note: 15s interval may cause some charts to be unable to display). - -2. Row. - - ![](../../../../resources/images/dashboard_row.png) - - In Grafana, the concept of Row is a set of graphs. As shown in the figure above, Overview and Cluster Overview are two different Rows. Row can be folded by clicking Row. Currently Dashboard has the following Rows (in continuous updates): - - 1. Overview: A summary display of all Doris clusters. - 2. Cluster Overview: A summary display of selected clusters. - 3. Query Statistic: Query-related monitoring of selected clusters. - 4. FE JVM: Select Frontend's JVM monitoring. - 5. BE: A summary display of the backends of the selected cluster. - 6. BE Task: Display of Backends Task Information for Selected Clusters. - -3. Charts - - ![](../../../../resources/images/dashboard_panel.png) - - A typical icon is divided into the following parts: - - 1. Hover the I icon in the upper left corner of the mouse to see the description of the chart. - 2. Click on the illustration below to view a monitoring item separately. Click again to display all. - 3. Dragging in the chart can select the time range. - 4. The selected cluster name is displayed in [] of the title. - 5. Some values correspond to the Y-axis on the left and some to the right, which can be distinguished by the `-right` at the end of the legend. - 6. Click on the name of the chart -> `Edit` to edit the chart. - -## Dashboard Update - -1. Click on `+` in the left column of Grafana and `Dashboard`. -2. Click `New dashboard` in the upper left corner, and `Import dashboard` appears on the right. -3. Click `Upload .json File` to select the latest template file. -4. Selecting Data Sources -5. Click on `Import (Overwrite)` to complete the template update. diff --git a/docs/documentation/en/administrator-guide/operation/multi-tenant_EN.md b/docs/documentation/en/administrator-guide/operation/multi-tenant_EN.md deleted file mode 100644 index 5b3c263e292663..00000000000000 --- a/docs/documentation/en/administrator-guide/operation/multi-tenant_EN.md +++ /dev/null @@ -1,231 +0,0 @@ - - -# Multi-tenancy(Exprimental) - -This function is experimental and is not recommended for use in production environment. - -## Background -Doris, as a PB-level online report and multi-dimensional analysis database, provides cloud-based database services through open cloud, and deploys a physical cluster for each client in the cloud. Internally, a physical cluster deploys multiple services, and separately builds clusters for services with high isolation requirements. In view of the above problems: - -- Deployment of multiple physical cluster maintenance costs a lot (upgrade, functionality on-line, bug repair). -- A user's query or a bug caused by a query often affects other users. -- In the actual production environment, only one BE process can be deployed on a single machine. Multiple BEs can better solve the problem of fat nodes. And for join, aggregation operations can provide higher concurrency. - -Together with the above three points, Doris needs a new multi-tenant scheme, which not only can achieve better resource isolation and fault isolation, but also can reduce the cost of maintenance to meet the needs of common and private clouds. - -## Design Principles - -- Easy to use -- Low Development Cost -- Convenient migration of existing clusters - -## Noun Interpretation - -- FE: Frontend, the module for metadata management or query planning in Doris. -- BE: Backend, the module used to store and query data in Doris. -- Master: A role for FE. A Doris cluster has only one Master and the other FE is Observer or Follower. -- instance: A BE process is an instance in time. -- host: a single physical machine -- Cluster: A cluster consisting of multiple instances. -- Tenant: A cluster belongs to a tenant. Cluster is a one-to-one relationship with tenants. -- database: A user-created database - -## Main Ideas - -- Deploy instances of multiple BEs on a host to isolate resources at the process level. -- Multiple instances form a cluster, and a cluster is assigned to a business-independent tenant. -- FE adds cluster level and is responsible for cluster management. -- CPU, IO, memory and other resources are segregated by cgroup. - -## Design scheme - -In order to achieve isolation, the concept of **virtual cluster** is introduced. - -1. Cluster represents a virtual cluster consisting of instances of multiple BEs. Multiple clusters share FE. -2. Multiple instances can be started on a host. When a cluster is created, an arbitrary number of instances are selected to form a cluster. -3. While creating a cluster, an account named superuser is created, which belongs to the cluster. Super user can manage clusters, create databases, assign privileges, and so on. -4. After Doris starts, the sink creates a default cluster: default_cluster. If the user does not want to use the function of multi-cluster, the default cluster is provided and other operational details of multi-cluster are hidden. - -The concrete structure is as follows: -![](../../../../resources/images/multi_tenant_arch.png) - -## SQL interface - -- Login - - Default cluster login name: user_name@default_cluster or user_name - - Custom cluster login name: user_name@cluster_name - - `mysqlclient -h host -P port -u user_name@cluster_name -p password` - -- Add, delete, decommission and cancel BE - - `ALTER SYSTEM ADD BACKEND "host:port"` - `ALTER SYSTEM DROP BACKEND "host:port"` - `ALTER SYSTEM DECOMMISSION BACKEND "host:port"` - `CANCEL DECOMMISSION BACKEND "host:port"` - - It is strongly recommended to use DECOMMISSION instead of DROP to delete BACKEND. The DECOMMISSION operation will first need to copy data from the offline node to other instances in the cluster. After that, they will be offline. - -- Create a cluster and specify the password for the superuser account - - `CREATE CLUSTER cluster_name PROPERTIES ("instance_num" = "10") identified by "password"` - -- Enter a cluster - - `ENTER cluster name` - -- Cluster Expansion and Shrinkage - - `ALTER CLUSTER cluster_name PROPERTIES ("instance_num" = "10")` - - When the number of instances specified is more than the number of existing be in cluster, it is expanded and if less than it is condensed. - -- Link, migrate DB - - `LINK DATABASE src_cluster_name.db_name dest_cluster_name.db_name` - - Soft-chain dB of one cluster to dB of another cluster can be used by users who need temporary access to dB of other clusters but do not need actual data migration. - - `MIGRATE DATABASE src_cluster_name.db_name dest_cluster_name.db_name` - - If you need to migrate dB across clusters, after linking, migrate the actual migration of data. - - Migration does not affect the query, import and other operations of the current two dbs. This is an asynchronous operation. You can see the progress of migration through `SHOW MIGRATIONS`. - -- Delete clusters - - `DROP CLUSTER cluster_name` - - Deleting a cluster requires that all databases in the cluster be deleted manually first. - -- Others - - `SHOW CLUSTERS` - - Show clusters that have been created in the system. Only root users have this permission. - - `SHOW BACKENDS` - - View the BE instance in the cluster. - - `SHOW MIGRATIONS` - - Show current DB migration tasks. After the migration of DB is completed, you can view the progress of the migration through this command. - -## Detailed design - -1. Namespace isolation - - In order to introduce multi-tenant, the namespaces between clusters in the system need to be isolated. - - Doris's existing metadata is image + Journal (metadata is designed in related documents). Doris records operations involving metadata as a journal, and then regularly writes images in the form of **Fig. 1** and reads them in the order in which they are written when loaded. But this brings a problem that the format that has been written is not easy to modify. For example, the metadata format for recording data distribution is: database + table + tablet + replica nesting. If we want to isolate the namespace between clusters in the past way, we need to add a layer of cluster on the database and the level of internal metadata. Change to cluster + database + table + tablet + replica, as shown in **Figure 2**. But the problems brought about by adding one layer are as follows: - - - The change of metadata brought by adding one layer is incompatible. It needs to be written in cluster+db+table+tablet+replica level in the way of Figure 2. This changes the way of metadata organization in the past. The upgrading of the old version will be more troublesome. The ideal way is to write cluster in the order of Figure 3 in the form of existing metadata. Metadata. - - - All the DB and user used in the code need to add a layer of cluster. There are many workload changes and deep levels. Most of the code acquires db. The existing functions almost need to be changed, and a layer of cluster locks need to be nested on the basis of DB locks. - - ![](../../../../resources/images/palo_meta.png) - - To sum up, we adopt a prefix to DB and user names to isolate the internal problems caused by the conflict of DB and user names between clusters. - - As follows, all SQL input involves db name and user name, and all SQL input needs to spell the full name of DB and user according to their cluster. - - ![](../../../../resources/images/cluster_namaspace.png) - - In this way, the above two problems no longer exist. Metadata is also organized in a relatively simple way. That is to say, use ** Figure 3 ** to record db, user and nodes belonging to their own cluster. - -2. BE 节点管理 - - Each cluster has its own set of instances, which can be viewed through `SHOW BACKENDS`. In order to distinguish which cluster the instance belongs to and how it is used, BE introduces several states: - - - Free: When a BE node is added to the system, be is idle when it does not belong to any cluster. - - Use: When creating a cluster or expanding capacity is selected into a cluster, it is in use. - - Cluster decommission: If a shrinkage is performed, the be that is executing the shrinkage is in this state. After that, the be state becomes free. - - System decommission: be is offline. When the offline is completed, the be will be permanently deleted. - - Only root users can check whether all be in the cluster is used through the cluster item in `SHOW PROC "/backends"`. To be free is to be idle, otherwise to be in use. `SHOW BACKENDS `can only see the nodes in the cluster. The following is a schematic diagram of the state changes of be nodes. - - ![](../../../../resources/images/backend_state.png) - -3. Creating Clusters - - Only root users can create a cluster and specify any number of BE instances. - - Supports selecting multiple instances on the same machine. The general principle of selecting instance is to select be on different machines as much as possible and to make the number of be used on all machines as uniform as possible. - - For use, each user and DB belongs to a cluster (except root). To create user and db, you first need to enter a cluster. When a cluster is created, the system defaults to the manager of the cluster, the superuser account. Supuser has the right to create db, user, and view the number of be nodes in the cluster to which it belongs. All non-root user logins must specify a cluster, namely `user_name@cluster_name`. - - Only root users can view all clusters in the system through `SHOW CLUSTER', and can enter different clusters through @ different cluster names. User clusters are invisible except root. - - In order to be compatible with the old version of Doris, a cluster named default_cluster was built in, which could not be used when creating the cluster. - - ![](../../../../resources/images/user_authority.png) - -4. Cluster Expansion - - The process of cluster expansion is the same as that of cluster creation. BE instance on hosts that are not outside the cluster is preferred. The selected principles are the same as creating clusters. - -5. 集群缩容、CLUSTER DECOMMISSION - - Users can scale clusters by setting instance num of clusters. - - Cluster shrinkage takes precedence over downlining instances on hosts with the largest number of BE instances. - - Users can also directly use `ALTER CLUSTER DECOMMISSION BACKEND` to specify BE for cluster scaling. - -![](../../../../resources/images/replica_recover.png) - -6. Create table - - To ensure high availability, each fragmented copy must be on a different machine. So when building a table, the strategy of choosing the be where the replica is located is to randomly select a be on each host. Then, the number of be copies needed is randomly selected from these be. On the whole, it can distribute patches evenly on each machine. - - Therefore, adding a fragment that needs to create a 3-copy fragment, even if the cluster contains three or more instances, but only two or less hosts, still cannot create the fragment. - -7. Load Balancing - - The granularity of load balancing is cluster level, and there is no load balancing between clusters. However, the computing load is carried out at the host level, and there may be BE instances of different clusters on a host. In the cluster, the load is calculated by the number of fragments on each host and the utilization of storage, and then the fragments on the machine with high load are copied to the machine with low load (see the load balancing documents for details). - -8. LINK DATABASE (Soft Chain) - - Multiple clusters can access each other's data through a soft chain. The link level is dB for different clusters. - - DB in other clusters is accessed by adding DB information of other clusters that need to be accessed in one cluster. - - When querying the linked db, the computing and storage resources used are those of the cluster where the source DB is located. - - DB that is soft-chained cannot be deleted in the source cluster. Only when the linked DB is deleted can the source DB be deleted. Deleting link DB will not delete source db. - -9. MIGRATE DATABASE - - DB can be physically migrated between clusters. - - To migrate db, you must first link db. After migration, the data will migrate to the cluster where the linked DB is located, and after migration, the source DB will be deleted and the link will be disconnected. - - Data migration reuses the process of replicating data in load balancing and replica recovery (see load balancing related documents for details). Specifically, after executing the `MIRAGTE` command, Doris will modify the cluster of all copies of the source DB to the destination cluster in the metadata. - - Doris regularly checks whether machines in the cluster are balanced, replicas are complete, and redundant replicas are available. The migration of DB borrows this process, checking whether the be where the replica is located belongs to the cluster while checking the replica is complete, and if not, it is recorded in the replica to be restored. And when the duplicate is redundant to be deleted, it will first delete the duplicate outside the cluster, and then choose according to the existing strategy: the duplicate of the downtime be -> the duplicate of clone -> the duplicate of the backward version - > the duplicate on the host with high load, until the duplicate is not redundant. - -![](../../../../resources/images/cluster_link_and_migrate_db.png) - -10. BE process isolation - - In order to isolate the actual cpu, IO and memory between be processes, we need to rely on the deployment of be. When deploying, you need to configure the CGroup on the periphery and write all the processes of be to be deployed to the cgroup. If the physical isolation of IO between the data storage paths of each be configuration requires different disks, there is no much introduction here. diff --git a/docs/documentation/en/administrator-guide/operation/tablet-meta-tool_EN.md b/docs/documentation/en/administrator-guide/operation/tablet-meta-tool_EN.md deleted file mode 100644 index c71d7ae3d76337..00000000000000 --- a/docs/documentation/en/administrator-guide/operation/tablet-meta-tool_EN.md +++ /dev/null @@ -1,105 +0,0 @@ - - -# Tablet metadata management tool - -## Background - -In the latest version of the code, we introduced RocksDB in BE to store meta-information of tablet, in order to solve various functional and performance problems caused by storing meta-information through header file. Currently, each data directory (root path) has a corresponding RocksDB instance, in which all tablets on the corresponding root path are stored in the key-value manner. - -To facilitate the maintenance of these metadata, we provide an online HTTP interface and an offline meta tool to complete related management operations. - -The HTTP interface is only used to view tablet metadata online, and can be used when the BE process is running. - -However, meta tool is only used for off-line metadata management operations. BE must be stopped before it can be used. - -The meta tool tool is stored in the Lib / directory of BE. - -## Operation - -### View Tablet Meta - -Viewing Tablet Meta information can be divided into online and offline methods - -#### On-line - -Access BE's HTTP interface to obtain the corresponding Tablet Meta information: - -api： - -`http://{host}:{port}/api/meta/header/{tablet_id}/{schema_hash}` - - -> Host: be Hostname -> -> port: BE's HTTP port -> -> tablet id: tablet id -> -> schema hash: tablet schema hash - -Give an example: - -`http://be_host:8040/api/meta/header/14156/2458238340` - -If the final query is successful, the Tablet Meta will be returned as json. - -#### Offline - -Get Tablet Meta on a disk based on the meta\ tool tool. - -Order: - -``` -./lib/meta_tool --root_path=/path/to/root_path --operation=get_meta --tablet_id=xxx --schema_hash=xxx -``` - -> root_path: The corresponding root_path path path configured in be.conf. - -The result is also a presentation of Tablet Meta in JSON format. - -### Load header - -The function of loading header is provided to realize manual migration of tablet. This function is based on Tablet Meta in JSON format, so if changes in the shard field and version information are involved, they can be changed directly in the JSON content of Tablet Meta. Then use the following commands to load. - -Order: - -``` -./lib/meta_tool --operation=load_meta --root_path=/path/to/root_path --json_header_path=path -``` - -### Delete header - -In order to realize the function of deleting a tablet from a disk of a be. - -Order: - -``` -./lib/meta_tool --operation=delete_meta --root_path=/path/to/root_path --tablet_id=xxx --schema_hash=xxx` -``` - -### TabletMeta in Pb format - -This command is to view the old file-based management PB format Tablet Meta, and to display Tablet Meta in JSON format. - -Order: - -``` -./lib/meta_tool --operation=show_meta --root_path=/path/to/root_path --pb_header_path=path -``` diff --git a/docs/documentation/en/administrator-guide/operation/tablet-repair-and-balance_EN.md b/docs/documentation/en/administrator-guide/operation/tablet-repair-and-balance_EN.md deleted file mode 100644 index 3db7238243c574..00000000000000 --- a/docs/documentation/en/administrator-guide/operation/tablet-repair-and-balance_EN.md +++ /dev/null @@ -1,660 +0,0 @@ - - -# Data replica management - -Beginning with version 0.9.0, Doris introduced an optimized replica management strategy and supported a richer replica status viewing tool. This document focuses on Doris data replica balancing, repair scheduling strategies, and replica management operations and maintenance methods. Help users to more easily master and manage the replica status in the cluster. - -> Repairing and balancing copies of tables with Collocation attributes can be referred to `docs/documentation/cn/administrator-guide/colocation-join.md'.` - -## Noun Interpretation - -1. Tablet: The logical fragmentation of a Doris table, where a table has multiple fragmentations. -2. Replica: A sliced copy, defaulting to three copies of a slice. -3. Healthy Replica: A healthy copy that survives at Backend and has a complete version. -4. Tablet Checker (TC): A resident background thread that scans all Tablets regularly, checks the status of these Tablets, and decides whether to send them to Tablet Scheduler based on the results. -5. Tablet Scheduler (TS): A resident background thread that handles Tablets sent by Tablet Checker that need to be repaired. At the same time, cluster replica balancing will be carried out. -6. Tablet SchedCtx (TSC): is a tablet encapsulation. When TC chooses a tablet, it encapsulates it as a TSC and sends it to TS. -7. Storage Medium: Storage medium. Doris supports specifying different storage media for partition granularity, including SSD and HDD. The replica scheduling strategy is also scheduled for different storage media. - -``` - - +--------+ +-----------+ - | Meta | | Backends | - +---^----+ +------^----+ - | | | 3. Send clone tasks - 1. Check tablets | | | - +--------v------+ +-----------------+ - | TabletChecker +--------> TabletScheduler | - +---------------+ +-----------------+ - 2. Waiting to be scheduled - - -``` -The figure above is a simplified workflow. - - -## Duplicate status - -Multiple copies of a Tablet may cause state inconsistencies due to certain circumstances. Doris will attempt to automatically fix the inconsistent copies of these states so that the cluster can recover from the wrong state as soon as possible. - -**The health status of a Replica is as follows:** - -1. BAD - - That is, the copy is damaged. Includes, but is not limited to, the irrecoverable damaged status of copies caused by disk failures, BUGs, etc. - -2. VERSION\_MISSING - - Version missing. Each batch of imports in Doris corresponds to a data version. A copy of the data consists of several consecutive versions. However, due to import errors, delays and other reasons, the data version of some copies may be incomplete. - -3. HEALTHY - - Health copy. That is, a copy of the normal data, and the BE node where the copy is located is in a normal state (heartbeat is normal and not in the offline process). - -**The health status of a Tablet is determined by the status of all its copies. There are the following categories:** - -1. REPLICA\_MISSING - - The copy is missing. That is, the number of surviving copies is less than the expected number of copies. - -2. VERSION\_INCOMPLETE - - The number of surviving copies is greater than or equal to the number of expected copies, but the number of healthy copies is less than the number of expected copies. - -3. REPLICA\_RELOCATING - - Have a full number of live copies of the replication num version, but the BE nodes where some copies are located are in unavailable state (such as decommission) - -4. REPLICA\_MISSING\_IN\_CLUSTER - - When using multi-cluster, the number of healthy replicas is greater than or equal to the expected number of replicas, but the number of replicas in the corresponding cluster is less than the expected number of replicas. - -5. REDUNDANT - - Duplicate redundancy. Healthy replicas are in the corresponding cluster, but the number of replicas is larger than the expected number. Or there's a spare copy of unavailable. - -6. FORCE\_REDUNDANT - - This is a special state. It only occurs when the number of expected replicas is greater than or equal to the number of available nodes, and when the Tablet is in the state of replica missing. In this case, you need to delete a copy first to ensure that there are available nodes for creating a new copy. - -7. COLOCATE\_MISMATCH - - Fragmentation status of tables for Collocation attributes. Represents that the distribution of fragmented copies is inconsistent with the specified distribution of Colocation Group. - -8. COLOCATE\_REDUNDANT - - Fragmentation status of tables for Collocation attributes. Represents the fragmented copy redundancy of the Colocation table. - -8. HEALTHY - - Healthy fragmentation, that is, conditions [1-5] are not satisfied. - -## Replica Repair - -As a resident background process, Tablet Checker regularly checks the status of all fragments. For unhealthy fragmentation, it will be sent to Tablet Scheduler for scheduling and repair. The actual operation of repair is accomplished by clone task on BE. FE is only responsible for generating these clone tasks. - -> Note 1: The main idea of replica repair is to make the number of fragmented replicas reach the desired value by creating or completing them first. Then delete the redundant copy. -> -> Note 2: A clone task is to complete the process of copying specified data from a specified remote end to a specified destination. - -For different states, we adopt different repair methods: - -1. REPLICA\_MISSING/REPLICA\_RELOCATING - - Select a low-load, available BE node as the destination. Choose a healthy copy as the source. Clone tasks copy a complete copy from the source to the destination. For replica completion, we will directly select an available BE node, regardless of the storage medium. - -2. VERSION\_INCOMPLETE - - Select a relatively complete copy as the destination. Choose a healthy copy as the source. The clone task attempts to copy the missing version from the source to the destination. - -3. REPLICA\_MISSING\_IN\_CLUSTER - - This state processing method is the same as REPLICAMISSING. - -4. REDUNDANT - - Usually, after repair, there will be redundant copies in fragmentation. We select a redundant copy to delete it. The selection of redundant copies follows the following priorities: - 1. The BE where the copy is located has been offline. - 2. The copy is damaged - 3. The copy is lost in BE or offline - 4. The replica is in the CLONE state (which is an intermediate state during clone task execution) - 5. The copy has version missing - 6. The cluster where the copy is located is incorrect - 7. The BE node where the replica is located has a high load - -5. FORCE\_REDUNDANT - - Unlike REDUNDANT, because at this point Tablet has a copy missing, because there are no additional available nodes for creating new copies. So at this point, a copy must be deleted to free up a available node for creating a new copy. - The order of deleting copies is the same as REDUNDANT. - -6. COLOCATE\_MISMATCH - - Select one of the replica distribution BE nodes specified in Colocation Group as the destination node for replica completion. - -7. COLOCATE\_REDUNDANT - - Delete a copy on a BE node that is distributed by a copy specified in a non-Colocation Group. - - Doris does not deploy a copy of the same Tablet on a different BE of the same host when selecting a replica node. It ensures that even if all BEs on the same host are deactivated, all copies will not be lost. - -### Scheduling priority - -Waiting for the scheduled fragments in Tablet Scheduler gives different priorities depending on the status. High priority fragments will be scheduled first. There are currently several priorities. - -1. VERY\_HIGH - - * REDUNDANT. For slices with duplicate redundancy, we give priority to them. Logically, duplicate redundancy is the least urgent, but because it is the fastest to handle and can quickly release resources (such as disk space, etc.), we give priority to it. - * FORCE\_REDUNDANT. Ditto. - -2. HIGH - - * REPLICA\_MISSING and most copies are missing (for example, 2 copies are missing in 3 copies) - * VERSION\_INCOMPLETE and most copies are missing - * COLOCATE\_MISMATCH We hope that the fragmentation related to the Collocation table can be repaired as soon as possible. - * COLOCATE\_REDUNDANT - -3. NORMAL - - * REPLICA\_MISSING, but most survive (for example, three copies lost one) - * VERSION\_INCOMPLETE, but most copies are complete - * REPLICA\_RELOCATING and relocate is required for most replicas (e.g. 3 replicas with 2 replicas) - -4. LOW - - * REPLICA\_MISSING\_IN\_CLUSTER - * REPLICA\_RELOCATING most copies stable - -### Manual priority - -The system will automatically determine the scheduling priority. Sometimes, however, users want the fragmentation of some tables or partitions to be repaired faster. So we provide a command that the user can specify that a slice of a table or partition is repaired first: - -`ADMIN REPAIR TABLE tbl [PARTITION (p1, p2, ...)];` - -This command tells TC to give VERY HIGH priority to the problematic tables or partitions that need to be repaired first when scanning Tablets. - -> Note: This command is only a hint, which does not guarantee that the repair will be successful, and the priority will change with the scheduling of TS. And when Master FE switches or restarts, this information will be lost. - -Priority can be cancelled by the following commands: - -`ADMIN CANCEL REPAIR TABLE tbl [PARTITION (p1, p2, ...)];` - -### Priority scheduling - -Priority ensures that severely damaged fragments can be repaired first, and improves system availability. But if the high priority repair task fails all the time, the low priority task will never be scheduled. Therefore, we will dynamically adjust the priority of tasks according to the running status of tasks, so as to ensure that all tasks have the opportunity to be scheduled. - -* If the scheduling fails for five consecutive times (e.g., no resources can be obtained, no suitable source or destination can be found, etc.), the priority will be lowered. -* If not scheduled for 30 minutes, priority will be raised. -* The priority of the same tablet task is adjusted at least five minutes apart. - -At the same time, in order to ensure the weight of the initial priority, we stipulate that the initial priority is VERY HIGH, and the lowest is lowered to NORMAL. When the initial priority is LOW, it is raised to HIGH at most. The priority adjustment here also adjusts the priority set manually by the user. - -## Duplicate Equilibrium - -Doris automatically balances replicas within the cluster. The main idea of balancing is to create a replica of some fragments on low-load nodes, and then delete the replicas of these fragments on high-load nodes. At the same time, because of the existence of different storage media, there may or may not exist one or two storage media on different BE nodes in the same cluster. We require that fragments of storage medium A be stored in storage medium A as far as possible after equalization. So we divide the BE nodes of the cluster according to the storage medium. Then load balancing scheduling is carried out for different BE node sets of storage media. - -Similarly, replica balancing ensures that a copy of the same table will not be deployed on the BE of the same host. - -### BE Node Load - -We use Cluster LoadStatistics (CLS) to represent the load balancing of each backend in a cluster. Tablet Scheduler triggers cluster equilibrium based on this statistic. We currently calculate a load Score for each BE as the BE load score by using **disk usage** and **number of copies**. The higher the score, the heavier the load on the BE. - -Disk usage and number of copies have a weight factor, which is **capacityCoefficient** and **replicaNumCoefficient**, respectively. The sum of them is **constant to 1**. Among them, capacityCoefficient will dynamically adjust according to actual disk utilization. When the overall disk utilization of a BE is below 50%, the capacityCoefficient value is 0.5, and if the disk utilization is above 75% (configurable through the FE configuration item `capacity_used_percent_high_water`), the value is 1. If the utilization rate is between 50% and 75%, the weight coefficient increases smoothly. The formula is as follows: - -`capacityCoefficient = 2 * Disk Utilization - 0.5` - -The weight coefficient ensures that when disk utilization is too high, the backend load score will be higher to ensure that the BE load is reduced as soon as possible. - -Tablet Scheduler updates CLS every 1 minute. - -### Equilibrium strategy - -Tablet Scheduler uses Load Balancer to select a certain number of healthy fragments as candidate fragments for balance in each round of scheduling. In the next scheduling, balanced scheduling will be attempted based on these candidate fragments. - -## Resource control - -Both replica repair and balancing are accomplished by replica copies between BEs. If the same BE performs too many tasks at the same time, it will bring a lot of IO pressure. Therefore, Doris controls the number of tasks that can be performed on each node during scheduling. The smallest resource control unit is the disk (that is, a data path specified in be.conf). By default, we configure two slots per disk for replica repair. A clone task occupies one slot at the source and one slot at the destination. If the number of slots is zero, no more tasks will be assigned to this disk. The number of slots can be configured by FE's `schedule_slot_num_per_path` parameter. - -In addition, by default, we provide two separate slots per disk for balancing tasks. The purpose is to prevent high-load nodes from losing space by balancing because slots are occupied by repair tasks. - -## Duplicate Status View - -Duplicate status view mainly looks at the status of the duplicate, as well as the status of the duplicate repair and balancing tasks. Most of these states **exist only in** Master FE nodes. Therefore, the following commands need to be executed directly to Master FE. - -### Duplicate status - -1. Global state checking - - Through `SHOW PROC'/ statistic'; `commands can view the replica status of the entire cluster. - - ``` - +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ - | DbId | DbName | TableNum | PartitionNum | IndexNum | TabletNum | ReplicaNum | UnhealthyTabletNum | InconsistentTabletNum | - +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ - | 35153636 | default_cluster:DF_Newrisk | 3 | 3 | 3 | 96 | 288 | 0 | 0 | - | 48297972 | default_cluster:PaperData | 0 | 0 | 0 | 0 | 0 | 0 | 0 | - | 5909381 | default_cluster:UM_TEST | 7 | 7 | 10 | 320 | 960 | 1 | 0 | - | Total | 240 | 10 | 10 | 13 | 416 | 1248 | 1 | 0 | - +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ - ``` - - The `UnhealthyTabletNum` column shows how many Tablets are in an unhealthy state in the corresponding database. `The Inconsistent Tablet Num` column shows how many Tablets are in an inconsistent replica state in the corresponding database. The last `Total` line counts the entire cluster. Normally `Unhealth Tablet Num` and `Inconsistent Tablet Num` should be 0. If it's not zero, you can further see which Tablets are there. As shown in the figure above, one table in the UM_TEST database is not healthy, you can use the following command to see which one is. - - `SHOW PROC '/statistic/5909381';` - - Among them `5909381'is the corresponding DbId. - - ``` - +------------------+---------------------+ - | UnhealthyTablets | InconsistentTablets | - +------------------+---------------------+ - | [40467980] | [] | - +------------------+---------------------+ - ``` - - The figure above shows the specific unhealthy Tablet ID (40467980). Later we'll show you how to view the status of each copy of a specific Tablet. - -2. Table (partition) level status checking - - Users can view the status of a copy of a specified table or partition through the following commands and filter the status through a WHERE statement. If you look at table tbl1, the state on partitions P1 and P2 is a copy of NORMAL: - - `ADMIN SHOW REPLICA STATUS FROM tbl1 PARTITION (p1, p2) WHERE STATUS = "NORMAL";` - - ``` - +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ - | TabletId | ReplicaId | BackendId | Version | LastFailedVersion | LastSuccessVersion | CommittedVersion | SchemaHash | VersionNum | IsBad | State | Status | - +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ - | 29502429 | 29502432 | 10006 | 2 | -1 | 2 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502429 | 36885996 | 10002 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502429 | 48100551 | 10007 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502433 | 29502434 | 10001 | 2 | -1 | 2 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502433 | 44900737 | 10004 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | - | 29502433 | 48369135 | 10006 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | - +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ - ``` - - The status of all copies is shown here. Where `IsBad` is listed as `true`, the copy is damaged. The `Status` column displays other states. Specific status description, you can see help through `HELP ADMIN SHOW REPLICA STATUS`. - - ` The ADMIN SHOW REPLICA STATUS `command is mainly used to view the health status of copies. Users can also view additional information about copies of a specified table by using the following commands: - - `SHOW TABLET FROM tbl1;` - - ``` - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - | TabletId | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | DataSize | RowCount | State | LstConsistencyCheckTime | CheckVersion | CheckVersionHash | VersionCount | PathHash | - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - | 29502429 | 29502432 | 10006 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -5822326203532286804 | - | 29502429 | 36885996 | 10002 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -1441285706148429853 | - | 29502429 | 48100551 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -4784691547051455525 | - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - ``` - - The figure above shows some additional information, including copy size, number of rows, number of versions, where the data path is located. - - > Note: The contents of the `State'column shown here do not represent the health status of the replica, but the status of the replica under certain tasks, such as CLONE, SCHEMA CHANGE, ROLLUP, etc. - - In addition, users can check the distribution of replicas in a specified table or partition by following commands. - - `ADMIN SHOW REPLICA DISTRIBUTION FROM tbl1;` - - ``` - +-----------+------------+-------+---------+ - | BackendId | ReplicaNum | Graph | Percent | - +-----------+------------+-------+---------+ - | 10000 | 7 | | 7.29 % | - | 10001 | 9 | | 9.38 % | - | 10002 | 7 | | 7.29 % | - | 10003 | 7 | | 7.29 % | - | 10004 | 9 | | 9.38 % | - | 10005 | 11 | > | 11.46 % | - | 10006 | 18 | > | 18.75 % | - | 10007 | 15 | > | 15.62 % | - | 10008 | 13 | > | 13.54 % | - +-----------+------------+-------+---------+ - ``` - - Here we show the number and percentage of replicas of table tbl1 on each BE node, as well as a simple graphical display. - -4. Tablet level status checking - - When we want to locate a specific Tablet, we can use the following command to view the status of a specific Tablet. For example, check the tablet with ID 2950253: - - `SHOW TABLET 29502553;` - - ``` - +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ - | DbName | TableName | PartitionName | IndexName | DbId | TableId | PartitionId | IndexId | IsSync | DetailCmd | - +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ - | default_cluster:test | test | test | test | 29502391 | 29502428 | 29502427 | 29502428 | true | SHOW PROC '/dbs/29502391/29502428/partitions/29502427/29502428/29502553'; | - +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ - ``` - - The figure above shows the database, tables, partitions, roll-up tables and other information corresponding to this tablet. The user can copy the command in the `DetailCmd` command to continue executing: - - `Show Proc'/DBS/29502391/29502428/Partitions/29502427/29502428/29502553;` - - ``` - +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ - | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | SchemaHash | DataSize | RowCount | State | IsBad | VersionCount | PathHash | - +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ - | 43734060 | 10004 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | -8566523878520798656 | - | 29502555 | 10002 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | 1885826196444191611 | - | 39279319 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | 1656508631294397870 | - +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ - ``` - - The figure above shows all replicas of the corresponding Tablet. The content shown here is the same as `SHOW TABLET FROM tbl1;`. But here you can clearly see the status of all copies of a specific Tablet. - -### Duplicate Scheduling Task - -1. View tasks waiting to be scheduled - - `SHOW PROC '/cluster_balance/pending_tablets';` - - ``` - +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ - | TabletId | Type | Status | State | OrigPrio | DynmPrio | SrcBe | SrcPath | DestBe | DestPath | Timeout | Create | LstSched | LstVisit | Finished | Rate | FailedSched | FailedRunning | LstAdjPrio | VisibleVer | VisibleVerHash | CmtVer | CmtVerHash | ErrMsg | - +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ - | 4203036 | REPAIR | REPLICA_MISSING | PENDING | HIGH | LOW | -1 | -1 | -1 | -1 | 0 | 2019-02-21 15:00:20 | 2019-02-24 11:18:41 | 2019-02-24 11:18:41 | N/A | N/A | 2 | 0 | 2019-02-21 15:00:43 | 1 | 0 | 2 | 0 | unable to find source replica | - +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ - ``` - - The specific meanings of each column are as follows: - - * TabletId: The ID of the Tablet waiting to be scheduled. A scheduling task is for only one Tablet - * Type: Task type, which can be REPAIR (repair) or BALANCE (balance) - * Status: The current status of the Tablet, such as REPLICAMISSING (copy missing) - * State: The status of the scheduling task may be PENDING/RUNNING/FINISHED/CANCELLED/TIMEOUT/UNEXPECTED - * OrigPrio: Initial Priority - * DynmPrio: Current dynamically adjusted priority - * SrcBe: ID of the BE node at the source end - * SrcPath: hash value of the path of the BE node at the source end - * DestBe: ID of destination BE node - * DestPath: hash value of the path of the destination BE node - * Timeout: When the task is scheduled successfully, the timeout time of the task is displayed here in units of seconds. - * Create: The time when the task was created - * LstSched: The last time a task was scheduled - * LstVisit: The last time a task was accessed. Here "accessed" refers to the processing time points associated with the task, including scheduling, task execution reporting, and so on. - * Finished: Task End Time - * Rate: Clone Task Data Copy Rate - * Failed Sched: Number of Task Scheduling Failures - * Failed Running: Number of task execution failures - * LstAdjPrio: Time of last priority adjustment - * CmtVer/CmtVerHash/VisibleVer/VisibleVerHash: version information for clone tasks - * ErrMsg: Error messages that occur when tasks are scheduled and run - -2. View running tasks - - `SHOW PROC '/cluster_balance/running_tablets';` - - The columns in the result have the same meaning as `pending_tablets`. - -3. View completed tasks - - `SHOW PROC '/cluster_balance/history_tablets';` - - By default, we reserve only the last 1,000 completed tasks. The columns in the result have the same meaning as `pending_tablets`. If `State` is listed as `FINISHED`, the task is normally completed. For others, you can see the specific reason based on the error information in the `ErrMsg` column. - -## Viewing Cluster Load and Scheduling Resources - -1. Cluster load - - You can view the current load of the cluster by following commands: - - `SHOW PROC '/cluster_balance/cluster_load_stat';` - - First of all, we can see the division of different storage media: - - ``` - +---------------+ - | StorageMedium | - +---------------+ - | HDD | - | SSD | - +---------------+ - ``` - - Click on a storage medium to see the equilibrium state of the BE node that contains the storage medium: - - `SHOW PROC '/cluster_balance/cluster_load_stat/HDD';` - - ``` - +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ - | BeId | Cluster | Available | UsedCapacity | Capacity | UsedPercent | ReplicaNum | CapCoeff | ReplCoeff | Score | Class | - +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ - | 10003 | default_cluster | true | 3477875259079 | 19377459077121 | 17.948 | 493477 | 0.5 | 0.5 | 0.9284678149967587 | MID | - | 10002 | default_cluster | true | 3607326225443 | 19377459077121 | 18.616 | 496928 | 0.5 | 0.5 | 0.948660871419998 | MID | - | 10005 | default_cluster | true | 3523518578241 | 19377459077121 | 18.184 | 545331 | 0.5 | 0.5 | 0.9843539990641831 | MID | - | 10001 | default_cluster | true | 3535547090016 | 19377459077121 | 18.246 | 558067 | 0.5 | 0.5 | 0.9981869446537612 | MID | - | 10006 | default_cluster | true | 3636050364835 | 19377459077121 | 18.764 | 547543 | 0.5 | 0.5 | 1.0011489897614072 | MID | - | 10004 | default_cluster | true | 3506558163744 | 15501967261697 | 22.620 | 468957 | 0.5 | 0.5 | 1.0228319835582569 | MID | - | 10007 | default_cluster | true | 4036460478905 | 19377459077121 | 20.831 | 551645 | 0.5 | 0.5 | 1.057279369420761 | MID | - | 10000 | default_cluster | true | 4369719923760 | 19377459077121 | 22.551 | 547175 | 0.5 | 0.5 | 1.0964036415787461 | MID | - +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ - ``` - - Some of these columns have the following meanings: - - * Available: True means that BE heartbeat is normal and not offline. - * UsedCapacity: Bytes, the size of disk space used on BE - * Capacity: Bytes, the total disk space size on BE - * UsedPercent: Percentage, disk space utilization on BE - * ReplicaNum: Number of copies on BE - * CapCoeff/ReplCoeff: Weight Coefficient of Disk Space and Copy Number - * Score: Load score. The higher the score, the heavier the load. - * Class: Classified by load, LOW/MID/HIGH. Balanced scheduling moves copies from high-load nodes to low-load nodes - - Users can further view the utilization of each path on a BE, such as the BE with ID 10001: - - `SHOW PROC '/cluster_balance/cluster_load_stat/HDD/10001';` - - ``` - +------------------+------------------+---------------+---------------+---------+--------+----------------------+ - | RootPath | DataUsedCapacity | AvailCapacity | TotalCapacity | UsedPct | State | PathHash | - +------------------+------------------+---------------+---------------+---------+--------+----------------------+ - | /home/disk4/palo | 498.757 GB | 3.033 TB | 3.525 TB | 13.94 % | ONLINE | 4883406271918338267 | - | /home/disk3/palo | 704.200 GB | 2.832 TB | 3.525 TB | 19.65 % | ONLINE | -5467083960906519443 | - | /home/disk1/palo | 512.833 GB | 3.007 TB | 3.525 TB | 14.69 % | ONLINE | -7733211489989964053 | - | /home/disk2/palo | 881.955 GB | 2.656 TB | 3.525 TB | 24.65 % | ONLINE | 4870995507205544622 | - | /home/disk5/palo | 694.992 GB | 2.842 TB | 3.525 TB | 19.36 % | ONLINE | 1916696897889786739 | - +------------------+------------------+---------------+---------------+---------+--------+----------------------+ - ``` - - The disk usage of each data path on the specified BE is shown here. - -2. Scheduling resources - - Users can view the current slot usage of each node through the following commands: - - `SHOW PROC '/cluster_balance/working_slots';` - - ``` - +----------+----------------------+------------+------------+-------------+----------------------+ - | BeId | PathHash | AvailSlots | TotalSlots | BalanceSlot | AvgRate | - +----------+----------------------+------------+------------+-------------+----------------------+ - | 10000 | 8110346074333016794 | 2 | 2 | 2 | 2.459007474009069E7 | - | 10000 | -5617618290584731137 | 2 | 2 | 2 | 2.4730105014001578E7 | - | 10001 | 4883406271918338267 | 2 | 2 | 2 | 1.6711402709780257E7 | - | 10001 | -5467083960906519443 | 2 | 2 | 2 | 2.7540126380326536E7 | - | 10002 | 9137404661108133814 | 2 | 2 | 2 | 2.417217089806745E7 | - | 10002 | 1885826196444191611 | 2 | 2 | 2 | 1.6327378456676323E7 | - +----------+----------------------+------------+------------+-------------+----------------------+ - ``` - - In this paper, data path is used as granularity to show the current use of slot. Among them, `AvgRate'is the copy rate of clone task in bytes/seconds on the path of historical statistics. - -3. Priority repair view - - The following command allows you to view the priority repaired tables or partitions set by the `ADMIN REPAIR TABLE'command. - - `SHOW PROC '/cluster_balance/priority_repair'`; - - Among them, `Remaining TimeMs'indicates that these priority fixes will be automatically removed from the priority fix queue after this time. In order to prevent resources from being occupied due to the failure of priority repair. - -### Scheduler Statistical Status View - -We have collected some statistics of Tablet Checker and Tablet Scheduler during their operation, which can be viewed through the following commands: - -`SHOW PROC '/cluster_balance/sched_stat'`; - -``` -+---------------------------------------------------+-------------+ -| Item | Value | -+---------------------------------------------------+-------------+ -| num of tablet check round | 12041 | -| cost of tablet check(ms) | 7162342 | -| num of tablet checked in tablet checker | 18793506362 | -| num of unhealthy tablet checked in tablet checker | 7043900 | -| num of tablet being added to tablet scheduler | 1153 | -| num of tablet schedule round | 49538 | -| cost of tablet schedule(ms) | 49822 | -| num of tablet being scheduled | 4356200 | -| num of tablet being scheduled succeeded | 320 | -| num of tablet being scheduled failed | 4355594 | -| num of tablet being scheduled discard | 286 | -| num of tablet priority upgraded | 0 | -| num of tablet priority downgraded | 1096 | -| num of clone task | 230 | -| num of clone task succeeded | 228 | -| num of clone task failed | 2 | -| num of clone task timeout | 2 | -| num of replica missing error | 4354857 | -| num of replica version missing error | 967 | -| num of replica relocating | 0 | -| num of replica redundant error | 90 | -| num of replica missing in cluster error | 0 | -| num of balance scheduled | 0 | -+---------------------------------------------------+-------------+ -``` - -The meanings of each line are as follows: - -* num of tablet check round：Tablet Checker 检查次数 -* cost of tablet check(ms)：Tablet Checker 检查总耗时 -* num of tablet checked in tablet checker：Tablet Checker 检查过的 tablet 数量 -* num of unhealthy tablet checked in tablet checker：Tablet Checker 检查过的不健康的 tablet 数量 -* num of tablet being added to tablet scheduler：被提交到 Tablet Scheduler 中的 tablet 数量 -* num of tablet schedule round：Tablet Scheduler 运行次数 -* cost of tablet schedule(ms)：Tablet Scheduler 运行总耗时 -* num of tablet being scheduled：被调度的 Tablet 总数量 -* num of tablet being scheduled succeeded：被成功调度的 Tablet 总数量 -* num of tablet being scheduled failed：调度失败的 Tablet 总数量 -* num of tablet being scheduled discard：调度失败且被抛弃的 Tablet 总数量 -* num of tablet priority upgraded：优先级上调次数 -* num of tablet priority downgraded：优先级下调次数 -* num of clone task: number of clone tasks generated -* num of clone task succeeded：clone 任务成功的数量 -* num of clone task failed：clone 任务失败的数量 -* num of clone task timeout：clone 任务超时的数量 -* num of replica missing error: the number of tablets whose status is checked is the missing copy -* num of replica version missing error：检查的状态为版本缺失的 tablet 的数量（该统计值包括了 num of replica relocating 和 num of replica missing in cluster error） -*num of replica relocation *29366;* 24577;*replica relocation tablet * -* num of replica redundant error: Number of tablets whose checked status is replica redundant -* num of replica missing in cluster error：检查的状态为不在对应 cluster 的 tablet 的数量 -* num of balance scheduled：均衡调度的次数 - -> Note: The above states are only historical accumulative values. We also print these statistics regularly in the FE logs, where the values in parentheses represent the number of changes in each statistical value since the last printing dependence of the statistical information. - -## Relevant configuration instructions - -### Adjustable parameters - -The following adjustable parameters are all configurable parameters in fe.conf. - -* use\_new\_tablet\_scheduler - - * Description: Whether to enable the new replica scheduling mode. The new replica scheduling method is the replica scheduling method introduced in this document. If turned on, `disable_colocate_join` must be `true`. Because the new scheduling strategy does not support data fragmentation scheduling of co-locotion tables for the time being. - * Default value:true - * Importance: High - -* tablet\_repair\_delay\_factor\_second - - * Note: For different scheduling priorities, we will delay different time to start repairing. In order to prevent a large number of unnecessary replica repair tasks from occurring in the process of routine restart and upgrade. This parameter is a reference coefficient. For HIGH priority, the delay is the reference coefficient * 1; for NORMAL priority, the delay is the reference coefficient * 2; for LOW priority, the delay is the reference coefficient * 3. That is, the lower the priority, the longer the delay waiting time. If the user wants to repair the copy as soon as possible, this parameter can be reduced appropriately. - * Default value: 60 seconds - * Importance: High - -* schedule\_slot\_num\_per\_path - - * Note: The default number of slots allocated to each disk for replica repair. This number represents the number of replica repair tasks that a disk can run simultaneously. If you want to repair the copy faster, you can adjust this parameter appropriately. The higher the single value, the greater the impact on IO. - * Default value: 2 - * Importance: High - -* balance\_load\_score\_threshold - - * Description: Threshold of Cluster Equilibrium. The default is 0.1, or 10%. When the load core of a BE node is not higher than or less than 10% of the average load core, we think that the node is balanced. If you want to make the cluster load more even, you can adjust this parameter appropriately. - * Default value: 0.1 - * Importance: - -* storage\_high\_watermark\_usage\_percent 和 storage\_min\_left\_capacity\_bytes - - * Description: These two parameters represent the upper limit of the maximum space utilization of a disk and the lower limit of the minimum space remaining, respectively. When the space utilization of a disk is greater than the upper limit or the remaining space is less than the lower limit, the disk will no longer be used as the destination address for balanced scheduling. - * Default values: 0.85 and 1048576000 (1GB) - * Importance: - -* disable\_balance - - * Description: Control whether to turn off the balancing function. When replicas are in equilibrium, some functions, such as ALTER TABLE, will be banned. Equilibrium can last for a long time. Therefore, if the user wants to do the prohibited operation as soon as possible. This parameter can be set to true to turn off balanced scheduling. - * Default value:true - * Importance: - -### Unadjustable parameters - -The following parameters do not support modification for the time being, just for illustration. - -* Tablet Checker scheduling interval - - Tablet Checker schedules checks every 20 seconds. - -* Tablet Scheduler scheduling interval - - Tablet Scheduler schedules every five seconds - -* Number of Tablet Scheduler Schedules per Batch - - Tablet Scheduler schedules up to 50 tablets at a time. - -* Tablet Scheduler Maximum Waiting Schedule and Number of Tasks in Operation - - The maximum number of waiting tasks and running tasks is 2000. When over 2000, Tablet Checker will no longer generate new scheduling tasks to Tablet Scheduler. - -* Tablet Scheduler Maximum Balanced Task Number - - The maximum number of balanced tasks is 500. When more than 500, there will be no new balancing tasks. - -* Number of slots per disk for balancing tasks - - The number of slots per disk for balancing tasks is 2. This slot is independent of the slot used for replica repair. - -* Update interval of cluster equilibrium - - Tablet Scheduler recalculates the load score of the cluster every 20 seconds. - -* Minimum and Maximum Timeout for Clone Tasks - - A clone task timeout time range is 3 minutes to 2 hours. The specific timeout is calculated by the size of the tablet. The formula is (tablet size)/ (5MB/s). When a clone task fails three times, the task terminates. - -* Dynamic Priority Adjustment Strategy - - The minimum priority adjustment interval is 5 minutes. When a tablet schedule fails five times, priority is lowered. When a tablet is not scheduled for 30 minutes, priority is raised. - -## Relevant issues - -* In some cases, the default replica repair and balancing strategy may cause the network to be full (mostly in the case of gigabit network cards and a large number of disks per BE). At this point, some parameters need to be adjusted to reduce the number of simultaneous balancing and repair tasks. - -* Current balancing strategies for copies of Colocate Table do not guarantee that copies of the same Tablet will not be distributed on the BE of the same host. However, the repair strategy of the copy of Colocate Table detects this distribution error and corrects it. However, it may occur that after correction, the balancing strategy regards the replicas as unbalanced and rebalances them. As a result, the Colocate Group can not achieve stability because of the continuous alternation between the two states. In view of this situation, we suggest that when using Colocate attribute, we try to ensure that the cluster is isomorphic, so as to reduce the probability that replicas are distributed on the same host. diff --git a/docs/documentation/en/administrator-guide/privilege_EN.md b/docs/documentation/en/administrator-guide/privilege_EN.md deleted file mode 100644 index 157aec67a28720..00000000000000 --- a/docs/documentation/en/administrator-guide/privilege_EN.md +++ /dev/null @@ -1,217 +0,0 @@ - - -# Authority Management - -Doris's new privilege management system refers to Mysql's privilege management mechanism, achieves table-level fine-grained privilege control, role-based privilege access control, and supports whitelist mechanism. - -## Noun Interpretation - -1. user_identity - - In a permission system, a user is identified as a User Identity. User ID consists of two parts: username and userhost. Username is a user name, which is composed of English upper and lower case. Userhost represents the IP from which the user link comes. User_identity is presented as username@'userhost', representing the username from userhost. - - Another expression of user_identity is username@['domain'], where domain is the domain name, which can be resolved into a set of IPS by DNS BNS (Baidu Name Service). The final expression is a set of username@'userhost', so we use username@'userhost'to represent it. - -2. Privilege - - The objects of permissions are nodes, databases or tables. Different permissions represent different operating permissions. - -3. Role - - Doris can create custom named roles. Roles can be seen as a set of permissions. When a newly created user can be assigned a role, the role's permissions are automatically granted. Subsequent changes in the role's permissions will also be reflected in all user permissions that belong to the role. - -4. user_property - - User attributes are directly attached to a user, not to a user identity. That is, both cmy@'192.%'and cmy@['domain'] have the same set of user attributes, which belong to user cmy, not cmy@'192.%' or cmy@['domain']. - - User attributes include, but are not limited to, the maximum number of user connections, import cluster configuration, and so on. - -## Supported operations - -1. Create users：CREATE USER -2. Delete users: DROP USER -3. Authorization: GRANT -4. Withdrawal: REVOKE -5. Create role：CREATE ROLE -6. Delete Roles: DROP ROLE -7. View current user privileges: SHOW GRANTS -8. View all user privilegesSHOW ALL GRANTS; -9. View the created roles: SHOW ROLES -10. View user attributes: SHOW PROPERTY - -For detailed help with the above commands, you can use help + command to get help after connecting Doris through the MySQL client. For example `HELP CREATE USER`. - -## Permission type - -Doris currently supports the following permissions - -1. Node_priv - - Nodes change permissions. Including FE, BE, BROKER node addition, deletion, offline operations. Currently, this permission can only be granted to Root users. - -2. Grant_priv - - Permissions change permissions. Allow the execution of operations including authorization, revocation, add/delete/change user/role, etc. - -3. Select_priv - - Read-only access to databases and tables. - -4. Load_priv - - Write permissions to databases and tables. Including Load, Insert, Delete and so on. - -5. Alter_priv - - Change permissions on databases and tables. It includes renaming libraries/tables, adding/deleting/changing columns, and adding/deleting partitions. - -6. Create_priv - - The right to create databases, tables, and views. - -7. Drop_priv - - Delete permissions for databases, tables, and views. - -## Permission hierarchy - -At the same time, according to the scope of application of permissions, we divide them into three levels: - -1. GLOBAL LEVEL: Global permissions. That is, permissions on `*.*` granted by GRANT statements. The granted permissions apply to any table in any database. -2. DATABASE LEVEL: Database-level permissions. That is, permissions on `db.*` granted by GRANT statements. The granted permissions apply to any table in the specified database. -3. TABLE LEVEL: Table-level permissions. That is, permissions on `db.tbl` granted by GRANT statements. The permissions granted apply to the specified tables in the specified database. - - -## ADMIN /GRANT - -ADMIN\_PRIV and GRANT\_PRIV have the authority of **"grant authority"** at the same time, which is more special. The operations related to these two privileges are described here one by one. - -1. CREATE USER - - * Users with ADMIN or GRANT privileges at any level can create new users. - -2. DROP USER - - * Only ADMIN privileges can delete users. - -3. CREATE/DROP ROLE - - * Only ADMIN privileges can create roles. - -4. GRANT /REVOKE - - * Users with ADMIN or GLOBAL GRANT privileges can grant or revoke the privileges of any user. - * Users with GRANT privileges at the DATABASE level can grant or revoke the privileges of any user on the specified database. - * Users with GRANT privileges at TABLE level can grant or revoke the privileges of any user on the specified tables in the specified database. - -5. SET PASSWORD - - * Users with ADMIN or GLOBAL GRANT privileges can set any user's password. - * Ordinary users can set their corresponding User Identity password. The corresponding User Identity can be viewed by `SELECT CURRENT_USER();`command. - * Users with GRANT privileges at non-GLOBAL level can not set the password of existing users, but can only specify the password when creating users. - - -## Some explanations - -1. When Doris initializes, the following users and roles are automatically created: - - 1. Operator role: This role has Node\_priv and Admin\_priv, i.e. all permissions for Doris. In a subsequent upgrade version, we may restrict the role's permissions to Node\_priv, which is to grant only node change permissions. To meet some cloud deployment requirements. - - 2. admin role: This role has Admin\_priv, which is all permissions except for node changes. - - 3. root@'%': root user, which allows login from any node, with the role of operator. - - 4. admin@'%': admin user, allowing login from any node, role admin. - -2. It is not supported to delete or change the permissions of default created roles or users. - -3. The user of the operator role has one and only one user. Users of admin roles can create multiple. - -4. Operational instructions for possible conflicts - - 1. Conflict between domain name and ip: - - Assume that the following users are created: - - CREATE USER cmy@['domain']; - - And authorize: - - GRANT SELECT_PRIV ON \*.\* TO cmy@['domain'] - - The domain is resolved into two ips: IP1 and IP2 - - Let's assume that we have a separate authorization for cmy@'ip1': - - GRANT ALTER_PRIV ON \*.\* TO cmy@'ip1'; - - The permissions of CMY @'ip1'will be changed to SELECT\_PRIV, ALTER\_PRIV. And when we change the permissions of cmy@['domain'] again, cmy@'ip1' will not follow. - - 2. duplicate IP conflicts: - - Assume that the following users are created: - - CREATE USER cmy@'%' IDENTIFIED BY "12345"; - - CREATE USER cmy@'192.%' IDENTIFIED BY "abcde"; - - In priority,'192.%'takes precedence over'%', so when user CMY tries to login Doris with password '12345' from 192.168.1.1, the machine will be rejected. - -5. Forget passwords - - If you forget your password and cannot log in to Doris, you can log in to Doris without a password using the following command on the machine where the Doris FE node is located: - - `mysql-client -h 127.0.0.1 -P query_port -uroot` - - After login, the password can be reset through the SET PASSWORD command. - -6. No user can reset the password of the root user except the root user himself. - -7. ADMIN\_PRIV permissions can only be granted or revoked at the GLOBAL level. - -8. Having GRANT\_PRIV at GLOBAL level is actually equivalent to having ADMIN\_PRIV, because GRANT\_PRIV at this level has the right to grant arbitrary permissions, please use it carefully. - -9. `current_user()` and `user()` - - Users can view `current_user` and `user` respectively by `SELECT current_user();` and `SELECT user();`. Where `current_user` indicates which identity the current user is passing through the authentication system, and `user` is the user's current actual `user_identity`. - -  For example, suppose the user `user1@'192.%'` is created, and then a user user1 from 192.168.10.1 is logged into the system. At this time, `current_user` is `user1@'192.%'`, and `user` is `user1@'192.168.10.1'`. - - All privileges are given to a `current_user`, and the real user has all the privileges of the corresponding `current_user`. - -## Best Practices - -Here are some usage scenarios of Doris privilege system. - -1. Scene 1 - - The users of Doris cluster are divided into Admin, RD and Client. Administrators have all the rights of the whole cluster, mainly responsible for cluster building, node management and so on. The development engineer is responsible for business modeling, including database building, data import and modification. Users access different databases and tables to get data. - - In this scenario, ADMIN or GRANT privileges can be granted to administrators. Give RD CREATE, DROP, ALTER, LOAD, SELECT permissions to any or specified database tables. Give Client SELECT permission to any or specified database table. At the same time, it can also simplify the authorization of multiple users by creating different roles. - -2. Scene 2 - - There are multiple services in a cluster, and each business may use one or more data. Each business needs to manage its own users. In this scenario. Administrator users can create a user with GRANT privileges at the DATABASE level for each database. The user can only authorize the specified database for the user. - -3. Blacklist - - Doris itself does not support blacklist, only whitelist, but we can simulate blacklist in some way. Suppose you first create a user named `user@'192.%'`, which allows users from `192.*` to login. At this time, if you want to prohibit users from `192.168.10.1` from logging in, you can create another user with `cmy@'192.168.10.1'` and set a new password. Since `192.168.10.1` has a higher priority than `192.%`, user can no longer login by using the old password from `192.168.10.1`. - - diff --git a/docs/documentation/en/administrator-guide/small-file-mgr_EN.md b/docs/documentation/en/administrator-guide/small-file-mgr_EN.md deleted file mode 100644 index 4a6d99486a31b3..00000000000000 --- a/docs/documentation/en/administrator-guide/small-file-mgr_EN.md +++ /dev/null @@ -1,97 +0,0 @@ - - -# File Manager - -Some functions in Doris require some user-defined files. For example, public keys, key files, certificate files and so on are used to access external data sources. The File Manager provides a function that allows users to upload these files in advance and save them in Doris system, which can then be referenced or accessed in other commands. - -## Noun Interpretation - -* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. -* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. -* BDBJE: Oracle Berkeley DB Java Edition. Distributed embedded database for persistent metadata in FE. -* SmallFileMgr: File Manager. Responsible for creating and maintaining user files. - -## Basic concepts - -Files are files created and saved by users in Doris. - -A file is located by `database`, `catalog`, `file_name`. At the same time, each file also has a globally unique ID (file_id), which serves as the identification in the system. - -File creation and deletion can only be performed by users with `admin` privileges. A file belongs to a database. Users who have access to a database (queries, imports, modifications, etc.) can use the files created under the database. - -## Specific operation - -File management has three main commands: `CREATE FILE`, `SHOW FILE` and `DROP FILE`, creating, viewing and deleting files respectively. The specific syntax of these three commands can be viewed by connecting to Doris and executing `HELP cmd;`. - -1. CREATE FILE - - In the command to create a file, the user must provide the following information: - - * file_name: File name. User-defined, unique within a catalog. - * Catalog: Category of files. User-defined, unique within a database. - - > Doris also has some special classification names for specific commands. - - > 1. Kafka - - > When the data source is specified as Kafka in the routine Import command and the file needs to be referenced, Doris defaults to looking for the file from the catalog category named "kafka". - - * url: the download address of the file. Currently, only unauthenticated HTTP download addresses are supported. This download address is only used to download files from this address when executing the create file command. When the file is successfully created and saved in Doris, the address will no longer be used. - * md5: optional. The MD5 value of the file. If the user provides this value, the MD5 value will be checked after the file is downloaded. File creation fails if validation fails. - - When the file is created successfully, the file-related information will be persisted in Doris. Users can view successfully created files through the `SHOW FILE` command. - -2. SHOW FILE - - This command allows you to view files that have been created successfully. Specific operations see: `HELP SHOW FILE;` - -3. DROP FILE - - This command can delete a file that has been created. Specific operations see: `HELP DROP FILE;` - -## Implementation details - -### Create and delete files - -When the user executes the `CREATE FILE` command, FE downloads the file from a given URL. The contents of the file are stored in FE memory directly in the form of Base64 encoding. At the same time, the file content and meta-information related to the file will be persisted in BDBJE. All created files, their meta-information and file content reside in FE memory. If the FE goes down and restarts, meta information and file content will also be loaded into memory from the BDBJE. When a file is deleted, the relevant information is deleted directly from FE memory and persistent information is deleted from BDBJE. - -### Use of documents - -If the FE side needs to use the created file, SmallFileMgr will directly save the data in FE memory as a local file, store it in the specified directory, and return the local file path for use. - -If the BE side needs to use the created file, BE will download the file content to the specified directory on BE through FE's HTTP interface `api/get_small_file` for use. At the same time, BE also records the information of the files that have been downloaded in memory. When BE requests a file, it first checks whether the local file exists and verifies it. If the validation passes, the local file path is returned directly. If the validation fails, the local file is deleted and downloaded from FE again. When BE restarts, local files are preloaded into memory. - -## Use restrictions - -Because the file meta-information and content are stored in FE memory. So by default, only files with size less than 1MB can be uploaded. And the total number of files is limited to 100. The configuration items described in the next section can be modified. - -## Relevant configuration - -1. FE configuration - -* `Small_file_dir`: The path used to store uploaded files, defaulting to the `small_files/` directory of the FE runtime directory. -* `max_small_file_size_bytes`: A single file size limit in bytes. The default is 1MB. File creation larger than this configuration will be rejected. -* `max_small_file_number`: The total number of files supported by a Doris cluster. The default is 100. When the number of files created exceeds this value, subsequent creation will be rejected. - - > If you need to upload more files or increase the size limit of a single file, you can modify the `max_small_file_size_bytes` and `max_small_file_number` parameters by using the `ADMIN SET CONFIG` command. However, the increase in the number and size of files will lead to an increase in FE memory usage. - -2. BE configuration - -* `Small_file_dir`: The path used to store files downloaded from FE by default is in the `lib/small_files/` directory of the BE runtime directory. diff --git a/docs/documentation/en/administrator-guide/sql-mode_EN.md b/docs/documentation/en/administrator-guide/sql-mode_EN.md deleted file mode 100644 index 58db62c4c4cd0d..00000000000000 --- a/docs/documentation/en/administrator-guide/sql-mode_EN.md +++ /dev/null @@ -1,69 +0,0 @@ - - -# SQL MODE - -The SQL MODE supported by Doris refers to the sql mode management mechanism of MySQL. Each client can set its own sql mode, and the database administrator with admin permission can set the global sql mode. - -## Sql mode introduction - -SQL MODE enables users to switch between different styles of SQL syntax and data verification strictness, making Doris more compatible with other databases. For example, in some databases, the '||' symbol is a string connector, but in Doris it is equivalent to 'or'. At this time, users only need to use SQL mode to switch to the style they want. Each client can set sql mode, which is valid in the current conversation. Only users with admin permission can set global SQL mode. - -## Theory - -SQL MODE is stored in session variables with a 64 bit long type. Each bit of this address represents the on / off (1 for on, 0 for off) state of a mode. As long as we know the specific bit of each mode, we can easily and quickly verify and operate SQL mode through bit operation. - -Every time you query sql mode, the long type will be parsed into a user-readable string. Similarly, the sql mode string sent by the user to the server will be parsed into a long type that can be stored in session variables. - -The set global sql mode will be persisted, so the operation on the global sql mode is always only once, even after the program is restarted, the last global sql mode can be recovered. - -## Operation - -1、set sql mode - -``` -set global sql_mode = "DEFAULT" -set session sql_mode = "DEFAULT" -``` ->At present, Doris's default sql mode is DEFAULT (but it will be changed in the future modification). ->Setting global sql mode requires admin permission and affects all clients that connect later. ->Setting session sql mode will only affect the current conversation client. The default setting way is session. - -2、select sql mode - -``` -select @@global.sql_mode -select @@session.sql_mode -``` ->In addition to this method, you can also view the current sql mode by returning all session variables as follows - -``` -show global variables -show session variables -``` - -## supported mode - -1. `PIPES_AS_CONCAT` - - Treat '||' as a string concatenation operator (same as CONCAT()) rather than as a synonym for OR. (e.g., `'a'||'b' = 'ab'`, `1||0 = '10'`) - -## combine mode - -(Work in progress) \ No newline at end of file diff --git a/docs/documentation/en/administrator-guide/time-zone_EN.md b/docs/documentation/en/administrator-guide/time-zone_EN.md deleted file mode 100644 index 8e174fba210e02..00000000000000 --- a/docs/documentation/en/administrator-guide/time-zone_EN.md +++ /dev/null @@ -1,91 +0,0 @@ - - -# Time zone - -Doris supports multiple time zone settings - -## Noun Interpretation - -* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. -* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. - -## Basic concepts - -There are multiple time zone related parameters in Doris - -* `system_time_zone`: - -When the server starts, it will be set automatically according to the time zone set by the machine, which can not be modified after setting. - -* `time_zone`: - -Server current time zone, set it at session level or global level. - -## Specific operations - -1. `SHOW VARIABLES LIKE '% time_zone%'` - - View the current time zone related configuration - -2. `SET time_zone = 'Asia/Shanghai'` - - This command can set the session level time zone, which will fail after disconnection. - -3. `SET global time_zone = 'Asia/Shanghai'` - - This command can set time zone parameters at the global level. The FE will persist the parameters and will not fail when the connection is disconnected. - -### Impact of time zone - -Time zone setting affects the display and storage of time zone sensitive values. - -It includes the values displayed by time functions such as `NOW()` or `CURTIME()`, as well as the time values in `SHOW LOAD` and `SHOW BACKENDS` statements. - -However, it does not affect the `LESS THAN VALUE` of the time-type partition column in the `CREATE TABLE` statement, nor does it affect the display of values stored as `DATE/DATETIME` type. - -Functions affected by time zone: - -* `FROM_UNIXTIME`: Given a UTC timestamp, return the date and time of the specified time zone, such as `FROM_UNIXTIME(0)`, return the CST time zone: `1970-01-08:00`. - -* `UNIX_TIMESTAMP`: Given a specified time zone date and time, return UTC timestamp, such as CST time zone `UNIX_TIMESTAMP('1970-01 08:00:00')`, return `0`. - -* `CURTIME`: Returns the datetime of specified time zone. - -* `NOW`: Returns the specified date and time of specified time zone. - -* `CONVERT_TZ`: Converts a date and time from one specified time zone to another. - -## Restrictions - -Time zone values can be given in several formats, case-insensitive: - -* A string representing UTC offset, such as '+10:00' or '-6:00'. - -* Standard time zone formats, such as "Asia/Shanghai", "America/Los_Angeles" - -* Abbreviated time zone formats such as MET and CTT are not supported. Because the abbreviated time zone is ambiguous in different scenarios, it is not recommended to use it. - -* In order to be compatible with Doris and support CST abbreviated time zone, CST will be internally transferred to "Asia/Shanghai", which is Chinese standard time zone. - -## Time zone format list - -[List of TZ database time zones] (https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) - -[Edit on GitHub](https://github.com/apache/incubator-doris/blob/master/docs/documentation/en/administrator-guide/time-zone_EN.md) \ No newline at end of file diff --git a/docs/documentation/en/administrator-guide/variables_EN.md b/docs/documentation/en/administrator-guide/variables_EN.md deleted file mode 100644 index e74ee1480f80d7..00000000000000 --- a/docs/documentation/en/administrator-guide/variables_EN.md +++ /dev/null @@ -1,321 +0,0 @@ - - -# Variable - -This document focuses on currently supported variables. - -Variables in Doris refer to variable settings in MySQL. However, some of the variables are only used to be compatible with some MySQL client protocols, and do not produce their actual meaning in the MySQL database. - -## Variable setting and viewing - -### View - -All or specified variables can be viewed via `SHOW VARIABLES [LIKE 'xxx'];`. Such as: - -``` -SHOW VARIABLES; -SHOW VARIABLES LIKE '%time_zone%'; -``` - -### Settings - -Some variables can be set at global-level or session-only. For global-level, the set value will be used in subsequent new session connections. For session-only, the variable only works for the current session. - -For session-only, set by the `SET var_name=xxx;` statement. Such as: - -``` -SET exec_mem_limit = 137438953472; -SET forward_to_master = true; -SET time_zone = "Asia/Shanghai"; -``` - -For global-level, set by `SET GLOBALE var_name=xxx;`. Such as: - -``` -SET GLOBAL exec_mem_limit = 137438953472 -``` - -> Note 1: Only ADMIN users can set variable at global-level. -> Note 2: Global-level variables do not affect variable values in the current session, only variables in new sessions. - -Variables that support global-level setting include: - -* `time_zone` -* `wait_timeout` -* `sql_mode` -* `is_report_success` -* `query_timeout` -* `exec_mem_limit` -* `batch_size` -* `parallel_fragment_exec_instance_num` -* `parallel_exchange_instance_num` - -At the same time, variable settings also support constant expressions. Such as: - -``` -SET exec_mem_limit = 10 * 1024 * 1024 * 1024; -SET forward_to_master = concat('tr', 'u', 'e'); -``` - -## Supported variables - -* `SQL_AUTO_IS_NULL` - - Used for compatible JDBC connection pool C3P0. No practical effect. - -* `auto_increment_increment` - - Used for compatibility with MySQL clients. No practical effect. - -* `autocommit` - - Used for compatibility with MySQL clients. No practical effect. - -* `batch_size` - - Used to specify the number of rows of a single packet transmitted by each node during query execution. By default, the number of rows of a packet is 1024 rows. That is, after the source node generates 1024 rows of data, it is packaged and sent to the destination node. - - A larger number of rows will increase the throughput of the query in the case of scanning large data volumes, but may increase the query delay in small query scenario. At the same time, it also increases the memory overhead of the query. The recommended setting range is 1024 to 4096. - -* `character_set_client` - -  Used for compatibility with MySQL clients. No practical effect. - -* `character_set_connection` - - Used for compatibility with MySQL clients. No practical effect. - -* `character_set_results` - - Used for compatibility with MySQL clients. No practical effect. - -* `character_set_server` - - Used for compatibility with MySQL clients. No practical effect. - -* `codegen_level` - - Used to set the level of LLVM codegen. (Not currently in effect). - -* `collation_connection` - - Used for compatibility with MySQL clients. No practical effect. - -* `collation_database` - - Used for compatibility with MySQL clients. No practical effect. - -* `collation_server` - - Used for compatibility with MySQL clients. No practical effect. - -* `disable_colocate_join` - - Controls whether the [Colocation Join] (./colocation-join.md) function is enabled. The default is false, which means that the feature is enabled. True means that the feature is disabled. When this feature is disabled, the query plan will not attempt to perform a Colocation Join. - -* `disable_streaming_preaggregations` - - Controls whether streaming pre-aggregation is turned on. The default is false, which is enabled. Currently not configurable and enabled by default. - -* `enable_insert_strict` - - Used to set the `strict` mode when loadingdata via INSERT statement. The default is false, which means that the `strict` mode is not turned on. For an introduction to this mode, see [here] (./load-data/insert-into-manual.md). - -* `enable_spilling` - - Used to set whether to enable external sorting. The default is false, which turns off the feature. This feature is enabled when the user does not specify a LIMIT condition for the ORDER BY clause and also sets `enable_spilling` to true. When this feature is enabled, the temporary data is stored in the `doris-scratch/` directory of the BE data directory and the temporary data is cleared after the query is completed. - - This feature is mainly used for sorting operations with large amounts of data using limited memory. - - Note that this feature is experimental and does not guarantee stability. Please turn it on carefully. - -* `exec_mem_limit` - - Used to set the memory limit for a single query. The default is 2GB, in bytes. - - This parameter is used to limit the memory that can be used by an instance of a single query fragment in a query plan. A query plan may have multiple instances, and a BE node may execute one or more instances. Therefore, this parameter does not accurately limit the memory usage of a query across the cluster, nor does it accurately limit the memory usage of a query on a single BE node. The specific needs need to be judged according to the generated query plan. - - Usually, only some blocking nodes (such as sorting node, aggregation node, and join node) consume more memory, while in other nodes (such as scan node), data is streamed and does not occupy much memory. - - When a `Memory Exceed Limit` error occurs, you can try to increase the parameter exponentially, such as 4G, 8G, 16G, and so on. - -* `forward_to_master` - - The user sets whether to forward some commands to the Master FE node for execution. The default is false, which means no forwarding. There are multiple FE nodes in Doris, one of which is the Master node. Usually users can connect to any FE node for full-featured operation. However, some of detail informationcan only be obtained from the Master FE node. - - For example, the `SHOW BACKENDS;` command, if not forwarded to the Master FE node, can only see some basic information such as whether the node is alive, and forwarded to the Master FE to obtain more detailed information including the node startup time and the last heartbeat time. - - The commands currently affected by this parameter are as follows: - - 1. `SHOW FRONTEND;` - - Forward to Master to view the last heartbeat information. - - 2. `SHOW BACKENDS;` - - Forward to Master to view startup time, last heartbeat information, and disk capacity information. - - 3. `SHOW BROKERS;` - - Forward to Master to view the start time and last heartbeat information. - - 4. `SHOW TABLET;`/`ADMIN SHOW REPLICA DISTRIBUTION;`/`ADMIN SHOW REPLICA STATUS;` - - Forward to Master to view the tablet information stored in the Master FE metadata. Under normal circumstances, the tablet information in different FE metadata should be consistent. When a problem occurs, this method can be used to compare the difference between the current FE and Master FE metadata. - - 5. `SHOW PROC;` - - Forward to Master to view information about the relevant PROC stored in the Master FE metadata. Mainly used for metadata comparison. - -* `init_connect` -    - Used for compatibility with MySQL clients. No practical effect. - -* `interactive_timeout` - - Used for compatibility with MySQL clients. No practical effect. - -* `is_report_success` - - Used to set whether you need to view the profile of the query. The default is false, which means no profile is required. - - By default, the BE sends a profile to the FE for viewing errors only if an error occurs in the query. A successful query will not send a profile. Sending a profile will incur a certain amount of network overhead, which is detrimental to a high concurrent query scenario. - - When the user wants to analyze the profile of a query, the query can be sent after this variable is set to true. After the query is finished, you can view the profile on the web page of the currently connected FE: - - `fe_host:fe_http:port/query` - - It will display the most recent 100 queries which `is_report_success` is set to true. - -* `language` - - Used for compatibility with MySQL clients. No practical effect. - -* `license` - - Show Doris's license. No other effect. - -* `load_mem_limit` - - Used to specify the memory limit of the load operation. The default is 0, which means that this variable is not used, and `exec_mem_limit` is used as the memory limit for the load operation. - - This variable is usually used for INSERT operations. Because the INSERT operation has both query and load part. If the user does not set this variable, the respective memory limits of the query and load part are `exec_mem_limit`. Otherwise, the memory of query part of INSERT is limited to `exec_mem_limit`, and the load part is limited to` load_mem_limit`. - - For other load methods, such as BROKER LOAD, STREAM LOAD, the memory limit still uses `exec_mem_limit`. - -* `lower_case_table_names` - - Used for compatibility with MySQL clients. Cannot be set. Table names in current Doris are case sensitive by default. - -* `max_allowed_packet` - - Used for compatible JDBC connection pool C3P0. No practical effect. - -* `net_buffer_length` - - Used for compatibility with MySQL clients. No practical effect. - -* `net_read_timeout` - - Used for compatibility with MySQL clients. No practical effect. - -* `net_write_timeout` - - Used for compatibility with MySQL clients. No practical effect. - -* `parallel_exchange_instance_num` - - Used to set the number of exchange nodes used by an upper node to receive data from the lower node in the execution plan. The default is -1, which means that the number of exchange nodes is equal to the number of execution instances of the lower nodes (default behavior). When the setting is greater than 0 and less than the number of execution instances of the lower node, the number of exchange nodes is equal to the set value. - - In a distributed query execution plan, the upper node usually has one or more exchange nodes for receiving data from the execution instances of the lower nodes on different BEs. Usually the number of exchange nodes is equal to the number of execution instances of the lower nodes. - - In some aggregate query scenarios, if the amount of data to be scanned at the bottom is large, but the amount of data after aggregation is small, you can try to modify this variable to a smaller value, which can reduce the resource overhead of such queries. Such as the scenario of aggregation query on the DUPLICATE KEY data model. - -* `parallel_fragment_exec_instance_num` - - For the scan node, set its number of instances to execute on each BE node. The default is 1. - - A query plan typically produces a set of scan ranges, the range of data that needs to be scanned. These data are distributed across multiple BE nodes. A BE node will have one or more scan ranges. By default, a set of scan ranges for each BE node is processed by only one execution instance. When the machine resources are abundant, you can increase the variable and let more execution instances process a set of scan ranges at the same time, thus improving query efficiency. - - The number of scan instances determines the number of other execution nodes in the upper layer, such as aggregate nodes and join nodes. Therefore, it is equivalent to increasing the concurrency of the entire query plan execution. Modifying this parameter will help improve the efficiency of large queries, but larger values will consume more machine resources, such as CPU, memory, and disk IO. - -* `query_cache_size` - - Used for compatibility with MySQL clients. No practical effect. - -* `query_cache_type` - - Used for compatible JDBC connection pool C3P0. No practical effect. - -* `query_timeout` - - Used to set the query timeout. This variable applies to all query statements in the current connection, as well as INSERT statements. The default is 5 minutes, in seconds. - -* `resource_group` - - Not used. - -* `sql_mode` - - Used to specify SQL mode to accommodate certain SQL dialects. For the SQL mode, see [here] (./sql-mode.md). - -* `sql_safe_updates` - - Used for compatibility with MySQL clients. No practical effect. - -* `sql_select_limit` - - Used for compatibility with MySQL clients. No practical effect. - -* `system_time_zone` - - Displays the current system time zone. Cannot be changed. - -* `time_zone` - - Used to set the time zone of the current session. The time zone has an effect on the results of certain time functions. For the time zone, see [here] (./time-zone.md). - -* `tx_isolation` - - Used for compatibility with MySQL clients. No practical effect. - -* `version` - - Used for compatibility with MySQL clients. No practical effect. - -* `version_comment` - - Used to display the version of Doris. Cannot be changed. - -* `wait_timeout` - - The length of the connection used to set up an idle connection. When an idle connection does not interact with Doris for that length of time, Doris will actively disconnect the link. The default is 8 hours, in seconds. - -* `default_rowset_type` - - Used for setting the default storage format of Backends storage engine. Valid options: alpha/beta - -* `use_v2_rollup` - - Used to control the sql query to use segment v2 rollup index to get data. This variable is only used for validation when upgrading to segment v2 feature. Otherwise, not recommended to use. - -* `rewrite_count_distinct_to_bitmap_hll` - - Whether to rewrite count distinct queries of bitmap and HLL types as bitmap_union_count and hll_union_agg. diff --git a/docs/documentation/en/community/gitter_EN.md b/docs/documentation/en/community/gitter_EN.md deleted file mode 100644 index 94833d95085b9e..00000000000000 --- a/docs/documentation/en/community/gitter_EN.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# Gitter Manual - -## Gitter introduction - -Gitter is a Markdown-enabled instant messaging software for developers. It can be seamlessly linked to github, PR on Github can be linked in chat, relevant historical records of discussions can be retained, historical records can be queried, and Chinese and English can be supported. - -Like many other open source projects, Doris can use Gitter as an instant messaging medium for technology exchange and community development. This article describes how to use Gitter to participate in Doris's open source development and community development. - -## Log in using links - -Entering [https://gitter.im/apache-doris/Lobby](https://gitter.im/apache-doris/Lobby) in the browser automatically jumps to the Doris community chat room interface on Gitter. - -Click on the `SIGN IN TO START TALKING` below to login. It can support two login modes, Github account or Twitter account. The author uses Github account to login, as follows: - -![](../../../resources/images/login-gitter1.png) - -After clicking on the red circle, enter the Github account and password to log into the chat room and start technical or community discussions: - -![](../../../resources/images/login-gitter2.PNG) - -You can use Gitter as well as Wechat, and get functions that are more comfortable for developers and technicians than Wechat, such as directly mentioning an activity for discussion, directly searching history chat records, etc. - -Don't forget to click on the Pentagon in the upper right corner to collect, which will make the chat room easier for you to find. - -For more gitter usage tips, you can refer to: - -[http://www.gitter.net.cn/book/gitter/roomsettings-1.html](http://www.gitter.net.cn/book/gitter/roomsettings-1.html) - -## Install Mobile Client - -You can download Gitter's mobile client and participate in technical discussions on your mobile phone at any time and anywhere. Download links: - -[https://gitter.im/home](https://gitter.im/home) - -## Search Gitter and join Doris Community Chat Room - -Partners already using Gitter log in directly to search for `apache-doris` and can join the chat room when they find it. Other functions are used in the same chapter, which is not discussed here. diff --git a/docs/documentation/en/community/how-to-contribute_EN.md b/docs/documentation/en/community/how-to-contribute_EN.md deleted file mode 100644 index 069c0c9c565db4..00000000000000 --- a/docs/documentation/en/community/how-to-contribute_EN.md +++ /dev/null @@ -1,76 +0,0 @@ - - -# Contribute to Doris - -Thank you very much for your interest in the Doris project. We welcome your suggestions, comments (including criticisms), comments and contributions to the Doris project. - -Your suggestions, comments and comments on Doris can be made directly through GitHub's [Issues] (https://github.com/apache/incubator-doris/issues/new/selection). - -There are many ways to participate in and contribute to Doris projects: code implementation, test writing, process tool improvement, document improvement, and so on. Any contribution will be welcomed and you will be added to the list of contributors. Further, with sufficient contributions, you will have the opportunity to become a Commiter of Aapche with Apache mailbox and be included in the list of [Apache Commiters] (http://people.apache.org/committer-index.html). - -Any questions, you can contact us to get timely answers, including Wechat, Gitter (GitHub instant messaging tool), e-mail and so on. - -## Initial contact - -For the first time in Doris community, you can: - -* Follow [Doris Github](https://github.com/apache/incubator-doris) -* Subscribe to our [mailing list] (./subscribe-mail-list.md); -* Join Doris Wechat Group (add micro-signal: morningman-cmy, note: join Doris Group) and ask questions at any time. -* Enter Doris's [Gitter] (./gitter.md) chat room; - -Learn the development trends of Doris project in time and give your opinions on the topics you are concerned about. - -## Doris's code and documentation - -As you can see from [GitHub] (https://github.com/apache/incubator-doris), Apache Doris (incubating) code base mainly consists of three parts: Frontend (FE), Backend (BE) and Broker (to support file reading on external storage systems such as HDFS). Documents are mainly the wiki on Doris website and GitHub, as well as the online help manual when running Doris. Details of these components can be found in the following table: - -| Component Name | Component Description | Related Language| -|--------|----------------------------|----------| -| [Frontend daemon (FE)] (https://github.com/apache/incubator-doris) | consists of a query coordinator and a metadata manager | Java| -| [Backend daemon (BE)] (https://github.com/apache/incubator-doris) | Responsible for storing data and executing query fragments | C++| -| [Broker] (https://github.com/apache/incubator-doris) | Read HDFS data to Doris | Java| -| [Website](https://github.com/apache/incubator-doris-website) | Doris Website | Markdown | -+ [Github Wiki] (https://github.com/apache/incubator-doris/wiki); Doris Github Wiki; Markdown_; -| Doris Runtime Help Document | Online Help Manual at Doris Runtime | Markdown| - -## Improving documentation - -Documentation is the most important way for you to understand Apache Doris, and it's where we need help most! - -Browse the document, you can deepen your understanding of Doris, can also help you understand Doris's function and technical details, if you find that the document has problems, please contact us in time; - -If you are interested in improving the quality of documents, whether it is revising the address of a page, correcting a link, and writing a better introductory document, we are very welcome! - -Most of our documents are written in markdown format, and you can modify and submit document changes directly through `docs/` in [GitHub] (https://github.com/apache/incubator-doris). If you submit code changes, you can refer to [Pull Request] (./pull-request.md). - -## If a Bug or problem is found - -If a Bug or problem is found, you can directly raise a new Issue through GitHub's [Issues] (https://github.com/apache/incubator-doris/issues/new/select), and we will have someone deal with it regularly. - -You can also fix it yourself by reading the analysis code (of course, it's better to talk to us before that, maybe someone has fixed the same problem) and submit a [Pull Request] (./pull-request.md). - -## Modify the code and submit PR (Pull Request) - -You can download the code, compile and install it, deploy and run it for a try (refer to the [compilation document] (./installing/compilation.md)) to see if it works as you expected. If you have problems, you can contact us directly, ask questions or fix them by reading and analyzing the source code. - -Whether it's fixing Bugs or adding Features, we're all very welcome. If you want to submit code to Doris, you need to create a new branch for your submitted code from the fork code library on GitHub to your project space, add the source project upstream, and submit PR. - -About how to submit a PR refer to [Pull Request] (./pull-request.md). diff --git a/docs/documentation/en/community/index.rst b/docs/documentation/en/community/index.rst deleted file mode 100644 index 5ff10aa4694a25..00000000000000 --- a/docs/documentation/en/community/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -=================== -Apache Commnity -=================== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/community/members_EN.md b/docs/documentation/en/community/members_EN.md deleted file mode 100644 index b779394975800a..00000000000000 --- a/docs/documentation/en/community/members_EN.md +++ /dev/null @@ -1,61 +0,0 @@ - - -# PMC Members & Committer - -## Mentors (3) - -|Apache ID|Github Username |Public Name| -|--------|-----------|----------| -|wave |dave2wave |Dave Fisher | -|shaofengshi |shaofengshi| Shao Feng Shi | -|ningjiang |WillemJiang |Willem Ning Jiang| - -## PPMC (9) -(the listing below excludes mentors) - -|Apache ID|Github Username |Public Name| -|--------|-----------|----------| -|lingbin| lingbin |Bin Ling | -|lichaoyong |chaoyli |Chaoyong Li | -|zhaoc |imay |Chun Zhao | -|lide |lide-reed, doris-ci |De Li | -|chenhao |chenhao7253886 |Hao Chen | -|morningman |morningman |Mingyu Chen| -|maruyue |maruyue| Ruyue Ma | -|sijie |sijie |Sijie Guo | -|zshao |zshao |Zheng Shao| - -## Committers (11) - -|Apache ID|Github Username |Public Name| -|--------|-----------|----------| -|lingbin| lingbin |Bin Ling | -|lichaoyong |chaoyli |Chaoyong Li | -|zhaoc |imay |Chun Zhao | -|lide |lide-reed, doris-ci |De Li | -|chenhao |chenhao7253886 |Hao Chen | -|morningman |morningman |Mingyu Chen| -|maruyue |maruyue| Ruyue Ma | -|sijie |sijie |Sijie Guo | -|zshao |zshao |Zheng Shao| -|kangkaisen|kangkaisen|Kaisen Kang| -|lingmiao|EmmyMiao87|Ling Miao| -|gaodayue|gaodayue|Dayue Gao| -|liuhangyuan|HangyuanLiu|Hangyuan Liu| diff --git a/docs/documentation/en/community/pull-request_EN.md b/docs/documentation/en/community/pull-request_EN.md deleted file mode 100644 index 9da220e14fa4b8..00000000000000 --- a/docs/documentation/en/community/pull-request_EN.md +++ /dev/null @@ -1,252 +0,0 @@ - - -# Code Submission Guide - -[Pull Request (PR)] (https://help.github.com/articles/about-pull-requests/) can be easily submitted on [Github] (https://github.com/apache/incubator-doris). The PR method of Doris project is described below. - -## Fork Repository - -Go to the [github page] (https://github.com/apache/incubator-doris) of apache/incubator-doris , and click the button `Fork` in the upper right corner for Fork. - -![Fork](../../../resources/images/fork-repo.png) - -### 2. Configuring GIT and submitting modifications - -#### (1) Clone the code locally: - -``` -git clone https://github.com//incubator-doris.git -``` - -Note: Please replace your GitHub name with your yourgithubname\\\\\\\\\\\\\\. - -When clone is completed, origin defaults to the remote fork address on github. - -#### (2) Add apache/incubator-doris to the remote branch upstream of the local warehouse: - -``` -cd incubator-doris -git remote add upstream https://github.com/apache/incubator-doris.git -``` - -#### (3) Check remote warehouse settings: - -``` -git remote -v -origin https://github.com//incubator-doris.git (fetch) -origin https://github.com//incubator-doris.git (push) -upstream https://github.com/apache/incubator-doris.git (fetch) -upstream https://github.com/apache/incubator-doris.git (push) -``` - -#### (4) New branches to modify them: - -``` -git checkout -b -``` - -Note: \ name is customized for you. - -Code changes can be made after creation. - -#### (5) Submit code to remote branch: - -``` -git commit -a -m "" -git push origin -``` - -For more git usage, please visit: [git usage] (https://www.atlassian.com/git/tutorials/set-up-a-repository), not to mention here. - -### 3. Create PR - -#### (1) New PR -Switch to your GitHub page in the browser, switch to the submitted branch yourbranchname\\ and click the `New pull request` button to create it, as shown in the following figure: - -![new PR](../../../resources/images/new-pr.png) - -#### (2) preparation branch -At this time, the `Create pull request` button will appear. If not, please check whether the branch is selected correctly or click on `compare across forks' to re-select the repo and branch. - -![create PR](../../../resources/images//create-pr.png) - -#### (3) Fill Commit Message -Here, please fill in the summary and details of the comment, and then click `Create pull request` to create it. - -For how to write Commit Message, here are some Tips: - -* Please use the form of English verb + object. The verb does not use the past tense and the sentence uses imperative sentence. -* Subject and body should be written, and they should be separated by blank lines (fill in separately on GitHub PR interface). -* Message topic length should not exceed **50** characters; -* Message content should not exceed **72** characters per line, and the excess should be replaced manually. -* Message content is used to explain what has been done, why and how. -* The first letter of the message subject should be **capitalized**, and the end of the sentence **should not** have a full stop. -* The message content specifies the associated issue (if any), such as # 233; - -For more details, see . - -![create PR](../../../resources/images/create-pr2.png) - -#### (4) Complete the creation -After successful creation, you can see that Doris project needs review, you can wait for us to review and join, you can also contact us directly. - -![create PR](../../../resources/images/create-pr3.png) - -So far, your PR creation is complete. Read more about PR [collaborating-with-issues-and-pull-requests] (https://help.github.com/categories/collaborating-with-issues-and-pull-requests/). - -### 4. Conflict Resolution - -When submitting PR, code conflicts are usually caused by multiple people editing the same file. The main steps to resolve conflicts are as follows: - -#### (1) Switch to the main branch - -``` -git checkout master -``` - -#### (2) Synchronize remote main branch to local - -``` -git pull upstream master -``` - -#### (3) Switch back to the previous branch (assuming the branch is named fix) - -``` -git checkout fix -``` - -#### (4) rebase - -``` -git rebase -i master -``` - -At this point, a file that modifies the record will pop up and can be saved directly. Then, we will prompt which files have conflicts. At this time, we can open the conflict file to modify the conflict part. After all the conflicts of the conflict files are resolved, we will execute them. - -``` -git add . -git rebase --continue -``` - -Then you can go back and forth until the screen appears something like * rebase successful * and then you can update the branch that submitted PR: - -``` -git push -f origin fix -``` - -### 5. An example - -#### (1) fetch to the latest code for the local branch of upstream that has been configured - -``` -$ git branch -* master - -$ git fetch upstream -remote: Counting objects: 195, done. -remote: Compressing objects: 100% (68/68), done. -remote: Total 141 (delta 75), reused 108 (delta 48) -Receiving objects: 100% (141/141), 58.28 KiB, done. -Resolving deltas: 100% (75/75), completed with 43 local objects. -From https://github.com/apache/incubator-doris - 9c36200..0c4edc2 master -> upstream/master -``` - -#### (2) rebase - -``` -$ git rebase upstream/master -First, rewinding head to replay your work on top of it... -Fast-forwarded master to upstream/master. -``` - -#### (3) Check to see if other submissions are not synchronized to their own repo submissions - -``` -$ git status -# On branch master -# Your branch is ahead of 'origin/master' by 8 commits. -# -# Untracked files: -# (use "git add ..." to include in what will be committed) -# -# custom_env.sh -nothing added to commit but untracked files present (use "git add" to track) -``` - -#### (4) Merge code submitted by others into their own repo - -``` -$ git push origin master -Counting objects: 195, done. -Delta compression using up to 32 threads. -Compressing objects: 100% (41/41), done. -Writing objects: 100% (141/141), 56.66 KiB, done. -Total 141 (delta 76), reused 140 (delta 75) -remote: Resolving deltas: 100% (76/76), completed with 44 local objects. -To https://lide-reed:fc35ff925bd8fd6629be3f6412bacee99d4e5f97@github.com/lide-reed/incubator-doris.git - 9c36200..0c4edc2 master -> master -``` - -#### (5) New branch, ready for development - -``` -$ git checkout -b my_branch -Switched to a new branch 'my_branch' - -$ git branch - master -* my_branch -``` - -#### (6) Prepare to submit after code modification is completed - -``` -$ git add -u -``` - -#### (7) Fill in the message and submit it it to the new local branch - -``` -$ git commit -m "Fix a typo" -[my_branch 55e0ba2] Fix a typo -1 files changed, 2 insertions(+), 2 deletions(-) -``` - -#### (8) Push the branch into GitHub's own repo far away - -``` -$ git push origin my_branch -Counting objects: 11, done. -Delta compression using up to 32 threads. -Compressing objects: 100% (6/6), done. -Writing objects: 100% (6/6), 534 bytes, done. -Total 6 (delta 4), reused 0 (delta 0) -remote: Resolving deltas: 100% (4/4), completed with 4 local objects. -remote: -remote: Create a pull request for 'my_branch' on GitHub by visiting: -remote: https://github.com/lide-reed/incubator-doris/pull/new/my_branch -remote: -To https://lide-reed:fc35ff925bd8fd6629be3f6412bacee99d4e5f97@github.com/lide-reed/incubator-doris.git - * [new branch] my_branch -> my_branch -``` - -At this point, you can create PR according to the previous process. diff --git a/docs/documentation/en/community/release-process_EN.md b/docs/documentation/en/community/release-process_EN.md deleted file mode 100644 index b1a7689a70adeb..00000000000000 --- a/docs/documentation/en/community/release-process_EN.md +++ /dev/null @@ -1,666 +0,0 @@ - - -# Publish of Apache Doris - -Apache publishing must be at least an IPMC member, a commiter with Apache mailboxes, a role called release manager. - -The general process of publication is as follows: - -1. Preparing your setup -2. Preparing for release candidates - 1. launching DISCUSS in the community - 2. cutting a release branch - 3. clean up issues - 4. merging necessary patch to release branch -3. Running the voting process for a release - 1. singing a tag and upload it to [Apache dev svn repo](https://dist.apache.org/repos/dist/dev/incubator/doris) - 2. calling votes from [Doris community](dev@doris.apache.org) - 3. send result email to [Doris community](dev@doris.apache.org) - 4. calling votes from [Incubator community](general@incubator.apache.org) - 5. send result email to general@incubator.apache.org -4. Finalizing and posting a release - 1. Upload the signature package to [Apache release repo](https://dist.apache.org/repos/dist/release/incubator/doris) and generate relevant links - 2. Prepare release note and send Announce mail to general@incubator.apache.org - 3. Publish download links on Doris website and GitHub - - -## prepare setup - -If you are a new Release Manager, you can read up on the process from the followings: - -1. release signing https://www.apache.org/dev/release-signing.html -2. gpg for signing https://www.apache.org/dev/openpgp.html -3. svn https://www.apache.org/dev/version-control.html#https-svn - -### preparing gpg key - -Release manager needs Mr. A to sign his own public key before publishing and upload it to the public key -server. Then he can use this public key to sign the package ready for publication. -If your key already exists in [key] (https://dist.apache.org/repos/dist/dev/initiator/doris/keys), you can skip this step. - - -#### Installation and configuration of signature software GnuPG -##### GnuPG - -In 1991, programmer Phil Zimmermann developed the encryption software PGP to avoid government surveillance. This software is very useful, spread quickly, has become a necessary tool for many programmers. However, it is commercial software and cannot be used freely. So the Free Software Foundation decided to develop a replacement for PGP, called GnuPG. This is the origin of GPG. - -##### Installation Configuration - -CentOS installation command: - -``` -yum install gnupg -``` -After installation, the default configuration file gpg.conf will be placed in the home directory. - -``` -~/.gnupg /gpg.conf -``` - -If this directory or file does not exist, you can create an empty file directly. -Edit gpg.conf, modify or add KeyServer configuration: - -``` -keyserver hkp http://keys.gnupg.net -``` - -Apache signature recommends SHA512, which can be done by configuring gpg. -Edit gpg.conf and add the following three lines: - -``` -personal-digest-preferences SHA512 -cert -digest -something SHA512 -default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed -``` - -#### Generating new signatures - -##### Prepare to Sign - -Recommended settings for generating new signatures: - -We must log in to user account directly through SecureCRT and other terminals. We can't transfer it through Su - user or ssh. Otherwise, the password input box will not show up and make an error. - -Let's first look at the version of GPG and whether it supports SHA512. - -``` -$ gpg --version -gpg (GnuPG) 2.0.22 -libgcrypt 1.5.3 -Copyright (C) 2013 Free Software Foundation, Inc. -License GPLv3+: GNU GPL version 3 or later -This is free software: you are free to change and redistribute it. -There is NO WARRANTY, to the extent permitted by law. - -Home: ~/.gnupg -Supported algorithms: -Pubkey: RSA, ?, ?, ELG, DSA -Cipher: IDEA, 3DES, CAST5, BLOWFISH, AES, AES192, AES256, TWOFISH, - CAMELLIA128, CAMELLIA192, CAMELLIA256 -Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224 -Compression: Uncompressed, ZIP, ZLIB, BZIP2 -``` - -##### Generating new signatures - -``` -$ gpg --gen-key -gpg (GnuPG) 2.0.22; Copyright (C) 2013 Free Software Foundation, Inc. -This is free software: you are free to change and redistribute it. -There is NO WARRANTY, to the extent permitted by law. - -Please select what kind of key you want: - (1) RSA and RSA (default) - (2) DSA and Elgamal - (3) DSA (sign only) - (4) RSA (sign only) -Your selection? 1 -RSA keys may be between 1024 and 4096 bits long. -What keysize do you want? (2048) 4096 -Requested keysize is 4096 bits -Please specify how long the key should be valid. - 0 = key does not expire - = key expires in n days - w = key expires in n weeks - m = key expires in n months - y = key expires in n years -Key is valid for? (0) -Key does not expire at all -Is this correct? (y/N) y - -GnuPG needs to construct a user ID to identify your key. - -Real name: xxx -Name must be at least 5 characters long -Real name: xxx-yyy -Email address: xxx@apache.org -Comment: xxx's key -You selected this USER-ID: - "xxx-yyy (xxx's key) " - -Change (N)ame, (C)omment, (E)mail or (O)kay/(Q)uit? o -``` - -Real name needs to be consistent with the ID shown in ID. apache. org. -Email address is apache's mailbox. - -##### View and Output - -The first line shows the name of the public key file (pubring. gpg), the second line shows the public key characteristics (4096 bits, Hash string and generation time), the third line shows the "user ID", and the fourth line shows the private key characteristics. - -``` -$ gpg --list-keys -/home/lide/.gnupg/pubring.gpg ------------------------------ -pub 4096R/33DBF2E0 2018-12-06 -uid xxx-yyy (xxx's key) -sub 4096R/0E8182E6 2018-12-06 -``` - -xxx-yy is the user ID. - -``` -gpg --armor --output public-key.txt --export [UserID] -``` - -``` -$ gpg --armor --output public-key.txt --export xxx-yyy -$ cat public-key.txt ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG V2.0.22 (GNU /Linux) - -mQINBFwJEQ0BEACwqLluHfjBqD/RWZ4uoYxNYHlIzZvbvxAlwS2mn53BirLIU/G3 -9opMWNplvmK+3+gNlRlFpiZ7EvHsF/YJOAP59HmI2Z... -``` - -#### Upload signature public key - -Public key servers are servers that store users'public keys exclusively on the network. The send-keys parameter uploads the public key to the server. - -``` -gpg --send-keys xxxx -``` - -Where XXX is the last step -- the string after pub in the list-keys result, as shown above: 33DBF2E0 - -You can also upload the contents of the above public-key.txt through the following website: - -``` -http://keys.gnupg.net -``` - -After successful upload, you can query the website and enter 0x33DBF2E0: - -http://keys.gnupg.net - -Queries on the site are delayed and may take an hour. - - -#### Generate fingerprint and upload it to Apache user information - -Because the public key server has no checking mechanism, anyone can upload the public key in your name, so there is no way to guarantee the reliability of the public key on the server. Usually, you can publish a public key fingerprint on the website and let others check whether the downloaded public key is true or not. - -Fingerprint parameter generates public key fingerprints: - -``` -gpg --fingerprint [UserID] -``` - -``` -$ gpg --fingerprint xxx-yyy -pub 4096R/33DBF2E0 2018-12-06 - Key fingerprint = 07AA E690 B01D 1A4B 469B 0BEF 5E29 CE39 33DB F2E0 -uid xxx-yyy (xxx's key) -sub 4096R/0E8182E6 2018-12-06 -``` - -Paste the fingerprint above (i.e. 07AA E690 B01D 1A4B 469B 0BEF 5E29 CE39 33DB F2E0) into your user information: - -https://id.apache.org -OpenPGP Public Key Primary Fingerprint: - -#### Generating keys - -``` -svn co //dist.apache.org/repos/dist/dev/incubator/doris/ -# edit doris/KEY file -gpg --list-sigs [用户 ID] >> doris/KEYS -gpg --armor --export [用户 ID] >> doris/KEYS -svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Update KEYS" -``` - -## Prepare for release - -### Launching DISCUSS in the Community - -If you think you've fixed a lot of bugs and developed more important features, any IPMC member can initiate DISCUSS discussions to release a new version. -An e-mail entitled [DISCUSS] x.y.z release can be launched to discuss within the community what bugs have been fixed and what features have been developed. -If DISCUSS mail is supported, we can proceed to the next step. - -### Preparatory Branch - -Before publishing, we need to build a new branch, For example: - -``` -$ git checkout -b branch-0.9 - -``` - -This branch needs to be fully tested to make functions available, bug convergence, and important bugs fixed. - -This process needs to wait for the community to see if a necessary patch needs to be merged in this version, and if so, it needs to be cherry picked to the release branch. - -### clean up issue - -Go through all the issues belonging to this version, close those that have been completed, and if they cannot be completed, postpone them to a later version. - -### Merge necessary patches - -During the release waiting process, there may be more important patch merging. If someone in the community says that there is an important bug to merge, then release manager needs to evaluate and merge the important patches into the release branch. - -## Running the voting process for a release - -### dozen Tags - -When the above branches are stable, tags can be made on them. -Remember to modify the `build_version` variable in `gensrc/script/gen_build_version.sh` when creating tags. For example, `build_version='0.10.0-release'.` - -For example: - -``` -$ git checkout branch-0.9 -$ git tag -a 0.9.0-rc01 -m "0.9.0 release candidate 01" -$ git push origin 0.9.0-rc01 -Counting objects: 1, done. -Writing objects: 100% (1/1), 165 bytes | 0 bytes/s, done. -Total 1 (delta 0), reused 0 (delta 0) -To git@github.com:apache/incubator-doris.git - * [new tag] 0.9.0-rc01 -> 0.9.0-rc01 - -$ git tag -``` - -### Packing Signature - -The following steps also need to log into user accounts directly through terminals such as SecureCRT, and can not be transferred through Su - user or ssh, otherwise the password input box will not show and error will be reported. - -``` -$ git checkout 0.9.0-rc01 - -$ git archive --format=tar 0.9.0-rc01 --prefix=apache-doris-0.9.0-incubating-src/ | gzip > apache-doris-0.9.0-incubating-src.tar.gz - -$ gpg -u xxx@apache.org --armor --output apache-doris-0.9.0-incubating-src.tar.gz.asc --detach-sign apache-doris-0.9.0-incubating-src.tar.gz - -$ gpg --verify apache-doris-0.9.0-incubating-src.tar.gz.asc apache-doris-0.9.0-incubating-src.tar.gz - -$ sha512sum apache-doris-0.9.0-incubating-src.tar.gz > apache-doris-0.9.0-incubating-src.tar.gz.sha512 - -$ sha512sum --check apache-doris-0.9.0-incubating-src.tar.gz.sha512 -``` - -### Upload signature packages and KEYS files to DEV SVN - -First, download the SVN library: - -``` -svn co https://dist.apache.org/repos/dist/dev/incubator/doris/ -``` - -Organize all previous files into the following SVN paths - -``` -./doris/ -|-- 0.11.0-rc1 -| |-- apache-doris-0.11.0-incubating-src.tar.gz -| |-- apache-doris-0.11.0-incubating-src.tar.gz.asc -| `-- apache-doris-0.11.0-incubating-src.tar.gz.sha512 -`-- KEYS -``` - -Upload these files - -``` -svn add 0.9.0-rc1 -svn commit -m "Release Apache Doris (incubating) 0.9.0 rc1" -``` - -### Send community voting emails - -[VOTE] Release Apache Doris 0.9.0-incubating-rc01 - - -``` -Hi all, - -Please review and vote on Apache Doris 0.9.0-incubating-rc01 release. - -The release candidate has been tagged in GitHub as 0.9.0-rc01, available -here: -https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 - -===== CHANGE LOG ===== - -New Features: -.... - -====================== - -Thanks to everyone who has contributed to this release. - -The artifacts (source, signature and checksum) corresponding to this release -candidate can be found here: -https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc1/ - -This has been signed with PGP key 33DBF2E0, corresponding to -lide@apache.org. -KEYS file is available here: -https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS -It is also listed here: -https://people.apache.org/keys/committer/lide.asc - -To verify and build, you can refer to following wiki: -https://github.com/apache/incubator-doris/wiki/How-to-verify-Apache-Release -https://wiki.apache.org/incubator/IncubatorReleaseChecklist - -The vote will be open for at least 72 hours. -[ ] +1 Approve the release -[ ] +0 No opinion -[ ] -1 Do not release this package because ... - -Best Regards, -xxx - ----- -DISCLAIMER-WIP: -Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), -sponsored by the Apache Incubator. Incubation is required of all newly accepted projects -until a further review indicates that the infrastructure, communications, and decision -making process have stabilized in a manner consistent with other successful ASF projects. -While incubation status is not necessarily a reflection of the completeness or stability -of the code, it does indicate that the project has yet to be fully endorsed by the ASF. - -Some of the incubating project’s releases may not be fully compliant with ASF policy. For -example, releases may have incomplete or un-reviewed licensing conditions. What follows is -a list of known issues the project is currently aware of (note that this list, by definition, -is likely to be incomplete): - - * Releases may have incomplete licensing conditions - -If you are planning to incorporate this work into your product/project, please be aware that -you will need to conduct a thorough licensing review to determine the overall implications of -including this work. For the current status of this project through the Apache Incubator -visit: https://incubator.apache.org/projects/doris.html -``` - -### Email Result after the vote is passed - -[Result][VOTE] Release Apache Doris 0.9.0-incubating-rc01 - -``` -Thanks to everyone, and this vote is now closed. - -It has passed with 4 +1 (binding) votes and no 0 or -1 votes. - -Binding: -Zhao Chun -+1 xxx -+ 1 Li Chaoyong -+1 Mingyu Chen - -Best Regards, -xxx - -``` - -### Send an e-mail to general@incubator.apache.org for a vote. - -[VOTE] Release Apache Doris 0.9.0-incubating-rc01 - -``` -Hi all, - -Please review and vote on Apache Doris 0.9.0-incubating-rc01 release. - -Apache Doris is an MPP-based interactive SQL data warehousing for reporting and analysis. - -The Apache Doris community has voted on and approved this release: -https://lists.apache.org/thread.html/d70f7c8a8ae448bf6680a15914646005c6483564464cfa15f4ddc2fc@%3Cdev.doris.apache.org%3E - -The vote result email thread: -https://lists.apache.org/thread.html/64d229f0ba15d66adc83306bc8d7b7ccd5910ecb7e842718ce6a61da@%3Cdev.doris.apache.org%3E - -The release candidate has been tagged in GitHub as 0.9.0-rc01, available here: -https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 - -There is no CHANGE LOG file because this is the first release of Apache Doris. -Thanks to everyone who has contributed to this release, and there is a simple release notes can be found here: -https://github.com/apache/incubator-doris/issues/406 - -The artifacts (source, signature and checksum) corresponding to this release candidate can be found here: -https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc01/ - -This has been signed with PGP key 33DBF2E0, corresponding to lide@apache.org. -KEYS file is available here: -https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS -It is also listed here: -https://people.apache.org/keys/committer/lide.asc - -The vote will be open for at least 72 hours. -[ ] +1 Approve the release -[ ] +0 No opinion -[ ] -1 Do not release this package because ... - -To verify and build, you can refer to following instruction: - -Firstly, you must be install and start docker service, and then you could build Doris as following steps: - -Step1: Pull the docker image with Doris building environment -$ docker pull apachedoris/doris-dev:build-env -You can check it by listing images, its size is about 3.28GB. - -Step2: Run the Docker image -You can run image directly: -$ docker run -it apachedoris/doris-dev:build-env - -Step3: Download Doris source -Now you should in docker environment, and you can download Doris source package. -(If you have downloaded source and it is not in image, you can map its path to image in Step2.) -$ wget https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc01/apache-doris-0.9.0.rc01-incubating-src.tar.gz - -Step4: Build Doris -Now you can decompress and enter Doris source path and build Doris. -$ tar zxvf apache-doris-0.9.0.rc01-incubating-src.tar.gz -$ cd apache-doris-0.9.0.rc01-incubating-src -$ sh build.sh - -Best Regards, -xxx - ----- -DISCLAIMER-WIP: -Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), -sponsored by the Apache Incubator. Incubation is required of all newly accepted projects -until a further review indicates that the infrastructure, communications, and decision -making process have stabilized in a manner consistent with other successful ASF projects. -While incubation status is not necessarily a reflection of the completeness or stability -of the code, it does indicate that the project has yet to be fully endorsed by the ASF. - -Some of the incubating project’s releases may not be fully compliant with ASF policy. For -example, releases may have incomplete or un-reviewed licensing conditions. What follows is -a list of known issues the project is currently aware of (note that this list, by definition, -is likely to be incomplete): - - * Releases may have incomplete licensing conditions - -If you are planning to incorporate this work into your product/project, please be aware that -you will need to conduct a thorough licensing review to determine the overall implications of -including this work. For the current status of this project through the Apache Incubator -visit: https://incubator.apache.org/projects/doris.html -``` - -The threaded connection for mail can be found here: - -`https://lists.apache.org/list.html?dev@doris.apache.org` - - -### Email Result to general@incubator.apache.org - -[RESULT][VOTE] Release Apache Doris 0.9.0-incubating-rc01 - -``` -Hi, - -Thanks to everyone, and the vote for releasing Apache Doris 0.9.0-incubating-rc01 is now closed. - -It has passed with 4 +1 (binding) votes and no 0 or -1 votes. - -Binding: -+1 Willem Jiang -+1 Justin Mclean -+1 ShaoFeng Shi -+1 Makoto Yui - -The vote thread: -https://lists.apache.org/thread.html/da05fdd8d84e35de527f27200b5690d7811a1e97d419d1ea66562130@%3Cgeneral.incubator.apache.org%3E - -Best Regards, -xxx -``` - -## Finalizing release - -### Upload package to release - -When the formal voting is successful, email [Result] first, and then prepare the release package. -Copy the source package, signature file and hash file from the corresponding RC folder published under dev to another directory 0.9.0-incubating. Note that the file name does not need rcxx (rename, but do not recalculate signatures, hash can recalculate, the results will not change) - -KEYS files also need to be copied if they are first released. Then add to SVN release. - -``` - -https://dist.apache.org/repos/dist/release/incubator/doris/0.9.0-incubating/ - -Eventually you can see it on apache's website: -http://www.apache.org/dist/incubator/doris/0.9.0-incubating/ - -``` - - -### Send Announce e-mail to general@incubator.apache.org - -Title: - -``` -[ANNOUNCE] Apache Doris (incubating) 0.9.0 Release -``` - -Send mail group: - -``` -general@incubator.apache.org -dev@doris.apache.org -``` - -Mail text: - -``` -Hi All, - -We are pleased to announce the release of Apache Doris 0.9.0-incubating. - -Apache Doris (incubating) is an MPP-based interactive SQL data warehousing for reporting and analysis. - -The release is available at: -http://doris.apache.org/downloads.html - -Thanks to everyone who has contributed to this release, and the release note can be found here: -https://github.com/apache/incubator-doris/releases - -Best Regards, - -On behalf of the Doris team, -xxx - ----- -DISCLAIMER-WIP: -Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), -sponsored by the Apache Incubator. Incubation is required of all newly accepted projects -until a further review indicates that the infrastructure, communications, and decision -making process have stabilized in a manner consistent with other successful ASF projects. -While incubation status is not necessarily a reflection of the completeness or stability -of the code, it does indicate that the project has yet to be fully endorsed by the ASF. - -Some of the incubating project’s releases may not be fully compliant with ASF policy. For -example, releases may have incomplete or un-reviewed licensing conditions. What follows is -a list of known issues the project is currently aware of (note that this list, by definition, -is likely to be incomplete): - - * Releases may have incomplete licensing conditions - -If you are planning to incorporate this work into your product/project, please be aware that -you will need to conduct a thorough licensing review to determine the overall implications of -including this work. For the current status of this project through the Apache Incubator -visit: https://incubator.apache.org/projects/doris.html - -``` - -### Publish links on Doris website and GitHub - -#### Create Download Links - -Download link: -http://www.apache.org/dyn/closer.cgi?filename=incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz&action=download - -wget --trust-server-names "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz" - -Original location: -https://www.apache.org/dist/incubator/doris/0.9.0-incubating/ - -http://www.apache.org/dyn/closer.cgi/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz - -Source package: -http://www.apache.org/dyn/closer.cgi/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz - -ASC: -http://archive.apache.org/dist/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.asc - -sha512: -http://archive.apache.org/dist/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.sha512 - -KEYS: -http://archive.apache.org /dist /incubator /doris /KEYS - -refer to: - -#### Prepare release note - -The following two areas need to be modified: - -1. Github's release page - -``` -https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 -``` - -2. Doris Official Website Download Page - -``` -http://doris.apache.org /downloads.html -``` diff --git a/docs/documentation/en/community/subscribe-mail-list_EN.md b/docs/documentation/en/community/subscribe-mail-list_EN.md deleted file mode 100644 index 305526cbb846de..00000000000000 --- a/docs/documentation/en/community/subscribe-mail-list_EN.md +++ /dev/null @@ -1,62 +0,0 @@ - - -# Subscribe to mailing lists - -Mail List is the most recognized form of communication in Apache community. Generally speaking, open source community questions and answers, technical discussions, transaction decisions are carried through mailing lists. The asynchronous and broadcasting features of mailing lists are also very suitable for communication in open source communities. So how do you subscribe to Apache Doris (incubating) mailing lists? It mainly includes the following five steps. - -## 1. Send Subscription Mail - -Open your own email, create a new email, and send an email to `dev-subscribe@doris.apache.org` (subject and content are arbitrary) - -![step1](../../../resources/images/subscribe-mail-list-step1.png) - -## 2. Receive confirmation emails from dev-help@doris.apache.org - -After the first step, you will receive a confirmation email from `dev-help@doris.apache.org`, which is shown below. (**If you fail to receive it for a long time, please confirm that the mail has been intercepted, or has been automatically grouped into "Subscribed Mail", "Spam Mail", "Promotional Mail" folders**) - -![step2](../../../resources/images/subscribe-mail-list-step2.png) - -## 3. Reply to confirmation mail - -For the mail received in the previous step, - -**a. Reply to this email directly** - -***or*** - -**B. Create a new `recipient` e-mail for the `reply address` in the previous step** - -Every subject is acceptable. - -![step3](../../../resources/images/subscribe-mail-list-step3.png) - - -## 4. Receiving Welcome Emails - -After completing the third step, you will receive a welcome email entitled **WELCOME to dev@doris.apache.org**. So far, the work of subscribing to mailing lists has been completed, and community dynamics will be notified by mail. - -![step4](../../../resources/images/subscribe-mail-list-step4.png) - - -## 5. Initiate e-mail discussion (optional) - -After successfully subscribing to the mailing list, if you want to initiate a discussion, send an email directly to `dev@doris.apache.org`. Anyone who subscribes to the mailing list receives the mail. -​ -​ diff --git a/docs/documentation/en/community/verify-apache-release_EN.md b/docs/documentation/en/community/verify-apache-release_EN.md deleted file mode 100644 index 805009f52ffe82..00000000000000 --- a/docs/documentation/en/community/verify-apache-release_EN.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# Verify Apache Release - -To verify the release, following checklist can used to reference: - -1. [ ] Download links are valid. -2. [ ] Checksums and PGP signatures are valid. -3. [ ] DISCLAIMER-WIP is included. -4. [ ] Source code artifacts have correct names matching the current release. -5. [ ] LICENSE and NOTICE files are correct for the repository. -6. [ ] All files have license headers if necessary. -7. [ ] No compiled archives bundled in source archive. -8. [ ] Building is OK. - -## 1. Download source package, signature file, hash file and KEYS - -Download all artifacts, take a.b.c-incubating as an example: - -``` -wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz - -wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz.sha512 - -wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz.asc - -wget https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS -``` - -## 2. Verify signature and hash - -GnuPG is recommended, which can install by yum install gnupg or apt-get install gnupg. - -``` -gpg --import KEYS -gpg --verify apache-doris-a.b.c-incubating-src.tar.gz.asc apache-doris-a.b.c-incubating-src.tar.gz -sha512sum --check apache-doris-a.b.c-incubating-src.tar.gz.sha512 -``` - -## 3. Verify license header - -Apache RAT is recommended to verify license headder, which can dowload as following command. - -``` -wget http://mirrors.tuna.tsinghua.edu.cn/apache//creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -tar zxvf apache -rat -0.12 -bin.tar.gz -``` - -Given your source dir is apache-doris-a.b.c-incubating-src, you can check with following command. -It will output a file list which don't include ASF license header, and these files used other licenses. - -``` -/usr/java/jdk/bin/java -jar apache-rat-0.12/apache-rat-0.12.jar -a -d apache-doris-a.b.c-incubating-src -E apache-doris-a.b.c-incubating-src/.rat-excudes -``` - -## 4. Verify building - -To compile the Doris, please read [Compilation](../installing/compilation_EN.html) diff --git a/docs/documentation/en/developer-guide/debug-tool.md b/docs/documentation/en/developer-guide/debug-tool.md deleted file mode 100644 index 1e221898d2ab4e..00000000000000 --- a/docs/documentation/en/developer-guide/debug-tool.md +++ /dev/null @@ -1,266 +0,0 @@ - - -# Debug Tool - -In the process of using and developing Doris, we often encounter scenarios that need to debug Doris. Here are some common debugging tools. - -## Preparing - -[pprof] (https://github.com/google/pprof): from gperftools, it is used to transform the content generated by gperftools into a format that is easy for people to read, such as PDF, SVG, text, etc. - -[graphviz] (http://www.graphviz.org/): in the absence of this library, pprof can only be converted to text format, but this way is not easy to view. After the library is installed, pprof can be converted to SVG, PDF and other formats, and the call relationship is clearer. - -[perf] (https://perf.wiki.kernel.org/index.php/main_page): Linux kernel comes with performance analysis tool. [here] (http://www.brendangregg.com/perf.html) there are some examples of perf usage. - -[flamegraph] (https://github.com/brendangregg/flamegraph): a visualization tool used to show the output of perf in the form of flame graph. - -## Memory - -Debugging memory is generally divided into two aspects. One is whether the total amount of memory use is reasonable. On the one hand, the excessive amount of memory use may be due to memory leak in the system, on the other hand, it may be due to improper use of program memory. The second is whether there is a problem of memory overrun and illegal access, such as program access to memory with an illegal address, use of uninitialized memory, etc. For the debugging of memory, we usually use the following ways to track the problems. - -### Log - -When we find that the memory usage is too large, we can first check the be.out log to see if there is a large memory application. Because of the TCMalloc currently used by Doris to manage memory, when a large memory application is encountered, the stack of the application will be printed to the be.out file. The general form is as follows: - -``` -tcmalloc: large alloc 1396277248 bytes == 0x3f3488000 @ 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed -``` - -This indicates that Doris be is trying to apply memory of '1396277248 bytes' on this stack. We can use the 'addr2line' command to restore the stack to a letter that we can understand. The specific example is shown below. - -``` -$ addr2line -e lib/palo_be 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed - -/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1335 -/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1357 -/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.cpp:267 -/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.hpp:86 -/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:239 -/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:213 -thread.cpp:? -``` - -### HEAP PROFILE - -Sometimes the application of memory is not caused by the application of large memory, but by the continuous accumulation of small memory. Then there is no way to locate the specific application information by viewing the log, so you need to get the information through other ways. - -At this time, we can take advantage of TCMalloc's [heapprofile](https://gperftools.github.io/gperftools/heapprofile.html). If the heapprofile function is set, we can get the overall memory application usage of the process. The usage is to set the 'heapprofile' environment variable before starting Doris be. For example: - -``` -export HEAPPROFILE=/tmp/doris_be.hprof -./bin/start_be.sh --daemon -``` - -In this way, when the dump condition of the heapprofile is met, the overall memory usage will be written to the file in the specified path. Later, we can use the 'pprof' tool to analyze the output content. - -``` -$ pprof --text lib/palo_be /tmp/doris_be.hprof.0012.heap | head -30 - -Using local file lib/palo_be. -Using local file /tmp/doris_be.hprof.0012.heap. -Total: 668.6 MB - 610.6 91.3% 91.3% 610.6 91.3% doris::SystemAllocator::allocate_via_malloc (inline) - 18.1 2.7% 94.0% 18.1 2.7% _objalloc_alloc - 5.6 0.8% 94.9% 63.4 9.5% doris::RowBatch::RowBatch - 5.1 0.8% 95.6% 7.1 1.1% butil::ResourcePool::add_block (inline) - 3.7 0.5% 96.2% 3.7 0.5% butil::iobuf::create_block (inline) - 3.4 0.5% 96.7% 3.4 0.5% butil::FlatMap::init - 3.2 0.5% 97.2% 5.2 0.8% butil::ObjectPool::add_block (inline) - 2.6 0.4% 97.6% 2.6 0.4% __gnu_cxx::new_allocator::allocate (inline) - 2.0 0.3% 97.9% 2.0 0.3% butil::ObjectPool::add_block_group (inline) - 2.0 0.3% 98.2% 2.0 0.3% butil::ResourcePool::add_block_group (inline) - 1.7 0.3% 98.4% 1.7 0.3% doris::SegmentReader::_load_index -``` - -Contents of each column of the above documents: - -* Column 1: the memory size directly applied by the function, in MB -* Column 4: the total memory size of the function and all the functions it calls. -* The second column and the fifth column are the proportion values of the first column and the fourth column respectively. -* The third column is the cumulative value of the second column. - -Of course, it can also generate call relation pictures, which is more convenient for analysis. For example, the following command can generate a call graph in SVG format. - -``` -pprof --svg lib/palo_be /tmp/doris_be.hprof.0012.heap > heap.svg -``` - -**NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.** - -### pprof remote server - -Although heapprofile can get all the memory usage information, it has some limitations. 1. Restart be. 2. You need to enable this command all the time, which will affect the performance of the whole process. - -For Doris be, you can also use the way of opening and closing the heap profile dynamically to analyze the memory application of the process. Doris supports the [remote server debugging of gperftools](https://gperftools.github.io/gperftools/pprof_remote_servers.html). Then you can use 'pprof' to directly perform dynamic head profile on the remote running Doris be. For example, we can check the memory usage increment of Doris through the following command - -``` -$ pprof --text --seconds=60 http://be_host:be_webport/pprof/heap - -Total: 1296.4 MB - 484.9 37.4% 37.4% 484.9 37.4% doris::StorageByteBuffer::create - 272.2 21.0% 58.4% 273.3 21.1% doris::RowBlock::init - 157.5 12.1% 70.5% 157.5 12.1% doris::RowBatch::RowBatch - 90.7 7.0% 77.5% 90.7 7.0% doris::SystemAllocator::allocate_via_malloc - 66.6 5.1% 82.7% 66.6 5.1% doris::IntegerColumnReader::init - 47.9 3.7% 86.4% 47.9 3.7% __gnu_cxx::new_allocator::allocate - 20.8 1.6% 88.0% 35.4 2.7% doris::SegmentReader::_load_index - 12.7 1.0% 89.0% 12.7 1.0% doris::DecimalColumnReader::init - 12.7 1.0% 89.9% 12.7 1.0% doris::LargeIntColumnReader::init - 12.7 1.0% 90.9% 12.7 1.0% doris::StringColumnDirectReader::init - 12.3 0.9% 91.9% 12.3 0.9% std::__cxx11::basic_string::_M_mutate - 10.4 0.8% 92.7% 10.4 0.8% doris::VectorizedRowBatch::VectorizedRowBatch - 10.0 0.8% 93.4% 10.0 0.8% doris::PlainTextLineReader::PlainTextLineReader -``` - -The output of this command is the same as the output and view mode of heap profile, which will not be described in detail here. Statistics will be enabled only during execution of this command, which has a limited impact on process performance compared with heap profile. - -### LSAN - -[LSAN](https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer) is an address checking tool, GCC has been integrated. When we compile the code, we can enable this function by turning on the corresponding compilation options. When the program has a determinable memory leak, it prints the leak stack. Doris be has integrated this tool, only need to compile with the following command to generate be binary with memory leak detection version. - -``` -BUILD_TYPE=LSAN ./build.sh -``` - -When the system detects a memory leak, it will output the corresponding information in be. Out. For the following demonstration, we intentionally insert a memory leak code into the code. We insert the following code into the `open` function of `StorageEngine`. - -``` - char* leak_buf = new char[1024]; - strcpy(leak_buf, "hello world"); - LOG(INFO) << leak_buf; -``` - -We get the following output in be.out - -``` -================================================================= -==24732==ERROR: LeakSanitizer: detected memory leaks - -Direct leak of 1024 byte(s) in 1 object(s) allocated from: - #0 0xd10586 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/lsan/lsan_interceptors.cc:164 - #1 0xe333a2 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104 - #2 0xd3cc96 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 - #3 0x7f573b5eebd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) - -SUMMARY: LeakSanitizer: 1024 byte(s) leaked in 1 allocation(s). -``` - -From the above output, we can see that 1024 bytes have been leaked, and the stack information of memory application has been printed out. - -**NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.** - -**NOTE: if the LSAN switch is turned on, the TCMalloc will be automatically turned off** - -### ASAN - -Except for the unreasonable use and leakage of memory. Sometimes there will be memory access illegal address and other errors. At this time, we can use [ASAN](https://github.com/google/sanitizers/wiki/addresssanitizer) to help us find the cause of the problem. Like LSAN, ASAN is integrated into GCC. Doris can open this function by compiling as follows - -``` -BUILD_TYPE=ASAN ./build.sh -``` - -Execute the binary generated by compilation. When the detection tool finds any abnormal access, it will immediately exit and output the stack illegally accessed in be.out. The output of ASAN is the same as that of LSAN. Here we also actively inject an address access error to show the specific content output. We still inject an illegal memory access into the 'open' function of 'storageengine'. The specific error code is as follows - -``` - char* invalid_buf = new char[1024]; - for (int i = 0; i < 1025; ++i) { - invalid_buf[i] = i; - } - LOG(INFO) << invalid_buf; -``` - -We get the following output in be.out - -``` -================================================================= -==23284==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x61900008bf80 at pc 0x00000129f56a bp 0x7fff546eed90 sp 0x7fff546eed88 -WRITE of size 1 at 0x61900008bf80 thread T0 - #0 0x129f569 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 - #1 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 - #2 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) - #3 0xd30794 (/home/ssd0/zc/palo/doris/core/output3/be/lib/palo_be+0xd30794) - -0x61900008bf80 is located 0 bytes to the right of 1024-byte region [0x61900008bb80,0x61900008bf80) -allocated by thread T0 here: - #0 0xdeb040 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/asan/asan_new_delete.cc:82 - #1 0x129f50d in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104 - #2 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 - #3 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) - -SUMMARY: AddressSanitizer: heap-buffer-overflow /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) -``` - -From this message, we can see that at the address of `0x61900008bf80`, we tried to write a byte, but this address is illegal. We can also see the application stack of the address `[0x61900008bb80, 0x61900008bf80]`. - -**NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.** - -**NOTE: if the ASAN switch is turned on, the TCMalloc will be automatically turned off** - -In addition, if stack information is output in be.out, but there is no function symbol, then we need to handle it manually to get readable stack information. The specific processing method needs a script to parse the output of ASAN. At this time, we need to use [asan_symbolize](https://llvm.org/svn/llvm-project/compiler-rt/trunk/lib/asan/scripts/asan_symbolize.py) to help with parsing. The specific usage is as follows: - -``` -cat be.out | python asan_symbolize.py | c++filt -``` - -With the above command, we can get readable stack information. - -## CPU - -When the CPU idle of the system is very low, it means that the CPU of the system has become the main bottleneck. At this time, it is necessary to analyze the current CPU usage. For the be of Doris, there are two ways to analyze the CPU bottleneck of Doris. - -### pprof - -Because Doris has integrated and compatible with GPERF rest interface, users can analyze remote Doris be through the 'pprof' tool. The specific usage is as follows: - -``` -pprof --svg --seconds=60 http://be_host:be_webport/pprof/profile > be.svg -``` - -In this way, a CPU consumption graph of be execution can be generated. - -![CPU Pprof](../../../resources/images/cpu-pprof-demo.png) - -### perf + flamegragh - -This is a quite common CPU analysis method. Compared with `pprof`, this method must be able to log in to the physical machine of the analysis object. However, compared with pprof, which can only collect points on time, perf can collect stack information through different events. The specific usage is as follows: - -``` -perf record -g -p be_pid -- sleep 60 -``` - -This command counts the CPU operation of be for 60 seconds and generates perf.data. For the analysis of perf.data, the command of perf can be used for analysis. - -``` -perf report -``` - -The analysis results in the following pictures - -![Perf Report](../../../resources/images/perf-report-demo.png) - -To analyze the generated content. Of course, you can also use flash graph to complete the visual display. - -``` -perf script | ./FlameGraph/stackcollapse-perf.pl | ./FlameGraph/flamegraph.pl > be.svg -``` - -This will also generate a graph of CPU consumption at that time. - -![CPU Flame](../../../resources/images/cpu-flame-demo.svg) diff --git a/docs/documentation/en/developer-guide/format-code.md b/docs/documentation/en/developer-guide/format-code.md deleted file mode 100644 index be5faa3f78d683..00000000000000 --- a/docs/documentation/en/developer-guide/format-code.md +++ /dev/null @@ -1,78 +0,0 @@ - - -# Format Code -To automatically format the code, clang-format is a good choice. - -## Code Style -Doris Code Style is based on Google's, makes a few changes. The customized .clang-format -file is in the root dir of Doris. -Now, .clang-format file only works on clang-format-8.0.1+. - -## Preparing -You should install clang-format, or you can use clang-format plugins which support by IDEs or Editors. - -### Install clang-format -Ubuntu: `apt-get install clang-format` - -The current release is 10.0, you can specify old version, e.g. - - `apt-get install clang-format-9` - -Centos 7: - -The version of clang-format installed by yum is too old. Compiling clang from source -is recommended. - -### Clang-format plugins -Clion IDE supports the plugin "ClangFormat", you can search in `File->Setting->Plugins` - and download it. -But the version is not match with clang-format. Judging from the options supported, -the version is lower than clang-format-9.0. - -## Usage - -### CMD -`clang-format --style=file -i $File$` - -When using `-style=file`, clang-format for each input file will try to find the -.clang-format file located in the closest parent directory of the input file. -When the standard input is used, the search is started from the current directory. - -Note: filter out the files which should not be formatted, when batch clang-formating - files. - - A example of how to filter \*.h/\*.cpp and exclude some dirs: - -`find . -type f -not \( -wholename ./env/* \) -regextype posix-egrep -regex - ".*\.(cpp|h)" | xargs clang-format -i -style=file` - -### Using clang-format in IDEs or Editors -#### Clion -If using the plugin 'ClangFormat' in Clion, choose `Reformat Code` or press the keyboard -shortcut. -#### VS Code -VS Code needs install the extension 'Clang-Format', and specify the executable path of -clang-format in settings. - -``` -"clang-format.executable": "$clang-format path$", -"clang-format.style": "file" -``` -Then, choose `Format Document`. \ No newline at end of file diff --git a/docs/documentation/en/developer-guide/index.rst b/docs/documentation/en/developer-guide/index.rst deleted file mode 100644 index 1eb091a1f379d5..00000000000000 --- a/docs/documentation/en/developer-guide/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -================ -Developer Guide -================ - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/downloads/downloads_EN.md b/docs/documentation/en/downloads/downloads_EN.md deleted file mode 100644 index ab2208bd5197b0..00000000000000 --- a/docs/documentation/en/downloads/downloads_EN.md +++ /dev/null @@ -1,14 +0,0 @@ -# Downloads - -You can download source code from following links, then compile and install Doirs. - -| Version | Release Date | Download Source from Mirror | -|---|---|---| -| 0.12.0 | 2020-04-24 | [Source](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz.sha512)) | -| 0.11.0 | 2019-11-29 | [Source](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz.sha512)) | -| 0.10.0 | 2019-07-02 | [Source](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz.sha512)) | -| 0.9.0 | 2019-02-18 | [Source](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.sha512)) | - -To verify the downloaded files, please read [Verify Apache Release](../community/verify-apache-release_EN.html). - -After verification, please read [Compilation](../installing/compilation_EN.html) and [Installation and deployment](../installing/install-deploy_EN.html) to compile and install Doris. diff --git a/docs/documentation/en/downloads/index.rst b/docs/documentation/en/downloads/index.rst deleted file mode 100644 index 78cc4ec69142cd..00000000000000 --- a/docs/documentation/en/downloads/index.rst +++ /dev/null @@ -1,10 +0,0 @@ -============= -Downloads -============= - -.. toctree:: - :maxdepth: 1 - :glob: - - * - diff --git a/docs/documentation/en/extending-doris/audit-plugin_EN.md b/docs/documentation/en/extending-doris/audit-plugin_EN.md deleted file mode 100644 index 84143cca5fccb1..00000000000000 --- a/docs/documentation/en/extending-doris/audit-plugin_EN.md +++ /dev/null @@ -1,89 +0,0 @@ - - -# Audit log plugin - -Doris's audit log plugin was developed based on FE's plugin framework. Is an optional plugin. Users can install or uninstall this plugin at runtime. - -This plugin can periodically import the FE audit log into the specified Doris cluster, so that users can easily view and analyze the audit log through SQL. - -## Compile, Configure and Deploy - -### Compile - -After executing `sh build_plugin.sh` in the Doris code directory, you will get the `auditloader.zip` file in the `fe_plugins/output` directory. - -### Configuration - -Unzip `auditloader.zip` and you will see three files: - -``` -auditloader.jar -plugin.properties -plugin.conf -``` - -Open `plugin.conf` for configuration. See the comments of the configuration items. - -After the configuration is complete, repackage the three files as `auditloader.zip`. - -### Deployment - -You can place this file on an http download server or copy it to the specified directory of all FEs. Here we use the latter. - -### Installation - -After deployment is complete, and before installing the plugin, you need to create the audit database and tables previously specified in `plugin.conf`. The table creation statement is as follows: - -``` -create table doris_audit_tbl__ -( -    query_id varchar (48) comment "Unique query id", -    time datetime not null comment "Query start time", -    client_ip varchar (32) comment "Client IP", -    user varchar (64) comment "User name", -    db varchar (96) comment "Database of this query", -    state varchar (8) comment "Query result state. EOF, ERR, OK", -    query_time bigint comment "Query execution time in millisecond", -    scan_bytes bigint comment "Total scan bytes of this query", -    scan_rows bigint comment "Total scan rows of this query", -    return_rows bigint comment "Returned rows of this query", -    stmt_id int comment "An incremental id of statement", -    is_query tinyint comment "Is this statemt a query. 1 or 0", -    frontend_ip varchar (32) comment "Frontend ip of executing this statement", -    stmt varchar (2048) comment "The original statement, trimed if longer than 2048 bytes" -) -partition by range (time) () -distributed by hash (query_id) buckets 1 -properties ( -    "dynamic_partition.time_unit" = "DAY", -    "dynamic_partition.start" = "-30", -    "dynamic_partition.end" = "3", -    "dynamic_partition.prefix" = "p", -    "dynamic_partition.buckets" = "1", -    "dynamic_partition.enable" = "true", -    "replication_num" = "1" -); -``` - -The `dynamic_partition` attribute selects the number of days to keep the audit log based on your needs. - -After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`. - -Upon completion, the plug-in will continuously import audit date into this table at specified intervals. \ No newline at end of file diff --git a/docs/documentation/en/extending-doris/doris-on-es_EN.md b/docs/documentation/en/extending-doris/doris-on-es_EN.md deleted file mode 100644 index 8503816b2b74c1..00000000000000 --- a/docs/documentation/en/extending-doris/doris-on-es_EN.md +++ /dev/null @@ -1,226 +0,0 @@ - - -# Doris On ES - -Doris-On-ES combines Doris's distributed query planning capability with ES (Elastic search)'s full-text search capability to provide a more complete OLAP scenario solution: - -1. Multi-index Distributed Join Query in ES -2. Joint Query of Tables in Doris and ES, More Complex Full-Text Retrieval and Filtering -3. Aggregated queries for fields of ES keyword type: suitable for frequent changes in index, tens of millions or more of single fragmented documents, and the cardinality of the field is very large - -This document mainly introduces the realization principle and usage of this function. - -## Noun Interpretation - -* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. -* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. -* Elastic search (ES): The most popular open source distributed search engine. -* DataNode: The data storage and computing node of ES. -* MasterNode: The Master node of ES, which manages metadata, nodes, data distribution, etc. -* scroll: The built-in data set cursor feature of ES for streaming scanning and filtering of data. - - -## How to use it - -### Create appearance - -``` -CREATE EXTERNAL TABLE `es_table` ( - `id` bigint(20) COMMENT "", - `k1` bigint(20) COMMENT "", - `k2` datetime COMMENT "", - `k3` varchar(20) COMMENT "", - `k4` varchar(100) COMMENT "", - `k5` float COMMENT "" -) ENGINE=ELASTICSEARCH -PARTITION BY RANGE(`id`) -() -PROPERTIES ( -"host" = "http://192.168.0.1:8200,http://192.168.0.2:8200", -"user" = "root", -"password" = "root", -"index" = "tindex”, -"type" = "doc" -); -``` - -Description of parameters: - -Parameter | description ----|--- -Host | ES Cluster Connection Address, which can specify one or more, through which Doris obtains the share distribution information of ES version number and index -User | Open the user name of the ES cluster authenticated by basic, you need to ensure that the user has access to: / cluster / state / nodes / HTTP and other path permissions and read permissions for index -Password | corresponding user's password information -The index name of the ES corresponding to the table in index | Doris can be alias -Type | Specifies the type of index, defaulting to _doc -Transport | Internal reservation, default to http - -### Query - -#### Basic Conditions Filtration - -``` -select * from es_table where k1 > 1000 and k3 ='term' or k4 like 'fu*z_' -``` - -#### Extended esquery SQL grammar -The first column name parameter of `esquery` is used to associate `index`, the second parameter is the JSON expression of the basic `Query DSL`, and the curly bracket `{}` is used to include `root` of json. There is and can only be one key of json, such as mat. Ch, geo_shape, bool, etc. - -Match query: - -``` -select * from es_table where esquery(k4, '{ - "match": { - "k4": "doris on elasticsearch" - } - }'); -``` -Geo related queries: - -``` -select * from es_table where esquery(k4, '{ - "geo_shape": { - "location": { - "shape": { - "type": "envelope", - "coordinates": [ - [ - 13, - 53 - ], - [ - 14, - 52 - ] - ] - }, - "relation": "within" - } - } - }'); -``` - -Bool query: - -``` -select * from es_table where esquery(k4, ' { - "bool": { - "must": [ - { - "terms": { - "k1": [ - 11, - 12 - ] - } - }, - { - "terms": { - "k2": [ - 100 - ] - } - } - ] - } - }'); -``` - - - -## Principle - -``` -+----------------------------------------------+ -| | -| Doris +------------------+ | -| | FE +--------------+-------+ -| | | Request Shard Location -| +--+-------------+-+ | | -| ^ ^ | | -| | | | | -| +-------------------+ +------------------+ | | -| | | | | | | | | -| | +----------+----+ | | +--+-----------+ | | | -| | | BE | | | | BE | | | | -| | +---------------+ | | +--------------+ | | | -+----------------------------------------------+ | - | | | | | | | - | | | | | | | - | HTTP SCROLL | | HTTP SCROLL | | -+-----------+---------------------+------------+ | -| | v | | v | | | -| | +------+--------+ | | +------+-------+ | | | -| | | | | | | | | | | -| | | DataNode | | | | DataNode +<-----------+ -| | | | | | | | | | | -| | | +<--------------------------------+ -| | +---------------+ | | |--------------| | | | -| +-------------------+ +------------------+ | | -| Same Physical Node | | -| | | -| +-----------------------+ | | -| | | | | -| | MasterNode +<-----------------+ -| ES | | | -| +-----------------------+ | -+----------------------------------------------+ - - -``` - -1. After the ES appearance is created, FE requests the host specified by the table to obtain HTTP port information of all nodes and share distribution information of index. If the request fails, it will traverse the host list sequentially until it succeeds or fails completely. - -2. When querying, the query plan will be generated and sent to the corresponding BE node according to some node information obtained by FE and metadata information of index. - -3. The BE node requests locally deployed ES nodes in accordance with the `proximity principle`. The BE receives data concurrently from each fragment of ES index in the `HTTP Scroll` mode. - -4. After calculating the result, return it to client - -## Push-Down operations -An important function of `Doris On Elastic` search is to push down filtering conditions: push ES under filtering conditions, so that only data that really meets the conditions can be returned, which can significantly improve query performance and reduce the CPU, memory and IO utilization of Doris and Elastic search. - -The following operators are optimized to push down filters as follows: - -| SQL syntax | ES 5.x+ syntax | -|-------|:---:| -| = | term query| -| in | terms query | -| > , < , >= , ⇐ | range | -| and | bool.filter | -| or | bool.should | -| not | bool.must_not | -| not in | bool.must_not + terms | -| esquery | ES Query DSL | - - -## Other notes - -1. ES Version Requirements - - The main version of ES is larger than 5. The scanning mode of ES data before 2. X and after 5. x is different. At present, the scanning mode of ES data after 5. x is supported. - -2. Does ES Cluster Support X-Pack Authentication - - Support all ES clusters using HTTP Basic authentication - -3. Some queries are much slower than requesting ES - - Yes, for example, query related to _count, etc., the ES internal will directly read the number of documents that meet the requirements of the relevant metadata, without the need to filter the real data. diff --git a/docs/documentation/en/extending-doris/index.rst b/docs/documentation/en/extending-doris/index.rst deleted file mode 100644 index 282d12575930a9..00000000000000 --- a/docs/documentation/en/extending-doris/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -==================== -Extending Ability -==================== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/extending-doris/plugin-development-manual_EN.md b/docs/documentation/en/extending-doris/plugin-development-manual_EN.md deleted file mode 100644 index 3ca7b0ef228414..00000000000000 --- a/docs/documentation/en/extending-doris/plugin-development-manual_EN.md +++ /dev/null @@ -1,293 +0,0 @@ - - -# Plugin Development Manual - -## Introduction - -Doris supports dynamic loading of plug-ins. Users can develop their own plug-ins to implement some extended functions. This manual mainly introduces the development, compilation and deployment methods of Frontend-side plug-ins. - - -`fe_plugins` is the parent module of the fe plugins. It can uniformly manage the third-party library information that the plugin depends on. Adding a plugin can add a submodule implementation under `fe_plugins`. - -## Plugin - -A FE Plugin can be a **.zip package** or a **directory**, which contains at least two parts: the `plugin.properties` and `.jar` files. The `plugin.properties` file is used to describe the plugin information. - -The file structure of a Plugin looks like this: - -``` -# plugin .zip -auditodemo.zip: - -plugin.properties - -auditdemo.jar - -xxx.config - -data/ - -test_data/ - -# plugin local directory -auditodemo/: - -plugin.properties - -auditdemo.jar - -xxx.config - -data/ - -test_data/ -``` - -`plugin.properties` example: - -``` -### required: -# -# the plugin name -name = audit_plugin_demo -# -# the plugin type -type = AUDIT -# -# simple summary of the plugin -description = just for test -# -# Doris's version, like: 0.11.0 -version = 0.11.0 - -### FE-Plugin optional: -# -# version of java the code is built against -# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4 -java.version = 1.8.31 -# -# the name of the class to load, fully-qualified. -classname = AuditPluginDemo - -### BE-Plugin optional: -# the name of the so to load -soName = example.so -``` - -## Write A Plugin - -The development environment of the FE plugin depends on the development environment of Doris. So please make sure Doris's compilation and development environment works normally. - -### Create module - -We can add a submodule in the `fe_plugins` directory to implement Plugin and create a project: - -``` -mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false -``` - -The command produces a new mvn project, and a new submodule is automatically added to `fe_plugins/pom.xml`: - -``` -    ..... -    org.apache -    doris-fe-plugins -    pom -    1.0-SNAPSHOT -     -        auditdemo -        # new plugin module -        doris-fe-test -     -    ..... -``` - -The new plugin project file structure is as follows: - -``` --doris-fe-test/ --pom.xml --src/ - ---- main/java/org/apache/ - ------- App.java # mvn auto generate, ignore - ---- test/java/org/apache -``` - -We will add an assembly folder under main to store `plugin.properties` and `zip.xml`. After completion, the file structure is as follows: - -``` --doris-fe-test/ --pom.xml --src/ ----- main/ ------- assembly/ --------- plugin.properties --------- zip.xml ------- java/org/apache/ ---------App.java # mvn auto generate, ignore ----- test/java/org/apache -``` - -### Add zip.xml - -`zip.xml`, used to describe the content of the final package of the plugin (.jar file, plugin.properties): - -``` - -    plugin -     -        zip -     -     -    false -     -         -            target -             -                *.jar -             -            / -         - -         -            src/main/assembly -             -                plugin.properties -             -            / -         -     - -``` - - -### Update pom.xml - -Then we need to update `pom.xml`, add doris-fe dependency, and modify maven packaging way: - -``` - - - - org.apache - doris-fe-plugins - 1.0-SNAPSHOT - - 4.0.0 - - auditloader - jar - - - - org.apache - doris-fe - - - - - ... - - - - - - auditloader - - - maven-assembly-plugin - 2.4.1 - - false - - src/main/assembly/zip.xml - - - - - make-assembly - package - - single - - - - - - - - -``` - -### Implement plugin - -Then we can happily implement Plugin according to the needs. Plugins need to implement the `Plugin` interface. For details, please refer to the `auditdemo` plugin sample code that comes with Doris. - -### Compile - -Before compiling the plugin, you must first execute `sh build.sh --fe` of Doris to complete the compilation of Doris FE. - -Finally, execute `sh build_plugin.sh` in the ${DORIS_HOME} path and you will find the `your_plugin_name.zip` file in `fe_plugins/output` - -Or you can execute `sh build_plugin.sh --plugin your_plugin_name` to only build your plugin. - -### Other way - -The easiest way, you can implement your plugin by modifying the example `auditdemo` - -## Deploy - -Doris's plugin can be deployed in three ways: - -* Http or Https .zip, like `http://xxx.xxxxxx.com/data/plugin.zip`, Doris will download this .zip file -* Local .zip, like `/home/work/data/plugin.zip`, need to be deployed on all FE and BE nodes -* Local directory, like `/home/work/data/plugin`, .zip decompressed folder, need to be deployed on all FE, BE nodes - -Note: Need to ensure that the plugin .zip file is available in the life cycle of doris! - -## Install and Uninstall - -Install and uninstall the plugin through the install/uninstall statements. More details, see `HELP INSTALL PLUGIN;` `HELP IUNNSTALL PLUGIN;` `HELP SHOW PLUGINS;` - -``` -mysql> install plugin from "/home/users/seaven/auditdemo.zip"; -Query OK, 0 rows affected (0.09 sec) - -mysql> mysql> show plugins\G -*************************** 1. row *************************** - Name: auditloader - Type: AUDIT -Description: load audit log to olap load, and user can view the statistic of queries - Version: 0.12.0 -JavaVersion: 1.8.31 - ClassName: AuditLoaderPlugin - SoName: NULL - Sources: /home/cmy/git/doris/core/fe_plugins/output/auditloader.zip - Status: INSTALLED -*************************** 2. row *************************** - Name: AuditLogBuilder - Type: AUDIT -Description: builtin audit logger - Version: 0.12.0 -JavaVersion: 1.8.31 - ClassName: org.apache.doris.qe.AuditLogBuilder - SoName: NULL - Sources: Builtin - Status: INSTALLED -2 rows in set (0.00 sec) - -mysql> uninstall plugin auditloader; -Query OK, 0 rows affected (0.05 sec) - -mysql> show plugins; -Empty set (0.00 sec) -``` diff --git a/docs/documentation/en/extending-doris/user-defined-function_EN.md b/docs/documentation/en/extending-doris/user-defined-function_EN.md deleted file mode 100644 index 82f8fe8974caa4..00000000000000 --- a/docs/documentation/en/extending-doris/user-defined-function_EN.md +++ /dev/null @@ -1,111 +0,0 @@ - - -# User Define Function - -Users can extend Doris's capabilities through UDF mechanisms. Through this document, users can create their own UDF. - -## Writing UDF functions - -Before using UDF, users need to write their own UDF functions in Doris's UDF framework. In the `be/src/udf_samples/udf_sample.h | cpp` file, it is a simple UDF Demo. - -Writing a UDF function requires the following steps. - -### Writing functions - -Create the corresponding header file, CPP file, and implement the logic you need in the CPP file. The corresponding relationship between the format of implementation function in CPP file and UDF. - -#### Non-variable parameters - -For UDF with non-variable parameters, the corresponding relationship between them is very direct. -For example, `INT MyADD'(INT, INT) ` UDF corresponds to `IntVal AddUdf(FunctionContext* context, const IntVal & arg1, const IntVal & arg2)`. - -1. `AddUdf` can be any name, as long as it is specified when UDF is created. -2. The first parameter in the implementation function is always `FunctionContext*`. The implementer can obtain some query-related content and apply for some memory to be used through this structure. Specific interfaces can be defined in `udf/udf.h`. -3. Implementing functions from the second parameter requires one-to-one correspondence with UDF parameters, such as `IntVal` corresponding to `INT` type. All types in this section are referenced by `const`. -4. Return parameters should correspond to the type of UDF parameters. - -#### Variable parameters - -For variable parameters, see the following example, UDF `String md5sum (String,...)` corresponds to -`StringVal md5sumUdf (FunctionContext * ctx, int num args, const StringVal * args)` - -1. The `md5sumUdf` can also be changed at will. It can be specified at the time of creation. -2. The first parameter, like a non-variable parameter function, is passed in a `FunctionContext*`. -3. The variable parameter part consists of two parts. First, an integer is passed in, which shows that there are several parameters. Later, an array of variable parameter parts is passed in. - -#### Type correspondence - -|UDF Type|Argument Type| -|----|---------| -|TinyInt|TinyIntVal| -|SmallInt|SmallIntVal| -|Int|IntVal| -|BigInt|BigIntVal| -|LargeInt|LargeIntVal| -|Float|FloatVal| -|Double|DoubleVal| -|Date|DateTimeVal| -|Datetime|DateTimeVal| -|Char|StringVal| -|Varchar|StringVal| -|Decimal|DecimalVal| - -## Compiling UDF functions - -### Compile Doris - -Executing `sh build.sh` in the Doris root directory generates the corresponding `headers|libs` in `output/udf/` - -### Edit CMakeLists.txt - -Based on the `headers | libs` generated in the previous step, users can introduce the dependency using tools such as `CMakeLists`; in `CMakeLists`, dynamic libraries can be added by adding `-I|L` to `CMAKE_CXX_FLAGS`, respectively. For example, in `be/src/udf_samples/CMakeLists.txt`, a `udf sample` dynamic library is added using `add_library` (udfsample SHARED udf_sample.cpp) `target_link_libraries`(udfsample -static-libstdc++ -static-libgcc). You need to write down all the source files involved later (no header files included). - -### Execute compilation - -Create a `build` directory under this directory and execute `cmake ../` generate `Makefile` under `build`, and execute `make` to generate corresponding dynamic libraries. - -## Create UDF functions - -Through the above steps, you can get a dynamic library. You need to put this dynamic library in a location that can be accessed through the HTTP protocol. Then execute the create UDF function to create a UDF inside the Doris system. You need AMDIN privileges to do this. - -``` -CREATE [AGGREGATE] FUNCTION - name ([argtype][,...]) - [RETURNS] rettype - PROPERTIES (["key"="value"][,...]) -``` -Explain: - -1. In PROPERTIES, `symbol` denotes the corresponding symbol for the execution of the entry function, which must be set. You can get the corresponding symbol by the `nm` command, such as `nm libudfsample.so`, `grep AddUdf`, `ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4`. -2. In PROPERTIES, `object_file` denotes where to download to the corresponding dynamic library. This parameter must be set. -3. name: A function belongs to a DB in the form of `dbName`. `funcName`. When `dbName` is not specified explicitly, the DB where the current session is located is used as `dbName`. - -For more details, see `CREATE FUNCTION`. - -## Using UDF - -Users using UDF/UDAF must have `SELECT` privileges for the corresponding database. - -UDF is used in the same way as normal functions. The only difference is that the scope of built-in functions is global, while the scope of UDF is internal to DB. When the link session is inside the data, using the UDF name directly will find the corresponding UDF within the current DB. Otherwise, the user needs to display the database name of the specified UDF, such as `dbName`. `funcName`. - - -## Delete UDF functions - -When you no longer need UDF functions, you can delete a UDF function by using the following command, referring to `DROP FUNCTION`. diff --git a/docs/documentation/en/getting-started/advance-usage_EN.md b/docs/documentation/en/getting-started/advance-usage_EN.md deleted file mode 100644 index b33765c8d4ac0f..00000000000000 --- a/docs/documentation/en/getting-started/advance-usage_EN.md +++ /dev/null @@ -1,265 +0,0 @@ - - -# Advanced Use Guide - -Here we introduce some of Doris's advanced features. - -## Table 1 Structural Change - -Schema of the table can be modified using the ALTER TABLE command, including the following modifications: - -* Additional columns -* Delete columns -* Modify column type -* Changing column order - -Examples are given below. - -Schema of Table 1 is as follows: - -``` -+----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+----------+-------------+------+-------+---------+-------+ -| siteid | int(11) | No | true | 10 | | -| citycode | smallint(6) | No | true | N/A | | -| username | varchar(32) | No | true | | | -| pv | bigint(20) | No | false | 0 | SUM | -+----------+-------------+------+-------+---------+-------+ -``` - -We added a new column of uv, type BIGINT, aggregation type SUM, default value is 0: - -`ALTER TABLE table1 ADD COLUMN uv BIGINT SUM DEFAULT '0' after pv;` - -After successful submission, you can view the progress of the job by following commands: - -`SHOW ALTER TABLE COLUMN;` - -When the job state is FINISHED, the job is completed. The new Schema is in force. - -After ALTER TABLE is completed, you can view the latest Schema through `DESC TABLE`. - -``` -mysql> DESC table1; -+----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+----------+-------------+------+-------+---------+-------+ -| siteid | int(11) | No | true | 10 | | -| citycode | smallint(6) | No | true | N/A | | -| username | varchar(32) | No | true | | | -| pv | bigint(20) | No | false | 0 | SUM | -| uv | bigint(20) | No | false | 0 | SUM | -+----------+-------------+------+-------+---------+-------+ -5 rows in set (0.00 sec) -``` - -The following command can be used to cancel the job currently being executed: - -`CANCEL ALTER TABLE COLUMN FROM table1` - -For more help, see `HELP ALTER TABLE'. - -## 2 Rollup - -Rollup can be understood as a physical index structure of Table. ** Physicalization ** is because its data is physically stored independently, and ** indexing ** means that Rollup can adjust column order to increase the hit rate of prefix index, or reduce key column to increase data aggregation. - -Examples are given below. - -Schema of Table 1 is as follows: - -``` -+----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+----------+-------------+------+-------+---------+-------+ -| siteid | int(11) | No | true | 10 | | -| citycode | smallint(6) | No | true | N/A | | -| username | varchar(32) | No | true | | | -| pv | bigint(20) | No | false | 0 | SUM | -| uv | bigint(20) | No | false | 0 | SUM | -+----------+-------------+------+-------+---------+-------+ -``` - -For table1 detailed data, siteid, citycode and username form a set of keys, which aggregate the PV field. If the business side often has the need to see the total amount of PV in the city, it can build a rollup with only citycode and pv. - -`ALTER TABLE table1 ADD ROLLUP rollup_city(citycode, pv);` - -After successful submission, you can view the progress of the job by following commands: - -`SHOW ALTER TABLE ROLLUP;` - -When the job state is FINISHED, the job is completed. - -When Rollup is established, you can use `DESC table1 ALL'to view the Rollup information of the table. - -``` -mysql> desc table1 all; -+-------------+----------+-------------+------+-------+--------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+-------------+----------+-------------+------+-------+---------+-------+ -| table1 | siteid | int(11) | No | true | 10 | | -| | citycode | smallint(6) | No | true | N/A | | -| | username | varchar(32) | No | true | | | -| | pv | bigint(20) | No | false | 0 | SUM | -| | uv | bigint(20) | No | false | 0 | SUM | -| | | | | | | | -| rollup_city | citycode | smallint(6) | No | true | N/A | | -| | pv | bigint(20) | No | false | 0 | SUM | -+-------------+----------+-------------+------+-------+---------+-------+ -8 rows in set (0.01 sec) -``` - -The following command can be used to cancel the job currently being executed: - -`CANCEL ALTER TABLE ROLLUP FROM table1;` - -After Rollup is established, the query does not need to specify Rollup to query. Or specify the original table for query. The program automatically determines whether Rollup should be used. Whether Rollup is hit or not can be viewed by the `EXPLAIN your_sql;'command. - -For more help, see `HELP ALTER TABLE`. - -## 2 Query of Data Table - -### 2.1 Memory Limitation - -To prevent a user's query from consuming too much memory. Queries are controlled in memory. A query task uses no more than 2GB of memory by default on a single BE node. - -When users use it, if they find a `Memory limit exceeded` error, they usually exceed the memory limit. - -Users should try to optimize their SQL statements when they encounter memory overrun. - -If it is found that 2GB memory cannot be satisfied, the memory parameters can be set manually. - -Display query memory limits: - -``` -mysql> SHOW VARIABLES LIKE "%mem_limit%"; -+---------------+------------+ -| Variable_name | Value | -+---------------+------------+ -| exec_mem_limit| 2147483648 | -+---------------+------------+ -1 row in set (0.00 sec) -``` - -The unit of `exec_mem_limit` is byte, and the value of `exec_mem_limit` can be changed by the `SET` command. If changed to 8GB. - -`SET exec_mem_limit = 8589934592;` - -``` -mysql> SHOW VARIABLES LIKE "%mem_limit%"; -+---------------+------------+ -| Variable_name | Value | -+---------------+------------+ -| exec_mem_limit| 8589934592 | -+---------------+------------+ -1 row in set (0.00 sec) -``` - ->* The above modification is session level and is only valid within the current connection session. Disconnecting and reconnecting will change back to the default value. ->* If you need to modify the global variable, you can set it as follows: `SET GLOBAL exec_mem_limit = 8589934592;` When the setup is complete, disconnect the session and log in again, and the parameters will take effect permanently. - -### 2.2 Query timeout - -The current default query time is set to 300 seconds. If a query is not completed within 300 seconds, the query will be cancelled by the Doris system. Users can use this parameter to customize the timeout time of their applications and achieve a blocking mode similar to wait (timeout). - -View the current timeout settings: - -``` -mysql> SHOW VARIABLES LIKE "%query_timeout%"; -+---------------+-------+ -| Variable_name | Value | -+---------------+-------+ -| QUERY_TIMEOUT | 300 | -+---------------+-------+ -1 row in set (0.00 sec) -``` - -Modify the timeout to 1 minute: - -`SET query timeout =60;` - ->* The current timeout check interval is 5 seconds, so timeouts less than 5 seconds are not very accurate. ->* The above modifications are also session level. Global validity can be modified by `SET GLOBAL`. - -### 2.3 Broadcast/Shuffle Join - -By default, the system implements Join by conditionally filtering small tables, broadcasting them to the nodes where the large tables are located, forming a memory Hash table, and then streaming out the data of the large tables Hash Join. However, if the amount of data filtered by small tables can not be put into memory, Join will not be able to complete at this time. The usual error should be caused by memory overrun first. - -If you encounter the above situation, it is recommended to use Shuffle Join, also known as Partitioned Join. That is, small and large tables are Hash according to Join's key, and then distributed Join. This memory consumption is allocated to all computing nodes in the cluster. - -Use Broadcast Join (default): - -``` -mysql> select sum(table1.pv) from table1 join table2 where table1.siteid = 2; -+--------------------+ -| sum(`table1`.`pv`) | -+--------------------+ -| 10 | -+--------------------+ -1 row in set (0.20 sec) -``` - -Use Broadcast Join (explicitly specified): - -``` -mysql> select sum(table1.pv) from table1 join [broadcast] table2 where table1.siteid = 2; -+--------------------+ -| sum(`table1`.`pv`) | -+--------------------+ -| 10 | -+--------------------+ -1 row in set (0.20 sec) -``` - -Shuffle Join: - -``` -mysql> select sum(table1.pv) from table1 join [shuffle] table2 where table1.siteid = 2; -+--------------------+ -| sum(`table1`.`pv`) | -+--------------------+ -| 10 | -+--------------------+ -1 row in set (0.15 sec) -``` - -### 2.4 Query Retry and High Availability - -When multiple FE nodes are deployed, users can deploy load balancing layers on top of multiple FEs to achieve high availability of Doris. - -Here are some highly available solutions: - -**The first** - -I retry and load balancing in application layer code. For example, if a connection is found to be dead, it will automatically retry on other connections. Application-level code retry requires the application to configure multiple Doris front-end node addresses. - -**Second** - -If you use MySQL JDBC connector to connect Doris, you can use jdbc's automatic retry mechanism: - -``` -jdbc:mysql:/[host:port],[host:port].../[database][? propertyName1][=propertyValue1][&propertyName2][=propertyValue2]... -``` - -**The third** - -Applications can connect to and deploy MySQL Proxy on the same machine by configuring MySQL Proxy's Failover and Load Balance functions. - -`http://dev.mysql.com/doc/refman/5.6/en/mysql-proxy-using.html` \ No newline at end of file diff --git a/docs/documentation/en/getting-started/basic-usage_EN.md b/docs/documentation/en/getting-started/basic-usage_EN.md deleted file mode 100644 index dbabc6bda2e409..00000000000000 --- a/docs/documentation/en/getting-started/basic-usage_EN.md +++ /dev/null @@ -1,375 +0,0 @@ - - - -# Guidelines for Basic Use - -Doris uses MySQL protocol to communicate. Users can connect to Doris cluster through MySQL client or MySQL JDBC. When selecting the MySQL client version, it is recommended to use the version after 5.1, because user names of more than 16 characters can not be supported before 5.1. This paper takes MySQL client as an example to show users the basic usage of Doris through a complete process. - -## 1 Create Users - -### 1.1 Root User Logon and Password Modification - -Doris has built-in root and admin users, and the password is empty by default. After starting the Doris program, you can connect to the Doris cluster through root or admin users. -Use the following command to log in to Doris: - -``` -mysql -h FE_HOST -P9030 -uroot -``` - ->` fe_host` is the IP address of any FE node. ` 9030 ` is the query_port configuration in fe.conf. - -After login, you can modify the root password by following commands - -``` -SET PASSWORD FOR 'root' = PASSWORD('your_password'); -``` - -### 1.3 Creating New Users - -Create an ordinary user with the following command. - -``` -CREATE USER 'test' IDENTIFIED BY 'test_passwd'; -``` - -Follow-up login can be done through the following connection commands. - -``` -mysql -h FE_HOST -P9030 -utest -ptest_passwd -``` - -> By default, the newly created common user does not have any permissions. Permission grants can be referred to later permission grants. - -## 2 Data Table Creation and Data Import - -### 2.1 Create a database - -Initially, a database can be created through root or admin users: - -`CREATE DATABASE example_db;` - -> All commands can use'HELP command;'to see detailed grammar help. For example: `HELP CREATE DATABASE;'` - -> If you don't know the full name of the command, you can use "help command a field" for fuzzy query. If you type'HELP CREATE', you can match commands like `CREATE DATABASE', `CREATE TABLE', `CREATE USER', etc. - -After the database is created, you can view the database information through `SHOW DATABASES'. - -``` -MySQL> SHOW DATABASES; -+--------------------+ -| Database | -+--------------------+ -| example_db | -| information_schema | -+--------------------+ -2 rows in set (0.00 sec) -``` - -Information_schema exists to be compatible with MySQL protocol. In practice, information may not be very accurate. Therefore, information about specific databases is suggested to be obtained by directly querying the corresponding databases. - -### 2.2 Account Authorization - -After the example_db is created, the read and write permissions of example_db can be authorized to ordinary accounts, such as test, through the root/admin account. After authorization, the example_db database can be operated by logging in with the test account. - -`GRANT ALL ON example_db TO test;` - -### 2.3 Formulation - -Create a table using the `CREATE TABLE'command. More detailed parameters can be seen: - -`HELP CREATE TABLE;` - -First switch the database: - -`USE example_db;` - -Doris supports single partition and composite partition. - -In the composite partition: - -* The first level is called Partition, or partition. Users can specify a dimension column as a partition column (currently only integer and time type columns are supported), and specify the range of values for each partition. - -* The second stage is called Distribution, or bucket division. Users can specify one or more dimension columns and the number of buckets for HASH distribution of data. - -Composite partitioning is recommended for the following scenarios - -* There are time dimensions or similar dimensions with ordered values, which can be used as partition columns. The partition granularity can be evaluated according to the frequency of importation and the amount of partition data. -* Historic data deletion requirements: If there is a need to delete historical data (for example, only the last N days of data are retained). Using composite partitions, you can achieve this by deleting historical partitions. Data can also be deleted by sending a DELETE statement within a specified partition. -* Solve the data skew problem: Each partition can specify the number of buckets separately. If dividing by day, when the amount of data varies greatly every day, we can divide the data of different partitions reasonably by the number of buckets in the specified partition. Bucket columns recommend choosing columns with high degree of differentiation. - -Users can also use no composite partitions, even single partitions. Then the data are only distributed by HASH. - -Taking the aggregation model as an example, the following two partitions are illustrated separately. - -#### Single partition - -Create a logical table with the name table1. The number of barrels is 10. - -The schema of this table is as follows: - -* Siteid: Type is INT (4 bytes), default value is 10 -* citycode: The type is SMALLINT (2 bytes) -* username: The type is VARCHAR, the maximum length is 32, and the default value is an empty string. -* pv: Type is BIGINT (8 bytes), default value is 0; this is an index column, Doris will aggregate the index column internally, the aggregation method of this column is SUM. - -The TABLE statement is as follows: -``` -CREATE TABLE table1 -( - siteid INT DEFAULT '10', - citycode SMALLINT, - username VARCHAR(32) DEFAULT '', - pv BIGINT SUM DEFAULT '0' -) -AGGREGATE KEY(siteid, citycode, username) -DISTRIBUTED BY HASH(siteid) BUCKETS 10 -PROPERTIES("replication_num" = "1"); -``` - -#### Composite partition - -Create a logical table named table2. - -The schema of this table is as follows: - -* event_day: Type DATE, no default -* Siteid: Type is INT (4 bytes), default value is 10 -* citycode: The type is SMALLINT (2 bytes) -* username: The type is VARCHAR, the maximum length is 32, and the default value is an empty string. -* pv: Type is BIGINT (8 bytes), default value is 0; this is an index column, Doris will aggregate the index column internally, the aggregation method of this column is SUM. - -We use the event_day column as the partition column to create three partitions: p201706, p201707, and p201708. - -* p201706: Range [Minimum, 2017-07-01) -* p201707: Scope [2017-07-01, 2017-08-01) -* p201708: Scope [2017-08-01, 2017-09-01) - -> Note that the interval is left closed and right open. - -Each partition uses siteid to hash buckets, with a bucket count of 10 - -The TABLE statement is as follows: -``` -CREATE TABLE table2 -( - event_day DATE, - siteid INT DEFAULT '10', - citycode SMALLINT, - username VARCHAR(32) DEFAULT '', - pv BIGINT SUM DEFAULT '0' -) -AGGREGATE KEY(event_day, siteid, citycode, username) -PARTITION BY RANGE(event_day) -( - PARTITION p201706 VALUES LESS THAN ('2017-07-01'), - PARTITION p201707 VALUES LESS THAN ('2017-08-01'), - PARTITION p201708 VALUES LESS THAN ('2017-09-01') -) -DISTRIBUTED BY HASH(siteid) BUCKETS 10 -PROPERTIES("replication_num" = "1"); -``` - -After the table is built, you can view the information of the table in example_db: - -``` -MySQL> SHOW TABLES; -+----------------------+ -| Tables_in_example_db | -+----------------------+ -| table1 | -| table2 | -+----------------------+ -2 rows in set (0.01 sec) - -MySQL> DESC table1; -+----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+----------+-------------+------+-------+---------+-------+ -| siteid | int(11) | Yes | true | 10 | | -| citycode | smallint(6) | Yes | true | N/A | | -| username | varchar(32) | Yes | true | | | -| pv | bigint(20) | Yes | false | 0 | SUM | -+----------+-------------+------+-------+---------+-------+ -4 rows in set (0.00 sec) - -MySQL> DESC table2; -+-----------+-------------+------+-------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+-----------+-------------+------+-------+---------+-------+ -| event_day | date | Yes | true | N/A | | -| siteid | int(11) | Yes | true | 10 | | -| citycode | smallint(6) | Yes | true | N/A | | -| username | varchar(32) | Yes | true | | | -| pv | bigint(20) | Yes | false | 0 | SUM | -+-----------+-------------+------+-------+---------+-------+ -5 rows in set (0.00 sec) -``` - -> Notes: -> -> 1. By setting replication_num, the above tables are all single-copy tables. Doris recommends that users adopt the default three-copy settings to ensure high availability. -> 2. Composite partition tables can be added or deleted dynamically. See the Partition section in `HELP ALTER TABLE`. -> 3. Data import can import the specified Partition. See `HELP LOAD'. -> 4. Schema of table can be dynamically modified. -> 5. Rollup can be added to Table to improve query performance. This section can be referred to the description of Rollup in Advanced Usage Guide. -> 6. The default value of Null property for column is true, which may result in poor scan performance. - -### 2.4 Import data - -Doris supports a variety of data import methods. Specifically, you can refer to the data import document. Here we use streaming import and Broker import as examples. - -#### Flow-in - -Streaming import transfers data to Doris via HTTP protocol. It can import local data directly without relying on other systems or components. Detailed grammar help can be found in `HELP STREAM LOAD;' - -Example 1: With "table1_20170707" as Label, import table1 tables using the local file table1_data. - -``` -curl --location-trusted -u test:test -H "label:table1_20170707" -H "column_separator:," -T table1_data http://FE_HOST:8030/api/example_db/table1/_stream_load -``` - -> 1. FE_HOST is the IP of any FE node and 8030 is http_port in fe.conf. -> 2. You can use the IP of any BE and the webserver_port in be.conf to connect the target left and right for import. For example: `BE_HOST:8040` - -The local file `table1_data` takes `,` as the separation between data, and the specific contents are as follows: - -``` -1,1,Jim,2 -2,1,grace,2 -3,2,tom,2 -4,3,bush,3 -5,3,helen,3 -``` - -Example 2: With "table2_20170707" as Label, import table2 tables using the local file table2_data. - -``` -curl --location-trusted -u test:test -H "label:table2_20170707" -H "column_separator:," -T table1_data http://127.0.0.1:8030/api/example_db/table2/_stream_load -``` - -The local file `table2_data'is separated by `t'. The details are as follows: - -``` -2017-07-03 1 1 jim 2 -2017-07-05 2 1 grace 2 -2017-07-12 3 2 tom 2 -2017-07-15 4 3 bush 3 -2017-07-12 5 3 helen 3 -``` - -> Notes: -> -> 1. The recommended file size for streaming import is limited to 10GB. Excessive file size will result in higher cost of retry failure. -> 2. Each batch of imported data needs to take a Label. Label is best a string related to a batch of data for easy reading and management. Doris based on Label guarantees that the same batch of data can be imported only once in a database. Label for failed tasks can be reused. -> 3. Streaming imports are synchronous commands. The successful return of the command indicates that the data has been imported, and the failure of the return indicates that the batch of data has not been imported. - -#### Broker Load - -Broker imports import data from external storage through deployed Broker processes. For more help, see `HELP BROKER LOAD;` - -Example: Import files on HDFS into table1 table with "table1_20170708" as Label - -``` -LOAD LABEL table1_20170708 -( - DATA INFILE("hdfs://your.namenode.host:port/dir/table1_data") - INTO TABLE table1 -) -WITH BROKER hdfs -( - "username"="hdfs_user", - "password"="hdfs_password" -) -PROPERTIES -( - "timeout"="3600", - "max_filter_ratio"="0.1" -); -``` - -Broker imports are asynchronous commands. Successful execution of the above commands only indicates successful submission of tasks. Successful imports need to be checked through `SHOW LOAD;' Such as: - -`SHOW LOAD WHERE LABLE = "table1_20170708";` - -In the return result, FINISHED in the `State'field indicates that the import was successful. - -For more instructions on `SHOW LOAD`, see` HELP SHOW LOAD; ` - -Asynchronous import tasks can be cancelled before the end: - -`CANCEL LOAD WHERE LABEL = "table1_20170708";` - -## 3 Data query - -### 3.1 Simple Query - -Examples: - -``` -MySQL> SELECT * FROM table1 LIMIT 3; -+--------+----------+----------+------+ -| siteid | citycode | username | pv | -+--------+----------+----------+------+ -| 2 | 1 | 'grace' | 2 | -| 5 | 3 | 'helen' | 3 | -| 3 | 2 | 'tom' | 2 | -+--------+----------+----------+------+ -5 rows in set (0.01 sec) - -MySQL> SELECT * FROM table1 ORDER BY citycode; -+--------+----------+----------+------+ -| siteid | citycode | username | pv | -+--------+----------+----------+------+ -| 2 | 1 | 'grace' | 2 | -| 1 | 1 | 'jim' | 2 | -| 3 | 2 | 'tom' | 2 | -| 4 | 3 | 'bush' | 3 | -| 5 | 3 | 'helen' | 3 | -+--------+----------+----------+------+ -5 rows in set (0.01 sec) -``` - -### 3.3 Join Query - -Examples: - -``` -MySQL> SELECT SUM(table1.pv) FROM table1 JOIN table2 WHERE table1.siteid = table2.siteid; -+--------------------+ -| sum(`table1`.`pv`) | -+--------------------+ -| 12 | -+--------------------+ -1 row in set (0.20 sec) -``` - -### 3.4 Subquery - -Examples: - -``` -MySQL> SELECT SUM(pv) FROM table2 WHERE siteid IN (SELECT siteid FROM table1 WHERE siteid > 2); -+-----------+ -| sum(`pv`) | -+-----------+ -| 8 | -+-----------+ -1 row in set (0.13 sec) -``` diff --git a/docs/documentation/en/getting-started/best-practice_EN.md b/docs/documentation/en/getting-started/best-practice_EN.md deleted file mode 100644 index 9bbf65b0c86977..00000000000000 --- a/docs/documentation/en/getting-started/best-practice_EN.md +++ /dev/null @@ -1,184 +0,0 @@ - - - -# Best Practices - -## 1 tabulation - -### 1.1 Data Model Selection - -Doris data model is currently divided into three categories: AGGREGATE KEY, UNIQUE KEY, DUPLICATE KEY. Data in all three models are sorted by KEY. - -1.1.1. AGGREGATE KEY - -When AGGREGATE KEY is the same, old and new records are aggregated. The aggregation functions currently supported are SUM, MIN, MAX, REPLACE. - -AGGREGATE KEY model can aggregate data in advance and is suitable for reporting and multi-dimensional analysis business. - -``` -CREATE TABLE site_visit -( -siteid INT, -City: SMALLINT, -username VARCHAR (32), -pv BIGINT SUM DEFAULT '0' -) -AGGREGATE KEY(siteid, city, username) -DISTRIBUTED BY HASH(siteid) BUCKETS 10; -``` - -1.1.2. KEY UNIQUE - -When UNIQUE KEY is the same, the new record covers the old record. At present, UNIQUE KEY implements the same RPLACE aggregation method as GGREGATE KEY, and they are essentially the same. Suitable for analytical business with updated requirements. - -``` -CREATE TABLE sales_order -( -orderid BIGINT, -status TINYINT, -username VARCHAR (32), -amount BIGINT DEFAULT '0' -) -KEY (orderid) UNIT -DISTRIBUTED BY HASH(orderid) BUCKETS 10; -``` - -1.1.3. DUPLICATE KEY - -Only sort columns are specified, and the same rows are not merged. It is suitable for the analysis business where data need not be aggregated in advance. - -``` -CREATE TABLE session_data -( -visitorid SMALLINT, -sessionid BIGINT, -visit time DATETIME, -City CHAR (20), -province CHAR(20), -ip. varchar (32), -brower CHAR(20), -url: VARCHAR (1024) -) -DUPLICATE KEY (visitor time, session time) -DISTRIBUTED BY HASH(sessionid, visitorid) BUCKETS 10; -``` - -### 1.2 Wide Table vs. Star Schema - -In order to adapt to the front-end business, business side often does not distinguish dimension information from indicator information, but defines Schema as a wide table. For Doris, the performance of such wide gauges is often unsatisfactory: - -* There are many fields in Schema, and there may be more key columns in the aggregation model. The number of columns that need to be sorted in the import process will increase. -* Dimensional information updates are reflected in the whole table, and the frequency of updates directly affects the efficiency of queries. - -In the process of using Star Schema, users are advised to use Star Schema to distinguish dimension tables from indicator tables as much as possible. Frequently updated dimension tables can also be placed in MySQL external tables. If there are only a few updates, they can be placed directly in Doris. When storing dimension tables in Doris, more copies of dimension tables can be set up to improve Join's performance. - -### 1.3 Partitioning and Bucketing - -Doris supports two-level partitioned storage. The first layer is RANGE partition and the second layer is HASH bucket. - -1.3.1. RANGE Partitioning - -The RANGE partition is used to divide data into different intervals, which can be logically understood as dividing the original table into multiple sub-tables. In business, most users will choose to partition on time, which has the following advantages: - -* Differentiable heat and cold data -* Availability of Doris Hierarchical Storage (SSD + SATA) -* Delete data by partition more quickly - -1.3.2. Hash Bucketing - -The data is divided into different buckets according to the hash value. - -* It is suggested that columns with large differentiation should be used as buckets to avoid data skew. -* In order to facilitate data recovery, it is suggested that the size of a single bucket should not be too large and should be kept within 10GB. Therefore, the number of buckets should be considered reasonably when building tables or increasing partitions, among which different partitions can specify different buckets. - -### 1.4 Sparse Index and Bloom Filter - -Doris stores the data in an orderly manner, and builds a sparse index for Doris on the basis of ordered data. The index granularity is block (1024 rows). - -Sparse index chooses fixed length prefix in schema as index content, and Doris currently chooses 36 bytes prefix as index. - -* When building tables, it is suggested that the common filter fields in queries should be placed in front of Schema. The more distinguishable the query fields are, the more frequent the query fields are. -* One particular feature of this is the varchar type field. The varchar type field can only be used as the last field of the sparse index. The index is truncated at varchar, so if varchar appears in front, the length of the index may be less than 36 bytes. Specifically, you can refer to [data model, ROLLUP and prefix index] (. / data-model-rollup. md). -* In addition to sparse index, Doris also provides bloomfilter index. Bloomfilter index has obvious filtering effect on columns with high discrimination. If you consider that varchar cannot be placed in a sparse index, you can create a bloomfilter index. - -### 1.5 Physical and Chemical View (rollup) - -Rollup can essentially be understood as a physical index of the original table. When creating Rollup, only some columns in Base Table can be selected as Schema. The order of fields in Schema can also be different from that in Base Table. - -Rollup can be considered in the following cases: - -1.5.1. Low ratio of data aggregation in the Base Table - -This is usually due to the fact that Base Table has more differentiated fields. At this point, you can consider selecting some columns and establishing Rollup. - -For the `site_visit'table: - -``` -site -u visit (siteid, city, username, pv) -``` - -Siteid may lead to a low degree of data aggregation. If business parties often base their PV needs on city statistics, they can build a city-only, PV-based ollup: - -``` -ALTER TABLE site_visit ADD ROLLUP rollup_city(city, pv); -``` - -1.5.2. The prefix index in Base Table cannot be hit - -Generally, the way Base Table is constructed cannot cover all query modes. At this point, you can consider adjusting the column order and establishing Rollup. - -Database Session - -``` -session -u data (visitorid, sessionid, visittime, city, province, ip, browser, url) -``` - -In addition to visitorid analysis, there are Brower and province analysis cases, Rollup can be established separately. - -``` -ALTER TABLE session_data ADD ROLLUP rollup_brower(brower,province,ip,url) DUPLICATE KEY(brower,province); -``` - -## 2 Schema Change - -There are three Schema Change in doris：Sorted Schema Change，Direct Schema Change, Linked Schema Change。 - -2.1. Sorted Schema Change - -The sorting of columns has been changed and the data needs to be reordered. For example, delete a column in a sorted column and reorder the fields. - -``` -ALTER TABLE site_visit DROP COLUMN city; -``` - -2.2. Direct Schema Change: There is no need to reorder, but there is a need to convert the data. For example, modify - the type of column, add a column to the sparse index, etc. - -``` -ALTER TABLE site_visit MODIFY COLUMN username varchar(64); -``` - -2.3. Linked Schema Change: No need to transform data, for example add columns. - -``` -ALTER TABLE site_visit ADD COLUMN click bigint SUM default '0'; -``` - -Schema is recommended to be considered when creating tables so that Schema can be changed more quickly. diff --git a/docs/documentation/en/getting-started/data-model-rollup_EN.md b/docs/documentation/en/getting-started/data-model-rollup_EN.md deleted file mode 100644 index 2ac0a8e1b89a57..00000000000000 --- a/docs/documentation/en/getting-started/data-model-rollup_EN.md +++ /dev/null @@ -1,630 +0,0 @@ - - -# Data Model, ROLLUP and Prefix Index - -This document describes Doris's data model, ROLLUP and prefix index concepts at the logical level to help users better use Doris to cope with different business scenarios. - -## Basic concepts - -In Doris, data is logically described in the form of tables. -A table consists of rows and columns. Row is a row of user data. Column is used to describe different fields in a row of data. - -Columns can be divided into two categories: Key and Value. From a business perspective, Key and Value can correspond to dimension columns and indicator columns, respectively. - -Doris's data model is divided into three main categories: - -* Aggregate -* Uniq -* Duplicate - -Let's introduce them separately. - -## Aggregate Model - -We illustrate what aggregation model is and how to use it correctly with practical examples. - -### Example 1: Importing data aggregation - -Assume that the business has the following data table schema: - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -| userid | LARGEINT | | user id| -| date | DATE | | date of data filling| -| City | VARCHAR (20) | | User City| -| age | SMALLINT | | User age| -| sex | TINYINT | | User gender| -| Last_visit_date | DATETIME | REPLACE | Last user access time| -| Cost | BIGINT | SUM | Total User Consumption| -| max dwell time | INT | MAX | Maximum user residence time| -| min dwell time | INT | MIN | User minimum residence time| - -If converted into a table-building statement, the following is done (omitting the Partition and Distribution information in the table-building statement) - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( - `user_id` LARGEINT NOT NULL COMMENT "user id", - `date` DATE NOT NULL COMMENT "data import time", - `city` VARCHAR(20) COMMENT "city", - `age` SMALLINT COMMENT "age", - `sex` TINYINT COMMENT "gender", - `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "last visit date time", - `cost` BIGINT SUM DEFAULT "0" COMMENT "user total cost", - `max_dwell_time` INT MAX DEFAULT "0" COMMENT "user max dwell time", - `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "user min dwell time", -) -AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) -... /* ignore Partition and Distribution */ -; -``` - -As you can see, this is a typical fact table of user information and access behavior. -In general star model, user information and access behavior are stored in dimension table and fact table respectively. Here, in order to explain Doris's data model more conveniently, we store the two parts of information in a single table. - -The columns in the table are divided into Key (dimension column) and Value (indicator column) according to whether `AggregationType`is set or not. No `AggregationType`, such as `user_id`, `date`, `age`, etc., is set as **Key**, while AggregationType'is set as **Value**. - -When we import data, the same rows and aggregates into one row for the Key column, while the Value column aggregates according to the set `AggregationType`. `AggregationType`currently has the following four ways of aggregation: - -1. SUM: Sum, multi-line Value accumulation. -2. REPLACE: Instead, Values in the next batch of data will replace Values in rows previously imported. -3. MAX: Keep the maximum. -4. MIN: Keep the minimum. - -Suppose we have the following imported data (raw data): - -|user\_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 06:00 | 20 | 10 | 10| -| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 07:00 | 15 | 2 | 2| -| 10001 | 2017-10-01 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| -| 10002 | 2017-10-02 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| -| 10003 | 2017-10-02 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| -| 10004 | 2017-10-01 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| -| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| - -Let's assume that this is a table that records the user's behavior in accessing a commodity page. Let's take the first row of data as an example and explain it as follows: - -| Data | Description| -|---|---| -| 10000 | User id, each user uniquely identifies id| -| 2017-10-01 | Data storage time, accurate to date| -| Beijing | User City| -| 20 | User Age| -| 0 | Gender male (1 for female)| -| 2017-10-01 06:00 | User's time to visit this page, accurate to seconds| -| 20 | Consumption generated by the user's current visit| -| 10 | User's visit, time to stay on the page| -| 10 | User's current visit, time spent on the page (redundancy)| - -Then when this batch of data is imported into Doris correctly, the final storage in Doris is is as follows: - -|user\_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 07:00 | 35 | 10 | 2| -| 10001 | 2017-10-01 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| -| 10002 | 2017-10-02 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| -| 10003 | 2017-10-02 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| -| 10004 | 2017-10-01 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| -| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| - -As you can see, there is only one line of aggregated data left for 10,000 users. The data of other users are consistent with the original data. Here we first explain the aggregated data of user 10000: - -The first five columns remain unchanged, starting with column 6 `last_visit_date': - -*`2017-10-01 07:00`: Because the `last_visit_date`column is aggregated by REPLACE, the `2017-10-01 07:00` column has been replaced by `2017-10-01 06:00'. -> Note: For data in the same import batch, the order of replacement is not guaranteed for the aggregation of REPLACE. For example, in this case, it may be `2017-10-01 06:00'. For data from different imported batches, it can be guaranteed that the data from the latter batch will replace the former batch. - -*`35`: Because the aggregation type of the `cost'column is SUM, 35 is accumulated from 20 + 15. -*`10`: Because the aggregation type of the`max_dwell_time'column is MAX, 10 and 2 take the maximum and get 10. -*`2`: Because the aggregation type of `min_dwell_time'column is MIN, 10 and 2 take the minimum value and get 2. - -After aggregation, Doris ultimately only stores aggregated data. In other words, detailed data will be lost and users can no longer query the detailed data before aggregation. - -### Example 2: Keep detailed data - -Following example 1, we modify the table structure as follows: - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -| userid | LARGEINT | | user id| -| date | DATE | | date of data filling| -| Time stamp | DATETIME | | Data filling time, accurate to seconds| -| City | VARCHAR (20) | | User City| -| age | SMALLINT | | User age| -| sex | TINYINT | | User gender| -| Last visit date | DATETIME | REPLACE | Last user access time| -| Cost | BIGINT | SUM | Total User Consumption| -| max dwell time | INT | MAX | Maximum user residence time| -| min dwell time | INT | MIN | User minimum residence time| - -That is to say, a column of `timestamp'has been added to record the data filling time accurate to seconds. - -The imported data are as follows: - -|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---|---| -| 10000 | 2017-10-01 | 2017-10-01 08:00:05 | Beijing | 20 | 0 | 2017-10-01 06:00 | 20 | 10 | 10| -| 10000 | 2017-10-01 | 2017-10-01 09:00:05 | Beijing | 20 | 0 | 2017-10-01 07:00 | 15 | 2 | 2| -| 10001 | 2017-10-01 | 2017-10-01 18:12:10 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| -| 10002 | 2017-10-02 | 2017-10-02 13:10:00 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| -| 10003 | 2017-10-02 | 2017-10-02 13:15:00 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| -| 10004 | 2017-10-01 | 2017-10-01 12:12:48 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| -| 10004 | 2017-10-03 | 2017-10-03 12:38:20 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| - -Then when this batch of data is imported into Doris correctly, the final storage in Doris is is as follows: - -|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---|---| -| 10000 | 2017-10-01 | 2017-10-01 08:00:05 | Beijing | 20 | 0 | 2017-10-01 06:00 | 20 | 10 | 10| -| 10000 | 2017-10-01 | 2017-10-01 09:00:05 | Beijing | 20 | 0 | 2017-10-01 07:00 | 15 | 2 | 2| -| 10001 | 2017-10-01 | 2017-10-01 18:12:10 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| -| 10002 | 2017-10-02 | 2017-10-02 13:10:00 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| -| 10003 | 2017-10-02 | 2017-10-02 13:15:00 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| -| 10004 | 2017-10-01 | 2017-10-01 12:12:48 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| -| 10004 | 2017-10-03 | 2017-10-03 12:38:20 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| - -We can see that the stored data, just like the imported data, does not aggregate at all. This is because, in this batch of data, because the `timestamp'column is added, the Keys of all rows are **not exactly the same**. That is, as long as the keys of each row are not identical in the imported data, Doris can save the complete detailed data even in the aggregation model. - -### Example 3: Importing data and aggregating existing data - -Take Example 1. Suppose that the data in the table are as follows: - -|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 07:00 | 35 | 10 | 2| -| 10001 | 2017-10-01 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| -| 10002 | 2017-10-02 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| -| 10003 | 2017-10-02 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| -| 10004 | 2017-10-01 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| -| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| - -We imported a new batch of data: - -|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 11:22:00 | 44 | 19 | 19| -| 10005 | 2017-10-03 | Changsha | 29 | 1 | 2017-10-03 18:11:02 | 3 | 1 | 1| - -Then when this batch of data is imported into Doris correctly, the final storage in Doris is is as follows: - -|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---| -| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 07:00 | 35 | 10 | 2| -| 10001 | 2017-10-01 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| -| 10002 | 2017-10-02 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| -| 10003 | 2017-10-02 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| -| 10004 | 2017-10-01 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| -| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 11:22:00 | 55 | 19 | 6| -| 10005 | 2017-10-03 | Changsha | 29 | 1 | 2017-10-03 18:11:02 | 3 | 1 | 1| - -As you can see, the existing data and the newly imported data of user 10004 have been aggregated. At the same time, 10005 new users'data were added. - -Data aggregation occurs in Doris in the following three stages: - -1. The ETL stage of data import for each batch. This phase aggregates data within each batch of imported data. -2. The stage in which the underlying BE performs data Compaction. At this stage, BE aggregates data from different batches that have been imported. -3. Data query stage. In data query, the data involved in the query will be aggregated accordingly. - -Data may be aggregated to varying degrees at different times. For example, when a batch of data is just imported, it may not be aggregated with the existing data. But for users, user**can only query aggregated data**. That is, different degrees of aggregation are transparent to user queries. Users should always assume that data exists in terms of the degree of aggregation that **ultimately completes**, and **should not assume that some aggregation has not yet occurred**. (See the section **Limitations of the aggregation model** for more details.) - -## Uniq Model - -In some multi-dimensional analysis scenarios, users are more concerned with how to ensure the uniqueness of Key, that is, how to obtain the Primary Key uniqueness constraint. Therefore, we introduce Uniq's data model. This model is essentially a special case of aggregation model and a simplified representation of table structure. Let's give an example. - -|ColumnName|Type|IsKey|Comment| -|---|---|---|---| -| user_id | BIGINT | Yes | user id| -| username | VARCHAR (50) | Yes | User nickname| -| City | VARCHAR (20) | No | User City| -| age | SMALLINT | No | User Age| -| sex | TINYINT | No | User Gender| -| Phone | LARGEINT | No | User Phone| -| address | VARCHAR (500) | No | User Address| -| register_time | DATETIME | No | user registration time| - -This is a typical user base information table. There is no aggregation requirement for this type of data, just the uniqueness of the primary key is guaranteed. (The primary key here is user_id + username). Then our statement is as follows: - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( -`user_id` LARGEINT NOT NULL COMMENT "用户id", -"username" VARCHAR (50) NOT NULL COMMENT "25143;" 261651;" -` City `VARCHAR (20) COMMENT `User City', -"Age" SMALLINT COMMENT "29992;" 25143;"24180;" 40836 ", -`sex` TINYINT COMMENT "用户性别", -`phone` LARGEINT COMMENT "用户电话", -'address ` VARCHAR (500) COMMENT'25143;', -'register 'or'time' DATETIME COMMENT "29992;" 25143;"27880;" 20876;"26102;" 38388;" -) -Unique Key ("User", "User", "Name") -... /* 省略 Partition 和 Distribution 信息 */ -; -``` - -This table structure is exactly the same as the following table structure described by the aggregation model: - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -| user_id | BIGINT | | user id| -| username | VARCHAR (50) | | User nickname| -| City | VARCHAR (20) | REPLACE | User City| -| age | SMALLINT | REPLACE | User Age| -| sex | TINYINT | REPLACE | User Gender| -| Phone | LARGEINT | REPLACE | User Phone| -| address | VARCHAR (500) | REPLACE | User Address| -| register_time | DATETIME | REPLACE | User registration time| - -And table-building statements: - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( -`user_id` LARGEINT NOT NULL COMMENT "用户id", -"username" VARCHAR (50) NOT NULL COMMENT "25143;" 261651;" -` City `VARCHAR (20) REPLACE COMMENT `User City', -What do you say when you are young? -`sex` TINYINT REPLACE COMMENT "用户性别", -"phone" LARGEINT REPLACE COMMENT "25143;" -`address` VARCHAR(500) REPLACE COMMENT "用户地址", -'register 'or'time' DATETIME REPLACE COMMENT "29992;" 25143;"27880;" 20876;"26102;" -) -AGGREGATE KEY(`user_id`, `user_name`) -... /* 省略 Partition 和 Distribution 信息 */ -; -``` - -That is to say, Uniq model can be completely replaced by REPLACE in aggregation model. Its internal implementation and data storage are exactly the same. No further examples will be given here. - -## Duplicate Model - -In some multidimensional analysis scenarios, data has neither primary keys nor aggregation requirements. Therefore, we introduce Duplicate data model to meet this kind of demand. Examples are given. - -|ColumnName|Type|SortKey|Comment| -|---|---|---|---| -| Timstamp | DATETIME | Yes | Logging Time| -| Type | INT | Yes | Log Type| -|error_code|INT|Yes|error code| -| Error_msg | VARCHAR (1024) | No | Error Details| -|op_id|BIGINT|No|operator id| -|op_time|DATETIME|No|operation time| - -The TABLE statement is as follows: -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( -`timestamp` DATETIME NOT NULL COMMENT "日志时间", -`type` INT NOT NULL COMMENT "日志类型", -"Error"\\\\\\\\\\\\\ -`error_msg` VARCHAR(1024) COMMENT "错误详细信息", -`op_id` BIGINT COMMENT "负责人id", -OP `op `time ` DATETIME COMMENT "22788;" 29702;"26102;" 388;" -) -DUPLICATE KEY(`timestamp`, `type`) -... /* 省略 Partition 和 Distribution 信息 */ -; -``` - -This data model is different from Aggregate and Uniq models. Data is stored entirely in accordance with the data in the imported file, without any aggregation. Even if the two rows of data are identical, they will be retained. -The DUPLICATE KEY specified in the table building statement is only used to specify which columns the underlying data is sorted according to. (The more appropriate name should be "Sorted Column", where the name "DUPLICATE KEY" is used to specify the data model used. For more explanations of "Sorted Column", see the section ** Prefix Index **. On the choice of DUPLICATE KEY, we recommend that the first 2-4 columns be selected appropriately. - -This data model is suitable for storing raw data without aggregation requirements and primary key uniqueness constraints. For more usage scenarios, see the ** Limitations of the Aggregation Model ** section. - -## ROLLUP - -ROLLUP in multidimensional analysis means "scroll up", which means that data is aggregated further at a specified granularity. - -### Basic concepts - -In Doris, we make the table created by the user through the table building statement a Base table. Base table holds the basic data stored in the way specified by the user's table-building statement. - -On top of the Base table, we can create any number of ROLLUP tables. These ROLLUP data are generated based on the Base table and physically **stored independently**. - -The basic function of ROLLUP tables is to obtain coarser aggregated data on the basis of Base tables. - -Let's illustrate the ROLLUP tables and their roles in different data models with examples. - -#### ROLLUP in Aggregate Model and Uniq Model - -Because Uniq is only a special case of the Aggregate model, we do not distinguish it here. - -Example 1: Get the total consumption per user - -Following **Example 2** in the **Aggregate Model** section, the Base table structure is as follows: - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -| user_id | LARGEINT | | user id| -| date | DATE | | date of data filling| -| Time stamp | DATETIME | | Data filling time, accurate to seconds| -| City | VARCHAR (20) | | User City| -| age | SMALLINT | | User age| -| sex | TINYINT | | User gender| -| Last_visit_date | DATETIME | REPLACE | Last user access time| -| Cost | BIGINT | SUM | Total User Consumption| -| max dwell time | INT | MAX | Maximum user residence time| -| min dwell time | INT | MIN | User minimum residence time| - -The data stored are as follows: - -|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---|---|---|---|---|---| -| 10000 | 2017-10-01 | 2017-10-01 08:00:05 | Beijing | 20 | 0 | 2017-10-01 06:00 | 20 | 10 | 10| -| 10000 | 2017-10-01 | 2017-10-01 09:00:05 | Beijing | 20 | 0 | 2017-10-01 07:00 | 15 | 2 | 2| -| 10001 | 2017-10-01 | 2017-10-01 18:12:10 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| -| 10002 | 2017-10-02 | 2017-10-02 13:10:00 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| -| 10003 | 2017-10-02 | 2017-10-02 13:15:00 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| -| 10004 | 2017-10-01 | 2017-10-01 12:12:48 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| -| 10004 | 2017-10-03 | 2017-10-03 12:38:20 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| - -On this basis, we create a ROLLUP: - -|ColumnName| -|---| -|user_id| -|cost| - -The ROLLUP contains only two columns: user_id and cost. After the creation, the data stored in the ROLLUP is as follows: - -|user\_id|cost| -|---|---| -|10000|35| -|10001|2| -|10002|200| -|10003|30| -|10004|111| - -As you can see, ROLLUP retains only the results of SUM on the cost column for each user_id. So when we do the following query: - -`SELECT user_id, sum(cost) FROM table GROUP BY user_id;` - -Doris automatically hits the ROLLUP table, thus completing the aggregated query by scanning only a very small amount of data. - -2. Example 2: Get the total consumption, the longest and shortest page residence time of users of different ages in different cities - -Follow example 1. Based on the Base table, we create a ROLLUP: - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -| City | VARCHAR (20) | | User City| -| age | SMALLINT | | User age| -| Cost | BIGINT | SUM | Total User Consumption| -| max dwell time | INT | MAX | Maximum user residence time| -| min dwell time | INT | MIN | User minimum residence time| - -After the creation, the data stored in the ROLLUP is as follows: - -|city|age|cost|max\_dwell\_time|min\_dwell\_time| -|---|---|---|---|---| -| Beijing | 20 | 0 | 30 | 10 | 2| -| Beijing | 30 | 1 | 2 | 22 | 22| -| Shanghai | 20 | 1 | 200 | 5 | 5| -| Guangzhou | 32 | 0 | 30 | 11 | 11| -| Shenzhen | 35 | 0 | 111 | 6 | 3| - -When we do the following queries: - -* Select City, Age, Sum (Cost), Max (Max dwell time), min (min dwell time) from table group by City, age;* -* `SELECT city, sum(cost), max(max_dwell_time), min(min_dwell_time) FROM table GROUP BY city;` -* `SELECT city, age, sum(cost), min(min_dwell_time) FROM table GROUP BY city, age;` - -Doris automatically hits the ROLLUP table. - -#### OLLUP in Duplicate Model - -Because the Duplicate model has no aggregate semantics. So the ROLLLUP in this model has lost the meaning of "scroll up". It's just to adjust the column order to hit the prefix index. In the next section, we will introduce prefix index in detail, and how to use ROLLUP to change prefix index in order to achieve better query efficiency. - -### Prefix Index and ROLLUP - -#### prefix index - -Unlike traditional database design, Doris does not support indexing on any column. OLAP databases based on MPP architecture such as Doris usually handle large amounts of data by improving concurrency. -In essence, Doris's data is stored in a data structure similar to SSTable (Sorted String Table). This structure is an ordered data structure, which can be sorted and stored according to the specified column. In this data structure, it is very efficient to search by sorting columns. - -In Aggregate, Uniq and Duplicate three data models. The underlying data storage is sorted and stored according to the columns specified in AGGREGATE KEY, UNIQ KEY and DUPLICATE KEY in their respective table-building statements. - -The prefix index, which is based on sorting, implements an index method to query data quickly according to a given prefix column. - -We use the prefix index of ** 36 bytes ** of a row of data as the prefix index of this row of data. When a VARCHAR type is encountered, the prefix index is truncated directly. We give examples to illustrate: - -1. The prefix index of the following table structure is user_id (8Byte) + age (8Bytes) + message (prefix 20 Bytes). - -|ColumnName|Type| -|---|---| -|user_id|BIGINT| -|age|INT| -|message|VARCHAR(100)| -|max\_dwell\_time|DATETIME| -|min\_dwell\_time|DATETIME| - -2. The prefix index of the following table structure is user_name (20 Bytes). Even if it does not reach 36 bytes, because it encounters VARCHAR, it truncates directly and no longer continues. - -|ColumnName|Type| -|---|---| -|user_name|VARCHAR(20)| -|age|INT| -|message|VARCHAR(100)| -|max\_dwell\_time|DATETIME| -|min\_dwell\_time|DATETIME| - -When our query condition is the prefix of ** prefix index **, it can greatly speed up the query speed. For example, in the first example, we execute the following queries: - -`SELECT * FROM table WHERE user_id=1829239 and age=20；` - -The efficiency of this query is much higher than that of ** the following queries: - -`SELECT * FROM table WHERE age=20；` - -Therefore, when constructing tables, ** correctly choosing column order can greatly improve query efficiency **. - -#### ROLLUP adjusts prefix index - -Because column order is specified when a table is built, there is only one prefix index for a table. This may be inefficient for queries that use other columns that cannot hit prefix indexes as conditions. Therefore, we can manually adjust the order of columns by creating ROLLUP. Examples are given. - -The structure of the Base table is as follows: - -|ColumnName|Type| -|---|---| -|user\_id|BIGINT| -|age|INT| -|message|VARCHAR(100)| -|max\_dwell\_time|DATETIME| -|min\_dwell\_time|DATETIME| - -On this basis, we can create a ROLLUP table: - -|ColumnName|Type| -|---|---| -|age|INT| -|user\_id|BIGINT| -|message|VARCHAR(100)| -|max\_dwell\_time|DATETIME| -|min\_dwell\_time|DATETIME| - -As you can see, the columns of ROLLUP and Base tables are exactly the same, just changing the order of user_id and age. So when we do the following query: - -`SELECT * FROM table where age=20 and massage LIKE "%error%";` - -The ROLLUP table is preferred because the prefix index of ROLLUP matches better. - -### Some Explanations of ROLLUP - -* The fundamental role of ROLLUP is to improve the query efficiency of some queries (whether by aggregating to reduce the amount of data or by modifying column order to match prefix indexes). Therefore, the meaning of ROLLUP has gone beyond the scope of "roll-up". That's why we named it Materized Index in the source code. -* ROLLUP is attached to the Base table and can be seen as an auxiliary data structure of the Base table. Users can create or delete ROLLUP based on the Base table, but cannot explicitly specify a query for a ROLLUP in the query. Whether ROLLUP is hit or not is entirely determined by the Doris system. -* ROLLUP data is stored in separate physical storage. Therefore, the more OLLUP you create, the more disk space you occupy. It also has an impact on the speed of import (the ETL phase of import automatically generates all ROLLUP data), but it does not reduce query efficiency (only better). -* Data updates for ROLLUP are fully synchronized with Base representations. Users need not care about this problem. -* Columns in ROLLUP are aggregated in exactly the same way as Base tables. There is no need to specify or modify ROLLUP when creating it. -* A necessary (inadequate) condition for a query to hit ROLLUP is that all columns ** (including the query condition columns in select list and where) involved in the query exist in the column of the ROLLUP. Otherwise, the query can only hit the Base table. -* Certain types of queries (such as count (*)) cannot hit ROLLUP under any conditions. See the next section **Limitations of the aggregation model**. -* The query execution plan can be obtained by `EXPLAIN your_sql;` command, and in the execution plan, whether ROLLUP has been hit or not can be checked. -* Base tables and all created ROLLUPs can be displayed by `DESC tbl_name ALL;` statement. - -In this document, you can see [Query how to hit Rollup] (hit-the-rollup) - -## Limitations of aggregation model - -Here we introduce the limitations of Aggregate model (including Uniq model). - -In the aggregation model, what the model presents is the aggregated data. That is to say, any data that has not yet been aggregated (for example, two different imported batches) must be presented in some way to ensure consistency. Let's give an example. - -The hypothesis table is structured as follows: - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -| userid | LARGEINT | | user id| -| date | DATE | | date of data filling| -| Cost | BIGINT | SUM | Total User Consumption| - -Assume that there are two batches of data that have been imported into the storage engine as follows: - -**batch 1** - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|50| -|10002|2017-11-21|39| - -**batch 2** - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|1| -|10001|2017-11-21|5| -|10003|2017-11-22|22| - -As you can see, data belonging to user 10001 in two import batches has not yet been aggregated. However, in order to ensure that users can only query the aggregated data as follows: - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|51| -|10001|2017-11-21|5| -|10002|2017-11-21|39| -|10003|2017-11-22|22| - -We add aggregation operator to query engine to ensure data consistency. - -In addition, on the aggregate column (Value), when executing aggregate class queries that are inconsistent with aggregate types, attention should be paid to semantics. For example, in the example above, we execute the following queries: - -`SELECT MIN(cost) FROM table;` - -The result is 5, not 1. - -At the same time, this consistency guarantee will greatly reduce the query efficiency in some queries. - -Let's take the most basic count (*) query as an example: - -`SELECT COUNT(*) FROM table;` - -In other databases, such queries return results quickly. Because in the implementation, we can get the query result by counting rows at the time of import and saving count statistics information, or by scanning only a column of data to get count value at the time of query, with very little overhead. But in Doris's aggregation model, the overhead of this query ** is very large **. - -Let's take the data as an example. - -**batch 1** - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|50| -|10002|2017-11-21|39| - -**batch 2** - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|1| -|10001|2017-11-21|5| -|10003|2017-11-22|22| - -Because the final aggregation result is: - -|user\_id|date|cost| -|---|---|---| -|10001|2017-11-20|51| -|10001|2017-11-21|5| -|10002|2017-11-21|39| -|10003|2017-11-22|22| - -So `select count (*) from table;` The correct result should be **4**. But if we only scan the `user_id'column and add query aggregation, the final result is **3** (10001, 10002, 10003). If aggregated without queries, the result is **5** (a total of five rows in two batches). It can be seen that both results are wrong. - -In order to get the correct result, we must read the data of `user_id` and `date`, and **together with aggregate** when querying, to return the correct result of **4**. That is to say, in the count (*) query, Doris must scan all AGGREGATE KEY columns (here are `user_id` and `date`) and aggregate them to get the semantically correct results. When aggregated columns are large, count (*) queries need to scan a large amount of data. - -Therefore, when there are frequent count (*) queries in the business, we recommend that users simulate count (*) by adding a column with a value of 1 and aggregation type of SUM. As the table structure in the previous example, we modify it as follows: - -|ColumnName|Type|AggregationType|Comment| -|---|---|---|---| -| user ID | BIGINT | | user id| -| date | DATE | | date of data filling| -| Cost | BIGINT | SUM | Total User Consumption| -| count | BIGINT | SUM | for counting| - -Add a count column and import the data with the column value **equal to 1**. The result of `select count (*) from table;`is equivalent to `select sum (count) from table;` The query efficiency of the latter is much higher than that of the former. However, this method also has limitations, that is, users need to guarantee that they will not import rows with the same AGGREGATE KEY column repeatedly. Otherwise, `select sum (count) from table;`can only express the number of rows originally imported, not the semantics of `select count (*) from table;` - -Another way is to **change the aggregation type of the count'column above to REPLACE, and still weigh 1**. Then`select sum (count) from table;` and `select count (*) from table;` the results will be consistent. And in this way, there is no restriction on importing duplicate rows. - -### Duplicate Model - -Duplicate model has no limitation of aggregation model. Because the model does not involve aggregate semantics, when doing count (*) query, we can get the correct semantics by choosing a column of queries arbitrarily. - -## Suggestions for Choosing Data Model - -Because the data model was established when the table was built, and **could not be modified **. Therefore, it is very important to select an appropriate data model**. - -1. Aggregate model can greatly reduce the amount of data scanned and the amount of query computation by pre-aggregation. It is very suitable for report query scenarios with fixed patterns. But this model is not very friendly for count (*) queries. At the same time, because the aggregation method on the Value column is fixed, semantic correctness should be considered in other types of aggregation queries. -2. Uniq model guarantees the uniqueness of primary key for scenarios requiring unique primary key constraints. However, the query advantage brought by pre-aggregation such as ROLLUP can not be exploited (because the essence is REPLACE, there is no such aggregation as SUM). -3. Duplicate is suitable for ad-hoc queries of any dimension. Although it is also impossible to take advantage of the pre-aggregation feature, it is not constrained by the aggregation model and can take advantage of the queue-store model (only reading related columns, but not all Key columns). diff --git a/docs/documentation/en/getting-started/data-partition_EN.md b/docs/documentation/en/getting-started/data-partition_EN.md deleted file mode 100644 index a06bb51c29e6df..00000000000000 --- a/docs/documentation/en/getting-started/data-partition_EN.md +++ /dev/null @@ -1,286 +0,0 @@ - - -# Data Partition - -This document mainly introduces Doris's table construction and data partitioning, as well as problems and solutions that may be encountered in the construction of the table. - -## Basic Concepts - -In Doris, data is logically described in the form of a table. - -### Row & Column - -A table includes rows (rows) and columns (columns). Row is a row of data for the user. Column is used to describe different fields in a row of data. - -Column can be divided into two broad categories: Key and Value. From a business perspective, Key and Value can correspond to dimension columns and metric columns, respectively. From the perspective of the aggregation model, the same row of Key columns will be aggregated into one row. The way the Value column is aggregated is specified by the user when the table is built. For an introduction to more aggregation models, see the [Doris Data Model] (./data-model-rollup.md). - -### Tablet & Partition - -In Doris's storage engine, user data is horizontally divided into several data slices (also known as data buckets). Each tablet contains several rows of data. The data between the individual tablets does not intersect and is physically stored independently. - -Multiple tablets are logically attributed to different partitions. A tablet belongs to only one Partition. And a Partition contains several Tablets. Because the tablet is physically stored independently, it can be considered that the Partition is physically independent. Tablet is the smallest physical storage unit for data movement, replication, and so on. - -Several Partitions form a Table. Partition can be thought of as the smallest logical unit of management. Importing and deleting data can be done for one Partition or only for one Partition. - -## Data division - -We use a table-building operation to illustrate Doris' data partitioning. - -Doris's built-in table is a synchronous command. If the command returns successfully, it means that the table is built successfully. - -See more help with `HELP CREATE TABLE;`. - -This section introduces Doris's approach to building tables with an example. - -``` -CREATE TABLE IF NOT EXISTS example_db.expamle_tbl -( - `user_id` LARGEINT NOT NULL COMMENT "user id", - `date` DATE NOT NULL COMMENT "Data fill in date time", - `timestamp` DATETIME NOT NULL COMMENT "Timestamp of data being poured", - `city` VARCHAR(20) COMMENT "The city where the user is located", - `age` SMALLINT COMMENT "user age", - `sex` TINYINT COMMENT "User Gender", - `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "User last visit time", - `cost` BIGINT SUM DEFAULT "0" COMMENT "Total user consumption", - `max_dwell_time` INT MAX DEFAULT "0" COMMENT "User maximum dwell time", - `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "User minimum dwell time" -) -ENGINE=olap -AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) -PARTITION BY RANGE(`date`) -( - PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), - PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), - PARTITION `p201703` VALUES LESS THAN ("2017-04-01") -) -DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 -PROPERTIES -( - "replication_num" = "3", - "storage_medium" = "SSD", - "storage_cooldown_time" = "2018-01-01 12:00:00" -); - -``` - -### Column Definition - -Here we only use the AGGREGATE KEY data model as an example. See the [Doris Data Model] (./data-model-rollup.md) for more data models. - -The basic type of column can be viewed by executing `HELP CREATE TABLE;` in mysql-client. - -In the AGGREGATE KEY data model, all columns that do not specify an aggregation mode (SUM, REPLACE, MAX, MIN) are treated as Key columns. The rest is the Value column. - -When defining columns, you can refer to the following suggestions: - -1. The Key column must precede all Value columns. -2. Try to choose the type of integer. Because integer type calculations and lookups are much more efficient than strings. -3. For the selection principle of integer types of different lengths, follow ** enough to **. -4. For lengths of type VARCHAR and STRING, follow ** is sufficient. -5. The total byte length of all columns (including Key and Value) cannot exceed 100KB. - -### Partitioning and binning - -Doris supports two levels of data partitioning. The first layer is Partition, which only supports the division of Range. The second layer is Bucket (Tablet), which only supports the way Hash is divided. - -It is also possible to use only one layer of partitioning. When using a layer partition, only Bucket partitioning is supported. - -1. Partition - - * The Partition column can specify one or more columns. The partition class must be a KEY column. The use of multi-column partitions is described later in the **Multi-column partitioning** summary.  - * Regardless of the type of partition column, double quotes are required when writing partition values. - * Partition columns are usually time columns for easy management of old and new data. - * There is no theoretical limit on the number of partitions. - * When you do not use Partition to build a table, the system will automatically generate a Partition with the same name as the table name. This Partition is not visible to the user and cannot be modified. - * Partition supports only the upper bound by `VALUES LESS THAN (...)`, the system will use the upper bound of the previous partition as the lower bound of the partition, and generate a left closed right open interval. Passing, also supports specifying the upper and lower bounds by `VALUES [...)`, and generating a left closed right open interval. - * It is easier to understand by specifying `VALUES [...)`. Here is an example of the change in partition range when adding or deleting partitions using the `VALUES LESS THAN (...)` statement: - * As the example above, when the table is built, the following 3 partitions are automatically generated: - ``` - P201701: [MIN_VALUE, 2017-02-01) - P201702: [2017-02-01, 2017-03-01) - P201703: [2017-03-01, 2017-04-01) - ``` - * When we add a partition p201705 VALUES LESS THAN ("2017-06-01"), the partition results are as follows: - - ``` - P201701: [MIN_VALUE, 2017-02-01) - P201702: [2017-02-01, 2017-03-01) - P201703: [2017-03-01, 2017-04-01) - P201705: [2017-04-01, 2017-06-01) - ``` - - * At this point we delete the partition p201703, the partition results are as follows: - - ``` - p201701: [MIN_VALUE, 2017-02-01) - p201702: [2017-02-01, 2017-03-01) - p201705: [2017-04-01, 2017-06-01) - ``` - - > Note that the partition range of p201702 and p201705 has not changed, and there is a hole between the two partitions: [2017-03-01, 2017-04-01). That is, if the imported data range is within this hole, it cannot be imported. - - * Continue to delete partition p201702, the partition results are as follows: - - ``` - p201701: [MIN_VALUE, 2017-02-01) - p201705: [2017-04-01, 2017-06-01) - The void range becomes: [2017-02-01, 2017-04-01) - ``` - - * Now add a partition p201702new VALUES LESS THAN ("2017-03-01"), the partition results are as follows: - - ``` - p201701: [MIN_VALUE, 2017-02-01) - p201702new: [2017-02-01, 2017-03-01) - p201705: [2017-04-01, 2017-06-01) - ``` - - > You can see that the hole size is reduced to: [2017-03-01, 2017-04-01) - - * Now delete partition p201701 and add partition p201612 VALUES LESS THAN ("2017-01-01"), the partition result is as follows: - - ``` - p201612: [MIN_VALUE, 2017-01-01) - p201702new: [2017-02-01, 2017-03-01) - p201705: [2017-04-01, 2017-06-01) - ``` - - > A new void appeared: [2017-01-01, 2017-02-01) - - In summary, the deletion of a partition does not change the scope of an existing partition. There may be holes in deleting partitions. When a partition is added by the `VALUES LESS THAN` statement, the lower bound of the partition immediately follows the upper bound of the previous partition. - - You cannot add partitions with overlapping ranges. - -2. Bucket - - * If a Partition is used, the `DISTRIBUTED ...` statement describes the division rules for the data in each partition. If you do not use Partition, it describes the rules for dividing the data of the entire table. - * The bucket column can be multiple columns, but it must be a Key column. The bucket column can be the same or different from the Partition column. - * The choice of bucket column is a trade-off between **query throughput** and **query concurrency**: - - 1. If you select multiple bucket columns, the data is more evenly distributed. However, if the query condition does not include the equivalent condition for all bucket columns, a query will scan all buckets. The throughput of such queries will increase, but the latency of a single query will increase. This method is suitable for large throughput and low concurrent query scenarios. - 2. If you select only one or a few bucket columns, the point query can query only one bucket. This approach is suitable for high-concurrency point query scenarios. - - * There is no theoretical limit on the number of buckets. - -3. Recommendations on the number and amount of data for Partitions and Buckets. - - * The total number of tablets in a table is equal to (Partition num * Bucket num). - * The number of tablets in a table, which is slightly more than the number of disks in the entire cluster, regardless of capacity expansion. - * The data volume of a single tablet does not theoretically have an upper and lower bound, but is recommended to be in the range of 1G - 10G. If the amount of data for a single tablet is too small, the aggregation of the data is not good and the metadata management pressure is high. If the amount of data is too large, it is not conducive to the migration, completion, and increase the cost of Schema Change or Rollup operation failure retry (the granularity of these operations failure retry is Tablet). - * When the tablet's data volume principle and quantity principle conflict, it is recommended to prioritize the data volume principle. - * When building a table, the number of Buckets for each partition is uniformly specified. However, when dynamically increasing partitions (`ADD PARTITION`), you can specify the number of Buckets for the new partition separately. This feature can be used to easily reduce or expand data. - * Once the number of Buckets for a Partition is specified, it cannot be changed. Therefore, when determining the number of Buckets, you need to consider the expansion of the cluster in advance. For example, there are currently only 3 hosts, and each host has 1 disk. If the number of Buckets is only set to 3 or less, then even if you add more machines later, you can't increase the concurrency. - * Give some examples: Suppose there are 10 BEs, one for each BE disk. If the total size of a table is 500MB, you can consider 4-8 shards. 5GB: 8-16. 50GB: 32. 500GB: Recommended partitions, each partition is about 50GB in size, with 16-32 shards per partition. 5TB: Recommended partitions, each with a size of around 50GB and 16-32 shards per partition. - - > Note: The amount of data in the table can be viewed by the `show data` command. The result is divided by the number of copies, which is the amount of data in the table. - -#### Multi-column partition - -Doris supports specifying multiple columns as partition columns, examples are as follows: - -``` -PARTITION BY RANGE(`date`, `id`) -( - PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), - PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), - PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") -) -``` - -In the above example, we specify `date` (DATE type) and `id` (INT type) as partition columns. The resulting partitions in the above example are as follows: - -``` -*p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) -*p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) -*p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) -``` - -Note that the last partition user defaults only the partition value of the `date` column, so the partition value of the `id` column will be filled with `MIN_VALUE` by default. When the user inserts data, the partition column values ​​are compared in order, and the corresponding partition is finally obtained. Examples are as follows: - -``` -* Data --> Partition -* 2017-01-01, 200 --> p201701_1000 -* 2017-01-01, 2000 --> p201701_1000 -* 2017-02-01, 100 --> p201701_1000 -* 2017-02-01, 2000 --> p201702_2000 -* 2017-02-15, 5000 --> p201702_2000 -* 2017-03-01, 2000 --> p201703_all -* 2017-03-10, 1 --> p201703_all -* 2017-04-01, 1000 --> Unable to import -* 2017-05-01, 1000 --> Unable to import -``` - -### PROPERTIES - -In the last PROPERTIES of the table statement, you can specify the following two parameters: - -Replication_num - - * The number of copies per tablet. The default is 3, it is recommended to keep the default. In the build statement, the number of Tablet copies in all Partitions is uniformly specified. When you add a new partition, you can individually specify the number of copies of the tablet in the new partition. - * The number of copies can be modified at runtime. It is strongly recommended to keep odd numbers. - * The maximum number of copies depends on the number of independent IPs in the cluster (note that it is not the number of BEs). The principle of replica distribution in Doris is that the copies of the same Tablet are not allowed to be distributed on the same physical machine, and the physical machine is identified as IP. Therefore, even if 3 or more BE instances are deployed on the same physical machine, if the BEs have the same IP, you can only set the number of copies to 1. - * For some small, and infrequently updated dimension tables, consider setting more copies. In this way, when joining queries, there is a greater probability of local data join. - -2. storage_medium & storage\_cooldown\_time - - * The BE data storage directory can be explicitly specified as SSD or HDD (differentiated by .SSD or .HDD suffix). When you build a table, you can uniformly specify the media for all Partition initial storage. Note that the suffix is ​​to explicitly specify the disk media without checking to see if it matches the actual media type. - * The default initial storage media can be specified by `default_storage_medium= XXX` in the fe configuration file `fe.conf`, or, if not, by default, HDD. If specified as an SSD, the data is initially stored on the SSD. - * If storage\_cooldown\_time is not specified, the data is automatically migrated from the SSD to the HDD after 30 days by default. If storage\_cooldown\_time is specified, the data will not migrate until the storage_cooldown_time time is reached. - * Note that this parameter is just a "best effort" setting when storage_medium is specified. Even if no SSD storage media is set in the cluster, no error is reported and it is automatically stored in the available data directory. Similarly, if the SSD media is inaccessible and out of space, the data may initially be stored directly on other available media. When the data expires and is migrated to the HDD, if the HDD media is inaccessible and there is not enough space, the migration may fail (but will continue to try). - -### ENGINE - -In this example, the type of ENGINE is olap, the default ENGINE type. In Doris, only this ENGINE type is managed and stored by Doris. Other ENGINE types, such as mysql, broker, es, etc., are essentially mappings to tables in other external databases or systems to ensure that Doris can read the data. And Doris itself does not create, manage, and store any tables and data of a non-olap ENGINE type. - -### Other - -`IF NOT EXISTS` indicates that if the table has not been created, it is created. Note that only the table name is judged here, and it is not determined whether the new table structure is the same as the existing table structure. So if there is a table with the same name but different structure, the command will also return success, but it does not mean that a new table and a new structure have been created. - -## common problem - -### Build Table Operations FAQ - -1. If a syntax error occurs in a long build statement, a syntax error may be incomplete. Here is a list of possible syntax errors for manual error correction: - - * The syntax is incorrect. Please read `HELP CREATE TABLE;` carefully to check the relevant syntax structure. - * Reserved words. When the user-defined name encounters a reserved word, it needs to be enclosed in the backquote ``. It is recommended that all custom names be generated using this symbol. - * Chinese characters or full-width characters. Non-utf8 encoded Chinese characters, or hidden full-width characters (spaces, punctuation, etc.) can cause syntax errors. It is recommended to check with a text editor with invisible characters. - -2. `Failed to create partition [xxx] . Timeout` - - Doris builds are created in order of Partition granularity. This error may be reported when a Partition creation fails. Even if you don't use Partition, you will report `Failed to create partition` when there is a problem with the built table, because as mentioned earlier, Doris will create an unchangeable default Partition for tables that do not have a Partition specified. - - When this error is encountered, it is usually the BE that has encountered problems creating data fragments. You can follow the steps below to troubleshoot: - - 1. In fe.log, find the `Failed to create partition` log for the corresponding point in time. In this log, a series of numbers like `{10001-10010}` will appear. The first number of the pair is the Backend ID and the second number is the Tablet ID. As for the pair of numbers above, on the Backend with ID 10001, creating a tablet with ID 10010 failed. - 2. Go to the be.INFO log corresponding to Backend and find the log related to the tablet id in the corresponding time period. You can find the error message. - 3. Listed below are some common tablet creation failure errors, including but not limited to: - * BE did not receive the relevant task, and the tablet id related log could not be found in be.INFO. Or the BE is created successfully, but the report fails. For the above questions, see [Deployment and Upgrade Documentation] to check the connectivity of FE and BE. - * Pre-allocated memory failed. It may be that the length of a line in a row in the table exceeds 100KB. - * `Too many open files`. The number of open file handles exceeds the Linux system limit. The handle limit of the Linux system needs to be modified. - - You can also extend the timeout by setting `tablet_create_timeout_second=xxx` in fe.conf. The default is 2 seconds. - -3. The build table command does not return results for a long time. - - Doris's table creation command is a synchronous command. The timeout of this command is currently set to be relatively simple, ie (tablet num * replication num) seconds. If you create more data fragments and have fragment creation failed, it may cause an error to be returned after waiting for a long timeout. - - Under normal circumstances, the statement will return in a few seconds or ten seconds. If it is more than one minute, it is recommended to cancel this operation directly and go to the FE or BE log to view the related errors. diff --git a/docs/documentation/en/getting-started/hit-the-rollup_EN.md b/docs/documentation/en/getting-started/hit-the-rollup_EN.md deleted file mode 100644 index c0bac60d94b5cf..00000000000000 --- a/docs/documentation/en/getting-started/hit-the-rollup_EN.md +++ /dev/null @@ -1,289 +0,0 @@ - - -# Rollup and query - -As a polymer view in Doris, Rollup can play two roles in queries: - -* Index -* Aggregate data (only for aggregate models, aggregate key) - -However, in order to hit Rollup, certain conditions need to be met, and the value of PreAggregation of ScanNdo node in the execution plan can be used to determine whether Rollup can be hit or not, and the Rollup field can be used to determine which Rollup table is hit. - -## Noun Interpretation - -Base: Base table. - -Rollup: Generally, it refers to the Rollup tables created based on Base tables, but in some scenarios, it includes Base and Rollup tables. - -## Index - -Doris's prefix index has been introduced in the previous query practice, that is, Doris will generate the first 36 bytes in the Base/Rollup table separately in the underlying storage engine (with varchar type, the prefix index may be less than 36 bytes, varchar will truncate the prefix index, and use up to 20 bytes of varchar). A sorted sparse index data (data is also sorted, positioned by index, and then searched by dichotomy in the data), and then matched each Base/Rollup prefix index according to the conditions in the query, and selected a Base/Rollup that matched the longest prefix index. - -``` - ---> matching from left to right -+----+----+----+----+----+----+ -| c1 | c2 | c3 | c4 | c5 |... | -``` - -As shown in the figure above, the conditions of where and on in the query are pushed up and down to ScanNode and matched from the first column of the prefix index. Check if there are any of these columns in the condition, and then accumulate the matching length until the matching cannot match or the end of 36 bytes (columns of varchar type can only match 20 bytes and match less than 36 words). Section truncates prefix index, and then chooses a Base/Rollup with the longest matching length. The following example shows how to create a Base table and four rollups: - -``` -+---------------+-------+--------------+------+-------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+---------------+-------+--------------+------+-------+---------+-------+ -| test | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup_index1 | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup_index2 | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k1 | TINYINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup_index3 | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup_index4 | k4 | BIGINT | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -+---------------+-------+--------------+------+-------+---------+-------+ -``` - -The prefix indexes of the three tables are - -``` -Base(k1 ,k2, k3, k4, k5, k6, k7) - -rollup_index1(k9)，rollup_index2(k9) - -rollup_index3(k4, k5, k6, k1, k2, k3, k7) - -rollup_index4(k4, k6, k5, k1, k2, k3, k7) -``` - -Conditions on columns that can be indexed with the prefix need to be `=` `<` `>` `<=` `>=` `in` `between`, and these conditions are side-by-side and the relationship uses `and` connections', which cannot be hit for `or`、`!=` and so on. Then look at the following query: - -``` -SELECT * FROM test WHERE k1 = 1 AND k2 > 3; -``` - -With the conditions on K1 and k2, check that only the first column of Base contains K1 in the condition, so match the longest prefix index, test, explain: - -``` -| 0:OlapScanNode -| TABLE: test -| PREAGGREGATION: OFF. Reason: No AggregateInfo -| PREDICATES: `k1` = 1, `k2` > 3 -| partitions=1/1 -| rollup: test -| buckets=1/10 -| cardinality=-1 -| avgRowSize=0.0 -| numNodes=0 -| tuple ids: 0 -``` - -Look again at the following queries: - -`SELECT * FROM test WHERE k4 =1 AND k5 > 3;` - -With K4 and K5 conditions, check that the first column of rollup_index3 and rollup_index4 contains k4, but the second column of rollup_index3 contains k5, so the matching prefix index is the longest. - -``` -| 0:OlapScanNode -| TABLE: test -| PREAGGREGATION: OFF. Reason: No AggregateInfo -| PREDICATES: `k4` = 1, `k5` > 3 -| partitions=1/1 -| rollup: rollup_index3 -| buckets=10/10 -| cardinality=-1 -| avgRowSize=0.0 -| numNodes=0 -| tuple ids: 0 -``` - -Now we try to match the conditions on the column containing varchar, as follows: - -`SELECT * FROM test WHERE k9 IN ("xxx", "yyyy") AND k1 = 10;` - -There are K9 and K1 conditions. The first column of rollup_index1 and rollup_index2 contains k9. It is reasonable to choose either rollup here to hit the prefix index and randomly select the same one (because there are just 20 bytes in varchar, and the prefix index is truncated in less than 36 bytes). The current strategy here will continue to match k1, because the second rollup_index1 is listed as k1, so rollup_index1 is chosen, in fact, the latter K1 condition will not play an accelerating role. (If the condition outside the prefix index needs to accelerate the query, it can be accelerated by establishing a Bloom Filter filter. Typically for string types, because Doris has a Block level for columns, a Min/Max index for shaping and dates.) The following is the result of explain. - -``` -| 0:OlapScanNode -| TABLE: test -| PREAGGREGATION: OFF. Reason: No AggregateInfo -| PREDICATES: `k9` IN ('xxx', 'yyyy'), `k1` = 10 -| partitions=1/1 -| rollup: rollup_index1 -| buckets=1/10 -| cardinality=-1 -| avgRowSize=0.0 -| numNodes=0 -| tuple ids: 0 -``` - -Finally, look at a query that can be hit by more than one Rollup: - -`Select * from test where K4 < 1000 and K5 = 80 and K6 = 10000;` - -There are three conditions: k4, K5 and k6. The first three columns of rollup_index3 and rollup_index4 contain these three columns respectively. So the prefix index length matched by them is the same. Both can be selected. The current default strategy is to select a rollup created earlier. Here is rollup_index3. - -``` -| 0:OlapScanNode -| TABLE: test -| PREAGGREGATION: OFF. Reason: No AggregateInfo -| PREDICATES: `k4` < 1000, `k5` = 80, `k6` >= 10000.0 -| partitions=1/1 -| rollup: rollup_index3 -| buckets=10/10 -| cardinality=-1 -| avgRowSize=0.0 -| numNodes=0 -| tuple ids: 0 -``` - -If you modify the above query slightly as follows: - -`SELECT * FROM test WHERE k4 < 1000 AND k5 = 80 OR k6 >= 10000;` - -The query here cannot hit the prefix index. (Even any Min/Max in the Doris storage engine, the BloomFilter index doesn't work.) - -## Aggregate data - -Of course, the function of aggregated data is indispensable for general polymer views. Such materialized views are very helpful for aggregated queries or report queries. To hit the polymer views, the following prerequisites are needed: - -1. There is a separate Rollup for all columns involved in a query or subquery. -2. If there is Join in a query or sub-query, the type of Join needs to be Inner join. - -The following are some types of aggregated queries that can hit Rollup. - -| Column type Query type | Sum | Distinct/Count Distinct | Min | Max | Ndv | -|--------------|-------|-------------------------|-------|-------|-------| -| Key | false | true | true | true | true | -| Value(Sum) | true | false | false | false | false | -|Value(Replace)| false | false | false | false | false | -| Value(Min) | false | false | true | false | false | -| Value(Max) | false | false | false | true | false | - - -If the above conditions are met, there will be two stages in judging the hit of Rollup for the aggregation model: - -1. Firstly, the Rollup table with the longest index hit by prefix index is matched by conditions. See the index strategy above. -2. Then compare the rows of Rollup and select the smallest Rollup. - -The following Base table and Rollup: - -``` -+-------------+-------+--------------+------+-------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+-------------+-------+--------------+------+-------+---------+-------+ -| test_rollup | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k6 | CHAR(5) | Yes | true | N/A | | -| | k7 | DATE | Yes | true | N/A | | -| | k8 | DATETIME | Yes | true | N/A | | -| | k9 | VARCHAR(20) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup2 | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -| | | | | | | | -| rollup1 | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | true | N/A | | -| | k3 | INT | Yes | true | N/A | | -| | k4 | BIGINT | Yes | true | N/A | | -| | k5 | DECIMAL(9,3) | Yes | true | N/A | | -| | k10 | DOUBLE | Yes | false | N/A | MAX | -| | k11 | FLOAT | Yes | false | N/A | SUM | -+-------------+-------+--------------+------+-------+---------+-------+ -``` - - -See the following queries: - -`SELECT SUM(k11) FROM test_rollup WHERE k1 = 10 AND k2 > 200 AND k3 in (1,2,3);` - -Firstly, it judges whether the query can hit the aggregated Rolup table. After checking the graph above, it is possible. Then the condition contains three conditions: k1, K2 and k3. The first three columns of test_rollup, rollup1 and rollup2 contain all the three conditions. So the prefix index length is the same. Then, it is obvious that the aggregation degree of rollup2 is the highest when comparing the number of rows. Row 2 is selected because of the minimum number of rows. - -``` -| 0:OlapScanNode | -| TABLE: test_rollup | -| PREAGGREGATION: ON | -| PREDICATES: `k1` = 10, `k2` > 200, `k3` IN (1, 2, 3) | -| partitions=1/1 | -| rollup: rollup2 | -| buckets=1/10 | -| cardinality=-1 | -| avgRowSize=0.0 | -| numNodes=0 | -| tuple ids: 0 | -``` diff --git a/docs/documentation/en/getting-started/index.rst b/docs/documentation/en/getting-started/index.rst deleted file mode 100644 index c9236f6ab35687..00000000000000 --- a/docs/documentation/en/getting-started/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -==================== -Getting Started -==================== - -.. toctree:: - - basic-usage_EN.md - advance-usage_EN.md - best-practice_EN.md - data-partition_EN.md - data-model-rollup_EN.md - hit-the-rollup_EN.md diff --git a/docs/documentation/en/index.rst b/docs/documentation/en/index.rst deleted file mode 100644 index 8b6e2fc3a09363..00000000000000 --- a/docs/documentation/en/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -============= -English -============= - -English version of Doris documents are mainly translated by machine, currently. -So please help us to improve the quality of English documents. -You can simply click "Edit on Github" at right top of a page, and edit the document on Github, then make a pull request directly. - -.. toctree:: - - downloads/index - installing/index - getting-started/index - administrator-guide/index - extending-doris/index - internal/index - sql-reference/index - developer-guide/index - community/index - diff --git a/docs/documentation/en/installing/compilation_EN.md b/docs/documentation/en/installing/compilation_EN.md deleted file mode 100644 index ca3dc4a7585581..00000000000000 --- a/docs/documentation/en/installing/compilation_EN.md +++ /dev/null @@ -1,100 +0,0 @@ - - - -# Compilation - -This document focuses on how to code Doris through source code. - -## Developing mirror compilation using Docker (recommended) - -### Use off-the-shelf mirrors - -1. Download Docker Mirror - - `$ docker pull apachedoris/doris-dev:build-env` - - Check mirror download completed: - - ``` - $ docker images - REPOSITORY TAG IMAGE ID CREATED SIZE - apachedoris/doris-dev build-env f8bc5d4024e0 21 hours ago 3.28GB - ``` - -Note: For different versions of Oris, you need to download the corresponding mirror version. - -| image version | commit id | release version | -|---|---|---| -| apachedoris/doris-dev:build-env | before [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.8.x, 0.9.x | -| apachedoris/doris-dev:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) or later | 0.10.x or later | - -2. Running Mirror - - `$ docker run -it apachedoris/doris-dev:build-env` - - If you want to compile the local Doris source code, you can mount the path: - - ``` - $ docker run -it -v /your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/ apachedoris/doris-dev:build-env - ``` - -3. Download source code - - After starting the mirror, you should be in the container. The Doris source code can be downloaded from the following command (local source directory mounted is not required): - - ``` - $ wget https://dist.apache.org/repos/dist/dev/incubator/doris/xxx.tar.gz - or - $ git clone https://github.com/apache/incubator-doris.git - ``` - -4. Compile Doris - - ``` - $ sh build.sh - ``` - - After compilation, the output file is in the `output/` directory. - -### Self-compiling Development Environment Mirror - -You can also create a Doris development environment mirror yourself, referring specifically to the `docker/README.md'file. - - -## Direct Compilation (CentOS/Ubuntu) - -You can try to compile Doris directly in your own Linux environment. - -1. System Dependence - - `GCC 5.3.1+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+` - - If you are using Ubuntu 16.04 or newer, you can use the following command to install the dependencies - - `sudo apt-get install build-essential openjdk-8-jdk maven cmake byacc flex automake libtool-bin bison binutils-dev libiberty-dev` - - After installation, set environment variables `PATH`, `JAVA_HOME`, etc. - -2. Compile Doris - - ``` - $ sh build.sh - ``` - After compilation, the output file is in the `output/` directory. diff --git a/docs/documentation/en/installing/index.rst b/docs/documentation/en/installing/index.rst deleted file mode 100644 index 6296287f45eb5f..00000000000000 --- a/docs/documentation/en/installing/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -=============================== -Compilation and Deployment -=============================== - -.. toctree:: - - compilation_EN.md - install-deploy_EN.md - upgrade_EN.md diff --git a/docs/documentation/en/installing/install-deploy_EN.md b/docs/documentation/en/installing/install-deploy_EN.md deleted file mode 100644 index 0504dbb4c617af..00000000000000 --- a/docs/documentation/en/installing/install-deploy_EN.md +++ /dev/null @@ -1,427 +0,0 @@ - - - -# Installation and deployment - -This document mainly introduces the hardware and software environment needed to deploy Doris, the proposed deployment mode, cluster expansion and scaling, and common problems in the process of cluster building and running. -Before reading this document, compile Doris according to the compiled document. - -## Software and hardware requirements - -### Overview - -Doris, as an open source MPP architecture OLAP database, can run on most mainstream commercial servers. In order to make full use of the concurrency advantages of MPP architecture and the high availability features of Doris, we recommend that the deployment of Doris follow the following requirements: - -#### Linux Operating System Version Requirements - -| Linux System | Version| -|---|---| -| Centers | 7.1 and above | -| Ubuntu | 16.04 and above | - -#### Software requirements - -| Soft | Version | -|---|---| -| Java | 1.8 and above | -| GCC | 4.8.2 and above | - -#### Development Test Environment - -| Module | CPU | Memory | Disk | Network | Instance Number| -|---|---|---|---|---|---| -| Frontend | 8 core + | 8GB + | SSD or SATA, 10GB + * | Gigabit Network Card | 1| -| Backend | 8-core + | 16GB + | SSD or SATA, 50GB + * | Gigabit Network Card | 1-3*| - -#### Production environment - -| Module | CPU | Memory | Disk | Network | Number of Instances (Minimum Requirements)| -|---|---|---|---|---|---| -| Frontend | 16 core + | 64GB + | SSD or RAID card, 100GB + * | 10,000 Mbp network card | 1-5*| -| Backend | 16 core + | 64GB + | SSD or SATA, 100G + * | 10-100 Mbp network card*| - -> Note 1: -> -> 1. The disk space of FE is mainly used to store metadata, including logs and images. Usually it ranges from several hundred MB to several GB. -> 2. BE's disk space is mainly used to store user data. The total disk space is calculated according to the user's total data * 3 (3 copies). Then an additional 40% of the space is reserved for background compaction and some intermediate data storage. -> 3. Multiple BE instances can be deployed on a single machine, but **can only deploy one FE**. If you need three copies of data, you need at least one BE instance per machine (instead of three BE instances per machine). **Clocks of multiple FE servers must be consistent (allowing a maximum of 5 seconds clock deviation)** -> 4. The test environment can also be tested with only one BE. In the actual production environment, the number of BE instances directly determines the overall query latency. -> 5. All deployment nodes close Swap. - -> Note 2: Number of FE nodes -> -> 1. FE roles are divided into Follower and Observer. (Leader is an elected role in the Follower group, hereinafter referred to as Follower, for the specific meaning, see [Metadata Design Document] (./internal/metadata-design).) -> 2. FE node data is at least 1 (1 Follower). When one Follower and one Observer are deployed, high read availability can be achieved. When three Followers are deployed, read-write high availability (HA) can be achieved. -> 3. The number of Followers **must be** odd, and the number of Observers is arbitrary. -> 4. According to past experience, when cluster availability requirements are high (e.g. providing online services), three Followers and one to three Observers can be deployed. For offline business, it is recommended to deploy 1 Follower and 1-3 Observers. - -* **Usually we recommend about 10 to 100 machines to give full play to Doris's performance (3 of them deploy FE (HA) and the rest deploy BE)** -* **Of course, Doris performance is positively correlated with the number and configuration of nodes. With a minimum of four machines (one FE, three BEs, one BE mixed with one Observer FE to provide metadata backup) and a lower configuration, Doris can still run smoothly.** -* **If FE and BE are mixed, we should pay attention to resource competition and ensure that metadata catalogue and data catalogue belong to different disks.** - -#### Broker deployment - -Broker is a process for accessing external data sources, such as hdfs. Usually, a broker instance is deployed on each machine. - -#### Network Requirements - -Doris instances communicate directly over the network. The following table shows all required ports - -| Instance Name | Port Name | Default Port | Communication Direction | Description| -| ---|---|---|---|---| -| BE | be_port | 9060 | FE - > BE | BE for receiving requests from FE| -| BE | webserver\_port | 8040 | BE <--> BE | BE| -| BE | heartbeat\_service_port | 9050 | FE - > BE | the heart beat service port (thrift) on BE, used to receive heartbeat from FE| -| BE | brpc\_port* | 8060 | FE < - > BE, BE < - > BE | BE for communication between BEs| -| FE | http_port* | 8030 | FE < - > FE, HTTP server port on user | FE| -| FE | rpc_port | 9020 | BE - > FE, FE < - > FE | thrift server port on FE| -| FE | query_port | 9030 | user | FE| -| FE | edit\_log_port | 9010 | FE <--> FE | FE| -| Broker | broker ipc_port | 8000 | FE - > Broker, BE - > Broker | Broker for receiving requests| - -> Note: -> -> 1. When deploying multiple FE instances, make sure that the http port configuration of FE is the same. -> 2. Make sure that each port has access in its proper direction before deployment. - -#### IP binding - -Because of the existence of multiple network cards, or the existence of virtual network cards caused by the installation of docker and other environments, the same host may have multiple different ips. Currently Doris does not automatically identify available IP. So when you encounter multiple IP on the deployment host, you must force the correct IP to be specified through the priority\_networks configuration item. - -Priority\_networks is a configuration that both FE and BE have, and the configuration items need to be written in fe.conf and be.conf. This configuration item is used to tell the process which IP should be bound when FE or BE starts. Examples are as follows: - -`priority_networks=10.1.3.0/24` - -This is a representation of [CIDR] (https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). FE or BE will find the matching IP based on this configuration item as their own local IP. - -**Note**: When priority networks is configured and FE or BE is started, only the correct IP binding of FE or BE is ensured. In ADD BACKEND or ADD FRONTEND statements, you also need to specify IP matching priority networks configuration, otherwise the cluster cannot be established. Give an example: - -BE is configured as `priority_networks = 10.1.3.0/24'.`. - -When you want to ADD BACKEND use ：`ALTER SYSTEM ADD BACKEND "192.168.0.1:9050";` - -Then FE and BE will not be able to communicate properly. - -At this point, DROP must remove the BE that added errors and re-use the correct IP to perform ADD BACKEND. - -FE is the same. - -BROKER does not currently have, nor does it need, priority\ networks. Broker's services are bound to 0.0.0 by default. Simply execute the correct accessible BROKER IP when ADD BROKER is used. - -## Cluster deployment - -### Manual deployment - -#### Deploy FE - -* Copy the FE deployment file to the specified node - - Copy the Fe folder under output generated by source code compilation to the node specified deployment path of FE. - -* Configure FE - - 1. The configuration file is conf/fe.conf. Note: `meta_dir`: Metadata storage location. The default is fe/doris-meta/. The directory needs to be **created manually** by. - 2. JAVA_OPTS in fe.conf defaults to a maximum heap memory of 4GB for java, and it is recommended that the production environment be adjusted to more than 8G. - -* Start FE - - `sh bin/start_fe.sh --daemon` - - The FE process starts and enters the background execution. Logs are stored in the fe/log/directory by default. If startup fails, you can view error messages by looking at fe/log/fe.log or fe/log/fe.out. - -* For deployment of multiple FEs, see the section "FE scaling and downsizing" - -#### Deploy BE - -* Copy BE deployment files to all nodes to deploy BE - - Copy the be folder under output generated by source code compilation to the specified deployment path of the BE node. - -* Modify all BE configurations - - Modify be/conf/be.conf. Mainly configure `storage_root_path`: data storage directory. The default is be/storage, this directory needs to be **created manually** by. In multi directories case, using `;` separation (do not add `;` after the last directory). - -* Add all BE nodes to FE - - BE nodes need to be added in FE before they can join the cluster. You can use mysql-client to connect to FE: - - `./mysql-client -h host -P port -uroot` - - The host is the node IP where FE is located; the port is the query_port in fe/conf/fe.conf; the root account is used by default and no password is used to login. - - After login, execute the following commands to add each BE: - - `ALTER SYSTEM ADD BACKEND "host:port";` - - If the multi-tenant function is used, the following command is executed to add BE: - - `ALTER SYSTEM ADD FREE BACKEND "host:port";` - - The host is the node IP where BE is located; the port is heartbeat_service_port in be/conf/be.conf. - - If the FREE keyword is not added, BE defaults to the automatically generated cluster, and the new BE does not belong to any cluster after adding the FREE keyword, so that when creating a new cluster, it can be selected from these free be, as detailed in [Multi-tenant Design Document] (./administrator-guide/operation/multi-tenant.md) - -* Start BE - - `sh bin/start_be.sh --daemon` - - The BE process will start and go into the background for execution. Logs are stored in be/log/directory by default. If startup fails, you can view error messages by looking at be/log/be.log or be/log/be.out. - -* View BE status - - Connect to FE using mysql-client and execute `SHOW PROC'/ backends'; `View BE operation. If everything is normal, the `isAlive`column should be `true`. - -#### (Optional) FS_Broker deployment - -Broker is deployed as a plug-in, independent of Doris. If you need to import data from a third-party storage system, you need to deploy the corresponding Broker. By default, it provides fs_broker to read HDFS and Baidu cloud BOS. Fs_broker is stateless and it is recommended that each FE and BE node deploy a Broker. - -* Copy the corresponding Broker directory in the output directory of the source fs_broker to all the nodes that need to be deployed. It is recommended to maintain the same level as the BE or FE directories. - -* Modify the corresponding Broker configuration - - In the corresponding broker/conf/directory configuration file, you can modify the corresponding configuration. - -* Start Broker - - `sh bin /start'u broker.sh --daemon ` start Broker - -* Add Broker - - To let Doris FE and BE know which nodes Broker is on, add a list of Broker nodes by SQL command. - - Use mysql-client to connect the FE started, and execute the following commands: - - `ALTER SYSTEM ADD BROKER broker_name "host1:port1","host2:port2",...;` - - The host is Broker's node ip; the port is brokeripcport in the Broker configuration file. - -* View Broker status - - Connect any booted FE using mysql-client and execute the following command to view Broker status: `SHOW PROC '/brokers';` - -**Note: In production environments, daemons should be used to start all instances to ensure that processes are automatically pulled up after they exit, such as [Supervisor] (http://supervisord.org/). For daemon startup, in 0.9.0 and previous versions, you need to modify the start_xx.sh scripts to remove the last & symbol**. Starting with version 0.10.0, call `sh start_xx.sh` directly to start. Also refer to [here] (https://www.cnblogs.com/lenmom/p/9973401.html) - -## Expansion and contraction - -Doris can easily expand and shrink FE, BE, Broker instances. - -### FE Expansion and Compression - -High availability of FE can be achieved by expanding FE to three top-one nodes. - -Users can login to Master FE through MySQL client. By: - -`SHOW PROC '/frontends';` - -To view the current FE node situation. - -You can also view the FE node through the front-end page connection: ``http://fe_hostname: fe_http_port/frontend`` or ```http://fe_hostname: fe_http_port/system? Path=//frontends```. - -All of the above methods require Doris's root user rights. - -The process of FE node expansion and contraction does not affect the current system operation. - -#### Adding FE nodes - -FE is divided into three roles: Leader, Follower and Observer. By default, a cluster can have only one Leader and multiple Followers and Observers. Leader and Follower form a Paxos selection group. If the Leader goes down, the remaining Followers will automatically select a new Leader to ensure high write availability. Observer synchronizes Leader data, but does not participate in the election. If only one FE is deployed, FE defaults to Leader. - -The first FE to start automatically becomes Leader. On this basis, several Followers and Observers can be added. - -Add Follower or Observer. Connect to the started FE using mysql-client and execute: - -`ALTER SYSTEM ADD FOLLOWER "host:port";` - -or - -`ALTER SYSTEM ADD OBSERVER "host:port";` - -The host is the node IP of Follower or Observer, and the port is edit\_log\_port in its configuration file fe.conf. - -Configure and start Follower or Observer. Follower and Observer are configured with Leader. The following commands need to be executed at the first startup: - -`./bin/start_fe.sh --helper host:port --daemon` - -The host is the node IP of Leader, and the port is edit\_log\_port in Lead's configuration file fe.conf. The --helper is only required when follower/observer is first startup. - -View the status of Follower or Observer. Connect to any booted FE using mysql-client and execute: SHOW PROC'/frontends'; you can view the FE currently joined the cluster and its corresponding roles. - -> Notes for FE expansion: -> -> 1. The number of Follower FEs (including Leaders) must be odd. It is recommended that a maximum of three constituent high availability (HA) modes be deployed. -> 2. When FE is in a highly available deployment (1 Leader, 2 Follower), we recommend that the reading service capability of FE be extended by adding Observer FE. Of course, you can continue to add Follower FE, but it's almost unnecessary. -> 3. Usually a FE node can handle 10-20 BE nodes. It is suggested that the total number of FE nodes should be less than 10. Usually three can meet most of the needs. - -#### Delete FE nodes - -Delete the corresponding FE node using the following command: - -```ALTER SYSTEM DROP FOLLOWER[OBSERVER] "fe_host:edit_log_port";``` - -> Notes for FE contraction: -> -> 1. When deleting Follower FE, make sure that the remaining Follower (including Leader) nodes are odd. - -### BE Expansion and Compression - -Users can login to Leader FE through mysql-client. By: - -```SHOW PROC '/backends';``` - -To see the current BE node situation. - -You can also view the BE node through the front-end page connection: ``http://fe_hostname: fe_http_port/backend`` or ``http://fe_hostname: fe_http_port/system? Path=//backends``. - -All of the above methods require Doris's root user rights. - -The expansion and scaling process of BE nodes does not affect the current system operation and the tasks being performed, and does not affect the performance of the current system. Data balancing is done automatically. Depending on the amount of data available in the cluster, the cluster will be restored to load balancing in a few hours to a day. For cluster load, see the [Tablet Load Balancing Document] (../administrator-guide/operation/tablet-repair-and-balance.md). - -#### Add BE nodes - -The BE node is added in the same way as in the **BE deployment** section. The BE node is added by the `ALTER SYSTEM ADD BACKEND` command. - -> Notes for BE expansion: -> -> 1. After BE expansion, Doris will automatically balance the data according to the load, without affecting the use during the period. - -#### Delete BE nodes - -There are two ways to delete BE nodes: DROP and DECOMMISSION - -The DROP statement is as follows: - -```ALTER SYSTEM DROP BACKEND "be_host:be_heartbeat_service_port";``` - -**Note: DROP BACKEND will delete the BE directly and the data on it will not be recovered!!! So we strongly do not recommend DROP BACKEND to delete BE nodes. When you use this statement, there will be corresponding error-proof operation hints.** - -DECOMMISSION clause： - -```ALTER SYSTEM DECOMMISSION BACKEND "be_host:be_heartbeat_service_port";``` - -> DECOMMISSION notes: -> -> 1. This command is used to safely delete BE nodes. After the command is issued, Doris attempts to migrate the data on the BE to other BE nodes, and when all data is migrated, Doris automatically deletes the node. -> 2. The command is an asynchronous operation. After execution, you can see that the BE node's isDecommission status is true through ``SHOW PROC '/backends';` Indicates that the node is offline. -> 3. The order **does not necessarily carry out successfully**. For example, when the remaining BE storage space is insufficient to accommodate the data on the offline BE, or when the number of remaining machines does not meet the minimum number of replicas, the command cannot be completed, and the BE will always be in the state of isDecommission as true. -> 4. The progress of DECOMMISSION can be viewed through `SHOW PROC '/backends';` Tablet Num, and if it is in progress, Tablet Num will continue to decrease. -> 5. The operation can be carried out by: -> ```CANCEL ALTER SYSTEM DECOMMISSION BACKEND "be_host:be_heartbeat_service_port";``` -> The order was cancelled. When cancelled, the data on the BE will maintain the current amount of data remaining. Follow-up Doris re-load balancing - -**For expansion and scaling of BE nodes in multi-tenant deployment environments, please refer to the [Multi-tenant Design Document] (./administrator-guide/operation/multi-tenant.md).** - -### Broker Expansion and Shrinkage - -There is no rigid requirement for the number of Broker instances. Usually one physical machine is deployed. Broker addition and deletion can be accomplished by following commands: - -```ALTER SYSTEM ADD BROKER broker_name "broker_host:broker_ipc_port";``` -```ALTER SYSTEM DROP BROKER broker_name "broker_host:broker_ipc_port";``` -```ALTER SYSTEM DROP ALL BROKER broker_name;``` - -Broker is a stateless process that can be started or stopped at will. Of course, when it stops, the job running on it will fail. Just try again. - -## Common Questions - -### Process correlation - -1. How to determine the success of FE process startup - - After the FE process starts, metadata is loaded first. According to the different roles of FE, you can see ```transfer from UNKNOWN to MASTER/FOLLOWER/OBSERVER```in the log. Eventually, you will see the ``thrift server started`` log and connect to FE through MySQL client, which indicates that FE started successfully. - - You can also check whether the startup was successful by connecting as follows: - - `http://fe_host:fe_http_port/api/bootstrap` - - If returned: - - `{"status":"OK","msg":"Success"}` - - The startup is successful, there may be problems in other cases. - - > Note: If you can't see the information of boot failure in fe. log, you may see it in fe. out. - -2. How to determine the success of BE process startup - - After the BE process starts, if there is data before, there may be several minutes of data index loading time. - - If BE is started for the first time or the BE has not joined any cluster, the BE log will periodically scroll the words `waiting to receive first heartbeat from frontend`. BE has not received Master's address through FE's heartbeat and is waiting passively. This error log will disappear after ADD BACKEND in FE sends the heartbeat. If the word `````master client', get client from cache failed. host:, port: 0, code: 7````` master client'appears again after receiving heartbeat, it indicates that FE has successfully connected BE, but BE cannot actively connect FE. It may be necessary to check the connectivity of rpc_port from BE to FE. - - If BE has been added to the cluster, the heartbeat log from FE should be scrolled every five seconds: ```get heartbeat, host:xx. xx.xx.xx, port:9020, cluster id:xxxxxxx```, indicating that the heartbeat is normal. - - Secondly, the word `finish report task success. return code: 0` should be scrolled every 10 seconds in the log to indicate that BE's communication to FE is normal. - - At the same time, if there is a data query, you should see the rolling logs, and have `execute time is xxx` logs, indicating that BE started successfully, and the query is normal. - - You can also check whether the startup was successful by connecting as follows: - - `http://be_host:be_http_port/api/health` - - If returned: - - `{"status": "OK","msg": "To Be Added"}` - - If the startup is successful, there may be problems in other cases. - - > Note: If you can't see the information of boot failure in be.INFO, you may see it in be.out. - -3. How to determine the normal connectivity of FE and BE after building the system - - Firstly, confirm that FE and BE processes have been started separately and normally, and confirm that all nodes have been added through `ADD BACKEND` or `ADD FOLLOWER/OBSERVER` statements. - - If the heartbeat is normal, BE logs will show ``get heartbeat, host:xx.xx.xx.xx, port:9020, cluster id:xxxxx`` If the heartbeat fails, the words ```backend [10001] get Exception: org.apache.thrift.transport.TTransportException``` will appear in FE's log, or other thrift communication abnormal log, indicating that the heartbeat fails from FE to 10001 BE. Here you need to check the connectivity of FE to BE host's heart-beating port. - - If BE's communication to FE is normal, the BE log will display the words `finish report task success. return code: 0`. Otherwise, the words `master client`, get client from cache failed` will appear. In this case, the connectivity of BE to the rpc_port of FE needs to be checked. - -4. Doris Node Authentication Mechanism - - In addition to Master FE, the other role nodes (Follower FE, Observer FE, Backend) need to register to the cluster through the `ALTER SYSTEM ADD` statement before joining the cluster. - - When Master FE is first started, a cluster_id is generated in the doris-meta/image/VERSION file. - - When FE first joins the cluster, it first retrieves the file from Master FE. Each subsequent reconnection between FEs (FE reboot) checks whether its cluster ID is the same as that of other existing FEs. If different, the FE will exit automatically. - - When BE first receives the heartbeat of Master FE, it gets the cluster ID from the heartbeat and records it in the `cluster_id` file of the data directory. Each heartbeat after that compares to the cluster ID sent by FE. If cluster IDs are not equal, BE will refuse to respond to FE's heartbeat. - - The heartbeat also contains Master FE's ip. When FE cuts the master, the new Master FE will carry its own IP to send the heartbeat to BE, BE will update its own saved Master FE ip. - - > **priority\_network** - > - > priority network is that both FE and BE have a configuration. Its main purpose is to assist FE or BE to identify their own IP addresses in the case of multi-network cards. Priority network is represented by CIDR: [RFC 4632] (https://tools.ietf.org/html/rfc4632) - > - > When the connectivity of FE and BE is confirmed to be normal, if the table Timeout still occurs, and the FE log has an error message with the words `backend does not find. host:xxxx.xxx.XXXX`. This means that there is a problem with the IP address that Doris automatically identifies and that priority\_network parameters need to be set manually. - > - > The main reason for this problem is that when the user adds BE through the `ADD BACKEND` statement, FE recognizes whether the statement specifies hostname or IP. If it is hostname, FE automatically converts it to an IP address and stores it in metadata. When BE reports on the completion of the task, it carries its own IP address. If FE finds that BE reports inconsistent IP addresses and metadata, it will make the above error. - > - > Solutions to this error: 1) Set **priority\_network** parameters in FE and BE respectively. Usually FE and BE are in a network segment, so this parameter can be set to the same. 2) Fill in the `ADD BACKEND` statement directly with the correct IP address of BE instead of hostname to avoid FE getting the wrong IP address. - -5. File descriptor number of BE process - - The number of file descriptor of BE process is controlled by the two parameters min_file_descriptor_number/max_file_descriptor_number. - - If it is not in the [min_file_descriptor_number, max_file_descriptor_number] interval, error will occurs when starting BE process. - - Please using ulimit command to set file descriptor under this circumstance. - - The default value of min_file_descriptor_number is 65536. - - The default value of max_file_descriptor_number is 131072. - - For Example : ulimit -n 65536; this command set file descriptor to 65536. - - After starting BE process, you can use **cat /proc/$pid/limits** to see the actual limit of process. diff --git a/docs/documentation/en/installing/upgrade_EN.md b/docs/documentation/en/installing/upgrade_EN.md deleted file mode 100644 index 5dde75586e053b..00000000000000 --- a/docs/documentation/en/installing/upgrade_EN.md +++ /dev/null @@ -1,57 +0,0 @@ - - - -# Cluster upgrade - -Doris can upgrade smoothly by rolling upgrades. The following steps are recommended for security upgrade. - -> Note: -> 1. The following approaches are based on highly available deployments. That is, data 3 replicas, FE high availability. - -## Test the correctness of BE upgrade - -1. Arbitrarily select a BE node and deploy the latest palo_be binary file. -2. Restart the BE node and check the BE log be.INFO to see if the boot was successful. -3. If the startup fails, you can check the reason first. If the error is not recoverable, you can delete the BE directly through DROP BACKEND, clean up the data, and restart the BE using the previous version of palo_be. Then re-ADD BACKEND. (**This method will result in the loss of a copy of the data, please make sure that three copies are complete, and perform this operation!!!** - -## Testing FE Metadata Compatibility - -0. **Important! Exceptional metadata compatibility is likely to cause data can not be restored!!** -1. Deploy a test FE process (such as your own local developer) using the new version alone. -2. Modify the FE configuration file fe.conf for testing and set all ports to **different from online**. -3. Add configuration in fe.conf: cluster_id=123456 -4. Add the configuration in fe.conf: metadatafailure_recovery=true -5. Copy the metadata directory palo-meta of the online environment Master FE to the test environment -6. Modify the cluster_id in the palo-meta/image/VERSION file copied into the test environment to 123456 (that is, the same as in Step 3) -7. "27979;" "35797;" "3681616;" sh bin /start fe.sh "21551;" FE -8. Observe whether the start-up is successful through FE log fe.log. -9. If the startup is successful, run sh bin/stop_fe.sh to stop the FE process of the test environment. -10. **The purpose of the above 2-6 steps is to prevent the FE of the test environment from being misconnected to the online environment after it starts.** - -## Upgrade preparation - -1. After data validation, the new version of BE and FE binary files are distributed to their respective directories. -2. Usually small version upgrade, BE only needs to upgrade palo_be; FE only needs to upgrade palo-fe.jar. If it is a large version upgrade, you may need to upgrade other files (including but not limited to bin / lib / etc.) If you are not sure whether you need to replace other files, it is recommended to replace all of them. - -## rolling upgrade - -1. Confirm that the new version of the file is deployed. Restart FE and BE instances one by one. -2. It is suggested that BE be restarted one by one and FE be restarted one by one. Because Doris usually guarantees backward compatibility between FE and BE, that is, the old version of FE can access the new version of BE. However, the old version of BE may not be supported to access the new version of FE. -3. It is recommended to restart the next instance after confirming that the previous instance started successfully. Refer to the Installation Deployment Document for the identification of successful instance startup. diff --git a/docs/documentation/en/internal/doris_storage_optimization_EN.md b/docs/documentation/en/internal/doris_storage_optimization_EN.md deleted file mode 100644 index 028bb361ab00c6..00000000000000 --- a/docs/documentation/en/internal/doris_storage_optimization_EN.md +++ /dev/null @@ -1,226 +0,0 @@ - - - -# Doris Storage File Format Optimization # - -## File format ## - -![](../../../resources/images/segment_v2.png) -
1. doris segment
- -Documents include: -- The file starts with an 8-byte magic code to identify the file format and version -- Data Region: Used to store data information for each column, where the data is loaded on demand by pages. -- Index Region: Doris stores the index data of each column in Index Region, where the data is loaded according to column granularity, so the data information of the following column is stored separately. -- Footer信息 - - FileFooterPB: Metadata Information for Definition Files - - Chesum of 4 bytes of footer Pb content - - Four bytes FileFooterPB message length for reading FileFooterPB - - The 8 byte MAGIC CODE is stored in the last bit to facilitate the identification of file types in different scenarios. - -The data in the file is organized in the form of page, which is the basic unit of coding and compression. Current page types include the following: - -### DataPage ### - -Data Page is divided into two types: nullable and non-nullable data pages. - -Nullable's data page includes: -``` - - +----------------+ - | value count | - |----------------| - | first row id | - |----------------| - | bitmap length | - |----------------| - | null bitmap | - |----------------| - | data | - |----------------| - | checksum | - +----------------+ -``` - -non -zero data page32467;- 26500;- 229140;- - -``` - |----------------| - | value count | - |----------------| - | first row id | - |----------------| - | data | - |----------------| - | checksum | - +----------------+ -``` - -The meanings of each field are as follows: - -- value count - - Represents the number of rows in a page -- First row id - - Line number of the first line in page -- bitmap length - - Represents the number of bytes in the next bitmap -- null bitmap - - bitmap representing null information -- Data - - Store data after encoding and compress - - You need to write in the header information of the data: is_compressed - - Various kinds of data encoded by different codes need to write some field information in the header information in order to achieve data parsing. - - TODO: Add header information for various encodings -- Checksum - - Store page granularity checksum, including page header and subsequent actual data - - -### Bloom Filter Pages ### - -For each bloom filter column, a page of the bloom filter is generated corresponding to the granularity of the page and saved in the bloom filter pages area. - -### Ordinal Index Page ### - -For each column, a sparse index of row numbers is established according to page granularity. The content is a pointer to the block (including offset and length) for the line number of the start line of the page - -### Short Key Index page ### - -We generate a sparse index of short key every N rows (configurable) with the contents of short key - > line number (ordinal) - -### Column's other indexes### - -The format design supports the subsequent expansion of other index information, such as bitmap index, spatial index, etc. It only needs to write the required data to the existing column data, and add the corresponding metadata fields to FileFooterPB. - -### Metadata Definition### -FileFooterPB is defined as: - -``` -message ColumnPB { - optional uint32 column_id = 1; // 这里使用column id，不使用column name是因为计划支持修改列名 - optional string type = 2; // 列类型 - optional string aggregation = 3; // 是否聚合 - optional uint32 length = 4; // 长度 - optional bool is_key = 5; // 是否是主键列 - optional string default_value = 6; // 默认值 - optional uint32 precision = 9 [default = 27]; // 精度 - optional uint32 frac = 10 [default = 9]; - optional bool is_nullable = 11 [default=false]; // 是否有null - optional bool is_bf_column = 15 [default=false]; // 是否有bf词典 - optional bool is_bitmap_column = 16 [default=false]; // 是否有bitmap索引 -} - -// page偏移 -message PagePointerPB { - required uint64 offset; // page在文件中的偏移 - required uint32 length; // page的大小 -} - -message MetadataPairPB { - optional string key = 1; - optional bytes value = 2; -} - -message ColumnMetaPB { - optional ColumnMessage encoding; // 编码方式 - - optional PagePointerPB dict_page // 词典page - repeated PagePointerPB bloom_filter_pages; // bloom filter词典信息 - optional PagePointerPB ordinal_index_page; // 行号索引数据 - optional PagePointerPB page_zone_map_page; // page级别统计信息索引数据 - - optional PagePointerPB bitmap_index_page; // bitmap索引数据 - - optional uint64 data_footprint; // 列中索引的大小 - optional uint64 index_footprint; // 列中数据的大小 - optional uint64 raw_data_footprint; // 原始列数据大小 - - optional CompressKind compress_kind; // 列的压缩方式 - - optional ZoneMapPB column_zone_map; //文件级别的过滤条件 - repeated MetadataPairPB column_meta_datas; -} - -message FileFooterPB { - optional uint32 version = 2 [default = 1]; // 用于版本兼容和升级使用 - repeated ColumnPB schema = 5; // 列Schema - optional uint64 num_values = 4; // 文件中保存的行数 - optional uint64 index_footprint = 7; // 索引大小 - optional uint64 data_footprint = 8; // 数据大小 - optional uint64 raw_data_footprint = 8; // 原始数据大小 - - optional CompressKind compress_kind = 9 [default = COMPRESS_LZO]; // 压缩方式 - repeated ColumnMetaPB column_metas = 10; // 列元数据 - optional PagePointerPB key_index_page; // short key索引page -} - -``` - -## Read-write logic## - -### Write ### - -The general writing process is as follows: -1. Write magic -2. Generate corresponding Column Writer according to schema information. Each Column Writer obtains corresponding encoding information (configurable) according to different types, and generates corresponding encoder according to encoding. -3. Call encoder - > add (value) for data writing. Each K line generates a short key index entry, and if the current page satisfies certain conditions (the size exceeds 1M or the number of rows is K), a new page is generated and cached in memory. -4. Continuous cycle step 3 until data writing is completed. Brush the data of each column into the file in sequence -5. Generate FileFooterPB information and write it to the file. - -Relevant issues: - -- How does the index of short key be generated? - - Now we still generate a short key sparse index according to how many rows are sparse, and keep a short sparse index generated every 1024 rows. The specific content is: short key - > ordinal - -- What should be stored in the ordinal index? - - Store the first ordinal to page pointer mapping information for pages -- What are stored in pages of different encoding types? - - Dictionary Compression - - plain - - rle - - bshuf - -### Read### - -1. Read the magic of the file and judge the type and version of the file. -2. Read FileFooterPB and check sum -3. Read short key index and data ordinal index information of corresponding columns according to required columns -4. Use start key and end key, locate the row number to be read through short key index, then determine the row ranges to be read through ordinal index, and filter the row ranges to be read through statistics, bitmap index and so on. -5. Then read row data through ordinal index according to row ranges - -Relevant issues: -1. How to quickly locate a row within the page? - - The data inside the page is encoding, so it can not locate the row-level data quickly. Different encoding methods have different schemes for fast line number positioning in-house, which need to be analyzed concretely: - - If it is rle-coded, skip is performed by resolving the head of RLE until the RLE block containing the row is reached, and then the reverse solution is performed. - - binary plain encoding: offset information will be stored in the page, and offset information will be specified in the page header. When reading, offset information will be parsed into the array first, so that you can quickly locate the data of a row of block through offset data information of each row. -2. How to achieve efficient block reading? Consider merging adjacent blocks while they are being read, one-time reading? -This requires judging whether the block is continuous at the time of reading, and if it is continuous, reading it once. - -## Coding## - -In the existing Doris storage, plain encoding is adopted for string type encoding, which is inefficient. After comparison, it is found that in Baidu statistics scenario, data will expand more than twice because of string type coding. Therefore, it is planned to introduce dictionary-based coding compression. - -## Compression## - -It implements a scalable compression framework, supports a variety of compression algorithms, facilitates the subsequent addition of new compression algorithms, and plans to introduce zstd compression. - -## TODO ## -1. How to implement nested types? How to locate line numbers in nested types? -2. How to optimize the downstream bitmap and column statistics statistics caused by ScanRange splitting? diff --git a/docs/documentation/en/internal/grouping_sets_design_EN.md b/docs/documentation/en/internal/grouping_sets_design_EN.md deleted file mode 100644 index e1c82b41e8bdb7..00000000000000 --- a/docs/documentation/en/internal/grouping_sets_design_EN.md +++ /dev/null @@ -1,494 +0,0 @@ - -# GROUPING SETS DESIGN - -## 1. GROUPING SETS Background - -The `CUBE`, `ROLLUP`, and `GROUPING` `SETS` extensions to SQL make querying and reporting easier and faster. `CUBE`, `ROLLUP`, and grouping sets produce a single result set that is equivalent to a `UNION` `ALL` of differently grouped rows. `ROLLUP` calculates aggregations such as `SUM`, `COUNT`, `MAX`, `MIN`, and `AVG` at increasing levels of aggregation, from the most detailed up to a grand total. `CUBE` is an extension similar to `ROLLUP`, enabling a single statement to calculate all possible combinations of aggregations. The `CUBE`, `ROLLUP`, and the `GROUPING` `SETS` extension lets you specify just the groupings needed in the `GROUP` `BY` clause. This allows efficient analysis across multiple dimensions without performing a `CUBE` operation. Computing a `CUBE` creates a heavy processing load, so replacing cubes with grouping sets can significantly increase performance. -To enhance performance, `CUBE`, `ROLLUP`, and `GROUPING SETS` can be parallelized: multiple processes can simultaneously execute all of these statements. These capabilities make aggregate calculations more efficient, thereby enhancing database performance, and scalability. - -The three `GROUPING` functions help you identify the group each row belongs to and enable sorting subtotal rows and filtering results. - -### 1.1 GROUPING SETS Syntax - -`GROUPING SETS` syntax lets you define multiple groupings in the same query. `GROUP BY` computes all the groupings specified and combines them with `UNION ALL`. For example, consider the following statement: - -``` -SELECT k1, k2, SUM( k3 ) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k1), (k2), ( ) ); -``` - - -This statement is equivalent to: - -``` -SELECT k1, k2, SUM( k3 ) FROM t GROUP BY k1, k2 -UNION -SELECT k1, null, SUM( k3 ) FROM t GROUP BY k1 -UNION -SELECT null, k2, SUM( k3 ) FROM t GROUP BY k2 -UNION -SELECT null, null, SUM( k3 ) FROM t -``` - -This is an example of real query： - -``` -mysql> SELECT * FROM t; -+------+------+------+ -| k1 | k2 | k3 | -+------+------+------+ -| a | A | 1 | -| a | A | 2 | -| a | B | 1 | -| a | B | 3 | -| b | A | 1 | -| b | A | 4 | -| b | B | 1 | -| b | B | 5 | -+------+------+------+ -8 rows in set (0.01 sec) - -mysql> SELECT k1, k2, SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); -+------+------+-----------+ -| k1 | k2 | sum(`k3`) | -+------+------+-----------+ -| b | B | 6 | -| a | B | 4 | -| a | A | 3 | -| b | A | 5 | -| NULL | B | 10 | -| NULL | A | 8 | -| a | NULL | 7 | -| b | NULL | 11 | -| NULL | NULL | 18 | -+------+------+-----------+ -9 rows in set (0.06 sec) -``` - -### 1.2 ROLLUP Syntax - -`ROLLUP` enables a `SELECT` statement to calculate multiple levels of subtotals across a specified group of dimensions. It also calculates a grand total. `ROLLUP` is a simple extension to the `GROUP` `BY` clause, so its syntax is extremely easy to use. The `ROLLUP` extension is highly efficient, adding minimal overhead to a query. - -`ROLLUP` appears in the `GROUP` `BY` clause in a `SELECT` statement. Its form is: - -``` -SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY ROLLUP(a,b,c) -``` - -This statement is equivalent to GROUPING SETS as followed： - -``` -GROUPING SETS ( -(a,b,c), -( a, b ), -( a), -( ) -) -``` - -### 1.3 CUBE Syntax - -Like `ROLLUP` `CUBE` generates all the subtotals that could be calculated for a data cube with the specified dimensions. - -``` -SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY CUBE(a,b,c) -``` - -e.g. CUBE ( a, b, c ) is equivalent to GROUPING SETS as followed: - -``` -GROUPING SETS ( -( a, b, c ), -( a, b ), -( a, c ), -( a ), -( b, c ), -( b ), -( c ), -( ) -) -``` - -### 1.4 GROUPING and GROUPING_ID Function - -Indicates whether a specified column expression in a `GROUP BY` list is aggregated or not. `GROUPING `returns 1 for aggregated or 0 for not aggregated in the result set. `GROUPING` can be used only in the `SELECT` list, `HAVING`, and `ORDER BY` clauses when `GROUP BY` is specified. - -`GROUPING_ID` describes which of a list of expressions are grouped in a row produced by a `GROUP BY` query. The `GROUPING_ID` function simply returns the decimal equivalent of the binary value formed as a result of the concatenation of the values returned by the `GROUPING` functions. - -Each `GROUPING_ID` argument must be an element of the `GROUP BY` list. `GROUPING_ID ()` returns an **integer** bitmap whose lowest N bits may be lit. A lit **bit** indicates the corresponding argument is not a grouping column for the given output row. The lowest-order **bit** corresponds to argument N, and the N-1th lowest-order **bit** corresponds to argument 1. If the column is a grouping column the bit is 0 else is 1. - -For example： - -``` -mysql> select * from t; -+------+------+------+ -| k1 | k2 | k3 | -+------+------+------+ -| a | A | 1 | -| a | A | 2 | -| a | B | 1 | -| a | B | 3 | -| b | A | 1 | -| b | A | 4 | -| b | B | 1 | -| b | B | 5 | -+------+------+------+ -``` - -grouping sets result： - -``` -mysql> SELECT k1, k2, GROUPING(k1), GROUPING(k2), SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); -+------+------+----------------+----------------+-----------+ -| k1 | k2 | grouping(`k1`) | grouping(`k2`) | sum(`k3`) | -+------+------+----------------+----------------+-----------+ -| a | A | 0 | 0 | 3 | -| a | B | 0 | 0 | 4 | -| a | NULL | 0 | 1 | 7 | -| b | A | 0 | 0 | 5 | -| b | B | 0 | 0 | 6 | -| b | NULL | 0 | 1 | 11 | -| NULL | A | 1 | 0 | 8 | -| NULL | B | 1 | 0 | 10 | -| NULL | NULL | 1 | 1 | 18 | -+------+------+----------------+----------------+-----------+ -9 rows in set (0.02 sec) - -mysql> SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); -+------+------+-------------------------+-----------+ -| k1 | k2 | grouping_id(`k1`, `k2`) | sum(`k3`) | -+------+------+-------------------------+-----------+ -| a | A | 0 | 3 | -| a | B | 0 | 4 | -| a | NULL | 1 | 7 | -| b | A | 0 | 5 | -| b | B | 0 | 6 | -| b | NULL | 1 | 11 | -| NULL | A | 2 | 8 | -| NULL | B | 2 | 10 | -| NULL | NULL | 3 | 18 | -+------+------+-------------------------+-----------+ -9 rows in set (0.02 sec) - -mysql> SELECT k1, k2, grouping(k1), grouping(k2), GROUPING_ID(k1,k2), SUM(k4) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ) order by k1, k2; -+------+------+----------------+----------------+-------------------------+-----------+ -| k1 | k2 | grouping(`k1`) | grouping(`k2`) | grouping_id(`k1`, `k2`) | sum(`k4`) | -+------+------+----------------+----------------+-------------------------+-----------+ -| a | A | 0 | 0 | 0 | 3 | -| a | B | 0 | 0 | 0 | 4 | -| a | NULL | 0 | 1 | 1 | 7 | -| b | A | 0 | 0 | 0 | 5 | -| b | B | 0 | 0 | 0 | 6 | -| b | NULL | 0 | 1 | 1 | 11 | -| NULL | A | 1 | 0 | 2 | 8 | -| NULL | B | 1 | 0 | 2 | 10 | -| NULL | NULL | 1 | 1 | 3 | 18 | -+------+------+----------------+----------------+-------------------------+-----------+ -9 rows in set (0.02 sec) - -``` -### 1.5 Composition and nesting of GROUPING SETS - -First of all, a GROUP BY clause is essentially a special case of GROUPING SETS, for example: - -``` - GROUP BY a -is equivalent to: - GROUP BY GROUPING SETS((a)) -also， - GROUP BY a,b,c -is equivalent to: - GROUP BY GROUPING SETS((a,b,c)) -``` - -Similarly, CUBE and ROLLUP can be expanded into GROUPING SETS, so the various combinations and nesting of GROUP BY, CUBE, ROLLUP, GROUPING SETS are essentially the combination and nesting of GROUPING SETS. - -For GROUPING SETS nesting, it is semantically equivalent to writing the statements inside the nest directly outside. (ref:) mentions: - -``` -The CUBE and ROLLUP constructs can be used either directly in the GROUP BY clause, or nested inside a GROUPING SETS clause. If one GROUPING SETS clause is nested inside another, the effect is the same as if all the elements of the inner clause had been written directly in the outer clause. -``` - -For a combined list of multiple GROUPING SETS, many databases consider it a cross product relationship. - -for example: - -``` -GROUP BY a, CUBE (b, c), GROUPING SETS ((d), (e)) - -is equivalent to: - -GROUP BY GROUPING SETS ( -(a, b, c, d), (a, b, c, e), -(a, b, d), (a, b, e), -(a, c, d), (a, c, e), -(a, d), (a, e) -) -``` - -For the combination and nesting of GROUPING SETS, each database support is not the same. For example snowflake does not support any combination and nesting. -（） - -Oracle supports both composition and nesting. -（） - -Presto supports composition, but not nesting. -（） - -## 2. Object - -Support `GROUPING SETS`， `ROLLUP` and `CUBE ` syntax，impliments 1.1, 1.2, 1.3 1.4, 1.5, not support the combination - and nesting of GROUPING SETS at current version. - -### 2.1 GROUPING SETS Syntax - -``` -SELECT ... -FROM ... -[ ... ] -GROUP BY GROUPING SETS ( groupSet [ , groupSet [ , ... ] ] ) -[ ... ] - -groupSet ::= { ( expr [ , expr [ , ... ] ] )} - - -Expression，column name. -``` - -### 2.2 ROLLUP Syntax - -``` -SELECT ... -FROM ... -[ ... ] -GROUP BY ROLLUP ( expr [ , expr [ , ... ] ] ) -[ ... ] - - -Expression，column name. -``` - -### 2.3 CUBE Syntax - -``` -SELECT ... -FROM ... -[ ... ] -GROUP BY CUBE ( expr [ , expr [ , ... ] ] ) -[ ... ] - - -Expression，column name. -``` - -## 3. Implementation - -### 3.1 Overall Design Approaches - -For `GROUPING SET` is equivalent to the `UNION` of `GROUP BY` . So we can expand input rows, and run an GROUP BY on these rows。 - -For example： - -``` -SELECT a, b FROM src GROUP BY a, b GROUPING SETS ((a, b), (a), (b), ()); -``` - -Data in table src : - -``` -1, 2 -3, 4 -``` - -Base on GROUPING SETS , we can expend the input to： - -``` -1, 2 (GROUPING_ID: a, b -> 00 -> 0) -1, null (GROUPING_ID: a, null -> 01 -> 1) -null, 2 (GROUPING_ID: null, b -> 10 -> 2) -null, null (GROUPING_ID: null, null -> 11 -> 3) - -3, 4 (GROUPING_ID: a, b -> 00 -> 0) -3, null (GROUPING_ID: a, null -> 01 -> 1) -null, 4 (GROUPING_ID: null, b -> 10 -> 2) -null, null (GROUPING_ID: null, null -> 11 -> 3) -``` - -And then use those row as input, then GROUP BY a, b, GROUPING_ID - -### 3.2 Example - -Table t： - -``` -mysql> select * from t; -+------+------+------+ -| k1 | k2 | k3 | -+------+------+------+ -| a | A | 1 | -| a | A | 2 | -| a | B | 1 | -| a | B | 3 | -| b | A | 1 | -| b | A | 4 | -| b | B | 1 | -| b | B | 5 | -+------+------+------+ -8 rows in set (0.01 sec) -``` - -for the query： - -``` -SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ()); -``` - -First，expand the input，every row expand into 4 rows ( the size of GROUPING SETS), and insert GROUPING_ID column - -e.g. a, A, 1 expanded to： - -``` -+------+------+------+-------------------------+ -| k1 | k2 | k3 | GROUPING_ID(`k1`, `k2`) | -+------+------+------+-------------------------+ -| a | A | 1 | 0 | -| a | NULL | 1 | 1 | -| NULL | A | 1 | 2 | -| NULL | NULL | 1 | 3 | -+------+------+------+-------------------------+ -``` - -Finally, all rows expended as follows (32 rows): - -``` -+------+------+------+-------------------------+ -| k1 | k2 | k3 | GROUPING_ID(`k1`, `k2`) | -+------+------+------+-------------------------+ -| a | A | 1 | 0 | -| a | A | 2 | 0 | -| a | B | 1 | 0 | -| a | B | 3 | 0 | -| b | A | 1 | 0 | -| b | A | 4 | 0 | -| b | B | 1 | 0 | -| b | B | 5 | 0 | -| a | NULL | 1 | 1 | -| a | NULL | 1 | 1 | -| a | NULL | 2 | 1 | -| a | NULL | 3 | 1 | -| b | NULL | 1 | 1 | -| b | NULL | 1 | 1 | -| b | NULL | 4 | 1 | -| b | NULL | 5 | 1 | -| NULL | A | 1 | 2 | -| NULL | A | 1 | 2 | -| NULL | A | 2 | 2 | -| NULL | A | 4 | 2 | -| NULL | B | 1 | 2 | -| NULL | B | 1 | 2 | -| NULL | B | 3 | 2 | -| NULL | B | 5 | 2 | -| NULL | NULL | 1 | 3 | -| NULL | NULL | 1 | 3 | -| NULL | NULL | 1 | 3 | -| NULL | NULL | 1 | 3 | -| NULL | NULL | 2 | 3 | -| NULL | NULL | 3 | 3 | -| NULL | NULL | 4 | 3 | -| NULL | NULL | 5 | 3 | -+------+------+------+-------------------------+ -32 rows in set. -``` - -now GROUP BY k1, k2, GROUPING_ID(k1,k2): - -``` -+------+------+-------------------------+-----------+ -| k1 | k2 | grouping_id(`k1`, `k2`) | sum(`k3`) | -+------+------+-------------------------+-----------+ -| a | A | 0 | 3 | -| a | B | 0 | 4 | -| a | NULL | 1 | 7 | -| b | A | 0 | 5 | -| b | B | 0 | 6 | -| b | NULL | 1 | 11 | -| NULL | A | 2 | 8 | -| NULL | B | 2 | 10 | -| NULL | NULL | 3 | 18 | -+------+------+-------------------------+-----------+ -9 rows in set (0.02 sec) -``` - -The result is equivalent to the UNION ALL - -``` -select k1, k2, sum(k3) from t group by k1, k2 -UNION ALL -select NULL, k2, sum(k3) from t group by k2 -UNION ALL -select k1, NULL, sum(k3) from t group by k1 -UNION ALL -select NULL, NULL, sum(k3) from t; - -+------+------+-----------+ -| k1 | k2 | sum(`k3`) | -+------+------+-----------+ -| b | B | 6 | -| b | A | 5 | -| a | A | 3 | -| a | B | 4 | -| a | NULL | 7 | -| b | NULL | 11 | -| NULL | B | 10 | -| NULL | A | 8 | -| NULL | NULL | 18 | -+------+------+-----------+ -9 rows in set (0.06 sec) -``` - -### 3.3 FE - -#### 3.3.1 Tasks - -1. Add GroupByClause, repalce groupingExprs. -2. Add Grouping Sets, Cube and RollUp syntax. -3. Add GroupByClause in SelectStmt. -4. Add GroupingFunctionCallExpr, impliments grouping grouping_id function call -5. Add VirtualSlot, generate the map of virtual slots and real slots -6. add virtual column GROUPING_ID and other virtual columns generated by grouping and grouping_id, insert into groupingExprs, -7. Add a PlanNode, name as RepeatNode. For GroupingSets aggregation insert RepeatNode to the plan. - -#### 3.3.2 Tuple - -In order to add GROUPING_ID to groupingExprs in GroupByClause, need to create virtual SlotRef, also, need tot create a tuple for this slot, named GROUPING\_\_ID Tuple. - -For the plannode RepeatNode, it's input is all the tuple of it's children, It's output tuple is the repeat data and GROUPING_ID. - - -#### 3.3.3 Expression and Function Substitution - -expr -> if(bitand(pos, grouping_id)=0, expr, null) for expr in extension grouping clause -grouping_id() -> grouping_id(grouping_id) for grouping_id function - -### 3.4 BE - -#### 3.4.1 Tasks - -1. Add RepeatNode executor, expend the input data and append GROUPING_ID to every row -2. Implements grouping_id() and grouping() function. \ No newline at end of file diff --git a/docs/documentation/en/internal/index.rst b/docs/documentation/en/internal/index.rst deleted file mode 100644 index ef52e59aee3447..00000000000000 --- a/docs/documentation/en/internal/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -==================== -Design Documents -==================== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/internal/metadata-design_EN.md b/docs/documentation/en/internal/metadata-design_EN.md deleted file mode 100644 index a2ff9ce64a5ba2..00000000000000 --- a/docs/documentation/en/internal/metadata-design_EN.md +++ /dev/null @@ -1,120 +0,0 @@ - - - -# Metadata Design Document - -## Noun Interpretation - -* FE: Frontend, the front-end node of Doris. Mainly responsible for receiving and returning client requests, metadata, cluster management, query plan generation and so on. -* BE: Backend, the back-end node of Doris. Mainly responsible for data storage and management, query plan execution and other work. -* bdbje: [Oracle Berkeley DB Java Edition] (http://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html). In Doris, we use bdbje to persist metadata operation logs and high availability of FE. - -## Overall architecture -![](../../../resources/images/palo_architecture.jpg) - -As shown above, Doris's overall architecture is divided into two layers. Multiple FEs form the first tier, providing lateral expansion and high availability of FE. Multiple BEs form the second layer, which is responsible for data storage and management. This paper mainly introduces the design and implementation of metadata in FE layer. - -1. FE 节点分为 follower 和 observer 两类。各个 FE 之间，通过 bdbje（[BerkeleyDB Java Edition](http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/overview/index-093405.html)）进行 leader 选举，数据同步等工作。 - -2. The follower node is elected, and one of the followers becomes the leader node, which is responsible for the writing of metadata. When the leader node goes down, other follower nodes re-elect a leader to ensure high availability of services. - -3. The observer node only synchronizes metadata from the leader node and does not participate in the election. It can be scaled horizontally to provide the extensibility of metadata reading services. - -> Note: The concepts of follower and observer corresponding to bdbje are replica and observer. You may use both names below. - -## Metadata structure - -Doris's metadata is in full memory. A complete metadata image is maintained in each FE memory. Within Baidu, a cluster of 2,500 tables and 1 million fragments (3 million copies) occupies only about 2GB of metadata in memory. (Of course, the memory overhead for querying intermediate objects and various job information needs to be estimated according to the actual situation. However, it still maintains a low memory overhead. - -At the same time, metadata is stored in the memory as a whole in a tree-like hierarchical structure. By adding auxiliary structure, metadata information at all levels can be accessed quickly. - -The following figure shows the contents stored in Doris meta-information. - -![](../../../resources/images/metadata_contents.png) - -As shown above, Doris's metadata mainly stores four types of data: - -1. User data information. Including database, table Schema, fragmentation information, etc. -2. All kinds of job information. For example, import jobs, Clone jobs, SchemaChange jobs, etc. -3. User and permission information. -4. Cluster and node information. - -## Data stream - -![](../../../resources/images/metadata_stream.png) - -The data flow of metadata is as follows: - -1. Only leader FE can write metadata. After modifying leader's memory, the write operation serializes into a log and writes to bdbje in the form of key-value. The key is a continuous integer, and as log id, value is the serialized operation log. - -2. After the log is written to bdbje, bdbje copies the log to other non-leader FE nodes according to the policy (write most/write all). The non-leader FE node modifies its metadata memory image by playback of the log, and completes the synchronization with the metadata of the leader node. - -3. When the number of log bars of the leader node reaches the threshold (default 10W bars), the checkpoint thread is started. Checkpoint reads existing image files and subsequent logs and replays a new mirror copy of metadata in memory. The copy is then written to disk to form a new image. The reason for this is to regenerate a mirror copy instead of writing an existing image to an image, mainly considering that the write operation will be blocked during writing the image plus read lock. So every checkpoint takes up twice as much memory space. - -4. After the image file is generated, the leader node notifies other non-leader nodes that a new image has been generated. Non-leader actively pulls the latest image files through HTTP to replace the old local files. - -5. The logs in bdbje will be deleted regularly after the image is completed. - -## Implementation details - -### Metadata catalogue - -1. The metadata directory is specified by the FE configuration item `meta_dir'. - -2. Data storage directory for bdbje under `bdb/` directory. - -3. The storage directory for image files under the `image/` directory. - -* `Image.[logid]`is the latest image file. The suffix `logid` indicates the ID of the last log contained in the image. -* `Image.ckpt` is the image file being written. If it is successfully written, it will be renamed `image.[logid]` and replaced with the original image file. -* The`cluster_id` is recorded in the `VERSION` file. `Cluster_id` uniquely identifies a Doris cluster. It is a 32-bit integer randomly generated at the first startup of leader. You can also specify a cluster ID through the Fe configuration item `cluster_id'. -* The role of FE itself recorded in the `ROLE` file. There are only `FOLLOWER` and `OBSERVER`. Where `FOLLOWER` denotes FE as an optional node. (Note: Even the leader node has a role of `FOLLOWER`) - -### Start-up process - -1. FE starts for the first time. If the startup script does not add any parameters, it will try to start as leader. You will eventually see `transfer from UNKNOWN to MASTER` in the FE startup log. - -2. FE starts for the first time. If the `-helper` parameter is specified in the startup script and points to the correct leader FE node, the FE first asks the leader node about its role (ROLE) and cluster_id through http. Then pull up the latest image file. After reading image file and generating metadata image, start bdbje and start bdbje log synchronization. After synchronization is completed, the log after image file in bdbje is replayed, and the final metadata image generation is completed. - - > Note 1: When starting with the `-helper` parameter, you need to first add the FE through the leader through the MySQL command, otherwise, the start will report an error. - - > Note 2: `-helper` can point to any follower node, even if it is not leader. - - > Note 3: In the process of synchronization log, the Fe log will show `xxx detached`. At this time, the log pull is in progress, which is a normal phenomenon. - -3. FE is not the first startup. If the startup script does not add any parameters, it will determine its identity according to the ROLE information stored locally. At the same time, according to the cluster information stored in the local bdbje, the leader information is obtained. Then read the local image file and the log in bdbje to complete the metadata image generation. (If the roles recorded in the local ROLE are inconsistent with those recorded in bdbje, an error will be reported.) - -4. FE is not the first boot, and the `-helper` parameter is specified in the boot script. Just like the first process started, the leader role is asked first. But it will be compared with the ROLE stored by itself. If they are inconsistent, they will report errors. - -#### Metadata Read-Write and Synchronization - -1. Users can use Mysql to connect any FE node to read and write metadata. If the connection is a non-leader node, the node forwards the write operation to the leader node. When the leader is successfully written, it returns a current and up-to-date log ID of the leader. Later, the non-leader node waits for the log ID it replays to be larger than the log ID it returns to the client before returning the message that the command succeeds. This approach guarantees Read-Your-Write semantics for any FE node. - - > Note: Some non-write operations are also forwarded to leader for execution. For example, `SHOW LOAD` operation. Because these commands usually need to read the intermediate states of some jobs, which are not written to bdbje, there are no such intermediate states in the memory of the non-leader node. (FE's direct metadata synchronization depends entirely on bdbje's log playback. If a metadata modification operation does not write bdbje's log, the result of the modification of the operation will not be seen in other non-leader nodes.) - -2. The leader node starts a TimePrinter thread. This thread periodically writes a key-value entry for the current time to bdbje. The remaining non-leader nodes read the recorded time in the log by playback and compare it with the local time. If the lag between the local time and the local time is found to be greater than the specified threshold (configuration item: `meta_delay_toleration_second`). If the write interval is half of the configuration item, the node will be in the **unreadable** state. This mechanism solves the problem that non-leader nodes still provide outdated metadata services after a long time of leader disconnection. - -3. The metadata of each FE only guarantees the final consistency. Normally, inconsistent window periods are only milliseconds. We guarantee the monotonous consistency of metadata access in the same session. But if the same client connects different FEs, metadata regression may occur. (But for batch update systems, this problem has little impact.) - -### Downtime recovery - -1. When the leader node goes down, the rest of the followers will immediately elect a new leader node to provide services. -2. Metadata cannot be written when most follower nodes are down. When metadata is not writable, if a write operation request occurs, the current process is that the **FE process exits**. This logic will be optimized in the future, and read services will still be provided in the non-writable state. -3. The downtime of observer node will not affect the state of any other node. It also does not affect metadata reading and writing at other nodes. diff --git a/docs/documentation/en/sql-reference/index.rst b/docs/documentation/en/sql-reference/index.rst deleted file mode 100644 index f00ee956159488..00000000000000 --- a/docs/documentation/en/sql-reference/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -============= -SQL Manual -============= - -.. toctree:: - :hidden: - - sql-functions/index - sql-statements/index diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/avg_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/avg_EN.md deleted file mode 100644 index 09ecffb5f634f9..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/avg_EN.md +++ /dev/null @@ -1,51 +0,0 @@ - - - -#AVG -## Description -### Syntax - -`AVG([DISTINCT] expr)` - - -Used to return the average value of the selected field - -Optional field DISTINCT parameters can be used to return the weighted average - -## example - -``` -mysql> SELECT datetime, AVG(cost_time) FROM log_statis group by datetime; -+---------------------+--------------------+ -| datetime | avg(`cost_time`) | -+---------------------+--------------------+ -| 2019-07-03 21:01:20 | 25.827794561933533 | -+---------------------+--------------------+ - -mysql> SELECT datetime, AVG(distinct cost_time) FROM log_statis group by datetime; -+---------------------+---------------------------+ -| datetime | avg(DISTINCT `cost_time`) | -+---------------------+---------------------------+ -| 2019-07-04 02:23:24 | 20.666666666666668 | -+---------------------+---------------------------+ - -``` -##keyword -AVG diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/bitmap_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/bitmap_EN.md deleted file mode 100644 index 3542e9fbf62b55..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/bitmap_EN.md +++ /dev/null @@ -1,139 +0,0 @@ - - - -# BITMAP - -## Create table - -The aggregation model needs to be used when creating the table. The data type is bitmap and the aggregation function is bitmap_union. -``` -CREATE TABLE `pv_bitmap` ( -  `dt` int (11) NULL COMMENT" ", -  `page` varchar (10) NULL COMMENT" ", -  `user_id` bitmap BITMAP_UNION NULL COMMENT" " -) ENGINE = OLAP -AGGREGATE KEY (`dt`,` page`) -COMMENT "OLAP" -DISTRIBUTED BY HASH (`dt`) BUCKETS 2; -``` - -Note: When the amount of data is large, it is best to create a corresponding rollup table for high-frequency bitmap_union queries - -``` -ALTER TABLE pv_bitmap ADD ROLLUP pv (page, user_id); -``` - -## Data Load - -`TO_BITMAP (expr)`: Convert 0 ~ 18446744073709551615 unsigned bigint to bitmap - -`BITMAP_EMPTY ()`: Generate empty bitmap columns, used for insert or import to fill the default value - -`BITMAP_HASH (expr)`: Convert any type of column to a bitmap by hashing - -### Stream Load - -``` -cat data | curl --location-trusted -u user: passwd -T--H "columns: dt, page, user_id, user_id = to_bitmap (user_id)" http: // host: 8410 / api / test / testDb / _stream_load -``` - -``` -cat data | curl --location-trusted -u user: passwd -T--H "columns: dt, page, user_id, user_id = bitmap_hash (user_id)" http: // host: 8410 / api / test / testDb / _stream_load -``` - -``` -cat data | curl --location-trusted -u user: passwd -T--H "columns: dt, page, user_id, user_id = bitmap_empty ()" http: // host: 8410 / api / test / testDb / _stream_load -``` - -### Insert Into - -id2's column type is bitmap -``` -insert into bitmap_table1 select id, id2 from bitmap_table2; -``` - -id2's column type is bitmap -``` -INSERT INTO bitmap_table1 (id, id2) VALUES (1001, to_bitmap (1000)), (1001, to_bitmap (2000)); -``` - -id2's column type is bitmap -``` -insert into bitmap_table1 select id, bitmap_union (id2) from bitmap_table2 group by id; -``` - -id2's column type is int -``` -insert into bitmap_table1 select id, to_bitmap (id2) from table; -``` - -id2's column type is String -``` -insert into bitmap_table1 select id, bitmap_hash (id_string) from table; -``` - - -## Data Query - -### Syntax - - -`BITMAP_UNION (expr)`: Calculate the union of two Bitmaps. The return value is the new Bitmap value. - -`BITMAP_UNION_COUNT (expr)`: Calculate the cardinality of the union of two Bitmaps, equivalent to BITMAP_COUNT (BITMAP_UNION (expr)). It is recommended to use the BITMAP_UNION_COUNT function first, its performance is better than BITMAP_COUNT (BITMAP_UNION (expr)). - -`BITMAP_UNION_INT (expr)`: Count the number of different values ​​in columns of type TINYINT, SMALLINT and INT, return the sum of COUNT (DISTINCT expr) same - -`INTERSECT_COUNT (bitmap_column_to_count, filter_column, filter_values ​​...)`: The calculation satisfies -filter_column The cardinality of the intersection of multiple bitmaps of the filter. -bitmap_column_to_count is a column of type bitmap, filter_column is a column of varying dimensions, and filter_values ​​is a list of dimension values. - -### Example - -The following SQL uses the pv_bitmap table above as an example: - -Calculate the deduplication value for user_id: - -``` -select bitmap_union_count (user_id) from pv_bitmap; - -select bitmap_count (bitmap_union (user_id)) from pv_bitmap; -``` - -Calculate the deduplication value of id: - -``` -select bitmap_union_int (id) from pv_bitmap; -``` - -Calculate the retention of user_id: - -``` -select intersect_count (user_id, page, 'meituan') as meituan_uv, -intersect_count (user_id, page, 'waimai') as waimai_uv, -intersect_count (user_id, page, 'meituan', 'waimai') as retention // Number of users appearing on both 'meituan' and 'waimai' pages -from pv_bitmap -where page in ('meituan', 'waimai'); -``` - - -## keyword - -BITMAP, BITMAP_COUNT, BITMAP_EMPTY, BITMAP_UNION, BITMAP_UNION_INT, TO_BITMAP, BITMAP_UNION_COUNT, INTERSECT_COUNT \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/count_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/count_EN.md deleted file mode 100644 index d2fa70f2dd4fcd..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/count_EN.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# COUNT -## Description -### Syntax - -`COUNT([DISTINCT] expr)` - - -Number of rows used to return the required rows - -## example - -``` -MySQL > select count(*) from log_statis group by datetime; -+----------+ -| count(*) | -+----------+ -| 28515903 | -+----------+ - -MySQL > select count(datetime) from log_statis group by datetime; -+-------------------+ -| count(`datetime`) | -+-------------------+ -| 28521682 | -+-------------------+ - -MySQL > select count(distinct datetime) from log_statis group by datetime; -+-------------------------------+ -| count(DISTINCT `datetime`) | -+-------------------------------+ -| 71045 | -+-------------------------------+ -``` -##keyword -COUNT diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/hll_union_agg_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/hll_union_agg_EN.md deleted file mode 100644 index 5b4699bb651973..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/hll_union_agg_EN.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# HLL_UNION_AGG -## description -### Syntax - -`HLL_UNION_AGG(hll)` - - -HLL is an engineering implementation based on HyperLog algorithm, which is used to save the intermediate results of HyperLog calculation process. - -It can only be used as the value column type of the table and reduce the amount of data through aggregation to achieve the purpose of speeding up the query. - -Based on this, we get an estimate with an error of about 1%. The HLL column is generated by other columns or data imported into the data. - -When importing, hll_hash function is used to specify which column in data is used to generate HLL column. It is often used to replace count distinct, and to calculate UV quickly in business by combining rollup. - -## example -``` -MySQL > select HLL_UNION_AGG(uv_set) from test_uv;; -+-------------------------+ -THE COURT OF JUSTICE OF THE EUROPEAN COMMUNITIES, -+-------------------------+ -| 17721 | -+-------------------------+ -``` -##keyword -HLL_UNION_AGG,HLL,UNION,AGG diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/index.rst b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/index.rst deleted file mode 100644 index 9cab40ca9f4885..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -======================== -Aggregate Functions -======================== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/max_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/max_EN.md deleted file mode 100644 index 4d26d48a2d61d5..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/max_EN.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# MAX -## description -### Syntax - -`MAX(expr)` - - -Returns the maximum value of an expr expression - -## example -``` -MySQL > select max(scan_rows) from log_statis group by datetime; -+------------------+ -| max(`scan_rows`) | -+------------------+ -| 4671587 | -+------------------+ -``` -##keyword -MAX diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/min_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/min_EN.md deleted file mode 100644 index fa5349d26fae07..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/min_EN.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# MIN -## Description -### Syntax - -`MIN(expr)` - - -Returns the minimum value of an expr expression - -## example -``` -MySQL > select min(scan_rows) from log_statis group by datetime; -+------------------+ -| min(`scan_rows`) | -+------------------+ -| 0 | -+------------------+ -``` -##keyword -MIN diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/ndv_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/ndv_EN.md deleted file mode 100644 index f4cf84f7b28533..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/ndv_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# NDV -## Description -### Syntax - -`NDV (expr)` - - -Returns an approximate aggregation function similar to the result of COUNT (DISTINCT col). - -It combines COUNT and DISTINCT faster and uses fixed-size memory, so less memory can be used for columns with high cardinality. - -## example -``` -MySQL > select ndv(query_id) from log_statis group by datetime; -+-----------------+ -| ndv(`query_id`) | -+-----------------+ -| 17721 | -+-----------------+ -``` -##keyword -NDV diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/percentile_approx_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/percentile_approx_EN.md deleted file mode 100644 index f57b208c321d2c..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/percentile_approx_EN.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# PERCENTILE_APPROX -## Description -### Syntax - -`PERCENTILE_APPROX(expr, DOUBLE p[, DOUBLE compression])` - -Return the approximation of the point p, where the value of P is between 0 and 1. - -Compression param is optional and can be setted to a value in the range of [2048, 10000]. The bigger compression you set, the more precise result and more time cost you will get. If it is not setted or not setted in the correct range, PERCENTILE_APPROX function will run with a default compression param of 10000. - -This function uses fixed size memory, so less memory can be used for columns with high cardinality, and can be used to calculate statistics such as tp99. - -## example -``` -MySQL > select `table`, percentile_approx(cost_time,0.99) from log_statis group by `table`; -+---------------------+---------------------------+ -| table | percentile_approx(`cost_time`, 0.99) | -+----------+--------------------------------------+ -| test | 54.22 | -+----------+--------------------------------------+ - -MySQL > select `table`, percentile_approx(cost_time,0.99, 4096) from log_statis group by `table`; -+---------------------+---------------------------+ -| table | percentile_approx(`cost_time`, 0.99, 4096.0) | -+----------+--------------------------------------+ -| test | 54.21 | -+----------+--------------------------------------+ -``` -##keyword -PERCENTILE_APPROX,PERCENTILE,APPROX diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/stddev_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/stddev_EN.md deleted file mode 100644 index 771e1d0c2142fe..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/stddev_EN.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# STDDEV,STDDEV_POP -## Description -### Syntax - -`stddev (expl)` - - -Returns the standard deviation of the expr expression - -## example -``` -MySQL > select stddev(scan_rows) from log_statis group by datetime; -+---------------------+ -| stddev(`scan_rows`) | -+---------------------+ -| 2.3736656687790934 | -+---------------------+ - -MySQL > select stddev_pop(scan_rows) from log_statis group by datetime; -+-------------------------+ -| stddev_pop(`scan_rows`) | -+-------------------------+ -| 2.3722760595994914 | -+-------------------------+ -``` -##keyword -STDDEV,STDDEV_POP,POP diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/stddev_samp_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/stddev_samp_EN.md deleted file mode 100644 index 3211ffa5e96bf9..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/stddev_samp_EN.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# STDDEV_SAMP -## Description -### Syntax - -`STDDEV SAMP (expr)` - - -Returns the sample standard deviation of the expr expression - -## example -``` -MySQL > select stddev_samp(scan_rows) from log_statis group by datetime; -+--------------------------+ -| stddev_samp(`scan_rows`) | -+--------------------------+ -| 2.372044195280762 | -+--------------------------+ -``` -##keyword -STDDEVu SAMP,STDDEV,SAMP diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/sum_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/sum_EN.md deleted file mode 100644 index b92d319c9f1431..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/sum_EN.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# SUM -## Description -### Syntax - -`Sum (Expr)` - - -Used to return the sum of all values of the selected field - -## example -``` -MySQL > select sum(scan_rows) from log_statis group by datetime; -+------------------+ -| sum(`scan_rows`) | -+------------------+ -| 8217360135 | -+------------------+ -``` -##keyword -SUM diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/var_samp_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/var_samp_EN.md deleted file mode 100644 index cf2bdb4ed71005..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/var_samp_EN.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# VARIANCE_SAMP,VARIANCE_SAMP -## Description -### Syntax - -`VAR SAMP (expr)` - - -Returns the sample variance of the expr expression - -## example -``` -MySQL > select var_samp(scan_rows) from log_statis group by datetime; -+-----------------------+ -| var_samp(`scan_rows`) | -+-----------------------+ -| 5.6227132145741789 | -+-----------------------+ -``` -##keyword -VAR SAMP, VARIANCE SAMP,VAR,SAMP,VARIANCE diff --git a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/variance_EN.md b/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/variance_EN.md deleted file mode 100644 index 360a893d662837..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/aggregate-functions/variance_EN.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# VARIANCE,VAR_POP,VARIANCE_POP -## Description -### Syntax - -`VARIANCE(expr)` - - -Returns the variance of the expr expression - -## example -``` -MySQL > select variance(scan_rows) from log_statis group by datetime; -+-----------------------+ -| variance(`scan_rows`) | -+-----------------------+ -| 5.6183332881176211 | -+-----------------------+ - -MySQL > select var_pop(scan_rows) from log_statis group by datetime; -+----------------------+ -| var_pop(`scan_rows`) | -+----------------------+ -| 5.6230744719006163 | -+----------------------+ -``` -##keyword -VARIANCE,VAR_POP,VARIANCE_POP,VAR,POP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_and_EN.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_and_EN.md deleted file mode 100644 index a7162e8b1c35b4..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_and_EN.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# bitmap_and -## description -### Syntax - -`BITMAP BITMAP_AND(BITMAP lhs, BITMAP rhs)` - -Compute intersection of two input bitmaps, return the new bitmap. - -## example - -``` -mysql> select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(2))) cnt; -+------+ -| cnt | -+------+ -| 0 | -+------+ - -mysql> select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(1))) cnt; -+------+ -| cnt | -+------+ -| 1 | -+------+ -``` - -## keyword - - BITMAP_AND,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_contains_EN.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_contains_EN.md deleted file mode 100644 index 0cb2e06ad5a713..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_contains_EN.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# bitmap_contains -## description -### Syntax - -`B00LEAN BITMAP_CONTAINS(BITMAP bitmap, BIGINT input)` - -Calculates whether the input value is in the Bitmap column and returns a Boolean value. - -## example - -``` -mysql> select bitmap_contains(to_bitmap(1),2) cnt; -+------+ -| cnt | -+------+ -| 0 | -+------+ - -mysql> select bitmap_contains(to_bitmap(1),1) cnt; -+------+ -| cnt | -+------+ -| 1 | -+------+ -``` - -## keyword - - BITMAP_CONTAINS,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_empty_EN.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_empty_EN.md deleted file mode 100644 index acb8d584541005..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_empty_EN.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# bitmap_empty -## description -### Syntax - -`BITMAP BITMAP_EMPTY()` - -Return an empty bitmap. Mainly be used to supply default value for bitmap column when loading, e.g., - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,v1,v2=bitmap_empty()" http://host:8410/api/test/testDb/_stream_load -``` - -## example - -``` -mysql> select bitmap_count(bitmap_empty()); -+------------------------------+ -| bitmap_count(bitmap_empty()) | -+------------------------------+ -| 0 | -+------------------------------+ -``` - -## keyword - - BITMAP_EMPTY,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md deleted file mode 100644 index 76c72c375a8f2c..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# bitmap_from_string - -## description -### Syntax - -`BITMAP BITMAP_FROM_STRING(VARCHAR input)` - -Convert a string into a bitmap. The input string should be a comma separated UNIT32. -For example: input string "0, 1, 2" will be converted to a Bitmap with bit 0, 1, 2 set. -If input string is invalid, return NULL. - -## example - -``` -mysql> select bitmap_to_string(bitmap_empty()); -+----------------------------------+ -| bitmap_to_string(bitmap_empty()) | -+----------------------------------+ -| | -+----------------------------------+ - -mysql> select bitmap_to_string(bitmap_from_string("0, 1, 2")); -+-------------------------------------------------+ -| bitmap_to_string(bitmap_from_string('0, 1, 2')) | -+-------------------------------------------------+ -| 0,1,2 | -+-------------------------------------------------+ - -mysql> select bitmap_from_string("-1, 0, 1, 2"); -+-----------------------------------+ -| bitmap_from_string('-1, 0, 1, 2') | -+-----------------------------------+ -| NULL | -+-----------------------------------+ -``` - -## keyword - - BITMAP_FROM_STRING,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_has_any_EN.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_has_any_EN.md deleted file mode 100644 index fc253d6097a84f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_has_any_EN.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# bitmap_has_any -## description -### Syntax - -`B00LEAN BITMAP_HAS_ANY(BITMAP lhs, BITMAP rhs)` - -Calculate whether there are intersecting elements in the two Bitmap columns. The return value is Boolean. - -## example - -``` -mysql> select bitmap_has_any(to_bitmap(1),to_bitmap(2)) cnt; -+------+ -| cnt | -+------+ -| 0 | -+------+ - -mysql> select bitmap_has_any(to_bitmap(1),to_bitmap(1)) cnt; -+------+ -| cnt | -+------+ -| 1 | -+------+ -``` - -## keyword - - BITMAP_HAS_ANY,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_hash_EN.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_hash_EN.md deleted file mode 100644 index 7c244adc956d38..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_hash_EN.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# bitmap_hash -## description -### Syntax - -`BITMAP BITMAP_HASH(expr)` - -Compute the 32-bits hash value of a expr of any type, then return a bitmap containing that hash value. Mainly be used to load non-integer value into bitmap column, e.g., - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,device_id, device_id=bitmap_hash(device_id)" http://host:8410/api/test/testDb/_stream_load -``` - -## example - -``` -mysql> select bitmap_count(bitmap_hash('hello')); -+------------------------------------+ -| bitmap_count(bitmap_hash('hello')) | -+------------------------------------+ -| 1 | -+------------------------------------+ -``` - -## keyword - - BITMAP_HASH,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_or_EN.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_or_EN.md deleted file mode 100644 index 5cfd58f3f1fd1e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_or_EN.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# bitmap_or -## description -### Syntax - -`BITMAP BITMAP_OR(BITMAP lhs, BITMAP rhs)` - -Compute union of two input bitmaps, returns the new bitmap. - -## example - -``` -mysql> select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt; -+------+ -| cnt | -+------+ -| 2 | -+------+ - -mysql> select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt; -+------+ -| cnt | -+------+ -| 1 | -+------+ -``` - -## keyword - - BITMAP_OR,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md deleted file mode 100644 index fbaf4b28a7a1a7..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md +++ /dev/null @@ -1,63 +0,0 @@ - - -# bitmap_to_string - -## description -### Syntax - -`VARCHAR BITMAP_TO_STRING(BITMAP input)` - -Convert a input BITMAP to a string. The string is a separated string, contains all set bits in Bitmap. -If input is null, return null. - -## example - -``` -mysql> select bitmap_to_string(null); -+------------------------+ -| bitmap_to_string(NULL) | -+------------------------+ -| NULL | -+------------------------+ - -mysql> select bitmap_to_string(bitmap_empty()); -+----------------------------------+ -| bitmap_to_string(bitmap_empty()) | -+----------------------------------+ -| | -+----------------------------------+ - -mysql> select bitmap_to_string(to_bitmap(1)); -+--------------------------------+ -| bitmap_to_string(to_bitmap(1)) | -+--------------------------------+ -| 1 | -+--------------------------------+ - -mysql> select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))); -+---------------------------------------------------------+ -| bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) | -+---------------------------------------------------------+ -| 1,2 | -+---------------------------------------------------------+ - -``` - -## keyword - - BITMAP_TO_STRING,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/index.rst b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/index.rst deleted file mode 100644 index 23bf4653632aa4..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -================ -bitmap functions -================ - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/to_bitmap_EN.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/to_bitmap_EN.md deleted file mode 100644 index 39f26ee0f52068..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/to_bitmap_EN.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# to_bitmap -## description -### Syntax - -`BITMAP TO_BITMAP(expr)` - -Convert an unsigned bigint (ranging from 0 to 18446744073709551615) to a bitmap containing that value. Mainly be used to load interger value into bitmap column, e.g., - -``` -cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=to_bitmap(user_id)" http://host:8410/api/test/testDb/_stream_load -``` - -## example - -``` -mysql> select bitmap_count(to_bitmap(10)); -+-----------------------------+ -| bitmap_count(to_bitmap(10)) | -+-----------------------------+ -| 1 | -+-----------------------------+ -``` - -## keyword - - TO_BITMAP,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/cast_EN.md b/docs/documentation/en/sql-reference/sql-functions/cast_EN.md deleted file mode 100644 index bddf93fe104c41..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/cast_EN.md +++ /dev/null @@ -1,76 +0,0 @@ - - - -# CAST -##Description - - -``` -cast (input as type) -``` - -### BIGINT type - -### Syntax (BIGINT) - -``` cast (input as BIGINT) ``` - - -Converts input to the specified type - - -Converting the current column input to BIGINT type - -## example - -1. Turn constant, or a column in a table - -``` -mysql> select cast (1 as BIGINT); -+-------------------+ -| CAST(1 AS BIGINT) | -+-------------------+ -| 1 | -+-------------------+ -``` - -2. Transferred raw data - -``` -curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(tmp_k1 as BIGINT)" http://host:port/api/test/bigint/_stream_load -``` - -* Note: In the import, because the original type is String, when the original data with floating point value is cast, the data will be converted to NULL, such as 12.0. Doris is currently not truncating raw data. * - -If you want to force this type of raw data cast to int. Look at the following words: - -``` -curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(cast(tmp_k1 as DOUBLE) as BIGINT)" http://host:port/api/test/bigint/_stream_load - -mysql> select cast(cast ("11.2" as double) as bigint); -+----------------------------------------+ -| CAST(CAST('11.2' AS DOUBLE) AS BIGINT) | -+----------------------------------------+ -| 11 | -+----------------------------------------+ -1 row in set (0.00 sec) -``` -##keyword -CAST diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/curdate_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/curdate_EN.md deleted file mode 100644 index cfa2a54b0abfe1..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/curdate_EN.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# curdate -## Description -### Syntax - -'DATE CURDATE()' - -Get the current date and return it in Date type - -## example - -``` -mysql> SELECT CURDATE(); -+------------+ -| CURDATE() | -+------------+ -| 2019-12-20 | -+------------+ - -mysql> SELECT CURDATE() + 0; -+---------------+ -| CURDATE() + 0 | -+---------------+ -| 20191220 | -+---------------+ -``` -##keyword -CURDATE diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/current_timestamp_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/current_timestamp_EN.md deleted file mode 100644 index 5ca078bc0c43eb..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/current_timestamp_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# current_timestamp -## Description -### Syntax - -`DATETIME CURRENT_TIMESTAMP()` - - -Get the current time and return it in Datetime type - -## example - -``` -mysql> select current_timestamp(); -+---------------------+ -| current_timestamp() | -+---------------------+ -| 2019-05-27 15:59:33 | -+---------------------+ -``` -##keyword -CURRENT_TIMESTAMP,CURRENT,TIMESTAMP diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_add_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_add_EN.md deleted file mode 100644 index 78fe62f1b79954..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_add_EN.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# date_add -## Description -### Syntax - -`INT DATE_ADD(DATETIME date,INTERVAL expr type)` - - -Adds a specified time interval to the date. - -The date parameter is a valid date expression. - -The expr parameter is the interval you want to add. - -Sweet, sweet, sweet - -## example - -``` -mysql> select date_add('2010-11-30 23:59:59', INTERVAL 2 DAY); -+-------------------------------------------------+ -| date_add('2010-11-30 23:59:59', INTERVAL 2 DAY) | -+-------------------------------------------------+ -| 2010-12-02 23:59:59 | -+-------------------------------------------------+ -``` -##keyword -DATE_ADD,DATE,ADD diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_format_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_format_EN.md deleted file mode 100644 index 4cd44102081402..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_format_EN.md +++ /dev/null @@ -1,151 +0,0 @@ - - -# date_format -## Description -### Syntax - -'WARCHAR DATE'U FORMAT (DATETIME DATE, WARCHAR Format)' - - -Convert the date type to a bit string according to the format type. -Currently supports a string with a maximum 128 bytes and returns NULL if the length of the return value exceeds 128 - -The date parameter is the valid date. Format specifies the date/time output format. - -The formats available are: - -% a | Abbreviation for Sunday Name - -% B | Abbreviated Monthly Name - -% C | Month, numerical value - -% D | Sky in the Moon with English Prefix - -% d | Monthly day, numerical value (00-31) - -% e | Monthly day, numerical value (0-31) - -% f | microseconds - -% H | Hours (00-23) - -% h | hour (01-12) - -% I | Hours (01-12) - -% I | min, numerical value (00-59) - -% J | Days of Year (001-366) - -% k | hours (0-23) - -% L | Hours (1-12) - -% M | Moon Name - -% m | month, numerical value (00-12) - -%p | AM or PM - -% R | Time, 12 - hour (hh: mm: SS AM or PM) - -% S | seconds (00-59) - -% s | seconds (00-59) - -% T | Time, 24 - hour (hh: mm: ss) - -% U | Week (00-53) Sunday is the first day of the week - -% U | Week (00 - 53) Monday is the first day of the week - -% V | Week (01-53) Sunday is the first day of the week, and% X is used. - -% v | Week (01 - 53) Monday is the first day of the week, and% x is used - -% W | Sunday - -% w | Weekly day (0 = Sunday, 6 = Saturday) - -% X | Year, where Sunday is the first day of the week, 4 places, and% V use - -% X | year, of which Monday is the first day of the week, 4 places, and% V - -% Y | Year, 4 - -% Y | Year, 2 - -%% | Represent % - -## example - -``` -mysql> select date_format('2009-10-04 22:23:00', '%W %M %Y'); -+------------------------------------------------+ -| date_format('2009-10-04 22:23:00', '%W %M %Y') | -+------------------------------------------------+ -| Sunday October 2009 | -+------------------------------------------------+ - -mysql> select date_format('2007-10-04 22:23:00', '%H:%i:%s'); -+------------------------------------------------+ -| date_format('2007-10-04 22:23:00', '%H:%i:%s') | -+------------------------------------------------+ -| 22:23:00 | -+------------------------------------------------+ - -mysql> select date_format('1900-10-04 22:23:00', '%D %y %a %d %m %b %j'); -+------------------------------------------------------------+ -| date_format('1900-10-04 22:23:00', '%D %y %a %d %m %b %j') | -+------------------------------------------------------------+ -| 4th 00 Thu 04 10 Oct 277 | -+------------------------------------------------------------+ - -mysql> select date_format('1997-10-04 22:23:00', '%H %k %I %r %T %S %w'); -+------------------------------------------------------------+ -| date_format('1997-10-04 22:23:00', '%H %k %I %r %T %S %w') | -+------------------------------------------------------------+ -| 22 22 10 10:23:00 PM 22:23:00 00 6 | -+------------------------------------------------------------+ - -mysql> select date_format('1999-01-01 00:00:00', '%X %V'); -+---------------------------------------------+ -| date_format('1999-01-01 00:00:00', '%X %V') | -+---------------------------------------------+ -| 1998 52 | -+---------------------------------------------+ - -mysql> select date_format('2006-06-01', '%d'); -+------------------------------------------+ -| date_format('2006-06-01 00:00:00', '%d') | -+------------------------------------------+ -| 01 | -+------------------------------------------+ - -mysql> select date_format('2006-06-01', '%%%d'); -+--------------------------------------------+ -| date_format('2006-06-01 00:00:00', '%%%d') | -+--------------------------------------------+ -| %01 | -+--------------------------------------------+ -``` -##keyword -DATE_FORMAT,DATE,FORMAT diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_sub_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_sub_EN.md deleted file mode 100644 index 2673c74848932a..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/date_sub_EN.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# date_sub -## Description -### Syntax - -`INT DATE_SUB(DATETIME date,INTERVAL expr type)` - - -Subtract the specified time interval from the date - -The date parameter is a valid date expression. - -The expr parameter is the interval you want to add. - -Sweet, sweet, sweet - -## example - -``` -mysql> select date_sub('2010-11-30 23:59:59', INTERVAL 2 DAY); -+-------------------------------------------------+ -| date_sub('2010-11-30 23:59:59', INTERVAL 2 DAY) | -+-------------------------------------------------+ -| 2010-11-28 23:59:59 | -+-------------------------------------------------+ -``` -##keyword -Date, date, date diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/datediff_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/datediff_EN.md deleted file mode 100644 index a3c3d7a630c6a8..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/datediff_EN.md +++ /dev/null @@ -1,52 +0,0 @@ - - - -# datediff -## Description -### Syntax - -'DATETIME DATEDIFF (DATETIME expr1,DATETIME expr2)' - - -Expr1 - expr2 is calculated and the result is accurate to the sky. - -Expr1 and expr2 parameters are valid date or date/time expressions. - -Note: Only the date part of the value participates in the calculation. - -### example - -``` -mysql> select datediff(CAST('2007-12-31 23:59:59' AS DATETIME), CAST('2007-12-30' AS DATETIME)); -+-----------------------------------------------------------------------------------+ -| datediff(CAST('2007-12-31 23:59:59' AS DATETIME), CAST('2007-12-30' AS DATETIME)) | -+-----------------------------------------------------------------------------------+ -| 1 | -+-----------------------------------------------------------------------------------+ - -mysql> select datediff(CAST('2010-11-30 23:59:59' AS DATETIME), CAST('2010-12-31' AS DATETIME)); -+-----------------------------------------------------------------------------------+ -| datediff(CAST('2010-11-30 23:59:59' AS DATETIME), CAST('2010-12-31' AS DATETIME)) | -+-----------------------------------------------------------------------------------+ -| -31 | -+-----------------------------------------------------------------------------------+ -``` -##keyword -DATEDIFF diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/day_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/day_EN.md deleted file mode 100644 index 5f04db3956add8..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/day_EN.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# day -## Description -### Syntax - -`INT DAY(DATETIME date)` - - -Get the day information in the date, and return values range from 1 to 31. - -The parameter is Date or Datetime type - -## example - -``` -mysql> select day('1987-01-31'); -+----------------------------+ -| day('1987-01-31 00:00:00') | -+----------------------------+ -| 31 | -+----------------------------+ -``` -##keyword -DAY diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayname_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayname_EN.md deleted file mode 100644 index 5c4be940df527e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayname_EN.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# dayname -## Description -### Syntax - -'VARCHAR DAYNAME (DATE)' - - -Date name corresponding to return date - -The parameter is Date or Datetime type - -## example - -``` -mysql> select dayname('2007-02-03 00:00:00'); -+--------------------------------+ -| dayname('2007-02-03 00:00:00') | -+--------------------------------+ -| Saturday | -+--------------------------------+ -``` -##keyword -DAYNAME diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofmonth_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofmonth_EN.md deleted file mode 100644 index ad5eed3f2c369a..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofmonth_EN.md +++ /dev/null @@ -1,43 +0,0 @@ - - - -# Dayofmonth -## Description -### Syntax - -'INT DAYOFMONTH (DATETIME date)' - - -Get the day information in the date, and return values range from 1 to 31. - -The parameter is Date or Datetime type - -## example - -``` -mysql> select dayofmonth('1987-01-31'); -+-----------------------------------+ -| dayofmonth('1987-01-31 00:00:00') | -+-----------------------------------+ -| 31 | -+-----------------------------------+ -``` -##keyword -DAYOFMONTH diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofweek_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofweek_EN.md deleted file mode 100644 index 29d5919c99e579..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofweek_EN.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# dayofweek -## Description -### Syntax - -INT DayOfWeek (DATETIME date) - - -The DAYOFWEEK function returns the index value of the working day of the date, that is, 1 on Sunday, 2 on Monday, and 7 on Saturday. - -The parameter is Date or Datetime type - -## example -``` -mysql> select dayofweek('2019-06-25'); -+----------------------------------+ -| dayofweek('2019-06-25 00:00:00') | -+----------------------------------+ -| 3 | -+----------------------------------+ - -mysql> select dayofweek(cast(20190625 as date)); -+-----------------------------------+ -| dayofweek(CAST(20190625 AS DATE)) | -+-----------------------------------+ -| 3 | -+-----------------------------------+ -``` -##keyword -DAYOFWEEK diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofyear_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofyear_EN.md deleted file mode 100644 index af6c363a58e876..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/dayofyear_EN.md +++ /dev/null @@ -1,43 +0,0 @@ - - -# Dayofyear -## Description -### Syntax - -'INT DAYOFYEAR (DATETIME date)' - - -The date of acquisition is the date of the corresponding year. - -The parameter is Date or Datetime type - -## example - - -``` -mysql> select dayofyear('2007-02-03 00:00:00'); -+----------------------------------+ -| dayofyear('2007-02-03 00:00:00') | -+----------------------------------+ -| 34 | -+----------------------------------+ -``` -##keyword -DAYOFYEAR diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/from_days_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/from_days_EN.md deleted file mode 100644 index c65f4a99b83e4a..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/from_days_EN.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# from_days -## Description -### Syntax - -`DATE FROM_DAYS(INT N)` - - -Calculate which day by the number of days from 0000-01-01 - -## example - -``` -mysql > select from u days (730669); -+-------------------+ -| from_days(730669) | -+-------------------+ -| 2000-07-03 | -+-------------------+ -##keyword -FROM_DAYS,FROM,DAYS diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/from_unixtime_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/from_unixtime_EN.md deleted file mode 100644 index c20b47869896f1..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/from_unixtime_EN.md +++ /dev/null @@ -1,69 +0,0 @@ - - -# from_unixtime -## description -### syntax - -`DATETIME FROM UNIXTIME (INT unix timestamp [, VARCHAR string format]` - -Convert the UNIX timestamp to the corresponding time format of bits, and the format returned is specified by `string_format` - -Input is an integer and return is a string type - -Currently, `string_format` supports following formats： - - %Y: Year. eg. 2014，1900 - %m: Month. eg. 12，09 - %d: Day. eg. 11，01 - %H: Hour. eg. 23，01，12 - %i: Minute. eg. 05，11 - %s: Second. eg. 59，01 - -Default is `%Y-%m-%d %H:%i:%s` - -Other `string_format` is illegal and will returns NULL. - -## example - -``` -mysql> select from_unixtime(1196440219); -+---------------------------+ -| from_unixtime(1196440219) | -+---------------------------+ -| 2007-12-01 00:30:19 | -+---------------------------+ - -mysql> select from_unixtime(1196440219, '%Y-%m-%d'); -+-----------------------------------------+ -| from_unixtime(1196440219, '%Y-%m-%d') | -+-----------------------------------------+ -| 2007-12-01 | -+-----------------------------------------+ - -mysql> select from_unixtime(1196440219, '%Y-%m-%d %H:%i:%s'); -+--------------------------------------------------+ -|From unixtime (1196440219,'%Y-%m-%d %H:%i:%s') | -+--------------------------------------------------+ -| 2007-12-01 00:30:19 | -+--------------------------------------------------+ - -##keyword - - FROM_UNIXTIME,FROM,UNIXTIME diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/hour_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/hour_EN.md deleted file mode 100644 index b245269b79c23e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/hour_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# hour -## description -### Syntax - -`INT HOUR(DATETIME date)` - -Returns hour information in the time type, ranging from 0,23 - -The parameter is Date or Datetime type - -## example - -``` -mysql> select hour('2018-12-31 23:59:59'); -+-----------------------------+ -| hour('2018-12-31 23:59:59') | -+-----------------------------+ -| 23 | -+-----------------------------+ -``` -##keyword -HOUR diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/index.rst b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/index.rst deleted file mode 100644 index ed3e2aa86792a3..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -========================= -Date Time Functions -========================= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/minute_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/minute_EN.md deleted file mode 100644 index c43c26c3fd2dea..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/minute_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# minute -## description -### Syntax - -`INT MINUTE(DATETIME date)` - -Returns minute information in the time type, ranging from 0,59 - -The parameter is Date or Datetime type - -## example - -``` -mysql> select minute('2018-12-31 23:59:59'); -+-----------------------------+ -| minute('2018-12-31 23:59:59') | -+-----------------------------+ -| 59 | -+-----------------------------+ -``` -##keyword -MINUTE diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/month_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/month_EN.md deleted file mode 100644 index 6ea1e89a5433b6..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/month_EN.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# month -## Description -### Syntax - -INT MONTH (DATETIME date) - - -Returns month information in the time type, ranging from 1,12 - -The parameter is Date or Datetime type - -## example - -``` -mysql> select month('1987-01-01'); -+-----------------------------+ -| month('1987-01-01 00:00:00') | -+-----------------------------+ -| 1 | -+-----------------------------+ -``` -##keyword -MONTH diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/monthname_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/monthname_EN.md deleted file mode 100644 index ecc850ab0d4da5..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/monthname_EN.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# Monthname -## Description -### Syntax - -'VARCHAR MONTHNAME (DATE)' - - -Month name corresponding to return date - -The parameter is Date or Datetime type - -## example - -``` -mysql> select monthname('2008-02-03 00:00:00'); -+----------------------------------+ -| monthname('2008-02-03 00:00:00') | -+----------------------------------+ -| February | -+----------------------------------+ -``` -##keyword -MONTHNAME diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/now_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/now_EN.md deleted file mode 100644 index 9ef4849bd0deb9..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/now_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# now -## Description -### Syntax - -'DATETIME NOW ()' - - -Get the current time and return it in Datetime type - -## example - -``` -mysql> select now(); -+---------------------+ -| now() | -+---------------------+ -| 2019-05-27 15:58:25 | -+---------------------+ -``` -##keyword -NOW diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/second_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/second_EN.md deleted file mode 100644 index 8318a1c11719c8..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/second_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# second -## description -### Syntax - -`INT SECOND(DATETIME date)` - -Returns second information in the time type, ranging from 0,59 - -The parameter is Date or Datetime type - -## example - -``` -mysql> select second('2018-12-31 23:59:59'); -+-----------------------------+ -| second('2018-12-31 23:59:59') | -+-----------------------------+ -| 59 | -+-----------------------------+ -``` -##keyword -SECOND diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/str_to_date_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/str_to_date_EN.md deleted file mode 100644 index 7957a6471df141..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/str_to_date_EN.md +++ /dev/null @@ -1,57 +0,0 @@ - - -# Str_to_date -## Description -### Syntax - -'DATETIME STR TWO DATES (VARCHAR STR, VARCHAR format)' - - -Convert STR to DATE type by format specified, if the conversion result does not return NULL - -The format format supported is consistent with date_format - -## example - -``` -mysql> select str_to_date('2014-12-21 12:34:56', '%Y-%m-%d %H:%i:%s'); -+---------------------------------------------------------+ -| str_to_date('2014-12-21 12:34:56', '%Y-%m-%d %H:%i:%s') | -+---------------------------------------------------------+ -| 2014-12-21 12:34:56 | -+---------------------------------------------------------+ - -mysql> select str_to_date('2014-12-21 12:34%3A56', '%Y-%m-%d %H:%i%%3A%s'); -+--------------------------------------------------------------+ -| str_to_date('2014-12-21 12:34%3A56', '%Y-%m-%d %H:%i%%3A%s') | -+--------------------------------------------------------------+ -| 2014-12-21 12:34:56 | -+--------------------------------------------------------------+ - -mysql> select str_to_date('200442 Monday', '%X%V %W'); -+-----------------------------------------+ -| str_to_date('200442 Monday', '%X%V %W') | -+-----------------------------------------+ -| 2004-10-18 | -+-----------------------------------------+ -``` -##keyword - - STR_TO_DATE,STR,TO,DATE diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timediff_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timediff_EN.md deleted file mode 100644 index b2d32c79f1f7ea..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timediff_EN.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# timediff -## Description -### Syntax - -'TIME TIMEDIFF (DATETIME expr1, DATETIME expr2)' - - -TIMEDIFF returns the difference between two DATETIMEs - -The TIMEDIFF function returns the result of expr1 - expr2 expressed as a time value, with a return value of TIME type - -The results are limited to TIME values ranging from - 838:59:59 to 838:59:59. - -### example - -``` -mysql> SELECT TIMEDIFF(now(),utc_timestamp()); -+----------------------------------+ -| timediff(now(), utc_timestamp()) | -+----------------------------------+ -| 08:00:00 | -+----------------------------------+ - -mysql> SELECT TIMEDIFF('2019-07-11 16:59:30','2019-07-11 16:59:21'); -+--------------------------------------------------------+ -| timediff('2019-07-11 16:59:30', '2019-07-11 16:59:21') | -+--------------------------------------------------------+ -| 00:00:09 | -+--------------------------------------------------------+ - -mysql> SELECT TIMEDIFF('2019-01-01 00:00:00', NULL); -+---------------------------------------+ -| timediff('2019-01-01 00:00:00', NULL) | -+---------------------------------------+ -| NULL | -+---------------------------------------+ -``` -##keyword -TIMEDIFF diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timestampadd_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timestampadd_EN.md deleted file mode 100644 index 71dcd7bc7df2e6..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timestampadd_EN.md +++ /dev/null @@ -1,51 +0,0 @@ - - -# timestampadd -## description -### Syntax - -`DATETIME TIMESTAMPADD(unit, interval, DATETIME datetime_expr)` - -Adds the integer expression interval to the date or datetime expression datetime_expr. - -The unit for interval is given by the unit argument, which should be one of the following values: - -SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, or YEAR. - -## example - -``` - -mysql> SELECT TIMESTAMPADD(MINUTE,1,'2019-01-02'); -+------------------------------------------------+ -| timestampadd(MINUTE, 1, '2019-01-02 00:00:00') | -+------------------------------------------------+ -| 2019-01-02 00:01:00 | -+------------------------------------------------+ - -mysql> SELECT TIMESTAMPADD(WEEK,1,'2019-01-02'); -+----------------------------------------------+ -| timestampadd(WEEK, 1, '2019-01-02 00:00:00') | -+----------------------------------------------+ -| 2019-01-09 00:00:00 | -+----------------------------------------------+ -``` -##keyword -TIMESTAMPADD diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timestampdiff_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timestampdiff_EN.md deleted file mode 100644 index 2eb8c0046e1527..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/timestampdiff_EN.md +++ /dev/null @@ -1,60 +0,0 @@ - - -# timestampdiff -## description -### Syntax - -`INT TIMESTAMPDIFF(unit,DATETIME datetime_expr1, DATETIME datetime_expr2)` - -Returns datetime_expr2 − datetime_expr1, where datetime_expr1 and datetime_expr2 are date or datetime expressions. - -The unit for the result (an integer) is given by the unit argument. - -The legal values for unit are the same as those listed in the description of the TIMESTAMPADD() function. - -## example - -``` - -MySQL> SELECT TIMESTAMPDIFF(MONTH,'2003-02-01','2003-05-01'); -+--------------------------------------------------------------------+ -| timestampdiff(MONTH, '2003-02-01 00:00:00', '2003-05-01 00:00:00') | -+--------------------------------------------------------------------+ -| 3 | -+--------------------------------------------------------------------+ - -MySQL> SELECT TIMESTAMPDIFF(YEAR,'2002-05-01','2001-01-01'); -+-------------------------------------------------------------------+ -| timestampdiff(YEAR, '2002-05-01 00:00:00', '2001-01-01 00:00:00') | -+-------------------------------------------------------------------+ -| -1 | -+-------------------------------------------------------------------+ - - -MySQL> SELECT TIMESTAMPDIFF(MINUTE,'2003-02-01','2003-05-01 12:05:55'); -+---------------------------------------------------------------------+ -| timestampdiff(MINUTE, '2003-02-01 00:00:00', '2003-05-01 12:05:55') | -+---------------------------------------------------------------------+ -| 128885 | -+---------------------------------------------------------------------+ - -``` -##keyword -TIMESTAMPDIFF diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/to_days_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/to_days_EN.md deleted file mode 100644 index 3c836c608604af..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/to_days_EN.md +++ /dev/null @@ -1,43 +0,0 @@ - - -# to_days -## Description -### Syntax - -'INT TO DAYS' - - -Days of returning date distance 0000-01-01 - -The parameter is Date or Datetime type - -## example - -``` -mysql> select to_days('2007-10-07'); -+-----------------------+ -| to_days('2007-10-07') | -+-----------------------+ -| 733321 | -+-----------------------+ -``` - -##keyword -TO_DAYS,TO,DAYS diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/unix_timestamp_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/unix_timestamp_EN.md deleted file mode 100644 index 611ba8a5f9ee8e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/unix_timestamp_EN.md +++ /dev/null @@ -1,79 +0,0 @@ - - -# unix_timestamp -## Description -### Syntax - -`INT UNIX_TIMESTAMP(), UNIX_TIMESTAMP(DATETIME date)` - -Converting a Date or Datetime type to a UNIX timestamp. - -If there are no parameters, the current time is converted into a timestamp. - -The parameter needs to be Date or Datetime type. - -Any date before 1970-01-01 00:00:00 or after 2038-01-19 03:14:07 will return 0. - -See `date_format` function to get Format explanation. - -This function is affected by time zone. - -## example - -``` -mysql> select unix_timestamp(); -+------------------+ -| unix_timestamp() | -+------------------+ -| 1558589570 | -+------------------+ - -mysql> select unix_timestamp('2007-11-30 10:30:19'); -+---------------------------------------+ -| unix_timestamp('2007-11-30 10:30:19') | -+---------------------------------------+ -| 1196389819 | -+---------------------------------------+ - -mysql> select unix_timestamp('2007-11-30 10:30-19', '%Y-%m-%d %H:%i-%s'); -+---------------------------------------+ -| unix_timestamp('2007-11-30 10:30-19') | -+---------------------------------------+ -| 1196389819 | -+---------------------------------------+ - -mysql> select unix_timestamp('2007-11-30 10:30%3A19', '%Y-%m-%d %H:%i%%3A%s'); -+---------------------------------------+ -| unix_timestamp('2007-11-30 10:30%3A19') | -+---------------------------------------+ -| 1196389819 | -+---------------------------------------+ - -mysql> select unix_timestamp('1969-01-01 00:00:00'); -+---------------------------------------+ -| unix_timestamp('1969-01-01 00:00:00') | -+---------------------------------------+ -| 0 | -+---------------------------------------+ -``` - -## keyword - - UNIX_TIMESTAMP,UNIX,TIMESTAMP diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/utc_timestamp_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/utc_timestamp_EN.md deleted file mode 100644 index bc05279863877e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/utc_timestamp_EN.md +++ /dev/null @@ -1,43 +0,0 @@ - - -# utc_timestamp -## Description -### Syntax - -`DATETIME UTC_TIMESTAMP()` - - -Returns the current UTC date and time in "YYYY-MM-DD HH: MM: SS" or - -A Value of "YYYYMMDDHMMSS" Format - -Depending on whether the function is used in a string or numeric context - -## example - -``` -mysql> select utc_timestamp(),utc_timestamp() + 1; -+---------------------+---------------------+ -| utc_timestamp() | utc_timestamp() + 1 | -+---------------------+---------------------+ -| 2019-07-10 12:31:18 | 20190710123119 | -+---------------------+---------------------+ -##keyword -UTC_TIMESTAMP,UTC,TIMESTAMP diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/workofyear_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/workofyear_EN.md deleted file mode 100644 index 88e6e75e89a877..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/workofyear_EN.md +++ /dev/null @@ -1,43 +0,0 @@ - - -# weekofyear -## Description -### Syntax - -'INT WEEKOFYEAR (DATETIME DATE)' - - - -Get the Weeks of the Year - -The parameter is Date or Datetime type - -## example - -``` -mysql> select weekofyear('2008-02-20 00:00:00'); -+-----------------------------------+ -| weekofyear('2008-02-20 00:00:00') | -+-----------------------------------+ -| 8 | -+-----------------------------------+ -``` -##keyword -WEEKOFYEAR diff --git a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/year_EN.md b/docs/documentation/en/sql-reference/sql-functions/date-time-functions/year_EN.md deleted file mode 100644 index 0d3d78905a6296..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/date-time-functions/year_EN.md +++ /dev/null @@ -1,43 +0,0 @@ - - - -# year -## Description -### Syntax - -`INT YEAR(DATETIME date)` - - -Returns the year part of the date type, ranging from 1000 to 9999 - -The parameter is Date or Datetime type - -## example - -``` -mysql> select year('1987-01-01'); -+-----------------------------+ -| year('1987-01-01 00:00:00') | -+-----------------------------+ -| 1987 | -+-----------------------------+ -``` -##keyword -YEAR diff --git a/docs/documentation/en/sql-reference/sql-functions/hash-functions/index.rst b/docs/documentation/en/sql-reference/sql-functions/hash-functions/index.rst deleted file mode 100644 index 5de92cdae4124e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/hash-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -======================= -Hash Functions -======================= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md b/docs/documentation/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md deleted file mode 100644 index 1be7ff0699d7ef..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# murmur_hash3_32 - -## description -### Syntax - -`INT MURMUR_HASH3_32(VARCHAR input, ...)` - -Return the 32 bits murmur3 hash of input string. - -## example - -``` -mysql> select murmur_hash3_32(null); -+-----------------------+ -| murmur_hash3_32(NULL) | -+-----------------------+ -| NULL | -+-----------------------+ - -mysql> select murmur_hash3_32("hello"); -+--------------------------+ -| murmur_hash3_32('hello') | -+--------------------------+ -| 1321743225 | -+--------------------------+ - -mysql> select murmur_hash3_32("hello", "world"); -+-----------------------------------+ -| murmur_hash3_32('hello', 'world') | -+-----------------------------------+ -| 984713481 | -+-----------------------------------+ -``` - -## keyword - - MURMUR_HASH3_32,HASH diff --git a/docs/documentation/en/sql-reference/sql-functions/index.rst b/docs/documentation/en/sql-reference/sql-functions/index.rst deleted file mode 100644 index d41a98032723f8..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -================== -SQL Functions -================== - -.. toctree:: - :glob: - - * - -.. toctree:: - :hidden: - - date-time-functions/index - spatial-functions/index - string-functions/index - aggregate-functions/index - bitmap-functions/index - hash-functions/index diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/index.rst b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/index.rst deleted file mode 100644 index f36f6cc29bbf56..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -======================= -Sptial Functions -======================= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_astext_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_astext_EN.md deleted file mode 100644 index 44d3894f985ab0..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_astext_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_AsText`,`ST_AsWKT` -## Description -### Syntax - -'VARCHAR ST'u AsText (GEOMETRY geo)' - - -Converting a geometric figure into a WKT (Well Known Text) representation - -## example - -``` -mysql> SELECT ST_AsText(ST_Point(24.7, 56.7)); -+---------------------------------+ -| st_astext(st_point(24.7, 56.7)) | -+---------------------------------+ -| POINT (24.7 56.7) | -+---------------------------------+ -``` -##keyword -ST_ASTEXT,ST_ASWKT,ST,ASTEXT,ASWKT diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_circle_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_circle_EN.md deleted file mode 100644 index 33b40558abf88f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_circle_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# `ST_Circle` -## Description -### Syntax - -`GEOMETRY ST_Circle(DOUBLE center_lng, DOUBLE center_lat, DOUBLE radius)` - - -Convert a WKT (Well Known Text) into a circle on the earth's sphere. Where `center_lng'denotes the longitude of the center of a circle, -` Center_lat` denotes the latitude of the center of a circle, radius` denotes the radius of a circle in meters. - -## example - -``` -mysql> SELECT ST_AsText(ST_Circle(111, 64, 10000)); -+--------------------------------------------+ -| st_astext(st_circle(111.0, 64.0, 10000.0)) | -+--------------------------------------------+ -| CIRCLE ((111 64), 10000) | -+--------------------------------------------+ -``` -##keyword -ST_CIRCLE,ST,CIRCLE diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_contains_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_contains_EN.md deleted file mode 100644 index ba5164d26e193f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_contains_EN.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# `ST_Contains' -## Description -### Syntax - -`BOOL ST_Contains(GEOMETRY shape1, GEOMETRY shape2)` - - -Judging whether geometric shape 1 can contain geometric shape 2 completely - -## example - - -``` -mysql> SELECT ST_Contains(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"), ST_Point(5, 5)); -+----------------------------------------------------------------------------------------+ -| st_contains(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))'), st_point(5.0, 5.0)) | -+----------------------------------------------------------------------------------------+ -| 1 | -+----------------------------------------------------------------------------------------+ - -mysql> SELECT ST_Contains(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"), ST_Point(50, 50)); -+------------------------------------------------------------------------------------------+ -| st_contains(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))'), st_point(50.0, 50.0)) | -+------------------------------------------------------------------------------------------+ -| 0 | -+------------------------------------------------------------------------------------------+ -``` -##keyword -ST_CONTAINS,ST,CONTAINS -w \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_distance_sphere_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_distance_sphere_EN.md deleted file mode 100644 index 8bcd8ba25fb34c..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_distance_sphere_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_Distance_Sphere` -## description -### Syntax - -`DOUBLE ST_Distance_Sphere(DOUBLE x_lng, DOUBLE x_lat, DOUBLE y_lng, DOUBLE x_lat)` - - -Calculate the spherical distance between two points of the earth in meters. The incoming parameters are the longitude of point X, the latitude of point X, the longitude of point Y and the latitude of point Y. - -## example - -``` -mysql> select st_distance_sphere(116.35620117, 39.939093, 116.4274406433, 39.9020987219); -+----------------------------------------------------------------------------+ -| st_distance_sphere(116.35620117, 39.939093, 116.4274406433, 39.9020987219) | -+----------------------------------------------------------------------------+ -| 7336.9135549995917 | -+----------------------------------------------------------------------------+ -``` -##keyword -ST_DISTANCE_SPHERE,ST,DISTANCE,SPHERE diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_geometryfromtext_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_geometryfromtext_EN.md deleted file mode 100644 index c146a769f6193b..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_geometryfromtext_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_GeometryFromText`,`ST GeomFromText` -## Description -### Syntax - -'GEOMETRY ST'u GeometryFromText (VARCHAR wkt)' - - -Converting a WKT (Well Known Text) into a corresponding memory geometry - -## example - -``` -mysql> SELECT ST_AsText(ST_GeometryFromText("LINESTRING (1 1, 2 2)")); -+---------------------------------------------------------+ -| st_astext(st_geometryfromtext('LINESTRING (1 1, 2 2)')) | -+---------------------------------------------------------+ -| LINESTRING (1 1, 2 2) | -+---------------------------------------------------------+ -``` -##keyword -ST_GEOMETRYFROMTEXT,ST_GEOMFROMTEXT,ST,GEOMETRYFROMTEXT,GEOMFROMTEXT diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_linefromtext_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_linefromtext_EN.md deleted file mode 100644 index ad0002fc6524aa..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_linefromtext_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_LineFromText`,`ST_LineStringFromText` -## Description -### Syntax - -'GEOMETRY ST LineFromText (VARCHAR wkt)' - - -Converting a WKT (Well Known Text) into a Line-style memory representation - -## example - -``` -mysql> SELECT ST_AsText(ST_LineFromText("LINESTRING (1 1, 2 2)")); -+---------------------------------------------------------+ -| st_astext(st_geometryfromtext('LINESTRING (1 1, 2 2)')) | -+---------------------------------------------------------+ -| LINESTRING (1 1, 2 2) | -+---------------------------------------------------------+ -``` -##keyword -ST_LINEFROMTEXT, ST_LINESTRINGFROMTEXT,ST,LINEFROMTEXT,LINESTRINGFROMTEXT diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_point_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_point_EN.md deleted file mode 100644 index b017f5787cf341..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_point_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# St_Point' -## Description -### Syntax - -`POINT ST_Point(DOUBLE x, DOUBLE y)` - - -Given the X coordinate value, the Y coordinate value returns the corresponding Point. -The current value is meaningful only for spherical sets, and X/Y corresponds to longitude/latitude. - -## example - -``` -mysql> SELECT ST_AsText(ST_Point(24.7, 56.7)); -+---------------------------------+ -| st_astext(st_point(24.7, 56.7)) | -+---------------------------------+ -| POINT (24.7 56.7) | -+---------------------------------+ -``` -##keyword -ST_POINT,ST,POINT diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_polygon_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_polygon_EN.md deleted file mode 100644 index 8237aa53cbc9d0..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_polygon_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# `ST_Polygon`,`ST_PolyFromText`,`ST_PolygonFromText` -## Description -### Syntax - -'GEOMETRY ST'u Polygon (VARCHAR wkt)' - - -Converting a WKT (Well Known Text) into a corresponding polygon memory form - - -### example - -``` -mysql> SELECT ST_AsText(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))")); -+------------------------------------------------------------------+ -| st_astext(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))')) | -+------------------------------------------------------------------+ -| POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0)) | -+------------------------------------------------------------------+ -``` -##keyword -ST_POLYGON,ST_POLYFROMTEXT,ST_POLYGONFROMTEXT,ST,POLYGON,POLYFROMTEXT,POLYGONFROMTEXT diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_x_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_x_EN.md deleted file mode 100644 index 656829949dc6ae..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_x_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_X` -## Description -### Syntax - -`DOUBLE ST_X(POINT point)` - - -When point is a valid POINT type, the corresponding X coordinate value is returned. - -## example - -``` -mysql> SELECT ST_X(ST_Point(24.7, 56.7)); -+----------------------------+ -| st_x(st_point(24.7, 56.7)) | -+----------------------------+ -| 24.7 | -+----------------------------+ -``` -##keyword -ST_X,ST,X diff --git a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_y_EN.md b/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_y_EN.md deleted file mode 100644 index cd0075af5ff225..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/spatial-functions/st_y_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# `ST_Y` -## Description -### Syntax - -`DOUBLE ST_Y(POINT point)` - - -When point is a valid POINT type, the corresponding Y coordinate value is returned. - -## example - -``` -mysql> SELECT ST_Y(ST_Point(24.7, 56.7)); -+----------------------------+ -| st_y(st_point(24.7, 56.7)) | -+----------------------------+ -| 56.7 | -+----------------------------+ -``` -##keyword -ST_Y,ST,Y diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/ascii_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/ascii_EN.md deleted file mode 100644 index 7cf1ebe1c6c440..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/ascii_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# ASCII -## Description -### Syntax - -`INT AXES (WARCHAR STR)` - - -Returns the ASCII code corresponding to the first character of the string - -## example - -``` -mysql> select ascii('1'); -+------------+ -| ascii('1') | -+------------+ -| 49 | -+------------+ - -mysql> select ascii('234'); -+--------------+ -| ascii('234') | -+--------------+ -| 50 | -+--------------+ -``` -##keyword -ASCII diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/concat_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/concat_EN.md deleted file mode 100644 index e45c002daecfc3..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/concat_EN.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# concat -## Description -### Syntax - -`VARCHAR concat (VARCHAR,...)` - - -Connect multiple strings and return NULL if any of the parameters is NULL - -## example - -``` -mysql> select concat("a", "b"); -+------------------+ -| concat('a', 'b') | -+------------------+ -| ab | -+------------------+ - -mysql> select concat("a", "b", "c"); -+-----------------------+ -| concat('a', 'b', 'c') | -+-----------------------+ -| abc | -+-----------------------+ - -mysql> select concat("a", null, "c"); -+------------------------+ -| concat('a', NULL, 'c') | -+------------------------+ -| NULL | -+------------------------+ -``` -##keyword -CONCAT diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/concat_ws_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/concat_ws_EN.md deleted file mode 100644 index 584853d576f075..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/concat_ws_EN.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# Concat_ws -## Description -### Syntax - -'VARCHAR concat ws (VARCHAR sep., VARCHAR str,...)' - - -Using the first parameter SEP as a connector, the second parameter and all subsequent parameters are spliced into a string. -If the separator is NULL, return NULL. -` The concat_ws` function does not skip empty strings, but NULL values. - -## example - -``` -mysql> select concat_ws("or", "d", "is"); -+----------------------------+ -| concat_ws('or', 'd', 'is') | -+----------------------------+ -| doris | -+----------------------------+ - -mysql> select concat_ws(NULL, "d", "is"); -+----------------------------+ -| concat_ws(NULL, 'd', 'is') | -+----------------------------+ -| NULL | -+----------------------------+ - -mysql> select concat_ws("or", "d", NULL,"is"); -+---------------------------------+ -| concat_ws("or", "d", NULL,"is") | -+---------------------------------+ -| doris | -+---------------------------------+ -``` -##keyword -CONCAT_WS,CONCAT,WS diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/ends_with_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/ends_with_EN.md deleted file mode 100644 index 7da9bd5c2fefdc..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/ends_with_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# ends_with -## Description -### Syntax - -`BOOLEAN ENDS_WITH (VARCHAR str, VARCHAR suffix)` - -It returns true if the string ends with the specified suffix, otherwise it returns false. -If any parameter is NULL, it returns NULL. - -## example - -``` -mysql> select ends_with("Hello doris", "doris"); -+-----------------------------------+ -| ends_with('Hello doris', 'doris') | -+-----------------------------------+ -| 1 | -+-----------------------------------+ - -mysql> select ends_with("Hello doris", "Hello"); -+-----------------------------------+ -| ends_with('Hello doris', 'Hello') | -+-----------------------------------+ -| 0 | -+-----------------------------------+ -``` -##keyword -ENDS_WITH diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/find_in_set_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/find_in_set_EN.md deleted file mode 100644 index 883dc6f8e5e48a..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/find_in_set_EN.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# find_in_set -## description -### Syntax - -`INT find_in_set(VARCHAR str, VARCHAR strlist)` - -"NOT found in set (VARCHAR str., VARCHAR strlist)" - - -Return to the location where the str first appears in strlist (counting from 1). Strlist is a comma-separated string. If not, return 0. Any parameter is NULL, returning NULL. - -## example - -``` -mysql> select find_in_set("b", "a,b,c"); -+---------------------------+ -| find_in_set('b', 'a,b,c') | -+---------------------------+ -| 2 | -+---------------------------+ -``` -##keyword -FIND_IN_SET,FIND,IN,SET diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_double_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_double_EN.md deleted file mode 100644 index 994d33d429f2ce..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_double_EN.md +++ /dev/null @@ -1,67 +0,0 @@ - - -# get_json_double -## description -### Syntax - -`DOUBLE get_json_double(VARCHAR json_str, VARCHAR json_path) - - -Parse and get the floating-point content of the specified path in the JSON string. -Where json_path must start with the $symbol and use. as the path splitter. If the path contains..., double quotation marks can be used to surround it. -Use [] to denote array subscripts, starting at 0. -The content of path cannot contain ",[and]. -If the json_string format is incorrect, or the json_path format is incorrect, or matches cannot be found, NULL is returned. - -## example - -1. Get the value of key as "k1" - -``` -mysql> SELECT get_json_double('{"k1":1.3, "k2":"2"}', "$.k1"); -+-------------------------------------------------+ -| get_json_double('{"k1":1.3, "k2":"2"}', '$.k1') | -+-------------------------------------------------+ -| 1.3 | -+-------------------------------------------------+ -``` - -2. Get the second element of the array whose key is "my. key" - -``` -mysql> SELECT get_json_double('{"k1":"v1", "my.key":[1.1, 2.2, 3.3]}', '$."my.key"[1]'); -+---------------------------------------------------------------------------+ -| get_json_double('{"k1":"v1", "my.key":[1.1, 2.2, 3.3]}', '$."my.key"[1]') | -+---------------------------------------------------------------------------+ -| 2.2 | -+---------------------------------------------------------------------------+ -``` - -3. Get the first element in an array whose secondary path is k1. key - > K2 -``` -mysql> SELECT get_json_double('{"k1.key":{"k2":[1.1, 2.2]}}', '$."k1.key".k2[0]'); -+---------------------------------------------------------------------+ -| get_json_double('{"k1.key":{"k2":[1.1, 2.2]}}', '$."k1.key".k2[0]') | -+---------------------------------------------------------------------+ -| 1.1 | -+---------------------------------------------------------------------+ -``` -##keyword -GET_JSON_DOUBLE,GET,JSON,DOUBLE diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_int_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_int_EN.md deleted file mode 100644 index 2c07f9a8c48b93..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_int_EN.md +++ /dev/null @@ -1,67 +0,0 @@ - - -# get_json_int -## Description -### Syntax - -`INT get_json_int(VARCHAR json_str, VARCHAR json_path) - - -Parse and retrieve the integer content of the specified path in the JSON string. -Where json_path must start with the $symbol and use. as the path splitter. If the path contains..., double quotation marks can be used to surround it. -Use [] to denote array subscripts, starting at 0. -The content of path cannot contain ",[and]. -If the json_string format is incorrect, or the json_path format is incorrect, or matches cannot be found, NULL is returned. - -## example - -1. Get the value of key as "k1" - -``` -mysql> SELECT get_json_int('{"k1":1, "k2":"2"}', "$.k1"); -+--------------------------------------------+ -| get_json_int('{"k1":1, "k2":"2"}', '$.k1') | -+--------------------------------------------+ -| 1 | -+--------------------------------------------+ -``` - -2. Get the second element of the array whose key is "my. key" - -``` -mysql> SELECT get_json_int('{"k1":"v1", "my.key":[1, 2, 3]}', '$."my.key"[1]'); -+------------------------------------------------------------------+ -| get_json_int('{"k1":"v1", "my.key":[1, 2, 3]}', '$."my.key"[1]') | -+------------------------------------------------------------------+ -| 2 | -+------------------------------------------------------------------+ -``` - -3. Get the first element in an array whose secondary path is k1. key - > K2 -``` -mysql> SELECT get_json_int('{"k1.key":{"k2":[1, 2]}}', '$."k1.key".k2[0]'); -+--------------------------------------------------------------+ -| get_json_int('{"k1.key":{"k2":[1, 2]}}', '$."k1.key".k2[0]') | -+--------------------------------------------------------------+ -| 1 | -+--------------------------------------------------------------+ -``` -##keyword -GET_JSON_INT,GET,JSON,INT diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_string_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_string_EN.md deleted file mode 100644 index a68685f0706a0f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/get_json_string_EN.md +++ /dev/null @@ -1,77 +0,0 @@ - - -# get_json_string -## description -### Syntax - -'VARCHAR get_json_string (VARCHAR json str, VARCHAR json path) - - -Parse and retrieve the string content of the specified path in the JSON string. -Where json_path must start with the $symbol and use. as the path splitter. If the path contains..., double quotation marks can be used to surround it. -Use [] to denote array subscripts, starting at 0. -The content of path cannot contain ",[and]. -If the json_string format is incorrect, or the json_path format is incorrect, or matches cannot be found, NULL is returned. - -## example - -1. Get the value of key as "k1" - -``` -mysql> SELECT get_json_string('{"k1":"v1", "k2":"v2"}', "$.k1"); -+---------------------------------------------------+ -| get_json_string('{"k1":"v1", "k2":"v2"}', '$.k1') | -+---------------------------------------------------+ -| v1 | -+---------------------------------------------------+ -``` - -2. Get the second element of the array whose key is "my. key" - -``` -mysql> SELECT get_json_string('{"k1":"v1", "my.key":["e1", "e2", "e3"]}', '$."my.key"[1]'); -+------------------------------------------------------------------------------+ -| get_json_string('{"k1":"v1", "my.key":["e1", "e2", "e3"]}', '$."my.key"[1]') | -+------------------------------------------------------------------------------+ -| e2 | -+------------------------------------------------------------------------------+ -``` - -3. Get the first element in an array whose secondary path is k1. key - > K2 -``` -mysql> SELECT get_json_string('{"k1.key":{"k2":["v1", "v2"]}}', '$."k1.key".k2[0]'); -+-----------------------------------------------------------------------+ -| get_json_string('{"k1.key":{"k2":["v1", "v2"]}}', '$."k1.key".k2[0]') | -+-----------------------------------------------------------------------+ -| v1 | -+-----------------------------------------------------------------------+ -``` - -4. Get all the values in the array where the key is "k1" -``` -mysql> SELECT get_json_string('[{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}]', '$.k1'); -+---------------------------------------------------------------------------------+ -| get_json_string('[{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}]', '$.k1') | -+---------------------------------------------------------------------------------+ -| ["v1","v3","v4"] | -+---------------------------------------------------------------------------------+ -``` -##keyword -GET_JSON_STRING,GET,JSON,STRING diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/group_concat_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/group_concat_EN.md deleted file mode 100644 index 2084a696156d58..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/group_concat_EN.md +++ /dev/null @@ -1,56 +0,0 @@ - - -# group_concat -## description -### Syntax - -`VARCHAR group_concat(VARCHAR str[, VARCHAR sep])` - - -This function is an aggregation function similar to sum (), and group_concat links multiple rows of results in the result set to a string. The second parameter, sep, is a connection symbol between strings, which can be omitted. This function usually needs to be used with group by statements. - -## example - -``` -mysql> select value from test; -+-------+ -| value | -+-------+ -| a | -| b | -| c | -+-------+ - -mysql> select group_concat(value) from test; -+-----------------------+ -| group_concat(`value`) | -+-----------------------+ -| a, b, c | -+-----------------------+ - -mysql> select group_concat(value, " ") from test; -+----------------------------+ -| group_concat(`value`, ' ') | -+----------------------------+ -| a b c | -+----------------------------+ -``` -##keyword -GROUP_CONCAT,GROUP,CONCAT diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/index.rst b/docs/documentation/en/sql-reference/sql-functions/string-functions/index.rst deleted file mode 100644 index 411341eaa1d732..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -======================= -String Functions -======================= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/instr_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/instr_EN.md deleted file mode 100644 index 8e4f5481977dfa..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/instr_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# instr -## Description -### Syntax - -'INT INSR (WARCHAR STR, WARCHAR substrate)' - - -Returns the location where substr first appeared in str (counting from 1). If substr does not appear in str, return 0. - -## example - -``` -mysql> select instr("abc", "b"); -+-------------------+ -| instr('abc', 'b') | -+-------------------+ -| 2 | -+-------------------+ - -mysql> select instr("abc", "d"); -+-------------------+ -| instr('abc', 'd') | -+-------------------+ -| 0 | -+-------------------+ -``` -##keyword -INSTR diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/lcase_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/lcase_EN.md deleted file mode 100644 index 5bf6dbccccba13..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/lcase_EN.md +++ /dev/null @@ -1,30 +0,0 @@ - - -# lcase -## Description -### Syntax - -`INT lcase (VARCHAR str)` - - -Consistent with `lower`. - -##keyword -LCASE diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/left_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/left_EN.md deleted file mode 100644 index 914e3d7838f691..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/left_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# left -## Description -### Syntax - -'VARCHAR left (VARCHAR str)' - - -It returns the left part of a string of specified length - -## example - -``` -mysql> select left("Hello doris",5); -+------------------------+ -| left('Hello doris', 5) | -+------------------------+ -| Hello | -+------------------------+ -``` -##keyword -LEFT diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/length_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/length_EN.md deleted file mode 100644 index 308fdcb98c8b33..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/length_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# length -## Description -### Syntax - -'INT length (VARCHAR str)' - - -Returns the length of the string and the number of characters returned for multi-byte characters. For example, five two-byte width words return a length of 10. - -## example - -``` -mysql> select length("abc"); -+---------------+ -| length('abc') | -+---------------+ -| 3 | -+---------------+ - -mysql> select length("中国"); -+------------------+ -| length('中国') | -+------------------+ -| 6 | -+------------------+ -``` -##keyword -LENGTH diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/locate_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/locate_EN.md deleted file mode 100644 index cdf8fc5b830c97..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/locate_EN.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# locate -## Description -### Syntax - -'INT LOCATION (WARCHAR substrate, WARCHAR str [, INT pos]]' - - -Returns where substr appears in str (counting from 1). If the third parameter POS is specified, the position where substr appears is found from the string where STR starts with POS subscript. If not found, return 0 - -## example - -``` -mysql> SELECT LOCATE('bar', 'foobarbar'); -+----------------------------+ -| locate('bar', 'foobarbar') | -+----------------------------+ -| 4 | -+----------------------------+ - -mysql> SELECT LOCATE('xbar', 'foobar'); -+--------------------------+ -| locate('xbar', 'foobar') | -+--------------------------+ -| 0 | -+--------------------------+ - -mysql> SELECT LOCATE('bar', 'foobarbar', 5); -+-------------------------------+ -| locate('bar', 'foobarbar', 5) | -+-------------------------------+ -| 7 | -+-------------------------------+ -``` -##keyword -LOCATE diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/lower_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/lower_EN.md deleted file mode 100644 index d1daa2dedbfad0..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/lower_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# lower -## Description -### Syntax - -'INT lower (WARCHAR str)' - - -Convert all strings in parameters to lowercase - -## example - -``` -mysql> SELECT lower("AbC123"); -+-----------------+ -| lower('AbC123') | -+-----------------+ -| abc123 | -+-----------------+ -``` -##keyword -LOWER diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/lpad_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/lpad_EN.md deleted file mode 100644 index d8d5edcc0d31ce..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/lpad_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# lpad -## Description -### Syntax - -'VARCHAR lpad (VARCHAR str., INT len, VARCHAR pad)' - - -Returns a string of length len in str, starting with the initials. If len is longer than str, pad characters are added to STR until the length of the string reaches len. If len is less than str's length, the function is equivalent to truncating STR strings and returning only strings of len's length. - -## example - -``` -mysql> SELECT lpad("hi", 5, "xy"); -+---------------------+ -| lpad('hi', 5, 'xy') | -+---------------------+ -| xyxhi | -+---------------------+ - -mysql> SELECT lpad("hi", 1, "xy"); -+---------------------+ -| lpad('hi', 1, 'xy') | -+---------------------+ -| h | -+---------------------+ -``` -##keyword -LPAD diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/ltrim_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/ltrim_EN.md deleted file mode 100644 index 633f5d00cd9ce2..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/ltrim_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# ltrim -## Description -### Syntax - -'VARCHAR ltrim (VARCHAR str)' - - -Remove the space that appears continuously from the beginning of the parameter str - -## example - -``` -mysql> SELECT ltrim(' ab d'); -+------------------+ -| ltrim(' ab d') | -+------------------+ -| ab d | -+------------------+ -``` -##keyword -LTRIM diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/money_format_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/money_format_EN.md deleted file mode 100644 index b294b59301466d..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/money_format_EN.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# money_format -## Description -### Syntax - -VARCHAR money format (Number) - - -The number is output in currency format, the integer part is separated by commas every three bits, and the decimal part is reserved for two bits. - -## example - -``` -mysql> select money_format(17014116); -+------------------------+ -| money_format(17014116) | -+------------------------+ -| 17,014,116.00 | -+------------------------+ - -mysql> select money_format(1123.456); -+------------------------+ -| money_format(1123.456) | -+------------------------+ -| 1,123.46 | -+------------------------+ - -mysql> select money_format(1123.4); -+----------------------+ -| money_format(1123.4) | -+----------------------+ -| 1,123.40 | -+----------------------+ -``` -##keyword -MONEY_FORMAT,MONEY,FORMAT diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/null_or_empty.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/null_or_empty.md deleted file mode 100644 index 847f2244a41417..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/null_or_empty.md +++ /dev/null @@ -1,53 +0,0 @@ - - -# null_or_empty -## description -### Syntax - -`BOOLEAN NULL_OR_EMPTY (VARCHAR str)` - -It returns true if the string is an empty string or NULL. Otherwise it returns false. - -## example - -``` -MySQL [(none)]> select null_or_empty(null); -+---------------------+ -| null_or_empty(NULL) | -+---------------------+ -| 1 | -+---------------------+ - -MySQL [(none)]> select null_or_empty(""); -+-------------------+ -| null_or_empty('') | -+-------------------+ -| 1 | -+-------------------+ - -MySQL [(none)]> select null_or_empty("a"); -+--------------------+ -| null_or_empty('a') | -+--------------------+ -| 0 | -+--------------------+ -``` -##keyword -NULL_OR_EMPTY \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/regexp_extract_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/regexp_extract_EN.md deleted file mode 100644 index 8cfc2828ce75a2..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/regexp_extract_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# regexp_extract -## Description -### Syntax - -'VARCHAR regexp 'extract (VARCHAR str, VARCHAR pattern, int pos) - - -The string STR is matched regularly and the POS matching part which conforms to pattern is extracted. Patterns need to match exactly some part of the STR to return to the matching part of the pattern. If there is no match, return an empty string. - -## example - -``` -mysql> SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 1); -+-------------------------------------------------------------+ -| regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 1) | -+-------------------------------------------------------------+ -| b | -+-------------------------------------------------------------+ - -mysql> SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 2); -+-------------------------------------------------------------+ -| regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 2) | -+-------------------------------------------------------------+ -| d | -+-------------------------------------------------------------+ -``` -##keyword -REGEXP_EXTRACT,REGEXP,EXTRACT diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/regexp_replace_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/regexp_replace_EN.md deleted file mode 100644 index 96b8a3a60b2b75..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/regexp_replace_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# regexp_replace -## description -### Syntax - -`VARCHAR regexp_replace(VARCHAR str, VARCHAR pattern, VARCHAR repl) - - -Regular matching of STR strings, replacing the part hitting pattern with repl - -## example - -``` -mysql> SELECT regexp_replace('a b c', " ", "-"); -+-----------------------------------+ -| regexp_replace('a b c', ' ', '-') | -+-----------------------------------+ -| a-b-c | -+-----------------------------------+ - -mysql> SELECT regexp_replace('a b c','(b)','<\\1>'); -+----------------------------------------+ -| regexp_replace('a b c', '(b)', '<\1>') | -+----------------------------------------+ -| a c | -+----------------------------------------+ -``` -##keyword -REGEXP_REPLACE,REGEXP,REPLACE diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/repeat_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/repeat_EN.md deleted file mode 100644 index bbc3dd8cd4befd..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/repeat_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# repeat -## Description -### Syntax - -'VARCHAR repeat (VARCHAR str, INT count) - - -Repeat the str of the string count times, return empty string when count is less than 1, return NULL when str, count is any NULL - -## example - -``` -mysql> SELECT repeat("a", 3); -+----------------+ -| repeat('a', 3) | -+----------------+ -| aaa | -+----------------+ - -mysql> SELECT repeat("a", -1); -+-----------------+ -| repeat('a', -1) | -+-----------------+ -| | -+-----------------+ -``` -##keyword -REPEAT, diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/right_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/right_EN.md deleted file mode 100644 index 6d4a06b56e1e74..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/right_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# right -## Description -### Syntax - -'VARCHAR RIGHT (VARCHAR STR)' - - -It returns the right part of a string of specified length - -## example - -``` -mysql> select right("Hello doris",5); -+-------------------------+ -| right('Hello doris', 5) | -+-------------------------+ -| doris | -+-------------------------+ -``` -##keyword -RIGHT diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/split_part_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/split_part_EN.md deleted file mode 100644 index a19025fc86df38..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/split_part_EN.md +++ /dev/null @@ -1,62 +0,0 @@ - - -# split_part -## Description -### Syntax - -'VARCHAR split party (VARCHAR content, VARCHAR delimiter, INT field)' - - -Returns the specified partition (counting from the beginning) by splitting the string according to the partitioner. - -## example - -``` -mysql> select split_part("hello world", " ", 1); -+----------------------------------+ -| split_part('hello world', ' ', 1) | -+----------------------------------+ -| hello | -+----------------------------------+ - - -mysql> select split_part("hello world", " ", 2); -+----------------------------------+ -| split_part('hello world', ' ', 2) | -+----------------------------------+ -| world | -+----------------------------------+ - -mysql> select split_part("2019年7月8号", "月", 1); -+-----------------------------------------+ -| split_part('2019年7月8号', '月', 1) | -+-----------------------------------------+ -| 2019年7 | -+-----------------------------------------+ - -mysql> select split_part("abca", "a", 1); -+----------------------------+ -| split_part('abca', 'a', 1) | -+----------------------------+ -| | -+----------------------------+ -``` -##keyword -SPLIT_PART,SPLIT,PART diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/starts_with_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/starts_with_EN.md deleted file mode 100644 index 81f1456c9160fc..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/starts_with_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# starts_with -## Description -### Syntax - -`BOOLEAN STARTS_WITH (VARCHAR str, VARCHAR prefix)` - -It returns true if the string starts with the specified prefix, otherwise it returns false. -If any parameter is NULL, it returns NULL. - -## example - -``` -MySQL [(none)]> select starts_with("hello world","hello"); -+-------------------------------------+ -| starts_with('hello world', 'hello') | -+-------------------------------------+ -| 1 | -+-------------------------------------+ - -MySQL [(none)]> select starts_with("hello world","world"); -+-------------------------------------+ -| starts_with('hello world', 'world') | -+-------------------------------------+ -| 0 | -+-------------------------------------+ -``` -##keyword -STARTS_WITH \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/strleft_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/strleft_EN.md deleted file mode 100644 index 2700327491db50..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/strleft_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# strleft -## Description -### Syntax - -'VARCHAR STRAIGHT (VARCHAR STR)' - - -It returns the left part of a string of specified length - -## example - -``` -mysql> select strleft("Hello doris",5); -+------------------------+ -| strleft('Hello doris', 5) | -+------------------------+ -| Hello | -+------------------------+ -``` -##keyword -STRLEFT diff --git a/docs/documentation/en/sql-reference/sql-functions/string-functions/strright_EN.md b/docs/documentation/en/sql-reference/sql-functions/string-functions/strright_EN.md deleted file mode 100644 index ed81e8e859dbcc..00000000000000 --- a/docs/documentation/en/sql-reference/sql-functions/string-functions/strright_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - - -# strright -## Description -### Syntax - -'VARCHAR strright (VARCHAR str)' - - -It returns the right part of a string of specified length - -## example - -``` -mysql> select strright("Hello doris",5); -+-------------------------+ -| strright('Hello doris', 5) | -+-------------------------+ -| doris | -+-------------------------+ -``` -##keyword -STRRIGHT diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/CREATE ROLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/CREATE ROLE_EN.md deleted file mode 100644 index 7cae6ef247c8c4..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/CREATE ROLE_EN.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# CREATE ROLE -## Description -The statement user creates a role - -Grammar: -CREATE ROLE role1; - -This statement creates an unauthorized role that can be subsequently granted permission through the GRANT command. - -## example - -1. Create a role - -CREATE ROLE role1; - -## keyword -CREATE, ROLE - - diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/CREATE USER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/CREATE USER_EN.md deleted file mode 100644 index a7d10946e5b1ed..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/CREATE USER_EN.md +++ /dev/null @@ -1,67 +0,0 @@ - - -# CREATE USER -##Description - -Syntax: - -CREATE USER user_identity [IDENTIFIED BY 'password'] [DEFAULT ROLE 'role_name'] - -user_identity: -'user_name'@'host' - -The CREATE USER command is used to create a Doris user. In Doris, a user_identity uniquely identifies a user. User_identity consists of two parts, user_name and host, where username is the user name. The host identifies the host address where the client connects. The host part can use% for fuzzy matching. If no host is specified, the default is'%', which means that the user can connect to Doris from any host. - -The host part can also be specified as a domain with the grammar:'user_name'@['domain']. Even if surrounded by brackets, Doris will think of it as a domain and try to parse its IP address. At present, it only supports BNS analysis within Baidu. - -If a role (ROLE) is specified, the permissions that the role has are automatically granted to the newly created user. If not specified, the user defaults to having no permissions. The specified ROLE must already exist. - -## example - -1. Create a passwordless user (without specifying host, it is equivalent to Jack @'%') - -CREATE USER 'jack'; - -2. Create a password user that allows login from'172.10.1.10' - -CREATE USER jack@'172.10.1.10' IDENTIFIED BY '123456'; - -3. To avoid passing plaintext, use case 2 can also be created in the following way - -CREATE USER jack@'172.10.1.10' IDENTIFIED BY PASSWORD '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9'; - -Later encrypted content can be obtained through PASSWORD (), for example: - -SELECT PASSWORD('123456'); - -4. Create a user who is allowed to log in from the'192.168'subnet and specify its role as example_role - -CREATE USER 'jack'@'192.168.%' DEFAULT ROLE 'example_role'; - -5. Create a user who is allowed to log in from the domain name'example_domain'. - -CREATE USER 'jack'@['example_domain'] IDENTIFIED BY '12345'; - -6. Create a user and specify a role - -CREATE USER 'jack'@'%' IDENTIFIED BY '12345' DEFAULT ROLE 'my_role'; - -## keyword -CREATE, USER diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/DROP ROLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/DROP ROLE_EN.md deleted file mode 100644 index 94247beb97cb76..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/DROP ROLE_EN.md +++ /dev/null @@ -1,36 +0,0 @@ - - -# DROP ROLE -## Description -The statement user deletes a role - -Grammar: -DROP ROLE role1; - -Deleting a role does not affect the permissions of users who previously belonged to that role. It is only equivalent to decoupling the role from the user. The permissions that the user has obtained from the role will not change. - -## example - -1. Delete a role - -DROP ROLE role1; - -## keyword -DROP, ROLE diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/DROP USER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/DROP USER_EN.md deleted file mode 100644 index 3e69a3907ece16..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/DROP USER_EN.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# DROP USER -## Description - -Syntax: - - DROP USER 'user_identity' - - `user_identity`: - - user@'host' - user@['domain'] - - Drop a specified user identity. - -## example - -1. Delete user jack@'192.%' - - DROP USER 'jack'@'192.%' - -## keyword - - DROP, USER diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/GRANT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/GRANT_EN.md deleted file mode 100644 index 2ec2efb4cf471f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/GRANT_EN.md +++ /dev/null @@ -1,74 +0,0 @@ - - -# Grant -## Description - -The GRANT command is used to give the specified user or role the specified permissions. - -Syntax: - -GRANT privilege_list ON db_name[.tbl_name] TO user_identity [ROLE role_name] - - -Privilege_list is a list of permissions that need to be granted, separated by commas. Currently Doris supports the following permissions: - -NODE_PRIV: Operational privileges of cluster nodes, including operation of nodes'up and down lines. Only root users have this privilege and can not be given to other users. -ADMIN_PRIV: All rights except NODE_PRIV. -GRANT_PRIV: Permission to operate permissions. Including the creation and deletion of users, roles, authorization and revocation, password settings and so on. -SELECT_PRIV: Read permissions for specified libraries or tables -LOAD_PRIV: Import permissions for specified libraries or tables -ALTER_PRIV: schema change permissions for specified libraries or tables -CREATE_PRIV: Creation permissions for specified libraries or tables -DROP_PRIV: Delete permissions for specified libraries or tables - -旧版权限中的 ALL 和 READ_WRITE 会被转换成：SELECT_PRIV,LOAD_PRIV,ALTER_PRIV,CREATE_PRIV,DROP_PRIV； -READ_ONLY is converted to SELECT_PRIV. - -Db_name [.tbl_name] supports the following three forms: - -1. *. * permissions can be applied to all libraries and all tables in them -2. db. * permissions can be applied to all tables under the specified library -3. db.tbl permissions can be applied to specified tables under specified Libraries - -The libraries or tables specified here can be non-existent libraries and tables. - -user_identity： - -The user_identity syntax here is the same as CREATE USER. And you must create user_identity for the user using CREATE USER. The host in user_identity can be a domain name. If it is a domain name, the validity time of permissions may be delayed by about one minute. - -You can also grant permissions to the specified ROLE, which is automatically created if the specified ROLE does not exist. - -## example - -1. Grant permissions to all libraries and tables to users - -GRANT SELECT_PRIV ON *.* TO 'jack'@'%'; - -2. Grant permissions to specified library tables to users - -GRANT SELECT_PRIV,ALTER_PRIV,LOAD_PRIV ON db1.tbl1 TO 'jack'@'192.8.%'; - -3. Grant permissions to specified library tables to roles - -GRANT LOAD_PRIV ON db1.* TO ROLE 'my_role'; - -## keyword -GRANT - diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/REVOKE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/REVOKE_EN.md deleted file mode 100644 index a1ba9401b60a32..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/REVOKE_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# REVOKE -## Description - -The REVOKE command is used to revoke the rights specified by the specified user or role. -Syntax -REVOKE privilege_list ON db_name[.tbl_name] FROM user_identity [ROLE role_name] - -user_identity： - -The user_identity syntax here is the same as CREATE USER. And you must create user_identity for the user using CREATE USER. The host in user_identity can be a domain name. If it is a domain name, the revocation time of permission may be delayed by about one minute. - -You can also revoke the permission of the specified ROLE, which must exist for execution. - -## example - -1. Revoke the rights of user Jack database testDb - -REVOKE SELECT_PRIV ON db1.* FROM 'jack'@'192.%'; - -## keyword - -REVOKE diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/SET PASSWORD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/SET PASSWORD_EN.md deleted file mode 100644 index 41226d3f2f1cae..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/SET PASSWORD_EN.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# SET PASSWORD -## Description - -Syntax: - -SET PASSWORD [FOR user_identity] = -[PASSWORD('plain password')]|['hashed password'] - -The SET PASSWORD command can be used to modify a user's login password. If the [FOR user_identity] field does not exist, modify the password of the current user. - -Note that the user_identity here must match exactly the user_identity specified when creating a user using CREATE USER, otherwise the user will be reported as non-existent. If user_identity is not specified, the current user is'username'@'ip', which may not match any user_identity. The current user can be viewed through SHOW GRANTS. - -PASSWORD () input is a plaintext password, and direct use of strings, you need to pass the encrypted password. -If you change the password of other users, you need to have administrator privileges. - -## example - -1. Modify the password of the current user - -SET PASSWORD = PASSWORD('123456') -SET PASSWORD = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' - -2. Modify the specified user password - -SET PASSWORD FOR 'jack'@'192.%' = PASSWORD('123456') -SET PASSWORD FOR 'jack'@['domain'] = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' - -## keyword -SET, PASSWORD diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/SET PROPERTY_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/SET PROPERTY_EN.md deleted file mode 100644 index 20dfa914f2d9f7..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/SET PROPERTY_EN.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# SET PROPERTY -## Description - -Syntax: - -SET PROPERTY [FOR 'user'] 'key' = 'value' [, 'key' = 'value'] - -Set user attributes, including resources allocated to users, import cluster, etc. The user attributes set here are for user, not user_identity. That is to say, if two users'jack'@'%' and'jack'@'192%'are created through the CREATE USER statement, the SET PROPERTY statement can only be used for the jack user, not'jack'@'%' or'jack'@'192%' - -Importing cluster is only applicable to Baidu internal users. - -key: - -Super user rights: -Max_user_connections: Maximum number of connections. -resource.cpu_share: cpu资源分配。 -Load_cluster. {cluster_name}. priority: assigns priority to a specified cluster, which can be HIGH or NORMAL - -Ordinary user rights: -Quota.normal: Resource allocation at the normal level. -Quota.high: Resource allocation at the high level. -Quota.low: Resource allocation at low level. - -Load_cluster. {cluster_name}. hadoop_palo_path: The Hadoop directory used by Palo needs to store ETL programs and intermediate data generated by ETL for Palo to import. After the import is completed, the intermediate data will be automatically cleaned up, and the ETL program will be automatically reserved for next use. -Load_cluster. {cluster_name}. hadoop_configs: configuration of hadoop, where fs. default. name, mapred. job. tracker, hadoop. job. UGI must be filled in. -Load ucluster. {cluster name}. hadoop port: Hadoop HDFS name node http} -Default_load_cluster: The default import cluster. - -## example - -1. Modify the maximum number of user jacks to 1000 -SET PROPERTY FOR 'jack' 'max_user_connections' = '1000'; - -2. Modify the cpu_share of user Jack to 1000 -SET PROPERTY FOR 'jack' 'resource.cpu_share' = '1000'; - -3. Modify the weight of the normal group of Jack users -Set property for'jack''quota. normal' = 400'; - -4. Add import cluster for user jack -SET PROPERTY FOR 'jack' -'load 'cluster.{cluster name}.hadoop'u palo path' ='/user /palo /palo path', -'load_cluster.{cluster_name}.hadoop_configs' = 'fs.default.name=hdfs://dpp.cluster.com:port;mapred.job.tracker=dpp.cluster.com:port;hadoop.job.ugi=user,password;mapred.job.queue.name=job_queue_name_in_hadoop;mapred.job.priority=HIGH;'; - -5. Delete the import cluster under user jack. -SET PROPERTY FOR 'jack' 'load_cluster.{cluster_name}' = ''; - -6. Modify user jack's default import cluster -SET PROPERTY FOR 'jack' 'default_load_cluster' = '{cluster_name}'; - -7. Modify the cluster priority of user Jack to HIGH -SET PROPERTY FOR 'jack' 'load_cluster.{cluster_name}.priority' = 'HIGH'; - -## keyword -SET, PROPERTY - diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/SHOW GRANTS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/SHOW GRANTS_EN.md deleted file mode 100644 index 74745a1646d3ef..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/SHOW GRANTS_EN.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# SHOW GRANTS -## Description - -This statement is used to view user rights. - -Grammar: -SHOW [ALL] GRANTS [FOR user_identity]; - -Explain: -1. SHOW ALL GRANTS can view the privileges of all users. -2. If you specify user_identity, view the permissions of the specified user. And the user_identity must be created for the CREATE USER command. -3. If you do not specify user_identity, view the permissions of the current user. - - -## example - -1. View all user rights information - -SHOW ALL GRANTS; - -2. View the permissions of the specified user - -SHOW GRANTS FOR jack@'%'; - -3. View the permissions of the current user - -SHOW GRANTS; - -## keyword -SHOW, GRANTS diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/SHOW ROLES_EN.md b/docs/documentation/en/sql-reference/sql-statements/Account Management/SHOW ROLES_EN.md deleted file mode 100644 index 57376862d6fa42..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/SHOW ROLES_EN.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# SHOW ROLES -## Description -This statement is used to display all created role information, including the role name, the user included, and the permissions. - -Grammar: -SHOW ROLES; - -## example - -1. View the created roles: - -SHOW ROLES; - -## keyword -SHOW,ROLES diff --git a/docs/documentation/en/sql-reference/sql-statements/Account Management/index.rst b/docs/documentation/en/sql-reference/sql-statements/Account Management/index.rst deleted file mode 100644 index 68c44caa012cb3..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Account Management/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -====================== -Account Management -====================== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR_EN.md deleted file mode 100644 index f4f6ca31da4b20..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# ADMIN CANCEL REPAIR -## Description - -This statement is used to cancel repairing a specified table or partition with high priority - -Grammar: - -ADMIN CANCEL REPAIR TABLE table_name[ PARTITION (p1,...)]; - -Explain: - -1. This statement only indicates that the system no longer repairs fragmented copies of specified tables or partitions with high priority. The system will still repair the copy by default scheduling. - -## example - -1. Cancel High Priority Repair - -ADMIN CANCEL REPAIR TABLE tbl PARTITION(p1); - -## keyword -ADMIN,CANCEL,REPAIR diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET_EN.md deleted file mode 100644 index 9d653ad059c3a9..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET_EN.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# ADMIN CHECK TABLET -## description - -This statement is used to perform a specified check operation on a list of tablets. - -Syntax： - -``` -ADMIN CHECK TABLE (tablet_id1, tablet_id2, ...) -PROPERTIES("type" = "..."); -``` - -说明： - -1. You must specify the list of tablet ids and the "type" property in PROPERTIES. -2. Currently "type" only supports: - - * consistency: Check the consistency of the replicas of the tablet. This command is asynchronous. After sending, Doris will start to perform the consistency check job of the corresponding tablet. The final result will be reflected in the "InconsistentTabletNum" column in the result of `SHOW PROC" / statistic "; - -## example - -1. Perform a replica consistency check on a specified set of tablets - - ``` - ADMIN CHECK TABLET (10000, 10001) - PROPERTIES("type" = "consistency"); - ``` - -## keyword - - ADMIN,CHECK,TABLET diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN REPAIR_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN REPAIR_EN.md deleted file mode 100644 index 628cec6c998d03..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN REPAIR_EN.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# ADMIN REPAIR -## Description - -This statement is used to try to fix the specified table or partition first - -Grammar: - -ADMIN REPAIR TABLE table_name[ PARTITION (p1,...)] - -Explain: - -1. This statement only means that the system attempts to repair a fragmented copy of a specified table or partition with high priority, and it is not guaranteed to be successful. Users can view the repair status through the ADMIN SHOW REPLICA STATUS command. -2. The default timeout is 14400 seconds (4 hours). Timeout means that the system will no longer repair fragmented copies of specified tables or partitions with high priority. The command settings need to be reused. - -## example - -1. Attempt to fix the specified table - -ADMIN REPAIR TABLE tbl1; - -2. Attempt to fix the specified partition - -ADMIN REPAIR TABLE tbl1 PARTITION (p1, p2); - -## keyword -ADMIN,REPAIR diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SET CONFIG_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SET CONFIG_EN.md deleted file mode 100644 index 6f7dc58c8f6938..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SET CONFIG_EN.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# ADMIN SET CONFIG -## Description - -This statement is used to set the configuration items for the cluster (currently only the configuration items for setting FE are supported). -Settable configuration items can be viewed through AMDIN SHOW FRONTEND CONFIG; commands. - -Grammar: - -ADMIN SET FRONTEND CONFIG ("key" = "value"); - -## example - -1. "disable balance" true - -ADMIN SET FRONTEND CONFIG ("disable_balance" = "true"); - -## keyword -ADMIN,SET,CONFIG diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS_EN.md deleted file mode 100644 index 5d89b47a5b4d0f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS_EN.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# ADMIN SET REPLICA STATUS -## description - - This commend is used to set the status of the specified replica. -    This command is currently only used to manually set the status of some replicas to BAD or OK, allowing the system to automatically repair these replicas. - - Syntax: - - ADMIN SET REPLICA STATUS - PROPERTIES ("key" = "value", ...); - - The following attributes are currently supported: - "tablet_id": required. Specify a Tablet Id. - "backend_id": required. Specify a Backend Id. - "status": required. Specify the status. Only "bad" and "ok" are currently supported. - - If the specified replica does not exist or the status is already bad or ok, it will be ignored. - - Notice: - - Replica set to Bad status may be dropped immediately, please proceed with caution. - -## example - - 1. Set the replica status of tablet 10003 on BE 10001 to bad. - - ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "10003", "backend_id" = "10001", "status" = "bad"); - - 2. Set the replica status of tablet 10003 on BE 10001 to ok. - - ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "10003", "backend_id" = "10001", "status" = "ok"); - -## keyword - - ADMIN,SET,REPLICA,STATUS - diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG_EN.md deleted file mode 100644 index 75eca91f0f7bfe..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG_EN.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# ADMIN SHOW CONFIG -## Description - -This statement is used to show the configuration of the current cluster (currently only supporting the display of FE configuration items) - -Grammar: - -ADMIN SHOW FRONTEND CONFIG; - -Explain: - -The implications of the results are as follows: -1. Key: Configuration item name -2. Value: Configuration item value -3. Type: Configuration item type -4. IsMutable： 是否可以通过 ADMIN SET CONFIG 命令设置 -5. MasterOnly： 是否仅适用于 Master FE -6. Comment: Configuration Item Description - -## example - -1. View the configuration of the current FE node - -ADMIN SHOW FRONTEND CONFIG; - -## keyword -ADMIN,SHOW,CONFIG diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION_EN.md deleted file mode 100644 index c17916ca53d503..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION_EN.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# ADMIN SHOW REPLICA DISTRIBUTION -## Description - -This statement is used to show the distribution status of a table or partition replica - -Grammar: - -ADMIN SHOW REPLICA DISTRIBUTION FROM [db_name.]tbl_name [PARTITION (p1, ...)]; - -Explain: - -The Graph column in the result shows the distribution ratio of replicas graphically - -## example - -1. View the distribution of replicas of tables - -ADMIN SHOW REPLICA DISTRIBUTION FROM tbl1; - -2. View the distribution of copies of partitions in the table - -ADMIN SHOW REPLICA DISTRIBUTION FROM db1.tbl1 PARTITION(p1, p2); - -## keyword -ADMIN,SHOW,REPLICA,DISTRIBUTION diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS_EN.md deleted file mode 100644 index 638032568b3326..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS_EN.md +++ /dev/null @@ -1,57 +0,0 @@ - - -# ADMIN SHOW REPLICA STATUS -## Description - -This statement is used to display copy status information for a table or partition - -Grammar: - -ADMIN SHOW REPLICA STATUS FROM [dbu name.]tbl name [PARTITION (p1,...)] -[where_clause]; - -where_clause: -WHERE STATUS [!]= "replica_status" - -Reply status: -OK: Replica 22788;'20581;' 29366;'24577; -DEAD: The Backend of replica is not available -VERSION_ERROR: The replica data version is missing -SCHEMA ERROR: replica schema hash -MISSING: replica does not exist - -## example - -1. View the status of all copies of the table - -ADMIN SHOW REPLICA STATUS FROM db1.tbl1; - -2. View a copy of a partition state of the table as VERSION_ERROR - -ADMIN SHOW REPLICA STATUS FROM tbl1 PARTITION (p1, p2) - - -3. Check all unhealthy copies of the table - -ADMIN SHOW REPLICA STATUS FROM tbl1 -WHERE STATUS != "OK"; - -## keyword -ADMIN,SHOW,REPLICA,STATUS diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ALTER CLUSTER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ALTER CLUSTER_EN.md deleted file mode 100644 index a6d015c3e5d370..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ALTER CLUSTER_EN.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# ALTER CLUSTER -## description - -This statement is used to update the logical cluster. Administrator privileges are required - -grammar - -ALTER CLUSTER cluster_name PROPERTIES ("key"="value", ...); - -1. Scaling, scaling (according to the number of be existing in the cluster, large is scaling, small is scaling), scaling for synchronous operation, scaling for asynchronous operation, through the state of backend can be known whether the scaling is completed. - -Proerties ("Instrume = Unum"= "3") - -Instancefn Microsoft Yahei - -## example - -1. Reduce the number of be of logical cluster test_cluster containing 3 be by 2. - -ALTER CLUSTER test_cluster PROPERTIES ("instance_num"="2"); - -2. Expansion, increase the number of be of logical cluster test_cluster containing 3 be to 4 - -ALTER CLUSTER test_cluster PROPERTIES ("instance_num"="4"); - -## keyword -ALTER,CLUSTER diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ALTER SYSTEM_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ALTER SYSTEM_EN.md deleted file mode 100644 index 209985f07b3644..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ALTER SYSTEM_EN.md +++ /dev/null @@ -1,112 +0,0 @@ - - -# ALTER SYSTEM -## Description - -This statement is used to operate on nodes in a system. (Administrator only!) -Grammar: -1) Adding nodes (without multi-tenant functionality, add in this way) -ALTER SYSTEM ADD BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; -2) Adding idle nodes (that is, adding BACKEND that does not belong to any cluster) -ALTER SYSTEM ADD FREE BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; -3) Adding nodes to a cluster -ALTER SYSTEM ADD BACKEND TO cluster_name "host:heartbeat_port"[,"host:heartbeat_port"...]; -4) Delete nodes -ALTER SYSTEM DROP BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; -5) Node offline -ALTER SYSTEM DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; -6)226;- 21152;-Broker -ALTER SYSTEM ADD BROKER broker_name "host:port"[,"host:port"...]; -(7) 20943;"23569;" Broker -ALTER SYSTEM DROP BROKER broker_name "host:port"[,"host:port"...]; -8) Delete all Brokers -ALTER SYSTEM DROP ALL BROKER broker_name -9) Set up a Load error hub for centralized display of import error information -ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES ("key" = "value"[, ...]); - -Explain: -1) Host can be hostname or IP address -2) heartbeat_port is the heartbeat port of the node -3) Adding and deleting nodes are synchronous operations. These two operations do not take into account the existing data on the node, the node is directly deleted from the metadata, please use cautiously. -4) Node offline operations are used to secure offline nodes. This operation is asynchronous. If successful, the node will eventually be removed from the metadata. If it fails, the offline will not be completed. -5) The downline operation of the node can be cancelled manually. See CANCEL DECOMMISSION for details -6) Load error hub: -Currently, two types of Hub are supported: Mysql and Broker. You need to specify "type" = "mysql" or "type" = "broker" in PROPERTIES. -If you need to delete the current load error hub, you can set type to null. -1) When using the Mysql type, the error information generated when importing will be inserted into the specified MySQL library table, and then the error information can be viewed directly through the show load warnings statement. - -Hub of Mysql type needs to specify the following parameters: -guest guest -port：mysql port -user：mysql user -password：mysql password -database mysql database -table：mysql table - -2) When the Broker type is used, the error information generated when importing will form a file and be written to the designated remote storage system through the broker. Make sure that the corresponding broker is deployed -Hub of Broker type needs to specify the following parameters: -Broker: Name of broker -Path: Remote Storage Path -Other properties: Other information necessary to access remote storage, such as authentication information. - -## example - -1. Add a node -ALTER SYSTEM ADD BACKEND "host:port"; - -2. Adding an idle node -ALTER SYSTEM ADD FREE BACKEND "host:port"; - -3. Delete two nodes -ALTER SYSTEM DROP BACKEND "host1:port", "host2:port"; - -4. Two downline nodes -ALTER SYSTEM DECOMMISSION BACKEND "host1:port", "host2:port"; - -5. Add two Hdfs Broker -ALTER SYSTEM ADD BROKER hdfs "host1:port", "host2:port"; - -6. Add a load error hub of Mysql type -ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES -("type"= "mysql", -"host" = "192.168.1.17" -"port" = "3306", -"User" = "my" name, -"password" = "my_passwd", -"database" = "doris_load", -"table" = "load_errors" -); - -7. 添加一个 Broker 类型的 load error hub -ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES -("type"= "broker", -"Name" = BOS, -"path" = "bos://backup-cmy/logs", -"bosu endpoint" ="http://gz.bcebos.com", -"bos_accesskey" = "069fc278xxxxxx24ddb522", -"bos_secret_accesskey"="700adb0c6xxxxxx74d59eaa980a" -); - -8. Delete the current load error hub -ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES -("type"= "null"); - -## keyword -AGE,SYSTEM,BACKGROUND,BROKER,FREE diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION_EN.md deleted file mode 100644 index 02ffd4d81f2674..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION_EN.md +++ /dev/null @@ -1,33 +0,0 @@ - - -# CANCEL DECOMMISSION -## Description - -This statement is used to undo a node's offline operation. (Administrator only!) -Grammar: -CANCEL DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - -## example - -1. Cancel the offline operation of two nodes: -CANCEL DECOMMISSION BACKEND "host1:port", "host2:port"; - -## keyword -CANCEL,DECOMMISSION,BACKEND diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/CREATE CLUSTER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/CREATE CLUSTER_EN.md deleted file mode 100644 index df2d279648daa0..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/CREATE CLUSTER_EN.md +++ /dev/null @@ -1,54 +0,0 @@ - - -# CREATE CLUSTER -## Description - -This statement is used to create a new logical cluster, requiring administrator privileges. If you don't use multiple tenants, create a cluster named default_cluster directly. Otherwise, create a cluster with a custom name. - -grammar - -CREATE CLUSTER [IF NOT EXISTS] cluster_name - -PROPERTIES ("key"="value", ...) - -IDENTIFIED BY 'password' - -1. PROPERTIES - -Specify attributes of logical clusters - -PROPERTIES ("instance_num" = "3") - -Instancefn Microsoft Yahei - -2. Identify by'password'each logical cluster contains a superuser whose password must be specified when creating a logical cluster - -## example - -1. Create a new test_cluster with three be nodes and specify its superuser password - -CREATE CLUSTER test_cluster PROPERTIES("instance_num"="3") IDENTIFIED BY 'test'; - -2. Create a new default_cluster with three be nodes (no multi-tenant is used) and specify its superuser password - -CREATE CLUSTER default_cluster PROPERTIES("instance_num"="3") IDENTIFIED BY 'test'; - -## keyword -CREATE,CLUSTER diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/CREATE FILE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/CREATE FILE_EN.md deleted file mode 100644 index 30dce72446d15f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/CREATE FILE_EN.md +++ /dev/null @@ -1,69 +0,0 @@ - - -# CREATE FILE -## Description - -This statement is used to create and upload a file to the Doris cluster. -This function is usually used to manage files that need to be used in some other commands, such as certificates, public key, private key, etc. - -This command can be executed by users with amdin privileges only. -A file belongs to a database. This file can be used by users who have access to database. - -The size of a single file is limited to 1MB. -A Doris cluster uploads up to 100 files. - -Grammar: - -CREATE FILE "File name" [IN database] -[properties] - -Explain: -File_name: Custom file name. -Database: The file belongs to a db, and if not specified, the DB of the current session is used. -properties 支持以下参数: - -Url: Must. Specify a download path for a file. Currently only unauthenticated HTTP download paths are supported. When the command line succeeds, the file will be saved in Doris and the URL will no longer be required. -Catalog: Yes. The classification name of the file can be customized. But in some commands, files in the specified catalog are looked up. For example, in a routine import, when the data source is kafka, the file under the name of catalog is looked up. -Md5: Optional. MD5 of the file. If specified, it will be checked after downloading the file. - -## example - -1. Create a file ca. pem, categorized as Kafka - -CREATE FILE "ca.pem" -PROPERTIES -( -"url" ="https://test.bj.bcebos.com /kafka -key /ca.pem", -"catalog" = "kafka" -); - -2. Create the file client. key, categorized as my_catalog - -CREATE FILE "client.key" -IN my database -PROPERTIES -( -"url" ="https://test.bj.bcebos.com /kafka -key /client.key", -"catalog" = "my_catalog", -"md5"= "b5bb901bf1099205b39a46ac3557dd9" -); - -## keyword -CREATE,FILE diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/DROP CLUSTER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/DROP CLUSTER_EN.md deleted file mode 100644 index 3cfe58e53dbd2b..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/DROP CLUSTER_EN.md +++ /dev/null @@ -1,36 +0,0 @@ - - -# DROP CLUSTER -## Description - -This statement is used to delete logical cluster. Successful deletion of logical cluster requires first deleting dB in the cluster and administrator privileges. - -grammar - -DROP CLUSTER [IF EXISTS] cluster_name - -## example - -Delete logical cluster test_cluster - -DROP CLUSTER test_cluster; - -## keyword -DROP,CLUSTER diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/DROP FILE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/DROP FILE_EN.md deleted file mode 100644 index 060fa0a1e4622c..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/DROP FILE_EN.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# DROP FILE -## Description - -This statement is used to delete an uploaded file. - -Grammar: - -DROP FILE "file_name" [FROM database] -[properties] - -Explain: -File_name: File name. -Database: A DB to which the file belongs, if not specified, uses the DB of the current session. -properties 支持以下参数: - -Catalog: Yes. Classification of documents. - -## example - -1. Delete the file ca.pem - -DROP FILE "ca.pem" properties("catalog" = "kafka"); - -## keyword -DROP,FILE diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/ENTER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/ENTER_EN.md deleted file mode 100644 index 09f98435318627..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/ENTER_EN.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# ENTER -## Description - -This statement is used to enter a logical cluster. All users and databases created need to be executed in a logical cluster. After creation, they belong to the logic. - -Cluster, need administrator privileges - -ENTER cluster name - -## example - -1. Enter the logical cluster test_cluster - -ENTER test cluster; - -## keyword -ENTER - diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN_EN.md deleted file mode 100644 index f4744302cf3f85..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN_EN.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# INSTALL PLUGIN -## description - - To install a plugin - - Syntax - - INSTALL PLUGIN FROM [source] - - source supports 3 kinds: - - 1. Point to a zip file with absolute path. - 2. Point to a plugin dir with absolute path. - 3. Point to a http/https download link of zip file. - -## example - - 1. Intall a plugin with a local zip file: - - INSTALL PLUGIN FROM "/home/users/seaven/auditdemo.zip"; - - 2. Intall a plugin with a local dir: - - INSTALL PLUGIN FROM "/home/users/seaven/auditdemo/"; - - 2. Download and install a plugin: - - INSTALL PLUGIN FROM "http://mywebsite.com/plugin.zip"; - -## keyword - INSTALL,PLUGIN diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/LINK DATABASE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/LINK DATABASE_EN.md deleted file mode 100644 index 3344161024ff32..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/LINK DATABASE_EN.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# LINK DATABASE -## Description - -This statement allows users to link a database of one logical cluster to another logical cluster. A database is only allowed to be linked once at the same time and the linked database is deleted. - -It does not delete data, and the linked database cannot be deleted. Administrator privileges are required. - -grammar - -LINK DATABASE src u cluster name.src db name of the cluster name.des db name - -## example - -1. Link test_db in test_cluster A to test_cluster B and name it link_test_db - -LINK DATABASE test_clusterA.test_db test_clusterB.link_test_db; - -2. Delete linked database link_test_db - -DROP DATABASE link_test_db; - -## keyword -LINK,DATABASE diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/MIGRATE DATABASE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/MIGRATE DATABASE_EN.md deleted file mode 100644 index 3229f0fd03e26a..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/MIGRATE DATABASE_EN.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# MIGRATE DATABASE -## Description - -This statement is used to migrate a logical cluster database to another logical cluster. Before performing this operation, the database must be in a link state and need to be managed. - -Membership authority - -grammar - -MIGRATE DATABASE src u cluster name.src db name of the cluster name.des db name - -## example - -1. 迁移test_clusterA中的test_db到test_clusterB - -MIGRATE DATABASE test_clusterA.test_db test_clusterB.link_test_db; - -## keyword -MIGRATE,DATABASE diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW BACKENDS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW BACKENDS_EN.md deleted file mode 100644 index 5a21669b198667..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW BACKENDS_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# SHOW BACKENDS -## Description -This statement is used to view BE nodes in the cluster -Grammar: -SHOW BACKENDS; - -Explain: -1. LastStartTime indicates the last BE start-up time. -2. LastHeartbeat represents the latest heartbeat. -3. Alive indicates whether the node survives. -4. System Decommissioned is true to indicate that the node is safely offline. -5. Cluster Decommissioned is true to indicate that the node is rushing downline in the current cluster. -6. TabletNum represents the number of fragments on the node. -7. Data Used Capacity represents the space occupied by the actual user data. -8. Avail Capacity represents the available space on the disk. -9. Total Capacity represents total disk space. Total Capacity = AvailCapacity + DataUsedCapacity + other non-user data files take up space. -10. UsedPct represents the percentage of disk usage. -11. ErrMsg is used to display error messages when a heartbeat fails. - -## keyword -SHOW, BACKENDS diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW BROKER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW BROKER_EN.md deleted file mode 100644 index deba72ac3bd50d..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW BROKER_EN.md +++ /dev/null @@ -1,33 +0,0 @@ - - -# SHOW BROKER -## Description -This statement is used to view the existing broker -Grammar: -SHOW BROKER; - -Explain: -1. LastStartTime indicates the last BE start-up time. -2. LastHeartbeat represents the latest heartbeat. -3. Alive indicates whether the node survives. -4. ErrMsg is used to display error messages when the heartbeat fails. - -## keyword -SHOW, BROKER diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FILE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FILE_EN.md deleted file mode 100644 index 927625f297684e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FILE_EN.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# SHOW FILE -## Description - -This statement is used to show a file created in a database - -Grammar: - -SHOW FILE [FROM database]; - -Explain: - -FileId: File ID, globally unique -DbName: The name of the database to which it belongs -Catalog: Custom Categories -FileName: File name -FileSize: File size, unit byte -MD5: Document MD5 - -## example - -1. View uploaded files in my_database - -SHOW FILE FROM my_database; - -## keyword -SHOW,FILE diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FRONTENDS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FRONTENDS_EN.md deleted file mode 100644 index 457cc8682b4aba..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FRONTENDS_EN.md +++ /dev/null @@ -1,36 +0,0 @@ - - -# SHOW FRONTENDS -## Description -This statement is used to view FE nodes -Grammar: -SHOW FRONTENDS; - -Explain: -1. name denotes the name of the FE node in bdbje. -2. Join is true to indicate that the node has joined the cluster. But it doesn't mean that it's still in the cluster (it may be out of touch) -3. Alive indicates whether the node survives. -4. Replayed Journal Id represents the maximum metadata log ID that the node has currently replayed. -5. LastHeartbeat is the latest heartbeat. -6. IsHelper indicates whether the node is a helper node in bdbje. -7. ErrMsg is used to display error messages when the heartbeat fails. - -## keyword -SHOW, FRONTENDS diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS_EN.md deleted file mode 100644 index dd47576f746c3c..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS_EN.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# SHOW FULL COLUMNS -## description - This statement is used to view some information about columns of a table. - - Syntax： - SHOW FULL COLUMNS FROM tbl; - -## example - - 1. View the column information of specified table - - SHOW FULL COLUMNS FROM tbl; - -## keyword - - SHOW,FULL,COLUMNS diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW INDEX_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW INDEX_EN.md deleted file mode 100644 index 996d5186ca4ab3..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW INDEX_EN.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# SHOW INDEX - -## description - - This statement is used to show all index(only bitmap index in current version) of a table - Grammar: - SHOW INDEX[ES] FROM [db_name.]table_name [FROM database]; - - OR - - SHOW KEY[S] FROM [db_name.]table_name [FROM database]; - -## example - - 1. dispaly all indexes in table table_name - SHOW INDEX FROM example_db.table_name; - -## keyword - - SHOW,INDEX diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW MIGRATIONS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW MIGRATIONS_EN.md deleted file mode 100644 index 3180c981969f88..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW MIGRATIONS_EN.md +++ /dev/null @@ -1,30 +0,0 @@ - - -# SHOW MIGRATIONS -## Description - -This statement is used to view the progress of database migration - -grammar - -SHOW MIGRATIONS - -## keyword -SHOW,MIGRATIONS diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW PLUGINS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW PLUGINS_EN.md deleted file mode 100644 index 1284ab7f900f03..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW PLUGINS_EN.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# SHOW PLUGINS -## description - - To view the installed plugins. - - Syntax - - SHOW PLUGINS; - - This command will show all builtin and custom plugins. - -## example - - 1. To view the installed plugins: - - SHOW PLUGINS; - -## keyword - SHOW PLUGINS \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW TABLE STATUS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW TABLE STATUS_EN.md deleted file mode 100644 index e9c228f720ebf8..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW TABLE STATUS_EN.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# SHOW TABLE STATUS - -## description - -This statement is used to view some information about Table. - - Syntax: - - SHOW TABLE STATUS - [FROM db] [LIKE "pattern"] - - Explain: - - 1. This statement is mainly used to be compatible with MySQL grammar. At present, only a small amount of information such as Comment is displayed. - -## Example - - 1. View the information of all tables under the current database - - SHOW TABLE STATUS; - - - 2. View the information of the table whose name contains example in the specified database - - SHOW TABLE STATUS FROM DB LIKE "% example%"; - -## Keyword - - SHOW,TABLE,STATUS \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN_EN.md deleted file mode 100644 index 48c52eb273cb88..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN_EN.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# UNINTALL PLUGIN -## description - - To uninstall a plugin. - - Syntax - - UNINSTALL PLUGIN plugin_name; - - plugin_name can be found by `SHOW PLUGINS;`. - - Can only uninstall non-builtin plugins. - -## example - - 1. Uninstall a plugin: - - UNINSTALL PLUGIN auditdemo; - -## keyword - UNINSTALL,PLUGIN \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/index.rst b/docs/documentation/en/sql-reference/sql-statements/Administration/index.rst deleted file mode 100644 index 29a3fe69ad79fd..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Administration/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -================== -Administration -================== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER DATABASE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER DATABASE_EN.md deleted file mode 100644 index daadef1aa4c192..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER DATABASE_EN.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# ALTER DATABASE -## description -This statement is used to set the properties of the specified database. (Administrators only) -Grammar: -1) Setting database data quota in B/K/KB/M/MB/G/GB/T/TB/P/PB -OTHER DATABASE dbu name SET DATA QUOTA quota; - -2) Rename the database -ALTER DATABASE db_name RENAME new_db_name; - -Explain: -After renaming the database, use REVOKE and GRANT commands to modify the corresponding user rights if necessary. -The database's default data quota is 1024GB, and the default replica quota is 1073741824. - -## example -1. Setting the specified database data quota -ALTER DATABASE example_db SET DATA QUOTA 10995116277760; -The above units are bytes, equivalent to -ALTER DATABASE example_db SET DATA QUOTA 10T; - -ALTER DATABASE example_db SET DATA QUOTA 100G; - -ALTER DATABASE example_db SET DATA QUOTA 200M; - -2. Rename the database example_db to example_db2 -ALTER DATABASE example_db RENAME example_db2; - -## keyword -ALTER,DATABASE,RENAME - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md deleted file mode 100644 index faf4da0086f362..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md +++ /dev/null @@ -1,356 +0,0 @@ - - -# ALTER TABLE - -## description - - This statement is used to modify an existing table. If no rollup index is specified, the base operation is the default. - The statement is divided into three types of operations: schema change, rollup, partition - These three types of operations cannot appear in an ALTER TABLE statement at the same time. - Where schema change and rollup are asynchronous operations and are returned if the task commits successfully. You can then use the SHOW ALTER command to view the progress. - Partition is a synchronous operation, and a command return indicates that execution is complete. - - grammar: - ALTER TABLE [database.]table - Alter_clause1[, alter_clause2, ...]; - - The alter_clause is divided into partition, rollup, schema change, rename and bimmap index. - - Partition supports the following modifications - Increase the partition - grammar: - ADD PARTITION [IF NOT EXISTS] partition_name - Partition_desc ["key"="value"] - [DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]] - note: - 1) partition_desc supports two ways of writing: - * VALUES LESS THAN [MAXVALUE|("value1", ...)] - * VALUES [("value1", ...), ("value1", ...)) - 1) The partition is the left closed right open interval. If the user only specifies the right boundary, the system will automatically determine the left boundary. - 2) If the bucket mode is not specified, the bucket method used by the built-in table is automatically used. - 3) If the bucket mode is specified, only the bucket number can be modified, and the bucket mode or bucket column cannot be modified. - 4) ["key"="value"] section can set some properties of the partition, see CREATE TABLE for details. - - 2. Delete the partition - grammar: - DROP PARTITION [IF EXISTS] partition_name - note: - 1) Use a partitioned table to keep at least one partition. - 2) Execute DROP PARTITION For a period of time, the deleted partition can be recovered by the RECOVER statement. See the RECOVER statement for details. - - 3. Modify the partition properties - grammar: - MODIFY PARTITION partition_name SET ("key" = "value", ...) - Description: - 1) The following attributes of the modified partition are currently supported. - - storage_medium - - storage_cooldown_time - - replication_num - — in_memory - 2) For single-partition tables, partition_name is the same as the table name. - - Rollup supports the following ways to create: - 1. Create a rollup index - grammar: - ADD ROLLUP rollup_name (column_name1, column_name2, ...) - [FROM from_index_name] - [PROPERTIES ("key"="value", ...)] - - properties: Support setting timeout time, the default timeout time is 1 day. - example: - ADD ROLLUP r1(col1,col2) from r0 - 1.2 Batch create rollup index - grammar: - ADD ROLLUP [rollup_name (column_name1, column_name2, ...) - [FROM from_index_name] - [PROPERTIES ("key"="value", ...)],...] - example： - ADD ROLLUP r1(col1,col2) from r0, r2(col3,col4) from r0 - 1.3 note: - 1) If from_index_name is not specified, it is created by default from base index - 2) The columns in the rollup table must be existing columns in from_index - 3) In properties, you can specify the storage format. See CREATE TABLE for details. - - 2. Delete the rollup index - grammar: - DROP ROLLUP rollup_name - [PROPERTIES ("key"="value", ...)] - example: - DROP ROLLUP r1 - 2.1 Batch Delete rollup index - grammar：DROP ROLLUP [rollup_name [PROPERTIES ("key"="value", ...)],...] - example：DROP ROLLUP r1,r2 - 2.2 note: - 1) Cannot delete base index - 2) Execute DROP ROLLUP For a period of time, the deleted rollup index can be restored by the RECOVER statement. See the RECOVER statement for details. - - - Schema change supports the following modifications: - 1. Add a column to the specified location of the specified index - grammar: - ADD COLUMN column_name column_type [KEY | agg_type] [DEFAULT "default_value"] - [AFTER column_name|FIRST] - [TO rollup_index_name] - [PROPERTIES ("key"="value", ...)] - note: - 1) Aggregate model If you add a value column, you need to specify agg_type - 2) Non-aggregate models (such as DUPLICATE KEY) If you add a key column, you need to specify the KEY keyword. - 3) You cannot add a column that already exists in the base index to the rollup index - Recreate a rollup index if needed - - 2. Add multiple columns to the specified index - grammar: - ADD COLUMN (column_name1 column_type [KEY | agg_type] DEFAULT "default_value", ...) - [TO rollup_index_name] - [PROPERTIES ("key"="value", ...)] - note: - 1) Aggregate model If you add a value column, you need to specify agg_type - 2) Non-aggregate model If you add a key column, you need to specify the KEY keyword. - 3) You cannot add a column that already exists in the base index to the rollup index - (You can recreate a rollup index if needed) - - 3. Remove a column from the specified index - grammar: - DROP COLUMN column_name - [FROM rollup_index_name] - note: - 1) Cannot delete partition column - 2) If the column is removed from the base index, it will also be deleted if the column is included in the rollup index - - 4. Modify the column type and column position of the specified index - grammar: - MODIFY COLUMN column_name column_type [KEY | agg_type] [NULL | NOT NULL] [DEFAULT "default_value"] - [AFTER column_name|FIRST] - [FROM rollup_index_name] - [PROPERTIES ("key"="value", ...)] - note: - 1) Aggregate model If you modify the value column, you need to specify agg_type - 2) Non-aggregate type If you modify the key column, you need to specify the KEY keyword. - 3) Only the type of the column can be modified. The other attributes of the column remain as they are (ie other attributes need to be explicitly written in the statement according to the original attribute, see example 8) - 4) The partition column cannot be modified - 5) The following types of conversions are currently supported (accuracy loss is guaranteed by the user) - TINYINT/SMALLINT/INT/BIGINT is converted to TINYINT/SMALLINT/INT/BIGINT/DOUBLE. - TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL is converted to VARCHAR - Convert LARGEINT to DOUBLE - VARCHAR supports modification of maximum length - Convert VARCHAR to TINYINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE. - Convert VARCHAR to DATE (currently support six formats: "%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d, "%y/%m/%d") - Convert DATETIME to DATE(Only year-month-day information is retained, For example: `2019-12-09 21:47:05` <--> `2019-12-09`) - Convert DATE to DATETIME(Set hour, minute, second to zero, For example: `2019-12-09` <--> `2019-12-09 00:00:00`) - Convert FLOAT to DOUBLE - Convert INT to DATE (If the INT data fails to convert, the original data remains the same) - 6) Does not support changing from NULL to NOT NULL - - 5. Reorder the columns of the specified index - grammar: - ORDER BY (column_name1, column_name2, ...) - [FROM rollup_index_name] - [PROPERTIES ("key"="value", ...)] - note: - 1) All columns in index must be written - 2) value is listed after the key column - - 6. Modify the properties of the table, currently supports modifying the bloom filter column, the colocate_with attribute and the dynamic_partition attribute， the replication_num and default.replication_num. - grammar: - PROPERTIES ("key"="value") - note: - Can also be merged into the above schema change operation to modify, see the example below -   - - Rename supports modification of the following names: - 1. Modify the table name - grammar: - RENAME new_table_name; - - 2. Modify the rollup index name - grammar: - RENAME ROLLUP old_rollup_name new_rollup_name; - - 3. Modify the partition name - grammar: - RENAME PARTITION old_partition_name new_partition_name; - - Bitmap index supports the following modifications: - 1. create bitmap index - grammar: - ADD INDEX index_name (column [, ...],) [USING BITMAP] [COMMENT 'balabala']; - note: - 1. only supports bitmap index for current version - 2. BITMAP index only supports apply on single column - 2. drop index - grammar: - DROP INDEX index_name； - -## example - - [table] - 1. Modify the default number of replications of the table, which is used as default number of replications while creating new partition. - ATLER TABLE example_db.my_table - SET ("default.replication_num" = "2"); - - 2. Modify the actual number of replications of a unpartitioned table (unpartitioned table only) - ALTER TABLE example_db.my_table - SET ("replication_num" = "3"); - - [partition] - 1. Add partition, existing partition [MIN, 2013-01-01), add partition [2013-01-01, 2014-01-01), use default bucket mode - ALTER TABLE example_db.my_table - ADD PARTITION p1 VALUES LESS THAN ("2014-01-01"); - - 2. Increase the partition and use the new number of buckets - ALTER TABLE example_db.my_table - ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") - DISTRIBUTED BY HASH(k1) BUCKETS 20; - - 3. Increase the partition and use the new number of copies - ALTER TABLE example_db.my_table - ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") - ("replication_num"="1"); - - 4. Modify the number of partition copies - ALTER TABLE example_db.my_table - MODIFY PARTITION p1 SET("replication_num"="1"); - - 5. Delete the partition - ALTER TABLE example_db.my_table - DROP PARTITION p1; - - 6. Add a partition that specifies the upper and lower bounds - - ALTER TABLE example_db.my_table - ADD PARTITION p1 VALUES [("2014-01-01"), ("2014-02-01")); - - [rollup] - 1. Create index: example_rollup_index, based on base index(k1,k2,k3,v1,v2). Columnar storage. - ALTER TABLE example_db.my_table - ADD ROLLUP example_rollup_index(k1, k3, v1, v2) - PROPERTIES("storage_type"="column"); - - 2. Create index: example_rollup_index2, based on example_rollup_index(k1,k3,v1,v2) - ALTER TABLE example_db.my_table - ADD ROLLUP example_rollup_index2 (k1, v1) - FROM example_rollup_index; - - 3. Create index: example_rollup_index3, based on base index (k1, k2, k3, v1), custom rollup timeout time is one hour. - - ALTER TABLE example_db.my_table - ADD ROLLUP example_rollup_index(k1, k3, v1) - PROPERTIES("storage_type"="column", "timeout" = "3600"); - - 3. Delete index: example_rollup_index2 - ALTER TABLE example_db.my_table - DROP ROLLUP example_rollup_index2; - - [schema change] - 1. Add a key column new_col to the col1 of example_rollup_index (non-aggregate model) - ALTER TABLE example_db.my_table - ADD COLUMN new_col INT KEY DEFAULT "0" AFTER col1 - TO example_rollup_index; - - 2. Add a value column new_col to the col1 of example_rollup_index (non-aggregate model) -   ALTER TABLE example_db.my_table -   ADD COLUMN new_col INT DEFAULT "0" AFTER col1 -   TO example_rollup_index; - - 3. Add a key column new_col (aggregation model) to col1 of example_rollup_index -   ALTER TABLE example_db.my_table -   ADD COLUMN new_col INT DEFAULT "0" AFTER col1 -   TO example_rollup_index; - - 4. Add a value column to the col1 of example_rollup_index. new_col SUM aggregation type (aggregation model) -   ALTER TABLE example_db.my_table -   ADD COLUMN new_col INT SUM DEFAULT "0" AFTER col1 -   TO example_rollup_index; - - 5. Add multiple columns to the example_rollup_index (aggregate model) - ALTER TABLE example_db.my_table - ADD COLUMN (col1 INT DEFAULT "1", col2 FLOAT SUM DEFAULT "2.3") - TO example_rollup_index; - - 6. Remove a column from example_rollup_index - ALTER TABLE example_db.my_table - DROP COLUMN col2 - FROM example_rollup_index; - - 7. Modify the base index's col1 column to be of type BIGINT and move to the col2 column - ALTER TABLE example_db.my_table - MODIFY COLUMN col1 BIGINT DEFAULT "1" AFTER col2; - - 8. Modify the maximum length of the val1 column of the base index. The original val1 is (val1 VARCHAR(32) REPLACE DEFAULT "abc") - ALTER TABLE example_db.my_table - MODIFY COLUMN val1 VARCHAR(64) REPLACE DEFAULT "abc"; - - 9. Reorder the columns in example_rollup_index (set the original column order: k1, k2, k3, v1, v2) - ALTER TABLE example_db.my_table - ORDER BY (k3, k1, k2, v2, v1) - FROM example_rollup_index; - - 10. Perform both operations simultaneously - ALTER TABLE example_db.my_table - ADD COLUMN v2 INT MAX DEFAULT "0" AFTER k2 TO example_rollup_index, - ORDER BY (k3,k1,k2,v2,v1) FROM example_rollup_index; - - 11. Modify the bloom filter column of the table - ALTER TABLE example_db.my_table SET ("bloom_filter_columns"="k1,k2,k3"); - - Can also be merged into the above schema change operation (note that the syntax of multiple clauses is slightly different) - ALTER TABLE example_db.my_table - DROP COLUMN col2 - PROPERTIES ("bloom_filter_columns"="k1,k2,k3"); - - 12. Modify the Colocate property of the table - - ALTER TABLE example_db.my_table set ("colocate_with" = "t1"); - - 13. Change the bucketing mode of the table from Random Distribution to Hash Distribution - - ALTER TABLE example_db.my_table set ("distribution_type" = "hash"); - - 14. Modify the dynamic partition properties of the table (support adding dynamic partition properties to tables without dynamic partition properties) - - ALTER TABLE example_db.my_table set ("dynamic_partition_enable" = "false"); - - If you need to add dynamic partition attributes to a table without dynamic partition attributes, you need to specify all dynamic partition attributes - - ALTER TABLE example_db.my_table set ("dynamic_partition. Enable "= "true", dynamic_partition. Time_unit" = "DAY", "dynamic_partition. End "= "3", "dynamic_partition. Prefix" = "p", "Dynamic_partition. Buckets" = "32"); - - 15. Modify the in_memory property of the table - - ALTER TABLE example_db.my_table set ("in_memory" = "true"); - - [rename] - 1. Modify the table named table1 to table2 - ALTER TABLE table1 RENAME table2; - - 2. Modify the rollup index named rollup1 in the table example_table to rollup2 - ALTER TABLE example_table RENAME ROLLUP rollup1 rollup2; - - 3. Modify the partition named p1 in the table example_table to p2 - ALTER TABLE example_table RENAME PARTITION p1 p2; - - [index] - 1. create index on table1 column siteid using bitmap - ALTER TABLE table1 ADD INDEX index_name [USING BITMAP] (siteid) COMMENT 'balabala'; - 2. drop bitmap index of table1 - ALTER TABLE table1 DROP INDEX index_name; - -## keyword - - ALTER, TABLE, ROLLUP, COLUMN, PARTITION, RENAME diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER VIEW_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER VIEW_EN.md deleted file mode 100644 index 03b849ec254ae1..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER VIEW_EN.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# ALTER VIEW -## description - This statement is used to modify the definition of a view - Syntax: - ALTER VIEW - [db_name.]view_name - (column1[ COMMENT "col comment"][, column2, ...]) - AS query_stmt - - Explain: - 1. View is logical, it isn't stored in the physical medium. When we querying, view will be embed as subqueries in query statement. Therefore, modifying the definition of views is equivalent to modifying query_stmt which is defined in view. - 2. query_stmt is arbitrarily supported SQL. - -## example - - 1. Modify example_view on the example_db - - ALTER VIEW example_db.example_view - ( - c1 COMMENT "column 1", - c2 COMMENT "column 2", - c3 COMMENT "column 3" - ) - AS SELECT k1, k2, SUM(v1) FROM example_table - GROUP BY k1, k2 \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/BACKUP_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/BACKUP_EN.md deleted file mode 100644 index 8c16fe02f14614..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/BACKUP_EN.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# BACKUP -## Description -This statement is used to backup data under the specified database. This command is an asynchronous operation. After successful submission, you need to check progress through the SHOW BACKUP command. Only tables of OLAP type are backed up. -Grammar: -BACKUP SNAPSHOT [db_name].{snapshot_name} -TO `repository_name` -ON ( -"`Table `U name'[Distriction (`P1',...)], -... -) -PROPERTIES ("key"="value", ...); - -Explain: -1. Only one BACKUP or RESTORE task can be performed under the same database. -2. The ON clause identifies the tables and partitions that need to be backed up. If no partition is specified, all partitions of the table are backed up by default. -3. PROPERTIES currently supports the following attributes: -"Type" = "full": means that this is a full update (default). -"Timeout" = "3600": Task timeout, default to one day. Unit seconds. - -## example - -1. Back up the table example_tbl under example_db in full to the warehouse example_repo: -BACKUP SNAPSHOT example_db.snapshot_label1 -TO example repo -On (example tbl) -PROPERTIES ("type" = "full"); - -2. Under full backup example_db, the P1 and P2 partitions of table example_tbl, and table example_tbl2 to warehouse example_repo: -BACKUP SNAPSHOT example_db.snapshot_label2 -TO example repo -ON -( -example_tbl PARTITION (p1,p2), -Example: -); - -## keyword -BACKUP - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL ALTER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL ALTER_EN.md deleted file mode 100644 index 76f9cc7cacbec4..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL ALTER_EN.md +++ /dev/null @@ -1,63 +0,0 @@ - - -# CANCEL ALTER -## Description -This statement is used to undo an ALTER operation. -1. 撤销 ALTER TABLE COLUMN 操作 -Grammar: -CANCEL ALTER TABLE COLUMN -FROM db_name.table_name - -2. 撤销 ALTER TABLE ROLLUP 操作 -Grammar: -CANCEL ALTER TABLE ROLLUP -FROM db_name.table_name - -3. batch cancel rollup by job id - Grammar: - CANCEL ALTER TABLE ROLLUP - FROM db_name.table_name (jobid,...) - Note: - Batch cancel rollup job is a async operation, use `show alter table rollup` to see whether it executes successfully - -2. OTHER CLUSTER -Grammar: -(To be realized... - - -## example -[CANCEL ALTER TABLE COLUMN] -1. 撤销针对 my_table 的 ALTER COLUMN 操作。 -CANCEL ALTER TABLE COLUMN -FROM example_db.my_table; - -[CANCEL ALTER TABLE ROLLUP] -1. 撤销 my_table 下的 ADD ROLLUP 操作。 -CANCEL ALTER TABLE ROLLUP -FROM example_db.my_table; - -[CANCEL ALTER TABLE ROLLUP] -1. cancel rollup alter job by job id -CANCEL ALTER TABLE ROLLUP -FROM example_db.my_table (12801,12802); - -## keyword -CANCEL,ALTER,TABLE,COLUMN,ROLLUP - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL BACKUP_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL BACKUP_EN.md deleted file mode 100644 index e79e6bbf16c6e8..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL BACKUP_EN.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# CANCEL BACKUP -## Description -This statement is used to cancel an ongoing BACKUP task. -Grammar: -CANCEL BACKUP FROM db_name; - -## example -1. Cancel the BACKUP task under example_db. -CANCEL BACKUP FROM example_db; - -## keyword -CANCEL, BACKUP - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL RESTORE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL RESTORE_EN.md deleted file mode 100644 index a288d161a840a0..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CANCEL RESTORE_EN.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# CANCEL RESTORE -## Description -This statement is used to cancel an ongoing RESTORE task. -Grammar: -CANCEL RESTORE FROM db_name; - -Be careful: -When the recovery is abolished around the COMMIT or later stage, the restored tables may be inaccessible. At this point, data recovery can only be done by performing the recovery operation again. - -## example -1. Cancel the RESTORE task under example_db. -CANCEL RESTORE FROM example_db; - -## keyword -CANCEL, RESTORE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE DATABASE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE DATABASE_EN.md deleted file mode 100644 index e9771000d46b37..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE DATABASE_EN.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# CREATE DATABASE -## Description -This statement is used to create a new database -Grammar: -CREATE DATABASE [IF NOT EXISTS] db_name; - -## example -1. New database db_test -CREATE DATABASE db_test; - -## keyword -CREATE,DATABASE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE INDEX_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE INDEX_EN.md deleted file mode 100644 index 819a5b5dbb434e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE INDEX_EN.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# CREATE INDEX - -## description - - This statement is used to create index - grammer: - CREATE INDEX index_name ON table_name (column [, ...],) [USING BITMAP] [COMMENT'balabala']; - note: - 1. only support bitmap index in current version - 2. BITMAP index only supports apply to single column - -## example - - 1. create index on table1 column siteid using bitmap - CREATE INDEX index_name ON table1 (siteid) USING BITMAP COMMENT 'balabala'; - -## keyword - - CREATE,INDEX diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md deleted file mode 100644 index 9d844b7ce64199..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md +++ /dev/null @@ -1,231 +0,0 @@ - - -# CREATE MATERIALIZED VIEW - -## description - -This statement is used to create a materialized view. - - Asynchronous syntax. After the call is successful, it only indicates that the task to create the materialized view is successfully submitted. The user needs to check the progress of the materialized view by using ```show alter table rollup```. - - After the progress is FINISHED, you can use the ```desc [table_name] all``` command to view the schema of the materialized view. - -syntax: - - ``` - - CREATE MATERIALIZED VIEW [MG name] as [query] - [PROPERTIES ("key" = "value")] - - ``` - -1. MV name - - Name of the materialized view. Required. - - Materialized view names in the same table cannot be duplicated. - -2. query - - The query used to construct the materialized view. The result of the query is the data of the materialized view. The query format currently supported is: - - ``` -    SELECT select_expr [, select_expr ...] -    FROM [Base view name] -    GROUP BY column_name [, column_name ...] -    ORDER BY column_name [, column_name ...] -     -    The syntax is the same as the query syntax. - ``` - - select_expr: All columns in the materialized view's schema. - - + Only single columns and aggregate columns without expression calculation are supported. - + The aggregate function currently only supports SUM, MIN, MAX, and the parameters of the aggregate function can only be a single column without expression calculation. - + Contains at least one single column. - + All involved columns can only appear once. - - base view name: The original table name of the materialized view. Required. - - + Must be a single table and not a subquery - - group by: Grouped column of materialized view, optional. - - + If not filled, the data will not be grouped. - - order by: Sort order of materialized view, optional. - - + The order of the column sort must be the same as the column declaration order in select_expr. - + If order by is not specified, sort columns are automatically supplemented according to the rules. - - + If the materialized view is an aggregate type, all grouping columns are automatically supplemented with sort columns. - + If the materialized view is a non-aggregated type, the first 36 bytes are automatically supplemented as a sorted column. If the number of sorts for automatic replenishment is less than three, the first three are sorted. - + If the query contains a grouping column, the sort order must be the same as the grouping column. - -3. properties - - Declare some configuration of materialized view, optional. - - ``` - PROPERTIES ("key" = "value", "key" = "value" ...) - - ``` - - The following configurations can be declared here: - - + short_key: the number of columns. - + timeout: timeout for materialized view construction. - -## example - -Base table structure is - -``` -mysql> desc duplicate_table; -+-------+--------+------+------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+-------+--------+------+------+---------+-------+ -| k1 | INT | Yes | true | N/A | | -| k2 | INT | Yes | true | N/A | | -| k3 | BIGINT | Yes | true | N/A | | -| k4 | BIGINT | Yes | true | N/A | | -+-------+--------+------+------+---------+-------+ -``` - -1. Create a materialized view containing only the columns of the original table (k1, k2) - - ``` - create materialized view k1_k2 as -select k1, k2 from duplicate_table; - ``` - - The materialized view's schema is shown below. The materialized view contains only two columns k1, k2 without any aggregation. - - ``` - +-----------------+-------+--------+------+------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +-----------------+-------+--------+------+------+---------+-------+ - | k1_k2 | k1 | INT | Yes | true | N/A | | - | | k2 | INT | Yes | true | N/A | | - +-----------------+-------+--------+------+------+---------+-------+ - ``` - -2. Create a materialized view sorted by k2 - - ``` - create materialized view k2_order as -select k2, k1 from duplicate_table order by k2; -``` - - The materialized view's schema is shown below. The materialized view contains only two columns k2, k1, where column k2 is a sorted column without any aggregation. - - ``` - +-----------------+-------+--------+------+-------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +-----------------+-------+--------+------+-------+---------+-------+ - | k2_order | k2 | INT | Yes | true | N/A | | - | | k1 | INT | Yes | false | N/A | NONE | - +-----------------+-------+--------+------+-------+---------+-------+ - ``` - -3. Create a materialized view grouped by k1, k2 with k3 as the SUM aggregate - - ``` - create materialized view k1_k2_sumk3 as -select k1, k2, sum (k3) from duplicate_table group by k1, k2; - ``` - - The materialized view's schema is shown below. The materialized view contains two columns k1, k2 and sum (k3), where k1, k2 are grouped columns, and sum (k3) is the sum of the k3 columns grouped according to k1, k2. - - Because the materialized view does not declare a sort column, and the materialized view has aggregate data, the system supplements the grouping columns k1 and k2 by default. - - ``` - +-----------------+-------+--------+------+-------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +-----------------+-------+--------+------+-------+---------+-------+ - | k1_k2_sumk3 | k1 | INT | Yes | true | N/A | | - | | k2 | INT | Yes | true | N/A | | - | | k3 | BIGINT | Yes | false | N/A | SUM | - +-----------------+-------+--------+------+-------+---------+-------+ - ``` - -4. Create a materialized view to remove duplicate rows - - ``` - create materialized view deduplicate as -select k1, k2, k3, k4 from duplicate_table group by k1, k2, k3, k4; - ``` - - The materialized view schema is shown below. The materialized view contains k1, k2, k3, and k4 columns, and there are no duplicate rows. - - ``` - +-----------------+-------+--------+------+-------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +-----------------+-------+--------+------+-------+---------+-------+ - | deduplicate | k1 | INT | Yes | true | N/A | | - | | k2 | INT | Yes | true | N/A | | - | | k3 | BIGINT | Yes | true | N/A | | - | | k4 | BIGINT | Yes | true | N/A | | - +-----------------+-------+--------+------+-------+---------+-------+ - ``` - -5. Create a non-aggregated materialized view that does not declare a sort column - - The schema of all_type_table is as follows: - - ``` - +-------+--------------+------+-------+---------+-------+ - | Field | Type | Null | Key | Default | Extra | - +-------+--------------+------+-------+---------+-------+ - | k1 | TINYINT | Yes | true | N/A | | - | k2 | SMALLINT | Yes | true | N/A | | - | k3 | INT | Yes | true | N/A | | - | k4 | BIGINT | Yes | true | N/A | | - | k5 | DECIMAL(9,0) | Yes | true | N/A | | - | k6 | DOUBLE | Yes | false | N/A | NONE | - | k7 | VARCHAR(20) | Yes | false | N/A | NONE | - +-------+--------------+------+-------+---------+-------+ - ``` - - The materialized view contains k3, k4, k5, k6, k7 columns, and no sort column is declared. The creation statement is as follows: - - ``` - create materialized view mv_1 as -select k3, k4, k5, k6, k7 from all_type_table; - ``` - - The system's default supplementary sort columns are k3, k4, and k5. The sum of the number of bytes for these three column types is 4 (INT) + 8 (BIGINT) + 16 (DECIMAL) = 28 <36. So these three columns are added as sort columns. - - The materialized view's schema is as follows. You can see that the key fields of the k3, k4, and k5 columns are true, which is the sort order. The key field of the k6, k7 columns is false, that is, non-sorted. - - ``` - +----------------+-------+--------------+------+-------+---------+-------+ - | IndexName | Field | Type | Null | Key | Default | Extra | - +----------------+-------+--------------+------+-------+---------+-------+ - | mv_1 | k3 | INT | Yes | true | N/A | | - | | k4 | BIGINT | Yes | true | N/A | | - | | k5 | DECIMAL(9,0) | Yes | true | N/A | | - | | k6 | DOUBLE | Yes | false | N/A | NONE | - | | k7 | VARCHAR(20) | Yes | false | N/A | NONE | - +----------------+-------+--------------+------+-------+---------+-------+ - ``` - -## keyword - CREATE, MATERIALIZED, VIEW diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY_EN.md deleted file mode 100644 index 59eaba92c080b3..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY_EN.md +++ /dev/null @@ -1,68 +0,0 @@ - - -# CREATE REPOSITORY -## Description -This statement is used to create the warehouse. The warehouse is used for backup or recovery. Only root or superuser users can create warehouses. -Grammar: -CREATE [READ ONLY] REPOSITORY `repo_name` -WITH BROKER `broker_name` -ON LOCATION `repo_location` -PROPERTIES ("key"="value", ...); - -Explain: -1. The creation of warehouses depends on existing brokers -2. If it is a read-only warehouse, it can only be restored on the warehouse. If not, you can backup and restore operations. -3. According to the different types of broker, PROPERTIES is different, see the example. - -## example -1. Create a warehouse named bos_repo, which relies on BOS broker "bos_broker", and the data root directory is: bos://palo_backup. -CREATE REPOSITORY `bos_repo` -WITH BROKER `bos_broker` -ON LOCATION "bos://palo_backup" -PROPERTIES -( -"bosu endpoint" ="http://gz.bcebos.com", -"bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", -"bos_secret_accesskey"="70999999999999de274d59eaa980a" -); - -2. Create the same warehouse as in Example 1, but with read-only attributes: -CREATE READ ONLY REPOSITORY `bos_repo` -WITH BROKER `bos_broker` -ON LOCATION "bos://palo_backup" -PROPERTIES -( -"bosu endpoint" ="http://gz.bcebos.com", -"bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", -"bos_secret_accesskey"="70999999999999de274d59eaa980a" -); - -3. Create a warehouse named hdfs_repo, which relies on Baidu HDFS broker "hdfs_broker", and the data root directory is: hdfs://hadoop-name-node:54310/path/to/repo./ -CREATE REPOSITORY `hdfs_repo` -WITH BROKER `hdfs_broker` -ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/" -PROPERTIES -( -"Username" = "User" -"password" = "password" -); - -## keyword -CREATE REPOSITORY diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md deleted file mode 100644 index 70755ca7d1da02..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md +++ /dev/null @@ -1,575 +0,0 @@ - - -# CREATE TABLE - -## description - -This statement is used to create table -Syntax: - -``` - CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [database.]table_name - (column_definition1[, column_definition2, ...] - [, index_definition1[, ndex_definition12,]]) - [ENGINE = [olap|mysql|broker]] - [key_desc] - [COMMENT "table comment"] - [partition_desc] - [distribution_desc] - [rollup_index] - [PROPERTIES ("key"="value", ...)] - [BROKER PROPERTIES ("key"="value", ...)]; -``` - -1. column_definition - Syntax: - `col_name col_type [agg_type] [NULL | NOT NULL] [DEFAULT "default_value"]` - Explain: - col_name: Name of column - col_type: Type of column - ``` - TINYINT(1 Byte) - Range: -2^7 + 1 ~ 2^7 - 1 - SMALLINT(2 Bytes) - Range: -2^15 + 1 ~ 2^15 - 1 - INT(4 Bytes) - Range: -2^31 + 1 ~ 2^31 - 1 - BIGINT(8 Bytes) - Range: -2^63 + 1 ~ 2^63 - 1 - LARGEINT(16 Bytes) - Range: -2^127 + 1 ~ 2^127 - 1 - FLOAT(4 Bytes) - Support scientific notation - DOUBLE(12 Bytes) - Support scientific notation - DECIMAL[(precision, scale)] (16 Bytes) - Default is DECIMAL(10, 0) - precision: 1 ~ 27 - scale: 0 ~ 9 - integer part: 1 ~ 18 - fractional part: 0 ~ 9 - Not support scientific notation - DATE(3 Bytes) - Range: 0000-01-01 ~ 9999-12-31 - DATETIME(8 Bytes) - Range: 0000-01-01 00:00:00 ~ 9999-12-31 23:59:59 - CHAR[(length)] - Fixed length string. Range: 1 ~ 255. Default: 1 - VARCHAR[(length)] - Variable length string. Range: 1 ~ 65533 - HLL (1~16385 Bytes) - HLL tpye, No need to specify length. - This type can only be queried by hll_union_agg, hll_cardinality, hll_hash functions. - BITMAP - BITMAP type, No need to specify length. Represent a set of unsigned bigint numbers, the largest element could be 2^64 - 1 - ``` - agg_type: Aggregation type. If not specified, the column is key column. Otherwise, the column is value column. - * SUM、MAX、MIN、REPLACE - * HLL_UNION: Only for HLL type - * REPLACE_IF_NOT_NULL: The meaning of this aggregation type is that substitution will occur if and only if the newly imported data is a non-null value. If the newly imported data is null, Doris will still retain the original value. Note: if NOT NULL is specified in the REPLACE_IF_NOT_NULL column when the user creates the table, Doris will convert it to NULL and will not report an error to the user. Users can leverage this aggregate type to achieve importing some of columns. - * BITMAP_UNION: Only for BITMAP type - Allow NULL: Default is NOT NULL. NULL value should be represented as `\N` in load source file. - Notice: - The origin value of BITMAP_UNION column should be TINYINT, SMALLINT, INT, BIGINT. -2. index_definition - Syntax: - `INDEX index_name (col_name[, col_name, ...]) [USING BITMAP] COMMENT 'xxxxxx'` - Explain: - index_name：index name - col_name：column name - Notice: - Only support BITMAP index in current version, BITMAP can only apply to single column -3. ENGINE type - Default is olap. Options are: olap, mysql, broker - 1) For mysql, properties should include: - - ``` - PROPERTIES ( - "host" = "mysql_server_host", - "port" = "mysql_server_port", - "user" = "your_user_name", - "password" = "your_password", - "database" = "database_name", - "table" = "table_name" - ) - ``` - - Notice: - "table_name" is the real table name in MySQL database. - table_name in CREATE TABLE stmt is table is Doris. They can be different or same. - MySQL table created in Doris is for accessing data in MySQL database. - Doris does not maintain and store any data from MySQL table. - 1) For broker, properties should include: - - ``` - PROPERTIES ( - "broker_name" = "broker_name", - "path" = "file_path1[,file_path2]", - "column_separator" = "value_separator" - "line_delimiter" = "value_delimiter" - ) - ``` - - ``` - BROKER PROPERTIES( - "username" = "name", - "password" = "password" - ) - ``` - - For different broker, the broker properties are different - Notice: - Files name in "path" is separated by ",". If file name includes ",", use "%2c" instead. If file name includes "%", use "%25" instead. - Support CSV and Parquet. Support GZ, BZ2, LZ4, LZO(LZOP) -4. key_desc - Syntax: - key_type(k1[,k2 ...]) - Explain: - Data is orderd by specified key columns. And has different behaviors for different key desc. - AGGREGATE KEY: - value columns will be aggregated is key columns are same. - UNIQUE KEY: - The new incoming rows will replace the old rows if key columns are same. - DUPLICATE KEY: - All incoming rows will be saved. - the default key_type is DUPLICATE KEY, and key columns are first 36 bytes of the columns in define order. - If the number of columns in the first 36 is less than 3, the first 3 columns will be used. - NOTICE: - Except for AGGREGATE KEY, no need to specify aggregation type for value columns. -5. partition_desc - Partition has two ways to use: - 1) LESS THAN - Syntax: - - ``` - PARTITION BY RANGE (k1, k2, ...) - ( - PARTITION partition_name1 VALUES LESS THAN MAXVALUE|("value1", "value2", ...), - PARTITION partition_name2 VALUES LESS THAN MAXVALUE|("value1", "value2", ...) - ... - ) - ``` - - Explain: - 1) Partition name only support [A-z0-9_] - 2) Partition key column's type should be: - TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME - 3) The range is [closed, open). And the lower bound of first partition is MIN VALUE of specifed column type. - 4) NULL values should be save in partition which includes MIN VALUE. - 5) Support multi partition columns, the the default partition value is MIN VALUE. - 2）Fixed Range - Syntax: - ``` - PARTITION BY RANGE (k1, k2, k3, ...) - ( - PARTITION partition_name1 VALUES [("k1-lower1", "k2-lower1", "k3-lower1",...), ("k1-upper1", "k2-upper1", "k3-upper1", ...)), - PARTITION partition_name2 VALUES [("k1-lower1-2", "k2-lower1-2", ...), ("k1-upper1-2", MAXVALUE, )) - "k3-upper1-2", ... - ) - ``` - Explain: - 1）The Fixed Range is more flexible than the LESS THAN, and the left and right intervals are completely determined by the user. - 2）Others are consistent with LESS THAN. -6. distribution_desc - 1) Hash - Syntax: - `DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]` - Explain: - The default buckets is 10. -7. PROPERTIES - 1) If ENGINE type is olap. User can specify storage medium, cooldown time and replication number: - - ``` - PROPERTIES ( - "storage_medium" = "[SSD|HDD]", - ["storage_cooldown_time" = "yyyy-MM-dd HH:mm:ss"], - ["replication_num" = "3"] - ) - ``` - - storage_medium: SSD or HDD, The default initial storage media can be specified by `default_storage_medium= XXX` in the fe configuration file `fe.conf`, or, if not, by default, HDD. - storage_cooldown_time: If storage_medium is SSD, data will be automatically moved to HDD when timeout. - Default is 30 days. - Format: "yyyy-MM-dd HH:mm:ss" - replication_num: Replication number of a partition. Default is 3. - If table is not range partitions. This property takes on Table level. Or it will takes on Partition level. - User can specify different properties for different partition by `ADD PARTITION` or `MODIFY PARTITION` statements. - 2) If Engine type is olap, user can set bloom filter index for column. - Bloom filter index will be used when query contains `IN` or `EQUAL`. - Bloom filter index support key columns with type except TINYINT FLOAT DOUBLE, also support value with REPLACE aggregation type. - - ``` - PROPERTIES ( - "bloom_filter_columns"="k1,k2,k3" - ) - ``` - - 3) For Colocation Join: - - ``` - PROPERTIES ( - "colocate_with"="table1" - ) - ``` - - 4) if you want to use the dynamic partitioning feature, specify it in properties - - ``` - PROPERTIES ( - "dynamic_partition.enable" = "true|false", - "dynamic_partition.time_unit" = "DAY|WEEK|MONTH", - "dynamic_partitoin.end" = "${integer_value}", - "dynamic_partition.prefix" = "${string_value}", - "dynamic_partition.buckets" = "${integer_value} - ) - ``` - - Dynamic_partition. Enable: specifies whether dynamic partitioning at the table level is enabled - - Dynamic_partition. Time_unit: used to specify the time unit for dynamically adding partitions, which can be selected as DAY, WEEK, and MONTH. - - Dynamic_partition. End: used to specify the number of partitions created in advance - - Dynamic_partition. Prefix: used to specify the partition name prefix to be created, such as the partition name prefix p, automatically creates the partition name p20200108 - - Dynamic_partition. Buckets: specifies the number of partition buckets that are automatically created -8. rollup_index - grammar: - ``` - ROLLUP (rollup_name (column_name1, column_name2, ...) - [FROM from_index_name] - [PROPERTIES ("key"="value", ...)],...) - ``` - - 5) if you want to use the inmemory table feature, specify it in properties - - ``` - PROPERTIES ( - "in_memory"="true" - ) - ``` -## example - -1. Create an olap table, distributed by hash, with aggregation type. - - ``` - CREATE TABLE example_db.table_hash - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 CHAR(10) REPLACE, - v2 INT SUM - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - COMMENT "my first doris table" - DISTRIBUTED BY HASH(k1) BUCKETS 32 - PROPERTIES ("storage_type"="column"); - ``` - -2. Create an olap table, distributed by hash, with aggregation type. Also set storage mediumand cooldown time. - - ``` - CREATE TABLE example_db.table_hash - ( - k1 BIGINT, - k2 LARGEINT, - v1 VARCHAR(2048) REPLACE, - v2 SMALLINT SUM DEFAULT "10" - ) - ENGINE=olap - UNIQUE KEY(k1, k2) - DISTRIBUTED BY HASH (k1, k2) BUCKETS 32 - PROPERTIES( - "storage_type"="column", - "storage_medium" = "SSD", - "storage_cooldown_time" = "2015-06-04 00:00:00" - ); - -3. Create an olap table, with range partitioned, distributed by hash. - -1) LESS THAN - - ``` - CREATE TABLE example_db.table_range - ( - k1 DATE, - k2 INT, - k3 SMALLINT, - v1 VARCHAR(2048), - v2 DATETIME DEFAULT "2014-02-04 15:36:00" - ) - ENGINE=olap - DUPLICATE KEY(k1, k2, k3) - PARTITION BY RANGE (k1) - ( - PARTITION p1 VALUES LESS THAN ("2014-01-01"), - PARTITION p2 VALUES LESS THAN ("2014-06-01"), - PARTITION p3 VALUES LESS THAN ("2014-12-01") - ) - DISTRIBUTED BY HASH(k2) BUCKETS 32 - PROPERTIES( - "storage_medium" = "SSD", "storage_cooldown_time" = "2015-06-04 00:00:00" - ); - ``` - - Explain: - This statement will create 3 partitions: - - ``` - ( { MIN }, {"2014-01-01"} ) - [ {"2014-01-01"}, {"2014-06-01"} ) - [ {"2014-06-01"}, {"2014-12-01"} ) - ``` - - Data outside these ranges will not be loaded. - -2) Fixed Range - CREATE TABLE table_range - ( - k1 DATE, - k2 INT, - k3 SMALLINT, - v1 VARCHAR(2048), - v2 DATETIME DEFAULT "2014-02-04 15:36:00" - ) - ENGINE=olap - DUPLICATE KEY(k1, k2, k3) - PARTITION BY RANGE (k1, k2, k3) - ( - PARTITION p1 VALUES [("2014-01-01", "10", "200"), ("2014-01-01", "20", "300")), - PARTITION p2 VALUES [("2014-06-01", "100", "200"), ("2014-07-01", "100", "300")) - ) - DISTRIBUTED BY HASH(k2) BUCKETS 32 - PROPERTIES( - "storage_medium" = "SSD" - ); - -4. Create a mysql table - - ``` - CREATE TABLE example_db.table_mysql - ( - k1 DATE, - k2 INT, - k3 SMALLINT, - k4 VARCHAR(2048), - k5 DATETIME - ) - ENGINE=mysql - PROPERTIES - ( - "host" = "127.0.0.1", - "port" = "8239", - "user" = "mysql_user", - "password" = "mysql_passwd", - "database" = "mysql_db_test", - "table" = "mysql_table_test" - ); - ``` - -5. Create a broker table, with file on HDFS, line delimit by "|", column separated by "\n" - - ``` - CREATE EXTERNAL TABLE example_db.table_broker ( - k1 DATE, - k2 INT, - k3 SMALLINT, - k4 VARCHAR(2048), - k5 DATETIME - ) - ENGINE=broker - PROPERTIES ( - "broker_name" = "hdfs", - "path" = "hdfs://hdfs_host:hdfs_port/data1,hdfs://hdfs_host:hdfs_port/data2,hdfs://hdfs_host:hdfs_port/data3%2c4", - "column_separator" = "|", - "line_delimiter" = "\n" - ) - BROKER PROPERTIES ( - "username" = "hdfs_user", - "password" = "hdfs_password" - ); - ``` - -6. Create table will HLL column - - ``` - CREATE TABLE example_db.example_table - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 HLL HLL_UNION, - v2 HLL HLL_UNION - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - DISTRIBUTED BY HASH(k1) BUCKETS 32; - ``` - -7. Create a table will BITMAP_UNION column - - ``` - CREATE TABLE example_db.example_table - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 BITMAP BITMAP_UNION, - v2 BITMAP BITMAP_UNION - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - DISTRIBUTED BY HASH(k1) BUCKETS 32; - ``` - -8. Create 2 colocate join table. - - ``` - CREATE TABLE `t1` ( - `id` int(11) COMMENT "", - `value` varchar(8) COMMENT "" - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "colocate_with" = "group1" - ); - CREATE TABLE `t2` ( - `id` int(11) COMMENT "", - `value` varchar(8) COMMENT "" - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "colocate_with" = "group1" - ); - ``` - -9. Create a broker table, with file on BOS. - - ``` - CREATE EXTERNAL TABLE example_db.table_broker ( - k1 DATE - ) - ENGINE=broker - PROPERTIES ( - "broker_name" = "bos", - "path" = "bos://my_bucket/input/file", - ) - BROKER PROPERTIES ( - "bos_endpoint" = "http://bj.bcebos.com", - "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "bos_secret_accesskey"="yyyyyyyyyyyyyyyyyyyy" - ); - ``` - -10. Create a table with a bitmap index - - ``` - CREATE TABLE example_db.table_hash - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 CHAR(10) REPLACE, - v2 INT SUM, - INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx' - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - COMMENT "my first doris table" - DISTRIBUTED BY HASH(k1) BUCKETS 32 - PROPERTIES ("storage_type"="column"); - ``` - -11. Create a dynamic partitioning table (dynamic partitioning needs to be enabled in FE configuration), which creates partitions 3 days in advance every day. For example, if today is' 2020-01-08 ', partitions named 'p20200108', 'p20200109', 'p20200110', 'p20200111' will be created. - - ``` - [types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) - [types: [DATE]; keys: [2020-01-09]; ‥types: [DATE]; keys: [2020-01-10]; ) - [types: [DATE]; keys: [2020-01-10]; ‥types: [DATE]; keys: [2020-01-11]; ) - [types: [DATE]; keys: [2020-01-11]; ‥types: [DATE]; keys: [2020-01-12]; ) - ``` - - ``` - CREATE TABLE example_db.dynamic_partition - ( - k1 DATE, - k2 INT, - k3 SMALLINT, - v1 VARCHAR(2048), - v2 DATETIME DEFAULT "2014-02-04 15:36:00" - ) - ENGINE=olap - DUPLICATE KEY(k1, k2, k3) - PARTITION BY RANGE (k1) - ( - PARTITION p1 VALUES LESS THAN ("2014-01-01"), - PARTITION p2 VALUES LESS THAN ("2014-06-01"), - PARTITION p3 VALUES LESS THAN ("2014-12-01") - ) - DISTRIBUTED BY HASH(k2) BUCKETS 32 - PROPERTIES( - "storage_medium" = "SSD", - "dynamic_partition.time_unit" = "DAY", - "dynamic_partition.end" = "3", - "dynamic_partition.prefix" = "p", - "dynamic_partition.buckets" = "32" - ); - ``` -12. Create a table with rollup index -``` - CREATE TABLE example_db.rolup_index_table - ( - event_day DATE, - siteid INT DEFAULT '10', - citycode SMALLINT, - username VARCHAR(32) DEFAULT '', - pv BIGINT SUM DEFAULT '0' - ) - AGGREGATE KEY(event_day, siteid, citycode, username) - DISTRIBUTED BY HASH(siteid) BUCKETS 10 - rollup ( - r1(event_day,siteid), - r2(event_day,citycode), - r3(event_day) - ) - PROPERTIES("replication_num" = "3"); -``` - -12. Create a inmemory table: - -``` - CREATE TABLE example_db.table_hash - ( - k1 TINYINT, - k2 DECIMAL(10, 2) DEFAULT "10.5", - v1 CHAR(10) REPLACE, - v2 INT SUM, - INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx' - ) - ENGINE=olap - AGGREGATE KEY(k1, k2) - COMMENT "my first doris table" - DISTRIBUTED BY HASH(k1) BUCKETS 32 - PROPERTIES ("in_memory"="true"); -``` - -## keyword - - CREATE,TABLE diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE VIEW_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE VIEW_EN.md deleted file mode 100644 index d14b0a43826755..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE VIEW_EN.md +++ /dev/null @@ -1,61 +0,0 @@ - - -# CREATE VIEW -## Description - This statement is used to create a logical view - Grammar: - - CREATE VIEW [IF NOT EXISTS] - [db_name.]view_name - (column1[ COMMENT "col comment"][, column2, ...]) - AS query_stmt - - Explain: - - 1. Views are logical views without physical storage. All queries on views are equivalent to sub-queries corresponding to views. - 2. query_stmt is arbitrarily supported SQL. - -## example - - 1. Create view example_view on example_db - - CREATE VIEW example_db.example_view (k1, k2, k3, v1) - AS - SELECT c1 as k1, k2, k3, SUM(v1) FROM example_table - WHERE k1 = 20160112 GROUP BY k1,k2,k3; - - 2. Create view with comment - - CREATE VIEW example_db.example_view - ( - k1 COMMENT "first key", - k2 COMMENT "second key", - k3 COMMENT "third key", - v1 COMMENT "first value" - ) - COMMENT "my first view" - AS - SELECT c1 as k1, k2, k3, SUM(v1) FROM example_table - WHERE k1 = 20160112 GROUP BY k1,k2,k3; - -## keyword - - CREATE,VIEW - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/Colocate Join_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/Colocate Join_EN.md deleted file mode 100644 index 8b2ecfe6f4e156..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/Colocate Join_EN.md +++ /dev/null @@ -1,91 +0,0 @@ - - -# Colocate Join -## Description -Colocate/Local Join means that when multiple nodes are Join, there is no data movement and network transmission, and each node is only Join locally. -The premise of Join locally is to import data from the same Join Key into a fixed node according to the same rules. - -1 How To Use: - -Simply add the property colocate_with when building a table. The value of colocate_with can be set to any one of the same set of colocate tables. -However, you need to ensure that tables in the colocate_with attribute are created first. - -If you need to Colocate Join table t1 and t2, you can build tables according to the following statements: - -CREATE TABLE `t1` ( -`id` int(11) COMMENT "", -'value ` varchar (8) COMMENT "" -) ENGINE=OLAP -DUPLICATE KEY(`id`) -DISTRIBUTED BY HASH(`id`) BUCKETS 10 -PROPERTIES ( -"colocate_with" = "t1" -); - -CREATE TABLE `t2` ( -`id` int(11) COMMENT "", -'value ` varchar (8) COMMENT "" -) ENGINE=OLAP -DUPLICATE KEY(`id`) -DISTRIBUTED BY HASH(`id`) BUCKETS 10 -PROPERTIES ( -"colocate_with" = "t1" -); - -2 Colocate Join 目前的限制: - -1. Colcoate Table must be an OLAP-type table -2. The BUCKET number of tables with the same colocate_with attribute must be the same -3. The number of copies of tables with the same colocate_with attribute must be the same -4. Data types of DISTRIBUTTED Columns for tables with the same colocate_with attribute must be the same - -3 Colocate Join's applicable scenario: - -Colocate Join is well suited for scenarios where tables are bucketed according to the same field and high frequency according to the same field Join. - -4 FAQ: - -Q: 支持多张表进行Colocate Join 吗? - -A: 25903;. 25345 - -Q: Do you support Colocate table and normal table Join? - -A: 25903;. 25345 - -Q: Does the Colocate table support Join with non-bucket Key? - -A: Support: Join that does not meet Colocate Join criteria will use Shuffle Join or Broadcast Join - -Q: How do you determine that Join is executed according to Colocate Join? - -A: The child node of Hash Join in the result of explain is Colocate Join if it is OlapScanNode directly without Exchange Node. - -Q: How to modify the colocate_with attribute? - -A: ALTER TABLE example_db.my_table set ("colocate_with"="target_table"); - -Q: 229144; colcoate join? - -A: set disable_colocate_join = true; 就可以禁用Colocate Join，查询时就会使用Shuffle Join 和Broadcast Join - -## keyword - -COLOCATE, JOIN, CREATE TABLE diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP DATABASE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP DATABASE_EN.md deleted file mode 100644 index dc39f129ee3337..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP DATABASE_EN.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# DROP DATABASE -##Description -This statement is used to delete the database -Grammar: -DROP DATABASE [IF EXISTS] db_name; - -Explain: -After executing DROP DATABASE for a period of time, the deleted database can be restored through the RECOVER statement. See RECOVER statement for details - -## example -1. Delete database db_test -DROP DATABASE db_test; - -## keyword -DROP,DATABASE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP INDEX_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP INDEX_EN.md deleted file mode 100644 index b13d79e8772e2b..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP INDEX_EN.md +++ /dev/null @@ -1,30 +0,0 @@ - - -# DROP INDEX - -## description - - This statement is used to delete index from table - grammer: - DROP INDEX index_name ON [db_name.]table_name; - -## keyword - - DROP,INDEX diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md deleted file mode 100644 index cd356a54a0da4d..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md +++ /dev/null @@ -1,103 +0,0 @@ - - -# DROP MATERIALIZED VIEW - -## description - This statement is used to delete a materialized view. Synchronization syntax - -syntax: - - ``` - DROP MATERIALIZED VIEW [IF EXISTS] mv_name ON table_name - ``` - -1. IF EXISTS - If the materialized view does not exist, doris will not throw an error. If this keyword is not declared, an error will be reported if the materialized view does not exist. -Ranch - -2. mv_name - The name of the materialized view to be deleted. Required. - -3. Table_name - Name of the table to which the materialized view to be deleted belongs. Required. - -## example - -Table structure is - -``` -mysql> desc all_type_table all; -+----------------+-------+----------+------+-------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+----------------+-------+----------+------+-------+---------+-------+ -| all_type_table | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | false | N/A | NONE | -| | k3 | INT | Yes | false | N/A | NONE | -| | k4 | BIGINT | Yes | false | N/A | NONE | -| | k5 | LARGEINT | Yes | false | N/A | NONE | -| | k6 | FLOAT | Yes | false | N/A | NONE | -| | k7 | DOUBLE | Yes | false | N/A | NONE | -| | | | | | | | -| k1_sumk2 | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | false | N/A | SUM | -+----------------+-------+----------+------+-------+---------+-------+ -``` - -1. Drop the materialized view named k1_sumk2 of the table all_type_table - - ``` - drop materialized view k1_sumk2 on all_type_table; - ``` - Table structure after materialized view is deleted as following: - - ``` -+----------------+-------+----------+------+-------+---------+-------+ -| IndexName | Field | Type | Null | Key | Default | Extra | -+----------------+-------+----------+------+-------+---------+-------+ -| all_type_table | k1 | TINYINT | Yes | true | N/A | | -| | k2 | SMALLINT | Yes | false | N/A | NONE | -| | k3 | INT | Yes | false | N/A | NONE | -| | k4 | BIGINT | Yes | false | N/A | NONE | -| | k5 | LARGEINT | Yes | false | N/A | NONE | -| | k6 | FLOAT | Yes | false | N/A | NONE | -| | k7 | DOUBLE | Yes | false | N/A | NONE | -+----------------+-------+----------+------+-------+---------+-------+ - ``` - -2. Delete a non-existing materialized view in the table all_type_table - - ``` - drop materialized view k1_k2 on all_type_table; -ERROR 1064 (HY000): errCode = 2, detailMessage = Materialized view [k1_k2] does not exist in table [all_type_table] - ``` - - The delete request directly reports an error - -3. Delete the materialized view k1_k2 in the table all_type_table. Materialized view does not exist and no error is reported. - - ``` - drop materialized view if exists k1_k2 on all_type_table; -Query OK, 0 rows affected (0.00 sec) - ``` - - If it exists, it will be deleted; If it does not exist, no error will be reported. - -## keyword - DROP, MATERILIAZED, VIEW diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP REPOSITORY_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP REPOSITORY_EN.md deleted file mode 100644 index 47751647332b9b..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP REPOSITORY_EN.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# DROP REPOSITORY -## Description -This statement is used to delete a created warehouse. Only root or superuser users can delete the warehouse. -Grammar: -DROP REPOSITORY `repo_name`; - -Explain: -1. Delete the warehouse, just delete the mapping of the warehouse in Palo, and do not delete the actual warehouse data. After deletion, you can map to the repository again by specifying the same broker and LOCATION. - -## example -1. Delete the warehouse named bos_repo: -DROP REPOSITORY `bos_repo`; - -## keyword -DROP REPOSITORY diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP TABLE_EN.md deleted file mode 100644 index c876930ed9b266..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP TABLE_EN.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# DROP TABLE -## Description -This statement is used to delete the table. -Grammar: -DROP TABLE [IF EXISTS] [db_name.]table_name; - -Explain: -After executing DROP TABLE for a period of time, the deleted table can be restored through the RECOVER statement. See RECOVER statement for details - -## example -1. Delete a table -DROP TABLE my_table; - -2. If it exists, delete the table that specifies the database -DROP TABLE IF EXISTS example_db.my_table; - -## keyword -DROP,TABLE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP VIEW_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP VIEW_EN.md deleted file mode 100644 index a24601b51e8736..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/DROP VIEW_EN.md +++ /dev/null @@ -1,33 +0,0 @@ - - -# DROP VIEW -## Description -This statement is used to delete a logical view VIEW -Grammar: -DROP VIEW [IF EXISTS] -[db_name.]view_name; - -## example -1. If it exists, delete view example_view on example_db -DROP VIEW IF EXISTS example_db.example_view; - -## keyword -DROP,VIEW - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/HLL_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/HLL_EN.md deleted file mode 100644 index 0a7e56eff898b2..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/HLL_EN.md +++ /dev/null @@ -1,104 +0,0 @@ - - -# HLL -## Description -HLL is an engineering implementation based on the HyperLogLog algorithm. It is used to store the intermediate results of the HyperLog calculation process. It can only be used as the value column type of the table. -By aggregating to reduce the amount of data continuously, in order to achieve the purpose of speeding up the query, based on which an estimated result, the error is about 1%. -The HLL column is generated by other columns or data in the imported data. When imported, the hll_hash function is used to specify which column in the data is used to generate the HLL column. -It is often used to replace count distinct, and to quickly calculate UV in business by combining rollup. - -The correlation function: - -TOTAL UNION -This function is an aggregation function, which is used to calculate the cardinality estimation of all data satisfying the conditions. This function can also be used to analyze functions. It only supports the default window and does not support the window clause. - -Coach L.u RAW AGG -This function is an aggregation function that aggregates HLL type fields and returns HLL type. - -HLL_CARDINALITY(hll) -This function is used to estimate the cardinality of a single HLL sequence - -HLL_HASH(column_name) -Generate HLL column types for insert or import, see the instructions for the use of imports - -EMPTY_HLL() -Generate empty HLL column types for insert or import, see the instructions for the use of imports - -## example -1. First create a table with HLL columns -create table test( -dt date, -id int, -name char(10), -Province of char (10), -The char (1), -the European Union, -European Union -distributed by hash(id) buckets 32; - -2. Import data. See help curl for the way you import it. - - A. Generate HLL columns using columns in tables - - curl --location-trusted -uname:password -T data -H "label:load_1" -H "columns:dt, id, name, province, os, set1=hll_hash(id), set2=hll_hash(name)" - http://host/api/test_db/test/_stream_load - - B. Generate HLL columns using a column in the data - - curl --location-trusted -uname:password -T data -H "label:load_1" -H "columns:dt, id, name, province, sex, cuid, os, set1=hll_hash(cuid), set2=hll_hash(os)" - http://host/api/test_db/test/_stream_load - -3. There are three common ways of aggregating data: (without aggregating the base table directly, the speed may be similar to that of using NDV directly) - -A. Create a rollup that allows HLL columns to generate aggregation. -alter table test add rollup test_rollup(dt, set1); - -B. Create another table dedicated to computing uv, and insert data) - -create table test_uv( -dt date, -uv_set hll hll_union) -distributed by hash(id) buckets 32; - -insert into test_uv select dt, set1 from test; - -C. Create another table dedicated to computing uv, then insert and generate HLL columns from other non-hll columns of test through hll_hash - -create table test_uv( -dt date, -id_set hll hll_union) -distributed by hash(id) buckets 32; - -insert into test_uv select dt, hll_hash(id) from test; - -4. Query, HLL column is not allowed to query its original value directly, it can be queried by matching functions. - -a. 27714; 24635; uv -select HLL_UNION_AGG(uv_set) from test_uv; - -B. Seek every day's UV -select dt, HLL_CARDINALITY(uv_set) from test_uv; - -C. Find the aggregate value of Set1 in the test table -select dt, HLL_CARDINALITY(uv) from (select dt, HLL_RAW_AGG(set1) as uv from test group by dt) tmp; -select dt, HLL_UNION_AGG(set1) as uv from test group by dt; - -## keyword -HLL diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/RECOVER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/RECOVER_EN.md deleted file mode 100644 index 07be2c5006e377..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/RECOVER_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# RECOVER -## Description -This statement is used to restore previously deleted databases, tables, or partitions -Grammar: -1)24674;"22797database; -RECOVER DATABASE db_name; -2) 恢复 table -RECOVER TABLE [db_name.]table_name; -3)24674;"22797partition -RECOVER PARTITION partition name FROM [dbu name.]table name; - -Explain: -1. This operation can only recover the meta-information deleted in the previous period of time. The default is 1 day.(You can configure it with the `catalog_trash_expire_second` parameter in fe.conf) -2. If new meta-information of the same name and type is created after deleting meta-information, the previously deleted meta-information cannot be restored. - -## example -1. Restore the database named example_db -RECOVER DATABASE example_db; - -2. Restore table named example_tbl -RECOVER TABLE example_db.example_tbl; - -3. Restore partition named P1 in example_tbl -RECOVER PARTITION p1 FROM example_tbl; - -## keyword -RECOVER - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/RESTORE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/RESTORE_EN.md deleted file mode 100644 index 889429763d2097..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/RESTORE_EN.md +++ /dev/null @@ -1,71 +0,0 @@ - - -# RESTORE -## Description -1. RESTOR -This statement is used to restore the data previously backed up by the BACKUP command to the specified database. This command is an asynchronous operation. After successful submission, you need to check progress through the SHOW RESTORE command. Restoring tables of OLAP type is supported only. -Grammar: -SNAPSHOT RESTORE [dbu name].{snapshot name} -FROM `repository_name` -ON ( -"`Table `uname'[`partition (`p1',...)] [as `tbl `uu alias'], -... -) -PROPERTIES ("key"="value", ...); - -Explain: -1. Only one BACKUP or RESTORE task can be performed under the same database. -2. The ON clause identifies the tables and partitions that need to be restored. If no partition is specified, all partitions of the table are restored by default. The specified tables and partitions must already exist in the warehouse backup. -3. The backup tables in the warehouse can be restored to new tables through AS statements. But the new table name cannot already exist in the database. Partition name cannot be changed. -4. The backup tables in the warehouse can be restored and replaced with the same-name tables in the database, but the table structure of the two tables must be completely consistent. Table structure includes: table name, column, partition, Rollup and so on. -5. Partitions of the recovery table can be specified, and the system checks whether the partition Range matches. -6. PROPERTIES currently supports the following attributes: -"Backup_timestamp" = "2018-05-04-16-45-08": specifies which version of the time to restore the corresponding backup must be filled in. This information can be obtained through the `SHOW SNAPSHOT ON repo;'statement. -"Replication_num" = "3": Specifies the number of replicas of the restored table or partition. The default is 3. If an existing table or partition is restored, the number of copies must be the same as the number of copies of an existing table or partition. At the same time, there must be enough hosts to accommodate multiple copies. -"Timeout" = "3600": Task timeout, default to one day. Unit seconds. -"Meta_version" = 40: Use the specified meta_version to read the previously backed up metadata. Note that as a temporary solution, this parameter is only used to restore the data backed up by the older version of Doris. The latest version of the backup data already contains metaversion, no need to specify. - -## example -1. Restore backup table backup_tbl in snapshot_1 from example_repo to database example_db1 with the time version of "2018-05-04-16-45-08". Restore to one copy: -RESTORE SNAPSHOT example_db1.`snapshot_1` -FROM `example 'u repo' -ON ( `backup_tbl` ) -PROPERTIES -( -"backup_timestamp"="2018-05-04-16-45-08", -"Replication\ num" = "1" -); - -2. Restore the partitions p1, P2 of table backup_tbl in snapshot_2 and table backup_tbl2 to database example_db1 from example_repo and rename it new_tbl. The time version is "2018-05-04-17-11-01". By default, three copies are restored: -RESTORE SNAPSHOT example_db1.`snapshot_2` -FROM `example 'u repo' -ON -( -`backup_tbl` PARTITION (`p1`, `p2`), -`backup_tbl2` AS `new_tbl` -) -PROPERTIES -( -"backup_timestamp"="2018-05-04-17-11-01" -); - -## keyword -RESTORE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE_EN.md deleted file mode 100644 index 7178e588a97476..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE_EN.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# TRUNCATE TABLES -## Description -This statement is used to empty the data of the specified table and partition -Grammar: - -TRUNCATE TABLE [db.]tbl[ PARTITION(p1, p2, ...)]; - -Explain: -1. The statement empties the data, but retains the table or partition. -2. Unlike DELETE, this statement can only empty the specified tables or partitions as a whole, without adding filtering conditions. -3. Unlike DELETE, using this method to clear data will not affect query performance. -4. The data deleted by this operation is not recoverable. -5. When using this command, the table state should be NORMAL, i.e. SCHEMA CHANGE operations are not allowed. - -## example - -1. Clear the table TBL under example_db - -TRUNCATE TABLE example_db.tbl; - -2. P1 and P2 partitions of clearing TABLE tbl - -TRUNCATE TABLE tbl PARTITION(p1, p2); - -## keyword -TRUNCATE,TABLE diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/create-function_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/create-function_EN.md deleted file mode 100644 index dc4a088b89d60b..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/create-function_EN.md +++ /dev/null @@ -1,104 +0,0 @@ - - -# CREATE FUNCTION -##Description -### Syntax - -``` -CREATE [AGGREGATE] FUNCTION function_name - (angry type [...]) - RETURNS ret_type - [INTERMEDIATE inter_type] - [PROPERTIES ("key" = "value" [, ...]) ] -``` - -### Parameters - ->`AGGREGATE`: If this is the case, it means that the created function is an aggregate function, otherwise it is a scalar function. -> ->`Function_name`: To create the name of the function, you can include the name of the database. For example: `db1.my_func'. -> ->` arg_type': The parameter type of the function is the same as the type defined at the time of table building. Variable-length parameters can be represented by `,...`. If it is a variable-length type, the type of the variable-length part of the parameters is the same as the last non-variable-length parameter type. -> ->`ret_type`: Function return type. -> ->`Inter_type`: A data type used to represent the intermediate stage of an aggregate function. -> ->`properties`: Used to set properties related to this function. Properties that can be set include -> -> "Object_file": Custom function dynamic library URL path, currently only supports HTTP/HTTPS protocol, this path needs to remain valid throughout the life cycle of the function. This option is mandatory -> -> "symbol": Function signature of scalar functions for finding function entries from dynamic libraries. This option is mandatory for scalar functions -> -> "init_fn": Initialization function signature of aggregate function. Necessary for aggregation functions -> -> "update_fn": Update function signature of aggregate function. Necessary for aggregation functions -> -> "merge_fn": Merge function signature of aggregate function. Necessary for aggregation functions -> -> "serialize_fn": Serialized function signature of aggregate function. For aggregation functions, it is optional, and if not specified, the default serialization function will be used -> -> "finalize_fn": A function signature that aggregates functions to obtain the final result. For aggregation functions, it is optional. If not specified, the default fetch result function will be used. -> -> "md5": The MD5 value of the function dynamic link library, which is used to verify that the downloaded content is correct. This option is optional -> -> "prepare_fn": Function signature of the prepare function for finding the entry from the dynamic library. This option is optional for custom functions -> -> "close_fn": Function signature of the close function for finding the entry from the dynamic library. This option is optional for custom functions - - -This statement creates a custom function. Executing this command requires that the user have `ADMIN` privileges. - -If the `function_name` contains the database name, the custom function will be created in the corresponding database, otherwise the function will be created in the database where the current session is located. The name and parameters of the new function cannot be the same as functions already existing in the current namespace, otherwise the creation will fail. But only with the same name and different parameters can the creation be successful. - -## example - -1. Create a custom scalar function - - ``` - CREATE FUNCTION my_add(INT, INT) RETURNS INT PROPERTIES ( - "symbol" = "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", - "object_file" ="http://host:port/libmyadd.so" - ); - ``` -2. Create a custom scalar function with prepare/close functions - - ``` - CREATE FUNCTION my_add(INT, INT) RETURNS INT PROPERTIES ( - "symbol" = "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", - "prepare_fn" = "_ZN9doris_udf14AddUdf_prepareEPNS_15FunctionContextENS0_18FunctionStateScopeE", - "close_fn" = "_ZN9doris_udf12AddUdf_closeEPNS_15FunctionContextENS0_18FunctionStateScopeE", - "object_file" = "http://host:port/libmyadd.so" - ); - ``` - -3. Create a custom aggregation function - - ``` - CREATE AGGREGATE FUNCTION my_count (BIGINT) RETURNS BIGINT PROPERTIES ( - "init_fn"= "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", - "update_fn" = "zn9dorisudf11CountupdateepnsFunctionContexterknsIntvalepnsbigintvale", - "merge_fn" = "zn9dorisudf10CountMergeepnsFunctionContexterknsBigintvaleps2 - "finalize_fn" = "zn9dorisudf13CountFinalizepnsFunctionContexterknsBigintvale", - "object_file" = "http://host:port/libudasample.so" - ); - ``` -##keyword -CREATE,FUNCTION diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/drop-function_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/drop-function_EN.md deleted file mode 100644 index eb48523be1bace..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/drop-function_EN.md +++ /dev/null @@ -1,47 +0,0 @@ - - -# DROP FUNCTION -##Description -### Syntax - -``` -DROP FUNCTION function_name -(angry type [...]) -``` - -### Parameters - ->` function_name': To delete the name of the function -> ->` arg_type`: To delete the parameter list of the function -> - - -Delete a custom function. The name of the function and the type of the parameter are exactly the same before they can be deleted. - -## example - -1. Delete a function - -``` -DROP FUNCTION my_add(INT, INT) -``` -##keyword -DROP,FUNCTION diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/index.rst b/docs/documentation/en/sql-reference/sql-statements/Data Definition/index.rst deleted file mode 100644 index 0e100c8758580f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -==================== -Data Definition -==================== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/show-functions_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/show-functions_EN.md deleted file mode 100644 index 3e393190130598..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/show-functions_EN.md +++ /dev/null @@ -1,70 +0,0 @@ - - -# SHOW FUNCTIONS -## Description -### Syntax - -``` -SHOW [FULL] [BUILTIN] FUNCTIONS [IN|FROM db] [LIKE 'function_pattern'] -``` - -### Parameters - ->`full`: Indicate to show the details of function ->`builtin`: Indicate to show the functions that doris provides ->`db`: The name of the database to query ->`function_pattern`: The parameter to filter function name - -Look at all the custom(builtin) functions under the database. If the user specifies the database, then look at the corresponding database, otherwise directly query the database where the current session is located. - -You need `SHOW'privileges for this database - -## example - -``` -mysql> show full functions in testDb\G -*************************** 1. row *************************** - Signature: my_add(INT,INT) - Return Type: INT - Function Type: Scalar -Intermediate Type: NULL - Properties: {"symbol":"_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_","object_file":"http://host:port/libudfsample.so","md5":"cfe7a362d10f3aaf6c49974ee0f1f878"} -*************************** 2. row *************************** - Signature: my_count(BIGINT) - Return Type: BIGINT - Function Type: Aggregate -Intermediate Type: NULL - Properties: {"object_file":"http://host:port/libudasample.so","finalize_fn":"_ZN9doris_udf13CountFinalizeEPNS_15FunctionContextERKNS_9BigIntValE","init_fn":"_ZN9doris_udf9CountInitEPNS_15FunctionContextEPNS_9BigIntValE","merge_fn":"_ZN9doris_udf10CountMergeEPNS_15FunctionContextERKNS_9BigIntValEPS2_","md5":"37d185f80f95569e2676da3d5b5b9d2f","update_fn":"_ZN9doris_udf11CountUpdateEPNS_15FunctionContextERKNS_6IntValEPNS_9BigIntValE"} - -2 rows in set (0.00 sec) -mysql> show builtin functions in testDb like 'year%'; -+---------------+ -| Function Name | -+---------------+ -| year | -| years_add | -| years_diff | -| years_sub | -+---------------+ -2 rows in set (0.00 sec) -``` - -##keyword -SHOW,FUNCTIONS diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/BROKER LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/BROKER LOAD_EN.md deleted file mode 100644 index d4bb70f77518fd..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/BROKER LOAD_EN.md +++ /dev/null @@ -1,421 +0,0 @@ - - -# BROKER LOAD -## description - - Broker load will load data into Doris via Broker. - Use `show broker;` to see the Broker deployed in cluster. - - Support following data sources: - - 1. Baidu HDFS: hdfs for Baidu. Only be used inside Baidu. - 2. Baidu AFS: afs for Baidu. Only be used inside Baidu. - 3. Baidu Object Storage(BOS): BOS on Baidu Cloud. - 4. Apache HDFS. - -### Syntax: - - LOAD LABEL load_label - ( - data_desc1[, data_desc2, ...] - ) - WITH BROKER broker_name - [broker_properties] - [opt_properties]; - - 1. load_label - - Unique load label within a database. - syntax: - [database_name.]your_label - - 2. data_desc - - To describe the data source. - syntax: - DATA INFILE - ( - "file_path1"[, file_path2, ...] - ) - [NEGATIVE] - INTO TABLE `table_name` - [PARTITION (p1, p2)] - [COLUMNS TERMINATED BY "column_separator"] - [FORMAT AS "file_type"] - [(column_list)] - [SET (k1 = func(k2))] - [WHERE predicate] - - Explain: - file_path: - - File path. Support wildcard. Must match to file, not directory. - - PARTITION: - - Data will only be loaded to specified partitions. Data out of partition's range will be filtered. If not specifed, all partitions will be loaded. - - NEGATIVE: - - If this parameter is specified, it is equivalent to importing a batch of "negative" data to offset the same batch of data loaded before. - - This parameter applies only to the case where there are value columns and the aggregation type of value columns is only SUM. - - column_separator: - - Used to specify the column separator in the import file. Default is `\t`. - If the character is invisible, it needs to be prefixed with `\\x`, using hexadecimal to represent the separator. - - For example, the separator `\x01` of the hive file is specified as `\\ x01` - - file_type: - - Used to specify the type of imported file, such as parquet, orc, csv. Default values are determined by the file suffix name. - - column_list: - - Used to specify the correspondence between columns in the import file and columns in the table. - - When you need to skip a column in the import file, specify it as a column name that does not exist in the table. - - syntax: - (col_name1, col_name2, ...) - - SET: - - If this parameter is specified, a column of the source file can be transformed according to a function, and then the transformed result can be loaded into the table. The grammar is `column_name = expression`. Some examples are given to help understand. - - Example 1: There are three columns "c1, c2, c3" in the table. The first two columns in the source file correspond in turn (c1, c2), and the last two columns correspond to c3. Then, column (c1, c2, tmp_c3, tmp_c4) SET (c3 = tmp_c3 + tmp_c4) should be specified. - - Example 2: There are three columns "year, month, day" in the table. There is only one time column in the source file, in the format of "2018-06-01:02:03". Then you can specify columns (tmp_time) set (year = year (tmp_time), month = month (tmp_time), day = day (tmp_time)) to complete the import. - - WHERE: - - After filtering the transformed data, data that meets where predicates can be loaded. Only column names in tables can be referenced in WHERE statements. - - 3. broker_name - - The name of the Broker used can be viewed through the `show broker` command. - - 4. broker_properties - - Used to provide Broker access to data sources. Different brokers, and different access methods, need to provide different information. - - 1. Baidu HDFS/AFS - - Access to Baidu's internal hdfs/afs currently only supports simple authentication, which needs to be provided: - - username: hdfs username - password: hdfs password - - 2. BOS - - bos_endpoint. - bos_accesskey: cloud user's accesskey - bos_secret_accesskey: cloud user's secret_accesskey - - 3. Apache HDFS - - Community version of HDFS supports simple authentication, Kerberos authentication, and HA configuration. - - Simple authentication: - hadoop.security.authentication = simple (default) - username: hdfs username - password: hdfs password - - kerberos authentication: - hadoop.security.authentication = kerberos - kerberos_principal: kerberos's principal - kerberos_keytab: path of kerberos's keytab file. This file should be able to access by Broker - kerberos_keytab_content: Specify the contents of the KeyTab file in Kerberos after base64 encoding. This option is optional from the kerberos_keytab configuration. - - namenode HA: - By configuring namenode HA, new namenode can be automatically identified when the namenode is switched - dfs.nameservices: hdfs service name，customize，eg: "dfs.nameservices" = "my_ha" - dfs.ha.namenodes.xxx: Customize the name of a namenode, separated by commas. XXX is a custom name in dfs. name services, such as "dfs. ha. namenodes. my_ha" = "my_nn" - dfs.namenode.rpc-address.xxx.nn: Specify RPC address information for namenode, where NN denotes the name of the namenode configured in dfs.ha.namenodes.xxxx, such as: "dfs.namenode.rpc-address.my_ha.my_nn"= "host:port" - dfs.client.failover.proxy.provider: Specify the provider that client connects to namenode by default: org. apache. hadoop. hdfs. server. namenode. ha. Configured Failover ProxyProvider. - - 4. opt_properties - - Used to specify some special parameters. - Syntax: - [PROPERTIES ("key"="value", ...)] - - You can specify the following parameters: - - timout: Specifies the timeout time for the import operation. The default timeout is 4 hours per second. - - max_filter_ratio: Data ratio of maximum tolerance filterable (data irregularity, etc.). Default zero tolerance. - - exc_mem_limit: Memory limit. Default is 2GB. Unit is Bytes. - - strict_mode: Whether the data is strictly restricted. The default is false. - - timezone: Specify time zones for functions affected by time zones, such as strftime/alignment_timestamp/from_unixtime, etc. See the documentation for details. If not specified, use the "Asia/Shanghai" time zone. - - 5. Load data format sample - - Integer（TINYINT/SMALLINT/INT/BIGINT/LARGEINT）: 1, 1000, 1234 - Float（FLOAT/DOUBLE/DECIMAL）: 1.1, 0.23, .356 - Date（DATE/DATETIME）: 2017-10-03, 2017-06-13 12:34:03. - (Note: If it's in other date formats, you can use strftime or time_format functions to convert in the import command) - - String（CHAR/VARCHAR）: "I am a student", "a" - NULL: \N - -## example - - 1. Load a batch of data from HDFS, specify timeout and filtering ratio. Use the broker with the inscription my_hdfs_broker. Simple authentication. - - LOAD LABEL example_db.label1 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - ) - WITH BROKER my_hdfs_broker - ( - "username" = "hdfs_user", - "password" = "hdfs_passwd" - ) - PROPERTIES - ( - "timeout" = "3600", - "max_filter_ratio" = "0.1" - ); - - Where hdfs_host is the host of the namenode and hdfs_port is the fs.defaultFS port (default 9000) - - 2. Load a batch of data from AFS contains multiple files. Import different tables, specify separators, and specify column correspondences. - - LOAD LABEL example_db.label2 - ( - DATA INFILE("afs://afs_host:hdfs_port/user/palo/data/input/file1") - INTO TABLE `my_table_1` - COLUMNS TERMINATED BY "," - (k1, k3, k2, v1, v2), - DATA INFILE("afs://afs_host:hdfs_port/user/palo/data/input/file2") - INTO TABLE `my_table_2` - COLUMNS TERMINATED BY "\t" - (k1, k2, k3, v2, v1) - ) - WITH BROKER my_afs_broker - ( - "username" = "afs_user", - "password" = "afs_passwd" - ) - PROPERTIES - ( - "timeout" = "3600", - "max_filter_ratio" = "0.1" - ); - - - 3. Load a batch of data from HDFS, specify hive's default delimiter \\x01, and use wildcard * to specify all files in the directory. Use simple authentication and configure namenode HA at the same time - - LOAD LABEL example_db.label3 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/*") - INTO TABLE `my_table` - COLUMNS TERMINATED BY "\\x01" - ) - WITH BROKER my_hdfs_broker - ( - "username" = "hdfs_user", - "password" = "hdfs_passwd", - "dfs.nameservices" = "my_ha", - "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", - "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", - "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", - "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" - ) - - 4. Load a batch of "negative" data from HDFS. Use Kerberos authentication to provide KeyTab file path. - - LOAD LABEL example_db.label4 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/old_file) - NEGATIVE - INTO TABLE `my_table` - COLUMNS TERMINATED BY "\t" - ) - WITH BROKER my_hdfs_broker - ( - "hadoop.security.authentication" = "kerberos", - "kerberos_principal"="doris@YOUR.COM", - "kerberos_keytab"="/home/palo/palo.keytab" - ) - - 5. Load a batch of data from HDFS, specify partition. At the same time, use Kerberos authentication mode. Provide the KeyTab file content encoded by base64. - - LOAD LABEL example_db.label5 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (k1, k3, k2, v1, v2) - ) - WITH BROKER my_hdfs_broker - ( - "hadoop.security.authentication"="kerberos", - "kerberos_principal"="doris@YOUR.COM", - "kerberos_keytab_content"="BQIAAABEAAEACUJBSURVLkNPTQAEcGFsbw" - ) - - 6. Load a batch of data from BOS, specify partitions, and make some transformations to the columns of the imported files, as follows: - - Table schema: - k1 varchar(20) - k2 int - - Assuming that the data file has only one row of data: - - Adele,1,1 - - The columns in the data file correspond to the columns specified in the load statement: - - k1,tmp_k2,tmp_k3 - - transform as: - - 1) k1: unchanged - 2) k2: sum of tmp_k2 and tmp_k3 - - LOAD LABEL example_db.label6 - ( - DATA INFILE("bos://my_bucket/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (k1, tmp_k2, tmp_k3) - SET ( - k2 = tmp_k2 + tmp_k3 - ) - ) - WITH BROKER my_bos_broker - ( - "bos_endpoint" = "http://bj.bcebos.com", - "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", - "bos_secret_accesskey"="yyyyyyyyyyyyyyyyyyyy" - ) - - 7. Load data into tables containing HLL columns, which can be columns in tables or columns in data - - If there are three columns in the table (id, v1, v2, v3). The V1 and V2 columns are HLL columns. The imported source file has three columns. Then (column_list) declares that the first column is id, and the second and third columns are temporarily named k1, k2. - - In SET, the HLL column in the table must be specifically declared hll_hash. The V1 column in the table is equal to the hll_hash (k1) column in the original data.The v3 column in the table does not have a corresponding value in the original data, and empty_hll is used to supplement the default value. - - LOAD LABEL example_db.label7 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (id, k1, k2) - SET ( - v1 = hll_hash(k1), - v2 = hll_hash(k2), - v3 = empty_hll() - ) - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - - LOAD LABEL example_db.label8 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - PARTITION (p1, p2) - COLUMNS TERMINATED BY "," - (k1, k2, tmp_k3, tmp_k4, v1, v2) - SET ( - v1 = hll_hash(tmp_k3), - v2 = hll_hash(tmp_k4) - ) - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - - 8. Data in load Parquet file specifies FORMAT as parquet. By default, it is judged by file suffix. - - LOAD LABEL example_db.label9 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - FORMAT AS "parquet" - (k1, k2, k3) - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - - 9. Extract partition fields in file paths - - If necessary, partitioned fields in the file path are resolved based on the field type defined in the table, similar to the Partition Discovery function in Spark. - - LOAD LABEL example_db.label10 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/*/*") - INTO TABLE `my_table` - FORMAT AS "csv" - (k1, k2, k3) - COLUMNS FROM PATH AS (city, utc_date) - SET (uniq_id = md5sum(k1, city)) - ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); - - Directory `hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing` contains following files: - - [hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/utc_date=2019-06-26/0000.csv, hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/utc_date=2019-06-26/0001.csv, ...] - - Extract city and utc_date fields in the file path - - 10. To filter the load data, columns whose K1 value is greater than K2 value can be imported. - - LOAD LABEL example_db.label10 - ( - DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") - INTO TABLE `my_table` - where k1 > k2 - ); - - 11. Extract date partition fields in file paths, and date time include %3A (in hdfs path, all ':' will be replaced by '%3A') - - Assume we have files: - - /user/data/data_time=2020-02-17 00%3A00%3A00/test.txt - /user/data/data_time=2020-02-18 00%3A00%3A00/test.txt - - Table schema is: - data_time DATETIME, - k2 INT, - k3 INT - - LOAD LABEL example_db.label12 - ( - DATA INFILE("hdfs://host:port/user/data/*/test.txt") - INTO TABLE `tbl12` - COLUMNS TERMINATED BY "," - (k2,k3) - COLUMNS FROM PATH AS (data_time) - SET (data_time=str_to_date(data_time, '%Y-%m-%d %H%%3A%i%%3A%s')) - ) - WITH BROKER "hdfs" ("username"="user", "password"="pass"); - -## keyword - - BROKER,LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL DELETE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL DELETE_EN.md deleted file mode 100644 index 923d8f076a80cd..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL DELETE_EN.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# CANCEL DELETE -Description - -This statement is used to undo a DELETE operation. (Administrator only!) (To be realized) - -'35;'35; example - -## keyword -CANCEL,DELETE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL LABEL_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL LABEL_EN.md deleted file mode 100644 index e385b1c19054f5..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL LABEL_EN.md +++ /dev/null @@ -1,46 +0,0 @@ - - -# Cancel Label -Description -NAME: -cancel_label: cancel a transaction with label - -SYNOPSIS -curl -u user:passwd -XPOST http://host:port/api/{db}/{label}/_cancel - -DESCRIPTION -This command is used to cancel a transaction corresponding to a specified Label, which can be successfully cancelled during the Prepare phase. - -RETURN VALUES -When the execution is complete, the relevant content of this import will be returned in Json format. Currently includes the following fields -Status: Successful cancel -Success: 成功cancel事务 -20854; 2018282: 22833; 361333; -Message: Specific Failure Information - -ERRORS - -'35;'35; example - -1. cancel testDb, testLabel20316;- 19994; -curl -u root -XPOST http://host:port/api/testDb/testLabel/_cancel - -## keyword -Cancel, Rabel diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD_EN.md deleted file mode 100644 index 55ff09834254e6..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD_EN.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# CANCEL LOAD -Description - -This statement is used to undo the import job for the batch of the specified load label. -This is an asynchronous operation, which returns if the task is submitted successfully. After execution, you can use the SHOW LOAD command to view progress. -Grammar: -CANCEL LOAD -[FROM both names] -WHERE LABEL = "load_label"; - -'35;'35; example - -1. Revoke the import job of example_db_test_load_label on the database example_db -CANCEL LOAD -FROM example_db -WHERE LABEL = "example_db_test_load_label"; - -## keyword -CANCEL,LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/DELETE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/DELETE_EN.md deleted file mode 100644 index b04cf0229fcb3f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/DELETE_EN.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# DELETE -Description - -This statement is used to conditionally delete data in the specified table (base index) partition. -This action deletes the rollup index data associated with this base index at the same time. -Grammar: -PART FROM table name [PARTITION partition name] -WHERE -column_name1 op value[ AND column_name2 op value ...]; - -Explain: -1) Optional types of OP include: =,>,<,>=,<=,<=,<=,!= -2) Conditions on key columns can only be specified. -2) When the selected key column does not exist in a rollup, delete cannot be performed. -3) The relationship between conditions can only be "and". -If you want to achieve the "or" relationship, you need to divide the conditions into two DELETE statements. -4) If you partition a table for RANGE, you must specify PARTITION. If it is a single partition table, you can not specify it. - -Be careful: -This statement may reduce query efficiency for a period of time after execution. -The degree of impact depends on the number of deletion conditions specified in the statement. -The more conditions specified, the greater the impact. - -'35;'35; example - -1. Delete rows whose K1 column value is 3 in my_table partition p 1 -DELETE FROM my_table PARTITION p1 -WHERE k1 = 3; - -2. Delete rows whose K1 column value is greater than or equal to 3 and whose K2 column value is "abc" in my_table partition P1 -DELETE FROM my_table PARTITION p1 -WHERE k1 >= 3 AND k2 = "abc"; - -## keyword -DELETE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/EXPORT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/EXPORT_EN.md deleted file mode 100644 index 8007b99a08349d..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/EXPORT_EN.md +++ /dev/null @@ -1,76 +0,0 @@ - - -# EXPORT -Description - -This statement is used to export data from a specified table to a specified location. -This function is implemented by broker process. For different purpose storage systems, different brokers need to be deployed. Deployed brokers can be viewed through SHOW BROKER. -This is an asynchronous operation, which returns if the task is submitted successfully. After execution, you can use the SHOW EXPORT command to view progress. - -Grammar: -EXPORT TABLE table_name -[PARTITION (p1 [,p2]] -TO export_path -[opt_properties] -broker; - -1. table_name -The table names to be exported currently support the export of tables with engine as OLAP and mysql. - -2. partition -You can export only certain specified partitions of the specified table - -3. export_path -The exported path needs to be a directory. At present, it can't be exported to local, so it needs to be exported to broker. - -4. opt_properties -Used to specify some special parameters. -Grammar: -[PROPERTIES ("key"="value", ...)] - -The following parameters can be specified: -Column_separator: Specifies the exported column separator, defaulting to t. -Line_delimiter: Specifies the exported line separator, defaulting to\n. -Exc_mem_limit: Exports the upper limit of memory usage for a single BE node, defaulting to 2GB in bytes. -Timeout: The time-out for importing jobs is 1 day by default, in seconds. -Tablet_num_per_task: The maximum number of tablets that each subtask can allocate. - -Five. debris -Broker used to specify export usage -Grammar: -WITH BROKER broker_name ("key"="value"[,...]) -Here you need to specify the specific broker name and the required broker attributes - -For brokers corresponding to different storage systems, the input parameters are different. Specific parameters can be referred to: `help broker load', broker required properties. - -'35;'35; example - -1. Export all data from the testTbl table to HDFS -EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); - -2. Export partitions P1 and P2 from the testTbl table to HDFS - -EXPORT TABLE testTbl PARTITION (p1,p2) TO "hdfs://hdfs_host:port/a/b/c" WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); -3. Export all data in the testTbl table to hdfs, using "," as column separator - -EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" PROPERTIES ("column_separator"=",") WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); - -## keyword -EXPORT diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/GET LABEL STATE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/GET LABEL STATE_EN.md deleted file mode 100644 index a72831a42536c7..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/GET LABEL STATE_EN.md +++ /dev/null @@ -1,51 +0,0 @@ - - -# GET LABEL STATE -## Description -NAME: -get_label_state: get label's state - -SYNOPSIS -curl -u user:passwd http://host:port /api /{db}/{label}// u state - -DESCRIPTION -This command is used to view the transaction status of a Label - -RETURN VALUES -After execution, the relevant content of this import will be returned in Json format. Currently includes the following fields -Label: The imported label, if not specified, is a uuid. -Status: Whether this command was successfully executed or not, Success indicates successful execution -Message: Specific execution information -State: It only makes sense if Status is Success -UNKNOWN: No corresponding Label was found -PREPARE: The corresponding transaction has been prepared, but not yet committed -COMMITTED: The transaction has been committed and cannot be canceled -VISIBLE: Transaction submission, and data visible, cannot be canceled -ABORTED: The transaction has been ROLLBACK and the import has failed. - -ERRORS - -'35;'35; example - -1. Obtain the state of testDb, testLabel -curl -u root http://host:port /api /testDb /testLabel / u state - -## keyword -GET, LABEL, STATE diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/GROUP BY_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/GROUP BY_EN.md deleted file mode 100644 index 3e90baa329cce4..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/GROUP BY_EN.md +++ /dev/null @@ -1,161 +0,0 @@ - - -# GROUP BY - -## description - - GROUP BY `GROUPING SETS` ｜ `CUBE` ｜ `ROLLUP` is an extension to GROUP BY clause. This syntax lets you define multiple groupings in the same query. GROUPING SETS produce a single result set that is equivalent to a UNION ALL of differently grouped rows - For example GROUPING SETS clause: - - ``` - SELECT a, b, SUM( c ) FROM tab1 GROUP BY GROUPING SETS ( (a, b), (a), (b), ( ) ); - ``` - - This statement is equivalent to: - - ``` - SELECT a, b, SUM( c ) FROM tab1 GROUP BY a, b - UNION - SELECT a, null, SUM( c ) FROM tab1 GROUP BY a - UNION - SELECT null, b, SUM( c ) FROM tab1 GROUP BY b - UNION - SELECT null, null, SUM( c ) FROM tab1 - ``` - - `GROUPING(expr)` indicates whether a specified column expression in a GROUP BY list is aggregated or not. GROUPING returns 1 for aggregated or 0 for not aggregated in the result set. - - `GROUPING_ID(expr [ , expr [ , ... ] ])` describes which of a list of expressions are grouped in a row produced by a GROUP BY query. The GROUPING_ID function simply returns the decimal equivalent of the binary value formed as a result of the concatenation of the values returned by the GROUPING functions. - -### Syntax - - ``` - SELECT ... - FROM ... - [ ... ] - GROUP BY [ - , ... | - GROUPING SETS [, ...] ( groupSet [ , groupSet [ , ... ] ] ) | - ROLLUP(expr [ , expr [ , ... ] ]) | - expr [ , expr [ , ... ] ] WITH ROLLUP | - CUBE(expr [ , expr [ , ... ] ]) | - expr [ , expr [ , ... ] ] WITH CUBE - ] - [ ... ] - ``` - -### Parameters - - `groupSet` is a set of expression or column or it's alias appearing in the query block’s SELECT list. `groupSet ::= { ( expr [ , expr [ , ... ] ] )}` - - `expr` is expression or column or it's alias appearing in the query block’s SELECT list. - -### Note - - doris supports PostgreSQL like syntax, for example: - - ``` - SELECT a, b, SUM( c ) FROM tab1 GROUP BY GROUPING SETS ( (a, b), (a), (b), ( ) ); - SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY ROLLUP(a,b,c) - SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY CUBE(a,b,c) - ``` - - `ROLLUP(a,b,c)` is equivalent to `GROUPING SETS` as follows: - - ``` - GROUPING SETS ( - (a,b,c), - ( a, b ), - ( a), - ( ) - ) - ``` - - `CUBE ( a, b, c )` is equivalent to `GROUPING SETS` as follows: - - ``` - GROUPING SETS ( - ( a, b, c ), - ( a, b ), - ( a, c ), - ( a ), - ( b, c ), - ( b ), - ( c ), - ( ) - ) - ``` - -## example - - This is a simple example - - ``` - > SELECT * FROM t; - +------+------+------+ - | k1 | k2 | k3 | - +------+------+------+ - | a | A | 1 | - | a | A | 2 | - | a | B | 1 | - | a | B | 3 | - | b | A | 1 | - | b | A | 4 | - | b | B | 1 | - | b | B | 5 | - +------+------+------+ - 8 rows in set (0.01 sec) - - > SELECT k1, k2, SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); - +------+------+-----------+ - | k1 | k2 | sum(`k3`) | - +------+------+-----------+ - | b | B | 6 | - | a | B | 4 | - | a | A | 3 | - | b | A | 5 | - | NULL | B | 10 | - | NULL | A | 8 | - | a | NULL | 7 | - | b | NULL | 11 | - | NULL | NULL | 18 | - +------+------+-----------+ - 9 rows in set (0.06 sec) - - > SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ()); - +------+------+---------------+----------------+ - | k1 | k2 | grouping_id(k1,k2) | sum(`k3`) | - +------+------+---------------+----------------+ - | a | A | 0 | 3 | - | a | B | 0 | 4 | - | a | NULL | 1 | 7 | - | b | A | 0 | 5 | - | b | B | 0 | 6 | - | b | NULL | 1 | 11 | - | NULL | A | 2 | 8 | - | NULL | B | 2 | 10 | - | NULL | NULL | 3 | 18 | - +------+------+---------------+----------------+ - 9 rows in set (0.02 sec) - ``` - -## keyword - - GROUP, GROUPING, GROUPING_ID, GROUPING_SETS, GROUPING SETS, CUBE, ROLLUP diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/LOAD_EN.md deleted file mode 100644 index 8ddc8aea9dab2c..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/LOAD_EN.md +++ /dev/null @@ -1,279 +0,0 @@ - - -# LOAD -## Description - -Palo currently supports the following four import methods: - -1. Hadoop Load: Importing ETL based on MR. -2. Broker Load: Use broker to import data. -3. Mini Load: Upload files through HTTP protocol for batch data import. -4. Stream Load: Stream data import through HTTP protocol. - -This help mainly describes the first import method, namely Hadoop Load related help information. The rest of the import methods can use the following commands to view help: - -This import method may not be supported in a subsequent version. It is recommended that other import methods be used for data import. !!! - -1. help broker load; -2. help mini load; -3. help stream load; - -Hadoop Load is only applicable to Baidu's internal environment. Public, private and open source environments cannot use this import approach. -The import method must set up a Hadoop computing queue for ETL, which can be viewed through the help set property command. - -Stream load only supports Baidu internal users for the time being. Open source communities and public cloud users will be supported in subsequent version updates. - -Grammar: - -LOAD LABEL load_label -( -Date of date of date of entry -) -[opt_properties]; - -1. load label - -The label of the current imported batch. Unique in a database. -Grammar: -[database_name.]your_label - -2. data_desc - -Used to describe a batch of imported data. -Grammar: -DATA INFILE -( -"file_path1"[, file_path2, ...] -) -[NEGATIVE] -INTO TABLE `table_name` -[PARTITION (p1, P2)] -[COLUMNS TERMINATED BY "column_separator"] -[FORMAT AS "file_type"] -[(column_list)] -[set (k1 = fun (k2)]] - -Explain: -file_path: - -File paths can be specified to a file, or * wildcards can be used to specify all files in a directory. Wildcards must match to files, not directories. - -PARTICIPATION: - -If this parameter is specified, only the specified partition will be imported, and data outside the imported partition will be filtered out. -If not specified, all partitions of the table are imported by default. - -NEGATIVE： -If this parameter is specified, it is equivalent to importing a batch of "negative" data. Used to offset the same batch of data imported before. -This parameter applies only to the case where there are value columns and the aggregation type of value columns is SUM only. - -Column U separator: - -Used to specify the column separator in the import file. Default tot -If the character is invisible, it needs to be prefixed with \x, using hexadecimal to represent the separator. -For example, the separator X01 of the hive file is specified as "\ x01" - -File type: - -Used to specify the type of imported file, such as parquet, orc, csv. The default value is determined by the file suffix name. - -column_list： - -Used to specify the correspondence between columns in the import file and columns in the table. -When you need to skip a column in the import file, specify it as a column name that does not exist in the table. -Grammar: -(col_name1, col_name2, ...) - -SET: - -If this parameter is specified, a column of the source file can be transformed according to a function, and then the transformed result can be imported into the table. -The functions currently supported are: - -Strftime (fmt, column) date conversion function -Fmt: Date format, such as% Y% m% d% H% M% S (year, month, day, hour, second) -Column: Column in column_list, which is the column in the input file. Storage content should be a digital timestamp. -If there is no column_list, the columns of the input file are entered by default in the column order of the Palo table. - -time_format(output_fmt, input_fmt, column) 日期格式转化 -Output_fmt: Converted date format, such as% Y% m% d% H% M% S (year, month, day, hour, second) -Input_fmt: The date format of the column before transformation, such as% Y% m% d% H% M% S (days, hours, seconds, months, years) -Column: Column in column_list, which is the column in the input file. Storage content should be a date string in input_fmt format. -If there is no column_list, the columns of the input file are entered by default in the column order of the Palo table. - -alignment_timestamp(precision, column) 将时间戳对齐到指定精度 -Precision: year 124month;124day;124hour; -Column: Column in column_list, which is the column in the input file. Storage content should be a digital timestamp. -If there is no column_list, the columns of the input file are entered by default in the column order of the Palo table. -Note: When the alignment accuracy is year and month, only the time stamps in the range of 20050101-20191231 are supported. - -Default_value (value) sets the default value for a column import -Use default values of columns when creating tables without specifying - -Md5sum (column1, column2,...) evaluates the value of the specified imported column to md5sum, returning a 32-bit hexadecimal string - -Replace_value (old_value [, new_value]) replaces old_value specified in the import file with new_value -New_value, if not specified, uses the default value of the column when building the table - -Hll_hash (column) is used to transform a column in a table or data into a data structure of a HLL column - -3. opt_properties - -Used to specify some special parameters. -Grammar: -[PROPERTIES ("key"="value", ...)] - -The following parameters can be specified: -Cluster: Import the Hadoop computed queue used. -Timeout: Specifies the timeout time of the import operation. The default timeout is 3 days. Unit seconds. -Max_filter_ratio: The ratio of data that is most tolerant of being filterable (for reasons such as data irregularities). Default zero tolerance. -Load_delete_flag: Specifies whether the import deletes data by importing the key column, which applies only to UNIQUE KEY. -Value column is not specified when importing. The default is false. - -5. Import data format sample - -Integer classes (TINYINT/SMALLINT/INT/BIGINT/LARGEINT): 1,1000,1234 -Floating Point Class (FLOAT/DOUBLE/DECIMAL): 1.1, 0.23, 356 -Date class (DATE/DATETIME): 2017-10-03, 2017-06-13 12:34:03. -(Note: If it's in other date formats, you can use strftime or time_format functions to convert in the import command) -字符串类（CHAR/VARCHAR）："I am a student", "a" -NULL value: N - -'35;'35; example - -1. Import a batch of data, specify timeout time and filtering ratio. Specify the import queue as my_cluster. - -LOAD LABEL example db.label1 -( -DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") -INTO TABLE `my_table` -) -PROPERTIES -( -"cluster" ="my" cluster, -Timeout ="3600", -"max_filter_ratio" = "0.1" -); - -Where hdfs_host is the host of the namenode and hdfs_port is the fs.defaultFS port (default 9000) - -2. Import a batch of data, including multiple files. Import different tables, specify separators, and specify column correspondences - -LOAD LABEL example db.label2 -( -DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file1") -INTO TABLE `my_table_1` -COLUMNS TERMINATED BY "," -(k1, k3, k2, v1, v2), -DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file2") -INTO TABLE `my_table_2` -COLUMNS TERMINATED BY "\t" -(k1, k2, k3, v2, v1) -); - -3. Import a batch of data, specify hive's default delimiter x01, and use wildcard * to specify all files in the directory - -LOAD LABEL example db.label3 -( -DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/*") -NEGATIVE -INTO TABLE `my_table` -COLUMNS TERMINATED BY "\\x01" -); - -4. Import a batch of "negative" data - -LOAD LABEL example db.label4 -( -DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/old_file) -NEGATIVE -INTO TABLE `my_table` -COLUMNS TERMINATED BY "\t" -); - -5. Import a batch of data and specify partitions - -LOAD LABEL example db.label5 -( -DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") -INTO TABLE `my_table` -PARTITION (p1, P2) -COLUMNS TERMINATED BY "," -(k1, k3, k2, v1, v2) -); - -6. Import a batch of data, specify partitions, and make some transformations to the columns of the imported files, as follows: -The table structure is as follows: -K1 date -date -k3 bigint -k4 varchar (20) -k5 varchar (64) -k6 int - -Assume that the data file has only one row of data, five columns, and comma-separated: - -1537002087,2018-08-09 11:12:13,1537002087,-,1 - -The columns in the data file correspond to the columns specified in the import statement: -tmp -u k1, tmp -u k2, tmp u k3, k6, v1 - -The conversion is as follows: - -1) k1: Transform tmp_k1 timestamp column into datetime type data -2) k2: Converting tmp_k2 datetime-type data into date data -3) k3: Transform tmp_k3 timestamp column into day-level timestamp -4) k4: Specify import default value of 1 -5) k5: Calculate MD5 values from tmp_k1, tmp_k2, tmp_k3 columns -6) k6: Replace the - value in the imported file with 10 - -LOAD LABEL example db.label6 -( -DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") -INTO TABLE `my_table` -PARTITION (p1, P2) -COLUMNS TERMINATED BY "," -(tmp /u k1, tmp /u k2, tmp /u k3, k6, v1) -SET ( -K1 = strftime (%Y -%m -%d%H:%M:%S ", TMP u K1), -K2 = Time = UFormat ("% Y-% M-% D% H:% M:% S", "% Y-% M-% D", "TMP = UK2), -k3 = alignment_timestamp("day", tmp_k3), -k4 = default_value("1"), -K5 = MD5Sum (TMP = UK1, TMP = UK2, TMP = UK3) -k6 = replace value ("-", "10") -) -); - -7. Import data into tables containing HLL columns, which can be columns in tables or columns in data - -LOAD LABEL example db.label7 -( -DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") -INTO TABLE `my_table` -PARTITION (p1, P2) -COLUMNS TERMINATED BY "," -SET ( -v1 = hll, u hash (k1), -v2 = hll, u hash (k2) -) -); - -## keyword -LOAD - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/MINI LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/MINI LOAD_EN.md deleted file mode 100644 index 1e5ebdbf1fa1ce..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/MINI LOAD_EN.md +++ /dev/null @@ -1,125 +0,0 @@ - - -# MINI LOAD -## Description - -MINI LOAD and STEAM LOAD are implemented in exactly the same way. MINI LOAD is a subset of STREAM LOAD in import support. -Subsequent imports of new features will only be supported in STEAM LOAD, MINI LOAD will no longer add features. It is suggested that STREAM LOAD be used instead. Please use HELP STREAM LOAD. - -MINI LOAD is imported through HTTP protocol. Users can import without relying on Hadoop or Mysql client. -The user describes the import through HTTP protocol, and the data is streamed into Doris in the process of receiving http requests. After the ** import job is completed, the ** returns to the user the imported results. - -* Note: In order to be compatible with the old version of mini load usage habits, users can still view the import results through the'SHOW LOAD'command. - -Grammar: -Import: - -curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table}/_load?label=xxx - -View import information - -curl -u user:passwd http://host:port/api/{db}/_load_info?label=xxx - -HTTP Protocol Specification - -Privilege Authentication Currently Doris uses the Basic mode of HTTP for privilege authentication. So you need to specify a username and password when importing -This way is to pass the password in plaintext, and does not support encrypted transmission for the time being. - -Expect Doris needs to send an HTTP request with the'Expect'header information,'100-continue'. -Why? Because we need to redirect the request, we have to transfer the data content before. -This can avoid causing multiple data transmission, thereby improving efficiency. - -Content-Length Doris needs to send a request with the header'Content-Length'. If the content ratio is sent -'Content-Length'is less, so Doris believes that if there is a transmission problem, the submission task fails. -NOTE: If you send more data than'Content-Length', Doris reads only'Content-Length'. -Length content and import - - -Description of parameters: - -User: User is user_name if the user is in default_cluster. Otherwise, it is user_name@cluster_name. - -Label: The label used to specify this batch of imports for later job queries, etc. -This parameter must be passed in. - -Columns: Used to describe the corresponding column name in the import file. -If it is not passed in, the column order in the file is considered to be the same as the order in which the table is built. -The specified method is comma-separated, such as columns = k1, k2, k3, K4 - -Column_separator: Used to specify the separator between columns, default is' t' -NOTE: Url encoding is required, for example -If you need to specify' t'as a separator, you should pass in'column_separator=% 09' -If you need to specify'x01'as a delimiter, you should pass in'column_separator=% 01' -If you need to specify','as a separator, you should pass in'column_separator=% 2c' - - -Max_filter_ratio: Used to specify the maximum percentage allowed to filter irregular data, default is 0, not allowed to filter -Custom specification should be as follows:'max_filter_ratio = 0.2', meaning that 20% error rate is allowed. - -Timeout: Specifies the timeout time of the load job in seconds. When the load execution time exceeds this threshold, it is automatically cancelled. The default timeout time is 86400 seconds. -It is recommended to specify a timeout time of less than 86400 seconds. - -Hll: Used to specify the corresponding relationship between the HLL columns in the data and the tables, the columns in the tables and the columns specified in the data. -(If columns are not specified, the columns of the data column surface can also be other non-HLL columns in the table.) By "partition" -Specify multiple HLL columns using ":" splitting, for example:'hll1, cuid: hll2, device' - -NOTE: -1. This method of importing is currently completed on a single machine, so it is not suitable to import a large amount of data. -It is recommended that the amount of data imported should not exceed 1 GB. - -2. Currently, it is not possible to submit multiple files in the form of `curl-T', `{file1, file2}', because curl splits them into multiple files. -Request sent, multiple requests can not share a label number, so it can not be used - -3. Miniload is imported in exactly the same way as streaming. It returns the results synchronously to users after the import of streaming is completed. -Although the information of mini load can be found in subsequent queries, it can not be operated on. The queries are only compatible with the old ways of use. - -4. When importing from the curl command line, you need to add escape before & or the parameter information will be lost. - -'35;'35; example - -1. Import the data from the local file'testData'into the table of'testTbl' in the database'testDb'(the user is in defalut_cluster) -curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123 - -2. Import the data from the local file'testData'into the table of'testTbl' in the database'testDb'(the user is in test_cluster). The timeout time is 3600 seconds. -curl --location-trusted -u root@test_cluster:root -T testData http://fe.host:port/api/testDb/testTbl/_load?label=123&timeout=3600 - -3. Import data from the local file'testData'into the'testTbl' table in the database'testDb', allowing a 20% error rate (the user is in defalut_cluster) -curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 - -4. Import the data from the local file'testData'into the table'testTbl' in the database'testDb', allowing a 20% error rate, and specify the column name of the file (the user is in defalut_cluster) -curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2\&columns=k1,k2,k3 - -5. Import in streaming mode (user is in defalut_cluster) -seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_load?label=123 - -6. Import tables containing HLL columns, which can be columns in tables or columns in data to generate HLL columns (users are in defalut_cluster) - - curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2\&hll=hll_column1,k1:hll_column2,k2 - \&columns=k1,k2,k3 - - curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 - \&hll=hll_column1,tmp_k4:hll_column2,tmp_k5\&columns=k1,k2,k3,tmp_k4,tmp_k5 - -7. View imports after submission - -curl -u root http://host:port/api/testDb/_load_info?label=123 - -## keyword -MINI, LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/MULTI LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/MULTI LOAD_EN.md deleted file mode 100644 index f690c0d2fe3814..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/MULTI LOAD_EN.md +++ /dev/null @@ -1,100 +0,0 @@ - - -# MULTI LOAD -## Description - -Syntax: -curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_start?label=xxx -curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table1}/_load?label=xxx\&sub_label=yyy -curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table2}/_load?label=xxx\&sub_label=zzz -curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_commit?label=xxx -curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_desc?label=xxx - -'MULTI LOAD'can support users to import multiple tables at the same time on the basis of'MINI LOAD'. The specific commands are shown above. -'/api/{db}/_multi_start'starts a multi-table import task -'/api/{db}/{table}/_load'adds a table to be imported to an import task. The main difference from'MINI LOAD' is that the'sub_label'parameter needs to be passed in. -'/api/{db}/_multi_commit'submits the entire multi-table import task and the background begins processing -'/api/{db}/_multi_abort'Abandons a multi-table import task -'/api/{db}/_multi_desc'shows the number of jobs submitted by a multi-table import task - -HTTP Protocol Specification -Privilege Authentication Currently Doris uses the Basic mode of HTTP for privilege authentication. So you need to specify a username and password when importing -This way is to pass passwords in plaintext, since we are all in the Intranet environment at present... - -Expect Doris needs to send an HTTP request, and needs the'Expect'header information with the content of'100-continue'. -Why? Because we need to redirect the request, we have to transfer the data content before. -This can avoid causing multiple data transmission, thereby improving efficiency. - -Content-Length Doris needs to send a request with the header'Content-Length'. If the content ratio is sent -If'Content-Length'is less, Palo believes that if there is a transmission problem, the submission of the task fails. -NOTE: If you send more data than'Content-Length', Doris reads only'Content-Length'. -Length content and import - -Description of parameters: -User: User is user_name if the user is in default_cluster. Otherwise, it is user_name@cluster_name. - -Label: Used to specify the label number imported in this batch for later job status queries, etc. -This parameter must be passed in. - -Sub_label: Used to specify a subversion number within a multi-table import task. For multi-table imported loads, this parameter must be passed in. - -Columns: Used to describe the corresponding column name in the import file. -If it is not passed in, the column order in the file is considered to be the same as the order in which the table is built. -The specified method is comma-separated, such as columns = k1, k2, k3, K4 - -Column_separator: Used to specify the separator between columns, default is' t' -NOTE: Url encoding is required, such as specifying't'as a delimiter. -Then you should pass in'column_separator=% 09' - -Max_filter_ratio: Used to specify the maximum percentage allowed to filter irregular data, default is 0, not allowed to filter -Custom specification should be as follows:'max_filter_ratio = 0.2', meaning that 20% error rate is allowed. -Pass in effect at'_multi_start' - -NOTE: -1. This method of importing is currently completed on a single machine, so it is not suitable to import a large amount of data. -It is recommended that the amount of data imported should not exceed 1GB - -2. Currently, it is not possible to submit multiple files in the form of `curl-T', `{file1, file2}', because curl splits them into multiple files. -Request sent, multiple requests can not share a label number, so it can not be used - -3. Supports streaming-like ways to use curl to import data into Doris, but Doris will have to wait until the streaming is over -Real import behavior will occur, and the amount of data in this way can not be too large. - -'35;'35; example - -1. Import the data from the local file'testData1'into the table of'testTbl1' in the database'testDb', and -Import the data from'testData2'into the table'testTbl2' in'testDb'(the user is in defalut_cluster) -curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 -curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 -curl --location-trusted -u root -T testData2 http://host:port/api/testDb/testTbl2/_load?label=123\&sub_label=2 -curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_commit?label=123 - -2. Multi-table Import Midway Abandon (User in defalut_cluster) -curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 -curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 -curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_abort?label=123 - -3. Multi-table import to see how much content has been submitted (user is in defalut_cluster) -curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 -curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 -curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_desc?label=123 - -## keyword -MULTI, MINI, LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD_EN.md deleted file mode 100644 index 3968b8c1bc42f3..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD_EN.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# PAUSE ROUTINE LOAD -## example - -1. Suspend the routine import operation named test 1. - -PAUSE ROUTINE LOAD FOR test1; - -## keyword -PAUSE,ROUTINE,LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/RESTORE TABLET_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/RESTORE TABLET_EN.md deleted file mode 100644 index ad2945f80b01e3..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/RESTORE TABLET_EN.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# RESTORE TABLET -Description - -This function is used to recover the tablet data that was deleted by mistake in the trash directory. - -Note: For the time being, this function only provides an HTTP interface in be service. If it is to be used, -A restore tablet API request needs to be sent to the HTTP port of the be machine for data recovery. The API format is as follows: -Method: Postal -URI: http://be_host:be_http_port/api/restore_tablet?tablet_id=xxx&schema_hash=xxx - -'35;'35; example - -Curl -X POST "http://hostname:8088 /api /restore" tablet? Tablet id =123456 &schema hash =1111111 " -##keyword -RESTORE,TABLET,RESTORE,TABLET diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD_EN.md deleted file mode 100644 index 9a4986f39afd9b..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD_EN.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# RESUME ROUTINE LOAD -## example - -1. Restore the routine import job named test 1. - -RESUME ROUTINE LOAD FOR test1; - -## keyword -RESUME,ROUTINE,LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD_EN.md deleted file mode 100644 index 3be3cdd0f33c39..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD_EN.md +++ /dev/null @@ -1,364 +0,0 @@ - - -# ROUTINE LOAD -## description - -Routine Load function allows users to submit a resident load task, and continuously load data into Doris by continuously reading data from the specified data source. Currently, only text data format (CSV) data is loaded from Kakfa by means of no authentication or SSL authentication. - -Syntax: - -``` -CREATE ROUTINE LOAD [db.]job_name ON tbl_name -[load_properties] -[job_properties] -FROM data_source -[data_source_properties] -``` - -1. [db.]job_name - - The name of the load job, in the same database, only one job can run with the same name. - -2. tbl_name - - Specifies the name of the table that needs to be loaded. - -3. load_properties - - Used to describe the load data. grammar: - - ``` - [column_separator], - [columns_mapping], - [where_predicates], - [partitions] - ``` - - 1. column_separator: - - Specify column separators, such as: - - `COLUMNS TERMINATED BY ","` - - The default is: `\t` -         - 2. columns_mapping: - - Specifies the mapping of columns in the source data and defines how the derived columns are generated. - - 1. Map column: - - Specify in order, which columns in the source data correspond to which columns in the destination table. For columns that you want to skip, you can specify a column name that does not exist. - - Suppose the destination table has three columns k1, k2, v1. The source data has 4 columns, of which columns 1, 2, and 4 correspond to k2, k1, and v1, respectively. Write as follows: - - `COLUMNS (k2, k1, xxx, v1)` - - Where xxx is a column that does not exist and is used to skip the third column in the source data. - - 2. Derived columns: - - A column represented in the form of col_name = expr, which we call a derived column. That is, the value of the corresponding column in the destination table is calculated by expr. - - Derived columns are usually arranged after the mapped column. Although this is not mandatory, Doris always parses the mapped columns first and then parses the derived columns. - - Following an example, assume that the destination table also has column 4, v2, which is generated by the sum of k1 and k2. You can write as follows: - - `COLUMNS (k2, k1, xxx, v1, v2 = k1 + k2);` - - 3. where_predicates - - Used to specify filter criteria to filter out unwanted columns. Filter columns can be either mapped columns or derived columns. - - For example, if we only want to load a column with k1 greater than 100 and k2 equal to 1000, we would write as follows: - - `WHERE k1 > 100 and k2 = 1000` -         - 4. partitions - - Specifies which partitions of the load destination table. If not specified, it will be automatically loaded into the corresponding partition. - - Example: - - `PARTITION(p1, p2, p3)` - -4. job_properties - - A generic parameter that specifies a routine load job. - - syntax: - - ``` - PROPERTIES ( - "key1" = "val1", - "key2" = "val2" - ) - ``` - - Currently we support the following parameters: - - 1. `desired_concurrent_number` - - The degree of concurrency desired. A routine load job is split into multiple subtasks. This parameter specifies how many tasks can be executed simultaneously in a job. Must be greater than 0. The default is 3. - - This concurrency is not the actual concurrency. The actual concurrency will be considered by the number of nodes in the cluster, the load, and the data source. - - example: - - `"desired_concurrent_number" = "3"` - - 2. `max_batch_interval/max_batch_rows/max_batch_size` - - These three parameters represent: - - 1) The maximum execution time of each subtask, in seconds. The range is 5 to 60. The default is 10. - - 2) The maximum number of rows read per subtask. Must be greater than or equal to 200,000. The default is 200000. - - 3) The maximum number of bytes read per subtask. The unit is byte and the range is 100MB to 1GB. The default is 100MB. - - These three parameters are used to control the execution time and throughput of a subtask. When either one reaches the threshold, the task ends. - - example: - - ``` - "max_batch_interval" = "20", - "max_batch_rows" = "300000", - "max_batch_size" = "209715200" - ``` - - 3. `max_error_number` - - The maximum number of error lines allowed in the sampling window. Must be greater than or equal to 0. The default is 0, which means that no error lines are allowed. - - The sampling window is max_batch_rows * 10. That is, if the number of error lines is greater than max_error_number in the sampling window, the routine job will be suspended, and manual intervention is required to check the data quality problem. - - Lines that are filtered by the where condition are not counted as error lines. - - 4. `strict_mode` - - Whether to enable strict mode, the default is on. If turned on, the column type transformation of non-null raw data is filtered if the result is NULL. Specified as "strict_mode" = "true" -             - 5. timezone - - Specifies the time zone in which the job will be loaded. The default by using session variable's timezone. This parameter affects all function results related to the time zone involved in the load. - -5. data_source - - The type of data source. Current support: - - KAFKA - -6. `data_source_properties` - - Specify information about the data source. - - syntax: -     - ``` - ( - "key1" = "val1", - "key2" = "val2" - ) - ``` - - 1. KAFKA data source - - `Kafka_broker_list` - - Kafka's broker connection information. The format is ip:host. Multiple brokare separated by commas. - - Example: - - `"kafka_broker_list" = "broker1:9092,broker2:9092"` - - 2. `kafka_topic` - - Specify the topic of Kafka to subscribe to. - - Example: - - `"kafka_topic" = "my_topic"` - - 3. `kafka_partitions/kafka_offsets` - - Specify the kafka partition to be subscribed to, and the corresponding star offset for each partition. - - Offset can specify a specific offset from 0 or greater, or: - - 1) OFFSET_BEGINNING: Subscribe from the location where the data is avaie. - - 2) OFFSET_END: ​​Subscribe from the end. - - If not specified, all partitions under topic are subscribed by default fromSET_END. - - Example: - - ``` - "kafka_partitions" = "0,1,2,3", - "kafka_offsets" = "101,0,OFFSET_BEGINNING,OFFSET_END" - ``` - - 4. property - - Specify custom kafka parameters. - - The function is equivalent to the "--property" parameter in the kafka shel - - When the value of the parameter is a file, you need to add the keyword: "FILbefore the value. - - For information on how to create a file, see "HELP CREATE FILE;" - - For more supported custom parameters, see the configuration items on the nt side in the official CONFIGURATION documentation for librdkafka. - - Example: - - ``` - "property.client.id" = "12345", - "property.ssl.ca.location" = "FILE:ca.pem" - ``` - - 1. When connecting to Kafka using SSL, you need to specify the follg parameters: - - ``` - "property.security.protocol" = "ssl", - "property.ssl.ca.location" = "FILE:ca.pem", - "property.ssl.certificate.location" = "FILE:client.pem", - "property.ssl.key.location" = "FILE:client.key", - "property.ssl.key.password" = "abcdefg" - ``` - - among them: - - "property.security.protocol" and "property.ssl.ca.location" are requ to indicate the connection method is SSL and the location of the CA certate. - - If the client authentication is enabled on the Kafka server, you alsod to set: - - ``` - "property.ssl.certificate.location" - "property.ssl.key.location" - "property.ssl.key.password" - ``` - - Used to specify the public key of the client, the private key, and the word of the private key. - - 2. Specify the default starting offset for kafka partition - - If kafka_partitions/kafka_offsets is not specified, all partitions are umed by default, and you can specify kafka_default_offsets to specify the star offset. The default is OFFSET_END, which starts at the end of the substion. - - Values: - - 1) OFFSET_BEGINNING: Subscribe from the location where the data is avaie. - - 2) OFFSET_END: Subscribe from the end. - - Example: - - `"property.kafka_default_offsets" = "OFFSET_BEGINNING"` - -7. load data format sample - - Integer class (TINYINT/SMALLINT/INT/BIGINT/LARGEINT): 1, 1000, 1234 - - Floating point class (FLOAT/DOUBLE/DECIMAL): 1.1, 0.23, .356 -  -  Date class (DATE/DATETIME): 2017-10-03, 2017-06-13 12:34:03. - - String class (CHAR/VARCHAR) (without quotes): I am a student, a - - NULL value: \N - -## example - -1. Create a Kafka routine load task named test1 for the example_tbl of example_db. Specify group.id and client.id, and automatically consume all partitions by default, with subscriptions starting at the end (OFFSET_END) - ``` - CREATE ROUTINE LOAD example_db.test1 ON example_tbl - COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100) - PROPERTIES - ( - "desired_concurrent_number"="3", - "max_batch_interval" = "20", - "max_batch_rows" = "300000", - "max_batch_size" = "209715200", - "strict_mode" = "false" - ) - FROM KAFKA - ( - "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", - "kafka_topic" = "my_topic", - "property.group.id" = "xxx", - "property.client.id" = "xxx" - ); - ``` - -2. Create a Kafka routine load task named test1 for the example_tbl of example_db. The load task is in strict mode. - - ``` - CREATE ROUTINE LOAD example_db.test1 ON example_tbl - COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), - WHERE k1 > 100 and k2 like "%doris%" - PROPERTIES - ( -     "desired_concurrent_number"="3", -     "max_batch_interval" = "20", -     "max_batch_rows" = "300000", -     "max_batch_size" = "209715200", -     "strict_mode" = "false" - ) - FROM KAFKA - ( -     "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", -     "kafka_topic" = "my_topic", -     "kafka_partitions" = "0,1,2,3", -     "kafka_offsets" = "101,0,0,200" - ); - ``` - -3. load data from Kafka clusters via SSL authentication. Also set the client.id parameter. The load task is in non-strict mode and the time zone is Africa/Abidjan - - ``` - CREATE ROUTINE LOAD example_db.test1 ON example_tbl - COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), - WHERE k1 > 100 and k2 like "%doris%" - PROPERTIES - ( -     "desired_concurrent_number"="3", -     "max_batch_interval" = "20", -     "max_batch_rows" = "300000", -     "max_batch_size" = "209715200", -     "strict_mode" = "false", -     "timezone" = "Africa/Abidjan" - ) - FROM KAFKA - ( -     "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", -     "kafka_topic" = "my_topic", -     "property.security.protocol" = "ssl", -     "property.ssl.ca.location" = "FILE:ca.pem", -     "property.ssl.certificate.location" = "FILE:client.pem", -     "property.ssl.key.location" = "FILE:client.key", -     "property.ssl.key.password" = "abcdefg", -     "property.client.id" = "my_client_id" - ); - ``` - -## keyword - - CREATE, ROUTINE, LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ALTER_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ALTER_EN.md deleted file mode 100644 index a0b9583bb7fea7..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ALTER_EN.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# SHOW ALTER -## Description -This statement is used to show the execution of various modification tasks currently under way. -Grammar: -SHOW ALTER [CLUSTER | TABLE [COLUMN | ROLLUP] [FROM db_name]]; - -Explain: -TABLE COLUMN：Shows the task of alter table column. - Support grammar [WHERE TableName|CreateTime|FinishTime|State] [ORDER BY] [LIMIT] -TABLE ROLLUP: Shows the task of creating or deleting ROLLUP index -If db_name is not specified, use the current default DB -CLUSTER: Show the cluster operation related tasks (only administrators use! To be realized... - -## example -1. Show the task execution of all modified columns of default DB -SHOW ALTER TABLE COLUMN; - -2. Show the last task execution of modified columns of some table -SHOW ALTER TABLE COLUMN WHERE TableName = "table1" ORDER BY CreateTime LIMIT 1; - -3. Show the execution of tasks to create or delete ROLLUP index for specified DB -SHOW ALTER TABLE ROLLUP FROM example_db; - -4. Show cluster operations related tasks (only administrators use! To be realized... -SHOW ALTER CLUSTER; - -## keyword -SHOW,ALTER - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP_EN.md deleted file mode 100644 index 7f72cd18d921cd..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP_EN.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# SHOW BACKUP -## Description -This statement is used to view BACKUP tasks -Grammar: -SHOW BACKUP [FROM db_name] - -Explain: -1. Only the last BACKUP task is saved in Palo. -2. Each column has the following meanings: -JobId: Unique job ID -SnapshotName: The name of the backup -DbName: Subordinate database -State: Current phase -PENDING: The initial state after submitting a job -SNAPSHOTING: In the execution snapshot -UPLOAD_SNAPSHOT: Snapshot completed, ready for upload -UPLOADING: Snapshot uploading -SAVE_META: Save job meta-information as a local file -UPLOAD_INFO: Upload job meta-information -FINISHED: Operation Successful -CANCELLED: Job Failure -Backup Objs: Backup tables and partitions -CreateTime: Task submission time -Snapshot Finished Time: Snapshot completion time -Upload Finished Time: Snapshot Upload Completion Time -FinishedTime: Job End Time -Unfinished Tasks: The unfinished sub-task ID is displayed in the SNAP HOTING and UPLOADING phases -Status: Display failure information if the job fails -Timeout: Job timeout, per second - -## example -1. See the last BACKUP task under example_db. -SHOW BACKUP FROM example_db; - -## keyword -SHOW, BACKUP diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES_EN.md deleted file mode 100644 index 8952f452faed77..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES_EN.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# SHOW DATABASES -## Description -This statement is used to show the currently visible DB -Grammar: -SHOW DATABASES; - -## keyword -SHOW,DATABASES - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DATA_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DATA_EN.md deleted file mode 100644 index d9af60c30489ce..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DATA_EN.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# SHOW DATA -## Description -This statement is used to show the amount of data and the number of replica -Grammar: -SHOW DATA [FROM db_name[.table_name]]; - -Explain: -1. If you do not specify the FROM clause, use the amount of data and the number of replica that shows the current DB subdivided into tables -2. If the FROM clause is specified, the amount of data and the number of replica subdivided into indices under the table is shown. -3. If you want to see the size of individual Partitions, see help show partitions - -## example -1. Display the data volume, replica size, aggregate data volume and aggregate replica count of each table of default DB -SHOW DATA; - -2. Display the subdivision data volume and replica count of the specified table below the specified DB -SHOW DATA FROM example_db.table_name; - -## keyword -SHOW,DATA diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DELETE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DELETE_EN.md deleted file mode 100644 index 2f3d23a72a0325..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DELETE_EN.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# SHOW DELETE -## Description -This statement is used to show successful historical delete tasks performed -Grammar: -SHOW DELETE [FROM db_name] - -## example -1. Show all historical delete tasks for database -SHOW DELETE FROM database; - -## keyword -SHOW,DELETE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES_EN.md deleted file mode 100644 index b0edcbf509b811..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES_EN.md +++ /dev/null @@ -1,29 +0,0 @@ - - - # SHOW DYNAMIC PARTITION TABLES -## description - This statement is used to display all dynamically partitioned table states under the current db - Grammar： - SHOW DYNAMIC PARTITION TABLES [FROM db_name]; - - ## example - 1. Displays all dynamically partitioned table states for the database - SHOW DYNAMIC PARTITION TABLES FROM database; - - ## keyword - SHOW,DYNAMIC,PARTITION,TABLES diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT_EN.md deleted file mode 100644 index 758e3e05a417be..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT_EN.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# SHOW EXPORT -## Description -This statement is used to show the execution of the specified export task -Grammar: -SHOW EXPORT -[FROM both names] -[ -WHERE -[EXPORT_JOB_ID = your_job_id] -[STATE = ["PENDING"|"EXPORTING"|"FINISHED"|"CANCELLED"]] -] -[ORDER BY ...] -[LIMIT limit]; - -Explain: -1) If db_name is not specified, use the current default DB -2) If STATE is specified, the EXPORT state is matched -3) Any column combination can be sorted using ORDER BY -4) If LIMIT is specified, the limit bar matching record is displayed. Otherwise, all of them will be displayed. - -## example -1. Show all export tasks of default DB -SHOW EXPORT; - -2. Show the export tasks of the specified db, sorted in descending order by StartTime -SHOW EXPORT FROM example_db ORDER BY StartTime DESC; - -3. Show the export task of the specified db, state is "exporting" and sorted in descending order by StartTime -SHOW EXPORT FROM example_db WHERE STATE = "exporting" ORDER BY StartTime DESC; - -4. Show the export task of specifying dB and job_id -SHOW EXPORT FROM example_db WHERE EXPORT_JOB_ID = job_id; - -## keyword -SHOW,EXPORT - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW LOAD_EN.md deleted file mode 100644 index da0fd565d513b4..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW LOAD_EN.md +++ /dev/null @@ -1,67 +0,0 @@ - - -# SHOW LOAD -## Description -This statement is used to show the execution of the specified import task -Grammar: -SHOW LOAD -[FROM both names] -[ -WHERE -[LABEL [ = "your_label" | LIKE "label_matcher"]] -[STATE = ["PENDING"|"ETL"|"LOADING"|"FINISHED"|"CANCELLED"|]] -] -[ORDER BY ...] -[LIMIT limit][OFFSET offset]; - -Explain: -1) If db_name is not specified, use the current default DB -2) If you use LABEL LIKE, the label that matches the import task contains the import task of label_matcher -3) If LABEL = is used, the specified label is matched accurately. -4) If STATE is specified, the LOAD state is matched -5) Arbitrary column combinations can be sorted using ORDER BY -6) If LIMIT is specified, the limit bar matching record is displayed. Otherwise, all of them will be displayed. -7) If OFFSET is specified, the query results are displayed from offset. By default, the offset is 0. -8) If broker/mini load is used, the connection in the URL column can be viewed using the following command: - -SHOW LOAD WARNINGS ON 'url' - -## example -1. Show all import tasks of default DB -SHOW LOAD; - -2. Show the import task of the specified db. The label contains the string "2014_01_02", showing the oldest 10 -SHOW LOAD FROM example_db WHERE LABEL LIKE "2014_01_02" LIMIT 10; - -3. Show the import task of the specified db, specify label as "load_example_db_20140102" and sort it in descending order by LoadStartTime -SHOW LOAD FROM example_db WHERE LABEL = "load_example_db_20140102" ORDER BY LoadStartTime DESC; - -4. Show the import task of the specified db, specify label as "load_example_db_20140102" and state as "load", and sort it in descending order by LoadStartTime -SHOW LOAD FROM example_db WHERE LABEL = "load_example_db_20140102" AND STATE = "loading" ORDER BY LoadStartTime DESC; - -5. Show the import task of the specified dB and sort it in descending order by LoadStartTime, and display 10 query results starting with offset 5 -SHOW LOAD FROM example_db ORDER BY LoadStartTime DESC limit 5,10; -SHOW LOAD FROM example_db ORDER BY LoadStartTime DESC limit 10 offset 5; - -6. Small batch import is a command to view the import status -curl --location-trusted -u {user}:{passwd} http://{hostname}:{port}/api/{database}/_load_info?label={labelname} - -## keyword -SHOW,LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS_EN.md deleted file mode 100644 index dde28ea3507689..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS_EN.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# SHOW PARTITIONS -## Description -This statement is used to display partition information -Grammar: -SHOW PARTITIONS FROM [db_name.]table_name [WHERE] [ORDER BY] [LIMIT]; -Explain: -Support filter with following columns: PartitionId,PartitionName,State,Buckets,ReplicationNum, -LastConsistencyCheckTime - -## example -1. Display partition information for the specified table below the specified DB -SHOW PARTITIONS FROM example_db.table_name; - -2. Display information about the specified partition of the specified table below the specified DB -SHOW PARTITIONS FROM example_db.table_name WHERE PartitionName = "p1"; - -3. Display information about the newest partition of the specified table below the specified DB -SHOW PARTITIONS FROM example_db.table_name ORDER BY PartitionId DESC LIMIT 1; - -## keyword -SHOW,PARTITIONS - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY_EN.md deleted file mode 100644 index b0c1cac59a8ee2..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY_EN.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# SHOW PROPERTY -## Description -This statement is used to view user attributes -Grammar: -SHOW PROPERTY [FOR user] [LIKE key] - -## example -1. View the attributes of the jack user -SHOW PROPERTY FOR 'jack' - -2. View Jack user import cluster related properties -SHOW PROPERTY FOR 'jack' LIKE '%load_cluster%' - -## keyword -SHOW, PROPERTY - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES_EN.md deleted file mode 100644 index 4158152b055907..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES_EN.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# SHOW REPOSITORIES -## Description -This statement is used to view the currently created warehouse. -Grammar: -SHOW REPOSITORIES; - -Explain: -1. Each column has the following meanings: -RepoId: Unique Warehouse ID -RepoName: Warehouse name -CreateTime: The first time the warehouse was created -IsReadOnly: Is it a read-only warehouse? -Location: The root directory in the repository for backing up data -Broker: Dependent Broker -ErrMsg: Palo regularly checks the connectivity of the warehouse, and if problems occur, error messages are displayed here. - -## example -1. View the warehouse that has been created: -SHOW REPOSITORIES; - -## keyword -SHOW, REPOSITORY, REPOSITORIES - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE_EN.md deleted file mode 100644 index 2d1adb16fe3aa0..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE_EN.md +++ /dev/null @@ -1,60 +0,0 @@ - - -# SHOW RESTORE -## Description -This statement is used to view RESTORE tasks -Grammar: -SHOW RESTORE [FROM db_name] - -Explain: -1. Palo -20165;- 20445;- 233844;-36817;- 27425RESTORE -21153s; -2. Each column has the following meanings: -JobId: Unique job ID -Label: The name of the backup to be restored -Timestamp: Time version of backup to be restored -DbName: Subordinate database -State: Current phase -PENDING: The initial state after submitting a job -SNAPSHOTING: In the execution snapshot -DOWNLOAD: The snapshot is complete, ready to download the snapshot in the warehouse -DOWNLOADING: Snapshot Download -COMMIT: Snapshot download completed, ready to take effect -COMMITING: In force -FINISHED: Operation Successful -CANCELLED: Job Failure -AllowLoad: Is import allowed on recovery (currently not supported) -ReplicationNum: Specifies the number of replicas recovered -Restore Jobs: Tables and partitions to be restored -CreateTime: Task submission time -MetaPreparedTime: Metadata Readiness Completion Time -Snapshot Finished Time: Snapshot completion time -Download Finished Time: Snapshot download completion time -FinishedTime: Job End Time -Unfinished Tasks: The unfinished sub-task ID is displayed in the SNAP HOTING, DOWNLOADING, and COMMITING phases -Status: Display failure information if the job fails -Timeout: Job timeout, per second - -## example -1. Check the last RESTORE task under example_db. -SHOW RESTORE FROM example_db; - -## keyword -SHOW, RESTORE - diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK_EN.md deleted file mode 100644 index 6737ce01af4753..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK_EN.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# SHOW ROUTINE LOAD TASK -## example - -1. Show sub-task information for a routine import task called test 1. - -SHOW ROUTINE LOAD TASK WHERE JobName = "test1"; - -## keyword -SHOW,ROUTINE,LOAD,TASK diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD_EN.md deleted file mode 100644 index f7914a3cfcf971..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD_EN.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# SHOW ROUTINE LOAD -## example - -1. Show all routine import jobs named test 1 (including stopped or cancelled jobs). The result is one or more lines. - -SHOW ALL ROUTINE LOAD FOR test1; - -2. Show the current running routine import job named test1 - -SHOW ROUTINE LOAD FOR test1; - -3. Display all routine import jobs (including stopped or cancelled jobs) under example_db. The result is one or more lines. - -use example_db; -SHOW ALL ROUTINE LOAD; - -4. Display all running routine import jobs under example_db - -use example_db; -SHOW ROUTINE LOAD; - -5. Display the current running routine import job named test1 under example_db - -SHOW ROUTINE LOAD FOR example_db.test1; - -6. Display all routine import jobs named test1 (including stopped or cancelled jobs) under example_db. The result is one or more lines. - -SHOW ALL ROUTINE LOAD FOR example_db.test1; - -## keyword -SHOW,ROUTINE,LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT_EN.md deleted file mode 100644 index dcd0111527554c..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT_EN.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# SHOW SNAPSHOT -## Description -This statement is used to view existing backups in the warehouse. -Grammar: -SHOW SNAPSHOT ON `repo_name` -[WHERE SNAPSHOT = "snapshot" [AND TIMESTAMP = "backup_timestamp"]]; - -Explain: -1. Each column has the following meanings: -Snapshot: The name of the backup -Timestamp: Time version for backup -Status: If the backup is normal, the OK will be displayed, otherwise the error message will be displayed. - -2. If TIMESTAMP is specified, the following additional information will be displayed: -Database: The name of the database where the backup data belongs -Details: Shows the entire backup data directory and file structure in the form of Json - -'35;'35; example -1. Check the existing backups in warehouse example_repo: -SHOW SNAPSHOT ON example_repo; - -2. View only the backup named backup1 in warehouse example_repo: -SHOW SNAPSHOT ON example_repo WHERE SNAPSHOT = "backup1"; - -2. Check the backup named backup1 in the warehouse example_repo for details of the time version "2018-05-05-15-34-26": -SHOW SNAPSHOT ON example_repo -WHERE SNAPSHOT = "backup1" AND TIMESTAMP = "2018-05-05-15-34-26"; - -## keyword -SHOW, SNAPSHOT diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLES_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLES_EN.md deleted file mode 100644 index 9883f69f7c0442..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLES_EN.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# SHOW TABLES -## Description -This statement is used to show all tables under the current DB -Grammar: -SHOW TABLES; - -## keyword -SHOW,TABLES diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLET_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLET_EN.md deleted file mode 100644 index 4c486f602dce76..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLET_EN.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# SHOW TABLET -## Description -This statement is used to display tablet-related information (for administrators only) -Grammar: -SHOW TABLET -[From [db-uu name]] table U name.; Table U Id] - -## example -1. Display all tablet information in the specified table below the specified DB -SHOW TABLET FROM example_db.table_name; - -2. Display parent level ID information of tablet with specified tablet ID of 10000 -Performance board 10000; - -## keyword -SHOW,TABLET diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION_EN.md deleted file mode 100644 index f6931db7b224dc..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION_EN.md +++ /dev/null @@ -1,79 +0,0 @@ - - -# SHOW TRANSACTION -## description - -This syntax is used to view transaction details for the specified transaction id. - -grammar: - -``` -SHOW TRANSACTION -[FROM db_name] -WHERE id = transaction_id; -``` - -Example return result: - -``` -     TransactionId: 4005 -             Label: insert_8d807d5d-bcdd-46eb-be6d-3fa87aa4952d -       Coordinator: FE: 10.74.167.16 - TransactionStatus: VISIBLE - LoadJobSourceType: INSERT_STREAMING -       PrepareTime: 2020-01-09 14:59:07 -        CommitTime: 2020-01-09 14:59:09 -        FinishTime: 2020-01-09 14:59:09 -            Reason: -ErrorReplicasCount: 0 -        ListenerId: -1 -         TimeoutMs: 300000 -``` - -* TransactionId: transaction id -* Label: the label of the corresponding load job -* Coordinator: the node responsible for transaction coordination -* TransactionStatus: transaction status -    * PREPARE: preparation stage -    * COMMITTED: The transaction was successful, but the data is not visible -    * VISIBLE: The transaction was successful and the data is visible -    * ABORTED: transaction failed -* LoadJobSourceType: The type of the load job. -* PrepareTime: transaction start time -* CommitTime: the time when the transaction was successfully committed -* FinishTime: The time when the data is visible -* Reason: error message -* ErrorReplicasCount: Number of replicas with errors -* ListenerId: the id of the related load job -* TimeoutMs: transaction timeout time in milliseconds - -## example - -1. View the transaction with id 4005: - - SHOW TRANSACTION WHERE ID = 4005; - -2. Specify the db and view the transaction with id 4005: - - SHOW TRANSACTION FROM db WHERE ID = 4005; - -## keyword - - SHOW, TRANSACTION \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD_EN.md deleted file mode 100644 index 31105827fca320..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD_EN.md +++ /dev/null @@ -1,28 +0,0 @@ - - -# STOP ROUTINE LOAD -## example - -1. Stop the routine import job named test 1. - -STOP ROUTINE LOAD FOR test1; - -## keyword -STOP,ROUTINE,LOAD diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/STREAM LOAD_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/STREAM LOAD_EN.md deleted file mode 100644 index 3d04ca9eb29d83..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/STREAM LOAD_EN.md +++ /dev/null @@ -1,186 +0,0 @@ - - -# STREAM LOAD -## description - -NAME - -load data to table in streaming - -SYNOPSIS - -Curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_host:http_port/api/{db}/{table}/_stream_load - -DESCRIPTION - -This statement is used to load data to the specified table. The difference from normal load is that this load method is synchronous load. - -This type of load still guarantees the atomicity of a batch of load tasks, either all data is loaded successfully or all fails. - -This operation also updates the data for the rollup table associated with this base table. - -This is a synchronous operation that returns the results to the user after the entire data load is completed. - -Currently, HTTP chunked and non-chunked uploads are supported. For non-chunked mode, Content-Length must be used to indicate the length of the uploaded content, which ensures data integrity. - -In addition, the user preferably sets the Content of the Expect Header field to 100-continue, which avoids unnecessary data transmission in certain error scenarios. - -OPTIONS - -Users can pass in the load parameters through the Header part of HTTP. - -`label` - -A label that is loaded at one time. The data of the same label cannot be loaded multiple times. Users can avoid the problem of repeated data load by specifying the label. - -Currently Palo internally retains the most recent successful label within 30 minutes. - -`column_separator` - -Used to specify the column separator in the load file. The default is `\t`. If it is an invisible character, you need to add `\x` as a prefix and hexadecimal to indicate the separator. - -For example, the separator `\x01` of the hive file needs to be specified as `-H "column_separator:\x01"` - -`columns` - -used to specify the correspondence between the columns in the load file and the columns in the table. If the column in the source file corresponds exactly to the contents of the table, then it is not necessary to specify the contents of this field. If the source file does not correspond to the table schema, then this field is required for some data conversion. There are two forms of column, one is directly corresponding to the field in the load file, directly using the field name to indicate. - -One is a derived column with the syntax `column_name` = expression. Give a few examples to help understand. - -Example 1: There are three columns "c1, c2, c3" in the table. The three columns in the source file correspond to "c3, c2, c1" at a time; then you need to specify `-H "columns: c3, c2, c1"` - -Example 2: There are three columns in the table, "c1, c2, c3". The first three columns in the source file correspond in turn, but there are more than one column; then you need to specify` -H "columns: c1, c2, c3, xxx"` - -The last column can optionally specify a name for the placeholder. - -Example 3: There are three columns in the table, "year, month, day". There is only one time column in the source file, which is "2018-06-01 01:02:03" format. Then you can specify `-H "columns: col, year = year(col), month=month(col), day=day(col)"` to complete the load. - -`where` - -Used to extract some data. If the user needs to filter out the unwanted data, it can be achieved by setting this option. - -Example 1: load only data larger than k1 column equal to 20180601, then you can specify -H "where: k1 = 20180601" when loading - -`max_filter_ratio` - -The maximum proportion of data that can be filtered (for reasons such as data irregularity). The default is zero tolerance. Data non-standard does not include rows that are filtered out by the where condition. - -`Partitions` - -Used to specify the partition designed for this load. If the user is able to determine the partition corresponding to the data, it is recommended to specify the item. Data that does not satisfy these partitions will be filtered out. - -For example, specify load to p1, p2 partition, `-H "partitions: p1, p2"` - -`Timeout` - -Specifies the timeout for the load. Unit seconds. The default is 600 seconds. The range is from 1 second to 259200 seconds. - -`strict_mode` - -The user specifies whether strict load mode is enabled for this load. The default is enabled. The shutdown mode is `-H "strict_mode: false"`. - -`timezone` - -Specifies the time zone used for this load. The default is East Eight District. This parameter affects all function results related to the time zone involved in the load. - -`exec_mem_limit` - -Memory limit. Default is 2GB. Unit is Bytes. - -RETURN VALUES - -After the load is completed, the related content of this load will be returned in Json format. Current field included - -* `Status`: load status. - - * Success: indicates that the load is successful and the data is visible. - - * Publish Timeout: Indicates that the load job has been successfully Commit, but for some reason it is not immediately visible. Users can be considered successful and do not have to retry load - - * Label Already Exists: Indicates that the Label is already occupied by another job, either the load was successful or it is being loaded. The user needs to use the get label state command to determine the subsequent operations. - - * Other: The load failed, the user can specify Label to retry the job. - -* Message: A detailed description of the load status. When it fails, it will return the specific reason for failure. - -* NumberTotalRows: The total number of rows read from the data stream - -* NumberLoadedRows: The number of data rows loaded this time, only valid when Success - -* NumberFilteredRows: The number of rows filtered by this load, that is, the number of rows with unqualified data quality. - -* NumberUnselectedRows: Number of rows that were filtered by the where condition for this load - -* LoadBytes: The amount of source file data loaded this time - -* LoadTimeMs: Time spent on this load - -* ErrorURL: The specific content of the filtered data, only the first 1000 items are retained - -ERRORS - -You can view the load error details by the following statement: - - ```SHOW LOAD WARNINGS ON 'url'``` - -Where url is the url given by ErrorURL. - -## example - -1. load the data from the local file 'testData' into the table 'testTbl' in the database 'testDb' and use Label for deduplication. Specify a timeout of 100 seconds - - ```Curl --location-trusted -u root -H "label:123" -H "timeout:100" -T testData http://host:port/api/testDb/testTbl/_stream_load``` - -2. load the data in the local file 'testData' into the table of 'testTbl' in the database 'testDb', use Label for deduplication, and load only data with k1 equal to 20180601 -         - ```Curl --location-trusted -u root -H "label:123" -H "where: k1=20180601" -T testData http://host:port/api/testDb/testTbl/_stream_load``` - -3. load the data from the local file 'testData' into the 'testTbl' table in the database 'testDb', allowing a 20% error rate (user is in defalut_cluster) - - ```Curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -T testData http://host:port/api/testDb/testTbl/_stream_load``` - -4. load the data from the local file 'testData' into the 'testTbl' table in the database 'testDb', allow a 20% error rate, and specify the column name of the file (user is in defalut_cluster) - - ```Curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "columns: k2, k1, v1" -T testData http://host:port/api/testDb/testTbl/_stream_load``` - -5. load the data from the local file 'testData' into the p1, p2 partition in the 'testTbl' table in the database 'testDb', allowing a 20% error rate. - - ```Curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "partitions: p1, p2" -T testData http://host:port/api/testDb/testTbl/stream_load``` - -6. load using streaming mode (user is in defalut_cluster) - - ```Seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_stream_load``` - -7. load a table with HLL columns, which can be columns in the table or columns in the data used to generate HLL columns,you can also use hll_empty to supplement columns that are not in the data - - ```Curl --location-trusted -u root -H "columns: k1, k2, v1=hll_hash(k1), v2=hll_empty()" -T testData http://host:port/api/testDb/testTbl/_stream_load``` - -8. load data for strict mode filtering and set the time zone to Africa/Abidjan - - ```Curl --location-trusted -u root -H "strict_mode: true" -H "timezone: Africa/Abidjan" -T testData http://host:port/api/testDb/testTbl/_stream_load``` - -9. load a table with BITMAP columns, which can be columns in the table or a column in the data used to generate BITMAP columns, you can also use bitmap_empty to supplement columns that are not in the data - - ```Curl --location-trusted -u root -H "columns: k1, k2, v1=to_bitmap(k1), v2=bitmap_empty()" -T testData http://host:port/api/testDb/testTbl/_stream_load``` - - -## keyword - - STREAM, LOAD \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/index.rst b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/index.rst deleted file mode 100644 index 92dd5a72cadf92..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -====================== -Data Manipulation -====================== - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/insert_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/insert_EN.md deleted file mode 100644 index 82264499fb6a3e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/insert_EN.md +++ /dev/null @@ -1,103 +0,0 @@ - - -# INSERT -## Description -### Syntax - -``` -INSERT INTO table_name -[ PARTITION (p1, ...)] -[ WITH LABEL label] -[ (column [, ...]) ] -[ [ hint [, ...] ] ] -{ VALUES ( { expression | DEFAULT } [, ...] ) [, ...] | query } -``` - -### Parameters - -> tablet_name: Target table for loading data. It can be in the form of `db_name.table_name`. -> -> partitions: Specifies the partitions to be loaded, with multiple partition names separated by commas. The partitions must exist in `table_name`, -> -> label: Specifies a label for Insert job. -> -> column_name: The specified destination columns must be columns that exists in `table_name`. -> -> expression: The corresponding expression that needs to be assigned to a column. -> -> DEFAULT: Let the corresponding columns use default values -> -> query: A common query whose results are written to the target -> -> hint: Indicators used to indicate `INSERT` execution. ` Both streaming `and default non `streaming'methods use synchronization to complete `INSERT' statement execution -> The non `streaming'mode returns a label after execution to facilitate users to query the imported status through `SHOW LOAD'. - -### Note - -When the `INSERT'statement is currently executed, the default behavior for data that does not conform to the target table is filtering, such as string length. However, for business scenarios where data is not filtered, the session variable `enable_insert_strict'can be set to `true' to ensure that `INSERT'will not be successfully executed when data is filtered out. - -## example - -` The test `table contains two columns `c1', `c2'. - -1. Import a row of data into the `test'table - -``` -INSERT INTO test VALUES (1, 2); -INSERT INTO test (c1, c2) VALUES (1, 2); -INSERT INTO test (c1, c2) VALUES (1, DEFAULT); -INSERT INTO test (c1) VALUES (1); -``` - -The first and second sentences have the same effect. When the target column is not specified, the column order in the table is used as the default target column. -The third and fourth statements express the same meaning, using the default value of `c2'column to complete data import. - -2. Import multiline data into the `test'table at one time - -``` -INSERT INTO test VALUES (1, 2), (3, 2 + 2) -INSERT INTO test (c1, c2) VALUES (1, 2), (3, 2 * 2) -INSERT INTO test (c1) VALUES (1), (3) -Insert in test (C1, C2) values (1, Default), (3, Default) -``` - -The effect of the first and second statements is the same, and two data are imported into the `test'table at one time. -The effect of the third and fourth statements is known, using the default value of the `c2'column to import two data into the `test' table. - - -3. Insert into table `test` with a query stmt. - -``` -INSERT INTO test SELECT * FROM test2 -INSERT INTO test (c1, c2) SELECT * from test2 -``` - -4. Insert into table `test` with specified partition and label - -``` -INSERT INTO test PARTITION(p1, p2) WITH LABEL `label1` SELECT * FROM test2; -INSERT INTO test WITH LABEL `label1` (c1, c2) SELECT * from test2; -``` - -Asynchronous imports are, in fact, encapsulated asynchronously by a synchronous import. Filling in streaming is as efficient as not filling in * execution. - -Since Doris used to import asynchronously, in order to be compatible with the old usage habits, the `INSERT'statement without streaming will still return a label. Users need to view the status of the `label' import job through the `SHOW LOAD command. -##keyword -INSERT diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/BIGINT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/BIGINT_EN.md deleted file mode 100644 index c441911960b98f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/BIGINT_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# BIGINT -## Description -BIGINT -8-byte signed integer, range [-9223372036854775808, 9223372036854775807] - -##keyword -BIGINT diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/BOOLEAN_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/BOOLEAN_EN.md deleted file mode 100644 index 1038fcc899924c..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/BOOLEAN_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# BOOLEAN -## Description -BOOL, BOOLEAN -Like TINYINT, 0 stands for false and 1 for true. - -##keyword -BOOLEAN diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/CHAR_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/CHAR_EN.md deleted file mode 100644 index 66b405b67ac3ad..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/CHAR_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# CHAR -## Description -CHAR(M) -A fixed-length string, M represents the length of a fixed-length string. The range of M is 1-255. - -##keyword -CHAR diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md deleted file mode 100644 index cfdc388dbfe766..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# DATETIME -## Description -DATETIME -Date and time type, value range is ['0000-01-01 00:00:00','9999-12-31 23:59:59']. -The form of printing is'YYYY-MM-DD HH:MM:SS' - -##keyword -DATETIME diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md deleted file mode 100644 index ff42017cf72cfd..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# date -## Description -DATE function -Syntax: -Date -Convert input type to DATE type -date -Date type, the current range of values is ['0000-01-01','9999-12-31'], and the default print form is'YYYYY-MM-DD'. - -## example -mysql> SELECT DATE('2003-12-31 01:02:03'); --> '2003-12-31' - -##keyword -DATE diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/DECIMAL_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/DECIMAL_EN.md deleted file mode 100644 index 1c2509349c0716..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/DECIMAL_EN.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# DECIMAL -## Description -DECIMAL (M [,D]) -High-precision fixed-point, M stands for the total number of significant numbers (precision), D stands for the maximum number of decimal points (scale) -The range of M is [1,27], the range of D is [1,9], in addition, M must be greater than or equal to the value of D. The default value of D is 0. - -##keyword -DECIMAL diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/DOUBLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/DOUBLE_EN.md deleted file mode 100644 index d97f88a5105c56..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/DOUBLE_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# Double -## Description -DOUBLE -8-byte floating point number - -##keyword -DOUBLE \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/FLOAT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/FLOAT_EN.md deleted file mode 100644 index 1a6a8a594ef6ef..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/FLOAT_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# FLOAT -## Description -FLOAT -4-byte floating point number - -##keyword -FLOAT diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/HLL(HyperLogLog)_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/HLL(HyperLogLog)_EN.md deleted file mode 100644 index df869d99a6a941..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/HLL(HyperLogLog)_EN.md +++ /dev/null @@ -1,28 +0,0 @@ - - -#HLL (Hyloglog) -## Description -MARKETING (M) -A variable length string, M represents the length of a variable length string. The range of M is 1-16385. -Users do not need to specify length and default values. Length is controlled within the system according to the aggregation degree of data -And HLL columns can only be queried or used by matching hll_union_agg, hll_raw_agg, hll_cardinality, hll_hash. - -##keyword -High loglog, hll, hyloglog diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/INT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/INT_EN.md deleted file mode 100644 index 2734088f413b6e..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/INT_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# INT -## Description -INT -4-byte signed integer, range [-2147483648, 2147483647] - -##keyword -INT diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/SMALLINT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/SMALLINT_EN.md deleted file mode 100644 index e4bc5e5b100831..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/SMALLINT_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# SMALLINT -## Description -SMALLINT -2-byte signed integer, range [-32768, 32767] - -##keyword -SMALLINT diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/TINYINT_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/TINYINT_EN.md deleted file mode 100644 index e9947c5c6412a8..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/TINYINT_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# TINYINT -## Description -TINYINT -1 byte signed integer, range [-128, 127] - -##keyword -TINYINT diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/VARCHAR_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/VARCHAR_EN.md deleted file mode 100644 index 31f4598196c793..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/VARCHAR_EN.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# VARCHAR -## Description -MARKETING (M) -A variable length string, M represents the length of a variable length string. The range of M is 1-65535. - -##keyword -VARCHAR diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/index.rst b/docs/documentation/en/sql-reference/sql-statements/Data Types/index.rst deleted file mode 100644 index f2ead57ce5559f..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -Data Types -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-statements/Utility/index.rst b/docs/documentation/en/sql-reference/sql-statements/Utility/index.rst deleted file mode 100644 index d00fb23d05ae9d..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Utility/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -============= -Utility -============= - -.. toctree:: - :glob: - - * diff --git a/docs/documentation/en/sql-reference/sql-statements/Utility/util_stmt_EN.md b/docs/documentation/en/sql-reference/sql-statements/Utility/util_stmt_EN.md deleted file mode 100644 index ef9208ce880859..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/Utility/util_stmt_EN.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# Describe -## Description -This statement is used to display schema information for the specified table -Grammar: -DESC [FISH] [dbu name.]table name [ALL]; - -Explain: -If ALL is specified, the schema of all indexes of the table is displayed - -## example - -## keyword -DESCRIBE,DESC diff --git a/docs/documentation/en/sql-reference/sql-statements/index.rst b/docs/documentation/en/sql-reference/sql-statements/index.rst deleted file mode 100644 index b968ec022c7acf..00000000000000 --- a/docs/documentation/en/sql-reference/sql-statements/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -================== -DDL Statements -================== - -.. toctree:: - :hidden: - - Account Management/index - Administration/index - Data Definition/index - Data Manipulation/index - Data Types/index - Utility/index diff --git a/docs/en/README.md b/docs/en/README.md new file mode 100644 index 00000000000000..96e654cab1f32b --- /dev/null +++ b/docs/en/README.md @@ -0,0 +1,8 @@ +--- +home: true +heroImage: /images/doris-logo.png +heroText: Apache Doris +tagline: MPP-based interactive SQL data warehousing for reporting and analysis. +actionText: Get Started → +actionLink: /en/installing/compilation +--- \ No newline at end of file diff --git a/docs/en/administrator-guide/alter-table/alter-table-bitmap-index.md b/docs/en/administrator-guide/alter-table/alter-table-bitmap-index.md new file mode 100644 index 00000000000000..25d6b1f8954979 --- /dev/null +++ b/docs/en/administrator-guide/alter-table/alter-table-bitmap-index.md @@ -0,0 +1,83 @@ +--- +{ + "title": "Bitmap Index", + "language": "en" +} +--- + + + +# Bitmap Index +Users can speed up queries by creating a bitmap index +This document focuses on how to create an index job, as well as some considerations and frequently asked questions when creating an index. + +## Glossary +* bitmap index：a fast data structure that speeds up queries + +## Basic Principles +Creating and droping index is essentially a schema change job. For details, please refer to +[Schema Change](alter-table-schema-change_EN.html)。 + +## Syntax +There are two forms of index creation and modification related syntax, one is integrated with alter table statement, and the other is using separate +create/drop index syntax +1. Create Index + + Please refer to [CREATE INDEX](../../sql-reference/sql-statements/Data%20Definition/CREATE%20INDEX_EN.html) + or [ALTER TABLE](../../sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE_EN.html), + You can also specify a bitmap index when creating a table，Please refer to [CREATE TABLE](../../sql-reference/sql-statements/Data%20Definition/CREATE%20TABLE_EN.html) + +2. Show Index + + Please refer to [SHOW INDEX](../../sql-reference/sql-statements/Administration/SHOW%20INDEX_EN.html) + +3. Drop Index + + Please refer to [DROP INDEX](../../sql-reference/sql-statements/Data%20Definition/DROP%20INDEX_EN.html) or [ALTER TABLE](../../sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE_EN.html) + +## Create Job +Please refer to [Scheam Change](alter-table-schema-change_EN.html) +## View Job +Please refer to [Scheam Change](alter-table-schema-change_EN.html) + +## Cancel Job +Please refer to [Scheam Change](alter-table-schema-change_EN.html) + +## Notice +* Currently only index of bitmap type is supported. +* The bitmap index is only created on a single column. +* Bitmap indexes can be applied to all columns of the `Duplicate` data model and key columns of the `Aggregate` and `Uniq` models. +* The data types supported by bitmap indexes are as follows: + * `TINYINT` + * `SMALLINT` + * `INT` + * `UNSIGNEDINT` + * `BIGINT` + * `CHAR` + * `VARCHAE` + * `DATE` + * `DATETIME` + * `LARGEINT` + * `DECIMAL` + * `BOOL` +* The bitmap index takes effect only in segmentV2. You need to add the following configuration to the configuration file of be + ``` + default_rowset_type=BETA + ``` diff --git a/docs/en/administrator-guide/alter-table/alter-table-rollup.md b/docs/en/administrator-guide/alter-table/alter-table-rollup.md new file mode 100644 index 00000000000000..8dcf9f465880e3 --- /dev/null +++ b/docs/en/administrator-guide/alter-table/alter-table-rollup.md @@ -0,0 +1,188 @@ +--- +{ + "title": "Rollup", + "language": "en" +} +--- + + + +# Rollup + +Users can speed up queries by creating rollup tables. For the concept and usage of Rollup, please refer to [Data + Model, ROLLUP and Prefix Index](../../getting-started/data-model-rollup_EN.md) and + [Rollup and query](../../getting-started/hit-the-rollup_EN.md). + +This document focuses on how to create a Rollup job, as well as some considerations and frequently asked questions about creating a Rollup. + +## Glossary + +* Base Table：When each table is created, it corresponds to a base table. The base table stores the complete data of this table. Rollups are usually created based on the data in the base table (and can also be created from other rollups). +* Index：Materialized index. Rollup or Base Table are both called materialized indexes. +* Transaction：Each import task is a transaction, and each transaction has a unique incrementing Transaction ID. + +## Basic Principles + +The basic process of creating a Rollup is to generate a new Rollup data containing the specified column from the data in the Base table. Among them, two parts of data conversion are needed. One is the conversion of existing historical data, and the other is the conversion of newly arrived imported data during Rollup execution. + +``` ++----------+ +| Load Job | ++----+-----+ + | + | Load job generates both base and rollup index data + | + | +------------------+ +---------------+ + | | Base Index | | Base Index | + +------> New Incoming Data| | History Data | + | +------------------+ +------+--------+ + | | + | | Convert history data + | | + | +------------------+ +------v--------+ + | | Rollup Index | | Rollup Index | + +------> New Incoming Data| | History Data | + +------------------+ +---------------+ +``` + +Before starting the conversion of historical data, Doris will obtain a latest transaction ID. And wait for all import transactions before this Transaction ID to complete. This Transaction ID becomes a watershed. This means that Doris guarantees that all import tasks after the watershed will generate data for the Rollup Index at the same time. In this way, after the historical data conversion is completed, the data of the Rollup and Base tables can be guaranteed to be flush. + +## Create Job + +The specific syntax for creating a Rollup can be found in the description of the Rollup section in the help `HELP ALTER TABLE`. + +The creation of Rollup is an asynchronous process. After the job is submitted successfully, the user needs to use the `SHOW ALTER TABLE ROLLUP` command to view the progress of the job. + +## View Job + +`SHOW ALTER TABLE ROLLUP` You can view rollup jobs that are currently executing or completed. For example: + +``` + JobId: 20037 + TableName: tbl1 + CreateTime: 2019-08-06 15:38:49 + FinishedTime: N/A + BaseIndexName: tbl1 +RollupIndexName: r1 + RollupId: 20038 + TransactionId: 10034 + State: PENDING + Msg: + Progress: N/A + Timeout: 86400 +``` + +* JobId: A unique ID for each Rollup job. +* TableName: The table name of the base table corresponding to Rollup. +* CreateTime: Job creation time. +* FinishedTime: The end time of the job. If it is not finished, "N / A" is displayed. +* BaseIndexName: The name of the source Index corresponding to Rollup. +* RollupIndexName: The name of the Rollup. +* RollupId: The unique ID of the Rollup. +* TransactionId: the watershed transaction ID of the conversion history data. +* State: The phase of the operation. +     * PENDING: The job is waiting in the queue to be scheduled. +     * WAITING_TXN: Wait for the import task before the watershed transaction ID to complete. +     * RUNNING: Historical data conversion. +     * FINISHED: The operation was successful. +     * CANCELLED: The job failed. +* Msg: If the job fails, a failure message is displayed here. +* Progress: operation progress. Progress is displayed only in the RUNNING state. Progress is displayed in M / N. Where N is the total number of copies of Rollup. M is the number of copies of historical data conversion completed. +* Timeout: Job timeout time. Unit of second. + +## Cancel Job + +In the case that the job status is not FINISHED or CANCELLED, you can cancel the Rollup job with the following command: + +`CANCEL ALTER TABLE ROLLUP FROM tbl_name;` + +## Notice + +* A table can have only one Rollup job running at a time. And only one rollup can be created in a job. + +* Rollup operations do not block import and query operations. + +* If a DELETE operation has a Key column in a where condition that does not exist in a Rollup, the DELETE is not allowed. + +    If a Key column does not exist in a Rollup, the DELETE operation cannot delete data from the Rollup, so the data consistency between the Rollup table and the Base table cannot be guaranteed. + +* Rollup columns must exist in the Base table. + +    Rollup columns are always a subset of the Base table columns. Columns that do not exist in the Base table cannot appear. + +* If a rollup contains columns of the REPLACE aggregation type, the rollup must contain all the key columns. + +    Assume the structure of the Base table is as follows: +     +    `` `(k1 INT, k2 INT, v1 INT REPLACE, v2 INT SUM)` `` +     +    If you need to create a Rollup that contains `v1` columns, you must include the` k1`, `k2` columns. Otherwise, the system cannot determine the value of `v1` listed in Rollup. +     +    Note that all Value columns in the Unique data model table are of the REPLACE aggregation type. +     +* Rollup of the DUPLICATE data model table, you can specify the DUPLICATE KEY of the rollup. + +    The DUPLICATE KEY in the DUPLICATE data model table is actually sorted. Rollup can specify its own sort order, but the sort order must be a prefix of the Rollup column order. If not specified, the system will check if the Rollup contains all sort columns of the Base table, and if it does not, it will report an error. For example: +     +    Base table structure: `(k1 INT, k2 INT, k3 INT) DUPLICATE KEY (k1, k2)` +     +    Rollup can be: `(k2 INT, k1 INT) DUPLICATE KEY (k2)` + +* Rollup does not need to include partitioned or bucket columns for the Base table. + +## FAQ + +* How many rollups can a table create + +    There is theoretically no limit to the number of rollups a table can create, but too many rollups can affect import performance. Because when importing, data will be generated for all rollups at the same time. At the same time, Rollup will take up physical storage space. Usually the number of rollups for a table is less than 10. +     +* Rollup creation speed + +    Rollup creation speed is currently estimated at about 10MB / s based on the worst efficiency. To be conservative, users can set the timeout for jobs based on this rate. + +* Submitting job error `Table xxx is not stable. ...` + +    Rollup can start only when the table data is complete and unbalanced. If some data shard copies of the table are incomplete, or if some copies are undergoing an equalization operation, the submission is rejected. +     +    Whether the data shard copy is complete can be checked with the following command: +     +    ```ADMIN SHOW REPLICA STATUS FROM tbl WHERE STATUS! =" OK ";``` +     +    If a result is returned, there is a problem with the copy. These problems are usually fixed automatically by the system. You can also use the following commands to repair this table first: +     +    ```ADMIN REPAIR TABLE tbl1; ``` +     +    You can check if there are running balancing tasks with the following command: +     +    ```SHOW PROC" / cluster_balance / pending_tablets ";``` +     +    You can wait for the balancing task to complete, or temporarily disable the balancing operation with the following command: + + ```ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");``` + +## Configurations + +### FE Configurations + +* `alter_table_timeout_second`：The default timeout for the job is 86400 seconds. + +### BE Configurations + +* `alter_tablet_worker_count`：Number of threads used to perform historical data conversion on the BE side. The default is 3. If you want to speed up the rollup job, you can increase this parameter appropriately and restart the BE. But too many conversion threads can cause increased IO pressure and affect other operations. This thread is shared with the Schema Change job. diff --git a/docs/en/administrator-guide/alter-table/alter-table-schema-change.md b/docs/en/administrator-guide/alter-table/alter-table-schema-change.md new file mode 100644 index 00000000000000..9424e0ea5076b8 --- /dev/null +++ b/docs/en/administrator-guide/alter-table/alter-table-schema-change.md @@ -0,0 +1,231 @@ +--- +{ + "title": "Scheam Change", + "language": "en" +} +--- + + + +# Scheam Change + +Users can modify the schema of existing tables through the Scheam Change operation. Doris currently supports the following modifications: + +* Add and delete columns +* Modify column type +* Adjust column order +* Add and modify Bloom Filter +* Add and delete bitmap index + +This document mainly describes how to create a Scheam Change job, as well as some considerations and frequently asked questions about Scheam Change. +## Glossary + +* Base Table：When each table is created, it corresponds to a base table. The base table stores the complete data of this table. Rollups are usually created based on the data in the base table (and can also be created from other rollups). +* Index：Materialized index. Rollup or Base Table are both called materialized indexes. +* Transaction：Each import task is a transaction, and each transaction has a unique incrementing Transaction ID. +* Rollup：Roll-up tables based on base tables or other rollups. + +## Basic Principles + +The basic process of executing a Schema Change is to generate a copy of the index data of the new schema from the data of the original index. Among them, two parts of data conversion are required. One is the conversion of existing historical data, and the other is the conversion of newly arrived imported data during the execution of Schema Change. +``` ++----------+ +| Load Job | ++----+-----+ + | + | Load job generates both origin and new index data + | + | +------------------+ +---------------+ + | | Origin Index | | Origin Index | + +------> New Incoming Data| | History Data | + | +------------------+ +------+--------+ + | | + | | Convert history data + | | + | +------------------+ +------v--------+ + | | New Index | | New Index | + +------> New Incoming Data| | History Data | + +------------------+ +---------------+ +``` + +Before starting the conversion of historical data, Doris will obtain a latest transaction ID. And wait for all import transactions before this Transaction ID to complete. This Transaction ID becomes a watershed. This means that Doris guarantees that all import tasks after the watershed will generate data for both the original Index and the new Index. In this way, when the historical data conversion is completed, the data in the new Index can be guaranteed to be complete. +## Create Job + +The specific syntax for creating a Scheam Change can be found in the description of the Scheam Change section in the help `HELP ALTER TABLE`. + +The creation of Scheam Change is an asynchronous process. After the job is submitted successfully, the user needs to view the job progress through the `SHOW ALTER TABLE COLUMN` command. +## View Job + +`SHOW ALTER TABLE COLUMN` You can view the Schema Change jobs that are currently executing or completed. When multiple indexes are involved in a Schema Change job, the command displays multiple lines, each corresponding to an index. For example: + +``` + JobId: 20021 + TableName: tbl1 + CreateTime: 2019-08-05 23:03:13 + FinishTime: 2019-08-05 23:03:42 + IndexName: tbl1 + IndexId: 20022 +OriginIndexId: 20017 +SchemaVersion: 2:792557838 +TransactionId: 10023 + State: FINISHED + Msg: + Progress: N/A + Timeout: 86400 +``` + +* JobId: A unique ID for each Schema Change job. +* TableName: The table name of the base table corresponding to Schema Change. +* CreateTime: Job creation time. +* FinishedTime: The end time of the job. If it is not finished, "N / A" is displayed. +* IndexName: The name of an Index involved in this modification. +* IndexId: The unique ID of the new Index. +* OriginIndexId: The unique ID of the old Index. +* SchemaVersion: Displayed in M: N format. M is the version of this Schema Change, and N is the corresponding hash value. With each Schema Change, the version is incremented. +* TransactionId: the watershed transaction ID of the conversion history data. +* State: The phase of the operation. +    * PENDING: The job is waiting in the queue to be scheduled. +    * WAITING_TXN: Wait for the import task before the watershed transaction ID to complete. +    * RUNNING: Historical data conversion. +    * FINISHED: The operation was successful. +    * CANCELLED: The job failed. +* Msg: If the job fails, a failure message is displayed here. +* Progress: operation progress. Progress is displayed only in the RUNNING state. Progress is displayed in M ​​/ N. Where N is the total number of copies involved in the Schema Change. M is the number of copies of historical data conversion completed. +* Timeout: Job timeout time. Unit of second. + +## Cancel Job + +In the case that the job status is not FINISHED or CANCELLED, you can cancel the Schema Change job with the following command: +`CANCEL ALTER TABLE COLUMN FROM tbl_name;` + +## Best Practice + +Schema Change can make multiple changes to multiple indexes in one job. For example: +Source Schema： + +``` ++-----------+-------+------+------+------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++-----------+-------+------+------+------+---------+-------+ +| tbl1 | k1 | INT | No | true | N/A | | +| | k2 | INT | No | true | N/A | | +| | k3 | INT | No | true | N/A | | +| | | | | | | | +| rollup2 | k2 | INT | No | true | N/A | | +| | | | | | | | +| rollup1 | k1 | INT | No | true | N/A | | +| | k2 | INT | No | true | N/A | | ++-----------+-------+------+------+------+---------+-------+ +``` + +You can add a row k4 to both rollup1 and rollup2 by adding the following k5 to rollup2: +``` +ALTER TABLE tbl1 +ADD COLUMN k4 INT default "1" to rollup1, +ADD COLUMN k4 INT default "1" to rollup2, +ADD COLUMN k5 INT default "1" to rollup2; +``` + +When completion, the Schema becomes: + +``` ++-----------+-------+------+------+------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++-----------+-------+------+------+------+---------+-------+ +| tbl1 | k1 | INT | No | true | N/A | | +| | k2 | INT | No | true | N/A | | +| | k3 | INT | No | true | N/A | | +| | k4 | INT | No | true | 1 | | +| | k5 | INT | No | true | 1 | | +| | | | | | | | +| rollup2 | k2 | INT | No | true | N/A | | +| | k4 | INT | No | true | 1 | | +| | k5 | INT | No | true | 1 | | +| | | | | | | | +| rollup1 | k1 | INT | No | true | N/A | | +| | k2 | INT | No | true | N/A | | +| | k4 | INT | No | true | 1 | | ++-----------+-------+------+------+------+---------+-------+ +``` + +As you can see, the base table tbl1 also automatically added k4, k5 columns. That is, columns added to any rollup are automatically added to the Base table. + +At the same time, columns that already exist in the Base table are not allowed to be added to Rollup. If you need to do this, you can re-create a Rollup with the new columns and then delete the original Rollup. +## Notice + +* Only one Schema Change job can be running on a table at a time. + +* Schema Change operation does not block import and query operations. + +* The partition column and bucket column cannot be modified. + +* If there is a value column aggregated by REPLACE in the schema, the Key column is not allowed to be deleted. + +     If the Key column is deleted, Doris cannot determine the value of the REPLACE column. +     +     All non-Key columns of the Unique data model table are REPLACE aggregated. +     +* When adding a value column whose aggregation type is SUM or REPLACE, the default value of this column has no meaning to historical data. + +     Because the historical data has lost the detailed information, the default value cannot actually reflect the aggregated value. +     +* When modifying the column type, fields other than Type need to be completed according to the information on the original column. + +     If you modify the column `k1 INT SUM NULL DEFAULT" 1 "` as type BIGINT, you need to execute the following command: + + ```ALTER TABLE tbl1 MODIFY COLUMN `k1` BIGINT SUM NULL DEFAULT "1"; ``` + + Note that in addition to the new column types, such as the aggregation mode, Nullable attributes, and default values must be completed according to the original information. + +* Modifying column names, aggregation types, nullable attributes, default values, and column comments is not supported. + +## FAQ + +* the execution speed of Schema Change + + At present, the execution speed of Schema Change is estimated to be about 10MB / s according to the worst efficiency. To be conservative, users can set the timeout for jobs based on this rate. + +* Submit job error `Table xxx is not stable. ...` + + Schema Change can only be started when the table data is complete and unbalanced. If some data shard copies of the table are incomplete, or if some copies are undergoing an equalization operation, the submission is rejected. +      + Whether the data shard copy is complete can be checked with the following command: + ```ADMIN SHOW REPLICA STATUS FROM tbl WHERE STATUS != "OK";``` + + If a result is returned, there is a problem with the copy. These problems are usually fixed automatically by the system. You can also use the following commands to repair this table first: + ```ADMIN REPAIR TABLE tbl1;``` + + You can check if there are running balancing tasks with the following command: + + ```SHOW PROC "/cluster_balance/pending_tablets";``` + + You can wait for the balancing task to complete, or temporarily disable the balancing operation with the following command: + + ```ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");``` + +## Configurations + +### FE Configurations + +* `alter_table_timeout_second`：The default timeout for the job is 86400 seconds. + +### BE Configurations + +* `alter_tablet_worker_count`：Number of threads used to perform historical data conversion on the BE side. The default is 3. If you want to speed up the Schema Change job, you can increase this parameter appropriately and restart the BE. But too many conversion threads can cause increased IO pressure and affect other operations. This thread is shared with the Rollup job. diff --git a/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md b/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md new file mode 100644 index 00000000000000..0c363c9700273e --- /dev/null +++ b/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md @@ -0,0 +1,241 @@ +--- +{ + "title": "Temporary partition", + "language": "en" +} +--- + + + +# Temporary partition + +Since version 0.12, Doris supports temporary partitioning. + +A temporary partition belongs to a partitioned table. Only partitioned tables can create temporary partitions. + +## Rules + +* The partition columns of the temporary partition is the same as the formal partition and cannot be modified. +* The partition ranges of all temporary partitions of a table cannot overlap, but the ranges of temporary partitions and formal partitions can overlap. +* The partition name of the temporary partition cannot be the same as the formal partitions and other temporary partitions. + +## Supported operations + +The temporary partition supports add, delete, and replace operations. + +### Add temporary partition + +You can add temporary partitions to a table with the `ALTER TABLE ADD TEMPORARY PARTITION` statement: + +``` +ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN ("2020-02-01"); + +ALTER TABLE tbl2 ADD TEMPORARY PARTITION tp1 VALUES [("2020-01-01"), ("2020-02-01")); + +ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN ("2020-02-01") +("in_memory" = "true", "replication_num" = "1") +DISTRIBUTED BY HASH (k1) BUCKETS 5; +``` + +See `HELP ALTER TABLE;` for more help and examples. + +Some instructions for adding operations: + +* Adding a temporary partition is similar to adding a formal partition. The partition range of the temporary partition is independent of the formal partition. +* Temporary partition can independently specify some attributes. Includes information such as the number of buckets, the number of replicas, whether it is a memory table, or the storage medium. + +### Delete temporary partition + +A table's temporary partition can be dropped with the `ALTER TABLE DROP TEMPORARY PARTITION` statement: + +``` +ALTER TABLE tbl1 DROP TEMPORARY PARTITION tp1; +``` + +See `HELP ALTER TABLE;` for more help and examples. + +Some instructions for the delete operation: + +* Deleting the temporary partition will not affect the data of the formal partition. + +### Replace partition + +You can replace formal partitions of a table with temporary partitions with the `ALTER TABLE REPLACE PARTITION` statement. + +``` +ALTER TABLE tbl1 REPLACE PARTITION (p1) WITH TEMPORARY PARTITION (tp1); + +ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1, tp2, tp3); + +ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1, tp2) +PROPERTIES ( +    "strict_range" = "false", +    "use_temp_partition_name" = "true" +); +``` + +See `HELP ALTER TABLE;` for more help and examples. + +The replace operation has two special optional parameters: + +1. `strict_range` + + The default is true. When this parameter is true, the range union of all formal partitions to be replaced needs to be the same as the range union of the temporary partitions to be replaced. When set to false, you only need to ensure that the range between the new formal partitions does not overlap after replacement. Here are some examples: + + * Example 1 + + Range of partitions p1, p2, p3 to be replaced (=> union): + + ``` + (10, 20), [20, 30), [40, 50) => [10, 30), [40, 50) + ``` + + Replace the range of partitions tp1, tp2 (=> union): + + ``` + (10, 30), [40, 45), [45, 50) => [10, 30), [40, 50) + ``` + + The union of ranges is the same, so you can use tp1 and tp2 to replace p1, p2, p3. + + * Example 2 + + Range of partition p1 to be replaced (=> union): + + ``` + (10, 50) => [10, 50) + ``` + + Replace the range of partitions tp1, tp2 (=> union): + + ``` + (10, 30), [40, 50) => [10, 30), [40, 50) + ``` + + The union of ranges is not the same. If `strict_range` is true, you cannot use tp1 and tp2 to replace p1. If false, and the two partition ranges `[10, 30), [40, 50)` and the other formal partitions do not overlap, they can be replaced. + +2. `use_temp_partition_name` + + The default is false. When this parameter is false, and the number of partitions to be replaced is the same as the number of replacement partitions, the name of the formal partition after the replacement remains unchanged. If true, after replacement, the name of the formal partition is the name of the replacement partition. Here are some examples: + + * Example 1 + + ``` + ALTER TABLE tbl1 REPLACE PARTITION (p1) WITH TEMPORARY PARTITION (tp1); + ``` + + `use_temp_partition_name` is false by default. After replacement, the partition name is still p1, but the related data and attributes are replaced with tp1. + + If `use_temp_partition_name` is true by default, the name of the partition is tp1 after replacement. The p1 partition no longer exists. + + * Example 2 + + ``` + ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1); + ``` + + `use_temp_partition_name` is false by default, but this parameter is invalid because the number of partitions to be replaced and the number of replacement partitions are different. After the replacement, the partition name is tp1, and p1 and p2 no longer exist. + +Some instructions for the replacement operation: + +* After the partition is replaced successfully, the replaced partition will be deleted and cannot be recovered. + +## Load and query of temporary partitions + +Users can load data into temporary partitions or specify temporary partitions for querying. + +1. Load temporary partition + + The syntax for specifying a temporary partition is slightly different depending on the load method. Here is a simple illustration through an example: + + ``` + INSERT INTO tbl TEMPORARY PARTITION (tp1, tp2, ...) SELECT .... + ``` + + ``` + curl --location-trusted -u root: -H "label: 123" -H "temporary_partition: tp1, tp2, ..." -T testData http: // host: port / api / testDb / testTbl / _stream_load + ``` + + ``` + LOAD LABEL example_db.label1 + ( + DATA INFILE ("hdfs: // hdfs_host: hdfs_port / user / palo / data / input / file") + INTO TABLE `my_table` + TEMPORARY PARTITION (tp1, tp2, ...) + ... + ) + WITH BROKER hdfs ("username" = "hdfs_user", "password" = "hdfs_password"); + ``` + + ``` + CREATE ROUTINE LOAD example_db.test1 ON example_tbl + COLUMNS (k1, k2, k3, v1, v2, v3 = k1 * 100), + TEMPORARY PARTITIONS (tp1, tp2, ...), + WHERE k1> 100 + PROPERTIES + (...) + FROM KAFKA + (...); + ``` + +2. Query the temporary partition + + ``` + SELECT ... FROM + tbl1 TEMPORARY PARTITION (tp1, tp2, ...) + JOIN + tbl2 TEMPORARY PARTITION (tp1, tp2, ...) + ON ... + WHERE ...; + ``` + +## Relationship to other operations + +### DROP + +* After using the `DROP` operation to directly drop the database or table, you can recover the database or table (within a limited time) through the `RECOVER` command, but the temporary partition will not be recovered. +* After the formal partition is dropped using the `ALTER` command, the partition can be recovered by the `RECOVER` command (within a limited time). Operating a formal partition is not related to a temporary partition. +* After the temporary partition is dropped using the `ALTER` command, the temporary partition cannot be recovered through the `RECOVER` command. + +### TRUNCATE + +* Use the `TRUNCATE` command to empty the table. The temporary partition of the table will be deleted and cannot be recovered. +* When using `TRUNCATE` command to empty the formal partition, it will not affect the temporary partition. +* You cannot use the `TRUNCATE` command to empty the temporary partition. + +### ALTER + +* When the table has a temporary partition, you cannot use the `ALTER` command to perform Schema Change, Rollup, etc. on the table. +* You cannot add temporary partitions to a table while the table is undergoing a alter operation. + + +## Best Practices + +1. Atomic overwrite + + In some cases, the user wants to be able to rewrite the data of a certain partition, but if it is dropped first and then loaded, there will be a period of time when the data cannot be seen. This is, the user can first create a corresponding temporary partition, load new data into the temporary partition, and then replace the original partition atomically through the `REPLACE` operation to achieve the purpose. +     +2. Modify the number of buckets + + In some cases, the user used an inappropriate number of buckets when creating a partition. The user can first create a temporary partition corresponding to the partition range and specify a new number of buckets. Then use the `INSERT INTO` command to load the data of the formal partition into the temporary partition. Through the replacement operation, the original partition is replaced atomically to achieve the purpose. +     +3. Merge or split partitions + + In some cases, users want to modify the range of partitions, such as merging two partitions, or splitting a large partition into multiple smaller partitions. Then the user can first create temporary partitions corresponding to the merged or divided range, and then load the data of the formal partition into the temporary partition through the `INSERT INTO` command. Through the replacement operation, the original partition is replaced atomically to achieve the purpose. \ No newline at end of file diff --git a/docs/en/administrator-guide/backup-restore.md b/docs/en/administrator-guide/backup-restore.md new file mode 100644 index 00000000000000..833505074b520f --- /dev/null +++ b/docs/en/administrator-guide/backup-restore.md @@ -0,0 +1,186 @@ +--- +{ + "title": "Backup and Recovery", + "language": "en" +} +--- + + + +# Backup and Recovery + +Doris supports the backup of current data in the form of files to remote storage systems via broker. The data can then be restored from the remote storage system to any Doris cluster by the restore command. With this feature, Doris can support regular snapshot backups of data. It can also be used to migrate data between different clusters. + +This feature requires Doris version 0.8.2+ + +Using this function, brokers corresponding to remote storage need to be deployed. Such as BOS, HDFS, etc. You can view the currently deployed broker through `SHOW BROKER;` + +## Brief Principle Description + +### Backup + +The backup operation is to upload the data of the specified table or partition directly to the remote warehouse in the form of files stored by Doris for storage. When a user submits a Backup request, the following actions will be done within the system: + +1. Snapshot and snapshot upload + + The snapshot phase takes a snapshot of the specified table or partition data file. Later, backups are all snapshots. After the snapshot, changes to tables, imports, and other operations no longer affect the results of the backup. Snapshots only produce a hard link to the current data file, which takes very little time. Once the snapshots are completed, they are uploaded one by one. Snapshot upload is done concurrently by each Backend. + +2. Metadata preparation and upload + + After the data file snapshot is uploaded, Frontend first writes the corresponding metadata to the local file, and then uploads the local metadata file to the remote warehouse through broker. Finish the final backup job. + +### Restore + +Recovery operations need to specify a backup that already exists in a remote repository, and then restore the backup content to the local cluster. When a user submits a Restore request, the following actions will be done within the system: + +1. Create corresponding metadata locally + + This step starts by creating structures such as restoring the corresponding table partitions in the local cluster. When created, the table is visible, but not accessible. + +2. Local snapshot + + This step is to take a snapshot of the table created in the previous step. This is actually an empty snapshot (because the tables just created have no data), and its main purpose is to generate the corresponding snapshot directory on the Backend for receiving the snapshot files downloaded from the remote repository later. + +3. Download snapshots + + The snapshot files in the remote warehouse are downloaded to the corresponding snapshot directory generated in the previous step. This step is done concurrently by each backend. + +4. Effective snapshot + + When the snapshot download is complete, we map each snapshot to the metadata of the current local table. These snapshots are then reloaded to take effect and complete the final recovery operation. + +## Best Practices + +### Backup + +We currently support full backup at the minimum partition granularity (incremental backup may be supported in future versions). If data need to be backed up regularly, first of all, it is necessary to plan the partition and bucket allocation of tables reasonably, such as partitioning according to time. Then in the subsequent run process, periodic data backup is performed according to partition granularity. + +### Data migration + +Users can first backup the data to the remote warehouse, and then restore the data to another cluster through the remote warehouse to complete data migration. Because data backup is done in the form of snapshots, new imported data after the snapshot phase of the backup job will not be backed up. Therefore, after the snapshot is completed, the data imported on the original cluster needs to be imported on the new cluster as well until the recovery job is completed. + +It is suggested that the new and old clusters be imported in parallel for a period of time after the migration is completed. After completing data and business correctness checks, the business is migrated to the new cluster. + +## Highlights + +1. Backup and recovery-related operations are currently only allowed to be performed by users with ADMIN privileges. +2. Within a database, only one backup or recovery job is allowed to be performed. +3. Both backup and recovery support the operation at the minimum partition level. When the table has a large amount of data, it is recommended to perform partition-by-partition to reduce the cost of failed retries. +4. Because backup and recovery operations, the operation is the actual data files. So when there are too many fragments of a table or too many small versions of a fragment, it may take a long time to backup or restore even if the total amount of data is very small. Users can estimate job execution time by `SHOW PARTITIONS FROM table_name;`, and `SHOW TABLET FROM table_name;`, viewing the number of partitions and the number of file versions of each partition. The number of files has a great impact on the execution time of the job, so it is suggested that the partition buckets should be planned reasonably in order to avoid excessive partitioning. +5. When viewing the job status through `SHOW BACKUP` or `SHOW RESTORE`. It is possible to see an error message in the `TaskErrMsg` column. But as long as the `State` column does not +`CANCELLED`, that means the job is still going on. These Tasks may succeed in retrying. Of course, some Task errors can also directly lead to job failure. +6. If the recovery operation is a coverage operation (specifying the recovery data to an existing table or partition), then starting from the `COMMIT` phase of the recovery operation, the data covered on the current cluster may not be restored. At this time, if the recovery operation fails or is cancelled, it may cause the previous data to be damaged and inaccessible. In this case, the recovery operation can only be performed again and wait for the job to complete. Therefore, we recommend that if it is not necessary, try not to use coverage to recover data unless it is confirmed that the current data is no longer in use. + +## Relevant orders + +The commands related to the backup recovery function are as follows. The following commands, you can use `help cmd;'to view detailed help after connecting Doris through mysql-client. + +1. CREATE REPOSITORY + + Create a remote warehouse Path for backup or recovery. + +1. BACKUP + + Perform a backup operation. + +3. SHOW BACKUP + + View the execution of the last backup job, including: + + * JobId: ID of this backup job. + * SnapshotName: User-specified name of this backup job (Label). + * DbName: The database corresponding to the backup job. + * State: The current stage of the backup job: + * PENDING: The initial state of the job. + * SNAPSHOTING: Snapshot operation is in progress. + * UPLOAD_SNAPSHOT: The snapshot is over and ready to upload. + * UPLOADING: Uploading snapshots. + * SAVE_META: Metadata files are being generated locally. + * UPLOAD_INFO: Upload metadata files and information for this backup job. + * FINISHED: The backup is complete. + * CANCELLED: Backup failed or cancelled. + * Backup Objs: List of tables and partitions involved in this backup. + * CreateTime: Job creation time. + * Snapshot Finished Time: Snapshot completion time. + * Upload Finished Time: Snapshot upload completion time. + * FinishedTime: The completion time of this assignment. + * Unfinished Tasks: In the `SNAPSHOTTING', `UPLOADING'and other stages, there will be multiple sub-tasks at the same time, the current stage shown here, the task ID of the unfinished sub-tasks. + * TaskErrMsg: If there is a sub-task execution error, the error message corresponding to the sub-task will be displayed here. + * Status: It is used to record some status information that may appear during the whole operation. + * Timeout: The timeout time of a job in seconds. + +4. SHOW SNAPSHOT + + View the backup that already exists in the remote warehouse. + + * Snapshot: The name of the backup specified at the time of backup (Label). + * Timestamp: Backup timestamp. + * Status: Is the backup normal? + + If the where clause is specified after `SHOW SNAPSHOT', more detailed backup information can be displayed. + + * Database: The database corresponding to backup. + * Details: Shows the complete data directory structure of the backup. + +5. RESTOR + + Perform a recovery operation. + +6. SHOW RESTORE + + View the execution of the last restore job, including: + + * JobId: ID of this resumption job. + * Label: The name of the backup in the user-specified warehouse (Label). + * Timestamp: The timestamp for backup in a user-specified warehouse. + * DbName: Restore the database corresponding to the job. + * State: The current stage of the recovery operation: + * PENDING: The initial state of the job. + * SNAPSHOTING: A snapshot of a new local table is in progress. + * DOWNLOAD: The download snapshot task is being sent. + * DOWNLOADING: Snapshot is downloading. + * COMMIT: Prepare to take effect the downloaded snapshot. + * COMMITTING: The downloaded snapshot is in effect. + * FINISHED: Recovery is complete. + * CANCELLED: Recovery failed or cancelled. + * AllowLoad: Is import allowed during recovery? + * ReplicationNum: Restores the specified number of copies. + * Restore Objs: List of tables and partitions involved in this recovery. + * CreateTime: Job creation time. + * MetaPreparedTime: Completion time of local metadata generation. + * Snapshot Finished Time: Local snapshot completion time. + * Download Finished Time: The download completion time of the remote snapshot. + * FinishedTime: The completion time of this assignment. + * Unfinished Tasks: In the `SNAPSHOTTING`, `DOWNLOADING`, `COMMITTING`, and other stages, there will be multiple sub-tasks at the same time, the current stage shown here, the task ID of the unfinished sub-tasks. + * TaskErrMsg: If there is a sub-task execution error, the error message corresponding to the sub-task will be displayed here. + * Status: It is used to record some status information that may appear during the whole operation. + * Timeout: The timeout time of a job in seconds. + +7. CANCEL BACKUP + + Cancel the backup job currently being performed. + +8. CANCEL RESTORE + + Cancel the recovery job currently being performed. + +9. DROP REPOSITORY + + Delete the created remote warehouse. Delete the warehouse, just delete the mapping of the warehouse in Doris, will not delete the actual warehouse data. diff --git a/docs/en/administrator-guide/broker.md b/docs/en/administrator-guide/broker.md new file mode 100644 index 00000000000000..d45d2ea1adda99 --- /dev/null +++ b/docs/en/administrator-guide/broker.md @@ -0,0 +1,293 @@ +--- +{ + "title": "Broker", + "language": "en" +} +--- + + + +# Broker + +Broker is an optional process in the Doris cluster. It is mainly used to support Doris to read and write files or directories on remote storage, such as HDFS, BOS, and AFS. + +Broker provides services through an RPC service port. It is a stateless JVM process that is responsible for encapsulating some POSIX-like file operations for read and write operations on remote storage, such as open, pred, pwrite, and so on. +In addition, the Broker does not record any other information, so the connection information, file information, permission information, and so on stored remotely need to be passed to the Broker process in the RPC call through parameters in order for the Broker to read and write files correctly . + +Broker only acts as a data channel and does not participate in any calculations, so it takes up less memory. Usually one or more Broker processes are deployed in a Doris system. And the same type of Broker will form a group and set a ** Broker name **. + +Broker's position in the Doris system architecture is as follows: + +``` ++----+ +----+ +| FE | | BE | ++-^--+ +--^-+ + | | + | | ++-v---------v-+ +| Broker | ++------^------+ + | + | ++------v------+ +|HDFS/BOS/AFS | ++-------------+ +``` + +This document mainly introduces the parameters that Broker needs when accessing different remote storages, such as connection information, +authorization information, and so on. + +## Supported Storage System + +Different types of brokers support different different storage systems。 + +1. Community HDFS + + * Support simple authentication access + * Support kerberos authentication access + * Support HDFS HA mode access + +2. Baidu HDFS / AFS (not supported by open source version) + + * Support UGI simple authentication access + +3. Baidu Object Storage BOS (not supported by open source version) + + * Support AK / SK authentication access + +## Function provided by Broker + +1. Broker Load + + The Broker Load function reads the file data on the remote storage through the Broker process and imports it into Doris. Examples are as follows: + + ``` + LOAD LABEL example_db.label6 + ( + DATA INFILE("bos://my_bucket/input/file") + INTO TABLE `my_table` + ) + WITH BROKER "broker_name" + ( + "bos_endpoint" = "http://bj.bcebos.com", + "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "bos_secret_accesskey" = "yyyyyyyyyyyyyyyyyyyy" + ) + ``` + + `WITH BROKER` and following Property Map are used to provide Broker's related information. + +2. Export + + The Export function export the data stored in Doris to a file stored in remote storage in text format through Broker process. Examples are as follows: + + ``` + EXPORT TABLE testTbl + TO "hdfs://hdfs_host:port/a/b/c" + WITH BROKER "broker_name" + ( + "username" = "xxx", + "password" = "yyy" + ); + ``` + + `WITH BROKER` and following Property Map are used to provide Broker's related information. + +3. Create Repository + + When users need to use the backup and restore function, they need to first create a "repository" with the `CREATE REPOSITORY` command,and the broker metadata and related information are recorded in the warehouse metadata. + Subsequent backup and restore operations will use Broker to back up data to this warehouse, or read data from this warehouse to restore to Doris. Examples are as follows: + + ``` + CREATE REPOSITORY `bos_repo` + WITH BROKER `broker_name` + ON LOCATION "bos://doris_backup" + PROPERTIES + ( + "bos_endpoint" = "http://gz.bcebos.com", + "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", + "bos_secret_accesskey" = "70999999999999de274d59eaa980a" + ); + ``` + + `WITH BROKER` and following Property Map are used to provide Broker's related information. + + +## Broker Information + +Broker information includes two parts: ** Broker name ** and ** Certification information **. The general syntax is as follows: + +``` +WITH BROKER "broker_name" +( + "username" = "xxx", + "password" = "yyy", + "other_prop" = "prop_value", + ... +); +``` + +### Broker Name + +Usually the user needs to specify an existing Broker Name through the `WITH BROKER" broker_name "` clause in the operation command. +Broker Name is a name that the user specifies when adding a Broker process through the ALTER SYSTEM ADD BROKER command. +A name usually corresponds to one or more broker processes. Doris selects available broker processes based on the name. +You can use the `SHOW BROKER` command to view the Brokers that currently exist in the cluster. + +**Note: Broker Name is just a user-defined name and does not represent the type of Broker.** + +### Certification Information + +Different broker types and different access methods need to provide different authentication information. +Authentication information is usually provided as a Key-Value in the Property Map after `WITH BROKER" broker_name "`. + +#### Community HDFS + +1. Simple Authentication + + Simple authentication means that Hadoop configures `hadoop.security.authentication` to` simple`. + + Use system users to access HDFS. Or add in the environment variable started by Broker：```HADOOP_USER_NAME```。 + + ``` + ( + "username" = "user", + "password" = "" + ); + ``` + + Just leave the password blank. + +2. Kerberos Authentication + + The authentication method needs to provide the following information:： + + * `hadoop.security.authentication`: Specify the authentication method as kerberos. + * `kerberos_principal`： Specify the principal of kerberos. + * `kerberos_keytab`: Specify the path to the keytab file for kerberos. The file must be an absolute path to a file on the server where the broker process is located. And can be accessed by the Broker process. + * `kerberos_keytab_content`: Specify the content of the keytab file in kerberos after base64 encoding. You can choose one of these with `kerberos_keytab` configuration. + + Examples are as follows： + + ``` + ( + "hadoop.security.authentication" = "kerberos", + "kerberos_principal" = "doris@YOUR.COM", + "kerberos_keytab" = "/home/doris/my.keytab" + ) + ``` + ``` + ( + "hadoop.security.authentication" = "kerberos", + "kerberos_principal" = "doris@YOUR.COM", + "kerberos_keytab_content" = "ASDOWHDLAWIDJHWLDKSALDJSDIWALD" + ) + ``` + If Kerberos authentication is used, the [krb5.conf](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html) file is required when deploying the Broker process. + The krb5.conf file contains Kerberos configuration information，Normally, you should install your krb5.conf file in the directory /etc. You can override the default location by setting the environment variable KRB5_CONFIG. + An example of the contents of the krb5.conf file is as follows: + ``` + [libdefaults] + default_realm = DORIS.HADOOP + default_tkt_enctypes = des3-hmac-sha1 des-cbc-crc + default_tgs_enctypes = des3-hmac-sha1 des-cbc-crc + dns_lookup_kdc = true + dns_lookup_realm = false + + [realms] + DORIS.HADOOP = { + kdc = kerberos-doris.hadoop.service:7005 + } + ``` + +3. HDFS HA Mode + + This configuration is used to access HDFS clusters deployed in HA mode. + + * `dfs.nameservices`: Specify the name of the hdfs service, custom, such as "dfs.nameservices" = "my_ha". + * `dfs.ha.namenodes.xxx`: Custom namenode names. Multiple names are separated by commas, where xxx is the custom name in `dfs.nameservices`, such as" dfs.ha.namenodes.my_ha "=" my_nn ". + * `dfs.namenode.rpc-address.xxx.nn`: Specify the rpc address information of namenode, Where nn represents the name of the namenode configured in `dfs.ha.namenodes.xxx`, such as: "dfs.namenode.rpc-address.my_ha.my_nn" = "host:port". + * `dfs.client.failover.proxy.provider`: Specify the provider for the client to connect to the namenode. The default is: org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider. + + Examples are as follows： + + ``` + ( + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ) + ``` + + The HA mode can be combined with the previous two authentication methods for cluster access. If you access HA HDFS with simple authentication: + + ``` + ( + "username"="user", + "password"="passwd", + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ) + ``` + The configuration for accessing the HDFS cluster can be written to the hdfs-site.xml file. When users use the Broker process to read data from the HDFS cluster, they only need to fill in the cluster file path and authentication information. + +#### Baidu Object Storage BOS + +**(Open source version is not supported)** + +1. Access via AK / SK + + * AK/SK: Access Key and Secret Key. You can check the user's AK / SK in Baidu Cloud Security Certification Center. + * Region Endpoint: Endpoint of the BOS region: + + * North China-Beijing: http://bj.bcebos.com + * North China-Baoding: http://bd.bcebos.com + * South China-Guangzhou: http://gz.bcebos.com + * East China-Suzhou: http://sz.bcebos.com + + Examples are as follows： + + ``` + ( + "bos_endpoint" = "http://bj.bcebos.com", + "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "bos_secret_accesskey" = "yyyyyyyyyyyyyyyyyyyyyyyyyy" + ) + ``` + +#### Baidu HDFS/AFS + +**（Open source version is not supported）** + +Baidu AFS and HDFS only support simple authentication access using UGI. Examples are as follows: + +``` +( + "username" = "user", + "password" = "passwd" +); +``` + +User and passwd are UGI configurations for Hadoop. \ No newline at end of file diff --git a/docs/en/administrator-guide/colocation-join.md b/docs/en/administrator-guide/colocation-join.md new file mode 100644 index 00000000000000..ea2e5118c3b8cd --- /dev/null +++ b/docs/en/administrator-guide/colocation-join.md @@ -0,0 +1,448 @@ +--- +{ + "title": "Colocation Join", + "language": "en" +} +--- + + + +# Colocation Join + +Colocation Join is a new feature introduced in Doris 0.9. The purpose of this paper is to provide local optimization for some Join queries to reduce data transmission time between nodes and speed up queries. + +The original design, implementation and effect can be referred to [ISSUE 245] (https://github.com/apache/incubator-doris/issues/245). + +The Colocation Join function has undergone a revision, and its design and use are slightly different from the original design. This document mainly introduces Colocation Join's principle, implementation, usage and precautions. + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. +* Colocation Group (CG): A CG contains one or more tables. Tables within the same group have the same Colocation Group Schema and the same data fragmentation distribution. +* Colocation Group Schema (CGS): Used to describe table in a CG and general Schema information related to Colocation. Including bucket column type, bucket number and copy number. + +## Principle + +The Colocation Join function is to make a CG of a set of tables with the same CGS. Ensure that the corresponding data fragments of these tables will fall on the same BE node. When tables in CG perform Join operations on bucket columns, local data Join can be directly performed to reduce data transmission time between nodes. + +The data of a table will eventually fall into a barrel according to the barrel column value Hash and the number of barrels modeled. Assuming that the number of buckets in a table is 8, there are eight buckets `[0, 1, 2, 3, 4, 5, 6, 7] `Buckets'. We call such a sequence a `Buckets Sequence`. Each Bucket has one or more Tablets. When a table is a single partitioned table, there is only one Tablet in a Bucket. If it is a multi-partition table, there will be more than one. + +In order for a table to have the same data distribution, the table in the same CG must ensure the following attributes are the same: + +1. Barrel row and number of barrels + + Bucket column, that is, the column specified in `DISTRIBUTED BY HASH (col1, col2,...)'in the table building statement. Bucket columns determine which column values are used to Hash data from a table into different Tablets. Tables in the same CG must ensure that the type and number of barrel columns are identical, and the number of barrels is identical, so that the data fragmentation of multiple tables can be controlled one by one. + +2. Number of copies + + The number of copies of all partitions of all tables in the same CG must be the same. If inconsistent, there may be a copy of a Tablet, and there is no corresponding copy of other table fragments on the same BE. + +Tables in the same CG do not require consistency in the number, scope, and type of partition columns. + +After fixing the number of bucket columns and buckets, the tables in the same CG will have the same Buckets Sequnce. The number of replicas determines the number of replicas of Tablets in each bucket, which BE they are stored on. Suppose that Buckets Sequnce is `[0, 1, 2, 3, 4, 5, 6, 7] `, and that BE nodes have `[A, B, C, D] `4. A possible distribution of data is as follows: + +``` ++---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +| 0 | | 1 | | 2 | | 3 | | 4 | | 5 | | 6 | | 7 | ++---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +| A | | B | | C | | D | | A | | B | | C | | D | +| | | | | | | | | | | | | | | | +| B | | C | | D | | A | | B | | C | | D | | A | +| | | | | | | | | | | | | | | | +| C | | D | | A | | B | | C | | D | | A | | B | ++---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +``` + +The data of all tables in CG will be uniformly distributed according to the above rules, which ensures that the data with the same barrel column value are on the same BE node, and local data Join can be carried out. + +## Usage + +### Establishment of tables + +When creating a table, you can specify the attribute `"colocate_with"="group_name"` in `PROPERTIES`, which means that the table is a Colocation Join table and belongs to a specified Colocation Group. + +Examples: + +``` +CREATE TABLE tbl (k1 int, v1 int sum) +DISTRIBUTED BY HASH(k1) +BUCKETS 8 +PROPERTIES( + "colocate_with" = "group1" +); +``` + +If the specified group does not exist, Doris automatically creates a group that contains only the current table. If the Group already exists, Doris checks whether the current table satisfies the Colocation Group Schema. If satisfied, the table is created and added to the Group. At the same time, tables create fragments and replicas based on existing data distribution rules in Groups. +Group belongs to a database, and its name is unique in a database. Internal storage is the full name of Group `dbId_groupName`, but users only perceive groupName. + +### Delete table + +When the last table in Group is deleted completely (deleting completely means deleting from the recycle bin). Usually, when a table is deleted by the `DROP TABLE` command, it will be deleted after the default one-day stay in the recycle bin, and the group will be deleted automatically. + +### View Group + +The following command allows you to view the existing Group information in the cluster. + +``` +SHOW PROC '/colocation_group'; + ++-------------+--------------+--------------+------------+----------------+----------+----------+ +| GroupId | GroupName | TableIds | BucketsNum | ReplicationNum | DistCols | IsStable | ++-------------+--------------+--------------+------------+----------------+----------+----------+ +| 10005.10008 | 10005_group1 | 10007, 10040 | 10 | 3 | int(11) | true | ++-------------+--------------+--------------+------------+----------------+----------+----------+ +``` + +* GroupId: The unique identity of a group's entire cluster, with DB ID in the first half and group ID in the second half. +* GroupName: The full name of Group. +* Tablet Ids: The group contains a list of Tables'ID. +* Buckets Num: Number of barrels. +* Replication Num: Number of copies. +* DistCols: Distribution columns, +* IsStable: Is the group stable (for the definition of stability, see section `Collocation replica balancing and repair'). + +You can further view the data distribution of a group by following commands: + +``` +SHOW PROC '/colocation_group/10005.10008'; + ++-------------+---------------------+ +| BucketIndex | BackendIds | ++-------------+---------------------+ +| 0 | 10004, 10002, 10001 | +| 1 | 10003, 10002, 10004 | +| 2 | 10002, 10004, 10001 | +| 3 | 10003, 10002, 10004 | +| 4 | 10002, 10004, 10003 | +| 5 | 10003, 10002, 10001 | +| 6 | 10003, 10004, 10001 | +| 7 | 10003, 10004, 10002 | ++-------------+---------------------+ +``` + +* BucketIndex: Subscript to the bucket sequence. +* Backend Ids: A list of BE node IDs where data fragments are located in buckets. + +> The above commands require AMDIN privileges. Normal user view is not supported at this time. + +### Modify Colocate Group + +You can modify the Colocation Group property of a table that has been created. Examples: + +`ALTER TABLE tbl SET ("colocate_with" = "group2");` + +* If the table has not previously specified a Group, the command checks the Schema and adds the table to the Group (if the Group does not exist, it will be created). +* If other groups are specified before the table, the command first removes the table from the original group and adds a new group (if the group does not exist, it will be created). + +You can also delete the Colocation attribute of a table by following commands: + +`ALTER TABLE tbl SET ("colocate_with" = "");` + +### Other related operations + +When an ADD PARTITION is added to a table with a Colocation attribute and the number of copies is modified, Doris checks whether the modification violates the Colocation Group Schema and rejects it if it does. + +## Colocation Duplicate Balancing and Repair + +Copy distribution of Colocation tables needs to follow the distribution specified in Group, so it is different from common fragmentation in replica repair and balancing. + +Group itself has a Stable attribute, when Stable is true, which indicates that all fragments of the table in the current Group are not changing, and the Colocation feature can be used normally. When Stable is false, it indicates that some tables in Group are being repaired or migrated. At this time, Colocation Join of related tables will degenerate into ordinary Join. + +### Replica Repair + +Copies can only be stored on specified BE nodes. So when a BE is unavailable (downtime, Decommission, etc.), a new BE is needed to replace it. Doris will first look for the BE with the lowest load to replace it. After replacement, all data fragments on the old BE in the Bucket will be repaired. During the migration process, Group is marked Unstable. + +### Duplicate Equilibrium + +Doris will try to distribute the fragments of the Collocation table evenly across all BE nodes. For the replica balancing of common tables, the granularity is single replica, that is to say, it is enough to find BE nodes with lower load for each replica alone. The equilibrium of the Colocation table is at the Bucket level, where all replicas within a Bucket migrate together. We adopt a simple equalization algorithm, which distributes Buckets Sequnce evenly on all BEs, regardless of the actual size of the replicas, but only according to the number of replicas. Specific algorithms can be referred to the code annotations in `ColocateTableBalancer.java`. + +> Note 1: Current Colocation replica balancing and repair algorithms may not work well for heterogeneous deployed Oris clusters. The so-called heterogeneous deployment, that is, the BE node's disk capacity, number, disk type (SSD and HDD) is inconsistent. In the case of heterogeneous deployment, small BE nodes and large BE nodes may store the same number of replicas. +> +> Note 2: When a group is in an Unstable state, the Join of the table in it will degenerate into a normal Join. At this time, the query performance of the cluster may be greatly reduced. If you do not want the system to balance automatically, you can set the FE configuration item `disable_colocate_balance` to prohibit automatic balancing. Then open it at the right time. (See Section `Advanced Operations` for details) + +## Query + +The Colocation table is queried in the same way as ordinary tables, and users do not need to perceive Colocation attributes. If the Group in which the Colocation table is located is in an Unstable state, it will automatically degenerate to a normal Join. + +Examples are given to illustrate: + +Table 1: + +``` +CREATE TABLE `tbl1` ( + `k1` date NOT NULL COMMENT "", + `k2` int(11) NOT NULL COMMENT "", + `v1` int(11) SUM NOT NULL COMMENT "" +) ENGINE=OLAP +AGGREGATE KEY(`k1`, `k2`) +PARTITION BY RANGE(`k1`) +( + PARTITION p1 VALUES LESS THAN ('2019-05-31'), + PARTITION p2 VALUES LESS THAN ('2019-06-30') +) +DISTRIBUTED BY HASH(`k2`) BUCKETS 8 +PROPERTIES ( + "colocate_with" = "group1" +); +``` + +Table 2: + +``` +CREATE TABLE `tbl2` ( + `k1` datetime NOT NULL COMMENT "", + `k2` int(11) NOT NULL COMMENT "", + `v1` double SUM NOT NULL COMMENT "" +) ENGINE=OLAP +AGGREGATE KEY(`k1`, `k2`) +DISTRIBUTED BY HASH(`k2`) BUCKETS 8 +PROPERTIES ( + "colocate_with" = "group1" +); +``` + +View the query plan: + +``` +DESC SELECT * FROM tbl1 INNER JOIN tbl2 ON (tbl1.k2 = tbl2.k2); + ++----------------------------------------------------+ +| Explain String | ++----------------------------------------------------+ +| PLAN FRAGMENT 0 | +| OUTPUT EXPRS:`tbl1`.`k1` | | +| PARTITION: RANDOM | +| | +| RESULT SINK | +| | +| 2:HASH JOIN | +| | join op: INNER JOIN | +| | hash predicates: | +| | colocate: true | +| | `tbl1`.`k2` = `tbl2`.`k2` | +| | tuple ids: 0 1 | +| | | +| |----1:OlapScanNode | +| | TABLE: tbl2 | +| | PREAGGREGATION: OFF. Reason: null | +| | partitions=0/1 | +| | rollup: null | +| | buckets=0/0 | +| | cardinality=-1 | +| | avgRowSize=0.0 | +| | numNodes=0 | +| | tuple ids: 1 | +| | | +| 0:OlapScanNode | +| TABLE: tbl1 | +| PREAGGREGATION: OFF. Reason: No AggregateInfo | +| partitions=0/2 | +| rollup: null | +| buckets=0/0 | +| cardinality=-1 | +| avgRowSize=0.0 | +| numNodes=0 | +| tuple ids: 0 | ++----------------------------------------------------+ +``` + +If Colocation Join works, the Hash Join Node will show `colocate: true`。 + +If not, the query plan is as follows: + +``` ++----------------------------------------------------+ +| Explain String | ++----------------------------------------------------+ +| PLAN FRAGMENT 0 | +| OUTPUT EXPRS:`tbl1`.`k1` | | +| PARTITION: RANDOM | +| | +| RESULT SINK | +| | +| 2:HASH JOIN | +| | join op: INNER JOIN (BROADCAST) | +| | hash predicates: | +| | colocate: false, reason: group is not stable | +| | `tbl1`.`k2` = `tbl2`.`k2` | +| | tuple ids: 0 1 | +| | | +| |----3:EXCHANGE | +| | tuple ids: 1 | +| | | +| 0:OlapScanNode | +| TABLE: tbl1 | +| PREAGGREGATION: OFF. Reason: No AggregateInfo | +| partitions=0/2 | +| rollup: null | +| buckets=0/0 | +| cardinality=-1 | +| avgRowSize=0.0 | +| numNodes=0 | +| tuple ids: 0 | +| | +| PLAN FRAGMENT 1 | +| OUTPUT EXPRS: | +| PARTITION: RANDOM | +| | +| STREAM DATA SINK | +| EXCHANGE ID: 03 | +| UNPARTITIONED | +| | +| 1:OlapScanNode | +| TABLE: tbl2 | +| PREAGGREGATION: OFF. Reason: null | +| partitions=0/1 | +| rollup: null | +| buckets=0/0 | +| cardinality=-1 | +| avgRowSize=0.0 | +| numNodes=0 | +| tuple ids: 1 | ++----------------------------------------------------+ +``` + +The HASH JOIN node displays the corresponding reason: `colocate: false, reason: group is not stable`. At the same time, an EXCHANGE node will be generated. + + +## Advanced Operations + +### FE Configuration Item + +* disable\_colocate\_relocate + +Whether to close Doris's automatic Colocation replica repair. The default is false, i.e. not closed. This parameter only affects the replica repair of the Colocation table, but does not affect the normal table. + +* disable\_colocate\_balance + +Whether to turn off automatic Colocation replica balancing for Doris. The default is false, i.e. not closed. This parameter only affects the replica balance of the Collocation table, but does not affect the common table. + +User can set these configurations at runtime. See `HELP ADMIN SHOW CONFIG;` and `HELP ADMIN SET CONFIG;`. + +* disable\_colocate\_join + +Whether to turn off the Colocation Join function or not. In 0.10 and previous versions, the default is true, that is, closed. In a later version, it will default to false, that is, open. + +* use\_new\_tablet\_scheduler + +In 0.10 and previous versions, the new replica scheduling logic is incompatible with the Colocation Join function, so in 0.10 and previous versions, if `disable_colocate_join = false`, you need to set `use_new_tablet_scheduler = false`, that is, close the new replica scheduler. In later versions, `use_new_tablet_scheduler` will be equal to true. + +###HTTP Restful API + +Doris provides several HTTP Restful APIs related to Colocation Join for viewing and modifying Colocation Group. + +The API is implemented on the FE side and accessed using `fe_host: fe_http_port`. ADMIN privileges are required. + +1. View all Colocation information for the cluster + + ``` + GET /api/colocate + + Return the internal Colocation info in JSON format: + + { + "colocate_meta": { + "groupName2Id": { + "g1": { + "dbId": 10005, + "grpId": 10008 + } + }, + "group2Tables": {}, + "table2Group": { + "10007": { + "dbId": 10005, + "grpId": 10008 + }, + "10040": { + "dbId": 10005, + "grpId": 10008 + } + }, + "group2Schema": { + "10005.10008": { + "groupId": { + "dbId": 10005, + "grpId": 10008 + }, + "distributionColTypes": [{ + "type": "INT", + "len": -1, + "isAssignedStrLenInColDefinition": false, + "precision": 0, + "scale": 0 + }], + "bucketsNum": 10, + "replicationNum": 2 + } + }, + "group2BackendsPerBucketSeq": { + "10005.10008": [ + [10004, 10002], + [10003, 10002], + [10002, 10004], + [10003, 10002], + [10002, 10004], + [10003, 10002], + [10003, 10004], + [10003, 10004], + [10003, 10004], + [10002, 10004] + ] + }, + "unstableGroups": [] + }, + "status": "OK" + } + ``` +2. Mark Group as Stable or Unstable + + * Mark as Stable + + ``` + POST /api/colocate/group_stable?db_id=10005&group_id=10008 + + Returns: 200 + ``` + + * Mark as Unstable + + ``` + DELETE /api/colocate/group_stable?db_id=10005&group_id=10008 + + Returns: 200 + ``` + +3. Setting Data Distribution for Group + + The interface can force the number distribution of a group. + + ``` + POST /api/colocate/bucketseq?db_id=10005&group_id= 10008 + + Body: + [[10004,10002],[10003,10002],[10002,10004],[10003,10002],[10002,10004],[10003,10002],[10003,10004],[10003,10004],[10003,10004],[10002,10004]] + + Returns: 200 + ``` + Body is a Buckets Sequence represented by a nested array and the ID of the BE where the fragments are distributed in each Bucket. + + Note that using this command, you may need to set the FE configuration `disable_colocate_relocate` and `disable_colocate_balance` to true. That is to shut down the system for automatic Colocation replica repair and balancing. Otherwise, it may be automatically reset by the system after modification. diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md new file mode 100644 index 00000000000000..4f0c0578ec1353 --- /dev/null +++ b/docs/en/administrator-guide/config/fe_config.md @@ -0,0 +1,41 @@ +--- +{ + "title": "Configuration", + "language": "en" +} +--- + + + +# Configuration + +## brpc_max_body_size + + This configuration is mainly used to modify the parameter max_body_size of brpc. The default configuration is 64M. It usually occurs in multi distinct + no group by + exceeds 1t data. In particular, if you find that the query is stuck, and be appears the word "body size is too large" in log. + + Because this is a brpc configuration, users can also directly modify this parameter on-the-fly by visiting ```http://host:brpc_port/flags``` + +## max_running_txn_num_per_db + +   This configuration is mainly used to control the number of concurrent load job in the same db. The default configuration is 100. When the number of concurrent load job exceeds the configured value, the load which is synchronously executed will fail, such as stream load. The load which is asynchronously will always be in a pending state such as broker load. + +   It is generally not recommended to change this property. If the current load concurrency exceeds this value, you need to first check if a single load job is too slow, or if there are too many small files, there is no problem of load after merging those small files. + +   Error information such as: current running txns on db xxx is xx, larger than limit xx. The above info is related by this property. diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md new file mode 100644 index 00000000000000..4cdffb2b185f83 --- /dev/null +++ b/docs/en/administrator-guide/dynamic-partition.md @@ -0,0 +1,192 @@ +--- +{ + "title": "Dynamic Partition", + "language": "en" +} +--- + + + +# Dynamic Partition + +Dynamic partition is a new feature introduced in Doris verion 0.12. It's designed to manage partition's Time-to-Life (TTL), reducing the burden on users. + +The original design, implementation and effect can be referred to [ISSUE 2262](https://github.com/apache/incubator-doris/issues/2262)。 + +Currently, the function of adding partitions dynamically is implemented, and the next version will support removing partitions dynamically. + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. + +## Principle + +In some scenarios, the user will create partitions for the table according to the day and perform routine tasks regularly every day. In this case, the user needs to manually manage the partition, otherwise the data import may fail because the partition is forgot to create, which brings additional maintenance costs to the user. + +The design of implementation is that FE will starts a background thread that determines whether or not to start the thread and the scheduling frequency of the thread based on the parameters `dynamic_partition_enable` and `dynamic_partition_check_interval_seconds` in `fe.conf`. + +When create a olap table, the `dynamic_partition` properties will be assigned. FE will parse `dynamic_partition` properties and check the legitimacy of the input parameters firstly, and then persist the properties to FE metadata, register the table to the list of dynamic partition at the same time. Daemon thread will scan the dynamic partition list periodically according to the configuration parameters, +read dynamic partition properties of the table, and doing the task of adding partitions. The scheduling information of each time will be kept in the memory of FE. You can check whether the scheduling task is successful through `SHOW DYNAMIC PARTITION TABLES`. + +## Usage + +### Establishment of tables + +When creating a table, you can specify the attribute `dynamic_partition` in `PROPERTIES`, which means that the table is a dynamic partition table. + +Examples: + +``` +CREATE TABLE example_db.dynamic_partition +( +k1 DATE, +k2 INT, +k3 SMALLINT, +v1 VARCHAR(2048), +v2 DATETIME DEFAULT "2014-02-04 15:36:00" +) +ENGINE=olap +DUPLICATE KEY(k1, k2, k3) +PARTITION BY RANGE (k1) +( +PARTITION p1 VALUES LESS THAN ("2014-01-01"), +PARTITION p2 VALUES LESS THAN ("2014-06-01"), +PARTITION p3 VALUES LESS THAN ("2014-12-01") +) +DISTRIBUTED BY HASH(k2) BUCKETS 32 +PROPERTIES( +"storage_medium" = "SSD", +"dynamic_partition.enable" = "true" +"dynamic_partition.time_unit" = "DAY", +"dynamic_partition.end" = "3", +"dynamic_partition.prefix" = "p", +"dynamic_partition.buckets" = "32" + ); +``` +Create a dynamic partition table, specify enable dynamic partition features, take today is 2020-01-08 for example, at every time of scheduling, will create today and after 3 days in advance of four partitions +(if the partition is existed, the task will be ignored), partition name respectively according to the specified prefix `p20200108` `p20200109` `p20200110` `p20200111`, each partition to 32 the number of points barrels, each partition scope is as follows: +``` +[types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) +[types: [DATE]; keys: [2020-01-09]; ‥types: [DATE]; keys: [2020-01-10]; ) +[types: [DATE]; keys: [2020-01-10]; ‥types: [DATE]; keys: [2020-01-11]; ) +[types: [DATE]; keys: [2020-01-11]; ‥types: [DATE]; keys: [2020-01-12]; ) +``` + +### Enable Dynamic Partition Feature + +1. First of all, `dynamic_partition_enable=true` needs to be set in fe.conf, which can be specified by modifying the configuration file when the cluster starts up, or dynamically modified by HTTP interface at run time + +2. If you need to add dynamic partitioning properties to a table prior to version 0.12, you need to modify the properties of the table with the following command + +``` +ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32"); +``` + +### Disable Dynamic Partition Feature + +If you need to stop dynamic partitioning for all dynamic partitioning tables in the cluster, you need to set 'dynamic_partition_enable=true' in fe.conf + +If you need to stop dynamic partitioning for a specified table, you can modify the properties of the table with the following command + +``` +ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "false") +``` + +### Modify Dynamic Partition Properties + +You can modify the properties of the dynamic partition with the following command + +``` +ALTER TABLE dynamic_partition set("key" = "value") +``` + +### Check Dynamic Partition Table Scheduling Status + +You can further view the scheduling of dynamic partitioned tables by using the following command: + +``` +SHOW DYNAMIC PARTITION TABLES; + ++-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ +| TableName | Enable | TimeUnit | End | Prefix | Buckets | LastUpdateTime | LastSchedulerTime | State | Msg | ++-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ +| dynamic_partition | true | DAY | 3 | p | 32 | 2020-01-08 20:19:09 | 2020-01-08 20:19:34 | NORMAL | N/A | ++-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ +1 row in set (0.00 sec) + +``` + +* LastUpdateTime: The last time of modifying dynamic partition properties +* LastSchedulerTime: The last time of performing dynamic partition scheduling +* State: The state of the last execution of dynamic partition scheduling +* Msg: Error message for the last time dynamic partition scheduling was performed + +## Advanced Operation + +### FE Configuration Item + +* dynamic\_partition\_enable + + Whether to enable Doris's dynamic partition feature. The default value is false, which is off. This parameter only affects the partitioning operation of dynamic partition tables, not normal tables. + +* dynamic\_partition\_check\_interval\_seconds + + The execution frequency of dynamically partitioned threads, by default 3600(1 hour), which means scheduled every 1 hour. + +### HTTP Restful API + +Doris provides an HTTP Restful API for modifying dynamic partition configuration parameters at run time. + +The API is implemented in FE, user can access it by `fe_host:fe_http_port`.The operation needs admin privilege. + +1. Set dynamic_partition_enable to true or false + + * Set to true + + ``` + GET /api/_set_config?dynamic_partition_enable=true + + For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true + + Return Code：200 + ``` + + * Set to false + + ``` + GET /api/_set_config?dynamic_partition_enable=false + + For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=false + + Return Code：200 + ``` + +2. Set the scheduling frequency for dynamic partition + + * Set schedule frequency to 12 hours. + + ``` + GET /api/_set_config?dynamic_partition_check_interval_seconds=432000 + + For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000 + + Return Code：200 + ``` diff --git a/docs/en/administrator-guide/export_manual.md b/docs/en/administrator-guide/export_manual.md new file mode 100644 index 00000000000000..b647e5aa5f8d78 --- /dev/null +++ b/docs/en/administrator-guide/export_manual.md @@ -0,0 +1,191 @@ +--- +{ + "title": "Data export", + "language": "en" +} +--- + + + +# Data export + +Export is a function provided by Doris to export data. This function can export user-specified table or partition data in text format to remote storage through Broker process, such as HDFS/BOS. + +This document mainly introduces the basic principles, usage, best practices and precautions of Export. + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. +* Broker: Doris can manipulate files for remote storage through the Broker process. +* Tablet: Data fragmentation. A table is divided into multiple data fragments. + +## Principle + +After the user submits an Export job. Doris counts all Tablets involved in this job. These tablets are then grouped to generate a special query plan for each group. The query plan reads the data on the included tablet and then writes the data to the specified path of the remote storage through Broker. + +The overall mode of dispatch is as follows: + +``` ++--------+ +| Client | ++---+----+ + | 1. Submit Job + | ++---v--------------------+ +| FE | +| | +| +-------------------+ | +| | ExportPendingTask | | +| +-------------------+ | +| | 2. Generate Tasks +| +--------------------+ | +| | ExportExporingTask | | +| +--------------------+ | +| | +| +-----------+ | +----+ +------+ +---------+ +| | QueryPlan +----------------> BE +--->Broker+---> | +| +-----------+ | +----+ +------+ | Remote | +| +-----------+ | +----+ +------+ | Storage | +| | QueryPlan +----------------> BE +--->Broker+---> | +| +-----------+ | +----+ +------+ +---------+ ++------------------------+ 3. Execute Tasks + +``` + +1. The user submits an Export job to FE. +2. FE's Export scheduler performs an Export job in two stages: + 1. PENDING: FE generates Export Pending Task, sends snapshot command to BE, and takes a snapshot of all Tablets involved. And generate multiple query plans. + 2. EXPORTING: FE generates Export ExporingTask and starts executing the query plan. + +### query plan splitting + +The Export job generates multiple query plans, each of which scans a portion of the Tablet. The number of Tablets scanned by each query plan is specified by the FE configuration parameter `export_tablet_num_per_task`, which defaults to 5. That is, assuming a total of 100 Tablets, 20 query plans will be generated. Users can also specify this number by the job attribute `tablet_num_per_task`, when submitting a job. + +Multiple query plans for a job are executed sequentially. + +### Query Plan Execution + +A query plan scans multiple fragments, organizes read data in rows, batches every 1024 actions, and writes Broker to remote storage. + +The query plan will automatically retry three times if it encounters errors. If a query plan fails three retries, the entire job fails. + +Doris will first create a temporary directory named `doris_export_tmp_12345` (where `12345` is the job id) in the specified remote storage path. The exported data is first written to this temporary directory. Each query plan generates a file with an example file name: + +`export-data-c69fcf2b6db5420f-a96b94c1ff8bccef-1561453713822` + +Among them, `c69fcf2b6db5420f-a96b94c1ff8bccef` is the query ID of the query plan. ` 1561453713822` Timestamp generated for the file. + +When all data is exported, Doris will rename these files to the user-specified path. + +## Use examples + +Export's detailed commands can be passed through `HELP EXPORT;` Examples are as follows: + +``` +EXPORT TABLE db1.tbl1 +PARTITION (p1,p2) +TO "bos://bj-test-cmy/export/" +PROPERTIES +( + "column_separator"=",", + "exec_mem_limit"="2147483648", + "timeout" = "3600" +) +WITH BROKER "hdfs" +( + "username" = "user", + "password" = "passwd", +); +``` + +* `column_separator`: Column separator. The default is `\t`. +* `line_delimiter`: Line separator. The default is `\n`. +* `exec_mem_limit`: Represents the memory usage limitation of a query plan on a single BE in an Export job. Default 2GB. Unit bytes. +* `timeout`: homework timeout. Default 2 hours. Unit seconds. +* `tablet_num_per_task`: The maximum number of fragments allocated per query plan. The default is 5. + +After submitting a job, the job status can be imported by querying the `SHOW EXPORT'command. The results are as follows: + +``` + JobId: 14008 + State: FINISHED + Progress: 100% + TaskInfo: {"partitions":["*"],"exec mem limit":2147483648,"column separator":",","line delimiter":"\n","tablet num":1,"broker":"hdfs","coord num":1,"db":"default_cluster:db1","tbl":"tbl3"} + Path: bos://bj-test-cmy/export/ +CreateTime: 2019-06-25 17:08:24 + StartTime: 2019-06-25 17:08:28 +FinishTime: 2019-06-25 17:08:34 + Timeout: 3600 + ErrorMsg: N/A +``` + + +* JobId: The unique ID of the job +* State: Job status: + * PENDING: Jobs to be Scheduled + * EXPORING: Data Export + * FINISHED: Operation Successful + * CANCELLED: Job Failure +* Progress: Work progress. The schedule is based on the query plan. Assuming a total of 10 query plans have been completed, the progress will be 30%. +* TaskInfo: Job information in Json format: + * db: database name + * tbl: Table name + * partitions: Specify the exported partition. `*` Represents all partitions. + * exec MEM limit: query plan memory usage limit. Unit bytes. + * column separator: The column separator for the exported file. + * line delimiter: The line separator for the exported file. + * tablet num: The total number of tablets involved. + * Broker: The name of the broker used. + * Coord num: Number of query plans. +* Path: Export path on remote storage. +* CreateTime/StartTime/FinishTime: Creation time, start scheduling time and end time of jobs. +* Timeout: Job timeout. The unit is seconds. This time is calculated from CreateTime. +* Error Msg: If there is an error in the job, the cause of the error is shown here. + +## Best Practices + +### Splitting Query Plans + +How many query plans need to be executed for an Export job depends on the total number of Tablets and how many Tablets can be allocated for a query plan at most. Since multiple query plans are executed serially, the execution time of jobs can be reduced if more fragments are processed by one query plan. However, if the query plan fails (e.g., the RPC fails to call Broker, the remote storage jitters, etc.), too many tablets can lead to a higher retry cost of a query plan. Therefore, it is necessary to arrange the number of query plans and the number of fragments to be scanned for each query plan in order to balance the execution time and the success rate of execution. It is generally recommended that the amount of data scanned by a query plan be within 3-5 GB (the size and number of tables in a table can be viewed by `SHOW TABLET FROM tbl_name;`statement. + +### exec\_mem\_limit + +Usually, a query plan for an Export job has only two parts `scan`- `export`, and does not involve computing logic that requires too much memory. So usually the default memory limit of 2GB can satisfy the requirement. But in some scenarios, such as a query plan, too many Tablets need to be scanned on the same BE, or too many data versions of Tablets, may lead to insufficient memory. At this point, larger memory needs to be set through this parameter, such as 4 GB, 8 GB, etc. + +## Notes + +* It is not recommended to export large amounts of data at one time. The maximum amount of exported data recommended by an Export job is tens of GB. Excessive export results in more junk files and higher retry costs. +* If the amount of table data is too large, it is recommended to export it by partition. +* During the operation of the Export job, if FE restarts or cuts the master, the Export job will fail, requiring the user to resubmit. +* If the Export job fails, the `__doris_export_tmp_xxx` temporary directory generated in the remote storage and the generated files will not be deleted, requiring the user to delete them manually. +* If the Export job runs successfully, the `__doris_export_tmp_xxx` directory generated in the remote storage may be retained or cleared according to the file system semantics of the remote storage. For example, in Baidu Object Storage (BOS), after removing the last file in a directory through rename operation, the directory will also be deleted. If the directory is not cleared, the user can clear it manually. +* When the Export runs successfully or fails, the FE reboots or cuts, then some information of the jobs displayed by `SHOW EXPORT` will be lost and can not be viewed. +* Export jobs only export data from Base tables, not Rollup Index. +* Export jobs scan data and occupy IO resources, which may affect the query latency of the system. + +## Relevant configuration + +### FE + +* `expo_checker_interval_second`: Scheduling interval of Export job scheduler, default is 5 seconds. Setting this parameter requires restarting FE. +* `export_running_job_num_limit `: Limit on the number of Export jobs running. If exceeded, the job will wait and be in PENDING state. The default is 5, which can be adjusted at run time. +* `Export_task_default_timeout_second`: Export job default timeout time. The default is 2 hours. It can be adjusted at run time. +* `export_tablet_num_per_task`: The maximum number of fragments that a query plan is responsible for. The default is 5. diff --git a/docs/en/administrator-guide/http-actions/cancel-label.md b/docs/en/administrator-guide/http-actions/cancel-label.md new file mode 100644 index 00000000000000..3ba84fce1bc1df --- /dev/null +++ b/docs/en/administrator-guide/http-actions/cancel-label.md @@ -0,0 +1,64 @@ +--- +{ + "title": "CANCEL LABEL", + "language": "en" +} +--- + + + +# CANCEL LABEL +## description + NAME: + cancel_label: cancel a transaction with label + + SYNOPSIS + curl -u user:passwd -XPOST http://host:port/api/{db}/{label}/_cancel + + DESCRIPTION + + This is to cancel a transaction with specified label. + + RETURN VALUES + + Return a JSON format string: + + Status: + Success: cancel succeed + Others: cancel failed + Message: Error message if cancel failed + + ERRORS + +## example + + 1. Cancel the transaction with label "testLabel" on database "testDb" + + curl -u root -XPOST http://host:port/api/testDb/testLabel/_cancel + +## keyword + + CANCEL，LABEL + + + + + + diff --git a/docs/en/administrator-guide/http-actions/compaction-action.md b/docs/en/administrator-guide/http-actions/compaction-action.md new file mode 100644 index 00000000000000..1b3d4f20a3dfb2 --- /dev/null +++ b/docs/en/administrator-guide/http-actions/compaction-action.md @@ -0,0 +1,85 @@ +--- +{ + "title": "Compaction Action", + "language": "en" +} +--- + + + +# Compaction Action + +This API is used to view the overall compaction status of a BE node or the compaction status of a specified tablet. It can also be used to manually trigger Compaction. + +## View Compaction status + +### The overall compaction status of the node + +(TODO) + +### Specify the compaction status of the tablet + +``` +curl -X GET http://be_host:webserver_port/api/compaction/show?tablet_id=xxxx\&schema_hash=yyyy +``` + +If the tablet does not exist, an error in JSON format is returned: + +``` +{ + "status": "Fail", + "msg": "Tablet not found" +} +``` + +If the tablet exists, the result is returned in JSON format: + +``` +{ + "cumulative point": 50, + "last cumulative failure time": "2019-12-16 18:13:43.224", + "last base failure time": "2019-12-16 18:13:23.320", + "last cumu success time": "2019-12-16 18:12:15.110", + "last base success time": "2019-12-16 18:11:50.780", + "rowsets": [ + "[0-48] 10 DATA OVERLAPPING", + "[49-49] 2 DATA OVERLAPPING", + "[50-50] 0 DELETE NONOVERLAPPING", + "[51-51] 5 DATA OVERLAPPING" + ] +} +``` + +Explanation of results: + +* cumulative point: The version boundary between base and cumulative compaction. Versions before (excluding) points are handled by base compaction. Versions after (inclusive) are handled by cumulative compaction. +* last cumulative failure time: The time when the last cumulative compaction failed. After 10 minutes by default, cumulative compaction is attempted on the this tablet again. +* last base failure time: The time when the last base compaction failed. After 10 minutes by default, base compaction is attempted on the this tablet again. +* rowsets: The current rowsets collection of this tablet. [0-48] means a rowset with version 0-48. The second number is the number of segments in a rowset. The `DELETE` indicates the delete version. `OVERLAPPING` and `NONOVERLAPPING` indicates whether data between segments is overlap. + +### Examples + +``` +curl -X GET http://192.168.10.24:8040/api/compaction/show?tablet_id=10015\&schema_hash=1294206575 +``` + +## Manually trigger Compaction + +(TODO) diff --git a/docs/en/administrator-guide/http-actions/fe-get-log-file.md b/docs/en/administrator-guide/http-actions/fe-get-log-file.md new file mode 100644 index 00000000000000..254e2ec548f89f --- /dev/null +++ b/docs/en/administrator-guide/http-actions/fe-get-log-file.md @@ -0,0 +1,74 @@ +--- +{ + "title": "get\\_log\\_file", + "language": "en" +} +--- + + + +# get\_log\_file + +To get FE log via HTTP + +## Types of FE log + +1. fe.audit.log (Audit log) + + The audit log records the all statements executed. Audit log's name format as follow: + + ``` + fe.audit.log # The latest audit log + fe.audit.log.20190603.1 # The historical audit log. The smaller the sequence number, the newer the log. + fe.audit.log.20190603.2 + fe.audit.log.20190602.1 + ... + ``` + +## Example + +1. Get the list of specified type of logs + + Example + + `curl -v -X HEAD -uuser:passwd http://fe_host:http_port/api/get_log_file?type=fe.audit.log` + + Returns: + + ``` + HTTP/1.1 200 OK + file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934} + content-type: text/html + connection: keep-alive + ``` + + In the header of result, the `file_infos` section saves the file list and file size in JSON format. + +2. Download files + + Example: + + ``` + curl -X GET -uuser:passwd http://fe_host:http_port/api/get_log_file?type=fe.audit.log\&file=fe.audit.log.20190528.1 + ``` + +## Notification + +Need ADMIN priviledge. diff --git a/docs/en/administrator-guide/http-actions/get-label-state.md b/docs/en/administrator-guide/http-actions/get-label-state.md new file mode 100644 index 00000000000000..ac24f3c339ab91 --- /dev/null +++ b/docs/en/administrator-guide/http-actions/get-label-state.md @@ -0,0 +1,59 @@ +--- +{ + "title": "GET LABEL STATE", + "language": "en" +} +--- + + + +# GET LABEL STATE +## description + NAME: + get_label_state: get label's state + + SYNOPSIS + curl -u user:passwd http://host:port/api/{db}/{label}/_state + + DESCRIPTION + + Check the status of a transaction + + RETURN VALUES + + Return of JSON format string of the status of specified transaction: + Label: The specified label. + Status: Success or not of this request. + Message: Error messages + State: + UNKNOWN/PREPARE/COMMITTED/VISIBLE/ABORTED + + ERRORS + +## example + + 1. Get status of label "testLabel" on database "testDb" + + curl -u root http://host:port/api/testDb/testLabel/_state + +## keyword + + GET, LABEL, STATE + diff --git a/docs/en/administrator-guide/http-actions/restore-tablet.md b/docs/en/administrator-guide/http-actions/restore-tablet.md new file mode 100644 index 00000000000000..ca55c81a596d94 --- /dev/null +++ b/docs/en/administrator-guide/http-actions/restore-tablet.md @@ -0,0 +1,41 @@ +--- +{ + "title": "RESTORE TABLET", + "language": "en" +} +--- + + + +# RESTORE TABLET +## description + + To restore the tablet data from trash dir on BE + + METHOD: POST + URI: http://be_host:be_http_port/api/restore_tablet?tablet_id=xxx&schema_hash=xxx + +## example + + curl -X POST "http://hostname:8088/api/restore_tablet?tablet_id=123456\&schema_hash=1111111" + +##keyword + + RESTORE,TABLET,RESTORE,TABLET diff --git a/docs/en/administrator-guide/load-data/broker-load-manual.md b/docs/en/administrator-guide/load-data/broker-load-manual.md new file mode 100644 index 00000000000000..6c0c02522647ea --- /dev/null +++ b/docs/en/administrator-guide/load-data/broker-load-manual.md @@ -0,0 +1,505 @@ +--- +{ + "title": "Broker Load", + "language": "en" +} +--- + + + +# Broker Load + +Broker load is an asynchronous import method, and the data source supported depends on the data source supported by the Broker process. + +Users need to create Broker load imports through MySQL protocol and check the import results by viewing the import commands. + +## Applicable scenarios + +* Source data in Broker accessible storage systems, such as HDFS. +* Data volumes range from tens to hundreds of GB. + +## Noun Interpretation + +1. Frontend (FE): Metadata and scheduling nodes of Doris system. In the import process, it is mainly responsible for the generation of import plan and the scheduling of import tasks. +2. Backend (BE): The computing and storage nodes of Doris system. In the import process, it is mainly responsible for ETL and storage of data. +3. Broker: Broker is an independent stateless process. It encapsulates the file system interface and provides Doris with the ability to read files in the remote storage system. +4. Plan: Import the execution plan, and BE executes the import execution plan to import data into Doris system. + +## Basic Principles + +After the user submits the import task, FE generates the corresponding plan and distributes the plan to several BEs according to the number of BEs and the size of the file. Each BE performs part of the import data. + +BE pulls data from Broker and imports it into the system after transforming the data. All BEs complete the import, and the FE decides whether the import is successful or not. + +``` + + + | 1. user create broker load + v + +----+----+ + | | + | FE | + | | + +----+----+ + | + | 2. BE etl and load the data + +--------------------------+ + | | | ++---v---+ +--v----+ +---v---+ +| | | | | | +| BE | | BE | | BE | +| | | | | | ++---+-^-+ +---+-^-+ +--+-^--+ + | | | | | | + | | | | | | 3. pull data from broker ++---v-+-+ +---v-+-+ +--v-+--+ +| | | | | | +|Broker | |Broker | |Broker | +| | | | | | ++---+-^-+ +---+-^-+ +---+-^-+ + | | | | | | ++---v-+-----------v-+----------v-+-+ +| HDFS/BOS/AFS cluster | +| | ++----------------------------------+ + +``` + +## Basic operations + +### Create a load + +Broker load create a data load job + +Grammar: + +``` +LOAD LABEL db_name.label_name +(data_desc, ...) +WITH BROKER broker_name broker_properties +[PROPERTIES (key1=value1, ... )] + +* data_desc: + + DATA INFILE ('file_path', ...) + [NEGATIVE] + INTO TABLE tbl_name + [PARTITION (p1, p2)] + [COLUMNS TERMINATED BY separator ] + [(col1, ...)] + [SET (k1=f1(xx), k2=f2(xx))] + [WHERE predicate] + +* broker_properties: + + (key1=value1, ...) +``` +Examples: + +``` +LOAD LABEL db1.label1 +( + DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file1") + INTO TABLE tbl1 + COLUMNS TERMINATED BY "," + (tmp_c1,tmp_c2) + SET + ( + id=tmp_c2, + name=tmp_c1) + ), + DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file2") + INTO TABLE tbl2 + COLUMNS TERMINATED BY "," + (col1, col2) + where col1 > 1 +) +WITH BROKER 'broker' +( + "username"="user", + "password"="pass" +) +PROPERTIES +( + "timeout" = "3600" +); + +``` + +Create the imported detailed grammar execution ``HELP BROKER LOAD `` View grammar help. This paper mainly introduces the parametric meaning and points for attention in Broker load's creation import grammar. + +#### Label + +Identity of import task. Each import task has a unique Label within a single database. Label is a user-defined name in the import command. With this Label, users can view the execution of the corresponding import task. + +Another function of Label is to prevent users from repeatedly importing the same data. **It is strongly recommended that users use the same label for the same batch of data. Thus, repeated requests for the same batch of data can only be accepted once, guaranteeing at-Most-One semantics** + +When the corresponding import job status of Label is CANCELLED, it can be used again to submit the import job. + +#### Data Description Class Parameters + +Data description class parameters mainly refer to the parameters belonging to ``data_desc`` in Broker load creating import statements. Each group of ```data_desc``` mainly describes the data source address, ETL function, target table and partition information involved in this import. + +The following is a detailed explanation of some parameters of the data description class: + ++ Multi-table import + + Broker load supports a single import task involving multiple tables, and each Broker load import task can implement multiple tables import by declaring multiple tables in multiple ``data_desc``. Each individual ```data_desc``` can also specify the data source address belonging to the table. Broker load guarantees atomic success or failure between multiple tables imported at a single time. + ++ negative + + ```data_desc``` can also set up data fetching and anti-importing. This function is mainly used when aggregated columns in data tables are of SUM type. If you want to revoke a batch of imported data. The `negative'parameter can be used as a batch of data. Doris automatically retrieves this batch of data on aggregated columns to eliminate the same batch of data. + ++ partition + + In `data_desc`, you can specify the partition information of the table to be imported, but it will not be imported if the data to be imported does not belong to the specified partition. At the same time, data that does not specify a Partition is considered error data. + ++ where predicate + + The where statement in ```data_desc``` is responsible for filtering the data that has been transformed. The unselected rows which is filtered by where predicate will not be calculated in ```max_filter_ratio``` . If there are more then one where predicate of the same table , the multi where predicate will be merged from different ```data_desc``` and the policy is AND. + +#### Import job parameters + +Import job parameters mainly refer to the parameters in Broker load creating import statement that belong to ``opt_properties``. Import operation parameters act on the whole import operation. + +The following is a detailed explanation of some parameters of the import operation parameters: + ++ time out + + The time-out of the import job (in seconds) allows the user to set the time-out of each import by himself in ``opt_properties``. If the import task is not completed within the set timeout time, it will be cancelled by the system and become CANCELLED. The default import timeout for Broker load is 4 hours. + + Usually, the user does not need to manually set the timeout of the import task. When the import cannot be completed within the default timeout time, the task timeout can be set manually. + + > Recommended timeout + > + > Total File Size (MB) / Slowest Import Speed (MB/s) > timeout >((MB) * Number of tables to be imported and related Roll up tables) / (10 * Number of concurrent imports) + + > The concurrency of imports can be seen in the final configuration of the import system in the document. The current import speed limit is 10MB/s in 10 of the formulas. + + > For example, a 1G data to be imported contains three Rollup tables, and the current concurrency of imports is 3. The minimum value of timeout is ```(1 * 1024 * 3) / (10 * 3) = 102 seconds.``` + + Because the machine environment of each Doris cluster is different and the concurrent query tasks of the cluster are different, the slowest import speed of the user Doris cluster requires the user to guess the import task speed according to the history. + ++ max\_filter\_ratio + + The maximum tolerance rate of the import task is 0 by default, and the range of values is 0-1. When the import error rate exceeds this value, the import fails. + + If the user wishes to ignore the wrong row, the import can be successful by setting this parameter greater than 0. + + The calculation formula is as follows: + + ``` (dpp.abnorm.ALL / (dpp.abnorm.ALL + dpp.norm.ALL ) ) > max_filter_ratio ``` + + ``` dpp.abnorm.ALL``` denotes the number of rows whose data quality is not up to standard. Such as type mismatch, column mismatch, length mismatch and so on. + + ``` dpp.norm.ALL ``` refers to the number of correct data in the import process. The correct amount of data for the import task can be queried by the ``SHOW LOAD`` command. + + The number of rows in the original file = `dpp.abnorm.ALL + dpp.norm.ALL` + +* exec\_mem\_limit + + Memory limit. Default is 2GB. Unit is Bytes. + ++ strict\_mode + + Broker load can use `strict mode`. Use ```properties ("strict_mode" = "true")``` to enable `strict mode`, default is false + + The strict mode means that the column type conversion in the import process is strictly filtered. The strategy of strict filtering is as follows: + + 1. For column type conversion, if strict mode is true, the wrong data will be filtered. Error data here refers to the kind of data that the original data is not null and the result is null after participating in column type conversion. + + 2. Strict mode does not affect the imported column when it is generated by a function transformation. + + 3. For a column type imported that contains scope restrictions, strict mode does not affect it if the original data can normally pass type conversion, but can not pass scope restrictions. For example, if the type is decimal (1,0) and the original data is 10, it falls within the scope of type conversion but not column declaration. This data strict has no effect on it. + +#### Import Relation between strict mode source data + +Here's an example of a column type TinyInt + +> Note: When columns in a table allow null values to be imported + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|---------| +|null | \N | N/A | true or false | NULL| +|not null | aaa or 2000 | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 | 1 | true or false | correct data| + +Here's an example of column type Decimal (1,0) + +> Note: When columns in a table allow null values to be imported + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|--------| +|null | \N | N/A | true or false | NULL| +|not null | aaa | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 or 10 | 1 | true or false | correct data| + +> Note: Although 10 is a value beyond the range, strict mode does not affect it because its type meets the requirements of decimal. 10 will eventually be filtered in other ETL processes. But it will not be filtered by strict mode. + +### View load + +Broker load import mode is asynchronous, so the user must create the imported Label record and use Label in the **view Import command to view the import result**. View import commands are common in all import modes. The specific syntax can be `HELP SHOW LOAD`. + +Examples: + +``` +mysql> show load order by createtime desc limit 1\G +*************************** 1. row *************************** + JobId: 76391 + Label: label1 + State: FINISHED + Progress: ETL:N/A; LOAD:100% + Type: BROKER + EtlInfo: dpp.abnorm.ALL=15; dpp.norm.ALL=28133376 + TaskInfo: cluster:N/A; timeout(s):10800; max_filter_ratio:5.0E-5 + ErrorMsg: N/A + CreateTime: 2019-07-27 11:46:42 + EtlStartTime: 2019-07-27 11:46:44 + EtlFinishTime: 2019-07-27 11:46:44 + LoadStartTime: 2019-07-27 11:46:44 +LoadFinishTime: 2019-07-27 11:50:16 + URL: http://192.168.1.1:8040/api/_load_error_log?file=__shard_4/error_log_insert_stmt_4bb00753932c491a-a6da6e2725415317_4bb00753932c491a_a6da6e2725415317 + JobDetails: {"Unfinished backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"ScannedRows":2390016,"TaskNumber":1,"All backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"FileNumber":1,"FileSize":1073741824} +``` + +The following is mainly about the significance of viewing the parameters in the return result set of the import command: + ++ JobId + + The unique ID of the import task is different for each import task, which is automatically generated by the system. Unlike Label, JobId will never be the same, while Label can be reused after the import task fails. + ++ Label + + Identity of import task. + ++ State + + Import the current phase of the task. In the Broker load import process, PENDING and LOADING are the two main import states. If the Broker load is in the PENDING state, it indicates that the current import task is waiting to be executed; the LOADING state indicates that it is executing. + + There are two final stages of the import task: CANCELLED and FINISHED. When Load job is in these two stages, the import is completed. CANCELLED is the import failure, FINISHED is the import success. + ++ Progress + + Import the progress description of the task. There are two kinds of progress: ETL and LOAD, which correspond to the two stages of the import process, ETL and LOADING. At present, Broker load only has the LOADING stage, so ETL will always be displayed as `N/A`. + + The progress range of LOAD is 0-100%. + + ``` LOAD Progress = Number of tables currently completed / Number of tables designed for this import task * 100%``` + + **If all import tables complete the import, then the progress of LOAD is 99%** import enters the final effective stage, and the progress of LOAD will only be changed to 100% after the entire import is completed. + + Import progress is not linear. So if there is no change in progress over a period of time, it does not mean that the import is not being implemented. + ++ Type + + Types of import tasks. The type value of Broker load is only BROKER. ++ Etlinfo + + It mainly shows the imported data quantity indicators `unselected.rows`, `dpp.norm.ALL` and `dpp.abnorm.ALL`. The first value shows the rows which has been filtered by where predicate. Users can verify that the error rate of the current import task exceeds max\_filter\_ratio based on these two indicators. + ++ TaskInfo + + It mainly shows the current import task parameters, that is, the user-specified import task parameters when creating the Broker load import task, including `cluster`, `timeout`, and `max_filter_ratio`. + ++ ErrorMsg + + When the import task status is CANCELLED, the reason for the failure is displayed in two parts: type and msg. If the import task succeeds, the `N/A` is displayed. + + The value meaning of type: + + ``` + USER_CANCEL: User Canceled Tasks + ETL_RUN_FAIL：Import tasks that failed in the ETL phase + ETL_QUALITY_UNSATISFIED：Data quality is not up to standard, that is, the error rate exceedsmax_filter_ratio + LOAD_RUN_FAIL：Import tasks that failed in the LOADING phase + TIMEOUT：Import task not completed in overtime + UNKNOWN：Unknown import error + ``` + ++ CreateTime /EtlStartTime /EtlFinishTime /LoadStartTime /LoadFinishTime + + These values represent the creation time of the import, the beginning time of the ETL phase, the completion time of the ETL phase, the beginning time of the Loading phase and the completion time of the entire import task, respectively. + + Broker load import has no ETL stage, so its EtlStartTime, EtlFinishTime, LoadStartTime are set to the same value. + + Import tasks stay in CreateTime for a long time, while LoadStartTime is N/A, which indicates that import tasks are heavily stacked at present. Users can reduce the frequency of import submissions. + + ``` + LoadFinishTime - CreateTime = Time consumed by the entire import task + LoadFinishTime - LoadStartTime = The entire Broker load import task execution time = the time consumed by the entire import task - the time the import task waits + ``` + ++ URL + + The error data sample of the import task can be obtained by accessing the URL address. When there is no error data in this import, the URL field is N/A. + ++ JobDetails + + Display some details of the running status of the job. Including file number, total file size(Bytes), num of sub tasks, scanned rows, related backend ids and unfinished backend ids. + + ``` + {"Unfinished backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"ScannedRows":2390016,"TaskNumber":1,"All backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"FileNumber":1,"FileSize":1073741824} + ``` + + This info will be updated every 5 seconds. the ScannedRows only for displaying the job progress, not indicate the real numbers. + +### Cancel load + +When the Broker load job status is not CANCELLED or FINISHED, it can be manually cancelled by the user. When canceling, you need to specify a Label for the import task to be cancelled. Canceling Import command syntax can perform `HELP CANCEL LOAD` view. + +## Relevant System Configuration + +### FE configuration + +The following configurations belong to the Broker load system-level configuration, which acts on all Broker load import tasks. Configuration values are adjusted mainly by modifying `fe.conf`. + ++ min\_bytes\_per\_broker\_scanner/max\_bytes\_per\_broker\_scanner/max\_broker\_concurrency + + The first two configurations limit the minimum and maximum amount of data processed by a single BE. The third configuration limits the maximum number of concurrent imports for a job. The minimum amount of data processed, the maximum number of concurrencies, the size of source files and the number of BEs in the current cluster **together determine the concurrency of this import**. + + ``` + The number of concurrent imports = Math. min (source file size / minimum throughput, maximum concurrency, current number of BE nodes) + Processing capacity of this import of a single BE = source file size / concurrency of this import + ``` + + Usually the maximum amount of data supported by an import job is `max_bytes_per_broker_scanner * number of BE nodes`. If you need to import a larger amount of data, you need to adjust the size of the `max_bytes_per_broker_scanner` parameter appropriately. + +Default configuration: + + ``` + Parameter name: min_bytes_per_broker_scanner, default 64MB, unit bytes. + Parameter name: max_broker_concurrency, default 10. + Parameter name: max_bytes_per_broker_scanner, default 3G, unit bytes. + ``` + +## Best Practices + +### Application scenarios + +The most appropriate scenario to use Broker load is the scenario of raw data in a file system (HDFS, BOS, AFS). Secondly, since Broker load is the only way of asynchronous import in a single import, users can also consider using Broker load if they need to use asynchronous access in importing large files. + +### Data volume + +We will only discuss the case of a single BE. If the user cluster has more than one BE, the amount of data in the heading below should be multiplied by the number of BEs. For example, if the user has three BEs, then the number below 3G (including) should be multiplied by 3, that is, under 9G (including). + ++ Below 3G (including) + + Users can submit Broker load to create import requests directly. + ++ Over 3G + + Since the maximum processing capacity of a single imported BE is 3G, the imported files over 3G need to be imported by adjusting the import parameters of Broker load to achieve the import of large files. + + 1. Modify the maximum number of scans and concurrency of a single BE according to the current number of BEs and the size of the original file. + + ``` + Modify the configuration in fe.conf + + max_broker_concurrency = BE number + The amount of data processed by a single BE for the current import task = the original file size / max_broker_concurrency + Max_bytes_per_broker_scanner >= the amount of data processed by a single BE of the current import task + + For example, a 100G file with 10 BEs in the cluster + max_broker_concurrency = 10 + Max================ + + ``` + + After modification, all BEs process import tasks concurrently, and each BE processes part of the original file. + + *Note: The configurations in both FEs are system configurations, that is to say, their modifications work on all Broker load tasks.* + + 2. Customize the timeout time of the current import task when creating the import + + ``` + Current import task single BE processing data volume / user Doris cluster slowest import speed (MB/s) >= current import task timeout time >= current import task single BE processing data volume / 10M/s + + For example, a 100G file with 10 BEs in the cluster + Timeout > 1000s = 10G / 10M /s + + ``` + + 3. When the user finds that the timeout time calculated in the second step exceeds the default maximum time-out time for importing the system by 4 hours. + + At this time, it is not recommended that users directly increase the maximum time-out to solve the problem. If the single import time exceeds the default maximum import timeout of 4 hours, it is better to solve the problem by splitting the file to be imported and importing it several times. The main reason is that if a single import exceeds 4 hours, the time cost of retry after import failure is very high. + + The maximum amount of imported file data expected by the Doris cluster can be calculated by the following formula: + + ``` + Expected maximum imported file data = 14400s * 10M / s * BE number + For example, the BE number of clusters is 10. + Expected maximum imported file data volume = 14400 * 10M / s * 10 = 1440000M 1440G + + Note: The average user's environment may not reach the speed of 10M/s, so it is recommended that more than 500G files be split and imported. + + ``` + +### Complete examples + +Data situation: User data in HDFS, file address is hdfs://abc.com:8888/store_sales, HDFS authentication user name is root, password is password, data size is about 30G, hope to import into database bj_sales table store_sales. + +Cluster situation: The number of BEs in the cluster is about 3, and the Broker name is broker. + ++ Step 1: After the calculation of the above method, the single BE import quantity is 10G, then the configuration of FE needs to be modified first, and the maximum amount of single BE import is changed to: + + ``` + max_bytes_per_broker_scanner = 10737418240 + + ``` + ++ Step 2: Calculated, the import time is about 1000s, which does not exceed the default timeout time. No custom timeout time for import can be configured. + ++ Step 3: Create import statements + + ``` + LOAD LABEL bj_sales.store_sales_broker_load_01 + ( + DATA INFILE("hdfs://abc.com:8888/store_sales") + INTO TABLE store_sales + ) + WITH BROKER 'broker' + ("username"="root", "password"="password"); + ``` + +## Common Questions + +* failed with : `Scan bytes per broker scanner exceed limit:xxx` + + Refer to the Best Practices section of the document to modify the FE configuration items `max_bytes_per_broker_scanner` and `max_broker_concurrency'.` + +* failed with ：`failed to send batch` or `TabletWriter add batch with unknown id` + + Refer to **General System Configuration** in **BE Configuration** in the Import Manual (./load-manual.md), and modify `query_timeout` and `streaming_load_rpc_max_alive_time_sec` appropriately. + +* failed with : `LOAD_RUN_FAIL; msg: Invalid Column Name: xxx` +     +     If it is PARQUET or ORC format data, you need to keep the column names in the file header consistent with the column names in the doris table, such as: +     `` ` +     (tmp_c1, tmp_c2) +     SET +     ( +         id = tmp_c2, +         name = tmp_c1 +     ) +     `` ` +     Represents getting the column with (tmp_c1, tmp_c2) as the column name in parquet or orc, which is mapped to the (id, name) column in the doris table. If set is not set, the column names in the column are used as the mapping relationship. + +     Note: If the orc file directly generated by some hive versions is used, the table header in the orc file is not the column name in the hive meta, but (_col0, _col1, _col2, ...), which may cause the Invalid Column Name error, then You need to use set for mapping. diff --git a/docs/en/administrator-guide/load-data/delete-manual.md b/docs/en/administrator-guide/load-data/delete-manual.md new file mode 100644 index 00000000000000..b080395a941b47 --- /dev/null +++ b/docs/en/administrator-guide/load-data/delete-manual.md @@ -0,0 +1,188 @@ +--- +{ + "title": "Delete", + "language": "en" +} +--- + + + +# Delete + +Unlike other import methods, delete is a synchronization process. Similar to insert into, all delete operations are an independent import job in Doris. Generally, delete statements need to specify tables, partitions and delete conditions to tell which data to be deleted, and the data on base index and rollup index will be deleted at the same time. + + +## Syntax + +The delete statement's syntax is as follows： + +``` +DELETE FROM table_name [PARTITION partition_name] +WHERE +column_name1 op value[ AND column_name2 op value ...]; +``` + +example 1： + +``` +DELETE FROM my_table PARTITION p1 WHERE k1 = 3; +``` + +example 2: + +``` +DELETE FROM my_table PARTITION p1 WHERE k1 < 3 AND k2 = "abc"; +``` + +The following describes the parameters used in the delete statement: + +* PARTITION + + The target partition of the delete statement. If not specified, the table must be a single partition table, otherwise it cannot be deleted + +* WHERE + + The conditiona of the delete statement. All delete statements must specify a where condition. + +说明： + +1. The type of `OP` in the WHERE condition can only include `=, >, <, > =, < =,!=`. Currently, where key in (value1, Value2, value3) mode is not supported yet, may be added this support later. +2. The column in the WHERE condition can only be the `key` column. +3. Cannot delete when the `key` column does not exist in any rollup table. +4. Each condition in WHERE condition can only be realated by `and`. If you want `or`, you are suggested to write these conditions into two delete statements. +5. If the specified table is a range partitioned table, `PARTITION` must be specified unless the table is a single partition table,. +6. Unlike the insert into command, delete statement cannot specify `label` manually. You can view the concept of `label` in [Insert Into] (./insert-into-manual.md) + +## Delete Result + +The delete command is an SQL command, and the returned results are synchronous. It can be divided into the following types: + +1. Successful visible + + If delete completes successfully and is visible, the following results will be returned, `query OK` indicates success. + + ``` + mysql> delete from test_tbl PARTITION p1 where k1 = 1; + Query OK, 0 rows affected (0.04 sec) + {'label':'delete_e7830c72-eb14-4cb9-bbb6-eebd4511d251', 'status':'VISIBLE', 'txnId':'4005'} + ``` + +2. Submitted successfully, but not visible + + + The transaction submission of Doris is divided into two steps: submission and publish version. Only after the publish version step is completed, the result will be visible to the user. If it has been submitted successfully, then it can be considered that the publish version step will eventually success. Doris will try to wait for publishing for a period of time after submitting. If it has timed out, even if the publishing version has not been completed, it will return to the user in priority and prompt the user that the submission has been completed but not visible. If delete has been committed and executed, but has not been published and visible, the following results will be returned. + + ``` + mysql> delete from test_tbl PARTITION p1 where k1 = 1; + Query OK, 0 rows affected (0.04 sec) + {'label':'delete_e7830c72-eb14-4cb9-bbb6-eebd4511d251', 'status':'VISIBLE', 'txnId':'4005', 'err':'delete job is committed but may be taking effect later' } + ``` + + The result will return a JSON string at the same time: + + `affected rows`: Indicates the row affected by this deletion. Since the deletion of Doris is currently a logical deletion, the value is always 0. + + `label`: The label generated automatically to be the signature of the delete jobs. Each job has a unique label within a single database. + + `status`: Indicates whether the data deletion is visible. If it is visible, `visible` will be displayed. If it is not visible, `committed` will be displayed. + + + `txnId`: The transaction ID corresponding to the delete job + + `err`: Field will display some details of this deletion + +3. Commit failed, transaction cancelled + + If the delete statement is not submitted successfully, it will be automatically aborted by Doris and the following results will be returned + + + ``` + mysql> delete from test_tbl partition p1 where k1 > 80; + ERROR 1064 (HY000): errCode = 2, detailMessage = {错误原因} + ``` + + example： + + A timeout deletion will return the timeout and unfinished replicas displayed as ` (tablet = replica)` + + + ``` + mysql> delete from test_tbl partition p1 where k1 > 80; + ERROR 1064 (HY000): errCode = 2, detailMessage = failed to delete replicas from job: 4005, Unfinished replicas:10000=60000, 10001=60000, 10002=60000 + ``` + + **The correct processing logic for the returned results of the delete operation is as follows:** + + 1. If `Error 1064 (HY000)` is returned, deletion fails + + 2. If the returned result is `Query OK`, the deletion is successful + + 1. If `status` is `committed`, the data deletion is committed and will be eventually invisible. Users can wait for a while and then use the `show delete` command to view the results. + 2. If `status` is `visible`, the data have been deleted successfully. + +## Relevant Configuration + +### FE configuration + +**TIMEOUT configuration** + +In general, Doris's deletion timeout is limited from 30 seconds to 5 minutes. The specific time can be adjusted through the following configuration items + +* tablet\_delete\_timeout\_second + + The timeout of delete itself can be elastically changed by the number of tablets in the specified partition. This configuration represents the average timeout contributed by a tablet. The default value is 2. + + Assuming that there are 5 tablets under the specified partition for this deletion, the timeout time available for the deletion is 10 seconds. Because the minimum timeout is 30 seconds which is higher than former timeout time, the final timeout is 30 seconds. + +* load\_straggler\_wait\_second + + If the user estimates a large amount of data, so that the upper limit of 5 minutes is insufficient, the user can adjust the upper limit of timeout through this item, and the default value is 300. + + **The specific calculation rule of timeout(seconds)** + + `TIMEOUT = MIN(load_straggler_wait_second, MAX(30, tablet_delete_timeout_second * tablet_num))` + +* query_timeout + + Because delete itself is an SQL command, the deletion statement is also limited by the session variables, and the timeout is also affected by the session value `query'timeout`. You can increase the value by `set query'timeout = xxx`. + +## Show delete history + +1. The user can view the deletion completed in history through the show delete statement. + + Syntax + + ``` + SHOW DELETE [FROM db_name] + ``` + + example + + ``` + mysql> show delete from test_db; + +-----------+---------------+---------------------+-----------------+----------+ + | TableName | PartitionName | CreateTime | DeleteCondition | State | + +-----------+---------------+---------------------+-----------------+----------+ + | empty_tbl | p3 | 2020-04-15 23:09:35 | k1 EQ "1" | FINISHED | + | test_tbl | p4 | 2020-04-15 23:09:53 | k1 GT "80" | FINISHED | + +-----------+---------------+---------------------+-----------------+----------+ + 2 rows in set (0.00 sec) + ``` + diff --git a/docs/en/administrator-guide/load-data/insert-into-manual.md b/docs/en/administrator-guide/load-data/insert-into-manual.md new file mode 100644 index 00000000000000..5795575acee4b8 --- /dev/null +++ b/docs/en/administrator-guide/load-data/insert-into-manual.md @@ -0,0 +1,275 @@ +--- +{ + "title": "Insert Into", + "language": "en" +} +--- + + + +# Insert Into + +The use of Insert Into statements is similar to that of Insert Into statements in databases such as MySQL. But in Doris, all data writing is a separate import job. So Insert Into is also introduced here as an import method. + +The main Insert Into command contains the following two kinds; + +* INSERT INTO tbl SELECT ... +* INSERT INTO tbl (col1, col2, ...) VALUES (1, 2, ...), (1,3, ...); + +The second command is for Demo only, not in a test or production environment. + +## Basic operations + +### Create a Load + +The Insert Into command needs to be submitted through MySQL protocol. Creating an import request returns the import result synchronously. + +Grammar: + +``` +INSERT INTO table_name [WITH LABEL label] [partition_info] [col_list] [query_stmt] [VALUES]; +``` + +Examples: + +``` +INSERT INTO tbl2 WITH LABEL label1 SELECT * FROM tbl3; +INSERT INTO tbl1 VALUES ("qweasdzxcqweasdzxc"), ("a"); +``` + +**Notice** + +When using `CTE(Common Table Expressions)` as the query part of insert operation, the `WITH LABEL` or column list part must be specified. +For example: + +``` +INSERT INTO tbl1 WITH LABEL label1 +WITH cte1 AS (SELECT * FROM tbl1), cte2 AS (SELECT * FROM tbl2) +SELECT k1 FROM cte1 JOIN cte2 WHERE cte1.k1 = 1; + +INSERT INTO tbl1 (k1) +WITH cte1 AS (SELECT * FROM tbl1), cte2 AS (SELECT * FROM tbl2) +SELECT k1 FROM cte1 JOIN cte2 WHERE cte1.k1 = 1; +``` + +The following is a brief introduction to the parameters used in creating import statements: + ++ partition\_info + + Import the target partition of the table. If the target partition is specified, only the data that matches the target partition will be imported. If not specified, the default value is all partitions of the table. + ++ col\_list + + The target column of the import table can exist in any order. If no target column is specified, the default value is all columns in this table. If a column in the table does not exist in the target column, the column needs a default value, otherwise Insert Into will fail. + + If the result column type of the query statement is inconsistent with the type of the target column, an implicit type conversion is invoked. If the conversion is not possible, the Insert Into statement will report a parsing error. + ++ query\_stmt + + Through a query statement, the results of the query statement are imported into other tables in Doris system. Query statements support any SQL query syntax supported by Doris. + ++ VALUES + + Users can insert one or more data through VALUES grammar. + + *Note: VALUES is only suitable for importing several pieces of data as DEMO. It is totally unsuitable for any test and production environment. Doris system itself is not suitable for single data import scenarios. It is recommended to use INSERT INTO SELECT for batch import.* + +* WITH LABEL + + INSERT as a load job, it can also be with a label. If not with a label, Doris will use a UUID as label. + + This feature needs Doris version 0.11+. + + *Note: It is recommended that Label be specified rather than automatically allocated by the system. If the system allocates automatically, but during the execution of the Insert Into statement, the connection is disconnected due to network errors, etc., then it is impossible to know whether Insert Into is successful. If you specify Label, you can view the task results again through Label.* + +### Load results + +Insert Into itself is a SQL command, and the return result is divided into the following types according to the different execution results: + +1. Result set is empty + + If the result set of the insert corresponding SELECT statement is empty, it is returned as follows: + + ``` + mysql> insert into tbl1 select * from empty_tbl; + Query OK, 0 rows affected (0.02 sec) + ``` + + `Query OK` indicates successful execution. `0 rows affected` means that no data was loaded. + +2. The result set is not empty + + In the case where the result set is not empty. The returned results are divided into the following situations: + + 1. Insert is successful and data is visible: + + ``` + mysql> insert into tbl1 select * from tbl2; + Query OK, 4 rows affected (0.38 sec) + {'label': 'insert_8510c568-9eda-4173-9e36-6adc7d35291c', 'status': 'visible', 'txnId': '4005'} + + mysql> insert into tbl1 with label my_label1 select * from tbl2; + Query OK, 4 rows affected (0.38 sec) + {'label': 'my_label1', 'status': 'visible', 'txnId': '4005'} + + mysql> insert into tbl1 select * from tbl2; + Query OK, 2 rows affected, 2 warnings (0.31 sec) + {'label': 'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status': 'visible', 'txnId': '4005'} + + mysql> insert into tbl1 select * from tbl2; + Query OK, 2 rows affected, 2 warnings (0.31 sec) + {'label': 'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status': 'committed', 'txnId': '4005'} + ``` + + `Query OK` indicates successful execution. `4 rows affected` means that a total of 4 rows of data were imported. `2 warnings` indicates the number of lines to be filtered. + + Also returns a json string: + + ``` + {'label': 'my_label1', 'status': 'visible', 'txnId': '4005'} + {'label': 'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status': 'committed', 'txnId': '4005'} + {'label': 'my_label1', 'status': 'visible', 'txnId': '4005', 'err': 'some other error'} + ``` + + `label` is a user-specified label or an automatically generated label. Label is the ID of this Insert Into load job. Each load job has a label that is unique within a single database. + + `status` indicates whether the loaded data is visible. If visible, show `visible`, if not, show` committed`. + + `txnId` is the id of the load transaction corresponding to this insert. + + The `err` field displays some other unexpected errors. + + When user need to view the filtered rows, the user can use the following statement + + ``` + show load where label = "xxx"; + ``` + + The URL in the returned result can be used to query the wrong data. For details, see the following **View Error Lines** Summary. +     + **"Data is not visible" is a temporary status, this batch of data must be visible eventually** + + You can view the visible status of this batch of data with the following statement: + + ``` + show transaction where id = 4005; + ``` + + If the `TransactionStatus` column in the returned result is `visible`, the data is visible. + + 2. Insert fails + + Execution failure indicates that no data was successfully loaded, and returns as follows: + + ``` + mysql> insert into tbl1 select * from tbl2 where k1 = "a"; + ERROR 1064 (HY000): all partitions have no load data. Url: http://10.74.167.16:8042/api/_load_error_log?file=__shard_2/error_log_insert_stmt_ba8bb9e158e4879-ae8de8507c0bf8a2_ba8bb9e158e4879_ae8de850e8de850 + ``` + + Where `ERROR 1064 (HY000): all partitions have no load data` shows the reason for the failure. The latter url can be used to query the wrong data. For details, see the following **View Error Lines** Summary. + +**In summary, the correct processing logic for the results returned by the insert operation should be:** + +1. If the returned result is `ERROR 1064 (HY000)`, it means that the import failed. +2. If the returned result is `Query OK`, it means the execution was successful. + + 1. If `rows affected` is 0, the result set is empty and no data is loaded. + 2. If `rows affected` is greater than 0: + 1. If `status` is` committed`, the data is not yet visible. You need to check the status through the `show transaction` statement until `visible`. + 2. If `status` is` visible`, the data is loaded successfully. + 3. If `warnings` is greater than 0, it means that some data is filtered. You can get the url through the `show load` statement to see the filtered rows. + +## Relevant System Configuration + +### FE configuration + ++ time out + + The timeout time of the import task (in seconds) will be cancelled by the system if the import task is not completed within the set timeout time, and will become CANCELLED. + + At present, Insert Into does not support custom import timeout time. All Insert Into imports have a uniform timeout time. The default timeout time is 1 hour. If the imported source file cannot complete the import within the specified time, the parameter ``insert_load_default_timeout_second`` of FE needs to be adjusted. + + At the same time, the Insert Into statement receives the restriction of the Session variable `query_timeout`. You can increase the timeout time by `SET query_timeout = xxx;` in seconds. + +### Session Variables + ++ enable\_insert\_strict + + The Insert Into import itself cannot control the tolerable error rate of the import. Users can only use the Session parameter `enable_insert_strict`. When this parameter is set to false, it indicates that at least one data has been imported correctly, and then it returns successfully. When this parameter is set to true, the import fails if there is a data error. The default is false. It can be set by `SET enable_insert_strict = true;`. + ++ query u timeout + + Insert Into itself is also an SQL command, so the Insert Into statement is also restricted by the Session variable `query_timeout`. You can increase the timeout time by `SET query_timeout = xxx;` in seconds. + +## Best Practices + +### Application scenarios +1. Users want to import only a few false data to verify the functionality of Doris system. The grammar of INSERT INTO VALUS is suitable at this time. +2. Users want to convert the data already in the Doris table into ETL and import it into a new Doris table, which is suitable for using INSERT INTO SELECT grammar. +3. Users can create an external table, such as MySQL external table mapping a table in MySQL system. Or create Broker external tables to map data files on HDFS. Then the data from the external table is imported into the Doris table for storage through the INSERT INTO SELECT grammar. + +### Data volume +Insert Into has no limitation on the amount of data, and large data imports can also be supported. However, Insert Into has a default timeout time, and the amount of imported data estimated by users is too large, so it is necessary to modify the system's Insert Into import timeout time. + +``` +Import data volume = 36G or less than 3600s*10M/s +Among them, 10M/s is the maximum import speed limit. Users need to calculate the average import speed according to the current cluster situation to replace 10M/s in the formula. +``` + +### Complete examples + +Users have a table store sales in the database sales. Users create a table called bj store sales in the database sales. Users want to import the data recorded in the store sales into the new table bj store sales. The amount of data imported is about 10G. + +``` +large sales scheme +(id, total, user_id, sale_timestamp, region) + +Order large sales schedule: +(id, total, user_id, sale_timestamp) + +``` + +Cluster situation: The average import speed of current user cluster is about 5M/s + ++ Step1: Determine whether you want to modify the default timeout of Insert Into + + ``` + Calculate the approximate time of import + 10G / 5M /s = 2000s + + Modify FE configuration + insert_load_default_timeout_second = 2000 + ``` + ++ Step2: Create Import Tasks + + Since users want to ETL data from a table and import it into the target table, they should use the Insert in query\\stmt mode to import it. + + ``` + INSERT INTO bj_store_sales SELECT id, total, user_id, sale_timestamp FROM store_sales where region = "bj"; + ``` + +## Common Questions + +* View the wrong line + + Because Insert Into can't control the error rate, it can only tolerate or ignore the error data completely by `enable_insert_strict`. So if `enable_insert_strict` is set to true, Insert Into may fail. If `enable_insert_strict` is set to false, then only some qualified data may be imported. However, in either case, Doris is currently unable to provide the ability to view substandard data rows. Therefore, the user cannot view the specific import error through the Insert Into statement. + + The causes of errors are usually: source data column length exceeds destination data column length, column type mismatch, partition mismatch, column order mismatch, etc. When it's still impossible to check for problems. At present, it is only recommended that the SELECT command in the Insert Into statement be run to export the data to a file, and then import the file through Stream load to see the specific errors. diff --git a/docs/en/administrator-guide/load-data/load-manual.md b/docs/en/administrator-guide/load-data/load-manual.md new file mode 100644 index 00000000000000..d502298a66a28c --- /dev/null +++ b/docs/en/administrator-guide/load-data/load-manual.md @@ -0,0 +1,219 @@ +--- +{ + "title": "Introduction Overview", + "language": "en" +} +--- + + + +# Introduction Overview + +The Load function is to import the user's raw data into Doris. After successful import, users can query data through Mysql client. + +Doris supports multiple imports. It is recommended to read this document in full first, and then to view the detailed documents of their respective import modes according to the selected import mode. + +## Basic concepts + +1. Frontend (FE): Metadata and scheduling nodes of Doris system. In the import process, it is mainly responsible for the generation of import planning and the scheduling of import tasks. +2. Backend (BE): The computing and storage nodes of Doris system. In the import process, it is mainly responsible for ETL and storage of data. +3. Broker: Broker is an independent stateless process. It encapsulates the file system interface and provides Doris with the ability to read files in the remote storage system. +4. Load job: The import job reads the source data submitted by the user, transforms or cleans it, and imports the data into the Doris system. After the import is completed, the data can be queried by the user. +5. Label: All import jobs have a Label. Label is unique in a database and can be specified by the user or automatically generated by the system to identify an import job. The same Label can only be used for a successful import job. +6. MySQL Protocol/HTTP Protocol: Doris provides two kinds of access protocol interfaces. MySQL protocol and HTTP protocol. Part of the import mode uses MySQL protocol interface to submit jobs, and part of the import mode uses HTTP protocol interface to submit jobs. + +## Load mode + +To adapt to different data import requirements, Doris system provides five different import methods. Each import mode supports different data sources and has different usage modes (asynchronous, synchronous). + +All import methods support CSV data format. Broker load also supports parquet and orc data format. + +For instructions on each import mode, please refer to the operation manual for a single import mode. + +* Broker load + + Access and read external data sources (such as HDFS) through the Broker process and import them into Doris. The user submits the import job through Mysql protocol and executes it asynchronously. View the import results through the `SHOW LOAD` command. + +* Stream load + + Users submit requests through HTTP protocol and create imports with raw data. It is mainly used to quickly import data from local files or data streams into Doris. The Import command returns the import result synchronously. + +* Insert + + Similar to the Insert statement in MySQL, Doris provides `INSERT INTO tbl SELECT ...;`reading data from Doris's table and importing it into another table. Or by `INSERT INTO tbl VALUES (...);` Insert a single piece of data. + +* Multi load + + Users submit multiple import jobs through HTTP protocol. Multi Load guarantees the atomic validity of multiple import jobs. + +* Routine load + + Users submit routine import jobs through MySQL protocol, generate a resident thread, read and import data from data sources (such as Kafka) uninterruptedly into Doris. + +## Basic Principles + +### Import execution process + + +``` ++---------+ +---------+ +----------+ +-----------+ +| | | | | | | | +| PENDING +----->+ ETL +----->+ LOADING +----->+ FINISHED | +| | | | | | | | ++---------+ +---+-----+ +----+-----+ +-----------+ + | | | + | | | + | | | + | | | +-----------+ + | | | | | + +---------------+-----------------+------------> CANCELLED | + | | + +-----------+ + +``` + +As shown above, an import operation mainly goes through the four stages above. + ++ PENDING (not required): Only Broker Load has this stage. Broker Load is submitted by the user and stays at this stage for a short time until it is scheduled by Scheduler in FE. Scheduler's schedule interval is 5 seconds. + ++ ETL (not required): This stage exists before version 0.10.0 (included), mainly for transforming raw data according to user declaration and filtering raw data that does not meet the requirements. In the version after 0.10.0, the ETL phase no longer exists, and the work of data transformation is merged into the LOADING phase. + ++ LOADING: This stage is mainly used to push the transformed data into the corresponding BE storage before version 0.10.0 (including). In the version after 0.10.0, the data is cleaned and changed first, and then sent to BE storage. When all imported data are imported, the process of waiting for validity enters, and Load job is still LOADING. + ++ FINISHED: After all the data involved in Load Job takes effect, the state of Load Job becomes FINISHED. Data imported after FINISHED can be queried. + ++ CANCELLED: Before job FINISH, jobs may be cancelled and entered the CANCELLED state. For example, the user manually cancels, or imports errors. CANCELLED is also the final state of Load Job and cannot be executed again. + +In the above stage, except for the PENDING to LOADING stage, which is scheduled by Scheduler, the transfer before other stages is implemented by callback mechanism. + +### Label and Atomicity + +Doris provides atomic assurance for all import methods. It ensures that the data in the same import operation is valid for atoms. There will be no case of importing only part of the data. + +At the same time, each import job has a Label designated by the user or automatically generated by the system. Label is unique in a database. When an import job corresponding to a Label is successful enough, the import job cannot be submitted repeatedly using the Label. If the import job corresponding to Label fails, it can be reused. + +Users can use Label mechanism to ensure that the data corresponding to Label can be imported at most once, at the level of At-Most-One semantics. + + +## Synchronization and asynchronization + +Doris's current import methods fall into two categories, synchronous and asynchronous. If an external program accesses Doris's import function, it is necessary to determine which type of import mode is used and then determine the access logic. + +### Synchronization + +Synchronized import means that users create import tasks, Doris executes import synchronously, and returns user import results after execution. Users can directly determine whether the import is successful or not by synchronizing the results returned by creating the import task command. + +The import methods of synchronous type are **Stream load**, **Insert**. + +Operation steps: + +1. Users (external systems) create import tasks. +2. Doris returns the import result. +3. The user (external system) judges the import result and can submit the import task again if it fails. + +*Note: If the user returns the import synchronously and the amount of data imported is too large, it may take a long time to create the import request to return the result.* + +### Asynchronism +Asynchronous import means that after the user creates the import task, Doris directly returns to the successful creation. **Successful creation does not mean that data has been imported into**. The import task will be executed asynchronously. After successful creation, users need to send a polling command to check the status of the import job. If the creation fails, you can judge whether it needs to be created again based on the failure information. + +The ways to import asynchronous types are: **Broker load**, **Multi load**. + +Operation steps: + +1. Users (external systems) create import tasks. +2. Doris returns the import creation result. +3. User (external system) judges the result of import creation, success enters 4, failure returns to retry to create import, return to 1. +4. The user (external system) polls to see the import task until the status changes to FINISHED or CANCELLED. + +### Notes +Neither asynchronous nor synchronous import types should be retried endlessly after Doris returns an import failure or an import creation failure. **After a limited number of retries and failures, the external system retains the failure information. Most of the retries fail because of the problem of using method or data itself.** + +## Memory Limit + +Users can limit the memory usage of a single load by setting parameters to prevent the system from taking up too much memory and causing the system OOM. +Different load methods restrict memory in a slightly different way. You can refer to the respective load manuals for viewing. + +An load job is usually distributed across multiple Backends. The load memory limit is the memory usage of load job on a single Backend, not memory usage across the cluster. + +At the same time, each Backend sets the overall upper limit of the memory available for load. See the General System Configuration section below for specific configuration. This configuration limits the overall memory usage limit for all load tasks running on this Backend. + +Smaller memory limits can affect load efficiency because the load process can frequently write in-memory data back to disk because memory reaches the upper limit. Excessive memory limits can cause system OOM when load concurrency is high. Therefore, you need to properly set the load memory limit according to your needs. + +## Best Practices + +When users access Doris import, they usually use program access mode to ensure that data is imported into Doris regularly. Below is a brief description of the best practices for program access to Doris. + +1. Choose the appropriate import mode: According to the location of the data source, choose the import mode. For example, if raw data is stored on HDFS, import it using Broker load. +2. Protocol for determining the import mode: If Broker load import mode is selected, external systems need to be able to submit and view import jobs regularly using MySQL protocol. +3. Determine the type of import mode: import mode is synchronous or asynchronous. For example, Broker load is an asynchronous import mode. After submitting the creation import, the external system must call the check import command to determine whether the import is successful or not based on the results of the check import command. +4. Label generation strategy: Label generation strategy needs to be satisfied, and each batch of data is unique and fixed. Doris can then guarantee At-Most-Once. +5. The program itself guarantees At-Least-Once: The external system needs to guarantee its own At-Least-Once, so that Exactly-Once of the import process can be guaranteed. + +## General System Configuration + +The following sections explain several system-level configurations that are common to all imports. + +### FE configuration + +The following configuration belongs to the system configuration of FE, which can be modified by modifying the configuration file ``fe.conf``. + ++ max\_load\_timeout\_second and min\_load\_timeout\_second + + The two configurations mean the maximum import timeout time and the minimum import timeout time in seconds. The default maximum timeout time is 3 days and the default minimum timeout time is 1 second. User-defined import timeouts should not exceed this range. This parameter is applicable to all import modes. + ++ desired\_max\_waiting\_jobs + + The maximum number of imported tasks in the waiting queue is 100 by default. New import requests are rejected when the number of imports in the PENDING state (i.e. waiting for execution) in FE exceeds that value. + + This configuration is only valid for asynchronous execution of imports. When the number of import waiting for asynchronous execution exceeds the default value, subsequent creation of import requests will be rejected. + ++ max\_running\_txn\_num\_per\_db + + The implication of this configuration is that the maximum number of imports running in each database (no distinction between import types, uniform counting). When the current database is running more than the maximum number of imports, subsequent imports will not be executed. If the job is imported synchronously, the import will be rejected. If it is an asynchronous import job. The job will wait in the queue. + +### BE configuration + +The following configuration belongs to the BE system configuration, which can be modified by modifying the BE configuration file `be.conf`. + ++ push\_write\_mbytes\_per\_sec + + Writing speed limit for a single Tablet on BE. The default is 10, or 10MB/s. Usually the maximum write speed of BE to a single Tablet is between 10 and 30 MB/s, depending on Schema and the system. This parameter can be adjusted appropriately to control the import speed. + ++ write\_buffer\_size + + The imported data will be written to a memtable on BE, and the memtable will not be written back to disk until it reaches the threshold. The default size is 100MB. Too small threshold may result in a large number of small files on BE. This threshold can be increased appropriately to reduce the number of files. However, excessive thresholds can lead to RPC timeouts, as shown in the configuration instructions below. + ++ tablet\_writer\_rpc\_timeout\_sec + + During the import process, a Batch (1024 rows) RPC timeout is sent. Default 600 seconds. Because the RPC may involve multiple memtable writes, it may cause RPC timeouts, which can be adjusted appropriately to reduce timeout errors (such as `send batch fail`). At the same time, if the `write_buffer_size` configuration is increased, this parameter needs to be adjusted appropriately. + ++ streaming\_load\_rpc\_max\_alive\_time\_sec + + During the import process, Doris opens a Writer for each Tablet to receive and write data. This parameter specifies Writer's waiting timeout time. If Writer does not receive any data at this time, Writer will be destroyed automatically. When the system processing speed is slow, Writer may not receive the next batch of data for a long time, resulting in import error: `Tablet Writer add batch with unknown id`. This configuration can be increased appropriately at this time. The default is 600 seconds. + ++ load\_process\_max\_memory\_limit\_bytes and load\_process\_max\_memory\_limit\_percent + + These two parameters limit the upper memory limit that can be used to load tasks on a single Backend. The maximum memory and maximum memory percentage are respectively. `load_process_max_memory_limit_percent` defaults to 80%, which is 80% of the `mem_limit` configuration. That is, if the physical memory is M, the default load memory limit is M * 80% * 80%. + +     `load_process_max_memory_limit_bytes` defaults to 100GB. The system takes the smaller of the two parameters as the final Backend load memory usage limit. + ++ label\_keep\_max\_second + + The retention time of load job which is FINISHED or CANCELLED. The record of load job will be kept in Doris system for a period of time which is determined by this parameter. The default time of this parameter is 3 days. This parameter is common to all types of load job. diff --git a/docs/en/administrator-guide/load-data/routine-load-manual.md b/docs/en/administrator-guide/load-data/routine-load-manual.md new file mode 100644 index 00000000000000..e13b39b1d63aa8 --- /dev/null +++ b/docs/en/administrator-guide/load-data/routine-load-manual.md @@ -0,0 +1,297 @@ +--- +{ + "title": "Routine Load", + "language": "en" +} +--- + + + +# Routine Load + +The Routine Load feature provides users with a way to automatically load data from a specified data source. + +This document describes the implementation principles, usage, and best practices of this feature. + +## Glossary + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, the backend node of Doris. Responsible for query execution and data storage. +* RoutineLoadJob: A routine load job submitted by the user. +* JobScheduler: A routine load job scheduler for scheduling and dividing a RoutineLoadJob into multiple Tasks. +* Task: RoutineLoadJob is divided by JobScheduler according to the rules. +* TaskScheduler: Task Scheduler. Used to schedule the execution of a Task. + +## Principle + +``` + +---------+ + | Client | + +----+----+ + | ++-----------------------------+ +| FE | | +| +-----------v------------+ | +| | | | +| | Routine Load Job | | +| | | | +| +---+--------+--------+--+ | +| | | | | +| +---v--+ +---v--+ +---v--+ | +| | task | | task | | task | | +| +--+---+ +---+--+ +---+--+ | +| | | | | ++-----------------------------+ + | | | + v v v + +---+--+ +--+---+ ++-----+ + | BE | | BE | | BE | + +------+ +------+ +------+ + +``` + +As shown above, the client submits a routine load job to FE. + +FE splits an load job into several Tasks via JobScheduler. Each Task is responsible for loading a specified portion of the data. The Task is assigned by the TaskScheduler to the specified BE. + +On the BE, a Task is treated as a normal load task and loaded via the Stream Load load mechanism. After the load is complete, report to FE. + +The JobScheduler in the FE continues to generate subsequent new Tasks based on the reported results, or retry the failed Task. + +The entire routine load job completes the uninterrupted load of data by continuously generating new Tasks. + +## Kafka Routine load + +Currently we only support routine load from the Kafka system. This section details Kafka's routine use and best practices. + +### Usage restrictions + +1. Support unauthenticated Kafka access and Kafka clusters certified by SSL. +2. The supported message format is csv text format. Each message is a line, and the end of the line does not contain a ** line break. +3. Only Kafka 0.10.0.0 or above is supported. + +### Create a routine load task + +The detailed syntax for creating a routine load task can be connected to Doris and execute `HELP ROUTINE LOAD;` to see the syntax help. Here is a detailed description of the precautions when creating a job. + +* columns_mapping + + `columns_mapping` is mainly used to specify the column structure of the table structure and message, as well as the conversion of some columns. If not specified, Doris will default to the columns in the message and the columns of the table structure in a one-to-one correspondence. Although under normal circumstances, if the source data is exactly one-to-one, normal data load can be performed without specifying. However, we still strongly recommend that users ** explicitly specify column mappings**. This way, when the table structure changes (such as adding a nullable column), or the source file changes (such as adding a column), the load task can continue. Otherwise, after the above changes occur, the load will report an error because the column mapping relationship is no longer one-to-one. + + In `columns_mapping` we can also use some built-in functions for column conversion. But you need to pay attention to the actual column type corresponding to the function parameters. for example: + + Suppose the user needs to load a table containing only a column of `k1` with a column type of `int`. And you need to convert the null value in the source file to 0. This feature can be implemented with the `ifnull` function. The correct way to use is as follows: + + `COLUMNS (xx, k1=ifnull(xx, "3"))` + + Note that we use `"3"` instead of `3`, although `k1` is of type `int`. Because the column type in the source data is `varchar` for the load task, the `xx` virtual column is also of type `varchar`. So we need to use `"3"` to match the match, otherwise the `ifnull` function can't find the function signature with the parameter `(varchar, int)`, and an error will occur. + + As another example, suppose the user needs to load a table containing only a column of `k1` with a column type of `int`. And you need to process the corresponding column in the source file: convert the negative number to a positive number and the positive number to 100. This function can be implemented with the `case when` function. The correct wording should be as follows: + + `COLUMNS (xx, case when xx < 0 than cast(-xx as varchar) else cast((xx + '100') as varchar) end)` + + Note that we need to convert all the parameters in `case when` to varchar in order to get the desired result. + +* where_predicates + + The type of the column in `where_predicates` is already the actual column type, so there is no need to cast to the varchar type as `columns_mapping`. Write according to the actual column type. + +* desired\_concurrent\_number + + `desired_concurrent_number` is used to specify the degree of concurrency expected for a routine job. That is, a job, at most how many tasks are executing at the same time. For Kafka load, the current actual concurrency is calculated as follows: + + ``` + Min(partition num, desired_concurrent_number, alive_backend_num, Config.max_routine_load_task_concurrrent_num) + ``` + + Where `Config.max_routine_load_task_concurrrent_num` is a default maximum concurrency limit for the system. This is a FE configuration that can be adjusted by changing the configuration. The default is 5. + + Where partition num refers to the number of partitions for the Kafka topic subscribed to. `alive_backend_num` is the current number of normal BE nodes. + +* max\_batch\_interval/max\_batch\_rows/max\_batch\_size + + These three parameters are used to control the execution time of a single task. If any of the thresholds is reached, the task ends. Where `max_batch_rows` is used to record the number of rows of data read from Kafka. `max_batch_size` is used to record the amount of data read from Kafka in bytes. The current consumption rate for a task is approximately 5-10MB/s. + + So assume a row of data 500B, the user wants to be a task every 100MB or 10 seconds. The expected processing time for 100MB is 10-20 seconds, and the corresponding number of rows is about 200000 rows. Then a reasonable configuration is: + + ``` + "max_batch_interval" = "10", + "max_batch_rows" = "200000", + "max_batch_size" = "104857600" + ``` + + The parameters in the above example are also the default parameters for these configurations. + +* max\_error\_number + + `max_error_number` is used to control the error rate. When the error rate is too high, the job will automatically pause. Because the entire job is stream-oriented, and because of the borderless nature of the data stream, we can't calculate the error rate with an error ratio like other load tasks. So here is a new way of calculating to calculate the proportion of errors in the data stream. + + We have set up a sampling window. The size of the window is `max_batch_rows * 10`. Within a sampling window, if the number of error lines exceeds `max_error_number`, the job is suspended. If it is not exceeded, the next window restarts counting the number of error lines. + + We assume that `max_batch_rows` is 200000 and the window size is 2000000. Let `max_error_number` be 20000, that is, the user expects an error behavior of 20000 for every 2000000 lines. That is, the error rate is 1%. But because not every batch of tasks consumes 200000 rows, the actual range of the window is [2000000, 2200000], which is 10% statistical error. + + The error line does not include rows that are filtered out by the where condition. But include rows that do not have a partition in the corresponding Doris table. + +* data\_source\_properties + + The specific Kakfa partition can be specified in `data_source_properties`. If not specified, all partitions of the subscribed topic are consumed by default. + + Note that when partition is explicitly specified, the load job will no longer dynamically detect changes to Kafka partition. If not specified, the partitions that need to be consumed are dynamically adjusted based on changes in the kafka partition. + +* strict\_mode + + Routine load load can turn on strict mode mode. The way to open it is to add ```"strict_mode" = "true"``` to job\_properties. The default strict mode is off. + + The strict mode mode means strict filtering of column type conversions during the load process. The strict filtering strategy is as follows: + + 1. For column type conversion, if strict mode is true, the wrong data will be filtered. The error data here refers to the fact that the original data is not null, and the result is a null value after participating in the column type conversion. + + 2. When a loaded column is generated by a function transformation, strict mode has no effect on it. + + 3. For a column type loaded with a range limit, if the original data can pass the type conversion normally, but cannot pass the range limit, strict mode will not affect it. For example, if the type is decimal(1,0) and the original data is 10, it is eligible for type conversion but not for column declarations. This data strict has no effect on it. + +#### strict mode and load relationship of source data + +Here is an example of a column type of TinyInt. + +> Note: When a column in a table allows a null value to be loaded + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|---------| +|null | \N | N/A | true or false | NULL| +|not null | aaa or 2000 | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 | 1 | true or false | correct data| + +Here the column type is Decimal(1,0) +  +> Note: When a column in a table allows a null value to be loaded + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|--------| +|null | \N | N/A | true or false | NULL| +|not null | aaa | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 or 10 | 1 | true or false | correct data| + +> Note: 10 Although it is a value that is out of range, because its type meets the requirements of decimal, strict mode has no effect on it. 10 will eventually be filtered in other ETL processing flows. But it will not be filtered by strict mode. + +#### Accessing SSL-certified Kafka clusters + +Accessing the SSL-certified Kafka cluster requires the user to provide a certificate file (ca.pem) for authenticating the Kafka Broker public key. If the Kafka cluster has both client authentication enabled, you will also need to provide the client's public key (client.pem), key file (client.key), and key password. The files needed here need to be uploaded to Doris via the `CREAE FILE` command, ** and the catalog name is `kafka`**. See `HELP CREATE FILE;` for specific help on the `CREATE FILE` command. Here is an example: + +1. Upload file + + ``` + CREATE FILE "ca.pem" PROPERTIES("url" = "https://example_url/kafka-key/ca.pem", "catalog" = "kafka"); + CREATE FILE "client.key" PROPERTIES("url" = "https://example_urlkafka-key/client.key", "catalog" = "kafka"); + CREATE FILE "client.pem" PROPERTIES("url" = "https://example_url/kafka-key/client.pem", "catalog" = "kafka"); +``` + +2. Create a routine load job + + ``` + CREATE ROUTINE LOAD db1.job1 on tbl1 + PROPERTIES + ( + "desired_concurrent_number"="1" + ) + FROM KAFKA + ( + "kafka_broker_list"= "broker1:9091,broker2:9091", + "kafka_topic" = "my_topic", + "property.security.protocol" = "ssl", + "property.ssl.ca.location" = "FILE:ca.pem", + "property.ssl.certificate.location" = "FILE:client.pem", + "property.ssl.key.location" = "FILE:client.key", + "property.ssl.key.password" = "abcdefg" + ); + ``` + +> Doris accesses Kafka clusters via Kafka's C++ API `librdkafka`. The parameters supported by `librdkafka` can be found. +> +> `https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md` + +### Viewing the status of the load job + +Specific commands and examples for viewing the status of the ** job** can be viewed with the `HELP SHOW ROUTINE LOAD;` command. + +Specific commands and examples for viewing the **Task** status can be viewed with the `HELP SHOW ROUTINE LOAD TASK;` command. + +You can only view tasks that are currently running, and tasks that have ended and are not started cannot be viewed. + +### Job Control + +The user can control the stop, pause and restart of the job by the three commands `STOP/PAUSE/RESUME`. You can view help and examples with the three commands `HELP STOP ROUTINE LOAD;`, `HELP PAUSE ROUTINE LOAD;` and `HELP RESUME ROUTINE LOAD;`. + +## other instructions + +1. The relationship between a routine load job and an ALTER TABLE operation + + * Routine load does not block SCHEMA CHANGE and ROLLUP operations. Note, however, that if the column mappings are not matched after SCHEMA CHANGE is completed, the job's erroneous data will spike and eventually cause the job to pause. It is recommended to reduce this type of problem by explicitly specifying column mappings in routine load jobs and by adding Nullable columns or columns with Default values. + * Deleting a Partition of a table may cause the loaded data to fail to find the corresponding Partition and the job will be paused. + +2. Relationship between routine load jobs and other load jobs (LOAD, DELETE, INSERT) + + * Routine load does not conflict with other LOAD jobs and INSERT operations. + * When performing a DELETE operation, the corresponding table partition cannot have any load tasks being executed. Therefore, before performing the DELETE operation, you may need to pause the routine load job and wait for the delivered task to complete before you can execute DELETE. + +3. Relationship between routine load jobs and DROP DATABASE/TABLE operations + + When the corresponding database or table is deleted, the job will automatically CANCEL. + +4. The relationship between the kafka type routine load job and kafka topic + + When the user creates a routine load declaration, the `kafka_topic` does not exist in the kafka cluster. + + * If the broker of the user kafka cluster has `auto.create.topics.enable = true` set, `kafka_topic` will be automatically created first, and the number of partitions created automatically will be in the kafka cluster** of the user side. The broker is configured with `num.partitions`. The routine job will continue to read the data of the topic continuously. + * If the broker of the user kafka cluster has `auto.create.topics.enable = false` set, topic will not be created automatically, and the routine will be paused before any data is read, with the status `PAUSED`. + + So, if the user wants to be automatically created by the routine when the kafka topic does not exist, just set the broker in the kafka cluster** of the user's side to set auto.create.topics.enable = true` . + +## Related parameters + +Some system configuration parameters can affect the use of routine loads. + +1. max\_routine\_load\_task\_concurrent\_num + + The FE configuration item, which defaults to 5, can be modified at runtime. This parameter limits the maximum number of subtask concurrency for a routine load job. It is recommended to maintain the default value. If the setting is too large, it may cause too many concurrent tasks and occupy cluster resources. + +2. max\_routine_load\_task\_num\_per\_be + + The FE configuration item, which defaults to 5, can be modified at runtime. This parameter limits the number of subtasks that can be executed concurrently by each BE node. It is recommended to maintain the default value. If the setting is too large, it may cause too many concurrent tasks and occupy cluster resources. + +3. max\_routine\_load\_job\_num + + The FE configuration item, which defaults to 100, can be modified at runtime. This parameter limits the total number of routine load jobs, including NEED_SCHEDULED, RUNNING, PAUSE. After the overtime, you cannot submit a new assignment. + +4. max\_consumer\_num\_per\_group + + BE configuration item, the default is 3. This parameter indicates that up to several consumers are generated in a subtask for data consumption. For a Kafka data source, a consumer may consume one or more kafka partitions. Suppose a task needs to consume 6 kafka partitions, it will generate 3 consumers, and each consumer consumes 2 partitions. If there are only 2 partitions, only 2 consumers will be generated, and each consumer will consume 1 partition. + +5. push\_write\_mbytes\_per\_sec + + BE configuration item. The default is 10, which is 10MB/s. This parameter is to load common parameters, not limited to routine load jobs. This parameter limits the speed at which loaded data is written to disk. For high-performance storage devices such as SSDs, this speed limit can be appropriately increased. + +6. max\_tolerable\_backend\_down\_num + FE configuration item, the default is 0. Under certain conditions, Doris can reschedule PAUSED tasks, that becomes RUNNING?This parameter is 0, which means that rescheduling is allowed only when all BE nodes are in alive state. + +7. period\_of\_auto\_resume\_min + FE configuration item, the default is 5 mins. Doris reschedules will only try at most 3 times in the 5 minute period. If all 3 times fail, the current task will be locked, and auto-scheduling will not be performed. However, manual intervention can be performed. diff --git a/docs/en/administrator-guide/load-data/stream-load-manual.md b/docs/en/administrator-guide/load-data/stream-load-manual.md new file mode 100644 index 00000000000000..f2bea8124d4e4c --- /dev/null +++ b/docs/en/administrator-guide/load-data/stream-load-manual.md @@ -0,0 +1,293 @@ +--- +{ + "title": "Stream load", + "language": "en" +} +--- + + + +# Stream load + +Stream load is a synchronous way of importing. Users import local files or data streams into Doris by sending HTTP protocol requests. Stream load synchronously executes the import and returns the import result. Users can directly determine whether the import is successful by the return body of the request. + +Stream load is mainly suitable for importing local files or data from data streams through procedures. + +## Basic Principles + +The following figure shows the main flow of Stream load, omitting some import details. + +``` + ^ + + | | + | | 1A. User submit load to FE + | | + | +--v-----------+ + | | FE | +5. Return result to user | +--+-----------+ + | | + | | 2. Redirect to BE + | | + | +--v-----------+ + +---+Coordinator BE| 1B. User submit load to BE + +-+-----+----+-+ + | | | + +-----+ | +-----+ + | | | 3. Distrbute data + | | | + +-v-+ +-v-+ +-v-+ + |BE | |BE | |BE | + +---+ +---+ +---+ +``` + +In Stream load, Doris selects a node as the Coordinator node. This node is responsible for receiving data and distributing data to other data nodes. + +Users submit import commands through HTTP protocol. If submitted to FE, FE forwards the request to a BE via the HTTP redirect instruction. Users can also submit import commands directly to a specified BE. + +The final result of the import is returned to the user by Coordinator BE. + +## Basic operations +### Create a Load + +Stream load submits and transfers data through HTTP protocol. Here, the `curl` command shows how to submit an import. + +Users can also operate through other HTTP clients. + +``` +curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_host:http_port/api/{db}/{table}/_stream_load + +The properties supported in the header are described in "Load Parameters" below +The format is: - H "key1: value1" +``` + +Examples: + +``` +curl --location-trusted -u root -T date -H "label:123" http://abc.com:8030/api/test/date/_stream_load +``` +The detailed syntax for creating imports helps to execute ``HELP STREAM LOAD`` view. The following section focuses on the significance of creating some parameters of Stream load. + +#### Signature parameters + ++ user/passwd + + Stream load uses the HTTP protocol to create the imported protocol and signs it through the Basic Access authentication. The Doris system verifies user identity and import permissions based on signatures. + +#### Load Parameters + +Stream load uses HTTP protocol, so all parameters related to import tasks are set in the header. The significance of some parameters of the import task parameters of Stream load is mainly introduced below. + ++ label + + Identity of import task. Each import task has a unique label inside a single database. Label is a user-defined name in the import command. With this label, users can view the execution of the corresponding import task. + + Another function of label is to prevent users from importing the same data repeatedly. **It is strongly recommended that users use the same label for the same batch of data. This way, repeated requests for the same batch of data will only be accepted once, guaranteeing at-Most-Once** + + When the corresponding import operation state of label is CANCELLED, the label can be used again. + ++ max\_filter\_ratio + + The maximum tolerance rate of the import task is 0 by default, and the range of values is 0-1. When the import error rate exceeds this value, the import fails. + + If the user wishes to ignore the wrong row, the import can be successful by setting this parameter greater than 0. + + The calculation formula is as follows: + + ``` (dpp.abnorm.ALL / (dpp.abnorm.ALL + dpp.norm.ALL ) ) > max_filter_ratio ``` + + ``` dpp.abnorm.ALL``` denotes the number of rows whose data quality is not up to standard. Such as type mismatch, column mismatch, length mismatch and so on. + + ``` dpp.norm.ALL ``` refers to the number of correct data in the import process. The correct amount of data for the import task can be queried by the ``SHOW LOAD` command. + +The number of rows in the original file = `dpp.abnorm.ALL + dpp.norm.ALL` + ++ where + + Import the filter conditions specified by the task. Stream load supports filtering of where statements specified for raw data. The filtered data will not be imported or participated in the calculation of filter ratio, but will be counted as `num_rows_unselected`. + ++ partition + + Partition information for tables to be imported will not be imported if the data to be imported does not belong to the specified Partition. These data will be included in `dpp.abnorm.ALL`. + ++ columns + + The function transformation configuration of data to be imported includes the sequence change of columns and the expression transformation, in which the expression transformation method is consistent with the query statement. + + ``` + Examples of column order transformation: There are two columns of original data, and there are also two columns (c1, c2) in the table at present. But the first column of the original file corresponds to the C2 column of the target table, while the second column of the original file corresponds to the C1 column of the target table, which is written as follows: + columns: c2,c1 + + Example of expression transformation: There are two columns in the original file and two columns in the target table (c1, c2). However, both columns in the original file need to be transformed by functions to correspond to the two columns in the target table. + columns: tmp_c1, tmp_c2, c1 = year(tmp_c1), c2 = mouth(tmp_c2) + Tmp_* is a placeholder, representing two original columns in the original file. + ``` + ++ exec\_mem\_limit + + Memory limit. Default is 2GB. Unit is Bytes + +### Return results + +Since Stream load is a synchronous import method, the result of the import is directly returned to the user by creating the return value of the import. + +Examples: + +``` +{ + "TxnId": 1003, + "Label": "b6f3bc78-0d2c-45d9-9e4c-faa0a0149bee", + "Status": "Success", + "ExistingJobStatus": "FINISHED", // optional + "Message": "OK", + "NumberTotalRows": 1000000, + "NumberLoadedRows": 1000000, + "NumberFilteredRows": 1, + "NumberUnselectedRows": 0, + "LoadBytes": 40888898, + "LoadTimeMs": 2144, + "ErrorURL": "http://192.168.1.1:8042/api/_load_error_log?file=__shard_0/error_log_insert_stmt_db18266d4d9b4ee5-abb00ddd64bdf005_db18266d4d9b4ee5_abb00ddd64bdf005" +} +``` + +The following main explanations are given for the Stream load import result parameters: + ++ TxnId: The imported transaction ID. Users do not perceive. + ++ Label: Import Label. User specified or automatically generated by the system. + ++ Status: Import completion status. + + "Success": Indicates successful import. + + "Publish Timeout": This state also indicates that the import has been completed, except that the data may be delayed and visible without retrying. + + "Label Already Exists"：Label duplicate, need to be replaced Label. + + "Fail": Import failed. + ++ ExistingJobStatus: The state of the load job corresponding to the existing Label. + + This field is displayed only when the status is "Label Already Exists". The user can know the status of the load job corresponding to Label through this state. "RUNNING" means that the job is still executing, and "FINISHED" means that the job is successful. + ++ Message: Import error messages. + ++ NumberTotalRows: Number of rows imported for total processing. + ++ NumberLoadedRows: Number of rows successfully imported. + ++ NumberFilteredRows: Number of rows that do not qualify for data quality. + ++ NumberUnselectedRows: Number of rows filtered by where condition. + ++ LoadBytes: Number of bytes imported. + ++ LoadTimeMs: Import completion time. Unit milliseconds. + ++ ErrorURL: If you have data quality problems, visit this URL to see specific error lines. + +> Note: Since Stream load is a synchronous import mode, import information will not be recorded in Doris system. Users cannot see Stream load asynchronously by looking at import commands. You need to listen for the return value of the create import request to get the import result. + +### Cancel Load + +Users can't cancel Stream load manually. Stream load will be cancelled automatically by the system after a timeout or import error. + +## Relevant System Configuration + +### FE configuration + ++ stream\_load\_default\_timeout\_second + + The timeout time of the import task (in seconds) will be cancelled by the system if the import task is not completed within the set timeout time, and will become CANCELLED. + + At present, Stream load does not support custom import timeout time. All Stream load import timeout time is uniform. The default timeout time is 300 seconds. If the imported source file can no longer complete the import within the specified time, the FE parameter ```stream_load_default_timeout_second``` needs to be adjusted. + +### BE configuration + ++ streaming\_load\_max\_mb + + The maximum import size of Stream load is 10G by default, in MB. If the user's original file exceeds this value, the BE parameter ```streaming_load_max_mb``` needs to be adjusted. + +## Best Practices + +### Application scenarios + +The most appropriate scenario for using Stream load is that the original file is in memory or on disk. Secondly, since Stream load is a synchronous import method, users can also use this import if they want to obtain the import results in a synchronous manner. + +### Data volume + +Since Stream load is based on the BE initiative to import and distribute data, the recommended amount of imported data is between 1G and 10G. Since the default maximum Stream load import data volume is 10G, the configuration of BE ```streaming_load_max_mb``` needs to be modified if files exceeding 10G are to be imported. + +``` +For example, the size of the file to be imported is 15G +Modify the BE configuration streaming_load_max_mb to 16000 +``` + +Stream load default timeout is 300 seconds, according to Doris currently the largest import speed limit, about more than 3G files need to modify the import task default timeout. + +``` +Import Task Timeout = Import Data Volume / 10M / s (Specific Average Import Speed Requires Users to Calculate Based on Their Cluster Conditions) +For example, import a 10G file +Timeout = 1000s -31561;. 20110G / 10M /s +``` + +### Complete examples +Data situation: In the local disk path / home / store_sales of the sending and importing requester, the imported data is about 15G, and it is hoped to be imported into the table store\_sales of the database bj_sales. + +Cluster situation: The concurrency of Stream load is not affected by cluster size. + ++ Step 1: Does the import file size exceed the default maximum import size of 10G + + ``` + BE conf + streaming_load_max_mb = 16000 + ``` ++ Step 2: Calculate whether the approximate import time exceeds the default timeout value + + ``` + Import time 15000/10 = 1500s + Over the default timeout time, you need to modify the FE configuration + stream_load_default_timeout_second = 1500 + ``` + ++ Step 3: Create Import Tasks + + ``` + curl --location-trusted -u user:password -T /home/store_sales -H "label:abc" http://abc.com:8000/api/bj_sales/store_sales/_stream_load + ``` + +## Common Questions + +* Label Already Exists + + The Label repeat checking steps of Stream load are as follows: + + 1. Is there an import Label conflict that already exists with other import methods? + + Because imported Label in Doris system does not distinguish between import methods, there is a problem that other import methods use the same Label. + + Through ``SHOW LOAD WHERE LABEL = "xxx"'``, where XXX is a duplicate Label string, see if there is already a Label imported by FINISHED that is the same as the Label created by the user. + + 2. Are Stream loads submitted repeatedly for the same job? + + Since Stream load is an HTTP protocol submission creation import task, HTTP Clients in various languages usually have their own request retry logic. After receiving the first request, the Doris system has started to operate Stream load, but because the result is not returned to the Client side in time, the Client side will retry to create the request. At this point, the Doris system is already operating on the first request, so the second request will be reported to Label Already Exists. + + To sort out the possible methods mentioned above: Search FE Master's log with Label to see if there are two ``redirect load action to destination = ``redirect load action to destination'cases in the same Label. If so, the request is submitted repeatedly by the Client side. + + It is suggested that the user calculate the approximate import time according to the data quantity of the current request, and change the request time-out time of the Client end according to the import time-out time, so as to avoid the request being submitted by the Client end many times. diff --git a/docs/en/administrator-guide/operation/metadata-operation.md b/docs/en/administrator-guide/operation/metadata-operation.md new file mode 100644 index 00000000000000..14a6231579cc9e --- /dev/null +++ b/docs/en/administrator-guide/operation/metadata-operation.md @@ -0,0 +1,342 @@ +--- +{ + "title": "Metadata Operations and Maintenance", + "language": "en" +} +--- + + + +# Metadata Operations and Maintenance + +This document focuses on how to manage Doris metadata in a real production environment. It includes the proposed deployment of FE nodes, some commonly used operational methods, and common error resolution methods. + +For the time being, read the [Doris metadata design document](../../internal/metadata-design_EN.md) to understand how Doris metadata works. + +## Important tips + +* Current metadata design is not backward compatible. That is, if the new version has a new metadata structure change (you can see whether there is a new VERSION in the `FeMetaVersion. java'file in the FE code), it is usually impossible to roll back to the old version after upgrading to the new version. Therefore, before upgrading FE, be sure to test metadata compatibility according to the operations in the [Upgrade Document](../../installing/upgrade_EN.md). + +## Metadata catalog structure + +Let's assume that the path of `meta_dir` specified in fe.conf is `path/to/palo-meta`. In a normal Doris cluster, the directory structure of metadata should be as follows: + +``` +/path/to/palo-meta/ + |-- bdb/ + | |-- 00000000.jdb + | |-- je.config.csv + | |-- je.info.0 + | |-- je.info.0.lck + | |-- je.lck + | `-- je.stat.csv + `-- image/ + |-- ROLE + |-- VERSION + `-- image.xxxx +``` + +1. bdb + + We use [bdbje] (https://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html) as a distributed kV system to store metadata journal. This BDB directory is equivalent to the "data directory" of bdbje. + + The `.jdb` suffix is the data file of bdbje. These data files will increase with the increasing number of metadata journals. When Doris regularly completes the image, the old log is deleted. So normally, the total size of these data files varies from several MB to several GB (depending on how Doris is used, such as import frequency). When the total size of the data file is larger than 10GB, you may need to wonder whether the image failed or the historical journals that failed to distribute the image could not be deleted. + + ` je.info.0 ` is the running log of bdbje. The time in this log is UTC + 0 time zone. We may fix this in a later version. From this log, you can also see how some bdbje works. + +2. image directory + + The image directory is used to store metadata mirrors generated regularly by Doris. Usually, you will see a `image.xxxxx` mirror file. Where `xxxxx` is a number. This number indicates that the image contains all metadata journal before `xxxx`. And the generation time of this file (viewed through `ls -al`) is usually the generation time of the mirror. + + You may also see a `image.ckpt` file. This is a metadata mirror being generated. The `du -sh` command should show that the file size is increasing, indicating that the mirror content is being written to the file. When the mirror is written, it automatically renames itself to a new `image.xxxxx` and replaces the old image file. + + Only FE with a Master role will actively generate image files on a regular basis. After each generation, FE is pushed to other non-Master roles. When it is confirmed that all other FEs have received this image, Master FE deletes the metadata journal in bdbje. Therefore, if image generation fails or image push fails to other FEs, data in bdbje will accumulate. + + `ROLE` file records the type of FE (FOLLOWER or OBSERVER), which is a text file. + + `VERSION` file records the cluster ID of the Doris cluster and the token used to access authentication between nodes, which is also a text file. + + `ROLE` file and `VERSION` file may only exist at the same time, or they may not exist at the same time (e.g. at the first startup). + +## Basic operations + +### Start single node FE + +Single node FE is the most basic deployment mode. A complete Doris cluster requires at least one FE node. When there is only one FE node, the type of the node is Follower and the role is Master. + +1. First start-up + + 1. Suppose the path of `meta_dir` specified in fe.conf is `path/to/palo-meta`. + 2. Ensure that `path/to/palo-meta` already exists, that the permissions are correct and that the directory is empty. + 3. Start directly through `sh bin/start_fe.sh`. + 4. After booting, you should be able to see the following log in fe.log: + + * Palo FE starting... + * image does not exist: /path/to/palo-meta/image/image.0 + * transfer from INIT to UNKNOWN + * transfer from UNKNOWN to MASTER + * the very first time to open bdb, dbname is 1 + * start fencing, epoch number is 1 + * finish replay in xxx msec + * QE service start + * thrift server started + + The above logs are not necessarily strictly in this order, but they are basically similar. + + 5. The first start-up of a single-node FE usually does not encounter problems. If you haven't seen the above logs, generally speaking, you haven't followed the document steps carefully, please read the relevant wiki carefully. + +2. Restart + + 1. Stopped FE nodes can be restarted by using `sh bin/start_fe.sh`. + 2. After restarting, you should be able to see the following log in fe.log: + + * Palo FE starting... + * finished to get cluster id: xxxx, role: FOLLOWER and node name: xxxx + * If no image has been generated before reboot, you will see: + * image does not exist: /path/to/palo-meta/image/image.0 + + * If an image is generated before the restart, you will see: + * start load image from /path/to/palo-meta/image/image.xxx. is ckpt: false + * finished load image in xxx ms + + * transfer from INIT to UNKNOWN + * replayed journal id is xxxx, replay to journal id is yyyy + * transfer from UNKNOWN to MASTER + * finish replay in xxx msec + * master finish replay journal, can write now. + * begin to generate new image: image.xxxx + * start save image to /path/to/palo-meta/image/image.ckpt. is ckpt: true + * finished save image /path/to/palo-meta/image/image.ckpt in xxx ms. checksum is xxxx + * push image.xxx to other nodes. totally xx nodes, push successed xx nodes + * QE service start + * thrift server started + + The above logs are not necessarily strictly in this order, but they are basically similar. + +3. Common problems + + For the deployment of single-node FE, start-stop usually does not encounter any problems. If you have any questions, please refer to the relevant Wiki and check your operation steps carefully. + +### Add FE + +Adding FE processes is described in detail in the [Deployment and Upgrade Documents] (https://github.com/apache/incubator-doris/wiki/Doris-Deploy-%26-Upgrade) and will not be repeated. Here are some points for attention, as well as common problems. + +1. Notes + + * Before adding a new FE, make sure that the current Master FE runs properly (connection is normal, JVM is normal, image generation is normal, bdbje data directory is too large, etc.) + * The first time you start a new FE, you must make sure that the `-helper` parameter is added to point to Master FE. There is no need to add `-helper` when restarting. (If `-helper` is specified, FE will directly ask the helper node for its role. If not, FE will try to obtain information from `ROLE` and `VERSION` files in the `palo-meta/image/` directory. + * The first time you start a new FE, you must make sure that the `meta_dir` of the FE is created, has correct permissions and is empty. + * Starting a new FE and executing the `ALTER SYSTEM ADD FOLLOWER/OBSERVER` statement adds FE to metadata in a sequence that is not required. If a new FE is started first and no statement is executed, the `current node is not added to the group. Please add it first.` in the new FE log. When the statement is executed, it enters the normal process. + * Make sure that after the previous FE is added successfully, the next FE is added. + * Connect to MASTER FE and execute `ALTER SYSTEM ADD FOLLOWER/OBSERVER` claus。 + +2. Common problems + + 1. this need is DETACHED + + When you first start a FE to be added, if the data in palo-meta/bdb on Master FE is large, you may see the words `this node is DETACHED`. in the FE log to be added. At this point, bdbje is copying data, and you can see that the `bdb/` directory of FE to be added is growing. This process usually takes several minutes (depending on the amount of data in bdbje). Later, there may be some bdbje-related error stack information in fe. log. If `QE service start` and `thrift server start` are displayed in the final log, the start is usually successful. You can try to connect this FE via mysql-client. If these words do not appear, it may be the problem of bdbje replication log timeout. At this point, restarting the FE directly will usually solve the problem. + + 2. Failure to add due to various reasons + + * If OBSERVER is added, because OBSERVER-type FE does not participate in the majority of metadata writing, it can theoretically start and stop at will. Therefore, for the case of adding OBSERVER failure. The process of OBSERVER FE can be killed directly. After clearing the metadata directory of OBSERVER, add the process again. + + * If FOLLOWER is added, because FOLLOWER is mostly written by participating metadata. So it is possible that FOLLOWER has joined the bdbje electoral team. If there are only two FOLLOWER nodes (including MASTER), then stopping one FE may cause another FE to quit because it cannot write most of the time. At this point, we should first delete the newly added FOLLOWER node from the metadata through the `ALTER SYSTEM DROP FOLLOWER` command, then kill the FOLLOWER process, empty the metadata and re-add the process. + + +### Delete FE + +The corresponding type of FE can be deleted by the `ALTER SYSTEM DROP FOLLOWER/OBSERVER` command. The following points should be noted: + +* For OBSERVER type FE, direct DROP is enough, without risk. + +* For FOLLOWER type FE. First, you should make sure that you start deleting an odd number of FOLLOWERs (three or more). + + 1. If the FE of non-MASTER role is deleted, it is recommended to connect to MASTER FE, execute DROP command, and then kill the process. + 2. If you want to delete MASTER FE, first confirm that there are odd FOLLOWER FE and it works properly. Then kill the MASTER FE process first. At this point, a FE will be elected MASTER. After confirming that the remaining FE is working properly, connect to the new MASTER FE and execute the DROP command to delete the old MASTER FE. + +## Advanced Operations + +### Failure recovery + +FE may fail to start bdbje and synchronize between FEs for some reasons. Phenomena include the inability to write metadata, the absence of MASTER, and so on. At this point, we need to manually restore the FE. The general principle of manual recovery of FE is to start a new MASTER through metadata in the current `meta_dir`, and then add other FEs one by one. Please follow the following steps strictly: + +1. First, stop all FE processes and all business access. Make sure that during metadata recovery, external access will not lead to other unexpected problems. + +2. Identify which FE node's metadata is up-to-date: + + * First of all, **be sure to back up all FE's `meta_dir` directories first.** + * Usually, Master FE's metadata is up to date. You can see the suffix of image.xxxx file in the `meta_dir/image` directory. The larger the number, the newer the metadata. + * Usually, by comparing all FOLLOWER FE image files, you can find the latest metadata. + * After that, we use the FE node with the latest metadata to recover. + * If using metadata of OBSERVER node to recover will be more troublesome, it is recommended to choose FOLLOWER node as far as possible. + +3. The following operations are performed on the FE nodes selected in step 2. + + 1. If the node is an OBSERVER, first change the `role=OBSERVER` in the `meta_dir/image/ROLE` file to `role=FOLLOWER`. (Recovery from the OBSERVER node will be more cumbersome, first follow the steps here, followed by a separate description) + 2. Add configuration in fe.conf: `metadata_failure_recovery=true`. + 3. Run `sh bin/start_fe.sh` to start the FE + 4. If normal, the FE will start in the role of MASTER, similar to the description in the previous section `Start a single node FE`. You should see the words `transfer from XXXX to MASTER` in fe.log. + 5. After the start-up is completed, connect to the FE first, and execute some query imports to check whether normal access is possible. If the operation is not normal, it may be wrong. It is recommended to read the above steps carefully and try again with the metadata previously backed up. If not, the problem may be more serious. + 6. If successful, through the `show frontends;` command, you should see all the FEs you added before, and the current FE is master. + 7. Delete the `metadata_failure_recovery=true` configuration item in fe.conf, or set it to `false`, and restart the FE (**Important**). + + + > If you are recovering metadata from an OBSERVER node, after completing the above steps, you will find that the current FE role is OBSERVER, but `IsMaster` appears as `true`. This is because the "OBSERVER" seen here is recorded in Doris's metadata, but whether it is master or not, is recorded in bdbje's metadata. Because we recovered from an OBSERVER node, there was inconsistency. Please take the following steps to fix this problem (we will fix it in a later version): + + > 1. First, all FE nodes except this "OBSERVER" are DROPed out. + > 2. A new FOLLOWER FE is added through the `ADD FOLLOWER` command, assuming that it is on hostA. + > 3. Start a new FE on hostA and join the cluster by `helper`. + > 4. After successful startup, you should see two FEs through the `show frontends;` statement, one is the previous OBSERVER, the other is the newly added FOLLOWER, and the OBSERVER is the master. + > 5. After confirming that the new FOLLOWER is working properly, the new FOLLOWER metadata is used to perform a failure recovery operation again. + > 6. The purpose of the above steps is to manufacture a metadata of FOLLOWER node artificially, and then use this metadata to restart fault recovery. This avoids inconsistencies in recovering metadata from OBSERVER. + + >The meaning of `metadata_failure_recovery = true` is to empty the metadata of `bdbje`. In this way, bdbje will not contact other FEs before, but start as a separate FE. This parameter needs to be set to true only when restoring startup. After recovery, it must be set to false. Otherwise, once restarted, the metadata of bdbje will be emptied again, which will make other FEs unable to work properly. + +4. After the successful execution of step 3, we delete the previous FEs from the metadata by using the `ALTER SYSTEM DROP FOLLOWER/OBSERVER` command and add them again by adding new FEs. + +5. If the above operation is normal, it will be restored. + +### FE type change + +If you need to change the existing FOLLOWER/OBSERVER type FE to OBSERVER/FOLLOWER type, please delete FE in the way described above, and then add the corresponding type FE. + +### FE Migration + +If you need to migrate one FE from the current node to another, there are several scenarios. + +1. FOLLOWER, or OBSERVER migration for non-MASTER nodes + + After adding a new FOLLOWER / OBSERVER directly, delete the old FOLLOWER / OBSERVER. + +2. Single-node MASTER migration + + When there is only one FE, refer to the `Failure Recovery` section. Copy the palo-meta directory of FE to the new node and start the new MASTER in Step 3 of the `Failure Recovery` section + +3. A set of FOLLOWER migrates from one set of nodes to another set of new nodes + + Deploy FE on the new node and add the new node first by adding FOLLOWER. The old nodes can be dropped by DROP one by one. In the process of DROP-by-DROP, MASTER automatically selects the new FOLLOWER node. + +### Replacement of FE port + +FE currently has the following ports + +* Ed_log_port: bdbje's communication port +* http_port: http port, also used to push image +* rpc_port：FE 的 thrift server port +* query_port: Mysql connection port + +1. edit_log_port + + If this port needs to be replaced, it needs to be restored with reference to the operations in the `Failure Recovery` section. Because the port has been persisted into bdbje's own metadata (also recorded in Doris's own metadata), it is necessary to clear bdbje's metadata by setting `metadata_failure_recovery=true`. + +2. http_port + + All FE http_ports must be consistent. So if you want to modify this port, all FEs need to be modified and restarted. Modifying this port will be more complex in the case of multiple FOLLOWER deployments (involving laying eggs and laying hens...), so this operation is not recommended. If necessary, follow the operation in the `Failure Recovery` section directly. + +3. rpc_port + + After modifying the configuration, restart FE directly. Master FE informs BE of the new port through heartbeat. Only this port of Master FE will be used. However, it is still recommended that all FE ports be consistent. + +4. query_port + + After modifying the configuration, restart FE directly. This only affects mysql's connection target. + +### Recover metadata from FE memory +In some extreme cases, the image file on the disk may be damaged, but the metadata in the memory is intact. At this point, we can dump the metadata from the memory and replace the image file on the disk to recover the metadata. the entire non-stop query service operation steps are as follows: + +1. Stop all Load, Create, Alter operations. + +2. Execute the following command to dump metadata from the Master FE memory: (hereafter called image_mem) +``` +curl -u $root_user:$password http://$master_hostname:8030/dump +``` +3. Replace the image file in the `meta_dir/image` directory on the OBSERVER FE node with the image_mem file, restart the OBSERVER FE node, and verify the integrity and correctness of the image_mem file. You can check whether the DB and Table metadata are normal on the FE Web page, whether there is an exception in `fe.log`, whether it is in a normal replayed jour. + +4. Replace the image file in the `meta_dir/image` directory on the FOLLOWER FE node with the image_mem file in turn, restart the FOLLOWER FE node, and confirm that the metadata and query services are normal. + +5. Replace the image file in the `meta_dir/image` directory on the Master FE node with the image_mem file, restart the Master FE node, and then confirm that the FE Master switch is normal and The Master FE node can generate a new image file through checkpoint. + +6. Recover all Load, Create, Alter operations. + +**Note: If the Image file is large, the entire process can take a long time, so during this time, make sure Master FE does not generate a new image file via checkpoint. When the image.ckpt file in the meta_dir/image directory on the Master FE node is observed to be as large as the image.xxx file, the image.ckpt file can be deleted directly.** + + +## Best Practices + +The deployment recommendation of FE is described in the Installation and [Deployment Document](../../installing/install-deploy_EN.md). Here are some supplements. + +* **If you don't know the operation logic of FE metadata very well, or you don't have enough experience in the operation and maintenance of FE metadata, we strongly recommend that only one FOLLOWER-type FE be deployed as MASTER in practice, and the other FEs are OBSERVER, which can reduce many complex operation and maintenance problems.** Don't worry too much about the failure of MASTER single point to write metadata. First, if you configure it properly, FE as a java process is very difficult to hang up. Secondly, if the MASTER disk is damaged (the probability is very low), we can also use the metadata on OBSERVER to recover manually through `fault recovery`. + +* The JVM of the FE process must ensure sufficient memory. We **strongly recommend** that FE's JVM memory should be at least 10GB and 32GB to 64GB. And deploy monitoring to monitor JVM memory usage. Because if OOM occurs in FE, metadata writing may fail, resulting in some failures that **cannot recover**! + +* FE nodes should have enough disk space to prevent the excessive metadata from causing insufficient disk space. At the same time, FE logs also take up more than a dozen gigabytes of disk space. + +## Other common problems + +1. Output `meta out of date. current time: xxx, synchronized time: xxx, has log: xxx, fe type: xxx` in fe.log + + This is usually because the FE cannot elect Master. For example, if three FOLLOWERs are configured, but only one FOLLOWER is started, this FOLLOWER will cause this problem. Usually, just start the remaining FOLLOWER. If the problem has not been solved after the start-up, manual recovery may be required in accordance with the way in the `Failure Recovery` section. + +2. `Clock delta: xxxx ms. between Feeder: xxxx and this Replica exceeds max permissible delta: xxxx ms.` + + Bdbje requires that clock errors between nodes should not exceed a certain threshold. If exceeded, the node will exit abnormally. The default threshold is 5000ms, which is controlled by FE parameter `max_bdbje_clock_delta_ms', and can be modified as appropriate. But we suggest using NTP and other clock synchronization methods to ensure the clock synchronization of Doris cluster hosts. + + +3. Mirror files in the `image/` directory have not been updated for a long time + + Master FE generates a mirror file by default for every 50,000 metadata journal. In a frequently used cluster, a new image file is usually generated every half to several days. If you find that the image file has not been updated for a long time (for example, more than a week), you can see the reasons in sequence as follows: + + 1. Search for `memory is not enough to do checkpoint. Committed memroy XXXX Bytes, used memory XXXX Bytes. ` in the fe.log of Master FE. If found, it indicates that the current FE's JVM memory is insufficient for image generation (usually we need to reserve half of the FE memory for image generation). Then you need to add JVM memory and restart FE before you can observe. Each time Master FE restarts, a new image is generated directly. This restart method can also be used to actively generate new images. Note that if there are multiple FOLLOWER deployments, then when you restart the current Master FE, another FOLLOWER FE will become MASTER, and subsequent image generation will be the responsibility of the new Master. Therefore, you may need to modify the JVM memory configuration of all FOLLOWER FE. + + 2. Search for `begin to generate new image: image.xxxx` in the fe.log of Master FE. If it is found, then the image is generated. Check the subsequent log of this thread, and if `checkpoint finished save image.xxxx` appears, the image is written successfully. If `Exception when generating new image file` occurs, the generation fails and specific error messages need to be viewed. + + +4. The size of the `bdb/` directory is very large, reaching several Gs or more. + + The BDB directory will remain large for some time after eliminating the error that the new image cannot be generated. Maybe it's because Master FE failed to push image. You can search `push image.XXXX to other nodes. totally XX nodes, push successed YY nodes` in the fe. log of Master FE. If YY is smaller than xx, then some FEs are not pushed successfully. You can see the specific error `Exception when pushing image file.url = xxx` in the fe. log. + + At the same time, you can add the configuration in the FE configuration file: `edit_log_roll_num = xxxx`. This parameter sets the number of metadata journals and makes an image once. The default is 50000. This number can be reduced appropriately to make images more frequent, thus speeding up the deletion of old journals. + +5. FOLLOWER FE hangs up one after another + + Because Doris's metadata adopts the majority writing strategy, that is, a metadata journal must be written to at least a number of FOLLOWER FEs (for example, three FOLLOWERs, two must be written successfully) before it can be considered successful. If the write fails, the FE process exits on its own initiative. So suppose there are three FOLLOWERs: A, B and C. C hangs up first, and then B hangs up, then A will hang up. So as described in the `Best Practices `section, if you don't have extensive experience in metadata operations and maintenance, it's not recommended to deploy multiple FOLLOWERs. + +6. fe.log 中出现 `get exception when try to close previously opened bdb database. ignore it` + + If there is the word `ignore it` behind it, there is usually no need to deal with it. If you are interested, you can search for this error in `BDBEnvironment.java`, and see the annotations. + +7. From `show frontends;` Look, the `Join` of a FE is listed as `true`, but actually the FE is abnormal. + + Through `show frontends;` see the `Join` information. If the column is `true`, it only means that the FE **has joined the** cluster. It does not mean that it still exists normally in the cluster. If `false`, it means that the FE **has never joined the** cluster. + +8. Configuration of FE `master_sync_policy`, `replica_sync_policy`, and `txn_rollback_limit.` + + `master_sync_policy` is used to specify whether fsync (), `replica_sync_policy` is called when Leader FE writes metadata log, and `replica_sync_policy` is used to specify whether other Follower FE calls fsync () when FE HA deploys synchronous metadata. In earlier versions of Oris, these two parameters defaulted to `WRITE_NO_SYNC`, i.e., fsync () was not called. In the latest version of Oris, the default has been changed to `SYNC`, that is, fsync () is called. Calling fsync () significantly reduces the efficiency of metadata disk writing. In some environments, IOPS may drop to several hundred and the latency increases to 2-3ms (but it's still enough for Doris metadata manipulation). Therefore, we recommend the following configuration: + + 1. For a single Follower FE deployment, `master_sync_policy` is set to `SYNC`, which prevents the loss of metadata due to the downtime of the FE system. + 2. For multi-Follower FE deployment, we can set `master_sync_policy` and `replica_sync_policy` to `WRITE_NO_SYNC`, because we think that the probability of simultaneous outage of multiple systems is very low. + + If `master_sync_policy` is set to `WRITE_NO_SYNC` in a single Follower FE deployment, then a FE system outage may occur, resulting in loss of metadata. At this point, if other Observer FE attempts to restart, it may report an error: + + ``` + Node xxx must rollback xx total commits(numPassedDurableCommits of which were durable) to the earliest point indicated by transaction xxxx in order to rejoin the replication group, but the transaction rollback limit of xxx prohibits this. + ``` + +This means that some transactions that have been persisted need to be rolled back, but the number of entries exceeds the upper limit. Here our default upper limit is 100, which can be changed by setting `txn_rollback_limit`. This operation is only used to attempt to start FE normally, but lost metadata cannot be recovered. diff --git a/docs/en/administrator-guide/operation/monitor-alert.md b/docs/en/administrator-guide/operation/monitor-alert.md new file mode 100644 index 00000000000000..16f4a7fadb1f87 --- /dev/null +++ b/docs/en/administrator-guide/operation/monitor-alert.md @@ -0,0 +1,309 @@ +--- +{ + "title": "Monitoring and alarming", + "language": "en" +} +--- + + + +# Monitoring and alarming + +This document mainly introduces Doris's monitoring items and how to collect and display them. And how to configure alarm (TODO) + +[Dashborad template click download](https://grafana.com/dashboards/9734/revisions) + +> Note: Before 0.9.0 (excluding), please use revision 1. For version 0.9.x, use revision 2. For version 0.10.x, use revision 3. + +Dashboard templates are updated from time to time. The way to update the template is shown in the last section. + +Welcome to provide better dashboard. + +## Components + +Doris uses [Prometheus] (https://prometheus.io/) and [Grafana] (https://grafana.com/) to collect and display input monitoring items. + +![](/images/dashboard_overview.png) + +1. Prometheus + + Prometheus is an open source system monitoring and alarm suite. It can collect monitored items by Pull or Push and store them in its own time series database. And through the rich multi-dimensional data query language, to meet the different data display needs of users. + +2. Grafana + + Grafana is an open source data analysis and display platform. Support multiple mainstream temporal database sources including Prometheus. Through the corresponding database query statements, the display data is obtained from the data source. With flexible and configurable dashboard, these data can be quickly presented to users in the form of graphs. + +> Note: This document only provides a way to collect and display Doris monitoring data using Prometheus and Grafana. In principle, these components are not developed or maintained. For more details on these components, please step through the corresponding official documents. + +## Monitoring data + +Doris's monitoring data is exposed through the HTTP interface of Frontend and Backend. Monitoring data is presented in the form of key-value text. Each Key may also be distinguished by different Labels. When the user has built Doris, the monitoring data of the node can be accessed in the browser through the following interfaces: + +* Frontend: `fe_host:fe_http_port/metrics` +* Backend: `be_host:be_web_server_port/metrics` +* Broker: Not available for now + +Users will see the following monitoring item results (for example, FE partial monitoring items): + + ``` + # HELP jvm_heap_size_bytes jvm heap stat + # TYPE jvm_heap_size_bytes gauge + jvm_heap_size_bytes{type="max"} 41661235200 + jvm_heap_size_bytes{type="committed"} 19785285632 + jvm_heap_size_bytes{type="used"} 10113221064 + # HELP jvm_non_heap_size_bytes jvm non heap stat + # TYPE jvm_non_heap_size_bytes gauge + jvm_non_heap_size_bytes{type="committed"} 105295872 + jvm_non_heap_size_bytes{type="used"} 103184784 + # HELP jvm_young_size_bytes jvm young mem pool stat + # TYPE jvm_young_size_bytes gauge + jvm_young_size_bytes{type="used"} 6505306808 + jvm_young_size_bytes{type="peak_used"} 10308026368 + jvm_young_size_bytes{type="max"} 10308026368 + # HELP jvm_old_size_bytes jvm old mem pool stat + # TYPE jvm_old_size_bytes gauge + jvm_old_size_bytes{type="used"} 3522435544 + jvm_old_size_bytes{type="peak_used"} 6561017832 + jvm_old_size_bytes{type="max"} 30064771072 + # HELP jvm_direct_buffer_pool_size_bytes jvm direct buffer pool stat + # TYPE jvm_direct_buffer_pool_size_bytes gauge + jvm_direct_buffer_pool_size_bytes{type="count"} 91 + jvm_direct_buffer_pool_size_bytes{type="used"} 226135222 + jvm_direct_buffer_pool_size_bytes{type="capacity"} 226135221 + # HELP jvm_young_gc jvm young gc stat + # TYPE jvm_young_gc gauge + jvm_young_gc{type="count"} 2186 + jvm_young_gc{type="time"} 93650 + # HELP jvm_old_gc jvm old gc stat + # TYPE jvm_old_gc gauge + jvm_old_gc{type="count"} 21 + jvm_old_gc{type="time"} 58268 + # HELP jvm_thread jvm thread stat + # TYPE jvm_thread gauge + jvm_thread{type="count"} 767 + jvm_thread{type="peak_count"} 831 + ... + ``` + +This is a monitoring data presented in [Promethus Format] (https://prometheus.io/docs/practices/naming/). We take one of these monitoring items as an example to illustrate: + +``` +# HELP jvm_heap_size_bytes jvm heap stat +# TYPE jvm_heap_size_bytes gauge +jvm_heap_size_bytes{type="max"} 41661235200 +jvm_heap_size_bytes{type="committed"} 19785285632 +jvm_heap_size_bytes{type="used"} 10113221064 +``` + +1. Behavior commentary line at the beginning of "#". HELP is the description of the monitored item; TYPE represents the data type of the monitored item, and Gauge is the scalar data in the example. There are also Counter, Histogram and other data types. Specifically, you can see [Prometheus Official Document] (https://prometheus.io/docs/practices/instrumentation/#counter-vs.-gauge,-summary-vs.-histogram). +2. `jvm_heap_size_bytes` is the name of the monitored item (Key); `type= "max"` is a label named `type`, with a value of `max`. A monitoring item can have multiple Labels. +3. The final number, such as `41661235200`, is the monitored value. + +## Monitoring Architecture + +The entire monitoring architecture is shown in the following figure: + +![](/images/monitor_arch.png) + +1. The yellow part is Prometheus related components. Prometheus Server is the main process of Prometheus. At present, Prometheus accesses the monitoring interface of Doris node by Pull, and then stores the time series data in the time series database TSDB (TSDB is included in the Prometheus process, and need not be deployed separately). Prometheus also supports building [Push Gateway] (https://github.com/prometheus/pushgateway) to allow monitored data to be pushed to Push Gateway by Push by monitoring system, and then data from Push Gateway by Prometheus Server through Pull. +2. [Alert Manager] (https://github.com/prometheus/alertmanager) is a Prometheus alarm component, which needs to be deployed separately (no solution is provided yet, but can be built by referring to official documents). Through Alert Manager, users can configure alarm strategy, receive mail, short messages and other alarms. +3. The green part is Grafana related components. Grafana Server is the main process of Grafana. After startup, users can configure Grafana through Web pages, including data source settings, user settings, Dashboard drawing, etc. This is also where end users view monitoring data. + + +## Start building + +Please start building the monitoring system after you have completed the deployment of Doris. + +Prometheus + +1. Download the latest version of Proetheus on the [Prometheus Website] (https://prometheus.io/download/). Here we take version 2.3.2-linux-amd64 as an example. +2. Unzip the downloaded tar file on the machine that is ready to run the monitoring service. +3. Open the configuration file promethues.yml. Here we provide an example configuration and explain it (the configuration file is in YML format, pay attention to uniform indentation and spaces): + + Here we use the simplest way of static files to monitor configuration. Prometheus supports a variety of [service discovery] (https://prometheus.io/docs/prometheus/latest/configuration/configuration/), which can dynamically sense the addition and deletion of nodes. + + ``` + # my global config + global: + scrape_interval: 15s # Global acquisition interval, default 1 m, set to 15s + evaluation_interval: 15s # Global rule trigger interval, default 1 m, set 15s here + + # Alertmanager configuration + alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + + # A scrape configuration containing exactly one endpoint to scrape: + # Here it's Prometheus itself. + scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'PALO_CLUSTER' # Each Doris cluster, we call it a job. Job can be given a name here as the name of Doris cluster in the monitoring system. + metrics_path: '/metrics' # Here you specify the restful API to get the monitors. With host: port in the following targets, Prometheus will eventually collect monitoring items through host: port/metrics_path. + static_configs: # Here we begin to configure the target addresses of FE and BE, respectively. All FE and BE are written into their respective groups. + - targets: ['fe_host1:8030', 'fe_host2:8030', 'fe_host3:8030'] + labels: + group: fe # Here configure the group of fe, which contains three Frontends + + - targets: ['be_host1:8040', 'be_host2:8040', 'be_host3:8040'] + labels: + group: be # Here configure the group of be, which contains three Backends + + - job_name: 'PALO_CLUSTER_2' # We can monitor multiple Doris clusters in a Prometheus, where we begin the configuration of another Doris cluster. Configuration is the same as above, the following is outlined. + metrics_path: '/metrics' + static_configs: + - targets: ['fe_host1:8030', 'fe_host2:8030', 'fe_host3:8030'] + labels: + group: fe + + - targets: ['be_host1:8040', 'be_host2:8040', 'be_host3:8040'] + labels: + group: be + + ``` + +4. start Promethues + + Start Promethues with the following command: + + `nohup ./prometheus --web.listen-address="0.0.0.0:8181" &` + + This command will run Prometheus in the background and specify its Web port as 8181. After startup, data is collected and stored in the data directory. + +5. stop Promethues + + At present, there is no formal way to stop the process, kill - 9 directly. Of course, Prometheus can also be set as a service to start and stop in a service way. + +6. access Prometheus + + Prometheus can be easily accessed through web pages. The page of Prometheus can be accessed by opening port 8181 through browser. Click on the navigation bar, `Status` -> `Targets`, and you can see all the monitoring host nodes of the grouped Jobs. Normally, all nodes should be `UP`, indicating that data acquisition is normal. Click on an `Endpoint` to see the current monitoring value. If the node state is not UP, you can first access Doris's metrics interface (see previous article) to check whether it is accessible, or query Prometheus related documents to try to resolve. + +7. So far, a simple Prometheus has been built and configured. For more advanced usage, see [Official Documents] (https://prometheus.io/docs/introduction/overview/) + +### Grafana + +1. Download the latest version of Grafana on [Grafana's official website] (https://grafana.com/grafana/download). Here we take version 5.2.1.linux-amd64 as an example. + +2. Unzip the downloaded tar file on the machine that is ready to run the monitoring service. + +3. Open the configuration file conf/defaults.ini. Here we only list the configuration items that need to be changed, and the other configurations can be used by default. + + ``` + # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) + data = data + + # Directory where grafana can store logs + logs = data/log + + # Protocol (http, https, socket) + protocal = http + + # The ip address to bind to, empty will bind to all interfaces + http_addr = + + # The http port to use + http_port = 8182 + ``` + +4. start Grafana + + Start Grafana with the following command + + `nohuo ./bin/grafana-server &` + + This command runs Grafana in the background, and the access port is 8182 configured above. + +5. stop Grafana + + At present, there is no formal way to stop the process, kill - 9 directly. Of course, you can also set Grafana as a service to start and stop as a service. + +6. access Grafana + + Through the browser, open port 8182, you can start accessing the Grafana page. The default username password is admin. + +7. Configure Grafana + + For the first landing, you need to set up the data source according to the prompt. Our data source here is Proetheus, which was configured in the previous step. + + The Setting page of the data source configuration is described as follows: + + 1. Name: Name of the data source, customized, such as doris_monitor_data_source + 2. Type: Select Prometheus + 3. URL: Fill in the web address of Prometheus, such as http://host:8181 + 4. Access: Here we choose the Server mode, which is to access Prometheus through the server where the Grafana process is located. + 5. The other options are available by default. + 6. Click `Save & Test` at the bottom. If `Data source is working`, it means that the data source is available. + 7. After confirming that the data source is available, click on the + number in the left navigation bar and start adding Dashboard. Here we have prepared Doris's dashboard template (at the beginning of this document). When the download is complete, click `New dashboard` -> `Import dashboard` -> `Upload.json File` above to import the downloaded JSON file. + 8. After importing, you can name Dashboard by default `Doris Overview`. At the same time, you need to select the data source, where you select the `doris_monitor_data_source` you created earlier. + 9. Click `Import` to complete the import. Later, you can see Doris's dashboard display. + +8. So far, a simple Grafana has been built and configured. For more advanced usage, see [Official Documents] (http://docs.grafana.org/) + + +## Dashboard + +Here we briefly introduce Doris Dashboard. The content of Dashboard may change with the upgrade of version. This document is not guaranteed to be the latest Dashboard description. + +1. Top Bar + + ![](/images/dashboard_navibar.png) + + * The upper left corner is the name of Dashboard. + * The upper right corner shows the current monitoring time range. You can choose different time ranges by dropping down. You can also specify a regular refresh page interval. + * Cluster name: Each job name in the Prometheus configuration file represents a Doris cluster. Select a different cluster, and the chart below shows the monitoring information for the corresponding cluster. + * fe_master: The Master Frontend node corresponding to the cluster. + * fe_instance: All Frontend nodes corresponding to the cluster. Select a different Frontend, and the chart below shows the monitoring information for the Frontend. + * be_instance: All Backend nodes corresponding to the cluster. Select a different Backend, and the chart below shows the monitoring information for the Backend. + * Interval: Some charts show rate-related monitoring items, where you can choose how much interval to sample and calculate the rate (Note: 15s interval may cause some charts to be unable to display). + +2. Row. + + ![](/images/dashboard_row.png) + + In Grafana, the concept of Row is a set of graphs. As shown in the figure above, Overview and Cluster Overview are two different Rows. Row can be folded by clicking Row. Currently Dashboard has the following Rows (in continuous updates): + + 1. Overview: A summary display of all Doris clusters. + 2. Cluster Overview: A summary display of selected clusters. + 3. Query Statistic: Query-related monitoring of selected clusters. + 4. FE JVM: Select Frontend's JVM monitoring. + 5. BE: A summary display of the backends of the selected cluster. + 6. BE Task: Display of Backends Task Information for Selected Clusters. + +3. Charts + + ![](/images/dashboard_panel.png) + + A typical icon is divided into the following parts: + + 1. Hover the I icon in the upper left corner of the mouse to see the description of the chart. + 2. Click on the illustration below to view a monitoring item separately. Click again to display all. + 3. Dragging in the chart can select the time range. + 4. The selected cluster name is displayed in [] of the title. + 5. Some values correspond to the Y-axis on the left and some to the right, which can be distinguished by the `-right` at the end of the legend. + 6. Click on the name of the chart -> `Edit` to edit the chart. + +## Dashboard Update + +1. Click on `+` in the left column of Grafana and `Dashboard`. +2. Click `New dashboard` in the upper left corner, and `Import dashboard` appears on the right. +3. Click `Upload .json File` to select the latest template file. +4. Selecting Data Sources +5. Click on `Import (Overwrite)` to complete the template update. diff --git a/docs/en/administrator-guide/operation/multi-tenant.md b/docs/en/administrator-guide/operation/multi-tenant.md new file mode 100644 index 00000000000000..076d58af38734c --- /dev/null +++ b/docs/en/administrator-guide/operation/multi-tenant.md @@ -0,0 +1,238 @@ +--- +{ + "title": "Multi-tenancy(Exprimental)", + "language": "en" +} +--- + + + +# Multi-tenancy(Exprimental) + +This function is experimental and is not recommended for use in production environment. + +## Background +Doris, as a PB-level online report and multi-dimensional analysis database, provides cloud-based database services through open cloud, and deploys a physical cluster for each client in the cloud. Internally, a physical cluster deploys multiple services, and separately builds clusters for services with high isolation requirements. In view of the above problems: + +- Deployment of multiple physical cluster maintenance costs a lot (upgrade, functionality on-line, bug repair). +- A user's query or a bug caused by a query often affects other users. +- In the actual production environment, only one BE process can be deployed on a single machine. Multiple BEs can better solve the problem of fat nodes. And for join, aggregation operations can provide higher concurrency. + +Together with the above three points, Doris needs a new multi-tenant scheme, which not only can achieve better resource isolation and fault isolation, but also can reduce the cost of maintenance to meet the needs of common and private clouds. + +## Design Principles + +- Easy to use +- Low Development Cost +- Convenient migration of existing clusters + +## Noun Interpretation + +- FE: Frontend, the module for metadata management or query planning in Doris. +- BE: Backend, the module used to store and query data in Doris. +- Master: A role for FE. A Doris cluster has only one Master and the other FE is Observer or Follower. +- instance: A BE process is an instance in time. +- host: a single physical machine +- Cluster: A cluster consisting of multiple instances. +- Tenant: A cluster belongs to a tenant. Cluster is a one-to-one relationship with tenants. +- database: A user-created database + +## Main Ideas + +- Deploy instances of multiple BEs on a host to isolate resources at the process level. +- Multiple instances form a cluster, and a cluster is assigned to a business-independent tenant. +- FE adds cluster level and is responsible for cluster management. +- CPU, IO, memory and other resources are segregated by cgroup. + +## Design scheme + +In order to achieve isolation, the concept of **virtual cluster** is introduced. + +1. Cluster represents a virtual cluster consisting of instances of multiple BEs. Multiple clusters share FE. +2. Multiple instances can be started on a host. When a cluster is created, an arbitrary number of instances are selected to form a cluster. +3. While creating a cluster, an account named superuser is created, which belongs to the cluster. Super user can manage clusters, create databases, assign privileges, and so on. +4. After Doris starts, the sink creates a default cluster: default_cluster. If the user does not want to use the function of multi-cluster, the default cluster is provided and other operational details of multi-cluster are hidden. + +The concrete structure is as follows: +![](/images/multi_tenant_arch.png) + +## SQL interface + +- Login + + Default cluster login name: user_name@default_cluster or user_name + + Custom cluster login name: user_name@cluster_name + + `mysqlclient -h host -P port -u user_name@cluster_name -p password` + +- Add, delete, decommission and cancel BE + + `ALTER SYSTEM ADD BACKEND "host:port"` + `ALTER SYSTEM DROP BACKEND "host:port"` + `ALTER SYSTEM DECOMMISSION BACKEND "host:port"` + `CANCEL DECOMMISSION BACKEND "host:port"` + + It is strongly recommended to use DECOMMISSION instead of DROP to delete BACKEND. The DECOMMISSION operation will first need to copy data from the offline node to other instances in the cluster. After that, they will be offline. + +- Create a cluster and specify the password for the superuser account + + `CREATE CLUSTER cluster_name PROPERTIES ("instance_num" = "10") identified by "password"` + +- Enter a cluster + + `ENTER cluster name` + +- Cluster Expansion and Shrinkage + + `ALTER CLUSTER cluster_name PROPERTIES ("instance_num" = "10")` + + When the number of instances specified is more than the number of existing be in cluster, it is expanded and if less than it is condensed. + +- Link, migrate DB + + `LINK DATABASE src_cluster_name.db_name dest_cluster_name.db_name` + + Soft-chain dB of one cluster to dB of another cluster can be used by users who need temporary access to dB of other clusters but do not need actual data migration. + + `MIGRATE DATABASE src_cluster_name.db_name dest_cluster_name.db_name` + + If you need to migrate dB across clusters, after linking, migrate the actual migration of data. + + Migration does not affect the query, import and other operations of the current two dbs. This is an asynchronous operation. You can see the progress of migration through `SHOW MIGRATIONS`. + +- Delete clusters + + `DROP CLUSTER cluster_name` + + Deleting a cluster requires that all databases in the cluster be deleted manually first. + +- Others + + `SHOW CLUSTERS` + + Show clusters that have been created in the system. Only root users have this permission. + + `SHOW BACKENDS` + + View the BE instance in the cluster. + + `SHOW MIGRATIONS` + + Show current DB migration tasks. After the migration of DB is completed, you can view the progress of the migration through this command. + +## Detailed design + +1. Namespace isolation + + In order to introduce multi-tenant, the namespaces between clusters in the system need to be isolated. + + Doris's existing metadata is image + Journal (metadata is designed in related documents). Doris records operations involving metadata as a journal, and then regularly writes images in the form of **Fig. 1** and reads them in the order in which they are written when loaded. But this brings a problem that the format that has been written is not easy to modify. For example, the metadata format for recording data distribution is: database + table + tablet + replica nesting. If we want to isolate the namespace between clusters in the past way, we need to add a layer of cluster on the database and the level of internal metadata. Change to cluster + database + table + tablet + replica, as shown in **Figure 2**. But the problems brought about by adding one layer are as follows: + + - The change of metadata brought by adding one layer is incompatible. It needs to be written in cluster+db+table+tablet+replica level in the way of Figure 2. This changes the way of metadata organization in the past. The upgrading of the old version will be more troublesome. The ideal way is to write cluster in the order of Figure 3 in the form of existing metadata. Metadata. + + - All the DB and user used in the code need to add a layer of cluster. There are many workload changes and deep levels. Most of the code acquires db. The existing functions almost need to be changed, and a layer of cluster locks need to be nested on the basis of DB locks. + + ![](/images/palo_meta.png) + + To sum up, we adopt a prefix to DB and user names to isolate the internal problems caused by the conflict of DB and user names between clusters. + + As follows, all SQL input involves db name and user name, and all SQL input needs to spell the full name of DB and user according to their cluster. + + ![](/images/cluster_namaspace.png) + + In this way, the above two problems no longer exist. Metadata is also organized in a relatively simple way. That is to say, use ** Figure 3 ** to record db, user and nodes belonging to their own cluster. + +2. BE 节点管理 + + Each cluster has its own set of instances, which can be viewed through `SHOW BACKENDS`. In order to distinguish which cluster the instance belongs to and how it is used, BE introduces several states: + + - Free: When a BE node is added to the system, be is idle when it does not belong to any cluster. + - Use: When creating a cluster or expanding capacity is selected into a cluster, it is in use. + - Cluster decommission: If a shrinkage is performed, the be that is executing the shrinkage is in this state. After that, the be state becomes free. + - System decommission: be is offline. When the offline is completed, the be will be permanently deleted. + + Only root users can check whether all be in the cluster is used through the cluster item in `SHOW PROC "/backends"`. To be free is to be idle, otherwise to be in use. `SHOW BACKENDS `can only see the nodes in the cluster. The following is a schematic diagram of the state changes of be nodes. + + ![](/images/backend_state.png) + +3. Creating Clusters + + Only root users can create a cluster and specify any number of BE instances. + + Supports selecting multiple instances on the same machine. The general principle of selecting instance is to select be on different machines as much as possible and to make the number of be used on all machines as uniform as possible. + + For use, each user and DB belongs to a cluster (except root). To create user and db, you first need to enter a cluster. When a cluster is created, the system defaults to the manager of the cluster, the superuser account. Supuser has the right to create db, user, and view the number of be nodes in the cluster to which it belongs. All non-root user logins must specify a cluster, namely `user_name@cluster_name`. + + Only root users can view all clusters in the system through `SHOW CLUSTER', and can enter different clusters through @ different cluster names. User clusters are invisible except root. + + In order to be compatible with the old version of Doris, a cluster named default_cluster was built in, which could not be used when creating the cluster. + + ![](/images/user_authority.png) + +4. Cluster Expansion + + The process of cluster expansion is the same as that of cluster creation. BE instance on hosts that are not outside the cluster is preferred. The selected principles are the same as creating clusters. + +5. 集群缩容、CLUSTER DECOMMISSION + + Users can scale clusters by setting instance num of clusters. + + Cluster shrinkage takes precedence over downlining instances on hosts with the largest number of BE instances. + + Users can also directly use `ALTER CLUSTER DECOMMISSION BACKEND` to specify BE for cluster scaling. + +![](/images/replica_recover.png) + +6. Create table + + To ensure high availability, each fragmented copy must be on a different machine. So when building a table, the strategy of choosing the be where the replica is located is to randomly select a be on each host. Then, the number of be copies needed is randomly selected from these be. On the whole, it can distribute patches evenly on each machine. + + Therefore, adding a fragment that needs to create a 3-copy fragment, even if the cluster contains three or more instances, but only two or less hosts, still cannot create the fragment. + +7. Load Balancing + + The granularity of load balancing is cluster level, and there is no load balancing between clusters. However, the computing load is carried out at the host level, and there may be BE instances of different clusters on a host. In the cluster, the load is calculated by the number of fragments on each host and the utilization of storage, and then the fragments on the machine with high load are copied to the machine with low load (see the load balancing documents for details). + +8. LINK DATABASE (Soft Chain) + + Multiple clusters can access each other's data through a soft chain. The link level is dB for different clusters. + + DB in other clusters is accessed by adding DB information of other clusters that need to be accessed in one cluster. + + When querying the linked db, the computing and storage resources used are those of the cluster where the source DB is located. + + DB that is soft-chained cannot be deleted in the source cluster. Only when the linked DB is deleted can the source DB be deleted. Deleting link DB will not delete source db. + +9. MIGRATE DATABASE + + DB can be physically migrated between clusters. + + To migrate db, you must first link db. After migration, the data will migrate to the cluster where the linked DB is located, and after migration, the source DB will be deleted and the link will be disconnected. + + Data migration reuses the process of replicating data in load balancing and replica recovery (see load balancing related documents for details). Specifically, after executing the `MIRAGTE` command, Doris will modify the cluster of all copies of the source DB to the destination cluster in the metadata. + + Doris regularly checks whether machines in the cluster are balanced, replicas are complete, and redundant replicas are available. The migration of DB borrows this process, checking whether the be where the replica is located belongs to the cluster while checking the replica is complete, and if not, it is recorded in the replica to be restored. And when the duplicate is redundant to be deleted, it will first delete the duplicate outside the cluster, and then choose according to the existing strategy: the duplicate of the downtime be -> the duplicate of clone -> the duplicate of the backward version - > the duplicate on the host with high load, until the duplicate is not redundant. + +![](/images/cluster_link_and_migrate_db.png) + +10. BE process isolation + + In order to isolate the actual cpu, IO and memory between be processes, we need to rely on the deployment of be. When deploying, you need to configure the CGroup on the periphery and write all the processes of be to be deployed to the cgroup. If the physical isolation of IO between the data storage paths of each be configuration requires different disks, there is no much introduction here. diff --git a/docs/en/administrator-guide/operation/tablet-meta-tool.md b/docs/en/administrator-guide/operation/tablet-meta-tool.md new file mode 100644 index 00000000000000..0ad12e0675bc4f --- /dev/null +++ b/docs/en/administrator-guide/operation/tablet-meta-tool.md @@ -0,0 +1,112 @@ +--- +{ + "title": "Tablet metadata management tool", + "language": "en" +} +--- + + + +# Tablet metadata management tool + +## Background + +In the latest version of the code, we introduced RocksDB in BE to store meta-information of tablet, in order to solve various functional and performance problems caused by storing meta-information through header file. Currently, each data directory (root path) has a corresponding RocksDB instance, in which all tablets on the corresponding root path are stored in the key-value manner. + +To facilitate the maintenance of these metadata, we provide an online HTTP interface and an offline meta tool to complete related management operations. + +The HTTP interface is only used to view tablet metadata online, and can be used when the BE process is running. + +However, meta tool is only used for off-line metadata management operations. BE must be stopped before it can be used. + +The meta tool tool is stored in the Lib / directory of BE. + +## Operation + +### View Tablet Meta + +Viewing Tablet Meta information can be divided into online and offline methods + +#### On-line + +Access BE's HTTP interface to obtain the corresponding Tablet Meta information: + +api： + +`http://{host}:{port}/api/meta/header/{tablet_id}/{schema_hash}` + + +> Host: be Hostname +> +> port: BE's HTTP port +> +> tablet id: tablet id +> +> schema hash: tablet schema hash + +Give an example: + +`http://be_host:8040/api/meta/header/14156/2458238340` + +If the final query is successful, the Tablet Meta will be returned as json. + +#### Offline + +Get Tablet Meta on a disk based on the meta\ tool tool. + +Order: + +``` +./lib/meta_tool --root_path=/path/to/root_path --operation=get_meta --tablet_id=xxx --schema_hash=xxx +``` + +> root_path: The corresponding root_path path path configured in be.conf. + +The result is also a presentation of Tablet Meta in JSON format. + +### Load header + +The function of loading header is provided to realize manual migration of tablet. This function is based on Tablet Meta in JSON format, so if changes in the shard field and version information are involved, they can be changed directly in the JSON content of Tablet Meta. Then use the following commands to load. + +Order: + +``` +./lib/meta_tool --operation=load_meta --root_path=/path/to/root_path --json_header_path=path +``` + +### Delete header + +In order to realize the function of deleting a tablet from a disk of a be. + +Order: + +``` +./lib/meta_tool --operation=delete_meta --root_path=/path/to/root_path --tablet_id=xxx --schema_hash=xxx` +``` + +### TabletMeta in Pb format + +This command is to view the old file-based management PB format Tablet Meta, and to display Tablet Meta in JSON format. + +Order: + +``` +./lib/meta_tool --operation=show_meta --root_path=/path/to/root_path --pb_header_path=path +``` diff --git a/docs/en/administrator-guide/operation/tablet-repair-and-balance.md b/docs/en/administrator-guide/operation/tablet-repair-and-balance.md new file mode 100644 index 00000000000000..802228bb978f1d --- /dev/null +++ b/docs/en/administrator-guide/operation/tablet-repair-and-balance.md @@ -0,0 +1,667 @@ +--- +{ + "title": "Data replica management", + "language": "en" +} +--- + + + +# Data replica management + +Beginning with version 0.9.0, Doris introduced an optimized replica management strategy and supported a richer replica status viewing tool. This document focuses on Doris data replica balancing, repair scheduling strategies, and replica management operations and maintenance methods. Help users to more easily master and manage the replica status in the cluster. + +> Repairing and balancing copies of tables with Collocation attributes can be referred to `docs/documentation/cn/administrator-guide/colocation-join.md'.` + +## Noun Interpretation + +1. Tablet: The logical fragmentation of a Doris table, where a table has multiple fragmentations. +2. Replica: A sliced copy, defaulting to three copies of a slice. +3. Healthy Replica: A healthy copy that survives at Backend and has a complete version. +4. Tablet Checker (TC): A resident background thread that scans all Tablets regularly, checks the status of these Tablets, and decides whether to send them to Tablet Scheduler based on the results. +5. Tablet Scheduler (TS): A resident background thread that handles Tablets sent by Tablet Checker that need to be repaired. At the same time, cluster replica balancing will be carried out. +6. Tablet SchedCtx (TSC): is a tablet encapsulation. When TC chooses a tablet, it encapsulates it as a TSC and sends it to TS. +7. Storage Medium: Storage medium. Doris supports specifying different storage media for partition granularity, including SSD and HDD. The replica scheduling strategy is also scheduled for different storage media. + +``` + + +--------+ +-----------+ + | Meta | | Backends | + +---^----+ +------^----+ + | | | 3. Send clone tasks + 1. Check tablets | | | + +--------v------+ +-----------------+ + | TabletChecker +--------> TabletScheduler | + +---------------+ +-----------------+ + 2. Waiting to be scheduled + + +``` +The figure above is a simplified workflow. + + +## Duplicate status + +Multiple copies of a Tablet may cause state inconsistencies due to certain circumstances. Doris will attempt to automatically fix the inconsistent copies of these states so that the cluster can recover from the wrong state as soon as possible. + +**The health status of a Replica is as follows:** + +1. BAD + + That is, the copy is damaged. Includes, but is not limited to, the irrecoverable damaged status of copies caused by disk failures, BUGs, etc. + +2. VERSION\_MISSING + + Version missing. Each batch of imports in Doris corresponds to a data version. A copy of the data consists of several consecutive versions. However, due to import errors, delays and other reasons, the data version of some copies may be incomplete. + +3. HEALTHY + + Health copy. That is, a copy of the normal data, and the BE node where the copy is located is in a normal state (heartbeat is normal and not in the offline process). + +**The health status of a Tablet is determined by the status of all its copies. There are the following categories:** + +1. REPLICA\_MISSING + + The copy is missing. That is, the number of surviving copies is less than the expected number of copies. + +2. VERSION\_INCOMPLETE + + The number of surviving copies is greater than or equal to the number of expected copies, but the number of healthy copies is less than the number of expected copies. + +3. REPLICA\_RELOCATING + + Have a full number of live copies of the replication num version, but the BE nodes where some copies are located are in unavailable state (such as decommission) + +4. REPLICA\_MISSING\_IN\_CLUSTER + + When using multi-cluster, the number of healthy replicas is greater than or equal to the expected number of replicas, but the number of replicas in the corresponding cluster is less than the expected number of replicas. + +5. REDUNDANT + + Duplicate redundancy. Healthy replicas are in the corresponding cluster, but the number of replicas is larger than the expected number. Or there's a spare copy of unavailable. + +6. FORCE\_REDUNDANT + + This is a special state. It only occurs when the number of expected replicas is greater than or equal to the number of available nodes, and when the Tablet is in the state of replica missing. In this case, you need to delete a copy first to ensure that there are available nodes for creating a new copy. + +7. COLOCATE\_MISMATCH + + Fragmentation status of tables for Collocation attributes. Represents that the distribution of fragmented copies is inconsistent with the specified distribution of Colocation Group. + +8. COLOCATE\_REDUNDANT + + Fragmentation status of tables for Collocation attributes. Represents the fragmented copy redundancy of the Colocation table. + +8. HEALTHY + + Healthy fragmentation, that is, conditions [1-5] are not satisfied. + +## Replica Repair + +As a resident background process, Tablet Checker regularly checks the status of all fragments. For unhealthy fragmentation, it will be sent to Tablet Scheduler for scheduling and repair. The actual operation of repair is accomplished by clone task on BE. FE is only responsible for generating these clone tasks. + +> Note 1: The main idea of replica repair is to make the number of fragmented replicas reach the desired value by creating or completing them first. Then delete the redundant copy. +> +> Note 2: A clone task is to complete the process of copying specified data from a specified remote end to a specified destination. + +For different states, we adopt different repair methods: + +1. REPLICA\_MISSING/REPLICA\_RELOCATING + + Select a low-load, available BE node as the destination. Choose a healthy copy as the source. Clone tasks copy a complete copy from the source to the destination. For replica completion, we will directly select an available BE node, regardless of the storage medium. + +2. VERSION\_INCOMPLETE + + Select a relatively complete copy as the destination. Choose a healthy copy as the source. The clone task attempts to copy the missing version from the source to the destination. + +3. REPLICA\_MISSING\_IN\_CLUSTER + + This state processing method is the same as REPLICAMISSING. + +4. REDUNDANT + + Usually, after repair, there will be redundant copies in fragmentation. We select a redundant copy to delete it. The selection of redundant copies follows the following priorities: + 1. The BE where the copy is located has been offline. + 2. The copy is damaged + 3. The copy is lost in BE or offline + 4. The replica is in the CLONE state (which is an intermediate state during clone task execution) + 5. The copy has version missing + 6. The cluster where the copy is located is incorrect + 7. The BE node where the replica is located has a high load + +5. FORCE\_REDUNDANT + + Unlike REDUNDANT, because at this point Tablet has a copy missing, because there are no additional available nodes for creating new copies. So at this point, a copy must be deleted to free up a available node for creating a new copy. + The order of deleting copies is the same as REDUNDANT. + +6. COLOCATE\_MISMATCH + + Select one of the replica distribution BE nodes specified in Colocation Group as the destination node for replica completion. + +7. COLOCATE\_REDUNDANT + + Delete a copy on a BE node that is distributed by a copy specified in a non-Colocation Group. + + Doris does not deploy a copy of the same Tablet on a different BE of the same host when selecting a replica node. It ensures that even if all BEs on the same host are deactivated, all copies will not be lost. + +### Scheduling priority + +Waiting for the scheduled fragments in Tablet Scheduler gives different priorities depending on the status. High priority fragments will be scheduled first. There are currently several priorities. + +1. VERY\_HIGH + + * REDUNDANT. For slices with duplicate redundancy, we give priority to them. Logically, duplicate redundancy is the least urgent, but because it is the fastest to handle and can quickly release resources (such as disk space, etc.), we give priority to it. + * FORCE\_REDUNDANT. Ditto. + +2. HIGH + + * REPLICA\_MISSING and most copies are missing (for example, 2 copies are missing in 3 copies) + * VERSION\_INCOMPLETE and most copies are missing + * COLOCATE\_MISMATCH We hope that the fragmentation related to the Collocation table can be repaired as soon as possible. + * COLOCATE\_REDUNDANT + +3. NORMAL + + * REPLICA\_MISSING, but most survive (for example, three copies lost one) + * VERSION\_INCOMPLETE, but most copies are complete + * REPLICA\_RELOCATING and relocate is required for most replicas (e.g. 3 replicas with 2 replicas) + +4. LOW + + * REPLICA\_MISSING\_IN\_CLUSTER + * REPLICA\_RELOCATING most copies stable + +### Manual priority + +The system will automatically determine the scheduling priority. Sometimes, however, users want the fragmentation of some tables or partitions to be repaired faster. So we provide a command that the user can specify that a slice of a table or partition is repaired first: + +`ADMIN REPAIR TABLE tbl [PARTITION (p1, p2, ...)];` + +This command tells TC to give VERY HIGH priority to the problematic tables or partitions that need to be repaired first when scanning Tablets. + +> Note: This command is only a hint, which does not guarantee that the repair will be successful, and the priority will change with the scheduling of TS. And when Master FE switches or restarts, this information will be lost. + +Priority can be cancelled by the following commands: + +`ADMIN CANCEL REPAIR TABLE tbl [PARTITION (p1, p2, ...)];` + +### Priority scheduling + +Priority ensures that severely damaged fragments can be repaired first, and improves system availability. But if the high priority repair task fails all the time, the low priority task will never be scheduled. Therefore, we will dynamically adjust the priority of tasks according to the running status of tasks, so as to ensure that all tasks have the opportunity to be scheduled. + +* If the scheduling fails for five consecutive times (e.g., no resources can be obtained, no suitable source or destination can be found, etc.), the priority will be lowered. +* If not scheduled for 30 minutes, priority will be raised. +* The priority of the same tablet task is adjusted at least five minutes apart. + +At the same time, in order to ensure the weight of the initial priority, we stipulate that the initial priority is VERY HIGH, and the lowest is lowered to NORMAL. When the initial priority is LOW, it is raised to HIGH at most. The priority adjustment here also adjusts the priority set manually by the user. + +## Duplicate Equilibrium + +Doris automatically balances replicas within the cluster. The main idea of balancing is to create a replica of some fragments on low-load nodes, and then delete the replicas of these fragments on high-load nodes. At the same time, because of the existence of different storage media, there may or may not exist one or two storage media on different BE nodes in the same cluster. We require that fragments of storage medium A be stored in storage medium A as far as possible after equalization. So we divide the BE nodes of the cluster according to the storage medium. Then load balancing scheduling is carried out for different BE node sets of storage media. + +Similarly, replica balancing ensures that a copy of the same table will not be deployed on the BE of the same host. + +### BE Node Load + +We use Cluster LoadStatistics (CLS) to represent the load balancing of each backend in a cluster. Tablet Scheduler triggers cluster equilibrium based on this statistic. We currently calculate a load Score for each BE as the BE load score by using **disk usage** and **number of copies**. The higher the score, the heavier the load on the BE. + +Disk usage and number of copies have a weight factor, which is **capacityCoefficient** and **replicaNumCoefficient**, respectively. The sum of them is **constant to 1**. Among them, capacityCoefficient will dynamically adjust according to actual disk utilization. When the overall disk utilization of a BE is below 50%, the capacityCoefficient value is 0.5, and if the disk utilization is above 75% (configurable through the FE configuration item `capacity_used_percent_high_water`), the value is 1. If the utilization rate is between 50% and 75%, the weight coefficient increases smoothly. The formula is as follows: + +`capacityCoefficient = 2 * Disk Utilization - 0.5` + +The weight coefficient ensures that when disk utilization is too high, the backend load score will be higher to ensure that the BE load is reduced as soon as possible. + +Tablet Scheduler updates CLS every 1 minute. + +### Equilibrium strategy + +Tablet Scheduler uses Load Balancer to select a certain number of healthy fragments as candidate fragments for balance in each round of scheduling. In the next scheduling, balanced scheduling will be attempted based on these candidate fragments. + +## Resource control + +Both replica repair and balancing are accomplished by replica copies between BEs. If the same BE performs too many tasks at the same time, it will bring a lot of IO pressure. Therefore, Doris controls the number of tasks that can be performed on each node during scheduling. The smallest resource control unit is the disk (that is, a data path specified in be.conf). By default, we configure two slots per disk for replica repair. A clone task occupies one slot at the source and one slot at the destination. If the number of slots is zero, no more tasks will be assigned to this disk. The number of slots can be configured by FE's `schedule_slot_num_per_path` parameter. + +In addition, by default, we provide two separate slots per disk for balancing tasks. The purpose is to prevent high-load nodes from losing space by balancing because slots are occupied by repair tasks. + +## Duplicate Status View + +Duplicate status view mainly looks at the status of the duplicate, as well as the status of the duplicate repair and balancing tasks. Most of these states **exist only in** Master FE nodes. Therefore, the following commands need to be executed directly to Master FE. + +### Duplicate status + +1. Global state checking + + Through `SHOW PROC'/ statistic'; `commands can view the replica status of the entire cluster. + + ``` + +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ + | DbId | DbName | TableNum | PartitionNum | IndexNum | TabletNum | ReplicaNum | UnhealthyTabletNum | InconsistentTabletNum | + +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ + | 35153636 | default_cluster:DF_Newrisk | 3 | 3 | 3 | 96 | 288 | 0 | 0 | + | 48297972 | default_cluster:PaperData | 0 | 0 | 0 | 0 | 0 | 0 | 0 | + | 5909381 | default_cluster:UM_TEST | 7 | 7 | 10 | 320 | 960 | 1 | 0 | + | Total | 240 | 10 | 10 | 13 | 416 | 1248 | 1 | 0 | + +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ + ``` + + The `UnhealthyTabletNum` column shows how many Tablets are in an unhealthy state in the corresponding database. `The Inconsistent Tablet Num` column shows how many Tablets are in an inconsistent replica state in the corresponding database. The last `Total` line counts the entire cluster. Normally `Unhealth Tablet Num` and `Inconsistent Tablet Num` should be 0. If it's not zero, you can further see which Tablets are there. As shown in the figure above, one table in the UM_TEST database is not healthy, you can use the following command to see which one is. + + `SHOW PROC '/statistic/5909381';` + + Among them `5909381'is the corresponding DbId. + + ``` + +------------------+---------------------+ + | UnhealthyTablets | InconsistentTablets | + +------------------+---------------------+ + | [40467980] | [] | + +------------------+---------------------+ + ``` + + The figure above shows the specific unhealthy Tablet ID (40467980). Later we'll show you how to view the status of each copy of a specific Tablet. + +2. Table (partition) level status checking + + Users can view the status of a copy of a specified table or partition through the following commands and filter the status through a WHERE statement. If you look at table tbl1, the state on partitions P1 and P2 is a copy of NORMAL: + + `ADMIN SHOW REPLICA STATUS FROM tbl1 PARTITION (p1, p2) WHERE STATUS = "NORMAL";` + + ``` + +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ + | TabletId | ReplicaId | BackendId | Version | LastFailedVersion | LastSuccessVersion | CommittedVersion | SchemaHash | VersionNum | IsBad | State | Status | + +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ + | 29502429 | 29502432 | 10006 | 2 | -1 | 2 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502429 | 36885996 | 10002 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502429 | 48100551 | 10007 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502433 | 29502434 | 10001 | 2 | -1 | 2 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502433 | 44900737 | 10004 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502433 | 48369135 | 10006 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | + +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ + ``` + + The status of all copies is shown here. Where `IsBad` is listed as `true`, the copy is damaged. The `Status` column displays other states. Specific status description, you can see help through `HELP ADMIN SHOW REPLICA STATUS`. + + ` The ADMIN SHOW REPLICA STATUS `command is mainly used to view the health status of copies. Users can also view additional information about copies of a specified table by using the following commands: + + `SHOW TABLET FROM tbl1;` + + ``` + +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ + | TabletId | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | DataSize | RowCount | State | LstConsistencyCheckTime | CheckVersion | CheckVersionHash | VersionCount | PathHash | + +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ + | 29502429 | 29502432 | 10006 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -5822326203532286804 | + | 29502429 | 36885996 | 10002 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -1441285706148429853 | + | 29502429 | 48100551 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -4784691547051455525 | + +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ + ``` + + The figure above shows some additional information, including copy size, number of rows, number of versions, where the data path is located. + + > Note: The contents of the `State'column shown here do not represent the health status of the replica, but the status of the replica under certain tasks, such as CLONE, SCHEMA CHANGE, ROLLUP, etc. + + In addition, users can check the distribution of replicas in a specified table or partition by following commands. + + `ADMIN SHOW REPLICA DISTRIBUTION FROM tbl1;` + + ``` + +-----------+------------+-------+---------+ + | BackendId | ReplicaNum | Graph | Percent | + +-----------+------------+-------+---------+ + | 10000 | 7 | | 7.29 % | + | 10001 | 9 | | 9.38 % | + | 10002 | 7 | | 7.29 % | + | 10003 | 7 | | 7.29 % | + | 10004 | 9 | | 9.38 % | + | 10005 | 11 | > | 11.46 % | + | 10006 | 18 | > | 18.75 % | + | 10007 | 15 | > | 15.62 % | + | 10008 | 13 | > | 13.54 % | + +-----------+------------+-------+---------+ + ``` + + Here we show the number and percentage of replicas of table tbl1 on each BE node, as well as a simple graphical display. + +4. Tablet level status checking + + When we want to locate a specific Tablet, we can use the following command to view the status of a specific Tablet. For example, check the tablet with ID 2950253: + + `SHOW TABLET 29502553;` + + ``` + +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ + | DbName | TableName | PartitionName | IndexName | DbId | TableId | PartitionId | IndexId | IsSync | DetailCmd | + +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ + | default_cluster:test | test | test | test | 29502391 | 29502428 | 29502427 | 29502428 | true | SHOW PROC '/dbs/29502391/29502428/partitions/29502427/29502428/29502553'; | + +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ + ``` + + The figure above shows the database, tables, partitions, roll-up tables and other information corresponding to this tablet. The user can copy the command in the `DetailCmd` command to continue executing: + + `Show Proc'/DBS/29502391/29502428/Partitions/29502427/29502428/29502553;` + + ``` + +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ + | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | SchemaHash | DataSize | RowCount | State | IsBad | VersionCount | PathHash | + +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ + | 43734060 | 10004 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | -8566523878520798656 | + | 29502555 | 10002 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | 1885826196444191611 | + | 39279319 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | 1656508631294397870 | + +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ + ``` + + The figure above shows all replicas of the corresponding Tablet. The content shown here is the same as `SHOW TABLET FROM tbl1;`. But here you can clearly see the status of all copies of a specific Tablet. + +### Duplicate Scheduling Task + +1. View tasks waiting to be scheduled + + `SHOW PROC '/cluster_balance/pending_tablets';` + + ``` + +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ + | TabletId | Type | Status | State | OrigPrio | DynmPrio | SrcBe | SrcPath | DestBe | DestPath | Timeout | Create | LstSched | LstVisit | Finished | Rate | FailedSched | FailedRunning | LstAdjPrio | VisibleVer | VisibleVerHash | CmtVer | CmtVerHash | ErrMsg | + +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ + | 4203036 | REPAIR | REPLICA_MISSING | PENDING | HIGH | LOW | -1 | -1 | -1 | -1 | 0 | 2019-02-21 15:00:20 | 2019-02-24 11:18:41 | 2019-02-24 11:18:41 | N/A | N/A | 2 | 0 | 2019-02-21 15:00:43 | 1 | 0 | 2 | 0 | unable to find source replica | + +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ + ``` + + The specific meanings of each column are as follows: + + * TabletId: The ID of the Tablet waiting to be scheduled. A scheduling task is for only one Tablet + * Type: Task type, which can be REPAIR (repair) or BALANCE (balance) + * Status: The current status of the Tablet, such as REPLICAMISSING (copy missing) + * State: The status of the scheduling task may be PENDING/RUNNING/FINISHED/CANCELLED/TIMEOUT/UNEXPECTED + * OrigPrio: Initial Priority + * DynmPrio: Current dynamically adjusted priority + * SrcBe: ID of the BE node at the source end + * SrcPath: hash value of the path of the BE node at the source end + * DestBe: ID of destination BE node + * DestPath: hash value of the path of the destination BE node + * Timeout: When the task is scheduled successfully, the timeout time of the task is displayed here in units of seconds. + * Create: The time when the task was created + * LstSched: The last time a task was scheduled + * LstVisit: The last time a task was accessed. Here "accessed" refers to the processing time points associated with the task, including scheduling, task execution reporting, and so on. + * Finished: Task End Time + * Rate: Clone Task Data Copy Rate + * Failed Sched: Number of Task Scheduling Failures + * Failed Running: Number of task execution failures + * LstAdjPrio: Time of last priority adjustment + * CmtVer/CmtVerHash/VisibleVer/VisibleVerHash: version information for clone tasks + * ErrMsg: Error messages that occur when tasks are scheduled and run + +2. View running tasks + + `SHOW PROC '/cluster_balance/running_tablets';` + + The columns in the result have the same meaning as `pending_tablets`. + +3. View completed tasks + + `SHOW PROC '/cluster_balance/history_tablets';` + + By default, we reserve only the last 1,000 completed tasks. The columns in the result have the same meaning as `pending_tablets`. If `State` is listed as `FINISHED`, the task is normally completed. For others, you can see the specific reason based on the error information in the `ErrMsg` column. + +## Viewing Cluster Load and Scheduling Resources + +1. Cluster load + + You can view the current load of the cluster by following commands: + + `SHOW PROC '/cluster_balance/cluster_load_stat';` + + First of all, we can see the division of different storage media: + + ``` + +---------------+ + | StorageMedium | + +---------------+ + | HDD | + | SSD | + +---------------+ + ``` + + Click on a storage medium to see the equilibrium state of the BE node that contains the storage medium: + + `SHOW PROC '/cluster_balance/cluster_load_stat/HDD';` + + ``` + +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ + | BeId | Cluster | Available | UsedCapacity | Capacity | UsedPercent | ReplicaNum | CapCoeff | ReplCoeff | Score | Class | + +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ + | 10003 | default_cluster | true | 3477875259079 | 19377459077121 | 17.948 | 493477 | 0.5 | 0.5 | 0.9284678149967587 | MID | + | 10002 | default_cluster | true | 3607326225443 | 19377459077121 | 18.616 | 496928 | 0.5 | 0.5 | 0.948660871419998 | MID | + | 10005 | default_cluster | true | 3523518578241 | 19377459077121 | 18.184 | 545331 | 0.5 | 0.5 | 0.9843539990641831 | MID | + | 10001 | default_cluster | true | 3535547090016 | 19377459077121 | 18.246 | 558067 | 0.5 | 0.5 | 0.9981869446537612 | MID | + | 10006 | default_cluster | true | 3636050364835 | 19377459077121 | 18.764 | 547543 | 0.5 | 0.5 | 1.0011489897614072 | MID | + | 10004 | default_cluster | true | 3506558163744 | 15501967261697 | 22.620 | 468957 | 0.5 | 0.5 | 1.0228319835582569 | MID | + | 10007 | default_cluster | true | 4036460478905 | 19377459077121 | 20.831 | 551645 | 0.5 | 0.5 | 1.057279369420761 | MID | + | 10000 | default_cluster | true | 4369719923760 | 19377459077121 | 22.551 | 547175 | 0.5 | 0.5 | 1.0964036415787461 | MID | + +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ + ``` + + Some of these columns have the following meanings: + + * Available: True means that BE heartbeat is normal and not offline. + * UsedCapacity: Bytes, the size of disk space used on BE + * Capacity: Bytes, the total disk space size on BE + * UsedPercent: Percentage, disk space utilization on BE + * ReplicaNum: Number of copies on BE + * CapCoeff/ReplCoeff: Weight Coefficient of Disk Space and Copy Number + * Score: Load score. The higher the score, the heavier the load. + * Class: Classified by load, LOW/MID/HIGH. Balanced scheduling moves copies from high-load nodes to low-load nodes + + Users can further view the utilization of each path on a BE, such as the BE with ID 10001: + + `SHOW PROC '/cluster_balance/cluster_load_stat/HDD/10001';` + + ``` + +------------------+------------------+---------------+---------------+---------+--------+----------------------+ + | RootPath | DataUsedCapacity | AvailCapacity | TotalCapacity | UsedPct | State | PathHash | + +------------------+------------------+---------------+---------------+---------+--------+----------------------+ + | /home/disk4/palo | 498.757 GB | 3.033 TB | 3.525 TB | 13.94 % | ONLINE | 4883406271918338267 | + | /home/disk3/palo | 704.200 GB | 2.832 TB | 3.525 TB | 19.65 % | ONLINE | -5467083960906519443 | + | /home/disk1/palo | 512.833 GB | 3.007 TB | 3.525 TB | 14.69 % | ONLINE | -7733211489989964053 | + | /home/disk2/palo | 881.955 GB | 2.656 TB | 3.525 TB | 24.65 % | ONLINE | 4870995507205544622 | + | /home/disk5/palo | 694.992 GB | 2.842 TB | 3.525 TB | 19.36 % | ONLINE | 1916696897889786739 | + +------------------+------------------+---------------+---------------+---------+--------+----------------------+ + ``` + + The disk usage of each data path on the specified BE is shown here. + +2. Scheduling resources + + Users can view the current slot usage of each node through the following commands: + + `SHOW PROC '/cluster_balance/working_slots';` + + ``` + +----------+----------------------+------------+------------+-------------+----------------------+ + | BeId | PathHash | AvailSlots | TotalSlots | BalanceSlot | AvgRate | + +----------+----------------------+------------+------------+-------------+----------------------+ + | 10000 | 8110346074333016794 | 2 | 2 | 2 | 2.459007474009069E7 | + | 10000 | -5617618290584731137 | 2 | 2 | 2 | 2.4730105014001578E7 | + | 10001 | 4883406271918338267 | 2 | 2 | 2 | 1.6711402709780257E7 | + | 10001 | -5467083960906519443 | 2 | 2 | 2 | 2.7540126380326536E7 | + | 10002 | 9137404661108133814 | 2 | 2 | 2 | 2.417217089806745E7 | + | 10002 | 1885826196444191611 | 2 | 2 | 2 | 1.6327378456676323E7 | + +----------+----------------------+------------+------------+-------------+----------------------+ + ``` + + In this paper, data path is used as granularity to show the current use of slot. Among them, `AvgRate'is the copy rate of clone task in bytes/seconds on the path of historical statistics. + +3. Priority repair view + + The following command allows you to view the priority repaired tables or partitions set by the `ADMIN REPAIR TABLE'command. + + `SHOW PROC '/cluster_balance/priority_repair'`; + + Among them, `Remaining TimeMs'indicates that these priority fixes will be automatically removed from the priority fix queue after this time. In order to prevent resources from being occupied due to the failure of priority repair. + +### Scheduler Statistical Status View + +We have collected some statistics of Tablet Checker and Tablet Scheduler during their operation, which can be viewed through the following commands: + +`SHOW PROC '/cluster_balance/sched_stat'`; + +``` ++---------------------------------------------------+-------------+ +| Item | Value | ++---------------------------------------------------+-------------+ +| num of tablet check round | 12041 | +| cost of tablet check(ms) | 7162342 | +| num of tablet checked in tablet checker | 18793506362 | +| num of unhealthy tablet checked in tablet checker | 7043900 | +| num of tablet being added to tablet scheduler | 1153 | +| num of tablet schedule round | 49538 | +| cost of tablet schedule(ms) | 49822 | +| num of tablet being scheduled | 4356200 | +| num of tablet being scheduled succeeded | 320 | +| num of tablet being scheduled failed | 4355594 | +| num of tablet being scheduled discard | 286 | +| num of tablet priority upgraded | 0 | +| num of tablet priority downgraded | 1096 | +| num of clone task | 230 | +| num of clone task succeeded | 228 | +| num of clone task failed | 2 | +| num of clone task timeout | 2 | +| num of replica missing error | 4354857 | +| num of replica version missing error | 967 | +| num of replica relocating | 0 | +| num of replica redundant error | 90 | +| num of replica missing in cluster error | 0 | +| num of balance scheduled | 0 | ++---------------------------------------------------+-------------+ +``` + +The meanings of each line are as follows: + +* num of tablet check round：Tablet Checker 检查次数 +* cost of tablet check(ms)：Tablet Checker 检查总耗时 +* num of tablet checked in tablet checker：Tablet Checker 检查过的 tablet 数量 +* num of unhealthy tablet checked in tablet checker：Tablet Checker 检查过的不健康的 tablet 数量 +* num of tablet being added to tablet scheduler：被提交到 Tablet Scheduler 中的 tablet 数量 +* num of tablet schedule round：Tablet Scheduler 运行次数 +* cost of tablet schedule(ms)：Tablet Scheduler 运行总耗时 +* num of tablet being scheduled：被调度的 Tablet 总数量 +* num of tablet being scheduled succeeded：被成功调度的 Tablet 总数量 +* num of tablet being scheduled failed：调度失败的 Tablet 总数量 +* num of tablet being scheduled discard：调度失败且被抛弃的 Tablet 总数量 +* num of tablet priority upgraded：优先级上调次数 +* num of tablet priority downgraded：优先级下调次数 +* num of clone task: number of clone tasks generated +* num of clone task succeeded：clone 任务成功的数量 +* num of clone task failed：clone 任务失败的数量 +* num of clone task timeout：clone 任务超时的数量 +* num of replica missing error: the number of tablets whose status is checked is the missing copy +* num of replica version missing error：检查的状态为版本缺失的 tablet 的数量（该统计值包括了 num of replica relocating 和 num of replica missing in cluster error） +*num of replica relocation *29366;* 24577;*replica relocation tablet * +* num of replica redundant error: Number of tablets whose checked status is replica redundant +* num of replica missing in cluster error：检查的状态为不在对应 cluster 的 tablet 的数量 +* num of balance scheduled：均衡调度的次数 + +> Note: The above states are only historical accumulative values. We also print these statistics regularly in the FE logs, where the values in parentheses represent the number of changes in each statistical value since the last printing dependence of the statistical information. + +## Relevant configuration instructions + +### Adjustable parameters + +The following adjustable parameters are all configurable parameters in fe.conf. + +* use\_new\_tablet\_scheduler + + * Description: Whether to enable the new replica scheduling mode. The new replica scheduling method is the replica scheduling method introduced in this document. If turned on, `disable_colocate_join` must be `true`. Because the new scheduling strategy does not support data fragmentation scheduling of co-locotion tables for the time being. + * Default value:true + * Importance: High + +* tablet\_repair\_delay\_factor\_second + + * Note: For different scheduling priorities, we will delay different time to start repairing. In order to prevent a large number of unnecessary replica repair tasks from occurring in the process of routine restart and upgrade. This parameter is a reference coefficient. For HIGH priority, the delay is the reference coefficient * 1; for NORMAL priority, the delay is the reference coefficient * 2; for LOW priority, the delay is the reference coefficient * 3. That is, the lower the priority, the longer the delay waiting time. If the user wants to repair the copy as soon as possible, this parameter can be reduced appropriately. + * Default value: 60 seconds + * Importance: High + +* schedule\_slot\_num\_per\_path + + * Note: The default number of slots allocated to each disk for replica repair. This number represents the number of replica repair tasks that a disk can run simultaneously. If you want to repair the copy faster, you can adjust this parameter appropriately. The higher the single value, the greater the impact on IO. + * Default value: 2 + * Importance: High + +* balance\_load\_score\_threshold + + * Description: Threshold of Cluster Equilibrium. The default is 0.1, or 10%. When the load core of a BE node is not higher than or less than 10% of the average load core, we think that the node is balanced. If you want to make the cluster load more even, you can adjust this parameter appropriately. + * Default value: 0.1 + * Importance: + +* storage\_high\_watermark\_usage\_percent 和 storage\_min\_left\_capacity\_bytes + + * Description: These two parameters represent the upper limit of the maximum space utilization of a disk and the lower limit of the minimum space remaining, respectively. When the space utilization of a disk is greater than the upper limit or the remaining space is less than the lower limit, the disk will no longer be used as the destination address for balanced scheduling. + * Default values: 0.85 and 1048576000 (1GB) + * Importance: + +* disable\_balance + + * Description: Control whether to turn off the balancing function. When replicas are in equilibrium, some functions, such as ALTER TABLE, will be banned. Equilibrium can last for a long time. Therefore, if the user wants to do the prohibited operation as soon as possible. This parameter can be set to true to turn off balanced scheduling. + * Default value:true + * Importance: + +### Unadjustable parameters + +The following parameters do not support modification for the time being, just for illustration. + +* Tablet Checker scheduling interval + + Tablet Checker schedules checks every 20 seconds. + +* Tablet Scheduler scheduling interval + + Tablet Scheduler schedules every five seconds + +* Number of Tablet Scheduler Schedules per Batch + + Tablet Scheduler schedules up to 50 tablets at a time. + +* Tablet Scheduler Maximum Waiting Schedule and Number of Tasks in Operation + + The maximum number of waiting tasks and running tasks is 2000. When over 2000, Tablet Checker will no longer generate new scheduling tasks to Tablet Scheduler. + +* Tablet Scheduler Maximum Balanced Task Number + + The maximum number of balanced tasks is 500. When more than 500, there will be no new balancing tasks. + +* Number of slots per disk for balancing tasks + + The number of slots per disk for balancing tasks is 2. This slot is independent of the slot used for replica repair. + +* Update interval of cluster equilibrium + + Tablet Scheduler recalculates the load score of the cluster every 20 seconds. + +* Minimum and Maximum Timeout for Clone Tasks + + A clone task timeout time range is 3 minutes to 2 hours. The specific timeout is calculated by the size of the tablet. The formula is (tablet size)/ (5MB/s). When a clone task fails three times, the task terminates. + +* Dynamic Priority Adjustment Strategy + + The minimum priority adjustment interval is 5 minutes. When a tablet schedule fails five times, priority is lowered. When a tablet is not scheduled for 30 minutes, priority is raised. + +## Relevant issues + +* In some cases, the default replica repair and balancing strategy may cause the network to be full (mostly in the case of gigabit network cards and a large number of disks per BE). At this point, some parameters need to be adjusted to reduce the number of simultaneous balancing and repair tasks. + +* Current balancing strategies for copies of Colocate Table do not guarantee that copies of the same Tablet will not be distributed on the BE of the same host. However, the repair strategy of the copy of Colocate Table detects this distribution error and corrects it. However, it may occur that after correction, the balancing strategy regards the replicas as unbalanced and rebalances them. As a result, the Colocate Group can not achieve stability because of the continuous alternation between the two states. In view of this situation, we suggest that when using Colocate attribute, we try to ensure that the cluster is isomorphic, so as to reduce the probability that replicas are distributed on the same host. diff --git a/docs/en/administrator-guide/privilege.md b/docs/en/administrator-guide/privilege.md new file mode 100644 index 00000000000000..82d0579f7156b7 --- /dev/null +++ b/docs/en/administrator-guide/privilege.md @@ -0,0 +1,224 @@ +--- +{ + "title": "Authority Management", + "language": "en" +} +--- + + + +# Authority Management + +Doris's new privilege management system refers to Mysql's privilege management mechanism, achieves table-level fine-grained privilege control, role-based privilege access control, and supports whitelist mechanism. + +## Noun Interpretation + +1. user_identity + + In a permission system, a user is identified as a User Identity. User ID consists of two parts: username and userhost. Username is a user name, which is composed of English upper and lower case. Userhost represents the IP from which the user link comes. User_identity is presented as username@'userhost', representing the username from userhost. + + Another expression of user_identity is username@['domain'], where domain is the domain name, which can be resolved into a set of IPS by DNS BNS (Baidu Name Service). The final expression is a set of username@'userhost', so we use username@'userhost'to represent it. + +2. Privilege + + The objects of permissions are nodes, databases or tables. Different permissions represent different operating permissions. + +3. Role + + Doris can create custom named roles. Roles can be seen as a set of permissions. When a newly created user can be assigned a role, the role's permissions are automatically granted. Subsequent changes in the role's permissions will also be reflected in all user permissions that belong to the role. + +4. user_property + + User attributes are directly attached to a user, not to a user identity. That is, both cmy@'192.%'and cmy@['domain'] have the same set of user attributes, which belong to user cmy, not cmy@'192.%' or cmy@['domain']. + + User attributes include, but are not limited to, the maximum number of user connections, import cluster configuration, and so on. + +## Supported operations + +1. Create users：CREATE USER +2. Delete users: DROP USER +3. Authorization: GRANT +4. Withdrawal: REVOKE +5. Create role：CREATE ROLE +6. Delete Roles: DROP ROLE +7. View current user privileges: SHOW GRANTS +8. View all user privilegesSHOW ALL GRANTS; +9. View the created roles: SHOW ROLES +10. View user attributes: SHOW PROPERTY + +For detailed help with the above commands, you can use help + command to get help after connecting Doris through the MySQL client. For example `HELP CREATE USER`. + +## Permission type + +Doris currently supports the following permissions + +1. Node_priv + + Nodes change permissions. Including FE, BE, BROKER node addition, deletion, offline operations. Currently, this permission can only be granted to Root users. + +2. Grant_priv + + Permissions change permissions. Allow the execution of operations including authorization, revocation, add/delete/change user/role, etc. + +3. Select_priv + + Read-only access to databases and tables. + +4. Load_priv + + Write permissions to databases and tables. Including Load, Insert, Delete and so on. + +5. Alter_priv + + Change permissions on databases and tables. It includes renaming libraries/tables, adding/deleting/changing columns, and adding/deleting partitions. + +6. Create_priv + + The right to create databases, tables, and views. + +7. Drop_priv + + Delete permissions for databases, tables, and views. + +## Permission hierarchy + +At the same time, according to the scope of application of permissions, we divide them into three levels: + +1. GLOBAL LEVEL: Global permissions. That is, permissions on `*.*` granted by GRANT statements. The granted permissions apply to any table in any database. +2. DATABASE LEVEL: Database-level permissions. That is, permissions on `db.*` granted by GRANT statements. The granted permissions apply to any table in the specified database. +3. TABLE LEVEL: Table-level permissions. That is, permissions on `db.tbl` granted by GRANT statements. The permissions granted apply to the specified tables in the specified database. + + +## ADMIN /GRANT + +ADMIN\_PRIV and GRANT\_PRIV have the authority of **"grant authority"** at the same time, which is more special. The operations related to these two privileges are described here one by one. + +1. CREATE USER + + * Users with ADMIN or GRANT privileges at any level can create new users. + +2. DROP USER + + * Only ADMIN privileges can delete users. + +3. CREATE/DROP ROLE + + * Only ADMIN privileges can create roles. + +4. GRANT /REVOKE + + * Users with ADMIN or GLOBAL GRANT privileges can grant or revoke the privileges of any user. + * Users with GRANT privileges at the DATABASE level can grant or revoke the privileges of any user on the specified database. + * Users with GRANT privileges at TABLE level can grant or revoke the privileges of any user on the specified tables in the specified database. + +5. SET PASSWORD + + * Users with ADMIN or GLOBAL GRANT privileges can set any user's password. + * Ordinary users can set their corresponding User Identity password. The corresponding User Identity can be viewed by `SELECT CURRENT_USER();`command. + * Users with GRANT privileges at non-GLOBAL level can not set the password of existing users, but can only specify the password when creating users. + + +## Some explanations + +1. When Doris initializes, the following users and roles are automatically created: + + 1. Operator role: This role has Node\_priv and Admin\_priv, i.e. all permissions for Doris. In a subsequent upgrade version, we may restrict the role's permissions to Node\_priv, which is to grant only node change permissions. To meet some cloud deployment requirements. + + 2. admin role: This role has Admin\_priv, which is all permissions except for node changes. + + 3. root@'%': root user, which allows login from any node, with the role of operator. + + 4. admin@'%': admin user, allowing login from any node, role admin. + +2. It is not supported to delete or change the permissions of default created roles or users. + +3. The user of the operator role has one and only one user. Users of admin roles can create multiple. + +4. Operational instructions for possible conflicts + + 1. Conflict between domain name and ip: + + Assume that the following users are created: + + CREATE USER cmy@['domain']; + + And authorize: + + GRANT SELECT_PRIV ON \*.\* TO cmy@['domain'] + + The domain is resolved into two ips: IP1 and IP2 + + Let's assume that we have a separate authorization for cmy@'ip1': + + GRANT ALTER_PRIV ON \*.\* TO cmy@'ip1'; + + The permissions of CMY @'ip1'will be changed to SELECT\_PRIV, ALTER\_PRIV. And when we change the permissions of cmy@['domain'] again, cmy@'ip1' will not follow. + + 2. duplicate IP conflicts: + + Assume that the following users are created: + + CREATE USER cmy@'%' IDENTIFIED BY "12345"; + + CREATE USER cmy@'192.%' IDENTIFIED BY "abcde"; + + In priority,'192.%'takes precedence over'%', so when user CMY tries to login Doris with password '12345' from 192.168.1.1, the machine will be rejected. + +5. Forget passwords + + If you forget your password and cannot log in to Doris, you can log in to Doris without a password using the following command on the machine where the Doris FE node is located: + + `mysql-client -h 127.0.0.1 -P query_port -uroot` + + After login, the password can be reset through the SET PASSWORD command. + +6. No user can reset the password of the root user except the root user himself. + +7. ADMIN\_PRIV permissions can only be granted or revoked at the GLOBAL level. + +8. Having GRANT\_PRIV at GLOBAL level is actually equivalent to having ADMIN\_PRIV, because GRANT\_PRIV at this level has the right to grant arbitrary permissions, please use it carefully. + +9. `current_user()` and `user()` + + Users can view `current_user` and `user` respectively by `SELECT current_user();` and `SELECT user();`. Where `current_user` indicates which identity the current user is passing through the authentication system, and `user` is the user's current actual `user_identity`. + +  For example, suppose the user `user1@'192.%'` is created, and then a user user1 from 192.168.10.1 is logged into the system. At this time, `current_user` is `user1@'192.%'`, and `user` is `user1@'192.168.10.1'`. + + All privileges are given to a `current_user`, and the real user has all the privileges of the corresponding `current_user`. + +## Best Practices + +Here are some usage scenarios of Doris privilege system. + +1. Scene 1 + + The users of Doris cluster are divided into Admin, RD and Client. Administrators have all the rights of the whole cluster, mainly responsible for cluster building, node management and so on. The development engineer is responsible for business modeling, including database building, data import and modification. Users access different databases and tables to get data. + + In this scenario, ADMIN or GRANT privileges can be granted to administrators. Give RD CREATE, DROP, ALTER, LOAD, SELECT permissions to any or specified database tables. Give Client SELECT permission to any or specified database table. At the same time, it can also simplify the authorization of multiple users by creating different roles. + +2. Scene 2 + + There are multiple services in a cluster, and each business may use one or more data. Each business needs to manage its own users. In this scenario. Administrator users can create a user with GRANT privileges at the DATABASE level for each database. The user can only authorize the specified database for the user. + +3. Blacklist + + Doris itself does not support blacklist, only whitelist, but we can simulate blacklist in some way. Suppose you first create a user named `user@'192.%'`, which allows users from `192.*` to login. At this time, if you want to prohibit users from `192.168.10.1` from logging in, you can create another user with `cmy@'192.168.10.1'` and set a new password. Since `192.168.10.1` has a higher priority than `192.%`, user can no longer login by using the old password from `192.168.10.1`. + + diff --git a/docs/en/administrator-guide/small-file-mgr.md b/docs/en/administrator-guide/small-file-mgr.md new file mode 100644 index 00000000000000..2a8226edda2837 --- /dev/null +++ b/docs/en/administrator-guide/small-file-mgr.md @@ -0,0 +1,104 @@ +--- +{ + "title": "File Manager", + "language": "en" +} +--- + + + +# File Manager + +Some functions in Doris require some user-defined files. For example, public keys, key files, certificate files and so on are used to access external data sources. The File Manager provides a function that allows users to upload these files in advance and save them in Doris system, which can then be referenced or accessed in other commands. + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. +* BDBJE: Oracle Berkeley DB Java Edition. Distributed embedded database for persistent metadata in FE. +* SmallFileMgr: File Manager. Responsible for creating and maintaining user files. + +## Basic concepts + +Files are files created and saved by users in Doris. + +A file is located by `database`, `catalog`, `file_name`. At the same time, each file also has a globally unique ID (file_id), which serves as the identification in the system. + +File creation and deletion can only be performed by users with `admin` privileges. A file belongs to a database. Users who have access to a database (queries, imports, modifications, etc.) can use the files created under the database. + +## Specific operation + +File management has three main commands: `CREATE FILE`, `SHOW FILE` and `DROP FILE`, creating, viewing and deleting files respectively. The specific syntax of these three commands can be viewed by connecting to Doris and executing `HELP cmd;`. + +1. CREATE FILE + + In the command to create a file, the user must provide the following information: + + * file_name: File name. User-defined, unique within a catalog. + * Catalog: Category of files. User-defined, unique within a database. + + > Doris also has some special classification names for specific commands. + + > 1. Kafka + + > When the data source is specified as Kafka in the routine Import command and the file needs to be referenced, Doris defaults to looking for the file from the catalog category named "kafka". + + * url: the download address of the file. Currently, only unauthenticated HTTP download addresses are supported. This download address is only used to download files from this address when executing the create file command. When the file is successfully created and saved in Doris, the address will no longer be used. + * md5: optional. The MD5 value of the file. If the user provides this value, the MD5 value will be checked after the file is downloaded. File creation fails if validation fails. + + When the file is created successfully, the file-related information will be persisted in Doris. Users can view successfully created files through the `SHOW FILE` command. + +2. SHOW FILE + + This command allows you to view files that have been created successfully. Specific operations see: `HELP SHOW FILE;` + +3. DROP FILE + + This command can delete a file that has been created. Specific operations see: `HELP DROP FILE;` + +## Implementation details + +### Create and delete files + +When the user executes the `CREATE FILE` command, FE downloads the file from a given URL. The contents of the file are stored in FE memory directly in the form of Base64 encoding. At the same time, the file content and meta-information related to the file will be persisted in BDBJE. All created files, their meta-information and file content reside in FE memory. If the FE goes down and restarts, meta information and file content will also be loaded into memory from the BDBJE. When a file is deleted, the relevant information is deleted directly from FE memory and persistent information is deleted from BDBJE. + +### Use of documents + +If the FE side needs to use the created file, SmallFileMgr will directly save the data in FE memory as a local file, store it in the specified directory, and return the local file path for use. + +If the BE side needs to use the created file, BE will download the file content to the specified directory on BE through FE's HTTP interface `api/get_small_file` for use. At the same time, BE also records the information of the files that have been downloaded in memory. When BE requests a file, it first checks whether the local file exists and verifies it. If the validation passes, the local file path is returned directly. If the validation fails, the local file is deleted and downloaded from FE again. When BE restarts, local files are preloaded into memory. + +## Use restrictions + +Because the file meta-information and content are stored in FE memory. So by default, only files with size less than 1MB can be uploaded. And the total number of files is limited to 100. The configuration items described in the next section can be modified. + +## Relevant configuration + +1. FE configuration + +* `Small_file_dir`: The path used to store uploaded files, defaulting to the `small_files/` directory of the FE runtime directory. +* `max_small_file_size_bytes`: A single file size limit in bytes. The default is 1MB. File creation larger than this configuration will be rejected. +* `max_small_file_number`: The total number of files supported by a Doris cluster. The default is 100. When the number of files created exceeds this value, subsequent creation will be rejected. + + > If you need to upload more files or increase the size limit of a single file, you can modify the `max_small_file_size_bytes` and `max_small_file_number` parameters by using the `ADMIN SET CONFIG` command. However, the increase in the number and size of files will lead to an increase in FE memory usage. + +2. BE configuration + +* `Small_file_dir`: The path used to store files downloaded from FE by default is in the `lib/small_files/` directory of the BE runtime directory. diff --git a/docs/en/administrator-guide/sql-mode.md b/docs/en/administrator-guide/sql-mode.md new file mode 100644 index 00000000000000..1564c194f533da --- /dev/null +++ b/docs/en/administrator-guide/sql-mode.md @@ -0,0 +1,76 @@ +--- +{ + "title": "SQL MODE", + "language": "en" +} +--- + + + +# SQL MODE + +The SQL MODE supported by Doris refers to the sql mode management mechanism of MySQL. Each client can set its own sql mode, and the database administrator with admin permission can set the global sql mode. + +## Sql mode introduction + +SQL MODE enables users to switch between different styles of SQL syntax and data verification strictness, making Doris more compatible with other databases. For example, in some databases, the '||' symbol is a string connector, but in Doris it is equivalent to 'or'. At this time, users only need to use SQL mode to switch to the style they want. Each client can set sql mode, which is valid in the current conversation. Only users with admin permission can set global SQL mode. + +## Theory + +SQL MODE is stored in session variables with a 64 bit long type. Each bit of this address represents the on / off (1 for on, 0 for off) state of a mode. As long as we know the specific bit of each mode, we can easily and quickly verify and operate SQL mode through bit operation. + +Every time you query sql mode, the long type will be parsed into a user-readable string. Similarly, the sql mode string sent by the user to the server will be parsed into a long type that can be stored in session variables. + +The set global sql mode will be persisted, so the operation on the global sql mode is always only once, even after the program is restarted, the last global sql mode can be recovered. + +## Operation + +1、set sql mode + +``` +set global sql_mode = "DEFAULT" +set session sql_mode = "DEFAULT" +``` +>At present, Doris's default sql mode is DEFAULT (but it will be changed in the future modification). +>Setting global sql mode requires admin permission and affects all clients that connect later. +>Setting session sql mode will only affect the current conversation client. The default setting way is session. + +2、select sql mode + +``` +select @@global.sql_mode +select @@session.sql_mode +``` +>In addition to this method, you can also view the current sql mode by returning all session variables as follows + +``` +show global variables +show session variables +``` + +## supported mode + +1. `PIPES_AS_CONCAT` + + Treat '||' as a string concatenation operator (same as CONCAT()) rather than as a synonym for OR. (e.g., `'a'||'b' = 'ab'`, `1||0 = '10'`) + +## combine mode + +(Work in progress) \ No newline at end of file diff --git a/docs/en/administrator-guide/time-zone.md b/docs/en/administrator-guide/time-zone.md new file mode 100644 index 00000000000000..eba38d55724010 --- /dev/null +++ b/docs/en/administrator-guide/time-zone.md @@ -0,0 +1,98 @@ +--- +{ + "title": "Time zone", + "language": "en" +} +--- + + + +# Time zone + +Doris supports multiple time zone settings + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. + +## Basic concepts + +There are multiple time zone related parameters in Doris + +* `system_time_zone`: + +When the server starts, it will be set automatically according to the time zone set by the machine, which can not be modified after setting. + +* `time_zone`: + +Server current time zone, set it at session level or global level. + +## Specific operations + +1. `SHOW VARIABLES LIKE '% time_zone%'` + + View the current time zone related configuration + +2. `SET time_zone = 'Asia/Shanghai'` + + This command can set the session level time zone, which will fail after disconnection. + +3. `SET global time_zone = 'Asia/Shanghai'` + + This command can set time zone parameters at the global level. The FE will persist the parameters and will not fail when the connection is disconnected. + +### Impact of time zone + +Time zone setting affects the display and storage of time zone sensitive values. + +It includes the values displayed by time functions such as `NOW()` or `CURTIME()`, as well as the time values in `SHOW LOAD` and `SHOW BACKENDS` statements. + +However, it does not affect the `LESS THAN VALUE` of the time-type partition column in the `CREATE TABLE` statement, nor does it affect the display of values stored as `DATE/DATETIME` type. + +Functions affected by time zone: + +* `FROM_UNIXTIME`: Given a UTC timestamp, return the date and time of the specified time zone, such as `FROM_UNIXTIME(0)`, return the CST time zone: `1970-01-08:00`. + +* `UNIX_TIMESTAMP`: Given a specified time zone date and time, return UTC timestamp, such as CST time zone `UNIX_TIMESTAMP('1970-01 08:00:00')`, return `0`. + +* `CURTIME`: Returns the datetime of specified time zone. + +* `NOW`: Returns the specified date and time of specified time zone. + +* `CONVERT_TZ`: Converts a date and time from one specified time zone to another. + +## Restrictions + +Time zone values can be given in several formats, case-insensitive: + +* A string representing UTC offset, such as '+10:00' or '-6:00'. + +* Standard time zone formats, such as "Asia/Shanghai", "America/Los_Angeles" + +* Abbreviated time zone formats such as MET and CTT are not supported. Because the abbreviated time zone is ambiguous in different scenarios, it is not recommended to use it. + +* In order to be compatible with Doris and support CST abbreviated time zone, CST will be internally transferred to "Asia/Shanghai", which is Chinese standard time zone. + +## Time zone format list + +[List of TZ database time zones] (https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + +[Edit on GitHub](https://github.com/apache/incubator-doris/blob/master/docs/documentation/en/administrator-guide/time-zone_EN.md) \ No newline at end of file diff --git a/docs/en/administrator-guide/variables.md b/docs/en/administrator-guide/variables.md new file mode 100644 index 00000000000000..2333f453cbb501 --- /dev/null +++ b/docs/en/administrator-guide/variables.md @@ -0,0 +1,328 @@ +--- +{ + "title": "Variable", + "language": "en" +} +--- + + + +# Variable + +This document focuses on currently supported variables. + +Variables in Doris refer to variable settings in MySQL. However, some of the variables are only used to be compatible with some MySQL client protocols, and do not produce their actual meaning in the MySQL database. + +## Variable setting and viewing + +### View + +All or specified variables can be viewed via `SHOW VARIABLES [LIKE 'xxx'];`. Such as: + +``` +SHOW VARIABLES; +SHOW VARIABLES LIKE '%time_zone%'; +``` + +### Settings + +Some variables can be set at global-level or session-only. For global-level, the set value will be used in subsequent new session connections. For session-only, the variable only works for the current session. + +For session-only, set by the `SET var_name=xxx;` statement. Such as: + +``` +SET exec_mem_limit = 137438953472; +SET forward_to_master = true; +SET time_zone = "Asia/Shanghai"; +``` + +For global-level, set by `SET GLOBALE var_name=xxx;`. Such as: + +``` +SET GLOBAL exec_mem_limit = 137438953472 +``` + +> Note 1: Only ADMIN users can set variable at global-level. +> Note 2: Global-level variables do not affect variable values in the current session, only variables in new sessions. + +Variables that support global-level setting include: + +* `time_zone` +* `wait_timeout` +* `sql_mode` +* `is_report_success` +* `query_timeout` +* `exec_mem_limit` +* `batch_size` +* `parallel_fragment_exec_instance_num` +* `parallel_exchange_instance_num` + +At the same time, variable settings also support constant expressions. Such as: + +``` +SET exec_mem_limit = 10 * 1024 * 1024 * 1024; +SET forward_to_master = concat('tr', 'u', 'e'); +``` + +## Supported variables + +* `SQL_AUTO_IS_NULL` + + Used for compatible JDBC connection pool C3P0. No practical effect. + +* `auto_increment_increment` + + Used for compatibility with MySQL clients. No practical effect. + +* `autocommit` + + Used for compatibility with MySQL clients. No practical effect. + +* `batch_size` + + Used to specify the number of rows of a single packet transmitted by each node during query execution. By default, the number of rows of a packet is 1024 rows. That is, after the source node generates 1024 rows of data, it is packaged and sent to the destination node. + + A larger number of rows will increase the throughput of the query in the case of scanning large data volumes, but may increase the query delay in small query scenario. At the same time, it also increases the memory overhead of the query. The recommended setting range is 1024 to 4096. + +* `character_set_client` + +  Used for compatibility with MySQL clients. No practical effect. + +* `character_set_connection` + + Used for compatibility with MySQL clients. No practical effect. + +* `character_set_results` + + Used for compatibility with MySQL clients. No practical effect. + +* `character_set_server` + + Used for compatibility with MySQL clients. No practical effect. + +* `codegen_level` + + Used to set the level of LLVM codegen. (Not currently in effect). + +* `collation_connection` + + Used for compatibility with MySQL clients. No practical effect. + +* `collation_database` + + Used for compatibility with MySQL clients. No practical effect. + +* `collation_server` + + Used for compatibility with MySQL clients. No practical effect. + +* `disable_colocate_join` + + Controls whether the [Colocation Join] (./colocation-join.md) function is enabled. The default is false, which means that the feature is enabled. True means that the feature is disabled. When this feature is disabled, the query plan will not attempt to perform a Colocation Join. + +* `disable_streaming_preaggregations` + + Controls whether streaming pre-aggregation is turned on. The default is false, which is enabled. Currently not configurable and enabled by default. + +* `enable_insert_strict` + + Used to set the `strict` mode when loadingdata via INSERT statement. The default is false, which means that the `strict` mode is not turned on. For an introduction to this mode, see [here] (./load-data/insert-into-manual.md). + +* `enable_spilling` + + Used to set whether to enable external sorting. The default is false, which turns off the feature. This feature is enabled when the user does not specify a LIMIT condition for the ORDER BY clause and also sets `enable_spilling` to true. When this feature is enabled, the temporary data is stored in the `doris-scratch/` directory of the BE data directory and the temporary data is cleared after the query is completed. + + This feature is mainly used for sorting operations with large amounts of data using limited memory. + + Note that this feature is experimental and does not guarantee stability. Please turn it on carefully. + +* `exec_mem_limit` + + Used to set the memory limit for a single query. The default is 2GB, in bytes. + + This parameter is used to limit the memory that can be used by an instance of a single query fragment in a query plan. A query plan may have multiple instances, and a BE node may execute one or more instances. Therefore, this parameter does not accurately limit the memory usage of a query across the cluster, nor does it accurately limit the memory usage of a query on a single BE node. The specific needs need to be judged according to the generated query plan. + + Usually, only some blocking nodes (such as sorting node, aggregation node, and join node) consume more memory, while in other nodes (such as scan node), data is streamed and does not occupy much memory. + + When a `Memory Exceed Limit` error occurs, you can try to increase the parameter exponentially, such as 4G, 8G, 16G, and so on. + +* `forward_to_master` + + The user sets whether to forward some commands to the Master FE node for execution. The default is false, which means no forwarding. There are multiple FE nodes in Doris, one of which is the Master node. Usually users can connect to any FE node for full-featured operation. However, some of detail informationcan only be obtained from the Master FE node. + + For example, the `SHOW BACKENDS;` command, if not forwarded to the Master FE node, can only see some basic information such as whether the node is alive, and forwarded to the Master FE to obtain more detailed information including the node startup time and the last heartbeat time. + + The commands currently affected by this parameter are as follows: + + 1. `SHOW FRONTEND;` + + Forward to Master to view the last heartbeat information. + + 2. `SHOW BACKENDS;` + + Forward to Master to view startup time, last heartbeat information, and disk capacity information. + + 3. `SHOW BROKERS;` + + Forward to Master to view the start time and last heartbeat information. + + 4. `SHOW TABLET;`/`ADMIN SHOW REPLICA DISTRIBUTION;`/`ADMIN SHOW REPLICA STATUS;` + + Forward to Master to view the tablet information stored in the Master FE metadata. Under normal circumstances, the tablet information in different FE metadata should be consistent. When a problem occurs, this method can be used to compare the difference between the current FE and Master FE metadata. + + 5. `SHOW PROC;` + + Forward to Master to view information about the relevant PROC stored in the Master FE metadata. Mainly used for metadata comparison. + +* `init_connect` +    + Used for compatibility with MySQL clients. No practical effect. + +* `interactive_timeout` + + Used for compatibility with MySQL clients. No practical effect. + +* `is_report_success` + + Used to set whether you need to view the profile of the query. The default is false, which means no profile is required. + + By default, the BE sends a profile to the FE for viewing errors only if an error occurs in the query. A successful query will not send a profile. Sending a profile will incur a certain amount of network overhead, which is detrimental to a high concurrent query scenario. + + When the user wants to analyze the profile of a query, the query can be sent after this variable is set to true. After the query is finished, you can view the profile on the web page of the currently connected FE: + + `fe_host:fe_http:port/query` + + It will display the most recent 100 queries which `is_report_success` is set to true. + +* `language` + + Used for compatibility with MySQL clients. No practical effect. + +* `license` + + Show Doris's license. No other effect. + +* `load_mem_limit` + + Used to specify the memory limit of the load operation. The default is 0, which means that this variable is not used, and `exec_mem_limit` is used as the memory limit for the load operation. + + This variable is usually used for INSERT operations. Because the INSERT operation has both query and load part. If the user does not set this variable, the respective memory limits of the query and load part are `exec_mem_limit`. Otherwise, the memory of query part of INSERT is limited to `exec_mem_limit`, and the load part is limited to` load_mem_limit`. + + For other load methods, such as BROKER LOAD, STREAM LOAD, the memory limit still uses `exec_mem_limit`. + +* `lower_case_table_names` + + Used for compatibility with MySQL clients. Cannot be set. Table names in current Doris are case sensitive by default. + +* `max_allowed_packet` + + Used for compatible JDBC connection pool C3P0. No practical effect. + +* `net_buffer_length` + + Used for compatibility with MySQL clients. No practical effect. + +* `net_read_timeout` + + Used for compatibility with MySQL clients. No practical effect. + +* `net_write_timeout` + + Used for compatibility with MySQL clients. No practical effect. + +* `parallel_exchange_instance_num` + + Used to set the number of exchange nodes used by an upper node to receive data from the lower node in the execution plan. The default is -1, which means that the number of exchange nodes is equal to the number of execution instances of the lower nodes (default behavior). When the setting is greater than 0 and less than the number of execution instances of the lower node, the number of exchange nodes is equal to the set value. + + In a distributed query execution plan, the upper node usually has one or more exchange nodes for receiving data from the execution instances of the lower nodes on different BEs. Usually the number of exchange nodes is equal to the number of execution instances of the lower nodes. + + In some aggregate query scenarios, if the amount of data to be scanned at the bottom is large, but the amount of data after aggregation is small, you can try to modify this variable to a smaller value, which can reduce the resource overhead of such queries. Such as the scenario of aggregation query on the DUPLICATE KEY data model. + +* `parallel_fragment_exec_instance_num` + + For the scan node, set its number of instances to execute on each BE node. The default is 1. + + A query plan typically produces a set of scan ranges, the range of data that needs to be scanned. These data are distributed across multiple BE nodes. A BE node will have one or more scan ranges. By default, a set of scan ranges for each BE node is processed by only one execution instance. When the machine resources are abundant, you can increase the variable and let more execution instances process a set of scan ranges at the same time, thus improving query efficiency. + + The number of scan instances determines the number of other execution nodes in the upper layer, such as aggregate nodes and join nodes. Therefore, it is equivalent to increasing the concurrency of the entire query plan execution. Modifying this parameter will help improve the efficiency of large queries, but larger values will consume more machine resources, such as CPU, memory, and disk IO. + +* `query_cache_size` + + Used for compatibility with MySQL clients. No practical effect. + +* `query_cache_type` + + Used for compatible JDBC connection pool C3P0. No practical effect. + +* `query_timeout` + + Used to set the query timeout. This variable applies to all query statements in the current connection, as well as INSERT statements. The default is 5 minutes, in seconds. + +* `resource_group` + + Not used. + +* `sql_mode` + + Used to specify SQL mode to accommodate certain SQL dialects. For the SQL mode, see [here] (./sql-mode.md). + +* `sql_safe_updates` + + Used for compatibility with MySQL clients. No practical effect. + +* `sql_select_limit` + + Used for compatibility with MySQL clients. No practical effect. + +* `system_time_zone` + + Displays the current system time zone. Cannot be changed. + +* `time_zone` + + Used to set the time zone of the current session. The time zone has an effect on the results of certain time functions. For the time zone, see [here] (./time-zone.md). + +* `tx_isolation` + + Used for compatibility with MySQL clients. No practical effect. + +* `version` + + Used for compatibility with MySQL clients. No practical effect. + +* `version_comment` + + Used to display the version of Doris. Cannot be changed. + +* `wait_timeout` + + The length of the connection used to set up an idle connection. When an idle connection does not interact with Doris for that length of time, Doris will actively disconnect the link. The default is 8 hours, in seconds. + +* `default_rowset_type` + + Used for setting the default storage format of Backends storage engine. Valid options: alpha/beta + +* `use_v2_rollup` + + Used to control the sql query to use segment v2 rollup index to get data. This variable is only used for validation when upgrading to segment v2 feature. Otherwise, not recommended to use. + +* `rewrite_count_distinct_to_bitmap_hll` + + Whether to rewrite count distinct queries of bitmap and HLL types as bitmap_union_count and hll_union_agg. diff --git a/docs/en/community/gitter.md b/docs/en/community/gitter.md new file mode 100644 index 00000000000000..d472461027197e --- /dev/null +++ b/docs/en/community/gitter.md @@ -0,0 +1,63 @@ +--- +{ + "title": "Gitter Manual", + "language": "en" +} +--- + + + +# Gitter Manual + +## Gitter introduction + +Gitter is a Markdown-enabled instant messaging software for developers. It can be seamlessly linked to github, PR on Github can be linked in chat, relevant historical records of discussions can be retained, historical records can be queried, and Chinese and English can be supported. + +Like many other open source projects, Doris can use Gitter as an instant messaging medium for technology exchange and community development. This article describes how to use Gitter to participate in Doris's open source development and community development. + +## Log in using links + +Entering [https://gitter.im/apache-doris/Lobby](https://gitter.im/apache-doris/Lobby) in the browser automatically jumps to the Doris community chat room interface on Gitter. + +Click on the `SIGN IN TO START TALKING` below to login. It can support two login modes, Github account or Twitter account. The author uses Github account to login, as follows: + +![](/images/login-gitter1.png) + +After clicking on the red circle, enter the Github account and password to log into the chat room and start technical or community discussions: + +![](/images/login-gitter2.png) + +You can use Gitter as well as Wechat, and get functions that are more comfortable for developers and technicians than Wechat, such as directly mentioning an activity for discussion, directly searching history chat records, etc. + +Don't forget to click on the Pentagon in the upper right corner to collect, which will make the chat room easier for you to find. + +For more gitter usage tips, you can refer to: + +[http://www.gitter.net.cn/book/gitter/roomsettings-1.html](http://www.gitter.net.cn/book/gitter/roomsettings-1.html) + +## Install Mobile Client + +You can download Gitter's mobile client and participate in technical discussions on your mobile phone at any time and anywhere. Download links: + +[https://gitter.im/home](https://gitter.im/home) + +## Search Gitter and join Doris Community Chat Room + +Partners already using Gitter log in directly to search for `apache-doris` and can join the chat room when they find it. Other functions are used in the same chapter, which is not discussed here. diff --git a/docs/en/community/how-to-contribute.md b/docs/en/community/how-to-contribute.md new file mode 100644 index 00000000000000..aa2f5fa2b80142 --- /dev/null +++ b/docs/en/community/how-to-contribute.md @@ -0,0 +1,83 @@ +--- +{ + "title": "Contribute to Doris", + "language": "en" +} +--- + + + +# Contribute to Doris + +Thank you very much for your interest in the Doris project. We welcome your suggestions, comments (including criticisms), comments and contributions to the Doris project. + +Your suggestions, comments and comments on Doris can be made directly through GitHub's [Issues] (https://github.com/apache/incubator-doris/issues/new/selection). + +There are many ways to participate in and contribute to Doris projects: code implementation, test writing, process tool improvement, document improvement, and so on. Any contribution will be welcomed and you will be added to the list of contributors. Further, with sufficient contributions, you will have the opportunity to become a Commiter of Aapche with Apache mailbox and be included in the list of [Apache Commiters] (http://people.apache.org/committer-index.html). + +Any questions, you can contact us to get timely answers, including Wechat, Gitter (GitHub instant messaging tool), e-mail and so on. + +## Initial contact + +For the first time in Doris community, you can: + +* Follow [Doris Github](https://github.com/apache/incubator-doris) +* Subscribe to our [mailing list] (./subscribe-mail-list.md); +* Join Doris Wechat Group (add micro-signal: morningman-cmy, note: join Doris Group) and ask questions at any time. +* Enter Doris's [Gitter] (./gitter.md) chat room; + +Learn the development trends of Doris project in time and give your opinions on the topics you are concerned about. + +## Doris's code and documentation + +As you can see from [GitHub] (https://github.com/apache/incubator-doris), Apache Doris (incubating) code base mainly consists of three parts: Frontend (FE), Backend (BE) and Broker (to support file reading on external storage systems such as HDFS). Documents are mainly the wiki on Doris website and GitHub, as well as the online help manual when running Doris. Details of these components can be found in the following table: + +| Component Name | Component Description | Related Language| +|--------|----------------------------|----------| +| [Frontend daemon (FE)] (https://github.com/apache/incubator-doris) | consists of a query coordinator and a metadata manager | Java| +| [Backend daemon (BE)] (https://github.com/apache/incubator-doris) | Responsible for storing data and executing query fragments | C++| +| [Broker] (https://github.com/apache/incubator-doris) | Read HDFS data to Doris | Java| +| [Website](https://github.com/apache/incubator-doris-website) | Doris Website | Markdown | ++ [Github Wiki] (https://github.com/apache/incubator-doris/wiki); Doris Github Wiki; Markdown_; +| Doris Runtime Help Document | Online Help Manual at Doris Runtime | Markdown| + +## Improving documentation + +Documentation is the most important way for you to understand Apache Doris, and it's where we need help most! + +Browse the document, you can deepen your understanding of Doris, can also help you understand Doris's function and technical details, if you find that the document has problems, please contact us in time; + +If you are interested in improving the quality of documents, whether it is revising the address of a page, correcting a link, and writing a better introductory document, we are very welcome! + +Most of our documents are written in markdown format, and you can modify and submit document changes directly through `docs/` in [GitHub] (https://github.com/apache/incubator-doris). If you submit code changes, you can refer to [Pull Request] (./pull-request.md). + +## If a Bug or problem is found + +If a Bug or problem is found, you can directly raise a new Issue through GitHub's [Issues] (https://github.com/apache/incubator-doris/issues/new/select), and we will have someone deal with it regularly. + +You can also fix it yourself by reading the analysis code (of course, it's better to talk to us before that, maybe someone has fixed the same problem) and submit a [Pull Request] (./pull-request.md). + +## Modify the code and submit PR (Pull Request) + +You can download the code, compile and install it, deploy and run it for a try (refer to the [compilation document] (./installing/compilation.md)) to see if it works as you expected. If you have problems, you can contact us directly, ask questions or fix them by reading and analyzing the source code. + +Whether it's fixing Bugs or adding Features, we're all very welcome. If you want to submit code to Doris, you need to create a new branch for your submitted code from the fork code library on GitHub to your project space, add the source project upstream, and submit PR. + +About how to submit a PR refer to [Pull Request] (./pull-request.md). diff --git a/docs/en/community/members.md b/docs/en/community/members.md new file mode 100644 index 00000000000000..7c71872dd13a9c --- /dev/null +++ b/docs/en/community/members.md @@ -0,0 +1,68 @@ +--- +{ + "title": "PMC Members & Committer", + "language": "en" +} +--- + + + +# PMC Members & Committer + +## Mentors (3) + +|Apache ID|Github Username |Public Name| +|--------|-----------|----------| +|wave |dave2wave |Dave Fisher | +|shaofengshi |shaofengshi| Shao Feng Shi | +|ningjiang |WillemJiang |Willem Ning Jiang| + +## PPMC (9) +(the listing below excludes mentors) + +|Apache ID|Github Username |Public Name| +|--------|-----------|----------| +|lingbin| lingbin |Bin Ling | +|lichaoyong |chaoyli |Chaoyong Li | +|zhaoc |imay |Chun Zhao | +|lide |lide-reed, doris-ci |De Li | +|chenhao |chenhao7253886 |Hao Chen | +|morningman |morningman |Mingyu Chen| +|maruyue |maruyue| Ruyue Ma | +|sijie |sijie |Sijie Guo | +|zshao |zshao |Zheng Shao| + +## Committers (11) + +|Apache ID|Github Username |Public Name| +|--------|-----------|----------| +|lingbin| lingbin |Bin Ling | +|lichaoyong |chaoyli |Chaoyong Li | +|zhaoc |imay |Chun Zhao | +|lide |lide-reed, doris-ci |De Li | +|chenhao |chenhao7253886 |Hao Chen | +|morningman |morningman |Mingyu Chen| +|maruyue |maruyue| Ruyue Ma | +|sijie |sijie |Sijie Guo | +|zshao |zshao |Zheng Shao| +|kangkaisen|kangkaisen|Kaisen Kang| +|lingmiao|EmmyMiao87|Ling Miao| +|gaodayue|gaodayue|Dayue Gao| +|liuhangyuan|HangyuanLiu|Hangyuan Liu| diff --git a/docs/en/community/pull-request.md b/docs/en/community/pull-request.md new file mode 100644 index 00000000000000..296e7318369e8d --- /dev/null +++ b/docs/en/community/pull-request.md @@ -0,0 +1,259 @@ +--- +{ + "title": "Code Submission Guide", + "language": "en" +} +--- + + + +# Code Submission Guide + +[Pull Request (PR)] (https://help.github.com/articles/about-pull-requests/) can be easily submitted on [Github] (https://github.com/apache/incubator-doris). The PR method of Doris project is described below. + +## Fork Repository + +Go to the [github page] (https://github.com/apache/incubator-doris) of apache/incubator-doris , and click the button `Fork` in the upper right corner for Fork. + +![Fork](/images/fork-repo.png) + +### 2. Configuring GIT and submitting modifications + +#### (1) Clone the code locally: + +``` +git clone https://github.com//incubator-doris.git +``` + +Note: Please replace your GitHub name with your yourgithubname\\\\\\\\\\\\\\. + +When clone is completed, origin defaults to the remote fork address on github. + +#### (2) Add apache/incubator-doris to the remote branch upstream of the local warehouse: + +``` +cd incubator-doris +git remote add upstream https://github.com/apache/incubator-doris.git +``` + +#### (3) Check remote warehouse settings: + +``` +git remote -v +origin https://github.com//incubator-doris.git (fetch) +origin https://github.com//incubator-doris.git (push) +upstream https://github.com/apache/incubator-doris.git (fetch) +upstream https://github.com/apache/incubator-doris.git (push) +``` + +#### (4) New branches to modify them: + +``` +git checkout -b +``` + +Note: \ name is customized for you. + +Code changes can be made after creation. + +#### (5) Submit code to remote branch: + +``` +git commit -a -m "" +git push origin +``` + +For more git usage, please visit: [git usage] (https://www.atlassian.com/git/tutorials/set-up-a-repository), not to mention here. + +### 3. Create PR + +#### (1) New PR +Switch to your GitHub page in the browser, switch to the submitted branch yourbranchname\\ and click the `New pull request` button to create it, as shown in the following figure: + +![new PR](/images/new-pr.png) + +#### (2) preparation branch +At this time, the `Create pull request` button will appear. If not, please check whether the branch is selected correctly or click on `compare across forks' to re-select the repo and branch. + +![create PR](/images//create-pr.png) + +#### (3) Fill Commit Message +Here, please fill in the summary and details of the comment, and then click `Create pull request` to create it. + +For how to write Commit Message, here are some Tips: + +* Please use the form of English verb + object. The verb does not use the past tense and the sentence uses imperative sentence. +* Subject and body should be written, and they should be separated by blank lines (fill in separately on GitHub PR interface). +* Message topic length should not exceed **50** characters; +* Message content should not exceed **72** characters per line, and the excess should be replaced manually. +* Message content is used to explain what has been done, why and how. +* The first letter of the message subject should be **capitalized**, and the end of the sentence **should not** have a full stop. +* The message content specifies the associated issue (if any), such as # 233; + +For more details, see . + +![create PR](/images/create-pr2.png) + +#### (4) Complete the creation +After successful creation, you can see that Doris project needs review, you can wait for us to review and join, you can also contact us directly. + +![create PR](/images/create-pr3.png) + +So far, your PR creation is complete. Read more about PR [collaborating-with-issues-and-pull-requests] (https://help.github.com/categories/collaborating-with-issues-and-pull-requests/). + +### 4. Conflict Resolution + +When submitting PR, code conflicts are usually caused by multiple people editing the same file. The main steps to resolve conflicts are as follows: + +#### (1) Switch to the main branch + +``` +git checkout master +``` + +#### (2) Synchronize remote main branch to local + +``` +git pull upstream master +``` + +#### (3) Switch back to the previous branch (assuming the branch is named fix) + +``` +git checkout fix +``` + +#### (4) rebase + +``` +git rebase -i master +``` + +At this point, a file that modifies the record will pop up and can be saved directly. Then, we will prompt which files have conflicts. At this time, we can open the conflict file to modify the conflict part. After all the conflicts of the conflict files are resolved, we will execute them. + +``` +git add . +git rebase --continue +``` + +Then you can go back and forth until the screen appears something like * rebase successful * and then you can update the branch that submitted PR: + +``` +git push -f origin fix +``` + +### 5. An example + +#### (1) fetch to the latest code for the local branch of upstream that has been configured + +``` +$ git branch +* master + +$ git fetch upstream +remote: Counting objects: 195, done. +remote: Compressing objects: 100% (68/68), done. +remote: Total 141 (delta 75), reused 108 (delta 48) +Receiving objects: 100% (141/141), 58.28 KiB, done. +Resolving deltas: 100% (75/75), completed with 43 local objects. +From https://github.com/apache/incubator-doris + 9c36200..0c4edc2 master -> upstream/master +``` + +#### (2) rebase + +``` +$ git rebase upstream/master +First, rewinding head to replay your work on top of it... +Fast-forwarded master to upstream/master. +``` + +#### (3) Check to see if other submissions are not synchronized to their own repo submissions + +``` +$ git status +# On branch master +# Your branch is ahead of 'origin/master' by 8 commits. +# +# Untracked files: +# (use "git add ..." to include in what will be committed) +# +# custom_env.sh +nothing added to commit but untracked files present (use "git add" to track) +``` + +#### (4) Merge code submitted by others into their own repo + +``` +$ git push origin master +Counting objects: 195, done. +Delta compression using up to 32 threads. +Compressing objects: 100% (41/41), done. +Writing objects: 100% (141/141), 56.66 KiB, done. +Total 141 (delta 76), reused 140 (delta 75) +remote: Resolving deltas: 100% (76/76), completed with 44 local objects. +To https://lide-reed:fc35ff925bd8fd6629be3f6412bacee99d4e5f97@github.com/lide-reed/incubator-doris.git + 9c36200..0c4edc2 master -> master +``` + +#### (5) New branch, ready for development + +``` +$ git checkout -b my_branch +Switched to a new branch 'my_branch' + +$ git branch + master +* my_branch +``` + +#### (6) Prepare to submit after code modification is completed + +``` +$ git add -u +``` + +#### (7) Fill in the message and submit it it to the new local branch + +``` +$ git commit -m "Fix a typo" +[my_branch 55e0ba2] Fix a typo +1 files changed, 2 insertions(+), 2 deletions(-) +``` + +#### (8) Push the branch into GitHub's own repo far away + +``` +$ git push origin my_branch +Counting objects: 11, done. +Delta compression using up to 32 threads. +Compressing objects: 100% (6/6), done. +Writing objects: 100% (6/6), 534 bytes, done. +Total 6 (delta 4), reused 0 (delta 0) +remote: Resolving deltas: 100% (4/4), completed with 4 local objects. +remote: +remote: Create a pull request for 'my_branch' on GitHub by visiting: +remote: https://github.com/lide-reed/incubator-doris/pull/new/my_branch +remote: +To https://lide-reed:fc35ff925bd8fd6629be3f6412bacee99d4e5f97@github.com/lide-reed/incubator-doris.git + * [new branch] my_branch -> my_branch +``` + +At this point, you can create PR according to the previous process. diff --git a/docs/en/community/release-process.md b/docs/en/community/release-process.md new file mode 100644 index 00000000000000..a0018ac707cee7 --- /dev/null +++ b/docs/en/community/release-process.md @@ -0,0 +1,673 @@ +--- +{ + "title": "Publish of Apache Doris", + "language": "en" +} +--- + + + +# Publish of Apache Doris + +Apache publishing must be at least an IPMC member, a commiter with Apache mailboxes, a role called release manager. + +The general process of publication is as follows: + +1. Preparing your setup +2. Preparing for release candidates + 1. launching DISCUSS in the community + 2. cutting a release branch + 3. clean up issues + 4. merging necessary patch to release branch +3. Running the voting process for a release + 1. singing a tag and upload it to [Apache dev svn repo](https://dist.apache.org/repos/dist/dev/incubator/doris) + 2. calling votes from [Doris community](dev@doris.apache.org) + 3. send result email to [Doris community](dev@doris.apache.org) + 4. calling votes from [Incubator community](general@incubator.apache.org) + 5. send result email to general@incubator.apache.org +4. Finalizing and posting a release + 1. Upload the signature package to [Apache release repo](https://dist.apache.org/repos/dist/release/incubator/doris) and generate relevant links + 2. Prepare release note and send Announce mail to general@incubator.apache.org + 3. Publish download links on Doris website and GitHub + + +## prepare setup + +If you are a new Release Manager, you can read up on the process from the followings: + +1. release signing https://www.apache.org/dev/release-signing.html +2. gpg for signing https://www.apache.org/dev/openpgp.html +3. svn https://www.apache.org/dev/version-control.html#https-svn + +### preparing gpg key + +Release manager needs Mr. A to sign his own public key before publishing and upload it to the public key +server. Then he can use this public key to sign the package ready for publication. +If your key already exists in [key] (https://dist.apache.org/repos/dist/dev/initiator/doris/keys), you can skip this step. + + +#### Installation and configuration of signature software GnuPG +##### GnuPG + +In 1991, programmer Phil Zimmermann developed the encryption software PGP to avoid government surveillance. This software is very useful, spread quickly, has become a necessary tool for many programmers. However, it is commercial software and cannot be used freely. So the Free Software Foundation decided to develop a replacement for PGP, called GnuPG. This is the origin of GPG. + +##### Installation Configuration + +CentOS installation command: + +``` +yum install gnupg +``` +After installation, the default configuration file gpg.conf will be placed in the home directory. + +``` +~/.gnupg /gpg.conf +``` + +If this directory or file does not exist, you can create an empty file directly. +Edit gpg.conf, modify or add KeyServer configuration: + +``` +keyserver hkp http://keys.gnupg.net +``` + +Apache signature recommends SHA512, which can be done by configuring gpg. +Edit gpg.conf and add the following three lines: + +``` +personal-digest-preferences SHA512 +cert -digest -something SHA512 +default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed +``` + +#### Generating new signatures + +##### Prepare to Sign + +Recommended settings for generating new signatures: + +We must log in to user account directly through SecureCRT and other terminals. We can't transfer it through Su - user or ssh. Otherwise, the password input box will not show up and make an error. + +Let's first look at the version of GPG and whether it supports SHA512. + +``` +$ gpg --version +gpg (GnuPG) 2.0.22 +libgcrypt 1.5.3 +Copyright (C) 2013 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. + +Home: ~/.gnupg +Supported algorithms: +Pubkey: RSA, ?, ?, ELG, DSA +Cipher: IDEA, 3DES, CAST5, BLOWFISH, AES, AES192, AES256, TWOFISH, + CAMELLIA128, CAMELLIA192, CAMELLIA256 +Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224 +Compression: Uncompressed, ZIP, ZLIB, BZIP2 +``` + +##### Generating new signatures + +``` +$ gpg --gen-key +gpg (GnuPG) 2.0.22; Copyright (C) 2013 Free Software Foundation, Inc. +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. + +Please select what kind of key you want: + (1) RSA and RSA (default) + (2) DSA and Elgamal + (3) DSA (sign only) + (4) RSA (sign only) +Your selection? 1 +RSA keys may be between 1024 and 4096 bits long. +What keysize do you want? (2048) 4096 +Requested keysize is 4096 bits +Please specify how long the key should be valid. + 0 = key does not expire + = key expires in n days + w = key expires in n weeks + m = key expires in n months + y = key expires in n years +Key is valid for? (0) +Key does not expire at all +Is this correct? (y/N) y + +GnuPG needs to construct a user ID to identify your key. + +Real name: xxx +Name must be at least 5 characters long +Real name: xxx-yyy +Email address: xxx@apache.org +Comment: xxx's key +You selected this USER-ID: + "xxx-yyy (xxx's key) " + +Change (N)ame, (C)omment, (E)mail or (O)kay/(Q)uit? o +``` + +Real name needs to be consistent with the ID shown in ID. apache. org. +Email address is apache's mailbox. + +##### View and Output + +The first line shows the name of the public key file (pubring. gpg), the second line shows the public key characteristics (4096 bits, Hash string and generation time), the third line shows the "user ID", and the fourth line shows the private key characteristics. + +``` +$ gpg --list-keys +/home/lide/.gnupg/pubring.gpg +----------------------------- +pub 4096R/33DBF2E0 2018-12-06 +uid xxx-yyy (xxx's key) +sub 4096R/0E8182E6 2018-12-06 +``` + +xxx-yy is the user ID. + +``` +gpg --armor --output public-key.txt --export [UserID] +``` + +``` +$ gpg --armor --output public-key.txt --export xxx-yyy +$ cat public-key.txt +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG V2.0.22 (GNU /Linux) + +mQINBFwJEQ0BEACwqLluHfjBqD/RWZ4uoYxNYHlIzZvbvxAlwS2mn53BirLIU/G3 +9opMWNplvmK+3+gNlRlFpiZ7EvHsF/YJOAP59HmI2Z... +``` + +#### Upload signature public key + +Public key servers are servers that store users'public keys exclusively on the network. The send-keys parameter uploads the public key to the server. + +``` +gpg --send-keys xxxx +``` + +Where XXX is the last step -- the string after pub in the list-keys result, as shown above: 33DBF2E0 + +You can also upload the contents of the above public-key.txt through the following website: + +``` +http://keys.gnupg.net +``` + +After successful upload, you can query the website and enter 0x33DBF2E0: + +http://keys.gnupg.net + +Queries on the site are delayed and may take an hour. + + +#### Generate fingerprint and upload it to Apache user information + +Because the public key server has no checking mechanism, anyone can upload the public key in your name, so there is no way to guarantee the reliability of the public key on the server. Usually, you can publish a public key fingerprint on the website and let others check whether the downloaded public key is true or not. + +Fingerprint parameter generates public key fingerprints: + +``` +gpg --fingerprint [UserID] +``` + +``` +$ gpg --fingerprint xxx-yyy +pub 4096R/33DBF2E0 2018-12-06 + Key fingerprint = 07AA E690 B01D 1A4B 469B 0BEF 5E29 CE39 33DB F2E0 +uid xxx-yyy (xxx's key) +sub 4096R/0E8182E6 2018-12-06 +``` + +Paste the fingerprint above (i.e. 07AA E690 B01D 1A4B 469B 0BEF 5E29 CE39 33DB F2E0) into your user information: + +https://id.apache.org +OpenPGP Public Key Primary Fingerprint: + +#### Generating keys + +``` +svn co //dist.apache.org/repos/dist/dev/incubator/doris/ +# edit doris/KEY file +gpg --list-sigs [用户 ID] >> doris/KEYS +gpg --armor --export [用户 ID] >> doris/KEYS +svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Update KEYS" +``` + +## Prepare for release + +### Launching DISCUSS in the Community + +If you think you've fixed a lot of bugs and developed more important features, any IPMC member can initiate DISCUSS discussions to release a new version. +An e-mail entitled [DISCUSS] x.y.z release can be launched to discuss within the community what bugs have been fixed and what features have been developed. +If DISCUSS mail is supported, we can proceed to the next step. + +### Preparatory Branch + +Before publishing, we need to build a new branch, For example: + +``` +$ git checkout -b branch-0.9 + +``` + +This branch needs to be fully tested to make functions available, bug convergence, and important bugs fixed. + +This process needs to wait for the community to see if a necessary patch needs to be merged in this version, and if so, it needs to be cherry picked to the release branch. + +### clean up issue + +Go through all the issues belonging to this version, close those that have been completed, and if they cannot be completed, postpone them to a later version. + +### Merge necessary patches + +During the release waiting process, there may be more important patch merging. If someone in the community says that there is an important bug to merge, then release manager needs to evaluate and merge the important patches into the release branch. + +## Running the voting process for a release + +### dozen Tags + +When the above branches are stable, tags can be made on them. +Remember to modify the `build_version` variable in `gensrc/script/gen_build_version.sh` when creating tags. For example, `build_version='0.10.0-release'.` + +For example: + +``` +$ git checkout branch-0.9 +$ git tag -a 0.9.0-rc01 -m "0.9.0 release candidate 01" +$ git push origin 0.9.0-rc01 +Counting objects: 1, done. +Writing objects: 100% (1/1), 165 bytes | 0 bytes/s, done. +Total 1 (delta 0), reused 0 (delta 0) +To git@github.com:apache/incubator-doris.git + * [new tag] 0.9.0-rc01 -> 0.9.0-rc01 + +$ git tag +``` + +### Packing Signature + +The following steps also need to log into user accounts directly through terminals such as SecureCRT, and can not be transferred through Su - user or ssh, otherwise the password input box will not show and error will be reported. + +``` +$ git checkout 0.9.0-rc01 + +$ git archive --format=tar 0.9.0-rc01 --prefix=apache-doris-0.9.0-incubating-src/ | gzip > apache-doris-0.9.0-incubating-src.tar.gz + +$ gpg -u xxx@apache.org --armor --output apache-doris-0.9.0-incubating-src.tar.gz.asc --detach-sign apache-doris-0.9.0-incubating-src.tar.gz + +$ gpg --verify apache-doris-0.9.0-incubating-src.tar.gz.asc apache-doris-0.9.0-incubating-src.tar.gz + +$ sha512sum apache-doris-0.9.0-incubating-src.tar.gz > apache-doris-0.9.0-incubating-src.tar.gz.sha512 + +$ sha512sum --check apache-doris-0.9.0-incubating-src.tar.gz.sha512 +``` + +### Upload signature packages and KEYS files to DEV SVN + +First, download the SVN library: + +``` +svn co https://dist.apache.org/repos/dist/dev/incubator/doris/ +``` + +Organize all previous files into the following SVN paths + +``` +./doris/ +|-- 0.11.0-rc1 +| |-- apache-doris-0.11.0-incubating-src.tar.gz +| |-- apache-doris-0.11.0-incubating-src.tar.gz.asc +| `-- apache-doris-0.11.0-incubating-src.tar.gz.sha512 +`-- KEYS +``` + +Upload these files + +``` +svn add 0.9.0-rc1 +svn commit -m "Release Apache Doris (incubating) 0.9.0 rc1" +``` + +### Send community voting emails + +[VOTE] Release Apache Doris 0.9.0-incubating-rc01 + + +``` +Hi all, + +Please review and vote on Apache Doris 0.9.0-incubating-rc01 release. + +The release candidate has been tagged in GitHub as 0.9.0-rc01, available +here: +https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 + +===== CHANGE LOG ===== + +New Features: +.... + +====================== + +Thanks to everyone who has contributed to this release. + +The artifacts (source, signature and checksum) corresponding to this release +candidate can be found here: +https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc1/ + +This has been signed with PGP key 33DBF2E0, corresponding to +lide@apache.org. +KEYS file is available here: +https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS +It is also listed here: +https://people.apache.org/keys/committer/lide.asc + +To verify and build, you can refer to following wiki: +https://github.com/apache/incubator-doris/wiki/How-to-verify-Apache-Release +https://wiki.apache.org/incubator/IncubatorReleaseChecklist + +The vote will be open for at least 72 hours. +[ ] +1 Approve the release +[ ] +0 No opinion +[ ] -1 Do not release this package because ... + +Best Regards, +xxx + +---- +DISCLAIMER-WIP: +Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), +sponsored by the Apache Incubator. Incubation is required of all newly accepted projects +until a further review indicates that the infrastructure, communications, and decision +making process have stabilized in a manner consistent with other successful ASF projects. +While incubation status is not necessarily a reflection of the completeness or stability +of the code, it does indicate that the project has yet to be fully endorsed by the ASF. + +Some of the incubating project’s releases may not be fully compliant with ASF policy. For +example, releases may have incomplete or un-reviewed licensing conditions. What follows is +a list of known issues the project is currently aware of (note that this list, by definition, +is likely to be incomplete): + + * Releases may have incomplete licensing conditions + +If you are planning to incorporate this work into your product/project, please be aware that +you will need to conduct a thorough licensing review to determine the overall implications of +including this work. For the current status of this project through the Apache Incubator +visit: https://incubator.apache.org/projects/doris.html +``` + +### Email Result after the vote is passed + +[Result][VOTE] Release Apache Doris 0.9.0-incubating-rc01 + +``` +Thanks to everyone, and this vote is now closed. + +It has passed with 4 +1 (binding) votes and no 0 or -1 votes. + +Binding: +Zhao Chun ++1 xxx ++ 1 Li Chaoyong ++1 Mingyu Chen + +Best Regards, +xxx + +``` + +### Send an e-mail to general@incubator.apache.org for a vote. + +[VOTE] Release Apache Doris 0.9.0-incubating-rc01 + +``` +Hi all, + +Please review and vote on Apache Doris 0.9.0-incubating-rc01 release. + +Apache Doris is an MPP-based interactive SQL data warehousing for reporting and analysis. + +The Apache Doris community has voted on and approved this release: +https://lists.apache.org/thread.html/d70f7c8a8ae448bf6680a15914646005c6483564464cfa15f4ddc2fc@%3Cdev.doris.apache.org%3E + +The vote result email thread: +https://lists.apache.org/thread.html/64d229f0ba15d66adc83306bc8d7b7ccd5910ecb7e842718ce6a61da@%3Cdev.doris.apache.org%3E + +The release candidate has been tagged in GitHub as 0.9.0-rc01, available here: +https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 + +There is no CHANGE LOG file because this is the first release of Apache Doris. +Thanks to everyone who has contributed to this release, and there is a simple release notes can be found here: +https://github.com/apache/incubator-doris/issues/406 + +The artifacts (source, signature and checksum) corresponding to this release candidate can be found here: +https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc01/ + +This has been signed with PGP key 33DBF2E0, corresponding to lide@apache.org. +KEYS file is available here: +https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS +It is also listed here: +https://people.apache.org/keys/committer/lide.asc + +The vote will be open for at least 72 hours. +[ ] +1 Approve the release +[ ] +0 No opinion +[ ] -1 Do not release this package because ... + +To verify and build, you can refer to following instruction: + +Firstly, you must be install and start docker service, and then you could build Doris as following steps: + +Step1: Pull the docker image with Doris building environment +$ docker pull apachedoris/doris-dev:build-env +You can check it by listing images, its size is about 3.28GB. + +Step2: Run the Docker image +You can run image directly: +$ docker run -it apachedoris/doris-dev:build-env + +Step3: Download Doris source +Now you should in docker environment, and you can download Doris source package. +(If you have downloaded source and it is not in image, you can map its path to image in Step2.) +$ wget https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc01/apache-doris-0.9.0.rc01-incubating-src.tar.gz + +Step4: Build Doris +Now you can decompress and enter Doris source path and build Doris. +$ tar zxvf apache-doris-0.9.0.rc01-incubating-src.tar.gz +$ cd apache-doris-0.9.0.rc01-incubating-src +$ sh build.sh + +Best Regards, +xxx + +---- +DISCLAIMER-WIP: +Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), +sponsored by the Apache Incubator. Incubation is required of all newly accepted projects +until a further review indicates that the infrastructure, communications, and decision +making process have stabilized in a manner consistent with other successful ASF projects. +While incubation status is not necessarily a reflection of the completeness or stability +of the code, it does indicate that the project has yet to be fully endorsed by the ASF. + +Some of the incubating project’s releases may not be fully compliant with ASF policy. For +example, releases may have incomplete or un-reviewed licensing conditions. What follows is +a list of known issues the project is currently aware of (note that this list, by definition, +is likely to be incomplete): + + * Releases may have incomplete licensing conditions + +If you are planning to incorporate this work into your product/project, please be aware that +you will need to conduct a thorough licensing review to determine the overall implications of +including this work. For the current status of this project through the Apache Incubator +visit: https://incubator.apache.org/projects/doris.html +``` + +The threaded connection for mail can be found here: + +`https://lists.apache.org/list.html?dev@doris.apache.org` + + +### Email Result to general@incubator.apache.org + +[RESULT][VOTE] Release Apache Doris 0.9.0-incubating-rc01 + +``` +Hi, + +Thanks to everyone, and the vote for releasing Apache Doris 0.9.0-incubating-rc01 is now closed. + +It has passed with 4 +1 (binding) votes and no 0 or -1 votes. + +Binding: ++1 Willem Jiang ++1 Justin Mclean ++1 ShaoFeng Shi ++1 Makoto Yui + +The vote thread: +https://lists.apache.org/thread.html/da05fdd8d84e35de527f27200b5690d7811a1e97d419d1ea66562130@%3Cgeneral.incubator.apache.org%3E + +Best Regards, +xxx +``` + +## Finalizing release + +### Upload package to release + +When the formal voting is successful, email [Result] first, and then prepare the release package. +Copy the source package, signature file and hash file from the corresponding RC folder published under dev to another directory 0.9.0-incubating. Note that the file name does not need rcxx (rename, but do not recalculate signatures, hash can recalculate, the results will not change) + +KEYS files also need to be copied if they are first released. Then add to SVN release. + +``` + +https://dist.apache.org/repos/dist/release/incubator/doris/0.9.0-incubating/ + +Eventually you can see it on apache's website: +http://www.apache.org/dist/incubator/doris/0.9.0-incubating/ + +``` + + +### Send Announce e-mail to general@incubator.apache.org + +Title: + +``` +[ANNOUNCE] Apache Doris (incubating) 0.9.0 Release +``` + +Send mail group: + +``` +general@incubator.apache.org +dev@doris.apache.org +``` + +Mail text: + +``` +Hi All, + +We are pleased to announce the release of Apache Doris 0.9.0-incubating. + +Apache Doris (incubating) is an MPP-based interactive SQL data warehousing for reporting and analysis. + +The release is available at: +http://doris.apache.org/downloads.html + +Thanks to everyone who has contributed to this release, and the release note can be found here: +https://github.com/apache/incubator-doris/releases + +Best Regards, + +On behalf of the Doris team, +xxx + +---- +DISCLAIMER-WIP: +Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), +sponsored by the Apache Incubator. Incubation is required of all newly accepted projects +until a further review indicates that the infrastructure, communications, and decision +making process have stabilized in a manner consistent with other successful ASF projects. +While incubation status is not necessarily a reflection of the completeness or stability +of the code, it does indicate that the project has yet to be fully endorsed by the ASF. + +Some of the incubating project’s releases may not be fully compliant with ASF policy. For +example, releases may have incomplete or un-reviewed licensing conditions. What follows is +a list of known issues the project is currently aware of (note that this list, by definition, +is likely to be incomplete): + + * Releases may have incomplete licensing conditions + +If you are planning to incorporate this work into your product/project, please be aware that +you will need to conduct a thorough licensing review to determine the overall implications of +including this work. For the current status of this project through the Apache Incubator +visit: https://incubator.apache.org/projects/doris.html + +``` + +### Publish links on Doris website and GitHub + +#### Create Download Links + +Download link: +http://www.apache.org/dyn/closer.cgi?filename=incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz&action=download + +wget --trust-server-names "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz" + +Original location: +https://www.apache.org/dist/incubator/doris/0.9.0-incubating/ + +http://www.apache.org/dyn/closer.cgi/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz + +Source package: +http://www.apache.org/dyn/closer.cgi/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz + +ASC: +http://archive.apache.org/dist/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.asc + +sha512: +http://archive.apache.org/dist/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.sha512 + +KEYS: +http://archive.apache.org /dist /incubator /doris /KEYS + +refer to: + +#### Prepare release note + +The following two areas need to be modified: + +1. Github's release page + +``` +https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 +``` + +2. Doris Official Website Download Page + +``` +http://doris.apache.org /downloads.html +``` diff --git a/docs/en/community/subscribe-mail-list.md b/docs/en/community/subscribe-mail-list.md new file mode 100644 index 00000000000000..bdd75a68d6f780 --- /dev/null +++ b/docs/en/community/subscribe-mail-list.md @@ -0,0 +1,69 @@ +--- +{ + "title": "Subscribe to mailing lists", + "language": "en" +} +--- + + + +# Subscribe to mailing lists + +Mail List is the most recognized form of communication in Apache community. Generally speaking, open source community questions and answers, technical discussions, transaction decisions are carried through mailing lists. The asynchronous and broadcasting features of mailing lists are also very suitable for communication in open source communities. So how do you subscribe to Apache Doris (incubating) mailing lists? It mainly includes the following five steps. + +## 1. Send Subscription Mail + +Open your own email, create a new email, and send an email to `dev-subscribe@doris.apache.org` (subject and content are arbitrary) + +![step1](/images/subscribe-mail-list-step1.png) + +## 2. Receive confirmation emails from dev-help@doris.apache.org + +After the first step, you will receive a confirmation email from `dev-help@doris.apache.org`, which is shown below. (**If you fail to receive it for a long time, please confirm that the mail has been intercepted, or has been automatically grouped into "Subscribed Mail", "Spam Mail", "Promotional Mail" folders**) + +![step2](/images/subscribe-mail-list-step2.png) + +## 3. Reply to confirmation mail + +For the mail received in the previous step, + +**a. Reply to this email directly** + +***or*** + +**B. Create a new `recipient` e-mail for the `reply address` in the previous step** + +Every subject is acceptable. + +![step3](/images/subscribe-mail-list-step3.png) + + +## 4. Receiving Welcome Emails + +After completing the third step, you will receive a welcome email entitled **WELCOME to dev@doris.apache.org**. So far, the work of subscribing to mailing lists has been completed, and community dynamics will be notified by mail. + +![step4](/images/subscribe-mail-list-step4.png) + + +## 5. Initiate e-mail discussion (optional) + +After successfully subscribing to the mailing list, if you want to initiate a discussion, send an email directly to `dev@doris.apache.org`. Anyone who subscribes to the mailing list receives the mail. +​ +​ diff --git a/docs/en/community/verify-apache-release.md b/docs/en/community/verify-apache-release.md new file mode 100644 index 00000000000000..2556638e203ad3 --- /dev/null +++ b/docs/en/community/verify-apache-release.md @@ -0,0 +1,82 @@ +--- +{ + "title": "Verify Apache Release", + "language": "en" +} +--- + + + +# Verify Apache Release + +To verify the release, following checklist can used to reference: + +1. [ ] Download links are valid. +2. [ ] Checksums and PGP signatures are valid. +3. [ ] DISCLAIMER-WIP is included. +4. [ ] Source code artifacts have correct names matching the current release. +5. [ ] LICENSE and NOTICE files are correct for the repository. +6. [ ] All files have license headers if necessary. +7. [ ] No compiled archives bundled in source archive. +8. [ ] Building is OK. + +## 1. Download source package, signature file, hash file and KEYS + +Download all artifacts, take a.b.c-incubating as an example: + +``` +wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz + +wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz.sha512 + +wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz.asc + +wget https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS +``` + +## 2. Verify signature and hash + +GnuPG is recommended, which can install by yum install gnupg or apt-get install gnupg. + +``` +gpg --import KEYS +gpg --verify apache-doris-a.b.c-incubating-src.tar.gz.asc apache-doris-a.b.c-incubating-src.tar.gz +sha512sum --check apache-doris-a.b.c-incubating-src.tar.gz.sha512 +``` + +## 3. Verify license header + +Apache RAT is recommended to verify license headder, which can dowload as following command. + +``` +wget http://mirrors.tuna.tsinghua.edu.cn/apache//creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz +tar zxvf apache -rat -0.12 -bin.tar.gz +``` + +Given your source dir is apache-doris-a.b.c-incubating-src, you can check with following command. +It will output a file list which don't include ASF license header, and these files used other licenses. + +``` +/usr/java/jdk/bin/java -jar apache-rat-0.12/apache-rat-0.12.jar -a -d apache-doris-a.b.c-incubating-src -E apache-doris-a.b.c-incubating-src/.rat-excudes +``` + +## 4. Verify building + +To compile the Doris, please read [Compilation](../installing/compilation_EN.html) diff --git a/docs/en/developer-guide/debug-tool.md b/docs/en/developer-guide/debug-tool.md new file mode 100644 index 00000000000000..8389fd61509055 --- /dev/null +++ b/docs/en/developer-guide/debug-tool.md @@ -0,0 +1,273 @@ +--- +{ + "title": "Debug Tool", + "language": "en" +} +--- + + + +# Debug Tool + +In the process of using and developing Doris, we often encounter scenarios that need to debug Doris. Here are some common debugging tools. + +## Preparing + +[pprof] (https://github.com/google/pprof): from gperftools, it is used to transform the content generated by gperftools into a format that is easy for people to read, such as PDF, SVG, text, etc. + +[graphviz] (http://www.graphviz.org/): in the absence of this library, pprof can only be converted to text format, but this way is not easy to view. After the library is installed, pprof can be converted to SVG, PDF and other formats, and the call relationship is clearer. + +[perf] (https://perf.wiki.kernel.org/index.php/main_page): Linux kernel comes with performance analysis tool. [here] (http://www.brendangregg.com/perf.html) there are some examples of perf usage. + +[flamegraph] (https://github.com/brendangregg/flamegraph): a visualization tool used to show the output of perf in the form of flame graph. + +## Memory + +Debugging memory is generally divided into two aspects. One is whether the total amount of memory use is reasonable. On the one hand, the excessive amount of memory use may be due to memory leak in the system, on the other hand, it may be due to improper use of program memory. The second is whether there is a problem of memory overrun and illegal access, such as program access to memory with an illegal address, use of uninitialized memory, etc. For the debugging of memory, we usually use the following ways to track the problems. + +### Log + +When we find that the memory usage is too large, we can first check the be.out log to see if there is a large memory application. Because of the TCMalloc currently used by Doris to manage memory, when a large memory application is encountered, the stack of the application will be printed to the be.out file. The general form is as follows: + +``` +tcmalloc: large alloc 1396277248 bytes == 0x3f3488000 @ 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed +``` + +This indicates that Doris be is trying to apply memory of '1396277248 bytes' on this stack. We can use the 'addr2line' command to restore the stack to a letter that we can understand. The specific example is shown below. + +``` +$ addr2line -e lib/palo_be 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed + +/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1335 +/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1357 +/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.cpp:267 +/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.hpp:86 +/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:239 +/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:213 +thread.cpp:? +``` + +### HEAP PROFILE + +Sometimes the application of memory is not caused by the application of large memory, but by the continuous accumulation of small memory. Then there is no way to locate the specific application information by viewing the log, so you need to get the information through other ways. + +At this time, we can take advantage of TCMalloc's [heapprofile](https://gperftools.github.io/gperftools/heapprofile.html). If the heapprofile function is set, we can get the overall memory application usage of the process. The usage is to set the 'heapprofile' environment variable before starting Doris be. For example: + +``` +export HEAPPROFILE=/tmp/doris_be.hprof +./bin/start_be.sh --daemon +``` + +In this way, when the dump condition of the heapprofile is met, the overall memory usage will be written to the file in the specified path. Later, we can use the 'pprof' tool to analyze the output content. + +``` +$ pprof --text lib/palo_be /tmp/doris_be.hprof.0012.heap | head -30 + +Using local file lib/palo_be. +Using local file /tmp/doris_be.hprof.0012.heap. +Total: 668.6 MB + 610.6 91.3% 91.3% 610.6 91.3% doris::SystemAllocator::allocate_via_malloc (inline) + 18.1 2.7% 94.0% 18.1 2.7% _objalloc_alloc + 5.6 0.8% 94.9% 63.4 9.5% doris::RowBatch::RowBatch + 5.1 0.8% 95.6% 7.1 1.1% butil::ResourcePool::add_block (inline) + 3.7 0.5% 96.2% 3.7 0.5% butil::iobuf::create_block (inline) + 3.4 0.5% 96.7% 3.4 0.5% butil::FlatMap::init + 3.2 0.5% 97.2% 5.2 0.8% butil::ObjectPool::add_block (inline) + 2.6 0.4% 97.6% 2.6 0.4% __gnu_cxx::new_allocator::allocate (inline) + 2.0 0.3% 97.9% 2.0 0.3% butil::ObjectPool::add_block_group (inline) + 2.0 0.3% 98.2% 2.0 0.3% butil::ResourcePool::add_block_group (inline) + 1.7 0.3% 98.4% 1.7 0.3% doris::SegmentReader::_load_index +``` + +Contents of each column of the above documents: + +* Column 1: the memory size directly applied by the function, in MB +* Column 4: the total memory size of the function and all the functions it calls. +* The second column and the fifth column are the proportion values of the first column and the fourth column respectively. +* The third column is the cumulative value of the second column. + +Of course, it can also generate call relation pictures, which is more convenient for analysis. For example, the following command can generate a call graph in SVG format. + +``` +pprof --svg lib/palo_be /tmp/doris_be.hprof.0012.heap > heap.svg +``` + +**NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.** + +### pprof remote server + +Although heapprofile can get all the memory usage information, it has some limitations. 1. Restart be. 2. You need to enable this command all the time, which will affect the performance of the whole process. + +For Doris be, you can also use the way of opening and closing the heap profile dynamically to analyze the memory application of the process. Doris supports the [remote server debugging of gperftools](https://gperftools.github.io/gperftools/pprof_remote_servers.html). Then you can use 'pprof' to directly perform dynamic head profile on the remote running Doris be. For example, we can check the memory usage increment of Doris through the following command + +``` +$ pprof --text --seconds=60 http://be_host:be_webport/pprof/heap + +Total: 1296.4 MB + 484.9 37.4% 37.4% 484.9 37.4% doris::StorageByteBuffer::create + 272.2 21.0% 58.4% 273.3 21.1% doris::RowBlock::init + 157.5 12.1% 70.5% 157.5 12.1% doris::RowBatch::RowBatch + 90.7 7.0% 77.5% 90.7 7.0% doris::SystemAllocator::allocate_via_malloc + 66.6 5.1% 82.7% 66.6 5.1% doris::IntegerColumnReader::init + 47.9 3.7% 86.4% 47.9 3.7% __gnu_cxx::new_allocator::allocate + 20.8 1.6% 88.0% 35.4 2.7% doris::SegmentReader::_load_index + 12.7 1.0% 89.0% 12.7 1.0% doris::DecimalColumnReader::init + 12.7 1.0% 89.9% 12.7 1.0% doris::LargeIntColumnReader::init + 12.7 1.0% 90.9% 12.7 1.0% doris::StringColumnDirectReader::init + 12.3 0.9% 91.9% 12.3 0.9% std::__cxx11::basic_string::_M_mutate + 10.4 0.8% 92.7% 10.4 0.8% doris::VectorizedRowBatch::VectorizedRowBatch + 10.0 0.8% 93.4% 10.0 0.8% doris::PlainTextLineReader::PlainTextLineReader +``` + +The output of this command is the same as the output and view mode of heap profile, which will not be described in detail here. Statistics will be enabled only during execution of this command, which has a limited impact on process performance compared with heap profile. + +### LSAN + +[LSAN](https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer) is an address checking tool, GCC has been integrated. When we compile the code, we can enable this function by turning on the corresponding compilation options. When the program has a determinable memory leak, it prints the leak stack. Doris be has integrated this tool, only need to compile with the following command to generate be binary with memory leak detection version. + +``` +BUILD_TYPE=LSAN ./build.sh +``` + +When the system detects a memory leak, it will output the corresponding information in be. Out. For the following demonstration, we intentionally insert a memory leak code into the code. We insert the following code into the `open` function of `StorageEngine`. + +``` + char* leak_buf = new char[1024]; + strcpy(leak_buf, "hello world"); + LOG(INFO) << leak_buf; +``` + +We get the following output in be.out + +``` +================================================================= +==24732==ERROR: LeakSanitizer: detected memory leaks + +Direct leak of 1024 byte(s) in 1 object(s) allocated from: + #0 0xd10586 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/lsan/lsan_interceptors.cc:164 + #1 0xe333a2 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104 + #2 0xd3cc96 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 + #3 0x7f573b5eebd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) + +SUMMARY: LeakSanitizer: 1024 byte(s) leaked in 1 allocation(s). +``` + +From the above output, we can see that 1024 bytes have been leaked, and the stack information of memory application has been printed out. + +**NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.** + +**NOTE: if the LSAN switch is turned on, the TCMalloc will be automatically turned off** + +### ASAN + +Except for the unreasonable use and leakage of memory. Sometimes there will be memory access illegal address and other errors. At this time, we can use [ASAN](https://github.com/google/sanitizers/wiki/addresssanitizer) to help us find the cause of the problem. Like LSAN, ASAN is integrated into GCC. Doris can open this function by compiling as follows + +``` +BUILD_TYPE=ASAN ./build.sh +``` + +Execute the binary generated by compilation. When the detection tool finds any abnormal access, it will immediately exit and output the stack illegally accessed in be.out. The output of ASAN is the same as that of LSAN. Here we also actively inject an address access error to show the specific content output. We still inject an illegal memory access into the 'open' function of 'storageengine'. The specific error code is as follows + +``` + char* invalid_buf = new char[1024]; + for (int i = 0; i < 1025; ++i) { + invalid_buf[i] = i; + } + LOG(INFO) << invalid_buf; +``` + +We get the following output in be.out + +``` +================================================================= +==23284==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x61900008bf80 at pc 0x00000129f56a bp 0x7fff546eed90 sp 0x7fff546eed88 +WRITE of size 1 at 0x61900008bf80 thread T0 + #0 0x129f569 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 + #1 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 + #2 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) + #3 0xd30794 (/home/ssd0/zc/palo/doris/core/output3/be/lib/palo_be+0xd30794) + +0x61900008bf80 is located 0 bytes to the right of 1024-byte region [0x61900008bb80,0x61900008bf80) +allocated by thread T0 here: + #0 0xdeb040 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/asan/asan_new_delete.cc:82 + #1 0x129f50d in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104 + #2 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 + #3 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) + +SUMMARY: AddressSanitizer: heap-buffer-overflow /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) +``` + +From this message, we can see that at the address of `0x61900008bf80`, we tried to write a byte, but this address is illegal. We can also see the application stack of the address `[0x61900008bb80, 0x61900008bf80]`. + +**NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.** + +**NOTE: if the ASAN switch is turned on, the TCMalloc will be automatically turned off** + +In addition, if stack information is output in be.out, but there is no function symbol, then we need to handle it manually to get readable stack information. The specific processing method needs a script to parse the output of ASAN. At this time, we need to use [asan_symbolize](https://llvm.org/svn/llvm-project/compiler-rt/trunk/lib/asan/scripts/asan_symbolize.py) to help with parsing. The specific usage is as follows: + +``` +cat be.out | python asan_symbolize.py | c++filt +``` + +With the above command, we can get readable stack information. + +## CPU + +When the CPU idle of the system is very low, it means that the CPU of the system has become the main bottleneck. At this time, it is necessary to analyze the current CPU usage. For the be of Doris, there are two ways to analyze the CPU bottleneck of Doris. + +### pprof + +Because Doris has integrated and compatible with GPERF rest interface, users can analyze remote Doris be through the 'pprof' tool. The specific usage is as follows: + +``` +pprof --svg --seconds=60 http://be_host:be_webport/pprof/profile > be.svg +``` + +In this way, a CPU consumption graph of be execution can be generated. + +![CPU Pprof](/images/cpu-pprof-demo.png) + +### perf + flamegragh + +This is a quite common CPU analysis method. Compared with `pprof`, this method must be able to log in to the physical machine of the analysis object. However, compared with pprof, which can only collect points on time, perf can collect stack information through different events. The specific usage is as follows: + +``` +perf record -g -p be_pid -- sleep 60 +``` + +This command counts the CPU operation of be for 60 seconds and generates perf.data. For the analysis of perf.data, the command of perf can be used for analysis. + +``` +perf report +``` + +The analysis results in the following pictures + +![Perf Report](/images/perf-report-demo.png) + +To analyze the generated content. Of course, you can also use flash graph to complete the visual display. + +``` +perf script | ./FlameGraph/stackcollapse-perf.pl | ./FlameGraph/flamegraph.pl > be.svg +``` + +This will also generate a graph of CPU consumption at that time. + +![CPU Flame](/images/cpu-flame-demo.svg) diff --git a/docs/en/developer-guide/format-code.md b/docs/en/developer-guide/format-code.md new file mode 100644 index 00000000000000..cac47fe7c0d2ad --- /dev/null +++ b/docs/en/developer-guide/format-code.md @@ -0,0 +1,85 @@ +--- +{ + "title": "Format Code", + "language": "en" +} +--- + + + +# Format Code +To automatically format the code, clang-format is a good choice. + +## Code Style +Doris Code Style is based on Google's, makes a few changes. The customized .clang-format +file is in the root dir of Doris. +Now, .clang-format file only works on clang-format-8.0.1+. + +## Preparing +You should install clang-format, or you can use clang-format plugins which support by IDEs or Editors. + +### Install clang-format +Ubuntu: `apt-get install clang-format` + +The current release is 10.0, you can specify old version, e.g. + + `apt-get install clang-format-9` + +Centos 7: + +The version of clang-format installed by yum is too old. Compiling clang from source +is recommended. + +### Clang-format plugins +Clion IDE supports the plugin "ClangFormat", you can search in `File->Setting->Plugins` + and download it. +But the version is not match with clang-format. Judging from the options supported, +the version is lower than clang-format-9.0. + +## Usage + +### CMD +`clang-format --style=file -i $File$` + +When using `-style=file`, clang-format for each input file will try to find the +.clang-format file located in the closest parent directory of the input file. +When the standard input is used, the search is started from the current directory. + +Note: filter out the files which should not be formatted, when batch clang-formating + files. + + A example of how to filter \*.h/\*.cpp and exclude some dirs: + +`find . -type f -not \( -wholename ./env/* \) -regextype posix-egrep -regex + ".*\.(cpp|h)" | xargs clang-format -i -style=file` + +### Using clang-format in IDEs or Editors +#### Clion +If using the plugin 'ClangFormat' in Clion, choose `Reformat Code` or press the keyboard +shortcut. +#### VS Code +VS Code needs install the extension 'Clang-Format', and specify the executable path of +clang-format in settings. + +``` +"clang-format.executable": "$clang-format path$", +"clang-format.style": "file" +``` +Then, choose `Format Document`. \ No newline at end of file diff --git a/docs/en/downloads/downloads.md b/docs/en/downloads/downloads.md new file mode 100644 index 00000000000000..cc4ae2f6b422d5 --- /dev/null +++ b/docs/en/downloads/downloads.md @@ -0,0 +1,21 @@ +--- +{ + "title": "Downloads", + "language": "en" +} +--- + +# Downloads + +You can download source code from following links, then compile and install Doirs. + +| Version | Release Date | Download Source from Mirror | +|---|---|---| +| 0.12.0 | 2020-04-24 | [Source](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz.sha512)) | +| 0.11.0 | 2019-11-29 | [Source](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz.sha512)) | +| 0.10.0 | 2019-07-02 | [Source](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz.sha512)) | +| 0.9.0 | 2019-02-18 | [Source](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.sha512)) | + +To verify the downloaded files, please read [Verify Apache Release](../community/verify-apache-release_EN.html). + +After verification, please read [Compilation](../installing/compilation_EN.html) and [Installation and deployment](../installing/install-deploy_EN.html) to compile and install Doris. diff --git a/docs/en/extending-doris/audit-plugin.md b/docs/en/extending-doris/audit-plugin.md new file mode 100644 index 00000000000000..bbc391c953c7e7 --- /dev/null +++ b/docs/en/extending-doris/audit-plugin.md @@ -0,0 +1,96 @@ +--- +{ + "title": "Audit log plugin", + "language": "en" +} +--- + + + +# Audit log plugin + +Doris's audit log plugin was developed based on FE's plugin framework. Is an optional plugin. Users can install or uninstall this plugin at runtime. + +This plugin can periodically import the FE audit log into the specified Doris cluster, so that users can easily view and analyze the audit log through SQL. + +## Compile, Configure and Deploy + +### Compile + +After executing `sh build_plugin.sh` in the Doris code directory, you will get the `auditloader.zip` file in the `fe_plugins/output` directory. + +### Configuration + +Unzip `auditloader.zip` and you will see three files: + +``` +auditloader.jar +plugin.properties +plugin.conf +``` + +Open `plugin.conf` for configuration. See the comments of the configuration items. + +After the configuration is complete, repackage the three files as `auditloader.zip`. + +### Deployment + +You can place this file on an http download server or copy it to the specified directory of all FEs. Here we use the latter. + +### Installation + +After deployment is complete, and before installing the plugin, you need to create the audit database and tables previously specified in `plugin.conf`. The table creation statement is as follows: + +``` +create table doris_audit_tbl__ +( +    query_id varchar (48) comment "Unique query id", +    time datetime not null comment "Query start time", +    client_ip varchar (32) comment "Client IP", +    user varchar (64) comment "User name", +    db varchar (96) comment "Database of this query", +    state varchar (8) comment "Query result state. EOF, ERR, OK", +    query_time bigint comment "Query execution time in millisecond", +    scan_bytes bigint comment "Total scan bytes of this query", +    scan_rows bigint comment "Total scan rows of this query", +    return_rows bigint comment "Returned rows of this query", +    stmt_id int comment "An incremental id of statement", +    is_query tinyint comment "Is this statemt a query. 1 or 0", +    frontend_ip varchar (32) comment "Frontend ip of executing this statement", +    stmt varchar (2048) comment "The original statement, trimed if longer than 2048 bytes" +) +partition by range (time) () +distributed by hash (query_id) buckets 1 +properties ( +    "dynamic_partition.time_unit" = "DAY", +    "dynamic_partition.start" = "-30", +    "dynamic_partition.end" = "3", +    "dynamic_partition.prefix" = "p", +    "dynamic_partition.buckets" = "1", +    "dynamic_partition.enable" = "true", +    "replication_num" = "1" +); +``` + +The `dynamic_partition` attribute selects the number of days to keep the audit log based on your needs. + +After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`. + +Upon completion, the plug-in will continuously import audit date into this table at specified intervals. \ No newline at end of file diff --git a/docs/en/extending-doris/doris-on-es.md b/docs/en/extending-doris/doris-on-es.md new file mode 100644 index 00000000000000..e46277ee94793b --- /dev/null +++ b/docs/en/extending-doris/doris-on-es.md @@ -0,0 +1,233 @@ +--- +{ + "title": "Doris On ES", + "language": "en" +} +--- + + + +# Doris On ES + +Doris-On-ES combines Doris's distributed query planning capability with ES (Elastic search)'s full-text search capability to provide a more complete OLAP scenario solution: + +1. Multi-index Distributed Join Query in ES +2. Joint Query of Tables in Doris and ES, More Complex Full-Text Retrieval and Filtering +3. Aggregated queries for fields of ES keyword type: suitable for frequent changes in index, tens of millions or more of single fragmented documents, and the cardinality of the field is very large + +This document mainly introduces the realization principle and usage of this function. + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. +* Elastic search (ES): The most popular open source distributed search engine. +* DataNode: The data storage and computing node of ES. +* MasterNode: The Master node of ES, which manages metadata, nodes, data distribution, etc. +* scroll: The built-in data set cursor feature of ES for streaming scanning and filtering of data. + + +## How to use it + +### Create appearance + +``` +CREATE EXTERNAL TABLE `es_table` ( + `id` bigint(20) COMMENT "", + `k1` bigint(20) COMMENT "", + `k2` datetime COMMENT "", + `k3` varchar(20) COMMENT "", + `k4` varchar(100) COMMENT "", + `k5` float COMMENT "" +) ENGINE=ELASTICSEARCH +PARTITION BY RANGE(`id`) +() +PROPERTIES ( +"host" = "http://192.168.0.1:8200,http://192.168.0.2:8200", +"user" = "root", +"password" = "root", +"index" = "tindex”, +"type" = "doc" +); +``` + +Description of parameters: + +Parameter | description +---|--- +Host | ES Cluster Connection Address, which can specify one or more, through which Doris obtains the share distribution information of ES version number and index +User | Open the user name of the ES cluster authenticated by basic, you need to ensure that the user has access to: / cluster / state / nodes / HTTP and other path permissions and read permissions for index +Password | corresponding user's password information +The index name of the ES corresponding to the table in index | Doris can be alias +Type | Specifies the type of index, defaulting to _doc +Transport | Internal reservation, default to http + +### Query + +#### Basic Conditions Filtration + +``` +select * from es_table where k1 > 1000 and k3 ='term' or k4 like 'fu*z_' +``` + +#### Extended esquery SQL grammar +The first column name parameter of `esquery` is used to associate `index`, the second parameter is the JSON expression of the basic `Query DSL`, and the curly bracket `{}` is used to include `root` of json. There is and can only be one key of json, such as mat. Ch, geo_shape, bool, etc. + +Match query: + +``` +select * from es_table where esquery(k4, '{ + "match": { + "k4": "doris on elasticsearch" + } + }'); +``` +Geo related queries: + +``` +select * from es_table where esquery(k4, '{ + "geo_shape": { + "location": { + "shape": { + "type": "envelope", + "coordinates": [ + [ + 13, + 53 + ], + [ + 14, + 52 + ] + ] + }, + "relation": "within" + } + } + }'); +``` + +Bool query: + +``` +select * from es_table where esquery(k4, ' { + "bool": { + "must": [ + { + "terms": { + "k1": [ + 11, + 12 + ] + } + }, + { + "terms": { + "k2": [ + 100 + ] + } + } + ] + } + }'); +``` + + + +## Principle + +``` ++----------------------------------------------+ +| | +| Doris +------------------+ | +| | FE +--------------+-------+ +| | | Request Shard Location +| +--+-------------+-+ | | +| ^ ^ | | +| | | | | +| +-------------------+ +------------------+ | | +| | | | | | | | | +| | +----------+----+ | | +--+-----------+ | | | +| | | BE | | | | BE | | | | +| | +---------------+ | | +--------------+ | | | ++----------------------------------------------+ | + | | | | | | | + | | | | | | | + | HTTP SCROLL | | HTTP SCROLL | | ++-----------+---------------------+------------+ | +| | v | | v | | | +| | +------+--------+ | | +------+-------+ | | | +| | | | | | | | | | | +| | | DataNode | | | | DataNode +<-----------+ +| | | | | | | | | | | +| | | +<--------------------------------+ +| | +---------------+ | | |--------------| | | | +| +-------------------+ +------------------+ | | +| Same Physical Node | | +| | | +| +-----------------------+ | | +| | | | | +| | MasterNode +<-----------------+ +| ES | | | +| +-----------------------+ | ++----------------------------------------------+ + + +``` + +1. After the ES appearance is created, FE requests the host specified by the table to obtain HTTP port information of all nodes and share distribution information of index. If the request fails, it will traverse the host list sequentially until it succeeds or fails completely. + +2. When querying, the query plan will be generated and sent to the corresponding BE node according to some node information obtained by FE and metadata information of index. + +3. The BE node requests locally deployed ES nodes in accordance with the `proximity principle`. The BE receives data concurrently from each fragment of ES index in the `HTTP Scroll` mode. + +4. After calculating the result, return it to client + +## Push-Down operations +An important function of `Doris On Elastic` search is to push down filtering conditions: push ES under filtering conditions, so that only data that really meets the conditions can be returned, which can significantly improve query performance and reduce the CPU, memory and IO utilization of Doris and Elastic search. + +The following operators are optimized to push down filters as follows: + +| SQL syntax | ES 5.x+ syntax | +|-------|:---:| +| = | term query| +| in | terms query | +| > , < , >= , ⇐ | range | +| and | bool.filter | +| or | bool.should | +| not | bool.must_not | +| not in | bool.must_not + terms | +| esquery | ES Query DSL | + + +## Other notes + +1. ES Version Requirements + + The main version of ES is larger than 5. The scanning mode of ES data before 2. X and after 5. x is different. At present, the scanning mode of ES data after 5. x is supported. + +2. Does ES Cluster Support X-Pack Authentication + + Support all ES clusters using HTTP Basic authentication + +3. Some queries are much slower than requesting ES + + Yes, for example, query related to _count, etc., the ES internal will directly read the number of documents that meet the requirements of the relevant metadata, without the need to filter the real data. diff --git a/docs/en/extending-doris/plugin-development-manual.md b/docs/en/extending-doris/plugin-development-manual.md new file mode 100644 index 00000000000000..5d8da3ef2ac302 --- /dev/null +++ b/docs/en/extending-doris/plugin-development-manual.md @@ -0,0 +1,300 @@ +--- +{ + "title": "Plugin Development Manual", + "language": "en" +} +--- + + + +# Plugin Development Manual + +## Introduction + +Doris supports dynamic loading of plug-ins. Users can develop their own plug-ins to implement some extended functions. This manual mainly introduces the development, compilation and deployment methods of Frontend-side plug-ins. + + +`fe_plugins` is the parent module of the fe plugins. It can uniformly manage the third-party library information that the plugin depends on. Adding a plugin can add a submodule implementation under `fe_plugins`. + +## Plugin + +A FE Plugin can be a **.zip package** or a **directory**, which contains at least two parts: the `plugin.properties` and `.jar` files. The `plugin.properties` file is used to describe the plugin information. + +The file structure of a Plugin looks like this: + +``` +# plugin .zip +auditodemo.zip: + -plugin.properties + -auditdemo.jar + -xxx.config + -data/ + -test_data/ + +# plugin local directory +auditodemo/: + -plugin.properties + -auditdemo.jar + -xxx.config + -data/ + -test_data/ +``` + +`plugin.properties` example: + +``` +### required: +# +# the plugin name +name = audit_plugin_demo +# +# the plugin type +type = AUDIT +# +# simple summary of the plugin +description = just for test +# +# Doris's version, like: 0.11.0 +version = 0.11.0 + +### FE-Plugin optional: +# +# version of java the code is built against +# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4 +java.version = 1.8.31 +# +# the name of the class to load, fully-qualified. +classname = AuditPluginDemo + +### BE-Plugin optional: +# the name of the so to load +soName = example.so +``` + +## Write A Plugin + +The development environment of the FE plugin depends on the development environment of Doris. So please make sure Doris's compilation and development environment works normally. + +### Create module + +We can add a submodule in the `fe_plugins` directory to implement Plugin and create a project: + +``` +mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false +``` + +The command produces a new mvn project, and a new submodule is automatically added to `fe_plugins/pom.xml`: + +``` +    ..... +    org.apache +    doris-fe-plugins +    pom +    1.0-SNAPSHOT +     +        auditdemo +        # new plugin module +        doris-fe-test +     +    ..... +``` + +The new plugin project file structure is as follows: + +``` +-doris-fe-test/ +-pom.xml +-src/ + ---- main/java/org/apache/ + ------- App.java # mvn auto generate, ignore + ---- test/java/org/apache +``` + +We will add an assembly folder under main to store `plugin.properties` and `zip.xml`. After completion, the file structure is as follows: + +``` +-doris-fe-test/ +-pom.xml +-src/ +---- main/ +------ assembly/ +-------- plugin.properties +-------- zip.xml +------ java/org/apache/ +--------App.java # mvn auto generate, ignore +---- test/java/org/apache +``` + +### Add zip.xml + +`zip.xml`, used to describe the content of the final package of the plugin (.jar file, plugin.properties): + +``` + +    plugin +     +        zip +     +     +    false +     +         +            target +             +                *.jar +             +            / +         + +         +            src/main/assembly +             +                plugin.properties +             +            / +         +     + +``` + + +### Update pom.xml + +Then we need to update `pom.xml`, add doris-fe dependency, and modify maven packaging way: + +``` + + + + org.apache + doris-fe-plugins + 1.0-SNAPSHOT + + 4.0.0 + + auditloader + jar + + + + org.apache + doris-fe + + + + + ... + + + + + + auditloader + + + maven-assembly-plugin + 2.4.1 + + false + + src/main/assembly/zip.xml + + + + + make-assembly + package + + single + + + + + + + + +``` + +### Implement plugin + +Then we can happily implement Plugin according to the needs. Plugins need to implement the `Plugin` interface. For details, please refer to the `auditdemo` plugin sample code that comes with Doris. + +### Compile + +Before compiling the plugin, you must first execute `sh build.sh --fe` of Doris to complete the compilation of Doris FE. + +Finally, execute `sh build_plugin.sh` in the ${DORIS_HOME} path and you will find the `your_plugin_name.zip` file in `fe_plugins/output` + +Or you can execute `sh build_plugin.sh --plugin your_plugin_name` to only build your plugin. + +### Other way + +The easiest way, you can implement your plugin by modifying the example `auditdemo` + +## Deploy + +Doris's plugin can be deployed in three ways: + +* Http or Https .zip, like `http://xxx.xxxxxx.com/data/plugin.zip`, Doris will download this .zip file +* Local .zip, like `/home/work/data/plugin.zip`, need to be deployed on all FE and BE nodes +* Local directory, like `/home/work/data/plugin`, .zip decompressed folder, need to be deployed on all FE, BE nodes + +Note: Need to ensure that the plugin .zip file is available in the life cycle of doris! + +## Install and Uninstall + +Install and uninstall the plugin through the install/uninstall statements. More details, see `HELP INSTALL PLUGIN;` `HELP IUNNSTALL PLUGIN;` `HELP SHOW PLUGINS;` + +``` +mysql> install plugin from "/home/users/seaven/auditdemo.zip"; +Query OK, 0 rows affected (0.09 sec) + +mysql> mysql> show plugins\G +*************************** 1. row *************************** + Name: auditloader + Type: AUDIT +Description: load audit log to olap load, and user can view the statistic of queries + Version: 0.12.0 +JavaVersion: 1.8.31 + ClassName: AuditLoaderPlugin + SoName: NULL + Sources: /home/cmy/git/doris/core/fe_plugins/output/auditloader.zip + Status: INSTALLED +*************************** 2. row *************************** + Name: AuditLogBuilder + Type: AUDIT +Description: builtin audit logger + Version: 0.12.0 +JavaVersion: 1.8.31 + ClassName: org.apache.doris.qe.AuditLogBuilder + SoName: NULL + Sources: Builtin + Status: INSTALLED +2 rows in set (0.00 sec) + +mysql> uninstall plugin auditloader; +Query OK, 0 rows affected (0.05 sec) + +mysql> show plugins; +Empty set (0.00 sec) +``` diff --git a/docs/en/extending-doris/user-defined-function.md b/docs/en/extending-doris/user-defined-function.md new file mode 100644 index 00000000000000..68791104b3689f --- /dev/null +++ b/docs/en/extending-doris/user-defined-function.md @@ -0,0 +1,118 @@ +--- +{ + "title": "User Define Function", + "language": "en" +} +--- + + + +# User Define Function + +Users can extend Doris's capabilities through UDF mechanisms. Through this document, users can create their own UDF. + +## Writing UDF functions + +Before using UDF, users need to write their own UDF functions in Doris's UDF framework. In the `be/src/udf_samples/udf_sample.h | cpp` file, it is a simple UDF Demo. + +Writing a UDF function requires the following steps. + +### Writing functions + +Create the corresponding header file, CPP file, and implement the logic you need in the CPP file. The corresponding relationship between the format of implementation function in CPP file and UDF. + +#### Non-variable parameters + +For UDF with non-variable parameters, the corresponding relationship between them is very direct. +For example, `INT MyADD'(INT, INT) ` UDF corresponds to `IntVal AddUdf(FunctionContext* context, const IntVal & arg1, const IntVal & arg2)`. + +1. `AddUdf` can be any name, as long as it is specified when UDF is created. +2. The first parameter in the implementation function is always `FunctionContext*`. The implementer can obtain some query-related content and apply for some memory to be used through this structure. Specific interfaces can be defined in `udf/udf.h`. +3. Implementing functions from the second parameter requires one-to-one correspondence with UDF parameters, such as `IntVal` corresponding to `INT` type. All types in this section are referenced by `const`. +4. Return parameters should correspond to the type of UDF parameters. + +#### Variable parameters + +For variable parameters, see the following example, UDF `String md5sum (String,...)` corresponds to +`StringVal md5sumUdf (FunctionContext * ctx, int num args, const StringVal * args)` + +1. The `md5sumUdf` can also be changed at will. It can be specified at the time of creation. +2. The first parameter, like a non-variable parameter function, is passed in a `FunctionContext*`. +3. The variable parameter part consists of two parts. First, an integer is passed in, which shows that there are several parameters. Later, an array of variable parameter parts is passed in. + +#### Type correspondence + +|UDF Type|Argument Type| +|----|---------| +|TinyInt|TinyIntVal| +|SmallInt|SmallIntVal| +|Int|IntVal| +|BigInt|BigIntVal| +|LargeInt|LargeIntVal| +|Float|FloatVal| +|Double|DoubleVal| +|Date|DateTimeVal| +|Datetime|DateTimeVal| +|Char|StringVal| +|Varchar|StringVal| +|Decimal|DecimalVal| + +## Compiling UDF functions + +### Compile Doris + +Executing `sh build.sh` in the Doris root directory generates the corresponding `headers|libs` in `output/udf/` + +### Edit CMakeLists.txt + +Based on the `headers | libs` generated in the previous step, users can introduce the dependency using tools such as `CMakeLists`; in `CMakeLists`, dynamic libraries can be added by adding `-I|L` to `CMAKE_CXX_FLAGS`, respectively. For example, in `be/src/udf_samples/CMakeLists.txt`, a `udf sample` dynamic library is added using `add_library` (udfsample SHARED udf_sample.cpp) `target_link_libraries`(udfsample -static-libstdc++ -static-libgcc). You need to write down all the source files involved later (no header files included). + +### Execute compilation + +Create a `build` directory under this directory and execute `cmake ../` generate `Makefile` under `build`, and execute `make` to generate corresponding dynamic libraries. + +## Create UDF functions + +Through the above steps, you can get a dynamic library. You need to put this dynamic library in a location that can be accessed through the HTTP protocol. Then execute the create UDF function to create a UDF inside the Doris system. You need AMDIN privileges to do this. + +``` +CREATE [AGGREGATE] FUNCTION + name ([argtype][,...]) + [RETURNS] rettype + PROPERTIES (["key"="value"][,...]) +``` +Explain: + +1. In PROPERTIES, `symbol` denotes the corresponding symbol for the execution of the entry function, which must be set. You can get the corresponding symbol by the `nm` command, such as `nm libudfsample.so`, `grep AddUdf`, `ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4`. +2. In PROPERTIES, `object_file` denotes where to download to the corresponding dynamic library. This parameter must be set. +3. name: A function belongs to a DB in the form of `dbName`. `funcName`. When `dbName` is not specified explicitly, the DB where the current session is located is used as `dbName`. + +For more details, see `CREATE FUNCTION`. + +## Using UDF + +Users using UDF/UDAF must have `SELECT` privileges for the corresponding database. + +UDF is used in the same way as normal functions. The only difference is that the scope of built-in functions is global, while the scope of UDF is internal to DB. When the link session is inside the data, using the UDF name directly will find the corresponding UDF within the current DB. Otherwise, the user needs to display the database name of the specified UDF, such as `dbName`. `funcName`. + + +## Delete UDF functions + +When you no longer need UDF functions, you can delete a UDF function by using the following command, referring to `DROP FUNCTION`. diff --git a/docs/en/getting-started/advance-usage.md b/docs/en/getting-started/advance-usage.md new file mode 100644 index 00000000000000..fdf2983dc05c77 --- /dev/null +++ b/docs/en/getting-started/advance-usage.md @@ -0,0 +1,272 @@ +--- +{ + "title": "Advanced Use Guide", + "language": "en" +} +--- + + + +# Advanced Use Guide + +Here we introduce some of Doris's advanced features. + +## Table 1 Structural Change + +Schema of the table can be modified using the ALTER TABLE command, including the following modifications: + +* Additional columns +* Delete columns +* Modify column type +* Changing column order + +Examples are given below. + +Schema of Table 1 is as follows: + +``` ++----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+-------------+------+-------+---------+-------+ +| siteid | int(11) | No | true | 10 | | +| citycode | smallint(6) | No | true | N/A | | +| username | varchar(32) | No | true | | | +| pv | bigint(20) | No | false | 0 | SUM | ++----------+-------------+------+-------+---------+-------+ +``` + +We added a new column of uv, type BIGINT, aggregation type SUM, default value is 0: + +`ALTER TABLE table1 ADD COLUMN uv BIGINT SUM DEFAULT '0' after pv;` + +After successful submission, you can view the progress of the job by following commands: + +`SHOW ALTER TABLE COLUMN;` + +When the job state is FINISHED, the job is completed. The new Schema is in force. + +After ALTER TABLE is completed, you can view the latest Schema through `DESC TABLE`. + +``` +mysql> DESC table1; ++----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+-------------+------+-------+---------+-------+ +| siteid | int(11) | No | true | 10 | | +| citycode | smallint(6) | No | true | N/A | | +| username | varchar(32) | No | true | | | +| pv | bigint(20) | No | false | 0 | SUM | +| uv | bigint(20) | No | false | 0 | SUM | ++----------+-------------+------+-------+---------+-------+ +5 rows in set (0.00 sec) +``` + +The following command can be used to cancel the job currently being executed: + +`CANCEL ALTER TABLE COLUMN FROM table1` + +For more help, see `HELP ALTER TABLE'. + +## 2 Rollup + +Rollup can be understood as a physical index structure of Table. ** Physicalization ** is because its data is physically stored independently, and ** indexing ** means that Rollup can adjust column order to increase the hit rate of prefix index, or reduce key column to increase data aggregation. + +Examples are given below. + +Schema of Table 1 is as follows: + +``` ++----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+-------------+------+-------+---------+-------+ +| siteid | int(11) | No | true | 10 | | +| citycode | smallint(6) | No | true | N/A | | +| username | varchar(32) | No | true | | | +| pv | bigint(20) | No | false | 0 | SUM | +| uv | bigint(20) | No | false | 0 | SUM | ++----------+-------------+------+-------+---------+-------+ +``` + +For table1 detailed data, siteid, citycode and username form a set of keys, which aggregate the PV field. If the business side often has the need to see the total amount of PV in the city, it can build a rollup with only citycode and pv. + +`ALTER TABLE table1 ADD ROLLUP rollup_city(citycode, pv);` + +After successful submission, you can view the progress of the job by following commands: + +`SHOW ALTER TABLE ROLLUP;` + +When the job state is FINISHED, the job is completed. + +When Rollup is established, you can use `DESC table1 ALL'to view the Rollup information of the table. + +``` +mysql> desc table1 all; ++-------------+----------+-------------+------+-------+--------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++-------------+----------+-------------+------+-------+---------+-------+ +| table1 | siteid | int(11) | No | true | 10 | | +| | citycode | smallint(6) | No | true | N/A | | +| | username | varchar(32) | No | true | | | +| | pv | bigint(20) | No | false | 0 | SUM | +| | uv | bigint(20) | No | false | 0 | SUM | +| | | | | | | | +| rollup_city | citycode | smallint(6) | No | true | N/A | | +| | pv | bigint(20) | No | false | 0 | SUM | ++-------------+----------+-------------+------+-------+---------+-------+ +8 rows in set (0.01 sec) +``` + +The following command can be used to cancel the job currently being executed: + +`CANCEL ALTER TABLE ROLLUP FROM table1;` + +After Rollup is established, the query does not need to specify Rollup to query. Or specify the original table for query. The program automatically determines whether Rollup should be used. Whether Rollup is hit or not can be viewed by the `EXPLAIN your_sql;'command. + +For more help, see `HELP ALTER TABLE`. + +## 2 Query of Data Table + +### 2.1 Memory Limitation + +To prevent a user's query from consuming too much memory. Queries are controlled in memory. A query task uses no more than 2GB of memory by default on a single BE node. + +When users use it, if they find a `Memory limit exceeded` error, they usually exceed the memory limit. + +Users should try to optimize their SQL statements when they encounter memory overrun. + +If it is found that 2GB memory cannot be satisfied, the memory parameters can be set manually. + +Display query memory limits: + +``` +mysql> SHOW VARIABLES LIKE "%mem_limit%"; ++---------------+------------+ +| Variable_name | Value | ++---------------+------------+ +| exec_mem_limit| 2147483648 | ++---------------+------------+ +1 row in set (0.00 sec) +``` + +The unit of `exec_mem_limit` is byte, and the value of `exec_mem_limit` can be changed by the `SET` command. If changed to 8GB. + +`SET exec_mem_limit = 8589934592;` + +``` +mysql> SHOW VARIABLES LIKE "%mem_limit%"; ++---------------+------------+ +| Variable_name | Value | ++---------------+------------+ +| exec_mem_limit| 8589934592 | ++---------------+------------+ +1 row in set (0.00 sec) +``` + +>* The above modification is session level and is only valid within the current connection session. Disconnecting and reconnecting will change back to the default value. +>* If you need to modify the global variable, you can set it as follows: `SET GLOBAL exec_mem_limit = 8589934592;` When the setup is complete, disconnect the session and log in again, and the parameters will take effect permanently. + +### 2.2 Query timeout + +The current default query time is set to 300 seconds. If a query is not completed within 300 seconds, the query will be cancelled by the Doris system. Users can use this parameter to customize the timeout time of their applications and achieve a blocking mode similar to wait (timeout). + +View the current timeout settings: + +``` +mysql> SHOW VARIABLES LIKE "%query_timeout%"; ++---------------+-------+ +| Variable_name | Value | ++---------------+-------+ +| QUERY_TIMEOUT | 300 | ++---------------+-------+ +1 row in set (0.00 sec) +``` + +Modify the timeout to 1 minute: + +`SET query timeout =60;` + +>* The current timeout check interval is 5 seconds, so timeouts less than 5 seconds are not very accurate. +>* The above modifications are also session level. Global validity can be modified by `SET GLOBAL`. + +### 2.3 Broadcast/Shuffle Join + +By default, the system implements Join by conditionally filtering small tables, broadcasting them to the nodes where the large tables are located, forming a memory Hash table, and then streaming out the data of the large tables Hash Join. However, if the amount of data filtered by small tables can not be put into memory, Join will not be able to complete at this time. The usual error should be caused by memory overrun first. + +If you encounter the above situation, it is recommended to use Shuffle Join, also known as Partitioned Join. That is, small and large tables are Hash according to Join's key, and then distributed Join. This memory consumption is allocated to all computing nodes in the cluster. + +Use Broadcast Join (default): + +``` +mysql> select sum(table1.pv) from table1 join table2 where table1.siteid = 2; ++--------------------+ +| sum(`table1`.`pv`) | ++--------------------+ +| 10 | ++--------------------+ +1 row in set (0.20 sec) +``` + +Use Broadcast Join (explicitly specified): + +``` +mysql> select sum(table1.pv) from table1 join [broadcast] table2 where table1.siteid = 2; ++--------------------+ +| sum(`table1`.`pv`) | ++--------------------+ +| 10 | ++--------------------+ +1 row in set (0.20 sec) +``` + +Shuffle Join: + +``` +mysql> select sum(table1.pv) from table1 join [shuffle] table2 where table1.siteid = 2; ++--------------------+ +| sum(`table1`.`pv`) | ++--------------------+ +| 10 | ++--------------------+ +1 row in set (0.15 sec) +``` + +### 2.4 Query Retry and High Availability + +When multiple FE nodes are deployed, users can deploy load balancing layers on top of multiple FEs to achieve high availability of Doris. + +Here are some highly available solutions: + +**The first** + +I retry and load balancing in application layer code. For example, if a connection is found to be dead, it will automatically retry on other connections. Application-level code retry requires the application to configure multiple Doris front-end node addresses. + +**Second** + +If you use MySQL JDBC connector to connect Doris, you can use jdbc's automatic retry mechanism: + +``` +jdbc:mysql:/[host:port],[host:port].../[database][? propertyName1][=propertyValue1][&propertyName2][=propertyValue2]... +``` + +**The third** + +Applications can connect to and deploy MySQL Proxy on the same machine by configuring MySQL Proxy's Failover and Load Balance functions. + +`http://dev.mysql.com/doc/refman/5.6/en/mysql-proxy-using.html` \ No newline at end of file diff --git a/docs/en/getting-started/basic-usage.md b/docs/en/getting-started/basic-usage.md new file mode 100644 index 00000000000000..aea122be9f15c2 --- /dev/null +++ b/docs/en/getting-started/basic-usage.md @@ -0,0 +1,382 @@ +--- +{ + "title": "Guidelines for Basic Use", + "language": "en" +} +--- + + + + +# Guidelines for Basic Use + +Doris uses MySQL protocol to communicate. Users can connect to Doris cluster through MySQL client or MySQL JDBC. When selecting the MySQL client version, it is recommended to use the version after 5.1, because user names of more than 16 characters can not be supported before 5.1. This paper takes MySQL client as an example to show users the basic usage of Doris through a complete process. + +## 1 Create Users + +### 1.1 Root User Logon and Password Modification + +Doris has built-in root and admin users, and the password is empty by default. After starting the Doris program, you can connect to the Doris cluster through root or admin users. +Use the following command to log in to Doris: + +``` +mysql -h FE_HOST -P9030 -uroot +``` + +>` fe_host` is the IP address of any FE node. ` 9030 ` is the query_port configuration in fe.conf. + +After login, you can modify the root password by following commands + +``` +SET PASSWORD FOR 'root' = PASSWORD('your_password'); +``` + +### 1.3 Creating New Users + +Create an ordinary user with the following command. + +``` +CREATE USER 'test' IDENTIFIED BY 'test_passwd'; +``` + +Follow-up login can be done through the following connection commands. + +``` +mysql -h FE_HOST -P9030 -utest -ptest_passwd +``` + +> By default, the newly created common user does not have any permissions. Permission grants can be referred to later permission grants. + +## 2 Data Table Creation and Data Import + +### 2.1 Create a database + +Initially, a database can be created through root or admin users: + +`CREATE DATABASE example_db;` + +> All commands can use'HELP command;'to see detailed grammar help. For example: `HELP CREATE DATABASE;'` + +> If you don't know the full name of the command, you can use "help command a field" for fuzzy query. If you type'HELP CREATE', you can match commands like `CREATE DATABASE', `CREATE TABLE', `CREATE USER', etc. + +After the database is created, you can view the database information through `SHOW DATABASES'. + +``` +MySQL> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| example_db | +| information_schema | ++--------------------+ +2 rows in set (0.00 sec) +``` + +Information_schema exists to be compatible with MySQL protocol. In practice, information may not be very accurate. Therefore, information about specific databases is suggested to be obtained by directly querying the corresponding databases. + +### 2.2 Account Authorization + +After the example_db is created, the read and write permissions of example_db can be authorized to ordinary accounts, such as test, through the root/admin account. After authorization, the example_db database can be operated by logging in with the test account. + +`GRANT ALL ON example_db TO test;` + +### 2.3 Formulation + +Create a table using the `CREATE TABLE'command. More detailed parameters can be seen: + +`HELP CREATE TABLE;` + +First switch the database: + +`USE example_db;` + +Doris supports single partition and composite partition. + +In the composite partition: + +* The first level is called Partition, or partition. Users can specify a dimension column as a partition column (currently only integer and time type columns are supported), and specify the range of values for each partition. + +* The second stage is called Distribution, or bucket division. Users can specify one or more dimension columns and the number of buckets for HASH distribution of data. + +Composite partitioning is recommended for the following scenarios + +* There are time dimensions or similar dimensions with ordered values, which can be used as partition columns. The partition granularity can be evaluated according to the frequency of importation and the amount of partition data. +* Historic data deletion requirements: If there is a need to delete historical data (for example, only the last N days of data are retained). Using composite partitions, you can achieve this by deleting historical partitions. Data can also be deleted by sending a DELETE statement within a specified partition. +* Solve the data skew problem: Each partition can specify the number of buckets separately. If dividing by day, when the amount of data varies greatly every day, we can divide the data of different partitions reasonably by the number of buckets in the specified partition. Bucket columns recommend choosing columns with high degree of differentiation. + +Users can also use no composite partitions, even single partitions. Then the data are only distributed by HASH. + +Taking the aggregation model as an example, the following two partitions are illustrated separately. + +#### Single partition + +Create a logical table with the name table1. The number of barrels is 10. + +The schema of this table is as follows: + +* Siteid: Type is INT (4 bytes), default value is 10 +* citycode: The type is SMALLINT (2 bytes) +* username: The type is VARCHAR, the maximum length is 32, and the default value is an empty string. +* pv: Type is BIGINT (8 bytes), default value is 0; this is an index column, Doris will aggregate the index column internally, the aggregation method of this column is SUM. + +The TABLE statement is as follows: +``` +CREATE TABLE table1 +( + siteid INT DEFAULT '10', + citycode SMALLINT, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' +) +AGGREGATE KEY(siteid, citycode, username) +DISTRIBUTED BY HASH(siteid) BUCKETS 10 +PROPERTIES("replication_num" = "1"); +``` + +#### Composite partition + +Create a logical table named table2. + +The schema of this table is as follows: + +* event_day: Type DATE, no default +* Siteid: Type is INT (4 bytes), default value is 10 +* citycode: The type is SMALLINT (2 bytes) +* username: The type is VARCHAR, the maximum length is 32, and the default value is an empty string. +* pv: Type is BIGINT (8 bytes), default value is 0; this is an index column, Doris will aggregate the index column internally, the aggregation method of this column is SUM. + +We use the event_day column as the partition column to create three partitions: p201706, p201707, and p201708. + +* p201706: Range [Minimum, 2017-07-01) +* p201707: Scope [2017-07-01, 2017-08-01) +* p201708: Scope [2017-08-01, 2017-09-01) + +> Note that the interval is left closed and right open. + +Each partition uses siteid to hash buckets, with a bucket count of 10 + +The TABLE statement is as follows: +``` +CREATE TABLE table2 +( + event_day DATE, + siteid INT DEFAULT '10', + citycode SMALLINT, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' +) +AGGREGATE KEY(event_day, siteid, citycode, username) +PARTITION BY RANGE(event_day) +( + PARTITION p201706 VALUES LESS THAN ('2017-07-01'), + PARTITION p201707 VALUES LESS THAN ('2017-08-01'), + PARTITION p201708 VALUES LESS THAN ('2017-09-01') +) +DISTRIBUTED BY HASH(siteid) BUCKETS 10 +PROPERTIES("replication_num" = "1"); +``` + +After the table is built, you can view the information of the table in example_db: + +``` +MySQL> SHOW TABLES; ++----------------------+ +| Tables_in_example_db | ++----------------------+ +| table1 | +| table2 | ++----------------------+ +2 rows in set (0.01 sec) + +MySQL> DESC table1; ++----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+-------------+------+-------+---------+-------+ +| siteid | int(11) | Yes | true | 10 | | +| citycode | smallint(6) | Yes | true | N/A | | +| username | varchar(32) | Yes | true | | | +| pv | bigint(20) | Yes | false | 0 | SUM | ++----------+-------------+------+-------+---------+-------+ +4 rows in set (0.00 sec) + +MySQL> DESC table2; ++-----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-----------+-------------+------+-------+---------+-------+ +| event_day | date | Yes | true | N/A | | +| siteid | int(11) | Yes | true | 10 | | +| citycode | smallint(6) | Yes | true | N/A | | +| username | varchar(32) | Yes | true | | | +| pv | bigint(20) | Yes | false | 0 | SUM | ++-----------+-------------+------+-------+---------+-------+ +5 rows in set (0.00 sec) +``` + +> Notes: +> +> 1. By setting replication_num, the above tables are all single-copy tables. Doris recommends that users adopt the default three-copy settings to ensure high availability. +> 2. Composite partition tables can be added or deleted dynamically. See the Partition section in `HELP ALTER TABLE`. +> 3. Data import can import the specified Partition. See `HELP LOAD'. +> 4. Schema of table can be dynamically modified. +> 5. Rollup can be added to Table to improve query performance. This section can be referred to the description of Rollup in Advanced Usage Guide. +> 6. The default value of Null property for column is true, which may result in poor scan performance. + +### 2.4 Import data + +Doris supports a variety of data import methods. Specifically, you can refer to the data import document. Here we use streaming import and Broker import as examples. + +#### Flow-in + +Streaming import transfers data to Doris via HTTP protocol. It can import local data directly without relying on other systems or components. Detailed grammar help can be found in `HELP STREAM LOAD;' + +Example 1: With "table1_20170707" as Label, import table1 tables using the local file table1_data. + +``` +curl --location-trusted -u test:test -H "label:table1_20170707" -H "column_separator:," -T table1_data http://FE_HOST:8030/api/example_db/table1/_stream_load +``` + +> 1. FE_HOST is the IP of any FE node and 8030 is http_port in fe.conf. +> 2. You can use the IP of any BE and the webserver_port in be.conf to connect the target left and right for import. For example: `BE_HOST:8040` + +The local file `table1_data` takes `,` as the separation between data, and the specific contents are as follows: + +``` +1,1,Jim,2 +2,1,grace,2 +3,2,tom,2 +4,3,bush,3 +5,3,helen,3 +``` + +Example 2: With "table2_20170707" as Label, import table2 tables using the local file table2_data. + +``` +curl --location-trusted -u test:test -H "label:table2_20170707" -H "column_separator:," -T table1_data http://127.0.0.1:8030/api/example_db/table2/_stream_load +``` + +The local file `table2_data'is separated by `t'. The details are as follows: + +``` +2017-07-03 1 1 jim 2 +2017-07-05 2 1 grace 2 +2017-07-12 3 2 tom 2 +2017-07-15 4 3 bush 3 +2017-07-12 5 3 helen 3 +``` + +> Notes: +> +> 1. The recommended file size for streaming import is limited to 10GB. Excessive file size will result in higher cost of retry failure. +> 2. Each batch of imported data needs to take a Label. Label is best a string related to a batch of data for easy reading and management. Doris based on Label guarantees that the same batch of data can be imported only once in a database. Label for failed tasks can be reused. +> 3. Streaming imports are synchronous commands. The successful return of the command indicates that the data has been imported, and the failure of the return indicates that the batch of data has not been imported. + +#### Broker Load + +Broker imports import data from external storage through deployed Broker processes. For more help, see `HELP BROKER LOAD;` + +Example: Import files on HDFS into table1 table with "table1_20170708" as Label + +``` +LOAD LABEL table1_20170708 +( + DATA INFILE("hdfs://your.namenode.host:port/dir/table1_data") + INTO TABLE table1 +) +WITH BROKER hdfs +( + "username"="hdfs_user", + "password"="hdfs_password" +) +PROPERTIES +( + "timeout"="3600", + "max_filter_ratio"="0.1" +); +``` + +Broker imports are asynchronous commands. Successful execution of the above commands only indicates successful submission of tasks. Successful imports need to be checked through `SHOW LOAD;' Such as: + +`SHOW LOAD WHERE LABLE = "table1_20170708";` + +In the return result, FINISHED in the `State'field indicates that the import was successful. + +For more instructions on `SHOW LOAD`, see` HELP SHOW LOAD; ` + +Asynchronous import tasks can be cancelled before the end: + +`CANCEL LOAD WHERE LABEL = "table1_20170708";` + +## 3 Data query + +### 3.1 Simple Query + +Examples: + +``` +MySQL> SELECT * FROM table1 LIMIT 3; ++--------+----------+----------+------+ +| siteid | citycode | username | pv | ++--------+----------+----------+------+ +| 2 | 1 | 'grace' | 2 | +| 5 | 3 | 'helen' | 3 | +| 3 | 2 | 'tom' | 2 | ++--------+----------+----------+------+ +5 rows in set (0.01 sec) + +MySQL> SELECT * FROM table1 ORDER BY citycode; ++--------+----------+----------+------+ +| siteid | citycode | username | pv | ++--------+----------+----------+------+ +| 2 | 1 | 'grace' | 2 | +| 1 | 1 | 'jim' | 2 | +| 3 | 2 | 'tom' | 2 | +| 4 | 3 | 'bush' | 3 | +| 5 | 3 | 'helen' | 3 | ++--------+----------+----------+------+ +5 rows in set (0.01 sec) +``` + +### 3.3 Join Query + +Examples: + +``` +MySQL> SELECT SUM(table1.pv) FROM table1 JOIN table2 WHERE table1.siteid = table2.siteid; ++--------------------+ +| sum(`table1`.`pv`) | ++--------------------+ +| 12 | ++--------------------+ +1 row in set (0.20 sec) +``` + +### 3.4 Subquery + +Examples: + +``` +MySQL> SELECT SUM(pv) FROM table2 WHERE siteid IN (SELECT siteid FROM table1 WHERE siteid > 2); ++-----------+ +| sum(`pv`) | ++-----------+ +| 8 | ++-----------+ +1 row in set (0.13 sec) +``` diff --git a/docs/en/getting-started/best-practice.md b/docs/en/getting-started/best-practice.md new file mode 100644 index 00000000000000..253695a038d212 --- /dev/null +++ b/docs/en/getting-started/best-practice.md @@ -0,0 +1,191 @@ +--- +{ + "title": "Best Practices", + "language": "en" +} +--- + + + + +# Best Practices + +## 1 tabulation + +### 1.1 Data Model Selection + +Doris data model is currently divided into three categories: AGGREGATE KEY, UNIQUE KEY, DUPLICATE KEY. Data in all three models are sorted by KEY. + +1.1.1. AGGREGATE KEY + +When AGGREGATE KEY is the same, old and new records are aggregated. The aggregation functions currently supported are SUM, MIN, MAX, REPLACE. + +AGGREGATE KEY model can aggregate data in advance and is suitable for reporting and multi-dimensional analysis business. + +``` +CREATE TABLE site_visit +( +siteid INT, +City: SMALLINT, +username VARCHAR (32), +pv BIGINT SUM DEFAULT '0' +) +AGGREGATE KEY(siteid, city, username) +DISTRIBUTED BY HASH(siteid) BUCKETS 10; +``` + +1.1.2. KEY UNIQUE + +When UNIQUE KEY is the same, the new record covers the old record. At present, UNIQUE KEY implements the same RPLACE aggregation method as GGREGATE KEY, and they are essentially the same. Suitable for analytical business with updated requirements. + +``` +CREATE TABLE sales_order +( +orderid BIGINT, +status TINYINT, +username VARCHAR (32), +amount BIGINT DEFAULT '0' +) +KEY (orderid) UNIT +DISTRIBUTED BY HASH(orderid) BUCKETS 10; +``` + +1.1.3. DUPLICATE KEY + +Only sort columns are specified, and the same rows are not merged. It is suitable for the analysis business where data need not be aggregated in advance. + +``` +CREATE TABLE session_data +( +visitorid SMALLINT, +sessionid BIGINT, +visit time DATETIME, +City CHAR (20), +province CHAR(20), +ip. varchar (32), +brower CHAR(20), +url: VARCHAR (1024) +) +DUPLICATE KEY (visitor time, session time) +DISTRIBUTED BY HASH(sessionid, visitorid) BUCKETS 10; +``` + +### 1.2 Wide Table vs. Star Schema + +In order to adapt to the front-end business, business side often does not distinguish dimension information from indicator information, but defines Schema as a wide table. For Doris, the performance of such wide gauges is often unsatisfactory: + +* There are many fields in Schema, and there may be more key columns in the aggregation model. The number of columns that need to be sorted in the import process will increase. +* Dimensional information updates are reflected in the whole table, and the frequency of updates directly affects the efficiency of queries. + +In the process of using Star Schema, users are advised to use Star Schema to distinguish dimension tables from indicator tables as much as possible. Frequently updated dimension tables can also be placed in MySQL external tables. If there are only a few updates, they can be placed directly in Doris. When storing dimension tables in Doris, more copies of dimension tables can be set up to improve Join's performance. + +### 1.3 Partitioning and Bucketing + +Doris supports two-level partitioned storage. The first layer is RANGE partition and the second layer is HASH bucket. + +1.3.1. RANGE Partitioning + +The RANGE partition is used to divide data into different intervals, which can be logically understood as dividing the original table into multiple sub-tables. In business, most users will choose to partition on time, which has the following advantages: + +* Differentiable heat and cold data +* Availability of Doris Hierarchical Storage (SSD + SATA) +* Delete data by partition more quickly + +1.3.2. Hash Bucketing + +The data is divided into different buckets according to the hash value. + +* It is suggested that columns with large differentiation should be used as buckets to avoid data skew. +* In order to facilitate data recovery, it is suggested that the size of a single bucket should not be too large and should be kept within 10GB. Therefore, the number of buckets should be considered reasonably when building tables or increasing partitions, among which different partitions can specify different buckets. + +### 1.4 Sparse Index and Bloom Filter + +Doris stores the data in an orderly manner, and builds a sparse index for Doris on the basis of ordered data. The index granularity is block (1024 rows). + +Sparse index chooses fixed length prefix in schema as index content, and Doris currently chooses 36 bytes prefix as index. + +* When building tables, it is suggested that the common filter fields in queries should be placed in front of Schema. The more distinguishable the query fields are, the more frequent the query fields are. +* One particular feature of this is the varchar type field. The varchar type field can only be used as the last field of the sparse index. The index is truncated at varchar, so if varchar appears in front, the length of the index may be less than 36 bytes. Specifically, you can refer to [data model, ROLLUP and prefix index] (. / data-model-rollup. md). +* In addition to sparse index, Doris also provides bloomfilter index. Bloomfilter index has obvious filtering effect on columns with high discrimination. If you consider that varchar cannot be placed in a sparse index, you can create a bloomfilter index. + +### 1.5 Physical and Chemical View (rollup) + +Rollup can essentially be understood as a physical index of the original table. When creating Rollup, only some columns in Base Table can be selected as Schema. The order of fields in Schema can also be different from that in Base Table. + +Rollup can be considered in the following cases: + +1.5.1. Low ratio of data aggregation in the Base Table + +This is usually due to the fact that Base Table has more differentiated fields. At this point, you can consider selecting some columns and establishing Rollup. + +For the `site_visit'table: + +``` +site -u visit (siteid, city, username, pv) +``` + +Siteid may lead to a low degree of data aggregation. If business parties often base their PV needs on city statistics, they can build a city-only, PV-based ollup: + +``` +ALTER TABLE site_visit ADD ROLLUP rollup_city(city, pv); +``` + +1.5.2. The prefix index in Base Table cannot be hit + +Generally, the way Base Table is constructed cannot cover all query modes. At this point, you can consider adjusting the column order and establishing Rollup. + +Database Session + +``` +session -u data (visitorid, sessionid, visittime, city, province, ip, browser, url) +``` + +In addition to visitorid analysis, there are Brower and province analysis cases, Rollup can be established separately. + +``` +ALTER TABLE session_data ADD ROLLUP rollup_brower(brower,province,ip,url) DUPLICATE KEY(brower,province); +``` + +## 2 Schema Change + +There are three Schema Change in doris：Sorted Schema Change，Direct Schema Change, Linked Schema Change。 + +2.1. Sorted Schema Change + +The sorting of columns has been changed and the data needs to be reordered. For example, delete a column in a sorted column and reorder the fields. + +``` +ALTER TABLE site_visit DROP COLUMN city; +``` + +2.2. Direct Schema Change: There is no need to reorder, but there is a need to convert the data. For example, modify + the type of column, add a column to the sparse index, etc. + +``` +ALTER TABLE site_visit MODIFY COLUMN username varchar(64); +``` + +2.3. Linked Schema Change: No need to transform data, for example add columns. + +``` +ALTER TABLE site_visit ADD COLUMN click bigint SUM default '0'; +``` + +Schema is recommended to be considered when creating tables so that Schema can be changed more quickly. diff --git a/docs/en/getting-started/data-model-rollup.md b/docs/en/getting-started/data-model-rollup.md new file mode 100644 index 00000000000000..2bdb60698dc8f1 --- /dev/null +++ b/docs/en/getting-started/data-model-rollup.md @@ -0,0 +1,637 @@ +--- +{ + "title": "Data Model, ROLLUP and Prefix Index", + "language": "en" +} +--- + + + +# Data Model, ROLLUP and Prefix Index + +This document describes Doris's data model, ROLLUP and prefix index concepts at the logical level to help users better use Doris to cope with different business scenarios. + +## Basic concepts + +In Doris, data is logically described in the form of tables. +A table consists of rows and columns. Row is a row of user data. Column is used to describe different fields in a row of data. + +Columns can be divided into two categories: Key and Value. From a business perspective, Key and Value can correspond to dimension columns and indicator columns, respectively. + +Doris's data model is divided into three main categories: + +* Aggregate +* Uniq +* Duplicate + +Let's introduce them separately. + +## Aggregate Model + +We illustrate what aggregation model is and how to use it correctly with practical examples. + +### Example 1: Importing data aggregation + +Assume that the business has the following data table schema: + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +| userid | LARGEINT | | user id| +| date | DATE | | date of data filling| +| City | VARCHAR (20) | | User City| +| age | SMALLINT | | User age| +| sex | TINYINT | | User gender| +| Last_visit_date | DATETIME | REPLACE | Last user access time| +| Cost | BIGINT | SUM | Total User Consumption| +| max dwell time | INT | MAX | Maximum user residence time| +| min dwell time | INT | MIN | User minimum residence time| + +If converted into a table-building statement, the following is done (omitting the Partition and Distribution information in the table-building statement) + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "user id", + `date` DATE NOT NULL COMMENT "data import time", + `city` VARCHAR(20) COMMENT "city", + `age` SMALLINT COMMENT "age", + `sex` TINYINT COMMENT "gender", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "last visit date time", + `cost` BIGINT SUM DEFAULT "0" COMMENT "user total cost", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "user max dwell time", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "user min dwell time", +) +AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) +... /* ignore Partition and Distribution */ +; +``` + +As you can see, this is a typical fact table of user information and access behavior. +In general star model, user information and access behavior are stored in dimension table and fact table respectively. Here, in order to explain Doris's data model more conveniently, we store the two parts of information in a single table. + +The columns in the table are divided into Key (dimension column) and Value (indicator column) according to whether `AggregationType`is set or not. No `AggregationType`, such as `user_id`, `date`, `age`, etc., is set as **Key**, while AggregationType'is set as **Value**. + +When we import data, the same rows and aggregates into one row for the Key column, while the Value column aggregates according to the set `AggregationType`. `AggregationType`currently has the following four ways of aggregation: + +1. SUM: Sum, multi-line Value accumulation. +2. REPLACE: Instead, Values in the next batch of data will replace Values in rows previously imported. +3. MAX: Keep the maximum. +4. MIN: Keep the minimum. + +Suppose we have the following imported data (raw data): + +|user\_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 06:00 | 20 | 10 | 10| +| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 07:00 | 15 | 2 | 2| +| 10001 | 2017-10-01 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| +| 10002 | 2017-10-02 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| +| 10003 | 2017-10-02 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| +| 10004 | 2017-10-01 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| +| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| + +Let's assume that this is a table that records the user's behavior in accessing a commodity page. Let's take the first row of data as an example and explain it as follows: + +| Data | Description| +|---|---| +| 10000 | User id, each user uniquely identifies id| +| 2017-10-01 | Data storage time, accurate to date| +| Beijing | User City| +| 20 | User Age| +| 0 | Gender male (1 for female)| +| 2017-10-01 06:00 | User's time to visit this page, accurate to seconds| +| 20 | Consumption generated by the user's current visit| +| 10 | User's visit, time to stay on the page| +| 10 | User's current visit, time spent on the page (redundancy)| + +Then when this batch of data is imported into Doris correctly, the final storage in Doris is is as follows: + +|user\_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 07:00 | 35 | 10 | 2| +| 10001 | 2017-10-01 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| +| 10002 | 2017-10-02 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| +| 10003 | 2017-10-02 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| +| 10004 | 2017-10-01 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| +| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| + +As you can see, there is only one line of aggregated data left for 10,000 users. The data of other users are consistent with the original data. Here we first explain the aggregated data of user 10000: + +The first five columns remain unchanged, starting with column 6 `last_visit_date': + +*`2017-10-01 07:00`: Because the `last_visit_date`column is aggregated by REPLACE, the `2017-10-01 07:00` column has been replaced by `2017-10-01 06:00'. +> Note: For data in the same import batch, the order of replacement is not guaranteed for the aggregation of REPLACE. For example, in this case, it may be `2017-10-01 06:00'. For data from different imported batches, it can be guaranteed that the data from the latter batch will replace the former batch. + +*`35`: Because the aggregation type of the `cost'column is SUM, 35 is accumulated from 20 + 15. +*`10`: Because the aggregation type of the`max_dwell_time'column is MAX, 10 and 2 take the maximum and get 10. +*`2`: Because the aggregation type of `min_dwell_time'column is MIN, 10 and 2 take the minimum value and get 2. + +After aggregation, Doris ultimately only stores aggregated data. In other words, detailed data will be lost and users can no longer query the detailed data before aggregation. + +### Example 2: Keep detailed data + +Following example 1, we modify the table structure as follows: + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +| userid | LARGEINT | | user id| +| date | DATE | | date of data filling| +| Time stamp | DATETIME | | Data filling time, accurate to seconds| +| City | VARCHAR (20) | | User City| +| age | SMALLINT | | User age| +| sex | TINYINT | | User gender| +| Last visit date | DATETIME | REPLACE | Last user access time| +| Cost | BIGINT | SUM | Total User Consumption| +| max dwell time | INT | MAX | Maximum user residence time| +| min dwell time | INT | MIN | User minimum residence time| + +That is to say, a column of `timestamp'has been added to record the data filling time accurate to seconds. + +The imported data are as follows: + +|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---|---| +| 10000 | 2017-10-01 | 2017-10-01 08:00:05 | Beijing | 20 | 0 | 2017-10-01 06:00 | 20 | 10 | 10| +| 10000 | 2017-10-01 | 2017-10-01 09:00:05 | Beijing | 20 | 0 | 2017-10-01 07:00 | 15 | 2 | 2| +| 10001 | 2017-10-01 | 2017-10-01 18:12:10 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| +| 10002 | 2017-10-02 | 2017-10-02 13:10:00 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| +| 10003 | 2017-10-02 | 2017-10-02 13:15:00 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| +| 10004 | 2017-10-01 | 2017-10-01 12:12:48 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| +| 10004 | 2017-10-03 | 2017-10-03 12:38:20 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| + +Then when this batch of data is imported into Doris correctly, the final storage in Doris is is as follows: + +|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---|---| +| 10000 | 2017-10-01 | 2017-10-01 08:00:05 | Beijing | 20 | 0 | 2017-10-01 06:00 | 20 | 10 | 10| +| 10000 | 2017-10-01 | 2017-10-01 09:00:05 | Beijing | 20 | 0 | 2017-10-01 07:00 | 15 | 2 | 2| +| 10001 | 2017-10-01 | 2017-10-01 18:12:10 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| +| 10002 | 2017-10-02 | 2017-10-02 13:10:00 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| +| 10003 | 2017-10-02 | 2017-10-02 13:15:00 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| +| 10004 | 2017-10-01 | 2017-10-01 12:12:48 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| +| 10004 | 2017-10-03 | 2017-10-03 12:38:20 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| + +We can see that the stored data, just like the imported data, does not aggregate at all. This is because, in this batch of data, because the `timestamp'column is added, the Keys of all rows are **not exactly the same**. That is, as long as the keys of each row are not identical in the imported data, Doris can save the complete detailed data even in the aggregation model. + +### Example 3: Importing data and aggregating existing data + +Take Example 1. Suppose that the data in the table are as follows: + +|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 07:00 | 35 | 10 | 2| +| 10001 | 2017-10-01 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| +| 10002 | 2017-10-02 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| +| 10003 | 2017-10-02 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| +| 10004 | 2017-10-01 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| +| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| + +We imported a new batch of data: + +|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 11:22:00 | 44 | 19 | 19| +| 10005 | 2017-10-03 | Changsha | 29 | 1 | 2017-10-03 18:11:02 | 3 | 1 | 1| + +Then when this batch of data is imported into Doris correctly, the final storage in Doris is is as follows: + +|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +| 10000 | 2017-10-01 | Beijing | 20 | 0 | 2017-10-01 07:00 | 35 | 10 | 2| +| 10001 | 2017-10-01 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| +| 10002 | 2017-10-02 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| +| 10003 | 2017-10-02 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| +| 10004 | 2017-10-01 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| +| 10004 | 2017-10-03 | Shenzhen | 35 | 0 | 2017-10-03 11:22:00 | 55 | 19 | 6| +| 10005 | 2017-10-03 | Changsha | 29 | 1 | 2017-10-03 18:11:02 | 3 | 1 | 1| + +As you can see, the existing data and the newly imported data of user 10004 have been aggregated. At the same time, 10005 new users'data were added. + +Data aggregation occurs in Doris in the following three stages: + +1. The ETL stage of data import for each batch. This phase aggregates data within each batch of imported data. +2. The stage in which the underlying BE performs data Compaction. At this stage, BE aggregates data from different batches that have been imported. +3. Data query stage. In data query, the data involved in the query will be aggregated accordingly. + +Data may be aggregated to varying degrees at different times. For example, when a batch of data is just imported, it may not be aggregated with the existing data. But for users, user**can only query aggregated data**. That is, different degrees of aggregation are transparent to user queries. Users should always assume that data exists in terms of the degree of aggregation that **ultimately completes**, and **should not assume that some aggregation has not yet occurred**. (See the section **Limitations of the aggregation model** for more details.) + +## Uniq Model + +In some multi-dimensional analysis scenarios, users are more concerned with how to ensure the uniqueness of Key, that is, how to obtain the Primary Key uniqueness constraint. Therefore, we introduce Uniq's data model. This model is essentially a special case of aggregation model and a simplified representation of table structure. Let's give an example. + +|ColumnName|Type|IsKey|Comment| +|---|---|---|---| +| user_id | BIGINT | Yes | user id| +| username | VARCHAR (50) | Yes | User nickname| +| City | VARCHAR (20) | No | User City| +| age | SMALLINT | No | User Age| +| sex | TINYINT | No | User Gender| +| Phone | LARGEINT | No | User Phone| +| address | VARCHAR (500) | No | User Address| +| register_time | DATETIME | No | user registration time| + +This is a typical user base information table. There is no aggregation requirement for this type of data, just the uniqueness of the primary key is guaranteed. (The primary key here is user_id + username). Then our statement is as follows: + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( +`user_id` LARGEINT NOT NULL COMMENT "用户id", +"username" VARCHAR (50) NOT NULL COMMENT "25143;" 261651;" +` City `VARCHAR (20) COMMENT `User City', +"Age" SMALLINT COMMENT "29992;" 25143;"24180;" 40836 ", +`sex` TINYINT COMMENT "用户性别", +`phone` LARGEINT COMMENT "用户电话", +'address ` VARCHAR (500) COMMENT'25143;', +'register 'or'time' DATETIME COMMENT "29992;" 25143;"27880;" 20876;"26102;" 38388;" +) +Unique Key ("User", "User", "Name") +... /* 省略 Partition 和 Distribution 信息 */ +; +``` + +This table structure is exactly the same as the following table structure described by the aggregation model: + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +| user_id | BIGINT | | user id| +| username | VARCHAR (50) | | User nickname| +| City | VARCHAR (20) | REPLACE | User City| +| age | SMALLINT | REPLACE | User Age| +| sex | TINYINT | REPLACE | User Gender| +| Phone | LARGEINT | REPLACE | User Phone| +| address | VARCHAR (500) | REPLACE | User Address| +| register_time | DATETIME | REPLACE | User registration time| + +And table-building statements: + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( +`user_id` LARGEINT NOT NULL COMMENT "用户id", +"username" VARCHAR (50) NOT NULL COMMENT "25143;" 261651;" +` City `VARCHAR (20) REPLACE COMMENT `User City', +What do you say when you are young? +`sex` TINYINT REPLACE COMMENT "用户性别", +"phone" LARGEINT REPLACE COMMENT "25143;" +`address` VARCHAR(500) REPLACE COMMENT "用户地址", +'register 'or'time' DATETIME REPLACE COMMENT "29992;" 25143;"27880;" 20876;"26102;" +) +AGGREGATE KEY(`user_id`, `user_name`) +... /* 省略 Partition 和 Distribution 信息 */ +; +``` + +That is to say, Uniq model can be completely replaced by REPLACE in aggregation model. Its internal implementation and data storage are exactly the same. No further examples will be given here. + +## Duplicate Model + +In some multidimensional analysis scenarios, data has neither primary keys nor aggregation requirements. Therefore, we introduce Duplicate data model to meet this kind of demand. Examples are given. + +|ColumnName|Type|SortKey|Comment| +|---|---|---|---| +| Timstamp | DATETIME | Yes | Logging Time| +| Type | INT | Yes | Log Type| +|error_code|INT|Yes|error code| +| Error_msg | VARCHAR (1024) | No | Error Details| +|op_id|BIGINT|No|operator id| +|op_time|DATETIME|No|operation time| + +The TABLE statement is as follows: +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( +`timestamp` DATETIME NOT NULL COMMENT "日志时间", +`type` INT NOT NULL COMMENT "日志类型", +"Error"\\\\\\\\\\\\\ +`error_msg` VARCHAR(1024) COMMENT "错误详细信息", +`op_id` BIGINT COMMENT "负责人id", +OP `op `time ` DATETIME COMMENT "22788;" 29702;"26102;" 388;" +) +DUPLICATE KEY(`timestamp`, `type`) +... /* 省略 Partition 和 Distribution 信息 */ +; +``` + +This data model is different from Aggregate and Uniq models. Data is stored entirely in accordance with the data in the imported file, without any aggregation. Even if the two rows of data are identical, they will be retained. +The DUPLICATE KEY specified in the table building statement is only used to specify which columns the underlying data is sorted according to. (The more appropriate name should be "Sorted Column", where the name "DUPLICATE KEY" is used to specify the data model used. For more explanations of "Sorted Column", see the section ** Prefix Index **. On the choice of DUPLICATE KEY, we recommend that the first 2-4 columns be selected appropriately. + +This data model is suitable for storing raw data without aggregation requirements and primary key uniqueness constraints. For more usage scenarios, see the ** Limitations of the Aggregation Model ** section. + +## ROLLUP + +ROLLUP in multidimensional analysis means "scroll up", which means that data is aggregated further at a specified granularity. + +### Basic concepts + +In Doris, we make the table created by the user through the table building statement a Base table. Base table holds the basic data stored in the way specified by the user's table-building statement. + +On top of the Base table, we can create any number of ROLLUP tables. These ROLLUP data are generated based on the Base table and physically **stored independently**. + +The basic function of ROLLUP tables is to obtain coarser aggregated data on the basis of Base tables. + +Let's illustrate the ROLLUP tables and their roles in different data models with examples. + +#### ROLLUP in Aggregate Model and Uniq Model + +Because Uniq is only a special case of the Aggregate model, we do not distinguish it here. + +Example 1: Get the total consumption per user + +Following **Example 2** in the **Aggregate Model** section, the Base table structure is as follows: + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +| user_id | LARGEINT | | user id| +| date | DATE | | date of data filling| +| Time stamp | DATETIME | | Data filling time, accurate to seconds| +| City | VARCHAR (20) | | User City| +| age | SMALLINT | | User age| +| sex | TINYINT | | User gender| +| Last_visit_date | DATETIME | REPLACE | Last user access time| +| Cost | BIGINT | SUM | Total User Consumption| +| max dwell time | INT | MAX | Maximum user residence time| +| min dwell time | INT | MIN | User minimum residence time| + +The data stored are as follows: + +|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---|---| +| 10000 | 2017-10-01 | 2017-10-01 08:00:05 | Beijing | 20 | 0 | 2017-10-01 06:00 | 20 | 10 | 10| +| 10000 | 2017-10-01 | 2017-10-01 09:00:05 | Beijing | 20 | 0 | 2017-10-01 07:00 | 15 | 2 | 2| +| 10001 | 2017-10-01 | 2017-10-01 18:12:10 | Beijing | 30 | 1 | 2017-10-01 17:05:45 | 2 | 22 | 22| +| 10002 | 2017-10-02 | 2017-10-02 13:10:00 | Shanghai | 20 | 1 | 2017-10-02 12:59:12 | 200 | 5 | 5| +| 10003 | 2017-10-02 | 2017-10-02 13:15:00 | Guangzhou | 32 | 0 | 2017-10-02 11:20:00 | 30 | 11 | 11| +| 10004 | 2017-10-01 | 2017-10-01 12:12:48 | Shenzhen | 35 | 0 | 2017-10-01 10:00:15 | 100 | 3 | 3| +| 10004 | 2017-10-03 | 2017-10-03 12:38:20 | Shenzhen | 35 | 0 | 2017-10-03 10:20:22 | 11 | 6 | 6| + +On this basis, we create a ROLLUP: + +|ColumnName| +|---| +|user_id| +|cost| + +The ROLLUP contains only two columns: user_id and cost. After the creation, the data stored in the ROLLUP is as follows: + +|user\_id|cost| +|---|---| +|10000|35| +|10001|2| +|10002|200| +|10003|30| +|10004|111| + +As you can see, ROLLUP retains only the results of SUM on the cost column for each user_id. So when we do the following query: + +`SELECT user_id, sum(cost) FROM table GROUP BY user_id;` + +Doris automatically hits the ROLLUP table, thus completing the aggregated query by scanning only a very small amount of data. + +2. Example 2: Get the total consumption, the longest and shortest page residence time of users of different ages in different cities + +Follow example 1. Based on the Base table, we create a ROLLUP: + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +| City | VARCHAR (20) | | User City| +| age | SMALLINT | | User age| +| Cost | BIGINT | SUM | Total User Consumption| +| max dwell time | INT | MAX | Maximum user residence time| +| min dwell time | INT | MIN | User minimum residence time| + +After the creation, the data stored in the ROLLUP is as follows: + +|city|age|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---| +| Beijing | 20 | 0 | 30 | 10 | 2| +| Beijing | 30 | 1 | 2 | 22 | 22| +| Shanghai | 20 | 1 | 200 | 5 | 5| +| Guangzhou | 32 | 0 | 30 | 11 | 11| +| Shenzhen | 35 | 0 | 111 | 6 | 3| + +When we do the following queries: + +* Select City, Age, Sum (Cost), Max (Max dwell time), min (min dwell time) from table group by City, age;* +* `SELECT city, sum(cost), max(max_dwell_time), min(min_dwell_time) FROM table GROUP BY city;` +* `SELECT city, age, sum(cost), min(min_dwell_time) FROM table GROUP BY city, age;` + +Doris automatically hits the ROLLUP table. + +#### OLLUP in Duplicate Model + +Because the Duplicate model has no aggregate semantics. So the ROLLLUP in this model has lost the meaning of "scroll up". It's just to adjust the column order to hit the prefix index. In the next section, we will introduce prefix index in detail, and how to use ROLLUP to change prefix index in order to achieve better query efficiency. + +### Prefix Index and ROLLUP + +#### prefix index + +Unlike traditional database design, Doris does not support indexing on any column. OLAP databases based on MPP architecture such as Doris usually handle large amounts of data by improving concurrency. +In essence, Doris's data is stored in a data structure similar to SSTable (Sorted String Table). This structure is an ordered data structure, which can be sorted and stored according to the specified column. In this data structure, it is very efficient to search by sorting columns. + +In Aggregate, Uniq and Duplicate three data models. The underlying data storage is sorted and stored according to the columns specified in AGGREGATE KEY, UNIQ KEY and DUPLICATE KEY in their respective table-building statements. + +The prefix index, which is based on sorting, implements an index method to query data quickly according to a given prefix column. + +We use the prefix index of ** 36 bytes ** of a row of data as the prefix index of this row of data. When a VARCHAR type is encountered, the prefix index is truncated directly. We give examples to illustrate: + +1. The prefix index of the following table structure is user_id (8Byte) + age (8Bytes) + message (prefix 20 Bytes). + +|ColumnName|Type| +|---|---| +|user_id|BIGINT| +|age|INT| +|message|VARCHAR(100)| +|max\_dwell\_time|DATETIME| +|min\_dwell\_time|DATETIME| + +2. The prefix index of the following table structure is user_name (20 Bytes). Even if it does not reach 36 bytes, because it encounters VARCHAR, it truncates directly and no longer continues. + +|ColumnName|Type| +|---|---| +|user_name|VARCHAR(20)| +|age|INT| +|message|VARCHAR(100)| +|max\_dwell\_time|DATETIME| +|min\_dwell\_time|DATETIME| + +When our query condition is the prefix of ** prefix index **, it can greatly speed up the query speed. For example, in the first example, we execute the following queries: + +`SELECT * FROM table WHERE user_id=1829239 and age=20；` + +The efficiency of this query is much higher than that of ** the following queries: + +`SELECT * FROM table WHERE age=20；` + +Therefore, when constructing tables, ** correctly choosing column order can greatly improve query efficiency **. + +#### ROLLUP adjusts prefix index + +Because column order is specified when a table is built, there is only one prefix index for a table. This may be inefficient for queries that use other columns that cannot hit prefix indexes as conditions. Therefore, we can manually adjust the order of columns by creating ROLLUP. Examples are given. + +The structure of the Base table is as follows: + +|ColumnName|Type| +|---|---| +|user\_id|BIGINT| +|age|INT| +|message|VARCHAR(100)| +|max\_dwell\_time|DATETIME| +|min\_dwell\_time|DATETIME| + +On this basis, we can create a ROLLUP table: + +|ColumnName|Type| +|---|---| +|age|INT| +|user\_id|BIGINT| +|message|VARCHAR(100)| +|max\_dwell\_time|DATETIME| +|min\_dwell\_time|DATETIME| + +As you can see, the columns of ROLLUP and Base tables are exactly the same, just changing the order of user_id and age. So when we do the following query: + +`SELECT * FROM table where age=20 and massage LIKE "%error%";` + +The ROLLUP table is preferred because the prefix index of ROLLUP matches better. + +### Some Explanations of ROLLUP + +* The fundamental role of ROLLUP is to improve the query efficiency of some queries (whether by aggregating to reduce the amount of data or by modifying column order to match prefix indexes). Therefore, the meaning of ROLLUP has gone beyond the scope of "roll-up". That's why we named it Materized Index in the source code. +* ROLLUP is attached to the Base table and can be seen as an auxiliary data structure of the Base table. Users can create or delete ROLLUP based on the Base table, but cannot explicitly specify a query for a ROLLUP in the query. Whether ROLLUP is hit or not is entirely determined by the Doris system. +* ROLLUP data is stored in separate physical storage. Therefore, the more OLLUP you create, the more disk space you occupy. It also has an impact on the speed of import (the ETL phase of import automatically generates all ROLLUP data), but it does not reduce query efficiency (only better). +* Data updates for ROLLUP are fully synchronized with Base representations. Users need not care about this problem. +* Columns in ROLLUP are aggregated in exactly the same way as Base tables. There is no need to specify or modify ROLLUP when creating it. +* A necessary (inadequate) condition for a query to hit ROLLUP is that all columns ** (including the query condition columns in select list and where) involved in the query exist in the column of the ROLLUP. Otherwise, the query can only hit the Base table. +* Certain types of queries (such as count (*)) cannot hit ROLLUP under any conditions. See the next section **Limitations of the aggregation model**. +* The query execution plan can be obtained by `EXPLAIN your_sql;` command, and in the execution plan, whether ROLLUP has been hit or not can be checked. +* Base tables and all created ROLLUPs can be displayed by `DESC tbl_name ALL;` statement. + +In this document, you can see [Query how to hit Rollup] (hit-the-rollup) + +## Limitations of aggregation model + +Here we introduce the limitations of Aggregate model (including Uniq model). + +In the aggregation model, what the model presents is the aggregated data. That is to say, any data that has not yet been aggregated (for example, two different imported batches) must be presented in some way to ensure consistency. Let's give an example. + +The hypothesis table is structured as follows: + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +| userid | LARGEINT | | user id| +| date | DATE | | date of data filling| +| Cost | BIGINT | SUM | Total User Consumption| + +Assume that there are two batches of data that have been imported into the storage engine as follows: + +**batch 1** + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|50| +|10002|2017-11-21|39| + +**batch 2** + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|1| +|10001|2017-11-21|5| +|10003|2017-11-22|22| + +As you can see, data belonging to user 10001 in two import batches has not yet been aggregated. However, in order to ensure that users can only query the aggregated data as follows: + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|51| +|10001|2017-11-21|5| +|10002|2017-11-21|39| +|10003|2017-11-22|22| + +We add aggregation operator to query engine to ensure data consistency. + +In addition, on the aggregate column (Value), when executing aggregate class queries that are inconsistent with aggregate types, attention should be paid to semantics. For example, in the example above, we execute the following queries: + +`SELECT MIN(cost) FROM table;` + +The result is 5, not 1. + +At the same time, this consistency guarantee will greatly reduce the query efficiency in some queries. + +Let's take the most basic count (*) query as an example: + +`SELECT COUNT(*) FROM table;` + +In other databases, such queries return results quickly. Because in the implementation, we can get the query result by counting rows at the time of import and saving count statistics information, or by scanning only a column of data to get count value at the time of query, with very little overhead. But in Doris's aggregation model, the overhead of this query ** is very large **. + +Let's take the data as an example. + +**batch 1** + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|50| +|10002|2017-11-21|39| + +**batch 2** + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|1| +|10001|2017-11-21|5| +|10003|2017-11-22|22| + +Because the final aggregation result is: + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|51| +|10001|2017-11-21|5| +|10002|2017-11-21|39| +|10003|2017-11-22|22| + +So `select count (*) from table;` The correct result should be **4**. But if we only scan the `user_id'column and add query aggregation, the final result is **3** (10001, 10002, 10003). If aggregated without queries, the result is **5** (a total of five rows in two batches). It can be seen that both results are wrong. + +In order to get the correct result, we must read the data of `user_id` and `date`, and **together with aggregate** when querying, to return the correct result of **4**. That is to say, in the count (*) query, Doris must scan all AGGREGATE KEY columns (here are `user_id` and `date`) and aggregate them to get the semantically correct results. When aggregated columns are large, count (*) queries need to scan a large amount of data. + +Therefore, when there are frequent count (*) queries in the business, we recommend that users simulate count (*) by adding a column with a value of 1 and aggregation type of SUM. As the table structure in the previous example, we modify it as follows: + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +| user ID | BIGINT | | user id| +| date | DATE | | date of data filling| +| Cost | BIGINT | SUM | Total User Consumption| +| count | BIGINT | SUM | for counting| + +Add a count column and import the data with the column value **equal to 1**. The result of `select count (*) from table;`is equivalent to `select sum (count) from table;` The query efficiency of the latter is much higher than that of the former. However, this method also has limitations, that is, users need to guarantee that they will not import rows with the same AGGREGATE KEY column repeatedly. Otherwise, `select sum (count) from table;`can only express the number of rows originally imported, not the semantics of `select count (*) from table;` + +Another way is to **change the aggregation type of the count'column above to REPLACE, and still weigh 1**. Then`select sum (count) from table;` and `select count (*) from table;` the results will be consistent. And in this way, there is no restriction on importing duplicate rows. + +### Duplicate Model + +Duplicate model has no limitation of aggregation model. Because the model does not involve aggregate semantics, when doing count (*) query, we can get the correct semantics by choosing a column of queries arbitrarily. + +## Suggestions for Choosing Data Model + +Because the data model was established when the table was built, and **could not be modified **. Therefore, it is very important to select an appropriate data model**. + +1. Aggregate model can greatly reduce the amount of data scanned and the amount of query computation by pre-aggregation. It is very suitable for report query scenarios with fixed patterns. But this model is not very friendly for count (*) queries. At the same time, because the aggregation method on the Value column is fixed, semantic correctness should be considered in other types of aggregation queries. +2. Uniq model guarantees the uniqueness of primary key for scenarios requiring unique primary key constraints. However, the query advantage brought by pre-aggregation such as ROLLUP can not be exploited (because the essence is REPLACE, there is no such aggregation as SUM). +3. Duplicate is suitable for ad-hoc queries of any dimension. Although it is also impossible to take advantage of the pre-aggregation feature, it is not constrained by the aggregation model and can take advantage of the queue-store model (only reading related columns, but not all Key columns). diff --git a/docs/en/getting-started/data-partition.md b/docs/en/getting-started/data-partition.md new file mode 100644 index 00000000000000..532ef6ba502bac --- /dev/null +++ b/docs/en/getting-started/data-partition.md @@ -0,0 +1,293 @@ +--- +{ + "title": "Data Partition", + "language": "en" +} +--- + + + +# Data Partition + +This document mainly introduces Doris's table construction and data partitioning, as well as problems and solutions that may be encountered in the construction of the table. + +## Basic Concepts + +In Doris, data is logically described in the form of a table. + +### Row & Column + +A table includes rows (rows) and columns (columns). Row is a row of data for the user. Column is used to describe different fields in a row of data. + +Column can be divided into two broad categories: Key and Value. From a business perspective, Key and Value can correspond to dimension columns and metric columns, respectively. From the perspective of the aggregation model, the same row of Key columns will be aggregated into one row. The way the Value column is aggregated is specified by the user when the table is built. For an introduction to more aggregation models, see the [Doris Data Model] (./data-model-rollup.md). + +### Tablet & Partition + +In Doris's storage engine, user data is horizontally divided into several data slices (also known as data buckets). Each tablet contains several rows of data. The data between the individual tablets does not intersect and is physically stored independently. + +Multiple tablets are logically attributed to different partitions. A tablet belongs to only one Partition. And a Partition contains several Tablets. Because the tablet is physically stored independently, it can be considered that the Partition is physically independent. Tablet is the smallest physical storage unit for data movement, replication, and so on. + +Several Partitions form a Table. Partition can be thought of as the smallest logical unit of management. Importing and deleting data can be done for one Partition or only for one Partition. + +## Data division + +We use a table-building operation to illustrate Doris' data partitioning. + +Doris's built-in table is a synchronous command. If the command returns successfully, it means that the table is built successfully. + +See more help with `HELP CREATE TABLE;`. + +This section introduces Doris's approach to building tables with an example. + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "user id", + `date` DATE NOT NULL COMMENT "Data fill in date time", + `timestamp` DATETIME NOT NULL COMMENT "Timestamp of data being poured", + `city` VARCHAR(20) COMMENT "The city where the user is located", + `age` SMALLINT COMMENT "user age", + `sex` TINYINT COMMENT "User Gender", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "User last visit time", + `cost` BIGINT SUM DEFAULT "0" COMMENT "Total user consumption", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "User maximum dwell time", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "User minimum dwell time" +) +ENGINE=olap +AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) +PARTITION BY RANGE(`date`) +( + PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), + PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), + PARTITION `p201703` VALUES LESS THAN ("2017-04-01") +) +DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 +PROPERTIES +( + "replication_num" = "3", + "storage_medium" = "SSD", + "storage_cooldown_time" = "2018-01-01 12:00:00" +); + +``` + +### Column Definition + +Here we only use the AGGREGATE KEY data model as an example. See the [Doris Data Model] (./data-model-rollup.md) for more data models. + +The basic type of column can be viewed by executing `HELP CREATE TABLE;` in mysql-client. + +In the AGGREGATE KEY data model, all columns that do not specify an aggregation mode (SUM, REPLACE, MAX, MIN) are treated as Key columns. The rest is the Value column. + +When defining columns, you can refer to the following suggestions: + +1. The Key column must precede all Value columns. +2. Try to choose the type of integer. Because integer type calculations and lookups are much more efficient than strings. +3. For the selection principle of integer types of different lengths, follow ** enough to **. +4. For lengths of type VARCHAR and STRING, follow ** is sufficient. +5. The total byte length of all columns (including Key and Value) cannot exceed 100KB. + +### Partitioning and binning + +Doris supports two levels of data partitioning. The first layer is Partition, which only supports the division of Range. The second layer is Bucket (Tablet), which only supports the way Hash is divided. + +It is also possible to use only one layer of partitioning. When using a layer partition, only Bucket partitioning is supported. + +1. Partition + + * The Partition column can specify one or more columns. The partition class must be a KEY column. The use of multi-column partitions is described later in the **Multi-column partitioning** summary.  + * Regardless of the type of partition column, double quotes are required when writing partition values. + * Partition columns are usually time columns for easy management of old and new data. + * There is no theoretical limit on the number of partitions. + * When you do not use Partition to build a table, the system will automatically generate a Partition with the same name as the table name. This Partition is not visible to the user and cannot be modified. + * Partition supports only the upper bound by `VALUES LESS THAN (...)`, the system will use the upper bound of the previous partition as the lower bound of the partition, and generate a left closed right open interval. Passing, also supports specifying the upper and lower bounds by `VALUES [...)`, and generating a left closed right open interval. + * It is easier to understand by specifying `VALUES [...)`. Here is an example of the change in partition range when adding or deleting partitions using the `VALUES LESS THAN (...)` statement: + * As the example above, when the table is built, the following 3 partitions are automatically generated: + ``` + P201701: [MIN_VALUE, 2017-02-01) + P201702: [2017-02-01, 2017-03-01) + P201703: [2017-03-01, 2017-04-01) + ``` + * When we add a partition p201705 VALUES LESS THAN ("2017-06-01"), the partition results are as follows: + + ``` + P201701: [MIN_VALUE, 2017-02-01) + P201702: [2017-02-01, 2017-03-01) + P201703: [2017-03-01, 2017-04-01) + P201705: [2017-04-01, 2017-06-01) + ``` + + * At this point we delete the partition p201703, the partition results are as follows: + + ``` + p201701: [MIN_VALUE, 2017-02-01) + p201702: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) + ``` + + > Note that the partition range of p201702 and p201705 has not changed, and there is a hole between the two partitions: [2017-03-01, 2017-04-01). That is, if the imported data range is within this hole, it cannot be imported. + + * Continue to delete partition p201702, the partition results are as follows: + + ``` + p201701: [MIN_VALUE, 2017-02-01) + p201705: [2017-04-01, 2017-06-01) + The void range becomes: [2017-02-01, 2017-04-01) + ``` + + * Now add a partition p201702new VALUES LESS THAN ("2017-03-01"), the partition results are as follows: + + ``` + p201701: [MIN_VALUE, 2017-02-01) + p201702new: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) + ``` + + > You can see that the hole size is reduced to: [2017-03-01, 2017-04-01) + + * Now delete partition p201701 and add partition p201612 VALUES LESS THAN ("2017-01-01"), the partition result is as follows: + + ``` + p201612: [MIN_VALUE, 2017-01-01) + p201702new: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) + ``` + + > A new void appeared: [2017-01-01, 2017-02-01) + + In summary, the deletion of a partition does not change the scope of an existing partition. There may be holes in deleting partitions. When a partition is added by the `VALUES LESS THAN` statement, the lower bound of the partition immediately follows the upper bound of the previous partition. + + You cannot add partitions with overlapping ranges. + +2. Bucket + + * If a Partition is used, the `DISTRIBUTED ...` statement describes the division rules for the data in each partition. If you do not use Partition, it describes the rules for dividing the data of the entire table. + * The bucket column can be multiple columns, but it must be a Key column. The bucket column can be the same or different from the Partition column. + * The choice of bucket column is a trade-off between **query throughput** and **query concurrency**: + + 1. If you select multiple bucket columns, the data is more evenly distributed. However, if the query condition does not include the equivalent condition for all bucket columns, a query will scan all buckets. The throughput of such queries will increase, but the latency of a single query will increase. This method is suitable for large throughput and low concurrent query scenarios. + 2. If you select only one or a few bucket columns, the point query can query only one bucket. This approach is suitable for high-concurrency point query scenarios. + + * There is no theoretical limit on the number of buckets. + +3. Recommendations on the number and amount of data for Partitions and Buckets. + + * The total number of tablets in a table is equal to (Partition num * Bucket num). + * The number of tablets in a table, which is slightly more than the number of disks in the entire cluster, regardless of capacity expansion. + * The data volume of a single tablet does not theoretically have an upper and lower bound, but is recommended to be in the range of 1G - 10G. If the amount of data for a single tablet is too small, the aggregation of the data is not good and the metadata management pressure is high. If the amount of data is too large, it is not conducive to the migration, completion, and increase the cost of Schema Change or Rollup operation failure retry (the granularity of these operations failure retry is Tablet). + * When the tablet's data volume principle and quantity principle conflict, it is recommended to prioritize the data volume principle. + * When building a table, the number of Buckets for each partition is uniformly specified. However, when dynamically increasing partitions (`ADD PARTITION`), you can specify the number of Buckets for the new partition separately. This feature can be used to easily reduce or expand data. + * Once the number of Buckets for a Partition is specified, it cannot be changed. Therefore, when determining the number of Buckets, you need to consider the expansion of the cluster in advance. For example, there are currently only 3 hosts, and each host has 1 disk. If the number of Buckets is only set to 3 or less, then even if you add more machines later, you can't increase the concurrency. + * Give some examples: Suppose there are 10 BEs, one for each BE disk. If the total size of a table is 500MB, you can consider 4-8 shards. 5GB: 8-16. 50GB: 32. 500GB: Recommended partitions, each partition is about 50GB in size, with 16-32 shards per partition. 5TB: Recommended partitions, each with a size of around 50GB and 16-32 shards per partition. + + > Note: The amount of data in the table can be viewed by the `show data` command. The result is divided by the number of copies, which is the amount of data in the table. + +#### Multi-column partition + +Doris supports specifying multiple columns as partition columns, examples are as follows: + +``` +PARTITION BY RANGE(`date`, `id`) +( + PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), + PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), + PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") +) +``` + +In the above example, we specify `date` (DATE type) and `id` (INT type) as partition columns. The resulting partitions in the above example are as follows: + +``` +*p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) +*p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) +*p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) +``` + +Note that the last partition user defaults only the partition value of the `date` column, so the partition value of the `id` column will be filled with `MIN_VALUE` by default. When the user inserts data, the partition column values ​​are compared in order, and the corresponding partition is finally obtained. Examples are as follows: + +``` +* Data --> Partition +* 2017-01-01, 200 --> p201701_1000 +* 2017-01-01, 2000 --> p201701_1000 +* 2017-02-01, 100 --> p201701_1000 +* 2017-02-01, 2000 --> p201702_2000 +* 2017-02-15, 5000 --> p201702_2000 +* 2017-03-01, 2000 --> p201703_all +* 2017-03-10, 1 --> p201703_all +* 2017-04-01, 1000 --> Unable to import +* 2017-05-01, 1000 --> Unable to import +``` + +### PROPERTIES + +In the last PROPERTIES of the table statement, you can specify the following two parameters: + +Replication_num + + * The number of copies per tablet. The default is 3, it is recommended to keep the default. In the build statement, the number of Tablet copies in all Partitions is uniformly specified. When you add a new partition, you can individually specify the number of copies of the tablet in the new partition. + * The number of copies can be modified at runtime. It is strongly recommended to keep odd numbers. + * The maximum number of copies depends on the number of independent IPs in the cluster (note that it is not the number of BEs). The principle of replica distribution in Doris is that the copies of the same Tablet are not allowed to be distributed on the same physical machine, and the physical machine is identified as IP. Therefore, even if 3 or more BE instances are deployed on the same physical machine, if the BEs have the same IP, you can only set the number of copies to 1. + * For some small, and infrequently updated dimension tables, consider setting more copies. In this way, when joining queries, there is a greater probability of local data join. + +2. storage_medium & storage\_cooldown\_time + + * The BE data storage directory can be explicitly specified as SSD or HDD (differentiated by .SSD or .HDD suffix). When you build a table, you can uniformly specify the media for all Partition initial storage. Note that the suffix is ​​to explicitly specify the disk media without checking to see if it matches the actual media type. + * The default initial storage media can be specified by `default_storage_medium= XXX` in the fe configuration file `fe.conf`, or, if not, by default, HDD. If specified as an SSD, the data is initially stored on the SSD. + * If storage\_cooldown\_time is not specified, the data is automatically migrated from the SSD to the HDD after 30 days by default. If storage\_cooldown\_time is specified, the data will not migrate until the storage_cooldown_time time is reached. + * Note that this parameter is just a "best effort" setting when storage_medium is specified. Even if no SSD storage media is set in the cluster, no error is reported and it is automatically stored in the available data directory. Similarly, if the SSD media is inaccessible and out of space, the data may initially be stored directly on other available media. When the data expires and is migrated to the HDD, if the HDD media is inaccessible and there is not enough space, the migration may fail (but will continue to try). + +### ENGINE + +In this example, the type of ENGINE is olap, the default ENGINE type. In Doris, only this ENGINE type is managed and stored by Doris. Other ENGINE types, such as mysql, broker, es, etc., are essentially mappings to tables in other external databases or systems to ensure that Doris can read the data. And Doris itself does not create, manage, and store any tables and data of a non-olap ENGINE type. + +### Other + +`IF NOT EXISTS` indicates that if the table has not been created, it is created. Note that only the table name is judged here, and it is not determined whether the new table structure is the same as the existing table structure. So if there is a table with the same name but different structure, the command will also return success, but it does not mean that a new table and a new structure have been created. + +## common problem + +### Build Table Operations FAQ + +1. If a syntax error occurs in a long build statement, a syntax error may be incomplete. Here is a list of possible syntax errors for manual error correction: + + * The syntax is incorrect. Please read `HELP CREATE TABLE;` carefully to check the relevant syntax structure. + * Reserved words. When the user-defined name encounters a reserved word, it needs to be enclosed in the backquote ``. It is recommended that all custom names be generated using this symbol. + * Chinese characters or full-width characters. Non-utf8 encoded Chinese characters, or hidden full-width characters (spaces, punctuation, etc.) can cause syntax errors. It is recommended to check with a text editor with invisible characters. + +2. `Failed to create partition [xxx] . Timeout` + + Doris builds are created in order of Partition granularity. This error may be reported when a Partition creation fails. Even if you don't use Partition, you will report `Failed to create partition` when there is a problem with the built table, because as mentioned earlier, Doris will create an unchangeable default Partition for tables that do not have a Partition specified. + + When this error is encountered, it is usually the BE that has encountered problems creating data fragments. You can follow the steps below to troubleshoot: + + 1. In fe.log, find the `Failed to create partition` log for the corresponding point in time. In this log, a series of numbers like `{10001-10010}` will appear. The first number of the pair is the Backend ID and the second number is the Tablet ID. As for the pair of numbers above, on the Backend with ID 10001, creating a tablet with ID 10010 failed. + 2. Go to the be.INFO log corresponding to Backend and find the log related to the tablet id in the corresponding time period. You can find the error message. + 3. Listed below are some common tablet creation failure errors, including but not limited to: + * BE did not receive the relevant task, and the tablet id related log could not be found in be.INFO. Or the BE is created successfully, but the report fails. For the above questions, see [Deployment and Upgrade Documentation] to check the connectivity of FE and BE. + * Pre-allocated memory failed. It may be that the length of a line in a row in the table exceeds 100KB. + * `Too many open files`. The number of open file handles exceeds the Linux system limit. The handle limit of the Linux system needs to be modified. + + You can also extend the timeout by setting `tablet_create_timeout_second=xxx` in fe.conf. The default is 2 seconds. + +3. The build table command does not return results for a long time. + + Doris's table creation command is a synchronous command. The timeout of this command is currently set to be relatively simple, ie (tablet num * replication num) seconds. If you create more data fragments and have fragment creation failed, it may cause an error to be returned after waiting for a long timeout. + + Under normal circumstances, the statement will return in a few seconds or ten seconds. If it is more than one minute, it is recommended to cancel this operation directly and go to the FE or BE log to view the related errors. diff --git a/docs/en/getting-started/hit-the-rollup.md b/docs/en/getting-started/hit-the-rollup.md new file mode 100644 index 00000000000000..23b5642227f43c --- /dev/null +++ b/docs/en/getting-started/hit-the-rollup.md @@ -0,0 +1,296 @@ +--- +{ + "title": "Rollup and query", + "language": "en" +} +--- + + + +# Rollup and query + +As a polymer view in Doris, Rollup can play two roles in queries: + +* Index +* Aggregate data (only for aggregate models, aggregate key) + +However, in order to hit Rollup, certain conditions need to be met, and the value of PreAggregation of ScanNdo node in the execution plan can be used to determine whether Rollup can be hit or not, and the Rollup field can be used to determine which Rollup table is hit. + +## Noun Interpretation + +Base: Base table. + +Rollup: Generally, it refers to the Rollup tables created based on Base tables, but in some scenarios, it includes Base and Rollup tables. + +## Index + +Doris's prefix index has been introduced in the previous query practice, that is, Doris will generate the first 36 bytes in the Base/Rollup table separately in the underlying storage engine (with varchar type, the prefix index may be less than 36 bytes, varchar will truncate the prefix index, and use up to 20 bytes of varchar). A sorted sparse index data (data is also sorted, positioned by index, and then searched by dichotomy in the data), and then matched each Base/Rollup prefix index according to the conditions in the query, and selected a Base/Rollup that matched the longest prefix index. + +``` + ---> matching from left to right ++----+----+----+----+----+----+ +| c1 | c2 | c3 | c4 | c5 |... | +``` + +As shown in the figure above, the conditions of where and on in the query are pushed up and down to ScanNode and matched from the first column of the prefix index. Check if there are any of these columns in the condition, and then accumulate the matching length until the matching cannot match or the end of 36 bytes (columns of varchar type can only match 20 bytes and match less than 36 words). Section truncates prefix index, and then chooses a Base/Rollup with the longest matching length. The following example shows how to create a Base table and four rollups: + +``` ++---------------+-------+--------------+------+-------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++---------------+-------+--------------+------+-------+---------+-------+ +| test | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup_index1 | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup_index2 | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k1 | TINYINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup_index3 | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup_index4 | k4 | BIGINT | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | ++---------------+-------+--------------+------+-------+---------+-------+ +``` + +The prefix indexes of the three tables are + +``` +Base(k1 ,k2, k3, k4, k5, k6, k7) + +rollup_index1(k9)，rollup_index2(k9) + +rollup_index3(k4, k5, k6, k1, k2, k3, k7) + +rollup_index4(k4, k6, k5, k1, k2, k3, k7) +``` + +Conditions on columns that can be indexed with the prefix need to be `=` `<` `>` `<=` `>=` `in` `between`, and these conditions are side-by-side and the relationship uses `and` connections', which cannot be hit for `or`、`!=` and so on. Then look at the following query: + +``` +SELECT * FROM test WHERE k1 = 1 AND k2 > 3; +``` + +With the conditions on K1 and k2, check that only the first column of Base contains K1 in the condition, so match the longest prefix index, test, explain: + +``` +| 0:OlapScanNode +| TABLE: test +| PREAGGREGATION: OFF. Reason: No AggregateInfo +| PREDICATES: `k1` = 1, `k2` > 3 +| partitions=1/1 +| rollup: test +| buckets=1/10 +| cardinality=-1 +| avgRowSize=0.0 +| numNodes=0 +| tuple ids: 0 +``` + +Look again at the following queries: + +`SELECT * FROM test WHERE k4 =1 AND k5 > 3;` + +With K4 and K5 conditions, check that the first column of rollup_index3 and rollup_index4 contains k4, but the second column of rollup_index3 contains k5, so the matching prefix index is the longest. + +``` +| 0:OlapScanNode +| TABLE: test +| PREAGGREGATION: OFF. Reason: No AggregateInfo +| PREDICATES: `k4` = 1, `k5` > 3 +| partitions=1/1 +| rollup: rollup_index3 +| buckets=10/10 +| cardinality=-1 +| avgRowSize=0.0 +| numNodes=0 +| tuple ids: 0 +``` + +Now we try to match the conditions on the column containing varchar, as follows: + +`SELECT * FROM test WHERE k9 IN ("xxx", "yyyy") AND k1 = 10;` + +There are K9 and K1 conditions. The first column of rollup_index1 and rollup_index2 contains k9. It is reasonable to choose either rollup here to hit the prefix index and randomly select the same one (because there are just 20 bytes in varchar, and the prefix index is truncated in less than 36 bytes). The current strategy here will continue to match k1, because the second rollup_index1 is listed as k1, so rollup_index1 is chosen, in fact, the latter K1 condition will not play an accelerating role. (If the condition outside the prefix index needs to accelerate the query, it can be accelerated by establishing a Bloom Filter filter. Typically for string types, because Doris has a Block level for columns, a Min/Max index for shaping and dates.) The following is the result of explain. + +``` +| 0:OlapScanNode +| TABLE: test +| PREAGGREGATION: OFF. Reason: No AggregateInfo +| PREDICATES: `k9` IN ('xxx', 'yyyy'), `k1` = 10 +| partitions=1/1 +| rollup: rollup_index1 +| buckets=1/10 +| cardinality=-1 +| avgRowSize=0.0 +| numNodes=0 +| tuple ids: 0 +``` + +Finally, look at a query that can be hit by more than one Rollup: + +`Select * from test where K4 < 1000 and K5 = 80 and K6 = 10000;` + +There are three conditions: k4, K5 and k6. The first three columns of rollup_index3 and rollup_index4 contain these three columns respectively. So the prefix index length matched by them is the same. Both can be selected. The current default strategy is to select a rollup created earlier. Here is rollup_index3. + +``` +| 0:OlapScanNode +| TABLE: test +| PREAGGREGATION: OFF. Reason: No AggregateInfo +| PREDICATES: `k4` < 1000, `k5` = 80, `k6` >= 10000.0 +| partitions=1/1 +| rollup: rollup_index3 +| buckets=10/10 +| cardinality=-1 +| avgRowSize=0.0 +| numNodes=0 +| tuple ids: 0 +``` + +If you modify the above query slightly as follows: + +`SELECT * FROM test WHERE k4 < 1000 AND k5 = 80 OR k6 >= 10000;` + +The query here cannot hit the prefix index. (Even any Min/Max in the Doris storage engine, the BloomFilter index doesn't work.) + +## Aggregate data + +Of course, the function of aggregated data is indispensable for general polymer views. Such materialized views are very helpful for aggregated queries or report queries. To hit the polymer views, the following prerequisites are needed: + +1. There is a separate Rollup for all columns involved in a query or subquery. +2. If there is Join in a query or sub-query, the type of Join needs to be Inner join. + +The following are some types of aggregated queries that can hit Rollup. + +| Column type Query type | Sum | Distinct/Count Distinct | Min | Max | Ndv | +|--------------|-------|-------------------------|-------|-------|-------| +| Key | false | true | true | true | true | +| Value(Sum) | true | false | false | false | false | +|Value(Replace)| false | false | false | false | false | +| Value(Min) | false | false | true | false | false | +| Value(Max) | false | false | false | true | false | + + +If the above conditions are met, there will be two stages in judging the hit of Rollup for the aggregation model: + +1. Firstly, the Rollup table with the longest index hit by prefix index is matched by conditions. See the index strategy above. +2. Then compare the rows of Rollup and select the smallest Rollup. + +The following Base table and Rollup: + +``` ++-------------+-------+--------------+------+-------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++-------------+-------+--------------+------+-------+---------+-------+ +| test_rollup | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup2 | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup1 | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | ++-------------+-------+--------------+------+-------+---------+-------+ +``` + + +See the following queries: + +`SELECT SUM(k11) FROM test_rollup WHERE k1 = 10 AND k2 > 200 AND k3 in (1,2,3);` + +Firstly, it judges whether the query can hit the aggregated Rolup table. After checking the graph above, it is possible. Then the condition contains three conditions: k1, K2 and k3. The first three columns of test_rollup, rollup1 and rollup2 contain all the three conditions. So the prefix index length is the same. Then, it is obvious that the aggregation degree of rollup2 is the highest when comparing the number of rows. Row 2 is selected because of the minimum number of rows. + +``` +| 0:OlapScanNode | +| TABLE: test_rollup | +| PREAGGREGATION: ON | +| PREDICATES: `k1` = 10, `k2` > 200, `k3` IN (1, 2, 3) | +| partitions=1/1 | +| rollup: rollup2 | +| buckets=1/10 | +| cardinality=-1 | +| avgRowSize=0.0 | +| numNodes=0 | +| tuple ids: 0 | +``` diff --git a/docs/en/installing/compilation.md b/docs/en/installing/compilation.md new file mode 100644 index 00000000000000..547683cc3dca87 --- /dev/null +++ b/docs/en/installing/compilation.md @@ -0,0 +1,107 @@ +--- +{ + "title": "Compilation", + "language": "en" +} +--- + + + + +# Compilation + +This document focuses on how to code Doris through source code. + +## Developing mirror compilation using Docker (recommended) + +### Use off-the-shelf mirrors + +1. Download Docker Mirror + + `$ docker pull apachedoris/doris-dev:build-env` + + Check mirror download completed: + + ``` + $ docker images + REPOSITORY TAG IMAGE ID CREATED SIZE + apachedoris/doris-dev build-env f8bc5d4024e0 21 hours ago 3.28GB + ``` + +Note: For different versions of Oris, you need to download the corresponding mirror version. + +| image version | commit id | release version | +|---|---|---| +| apachedoris/doris-dev:build-env | before [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.8.x, 0.9.x | +| apachedoris/doris-dev:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) or later | 0.10.x or later | + +2. Running Mirror + + `$ docker run -it apachedoris/doris-dev:build-env` + + If you want to compile the local Doris source code, you can mount the path: + + ``` + $ docker run -it -v /your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/ apachedoris/doris-dev:build-env + ``` + +3. Download source code + + After starting the mirror, you should be in the container. The Doris source code can be downloaded from the following command (local source directory mounted is not required): + + ``` + $ wget https://dist.apache.org/repos/dist/dev/incubator/doris/xxx.tar.gz + or + $ git clone https://github.com/apache/incubator-doris.git + ``` + +4. Compile Doris + + ``` + $ sh build.sh + ``` + + After compilation, the output file is in the `output/` directory. + +### Self-compiling Development Environment Mirror + +You can also create a Doris development environment mirror yourself, referring specifically to the `docker/README.md'file. + + +## Direct Compilation (CentOS/Ubuntu) + +You can try to compile Doris directly in your own Linux environment. + +1. System Dependence + + `GCC 5.3.1+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+` + + If you are using Ubuntu 16.04 or newer, you can use the following command to install the dependencies + + `sudo apt-get install build-essential openjdk-8-jdk maven cmake byacc flex automake libtool-bin bison binutils-dev libiberty-dev` + + After installation, set environment variables `PATH`, `JAVA_HOME`, etc. + +2. Compile Doris + + ``` + $ sh build.sh + ``` + After compilation, the output file is in the `output/` directory. diff --git a/docs/en/installing/install-deploy.md b/docs/en/installing/install-deploy.md new file mode 100644 index 00000000000000..ea9cfec0ff5e40 --- /dev/null +++ b/docs/en/installing/install-deploy.md @@ -0,0 +1,434 @@ +--- +{ + "title": "Installation and deployment", + "language": "en" +} +--- + + + + +# Installation and deployment + +This document mainly introduces the hardware and software environment needed to deploy Doris, the proposed deployment mode, cluster expansion and scaling, and common problems in the process of cluster building and running. +Before reading this document, compile Doris according to the compiled document. + +## Software and hardware requirements + +### Overview + +Doris, as an open source MPP architecture OLAP database, can run on most mainstream commercial servers. In order to make full use of the concurrency advantages of MPP architecture and the high availability features of Doris, we recommend that the deployment of Doris follow the following requirements: + +#### Linux Operating System Version Requirements + +| Linux System | Version| +|---|---| +| Centers | 7.1 and above | +| Ubuntu | 16.04 and above | + +#### Software requirements + +| Soft | Version | +|---|---| +| Java | 1.8 and above | +| GCC | 4.8.2 and above | + +#### Development Test Environment + +| Module | CPU | Memory | Disk | Network | Instance Number| +|---|---|---|---|---|---| +| Frontend | 8 core + | 8GB + | SSD or SATA, 10GB + * | Gigabit Network Card | 1| +| Backend | 8-core + | 16GB + | SSD or SATA, 50GB + * | Gigabit Network Card | 1-3*| + +#### Production environment + +| Module | CPU | Memory | Disk | Network | Number of Instances (Minimum Requirements)| +|---|---|---|---|---|---| +| Frontend | 16 core + | 64GB + | SSD or RAID card, 100GB + * | 10,000 Mbp network card | 1-5*| +| Backend | 16 core + | 64GB + | SSD or SATA, 100G + * | 10-100 Mbp network card*| + +> Note 1: +> +> 1. The disk space of FE is mainly used to store metadata, including logs and images. Usually it ranges from several hundred MB to several GB. +> 2. BE's disk space is mainly used to store user data. The total disk space is calculated according to the user's total data * 3 (3 copies). Then an additional 40% of the space is reserved for background compaction and some intermediate data storage. +> 3. Multiple BE instances can be deployed on a single machine, but **can only deploy one FE**. If you need three copies of data, you need at least one BE instance per machine (instead of three BE instances per machine). **Clocks of multiple FE servers must be consistent (allowing a maximum of 5 seconds clock deviation)** +> 4. The test environment can also be tested with only one BE. In the actual production environment, the number of BE instances directly determines the overall query latency. +> 5. All deployment nodes close Swap. + +> Note 2: Number of FE nodes +> +> 1. FE roles are divided into Follower and Observer. (Leader is an elected role in the Follower group, hereinafter referred to as Follower, for the specific meaning, see [Metadata Design Document] (./internal/metadata-design).) +> 2. FE node data is at least 1 (1 Follower). When one Follower and one Observer are deployed, high read availability can be achieved. When three Followers are deployed, read-write high availability (HA) can be achieved. +> 3. The number of Followers **must be** odd, and the number of Observers is arbitrary. +> 4. According to past experience, when cluster availability requirements are high (e.g. providing online services), three Followers and one to three Observers can be deployed. For offline business, it is recommended to deploy 1 Follower and 1-3 Observers. + +* **Usually we recommend about 10 to 100 machines to give full play to Doris's performance (3 of them deploy FE (HA) and the rest deploy BE)** +* **Of course, Doris performance is positively correlated with the number and configuration of nodes. With a minimum of four machines (one FE, three BEs, one BE mixed with one Observer FE to provide metadata backup) and a lower configuration, Doris can still run smoothly.** +* **If FE and BE are mixed, we should pay attention to resource competition and ensure that metadata catalogue and data catalogue belong to different disks.** + +#### Broker deployment + +Broker is a process for accessing external data sources, such as hdfs. Usually, a broker instance is deployed on each machine. + +#### Network Requirements + +Doris instances communicate directly over the network. The following table shows all required ports + +| Instance Name | Port Name | Default Port | Communication Direction | Description| +| ---|---|---|---|---| +| BE | be_port | 9060 | FE - > BE | BE for receiving requests from FE| +| BE | webserver\_port | 8040 | BE <--> BE | BE| +| BE | heartbeat\_service_port | 9050 | FE - > BE | the heart beat service port (thrift) on BE, used to receive heartbeat from FE| +| BE | brpc\_port* | 8060 | FE < - > BE, BE < - > BE | BE for communication between BEs| +| FE | http_port* | 8030 | FE < - > FE, HTTP server port on user | FE| +| FE | rpc_port | 9020 | BE - > FE, FE < - > FE | thrift server port on FE| +| FE | query_port | 9030 | user | FE| +| FE | edit\_log_port | 9010 | FE <--> FE | FE| +| Broker | broker ipc_port | 8000 | FE - > Broker, BE - > Broker | Broker for receiving requests| + +> Note: +> +> 1. When deploying multiple FE instances, make sure that the http port configuration of FE is the same. +> 2. Make sure that each port has access in its proper direction before deployment. + +#### IP binding + +Because of the existence of multiple network cards, or the existence of virtual network cards caused by the installation of docker and other environments, the same host may have multiple different ips. Currently Doris does not automatically identify available IP. So when you encounter multiple IP on the deployment host, you must force the correct IP to be specified through the priority\_networks configuration item. + +Priority\_networks is a configuration that both FE and BE have, and the configuration items need to be written in fe.conf and be.conf. This configuration item is used to tell the process which IP should be bound when FE or BE starts. Examples are as follows: + +`priority_networks=10.1.3.0/24` + +This is a representation of [CIDR] (https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). FE or BE will find the matching IP based on this configuration item as their own local IP. + +**Note**: When priority networks is configured and FE or BE is started, only the correct IP binding of FE or BE is ensured. In ADD BACKEND or ADD FRONTEND statements, you also need to specify IP matching priority networks configuration, otherwise the cluster cannot be established. Give an example: + +BE is configured as `priority_networks = 10.1.3.0/24'.`. + +When you want to ADD BACKEND use ：`ALTER SYSTEM ADD BACKEND "192.168.0.1:9050";` + +Then FE and BE will not be able to communicate properly. + +At this point, DROP must remove the BE that added errors and re-use the correct IP to perform ADD BACKEND. + +FE is the same. + +BROKER does not currently have, nor does it need, priority\ networks. Broker's services are bound to 0.0.0 by default. Simply execute the correct accessible BROKER IP when ADD BROKER is used. + +## Cluster deployment + +### Manual deployment + +#### Deploy FE + +* Copy the FE deployment file to the specified node + + Copy the Fe folder under output generated by source code compilation to the node specified deployment path of FE. + +* Configure FE + + 1. The configuration file is conf/fe.conf. Note: `meta_dir`: Metadata storage location. The default is fe/doris-meta/. The directory needs to be **created manually** by. + 2. JAVA_OPTS in fe.conf defaults to a maximum heap memory of 4GB for java, and it is recommended that the production environment be adjusted to more than 8G. + +* Start FE + + `sh bin/start_fe.sh --daemon` + + The FE process starts and enters the background execution. Logs are stored in the fe/log/directory by default. If startup fails, you can view error messages by looking at fe/log/fe.log or fe/log/fe.out. + +* For deployment of multiple FEs, see the section "FE scaling and downsizing" + +#### Deploy BE + +* Copy BE deployment files to all nodes to deploy BE + + Copy the be folder under output generated by source code compilation to the specified deployment path of the BE node. + +* Modify all BE configurations + + Modify be/conf/be.conf. Mainly configure `storage_root_path`: data storage directory. The default is be/storage, this directory needs to be **created manually** by. In multi directories case, using `;` separation (do not add `;` after the last directory). + +* Add all BE nodes to FE + + BE nodes need to be added in FE before they can join the cluster. You can use mysql-client to connect to FE: + + `./mysql-client -h host -P port -uroot` + + The host is the node IP where FE is located; the port is the query_port in fe/conf/fe.conf; the root account is used by default and no password is used to login. + + After login, execute the following commands to add each BE: + + `ALTER SYSTEM ADD BACKEND "host:port";` + + If the multi-tenant function is used, the following command is executed to add BE: + + `ALTER SYSTEM ADD FREE BACKEND "host:port";` + + The host is the node IP where BE is located; the port is heartbeat_service_port in be/conf/be.conf. + + If the FREE keyword is not added, BE defaults to the automatically generated cluster, and the new BE does not belong to any cluster after adding the FREE keyword, so that when creating a new cluster, it can be selected from these free be, as detailed in [Multi-tenant Design Document] (./administrator-guide/operation/multi-tenant.md) + +* Start BE + + `sh bin/start_be.sh --daemon` + + The BE process will start and go into the background for execution. Logs are stored in be/log/directory by default. If startup fails, you can view error messages by looking at be/log/be.log or be/log/be.out. + +* View BE status + + Connect to FE using mysql-client and execute `SHOW PROC'/ backends'; `View BE operation. If everything is normal, the `isAlive`column should be `true`. + +#### (Optional) FS_Broker deployment + +Broker is deployed as a plug-in, independent of Doris. If you need to import data from a third-party storage system, you need to deploy the corresponding Broker. By default, it provides fs_broker to read HDFS and Baidu cloud BOS. Fs_broker is stateless and it is recommended that each FE and BE node deploy a Broker. + +* Copy the corresponding Broker directory in the output directory of the source fs_broker to all the nodes that need to be deployed. It is recommended to maintain the same level as the BE or FE directories. + +* Modify the corresponding Broker configuration + + In the corresponding broker/conf/directory configuration file, you can modify the corresponding configuration. + +* Start Broker + + `sh bin /start'u broker.sh --daemon ` start Broker + +* Add Broker + + To let Doris FE and BE know which nodes Broker is on, add a list of Broker nodes by SQL command. + + Use mysql-client to connect the FE started, and execute the following commands: + + `ALTER SYSTEM ADD BROKER broker_name "host1:port1","host2:port2",...;` + + The host is Broker's node ip; the port is brokeripcport in the Broker configuration file. + +* View Broker status + + Connect any booted FE using mysql-client and execute the following command to view Broker status: `SHOW PROC '/brokers';` + +**Note: In production environments, daemons should be used to start all instances to ensure that processes are automatically pulled up after they exit, such as [Supervisor] (http://supervisord.org/). For daemon startup, in 0.9.0 and previous versions, you need to modify the start_xx.sh scripts to remove the last & symbol**. Starting with version 0.10.0, call `sh start_xx.sh` directly to start. Also refer to [here] (https://www.cnblogs.com/lenmom/p/9973401.html) + +## Expansion and contraction + +Doris can easily expand and shrink FE, BE, Broker instances. + +### FE Expansion and Compression + +High availability of FE can be achieved by expanding FE to three top-one nodes. + +Users can login to Master FE through MySQL client. By: + +`SHOW PROC '/frontends';` + +To view the current FE node situation. + +You can also view the FE node through the front-end page connection: ``http://fe_hostname: fe_http_port/frontend`` or ```http://fe_hostname: fe_http_port/system? Path=//frontends```. + +All of the above methods require Doris's root user rights. + +The process of FE node expansion and contraction does not affect the current system operation. + +#### Adding FE nodes + +FE is divided into three roles: Leader, Follower and Observer. By default, a cluster can have only one Leader and multiple Followers and Observers. Leader and Follower form a Paxos selection group. If the Leader goes down, the remaining Followers will automatically select a new Leader to ensure high write availability. Observer synchronizes Leader data, but does not participate in the election. If only one FE is deployed, FE defaults to Leader. + +The first FE to start automatically becomes Leader. On this basis, several Followers and Observers can be added. + +Add Follower or Observer. Connect to the started FE using mysql-client and execute: + +`ALTER SYSTEM ADD FOLLOWER "host:port";` + +or + +`ALTER SYSTEM ADD OBSERVER "host:port";` + +The host is the node IP of Follower or Observer, and the port is edit\_log\_port in its configuration file fe.conf. + +Configure and start Follower or Observer. Follower and Observer are configured with Leader. The following commands need to be executed at the first startup: + +`./bin/start_fe.sh --helper host:port --daemon` + +The host is the node IP of Leader, and the port is edit\_log\_port in Lead's configuration file fe.conf. The --helper is only required when follower/observer is first startup. + +View the status of Follower or Observer. Connect to any booted FE using mysql-client and execute: SHOW PROC'/frontends'; you can view the FE currently joined the cluster and its corresponding roles. + +> Notes for FE expansion: +> +> 1. The number of Follower FEs (including Leaders) must be odd. It is recommended that a maximum of three constituent high availability (HA) modes be deployed. +> 2. When FE is in a highly available deployment (1 Leader, 2 Follower), we recommend that the reading service capability of FE be extended by adding Observer FE. Of course, you can continue to add Follower FE, but it's almost unnecessary. +> 3. Usually a FE node can handle 10-20 BE nodes. It is suggested that the total number of FE nodes should be less than 10. Usually three can meet most of the needs. + +#### Delete FE nodes + +Delete the corresponding FE node using the following command: + +```ALTER SYSTEM DROP FOLLOWER[OBSERVER] "fe_host:edit_log_port";``` + +> Notes for FE contraction: +> +> 1. When deleting Follower FE, make sure that the remaining Follower (including Leader) nodes are odd. + +### BE Expansion and Compression + +Users can login to Leader FE through mysql-client. By: + +```SHOW PROC '/backends';``` + +To see the current BE node situation. + +You can also view the BE node through the front-end page connection: ``http://fe_hostname: fe_http_port/backend`` or ``http://fe_hostname: fe_http_port/system? Path=//backends``. + +All of the above methods require Doris's root user rights. + +The expansion and scaling process of BE nodes does not affect the current system operation and the tasks being performed, and does not affect the performance of the current system. Data balancing is done automatically. Depending on the amount of data available in the cluster, the cluster will be restored to load balancing in a few hours to a day. For cluster load, see the [Tablet Load Balancing Document] (../administrator-guide/operation/tablet-repair-and-balance.md). + +#### Add BE nodes + +The BE node is added in the same way as in the **BE deployment** section. The BE node is added by the `ALTER SYSTEM ADD BACKEND` command. + +> Notes for BE expansion: +> +> 1. After BE expansion, Doris will automatically balance the data according to the load, without affecting the use during the period. + +#### Delete BE nodes + +There are two ways to delete BE nodes: DROP and DECOMMISSION + +The DROP statement is as follows: + +```ALTER SYSTEM DROP BACKEND "be_host:be_heartbeat_service_port";``` + +**Note: DROP BACKEND will delete the BE directly and the data on it will not be recovered!!! So we strongly do not recommend DROP BACKEND to delete BE nodes. When you use this statement, there will be corresponding error-proof operation hints.** + +DECOMMISSION clause： + +```ALTER SYSTEM DECOMMISSION BACKEND "be_host:be_heartbeat_service_port";``` + +> DECOMMISSION notes: +> +> 1. This command is used to safely delete BE nodes. After the command is issued, Doris attempts to migrate the data on the BE to other BE nodes, and when all data is migrated, Doris automatically deletes the node. +> 2. The command is an asynchronous operation. After execution, you can see that the BE node's isDecommission status is true through ``SHOW PROC '/backends';` Indicates that the node is offline. +> 3. The order **does not necessarily carry out successfully**. For example, when the remaining BE storage space is insufficient to accommodate the data on the offline BE, or when the number of remaining machines does not meet the minimum number of replicas, the command cannot be completed, and the BE will always be in the state of isDecommission as true. +> 4. The progress of DECOMMISSION can be viewed through `SHOW PROC '/backends';` Tablet Num, and if it is in progress, Tablet Num will continue to decrease. +> 5. The operation can be carried out by: +> ```CANCEL ALTER SYSTEM DECOMMISSION BACKEND "be_host:be_heartbeat_service_port";``` +> The order was cancelled. When cancelled, the data on the BE will maintain the current amount of data remaining. Follow-up Doris re-load balancing + +**For expansion and scaling of BE nodes in multi-tenant deployment environments, please refer to the [Multi-tenant Design Document] (./administrator-guide/operation/multi-tenant.md).** + +### Broker Expansion and Shrinkage + +There is no rigid requirement for the number of Broker instances. Usually one physical machine is deployed. Broker addition and deletion can be accomplished by following commands: + +```ALTER SYSTEM ADD BROKER broker_name "broker_host:broker_ipc_port";``` +```ALTER SYSTEM DROP BROKER broker_name "broker_host:broker_ipc_port";``` +```ALTER SYSTEM DROP ALL BROKER broker_name;``` + +Broker is a stateless process that can be started or stopped at will. Of course, when it stops, the job running on it will fail. Just try again. + +## Common Questions + +### Process correlation + +1. How to determine the success of FE process startup + + After the FE process starts, metadata is loaded first. According to the different roles of FE, you can see ```transfer from UNKNOWN to MASTER/FOLLOWER/OBSERVER```in the log. Eventually, you will see the ``thrift server started`` log and connect to FE through MySQL client, which indicates that FE started successfully. + + You can also check whether the startup was successful by connecting as follows: + + `http://fe_host:fe_http_port/api/bootstrap` + + If returned: + + `{"status":"OK","msg":"Success"}` + + The startup is successful, there may be problems in other cases. + + > Note: If you can't see the information of boot failure in fe. log, you may see it in fe. out. + +2. How to determine the success of BE process startup + + After the BE process starts, if there is data before, there may be several minutes of data index loading time. + + If BE is started for the first time or the BE has not joined any cluster, the BE log will periodically scroll the words `waiting to receive first heartbeat from frontend`. BE has not received Master's address through FE's heartbeat and is waiting passively. This error log will disappear after ADD BACKEND in FE sends the heartbeat. If the word `````master client', get client from cache failed. host:, port: 0, code: 7````` master client'appears again after receiving heartbeat, it indicates that FE has successfully connected BE, but BE cannot actively connect FE. It may be necessary to check the connectivity of rpc_port from BE to FE. + + If BE has been added to the cluster, the heartbeat log from FE should be scrolled every five seconds: ```get heartbeat, host:xx. xx.xx.xx, port:9020, cluster id:xxxxxxx```, indicating that the heartbeat is normal. + + Secondly, the word `finish report task success. return code: 0` should be scrolled every 10 seconds in the log to indicate that BE's communication to FE is normal. + + At the same time, if there is a data query, you should see the rolling logs, and have `execute time is xxx` logs, indicating that BE started successfully, and the query is normal. + + You can also check whether the startup was successful by connecting as follows: + + `http://be_host:be_http_port/api/health` + + If returned: + + `{"status": "OK","msg": "To Be Added"}` + + If the startup is successful, there may be problems in other cases. + + > Note: If you can't see the information of boot failure in be.INFO, you may see it in be.out. + +3. How to determine the normal connectivity of FE and BE after building the system + + Firstly, confirm that FE and BE processes have been started separately and normally, and confirm that all nodes have been added through `ADD BACKEND` or `ADD FOLLOWER/OBSERVER` statements. + + If the heartbeat is normal, BE logs will show ``get heartbeat, host:xx.xx.xx.xx, port:9020, cluster id:xxxxx`` If the heartbeat fails, the words ```backend [10001] get Exception: org.apache.thrift.transport.TTransportException``` will appear in FE's log, or other thrift communication abnormal log, indicating that the heartbeat fails from FE to 10001 BE. Here you need to check the connectivity of FE to BE host's heart-beating port. + + If BE's communication to FE is normal, the BE log will display the words `finish report task success. return code: 0`. Otherwise, the words `master client`, get client from cache failed` will appear. In this case, the connectivity of BE to the rpc_port of FE needs to be checked. + +4. Doris Node Authentication Mechanism + + In addition to Master FE, the other role nodes (Follower FE, Observer FE, Backend) need to register to the cluster through the `ALTER SYSTEM ADD` statement before joining the cluster. + + When Master FE is first started, a cluster_id is generated in the doris-meta/image/VERSION file. + + When FE first joins the cluster, it first retrieves the file from Master FE. Each subsequent reconnection between FEs (FE reboot) checks whether its cluster ID is the same as that of other existing FEs. If different, the FE will exit automatically. + + When BE first receives the heartbeat of Master FE, it gets the cluster ID from the heartbeat and records it in the `cluster_id` file of the data directory. Each heartbeat after that compares to the cluster ID sent by FE. If cluster IDs are not equal, BE will refuse to respond to FE's heartbeat. + + The heartbeat also contains Master FE's ip. When FE cuts the master, the new Master FE will carry its own IP to send the heartbeat to BE, BE will update its own saved Master FE ip. + + > **priority\_network** + > + > priority network is that both FE and BE have a configuration. Its main purpose is to assist FE or BE to identify their own IP addresses in the case of multi-network cards. Priority network is represented by CIDR: [RFC 4632] (https://tools.ietf.org/html/rfc4632) + > + > When the connectivity of FE and BE is confirmed to be normal, if the table Timeout still occurs, and the FE log has an error message with the words `backend does not find. host:xxxx.xxx.XXXX`. This means that there is a problem with the IP address that Doris automatically identifies and that priority\_network parameters need to be set manually. + > + > The main reason for this problem is that when the user adds BE through the `ADD BACKEND` statement, FE recognizes whether the statement specifies hostname or IP. If it is hostname, FE automatically converts it to an IP address and stores it in metadata. When BE reports on the completion of the task, it carries its own IP address. If FE finds that BE reports inconsistent IP addresses and metadata, it will make the above error. + > + > Solutions to this error: 1) Set **priority\_network** parameters in FE and BE respectively. Usually FE and BE are in a network segment, so this parameter can be set to the same. 2) Fill in the `ADD BACKEND` statement directly with the correct IP address of BE instead of hostname to avoid FE getting the wrong IP address. + +5. File descriptor number of BE process + + The number of file descriptor of BE process is controlled by the two parameters min_file_descriptor_number/max_file_descriptor_number. + + If it is not in the [min_file_descriptor_number, max_file_descriptor_number] interval, error will occurs when starting BE process. + + Please using ulimit command to set file descriptor under this circumstance. + + The default value of min_file_descriptor_number is 65536. + + The default value of max_file_descriptor_number is 131072. + + For Example : ulimit -n 65536; this command set file descriptor to 65536. + + After starting BE process, you can use **cat /proc/$pid/limits** to see the actual limit of process. diff --git a/docs/en/installing/upgrade.md b/docs/en/installing/upgrade.md new file mode 100644 index 00000000000000..9ff45729fa8ce9 --- /dev/null +++ b/docs/en/installing/upgrade.md @@ -0,0 +1,64 @@ +--- +{ + "title": "Cluster upgrade", + "language": "en" +} +--- + + + + +# Cluster upgrade + +Doris can upgrade smoothly by rolling upgrades. The following steps are recommended for security upgrade. + +> Note: +> 1. The following approaches are based on highly available deployments. That is, data 3 replicas, FE high availability. + +## Test the correctness of BE upgrade + +1. Arbitrarily select a BE node and deploy the latest palo_be binary file. +2. Restart the BE node and check the BE log be.INFO to see if the boot was successful. +3. If the startup fails, you can check the reason first. If the error is not recoverable, you can delete the BE directly through DROP BACKEND, clean up the data, and restart the BE using the previous version of palo_be. Then re-ADD BACKEND. (**This method will result in the loss of a copy of the data, please make sure that three copies are complete, and perform this operation!!!** + +## Testing FE Metadata Compatibility + +0. **Important! Exceptional metadata compatibility is likely to cause data can not be restored!!** +1. Deploy a test FE process (such as your own local developer) using the new version alone. +2. Modify the FE configuration file fe.conf for testing and set all ports to **different from online**. +3. Add configuration in fe.conf: cluster_id=123456 +4. Add the configuration in fe.conf: metadatafailure_recovery=true +5. Copy the metadata directory palo-meta of the online environment Master FE to the test environment +6. Modify the cluster_id in the palo-meta/image/VERSION file copied into the test environment to 123456 (that is, the same as in Step 3) +7. "27979;" "35797;" "3681616;" sh bin /start fe.sh "21551;" FE +8. Observe whether the start-up is successful through FE log fe.log. +9. If the startup is successful, run sh bin/stop_fe.sh to stop the FE process of the test environment. +10. **The purpose of the above 2-6 steps is to prevent the FE of the test environment from being misconnected to the online environment after it starts.** + +## Upgrade preparation + +1. After data validation, the new version of BE and FE binary files are distributed to their respective directories. +2. Usually small version upgrade, BE only needs to upgrade palo_be; FE only needs to upgrade palo-fe.jar. If it is a large version upgrade, you may need to upgrade other files (including but not limited to bin / lib / etc.) If you are not sure whether you need to replace other files, it is recommended to replace all of them. + +## rolling upgrade + +1. Confirm that the new version of the file is deployed. Restart FE and BE instances one by one. +2. It is suggested that BE be restarted one by one and FE be restarted one by one. Because Doris usually guarantees backward compatibility between FE and BE, that is, the old version of FE can access the new version of BE. However, the old version of BE may not be supported to access the new version of FE. +3. It is recommended to restart the next instance after confirming that the previous instance started successfully. Refer to the Installation Deployment Document for the identification of successful instance startup. diff --git a/docs/en/internal/doris_storage_optimization.md b/docs/en/internal/doris_storage_optimization.md new file mode 100644 index 00000000000000..c62b6fa00689a2 --- /dev/null +++ b/docs/en/internal/doris_storage_optimization.md @@ -0,0 +1,233 @@ +--- +{ + "title": "Doris Storage File Format Optimization", + "language": "en" +} +--- + + + + +# Doris Storage File Format Optimization # + +## File format ## + +![](/images/segment_v2.png) +
1. doris segment
+ +Documents include: +- The file starts with an 8-byte magic code to identify the file format and version +- Data Region: Used to store data information for each column, where the data is loaded on demand by pages. +- Index Region: Doris stores the index data of each column in Index Region, where the data is loaded according to column granularity, so the data information of the following column is stored separately. +- Footer信息 + - FileFooterPB: Metadata Information for Definition Files + - Chesum of 4 bytes of footer Pb content + - Four bytes FileFooterPB message length for reading FileFooterPB + - The 8 byte MAGIC CODE is stored in the last bit to facilitate the identification of file types in different scenarios. + +The data in the file is organized in the form of page, which is the basic unit of coding and compression. Current page types include the following: + +### DataPage ### + +Data Page is divided into two types: nullable and non-nullable data pages. + +Nullable's data page includes: +``` + + +----------------+ + | value count | + |----------------| + | first row id | + |----------------| + | bitmap length | + |----------------| + | null bitmap | + |----------------| + | data | + |----------------| + | checksum | + +----------------+ +``` + +non -zero data page32467;- 26500;- 229140;- + +``` + |----------------| + | value count | + |----------------| + | first row id | + |----------------| + | data | + |----------------| + | checksum | + +----------------+ +``` + +The meanings of each field are as follows: + +- value count + - Represents the number of rows in a page +- First row id + - Line number of the first line in page +- bitmap length + - Represents the number of bytes in the next bitmap +- null bitmap + - bitmap representing null information +- Data + - Store data after encoding and compress + - You need to write in the header information of the data: is_compressed + - Various kinds of data encoded by different codes need to write some field information in the header information in order to achieve data parsing. + - TODO: Add header information for various encodings +- Checksum + - Store page granularity checksum, including page header and subsequent actual data + + +### Bloom Filter Pages ### + +For each bloom filter column, a page of the bloom filter is generated corresponding to the granularity of the page and saved in the bloom filter pages area. + +### Ordinal Index Page ### + +For each column, a sparse index of row numbers is established according to page granularity. The content is a pointer to the block (including offset and length) for the line number of the start line of the page + +### Short Key Index page ### + +We generate a sparse index of short key every N rows (configurable) with the contents of short key - > line number (ordinal) + +### Column's other indexes### + +The format design supports the subsequent expansion of other index information, such as bitmap index, spatial index, etc. It only needs to write the required data to the existing column data, and add the corresponding metadata fields to FileFooterPB. + +### Metadata Definition### +FileFooterPB is defined as: + +``` +message ColumnPB { + optional uint32 column_id = 1; // 这里使用column id，不使用column name是因为计划支持修改列名 + optional string type = 2; // 列类型 + optional string aggregation = 3; // 是否聚合 + optional uint32 length = 4; // 长度 + optional bool is_key = 5; // 是否是主键列 + optional string default_value = 6; // 默认值 + optional uint32 precision = 9 [default = 27]; // 精度 + optional uint32 frac = 10 [default = 9]; + optional bool is_nullable = 11 [default=false]; // 是否有null + optional bool is_bf_column = 15 [default=false]; // 是否有bf词典 + optional bool is_bitmap_column = 16 [default=false]; // 是否有bitmap索引 +} + +// page偏移 +message PagePointerPB { + required uint64 offset; // page在文件中的偏移 + required uint32 length; // page的大小 +} + +message MetadataPairPB { + optional string key = 1; + optional bytes value = 2; +} + +message ColumnMetaPB { + optional ColumnMessage encoding; // 编码方式 + + optional PagePointerPB dict_page // 词典page + repeated PagePointerPB bloom_filter_pages; // bloom filter词典信息 + optional PagePointerPB ordinal_index_page; // 行号索引数据 + optional PagePointerPB page_zone_map_page; // page级别统计信息索引数据 + + optional PagePointerPB bitmap_index_page; // bitmap索引数据 + + optional uint64 data_footprint; // 列中索引的大小 + optional uint64 index_footprint; // 列中数据的大小 + optional uint64 raw_data_footprint; // 原始列数据大小 + + optional CompressKind compress_kind; // 列的压缩方式 + + optional ZoneMapPB column_zone_map; //文件级别的过滤条件 + repeated MetadataPairPB column_meta_datas; +} + +message FileFooterPB { + optional uint32 version = 2 [default = 1]; // 用于版本兼容和升级使用 + repeated ColumnPB schema = 5; // 列Schema + optional uint64 num_values = 4; // 文件中保存的行数 + optional uint64 index_footprint = 7; // 索引大小 + optional uint64 data_footprint = 8; // 数据大小 + optional uint64 raw_data_footprint = 8; // 原始数据大小 + + optional CompressKind compress_kind = 9 [default = COMPRESS_LZO]; // 压缩方式 + repeated ColumnMetaPB column_metas = 10; // 列元数据 + optional PagePointerPB key_index_page; // short key索引page +} + +``` + +## Read-write logic## + +### Write ### + +The general writing process is as follows: +1. Write magic +2. Generate corresponding Column Writer according to schema information. Each Column Writer obtains corresponding encoding information (configurable) according to different types, and generates corresponding encoder according to encoding. +3. Call encoder - > add (value) for data writing. Each K line generates a short key index entry, and if the current page satisfies certain conditions (the size exceeds 1M or the number of rows is K), a new page is generated and cached in memory. +4. Continuous cycle step 3 until data writing is completed. Brush the data of each column into the file in sequence +5. Generate FileFooterPB information and write it to the file. + +Relevant issues: + +- How does the index of short key be generated? + - Now we still generate a short key sparse index according to how many rows are sparse, and keep a short sparse index generated every 1024 rows. The specific content is: short key - > ordinal + +- What should be stored in the ordinal index? + - Store the first ordinal to page pointer mapping information for pages +- What are stored in pages of different encoding types? + - Dictionary Compression + - plain + - rle + - bshuf + +### Read### + +1. Read the magic of the file and judge the type and version of the file. +2. Read FileFooterPB and check sum +3. Read short key index and data ordinal index information of corresponding columns according to required columns +4. Use start key and end key, locate the row number to be read through short key index, then determine the row ranges to be read through ordinal index, and filter the row ranges to be read through statistics, bitmap index and so on. +5. Then read row data through ordinal index according to row ranges + +Relevant issues: +1. How to quickly locate a row within the page? + + The data inside the page is encoding, so it can not locate the row-level data quickly. Different encoding methods have different schemes for fast line number positioning in-house, which need to be analyzed concretely: + - If it is rle-coded, skip is performed by resolving the head of RLE until the RLE block containing the row is reached, and then the reverse solution is performed. + - binary plain encoding: offset information will be stored in the page, and offset information will be specified in the page header. When reading, offset information will be parsed into the array first, so that you can quickly locate the data of a row of block through offset data information of each row. +2. How to achieve efficient block reading? Consider merging adjacent blocks while they are being read, one-time reading? +This requires judging whether the block is continuous at the time of reading, and if it is continuous, reading it once. + +## Coding## + +In the existing Doris storage, plain encoding is adopted for string type encoding, which is inefficient. After comparison, it is found that in Baidu statistics scenario, data will expand more than twice because of string type coding. Therefore, it is planned to introduce dictionary-based coding compression. + +## Compression## + +It implements a scalable compression framework, supports a variety of compression algorithms, facilitates the subsequent addition of new compression algorithms, and plans to introduce zstd compression. + +## TODO ## +1. How to implement nested types? How to locate line numbers in nested types? +2. How to optimize the downstream bitmap and column statistics statistics caused by ScanRange splitting? diff --git a/docs/en/internal/grouping_sets_design.md b/docs/en/internal/grouping_sets_design.md new file mode 100644 index 00000000000000..e680362a2e7324 --- /dev/null +++ b/docs/en/internal/grouping_sets_design.md @@ -0,0 +1,501 @@ +--- +{ + "title": "GROUPING SETS DESIGN", + "language": "en" +} +--- + + +# GROUPING SETS DESIGN + +## 1. GROUPING SETS Background + +The `CUBE`, `ROLLUP`, and `GROUPING` `SETS` extensions to SQL make querying and reporting easier and faster. `CUBE`, `ROLLUP`, and grouping sets produce a single result set that is equivalent to a `UNION` `ALL` of differently grouped rows. `ROLLUP` calculates aggregations such as `SUM`, `COUNT`, `MAX`, `MIN`, and `AVG` at increasing levels of aggregation, from the most detailed up to a grand total. `CUBE` is an extension similar to `ROLLUP`, enabling a single statement to calculate all possible combinations of aggregations. The `CUBE`, `ROLLUP`, and the `GROUPING` `SETS` extension lets you specify just the groupings needed in the `GROUP` `BY` clause. This allows efficient analysis across multiple dimensions without performing a `CUBE` operation. Computing a `CUBE` creates a heavy processing load, so replacing cubes with grouping sets can significantly increase performance. +To enhance performance, `CUBE`, `ROLLUP`, and `GROUPING SETS` can be parallelized: multiple processes can simultaneously execute all of these statements. These capabilities make aggregate calculations more efficient, thereby enhancing database performance, and scalability. + +The three `GROUPING` functions help you identify the group each row belongs to and enable sorting subtotal rows and filtering results. + +### 1.1 GROUPING SETS Syntax + +`GROUPING SETS` syntax lets you define multiple groupings in the same query. `GROUP BY` computes all the groupings specified and combines them with `UNION ALL`. For example, consider the following statement: + +``` +SELECT k1, k2, SUM( k3 ) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k1), (k2), ( ) ); +``` + + +This statement is equivalent to: + +``` +SELECT k1, k2, SUM( k3 ) FROM t GROUP BY k1, k2 +UNION +SELECT k1, null, SUM( k3 ) FROM t GROUP BY k1 +UNION +SELECT null, k2, SUM( k3 ) FROM t GROUP BY k2 +UNION +SELECT null, null, SUM( k3 ) FROM t +``` + +This is an example of real query： + +``` +mysql> SELECT * FROM t; ++------+------+------+ +| k1 | k2 | k3 | ++------+------+------+ +| a | A | 1 | +| a | A | 2 | +| a | B | 1 | +| a | B | 3 | +| b | A | 1 | +| b | A | 4 | +| b | B | 1 | +| b | B | 5 | ++------+------+------+ +8 rows in set (0.01 sec) + +mysql> SELECT k1, k2, SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); ++------+------+-----------+ +| k1 | k2 | sum(`k3`) | ++------+------+-----------+ +| b | B | 6 | +| a | B | 4 | +| a | A | 3 | +| b | A | 5 | +| NULL | B | 10 | +| NULL | A | 8 | +| a | NULL | 7 | +| b | NULL | 11 | +| NULL | NULL | 18 | ++------+------+-----------+ +9 rows in set (0.06 sec) +``` + +### 1.2 ROLLUP Syntax + +`ROLLUP` enables a `SELECT` statement to calculate multiple levels of subtotals across a specified group of dimensions. It also calculates a grand total. `ROLLUP` is a simple extension to the `GROUP` `BY` clause, so its syntax is extremely easy to use. The `ROLLUP` extension is highly efficient, adding minimal overhead to a query. + +`ROLLUP` appears in the `GROUP` `BY` clause in a `SELECT` statement. Its form is: + +``` +SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY ROLLUP(a,b,c) +``` + +This statement is equivalent to GROUPING SETS as followed： + +``` +GROUPING SETS ( +(a,b,c), +( a, b ), +( a), +( ) +) +``` + +### 1.3 CUBE Syntax + +Like `ROLLUP` `CUBE` generates all the subtotals that could be calculated for a data cube with the specified dimensions. + +``` +SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY CUBE(a,b,c) +``` + +e.g. CUBE ( a, b, c ) is equivalent to GROUPING SETS as followed: + +``` +GROUPING SETS ( +( a, b, c ), +( a, b ), +( a, c ), +( a ), +( b, c ), +( b ), +( c ), +( ) +) +``` + +### 1.4 GROUPING and GROUPING_ID Function + +Indicates whether a specified column expression in a `GROUP BY` list is aggregated or not. `GROUPING `returns 1 for aggregated or 0 for not aggregated in the result set. `GROUPING` can be used only in the `SELECT` list, `HAVING`, and `ORDER BY` clauses when `GROUP BY` is specified. + +`GROUPING_ID` describes which of a list of expressions are grouped in a row produced by a `GROUP BY` query. The `GROUPING_ID` function simply returns the decimal equivalent of the binary value formed as a result of the concatenation of the values returned by the `GROUPING` functions. + +Each `GROUPING_ID` argument must be an element of the `GROUP BY` list. `GROUPING_ID ()` returns an **integer** bitmap whose lowest N bits may be lit. A lit **bit** indicates the corresponding argument is not a grouping column for the given output row. The lowest-order **bit** corresponds to argument N, and the N-1th lowest-order **bit** corresponds to argument 1. If the column is a grouping column the bit is 0 else is 1. + +For example： + +``` +mysql> select * from t; ++------+------+------+ +| k1 | k2 | k3 | ++------+------+------+ +| a | A | 1 | +| a | A | 2 | +| a | B | 1 | +| a | B | 3 | +| b | A | 1 | +| b | A | 4 | +| b | B | 1 | +| b | B | 5 | ++------+------+------+ +``` + +grouping sets result： + +``` +mysql> SELECT k1, k2, GROUPING(k1), GROUPING(k2), SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); ++------+------+----------------+----------------+-----------+ +| k1 | k2 | grouping(`k1`) | grouping(`k2`) | sum(`k3`) | ++------+------+----------------+----------------+-----------+ +| a | A | 0 | 0 | 3 | +| a | B | 0 | 0 | 4 | +| a | NULL | 0 | 1 | 7 | +| b | A | 0 | 0 | 5 | +| b | B | 0 | 0 | 6 | +| b | NULL | 0 | 1 | 11 | +| NULL | A | 1 | 0 | 8 | +| NULL | B | 1 | 0 | 10 | +| NULL | NULL | 1 | 1 | 18 | ++------+------+----------------+----------------+-----------+ +9 rows in set (0.02 sec) + +mysql> SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); ++------+------+-------------------------+-----------+ +| k1 | k2 | grouping_id(`k1`, `k2`) | sum(`k3`) | ++------+------+-------------------------+-----------+ +| a | A | 0 | 3 | +| a | B | 0 | 4 | +| a | NULL | 1 | 7 | +| b | A | 0 | 5 | +| b | B | 0 | 6 | +| b | NULL | 1 | 11 | +| NULL | A | 2 | 8 | +| NULL | B | 2 | 10 | +| NULL | NULL | 3 | 18 | ++------+------+-------------------------+-----------+ +9 rows in set (0.02 sec) + +mysql> SELECT k1, k2, grouping(k1), grouping(k2), GROUPING_ID(k1,k2), SUM(k4) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ) order by k1, k2; ++------+------+----------------+----------------+-------------------------+-----------+ +| k1 | k2 | grouping(`k1`) | grouping(`k2`) | grouping_id(`k1`, `k2`) | sum(`k4`) | ++------+------+----------------+----------------+-------------------------+-----------+ +| a | A | 0 | 0 | 0 | 3 | +| a | B | 0 | 0 | 0 | 4 | +| a | NULL | 0 | 1 | 1 | 7 | +| b | A | 0 | 0 | 0 | 5 | +| b | B | 0 | 0 | 0 | 6 | +| b | NULL | 0 | 1 | 1 | 11 | +| NULL | A | 1 | 0 | 2 | 8 | +| NULL | B | 1 | 0 | 2 | 10 | +| NULL | NULL | 1 | 1 | 3 | 18 | ++------+------+----------------+----------------+-------------------------+-----------+ +9 rows in set (0.02 sec) + +``` +### 1.5 Composition and nesting of GROUPING SETS + +First of all, a GROUP BY clause is essentially a special case of GROUPING SETS, for example: + +``` + GROUP BY a +is equivalent to: + GROUP BY GROUPING SETS((a)) +also， + GROUP BY a,b,c +is equivalent to: + GROUP BY GROUPING SETS((a,b,c)) +``` + +Similarly, CUBE and ROLLUP can be expanded into GROUPING SETS, so the various combinations and nesting of GROUP BY, CUBE, ROLLUP, GROUPING SETS are essentially the combination and nesting of GROUPING SETS. + +For GROUPING SETS nesting, it is semantically equivalent to writing the statements inside the nest directly outside. (ref:) mentions: + +``` +The CUBE and ROLLUP constructs can be used either directly in the GROUP BY clause, or nested inside a GROUPING SETS clause. If one GROUPING SETS clause is nested inside another, the effect is the same as if all the elements of the inner clause had been written directly in the outer clause. +``` + +For a combined list of multiple GROUPING SETS, many databases consider it a cross product relationship. + +for example: + +``` +GROUP BY a, CUBE (b, c), GROUPING SETS ((d), (e)) + +is equivalent to: + +GROUP BY GROUPING SETS ( +(a, b, c, d), (a, b, c, e), +(a, b, d), (a, b, e), +(a, c, d), (a, c, e), +(a, d), (a, e) +) +``` + +For the combination and nesting of GROUPING SETS, each database support is not the same. For example snowflake does not support any combination and nesting. +（） + +Oracle supports both composition and nesting. +（） + +Presto supports composition, but not nesting. +（） + +## 2. Object + +Support `GROUPING SETS`， `ROLLUP` and `CUBE ` syntax，impliments 1.1, 1.2, 1.3 1.4, 1.5, not support the combination + and nesting of GROUPING SETS at current version. + +### 2.1 GROUPING SETS Syntax + +``` +SELECT ... +FROM ... +[ ... ] +GROUP BY GROUPING SETS ( groupSet [ , groupSet [ , ... ] ] ) +[ ... ] + +groupSet ::= { ( expr [ , expr [ , ... ] ] )} + + +Expression，column name. +``` + +### 2.2 ROLLUP Syntax + +``` +SELECT ... +FROM ... +[ ... ] +GROUP BY ROLLUP ( expr [ , expr [ , ... ] ] ) +[ ... ] + + +Expression，column name. +``` + +### 2.3 CUBE Syntax + +``` +SELECT ... +FROM ... +[ ... ] +GROUP BY CUBE ( expr [ , expr [ , ... ] ] ) +[ ... ] + + +Expression，column name. +``` + +## 3. Implementation + +### 3.1 Overall Design Approaches + +For `GROUPING SET` is equivalent to the `UNION` of `GROUP BY` . So we can expand input rows, and run an GROUP BY on these rows。 + +For example： + +``` +SELECT a, b FROM src GROUP BY a, b GROUPING SETS ((a, b), (a), (b), ()); +``` + +Data in table src : + +``` +1, 2 +3, 4 +``` + +Base on GROUPING SETS , we can expend the input to： + +``` +1, 2 (GROUPING_ID: a, b -> 00 -> 0) +1, null (GROUPING_ID: a, null -> 01 -> 1) +null, 2 (GROUPING_ID: null, b -> 10 -> 2) +null, null (GROUPING_ID: null, null -> 11 -> 3) + +3, 4 (GROUPING_ID: a, b -> 00 -> 0) +3, null (GROUPING_ID: a, null -> 01 -> 1) +null, 4 (GROUPING_ID: null, b -> 10 -> 2) +null, null (GROUPING_ID: null, null -> 11 -> 3) +``` + +And then use those row as input, then GROUP BY a, b, GROUPING_ID + +### 3.2 Example + +Table t： + +``` +mysql> select * from t; ++------+------+------+ +| k1 | k2 | k3 | ++------+------+------+ +| a | A | 1 | +| a | A | 2 | +| a | B | 1 | +| a | B | 3 | +| b | A | 1 | +| b | A | 4 | +| b | B | 1 | +| b | B | 5 | ++------+------+------+ +8 rows in set (0.01 sec) +``` + +for the query： + +``` +SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ()); +``` + +First，expand the input，every row expand into 4 rows ( the size of GROUPING SETS), and insert GROUPING_ID column + +e.g. a, A, 1 expanded to： + +``` ++------+------+------+-------------------------+ +| k1 | k2 | k3 | GROUPING_ID(`k1`, `k2`) | ++------+------+------+-------------------------+ +| a | A | 1 | 0 | +| a | NULL | 1 | 1 | +| NULL | A | 1 | 2 | +| NULL | NULL | 1 | 3 | ++------+------+------+-------------------------+ +``` + +Finally, all rows expended as follows (32 rows): + +``` ++------+------+------+-------------------------+ +| k1 | k2 | k3 | GROUPING_ID(`k1`, `k2`) | ++------+------+------+-------------------------+ +| a | A | 1 | 0 | +| a | A | 2 | 0 | +| a | B | 1 | 0 | +| a | B | 3 | 0 | +| b | A | 1 | 0 | +| b | A | 4 | 0 | +| b | B | 1 | 0 | +| b | B | 5 | 0 | +| a | NULL | 1 | 1 | +| a | NULL | 1 | 1 | +| a | NULL | 2 | 1 | +| a | NULL | 3 | 1 | +| b | NULL | 1 | 1 | +| b | NULL | 1 | 1 | +| b | NULL | 4 | 1 | +| b | NULL | 5 | 1 | +| NULL | A | 1 | 2 | +| NULL | A | 1 | 2 | +| NULL | A | 2 | 2 | +| NULL | A | 4 | 2 | +| NULL | B | 1 | 2 | +| NULL | B | 1 | 2 | +| NULL | B | 3 | 2 | +| NULL | B | 5 | 2 | +| NULL | NULL | 1 | 3 | +| NULL | NULL | 1 | 3 | +| NULL | NULL | 1 | 3 | +| NULL | NULL | 1 | 3 | +| NULL | NULL | 2 | 3 | +| NULL | NULL | 3 | 3 | +| NULL | NULL | 4 | 3 | +| NULL | NULL | 5 | 3 | ++------+------+------+-------------------------+ +32 rows in set. +``` + +now GROUP BY k1, k2, GROUPING_ID(k1,k2): + +``` ++------+------+-------------------------+-----------+ +| k1 | k2 | grouping_id(`k1`, `k2`) | sum(`k3`) | ++------+------+-------------------------+-----------+ +| a | A | 0 | 3 | +| a | B | 0 | 4 | +| a | NULL | 1 | 7 | +| b | A | 0 | 5 | +| b | B | 0 | 6 | +| b | NULL | 1 | 11 | +| NULL | A | 2 | 8 | +| NULL | B | 2 | 10 | +| NULL | NULL | 3 | 18 | ++------+------+-------------------------+-----------+ +9 rows in set (0.02 sec) +``` + +The result is equivalent to the UNION ALL + +``` +select k1, k2, sum(k3) from t group by k1, k2 +UNION ALL +select NULL, k2, sum(k3) from t group by k2 +UNION ALL +select k1, NULL, sum(k3) from t group by k1 +UNION ALL +select NULL, NULL, sum(k3) from t; + ++------+------+-----------+ +| k1 | k2 | sum(`k3`) | ++------+------+-----------+ +| b | B | 6 | +| b | A | 5 | +| a | A | 3 | +| a | B | 4 | +| a | NULL | 7 | +| b | NULL | 11 | +| NULL | B | 10 | +| NULL | A | 8 | +| NULL | NULL | 18 | ++------+------+-----------+ +9 rows in set (0.06 sec) +``` + +### 3.3 FE + +#### 3.3.1 Tasks + +1. Add GroupByClause, repalce groupingExprs. +2. Add Grouping Sets, Cube and RollUp syntax. +3. Add GroupByClause in SelectStmt. +4. Add GroupingFunctionCallExpr, impliments grouping grouping_id function call +5. Add VirtualSlot, generate the map of virtual slots and real slots +6. add virtual column GROUPING_ID and other virtual columns generated by grouping and grouping_id, insert into groupingExprs, +7. Add a PlanNode, name as RepeatNode. For GroupingSets aggregation insert RepeatNode to the plan. + +#### 3.3.2 Tuple + +In order to add GROUPING_ID to groupingExprs in GroupByClause, need to create virtual SlotRef, also, need tot create a tuple for this slot, named GROUPING\_\_ID Tuple. + +For the plannode RepeatNode, it's input is all the tuple of it's children, It's output tuple is the repeat data and GROUPING_ID. + + +#### 3.3.3 Expression and Function Substitution + +expr -> if(bitand(pos, grouping_id)=0, expr, null) for expr in extension grouping clause +grouping_id() -> grouping_id(grouping_id) for grouping_id function + +### 3.4 BE + +#### 3.4.1 Tasks + +1. Add RepeatNode executor, expend the input data and append GROUPING_ID to every row +2. Implements grouping_id() and grouping() function. \ No newline at end of file diff --git a/docs/en/internal/metadata-design.md b/docs/en/internal/metadata-design.md new file mode 100644 index 00000000000000..d02d5d3f7b8534 --- /dev/null +++ b/docs/en/internal/metadata-design.md @@ -0,0 +1,127 @@ +--- +{ + "title": "Metadata Design Document", + "language": "en" +} +--- + + + + +# Metadata Design Document + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Mainly responsible for receiving and returning client requests, metadata, cluster management, query plan generation and so on. +* BE: Backend, the back-end node of Doris. Mainly responsible for data storage and management, query plan execution and other work. +* bdbje: [Oracle Berkeley DB Java Edition] (http://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html). In Doris, we use bdbje to persist metadata operation logs and high availability of FE. + +## Overall architecture +![](/images/palo_architecture.jpg) + +As shown above, Doris's overall architecture is divided into two layers. Multiple FEs form the first tier, providing lateral expansion and high availability of FE. Multiple BEs form the second layer, which is responsible for data storage and management. This paper mainly introduces the design and implementation of metadata in FE layer. + +1. FE 节点分为 follower 和 observer 两类。各个 FE 之间，通过 bdbje（[BerkeleyDB Java Edition](http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/overview/index-093405.html)）进行 leader 选举，数据同步等工作。 + +2. The follower node is elected, and one of the followers becomes the leader node, which is responsible for the writing of metadata. When the leader node goes down, other follower nodes re-elect a leader to ensure high availability of services. + +3. The observer node only synchronizes metadata from the leader node and does not participate in the election. It can be scaled horizontally to provide the extensibility of metadata reading services. + +> Note: The concepts of follower and observer corresponding to bdbje are replica and observer. You may use both names below. + +## Metadata structure + +Doris's metadata is in full memory. A complete metadata image is maintained in each FE memory. Within Baidu, a cluster of 2,500 tables and 1 million fragments (3 million copies) occupies only about 2GB of metadata in memory. (Of course, the memory overhead for querying intermediate objects and various job information needs to be estimated according to the actual situation. However, it still maintains a low memory overhead. + +At the same time, metadata is stored in the memory as a whole in a tree-like hierarchical structure. By adding auxiliary structure, metadata information at all levels can be accessed quickly. + +The following figure shows the contents stored in Doris meta-information. + +![](/images/metadata_contents.png) + +As shown above, Doris's metadata mainly stores four types of data: + +1. User data information. Including database, table Schema, fragmentation information, etc. +2. All kinds of job information. For example, import jobs, Clone jobs, SchemaChange jobs, etc. +3. User and permission information. +4. Cluster and node information. + +## Data stream + +![](/images/metadata_stream.png) + +The data flow of metadata is as follows: + +1. Only leader FE can write metadata. After modifying leader's memory, the write operation serializes into a log and writes to bdbje in the form of key-value. The key is a continuous integer, and as log id, value is the serialized operation log. + +2. After the log is written to bdbje, bdbje copies the log to other non-leader FE nodes according to the policy (write most/write all). The non-leader FE node modifies its metadata memory image by playback of the log, and completes the synchronization with the metadata of the leader node. + +3. When the number of log bars of the leader node reaches the threshold (default 10W bars), the checkpoint thread is started. Checkpoint reads existing image files and subsequent logs and replays a new mirror copy of metadata in memory. The copy is then written to disk to form a new image. The reason for this is to regenerate a mirror copy instead of writing an existing image to an image, mainly considering that the write operation will be blocked during writing the image plus read lock. So every checkpoint takes up twice as much memory space. + +4. After the image file is generated, the leader node notifies other non-leader nodes that a new image has been generated. Non-leader actively pulls the latest image files through HTTP to replace the old local files. + +5. The logs in bdbje will be deleted regularly after the image is completed. + +## Implementation details + +### Metadata catalogue + +1. The metadata directory is specified by the FE configuration item `meta_dir'. + +2. Data storage directory for bdbje under `bdb/` directory. + +3. The storage directory for image files under the `image/` directory. + +* `Image.[logid]`is the latest image file. The suffix `logid` indicates the ID of the last log contained in the image. +* `Image.ckpt` is the image file being written. If it is successfully written, it will be renamed `image.[logid]` and replaced with the original image file. +* The`cluster_id` is recorded in the `VERSION` file. `Cluster_id` uniquely identifies a Doris cluster. It is a 32-bit integer randomly generated at the first startup of leader. You can also specify a cluster ID through the Fe configuration item `cluster_id'. +* The role of FE itself recorded in the `ROLE` file. There are only `FOLLOWER` and `OBSERVER`. Where `FOLLOWER` denotes FE as an optional node. (Note: Even the leader node has a role of `FOLLOWER`) + +### Start-up process + +1. FE starts for the first time. If the startup script does not add any parameters, it will try to start as leader. You will eventually see `transfer from UNKNOWN to MASTER` in the FE startup log. + +2. FE starts for the first time. If the `-helper` parameter is specified in the startup script and points to the correct leader FE node, the FE first asks the leader node about its role (ROLE) and cluster_id through http. Then pull up the latest image file. After reading image file and generating metadata image, start bdbje and start bdbje log synchronization. After synchronization is completed, the log after image file in bdbje is replayed, and the final metadata image generation is completed. + + > Note 1: When starting with the `-helper` parameter, you need to first add the FE through the leader through the MySQL command, otherwise, the start will report an error. + + > Note 2: `-helper` can point to any follower node, even if it is not leader. + + > Note 3: In the process of synchronization log, the Fe log will show `xxx detached`. At this time, the log pull is in progress, which is a normal phenomenon. + +3. FE is not the first startup. If the startup script does not add any parameters, it will determine its identity according to the ROLE information stored locally. At the same time, according to the cluster information stored in the local bdbje, the leader information is obtained. Then read the local image file and the log in bdbje to complete the metadata image generation. (If the roles recorded in the local ROLE are inconsistent with those recorded in bdbje, an error will be reported.) + +4. FE is not the first boot, and the `-helper` parameter is specified in the boot script. Just like the first process started, the leader role is asked first. But it will be compared with the ROLE stored by itself. If they are inconsistent, they will report errors. + +#### Metadata Read-Write and Synchronization + +1. Users can use Mysql to connect any FE node to read and write metadata. If the connection is a non-leader node, the node forwards the write operation to the leader node. When the leader is successfully written, it returns a current and up-to-date log ID of the leader. Later, the non-leader node waits for the log ID it replays to be larger than the log ID it returns to the client before returning the message that the command succeeds. This approach guarantees Read-Your-Write semantics for any FE node. + + > Note: Some non-write operations are also forwarded to leader for execution. For example, `SHOW LOAD` operation. Because these commands usually need to read the intermediate states of some jobs, which are not written to bdbje, there are no such intermediate states in the memory of the non-leader node. (FE's direct metadata synchronization depends entirely on bdbje's log playback. If a metadata modification operation does not write bdbje's log, the result of the modification of the operation will not be seen in other non-leader nodes.) + +2. The leader node starts a TimePrinter thread. This thread periodically writes a key-value entry for the current time to bdbje. The remaining non-leader nodes read the recorded time in the log by playback and compare it with the local time. If the lag between the local time and the local time is found to be greater than the specified threshold (configuration item: `meta_delay_toleration_second`). If the write interval is half of the configuration item, the node will be in the **unreadable** state. This mechanism solves the problem that non-leader nodes still provide outdated metadata services after a long time of leader disconnection. + +3. The metadata of each FE only guarantees the final consistency. Normally, inconsistent window periods are only milliseconds. We guarantee the monotonous consistency of metadata access in the same session. But if the same client connects different FEs, metadata regression may occur. (But for batch update systems, this problem has little impact.) + +### Downtime recovery + +1. When the leader node goes down, the rest of the followers will immediately elect a new leader node to provide services. +2. Metadata cannot be written when most follower nodes are down. When metadata is not writable, if a write operation request occurs, the current process is that the **FE process exits**. This logic will be optimized in the future, and read services will still be provided in the non-writable state. +3. The downtime of observer node will not affect the state of any other node. It also does not affect metadata reading and writing at other nodes. diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/avg.md b/docs/en/sql-reference/sql-functions/aggregate-functions/avg.md new file mode 100644 index 00000000000000..315e8e2f7eb81e --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/avg.md @@ -0,0 +1,58 @@ +--- +{ + "title": "AVG", + "language": "en" +} +--- + + + + +#AVG +## Description +### Syntax + +`AVG([DISTINCT] expr)` + + +Used to return the average value of the selected field + +Optional field DISTINCT parameters can be used to return the weighted average + +## example + +``` +mysql> SELECT datetime, AVG(cost_time) FROM log_statis group by datetime; ++---------------------+--------------------+ +| datetime | avg(`cost_time`) | ++---------------------+--------------------+ +| 2019-07-03 21:01:20 | 25.827794561933533 | ++---------------------+--------------------+ + +mysql> SELECT datetime, AVG(distinct cost_time) FROM log_statis group by datetime; ++---------------------+---------------------------+ +| datetime | avg(DISTINCT `cost_time`) | ++---------------------+---------------------------+ +| 2019-07-04 02:23:24 | 20.666666666666668 | ++---------------------+---------------------------+ + +``` +##keyword +AVG diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/bitmap.md b/docs/en/sql-reference/sql-functions/aggregate-functions/bitmap.md new file mode 100644 index 00000000000000..a53488be34032a --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/bitmap.md @@ -0,0 +1,146 @@ +--- +{ + "title": "BITMAP", + "language": "en" +} +--- + + + + +# BITMAP + +## Create table + +The aggregation model needs to be used when creating the table. The data type is bitmap and the aggregation function is bitmap_union. +``` +CREATE TABLE `pv_bitmap` ( +  `dt` int (11) NULL COMMENT" ", +  `page` varchar (10) NULL COMMENT" ", +  `user_id` bitmap BITMAP_UNION NULL COMMENT" " +) ENGINE = OLAP +AGGREGATE KEY (`dt`,` page`) +COMMENT "OLAP" +DISTRIBUTED BY HASH (`dt`) BUCKETS 2; +``` + +Note: When the amount of data is large, it is best to create a corresponding rollup table for high-frequency bitmap_union queries + +``` +ALTER TABLE pv_bitmap ADD ROLLUP pv (page, user_id); +``` + +## Data Load + +`TO_BITMAP (expr)`: Convert 0 ~ 18446744073709551615 unsigned bigint to bitmap + +`BITMAP_EMPTY ()`: Generate empty bitmap columns, used for insert or import to fill the default value + +`BITMAP_HASH (expr)`: Convert any type of column to a bitmap by hashing + +### Stream Load + +``` +cat data | curl --location-trusted -u user: passwd -T--H "columns: dt, page, user_id, user_id = to_bitmap (user_id)" http: // host: 8410 / api / test / testDb / _stream_load +``` + +``` +cat data | curl --location-trusted -u user: passwd -T--H "columns: dt, page, user_id, user_id = bitmap_hash (user_id)" http: // host: 8410 / api / test / testDb / _stream_load +``` + +``` +cat data | curl --location-trusted -u user: passwd -T--H "columns: dt, page, user_id, user_id = bitmap_empty ()" http: // host: 8410 / api / test / testDb / _stream_load +``` + +### Insert Into + +id2's column type is bitmap +``` +insert into bitmap_table1 select id, id2 from bitmap_table2; +``` + +id2's column type is bitmap +``` +INSERT INTO bitmap_table1 (id, id2) VALUES (1001, to_bitmap (1000)), (1001, to_bitmap (2000)); +``` + +id2's column type is bitmap +``` +insert into bitmap_table1 select id, bitmap_union (id2) from bitmap_table2 group by id; +``` + +id2's column type is int +``` +insert into bitmap_table1 select id, to_bitmap (id2) from table; +``` + +id2's column type is String +``` +insert into bitmap_table1 select id, bitmap_hash (id_string) from table; +``` + + +## Data Query + +### Syntax + + +`BITMAP_UNION (expr)`: Calculate the union of two Bitmaps. The return value is the new Bitmap value. + +`BITMAP_UNION_COUNT (expr)`: Calculate the cardinality of the union of two Bitmaps, equivalent to BITMAP_COUNT (BITMAP_UNION (expr)). It is recommended to use the BITMAP_UNION_COUNT function first, its performance is better than BITMAP_COUNT (BITMAP_UNION (expr)). + +`BITMAP_UNION_INT (expr)`: Count the number of different values ​​in columns of type TINYINT, SMALLINT and INT, return the sum of COUNT (DISTINCT expr) same + +`INTERSECT_COUNT (bitmap_column_to_count, filter_column, filter_values ​​...)`: The calculation satisfies +filter_column The cardinality of the intersection of multiple bitmaps of the filter. +bitmap_column_to_count is a column of type bitmap, filter_column is a column of varying dimensions, and filter_values ​​is a list of dimension values. + +### Example + +The following SQL uses the pv_bitmap table above as an example: + +Calculate the deduplication value for user_id: + +``` +select bitmap_union_count (user_id) from pv_bitmap; + +select bitmap_count (bitmap_union (user_id)) from pv_bitmap; +``` + +Calculate the deduplication value of id: + +``` +select bitmap_union_int (id) from pv_bitmap; +``` + +Calculate the retention of user_id: + +``` +select intersect_count (user_id, page, 'meituan') as meituan_uv, +intersect_count (user_id, page, 'waimai') as waimai_uv, +intersect_count (user_id, page, 'meituan', 'waimai') as retention // Number of users appearing on both 'meituan' and 'waimai' pages +from pv_bitmap +where page in ('meituan', 'waimai'); +``` + + +## keyword + +BITMAP, BITMAP_COUNT, BITMAP_EMPTY, BITMAP_UNION, BITMAP_UNION_INT, TO_BITMAP, BITMAP_UNION_COUNT, INTERSECT_COUNT \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/count.md b/docs/en/sql-reference/sql-functions/aggregate-functions/count.md new file mode 100644 index 00000000000000..c14d216b0ed058 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/count.md @@ -0,0 +1,61 @@ +--- +{ + "title": "COUNT", + "language": "en" +} +--- + + + +# COUNT +## Description +### Syntax + +`COUNT([DISTINCT] expr)` + + +Number of rows used to return the required rows + +## example + +``` +MySQL > select count(*) from log_statis group by datetime; ++----------+ +| count(*) | ++----------+ +| 28515903 | ++----------+ + +MySQL > select count(datetime) from log_statis group by datetime; ++-------------------+ +| count(`datetime`) | ++-------------------+ +| 28521682 | ++-------------------+ + +MySQL > select count(distinct datetime) from log_statis group by datetime; ++-------------------------------+ +| count(DISTINCT `datetime`) | ++-------------------------------+ +| 71045 | ++-------------------------------+ +``` +##keyword +COUNT diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md b/docs/en/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md new file mode 100644 index 00000000000000..06db7853c40657 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md @@ -0,0 +1,52 @@ +--- +{ + "title": "HLL_UNION_AGG", + "language": "en" +} +--- + + + +# HLL_UNION_AGG +## description +### Syntax + +`HLL_UNION_AGG(hll)` + + +HLL is an engineering implementation based on HyperLog algorithm, which is used to save the intermediate results of HyperLog calculation process. + +It can only be used as the value column type of the table and reduce the amount of data through aggregation to achieve the purpose of speeding up the query. + +Based on this, we get an estimate with an error of about 1%. The HLL column is generated by other columns or data imported into the data. + +When importing, hll_hash function is used to specify which column in data is used to generate HLL column. It is often used to replace count distinct, and to calculate UV quickly in business by combining rollup. + +## example +``` +MySQL > select HLL_UNION_AGG(uv_set) from test_uv;; ++-------------------------+ +THE COURT OF JUSTICE OF THE EUROPEAN COMMUNITIES, ++-------------------------+ +| 17721 | ++-------------------------+ +``` +##keyword +HLL_UNION_AGG,HLL,UNION,AGG diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/max.md b/docs/en/sql-reference/sql-functions/aggregate-functions/max.md new file mode 100644 index 00000000000000..74cae6eae5e1d6 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/max.md @@ -0,0 +1,46 @@ +--- +{ + "title": "MAX", + "language": "en" +} +--- + + + +# MAX +## description +### Syntax + +`MAX(expr)` + + +Returns the maximum value of an expr expression + +## example +``` +MySQL > select max(scan_rows) from log_statis group by datetime; ++------------------+ +| max(`scan_rows`) | ++------------------+ +| 4671587 | ++------------------+ +``` +##keyword +MAX diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/min.md b/docs/en/sql-reference/sql-functions/aggregate-functions/min.md new file mode 100644 index 00000000000000..8a1ec850c4a4a0 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/min.md @@ -0,0 +1,46 @@ +--- +{ + "title": "MIN", + "language": "en" +} +--- + + + +# MIN +## Description +### Syntax + +`MIN(expr)` + + +Returns the minimum value of an expr expression + +## example +``` +MySQL > select min(scan_rows) from log_statis group by datetime; ++------------------+ +| min(`scan_rows`) | ++------------------+ +| 0 | ++------------------+ +``` +##keyword +MIN diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/ndv.md b/docs/en/sql-reference/sql-functions/aggregate-functions/ndv.md new file mode 100644 index 00000000000000..e8cdbc2df3b795 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/ndv.md @@ -0,0 +1,48 @@ +--- +{ + "title": "NDV", + "language": "en" +} +--- + + + +# NDV +## Description +### Syntax + +`NDV (expr)` + + +Returns an approximate aggregation function similar to the result of COUNT (DISTINCT col). + +It combines COUNT and DISTINCT faster and uses fixed-size memory, so less memory can be used for columns with high cardinality. + +## example +``` +MySQL > select ndv(query_id) from log_statis group by datetime; ++-----------------+ +| ndv(`query_id`) | ++-----------------+ +| 17721 | ++-----------------+ +``` +##keyword +NDV diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/percentile_approx.md b/docs/en/sql-reference/sql-functions/aggregate-functions/percentile_approx.md new file mode 100644 index 00000000000000..4924b8e905e5b0 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/percentile_approx.md @@ -0,0 +1,56 @@ +--- +{ + "title": "PERCENTILE_APPROX", + "language": "en" +} +--- + + + +# PERCENTILE_APPROX +## Description +### Syntax + +`PERCENTILE_APPROX(expr, DOUBLE p[, DOUBLE compression])` + +Return the approximation of the point p, where the value of P is between 0 and 1. + +Compression param is optional and can be setted to a value in the range of [2048, 10000]. The bigger compression you set, the more precise result and more time cost you will get. If it is not setted or not setted in the correct range, PERCENTILE_APPROX function will run with a default compression param of 10000. + +This function uses fixed size memory, so less memory can be used for columns with high cardinality, and can be used to calculate statistics such as tp99. + +## example +``` +MySQL > select `table`, percentile_approx(cost_time,0.99) from log_statis group by `table`; ++---------------------+---------------------------+ +| table | percentile_approx(`cost_time`, 0.99) | ++----------+--------------------------------------+ +| test | 54.22 | ++----------+--------------------------------------+ + +MySQL > select `table`, percentile_approx(cost_time,0.99, 4096) from log_statis group by `table`; ++---------------------+---------------------------+ +| table | percentile_approx(`cost_time`, 0.99, 4096.0) | ++----------+--------------------------------------+ +| test | 54.21 | ++----------+--------------------------------------+ +``` +##keyword +PERCENTILE_APPROX,PERCENTILE,APPROX diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/stddev.md b/docs/en/sql-reference/sql-functions/aggregate-functions/stddev.md new file mode 100644 index 00000000000000..08a0adf6481003 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/stddev.md @@ -0,0 +1,53 @@ +--- +{ + "title": "STDDEV,STDDEV_POP", + "language": "en" +} +--- + + + +# STDDEV,STDDEV_POP +## Description +### Syntax + +`stddev (expl)` + + +Returns the standard deviation of the expr expression + +## example +``` +MySQL > select stddev(scan_rows) from log_statis group by datetime; ++---------------------+ +| stddev(`scan_rows`) | ++---------------------+ +| 2.3736656687790934 | ++---------------------+ + +MySQL > select stddev_pop(scan_rows) from log_statis group by datetime; ++-------------------------+ +| stddev_pop(`scan_rows`) | ++-------------------------+ +| 2.3722760595994914 | ++-------------------------+ +``` +##keyword +STDDEV,STDDEV_POP,POP diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/stddev_samp.md b/docs/en/sql-reference/sql-functions/aggregate-functions/stddev_samp.md new file mode 100644 index 00000000000000..3ca8f5ad0497b9 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/stddev_samp.md @@ -0,0 +1,46 @@ +--- +{ + "title": "STDDEV_SAMP", + "language": "en" +} +--- + + + +# STDDEV_SAMP +## Description +### Syntax + +`STDDEV SAMP (expr)` + + +Returns the sample standard deviation of the expr expression + +## example +``` +MySQL > select stddev_samp(scan_rows) from log_statis group by datetime; ++--------------------------+ +| stddev_samp(`scan_rows`) | ++--------------------------+ +| 2.372044195280762 | ++--------------------------+ +``` +##keyword +STDDEVu SAMP,STDDEV,SAMP diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/sum.md b/docs/en/sql-reference/sql-functions/aggregate-functions/sum.md new file mode 100644 index 00000000000000..91dec04d03206e --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/sum.md @@ -0,0 +1,46 @@ +--- +{ + "title": "SUM", + "language": "en" +} +--- + + + +# SUM +## Description +### Syntax + +`Sum (Expr)` + + +Used to return the sum of all values of the selected field + +## example +``` +MySQL > select sum(scan_rows) from log_statis group by datetime; ++------------------+ +| sum(`scan_rows`) | ++------------------+ +| 8217360135 | ++------------------+ +``` +##keyword +SUM diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/var_samp.md b/docs/en/sql-reference/sql-functions/aggregate-functions/var_samp.md new file mode 100644 index 00000000000000..3a0b91179c0ccc --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/var_samp.md @@ -0,0 +1,46 @@ +--- +{ + "title": "VARIANCE_SAMP,VARIANCE_SAMP", + "language": "en" +} +--- + + + +# VARIANCE_SAMP,VARIANCE_SAMP +## Description +### Syntax + +`VAR SAMP (expr)` + + +Returns the sample variance of the expr expression + +## example +``` +MySQL > select var_samp(scan_rows) from log_statis group by datetime; ++-----------------------+ +| var_samp(`scan_rows`) | ++-----------------------+ +| 5.6227132145741789 | ++-----------------------+ +``` +##keyword +VAR SAMP, VARIANCE SAMP,VAR,SAMP,VARIANCE diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/variance.md b/docs/en/sql-reference/sql-functions/aggregate-functions/variance.md new file mode 100644 index 00000000000000..1716d4b8d65360 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/aggregate-functions/variance.md @@ -0,0 +1,53 @@ +--- +{ + "title": "VARIANCE,VAR_POP,VARIANCE_POP", + "language": "en" +} +--- + + + +# VARIANCE,VAR_POP,VARIANCE_POP +## Description +### Syntax + +`VARIANCE(expr)` + + +Returns the variance of the expr expression + +## example +``` +MySQL > select variance(scan_rows) from log_statis group by datetime; ++-----------------------+ +| variance(`scan_rows`) | ++-----------------------+ +| 5.6183332881176211 | ++-----------------------+ + +MySQL > select var_pop(scan_rows) from log_statis group by datetime; ++----------------------+ +| var_pop(`scan_rows`) | ++----------------------+ +| 5.6230744719006163 | ++----------------------+ +``` +##keyword +VARIANCE,VAR_POP,VARIANCE_POP,VAR,POP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_and.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_and.md new file mode 100644 index 00000000000000..a7a56cbb3de825 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_and.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_and", + "language": "en" +} +--- + + + +# bitmap_and +## description +### Syntax + +`BITMAP BITMAP_AND(BITMAP lhs, BITMAP rhs)` + +Compute intersection of two input bitmaps, return the new bitmap. + +## example + +``` +mysql> select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(2))) cnt; ++------+ +| cnt | ++------+ +| 0 | ++------+ + +mysql> select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(1))) cnt; ++------+ +| cnt | ++------+ +| 1 | ++------+ +``` + +## keyword + + BITMAP_AND,BITMAP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md new file mode 100644 index 00000000000000..5537ddbc0758a9 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_contains", + "language": "en" +} +--- + + + +# bitmap_contains +## description +### Syntax + +`B00LEAN BITMAP_CONTAINS(BITMAP bitmap, BIGINT input)` + +Calculates whether the input value is in the Bitmap column and returns a Boolean value. + +## example + +``` +mysql> select bitmap_contains(to_bitmap(1),2) cnt; ++------+ +| cnt | ++------+ +| 0 | ++------+ + +mysql> select bitmap_contains(to_bitmap(1),1) cnt; ++------+ +| cnt | ++------+ +| 1 | ++------+ +``` + +## keyword + + BITMAP_CONTAINS,BITMAP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md new file mode 100644 index 00000000000000..0b332335e2d082 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md @@ -0,0 +1,52 @@ +--- +{ + "title": "bitmap_empty", + "language": "en" +} +--- + + + +# bitmap_empty +## description +### Syntax + +`BITMAP BITMAP_EMPTY()` + +Return an empty bitmap. Mainly be used to supply default value for bitmap column when loading, e.g., + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,v1,v2=bitmap_empty()" http://host:8410/api/test/testDb/_stream_load +``` + +## example + +``` +mysql> select bitmap_count(bitmap_empty()); ++------------------------------+ +| bitmap_count(bitmap_empty()) | ++------------------------------+ +| 0 | ++------------------------------+ +``` + +## keyword + + BITMAP_EMPTY,BITMAP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md new file mode 100644 index 00000000000000..2dc75c3d43bcda --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md @@ -0,0 +1,63 @@ +--- +{ + "title": "bitmap_from_string", + "language": "en" +} +--- + + + +# bitmap_from_string + +## description +### Syntax + +`BITMAP BITMAP_FROM_STRING(VARCHAR input)` + +Convert a string into a bitmap. The input string should be a comma separated UNIT32. +For example: input string "0, 1, 2" will be converted to a Bitmap with bit 0, 1, 2 set. +If input string is invalid, return NULL. + +## example + +``` +mysql> select bitmap_to_string(bitmap_empty()); ++----------------------------------+ +| bitmap_to_string(bitmap_empty()) | ++----------------------------------+ +| | ++----------------------------------+ + +mysql> select bitmap_to_string(bitmap_from_string("0, 1, 2")); ++-------------------------------------------------+ +| bitmap_to_string(bitmap_from_string('0, 1, 2')) | ++-------------------------------------------------+ +| 0,1,2 | ++-------------------------------------------------+ + +mysql> select bitmap_from_string("-1, 0, 1, 2"); ++-----------------------------------+ +| bitmap_from_string('-1, 0, 1, 2') | ++-----------------------------------+ +| NULL | ++-----------------------------------+ +``` + +## keyword + + BITMAP_FROM_STRING,BITMAP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md new file mode 100644 index 00000000000000..8138c467b85322 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_has_any", + "language": "en" +} +--- + + + +# bitmap_has_any +## description +### Syntax + +`B00LEAN BITMAP_HAS_ANY(BITMAP lhs, BITMAP rhs)` + +Calculate whether there are intersecting elements in the two Bitmap columns. The return value is Boolean. + +## example + +``` +mysql> select bitmap_has_any(to_bitmap(1),to_bitmap(2)) cnt; ++------+ +| cnt | ++------+ +| 0 | ++------+ + +mysql> select bitmap_has_any(to_bitmap(1),to_bitmap(1)) cnt; ++------+ +| cnt | ++------+ +| 1 | ++------+ +``` + +## keyword + + BITMAP_HAS_ANY,BITMAP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md new file mode 100644 index 00000000000000..ff3ed8b66c900b --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md @@ -0,0 +1,52 @@ +--- +{ + "title": "bitmap_hash", + "language": "en" +} +--- + + + +# bitmap_hash +## description +### Syntax + +`BITMAP BITMAP_HASH(expr)` + +Compute the 32-bits hash value of a expr of any type, then return a bitmap containing that hash value. Mainly be used to load non-integer value into bitmap column, e.g., + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,device_id, device_id=bitmap_hash(device_id)" http://host:8410/api/test/testDb/_stream_load +``` + +## example + +``` +mysql> select bitmap_count(bitmap_hash('hello')); ++------------------------------------+ +| bitmap_count(bitmap_hash('hello')) | ++------------------------------------+ +| 1 | ++------------------------------------+ +``` + +## keyword + + BITMAP_HASH,BITMAP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_or.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_or.md new file mode 100644 index 00000000000000..77c4441b024135 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_or.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_or", + "language": "en" +} +--- + + + +# bitmap_or +## description +### Syntax + +`BITMAP BITMAP_OR(BITMAP lhs, BITMAP rhs)` + +Compute union of two input bitmaps, returns the new bitmap. + +## example + +``` +mysql> select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt; ++------+ +| cnt | ++------+ +| 2 | ++------+ + +mysql> select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt; ++------+ +| cnt | ++------+ +| 1 | ++------+ +``` + +## keyword + + BITMAP_OR,BITMAP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md new file mode 100644 index 00000000000000..c9f551011f99cf --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md @@ -0,0 +1,70 @@ +--- +{ + "title": "bitmap_to_string", + "language": "en" +} +--- + + + +# bitmap_to_string + +## description +### Syntax + +`VARCHAR BITMAP_TO_STRING(BITMAP input)` + +Convert a input BITMAP to a string. The string is a separated string, contains all set bits in Bitmap. +If input is null, return null. + +## example + +``` +mysql> select bitmap_to_string(null); ++------------------------+ +| bitmap_to_string(NULL) | ++------------------------+ +| NULL | ++------------------------+ + +mysql> select bitmap_to_string(bitmap_empty()); ++----------------------------------+ +| bitmap_to_string(bitmap_empty()) | ++----------------------------------+ +| | ++----------------------------------+ + +mysql> select bitmap_to_string(to_bitmap(1)); ++--------------------------------+ +| bitmap_to_string(to_bitmap(1)) | ++--------------------------------+ +| 1 | ++--------------------------------+ + +mysql> select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))); ++---------------------------------------------------------+ +| bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) | ++---------------------------------------------------------+ +| 1,2 | ++---------------------------------------------------------+ + +``` + +## keyword + + BITMAP_TO_STRING,BITMAP diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/to_bitmap.md b/docs/en/sql-reference/sql-functions/bitmap-functions/to_bitmap.md new file mode 100644 index 00000000000000..2acef169039414 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/to_bitmap.md @@ -0,0 +1,52 @@ +--- +{ + "title": "to_bitmap", + "language": "en" +} +--- + + + +# to_bitmap +## description +### Syntax + +`BITMAP TO_BITMAP(expr)` + +Convert an unsigned bigint (ranging from 0 to 18446744073709551615) to a bitmap containing that value. Mainly be used to load interger value into bitmap column, e.g., + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=to_bitmap(user_id)" http://host:8410/api/test/testDb/_stream_load +``` + +## example + +``` +mysql> select bitmap_count(to_bitmap(10)); ++-----------------------------+ +| bitmap_count(to_bitmap(10)) | ++-----------------------------+ +| 1 | ++-----------------------------+ +``` + +## keyword + + TO_BITMAP,BITMAP diff --git a/docs/en/sql-reference/sql-functions/cast.md b/docs/en/sql-reference/sql-functions/cast.md new file mode 100644 index 00000000000000..1dba27ec8f5f07 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/cast.md @@ -0,0 +1,83 @@ +--- +{ + "title": "CAST", + "language": "en" +} +--- + + + + +# CAST +##Description + + +``` +cast (input as type) +``` + +### BIGINT type + +### Syntax (BIGINT) + +``` cast (input as BIGINT) ``` + + +Converts input to the specified type + + +Converting the current column input to BIGINT type + +## example + +1. Turn constant, or a column in a table + +``` +mysql> select cast (1 as BIGINT); ++-------------------+ +| CAST(1 AS BIGINT) | ++-------------------+ +| 1 | ++-------------------+ +``` + +2. Transferred raw data + +``` +curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(tmp_k1 as BIGINT)" http://host:port/api/test/bigint/_stream_load +``` + +* Note: In the import, because the original type is String, when the original data with floating point value is cast, the data will be converted to NULL, such as 12.0. Doris is currently not truncating raw data. * + +If you want to force this type of raw data cast to int. Look at the following words: + +``` +curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(cast(tmp_k1 as DOUBLE) as BIGINT)" http://host:port/api/test/bigint/_stream_load + +mysql> select cast(cast ("11.2" as double) as bigint); ++----------------------------------------+ +| CAST(CAST('11.2' AS DOUBLE) AS BIGINT) | ++----------------------------------------+ +| 11 | ++----------------------------------------+ +1 row in set (0.00 sec) +``` +##keyword +CAST diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/curdate.md b/docs/en/sql-reference/sql-functions/date-time-functions/curdate.md new file mode 100644 index 00000000000000..1989c198bfc077 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/curdate.md @@ -0,0 +1,53 @@ +--- +{ + "title": "curdate", + "language": "en" +} +--- + + + +# curdate +## Description +### Syntax + +'DATE CURDATE()' + +Get the current date and return it in Date type + +## example + +``` +mysql> SELECT CURDATE(); ++------------+ +| CURDATE() | ++------------+ +| 2019-12-20 | ++------------+ + +mysql> SELECT CURDATE() + 0; ++---------------+ +| CURDATE() + 0 | ++---------------+ +| 20191220 | ++---------------+ +``` +##keyword +CURDATE diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/current_timestamp.md b/docs/en/sql-reference/sql-functions/date-time-functions/current_timestamp.md new file mode 100644 index 00000000000000..97ea6124a7c3e9 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/current_timestamp.md @@ -0,0 +1,47 @@ +--- +{ + "title": "current_timestamp", + "language": "en" +} +--- + + + +# current_timestamp +## Description +### Syntax + +`DATETIME CURRENT_TIMESTAMP()` + + +Get the current time and return it in Datetime type + +## example + +``` +mysql> select current_timestamp(); ++---------------------+ +| current_timestamp() | ++---------------------+ +| 2019-05-27 15:59:33 | ++---------------------+ +``` +##keyword +CURRENT_TIMESTAMP,CURRENT,TIMESTAMP diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/date_add.md b/docs/en/sql-reference/sql-functions/date-time-functions/date_add.md new file mode 100644 index 00000000000000..0a477d40b1c783 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/date_add.md @@ -0,0 +1,53 @@ +--- +{ + "title": "date_add", + "language": "en" +} +--- + + + +# date_add +## Description +### Syntax + +`INT DATE_ADD(DATETIME date,INTERVAL expr type)` + + +Adds a specified time interval to the date. + +The date parameter is a valid date expression. + +The expr parameter is the interval you want to add. + +Sweet, sweet, sweet + +## example + +``` +mysql> select date_add('2010-11-30 23:59:59', INTERVAL 2 DAY); ++-------------------------------------------------+ +| date_add('2010-11-30 23:59:59', INTERVAL 2 DAY) | ++-------------------------------------------------+ +| 2010-12-02 23:59:59 | ++-------------------------------------------------+ +``` +##keyword +DATE_ADD,DATE,ADD diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/date_format.md b/docs/en/sql-reference/sql-functions/date-time-functions/date_format.md new file mode 100644 index 00000000000000..36924fbde3cf5b --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/date_format.md @@ -0,0 +1,158 @@ +--- +{ + "title": "date_format", + "language": "en" +} +--- + + + +# date_format +## Description +### Syntax + +'WARCHAR DATE'U FORMAT (DATETIME DATE, WARCHAR Format)' + + +Convert the date type to a bit string according to the format type. +Currently supports a string with a maximum 128 bytes and returns NULL if the length of the return value exceeds 128 + +The date parameter is the valid date. Format specifies the date/time output format. + +The formats available are: + +% a | Abbreviation for Sunday Name + +% B | Abbreviated Monthly Name + +% C | Month, numerical value + +% D | Sky in the Moon with English Prefix + +% d | Monthly day, numerical value (00-31) + +% e | Monthly day, numerical value (0-31) + +% f | microseconds + +% H | Hours (00-23) + +% h | hour (01-12) + +% I | Hours (01-12) + +% I | min, numerical value (00-59) + +% J | Days of Year (001-366) + +% k | hours (0-23) + +% L | Hours (1-12) + +% M | Moon Name + +% m | month, numerical value (00-12) + +%p | AM or PM + +% R | Time, 12 - hour (hh: mm: SS AM or PM) + +% S | seconds (00-59) + +% s | seconds (00-59) + +% T | Time, 24 - hour (hh: mm: ss) + +% U | Week (00-53) Sunday is the first day of the week + +% U | Week (00 - 53) Monday is the first day of the week + +% V | Week (01-53) Sunday is the first day of the week, and% X is used. + +% v | Week (01 - 53) Monday is the first day of the week, and% x is used + +% W | Sunday + +% w | Weekly day (0 = Sunday, 6 = Saturday) + +% X | Year, where Sunday is the first day of the week, 4 places, and% V use + +% X | year, of which Monday is the first day of the week, 4 places, and% V + +% Y | Year, 4 + +% Y | Year, 2 + +%% | Represent % + +## example + +``` +mysql> select date_format('2009-10-04 22:23:00', '%W %M %Y'); ++------------------------------------------------+ +| date_format('2009-10-04 22:23:00', '%W %M %Y') | ++------------------------------------------------+ +| Sunday October 2009 | ++------------------------------------------------+ + +mysql> select date_format('2007-10-04 22:23:00', '%H:%i:%s'); ++------------------------------------------------+ +| date_format('2007-10-04 22:23:00', '%H:%i:%s') | ++------------------------------------------------+ +| 22:23:00 | ++------------------------------------------------+ + +mysql> select date_format('1900-10-04 22:23:00', '%D %y %a %d %m %b %j'); ++------------------------------------------------------------+ +| date_format('1900-10-04 22:23:00', '%D %y %a %d %m %b %j') | ++------------------------------------------------------------+ +| 4th 00 Thu 04 10 Oct 277 | ++------------------------------------------------------------+ + +mysql> select date_format('1997-10-04 22:23:00', '%H %k %I %r %T %S %w'); ++------------------------------------------------------------+ +| date_format('1997-10-04 22:23:00', '%H %k %I %r %T %S %w') | ++------------------------------------------------------------+ +| 22 22 10 10:23:00 PM 22:23:00 00 6 | ++------------------------------------------------------------+ + +mysql> select date_format('1999-01-01 00:00:00', '%X %V'); ++---------------------------------------------+ +| date_format('1999-01-01 00:00:00', '%X %V') | ++---------------------------------------------+ +| 1998 52 | ++---------------------------------------------+ + +mysql> select date_format('2006-06-01', '%d'); ++------------------------------------------+ +| date_format('2006-06-01 00:00:00', '%d') | ++------------------------------------------+ +| 01 | ++------------------------------------------+ + +mysql> select date_format('2006-06-01', '%%%d'); ++--------------------------------------------+ +| date_format('2006-06-01 00:00:00', '%%%d') | ++--------------------------------------------+ +| %01 | ++--------------------------------------------+ +``` +##keyword +DATE_FORMAT,DATE,FORMAT diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/date_sub.md b/docs/en/sql-reference/sql-functions/date-time-functions/date_sub.md new file mode 100644 index 00000000000000..8d775ce51abaa0 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/date_sub.md @@ -0,0 +1,53 @@ +--- +{ + "title": "date_sub", + "language": "en" +} +--- + + + +# date_sub +## Description +### Syntax + +`INT DATE_SUB(DATETIME date,INTERVAL expr type)` + + +Subtract the specified time interval from the date + +The date parameter is a valid date expression. + +The expr parameter is the interval you want to add. + +Sweet, sweet, sweet + +## example + +``` +mysql> select date_sub('2010-11-30 23:59:59', INTERVAL 2 DAY); ++-------------------------------------------------+ +| date_sub('2010-11-30 23:59:59', INTERVAL 2 DAY) | ++-------------------------------------------------+ +| 2010-11-28 23:59:59 | ++-------------------------------------------------+ +``` +##keyword +Date, date, date diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/datediff.md b/docs/en/sql-reference/sql-functions/date-time-functions/datediff.md new file mode 100644 index 00000000000000..69abb5fa4c7389 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/datediff.md @@ -0,0 +1,59 @@ +--- +{ + "title": "datediff", + "language": "en" +} +--- + + + + +# datediff +## Description +### Syntax + +'DATETIME DATEDIFF (DATETIME expr1,DATETIME expr2)' + + +Expr1 - expr2 is calculated and the result is accurate to the sky. + +Expr1 and expr2 parameters are valid date or date/time expressions. + +Note: Only the date part of the value participates in the calculation. + +### example + +``` +mysql> select datediff(CAST('2007-12-31 23:59:59' AS DATETIME), CAST('2007-12-30' AS DATETIME)); ++-----------------------------------------------------------------------------------+ +| datediff(CAST('2007-12-31 23:59:59' AS DATETIME), CAST('2007-12-30' AS DATETIME)) | ++-----------------------------------------------------------------------------------+ +| 1 | ++-----------------------------------------------------------------------------------+ + +mysql> select datediff(CAST('2010-11-30 23:59:59' AS DATETIME), CAST('2010-12-31' AS DATETIME)); ++-----------------------------------------------------------------------------------+ +| datediff(CAST('2010-11-30 23:59:59' AS DATETIME), CAST('2010-12-31' AS DATETIME)) | ++-----------------------------------------------------------------------------------+ +| -31 | ++-----------------------------------------------------------------------------------+ +``` +##keyword +DATEDIFF diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/day.md b/docs/en/sql-reference/sql-functions/date-time-functions/day.md new file mode 100644 index 00000000000000..198abed0d919f7 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/day.md @@ -0,0 +1,49 @@ +--- +{ + "title": "day", + "language": "en" +} +--- + + + +# day +## Description +### Syntax + +`INT DAY(DATETIME date)` + + +Get the day information in the date, and return values range from 1 to 31. + +The parameter is Date or Datetime type + +## example + +``` +mysql> select day('1987-01-31'); ++----------------------------+ +| day('1987-01-31 00:00:00') | ++----------------------------+ +| 31 | ++----------------------------+ +``` +##keyword +DAY diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/dayname.md b/docs/en/sql-reference/sql-functions/date-time-functions/dayname.md new file mode 100644 index 00000000000000..9420b2629d2a07 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/dayname.md @@ -0,0 +1,49 @@ +--- +{ + "title": "dayname", + "language": "en" +} +--- + + + +# dayname +## Description +### Syntax + +'VARCHAR DAYNAME (DATE)' + + +Date name corresponding to return date + +The parameter is Date or Datetime type + +## example + +``` +mysql> select dayname('2007-02-03 00:00:00'); ++--------------------------------+ +| dayname('2007-02-03 00:00:00') | ++--------------------------------+ +| Saturday | ++--------------------------------+ +``` +##keyword +DAYNAME diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/dayofmonth.md b/docs/en/sql-reference/sql-functions/date-time-functions/dayofmonth.md new file mode 100644 index 00000000000000..29c39dbdc83bc7 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/dayofmonth.md @@ -0,0 +1,50 @@ +--- +{ + "title": "Dayofmonth", + "language": "en" +} +--- + + + + +# Dayofmonth +## Description +### Syntax + +'INT DAYOFMONTH (DATETIME date)' + + +Get the day information in the date, and return values range from 1 to 31. + +The parameter is Date or Datetime type + +## example + +``` +mysql> select dayofmonth('1987-01-31'); ++-----------------------------------+ +| dayofmonth('1987-01-31 00:00:00') | ++-----------------------------------+ +| 31 | ++-----------------------------------+ +``` +##keyword +DAYOFMONTH diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/dayofweek.md b/docs/en/sql-reference/sql-functions/date-time-functions/dayofweek.md new file mode 100644 index 00000000000000..09ea9692a9b9ef --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/dayofweek.md @@ -0,0 +1,55 @@ +--- +{ + "title": "dayofweek", + "language": "en" +} +--- + + + +# dayofweek +## Description +### Syntax + +INT DayOfWeek (DATETIME date) + + +The DAYOFWEEK function returns the index value of the working day of the date, that is, 1 on Sunday, 2 on Monday, and 7 on Saturday. + +The parameter is Date or Datetime type + +## example +``` +mysql> select dayofweek('2019-06-25'); ++----------------------------------+ +| dayofweek('2019-06-25 00:00:00') | ++----------------------------------+ +| 3 | ++----------------------------------+ + +mysql> select dayofweek(cast(20190625 as date)); ++-----------------------------------+ +| dayofweek(CAST(20190625 AS DATE)) | ++-----------------------------------+ +| 3 | ++-----------------------------------+ +``` +##keyword +DAYOFWEEK diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/dayofyear.md b/docs/en/sql-reference/sql-functions/date-time-functions/dayofyear.md new file mode 100644 index 00000000000000..83d88015e7052e --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/dayofyear.md @@ -0,0 +1,50 @@ +--- +{ + "title": "Dayofyear", + "language": "en" +} +--- + + + +# Dayofyear +## Description +### Syntax + +'INT DAYOFYEAR (DATETIME date)' + + +The date of acquisition is the date of the corresponding year. + +The parameter is Date or Datetime type + +## example + + +``` +mysql> select dayofyear('2007-02-03 00:00:00'); ++----------------------------------+ +| dayofyear('2007-02-03 00:00:00') | ++----------------------------------+ +| 34 | ++----------------------------------+ +``` +##keyword +DAYOFYEAR diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/from_days.md b/docs/en/sql-reference/sql-functions/date-time-functions/from_days.md new file mode 100644 index 00000000000000..d14a8fd92a40a8 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/from_days.md @@ -0,0 +1,46 @@ +--- +{ + "title": "from_days", + "language": "en" +} +--- + + + +# from_days +## Description +### Syntax + +`DATE FROM_DAYS(INT N)` + + +Calculate which day by the number of days from 0000-01-01 + +## example + +``` +mysql > select from u days (730669); ++-------------------+ +| from_days(730669) | ++-------------------+ +| 2000-07-03 | ++-------------------+ +##keyword +FROM_DAYS,FROM,DAYS diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/from_unixtime.md b/docs/en/sql-reference/sql-functions/date-time-functions/from_unixtime.md new file mode 100644 index 00000000000000..eb8c064e0b466e --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/from_unixtime.md @@ -0,0 +1,76 @@ +--- +{ + "title": "from_unixtime", + "language": "en" +} +--- + + + +# from_unixtime +## description +### syntax + +`DATETIME FROM UNIXTIME (INT unix timestamp [, VARCHAR string format]` + +Convert the UNIX timestamp to the corresponding time format of bits, and the format returned is specified by `string_format` + +Input is an integer and return is a string type + +Currently, `string_format` supports following formats： + + %Y: Year. eg. 2014，1900 + %m: Month. eg. 12，09 + %d: Day. eg. 11，01 + %H: Hour. eg. 23，01，12 + %i: Minute. eg. 05，11 + %s: Second. eg. 59，01 + +Default is `%Y-%m-%d %H:%i:%s` + +Other `string_format` is illegal and will returns NULL. + +## example + +``` +mysql> select from_unixtime(1196440219); ++---------------------------+ +| from_unixtime(1196440219) | ++---------------------------+ +| 2007-12-01 00:30:19 | ++---------------------------+ + +mysql> select from_unixtime(1196440219, '%Y-%m-%d'); ++-----------------------------------------+ +| from_unixtime(1196440219, '%Y-%m-%d') | ++-----------------------------------------+ +| 2007-12-01 | ++-----------------------------------------+ + +mysql> select from_unixtime(1196440219, '%Y-%m-%d %H:%i:%s'); ++--------------------------------------------------+ +|From unixtime (1196440219,'%Y-%m-%d %H:%i:%s') | ++--------------------------------------------------+ +| 2007-12-01 00:30:19 | ++--------------------------------------------------+ + +##keyword + + FROM_UNIXTIME,FROM,UNIXTIME diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/hour.md b/docs/en/sql-reference/sql-functions/date-time-functions/hour.md new file mode 100644 index 00000000000000..78670a6f1c614b --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/hour.md @@ -0,0 +1,48 @@ +--- +{ + "title": "hour", + "language": "en" +} +--- + + + +# hour +## description +### Syntax + +`INT HOUR(DATETIME date)` + +Returns hour information in the time type, ranging from 0,23 + +The parameter is Date or Datetime type + +## example + +``` +mysql> select hour('2018-12-31 23:59:59'); ++-----------------------------+ +| hour('2018-12-31 23:59:59') | ++-----------------------------+ +| 23 | ++-----------------------------+ +``` +##keyword +HOUR diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/minute.md b/docs/en/sql-reference/sql-functions/date-time-functions/minute.md new file mode 100644 index 00000000000000..a3649cfea8cd2f --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/minute.md @@ -0,0 +1,48 @@ +--- +{ + "title": "minute", + "language": "en" +} +--- + + + +# minute +## description +### Syntax + +`INT MINUTE(DATETIME date)` + +Returns minute information in the time type, ranging from 0,59 + +The parameter is Date or Datetime type + +## example + +``` +mysql> select minute('2018-12-31 23:59:59'); ++-----------------------------+ +| minute('2018-12-31 23:59:59') | ++-----------------------------+ +| 59 | ++-----------------------------+ +``` +##keyword +MINUTE diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/month.md b/docs/en/sql-reference/sql-functions/date-time-functions/month.md new file mode 100644 index 00000000000000..29644829c0685e --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/month.md @@ -0,0 +1,49 @@ +--- +{ + "title": "month", + "language": "en" +} +--- + + + +# month +## Description +### Syntax + +INT MONTH (DATETIME date) + + +Returns month information in the time type, ranging from 1,12 + +The parameter is Date or Datetime type + +## example + +``` +mysql> select month('1987-01-01'); ++-----------------------------+ +| month('1987-01-01 00:00:00') | ++-----------------------------+ +| 1 | ++-----------------------------+ +``` +##keyword +MONTH diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/monthname.md b/docs/en/sql-reference/sql-functions/date-time-functions/monthname.md new file mode 100644 index 00000000000000..b0c0d3fdf90662 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/monthname.md @@ -0,0 +1,49 @@ +--- +{ + "title": "Monthname", + "language": "en" +} +--- + + + +# Monthname +## Description +### Syntax + +'VARCHAR MONTHNAME (DATE)' + + +Month name corresponding to return date + +The parameter is Date or Datetime type + +## example + +``` +mysql> select monthname('2008-02-03 00:00:00'); ++----------------------------------+ +| monthname('2008-02-03 00:00:00') | ++----------------------------------+ +| February | ++----------------------------------+ +``` +##keyword +MONTHNAME diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/now.md b/docs/en/sql-reference/sql-functions/date-time-functions/now.md new file mode 100644 index 00000000000000..6f9c79d45da4bb --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/now.md @@ -0,0 +1,47 @@ +--- +{ + "title": "now", + "language": "en" +} +--- + + + +# now +## Description +### Syntax + +'DATETIME NOW ()' + + +Get the current time and return it in Datetime type + +## example + +``` +mysql> select now(); ++---------------------+ +| now() | ++---------------------+ +| 2019-05-27 15:58:25 | ++---------------------+ +``` +##keyword +NOW diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/second.md b/docs/en/sql-reference/sql-functions/date-time-functions/second.md new file mode 100644 index 00000000000000..5553db40a4c47a --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/second.md @@ -0,0 +1,48 @@ +--- +{ + "title": "second", + "language": "en" +} +--- + + + +# second +## description +### Syntax + +`INT SECOND(DATETIME date)` + +Returns second information in the time type, ranging from 0,59 + +The parameter is Date or Datetime type + +## example + +``` +mysql> select second('2018-12-31 23:59:59'); ++-----------------------------+ +| second('2018-12-31 23:59:59') | ++-----------------------------+ +| 59 | ++-----------------------------+ +``` +##keyword +SECOND diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/str_to_date.md b/docs/en/sql-reference/sql-functions/date-time-functions/str_to_date.md new file mode 100644 index 00000000000000..d74d5eee12a8ab --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/str_to_date.md @@ -0,0 +1,64 @@ +--- +{ + "title": "Str_to_date", + "language": "en" +} +--- + + + +# Str_to_date +## Description +### Syntax + +'DATETIME STR TWO DATES (VARCHAR STR, VARCHAR format)' + + +Convert STR to DATE type by format specified, if the conversion result does not return NULL + +The format format supported is consistent with date_format + +## example + +``` +mysql> select str_to_date('2014-12-21 12:34:56', '%Y-%m-%d %H:%i:%s'); ++---------------------------------------------------------+ +| str_to_date('2014-12-21 12:34:56', '%Y-%m-%d %H:%i:%s') | ++---------------------------------------------------------+ +| 2014-12-21 12:34:56 | ++---------------------------------------------------------+ + +mysql> select str_to_date('2014-12-21 12:34%3A56', '%Y-%m-%d %H:%i%%3A%s'); ++--------------------------------------------------------------+ +| str_to_date('2014-12-21 12:34%3A56', '%Y-%m-%d %H:%i%%3A%s') | ++--------------------------------------------------------------+ +| 2014-12-21 12:34:56 | ++--------------------------------------------------------------+ + +mysql> select str_to_date('200442 Monday', '%X%V %W'); ++-----------------------------------------+ +| str_to_date('200442 Monday', '%X%V %W') | ++-----------------------------------------+ +| 2004-10-18 | ++-----------------------------------------+ +``` +##keyword + + STR_TO_DATE,STR,TO,DATE diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/timediff.md b/docs/en/sql-reference/sql-functions/date-time-functions/timediff.md new file mode 100644 index 00000000000000..86c6efb8a23205 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/timediff.md @@ -0,0 +1,65 @@ +--- +{ + "title": "timediff", + "language": "en" +} +--- + + + +# timediff +## Description +### Syntax + +'TIME TIMEDIFF (DATETIME expr1, DATETIME expr2)' + + +TIMEDIFF returns the difference between two DATETIMEs + +The TIMEDIFF function returns the result of expr1 - expr2 expressed as a time value, with a return value of TIME type + +The results are limited to TIME values ranging from - 838:59:59 to 838:59:59. + +### example + +``` +mysql> SELECT TIMEDIFF(now(),utc_timestamp()); ++----------------------------------+ +| timediff(now(), utc_timestamp()) | ++----------------------------------+ +| 08:00:00 | ++----------------------------------+ + +mysql> SELECT TIMEDIFF('2019-07-11 16:59:30','2019-07-11 16:59:21'); ++--------------------------------------------------------+ +| timediff('2019-07-11 16:59:30', '2019-07-11 16:59:21') | ++--------------------------------------------------------+ +| 00:00:09 | ++--------------------------------------------------------+ + +mysql> SELECT TIMEDIFF('2019-01-01 00:00:00', NULL); ++---------------------------------------+ +| timediff('2019-01-01 00:00:00', NULL) | ++---------------------------------------+ +| NULL | ++---------------------------------------+ +``` +##keyword +TIMEDIFF diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/timestampadd.md b/docs/en/sql-reference/sql-functions/date-time-functions/timestampadd.md new file mode 100644 index 00000000000000..4ca118563785f1 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/timestampadd.md @@ -0,0 +1,58 @@ +--- +{ + "title": "timestampadd", + "language": "en" +} +--- + + + +# timestampadd +## description +### Syntax + +`DATETIME TIMESTAMPADD(unit, interval, DATETIME datetime_expr)` + +Adds the integer expression interval to the date or datetime expression datetime_expr. + +The unit for interval is given by the unit argument, which should be one of the following values: + +SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, or YEAR. + +## example + +``` + +mysql> SELECT TIMESTAMPADD(MINUTE,1,'2019-01-02'); ++------------------------------------------------+ +| timestampadd(MINUTE, 1, '2019-01-02 00:00:00') | ++------------------------------------------------+ +| 2019-01-02 00:01:00 | ++------------------------------------------------+ + +mysql> SELECT TIMESTAMPADD(WEEK,1,'2019-01-02'); ++----------------------------------------------+ +| timestampadd(WEEK, 1, '2019-01-02 00:00:00') | ++----------------------------------------------+ +| 2019-01-09 00:00:00 | ++----------------------------------------------+ +``` +##keyword +TIMESTAMPADD diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/timestampdiff.md b/docs/en/sql-reference/sql-functions/date-time-functions/timestampdiff.md new file mode 100644 index 00000000000000..9d6c4f8e2438eb --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/timestampdiff.md @@ -0,0 +1,67 @@ +--- +{ + "title": "timestampdiff", + "language": "en" +} +--- + + + +# timestampdiff +## description +### Syntax + +`INT TIMESTAMPDIFF(unit,DATETIME datetime_expr1, DATETIME datetime_expr2)` + +Returns datetime_expr2 − datetime_expr1, where datetime_expr1 and datetime_expr2 are date or datetime expressions. + +The unit for the result (an integer) is given by the unit argument. + +The legal values for unit are the same as those listed in the description of the TIMESTAMPADD() function. + +## example + +``` + +MySQL> SELECT TIMESTAMPDIFF(MONTH,'2003-02-01','2003-05-01'); ++--------------------------------------------------------------------+ +| timestampdiff(MONTH, '2003-02-01 00:00:00', '2003-05-01 00:00:00') | ++--------------------------------------------------------------------+ +| 3 | ++--------------------------------------------------------------------+ + +MySQL> SELECT TIMESTAMPDIFF(YEAR,'2002-05-01','2001-01-01'); ++-------------------------------------------------------------------+ +| timestampdiff(YEAR, '2002-05-01 00:00:00', '2001-01-01 00:00:00') | ++-------------------------------------------------------------------+ +| -1 | ++-------------------------------------------------------------------+ + + +MySQL> SELECT TIMESTAMPDIFF(MINUTE,'2003-02-01','2003-05-01 12:05:55'); ++---------------------------------------------------------------------+ +| timestampdiff(MINUTE, '2003-02-01 00:00:00', '2003-05-01 12:05:55') | ++---------------------------------------------------------------------+ +| 128885 | ++---------------------------------------------------------------------+ + +``` +##keyword +TIMESTAMPDIFF diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/to_days.md b/docs/en/sql-reference/sql-functions/date-time-functions/to_days.md new file mode 100644 index 00000000000000..8ecad8d408a972 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/to_days.md @@ -0,0 +1,50 @@ +--- +{ + "title": "to_days", + "language": "en" +} +--- + + + +# to_days +## Description +### Syntax + +'INT TO DAYS' + + +Days of returning date distance 0000-01-01 + +The parameter is Date or Datetime type + +## example + +``` +mysql> select to_days('2007-10-07'); ++-----------------------+ +| to_days('2007-10-07') | ++-----------------------+ +| 733321 | ++-----------------------+ +``` + +##keyword +TO_DAYS,TO,DAYS diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/unix_timestamp.md b/docs/en/sql-reference/sql-functions/date-time-functions/unix_timestamp.md new file mode 100644 index 00000000000000..e4f920005ddcb4 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/unix_timestamp.md @@ -0,0 +1,86 @@ +--- +{ + "title": "unix_timestamp", + "language": "en" +} +--- + + + +# unix_timestamp +## Description +### Syntax + +`INT UNIX_TIMESTAMP(), UNIX_TIMESTAMP(DATETIME date)` + +Converting a Date or Datetime type to a UNIX timestamp. + +If there are no parameters, the current time is converted into a timestamp. + +The parameter needs to be Date or Datetime type. + +Any date before 1970-01-01 00:00:00 or after 2038-01-19 03:14:07 will return 0. + +See `date_format` function to get Format explanation. + +This function is affected by time zone. + +## example + +``` +mysql> select unix_timestamp(); ++------------------+ +| unix_timestamp() | ++------------------+ +| 1558589570 | ++------------------+ + +mysql> select unix_timestamp('2007-11-30 10:30:19'); ++---------------------------------------+ +| unix_timestamp('2007-11-30 10:30:19') | ++---------------------------------------+ +| 1196389819 | ++---------------------------------------+ + +mysql> select unix_timestamp('2007-11-30 10:30-19', '%Y-%m-%d %H:%i-%s'); ++---------------------------------------+ +| unix_timestamp('2007-11-30 10:30-19') | ++---------------------------------------+ +| 1196389819 | ++---------------------------------------+ + +mysql> select unix_timestamp('2007-11-30 10:30%3A19', '%Y-%m-%d %H:%i%%3A%s'); ++---------------------------------------+ +| unix_timestamp('2007-11-30 10:30%3A19') | ++---------------------------------------+ +| 1196389819 | ++---------------------------------------+ + +mysql> select unix_timestamp('1969-01-01 00:00:00'); ++---------------------------------------+ +| unix_timestamp('1969-01-01 00:00:00') | ++---------------------------------------+ +| 0 | ++---------------------------------------+ +``` + +## keyword + + UNIX_TIMESTAMP,UNIX,TIMESTAMP diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/utc_timestamp.md b/docs/en/sql-reference/sql-functions/date-time-functions/utc_timestamp.md new file mode 100644 index 00000000000000..42c8d0a6bbbd03 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/utc_timestamp.md @@ -0,0 +1,50 @@ +--- +{ + "title": "utc_timestamp", + "language": "en" +} +--- + + + +# utc_timestamp +## Description +### Syntax + +`DATETIME UTC_TIMESTAMP()` + + +Returns the current UTC date and time in "YYYY-MM-DD HH: MM: SS" or + +A Value of "YYYYMMDDHMMSS" Format + +Depending on whether the function is used in a string or numeric context + +## example + +``` +mysql> select utc_timestamp(),utc_timestamp() + 1; ++---------------------+---------------------+ +| utc_timestamp() | utc_timestamp() + 1 | ++---------------------+---------------------+ +| 2019-07-10 12:31:18 | 20190710123119 | ++---------------------+---------------------+ +##keyword +UTC_TIMESTAMP,UTC,TIMESTAMP diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/workofyear.md b/docs/en/sql-reference/sql-functions/date-time-functions/workofyear.md new file mode 100644 index 00000000000000..9b602e39bc8ce9 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/workofyear.md @@ -0,0 +1,50 @@ +--- +{ + "title": "weekofyear", + "language": "en" +} +--- + + + +# weekofyear +## Description +### Syntax + +'INT WEEKOFYEAR (DATETIME DATE)' + + + +Get the Weeks of the Year + +The parameter is Date or Datetime type + +## example + +``` +mysql> select weekofyear('2008-02-20 00:00:00'); ++-----------------------------------+ +| weekofyear('2008-02-20 00:00:00') | ++-----------------------------------+ +| 8 | ++-----------------------------------+ +``` +##keyword +WEEKOFYEAR diff --git a/docs/en/sql-reference/sql-functions/date-time-functions/year.md b/docs/en/sql-reference/sql-functions/date-time-functions/year.md new file mode 100644 index 00000000000000..92e6f644597265 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/date-time-functions/year.md @@ -0,0 +1,50 @@ +--- +{ + "title": "year", + "language": "en" +} +--- + + + + +# year +## Description +### Syntax + +`INT YEAR(DATETIME date)` + + +Returns the year part of the date type, ranging from 1000 to 9999 + +The parameter is Date or Datetime type + +## example + +``` +mysql> select year('1987-01-01'); ++-----------------------------+ +| year('1987-01-01 00:00:00') | ++-----------------------------+ +| 1987 | ++-----------------------------+ +``` +##keyword +YEAR diff --git a/docs/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md b/docs/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md new file mode 100644 index 00000000000000..d75d9380b7e6e5 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md @@ -0,0 +1,61 @@ +--- +{ + "title": "murmur_hash3_32", + "language": "en" +} +--- + + + +# murmur_hash3_32 + +## description +### Syntax + +`INT MURMUR_HASH3_32(VARCHAR input, ...)` + +Return the 32 bits murmur3 hash of input string. + +## example + +``` +mysql> select murmur_hash3_32(null); ++-----------------------+ +| murmur_hash3_32(NULL) | ++-----------------------+ +| NULL | ++-----------------------+ + +mysql> select murmur_hash3_32("hello"); ++--------------------------+ +| murmur_hash3_32('hello') | ++--------------------------+ +| 1321743225 | ++--------------------------+ + +mysql> select murmur_hash3_32("hello", "world"); ++-----------------------------------+ +| murmur_hash3_32('hello', 'world') | ++-----------------------------------+ +| 984713481 | ++-----------------------------------+ +``` + +## keyword + + MURMUR_HASH3_32,HASH diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_astext.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_astext.md new file mode 100644 index 00000000000000..ef489691284746 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_astext.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_AsText`,`ST_AsWKT`", + "language": "en" +} +--- + + + +# `ST_AsText`,`ST_AsWKT` +## Description +### Syntax + +'VARCHAR ST'u AsText (GEOMETRY geo)' + + +Converting a geometric figure into a WKT (Well Known Text) representation + +## example + +``` +mysql> SELECT ST_AsText(ST_Point(24.7, 56.7)); ++---------------------------------+ +| st_astext(st_point(24.7, 56.7)) | ++---------------------------------+ +| POINT (24.7 56.7) | ++---------------------------------+ +``` +##keyword +ST_ASTEXT,ST_ASWKT,ST,ASTEXT,ASWKT diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_circle.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_circle.md new file mode 100644 index 00000000000000..e11bf0f0115b32 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_circle.md @@ -0,0 +1,48 @@ +--- +{ + "title": "`ST_Circle`", + "language": "en" +} +--- + + + +# `ST_Circle` +## Description +### Syntax + +`GEOMETRY ST_Circle(DOUBLE center_lng, DOUBLE center_lat, DOUBLE radius)` + + +Convert a WKT (Well Known Text) into a circle on the earth's sphere. Where `center_lng'denotes the longitude of the center of a circle, +` Center_lat` denotes the latitude of the center of a circle, radius` denotes the radius of a circle in meters. + +## example + +``` +mysql> SELECT ST_AsText(ST_Circle(111, 64, 10000)); ++--------------------------------------------+ +| st_astext(st_circle(111.0, 64.0, 10000.0)) | ++--------------------------------------------+ +| CIRCLE ((111 64), 10000) | ++--------------------------------------------+ +``` +##keyword +ST_CIRCLE,ST,CIRCLE diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_contains.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_contains.md new file mode 100644 index 00000000000000..4b916415bee287 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_contains.md @@ -0,0 +1,56 @@ +--- +{ + "title": "`ST_Contains'", + "language": "en" +} +--- + + + +# `ST_Contains' +## Description +### Syntax + +`BOOL ST_Contains(GEOMETRY shape1, GEOMETRY shape2)` + + +Judging whether geometric shape 1 can contain geometric shape 2 completely + +## example + + +``` +mysql> SELECT ST_Contains(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"), ST_Point(5, 5)); ++----------------------------------------------------------------------------------------+ +| st_contains(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))'), st_point(5.0, 5.0)) | ++----------------------------------------------------------------------------------------+ +| 1 | ++----------------------------------------------------------------------------------------+ + +mysql> SELECT ST_Contains(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"), ST_Point(50, 50)); ++------------------------------------------------------------------------------------------+ +| st_contains(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))'), st_point(50.0, 50.0)) | ++------------------------------------------------------------------------------------------+ +| 0 | ++------------------------------------------------------------------------------------------+ +``` +##keyword +ST_CONTAINS,ST,CONTAINS +w \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md new file mode 100644 index 00000000000000..9f0a211b5be0a5 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_Distance_Sphere`", + "language": "en" +} +--- + + + +# `ST_Distance_Sphere` +## description +### Syntax + +`DOUBLE ST_Distance_Sphere(DOUBLE x_lng, DOUBLE x_lat, DOUBLE y_lng, DOUBLE x_lat)` + + +Calculate the spherical distance between two points of the earth in meters. The incoming parameters are the longitude of point X, the latitude of point X, the longitude of point Y and the latitude of point Y. + +## example + +``` +mysql> select st_distance_sphere(116.35620117, 39.939093, 116.4274406433, 39.9020987219); ++----------------------------------------------------------------------------+ +| st_distance_sphere(116.35620117, 39.939093, 116.4274406433, 39.9020987219) | ++----------------------------------------------------------------------------+ +| 7336.9135549995917 | ++----------------------------------------------------------------------------+ +``` +##keyword +ST_DISTANCE_SPHERE,ST,DISTANCE,SPHERE diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md new file mode 100644 index 00000000000000..7860f73d26ad23 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_GeometryFromText`,`ST GeomFromText`", + "language": "en" +} +--- + + + +# `ST_GeometryFromText`,`ST GeomFromText` +## Description +### Syntax + +'GEOMETRY ST'u GeometryFromText (VARCHAR wkt)' + + +Converting a WKT (Well Known Text) into a corresponding memory geometry + +## example + +``` +mysql> SELECT ST_AsText(ST_GeometryFromText("LINESTRING (1 1, 2 2)")); ++---------------------------------------------------------+ +| st_astext(st_geometryfromtext('LINESTRING (1 1, 2 2)')) | ++---------------------------------------------------------+ +| LINESTRING (1 1, 2 2) | ++---------------------------------------------------------+ +``` +##keyword +ST_GEOMETRYFROMTEXT,ST_GEOMFROMTEXT,ST,GEOMETRYFROMTEXT,GEOMFROMTEXT diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_linefromtext.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_linefromtext.md new file mode 100644 index 00000000000000..72f286571ccebd --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_linefromtext.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_LineFromText`,`ST_LineStringFromText`", + "language": "en" +} +--- + + + +# `ST_LineFromText`,`ST_LineStringFromText` +## Description +### Syntax + +'GEOMETRY ST LineFromText (VARCHAR wkt)' + + +Converting a WKT (Well Known Text) into a Line-style memory representation + +## example + +``` +mysql> SELECT ST_AsText(ST_LineFromText("LINESTRING (1 1, 2 2)")); ++---------------------------------------------------------+ +| st_astext(st_geometryfromtext('LINESTRING (1 1, 2 2)')) | ++---------------------------------------------------------+ +| LINESTRING (1 1, 2 2) | ++---------------------------------------------------------+ +``` +##keyword +ST_LINEFROMTEXT, ST_LINESTRINGFROMTEXT,ST,LINEFROMTEXT,LINESTRINGFROMTEXT diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_point.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_point.md new file mode 100644 index 00000000000000..d223a7836b2d74 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_point.md @@ -0,0 +1,48 @@ +--- +{ + "title": "St_Point'", + "language": "en" +} +--- + + + +# St_Point' +## Description +### Syntax + +`POINT ST_Point(DOUBLE x, DOUBLE y)` + + +Given the X coordinate value, the Y coordinate value returns the corresponding Point. +The current value is meaningful only for spherical sets, and X/Y corresponds to longitude/latitude. + +## example + +``` +mysql> SELECT ST_AsText(ST_Point(24.7, 56.7)); ++---------------------------------+ +| st_astext(st_point(24.7, 56.7)) | ++---------------------------------+ +| POINT (24.7 56.7) | ++---------------------------------+ +``` +##keyword +ST_POINT,ST,POINT diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_polygon.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_polygon.md new file mode 100644 index 00000000000000..9cc72bdaed8024 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_polygon.md @@ -0,0 +1,48 @@ +--- +{ + "title": "`ST_Polygon`,`ST_PolyFromText`,`ST_PolygonFromText`", + "language": "en" +} +--- + + + +# `ST_Polygon`,`ST_PolyFromText`,`ST_PolygonFromText` +## Description +### Syntax + +'GEOMETRY ST'u Polygon (VARCHAR wkt)' + + +Converting a WKT (Well Known Text) into a corresponding polygon memory form + + +### example + +``` +mysql> SELECT ST_AsText(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))")); ++------------------------------------------------------------------+ +| st_astext(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))')) | ++------------------------------------------------------------------+ +| POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0)) | ++------------------------------------------------------------------+ +``` +##keyword +ST_POLYGON,ST_POLYFROMTEXT,ST_POLYGONFROMTEXT,ST,POLYGON,POLYFROMTEXT,POLYGONFROMTEXT diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_x.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_x.md new file mode 100644 index 00000000000000..87ac54bf158bcc --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_x.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_X`", + "language": "en" +} +--- + + + +# `ST_X` +## Description +### Syntax + +`DOUBLE ST_X(POINT point)` + + +When point is a valid POINT type, the corresponding X coordinate value is returned. + +## example + +``` +mysql> SELECT ST_X(ST_Point(24.7, 56.7)); ++----------------------------+ +| st_x(st_point(24.7, 56.7)) | ++----------------------------+ +| 24.7 | ++----------------------------+ +``` +##keyword +ST_X,ST,X diff --git a/docs/en/sql-reference/sql-functions/spatial-functions/st_y.md b/docs/en/sql-reference/sql-functions/spatial-functions/st_y.md new file mode 100644 index 00000000000000..9599c5e4c1e720 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/spatial-functions/st_y.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_Y`", + "language": "en" +} +--- + + + +# `ST_Y` +## Description +### Syntax + +`DOUBLE ST_Y(POINT point)` + + +When point is a valid POINT type, the corresponding Y coordinate value is returned. + +## example + +``` +mysql> SELECT ST_Y(ST_Point(24.7, 56.7)); ++----------------------------+ +| st_y(st_point(24.7, 56.7)) | ++----------------------------+ +| 56.7 | ++----------------------------+ +``` +##keyword +ST_Y,ST,Y diff --git a/docs/en/sql-reference/sql-functions/string-functions/ascii.md b/docs/en/sql-reference/sql-functions/string-functions/ascii.md new file mode 100644 index 00000000000000..53f166e3a9af1e --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/ascii.md @@ -0,0 +1,54 @@ +--- +{ + "title": "ASCII", + "language": "en" +} +--- + + + +# ASCII +## Description +### Syntax + +`INT AXES (WARCHAR STR)` + + +Returns the ASCII code corresponding to the first character of the string + +## example + +``` +mysql> select ascii('1'); ++------------+ +| ascii('1') | ++------------+ +| 49 | ++------------+ + +mysql> select ascii('234'); ++--------------+ +| ascii('234') | ++--------------+ +| 50 | ++--------------+ +``` +##keyword +ASCII diff --git a/docs/en/sql-reference/sql-functions/string-functions/concat.md b/docs/en/sql-reference/sql-functions/string-functions/concat.md new file mode 100644 index 00000000000000..91fe8e1c982898 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/concat.md @@ -0,0 +1,61 @@ +--- +{ + "title": "concat", + "language": "en" +} +--- + + + +# concat +## Description +### Syntax + +`VARCHAR concat (VARCHAR,...)` + + +Connect multiple strings and return NULL if any of the parameters is NULL + +## example + +``` +mysql> select concat("a", "b"); ++------------------+ +| concat('a', 'b') | ++------------------+ +| ab | ++------------------+ + +mysql> select concat("a", "b", "c"); ++-----------------------+ +| concat('a', 'b', 'c') | ++-----------------------+ +| abc | ++-----------------------+ + +mysql> select concat("a", null, "c"); ++------------------------+ +| concat('a', NULL, 'c') | ++------------------------+ +| NULL | ++------------------------+ +``` +##keyword +CONCAT diff --git a/docs/en/sql-reference/sql-functions/string-functions/concat_ws.md b/docs/en/sql-reference/sql-functions/string-functions/concat_ws.md new file mode 100644 index 00000000000000..c16539e03ed973 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/concat_ws.md @@ -0,0 +1,63 @@ +--- +{ + "title": "Concat_ws", + "language": "en" +} +--- + + + +# Concat_ws +## Description +### Syntax + +'VARCHAR concat ws (VARCHAR sep., VARCHAR str,...)' + + +Using the first parameter SEP as a connector, the second parameter and all subsequent parameters are spliced into a string. +If the separator is NULL, return NULL. +` The concat_ws` function does not skip empty strings, but NULL values. + +## example + +``` +mysql> select concat_ws("or", "d", "is"); ++----------------------------+ +| concat_ws('or', 'd', 'is') | ++----------------------------+ +| doris | ++----------------------------+ + +mysql> select concat_ws(NULL, "d", "is"); ++----------------------------+ +| concat_ws(NULL, 'd', 'is') | ++----------------------------+ +| NULL | ++----------------------------+ + +mysql> select concat_ws("or", "d", NULL,"is"); ++---------------------------------+ +| concat_ws("or", "d", NULL,"is") | ++---------------------------------+ +| doris | ++---------------------------------+ +``` +##keyword +CONCAT_WS,CONCAT,WS diff --git a/docs/en/sql-reference/sql-functions/string-functions/ends_with.md b/docs/en/sql-reference/sql-functions/string-functions/ends_with.md new file mode 100644 index 00000000000000..d11ee6bdacab6b --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/ends_with.md @@ -0,0 +1,54 @@ +--- +{ + "title": "ends_with", + "language": "en" +} +--- + + + +# ends_with +## Description +### Syntax + +`BOOLEAN ENDS_WITH (VARCHAR str, VARCHAR suffix)` + +It returns true if the string ends with the specified suffix, otherwise it returns false. +If any parameter is NULL, it returns NULL. + +## example + +``` +mysql> select ends_with("Hello doris", "doris"); ++-----------------------------------+ +| ends_with('Hello doris', 'doris') | ++-----------------------------------+ +| 1 | ++-----------------------------------+ + +mysql> select ends_with("Hello doris", "Hello"); ++-----------------------------------+ +| ends_with('Hello doris', 'Hello') | ++-----------------------------------+ +| 0 | ++-----------------------------------+ +``` +##keyword +ENDS_WITH diff --git a/docs/en/sql-reference/sql-functions/string-functions/find_in_set.md b/docs/en/sql-reference/sql-functions/string-functions/find_in_set.md new file mode 100644 index 00000000000000..a4a0dac8e45f6c --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/find_in_set.md @@ -0,0 +1,49 @@ +--- +{ + "title": "find_in_set", + "language": "en" +} +--- + + + +# find_in_set +## description +### Syntax + +`INT find_in_set(VARCHAR str, VARCHAR strlist)` + +"NOT found in set (VARCHAR str., VARCHAR strlist)" + + +Return to the location where the str first appears in strlist (counting from 1). Strlist is a comma-separated string. If not, return 0. Any parameter is NULL, returning NULL. + +## example + +``` +mysql> select find_in_set("b", "a,b,c"); ++---------------------------+ +| find_in_set('b', 'a,b,c') | ++---------------------------+ +| 2 | ++---------------------------+ +``` +##keyword +FIND_IN_SET,FIND,IN,SET diff --git a/docs/en/sql-reference/sql-functions/string-functions/get_json_double.md b/docs/en/sql-reference/sql-functions/string-functions/get_json_double.md new file mode 100644 index 00000000000000..d0177959ca4791 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/get_json_double.md @@ -0,0 +1,74 @@ +--- +{ + "title": "get_json_double", + "language": "en" +} +--- + + + +# get_json_double +## description +### Syntax + +`DOUBLE get_json_double(VARCHAR json_str, VARCHAR json_path) + + +Parse and get the floating-point content of the specified path in the JSON string. +Where json_path must start with the $symbol and use. as the path splitter. If the path contains..., double quotation marks can be used to surround it. +Use [] to denote array subscripts, starting at 0. +The content of path cannot contain ",[and]. +If the json_string format is incorrect, or the json_path format is incorrect, or matches cannot be found, NULL is returned. + +## example + +1. Get the value of key as "k1" + +``` +mysql> SELECT get_json_double('{"k1":1.3, "k2":"2"}', "$.k1"); ++-------------------------------------------------+ +| get_json_double('{"k1":1.3, "k2":"2"}', '$.k1') | ++-------------------------------------------------+ +| 1.3 | ++-------------------------------------------------+ +``` + +2. Get the second element of the array whose key is "my. key" + +``` +mysql> SELECT get_json_double('{"k1":"v1", "my.key":[1.1, 2.2, 3.3]}', '$."my.key"[1]'); ++---------------------------------------------------------------------------+ +| get_json_double('{"k1":"v1", "my.key":[1.1, 2.2, 3.3]}', '$."my.key"[1]') | ++---------------------------------------------------------------------------+ +| 2.2 | ++---------------------------------------------------------------------------+ +``` + +3. Get the first element in an array whose secondary path is k1. key - > K2 +``` +mysql> SELECT get_json_double('{"k1.key":{"k2":[1.1, 2.2]}}', '$."k1.key".k2[0]'); ++---------------------------------------------------------------------+ +| get_json_double('{"k1.key":{"k2":[1.1, 2.2]}}', '$."k1.key".k2[0]') | ++---------------------------------------------------------------------+ +| 1.1 | ++---------------------------------------------------------------------+ +``` +##keyword +GET_JSON_DOUBLE,GET,JSON,DOUBLE diff --git a/docs/en/sql-reference/sql-functions/string-functions/get_json_int.md b/docs/en/sql-reference/sql-functions/string-functions/get_json_int.md new file mode 100644 index 00000000000000..18297a8c2ba2d8 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/get_json_int.md @@ -0,0 +1,74 @@ +--- +{ + "title": "get_json_int", + "language": "en" +} +--- + + + +# get_json_int +## Description +### Syntax + +`INT get_json_int(VARCHAR json_str, VARCHAR json_path) + + +Parse and retrieve the integer content of the specified path in the JSON string. +Where json_path must start with the $symbol and use. as the path splitter. If the path contains..., double quotation marks can be used to surround it. +Use [] to denote array subscripts, starting at 0. +The content of path cannot contain ",[and]. +If the json_string format is incorrect, or the json_path format is incorrect, or matches cannot be found, NULL is returned. + +## example + +1. Get the value of key as "k1" + +``` +mysql> SELECT get_json_int('{"k1":1, "k2":"2"}', "$.k1"); ++--------------------------------------------+ +| get_json_int('{"k1":1, "k2":"2"}', '$.k1') | ++--------------------------------------------+ +| 1 | ++--------------------------------------------+ +``` + +2. Get the second element of the array whose key is "my. key" + +``` +mysql> SELECT get_json_int('{"k1":"v1", "my.key":[1, 2, 3]}', '$."my.key"[1]'); ++------------------------------------------------------------------+ +| get_json_int('{"k1":"v1", "my.key":[1, 2, 3]}', '$."my.key"[1]') | ++------------------------------------------------------------------+ +| 2 | ++------------------------------------------------------------------+ +``` + +3. Get the first element in an array whose secondary path is k1. key - > K2 +``` +mysql> SELECT get_json_int('{"k1.key":{"k2":[1, 2]}}', '$."k1.key".k2[0]'); ++--------------------------------------------------------------+ +| get_json_int('{"k1.key":{"k2":[1, 2]}}', '$."k1.key".k2[0]') | ++--------------------------------------------------------------+ +| 1 | ++--------------------------------------------------------------+ +``` +##keyword +GET_JSON_INT,GET,JSON,INT diff --git a/docs/en/sql-reference/sql-functions/string-functions/get_json_string.md b/docs/en/sql-reference/sql-functions/string-functions/get_json_string.md new file mode 100644 index 00000000000000..8d921725592dbc --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/get_json_string.md @@ -0,0 +1,84 @@ +--- +{ + "title": "get_json_string", + "language": "en" +} +--- + + + +# get_json_string +## description +### Syntax + +'VARCHAR get_json_string (VARCHAR json str, VARCHAR json path) + + +Parse and retrieve the string content of the specified path in the JSON string. +Where json_path must start with the $symbol and use. as the path splitter. If the path contains..., double quotation marks can be used to surround it. +Use [] to denote array subscripts, starting at 0. +The content of path cannot contain ",[and]. +If the json_string format is incorrect, or the json_path format is incorrect, or matches cannot be found, NULL is returned. + +## example + +1. Get the value of key as "k1" + +``` +mysql> SELECT get_json_string('{"k1":"v1", "k2":"v2"}', "$.k1"); ++---------------------------------------------------+ +| get_json_string('{"k1":"v1", "k2":"v2"}', '$.k1') | ++---------------------------------------------------+ +| v1 | ++---------------------------------------------------+ +``` + +2. Get the second element of the array whose key is "my. key" + +``` +mysql> SELECT get_json_string('{"k1":"v1", "my.key":["e1", "e2", "e3"]}', '$."my.key"[1]'); ++------------------------------------------------------------------------------+ +| get_json_string('{"k1":"v1", "my.key":["e1", "e2", "e3"]}', '$."my.key"[1]') | ++------------------------------------------------------------------------------+ +| e2 | ++------------------------------------------------------------------------------+ +``` + +3. Get the first element in an array whose secondary path is k1. key - > K2 +``` +mysql> SELECT get_json_string('{"k1.key":{"k2":["v1", "v2"]}}', '$."k1.key".k2[0]'); ++-----------------------------------------------------------------------+ +| get_json_string('{"k1.key":{"k2":["v1", "v2"]}}', '$."k1.key".k2[0]') | ++-----------------------------------------------------------------------+ +| v1 | ++-----------------------------------------------------------------------+ +``` + +4. Get all the values in the array where the key is "k1" +``` +mysql> SELECT get_json_string('[{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}]', '$.k1'); ++---------------------------------------------------------------------------------+ +| get_json_string('[{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}]', '$.k1') | ++---------------------------------------------------------------------------------+ +| ["v1","v3","v4"] | ++---------------------------------------------------------------------------------+ +``` +##keyword +GET_JSON_STRING,GET,JSON,STRING diff --git a/docs/en/sql-reference/sql-functions/string-functions/group_concat.md b/docs/en/sql-reference/sql-functions/string-functions/group_concat.md new file mode 100644 index 00000000000000..aa535261fd7978 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/group_concat.md @@ -0,0 +1,63 @@ +--- +{ + "title": "group_concat", + "language": "en" +} +--- + + + +# group_concat +## description +### Syntax + +`VARCHAR group_concat(VARCHAR str[, VARCHAR sep])` + + +This function is an aggregation function similar to sum (), and group_concat links multiple rows of results in the result set to a string. The second parameter, sep, is a connection symbol between strings, which can be omitted. This function usually needs to be used with group by statements. + +## example + +``` +mysql> select value from test; ++-------+ +| value | ++-------+ +| a | +| b | +| c | ++-------+ + +mysql> select group_concat(value) from test; ++-----------------------+ +| group_concat(`value`) | ++-----------------------+ +| a, b, c | ++-----------------------+ + +mysql> select group_concat(value, " ") from test; ++----------------------------+ +| group_concat(`value`, ' ') | ++----------------------------+ +| a b c | ++----------------------------+ +``` +##keyword +GROUP_CONCAT,GROUP,CONCAT diff --git a/docs/en/sql-reference/sql-functions/string-functions/instr.md b/docs/en/sql-reference/sql-functions/string-functions/instr.md new file mode 100644 index 00000000000000..1a2b2476b94342 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/instr.md @@ -0,0 +1,54 @@ +--- +{ + "title": "instr", + "language": "en" +} +--- + + + +# instr +## Description +### Syntax + +'INT INSR (WARCHAR STR, WARCHAR substrate)' + + +Returns the location where substr first appeared in str (counting from 1). If substr does not appear in str, return 0. + +## example + +``` +mysql> select instr("abc", "b"); ++-------------------+ +| instr('abc', 'b') | ++-------------------+ +| 2 | ++-------------------+ + +mysql> select instr("abc", "d"); ++-------------------+ +| instr('abc', 'd') | ++-------------------+ +| 0 | ++-------------------+ +``` +##keyword +INSTR diff --git a/docs/en/sql-reference/sql-functions/string-functions/lcase.md b/docs/en/sql-reference/sql-functions/string-functions/lcase.md new file mode 100644 index 00000000000000..2f53d8546fa0ee --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/lcase.md @@ -0,0 +1,37 @@ +--- +{ + "title": "lcase", + "language": "en" +} +--- + + + +# lcase +## Description +### Syntax + +`INT lcase (VARCHAR str)` + + +Consistent with `lower`. + +##keyword +LCASE diff --git a/docs/en/sql-reference/sql-functions/string-functions/left.md b/docs/en/sql-reference/sql-functions/string-functions/left.md new file mode 100644 index 00000000000000..074bc349be4039 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/left.md @@ -0,0 +1,47 @@ +--- +{ + "title": "left", + "language": "en" +} +--- + + + +# left +## Description +### Syntax + +'VARCHAR left (VARCHAR str)' + + +It returns the left part of a string of specified length + +## example + +``` +mysql> select left("Hello doris",5); ++------------------------+ +| left('Hello doris', 5) | ++------------------------+ +| Hello | ++------------------------+ +``` +##keyword +LEFT diff --git a/docs/en/sql-reference/sql-functions/string-functions/length.md b/docs/en/sql-reference/sql-functions/string-functions/length.md new file mode 100644 index 00000000000000..ff2c961d4b5781 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/length.md @@ -0,0 +1,54 @@ +--- +{ + "title": "length", + "language": "en" +} +--- + + + +# length +## Description +### Syntax + +'INT length (VARCHAR str)' + + +Returns the length of the string and the number of characters returned for multi-byte characters. For example, five two-byte width words return a length of 10. + +## example + +``` +mysql> select length("abc"); ++---------------+ +| length('abc') | ++---------------+ +| 3 | ++---------------+ + +mysql> select length("中国"); ++------------------+ +| length('中国') | ++------------------+ +| 6 | ++------------------+ +``` +##keyword +LENGTH diff --git a/docs/en/sql-reference/sql-functions/string-functions/locate.md b/docs/en/sql-reference/sql-functions/string-functions/locate.md new file mode 100644 index 00000000000000..e5c89857d3b30c --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/locate.md @@ -0,0 +1,61 @@ +--- +{ + "title": "locate", + "language": "en" +} +--- + + + +# locate +## Description +### Syntax + +'INT LOCATION (WARCHAR substrate, WARCHAR str [, INT pos]]' + + +Returns where substr appears in str (counting from 1). If the third parameter POS is specified, the position where substr appears is found from the string where STR starts with POS subscript. If not found, return 0 + +## example + +``` +mysql> SELECT LOCATE('bar', 'foobarbar'); ++----------------------------+ +| locate('bar', 'foobarbar') | ++----------------------------+ +| 4 | ++----------------------------+ + +mysql> SELECT LOCATE('xbar', 'foobar'); ++--------------------------+ +| locate('xbar', 'foobar') | ++--------------------------+ +| 0 | ++--------------------------+ + +mysql> SELECT LOCATE('bar', 'foobarbar', 5); ++-------------------------------+ +| locate('bar', 'foobarbar', 5) | ++-------------------------------+ +| 7 | ++-------------------------------+ +``` +##keyword +LOCATE diff --git a/docs/en/sql-reference/sql-functions/string-functions/lower.md b/docs/en/sql-reference/sql-functions/string-functions/lower.md new file mode 100644 index 00000000000000..7a855ee8926a68 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/lower.md @@ -0,0 +1,47 @@ +--- +{ + "title": "lower", + "language": "en" +} +--- + + + +# lower +## Description +### Syntax + +'INT lower (WARCHAR str)' + + +Convert all strings in parameters to lowercase + +## example + +``` +mysql> SELECT lower("AbC123"); ++-----------------+ +| lower('AbC123') | ++-----------------+ +| abc123 | ++-----------------+ +``` +##keyword +LOWER diff --git a/docs/en/sql-reference/sql-functions/string-functions/lpad.md b/docs/en/sql-reference/sql-functions/string-functions/lpad.md new file mode 100644 index 00000000000000..1751cffc453f2b --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/lpad.md @@ -0,0 +1,54 @@ +--- +{ + "title": "lpad", + "language": "en" +} +--- + + + +# lpad +## Description +### Syntax + +'VARCHAR lpad (VARCHAR str., INT len, VARCHAR pad)' + + +Returns a string of length len in str, starting with the initials. If len is longer than str, pad characters are added to STR until the length of the string reaches len. If len is less than str's length, the function is equivalent to truncating STR strings and returning only strings of len's length. + +## example + +``` +mysql> SELECT lpad("hi", 5, "xy"); ++---------------------+ +| lpad('hi', 5, 'xy') | ++---------------------+ +| xyxhi | ++---------------------+ + +mysql> SELECT lpad("hi", 1, "xy"); ++---------------------+ +| lpad('hi', 1, 'xy') | ++---------------------+ +| h | ++---------------------+ +``` +##keyword +LPAD diff --git a/docs/en/sql-reference/sql-functions/string-functions/ltrim.md b/docs/en/sql-reference/sql-functions/string-functions/ltrim.md new file mode 100644 index 00000000000000..d6f69debca8155 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/ltrim.md @@ -0,0 +1,47 @@ +--- +{ + "title": "ltrim", + "language": "en" +} +--- + + + +# ltrim +## Description +### Syntax + +'VARCHAR ltrim (VARCHAR str)' + + +Remove the space that appears continuously from the beginning of the parameter str + +## example + +``` +mysql> SELECT ltrim(' ab d'); ++------------------+ +| ltrim(' ab d') | ++------------------+ +| ab d | ++------------------+ +``` +##keyword +LTRIM diff --git a/docs/en/sql-reference/sql-functions/string-functions/money_format.md b/docs/en/sql-reference/sql-functions/string-functions/money_format.md new file mode 100644 index 00000000000000..f7f72eeac33f88 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/money_format.md @@ -0,0 +1,61 @@ +--- +{ + "title": "money_format", + "language": "en" +} +--- + + + +# money_format +## Description +### Syntax + +VARCHAR money format (Number) + + +The number is output in currency format, the integer part is separated by commas every three bits, and the decimal part is reserved for two bits. + +## example + +``` +mysql> select money_format(17014116); ++------------------------+ +| money_format(17014116) | ++------------------------+ +| 17,014,116.00 | ++------------------------+ + +mysql> select money_format(1123.456); ++------------------------+ +| money_format(1123.456) | ++------------------------+ +| 1,123.46 | ++------------------------+ + +mysql> select money_format(1123.4); ++----------------------+ +| money_format(1123.4) | ++----------------------+ +| 1,123.40 | ++----------------------+ +``` +##keyword +MONEY_FORMAT,MONEY,FORMAT diff --git a/docs/en/sql-reference/sql-functions/string-functions/null_or_empty.md b/docs/en/sql-reference/sql-functions/string-functions/null_or_empty.md new file mode 100644 index 00000000000000..e712fbec8474da --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/null_or_empty.md @@ -0,0 +1,60 @@ +--- +{ + "title": "null_or_empty", + "language": "en" +} +--- + + + +# null_or_empty +## description +### Syntax + +`BOOLEAN NULL_OR_EMPTY (VARCHAR str)` + +It returns true if the string is an empty string or NULL. Otherwise it returns false. + +## example + +``` +MySQL [(none)]> select null_or_empty(null); ++---------------------+ +| null_or_empty(NULL) | ++---------------------+ +| 1 | ++---------------------+ + +MySQL [(none)]> select null_or_empty(""); ++-------------------+ +| null_or_empty('') | ++-------------------+ +| 1 | ++-------------------+ + +MySQL [(none)]> select null_or_empty("a"); ++--------------------+ +| null_or_empty('a') | ++--------------------+ +| 0 | ++--------------------+ +``` +##keyword +NULL_OR_EMPTY \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/string-functions/regexp_extract.md b/docs/en/sql-reference/sql-functions/string-functions/regexp_extract.md new file mode 100644 index 00000000000000..0500bad385bfd0 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/regexp_extract.md @@ -0,0 +1,54 @@ +--- +{ + "title": "regexp_extract", + "language": "en" +} +--- + + + +# regexp_extract +## Description +### Syntax + +'VARCHAR regexp 'extract (VARCHAR str, VARCHAR pattern, int pos) + + +The string STR is matched regularly and the POS matching part which conforms to pattern is extracted. Patterns need to match exactly some part of the STR to return to the matching part of the pattern. If there is no match, return an empty string. + +## example + +``` +mysql> SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 1); ++-------------------------------------------------------------+ +| regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 1) | ++-------------------------------------------------------------+ +| b | ++-------------------------------------------------------------+ + +mysql> SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 2); ++-------------------------------------------------------------+ +| regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 2) | ++-------------------------------------------------------------+ +| d | ++-------------------------------------------------------------+ +``` +##keyword +REGEXP_EXTRACT,REGEXP,EXTRACT diff --git a/docs/en/sql-reference/sql-functions/string-functions/regexp_replace.md b/docs/en/sql-reference/sql-functions/string-functions/regexp_replace.md new file mode 100644 index 00000000000000..713f3c338cf4ac --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/regexp_replace.md @@ -0,0 +1,54 @@ +--- +{ + "title": "regexp_replace", + "language": "en" +} +--- + + + +# regexp_replace +## description +### Syntax + +`VARCHAR regexp_replace(VARCHAR str, VARCHAR pattern, VARCHAR repl) + + +Regular matching of STR strings, replacing the part hitting pattern with repl + +## example + +``` +mysql> SELECT regexp_replace('a b c', " ", "-"); ++-----------------------------------+ +| regexp_replace('a b c', ' ', '-') | ++-----------------------------------+ +| a-b-c | ++-----------------------------------+ + +mysql> SELECT regexp_replace('a b c','(b)','<\\1>'); ++----------------------------------------+ +| regexp_replace('a b c', '(b)', '<\1>') | ++----------------------------------------+ +| a c | ++----------------------------------------+ +``` +##keyword +REGEXP_REPLACE,REGEXP,REPLACE diff --git a/docs/en/sql-reference/sql-functions/string-functions/repeat.md b/docs/en/sql-reference/sql-functions/string-functions/repeat.md new file mode 100644 index 00000000000000..ec828596527bfe --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/repeat.md @@ -0,0 +1,54 @@ +--- +{ + "title": "repeat", + "language": "en" +} +--- + + + +# repeat +## Description +### Syntax + +'VARCHAR repeat (VARCHAR str, INT count) + + +Repeat the str of the string count times, return empty string when count is less than 1, return NULL when str, count is any NULL + +## example + +``` +mysql> SELECT repeat("a", 3); ++----------------+ +| repeat('a', 3) | ++----------------+ +| aaa | ++----------------+ + +mysql> SELECT repeat("a", -1); ++-----------------+ +| repeat('a', -1) | ++-----------------+ +| | ++-----------------+ +``` +##keyword +REPEAT, diff --git a/docs/en/sql-reference/sql-functions/string-functions/right.md b/docs/en/sql-reference/sql-functions/string-functions/right.md new file mode 100644 index 00000000000000..d8e356a9666fac --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/right.md @@ -0,0 +1,47 @@ +--- +{ + "title": "right", + "language": "en" +} +--- + + + +# right +## Description +### Syntax + +'VARCHAR RIGHT (VARCHAR STR)' + + +It returns the right part of a string of specified length + +## example + +``` +mysql> select right("Hello doris",5); ++-------------------------+ +| right('Hello doris', 5) | ++-------------------------+ +| doris | ++-------------------------+ +``` +##keyword +RIGHT diff --git a/docs/en/sql-reference/sql-functions/string-functions/split_part.md b/docs/en/sql-reference/sql-functions/string-functions/split_part.md new file mode 100644 index 00000000000000..e7ff351807dd39 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/split_part.md @@ -0,0 +1,69 @@ +--- +{ + "title": "split_part", + "language": "en" +} +--- + + + +# split_part +## Description +### Syntax + +'VARCHAR split party (VARCHAR content, VARCHAR delimiter, INT field)' + + +Returns the specified partition (counting from the beginning) by splitting the string according to the partitioner. + +## example + +``` +mysql> select split_part("hello world", " ", 1); ++----------------------------------+ +| split_part('hello world', ' ', 1) | ++----------------------------------+ +| hello | ++----------------------------------+ + + +mysql> select split_part("hello world", " ", 2); ++----------------------------------+ +| split_part('hello world', ' ', 2) | ++----------------------------------+ +| world | ++----------------------------------+ + +mysql> select split_part("2019年7月8号", "月", 1); ++-----------------------------------------+ +| split_part('2019年7月8号', '月', 1) | ++-----------------------------------------+ +| 2019年7 | ++-----------------------------------------+ + +mysql> select split_part("abca", "a", 1); ++----------------------------+ +| split_part('abca', 'a', 1) | ++----------------------------+ +| | ++----------------------------+ +``` +##keyword +SPLIT_PART,SPLIT,PART diff --git a/docs/en/sql-reference/sql-functions/string-functions/starts_with.md b/docs/en/sql-reference/sql-functions/string-functions/starts_with.md new file mode 100644 index 00000000000000..c01e6b9395fd47 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/starts_with.md @@ -0,0 +1,54 @@ +--- +{ + "title": "starts_with", + "language": "en" +} +--- + + + +# starts_with +## Description +### Syntax + +`BOOLEAN STARTS_WITH (VARCHAR str, VARCHAR prefix)` + +It returns true if the string starts with the specified prefix, otherwise it returns false. +If any parameter is NULL, it returns NULL. + +## example + +``` +MySQL [(none)]> select starts_with("hello world","hello"); ++-------------------------------------+ +| starts_with('hello world', 'hello') | ++-------------------------------------+ +| 1 | ++-------------------------------------+ + +MySQL [(none)]> select starts_with("hello world","world"); ++-------------------------------------+ +| starts_with('hello world', 'world') | ++-------------------------------------+ +| 0 | ++-------------------------------------+ +``` +##keyword +STARTS_WITH \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/string-functions/strleft.md b/docs/en/sql-reference/sql-functions/string-functions/strleft.md new file mode 100644 index 00000000000000..81d6ee8023f6e5 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/strleft.md @@ -0,0 +1,47 @@ +--- +{ + "title": "strleft", + "language": "en" +} +--- + + + +# strleft +## Description +### Syntax + +'VARCHAR STRAIGHT (VARCHAR STR)' + + +It returns the left part of a string of specified length + +## example + +``` +mysql> select strleft("Hello doris",5); ++------------------------+ +| strleft('Hello doris', 5) | ++------------------------+ +| Hello | ++------------------------+ +``` +##keyword +STRLEFT diff --git a/docs/en/sql-reference/sql-functions/string-functions/strright.md b/docs/en/sql-reference/sql-functions/string-functions/strright.md new file mode 100644 index 00000000000000..25707cdbc001e9 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/strright.md @@ -0,0 +1,48 @@ +--- +{ + "title": "strright", + "language": "en" +} +--- + + + + +# strright +## Description +### Syntax + +'VARCHAR strright (VARCHAR str)' + + +It returns the right part of a string of specified length + +## example + +``` +mysql> select strright("Hello doris",5); ++-------------------------+ +| strright('Hello doris', 5) | ++-------------------------+ +| doris | ++-------------------------+ +``` +##keyword +STRRIGHT diff --git a/docs/en/sql-reference/sql-statements/Account Management/CREATE ROLE.md b/docs/en/sql-reference/sql-statements/Account Management/CREATE ROLE.md new file mode 100644 index 00000000000000..1ee11b440c42e5 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/CREATE ROLE.md @@ -0,0 +1,45 @@ +--- +{ + "title": "CREATE ROLE", + "language": "en" +} +--- + + + +# CREATE ROLE +## Description +The statement user creates a role + +Grammar: +CREATE ROLE role1; + +This statement creates an unauthorized role that can be subsequently granted permission through the GRANT command. + +## example + +1. Create a role + +CREATE ROLE role1; + +## keyword +CREATE, ROLE + + diff --git a/docs/en/sql-reference/sql-statements/Account Management/CREATE USER.md b/docs/en/sql-reference/sql-statements/Account Management/CREATE USER.md new file mode 100644 index 00000000000000..1cc0e90309f138 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/CREATE USER.md @@ -0,0 +1,74 @@ +--- +{ + "title": "CREATE USER", + "language": "en" +} +--- + + + +# CREATE USER +##Description + +Syntax: + +CREATE USER user_identity [IDENTIFIED BY 'password'] [DEFAULT ROLE 'role_name'] + +user_identity: +'user_name'@'host' + +The CREATE USER command is used to create a Doris user. In Doris, a user_identity uniquely identifies a user. User_identity consists of two parts, user_name and host, where username is the user name. The host identifies the host address where the client connects. The host part can use% for fuzzy matching. If no host is specified, the default is'%', which means that the user can connect to Doris from any host. + +The host part can also be specified as a domain with the grammar:'user_name'@['domain']. Even if surrounded by brackets, Doris will think of it as a domain and try to parse its IP address. At present, it only supports BNS analysis within Baidu. + +If a role (ROLE) is specified, the permissions that the role has are automatically granted to the newly created user. If not specified, the user defaults to having no permissions. The specified ROLE must already exist. + +## example + +1. Create a passwordless user (without specifying host, it is equivalent to Jack @'%') + +CREATE USER 'jack'; + +2. Create a password user that allows login from'172.10.1.10' + +CREATE USER jack@'172.10.1.10' IDENTIFIED BY '123456'; + +3. To avoid passing plaintext, use case 2 can also be created in the following way + +CREATE USER jack@'172.10.1.10' IDENTIFIED BY PASSWORD '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9'; + +Later encrypted content can be obtained through PASSWORD (), for example: + +SELECT PASSWORD('123456'); + +4. Create a user who is allowed to log in from the'192.168'subnet and specify its role as example_role + +CREATE USER 'jack'@'192.168.%' DEFAULT ROLE 'example_role'; + +5. Create a user who is allowed to log in from the domain name'example_domain'. + +CREATE USER 'jack'@['example_domain'] IDENTIFIED BY '12345'; + +6. Create a user and specify a role + +CREATE USER 'jack'@'%' IDENTIFIED BY '12345' DEFAULT ROLE 'my_role'; + +## keyword +CREATE, USER diff --git a/docs/en/sql-reference/sql-statements/Account Management/DROP ROLE.md b/docs/en/sql-reference/sql-statements/Account Management/DROP ROLE.md new file mode 100644 index 00000000000000..6966f8241a7649 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/DROP ROLE.md @@ -0,0 +1,43 @@ +--- +{ + "title": "DROP ROLE", + "language": "en" +} +--- + + + +# DROP ROLE +## Description +The statement user deletes a role + +Grammar: +DROP ROLE role1; + +Deleting a role does not affect the permissions of users who previously belonged to that role. It is only equivalent to decoupling the role from the user. The permissions that the user has obtained from the role will not change. + +## example + +1. Delete a role + +DROP ROLE role1; + +## keyword +DROP, ROLE diff --git a/docs/en/sql-reference/sql-statements/Account Management/DROP USER.md b/docs/en/sql-reference/sql-statements/Account Management/DROP USER.md new file mode 100644 index 00000000000000..8860140ddcc4b5 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/DROP USER.md @@ -0,0 +1,49 @@ +--- +{ + "title": "DROP USER", + "language": "en" +} +--- + + + +# DROP USER +## Description + +Syntax: + + DROP USER 'user_identity' + + `user_identity`: + + user@'host' + user@['domain'] + + Drop a specified user identity. + +## example + +1. Delete user jack@'192.%' + + DROP USER 'jack'@'192.%' + +## keyword + + DROP, USER diff --git a/docs/en/sql-reference/sql-statements/Account Management/GRANT.md b/docs/en/sql-reference/sql-statements/Account Management/GRANT.md new file mode 100644 index 00000000000000..8c2ae659d2de82 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/GRANT.md @@ -0,0 +1,81 @@ +--- +{ + "title": "Grant", + "language": "en" +} +--- + + + +# Grant +## Description + +The GRANT command is used to give the specified user or role the specified permissions. + +Syntax: + +GRANT privilege_list ON db_name[.tbl_name] TO user_identity [ROLE role_name] + + +Privilege_list is a list of permissions that need to be granted, separated by commas. Currently Doris supports the following permissions: + +NODE_PRIV: Operational privileges of cluster nodes, including operation of nodes'up and down lines. Only root users have this privilege and can not be given to other users. +ADMIN_PRIV: All rights except NODE_PRIV. +GRANT_PRIV: Permission to operate permissions. Including the creation and deletion of users, roles, authorization and revocation, password settings and so on. +SELECT_PRIV: Read permissions for specified libraries or tables +LOAD_PRIV: Import permissions for specified libraries or tables +ALTER_PRIV: schema change permissions for specified libraries or tables +CREATE_PRIV: Creation permissions for specified libraries or tables +DROP_PRIV: Delete permissions for specified libraries or tables + +旧版权限中的 ALL 和 READ_WRITE 会被转换成：SELECT_PRIV,LOAD_PRIV,ALTER_PRIV,CREATE_PRIV,DROP_PRIV； +READ_ONLY is converted to SELECT_PRIV. + +Db_name [.tbl_name] supports the following three forms: + +1. *. * permissions can be applied to all libraries and all tables in them +2. db. * permissions can be applied to all tables under the specified library +3. db.tbl permissions can be applied to specified tables under specified Libraries + +The libraries or tables specified here can be non-existent libraries and tables. + +user_identity： + +The user_identity syntax here is the same as CREATE USER. And you must create user_identity for the user using CREATE USER. The host in user_identity can be a domain name. If it is a domain name, the validity time of permissions may be delayed by about one minute. + +You can also grant permissions to the specified ROLE, which is automatically created if the specified ROLE does not exist. + +## example + +1. Grant permissions to all libraries and tables to users + +GRANT SELECT_PRIV ON *.* TO 'jack'@'%'; + +2. Grant permissions to specified library tables to users + +GRANT SELECT_PRIV,ALTER_PRIV,LOAD_PRIV ON db1.tbl1 TO 'jack'@'192.8.%'; + +3. Grant permissions to specified library tables to roles + +GRANT LOAD_PRIV ON db1.* TO ROLE 'my_role'; + +## keyword +GRANT + diff --git a/docs/en/sql-reference/sql-statements/Account Management/REVOKE.md b/docs/en/sql-reference/sql-statements/Account Management/REVOKE.md new file mode 100644 index 00000000000000..8c0734fb21f3b8 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/REVOKE.md @@ -0,0 +1,48 @@ +--- +{ + "title": "REVOKE", + "language": "en" +} +--- + + + +# REVOKE +## Description + +The REVOKE command is used to revoke the rights specified by the specified user or role. +Syntax +REVOKE privilege_list ON db_name[.tbl_name] FROM user_identity [ROLE role_name] + +user_identity： + +The user_identity syntax here is the same as CREATE USER. And you must create user_identity for the user using CREATE USER. The host in user_identity can be a domain name. If it is a domain name, the revocation time of permission may be delayed by about one minute. + +You can also revoke the permission of the specified ROLE, which must exist for execution. + +## example + +1. Revoke the rights of user Jack database testDb + +REVOKE SELECT_PRIV ON db1.* FROM 'jack'@'192.%'; + +## keyword + +REVOKE diff --git a/docs/en/sql-reference/sql-statements/Account Management/SET PASSWORD.md b/docs/en/sql-reference/sql-statements/Account Management/SET PASSWORD.md new file mode 100644 index 00000000000000..37f4b2e25f6135 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/SET PASSWORD.md @@ -0,0 +1,55 @@ +--- +{ + "title": "SET PASSWORD", + "language": "en" +} +--- + + + +# SET PASSWORD +## Description + +Syntax: + +SET PASSWORD [FOR user_identity] = +[PASSWORD('plain password')]|['hashed password'] + +The SET PASSWORD command can be used to modify a user's login password. If the [FOR user_identity] field does not exist, modify the password of the current user. + +Note that the user_identity here must match exactly the user_identity specified when creating a user using CREATE USER, otherwise the user will be reported as non-existent. If user_identity is not specified, the current user is'username'@'ip', which may not match any user_identity. The current user can be viewed through SHOW GRANTS. + +PASSWORD () input is a plaintext password, and direct use of strings, you need to pass the encrypted password. +If you change the password of other users, you need to have administrator privileges. + +## example + +1. Modify the password of the current user + +SET PASSWORD = PASSWORD('123456') +SET PASSWORD = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' + +2. Modify the specified user password + +SET PASSWORD FOR 'jack'@'192.%' = PASSWORD('123456') +SET PASSWORD FOR 'jack'@['domain'] = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' + +## keyword +SET, PASSWORD diff --git a/docs/en/sql-reference/sql-statements/Account Management/SET PROPERTY.md b/docs/en/sql-reference/sql-statements/Account Management/SET PROPERTY.md new file mode 100644 index 00000000000000..f524ff00c0215d --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/SET PROPERTY.md @@ -0,0 +1,82 @@ +--- +{ + "title": "SET PROPERTY", + "language": "en" +} +--- + + + +# SET PROPERTY +## Description + +Syntax: + +SET PROPERTY [FOR 'user'] 'key' = 'value' [, 'key' = 'value'] + +Set user attributes, including resources allocated to users, import cluster, etc. The user attributes set here are for user, not user_identity. That is to say, if two users'jack'@'%' and'jack'@'192%'are created through the CREATE USER statement, the SET PROPERTY statement can only be used for the jack user, not'jack'@'%' or'jack'@'192%' + +Importing cluster is only applicable to Baidu internal users. + +key: + +Super user rights: +Max_user_connections: Maximum number of connections. +resource.cpu_share: cpu资源分配。 +Load_cluster. {cluster_name}. priority: assigns priority to a specified cluster, which can be HIGH or NORMAL + +Ordinary user rights: +Quota.normal: Resource allocation at the normal level. +Quota.high: Resource allocation at the high level. +Quota.low: Resource allocation at low level. + +Load_cluster. {cluster_name}. hadoop_palo_path: The Hadoop directory used by Palo needs to store ETL programs and intermediate data generated by ETL for Palo to import. After the import is completed, the intermediate data will be automatically cleaned up, and the ETL program will be automatically reserved for next use. +Load_cluster. {cluster_name}. hadoop_configs: configuration of hadoop, where fs. default. name, mapred. job. tracker, hadoop. job. UGI must be filled in. +Load ucluster. {cluster name}. hadoop port: Hadoop HDFS name node http} +Default_load_cluster: The default import cluster. + +## example + +1. Modify the maximum number of user jacks to 1000 +SET PROPERTY FOR 'jack' 'max_user_connections' = '1000'; + +2. Modify the cpu_share of user Jack to 1000 +SET PROPERTY FOR 'jack' 'resource.cpu_share' = '1000'; + +3. Modify the weight of the normal group of Jack users +Set property for'jack''quota. normal' = 400'; + +4. Add import cluster for user jack +SET PROPERTY FOR 'jack' +'load 'cluster.{cluster name}.hadoop'u palo path' ='/user /palo /palo path', +'load_cluster.{cluster_name}.hadoop_configs' = 'fs.default.name=hdfs://dpp.cluster.com:port;mapred.job.tracker=dpp.cluster.com:port;hadoop.job.ugi=user,password;mapred.job.queue.name=job_queue_name_in_hadoop;mapred.job.priority=HIGH;'; + +5. Delete the import cluster under user jack. +SET PROPERTY FOR 'jack' 'load_cluster.{cluster_name}' = ''; + +6. Modify user jack's default import cluster +SET PROPERTY FOR 'jack' 'default_load_cluster' = '{cluster_name}'; + +7. Modify the cluster priority of user Jack to HIGH +SET PROPERTY FOR 'jack' 'load_cluster.{cluster_name}.priority' = 'HIGH'; + +## keyword +SET, PROPERTY + diff --git a/docs/en/sql-reference/sql-statements/Account Management/SHOW GRANTS.md b/docs/en/sql-reference/sql-statements/Account Management/SHOW GRANTS.md new file mode 100644 index 00000000000000..a12ac9bf913590 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/SHOW GRANTS.md @@ -0,0 +1,56 @@ +--- +{ + "title": "SHOW GRANTS", + "language": "en" +} +--- + + + +# SHOW GRANTS +## Description + +This statement is used to view user rights. + +Grammar: +SHOW [ALL] GRANTS [FOR user_identity]; + +Explain: +1. SHOW ALL GRANTS can view the privileges of all users. +2. If you specify user_identity, view the permissions of the specified user. And the user_identity must be created for the CREATE USER command. +3. If you do not specify user_identity, view the permissions of the current user. + + +## example + +1. View all user rights information + +SHOW ALL GRANTS; + +2. View the permissions of the specified user + +SHOW GRANTS FOR jack@'%'; + +3. View the permissions of the current user + +SHOW GRANTS; + +## keyword +SHOW, GRANTS diff --git a/docs/en/sql-reference/sql-statements/Account Management/SHOW ROLES.md b/docs/en/sql-reference/sql-statements/Account Management/SHOW ROLES.md new file mode 100644 index 00000000000000..a27d30dbd1ac83 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Account Management/SHOW ROLES.md @@ -0,0 +1,41 @@ +--- +{ + "title": "SHOW ROLES", + "language": "en" +} +--- + + + +# SHOW ROLES +## Description +This statement is used to display all created role information, including the role name, the user included, and the permissions. + +Grammar: +SHOW ROLES; + +## example + +1. View the created roles: + +SHOW ROLES; + +## keyword +SHOW,ROLES diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md new file mode 100644 index 00000000000000..db001c90b3d03b --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md @@ -0,0 +1,47 @@ +--- +{ + "title": "ADMIN CANCEL REPAIR", + "language": "en" +} +--- + + + +# ADMIN CANCEL REPAIR +## Description + +This statement is used to cancel repairing a specified table or partition with high priority + +Grammar: + +ADMIN CANCEL REPAIR TABLE table_name[ PARTITION (p1,...)]; + +Explain: + +1. This statement only indicates that the system no longer repairs fragmented copies of specified tables or partitions with high priority. The system will still repair the copy by default scheduling. + +## example + +1. Cancel High Priority Repair + +ADMIN CANCEL REPAIR TABLE tbl PARTITION(p1); + +## keyword +ADMIN,CANCEL,REPAIR diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md new file mode 100644 index 00000000000000..8301791b4ebacd --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md @@ -0,0 +1,57 @@ +--- +{ + "title": "ADMIN CHECK TABLET", + "language": "en" +} +--- + + + +# ADMIN CHECK TABLET +## description + +This statement is used to perform a specified check operation on a list of tablets. + +Syntax： + +``` +ADMIN CHECK TABLE (tablet_id1, tablet_id2, ...) +PROPERTIES("type" = "..."); +``` + +说明： + +1. You must specify the list of tablet ids and the "type" property in PROPERTIES. +2. Currently "type" only supports: + + * consistency: Check the consistency of the replicas of the tablet. This command is asynchronous. After sending, Doris will start to perform the consistency check job of the corresponding tablet. The final result will be reflected in the "InconsistentTabletNum" column in the result of `SHOW PROC" / statistic "; + +## example + +1. Perform a replica consistency check on a specified set of tablets + + ``` + ADMIN CHECK TABLET (10000, 10001) + PROPERTIES("type" = "consistency"); + ``` + +## keyword + + ADMIN,CHECK,TABLET diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN REPAIR.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN REPAIR.md new file mode 100644 index 00000000000000..8cb037be6537c9 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN REPAIR.md @@ -0,0 +1,52 @@ +--- +{ + "title": "ADMIN REPAIR", + "language": "en" +} +--- + + + +# ADMIN REPAIR +## Description + +This statement is used to try to fix the specified table or partition first + +Grammar: + +ADMIN REPAIR TABLE table_name[ PARTITION (p1,...)] + +Explain: + +1. This statement only means that the system attempts to repair a fragmented copy of a specified table or partition with high priority, and it is not guaranteed to be successful. Users can view the repair status through the ADMIN SHOW REPLICA STATUS command. +2. The default timeout is 14400 seconds (4 hours). Timeout means that the system will no longer repair fragmented copies of specified tables or partitions with high priority. The command settings need to be reused. + +## example + +1. Attempt to fix the specified table + +ADMIN REPAIR TABLE tbl1; + +2. Attempt to fix the specified partition + +ADMIN REPAIR TABLE tbl1 PARTITION (p1, p2); + +## keyword +ADMIN,REPAIR diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md new file mode 100644 index 00000000000000..02f19b9fba87a3 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md @@ -0,0 +1,44 @@ +--- +{ + "title": "ADMIN SET CONFIG", + "language": "en" +} +--- + + + +# ADMIN SET CONFIG +## Description + +This statement is used to set the configuration items for the cluster (currently only the configuration items for setting FE are supported). +Settable configuration items can be viewed through AMDIN SHOW FRONTEND CONFIG; commands. + +Grammar: + +ADMIN SET FRONTEND CONFIG ("key" = "value"); + +## example + +1. "disable balance" true + +ADMIN SET FRONTEND CONFIG ("disable_balance" = "true"); + +## keyword +ADMIN,SET,CONFIG diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md new file mode 100644 index 00000000000000..03764b53d287a9 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md @@ -0,0 +1,62 @@ +--- +{ + "title": "ADMIN SET REPLICA STATUS", + "language": "en" +} +--- + + + +# ADMIN SET REPLICA STATUS +## description + + This commend is used to set the status of the specified replica. +    This command is currently only used to manually set the status of some replicas to BAD or OK, allowing the system to automatically repair these replicas. + + Syntax: + + ADMIN SET REPLICA STATUS + PROPERTIES ("key" = "value", ...); + + The following attributes are currently supported: + "tablet_id": required. Specify a Tablet Id. + "backend_id": required. Specify a Backend Id. + "status": required. Specify the status. Only "bad" and "ok" are currently supported. + + If the specified replica does not exist or the status is already bad or ok, it will be ignored. + + Notice: + + Replica set to Bad status may be dropped immediately, please proceed with caution. + +## example + + 1. Set the replica status of tablet 10003 on BE 10001 to bad. + + ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "10003", "backend_id" = "10001", "status" = "bad"); + + 2. Set the replica status of tablet 10003 on BE 10001 to ok. + + ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "10003", "backend_id" = "10001", "status" = "ok"); + +## keyword + + ADMIN,SET,REPLICA,STATUS + diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md new file mode 100644 index 00000000000000..02fa5ee7c99914 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md @@ -0,0 +1,53 @@ +--- +{ + "title": "ADMIN SHOW CONFIG", + "language": "en" +} +--- + + + +# ADMIN SHOW CONFIG +## Description + +This statement is used to show the configuration of the current cluster (currently only supporting the display of FE configuration items) + +Grammar: + +ADMIN SHOW FRONTEND CONFIG; + +Explain: + +The implications of the results are as follows: +1. Key: Configuration item name +2. Value: Configuration item value +3. Type: Configuration item type +4. IsMutable： 是否可以通过 ADMIN SET CONFIG 命令设置 +5. MasterOnly： 是否仅适用于 Master FE +6. Comment: Configuration Item Description + +## example + +1. View the configuration of the current FE node + +ADMIN SHOW FRONTEND CONFIG; + +## keyword +ADMIN,SHOW,CONFIG diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md new file mode 100644 index 00000000000000..b521ee0889ff34 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md @@ -0,0 +1,51 @@ +--- +{ + "title": "ADMIN SHOW REPLICA DISTRIBUTION", + "language": "en" +} +--- + + + +# ADMIN SHOW REPLICA DISTRIBUTION +## Description + +This statement is used to show the distribution status of a table or partition replica + +Grammar: + +ADMIN SHOW REPLICA DISTRIBUTION FROM [db_name.]tbl_name [PARTITION (p1, ...)]; + +Explain: + +The Graph column in the result shows the distribution ratio of replicas graphically + +## example + +1. View the distribution of replicas of tables + +ADMIN SHOW REPLICA DISTRIBUTION FROM tbl1; + +2. View the distribution of copies of partitions in the table + +ADMIN SHOW REPLICA DISTRIBUTION FROM db1.tbl1 PARTITION(p1, p2); + +## keyword +ADMIN,SHOW,REPLICA,DISTRIBUTION diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md new file mode 100644 index 00000000000000..26486e9f5c02a8 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md @@ -0,0 +1,64 @@ +--- +{ + "title": "ADMIN SHOW REPLICA STATUS", + "language": "en" +} +--- + + + +# ADMIN SHOW REPLICA STATUS +## Description + +This statement is used to display copy status information for a table or partition + +Grammar: + +ADMIN SHOW REPLICA STATUS FROM [dbu name.]tbl name [PARTITION (p1,...)] +[where_clause]; + +where_clause: +WHERE STATUS [!]= "replica_status" + +Reply status: +OK: Replica 22788;'20581;' 29366;'24577; +DEAD: The Backend of replica is not available +VERSION_ERROR: The replica data version is missing +SCHEMA ERROR: replica schema hash +MISSING: replica does not exist + +## example + +1. View the status of all copies of the table + +ADMIN SHOW REPLICA STATUS FROM db1.tbl1; + +2. View a copy of a partition state of the table as VERSION_ERROR + +ADMIN SHOW REPLICA STATUS FROM tbl1 PARTITION (p1, p2) + + +3. Check all unhealthy copies of the table + +ADMIN SHOW REPLICA STATUS FROM tbl1 +WHERE STATUS != "OK"; + +## keyword +ADMIN,SHOW,REPLICA,STATUS diff --git a/docs/en/sql-reference/sql-statements/Administration/ALTER CLUSTER.md b/docs/en/sql-reference/sql-statements/Administration/ALTER CLUSTER.md new file mode 100644 index 00000000000000..618f033b686529 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ALTER CLUSTER.md @@ -0,0 +1,53 @@ +--- +{ + "title": "ALTER CLUSTER", + "language": "en" +} +--- + + + +# ALTER CLUSTER +## description + +This statement is used to update the logical cluster. Administrator privileges are required + +grammar + +ALTER CLUSTER cluster_name PROPERTIES ("key"="value", ...); + +1. Scaling, scaling (according to the number of be existing in the cluster, large is scaling, small is scaling), scaling for synchronous operation, scaling for asynchronous operation, through the state of backend can be known whether the scaling is completed. + +Proerties ("Instrume = Unum"= "3") + +Instancefn Microsoft Yahei + +## example + +1. Reduce the number of be of logical cluster test_cluster containing 3 be by 2. + +ALTER CLUSTER test_cluster PROPERTIES ("instance_num"="2"); + +2. Expansion, increase the number of be of logical cluster test_cluster containing 3 be to 4 + +ALTER CLUSTER test_cluster PROPERTIES ("instance_num"="4"); + +## keyword +ALTER,CLUSTER diff --git a/docs/en/sql-reference/sql-statements/Administration/ALTER SYSTEM.md b/docs/en/sql-reference/sql-statements/Administration/ALTER SYSTEM.md new file mode 100644 index 00000000000000..79844775754c40 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ALTER SYSTEM.md @@ -0,0 +1,119 @@ +--- +{ + "title": "ALTER SYSTEM", + "language": "en" +} +--- + + + +# ALTER SYSTEM +## Description + +This statement is used to operate on nodes in a system. (Administrator only!) +Grammar: +1) Adding nodes (without multi-tenant functionality, add in this way) +ALTER SYSTEM ADD BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; +2) Adding idle nodes (that is, adding BACKEND that does not belong to any cluster) +ALTER SYSTEM ADD FREE BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; +3) Adding nodes to a cluster +ALTER SYSTEM ADD BACKEND TO cluster_name "host:heartbeat_port"[,"host:heartbeat_port"...]; +4) Delete nodes +ALTER SYSTEM DROP BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; +5) Node offline +ALTER SYSTEM DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; +6)226;- 21152;-Broker +ALTER SYSTEM ADD BROKER broker_name "host:port"[,"host:port"...]; +(7) 20943;"23569;" Broker +ALTER SYSTEM DROP BROKER broker_name "host:port"[,"host:port"...]; +8) Delete all Brokers +ALTER SYSTEM DROP ALL BROKER broker_name +9) Set up a Load error hub for centralized display of import error information +ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES ("key" = "value"[, ...]); + +Explain: +1) Host can be hostname or IP address +2) heartbeat_port is the heartbeat port of the node +3) Adding and deleting nodes are synchronous operations. These two operations do not take into account the existing data on the node, the node is directly deleted from the metadata, please use cautiously. +4) Node offline operations are used to secure offline nodes. This operation is asynchronous. If successful, the node will eventually be removed from the metadata. If it fails, the offline will not be completed. +5) The downline operation of the node can be cancelled manually. See CANCEL DECOMMISSION for details +6) Load error hub: +Currently, two types of Hub are supported: Mysql and Broker. You need to specify "type" = "mysql" or "type" = "broker" in PROPERTIES. +If you need to delete the current load error hub, you can set type to null. +1) When using the Mysql type, the error information generated when importing will be inserted into the specified MySQL library table, and then the error information can be viewed directly through the show load warnings statement. + +Hub of Mysql type needs to specify the following parameters: +guest guest +port：mysql port +user：mysql user +password：mysql password +database mysql database +table：mysql table + +2) When the Broker type is used, the error information generated when importing will form a file and be written to the designated remote storage system through the broker. Make sure that the corresponding broker is deployed +Hub of Broker type needs to specify the following parameters: +Broker: Name of broker +Path: Remote Storage Path +Other properties: Other information necessary to access remote storage, such as authentication information. + +## example + +1. Add a node +ALTER SYSTEM ADD BACKEND "host:port"; + +2. Adding an idle node +ALTER SYSTEM ADD FREE BACKEND "host:port"; + +3. Delete two nodes +ALTER SYSTEM DROP BACKEND "host1:port", "host2:port"; + +4. Two downline nodes +ALTER SYSTEM DECOMMISSION BACKEND "host1:port", "host2:port"; + +5. Add two Hdfs Broker +ALTER SYSTEM ADD BROKER hdfs "host1:port", "host2:port"; + +6. Add a load error hub of Mysql type +ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES +("type"= "mysql", +"host" = "192.168.1.17" +"port" = "3306", +"User" = "my" name, +"password" = "my_passwd", +"database" = "doris_load", +"table" = "load_errors" +); + +7. 添加一个 Broker 类型的 load error hub +ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES +("type"= "broker", +"Name" = BOS, +"path" = "bos://backup-cmy/logs", +"bosu endpoint" ="http://gz.bcebos.com", +"bos_accesskey" = "069fc278xxxxxx24ddb522", +"bos_secret_accesskey"="700adb0c6xxxxxx74d59eaa980a" +); + +8. Delete the current load error hub +ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES +("type"= "null"); + +## keyword +AGE,SYSTEM,BACKGROUND,BROKER,FREE diff --git a/docs/en/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md b/docs/en/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md new file mode 100644 index 00000000000000..6965e6b80dbbca --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md @@ -0,0 +1,40 @@ +--- +{ + "title": "CANCEL DECOMMISSION", + "language": "en" +} +--- + + + +# CANCEL DECOMMISSION +## Description + +This statement is used to undo a node's offline operation. (Administrator only!) +Grammar: +CANCEL DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; + +## example + +1. Cancel the offline operation of two nodes: +CANCEL DECOMMISSION BACKEND "host1:port", "host2:port"; + +## keyword +CANCEL,DECOMMISSION,BACKEND diff --git a/docs/en/sql-reference/sql-statements/Administration/CREATE CLUSTER.md b/docs/en/sql-reference/sql-statements/Administration/CREATE CLUSTER.md new file mode 100644 index 00000000000000..b0c0cfa24704ba --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/CREATE CLUSTER.md @@ -0,0 +1,61 @@ +--- +{ + "title": "CREATE CLUSTER", + "language": "en" +} +--- + + + +# CREATE CLUSTER +## Description + +This statement is used to create a new logical cluster, requiring administrator privileges. If you don't use multiple tenants, create a cluster named default_cluster directly. Otherwise, create a cluster with a custom name. + +grammar + +CREATE CLUSTER [IF NOT EXISTS] cluster_name + +PROPERTIES ("key"="value", ...) + +IDENTIFIED BY 'password' + +1. PROPERTIES + +Specify attributes of logical clusters + +PROPERTIES ("instance_num" = "3") + +Instancefn Microsoft Yahei + +2. Identify by'password'each logical cluster contains a superuser whose password must be specified when creating a logical cluster + +## example + +1. Create a new test_cluster with three be nodes and specify its superuser password + +CREATE CLUSTER test_cluster PROPERTIES("instance_num"="3") IDENTIFIED BY 'test'; + +2. Create a new default_cluster with three be nodes (no multi-tenant is used) and specify its superuser password + +CREATE CLUSTER default_cluster PROPERTIES("instance_num"="3") IDENTIFIED BY 'test'; + +## keyword +CREATE,CLUSTER diff --git a/docs/en/sql-reference/sql-statements/Administration/CREATE FILE.md b/docs/en/sql-reference/sql-statements/Administration/CREATE FILE.md new file mode 100644 index 00000000000000..b428b01e5d42ba --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/CREATE FILE.md @@ -0,0 +1,76 @@ +--- +{ + "title": "CREATE FILE", + "language": "en" +} +--- + + + +# CREATE FILE +## Description + +This statement is used to create and upload a file to the Doris cluster. +This function is usually used to manage files that need to be used in some other commands, such as certificates, public key, private key, etc. + +This command can be executed by users with amdin privileges only. +A file belongs to a database. This file can be used by users who have access to database. + +The size of a single file is limited to 1MB. +A Doris cluster uploads up to 100 files. + +Grammar: + +CREATE FILE "File name" [IN database] +[properties] + +Explain: +File_name: Custom file name. +Database: The file belongs to a db, and if not specified, the DB of the current session is used. +properties 支持以下参数: + +Url: Must. Specify a download path for a file. Currently only unauthenticated HTTP download paths are supported. When the command line succeeds, the file will be saved in Doris and the URL will no longer be required. +Catalog: Yes. The classification name of the file can be customized. But in some commands, files in the specified catalog are looked up. For example, in a routine import, when the data source is kafka, the file under the name of catalog is looked up. +Md5: Optional. MD5 of the file. If specified, it will be checked after downloading the file. + +## example + +1. Create a file ca. pem, categorized as Kafka + +CREATE FILE "ca.pem" +PROPERTIES +( +"url" ="https://test.bj.bcebos.com /kafka -key /ca.pem", +"catalog" = "kafka" +); + +2. Create the file client. key, categorized as my_catalog + +CREATE FILE "client.key" +IN my database +PROPERTIES +( +"url" ="https://test.bj.bcebos.com /kafka -key /client.key", +"catalog" = "my_catalog", +"md5"= "b5bb901bf1099205b39a46ac3557dd9" +); + +## keyword +CREATE,FILE diff --git a/docs/en/sql-reference/sql-statements/Administration/DROP CLUSTER.md b/docs/en/sql-reference/sql-statements/Administration/DROP CLUSTER.md new file mode 100644 index 00000000000000..4330caa93d23f2 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/DROP CLUSTER.md @@ -0,0 +1,43 @@ +--- +{ + "title": "DROP CLUSTER", + "language": "en" +} +--- + + + +# DROP CLUSTER +## Description + +This statement is used to delete logical cluster. Successful deletion of logical cluster requires first deleting dB in the cluster and administrator privileges. + +grammar + +DROP CLUSTER [IF EXISTS] cluster_name + +## example + +Delete logical cluster test_cluster + +DROP CLUSTER test_cluster; + +## keyword +DROP,CLUSTER diff --git a/docs/en/sql-reference/sql-statements/Administration/DROP FILE.md b/docs/en/sql-reference/sql-statements/Administration/DROP FILE.md new file mode 100644 index 00000000000000..4d259fa980b1d4 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/DROP FILE.md @@ -0,0 +1,51 @@ +--- +{ + "title": "DROP FILE", + "language": "en" +} +--- + + + +# DROP FILE +## Description + +This statement is used to delete an uploaded file. + +Grammar: + +DROP FILE "file_name" [FROM database] +[properties] + +Explain: +File_name: File name. +Database: A DB to which the file belongs, if not specified, uses the DB of the current session. +properties 支持以下参数: + +Catalog: Yes. Classification of documents. + +## example + +1. Delete the file ca.pem + +DROP FILE "ca.pem" properties("catalog" = "kafka"); + +## keyword +DROP,FILE diff --git a/docs/en/sql-reference/sql-statements/Administration/ENTER.md b/docs/en/sql-reference/sql-statements/Administration/ENTER.md new file mode 100644 index 00000000000000..22b3a10c1d9100 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/ENTER.md @@ -0,0 +1,44 @@ +--- +{ + "title": "ENTER", + "language": "en" +} +--- + + + +# ENTER +## Description + +This statement is used to enter a logical cluster. All users and databases created need to be executed in a logical cluster. After creation, they belong to the logic. + +Cluster, need administrator privileges + +ENTER cluster name + +## example + +1. Enter the logical cluster test_cluster + +ENTER test cluster; + +## keyword +ENTER + diff --git a/docs/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md b/docs/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md new file mode 100644 index 00000000000000..5ad18861055c03 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md @@ -0,0 +1,57 @@ +--- +{ + "title": "INSTALL PLUGIN", + "language": "en" +} +--- + + + +# INSTALL PLUGIN +## description + + To install a plugin + + Syntax + + INSTALL PLUGIN FROM [source] + + source supports 3 kinds: + + 1. Point to a zip file with absolute path. + 2. Point to a plugin dir with absolute path. + 3. Point to a http/https download link of zip file. + +## example + + 1. Intall a plugin with a local zip file: + + INSTALL PLUGIN FROM "/home/users/seaven/auditdemo.zip"; + + 2. Intall a plugin with a local dir: + + INSTALL PLUGIN FROM "/home/users/seaven/auditdemo/"; + + 2. Download and install a plugin: + + INSTALL PLUGIN FROM "http://mywebsite.com/plugin.zip"; + +## keyword + INSTALL,PLUGIN diff --git a/docs/en/sql-reference/sql-statements/Administration/LINK DATABASE.md b/docs/en/sql-reference/sql-statements/Administration/LINK DATABASE.md new file mode 100644 index 00000000000000..cf5b87ddbd0783 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/LINK DATABASE.md @@ -0,0 +1,49 @@ +--- +{ + "title": "LINK DATABASE", + "language": "en" +} +--- + + + +# LINK DATABASE +## Description + +This statement allows users to link a database of one logical cluster to another logical cluster. A database is only allowed to be linked once at the same time and the linked database is deleted. + +It does not delete data, and the linked database cannot be deleted. Administrator privileges are required. + +grammar + +LINK DATABASE src u cluster name.src db name of the cluster name.des db name + +## example + +1. Link test_db in test_cluster A to test_cluster B and name it link_test_db + +LINK DATABASE test_clusterA.test_db test_clusterB.link_test_db; + +2. Delete linked database link_test_db + +DROP DATABASE link_test_db; + +## keyword +LINK,DATABASE diff --git a/docs/en/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md b/docs/en/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md new file mode 100644 index 00000000000000..bd6d1299a13048 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md @@ -0,0 +1,45 @@ +--- +{ + "title": "MIGRATE DATABASE", + "language": "en" +} +--- + + + +# MIGRATE DATABASE +## Description + +This statement is used to migrate a logical cluster database to another logical cluster. Before performing this operation, the database must be in a link state and need to be managed. + +Membership authority + +grammar + +MIGRATE DATABASE src u cluster name.src db name of the cluster name.des db name + +## example + +1. 迁移test_clusterA中的test_db到test_clusterB + +MIGRATE DATABASE test_clusterA.test_db test_clusterB.link_test_db; + +## keyword +MIGRATE,DATABASE diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW BACKENDS.md b/docs/en/sql-reference/sql-statements/Administration/SHOW BACKENDS.md new file mode 100644 index 00000000000000..8d414e6532add6 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW BACKENDS.md @@ -0,0 +1,47 @@ +--- +{ + "title": "SHOW BACKENDS", + "language": "en" +} +--- + + + +# SHOW BACKENDS +## Description +This statement is used to view BE nodes in the cluster +Grammar: +SHOW BACKENDS; + +Explain: +1. LastStartTime indicates the last BE start-up time. +2. LastHeartbeat represents the latest heartbeat. +3. Alive indicates whether the node survives. +4. System Decommissioned is true to indicate that the node is safely offline. +5. Cluster Decommissioned is true to indicate that the node is rushing downline in the current cluster. +6. TabletNum represents the number of fragments on the node. +7. Data Used Capacity represents the space occupied by the actual user data. +8. Avail Capacity represents the available space on the disk. +9. Total Capacity represents total disk space. Total Capacity = AvailCapacity + DataUsedCapacity + other non-user data files take up space. +10. UsedPct represents the percentage of disk usage. +11. ErrMsg is used to display error messages when a heartbeat fails. + +## keyword +SHOW, BACKENDS diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW BROKER.md b/docs/en/sql-reference/sql-statements/Administration/SHOW BROKER.md new file mode 100644 index 00000000000000..20d8238840aa4c --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW BROKER.md @@ -0,0 +1,40 @@ +--- +{ + "title": "SHOW BROKER", + "language": "en" +} +--- + + + +# SHOW BROKER +## Description +This statement is used to view the existing broker +Grammar: +SHOW BROKER; + +Explain: +1. LastStartTime indicates the last BE start-up time. +2. LastHeartbeat represents the latest heartbeat. +3. Alive indicates whether the node survives. +4. ErrMsg is used to display error messages when the heartbeat fails. + +## keyword +SHOW, BROKER diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW FILE.md b/docs/en/sql-reference/sql-statements/Administration/SHOW FILE.md new file mode 100644 index 00000000000000..0fabe4e9edd5f5 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW FILE.md @@ -0,0 +1,52 @@ +--- +{ + "title": "SHOW FILE", + "language": "en" +} +--- + + + +# SHOW FILE +## Description + +This statement is used to show a file created in a database + +Grammar: + +SHOW FILE [FROM database]; + +Explain: + +FileId: File ID, globally unique +DbName: The name of the database to which it belongs +Catalog: Custom Categories +FileName: File name +FileSize: File size, unit byte +MD5: Document MD5 + +## example + +1. View uploaded files in my_database + +SHOW FILE FROM my_database; + +## keyword +SHOW,FILE diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md b/docs/en/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md new file mode 100644 index 00000000000000..097829522ef302 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md @@ -0,0 +1,43 @@ +--- +{ + "title": "SHOW FRONTENDS", + "language": "en" +} +--- + + + +# SHOW FRONTENDS +## Description +This statement is used to view FE nodes +Grammar: +SHOW FRONTENDS; + +Explain: +1. name denotes the name of the FE node in bdbje. +2. Join is true to indicate that the node has joined the cluster. But it doesn't mean that it's still in the cluster (it may be out of touch) +3. Alive indicates whether the node survives. +4. Replayed Journal Id represents the maximum metadata log ID that the node has currently replayed. +5. LastHeartbeat is the latest heartbeat. +6. IsHelper indicates whether the node is a helper node in bdbje. +7. ErrMsg is used to display error messages when the heartbeat fails. + +## keyword +SHOW, FRONTENDS diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md b/docs/en/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md new file mode 100644 index 00000000000000..95be5dfc9f346a --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md @@ -0,0 +1,42 @@ +--- +{ + "title": "SHOW FULL COLUMNS", + "language": "en" +} +--- + + + +# SHOW FULL COLUMNS +## description + This statement is used to view some information about columns of a table. + + Syntax： + SHOW FULL COLUMNS FROM tbl; + +## example + + 1. View the column information of specified table + + SHOW FULL COLUMNS FROM tbl; + +## keyword + + SHOW,FULL,COLUMNS diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW INDEX.md b/docs/en/sql-reference/sql-statements/Administration/SHOW INDEX.md new file mode 100644 index 00000000000000..80a4b00a6b38e0 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW INDEX.md @@ -0,0 +1,46 @@ +--- +{ + "title": "SHOW INDEX", + "language": "en" +} +--- + + + +# SHOW INDEX + +## description + + This statement is used to show all index(only bitmap index in current version) of a table + Grammar: + SHOW INDEX[ES] FROM [db_name.]table_name [FROM database]; + + OR + + SHOW KEY[S] FROM [db_name.]table_name [FROM database]; + +## example + + 1. dispaly all indexes in table table_name + SHOW INDEX FROM example_db.table_name; + +## keyword + + SHOW,INDEX diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md b/docs/en/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md new file mode 100644 index 00000000000000..d844d423ce7ec4 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md @@ -0,0 +1,37 @@ +--- +{ + "title": "SHOW MIGRATIONS", + "language": "en" +} +--- + + + +# SHOW MIGRATIONS +## Description + +This statement is used to view the progress of database migration + +grammar + +SHOW MIGRATIONS + +## keyword +SHOW,MIGRATIONS diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW PLUGINS.md b/docs/en/sql-reference/sql-statements/Administration/SHOW PLUGINS.md new file mode 100644 index 00000000000000..c1f00a2011d604 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW PLUGINS.md @@ -0,0 +1,45 @@ +--- +{ + "title": "SHOW PLUGINS", + "language": "en" +} +--- + + + +# SHOW PLUGINS +## description + + To view the installed plugins. + + Syntax + + SHOW PLUGINS; + + This command will show all builtin and custom plugins. + +## example + + 1. To view the installed plugins: + + SHOW PLUGINS; + +## keyword + SHOW PLUGINS \ No newline at end of file diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md b/docs/en/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md new file mode 100644 index 00000000000000..a396e1b43fd8c2 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md @@ -0,0 +1,55 @@ +--- +{ + "title": "SHOW TABLE STATUS", + "language": "en" +} +--- + + + +# SHOW TABLE STATUS + +## description + +This statement is used to view some information about Table. + + Syntax: + + SHOW TABLE STATUS + [FROM db] [LIKE "pattern"] + + Explain: + + 1. This statement is mainly used to be compatible with MySQL grammar. At present, only a small amount of information such as Comment is displayed. + +## Example + + 1. View the information of all tables under the current database + + SHOW TABLE STATUS; + + + 2. View the information of the table whose name contains example in the specified database + + SHOW TABLE STATUS FROM DB LIKE "% example%"; + +## Keyword + + SHOW,TABLE,STATUS \ No newline at end of file diff --git a/docs/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN.md b/docs/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN.md new file mode 100644 index 00000000000000..d9b8f1fe1b99d8 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN.md @@ -0,0 +1,47 @@ +--- +{ + "title": "UNINTALL PLUGIN", + "language": "en" +} +--- + + + +# UNINTALL PLUGIN +## description + + To uninstall a plugin. + + Syntax + + UNINSTALL PLUGIN plugin_name; + + plugin_name can be found by `SHOW PLUGINS;`. + + Can only uninstall non-builtin plugins. + +## example + + 1. Uninstall a plugin: + + UNINSTALL PLUGIN auditdemo; + +## keyword + UNINSTALL,PLUGIN \ No newline at end of file diff --git a/docs/en/sql-reference/sql-statements/Data Definition/ALTER DATABASE.md b/docs/en/sql-reference/sql-statements/Data Definition/ALTER DATABASE.md new file mode 100644 index 00000000000000..6d6c2dd28f2428 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/ALTER DATABASE.md @@ -0,0 +1,56 @@ +--- +{ + "title": "ALTER DATABASE", + "language": "en" +} +--- + + + +# ALTER DATABASE +## description +This statement is used to set the properties of the specified database. (Administrators only) +Grammar: +1) Setting database data quota in B/K/KB/M/MB/G/GB/T/TB/P/PB +OTHER DATABASE dbu name SET DATA QUOTA quota; + +2) Rename the database +ALTER DATABASE db_name RENAME new_db_name; + +Explain: +After renaming the database, use REVOKE and GRANT commands to modify the corresponding user rights if necessary. +The database's default data quota is 1024GB, and the default replica quota is 1073741824. + +## example +1. Setting the specified database data quota +ALTER DATABASE example_db SET DATA QUOTA 10995116277760; +The above units are bytes, equivalent to +ALTER DATABASE example_db SET DATA QUOTA 10T; + +ALTER DATABASE example_db SET DATA QUOTA 100G; + +ALTER DATABASE example_db SET DATA QUOTA 200M; + +2. Rename the database example_db to example_db2 +ALTER DATABASE example_db RENAME example_db2; + +## keyword +ALTER,DATABASE,RENAME + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/ALTER TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/ALTER TABLE.md new file mode 100644 index 00000000000000..73f1c362ca9f9e --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/ALTER TABLE.md @@ -0,0 +1,363 @@ +--- +{ + "title": "ALTER TABLE", + "language": "en" +} +--- + + + +# ALTER TABLE + +## description + + This statement is used to modify an existing table. If no rollup index is specified, the base operation is the default. + The statement is divided into three types of operations: schema change, rollup, partition + These three types of operations cannot appear in an ALTER TABLE statement at the same time. + Where schema change and rollup are asynchronous operations and are returned if the task commits successfully. You can then use the SHOW ALTER command to view the progress. + Partition is a synchronous operation, and a command return indicates that execution is complete. + + grammar: + ALTER TABLE [database.]table + Alter_clause1[, alter_clause2, ...]; + + The alter_clause is divided into partition, rollup, schema change, rename and bimmap index. + + Partition supports the following modifications + Increase the partition + grammar: + ADD PARTITION [IF NOT EXISTS] partition_name + Partition_desc ["key"="value"] + [DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]] + note: + 1) partition_desc supports two ways of writing: + * VALUES LESS THAN [MAXVALUE|("value1", ...)] + * VALUES [("value1", ...), ("value1", ...)) + 1) The partition is the left closed right open interval. If the user only specifies the right boundary, the system will automatically determine the left boundary. + 2) If the bucket mode is not specified, the bucket method used by the built-in table is automatically used. + 3) If the bucket mode is specified, only the bucket number can be modified, and the bucket mode or bucket column cannot be modified. + 4) ["key"="value"] section can set some properties of the partition, see CREATE TABLE for details. + + 2. Delete the partition + grammar: + DROP PARTITION [IF EXISTS] partition_name + note: + 1) Use a partitioned table to keep at least one partition. + 2) Execute DROP PARTITION For a period of time, the deleted partition can be recovered by the RECOVER statement. See the RECOVER statement for details. + + 3. Modify the partition properties + grammar: + MODIFY PARTITION partition_name SET ("key" = "value", ...) + Description: + 1) The following attributes of the modified partition are currently supported. + - storage_medium + - storage_cooldown_time + - replication_num + — in_memory + 2) For single-partition tables, partition_name is the same as the table name. + + Rollup supports the following ways to create: + 1. Create a rollup index + grammar: + ADD ROLLUP rollup_name (column_name1, column_name2, ...) + [FROM from_index_name] + [PROPERTIES ("key"="value", ...)] + + properties: Support setting timeout time, the default timeout time is 1 day. + example: + ADD ROLLUP r1(col1,col2) from r0 + 1.2 Batch create rollup index + grammar: + ADD ROLLUP [rollup_name (column_name1, column_name2, ...) + [FROM from_index_name] + [PROPERTIES ("key"="value", ...)],...] + example： + ADD ROLLUP r1(col1,col2) from r0, r2(col3,col4) from r0 + 1.3 note: + 1) If from_index_name is not specified, it is created by default from base index + 2) The columns in the rollup table must be existing columns in from_index + 3) In properties, you can specify the storage format. See CREATE TABLE for details. + + 2. Delete the rollup index + grammar: + DROP ROLLUP rollup_name + [PROPERTIES ("key"="value", ...)] + example: + DROP ROLLUP r1 + 2.1 Batch Delete rollup index + grammar：DROP ROLLUP [rollup_name [PROPERTIES ("key"="value", ...)],...] + example：DROP ROLLUP r1,r2 + 2.2 note: + 1) Cannot delete base index + 2) Execute DROP ROLLUP For a period of time, the deleted rollup index can be restored by the RECOVER statement. See the RECOVER statement for details. + + + Schema change supports the following modifications: + 1. Add a column to the specified location of the specified index + grammar: + ADD COLUMN column_name column_type [KEY | agg_type] [DEFAULT "default_value"] + [AFTER column_name|FIRST] + [TO rollup_index_name] + [PROPERTIES ("key"="value", ...)] + note: + 1) Aggregate model If you add a value column, you need to specify agg_type + 2) Non-aggregate models (such as DUPLICATE KEY) If you add a key column, you need to specify the KEY keyword. + 3) You cannot add a column that already exists in the base index to the rollup index + Recreate a rollup index if needed + + 2. Add multiple columns to the specified index + grammar: + ADD COLUMN (column_name1 column_type [KEY | agg_type] DEFAULT "default_value", ...) + [TO rollup_index_name] + [PROPERTIES ("key"="value", ...)] + note: + 1) Aggregate model If you add a value column, you need to specify agg_type + 2) Non-aggregate model If you add a key column, you need to specify the KEY keyword. + 3) You cannot add a column that already exists in the base index to the rollup index + (You can recreate a rollup index if needed) + + 3. Remove a column from the specified index + grammar: + DROP COLUMN column_name + [FROM rollup_index_name] + note: + 1) Cannot delete partition column + 2) If the column is removed from the base index, it will also be deleted if the column is included in the rollup index + + 4. Modify the column type and column position of the specified index + grammar: + MODIFY COLUMN column_name column_type [KEY | agg_type] [NULL | NOT NULL] [DEFAULT "default_value"] + [AFTER column_name|FIRST] + [FROM rollup_index_name] + [PROPERTIES ("key"="value", ...)] + note: + 1) Aggregate model If you modify the value column, you need to specify agg_type + 2) Non-aggregate type If you modify the key column, you need to specify the KEY keyword. + 3) Only the type of the column can be modified. The other attributes of the column remain as they are (ie other attributes need to be explicitly written in the statement according to the original attribute, see example 8) + 4) The partition column cannot be modified + 5) The following types of conversions are currently supported (accuracy loss is guaranteed by the user) + TINYINT/SMALLINT/INT/BIGINT is converted to TINYINT/SMALLINT/INT/BIGINT/DOUBLE. + TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL is converted to VARCHAR + Convert LARGEINT to DOUBLE + VARCHAR supports modification of maximum length + Convert VARCHAR to TINYINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE. + Convert VARCHAR to DATE (currently support six formats: "%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d, "%y/%m/%d") + Convert DATETIME to DATE(Only year-month-day information is retained, For example: `2019-12-09 21:47:05` <--> `2019-12-09`) + Convert DATE to DATETIME(Set hour, minute, second to zero, For example: `2019-12-09` <--> `2019-12-09 00:00:00`) + Convert FLOAT to DOUBLE + Convert INT to DATE (If the INT data fails to convert, the original data remains the same) + 6) Does not support changing from NULL to NOT NULL + + 5. Reorder the columns of the specified index + grammar: + ORDER BY (column_name1, column_name2, ...) + [FROM rollup_index_name] + [PROPERTIES ("key"="value", ...)] + note: + 1) All columns in index must be written + 2) value is listed after the key column + + 6. Modify the properties of the table, currently supports modifying the bloom filter column, the colocate_with attribute and the dynamic_partition attribute， the replication_num and default.replication_num. + grammar: + PROPERTIES ("key"="value") + note: + Can also be merged into the above schema change operation to modify, see the example below +   + + Rename supports modification of the following names: + 1. Modify the table name + grammar: + RENAME new_table_name; + + 2. Modify the rollup index name + grammar: + RENAME ROLLUP old_rollup_name new_rollup_name; + + 3. Modify the partition name + grammar: + RENAME PARTITION old_partition_name new_partition_name; + + Bitmap index supports the following modifications: + 1. create bitmap index + grammar: + ADD INDEX index_name (column [, ...],) [USING BITMAP] [COMMENT 'balabala']; + note: + 1. only supports bitmap index for current version + 2. BITMAP index only supports apply on single column + 2. drop index + grammar: + DROP INDEX index_name； + +## example + + [table] + 1. Modify the default number of replications of the table, which is used as default number of replications while creating new partition. + ATLER TABLE example_db.my_table + SET ("default.replication_num" = "2"); + + 2. Modify the actual number of replications of a unpartitioned table (unpartitioned table only) + ALTER TABLE example_db.my_table + SET ("replication_num" = "3"); + + [partition] + 1. Add partition, existing partition [MIN, 2013-01-01), add partition [2013-01-01, 2014-01-01), use default bucket mode + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES LESS THAN ("2014-01-01"); + + 2. Increase the partition and use the new number of buckets + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") + DISTRIBUTED BY HASH(k1) BUCKETS 20; + + 3. Increase the partition and use the new number of copies + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") + ("replication_num"="1"); + + 4. Modify the number of partition copies + ALTER TABLE example_db.my_table + MODIFY PARTITION p1 SET("replication_num"="1"); + + 5. Delete the partition + ALTER TABLE example_db.my_table + DROP PARTITION p1; + + 6. Add a partition that specifies the upper and lower bounds + + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES [("2014-01-01"), ("2014-02-01")); + + [rollup] + 1. Create index: example_rollup_index, based on base index(k1,k2,k3,v1,v2). Columnar storage. + ALTER TABLE example_db.my_table + ADD ROLLUP example_rollup_index(k1, k3, v1, v2) + PROPERTIES("storage_type"="column"); + + 2. Create index: example_rollup_index2, based on example_rollup_index(k1,k3,v1,v2) + ALTER TABLE example_db.my_table + ADD ROLLUP example_rollup_index2 (k1, v1) + FROM example_rollup_index; + + 3. Create index: example_rollup_index3, based on base index (k1, k2, k3, v1), custom rollup timeout time is one hour. + + ALTER TABLE example_db.my_table + ADD ROLLUP example_rollup_index(k1, k3, v1) + PROPERTIES("storage_type"="column", "timeout" = "3600"); + + 3. Delete index: example_rollup_index2 + ALTER TABLE example_db.my_table + DROP ROLLUP example_rollup_index2; + + [schema change] + 1. Add a key column new_col to the col1 of example_rollup_index (non-aggregate model) + ALTER TABLE example_db.my_table + ADD COLUMN new_col INT KEY DEFAULT "0" AFTER col1 + TO example_rollup_index; + + 2. Add a value column new_col to the col1 of example_rollup_index (non-aggregate model) +   ALTER TABLE example_db.my_table +   ADD COLUMN new_col INT DEFAULT "0" AFTER col1 +   TO example_rollup_index; + + 3. Add a key column new_col (aggregation model) to col1 of example_rollup_index +   ALTER TABLE example_db.my_table +   ADD COLUMN new_col INT DEFAULT "0" AFTER col1 +   TO example_rollup_index; + + 4. Add a value column to the col1 of example_rollup_index. new_col SUM aggregation type (aggregation model) +   ALTER TABLE example_db.my_table +   ADD COLUMN new_col INT SUM DEFAULT "0" AFTER col1 +   TO example_rollup_index; + + 5. Add multiple columns to the example_rollup_index (aggregate model) + ALTER TABLE example_db.my_table + ADD COLUMN (col1 INT DEFAULT "1", col2 FLOAT SUM DEFAULT "2.3") + TO example_rollup_index; + + 6. Remove a column from example_rollup_index + ALTER TABLE example_db.my_table + DROP COLUMN col2 + FROM example_rollup_index; + + 7. Modify the base index's col1 column to be of type BIGINT and move to the col2 column + ALTER TABLE example_db.my_table + MODIFY COLUMN col1 BIGINT DEFAULT "1" AFTER col2; + + 8. Modify the maximum length of the val1 column of the base index. The original val1 is (val1 VARCHAR(32) REPLACE DEFAULT "abc") + ALTER TABLE example_db.my_table + MODIFY COLUMN val1 VARCHAR(64) REPLACE DEFAULT "abc"; + + 9. Reorder the columns in example_rollup_index (set the original column order: k1, k2, k3, v1, v2) + ALTER TABLE example_db.my_table + ORDER BY (k3, k1, k2, v2, v1) + FROM example_rollup_index; + + 10. Perform both operations simultaneously + ALTER TABLE example_db.my_table + ADD COLUMN v2 INT MAX DEFAULT "0" AFTER k2 TO example_rollup_index, + ORDER BY (k3,k1,k2,v2,v1) FROM example_rollup_index; + + 11. Modify the bloom filter column of the table + ALTER TABLE example_db.my_table SET ("bloom_filter_columns"="k1,k2,k3"); + + Can also be merged into the above schema change operation (note that the syntax of multiple clauses is slightly different) + ALTER TABLE example_db.my_table + DROP COLUMN col2 + PROPERTIES ("bloom_filter_columns"="k1,k2,k3"); + + 12. Modify the Colocate property of the table + + ALTER TABLE example_db.my_table set ("colocate_with" = "t1"); + + 13. Change the bucketing mode of the table from Random Distribution to Hash Distribution + + ALTER TABLE example_db.my_table set ("distribution_type" = "hash"); + + 14. Modify the dynamic partition properties of the table (support adding dynamic partition properties to tables without dynamic partition properties) + + ALTER TABLE example_db.my_table set ("dynamic_partition_enable" = "false"); + + If you need to add dynamic partition attributes to a table without dynamic partition attributes, you need to specify all dynamic partition attributes + + ALTER TABLE example_db.my_table set ("dynamic_partition. Enable "= "true", dynamic_partition. Time_unit" = "DAY", "dynamic_partition. End "= "3", "dynamic_partition. Prefix" = "p", "Dynamic_partition. Buckets" = "32"); + + 15. Modify the in_memory property of the table + + ALTER TABLE example_db.my_table set ("in_memory" = "true"); + + [rename] + 1. Modify the table named table1 to table2 + ALTER TABLE table1 RENAME table2; + + 2. Modify the rollup index named rollup1 in the table example_table to rollup2 + ALTER TABLE example_table RENAME ROLLUP rollup1 rollup2; + + 3. Modify the partition named p1 in the table example_table to p2 + ALTER TABLE example_table RENAME PARTITION p1 p2; + + [index] + 1. create index on table1 column siteid using bitmap + ALTER TABLE table1 ADD INDEX index_name [USING BITMAP] (siteid) COMMENT 'balabala'; + 2. drop bitmap index of table1 + ALTER TABLE table1 DROP INDEX index_name; + +## keyword + + ALTER, TABLE, ROLLUP, COLUMN, PARTITION, RENAME diff --git a/docs/en/sql-reference/sql-statements/Data Definition/ALTER VIEW.md b/docs/en/sql-reference/sql-statements/Data Definition/ALTER VIEW.md new file mode 100644 index 00000000000000..03ff25f4021c08 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/ALTER VIEW.md @@ -0,0 +1,51 @@ +--- +{ + "title": "ALTER VIEW", + "language": "en" +} +--- + + + +# ALTER VIEW +## description + This statement is used to modify the definition of a view + Syntax: + ALTER VIEW + [db_name.]view_name + (column1[ COMMENT "col comment"][, column2, ...]) + AS query_stmt + + Explain: + 1. View is logical, it isn't stored in the physical medium. When we querying, view will be embed as subqueries in query statement. Therefore, modifying the definition of views is equivalent to modifying query_stmt which is defined in view. + 2. query_stmt is arbitrarily supported SQL. + +## example + + 1. Modify example_view on the example_db + + ALTER VIEW example_db.example_view + ( + c1 COMMENT "column 1", + c2 COMMENT "column 2", + c3 COMMENT "column 3" + ) + AS SELECT k1, k2, SUM(v1) FROM example_table + GROUP BY k1, k2 \ No newline at end of file diff --git a/docs/en/sql-reference/sql-statements/Data Definition/BACKUP.md b/docs/en/sql-reference/sql-statements/Data Definition/BACKUP.md new file mode 100644 index 00000000000000..c588e2cb4963d9 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/BACKUP.md @@ -0,0 +1,65 @@ +--- +{ + "title": "BACKUP", + "language": "en" +} +--- + + + +# BACKUP +## Description +This statement is used to backup data under the specified database. This command is an asynchronous operation. After successful submission, you need to check progress through the SHOW BACKUP command. Only tables of OLAP type are backed up. +Grammar: +BACKUP SNAPSHOT [db_name].{snapshot_name} +TO `repository_name` +ON ( +"`Table `U name'[Distriction (`P1',...)], +... +) +PROPERTIES ("key"="value", ...); + +Explain: +1. Only one BACKUP or RESTORE task can be performed under the same database. +2. The ON clause identifies the tables and partitions that need to be backed up. If no partition is specified, all partitions of the table are backed up by default. +3. PROPERTIES currently supports the following attributes: +"Type" = "full": means that this is a full update (default). +"Timeout" = "3600": Task timeout, default to one day. Unit seconds. + +## example + +1. Back up the table example_tbl under example_db in full to the warehouse example_repo: +BACKUP SNAPSHOT example_db.snapshot_label1 +TO example repo +On (example tbl) +PROPERTIES ("type" = "full"); + +2. Under full backup example_db, the P1 and P2 partitions of table example_tbl, and table example_tbl2 to warehouse example_repo: +BACKUP SNAPSHOT example_db.snapshot_label2 +TO example repo +ON +( +example_tbl PARTITION (p1,p2), +Example: +); + +## keyword +BACKUP + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md b/docs/en/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md new file mode 100644 index 00000000000000..346c77e9047e20 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md @@ -0,0 +1,70 @@ +--- +{ + "title": "CANCEL ALTER", + "language": "en" +} +--- + + + +# CANCEL ALTER +## Description +This statement is used to undo an ALTER operation. +1. 撤销 ALTER TABLE COLUMN 操作 +Grammar: +CANCEL ALTER TABLE COLUMN +FROM db_name.table_name + +2. 撤销 ALTER TABLE ROLLUP 操作 +Grammar: +CANCEL ALTER TABLE ROLLUP +FROM db_name.table_name + +3. batch cancel rollup by job id + Grammar: + CANCEL ALTER TABLE ROLLUP + FROM db_name.table_name (jobid,...) + Note: + Batch cancel rollup job is a async operation, use `show alter table rollup` to see whether it executes successfully + +2. OTHER CLUSTER +Grammar: +(To be realized... + + +## example +[CANCEL ALTER TABLE COLUMN] +1. 撤销针对 my_table 的 ALTER COLUMN 操作。 +CANCEL ALTER TABLE COLUMN +FROM example_db.my_table; + +[CANCEL ALTER TABLE ROLLUP] +1. 撤销 my_table 下的 ADD ROLLUP 操作。 +CANCEL ALTER TABLE ROLLUP +FROM example_db.my_table; + +[CANCEL ALTER TABLE ROLLUP] +1. cancel rollup alter job by job id +CANCEL ALTER TABLE ROLLUP +FROM example_db.my_table (12801,12802); + +## keyword +CANCEL,ALTER,TABLE,COLUMN,ROLLUP + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md b/docs/en/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md new file mode 100644 index 00000000000000..a0e379ad1775b2 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md @@ -0,0 +1,39 @@ +--- +{ + "title": "CANCEL BACKUP", + "language": "en" +} +--- + + + +# CANCEL BACKUP +## Description +This statement is used to cancel an ongoing BACKUP task. +Grammar: +CANCEL BACKUP FROM db_name; + +## example +1. Cancel the BACKUP task under example_db. +CANCEL BACKUP FROM example_db; + +## keyword +CANCEL, BACKUP + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md b/docs/en/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md new file mode 100644 index 00000000000000..b4bf1bb5692647 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md @@ -0,0 +1,42 @@ +--- +{ + "title": "CANCEL RESTORE", + "language": "en" +} +--- + + + +# CANCEL RESTORE +## Description +This statement is used to cancel an ongoing RESTORE task. +Grammar: +CANCEL RESTORE FROM db_name; + +Be careful: +When the recovery is abolished around the COMMIT or later stage, the restored tables may be inaccessible. At this point, data recovery can only be done by performing the recovery operation again. + +## example +1. Cancel the RESTORE task under example_db. +CANCEL RESTORE FROM example_db; + +## keyword +CANCEL, RESTORE + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md new file mode 100644 index 00000000000000..012aabb97bc42e --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md @@ -0,0 +1,39 @@ +--- +{ + "title": "CREATE DATABASE", + "language": "en" +} +--- + + + +# CREATE DATABASE +## Description +This statement is used to create a new database +Grammar: +CREATE DATABASE [IF NOT EXISTS] db_name; + +## example +1. New database db_test +CREATE DATABASE db_test; + +## keyword +CREATE,DATABASE + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE INDEX.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE INDEX.md new file mode 100644 index 00000000000000..2f13d24f7e42ff --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE INDEX.md @@ -0,0 +1,45 @@ +--- +{ + "title": "CREATE INDEX", + "language": "en" +} +--- + + + +# CREATE INDEX + +## description + + This statement is used to create index + grammer: + CREATE INDEX index_name ON table_name (column [, ...],) [USING BITMAP] [COMMENT'balabala']; + note: + 1. only support bitmap index in current version + 2. BITMAP index only supports apply to single column + +## example + + 1. create index on table1 column siteid using bitmap + CREATE INDEX index_name ON table1 (siteid) USING BITMAP COMMENT 'balabala'; + +## keyword + + CREATE,INDEX diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md new file mode 100644 index 00000000000000..4c0afa2c651d9a --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md @@ -0,0 +1,238 @@ +--- +{ + "title": "CREATE MATERIALIZED VIEW", + "language": "en" +} +--- + + + +# CREATE MATERIALIZED VIEW + +## description + +This statement is used to create a materialized view. + + Asynchronous syntax. After the call is successful, it only indicates that the task to create the materialized view is successfully submitted. The user needs to check the progress of the materialized view by using ```show alter table rollup```. + + After the progress is FINISHED, you can use the ```desc [table_name] all``` command to view the schema of the materialized view. + +syntax: + + ``` + + CREATE MATERIALIZED VIEW [MG name] as [query] + [PROPERTIES ("key" = "value")] + + ``` + +1. MV name + + Name of the materialized view. Required. + + Materialized view names in the same table cannot be duplicated. + +2. query + + The query used to construct the materialized view. The result of the query is the data of the materialized view. The query format currently supported is: + + ``` +    SELECT select_expr [, select_expr ...] +    FROM [Base view name] +    GROUP BY column_name [, column_name ...] +    ORDER BY column_name [, column_name ...] +     +    The syntax is the same as the query syntax. + ``` + + select_expr: All columns in the materialized view's schema. + + + Only single columns and aggregate columns without expression calculation are supported. + + The aggregate function currently only supports SUM, MIN, MAX, and the parameters of the aggregate function can only be a single column without expression calculation. + + Contains at least one single column. + + All involved columns can only appear once. + + base view name: The original table name of the materialized view. Required. + + + Must be a single table and not a subquery + + group by: Grouped column of materialized view, optional. + + + If not filled, the data will not be grouped. + + order by: Sort order of materialized view, optional. + + + The order of the column sort must be the same as the column declaration order in select_expr. + + If order by is not specified, sort columns are automatically supplemented according to the rules. + + + If the materialized view is an aggregate type, all grouping columns are automatically supplemented with sort columns. + + If the materialized view is a non-aggregated type, the first 36 bytes are automatically supplemented as a sorted column. If the number of sorts for automatic replenishment is less than three, the first three are sorted. + + If the query contains a grouping column, the sort order must be the same as the grouping column. + +3. properties + + Declare some configuration of materialized view, optional. + + ``` + PROPERTIES ("key" = "value", "key" = "value" ...) + + ``` + + The following configurations can be declared here: + + + short_key: the number of columns. + + timeout: timeout for materialized view construction. + +## example + +Base table structure is + +``` +mysql> desc duplicate_table; ++-------+--------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-------+--------+------+------+---------+-------+ +| k1 | INT | Yes | true | N/A | | +| k2 | INT | Yes | true | N/A | | +| k3 | BIGINT | Yes | true | N/A | | +| k4 | BIGINT | Yes | true | N/A | | ++-------+--------+------+------+---------+-------+ +``` + +1. Create a materialized view containing only the columns of the original table (k1, k2) + + ``` + create materialized view k1_k2 as +select k1, k2 from duplicate_table; + ``` + + The materialized view's schema is shown below. The materialized view contains only two columns k1, k2 without any aggregation. + + ``` + +-----------------+-------+--------+------+------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +-----------------+-------+--------+------+------+---------+-------+ + | k1_k2 | k1 | INT | Yes | true | N/A | | + | | k2 | INT | Yes | true | N/A | | + +-----------------+-------+--------+------+------+---------+-------+ + ``` + +2. Create a materialized view sorted by k2 + + ``` + create materialized view k2_order as +select k2, k1 from duplicate_table order by k2; +``` + + The materialized view's schema is shown below. The materialized view contains only two columns k2, k1, where column k2 is a sorted column without any aggregation. + + ``` + +-----------------+-------+--------+------+-------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +-----------------+-------+--------+------+-------+---------+-------+ + | k2_order | k2 | INT | Yes | true | N/A | | + | | k1 | INT | Yes | false | N/A | NONE | + +-----------------+-------+--------+------+-------+---------+-------+ + ``` + +3. Create a materialized view grouped by k1, k2 with k3 as the SUM aggregate + + ``` + create materialized view k1_k2_sumk3 as +select k1, k2, sum (k3) from duplicate_table group by k1, k2; + ``` + + The materialized view's schema is shown below. The materialized view contains two columns k1, k2 and sum (k3), where k1, k2 are grouped columns, and sum (k3) is the sum of the k3 columns grouped according to k1, k2. + + Because the materialized view does not declare a sort column, and the materialized view has aggregate data, the system supplements the grouping columns k1 and k2 by default. + + ``` + +-----------------+-------+--------+------+-------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +-----------------+-------+--------+------+-------+---------+-------+ + | k1_k2_sumk3 | k1 | INT | Yes | true | N/A | | + | | k2 | INT | Yes | true | N/A | | + | | k3 | BIGINT | Yes | false | N/A | SUM | + +-----------------+-------+--------+------+-------+---------+-------+ + ``` + +4. Create a materialized view to remove duplicate rows + + ``` + create materialized view deduplicate as +select k1, k2, k3, k4 from duplicate_table group by k1, k2, k3, k4; + ``` + + The materialized view schema is shown below. The materialized view contains k1, k2, k3, and k4 columns, and there are no duplicate rows. + + ``` + +-----------------+-------+--------+------+-------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +-----------------+-------+--------+------+-------+---------+-------+ + | deduplicate | k1 | INT | Yes | true | N/A | | + | | k2 | INT | Yes | true | N/A | | + | | k3 | BIGINT | Yes | true | N/A | | + | | k4 | BIGINT | Yes | true | N/A | | + +-----------------+-------+--------+------+-------+---------+-------+ + ``` + +5. Create a non-aggregated materialized view that does not declare a sort column + + The schema of all_type_table is as follows: + + ``` + +-------+--------------+------+-------+---------+-------+ + | Field | Type | Null | Key | Default | Extra | + +-------+--------------+------+-------+---------+-------+ + | k1 | TINYINT | Yes | true | N/A | | + | k2 | SMALLINT | Yes | true | N/A | | + | k3 | INT | Yes | true | N/A | | + | k4 | BIGINT | Yes | true | N/A | | + | k5 | DECIMAL(9,0) | Yes | true | N/A | | + | k6 | DOUBLE | Yes | false | N/A | NONE | + | k7 | VARCHAR(20) | Yes | false | N/A | NONE | + +-------+--------------+------+-------+---------+-------+ + ``` + + The materialized view contains k3, k4, k5, k6, k7 columns, and no sort column is declared. The creation statement is as follows: + + ``` + create materialized view mv_1 as +select k3, k4, k5, k6, k7 from all_type_table; + ``` + + The system's default supplementary sort columns are k3, k4, and k5. The sum of the number of bytes for these three column types is 4 (INT) + 8 (BIGINT) + 16 (DECIMAL) = 28 <36. So these three columns are added as sort columns. + + The materialized view's schema is as follows. You can see that the key fields of the k3, k4, and k5 columns are true, which is the sort order. The key field of the k6, k7 columns is false, that is, non-sorted. + + ``` + +----------------+-------+--------------+------+-------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +----------------+-------+--------------+------+-------+---------+-------+ + | mv_1 | k3 | INT | Yes | true | N/A | | + | | k4 | BIGINT | Yes | true | N/A | | + | | k5 | DECIMAL(9,0) | Yes | true | N/A | | + | | k6 | DOUBLE | Yes | false | N/A | NONE | + | | k7 | VARCHAR(20) | Yes | false | N/A | NONE | + +----------------+-------+--------------+------+-------+---------+-------+ + ``` + +## keyword + CREATE, MATERIALIZED, VIEW diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md new file mode 100644 index 00000000000000..b27b50cf0dd7f7 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md @@ -0,0 +1,75 @@ +--- +{ + "title": "CREATE REPOSITORY", + "language": "en" +} +--- + + + +# CREATE REPOSITORY +## Description +This statement is used to create the warehouse. The warehouse is used for backup or recovery. Only root or superuser users can create warehouses. +Grammar: +CREATE [READ ONLY] REPOSITORY `repo_name` +WITH BROKER `broker_name` +ON LOCATION `repo_location` +PROPERTIES ("key"="value", ...); + +Explain: +1. The creation of warehouses depends on existing brokers +2. If it is a read-only warehouse, it can only be restored on the warehouse. If not, you can backup and restore operations. +3. According to the different types of broker, PROPERTIES is different, see the example. + +## example +1. Create a warehouse named bos_repo, which relies on BOS broker "bos_broker", and the data root directory is: bos://palo_backup. +CREATE REPOSITORY `bos_repo` +WITH BROKER `bos_broker` +ON LOCATION "bos://palo_backup" +PROPERTIES +( +"bosu endpoint" ="http://gz.bcebos.com", +"bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", +"bos_secret_accesskey"="70999999999999de274d59eaa980a" +); + +2. Create the same warehouse as in Example 1, but with read-only attributes: +CREATE READ ONLY REPOSITORY `bos_repo` +WITH BROKER `bos_broker` +ON LOCATION "bos://palo_backup" +PROPERTIES +( +"bosu endpoint" ="http://gz.bcebos.com", +"bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", +"bos_secret_accesskey"="70999999999999de274d59eaa980a" +); + +3. Create a warehouse named hdfs_repo, which relies on Baidu HDFS broker "hdfs_broker", and the data root directory is: hdfs://hadoop-name-node:54310/path/to/repo./ +CREATE REPOSITORY `hdfs_repo` +WITH BROKER `hdfs_broker` +ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/" +PROPERTIES +( +"Username" = "User" +"password" = "password" +); + +## keyword +CREATE REPOSITORY diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md new file mode 100644 index 00000000000000..82873a581517ad --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -0,0 +1,582 @@ +--- +{ + "title": "CREATE TABLE", + "language": "en" +} +--- + + + +# CREATE TABLE + +## description + +This statement is used to create table +Syntax: + +``` + CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [database.]table_name + (column_definition1[, column_definition2, ...] + [, index_definition1[, ndex_definition12,]]) + [ENGINE = [olap|mysql|broker]] + [key_desc] + [COMMENT "table comment"] + [partition_desc] + [distribution_desc] + [rollup_index] + [PROPERTIES ("key"="value", ...)] + [BROKER PROPERTIES ("key"="value", ...)]; +``` + +1. column_definition + Syntax: + `col_name col_type [agg_type] [NULL | NOT NULL] [DEFAULT "default_value"]` + Explain: + col_name: Name of column + col_type: Type of column + ``` + TINYINT(1 Byte) + Range: -2^7 + 1 ~ 2^7 - 1 + SMALLINT(2 Bytes) + Range: -2^15 + 1 ~ 2^15 - 1 + INT(4 Bytes) + Range: -2^31 + 1 ~ 2^31 - 1 + BIGINT(8 Bytes) + Range: -2^63 + 1 ~ 2^63 - 1 + LARGEINT(16 Bytes) + Range: -2^127 + 1 ~ 2^127 - 1 + FLOAT(4 Bytes) + Support scientific notation + DOUBLE(12 Bytes) + Support scientific notation + DECIMAL[(precision, scale)] (16 Bytes) + Default is DECIMAL(10, 0) + precision: 1 ~ 27 + scale: 0 ~ 9 + integer part: 1 ~ 18 + fractional part: 0 ~ 9 + Not support scientific notation + DATE(3 Bytes) + Range: 0000-01-01 ~ 9999-12-31 + DATETIME(8 Bytes) + Range: 0000-01-01 00:00:00 ~ 9999-12-31 23:59:59 + CHAR[(length)] + Fixed length string. Range: 1 ~ 255. Default: 1 + VARCHAR[(length)] + Variable length string. Range: 1 ~ 65533 + HLL (1~16385 Bytes) + HLL tpye, No need to specify length. + This type can only be queried by hll_union_agg, hll_cardinality, hll_hash functions. + BITMAP + BITMAP type, No need to specify length. Represent a set of unsigned bigint numbers, the largest element could be 2^64 - 1 + ``` + agg_type: Aggregation type. If not specified, the column is key column. Otherwise, the column is value column. + * SUM、MAX、MIN、REPLACE + * HLL_UNION: Only for HLL type + * REPLACE_IF_NOT_NULL: The meaning of this aggregation type is that substitution will occur if and only if the newly imported data is a non-null value. If the newly imported data is null, Doris will still retain the original value. Note: if NOT NULL is specified in the REPLACE_IF_NOT_NULL column when the user creates the table, Doris will convert it to NULL and will not report an error to the user. Users can leverage this aggregate type to achieve importing some of columns. + * BITMAP_UNION: Only for BITMAP type + Allow NULL: Default is NOT NULL. NULL value should be represented as `\N` in load source file. + Notice: + The origin value of BITMAP_UNION column should be TINYINT, SMALLINT, INT, BIGINT. +2. index_definition + Syntax: + `INDEX index_name (col_name[, col_name, ...]) [USING BITMAP] COMMENT 'xxxxxx'` + Explain: + index_name：index name + col_name：column name + Notice: + Only support BITMAP index in current version, BITMAP can only apply to single column +3. ENGINE type + Default is olap. Options are: olap, mysql, broker + 1) For mysql, properties should include: + + ``` + PROPERTIES ( + "host" = "mysql_server_host", + "port" = "mysql_server_port", + "user" = "your_user_name", + "password" = "your_password", + "database" = "database_name", + "table" = "table_name" + ) + ``` + + Notice: + "table_name" is the real table name in MySQL database. + table_name in CREATE TABLE stmt is table is Doris. They can be different or same. + MySQL table created in Doris is for accessing data in MySQL database. + Doris does not maintain and store any data from MySQL table. + 1) For broker, properties should include: + + ``` + PROPERTIES ( + "broker_name" = "broker_name", + "path" = "file_path1[,file_path2]", + "column_separator" = "value_separator" + "line_delimiter" = "value_delimiter" + ) + ``` + + ``` + BROKER PROPERTIES( + "username" = "name", + "password" = "password" + ) + ``` + + For different broker, the broker properties are different + Notice: + Files name in "path" is separated by ",". If file name includes ",", use "%2c" instead. If file name includes "%", use "%25" instead. + Support CSV and Parquet. Support GZ, BZ2, LZ4, LZO(LZOP) +4. key_desc + Syntax: + key_type(k1[,k2 ...]) + Explain: + Data is orderd by specified key columns. And has different behaviors for different key desc. + AGGREGATE KEY: + value columns will be aggregated is key columns are same. + UNIQUE KEY: + The new incoming rows will replace the old rows if key columns are same. + DUPLICATE KEY: + All incoming rows will be saved. + the default key_type is DUPLICATE KEY, and key columns are first 36 bytes of the columns in define order. + If the number of columns in the first 36 is less than 3, the first 3 columns will be used. + NOTICE: + Except for AGGREGATE KEY, no need to specify aggregation type for value columns. +5. partition_desc + Partition has two ways to use: + 1) LESS THAN + Syntax: + + ``` + PARTITION BY RANGE (k1, k2, ...) + ( + PARTITION partition_name1 VALUES LESS THAN MAXVALUE|("value1", "value2", ...), + PARTITION partition_name2 VALUES LESS THAN MAXVALUE|("value1", "value2", ...) + ... + ) + ``` + + Explain: + 1) Partition name only support [A-z0-9_] + 2) Partition key column's type should be: + TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME + 3) The range is [closed, open). And the lower bound of first partition is MIN VALUE of specifed column type. + 4) NULL values should be save in partition which includes MIN VALUE. + 5) Support multi partition columns, the the default partition value is MIN VALUE. + 2）Fixed Range + Syntax: + ``` + PARTITION BY RANGE (k1, k2, k3, ...) + ( + PARTITION partition_name1 VALUES [("k1-lower1", "k2-lower1", "k3-lower1",...), ("k1-upper1", "k2-upper1", "k3-upper1", ...)), + PARTITION partition_name2 VALUES [("k1-lower1-2", "k2-lower1-2", ...), ("k1-upper1-2", MAXVALUE, )) + "k3-upper1-2", ... + ) + ``` + Explain: + 1）The Fixed Range is more flexible than the LESS THAN, and the left and right intervals are completely determined by the user. + 2）Others are consistent with LESS THAN. +6. distribution_desc + 1) Hash + Syntax: + `DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]` + Explain: + The default buckets is 10. +7. PROPERTIES + 1) If ENGINE type is olap. User can specify storage medium, cooldown time and replication number: + + ``` + PROPERTIES ( + "storage_medium" = "[SSD|HDD]", + ["storage_cooldown_time" = "yyyy-MM-dd HH:mm:ss"], + ["replication_num" = "3"] + ) + ``` + + storage_medium: SSD or HDD, The default initial storage media can be specified by `default_storage_medium= XXX` in the fe configuration file `fe.conf`, or, if not, by default, HDD. + storage_cooldown_time: If storage_medium is SSD, data will be automatically moved to HDD when timeout. + Default is 30 days. + Format: "yyyy-MM-dd HH:mm:ss" + replication_num: Replication number of a partition. Default is 3. + If table is not range partitions. This property takes on Table level. Or it will takes on Partition level. + User can specify different properties for different partition by `ADD PARTITION` or `MODIFY PARTITION` statements. + 2) If Engine type is olap, user can set bloom filter index for column. + Bloom filter index will be used when query contains `IN` or `EQUAL`. + Bloom filter index support key columns with type except TINYINT FLOAT DOUBLE, also support value with REPLACE aggregation type. + + ``` + PROPERTIES ( + "bloom_filter_columns"="k1,k2,k3" + ) + ``` + + 3) For Colocation Join: + + ``` + PROPERTIES ( + "colocate_with"="table1" + ) + ``` + + 4) if you want to use the dynamic partitioning feature, specify it in properties + + ``` + PROPERTIES ( + "dynamic_partition.enable" = "true|false", + "dynamic_partition.time_unit" = "DAY|WEEK|MONTH", + "dynamic_partitoin.end" = "${integer_value}", + "dynamic_partition.prefix" = "${string_value}", + "dynamic_partition.buckets" = "${integer_value} + ) + ``` + + Dynamic_partition. Enable: specifies whether dynamic partitioning at the table level is enabled + + Dynamic_partition. Time_unit: used to specify the time unit for dynamically adding partitions, which can be selected as DAY, WEEK, and MONTH. + + Dynamic_partition. End: used to specify the number of partitions created in advance + + Dynamic_partition. Prefix: used to specify the partition name prefix to be created, such as the partition name prefix p, automatically creates the partition name p20200108 + + Dynamic_partition. Buckets: specifies the number of partition buckets that are automatically created +8. rollup_index + grammar: + ``` + ROLLUP (rollup_name (column_name1, column_name2, ...) + [FROM from_index_name] + [PROPERTIES ("key"="value", ...)],...) + ``` + + 5) if you want to use the inmemory table feature, specify it in properties + + ``` + PROPERTIES ( + "in_memory"="true" + ) + ``` +## example + +1. Create an olap table, distributed by hash, with aggregation type. + + ``` + CREATE TABLE example_db.table_hash + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 CHAR(10) REPLACE, + v2 INT SUM + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + COMMENT "my first doris table" + DISTRIBUTED BY HASH(k1) BUCKETS 32 + PROPERTIES ("storage_type"="column"); + ``` + +2. Create an olap table, distributed by hash, with aggregation type. Also set storage mediumand cooldown time. + + ``` + CREATE TABLE example_db.table_hash + ( + k1 BIGINT, + k2 LARGEINT, + v1 VARCHAR(2048) REPLACE, + v2 SMALLINT SUM DEFAULT "10" + ) + ENGINE=olap + UNIQUE KEY(k1, k2) + DISTRIBUTED BY HASH (k1, k2) BUCKETS 32 + PROPERTIES( + "storage_type"="column", + "storage_medium" = "SSD", + "storage_cooldown_time" = "2015-06-04 00:00:00" + ); + +3. Create an olap table, with range partitioned, distributed by hash. + +1) LESS THAN + + ``` + CREATE TABLE example_db.table_range + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY RANGE (k1) + ( + PARTITION p1 VALUES LESS THAN ("2014-01-01"), + PARTITION p2 VALUES LESS THAN ("2014-06-01"), + PARTITION p3 VALUES LESS THAN ("2014-12-01") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", "storage_cooldown_time" = "2015-06-04 00:00:00" + ); + ``` + + Explain: + This statement will create 3 partitions: + + ``` + ( { MIN }, {"2014-01-01"} ) + [ {"2014-01-01"}, {"2014-06-01"} ) + [ {"2014-06-01"}, {"2014-12-01"} ) + ``` + + Data outside these ranges will not be loaded. + +2) Fixed Range + CREATE TABLE table_range + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY RANGE (k1, k2, k3) + ( + PARTITION p1 VALUES [("2014-01-01", "10", "200"), ("2014-01-01", "20", "300")), + PARTITION p2 VALUES [("2014-06-01", "100", "200"), ("2014-07-01", "100", "300")) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD" + ); + +4. Create a mysql table + + ``` + CREATE TABLE example_db.table_mysql + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=mysql + PROPERTIES + ( + "host" = "127.0.0.1", + "port" = "8239", + "user" = "mysql_user", + "password" = "mysql_passwd", + "database" = "mysql_db_test", + "table" = "mysql_table_test" + ); + ``` + +5. Create a broker table, with file on HDFS, line delimit by "|", column separated by "\n" + + ``` + CREATE EXTERNAL TABLE example_db.table_broker ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=broker + PROPERTIES ( + "broker_name" = "hdfs", + "path" = "hdfs://hdfs_host:hdfs_port/data1,hdfs://hdfs_host:hdfs_port/data2,hdfs://hdfs_host:hdfs_port/data3%2c4", + "column_separator" = "|", + "line_delimiter" = "\n" + ) + BROKER PROPERTIES ( + "username" = "hdfs_user", + "password" = "hdfs_password" + ); + ``` + +6. Create table will HLL column + + ``` + CREATE TABLE example_db.example_table + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 HLL HLL_UNION, + v2 HLL HLL_UNION + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + DISTRIBUTED BY HASH(k1) BUCKETS 32; + ``` + +7. Create a table will BITMAP_UNION column + + ``` + CREATE TABLE example_db.example_table + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 BITMAP BITMAP_UNION, + v2 BITMAP BITMAP_UNION + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + DISTRIBUTED BY HASH(k1) BUCKETS 32; + ``` + +8. Create 2 colocate join table. + + ``` + CREATE TABLE `t1` ( + `id` int(11) COMMENT "", + `value` varchar(8) COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "colocate_with" = "group1" + ); + CREATE TABLE `t2` ( + `id` int(11) COMMENT "", + `value` varchar(8) COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "colocate_with" = "group1" + ); + ``` + +9. Create a broker table, with file on BOS. + + ``` + CREATE EXTERNAL TABLE example_db.table_broker ( + k1 DATE + ) + ENGINE=broker + PROPERTIES ( + "broker_name" = "bos", + "path" = "bos://my_bucket/input/file", + ) + BROKER PROPERTIES ( + "bos_endpoint" = "http://bj.bcebos.com", + "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "bos_secret_accesskey"="yyyyyyyyyyyyyyyyyyyy" + ); + ``` + +10. Create a table with a bitmap index + + ``` + CREATE TABLE example_db.table_hash + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 CHAR(10) REPLACE, + v2 INT SUM, + INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx' + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + COMMENT "my first doris table" + DISTRIBUTED BY HASH(k1) BUCKETS 32 + PROPERTIES ("storage_type"="column"); + ``` + +11. Create a dynamic partitioning table (dynamic partitioning needs to be enabled in FE configuration), which creates partitions 3 days in advance every day. For example, if today is' 2020-01-08 ', partitions named 'p20200108', 'p20200109', 'p20200110', 'p20200111' will be created. + + ``` + [types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) + [types: [DATE]; keys: [2020-01-09]; ‥types: [DATE]; keys: [2020-01-10]; ) + [types: [DATE]; keys: [2020-01-10]; ‥types: [DATE]; keys: [2020-01-11]; ) + [types: [DATE]; keys: [2020-01-11]; ‥types: [DATE]; keys: [2020-01-12]; ) + ``` + + ``` + CREATE TABLE example_db.dynamic_partition + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY RANGE (k1) + ( + PARTITION p1 VALUES LESS THAN ("2014-01-01"), + PARTITION p2 VALUES LESS THAN ("2014-06-01"), + PARTITION p3 VALUES LESS THAN ("2014-12-01") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "32" + ); + ``` +12. Create a table with rollup index +``` + CREATE TABLE example_db.rolup_index_table + ( + event_day DATE, + siteid INT DEFAULT '10', + citycode SMALLINT, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(event_day, siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 10 + rollup ( + r1(event_day,siteid), + r2(event_day,citycode), + r3(event_day) + ) + PROPERTIES("replication_num" = "3"); +``` + +12. Create a inmemory table: + +``` + CREATE TABLE example_db.table_hash + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 CHAR(10) REPLACE, + v2 INT SUM, + INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx' + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + COMMENT "my first doris table" + DISTRIBUTED BY HASH(k1) BUCKETS 32 + PROPERTIES ("in_memory"="true"); +``` + +## keyword + + CREATE,TABLE diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE VIEW.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE VIEW.md new file mode 100644 index 00000000000000..fa6e245ed70170 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE VIEW.md @@ -0,0 +1,68 @@ +--- +{ + "title": "CREATE VIEW", + "language": "en" +} +--- + + + +# CREATE VIEW +## Description + This statement is used to create a logical view + Grammar: + + CREATE VIEW [IF NOT EXISTS] + [db_name.]view_name + (column1[ COMMENT "col comment"][, column2, ...]) + AS query_stmt + + Explain: + + 1. Views are logical views without physical storage. All queries on views are equivalent to sub-queries corresponding to views. + 2. query_stmt is arbitrarily supported SQL. + +## example + + 1. Create view example_view on example_db + + CREATE VIEW example_db.example_view (k1, k2, k3, v1) + AS + SELECT c1 as k1, k2, k3, SUM(v1) FROM example_table + WHERE k1 = 20160112 GROUP BY k1,k2,k3; + + 2. Create view with comment + + CREATE VIEW example_db.example_view + ( + k1 COMMENT "first key", + k2 COMMENT "second key", + k3 COMMENT "third key", + v1 COMMENT "first value" + ) + COMMENT "my first view" + AS + SELECT c1 as k1, k2, k3, SUM(v1) FROM example_table + WHERE k1 = 20160112 GROUP BY k1,k2,k3; + +## keyword + + CREATE,VIEW + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/Colocate Join.md b/docs/en/sql-reference/sql-statements/Data Definition/Colocate Join.md new file mode 100644 index 00000000000000..f2027187554287 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/Colocate Join.md @@ -0,0 +1,98 @@ +--- +{ + "title": "Colocate Join", + "language": "en" +} +--- + + + +# Colocate Join +## Description +Colocate/Local Join means that when multiple nodes are Join, there is no data movement and network transmission, and each node is only Join locally. +The premise of Join locally is to import data from the same Join Key into a fixed node according to the same rules. + +1 How To Use: + +Simply add the property colocate_with when building a table. The value of colocate_with can be set to any one of the same set of colocate tables. +However, you need to ensure that tables in the colocate_with attribute are created first. + +If you need to Colocate Join table t1 and t2, you can build tables according to the following statements: + +CREATE TABLE `t1` ( +`id` int(11) COMMENT "", +'value ` varchar (8) COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 10 +PROPERTIES ( +"colocate_with" = "t1" +); + +CREATE TABLE `t2` ( +`id` int(11) COMMENT "", +'value ` varchar (8) COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 10 +PROPERTIES ( +"colocate_with" = "t1" +); + +2 Colocate Join 目前的限制: + +1. Colcoate Table must be an OLAP-type table +2. The BUCKET number of tables with the same colocate_with attribute must be the same +3. The number of copies of tables with the same colocate_with attribute must be the same +4. Data types of DISTRIBUTTED Columns for tables with the same colocate_with attribute must be the same + +3 Colocate Join's applicable scenario: + +Colocate Join is well suited for scenarios where tables are bucketed according to the same field and high frequency according to the same field Join. + +4 FAQ: + +Q: 支持多张表进行Colocate Join 吗? + +A: 25903;. 25345 + +Q: Do you support Colocate table and normal table Join? + +A: 25903;. 25345 + +Q: Does the Colocate table support Join with non-bucket Key? + +A: Support: Join that does not meet Colocate Join criteria will use Shuffle Join or Broadcast Join + +Q: How do you determine that Join is executed according to Colocate Join? + +A: The child node of Hash Join in the result of explain is Colocate Join if it is OlapScanNode directly without Exchange Node. + +Q: How to modify the colocate_with attribute? + +A: ALTER TABLE example_db.my_table set ("colocate_with"="target_table"); + +Q: 229144; colcoate join? + +A: set disable_colocate_join = true; 就可以禁用Colocate Join，查询时就会使用Shuffle Join 和Broadcast Join + +## keyword + +COLOCATE, JOIN, CREATE TABLE diff --git a/docs/en/sql-reference/sql-statements/Data Definition/DROP DATABASE.md b/docs/en/sql-reference/sql-statements/Data Definition/DROP DATABASE.md new file mode 100644 index 00000000000000..9a1b09560061d0 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/DROP DATABASE.md @@ -0,0 +1,42 @@ +--- +{ + "title": "DROP DATABASE", + "language": "en" +} +--- + + + +# DROP DATABASE +##Description +This statement is used to delete the database +Grammar: +DROP DATABASE [IF EXISTS] db_name; + +Explain: +After executing DROP DATABASE for a period of time, the deleted database can be restored through the RECOVER statement. See RECOVER statement for details + +## example +1. Delete database db_test +DROP DATABASE db_test; + +## keyword +DROP,DATABASE + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/DROP INDEX.md b/docs/en/sql-reference/sql-statements/Data Definition/DROP INDEX.md new file mode 100644 index 00000000000000..732d6658985d5a --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/DROP INDEX.md @@ -0,0 +1,37 @@ +--- +{ + "title": "DROP INDEX", + "language": "en" +} +--- + + + +# DROP INDEX + +## description + + This statement is used to delete index from table + grammer: + DROP INDEX index_name ON [db_name.]table_name; + +## keyword + + DROP,INDEX diff --git a/docs/en/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md b/docs/en/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md new file mode 100644 index 00000000000000..bc90ec20db3743 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md @@ -0,0 +1,110 @@ +--- +{ + "title": "DROP MATERIALIZED VIEW", + "language": "en" +} +--- + + + +# DROP MATERIALIZED VIEW + +## description + This statement is used to delete a materialized view. Synchronization syntax + +syntax: + + ``` + DROP MATERIALIZED VIEW [IF EXISTS] mv_name ON table_name + ``` + +1. IF EXISTS + If the materialized view does not exist, doris will not throw an error. If this keyword is not declared, an error will be reported if the materialized view does not exist. +Ranch + +2. mv_name + The name of the materialized view to be deleted. Required. + +3. Table_name + Name of the table to which the materialized view to be deleted belongs. Required. + +## example + +Table structure is + +``` +mysql> desc all_type_table all; ++----------------+-------+----------+------+-------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++----------------+-------+----------+------+-------+---------+-------+ +| all_type_table | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | false | N/A | NONE | +| | k3 | INT | Yes | false | N/A | NONE | +| | k4 | BIGINT | Yes | false | N/A | NONE | +| | k5 | LARGEINT | Yes | false | N/A | NONE | +| | k6 | FLOAT | Yes | false | N/A | NONE | +| | k7 | DOUBLE | Yes | false | N/A | NONE | +| | | | | | | | +| k1_sumk2 | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | false | N/A | SUM | ++----------------+-------+----------+------+-------+---------+-------+ +``` + +1. Drop the materialized view named k1_sumk2 of the table all_type_table + + ``` + drop materialized view k1_sumk2 on all_type_table; + ``` + Table structure after materialized view is deleted as following: + + ``` ++----------------+-------+----------+------+-------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++----------------+-------+----------+------+-------+---------+-------+ +| all_type_table | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | false | N/A | NONE | +| | k3 | INT | Yes | false | N/A | NONE | +| | k4 | BIGINT | Yes | false | N/A | NONE | +| | k5 | LARGEINT | Yes | false | N/A | NONE | +| | k6 | FLOAT | Yes | false | N/A | NONE | +| | k7 | DOUBLE | Yes | false | N/A | NONE | ++----------------+-------+----------+------+-------+---------+-------+ + ``` + +2. Delete a non-existing materialized view in the table all_type_table + + ``` + drop materialized view k1_k2 on all_type_table; +ERROR 1064 (HY000): errCode = 2, detailMessage = Materialized view [k1_k2] does not exist in table [all_type_table] + ``` + + The delete request directly reports an error + +3. Delete the materialized view k1_k2 in the table all_type_table. Materialized view does not exist and no error is reported. + + ``` + drop materialized view if exists k1_k2 on all_type_table; +Query OK, 0 rows affected (0.00 sec) + ``` + + If it exists, it will be deleted; If it does not exist, no error will be reported. + +## keyword + DROP, MATERILIAZED, VIEW diff --git a/docs/en/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md b/docs/en/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md new file mode 100644 index 00000000000000..c673762169f2c7 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md @@ -0,0 +1,41 @@ +--- +{ + "title": "DROP REPOSITORY", + "language": "en" +} +--- + + + +# DROP REPOSITORY +## Description +This statement is used to delete a created warehouse. Only root or superuser users can delete the warehouse. +Grammar: +DROP REPOSITORY `repo_name`; + +Explain: +1. Delete the warehouse, just delete the mapping of the warehouse in Palo, and do not delete the actual warehouse data. After deletion, you can map to the repository again by specifying the same broker and LOCATION. + +## example +1. Delete the warehouse named bos_repo: +DROP REPOSITORY `bos_repo`; + +## keyword +DROP REPOSITORY diff --git a/docs/en/sql-reference/sql-statements/Data Definition/DROP TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/DROP TABLE.md new file mode 100644 index 00000000000000..6900d6ed2aec8b --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/DROP TABLE.md @@ -0,0 +1,45 @@ +--- +{ + "title": "DROP TABLE", + "language": "en" +} +--- + + + +# DROP TABLE +## Description +This statement is used to delete the table. +Grammar: +DROP TABLE [IF EXISTS] [db_name.]table_name; + +Explain: +After executing DROP TABLE for a period of time, the deleted table can be restored through the RECOVER statement. See RECOVER statement for details + +## example +1. Delete a table +DROP TABLE my_table; + +2. If it exists, delete the table that specifies the database +DROP TABLE IF EXISTS example_db.my_table; + +## keyword +DROP,TABLE + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/DROP VIEW.md b/docs/en/sql-reference/sql-statements/Data Definition/DROP VIEW.md new file mode 100644 index 00000000000000..b34a402b4f6207 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/DROP VIEW.md @@ -0,0 +1,40 @@ +--- +{ + "title": "DROP VIEW", + "language": "en" +} +--- + + + +# DROP VIEW +## Description +This statement is used to delete a logical view VIEW +Grammar: +DROP VIEW [IF EXISTS] +[db_name.]view_name; + +## example +1. If it exists, delete view example_view on example_db +DROP VIEW IF EXISTS example_db.example_view; + +## keyword +DROP,VIEW + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/HLL.md b/docs/en/sql-reference/sql-statements/Data Definition/HLL.md new file mode 100644 index 00000000000000..f8d61a954da860 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/HLL.md @@ -0,0 +1,111 @@ +--- +{ + "title": "HLL", + "language": "en" +} +--- + + + +# HLL +## Description +HLL is an engineering implementation based on the HyperLogLog algorithm. It is used to store the intermediate results of the HyperLog calculation process. It can only be used as the value column type of the table. +By aggregating to reduce the amount of data continuously, in order to achieve the purpose of speeding up the query, based on which an estimated result, the error is about 1%. +The HLL column is generated by other columns or data in the imported data. When imported, the hll_hash function is used to specify which column in the data is used to generate the HLL column. +It is often used to replace count distinct, and to quickly calculate UV in business by combining rollup. + +The correlation function: + +TOTAL UNION +This function is an aggregation function, which is used to calculate the cardinality estimation of all data satisfying the conditions. This function can also be used to analyze functions. It only supports the default window and does not support the window clause. + +Coach L.u RAW AGG +This function is an aggregation function that aggregates HLL type fields and returns HLL type. + +HLL_CARDINALITY(hll) +This function is used to estimate the cardinality of a single HLL sequence + +HLL_HASH(column_name) +Generate HLL column types for insert or import, see the instructions for the use of imports + +EMPTY_HLL() +Generate empty HLL column types for insert or import, see the instructions for the use of imports + +## example +1. First create a table with HLL columns +create table test( +dt date, +id int, +name char(10), +Province of char (10), +The char (1), +the European Union, +European Union +distributed by hash(id) buckets 32; + +2. Import data. See help curl for the way you import it. + + A. Generate HLL columns using columns in tables + + curl --location-trusted -uname:password -T data -H "label:load_1" -H "columns:dt, id, name, province, os, set1=hll_hash(id), set2=hll_hash(name)" + http://host/api/test_db/test/_stream_load + + B. Generate HLL columns using a column in the data + + curl --location-trusted -uname:password -T data -H "label:load_1" -H "columns:dt, id, name, province, sex, cuid, os, set1=hll_hash(cuid), set2=hll_hash(os)" + http://host/api/test_db/test/_stream_load + +3. There are three common ways of aggregating data: (without aggregating the base table directly, the speed may be similar to that of using NDV directly) + +A. Create a rollup that allows HLL columns to generate aggregation. +alter table test add rollup test_rollup(dt, set1); + +B. Create another table dedicated to computing uv, and insert data) + +create table test_uv( +dt date, +uv_set hll hll_union) +distributed by hash(id) buckets 32; + +insert into test_uv select dt, set1 from test; + +C. Create another table dedicated to computing uv, then insert and generate HLL columns from other non-hll columns of test through hll_hash + +create table test_uv( +dt date, +id_set hll hll_union) +distributed by hash(id) buckets 32; + +insert into test_uv select dt, hll_hash(id) from test; + +4. Query, HLL column is not allowed to query its original value directly, it can be queried by matching functions. + +a. 27714; 24635; uv +select HLL_UNION_AGG(uv_set) from test_uv; + +B. Seek every day's UV +select dt, HLL_CARDINALITY(uv_set) from test_uv; + +C. Find the aggregate value of Set1 in the test table +select dt, HLL_CARDINALITY(uv) from (select dt, HLL_RAW_AGG(set1) as uv from test group by dt) tmp; +select dt, HLL_UNION_AGG(set1) as uv from test group by dt; + +## keyword +HLL diff --git a/docs/en/sql-reference/sql-statements/Data Definition/RECOVER.md b/docs/en/sql-reference/sql-statements/Data Definition/RECOVER.md new file mode 100644 index 00000000000000..d2f1f350194d34 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/RECOVER.md @@ -0,0 +1,54 @@ +--- +{ + "title": "RECOVER", + "language": "en" +} +--- + + + +# RECOVER +## Description +This statement is used to restore previously deleted databases, tables, or partitions +Grammar: +1)24674;"22797database; +RECOVER DATABASE db_name; +2) 恢复 table +RECOVER TABLE [db_name.]table_name; +3)24674;"22797partition +RECOVER PARTITION partition name FROM [dbu name.]table name; + +Explain: +1. This operation can only recover the meta-information deleted in the previous period of time. The default is 1 day.(You can configure it with the `catalog_trash_expire_second` parameter in fe.conf) +2. If new meta-information of the same name and type is created after deleting meta-information, the previously deleted meta-information cannot be restored. + +## example +1. Restore the database named example_db +RECOVER DATABASE example_db; + +2. Restore table named example_tbl +RECOVER TABLE example_db.example_tbl; + +3. Restore partition named P1 in example_tbl +RECOVER PARTITION p1 FROM example_tbl; + +## keyword +RECOVER + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/RESTORE.md b/docs/en/sql-reference/sql-statements/Data Definition/RESTORE.md new file mode 100644 index 00000000000000..0011187ae2ade3 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/RESTORE.md @@ -0,0 +1,78 @@ +--- +{ + "title": "RESTORE", + "language": "en" +} +--- + + + +# RESTORE +## Description +1. RESTOR +This statement is used to restore the data previously backed up by the BACKUP command to the specified database. This command is an asynchronous operation. After successful submission, you need to check progress through the SHOW RESTORE command. Restoring tables of OLAP type is supported only. +Grammar: +SNAPSHOT RESTORE [dbu name].{snapshot name} +FROM `repository_name` +ON ( +"`Table `uname'[`partition (`p1',...)] [as `tbl `uu alias'], +... +) +PROPERTIES ("key"="value", ...); + +Explain: +1. Only one BACKUP or RESTORE task can be performed under the same database. +2. The ON clause identifies the tables and partitions that need to be restored. If no partition is specified, all partitions of the table are restored by default. The specified tables and partitions must already exist in the warehouse backup. +3. The backup tables in the warehouse can be restored to new tables through AS statements. But the new table name cannot already exist in the database. Partition name cannot be changed. +4. The backup tables in the warehouse can be restored and replaced with the same-name tables in the database, but the table structure of the two tables must be completely consistent. Table structure includes: table name, column, partition, Rollup and so on. +5. Partitions of the recovery table can be specified, and the system checks whether the partition Range matches. +6. PROPERTIES currently supports the following attributes: +"Backup_timestamp" = "2018-05-04-16-45-08": specifies which version of the time to restore the corresponding backup must be filled in. This information can be obtained through the `SHOW SNAPSHOT ON repo;'statement. +"Replication_num" = "3": Specifies the number of replicas of the restored table or partition. The default is 3. If an existing table or partition is restored, the number of copies must be the same as the number of copies of an existing table or partition. At the same time, there must be enough hosts to accommodate multiple copies. +"Timeout" = "3600": Task timeout, default to one day. Unit seconds. +"Meta_version" = 40: Use the specified meta_version to read the previously backed up metadata. Note that as a temporary solution, this parameter is only used to restore the data backed up by the older version of Doris. The latest version of the backup data already contains metaversion, no need to specify. + +## example +1. Restore backup table backup_tbl in snapshot_1 from example_repo to database example_db1 with the time version of "2018-05-04-16-45-08". Restore to one copy: +RESTORE SNAPSHOT example_db1.`snapshot_1` +FROM `example 'u repo' +ON ( `backup_tbl` ) +PROPERTIES +( +"backup_timestamp"="2018-05-04-16-45-08", +"Replication\ num" = "1" +); + +2. Restore the partitions p1, P2 of table backup_tbl in snapshot_2 and table backup_tbl2 to database example_db1 from example_repo and rename it new_tbl. The time version is "2018-05-04-17-11-01". By default, three copies are restored: +RESTORE SNAPSHOT example_db1.`snapshot_2` +FROM `example 'u repo' +ON +( +`backup_tbl` PARTITION (`p1`, `p2`), +`backup_tbl2` AS `new_tbl` +) +PROPERTIES +( +"backup_timestamp"="2018-05-04-17-11-01" +); + +## keyword +RESTORE + diff --git a/docs/en/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md new file mode 100644 index 00000000000000..247f129d15a15b --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md @@ -0,0 +1,52 @@ +--- +{ + "title": "TRUNCATE TABLES", + "language": "en" +} +--- + + + +# TRUNCATE TABLES +## Description +This statement is used to empty the data of the specified table and partition +Grammar: + +TRUNCATE TABLE [db.]tbl[ PARTITION(p1, p2, ...)]; + +Explain: +1. The statement empties the data, but retains the table or partition. +2. Unlike DELETE, this statement can only empty the specified tables or partitions as a whole, without adding filtering conditions. +3. Unlike DELETE, using this method to clear data will not affect query performance. +4. The data deleted by this operation is not recoverable. +5. When using this command, the table state should be NORMAL, i.e. SCHEMA CHANGE operations are not allowed. + +## example + +1. Clear the table TBL under example_db + +TRUNCATE TABLE example_db.tbl; + +2. P1 and P2 partitions of clearing TABLE tbl + +TRUNCATE TABLE tbl PARTITION(p1, p2); + +## keyword +TRUNCATE,TABLE diff --git a/docs/en/sql-reference/sql-statements/Data Definition/create-function.md b/docs/en/sql-reference/sql-statements/Data Definition/create-function.md new file mode 100644 index 00000000000000..183fc6283d4539 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/create-function.md @@ -0,0 +1,111 @@ +--- +{ + "title": "CREATE FUNCTION", + "language": "en" +} +--- + + + +# CREATE FUNCTION +##Description +### Syntax + +``` +CREATE [AGGREGATE] FUNCTION function_name + (angry type [...]) + RETURNS ret_type + [INTERMEDIATE inter_type] + [PROPERTIES ("key" = "value" [, ...]) ] +``` + +### Parameters + +>`AGGREGATE`: If this is the case, it means that the created function is an aggregate function, otherwise it is a scalar function. +> +>`Function_name`: To create the name of the function, you can include the name of the database. For example: `db1.my_func'. +> +>` arg_type': The parameter type of the function is the same as the type defined at the time of table building. Variable-length parameters can be represented by `,...`. If it is a variable-length type, the type of the variable-length part of the parameters is the same as the last non-variable-length parameter type. +> +>`ret_type`: Function return type. +> +>`Inter_type`: A data type used to represent the intermediate stage of an aggregate function. +> +>`properties`: Used to set properties related to this function. Properties that can be set include +> +> "Object_file": Custom function dynamic library URL path, currently only supports HTTP/HTTPS protocol, this path needs to remain valid throughout the life cycle of the function. This option is mandatory +> +> "symbol": Function signature of scalar functions for finding function entries from dynamic libraries. This option is mandatory for scalar functions +> +> "init_fn": Initialization function signature of aggregate function. Necessary for aggregation functions +> +> "update_fn": Update function signature of aggregate function. Necessary for aggregation functions +> +> "merge_fn": Merge function signature of aggregate function. Necessary for aggregation functions +> +> "serialize_fn": Serialized function signature of aggregate function. For aggregation functions, it is optional, and if not specified, the default serialization function will be used +> +> "finalize_fn": A function signature that aggregates functions to obtain the final result. For aggregation functions, it is optional. If not specified, the default fetch result function will be used. +> +> "md5": The MD5 value of the function dynamic link library, which is used to verify that the downloaded content is correct. This option is optional +> +> "prepare_fn": Function signature of the prepare function for finding the entry from the dynamic library. This option is optional for custom functions +> +> "close_fn": Function signature of the close function for finding the entry from the dynamic library. This option is optional for custom functions + + +This statement creates a custom function. Executing this command requires that the user have `ADMIN` privileges. + +If the `function_name` contains the database name, the custom function will be created in the corresponding database, otherwise the function will be created in the database where the current session is located. The name and parameters of the new function cannot be the same as functions already existing in the current namespace, otherwise the creation will fail. But only with the same name and different parameters can the creation be successful. + +## example + +1. Create a custom scalar function + + ``` + CREATE FUNCTION my_add(INT, INT) RETURNS INT PROPERTIES ( + "symbol" = "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", + "object_file" ="http://host:port/libmyadd.so" + ); + ``` +2. Create a custom scalar function with prepare/close functions + + ``` + CREATE FUNCTION my_add(INT, INT) RETURNS INT PROPERTIES ( + "symbol" = "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", + "prepare_fn" = "_ZN9doris_udf14AddUdf_prepareEPNS_15FunctionContextENS0_18FunctionStateScopeE", + "close_fn" = "_ZN9doris_udf12AddUdf_closeEPNS_15FunctionContextENS0_18FunctionStateScopeE", + "object_file" = "http://host:port/libmyadd.so" + ); + ``` + +3. Create a custom aggregation function + + ``` + CREATE AGGREGATE FUNCTION my_count (BIGINT) RETURNS BIGINT PROPERTIES ( + "init_fn"= "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", + "update_fn" = "zn9dorisudf11CountupdateepnsFunctionContexterknsIntvalepnsbigintvale", + "merge_fn" = "zn9dorisudf10CountMergeepnsFunctionContexterknsBigintvaleps2 + "finalize_fn" = "zn9dorisudf13CountFinalizepnsFunctionContexterknsBigintvale", + "object_file" = "http://host:port/libudasample.so" + ); + ``` +##keyword +CREATE,FUNCTION diff --git a/docs/en/sql-reference/sql-statements/Data Definition/drop-function.md b/docs/en/sql-reference/sql-statements/Data Definition/drop-function.md new file mode 100644 index 00000000000000..bd84bc512e8a51 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/drop-function.md @@ -0,0 +1,54 @@ +--- +{ + "title": "DROP FUNCTION", + "language": "en" +} +--- + + + +# DROP FUNCTION +##Description +### Syntax + +``` +DROP FUNCTION function_name +(angry type [...]) +``` + +### Parameters + +>` function_name': To delete the name of the function +> +>` arg_type`: To delete the parameter list of the function +> + + +Delete a custom function. The name of the function and the type of the parameter are exactly the same before they can be deleted. + +## example + +1. Delete a function + +``` +DROP FUNCTION my_add(INT, INT) +``` +##keyword +DROP,FUNCTION diff --git a/docs/en/sql-reference/sql-statements/Data Definition/show-functions.md b/docs/en/sql-reference/sql-statements/Data Definition/show-functions.md new file mode 100644 index 00000000000000..b32e02d8a92d11 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Definition/show-functions.md @@ -0,0 +1,77 @@ +--- +{ + "title": "SHOW FUNCTIONS", + "language": "en" +} +--- + + + +# SHOW FUNCTIONS +## Description +### Syntax + +``` +SHOW [FULL] [BUILTIN] FUNCTIONS [IN|FROM db] [LIKE 'function_pattern'] +``` + +### Parameters + +>`full`: Indicate to show the details of function +>`builtin`: Indicate to show the functions that doris provides +>`db`: The name of the database to query +>`function_pattern`: The parameter to filter function name + +Look at all the custom(builtin) functions under the database. If the user specifies the database, then look at the corresponding database, otherwise directly query the database where the current session is located. + +You need `SHOW'privileges for this database + +## example + +``` +mysql> show full functions in testDb\G +*************************** 1. row *************************** + Signature: my_add(INT,INT) + Return Type: INT + Function Type: Scalar +Intermediate Type: NULL + Properties: {"symbol":"_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_","object_file":"http://host:port/libudfsample.so","md5":"cfe7a362d10f3aaf6c49974ee0f1f878"} +*************************** 2. row *************************** + Signature: my_count(BIGINT) + Return Type: BIGINT + Function Type: Aggregate +Intermediate Type: NULL + Properties: {"object_file":"http://host:port/libudasample.so","finalize_fn":"_ZN9doris_udf13CountFinalizeEPNS_15FunctionContextERKNS_9BigIntValE","init_fn":"_ZN9doris_udf9CountInitEPNS_15FunctionContextEPNS_9BigIntValE","merge_fn":"_ZN9doris_udf10CountMergeEPNS_15FunctionContextERKNS_9BigIntValEPS2_","md5":"37d185f80f95569e2676da3d5b5b9d2f","update_fn":"_ZN9doris_udf11CountUpdateEPNS_15FunctionContextERKNS_6IntValEPNS_9BigIntValE"} + +2 rows in set (0.00 sec) +mysql> show builtin functions in testDb like 'year%'; ++---------------+ +| Function Name | ++---------------+ +| year | +| years_add | +| years_diff | +| years_sub | ++---------------+ +2 rows in set (0.00 sec) +``` + +##keyword +SHOW,FUNCTIONS diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md new file mode 100644 index 00000000000000..e8a8fa778672f9 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md @@ -0,0 +1,428 @@ +--- +{ + "title": "BROKER LOAD", + "language": "en" +} +--- + + + +# BROKER LOAD +## description + + Broker load will load data into Doris via Broker. + Use `show broker;` to see the Broker deployed in cluster. + + Support following data sources: + + 1. Baidu HDFS: hdfs for Baidu. Only be used inside Baidu. + 2. Baidu AFS: afs for Baidu. Only be used inside Baidu. + 3. Baidu Object Storage(BOS): BOS on Baidu Cloud. + 4. Apache HDFS. + +### Syntax: + + LOAD LABEL load_label + ( + data_desc1[, data_desc2, ...] + ) + WITH BROKER broker_name + [broker_properties] + [opt_properties]; + + 1. load_label + + Unique load label within a database. + syntax: + [database_name.]your_label + + 2. data_desc + + To describe the data source. + syntax: + DATA INFILE + ( + "file_path1"[, file_path2, ...] + ) + [NEGATIVE] + INTO TABLE `table_name` + [PARTITION (p1, p2)] + [COLUMNS TERMINATED BY "column_separator"] + [FORMAT AS "file_type"] + [(column_list)] + [SET (k1 = func(k2))] + [WHERE predicate] + + Explain: + file_path: + + File path. Support wildcard. Must match to file, not directory. + + PARTITION: + + Data will only be loaded to specified partitions. Data out of partition's range will be filtered. If not specifed, all partitions will be loaded. + + NEGATIVE: + + If this parameter is specified, it is equivalent to importing a batch of "negative" data to offset the same batch of data loaded before. + + This parameter applies only to the case where there are value columns and the aggregation type of value columns is only SUM. + + column_separator: + + Used to specify the column separator in the import file. Default is `\t`. + If the character is invisible, it needs to be prefixed with `\\x`, using hexadecimal to represent the separator. + + For example, the separator `\x01` of the hive file is specified as `\\ x01` + + file_type: + + Used to specify the type of imported file, such as parquet, orc, csv. Default values are determined by the file suffix name. + + column_list: + + Used to specify the correspondence between columns in the import file and columns in the table. + + When you need to skip a column in the import file, specify it as a column name that does not exist in the table. + + syntax: + (col_name1, col_name2, ...) + + SET: + + If this parameter is specified, a column of the source file can be transformed according to a function, and then the transformed result can be loaded into the table. The grammar is `column_name = expression`. Some examples are given to help understand. + + Example 1: There are three columns "c1, c2, c3" in the table. The first two columns in the source file correspond in turn (c1, c2), and the last two columns correspond to c3. Then, column (c1, c2, tmp_c3, tmp_c4) SET (c3 = tmp_c3 + tmp_c4) should be specified. + + Example 2: There are three columns "year, month, day" in the table. There is only one time column in the source file, in the format of "2018-06-01:02:03". Then you can specify columns (tmp_time) set (year = year (tmp_time), month = month (tmp_time), day = day (tmp_time)) to complete the import. + + WHERE: + + After filtering the transformed data, data that meets where predicates can be loaded. Only column names in tables can be referenced in WHERE statements. + + 3. broker_name + + The name of the Broker used can be viewed through the `show broker` command. + + 4. broker_properties + + Used to provide Broker access to data sources. Different brokers, and different access methods, need to provide different information. + + 1. Baidu HDFS/AFS + + Access to Baidu's internal hdfs/afs currently only supports simple authentication, which needs to be provided: + + username: hdfs username + password: hdfs password + + 2. BOS + + bos_endpoint. + bos_accesskey: cloud user's accesskey + bos_secret_accesskey: cloud user's secret_accesskey + + 3. Apache HDFS + + Community version of HDFS supports simple authentication, Kerberos authentication, and HA configuration. + + Simple authentication: + hadoop.security.authentication = simple (default) + username: hdfs username + password: hdfs password + + kerberos authentication: + hadoop.security.authentication = kerberos + kerberos_principal: kerberos's principal + kerberos_keytab: path of kerberos's keytab file. This file should be able to access by Broker + kerberos_keytab_content: Specify the contents of the KeyTab file in Kerberos after base64 encoding. This option is optional from the kerberos_keytab configuration. + + namenode HA: + By configuring namenode HA, new namenode can be automatically identified when the namenode is switched + dfs.nameservices: hdfs service name，customize，eg: "dfs.nameservices" = "my_ha" + dfs.ha.namenodes.xxx: Customize the name of a namenode, separated by commas. XXX is a custom name in dfs. name services, such as "dfs. ha. namenodes. my_ha" = "my_nn" + dfs.namenode.rpc-address.xxx.nn: Specify RPC address information for namenode, where NN denotes the name of the namenode configured in dfs.ha.namenodes.xxxx, such as: "dfs.namenode.rpc-address.my_ha.my_nn"= "host:port" + dfs.client.failover.proxy.provider: Specify the provider that client connects to namenode by default: org. apache. hadoop. hdfs. server. namenode. ha. Configured Failover ProxyProvider. + + 4. opt_properties + + Used to specify some special parameters. + Syntax: + [PROPERTIES ("key"="value", ...)] + + You can specify the following parameters: + + timout: Specifies the timeout time for the import operation. The default timeout is 4 hours per second. + + max_filter_ratio: Data ratio of maximum tolerance filterable (data irregularity, etc.). Default zero tolerance. + + exc_mem_limit: Memory limit. Default is 2GB. Unit is Bytes. + + strict_mode: Whether the data is strictly restricted. The default is false. + + timezone: Specify time zones for functions affected by time zones, such as strftime/alignment_timestamp/from_unixtime, etc. See the documentation for details. If not specified, use the "Asia/Shanghai" time zone. + + 5. Load data format sample + + Integer（TINYINT/SMALLINT/INT/BIGINT/LARGEINT）: 1, 1000, 1234 + Float（FLOAT/DOUBLE/DECIMAL）: 1.1, 0.23, .356 + Date（DATE/DATETIME）: 2017-10-03, 2017-06-13 12:34:03. + (Note: If it's in other date formats, you can use strftime or time_format functions to convert in the import command) + + String（CHAR/VARCHAR）: "I am a student", "a" + NULL: \N + +## example + + 1. Load a batch of data from HDFS, specify timeout and filtering ratio. Use the broker with the inscription my_hdfs_broker. Simple authentication. + + LOAD LABEL example_db.label1 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + ) + WITH BROKER my_hdfs_broker + ( + "username" = "hdfs_user", + "password" = "hdfs_passwd" + ) + PROPERTIES + ( + "timeout" = "3600", + "max_filter_ratio" = "0.1" + ); + + Where hdfs_host is the host of the namenode and hdfs_port is the fs.defaultFS port (default 9000) + + 2. Load a batch of data from AFS contains multiple files. Import different tables, specify separators, and specify column correspondences. + + LOAD LABEL example_db.label2 + ( + DATA INFILE("afs://afs_host:hdfs_port/user/palo/data/input/file1") + INTO TABLE `my_table_1` + COLUMNS TERMINATED BY "," + (k1, k3, k2, v1, v2), + DATA INFILE("afs://afs_host:hdfs_port/user/palo/data/input/file2") + INTO TABLE `my_table_2` + COLUMNS TERMINATED BY "\t" + (k1, k2, k3, v2, v1) + ) + WITH BROKER my_afs_broker + ( + "username" = "afs_user", + "password" = "afs_passwd" + ) + PROPERTIES + ( + "timeout" = "3600", + "max_filter_ratio" = "0.1" + ); + + + 3. Load a batch of data from HDFS, specify hive's default delimiter \\x01, and use wildcard * to specify all files in the directory. Use simple authentication and configure namenode HA at the same time + + LOAD LABEL example_db.label3 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/*") + INTO TABLE `my_table` + COLUMNS TERMINATED BY "\\x01" + ) + WITH BROKER my_hdfs_broker + ( + "username" = "hdfs_user", + "password" = "hdfs_passwd", + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ) + + 4. Load a batch of "negative" data from HDFS. Use Kerberos authentication to provide KeyTab file path. + + LOAD LABEL example_db.label4 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/old_file) + NEGATIVE + INTO TABLE `my_table` + COLUMNS TERMINATED BY "\t" + ) + WITH BROKER my_hdfs_broker + ( + "hadoop.security.authentication" = "kerberos", + "kerberos_principal"="doris@YOUR.COM", + "kerberos_keytab"="/home/palo/palo.keytab" + ) + + 5. Load a batch of data from HDFS, specify partition. At the same time, use Kerberos authentication mode. Provide the KeyTab file content encoded by base64. + + LOAD LABEL example_db.label5 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (k1, k3, k2, v1, v2) + ) + WITH BROKER my_hdfs_broker + ( + "hadoop.security.authentication"="kerberos", + "kerberos_principal"="doris@YOUR.COM", + "kerberos_keytab_content"="BQIAAABEAAEACUJBSURVLkNPTQAEcGFsbw" + ) + + 6. Load a batch of data from BOS, specify partitions, and make some transformations to the columns of the imported files, as follows: + + Table schema: + k1 varchar(20) + k2 int + + Assuming that the data file has only one row of data: + + Adele,1,1 + + The columns in the data file correspond to the columns specified in the load statement: + + k1,tmp_k2,tmp_k3 + + transform as: + + 1) k1: unchanged + 2) k2: sum of tmp_k2 and tmp_k3 + + LOAD LABEL example_db.label6 + ( + DATA INFILE("bos://my_bucket/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (k1, tmp_k2, tmp_k3) + SET ( + k2 = tmp_k2 + tmp_k3 + ) + ) + WITH BROKER my_bos_broker + ( + "bos_endpoint" = "http://bj.bcebos.com", + "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "bos_secret_accesskey"="yyyyyyyyyyyyyyyyyyyy" + ) + + 7. Load data into tables containing HLL columns, which can be columns in tables or columns in data + + If there are three columns in the table (id, v1, v2, v3). The V1 and V2 columns are HLL columns. The imported source file has three columns. Then (column_list) declares that the first column is id, and the second and third columns are temporarily named k1, k2. + + In SET, the HLL column in the table must be specifically declared hll_hash. The V1 column in the table is equal to the hll_hash (k1) column in the original data.The v3 column in the table does not have a corresponding value in the original data, and empty_hll is used to supplement the default value. + + LOAD LABEL example_db.label7 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (id, k1, k2) + SET ( + v1 = hll_hash(k1), + v2 = hll_hash(k2), + v3 = empty_hll() + ) + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + + LOAD LABEL example_db.label8 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (k1, k2, tmp_k3, tmp_k4, v1, v2) + SET ( + v1 = hll_hash(tmp_k3), + v2 = hll_hash(tmp_k4) + ) + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + + 8. Data in load Parquet file specifies FORMAT as parquet. By default, it is judged by file suffix. + + LOAD LABEL example_db.label9 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + FORMAT AS "parquet" + (k1, k2, k3) + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + + 9. Extract partition fields in file paths + + If necessary, partitioned fields in the file path are resolved based on the field type defined in the table, similar to the Partition Discovery function in Spark. + + LOAD LABEL example_db.label10 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/*/*") + INTO TABLE `my_table` + FORMAT AS "csv" + (k1, k2, k3) + COLUMNS FROM PATH AS (city, utc_date) + SET (uniq_id = md5sum(k1, city)) + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + + Directory `hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing` contains following files: + + [hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/utc_date=2019-06-26/0000.csv, hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/utc_date=2019-06-26/0001.csv, ...] + + Extract city and utc_date fields in the file path + + 10. To filter the load data, columns whose K1 value is greater than K2 value can be imported. + + LOAD LABEL example_db.label10 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + where k1 > k2 + ); + + 11. Extract date partition fields in file paths, and date time include %3A (in hdfs path, all ':' will be replaced by '%3A') + + Assume we have files: + + /user/data/data_time=2020-02-17 00%3A00%3A00/test.txt + /user/data/data_time=2020-02-18 00%3A00%3A00/test.txt + + Table schema is: + data_time DATETIME, + k2 INT, + k3 INT + + LOAD LABEL example_db.label12 + ( + DATA INFILE("hdfs://host:port/user/data/*/test.txt") + INTO TABLE `tbl12` + COLUMNS TERMINATED BY "," + (k2,k3) + COLUMNS FROM PATH AS (data_time) + SET (data_time=str_to_date(data_time, '%Y-%m-%d %H%%3A%i%%3A%s')) + ) + WITH BROKER "hdfs" ("username"="user", "password"="pass"); + +## keyword + + BROKER,LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL DELETE.md b/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL DELETE.md new file mode 100644 index 00000000000000..0ab4c638881486 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL DELETE.md @@ -0,0 +1,36 @@ +--- +{ + "title": "CANCEL DELETE", + "language": "en" +} +--- + + + +# CANCEL DELETE +Description + +This statement is used to undo a DELETE operation. (Administrator only!) (To be realized) + +'35;'35; example + +## keyword +CANCEL,DELETE + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL LABEL.md b/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL LABEL.md new file mode 100644 index 00000000000000..5b7ff63b6a788b --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL LABEL.md @@ -0,0 +1,53 @@ +--- +{ + "title": "Cancel Label", + "language": "en" +} +--- + + + +# Cancel Label +Description +NAME: +cancel_label: cancel a transaction with label + +SYNOPSIS +curl -u user:passwd -XPOST http://host:port/api/{db}/{label}/_cancel + +DESCRIPTION +This command is used to cancel a transaction corresponding to a specified Label, which can be successfully cancelled during the Prepare phase. + +RETURN VALUES +When the execution is complete, the relevant content of this import will be returned in Json format. Currently includes the following fields +Status: Successful cancel +Success: 成功cancel事务 +20854; 2018282: 22833; 361333; +Message: Specific Failure Information + +ERRORS + +'35;'35; example + +1. cancel testDb, testLabel20316;- 19994; +curl -u root -XPOST http://host:port/api/testDb/testLabel/_cancel + +## keyword +Cancel, Rabel diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md new file mode 100644 index 00000000000000..3a122d4bcb08c4 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md @@ -0,0 +1,45 @@ +--- +{ + "title": "CANCEL LOAD", + "language": "en" +} +--- + + + +# CANCEL LOAD +Description + +This statement is used to undo the import job for the batch of the specified load label. +This is an asynchronous operation, which returns if the task is submitted successfully. After execution, you can use the SHOW LOAD command to view progress. +Grammar: +CANCEL LOAD +[FROM both names] +WHERE LABEL = "load_label"; + +'35;'35; example + +1. Revoke the import job of example_db_test_load_label on the database example_db +CANCEL LOAD +FROM example_db +WHERE LABEL = "example_db_test_load_label"; + +## keyword +CANCEL,LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/DELETE.md b/docs/en/sql-reference/sql-statements/Data Manipulation/DELETE.md new file mode 100644 index 00000000000000..42ae302b643130 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/DELETE.md @@ -0,0 +1,62 @@ +--- +{ + "title": "DELETE", + "language": "en" +} +--- + + + +# DELETE +Description + +This statement is used to conditionally delete data in the specified table (base index) partition. +This action deletes the rollup index data associated with this base index at the same time. +Grammar: +PART FROM table name [PARTITION partition name] +WHERE +column_name1 op value[ AND column_name2 op value ...]; + +Explain: +1) Optional types of OP include: =,>,<,>=,<=,<=,<=,!= +2) Conditions on key columns can only be specified. +2) When the selected key column does not exist in a rollup, delete cannot be performed. +3) The relationship between conditions can only be "and". +If you want to achieve the "or" relationship, you need to divide the conditions into two DELETE statements. +4) If you partition a table for RANGE, you must specify PARTITION. If it is a single partition table, you can not specify it. + +Be careful: +This statement may reduce query efficiency for a period of time after execution. +The degree of impact depends on the number of deletion conditions specified in the statement. +The more conditions specified, the greater the impact. + +'35;'35; example + +1. Delete rows whose K1 column value is 3 in my_table partition p 1 +DELETE FROM my_table PARTITION p1 +WHERE k1 = 3; + +2. Delete rows whose K1 column value is greater than or equal to 3 and whose K2 column value is "abc" in my_table partition P1 +DELETE FROM my_table PARTITION p1 +WHERE k1 >= 3 AND k2 = "abc"; + +## keyword +DELETE + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/EXPORT.md b/docs/en/sql-reference/sql-statements/Data Manipulation/EXPORT.md new file mode 100644 index 00000000000000..f29a6bfa4cc947 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/EXPORT.md @@ -0,0 +1,83 @@ +--- +{ + "title": "EXPORT", + "language": "en" +} +--- + + + +# EXPORT +Description + +This statement is used to export data from a specified table to a specified location. +This function is implemented by broker process. For different purpose storage systems, different brokers need to be deployed. Deployed brokers can be viewed through SHOW BROKER. +This is an asynchronous operation, which returns if the task is submitted successfully. After execution, you can use the SHOW EXPORT command to view progress. + +Grammar: +EXPORT TABLE table_name +[PARTITION (p1 [,p2]] +TO export_path +[opt_properties] +broker; + +1. table_name +The table names to be exported currently support the export of tables with engine as OLAP and mysql. + +2. partition +You can export only certain specified partitions of the specified table + +3. export_path +The exported path needs to be a directory. At present, it can't be exported to local, so it needs to be exported to broker. + +4. opt_properties +Used to specify some special parameters. +Grammar: +[PROPERTIES ("key"="value", ...)] + +The following parameters can be specified: +Column_separator: Specifies the exported column separator, defaulting to t. +Line_delimiter: Specifies the exported line separator, defaulting to\n. +Exc_mem_limit: Exports the upper limit of memory usage for a single BE node, defaulting to 2GB in bytes. +Timeout: The time-out for importing jobs is 1 day by default, in seconds. +Tablet_num_per_task: The maximum number of tablets that each subtask can allocate. + +Five. debris +Broker used to specify export usage +Grammar: +WITH BROKER broker_name ("key"="value"[,...]) +Here you need to specify the specific broker name and the required broker attributes + +For brokers corresponding to different storage systems, the input parameters are different. Specific parameters can be referred to: `help broker load', broker required properties. + +'35;'35; example + +1. Export all data from the testTbl table to HDFS +EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); + +2. Export partitions P1 and P2 from the testTbl table to HDFS + +EXPORT TABLE testTbl PARTITION (p1,p2) TO "hdfs://hdfs_host:port/a/b/c" WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); +3. Export all data in the testTbl table to hdfs, using "," as column separator + +EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" PROPERTIES ("column_separator"=",") WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); + +## keyword +EXPORT diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/GET LABEL STATE.md b/docs/en/sql-reference/sql-statements/Data Manipulation/GET LABEL STATE.md new file mode 100644 index 00000000000000..21b46c73b15dd9 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/GET LABEL STATE.md @@ -0,0 +1,58 @@ +--- +{ + "title": "GET LABEL STATE", + "language": "en" +} +--- + + + +# GET LABEL STATE +## Description +NAME: +get_label_state: get label's state + +SYNOPSIS +curl -u user:passwd http://host:port /api /{db}/{label}// u state + +DESCRIPTION +This command is used to view the transaction status of a Label + +RETURN VALUES +After execution, the relevant content of this import will be returned in Json format. Currently includes the following fields +Label: The imported label, if not specified, is a uuid. +Status: Whether this command was successfully executed or not, Success indicates successful execution +Message: Specific execution information +State: It only makes sense if Status is Success +UNKNOWN: No corresponding Label was found +PREPARE: The corresponding transaction has been prepared, but not yet committed +COMMITTED: The transaction has been committed and cannot be canceled +VISIBLE: Transaction submission, and data visible, cannot be canceled +ABORTED: The transaction has been ROLLBACK and the import has failed. + +ERRORS + +'35;'35; example + +1. Obtain the state of testDb, testLabel +curl -u root http://host:port /api /testDb /testLabel / u state + +## keyword +GET, LABEL, STATE diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/GROUP BY.md b/docs/en/sql-reference/sql-statements/Data Manipulation/GROUP BY.md new file mode 100644 index 00000000000000..8fcb20e568c3c6 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/GROUP BY.md @@ -0,0 +1,168 @@ +--- +{ + "title": "GROUP BY", + "language": "en" +} +--- + + + +# GROUP BY + +## description + + GROUP BY `GROUPING SETS` ｜ `CUBE` ｜ `ROLLUP` is an extension to GROUP BY clause. This syntax lets you define multiple groupings in the same query. GROUPING SETS produce a single result set that is equivalent to a UNION ALL of differently grouped rows + For example GROUPING SETS clause: + + ``` + SELECT a, b, SUM( c ) FROM tab1 GROUP BY GROUPING SETS ( (a, b), (a), (b), ( ) ); + ``` + + This statement is equivalent to: + + ``` + SELECT a, b, SUM( c ) FROM tab1 GROUP BY a, b + UNION + SELECT a, null, SUM( c ) FROM tab1 GROUP BY a + UNION + SELECT null, b, SUM( c ) FROM tab1 GROUP BY b + UNION + SELECT null, null, SUM( c ) FROM tab1 + ``` + + `GROUPING(expr)` indicates whether a specified column expression in a GROUP BY list is aggregated or not. GROUPING returns 1 for aggregated or 0 for not aggregated in the result set. + + `GROUPING_ID(expr [ , expr [ , ... ] ])` describes which of a list of expressions are grouped in a row produced by a GROUP BY query. The GROUPING_ID function simply returns the decimal equivalent of the binary value formed as a result of the concatenation of the values returned by the GROUPING functions. + +### Syntax + + ``` + SELECT ... + FROM ... + [ ... ] + GROUP BY [ + , ... | + GROUPING SETS [, ...] ( groupSet [ , groupSet [ , ... ] ] ) | + ROLLUP(expr [ , expr [ , ... ] ]) | + expr [ , expr [ , ... ] ] WITH ROLLUP | + CUBE(expr [ , expr [ , ... ] ]) | + expr [ , expr [ , ... ] ] WITH CUBE + ] + [ ... ] + ``` + +### Parameters + + `groupSet` is a set of expression or column or it's alias appearing in the query block’s SELECT list. `groupSet ::= { ( expr [ , expr [ , ... ] ] )}` + + `expr` is expression or column or it's alias appearing in the query block’s SELECT list. + +### Note + + doris supports PostgreSQL like syntax, for example: + + ``` + SELECT a, b, SUM( c ) FROM tab1 GROUP BY GROUPING SETS ( (a, b), (a), (b), ( ) ); + SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY ROLLUP(a,b,c) + SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY CUBE(a,b,c) + ``` + + `ROLLUP(a,b,c)` is equivalent to `GROUPING SETS` as follows: + + ``` + GROUPING SETS ( + (a,b,c), + ( a, b ), + ( a), + ( ) + ) + ``` + + `CUBE ( a, b, c )` is equivalent to `GROUPING SETS` as follows: + + ``` + GROUPING SETS ( + ( a, b, c ), + ( a, b ), + ( a, c ), + ( a ), + ( b, c ), + ( b ), + ( c ), + ( ) + ) + ``` + +## example + + This is a simple example + + ``` + > SELECT * FROM t; + +------+------+------+ + | k1 | k2 | k3 | + +------+------+------+ + | a | A | 1 | + | a | A | 2 | + | a | B | 1 | + | a | B | 3 | + | b | A | 1 | + | b | A | 4 | + | b | B | 1 | + | b | B | 5 | + +------+------+------+ + 8 rows in set (0.01 sec) + + > SELECT k1, k2, SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); + +------+------+-----------+ + | k1 | k2 | sum(`k3`) | + +------+------+-----------+ + | b | B | 6 | + | a | B | 4 | + | a | A | 3 | + | b | A | 5 | + | NULL | B | 10 | + | NULL | A | 8 | + | a | NULL | 7 | + | b | NULL | 11 | + | NULL | NULL | 18 | + +------+------+-----------+ + 9 rows in set (0.06 sec) + + > SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ()); + +------+------+---------------+----------------+ + | k1 | k2 | grouping_id(k1,k2) | sum(`k3`) | + +------+------+---------------+----------------+ + | a | A | 0 | 3 | + | a | B | 0 | 4 | + | a | NULL | 1 | 7 | + | b | A | 0 | 5 | + | b | B | 0 | 6 | + | b | NULL | 1 | 11 | + | NULL | A | 2 | 8 | + | NULL | B | 2 | 10 | + | NULL | NULL | 3 | 18 | + +------+------+---------------+----------------+ + 9 rows in set (0.02 sec) + ``` + +## keyword + + GROUP, GROUPING, GROUPING_ID, GROUPING_SETS, GROUPING SETS, CUBE, ROLLUP diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/LOAD.md new file mode 100644 index 00000000000000..4ea2880c63a978 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/LOAD.md @@ -0,0 +1,286 @@ +--- +{ + "title": "LOAD", + "language": "en" +} +--- + + + +# LOAD +## Description + +Palo currently supports the following four import methods: + +1. Hadoop Load: Importing ETL based on MR. +2. Broker Load: Use broker to import data. +3. Mini Load: Upload files through HTTP protocol for batch data import. +4. Stream Load: Stream data import through HTTP protocol. + +This help mainly describes the first import method, namely Hadoop Load related help information. The rest of the import methods can use the following commands to view help: + +This import method may not be supported in a subsequent version. It is recommended that other import methods be used for data import. !!! + +1. help broker load; +2. help mini load; +3. help stream load; + +Hadoop Load is only applicable to Baidu's internal environment. Public, private and open source environments cannot use this import approach. +The import method must set up a Hadoop computing queue for ETL, which can be viewed through the help set property command. + +Stream load only supports Baidu internal users for the time being. Open source communities and public cloud users will be supported in subsequent version updates. + +Grammar: + +LOAD LABEL load_label +( +Date of date of date of entry +) +[opt_properties]; + +1. load label + +The label of the current imported batch. Unique in a database. +Grammar: +[database_name.]your_label + +2. data_desc + +Used to describe a batch of imported data. +Grammar: +DATA INFILE +( +"file_path1"[, file_path2, ...] +) +[NEGATIVE] +INTO TABLE `table_name` +[PARTITION (p1, P2)] +[COLUMNS TERMINATED BY "column_separator"] +[FORMAT AS "file_type"] +[(column_list)] +[set (k1 = fun (k2)]] + +Explain: +file_path: + +File paths can be specified to a file, or * wildcards can be used to specify all files in a directory. Wildcards must match to files, not directories. + +PARTICIPATION: + +If this parameter is specified, only the specified partition will be imported, and data outside the imported partition will be filtered out. +If not specified, all partitions of the table are imported by default. + +NEGATIVE： +If this parameter is specified, it is equivalent to importing a batch of "negative" data. Used to offset the same batch of data imported before. +This parameter applies only to the case where there are value columns and the aggregation type of value columns is SUM only. + +Column U separator: + +Used to specify the column separator in the import file. Default tot +If the character is invisible, it needs to be prefixed with \x, using hexadecimal to represent the separator. +For example, the separator X01 of the hive file is specified as "\ x01" + +File type: + +Used to specify the type of imported file, such as parquet, orc, csv. The default value is determined by the file suffix name. + +column_list： + +Used to specify the correspondence between columns in the import file and columns in the table. +When you need to skip a column in the import file, specify it as a column name that does not exist in the table. +Grammar: +(col_name1, col_name2, ...) + +SET: + +If this parameter is specified, a column of the source file can be transformed according to a function, and then the transformed result can be imported into the table. +The functions currently supported are: + +Strftime (fmt, column) date conversion function +Fmt: Date format, such as% Y% m% d% H% M% S (year, month, day, hour, second) +Column: Column in column_list, which is the column in the input file. Storage content should be a digital timestamp. +If there is no column_list, the columns of the input file are entered by default in the column order of the Palo table. + +time_format(output_fmt, input_fmt, column) 日期格式转化 +Output_fmt: Converted date format, such as% Y% m% d% H% M% S (year, month, day, hour, second) +Input_fmt: The date format of the column before transformation, such as% Y% m% d% H% M% S (days, hours, seconds, months, years) +Column: Column in column_list, which is the column in the input file. Storage content should be a date string in input_fmt format. +If there is no column_list, the columns of the input file are entered by default in the column order of the Palo table. + +alignment_timestamp(precision, column) 将时间戳对齐到指定精度 +Precision: year 124month;124day;124hour; +Column: Column in column_list, which is the column in the input file. Storage content should be a digital timestamp. +If there is no column_list, the columns of the input file are entered by default in the column order of the Palo table. +Note: When the alignment accuracy is year and month, only the time stamps in the range of 20050101-20191231 are supported. + +Default_value (value) sets the default value for a column import +Use default values of columns when creating tables without specifying + +Md5sum (column1, column2,...) evaluates the value of the specified imported column to md5sum, returning a 32-bit hexadecimal string + +Replace_value (old_value [, new_value]) replaces old_value specified in the import file with new_value +New_value, if not specified, uses the default value of the column when building the table + +Hll_hash (column) is used to transform a column in a table or data into a data structure of a HLL column + +3. opt_properties + +Used to specify some special parameters. +Grammar: +[PROPERTIES ("key"="value", ...)] + +The following parameters can be specified: +Cluster: Import the Hadoop computed queue used. +Timeout: Specifies the timeout time of the import operation. The default timeout is 3 days. Unit seconds. +Max_filter_ratio: The ratio of data that is most tolerant of being filterable (for reasons such as data irregularities). Default zero tolerance. +Load_delete_flag: Specifies whether the import deletes data by importing the key column, which applies only to UNIQUE KEY. +Value column is not specified when importing. The default is false. + +5. Import data format sample + +Integer classes (TINYINT/SMALLINT/INT/BIGINT/LARGEINT): 1,1000,1234 +Floating Point Class (FLOAT/DOUBLE/DECIMAL): 1.1, 0.23, 356 +Date class (DATE/DATETIME): 2017-10-03, 2017-06-13 12:34:03. +(Note: If it's in other date formats, you can use strftime or time_format functions to convert in the import command) +字符串类（CHAR/VARCHAR）："I am a student", "a" +NULL value: N + +'35;'35; example + +1. Import a batch of data, specify timeout time and filtering ratio. Specify the import queue as my_cluster. + +LOAD LABEL example db.label1 +( +DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") +INTO TABLE `my_table` +) +PROPERTIES +( +"cluster" ="my" cluster, +Timeout ="3600", +"max_filter_ratio" = "0.1" +); + +Where hdfs_host is the host of the namenode and hdfs_port is the fs.defaultFS port (default 9000) + +2. Import a batch of data, including multiple files. Import different tables, specify separators, and specify column correspondences + +LOAD LABEL example db.label2 +( +DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file1") +INTO TABLE `my_table_1` +COLUMNS TERMINATED BY "," +(k1, k3, k2, v1, v2), +DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file2") +INTO TABLE `my_table_2` +COLUMNS TERMINATED BY "\t" +(k1, k2, k3, v2, v1) +); + +3. Import a batch of data, specify hive's default delimiter x01, and use wildcard * to specify all files in the directory + +LOAD LABEL example db.label3 +( +DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/*") +NEGATIVE +INTO TABLE `my_table` +COLUMNS TERMINATED BY "\\x01" +); + +4. Import a batch of "negative" data + +LOAD LABEL example db.label4 +( +DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/old_file) +NEGATIVE +INTO TABLE `my_table` +COLUMNS TERMINATED BY "\t" +); + +5. Import a batch of data and specify partitions + +LOAD LABEL example db.label5 +( +DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") +INTO TABLE `my_table` +PARTITION (p1, P2) +COLUMNS TERMINATED BY "," +(k1, k3, k2, v1, v2) +); + +6. Import a batch of data, specify partitions, and make some transformations to the columns of the imported files, as follows: +The table structure is as follows: +K1 date +date +k3 bigint +k4 varchar (20) +k5 varchar (64) +k6 int + +Assume that the data file has only one row of data, five columns, and comma-separated: + +1537002087,2018-08-09 11:12:13,1537002087,-,1 + +The columns in the data file correspond to the columns specified in the import statement: +tmp -u k1, tmp -u k2, tmp u k3, k6, v1 + +The conversion is as follows: + +1) k1: Transform tmp_k1 timestamp column into datetime type data +2) k2: Converting tmp_k2 datetime-type data into date data +3) k3: Transform tmp_k3 timestamp column into day-level timestamp +4) k4: Specify import default value of 1 +5) k5: Calculate MD5 values from tmp_k1, tmp_k2, tmp_k3 columns +6) k6: Replace the - value in the imported file with 10 + +LOAD LABEL example db.label6 +( +DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") +INTO TABLE `my_table` +PARTITION (p1, P2) +COLUMNS TERMINATED BY "," +(tmp /u k1, tmp /u k2, tmp /u k3, k6, v1) +SET ( +K1 = strftime (%Y -%m -%d%H:%M:%S ", TMP u K1), +K2 = Time = UFormat ("% Y-% M-% D% H:% M:% S", "% Y-% M-% D", "TMP = UK2), +k3 = alignment_timestamp("day", tmp_k3), +k4 = default_value("1"), +K5 = MD5Sum (TMP = UK1, TMP = UK2, TMP = UK3) +k6 = replace value ("-", "10") +) +); + +7. Import data into tables containing HLL columns, which can be columns in tables or columns in data + +LOAD LABEL example db.label7 +( +DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") +INTO TABLE `my_table` +PARTITION (p1, P2) +COLUMNS TERMINATED BY "," +SET ( +v1 = hll, u hash (k1), +v2 = hll, u hash (k2) +) +); + +## keyword +LOAD + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md new file mode 100644 index 00000000000000..c1c71ca7069c57 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md @@ -0,0 +1,132 @@ +--- +{ + "title": "MINI LOAD", + "language": "en" +} +--- + + + +# MINI LOAD +## Description + +MINI LOAD and STEAM LOAD are implemented in exactly the same way. MINI LOAD is a subset of STREAM LOAD in import support. +Subsequent imports of new features will only be supported in STEAM LOAD, MINI LOAD will no longer add features. It is suggested that STREAM LOAD be used instead. Please use HELP STREAM LOAD. + +MINI LOAD is imported through HTTP protocol. Users can import without relying on Hadoop or Mysql client. +The user describes the import through HTTP protocol, and the data is streamed into Doris in the process of receiving http requests. After the ** import job is completed, the ** returns to the user the imported results. + +* Note: In order to be compatible with the old version of mini load usage habits, users can still view the import results through the'SHOW LOAD'command. + +Grammar: +Import: + +curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table}/_load?label=xxx + +View import information + +curl -u user:passwd http://host:port/api/{db}/_load_info?label=xxx + +HTTP Protocol Specification + +Privilege Authentication Currently Doris uses the Basic mode of HTTP for privilege authentication. So you need to specify a username and password when importing +This way is to pass the password in plaintext, and does not support encrypted transmission for the time being. + +Expect Doris needs to send an HTTP request with the'Expect'header information,'100-continue'. +Why? Because we need to redirect the request, we have to transfer the data content before. +This can avoid causing multiple data transmission, thereby improving efficiency. + +Content-Length Doris needs to send a request with the header'Content-Length'. If the content ratio is sent +'Content-Length'is less, so Doris believes that if there is a transmission problem, the submission task fails. +NOTE: If you send more data than'Content-Length', Doris reads only'Content-Length'. +Length content and import + + +Description of parameters: + +User: User is user_name if the user is in default_cluster. Otherwise, it is user_name@cluster_name. + +Label: The label used to specify this batch of imports for later job queries, etc. +This parameter must be passed in. + +Columns: Used to describe the corresponding column name in the import file. +If it is not passed in, the column order in the file is considered to be the same as the order in which the table is built. +The specified method is comma-separated, such as columns = k1, k2, k3, K4 + +Column_separator: Used to specify the separator between columns, default is' t' +NOTE: Url encoding is required, for example +If you need to specify' t'as a separator, you should pass in'column_separator=% 09' +If you need to specify'x01'as a delimiter, you should pass in'column_separator=% 01' +If you need to specify','as a separator, you should pass in'column_separator=% 2c' + + +Max_filter_ratio: Used to specify the maximum percentage allowed to filter irregular data, default is 0, not allowed to filter +Custom specification should be as follows:'max_filter_ratio = 0.2', meaning that 20% error rate is allowed. + +Timeout: Specifies the timeout time of the load job in seconds. When the load execution time exceeds this threshold, it is automatically cancelled. The default timeout time is 86400 seconds. +It is recommended to specify a timeout time of less than 86400 seconds. + +Hll: Used to specify the corresponding relationship between the HLL columns in the data and the tables, the columns in the tables and the columns specified in the data. +(If columns are not specified, the columns of the data column surface can also be other non-HLL columns in the table.) By "partition" +Specify multiple HLL columns using ":" splitting, for example:'hll1, cuid: hll2, device' + +NOTE: +1. This method of importing is currently completed on a single machine, so it is not suitable to import a large amount of data. +It is recommended that the amount of data imported should not exceed 1 GB. + +2. Currently, it is not possible to submit multiple files in the form of `curl-T', `{file1, file2}', because curl splits them into multiple files. +Request sent, multiple requests can not share a label number, so it can not be used + +3. Miniload is imported in exactly the same way as streaming. It returns the results synchronously to users after the import of streaming is completed. +Although the information of mini load can be found in subsequent queries, it can not be operated on. The queries are only compatible with the old ways of use. + +4. When importing from the curl command line, you need to add escape before & or the parameter information will be lost. + +'35;'35; example + +1. Import the data from the local file'testData'into the table of'testTbl' in the database'testDb'(the user is in defalut_cluster) +curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123 + +2. Import the data from the local file'testData'into the table of'testTbl' in the database'testDb'(the user is in test_cluster). The timeout time is 3600 seconds. +curl --location-trusted -u root@test_cluster:root -T testData http://fe.host:port/api/testDb/testTbl/_load?label=123&timeout=3600 + +3. Import data from the local file'testData'into the'testTbl' table in the database'testDb', allowing a 20% error rate (the user is in defalut_cluster) +curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 + +4. Import the data from the local file'testData'into the table'testTbl' in the database'testDb', allowing a 20% error rate, and specify the column name of the file (the user is in defalut_cluster) +curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2\&columns=k1,k2,k3 + +5. Import in streaming mode (user is in defalut_cluster) +seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_load?label=123 + +6. Import tables containing HLL columns, which can be columns in tables or columns in data to generate HLL columns (users are in defalut_cluster) + + curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2\&hll=hll_column1,k1:hll_column2,k2 + \&columns=k1,k2,k3 + + curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 + \&hll=hll_column1,tmp_k4:hll_column2,tmp_k5\&columns=k1,k2,k3,tmp_k4,tmp_k5 + +7. View imports after submission + +curl -u root http://host:port/api/testDb/_load_info?label=123 + +## keyword +MINI, LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md new file mode 100644 index 00000000000000..1676cfb5c1fb4b --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md @@ -0,0 +1,107 @@ +--- +{ + "title": "MULTI LOAD", + "language": "en" +} +--- + + + +# MULTI LOAD +## Description + +Syntax: +curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_start?label=xxx +curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table1}/_load?label=xxx\&sub_label=yyy +curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table2}/_load?label=xxx\&sub_label=zzz +curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_commit?label=xxx +curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_desc?label=xxx + +'MULTI LOAD'can support users to import multiple tables at the same time on the basis of'MINI LOAD'. The specific commands are shown above. +'/api/{db}/_multi_start'starts a multi-table import task +'/api/{db}/{table}/_load'adds a table to be imported to an import task. The main difference from'MINI LOAD' is that the'sub_label'parameter needs to be passed in. +'/api/{db}/_multi_commit'submits the entire multi-table import task and the background begins processing +'/api/{db}/_multi_abort'Abandons a multi-table import task +'/api/{db}/_multi_desc'shows the number of jobs submitted by a multi-table import task + +HTTP Protocol Specification +Privilege Authentication Currently Doris uses the Basic mode of HTTP for privilege authentication. So you need to specify a username and password when importing +This way is to pass passwords in plaintext, since we are all in the Intranet environment at present... + +Expect Doris needs to send an HTTP request, and needs the'Expect'header information with the content of'100-continue'. +Why? Because we need to redirect the request, we have to transfer the data content before. +This can avoid causing multiple data transmission, thereby improving efficiency. + +Content-Length Doris needs to send a request with the header'Content-Length'. If the content ratio is sent +If'Content-Length'is less, Palo believes that if there is a transmission problem, the submission of the task fails. +NOTE: If you send more data than'Content-Length', Doris reads only'Content-Length'. +Length content and import + +Description of parameters: +User: User is user_name if the user is in default_cluster. Otherwise, it is user_name@cluster_name. + +Label: Used to specify the label number imported in this batch for later job status queries, etc. +This parameter must be passed in. + +Sub_label: Used to specify a subversion number within a multi-table import task. For multi-table imported loads, this parameter must be passed in. + +Columns: Used to describe the corresponding column name in the import file. +If it is not passed in, the column order in the file is considered to be the same as the order in which the table is built. +The specified method is comma-separated, such as columns = k1, k2, k3, K4 + +Column_separator: Used to specify the separator between columns, default is' t' +NOTE: Url encoding is required, such as specifying't'as a delimiter. +Then you should pass in'column_separator=% 09' + +Max_filter_ratio: Used to specify the maximum percentage allowed to filter irregular data, default is 0, not allowed to filter +Custom specification should be as follows:'max_filter_ratio = 0.2', meaning that 20% error rate is allowed. +Pass in effect at'_multi_start' + +NOTE: +1. This method of importing is currently completed on a single machine, so it is not suitable to import a large amount of data. +It is recommended that the amount of data imported should not exceed 1GB + +2. Currently, it is not possible to submit multiple files in the form of `curl-T', `{file1, file2}', because curl splits them into multiple files. +Request sent, multiple requests can not share a label number, so it can not be used + +3. Supports streaming-like ways to use curl to import data into Doris, but Doris will have to wait until the streaming is over +Real import behavior will occur, and the amount of data in this way can not be too large. + +'35;'35; example + +1. Import the data from the local file'testData1'into the table of'testTbl1' in the database'testDb', and +Import the data from'testData2'into the table'testTbl2' in'testDb'(the user is in defalut_cluster) +curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 +curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 +curl --location-trusted -u root -T testData2 http://host:port/api/testDb/testTbl2/_load?label=123\&sub_label=2 +curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_commit?label=123 + +2. Multi-table Import Midway Abandon (User in defalut_cluster) +curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 +curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 +curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_abort?label=123 + +3. Multi-table import to see how much content has been submitted (user is in defalut_cluster) +curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 +curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 +curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_desc?label=123 + +## keyword +MULTI, MINI, LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md new file mode 100644 index 00000000000000..7b98035a136ee4 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md @@ -0,0 +1,35 @@ +--- +{ + "title": "PAUSE ROUTINE LOAD", + "language": "en" +} +--- + + + +# PAUSE ROUTINE LOAD +## example + +1. Suspend the routine import operation named test 1. + +PAUSE ROUTINE LOAD FOR test1; + +## keyword +PAUSE,ROUTINE,LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/RESTORE TABLET.md b/docs/en/sql-reference/sql-statements/Data Manipulation/RESTORE TABLET.md new file mode 100644 index 00000000000000..75ecee7da9a92d --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/RESTORE TABLET.md @@ -0,0 +1,41 @@ +--- +{ + "title": "RESTORE TABLET", + "language": "en" +} +--- + + + +# RESTORE TABLET +Description + +This function is used to recover the tablet data that was deleted by mistake in the trash directory. + +Note: For the time being, this function only provides an HTTP interface in be service. If it is to be used, +A restore tablet API request needs to be sent to the HTTP port of the be machine for data recovery. The API format is as follows: +Method: Postal +URI: http://be_host:be_http_port/api/restore_tablet?tablet_id=xxx&schema_hash=xxx + +'35;'35; example + +Curl -X POST "http://hostname:8088 /api /restore" tablet? Tablet id =123456 &schema hash =1111111 " +##keyword +RESTORE,TABLET,RESTORE,TABLET diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md new file mode 100644 index 00000000000000..78d475545398b4 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md @@ -0,0 +1,35 @@ +--- +{ + "title": "RESUME ROUTINE LOAD", + "language": "en" +} +--- + + + +# RESUME ROUTINE LOAD +## example + +1. Restore the routine import job named test 1. + +RESUME ROUTINE LOAD FOR test1; + +## keyword +RESUME,ROUTINE,LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md new file mode 100644 index 00000000000000..35d61dfcbacee4 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md @@ -0,0 +1,371 @@ +--- +{ + "title": "ROUTINE LOAD", + "language": "en" +} +--- + + + +# ROUTINE LOAD +## description + +Routine Load function allows users to submit a resident load task, and continuously load data into Doris by continuously reading data from the specified data source. Currently, only text data format (CSV) data is loaded from Kakfa by means of no authentication or SSL authentication. + +Syntax: + +``` +CREATE ROUTINE LOAD [db.]job_name ON tbl_name +[load_properties] +[job_properties] +FROM data_source +[data_source_properties] +``` + +1. [db.]job_name + + The name of the load job, in the same database, only one job can run with the same name. + +2. tbl_name + + Specifies the name of the table that needs to be loaded. + +3. load_properties + + Used to describe the load data. grammar: + + ``` + [column_separator], + [columns_mapping], + [where_predicates], + [partitions] + ``` + + 1. column_separator: + + Specify column separators, such as: + + `COLUMNS TERMINATED BY ","` + + The default is: `\t` +         + 2. columns_mapping: + + Specifies the mapping of columns in the source data and defines how the derived columns are generated. + + 1. Map column: + + Specify in order, which columns in the source data correspond to which columns in the destination table. For columns that you want to skip, you can specify a column name that does not exist. + + Suppose the destination table has three columns k1, k2, v1. The source data has 4 columns, of which columns 1, 2, and 4 correspond to k2, k1, and v1, respectively. Write as follows: + + `COLUMNS (k2, k1, xxx, v1)` + + Where xxx is a column that does not exist and is used to skip the third column in the source data. + + 2. Derived columns: + + A column represented in the form of col_name = expr, which we call a derived column. That is, the value of the corresponding column in the destination table is calculated by expr. + + Derived columns are usually arranged after the mapped column. Although this is not mandatory, Doris always parses the mapped columns first and then parses the derived columns. + + Following an example, assume that the destination table also has column 4, v2, which is generated by the sum of k1 and k2. You can write as follows: + + `COLUMNS (k2, k1, xxx, v1, v2 = k1 + k2);` + + 3. where_predicates + + Used to specify filter criteria to filter out unwanted columns. Filter columns can be either mapped columns or derived columns. + + For example, if we only want to load a column with k1 greater than 100 and k2 equal to 1000, we would write as follows: + + `WHERE k1 > 100 and k2 = 1000` +         + 4. partitions + + Specifies which partitions of the load destination table. If not specified, it will be automatically loaded into the corresponding partition. + + Example: + + `PARTITION(p1, p2, p3)` + +4. job_properties + + A generic parameter that specifies a routine load job. + + syntax: + + ``` + PROPERTIES ( + "key1" = "val1", + "key2" = "val2" + ) + ``` + + Currently we support the following parameters: + + 1. `desired_concurrent_number` + + The degree of concurrency desired. A routine load job is split into multiple subtasks. This parameter specifies how many tasks can be executed simultaneously in a job. Must be greater than 0. The default is 3. + + This concurrency is not the actual concurrency. The actual concurrency will be considered by the number of nodes in the cluster, the load, and the data source. + + example: + + `"desired_concurrent_number" = "3"` + + 2. `max_batch_interval/max_batch_rows/max_batch_size` + + These three parameters represent: + + 1) The maximum execution time of each subtask, in seconds. The range is 5 to 60. The default is 10. + + 2) The maximum number of rows read per subtask. Must be greater than or equal to 200,000. The default is 200000. + + 3) The maximum number of bytes read per subtask. The unit is byte and the range is 100MB to 1GB. The default is 100MB. + + These three parameters are used to control the execution time and throughput of a subtask. When either one reaches the threshold, the task ends. + + example: + + ``` + "max_batch_interval" = "20", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200" + ``` + + 3. `max_error_number` + + The maximum number of error lines allowed in the sampling window. Must be greater than or equal to 0. The default is 0, which means that no error lines are allowed. + + The sampling window is max_batch_rows * 10. That is, if the number of error lines is greater than max_error_number in the sampling window, the routine job will be suspended, and manual intervention is required to check the data quality problem. + + Lines that are filtered by the where condition are not counted as error lines. + + 4. `strict_mode` + + Whether to enable strict mode, the default is on. If turned on, the column type transformation of non-null raw data is filtered if the result is NULL. Specified as "strict_mode" = "true" +             + 5. timezone + + Specifies the time zone in which the job will be loaded. The default by using session variable's timezone. This parameter affects all function results related to the time zone involved in the load. + +5. data_source + + The type of data source. Current support: + + KAFKA + +6. `data_source_properties` + + Specify information about the data source. + + syntax: +     + ``` + ( + "key1" = "val1", + "key2" = "val2" + ) + ``` + + 1. KAFKA data source + + `Kafka_broker_list` + + Kafka's broker connection information. The format is ip:host. Multiple brokare separated by commas. + + Example: + + `"kafka_broker_list" = "broker1:9092,broker2:9092"` + + 2. `kafka_topic` + + Specify the topic of Kafka to subscribe to. + + Example: + + `"kafka_topic" = "my_topic"` + + 3. `kafka_partitions/kafka_offsets` + + Specify the kafka partition to be subscribed to, and the corresponding star offset for each partition. + + Offset can specify a specific offset from 0 or greater, or: + + 1) OFFSET_BEGINNING: Subscribe from the location where the data is avaie. + + 2) OFFSET_END: ​​Subscribe from the end. + + If not specified, all partitions under topic are subscribed by default fromSET_END. + + Example: + + ``` + "kafka_partitions" = "0,1,2,3", + "kafka_offsets" = "101,0,OFFSET_BEGINNING,OFFSET_END" + ``` + + 4. property + + Specify custom kafka parameters. + + The function is equivalent to the "--property" parameter in the kafka shel + + When the value of the parameter is a file, you need to add the keyword: "FILbefore the value. + + For information on how to create a file, see "HELP CREATE FILE;" + + For more supported custom parameters, see the configuration items on the nt side in the official CONFIGURATION documentation for librdkafka. + + Example: + + ``` + "property.client.id" = "12345", + "property.ssl.ca.location" = "FILE:ca.pem" + ``` + + 1. When connecting to Kafka using SSL, you need to specify the follg parameters: + + ``` + "property.security.protocol" = "ssl", + "property.ssl.ca.location" = "FILE:ca.pem", + "property.ssl.certificate.location" = "FILE:client.pem", + "property.ssl.key.location" = "FILE:client.key", + "property.ssl.key.password" = "abcdefg" + ``` + + among them: + + "property.security.protocol" and "property.ssl.ca.location" are requ to indicate the connection method is SSL and the location of the CA certate. + + If the client authentication is enabled on the Kafka server, you alsod to set: + + ``` + "property.ssl.certificate.location" + "property.ssl.key.location" + "property.ssl.key.password" + ``` + + Used to specify the public key of the client, the private key, and the word of the private key. + + 2. Specify the default starting offset for kafka partition + + If kafka_partitions/kafka_offsets is not specified, all partitions are umed by default, and you can specify kafka_default_offsets to specify the star offset. The default is OFFSET_END, which starts at the end of the substion. + + Values: + + 1) OFFSET_BEGINNING: Subscribe from the location where the data is avaie. + + 2) OFFSET_END: Subscribe from the end. + + Example: + + `"property.kafka_default_offsets" = "OFFSET_BEGINNING"` + +7. load data format sample + + Integer class (TINYINT/SMALLINT/INT/BIGINT/LARGEINT): 1, 1000, 1234 + + Floating point class (FLOAT/DOUBLE/DECIMAL): 1.1, 0.23, .356 +  +  Date class (DATE/DATETIME): 2017-10-03, 2017-06-13 12:34:03. + + String class (CHAR/VARCHAR) (without quotes): I am a student, a + + NULL value: \N + +## example + +1. Create a Kafka routine load task named test1 for the example_tbl of example_db. Specify group.id and client.id, and automatically consume all partitions by default, with subscriptions starting at the end (OFFSET_END) + ``` + CREATE ROUTINE LOAD example_db.test1 ON example_tbl + COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100) + PROPERTIES + ( + "desired_concurrent_number"="3", + "max_batch_interval" = "20", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200", + "strict_mode" = "false" + ) + FROM KAFKA + ( + "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", + "kafka_topic" = "my_topic", + "property.group.id" = "xxx", + "property.client.id" = "xxx" + ); + ``` + +2. Create a Kafka routine load task named test1 for the example_tbl of example_db. The load task is in strict mode. + + ``` + CREATE ROUTINE LOAD example_db.test1 ON example_tbl + COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), + WHERE k1 > 100 and k2 like "%doris%" + PROPERTIES + ( +     "desired_concurrent_number"="3", +     "max_batch_interval" = "20", +     "max_batch_rows" = "300000", +     "max_batch_size" = "209715200", +     "strict_mode" = "false" + ) + FROM KAFKA + ( +     "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", +     "kafka_topic" = "my_topic", +     "kafka_partitions" = "0,1,2,3", +     "kafka_offsets" = "101,0,0,200" + ); + ``` + +3. load data from Kafka clusters via SSL authentication. Also set the client.id parameter. The load task is in non-strict mode and the time zone is Africa/Abidjan + + ``` + CREATE ROUTINE LOAD example_db.test1 ON example_tbl + COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), + WHERE k1 > 100 and k2 like "%doris%" + PROPERTIES + ( +     "desired_concurrent_number"="3", +     "max_batch_interval" = "20", +     "max_batch_rows" = "300000", +     "max_batch_size" = "209715200", +     "strict_mode" = "false", +     "timezone" = "Africa/Abidjan" + ) + FROM KAFKA + ( +     "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", +     "kafka_topic" = "my_topic", +     "property.security.protocol" = "ssl", +     "property.ssl.ca.location" = "FILE:ca.pem", +     "property.ssl.certificate.location" = "FILE:client.pem", +     "property.ssl.key.location" = "FILE:client.key", +     "property.ssl.key.password" = "abcdefg", +     "property.client.id" = "my_client_id" + ); + ``` + +## keyword + + CREATE, ROUTINE, LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md new file mode 100644 index 00000000000000..8ae554eda01357 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md @@ -0,0 +1,55 @@ +--- +{ + "title": "SHOW ALTER", + "language": "en" +} +--- + + + +# SHOW ALTER +## Description +This statement is used to show the execution of various modification tasks currently under way. +Grammar: +SHOW ALTER [CLUSTER | TABLE [COLUMN | ROLLUP] [FROM db_name]]; + +Explain: +TABLE COLUMN：Shows the task of alter table column. + Support grammar [WHERE TableName|CreateTime|FinishTime|State] [ORDER BY] [LIMIT] +TABLE ROLLUP: Shows the task of creating or deleting ROLLUP index +If db_name is not specified, use the current default DB +CLUSTER: Show the cluster operation related tasks (only administrators use! To be realized... + +## example +1. Show the task execution of all modified columns of default DB +SHOW ALTER TABLE COLUMN; + +2. Show the last task execution of modified columns of some table +SHOW ALTER TABLE COLUMN WHERE TableName = "table1" ORDER BY CreateTime LIMIT 1; + +3. Show the execution of tasks to create or delete ROLLUP index for specified DB +SHOW ALTER TABLE ROLLUP FROM example_db; + +4. Show cluster operations related tasks (only administrators use! To be realized... +SHOW ALTER CLUSTER; + +## keyword +SHOW,ALTER + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md new file mode 100644 index 00000000000000..73b00422297daa --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md @@ -0,0 +1,62 @@ +--- +{ + "title": "SHOW BACKUP", + "language": "en" +} +--- + + + +# SHOW BACKUP +## Description +This statement is used to view BACKUP tasks +Grammar: +SHOW BACKUP [FROM db_name] + +Explain: +1. Only the last BACKUP task is saved in Palo. +2. Each column has the following meanings: +JobId: Unique job ID +SnapshotName: The name of the backup +DbName: Subordinate database +State: Current phase +PENDING: The initial state after submitting a job +SNAPSHOTING: In the execution snapshot +UPLOAD_SNAPSHOT: Snapshot completed, ready for upload +UPLOADING: Snapshot uploading +SAVE_META: Save job meta-information as a local file +UPLOAD_INFO: Upload job meta-information +FINISHED: Operation Successful +CANCELLED: Job Failure +Backup Objs: Backup tables and partitions +CreateTime: Task submission time +Snapshot Finished Time: Snapshot completion time +Upload Finished Time: Snapshot Upload Completion Time +FinishedTime: Job End Time +Unfinished Tasks: The unfinished sub-task ID is displayed in the SNAP HOTING and UPLOADING phases +Status: Display failure information if the job fails +Timeout: Job timeout, per second + +## example +1. See the last BACKUP task under example_db. +SHOW BACKUP FROM example_db; + +## keyword +SHOW, BACKUP diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md new file mode 100644 index 00000000000000..c29bac4bea85d3 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md @@ -0,0 +1,46 @@ +--- +{ + "title": "SHOW DATA", + "language": "en" +} +--- + + + +# SHOW DATA +## Description +This statement is used to show the amount of data and the number of replica +Grammar: +SHOW DATA [FROM db_name[.table_name]]; + +Explain: +1. If you do not specify the FROM clause, use the amount of data and the number of replica that shows the current DB subdivided into tables +2. If the FROM clause is specified, the amount of data and the number of replica subdivided into indices under the table is shown. +3. If you want to see the size of individual Partitions, see help show partitions + +## example +1. Display the data volume, replica size, aggregate data volume and aggregate replica count of each table of default DB +SHOW DATA; + +2. Display the subdivision data volume and replica count of the specified table below the specified DB +SHOW DATA FROM example_db.table_name; + +## keyword +SHOW,DATA diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md new file mode 100644 index 00000000000000..b9ebd362731c3b --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md @@ -0,0 +1,35 @@ +--- +{ + "title": "SHOW DATABASES", + "language": "en" +} +--- + + + +# SHOW DATABASES +## Description +This statement is used to show the currently visible DB +Grammar: +SHOW DATABASES; + +## keyword +SHOW,DATABASES + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md new file mode 100644 index 00000000000000..2e397d52238cc5 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md @@ -0,0 +1,39 @@ +--- +{ + "title": "SHOW DELETE", + "language": "en" +} +--- + + + +# SHOW DELETE +## Description +This statement is used to show successful historical delete tasks performed +Grammar: +SHOW DELETE [FROM db_name] + +## example +1. Show all historical delete tasks for database +SHOW DELETE FROM database; + +## keyword +SHOW,DELETE + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md new file mode 100644 index 00000000000000..fc1c29168edadd --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md @@ -0,0 +1,36 @@ +--- +{ + "title": "SHOW DYNAMIC PARTITION TABLES", + "language": "en" +} +--- + + + + # SHOW DYNAMIC PARTITION TABLES +## description + This statement is used to display all dynamically partitioned table states under the current db + Grammar： + SHOW DYNAMIC PARTITION TABLES [FROM db_name]; + + ## example + 1. Displays all dynamically partitioned table states for the database + SHOW DYNAMIC PARTITION TABLES FROM database; + + ## keyword + SHOW,DYNAMIC,PARTITION,TABLES diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md new file mode 100644 index 00000000000000..09165e0bd08fc0 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md @@ -0,0 +1,62 @@ +--- +{ + "title": "SHOW EXPORT", + "language": "en" +} +--- + + + +# SHOW EXPORT +## Description +This statement is used to show the execution of the specified export task +Grammar: +SHOW EXPORT +[FROM both names] +[ +WHERE +[EXPORT_JOB_ID = your_job_id] +[STATE = ["PENDING"|"EXPORTING"|"FINISHED"|"CANCELLED"]] +] +[ORDER BY ...] +[LIMIT limit]; + +Explain: +1) If db_name is not specified, use the current default DB +2) If STATE is specified, the EXPORT state is matched +3) Any column combination can be sorted using ORDER BY +4) If LIMIT is specified, the limit bar matching record is displayed. Otherwise, all of them will be displayed. + +## example +1. Show all export tasks of default DB +SHOW EXPORT; + +2. Show the export tasks of the specified db, sorted in descending order by StartTime +SHOW EXPORT FROM example_db ORDER BY StartTime DESC; + +3. Show the export task of the specified db, state is "exporting" and sorted in descending order by StartTime +SHOW EXPORT FROM example_db WHERE STATE = "exporting" ORDER BY StartTime DESC; + +4. Show the export task of specifying dB and job_id +SHOW EXPORT FROM example_db WHERE EXPORT_JOB_ID = job_id; + +## keyword +SHOW,EXPORT + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md new file mode 100644 index 00000000000000..e5ef14acea30ba --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md @@ -0,0 +1,74 @@ +--- +{ + "title": "SHOW LOAD", + "language": "en" +} +--- + + + +# SHOW LOAD +## Description +This statement is used to show the execution of the specified import task +Grammar: +SHOW LOAD +[FROM both names] +[ +WHERE +[LABEL [ = "your_label" | LIKE "label_matcher"]] +[STATE = ["PENDING"|"ETL"|"LOADING"|"FINISHED"|"CANCELLED"|]] +] +[ORDER BY ...] +[LIMIT limit][OFFSET offset]; + +Explain: +1) If db_name is not specified, use the current default DB +2) If you use LABEL LIKE, the label that matches the import task contains the import task of label_matcher +3) If LABEL = is used, the specified label is matched accurately. +4) If STATE is specified, the LOAD state is matched +5) Arbitrary column combinations can be sorted using ORDER BY +6) If LIMIT is specified, the limit bar matching record is displayed. Otherwise, all of them will be displayed. +7) If OFFSET is specified, the query results are displayed from offset. By default, the offset is 0. +8) If broker/mini load is used, the connection in the URL column can be viewed using the following command: + +SHOW LOAD WARNINGS ON 'url' + +## example +1. Show all import tasks of default DB +SHOW LOAD; + +2. Show the import task of the specified db. The label contains the string "2014_01_02", showing the oldest 10 +SHOW LOAD FROM example_db WHERE LABEL LIKE "2014_01_02" LIMIT 10; + +3. Show the import task of the specified db, specify label as "load_example_db_20140102" and sort it in descending order by LoadStartTime +SHOW LOAD FROM example_db WHERE LABEL = "load_example_db_20140102" ORDER BY LoadStartTime DESC; + +4. Show the import task of the specified db, specify label as "load_example_db_20140102" and state as "load", and sort it in descending order by LoadStartTime +SHOW LOAD FROM example_db WHERE LABEL = "load_example_db_20140102" AND STATE = "loading" ORDER BY LoadStartTime DESC; + +5. Show the import task of the specified dB and sort it in descending order by LoadStartTime, and display 10 query results starting with offset 5 +SHOW LOAD FROM example_db ORDER BY LoadStartTime DESC limit 5,10; +SHOW LOAD FROM example_db ORDER BY LoadStartTime DESC limit 10 offset 5; + +6. Small batch import is a command to view the import status +curl --location-trusted -u {user}:{passwd} http://{hostname}:{port}/api/{database}/_load_info?label={labelname} + +## keyword +SHOW,LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md new file mode 100644 index 00000000000000..0a75f56116cfa3 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md @@ -0,0 +1,48 @@ +--- +{ + "title": "SHOW PARTITIONS", + "language": "en" +} +--- + + + +# SHOW PARTITIONS +## Description +This statement is used to display partition information +Grammar: +SHOW PARTITIONS FROM [db_name.]table_name [WHERE] [ORDER BY] [LIMIT]; +Explain: +Support filter with following columns: PartitionId,PartitionName,State,Buckets,ReplicationNum, +LastConsistencyCheckTime + +## example +1. Display partition information for the specified table below the specified DB +SHOW PARTITIONS FROM example_db.table_name; + +2. Display information about the specified partition of the specified table below the specified DB +SHOW PARTITIONS FROM example_db.table_name WHERE PartitionName = "p1"; + +3. Display information about the newest partition of the specified table below the specified DB +SHOW PARTITIONS FROM example_db.table_name ORDER BY PartitionId DESC LIMIT 1; + +## keyword +SHOW,PARTITIONS + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md new file mode 100644 index 00000000000000..3c55e9f6c2cfbe --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md @@ -0,0 +1,42 @@ +--- +{ + "title": "SHOW PROPERTY", + "language": "en" +} +--- + + + +# SHOW PROPERTY +## Description +This statement is used to view user attributes +Grammar: +SHOW PROPERTY [FOR user] [LIKE key] + +## example +1. View the attributes of the jack user +SHOW PROPERTY FOR 'jack' + +2. View Jack user import cluster related properties +SHOW PROPERTY FOR 'jack' LIKE '%load_cluster%' + +## keyword +SHOW, PROPERTY + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md new file mode 100644 index 00000000000000..24c1197ea69cfd --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md @@ -0,0 +1,49 @@ +--- +{ + "title": "SHOW REPOSITORIES", + "language": "en" +} +--- + + + +# SHOW REPOSITORIES +## Description +This statement is used to view the currently created warehouse. +Grammar: +SHOW REPOSITORIES; + +Explain: +1. Each column has the following meanings: +RepoId: Unique Warehouse ID +RepoName: Warehouse name +CreateTime: The first time the warehouse was created +IsReadOnly: Is it a read-only warehouse? +Location: The root directory in the repository for backing up data +Broker: Dependent Broker +ErrMsg: Palo regularly checks the connectivity of the warehouse, and if problems occur, error messages are displayed here. + +## example +1. View the warehouse that has been created: +SHOW REPOSITORIES; + +## keyword +SHOW, REPOSITORY, REPOSITORIES + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md new file mode 100644 index 00000000000000..af69744df06638 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md @@ -0,0 +1,67 @@ +--- +{ + "title": "SHOW RESTORE", + "language": "en" +} +--- + + + +# SHOW RESTORE +## Description +This statement is used to view RESTORE tasks +Grammar: +SHOW RESTORE [FROM db_name] + +Explain: +1. Palo -20165;- 20445;- 233844;-36817;- 27425RESTORE -21153s; +2. Each column has the following meanings: +JobId: Unique job ID +Label: The name of the backup to be restored +Timestamp: Time version of backup to be restored +DbName: Subordinate database +State: Current phase +PENDING: The initial state after submitting a job +SNAPSHOTING: In the execution snapshot +DOWNLOAD: The snapshot is complete, ready to download the snapshot in the warehouse +DOWNLOADING: Snapshot Download +COMMIT: Snapshot download completed, ready to take effect +COMMITING: In force +FINISHED: Operation Successful +CANCELLED: Job Failure +AllowLoad: Is import allowed on recovery (currently not supported) +ReplicationNum: Specifies the number of replicas recovered +Restore Jobs: Tables and partitions to be restored +CreateTime: Task submission time +MetaPreparedTime: Metadata Readiness Completion Time +Snapshot Finished Time: Snapshot completion time +Download Finished Time: Snapshot download completion time +FinishedTime: Job End Time +Unfinished Tasks: The unfinished sub-task ID is displayed in the SNAP HOTING, DOWNLOADING, and COMMITING phases +Status: Display failure information if the job fails +Timeout: Job timeout, per second + +## example +1. Check the last RESTORE task under example_db. +SHOW RESTORE FROM example_db; + +## keyword +SHOW, RESTORE + diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md new file mode 100644 index 00000000000000..e44d3c92467066 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md @@ -0,0 +1,35 @@ +--- +{ + "title": "SHOW ROUTINE LOAD TASK", + "language": "en" +} +--- + + + +# SHOW ROUTINE LOAD TASK +## example + +1. Show sub-task information for a routine import task called test 1. + +SHOW ROUTINE LOAD TASK WHERE JobName = "test1"; + +## keyword +SHOW,ROUTINE,LOAD,TASK diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md new file mode 100644 index 00000000000000..68284d900a33b5 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md @@ -0,0 +1,57 @@ +--- +{ + "title": "SHOW ROUTINE LOAD", + "language": "en" +} +--- + + + +# SHOW ROUTINE LOAD +## example + +1. Show all routine import jobs named test 1 (including stopped or cancelled jobs). The result is one or more lines. + +SHOW ALL ROUTINE LOAD FOR test1; + +2. Show the current running routine import job named test1 + +SHOW ROUTINE LOAD FOR test1; + +3. Display all routine import jobs (including stopped or cancelled jobs) under example_db. The result is one or more lines. + +use example_db; +SHOW ALL ROUTINE LOAD; + +4. Display all running routine import jobs under example_db + +use example_db; +SHOW ROUTINE LOAD; + +5. Display the current running routine import job named test1 under example_db + +SHOW ROUTINE LOAD FOR example_db.test1; + +6. Display all routine import jobs named test1 (including stopped or cancelled jobs) under example_db. The result is one or more lines. + +SHOW ALL ROUTINE LOAD FOR example_db.test1; + +## keyword +SHOW,ROUTINE,LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md new file mode 100644 index 00000000000000..84ae5373dc83f1 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md @@ -0,0 +1,56 @@ +--- +{ + "title": "SHOW SNAPSHOT", + "language": "en" +} +--- + + + +# SHOW SNAPSHOT +## Description +This statement is used to view existing backups in the warehouse. +Grammar: +SHOW SNAPSHOT ON `repo_name` +[WHERE SNAPSHOT = "snapshot" [AND TIMESTAMP = "backup_timestamp"]]; + +Explain: +1. Each column has the following meanings: +Snapshot: The name of the backup +Timestamp: Time version for backup +Status: If the backup is normal, the OK will be displayed, otherwise the error message will be displayed. + +2. If TIMESTAMP is specified, the following additional information will be displayed: +Database: The name of the database where the backup data belongs +Details: Shows the entire backup data directory and file structure in the form of Json + +'35;'35; example +1. Check the existing backups in warehouse example_repo: +SHOW SNAPSHOT ON example_repo; + +2. View only the backup named backup1 in warehouse example_repo: +SHOW SNAPSHOT ON example_repo WHERE SNAPSHOT = "backup1"; + +2. Check the backup named backup1 in the warehouse example_repo for details of the time version "2018-05-05-15-34-26": +SHOW SNAPSHOT ON example_repo +WHERE SNAPSHOT = "backup1" AND TIMESTAMP = "2018-05-05-15-34-26"; + +## keyword +SHOW, SNAPSHOT diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md new file mode 100644 index 00000000000000..e1a5d3a627eaf6 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md @@ -0,0 +1,34 @@ +--- +{ + "title": "SHOW TABLES", + "language": "en" +} +--- + + + +# SHOW TABLES +## Description +This statement is used to show all tables under the current DB +Grammar: +SHOW TABLES; + +## keyword +SHOW,TABLES diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md new file mode 100644 index 00000000000000..17a559e15ec835 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md @@ -0,0 +1,42 @@ +--- +{ + "title": "SHOW TABLET", + "language": "en" +} +--- + + + +# SHOW TABLET +## Description +This statement is used to display tablet-related information (for administrators only) +Grammar: +SHOW TABLET +[From [db-uu name]] table U name.; Table U Id] + +## example +1. Display all tablet information in the specified table below the specified DB +SHOW TABLET FROM example_db.table_name; + +2. Display parent level ID information of tablet with specified tablet ID of 10000 +Performance board 10000; + +## keyword +SHOW,TABLET diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md new file mode 100644 index 00000000000000..0639f15356e78d --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md @@ -0,0 +1,86 @@ +--- +{ + "title": "SHOW TRANSACTION", + "language": "en" +} +--- + + + +# SHOW TRANSACTION +## description + +This syntax is used to view transaction details for the specified transaction id. + +grammar: + +``` +SHOW TRANSACTION +[FROM db_name] +WHERE id = transaction_id; +``` + +Example return result: + +``` +     TransactionId: 4005 +             Label: insert_8d807d5d-bcdd-46eb-be6d-3fa87aa4952d +       Coordinator: FE: 10.74.167.16 + TransactionStatus: VISIBLE + LoadJobSourceType: INSERT_STREAMING +       PrepareTime: 2020-01-09 14:59:07 +        CommitTime: 2020-01-09 14:59:09 +        FinishTime: 2020-01-09 14:59:09 +            Reason: +ErrorReplicasCount: 0 +        ListenerId: -1 +         TimeoutMs: 300000 +``` + +* TransactionId: transaction id +* Label: the label of the corresponding load job +* Coordinator: the node responsible for transaction coordination +* TransactionStatus: transaction status +    * PREPARE: preparation stage +    * COMMITTED: The transaction was successful, but the data is not visible +    * VISIBLE: The transaction was successful and the data is visible +    * ABORTED: transaction failed +* LoadJobSourceType: The type of the load job. +* PrepareTime: transaction start time +* CommitTime: the time when the transaction was successfully committed +* FinishTime: The time when the data is visible +* Reason: error message +* ErrorReplicasCount: Number of replicas with errors +* ListenerId: the id of the related load job +* TimeoutMs: transaction timeout time in milliseconds + +## example + +1. View the transaction with id 4005: + + SHOW TRANSACTION WHERE ID = 4005; + +2. Specify the db and view the transaction with id 4005: + + SHOW TRANSACTION FROM db WHERE ID = 4005; + +## keyword + + SHOW, TRANSACTION \ No newline at end of file diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md new file mode 100644 index 00000000000000..487aaaed4ccd80 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md @@ -0,0 +1,35 @@ +--- +{ + "title": "STOP ROUTINE LOAD", + "language": "en" +} +--- + + + +# STOP ROUTINE LOAD +## example + +1. Stop the routine import job named test 1. + +STOP ROUTINE LOAD FOR test1; + +## keyword +STOP,ROUTINE,LOAD diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md b/docs/en/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md new file mode 100644 index 00000000000000..b4597b250baa54 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md @@ -0,0 +1,193 @@ +--- +{ + "title": "STREAM LOAD", + "language": "en" +} +--- + + + +# STREAM LOAD +## description + +NAME + +load data to table in streaming + +SYNOPSIS + +Curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_host:http_port/api/{db}/{table}/_stream_load + +DESCRIPTION + +This statement is used to load data to the specified table. The difference from normal load is that this load method is synchronous load. + +This type of load still guarantees the atomicity of a batch of load tasks, either all data is loaded successfully or all fails. + +This operation also updates the data for the rollup table associated with this base table. + +This is a synchronous operation that returns the results to the user after the entire data load is completed. + +Currently, HTTP chunked and non-chunked uploads are supported. For non-chunked mode, Content-Length must be used to indicate the length of the uploaded content, which ensures data integrity. + +In addition, the user preferably sets the Content of the Expect Header field to 100-continue, which avoids unnecessary data transmission in certain error scenarios. + +OPTIONS + +Users can pass in the load parameters through the Header part of HTTP. + +`label` + +A label that is loaded at one time. The data of the same label cannot be loaded multiple times. Users can avoid the problem of repeated data load by specifying the label. + +Currently Palo internally retains the most recent successful label within 30 minutes. + +`column_separator` + +Used to specify the column separator in the load file. The default is `\t`. If it is an invisible character, you need to add `\x` as a prefix and hexadecimal to indicate the separator. + +For example, the separator `\x01` of the hive file needs to be specified as `-H "column_separator:\x01"` + +`columns` + +used to specify the correspondence between the columns in the load file and the columns in the table. If the column in the source file corresponds exactly to the contents of the table, then it is not necessary to specify the contents of this field. If the source file does not correspond to the table schema, then this field is required for some data conversion. There are two forms of column, one is directly corresponding to the field in the load file, directly using the field name to indicate. + +One is a derived column with the syntax `column_name` = expression. Give a few examples to help understand. + +Example 1: There are three columns "c1, c2, c3" in the table. The three columns in the source file correspond to "c3, c2, c1" at a time; then you need to specify `-H "columns: c3, c2, c1"` + +Example 2: There are three columns in the table, "c1, c2, c3". The first three columns in the source file correspond in turn, but there are more than one column; then you need to specify` -H "columns: c1, c2, c3, xxx"` + +The last column can optionally specify a name for the placeholder. + +Example 3: There are three columns in the table, "year, month, day". There is only one time column in the source file, which is "2018-06-01 01:02:03" format. Then you can specify `-H "columns: col, year = year(col), month=month(col), day=day(col)"` to complete the load. + +`where` + +Used to extract some data. If the user needs to filter out the unwanted data, it can be achieved by setting this option. + +Example 1: load only data larger than k1 column equal to 20180601, then you can specify -H "where: k1 = 20180601" when loading + +`max_filter_ratio` + +The maximum proportion of data that can be filtered (for reasons such as data irregularity). The default is zero tolerance. Data non-standard does not include rows that are filtered out by the where condition. + +`Partitions` + +Used to specify the partition designed for this load. If the user is able to determine the partition corresponding to the data, it is recommended to specify the item. Data that does not satisfy these partitions will be filtered out. + +For example, specify load to p1, p2 partition, `-H "partitions: p1, p2"` + +`Timeout` + +Specifies the timeout for the load. Unit seconds. The default is 600 seconds. The range is from 1 second to 259200 seconds. + +`strict_mode` + +The user specifies whether strict load mode is enabled for this load. The default is enabled. The shutdown mode is `-H "strict_mode: false"`. + +`timezone` + +Specifies the time zone used for this load. The default is East Eight District. This parameter affects all function results related to the time zone involved in the load. + +`exec_mem_limit` + +Memory limit. Default is 2GB. Unit is Bytes. + +RETURN VALUES + +After the load is completed, the related content of this load will be returned in Json format. Current field included + +* `Status`: load status. + + * Success: indicates that the load is successful and the data is visible. + + * Publish Timeout: Indicates that the load job has been successfully Commit, but for some reason it is not immediately visible. Users can be considered successful and do not have to retry load + + * Label Already Exists: Indicates that the Label is already occupied by another job, either the load was successful or it is being loaded. The user needs to use the get label state command to determine the subsequent operations. + + * Other: The load failed, the user can specify Label to retry the job. + +* Message: A detailed description of the load status. When it fails, it will return the specific reason for failure. + +* NumberTotalRows: The total number of rows read from the data stream + +* NumberLoadedRows: The number of data rows loaded this time, only valid when Success + +* NumberFilteredRows: The number of rows filtered by this load, that is, the number of rows with unqualified data quality. + +* NumberUnselectedRows: Number of rows that were filtered by the where condition for this load + +* LoadBytes: The amount of source file data loaded this time + +* LoadTimeMs: Time spent on this load + +* ErrorURL: The specific content of the filtered data, only the first 1000 items are retained + +ERRORS + +You can view the load error details by the following statement: + + ```SHOW LOAD WARNINGS ON 'url'``` + +Where url is the url given by ErrorURL. + +## example + +1. load the data from the local file 'testData' into the table 'testTbl' in the database 'testDb' and use Label for deduplication. Specify a timeout of 100 seconds + + ```Curl --location-trusted -u root -H "label:123" -H "timeout:100" -T testData http://host:port/api/testDb/testTbl/_stream_load``` + +2. load the data in the local file 'testData' into the table of 'testTbl' in the database 'testDb', use Label for deduplication, and load only data with k1 equal to 20180601 +         + ```Curl --location-trusted -u root -H "label:123" -H "where: k1=20180601" -T testData http://host:port/api/testDb/testTbl/_stream_load``` + +3. load the data from the local file 'testData' into the 'testTbl' table in the database 'testDb', allowing a 20% error rate (user is in defalut_cluster) + + ```Curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -T testData http://host:port/api/testDb/testTbl/_stream_load``` + +4. load the data from the local file 'testData' into the 'testTbl' table in the database 'testDb', allow a 20% error rate, and specify the column name of the file (user is in defalut_cluster) + + ```Curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "columns: k2, k1, v1" -T testData http://host:port/api/testDb/testTbl/_stream_load``` + +5. load the data from the local file 'testData' into the p1, p2 partition in the 'testTbl' table in the database 'testDb', allowing a 20% error rate. + + ```Curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "partitions: p1, p2" -T testData http://host:port/api/testDb/testTbl/stream_load``` + +6. load using streaming mode (user is in defalut_cluster) + + ```Seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_stream_load``` + +7. load a table with HLL columns, which can be columns in the table or columns in the data used to generate HLL columns,you can also use hll_empty to supplement columns that are not in the data + + ```Curl --location-trusted -u root -H "columns: k1, k2, v1=hll_hash(k1), v2=hll_empty()" -T testData http://host:port/api/testDb/testTbl/_stream_load``` + +8. load data for strict mode filtering and set the time zone to Africa/Abidjan + + ```Curl --location-trusted -u root -H "strict_mode: true" -H "timezone: Africa/Abidjan" -T testData http://host:port/api/testDb/testTbl/_stream_load``` + +9. load a table with BITMAP columns, which can be columns in the table or a column in the data used to generate BITMAP columns, you can also use bitmap_empty to supplement columns that are not in the data + + ```Curl --location-trusted -u root -H "columns: k1, k2, v1=to_bitmap(k1), v2=bitmap_empty()" -T testData http://host:port/api/testDb/testTbl/_stream_load``` + + +## keyword + + STREAM, LOAD \ No newline at end of file diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/insert.md b/docs/en/sql-reference/sql-statements/Data Manipulation/insert.md new file mode 100644 index 00000000000000..43dec1224d5a71 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/insert.md @@ -0,0 +1,110 @@ +--- +{ + "title": "INSERT", + "language": "en" +} +--- + + + +# INSERT +## Description +### Syntax + +``` +INSERT INTO table_name +[ PARTITION (p1, ...)] +[ WITH LABEL label] +[ (column [, ...]) ] +[ [ hint [, ...] ] ] +{ VALUES ( { expression | DEFAULT } [, ...] ) [, ...] | query } +``` + +### Parameters + +> tablet_name: Target table for loading data. It can be in the form of `db_name.table_name`. +> +> partitions: Specifies the partitions to be loaded, with multiple partition names separated by commas. The partitions must exist in `table_name`, +> +> label: Specifies a label for Insert job. +> +> column_name: The specified destination columns must be columns that exists in `table_name`. +> +> expression: The corresponding expression that needs to be assigned to a column. +> +> DEFAULT: Let the corresponding columns use default values +> +> query: A common query whose results are written to the target +> +> hint: Indicators used to indicate `INSERT` execution. ` Both streaming `and default non `streaming'methods use synchronization to complete `INSERT' statement execution +> The non `streaming'mode returns a label after execution to facilitate users to query the imported status through `SHOW LOAD'. + +### Note + +When the `INSERT'statement is currently executed, the default behavior for data that does not conform to the target table is filtering, such as string length. However, for business scenarios where data is not filtered, the session variable `enable_insert_strict'can be set to `true' to ensure that `INSERT'will not be successfully executed when data is filtered out. + +## example + +` The test `table contains two columns `c1', `c2'. + +1. Import a row of data into the `test'table + +``` +INSERT INTO test VALUES (1, 2); +INSERT INTO test (c1, c2) VALUES (1, 2); +INSERT INTO test (c1, c2) VALUES (1, DEFAULT); +INSERT INTO test (c1) VALUES (1); +``` + +The first and second sentences have the same effect. When the target column is not specified, the column order in the table is used as the default target column. +The third and fourth statements express the same meaning, using the default value of `c2'column to complete data import. + +2. Import multiline data into the `test'table at one time + +``` +INSERT INTO test VALUES (1, 2), (3, 2 + 2) +INSERT INTO test (c1, c2) VALUES (1, 2), (3, 2 * 2) +INSERT INTO test (c1) VALUES (1), (3) +Insert in test (C1, C2) values (1, Default), (3, Default) +``` + +The effect of the first and second statements is the same, and two data are imported into the `test'table at one time. +The effect of the third and fourth statements is known, using the default value of the `c2'column to import two data into the `test' table. + + +3. Insert into table `test` with a query stmt. + +``` +INSERT INTO test SELECT * FROM test2 +INSERT INTO test (c1, c2) SELECT * from test2 +``` + +4. Insert into table `test` with specified partition and label + +``` +INSERT INTO test PARTITION(p1, p2) WITH LABEL `label1` SELECT * FROM test2; +INSERT INTO test WITH LABEL `label1` (c1, c2) SELECT * from test2; +``` + +Asynchronous imports are, in fact, encapsulated asynchronously by a synchronous import. Filling in streaming is as efficient as not filling in * execution. + +Since Doris used to import asynchronously, in order to be compatible with the old usage habits, the `INSERT'statement without streaming will still return a label. Users need to view the status of the `label' import job through the `SHOW LOAD command. +##keyword +INSERT diff --git a/docs/en/sql-reference/sql-statements/Data Types/BIGINT.md b/docs/en/sql-reference/sql-statements/Data Types/BIGINT.md new file mode 100644 index 00000000000000..891964e3988f3b --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/BIGINT.md @@ -0,0 +1,33 @@ +--- +{ + "title": "BIGINT", + "language": "en" +} +--- + + + +# BIGINT +## Description +BIGINT +8-byte signed integer, range [-9223372036854775808, 9223372036854775807] + +##keyword +BIGINT diff --git a/docs/en/sql-reference/sql-statements/Data Types/BOOLEAN.md b/docs/en/sql-reference/sql-statements/Data Types/BOOLEAN.md new file mode 100644 index 00000000000000..79faa6e14e9875 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/BOOLEAN.md @@ -0,0 +1,33 @@ +--- +{ + "title": "BOOLEAN", + "language": "en" +} +--- + + + +# BOOLEAN +## Description +BOOL, BOOLEAN +Like TINYINT, 0 stands for false and 1 for true. + +##keyword +BOOLEAN diff --git a/docs/en/sql-reference/sql-statements/Data Types/CHAR.md b/docs/en/sql-reference/sql-statements/Data Types/CHAR.md new file mode 100644 index 00000000000000..a0ff4b849c2b52 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/CHAR.md @@ -0,0 +1,33 @@ +--- +{ + "title": "CHAR", + "language": "en" +} +--- + + + +# CHAR +## Description +CHAR(M) +A fixed-length string, M represents the length of a fixed-length string. The range of M is 1-255. + +##keyword +CHAR diff --git a/docs/en/sql-reference/sql-statements/Data Types/DATE.md b/docs/en/sql-reference/sql-statements/Data Types/DATE.md new file mode 100644 index 00000000000000..f6aee7f807fcb9 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/DATE.md @@ -0,0 +1,41 @@ +--- +{ + "title": "date", + "language": "en" +} +--- + + + +# date +## Description +DATE function +Syntax: +Date +Convert input type to DATE type +date +Date type, the current range of values is ['0000-01-01','9999-12-31'], and the default print form is'YYYYY-MM-DD'. + +## example +mysql> SELECT DATE('2003-12-31 01:02:03'); +-> '2003-12-31' + +##keyword +DATE diff --git a/docs/en/sql-reference/sql-statements/Data Types/DATETIME.md b/docs/en/sql-reference/sql-statements/Data Types/DATETIME.md new file mode 100644 index 00000000000000..92f6280caf846f --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/DATETIME.md @@ -0,0 +1,34 @@ +--- +{ + "title": "DATETIME", + "language": "en" +} +--- + + + +# DATETIME +## Description +DATETIME +Date and time type, value range is ['0000-01-01 00:00:00','9999-12-31 23:59:59']. +The form of printing is'YYYY-MM-DD HH:MM:SS' + +##keyword +DATETIME diff --git a/docs/en/sql-reference/sql-statements/Data Types/DECIMAL.md b/docs/en/sql-reference/sql-statements/Data Types/DECIMAL.md new file mode 100644 index 00000000000000..b5880f802f25f3 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/DECIMAL.md @@ -0,0 +1,34 @@ +--- +{ + "title": "DECIMAL", + "language": "en" +} +--- + + + +# DECIMAL +## Description +DECIMAL (M [,D]) +High-precision fixed-point, M stands for the total number of significant numbers (precision), D stands for the maximum number of decimal points (scale) +The range of M is [1,27], the range of D is [1,9], in addition, M must be greater than or equal to the value of D. The default value of D is 0. + +##keyword +DECIMAL diff --git a/docs/en/sql-reference/sql-statements/Data Types/DOUBLE.md b/docs/en/sql-reference/sql-statements/Data Types/DOUBLE.md new file mode 100644 index 00000000000000..da5042b9434c7c --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/DOUBLE.md @@ -0,0 +1,33 @@ +--- +{ + "title": "Double", + "language": "en" +} +--- + + + +# Double +## Description +DOUBLE +8-byte floating point number + +##keyword +DOUBLE \ No newline at end of file diff --git a/docs/en/sql-reference/sql-statements/Data Types/FLOAT.md b/docs/en/sql-reference/sql-statements/Data Types/FLOAT.md new file mode 100644 index 00000000000000..d5d9b0ac0506eb --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/FLOAT.md @@ -0,0 +1,33 @@ +--- +{ + "title": "FLOAT", + "language": "en" +} +--- + + + +# FLOAT +## Description +FLOAT +4-byte floating point number + +##keyword +FLOAT diff --git a/docs/en/sql-reference/sql-statements/Data Types/HLL(HyperLogLog).md b/docs/en/sql-reference/sql-statements/Data Types/HLL(HyperLogLog).md new file mode 100644 index 00000000000000..920fd74fb24d46 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/HLL(HyperLogLog).md @@ -0,0 +1,35 @@ +--- +{ + "title": "HLL (Hyloglog)", + "language": "en" +} +--- + + + +#HLL (Hyloglog) +## Description +MARKETING (M) +A variable length string, M represents the length of a variable length string. The range of M is 1-16385. +Users do not need to specify length and default values. Length is controlled within the system according to the aggregation degree of data +And HLL columns can only be queried or used by matching hll_union_agg, hll_raw_agg, hll_cardinality, hll_hash. + +##keyword +High loglog, hll, hyloglog diff --git a/docs/en/sql-reference/sql-statements/Data Types/INT.md b/docs/en/sql-reference/sql-statements/Data Types/INT.md new file mode 100644 index 00000000000000..06b0da905c15c0 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/INT.md @@ -0,0 +1,33 @@ +--- +{ + "title": "INT", + "language": "en" +} +--- + + + +# INT +## Description +INT +4-byte signed integer, range [-2147483648, 2147483647] + +##keyword +INT diff --git a/docs/en/sql-reference/sql-statements/Data Types/SMALLINT.md b/docs/en/sql-reference/sql-statements/Data Types/SMALLINT.md new file mode 100644 index 00000000000000..4e2663eccff7f4 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/SMALLINT.md @@ -0,0 +1,33 @@ +--- +{ + "title": "SMALLINT", + "language": "en" +} +--- + + + +# SMALLINT +## Description +SMALLINT +2-byte signed integer, range [-32768, 32767] + +##keyword +SMALLINT diff --git a/docs/en/sql-reference/sql-statements/Data Types/TINYINT.md b/docs/en/sql-reference/sql-statements/Data Types/TINYINT.md new file mode 100644 index 00000000000000..680af624aaa96e --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/TINYINT.md @@ -0,0 +1,33 @@ +--- +{ + "title": "TINYINT", + "language": "en" +} +--- + + + +# TINYINT +## Description +TINYINT +1 byte signed integer, range [-128, 127] + +##keyword +TINYINT diff --git a/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md b/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md new file mode 100644 index 00000000000000..8dfa67cb6e141b --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md @@ -0,0 +1,33 @@ +--- +{ + "title": "VARCHAR", + "language": "en" +} +--- + + + +# VARCHAR +## Description +MARKETING (M) +A variable length string, M represents the length of a variable length string. The range of M is 1-65535. + +##keyword +VARCHAR diff --git a/docs/en/sql-reference/sql-statements/Utility/util_stmt.md b/docs/en/sql-reference/sql-statements/Utility/util_stmt.md new file mode 100644 index 00000000000000..955b2dd9c180d0 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Utility/util_stmt.md @@ -0,0 +1,39 @@ +--- +{ + "title": "Describe", + "language": "en" +} +--- + + + +# Describe +## Description +This statement is used to display schema information for the specified table +Grammar: +DESC [FISH] [dbu name.]table name [ALL]; + +Explain: +If ALL is specified, the schema of all indexes of the table is displayed + +## example + +## keyword +DESCRIBE,DESC diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 00000000000000..851aff5b3aa10d --- /dev/null +++ b/docs/package.json @@ -0,0 +1,36 @@ +{ + "name": "docs", + "private": true, + "description": "docs of doris", + "keywords": [ + "doris", + "mysql" + ], + "homepage": "https://github.com/apache/incubator-doris", + "bugs": { + "url": "https://github.com/apache/incubator-doris/issues" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/apache/incubator-doris.git" + }, + "license": "Apache", + "author": "Apache Doris", + "scripts": { + "build": "vuepress build . --temp .temp", + "dev": "vuepress dev . --temp .temp", + "lint": "markdownlint '**/*.md' -f" + }, + "devDependencies": { + "@vuepress/plugin-back-to-top": "^1.3.1", + "@vuepress/plugin-medium-zoom": "^1.3.1", + "@vuepress/theme-vue": "^1.3.1", + "markdownlint": "^0.19.0", + "markdownlint-cli": "^0.22.0", + "vue-toasted": "^1.1.25", + "vuepress": "^1.3.1" + }, + "dependencies": { + "axios": "^0.19.2" + } +} diff --git a/docs/readme.md b/docs/readme.md index b87bc3be5882fe..fa8219589bb417 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -17,100 +17,231 @@ specific language governing permissions and limitations under the License. --> -## Philosophy +# Doris Document -**write once, use everywhere** +[Vuepress](https://github.com/vuejs/vuepress.git) is used as our document site generator, configurations are in `./docs/.vuepress` folder. -Documentations will be written once, and will be converted to other format according to different application scenarios. +## Getting Started -## Implementation +Download and install [nodejs](http://nodejs.cn/download/) +```bash +npm config set registry https://registry.npm.taobao.org // Only if you are in Mainland China. +cd docs && npm install +npm run dev ``` - +---------------+ - | Documentation | - +-------+-------+ - | - +-------+-------+ - | Doc Builder | - +-------+-------+ - | - +--------------------------------+ - | | | -+---+---+ +---+----+ +-----+----+ -| PDF | | HTML | .... | Help Doc | -+-------+ +--------+ +----------+ +Open your browser and navigate to `localhost:8080/en/` or `localhost:8080/zh-CN/`. + +## Docs' Directories + +```bash + . + ├─ docs/ + │ ├─ .vuepress + │ │ ├─ dist // Built site files. + │ │ ├─ public // Assets + │ │ ├─ sidebar // Side bar configurations. + │ │ │ ├─ en.js + │ │ │ └─ zh-CN.js + │ ├─ theme // Global styles and customizations. + │ └─ config.js // Vuepress configurations. + ├─ zh-CN/ + │ ├─ xxxx.md + │ └─ README.md // Will be rendered as entry page. + └─ en/ + ├─ one.md + └─ README.md // Will be rendered as entry page. ``` -> Documentation：Text contents which is written by human. And this is the only place for documentation. -> Doc Builder: Tools that convert documentations to other format, such as PDF, HTML. There could be many tools, and we can use different tools to convert documentation to different formats. +## Start Writing + +1. Write markdown files in multi languages and put them in separated folders `./en/` and `./zh-CN/`. **But they should be with the same name.** + + ```bash + . + ├─ en/ + │ ├─ one.md + │ └─ two.md + └─ zh-CN/ + │ ├─ one.md + │ └─ two.md + ``` + +2. Frontmatters like below should always be on the top of each file: + + ```markdown + --- + { + "title": "Backup and Recovery", // sidebar title + "language": "en" // writing language + } + --- + ``` + +3. Assets are in `.vuepress/public/`. + + Assuming that there exists a png `.vuepress/public/images/image_x.png`, then it can be used like: + + ```markdown + ![alter text](/images/image_x.png) + ``` + +4. Remember to update the sidebar configurations in `.vuepress/sidebar/` after adding a new file or a folder. + + Assuming that the directories are: + + ```bash + . + ├─ en/ + │ ├─ subfolder + │ │ ├─ one.md + │ │ └─ two.md + │ └─ three.md + └─ zh-CN/ + ├─ subfolder + │ ├─ one.md + │ └─ two.md + └─ three.md + ``` + + Then the sidebar configurations would be like: + + ```javascript + // .vuepress/sidebar/en.js` + module.exports = [ + { + title: "subfolder name", + directoryPath: "subfolder/", + children: ["one", "two"] + }, + "three" + ] + ``` + + ```javascript + // .vuepress/sidebar/zh-CN.js + module.exports = [ + { + title: "文件夹名称", + directoryPath: "subfolder/", + children: ["one", "two"] + }, + "three" + ] + ``` + +5. Run `npm run lint` before starting a PR. + + Surely that there will be lots of error logs if the mardown files are not following the rules, and these logs will all be printed in the console: + +```shell + +en/administrator-guide/alter-table/alter-table-bitmap-index.md:92 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: " ```"] +en/administrator-guide/alter-table/alter-table-rollup.md:45 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-rollup.md:77 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-rollup.md:178 MD046/code-block-style Code block style [Expected: fenced; Actual: indented] +en/administrator-guide/alter-table/alter-table-schema-change.md:50 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:82 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:127 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:144 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:153 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"] +en/administrator-guide/alter-table/alter-table-schema-change.md:199 MD046/code-block-style Code block style [Expected: fenced; Actual: indented] +en/administrator-guide/backup-restore.md:45:1 MD029/ol-prefix Ordered list item prefix [Expected: 1; Actual: 2; Style: 1/1/1] +en/administrator-guide/backup-restore.md:57:1 MD029/ol-prefix Ordered list item prefix [Expected: 1; Actual: 2; Style: 1/1/1] +en/administrator-guide/backup-restore.md:61:1 MD029/ol-prefix Ordered list item prefix [Expected: 1; Actual: 3; Style: 1/1/1] +npm ERR! code ELIFECYCLE +npm ERR! errno 1 +npm ERR! docs@ lint: `markdownlint '**/*.md' -f` +npm ERR! Exit status 1 +npm ERR! +npm ERR! Failed at the docs@ lint script. -## Organization +``` -> `docs/documentation`: Root directory for documentation. And for different languages, there is a root directory for it. For example, `docs/documentation/cn` is the Chinese documentation's root directory. -> `docs/scripts`: Place of `Doc Builder`. -> `docs/resources`: Resources that are referenced in documentation, such as pictures. -> `docs/website`: A website for documentations built with [Sphinx](http://www.sphinx-doc.org) using a theme provided by [Read-the-Docs](https://readthedocs.org/). +## Deployment -## Constraints +Just start a PR, and all things will be done automatically. -1. All documents are written in Markdown format, and file name is end with ".md". -2. All documents are started with level 1 title `# Title`, and should have only one level 1 title. -3. Names of file and directory are in lowercase letters, and use dashes as separator. -4. Documentation can be constructed as a directory or a single Markdown file, these two formats equal with each other in logical. Relationship is represented by parent-child directory in directory format, and by title level in file format. It is recommended to use directory format to manage a large documentation, because it is easy to maintain. -3. A directory corresponds to a title, and readme.md in this directory is its content. Other documents in this directory is its sub-sections. -4. For manual like section, such as function description, there should be `Description`, `Syntax`, `Examples` section in documents. +## What Travis Does -## Level Directories +Once a PR accepted, travis ci will be triggered to build and deploy the whole website within its own branch. Here is what `.travis.yml` does: -1. doris-concepts -2. installing -3. getting-started -4. administrator-guide -5. sql-references -6. best-practices -7. internals -8. community +1. Prepare nodejs and vuepress enviorment. -Each directory, or its sub directories should contain a file `index.rst`, for constructing the navibar of the website. For example: +2. Use current branch's name as the relative url path in `.vuepress/config.js`(which is the `base` property). -``` -documentation/ -└── cn - ├── administrator-guide - │   ├── index.rst - │   ├── http-actions - │   │   └── index.rst - │   ├── load-data - │   │   ├── index.rst - │   ├── operation - │   │   ├── index.rst - ├── extending-doris - │   ├── index.rst - └── sql-reference - ├── index.rst - │   ├── date-time-functions - │   │   ├── index.rst -``` +3. Build the documents into a website all by vuepress. -## Docs Styles +4. Fetch asf-site repo to local directory, and copy `.vupress/dist/` into `{BRANCH}/`. -There are some styles need to be followed. +5. Push the new site to asf-site repo with `GitHub Token`(which is preset in Travis console as a variable used in .travis.yml). -### SQL-Statement +## asf-site repository -Docs under `documentation/cn/sql-reference/sql-statements/` must obey the following style +Finally the asf-site repository will be like: +```bash +. +├─ master/ +│ ├─ en/ +│ │ ├─ subfolder +│ │ │ ├─ one.md +│ │ └─ three.md +│ └─ zh-CN/ +│ ├─ subfolder +│ │ ├─ one.md +│ └─ three.md +├─ incubating-0.11/ +│ ├─ en/ +│ │ ├─ subfolder +│ │ │ ├─ one.md +│ │ └─ three.md +│ └─ zh-CN/ +│ ├─ subfolder +│ │ ├─ one.md +│ └─ three.md +├─ index.html // user entry, and auto redirected to master folder +└─ versions.json // all versions that can be seleted on the website are defined here ``` -# TITLE(capital) - -## description - The description of this doc. The "## description" must be reserved, with a following empty line. - -## keyword - - The keyword of this doc. Usually, this can be the title of this doc. - The "## keyword" must be reserved, with a following empty line. +And the `versions.json` is like: + +```json +{ + "en": [ + { + "text": "Versions", // dropdown label + "items": [ + { + "text": "master", // dropdown-item label + "link": "/../master/en/installing/compilation.html", // entry page for this version + "target": "_blank" + }, + { + "text": "branch-0.11", + "link": "/../branch-0.11/en/installing/compilation.html", + "target": "_blank" + } + ] + } + ], + "zh-CN": [ + { + "text": "版本", + "items": [ + { + "text": "master", + "link": "/../master/zh-CN/installing/compilation.html", + "target": "_blank" + }, + { + "text": "branch-0.11", + "link": "/../branch-0.11/zh-CN/installing/compilation.html", + "target": "_blank" + } + ] + } + ] +} ``` - diff --git a/docs/resources/palo_export_mind_map.svg b/docs/resources/palo_export_mind_map.svg deleted file mode 100644 index 278973a21b4008..00000000000000 --- a/docs/resources/palo_export_mind_map.svg +++ /dev/null @@ -1 +0,0 @@ -导出导出内容全量1部分指定partition进行导出2任意sql查询结果 3导出目标存储localcloudBaidu HadoopBOSCommunity Hadoop导出后访问方式palo的外部表其它导出后数据格式csv文本压缩文件其它格式其它导出效率失败重试 \ No newline at end of file diff --git a/docs/website/Makefile b/docs/website/Makefile deleted file mode 100644 index 76d13b43b00cda..00000000000000 --- a/docs/website/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/website/README.md b/docs/website/README.md deleted file mode 100644 index 9058cfdf00c815..00000000000000 --- a/docs/website/README.md +++ /dev/null @@ -1,61 +0,0 @@ - - - -# Build the website for Doris documentations - -## Prerequisites - -1. Install PiPy (If not installed) - - ``` - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - python get-pip.py - ``` - -2. Install sphinx and other dependencies - - ``` - pip install sphinx # Sphinx main program - pip install recommonmark # Sphinx markdown extension - pip install sphinx-markdown-tables # Sphinx markdown table render extension - pip install jieba # Sphinx Chinese tokenizer - pip install sphinx_rtd_theme # Sphinx Read-the-Docs theme - ``` - -## Build the website - -``` -sh build_site.sh -``` - -## Start web server - -``` -cd build/html/ -nohup python -m SimpleHTTPServer & -``` - -You can start any web server you like. - -## Browse website - -``` -http://localhost:8000/ -``` diff --git a/docs/website/build_site.sh b/docs/website/build_site.sh deleted file mode 100644 index d9936e9ff7b5e2..00000000000000 --- a/docs/website/build_site.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -CUR=`dirname "$0"` -CUR=`cd "$CUR"; pwd` - -export DORIS_HOME=${CUR}/../../ - -# Check args -usage() { - echo " -Usage: $0 - Optional options: - --cn build Chinese Documentation - --en build English Documentation - " - exit 1 -} - -OPTS=$(getopt \ - -n $0 \ - -o '' \ - -l 'cn' \ - -l 'en' \ - -l 'help' \ - -- "$@") - -if [ $? != 0 ] ; then - usage -fi - -eval set -- "$OPTS" - -BUILD_CN= -BUILD_EN= -HELP=0 -if [ $# == 1 ] ; then - # default - BUILD_CN=1 - BUILD_EN=0 -else - BUILD_CN=0 - BUILD_EN=0 - while true; do - case "$1" in - --cn) BUILD_CN=1 ; shift ;; - --en) BUILD_EN=1 ; shift ;; - --help) HELP=1; shift ;; - --) shift ; break ;; - *) ehco "Internal error" ; exit 1 ;; - esac - done -fi - -if [[ ${HELP} -eq 1 ]]; then - usage - exit -fi - - -rm -rf ${CUR}/source/documentation/ -rm -rf ${CUR}/source/resources/ -mkdir -p ${CUR}/source/documentation -mkdir -p ${CUR}/source/resources -cp -r ${DORIS_HOME}/docs/documentation/* ${CUR}/source/documentation/ -cp -r ${DORIS_HOME}/docs/resources/* ${CUR}/source/resources/ -make clean && make html - diff --git a/docs/website/source/_templates/footer.html b/docs/website/source/_templates/footer.html deleted file mode 100644 index 21beb5cd484ef4..00000000000000 --- a/docs/website/source/_templates/footer.html +++ /dev/null @@ -1,28 +0,0 @@ - - -{% extends '!footer.html' %} -{% block extrafooter %} -
-

-

- - Apache Doris(incubating) is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. - -

-
-{% endblock %} diff --git a/docs/website/source/conf.py b/docs/website/source/conf.py deleted file mode 100644 index 72c107cd305b66..00000000000000 --- a/docs/website/source/conf.py +++ /dev/null @@ -1,199 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Configuration file for the Sphinx documentation builder. -# -# This file does only contain a selection of the most common options. For a -# full list see the documentation: -# http://www.sphinx-doc.org/en/master/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - -#import t3SphinxThemeRtd - -# -- Project information ----------------------------------------------------- - -project = u'Doris Documentations' -copyright = u'2019, Apache Doris(Incubating)' -author = u'Apache Doris(Incubating)' - -# The short X.Y version -version = u'0.11.0' -# The full version, including alpha/beta/rc tags -release = u'0.11.0' - - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'recommonmark', - 'sphinx_markdown_tables' -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -# source_suffix = '.rst' -source_suffix = { - '.rst': 'restructuredtext', - '.txt': 'markdown', - '.md': 'markdown', -} - -# The master toctree document. -master_doc = 'index' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "sphinx_rtd_theme" -#html_theme = "t3SphinxThemeRtd" -#html_theme_path = [t3SphinxThemeRtd.get_html_theme_path()] - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - - -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = 'dorisdoc' - - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'doris.tex', u'Doris Documentation', - u'Apache Doris(Incubating)', 'manual'), -] - - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'Doris', u'Doris Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'Doris', u'Doris Documentation', - author, 'Apache Doris(Incubating)', 'An MPP-based interactive SQL data warehouse for reporting and analysis', - 'Miscellaneous'), -] - - -# -- Options for Epub output ------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] - -html_search_language = 'zh' - -html_context = { - # Enable the "Edit in GitHub link within the header of each page. - 'display_github': True, - # Set the following variables to generate the resulting github URL for each page. - # Format Template: https://{{ github_host|default("github.com") }}/{{ github_user }} - #/{{ github_repo }}/blob/{{ github_version }}{{ conf_py_path }}{{ pagename }}{{ suffix }} - 'github_user': 'apache', - 'github_repo': 'incubator-doris', - 'github_version': 'master' , - 'conf_py_path' : '/docs/', -} - -# -- Extension configuration ------------------------------------------------- diff --git a/docs/website/source/index.rst b/docs/website/source/index.rst deleted file mode 100644 index 16bbc299d736f1..00000000000000 --- a/docs/website/source/index.rst +++ /dev/null @@ -1,590 +0,0 @@ -Introduction to Apache Doris (incubating) -========================================= - -Apache Doris is an MPP-based interactive SQL data warehousing for -reporting and analysis. Doris mainly integrates the technology of Google -Mesa and Apache Impala. Unlike other popular SQL-on-Hadoop systems, -Doris is designed to be a simple and single tightly coupled system, not -depending on other systems. Doris not only provides high concurrent low -latency point query performance, but also provides high throughput -queries of ad-hoc analysis. Doris not only provides batch data loading, -but also provides near real-time mini-batch data loading. Doris also -provides high availability, reliability, fault tolerance, and -scalability. The simplicity (of developing, deploying and using) and -meeting many data serving requirements in single system are the main -features of Doris. - -1. Background -------------- - -In Baidu, the largest Chinese search engine, we run a two-tiered data -warehousing system for data processing, reporting and analysis. Similar -to lambda architecture, the whole data warehouse comprises data -processing and data serving. Data processing does the heavy lifting of -big data: cleaning data, merging and transforming it, analyzing it and -preparing it for use by end user queries; data serving is designed to -serve queries against that data for different use cases. Currently data -processing includes batch data processing and stream data processing -technology, like Hadoop, Spark and Storm; Doris is a SQL data warehouse -for serving online and interactive data reporting and analysis querying. - -Prior to Doris, different tools were deployed to solve diverse -requirements in many ways. For example, the advertising platform needs -to provide some detailed statistics associated with each served ad for -every advertiser. The platform must support continuous updates, both new -rows and incremental updates to existing rows within minutes. It must -support latency-sensitive users serving live customer reports with very -low latency requirements and batch ad-hoc multiple dimensions data -analysis requiring very high throughput. In the past,this platform was -built on top of sharded MySQL. But with the growth of data, MySQL cannot -meet the requirements. Then, based on our existing KV system, we -developed our own proprietary distributed statistical database. But, the -simple KV storage was not efficient on scan performance. Because the -system depends on many other systems, it is very complex to operate and -maintain. Using RPC API, more complex querying usually required code -programming, but users wants an MPP SQL engine. In addition to -advertising system, a large number of internal BI Reporting / Analysis, -also used a variety of tools. Some used the combination of SparkSQL / -Impala + HDFS / HBASE. Some used MySQL to store the results that were -prepared by distributed MapReduce computing. Some also bought commercial -databases to use. - -However, when a use case requires the simultaneous availability of -capabilities that cannot all be provided by a single tool, users were -forced to build hybrid architectures that stitch multiple tools -together. Users often choose to ingest and update data in one storage -system, but later reorganize this data to optimize for an analytical -reporting use-case served from another. Our users had been successfully -deploying and maintaining these hybrid architectures, but we believe -that they shouldn’t need to accept their inherent complexity. A storage -system built to provide great performance across a broad range of -workloads provides a more elegant solution to the problems that hybrid -architectures aim to solve. Doris is the solution. Doris is designed to -be a simple and single tightly coupled system, not depending on other -systems. Doris provides high concurrent low latency point query -performance, but also provides high throughput queries of ad-hoc -analysis. Doris provides bulk-batch data loading, but also provides near -real-time mini-batch data loading. Doris also provides high -availability, reliability, fault tolerance, and scalability. - -Generally speaking, Doris is the technology combination of Google Mesa -and Apache Impala. Mesa is a highly scalable analytic data storage -system that stores critical measurement data related to Google’s -Internet advertising business. Mesa is designed to satisfy complex and -challenging set of users’ and systems’ requirements, including near -real-time data ingestion and query ability, as well as high -availability, reliability, fault tolerance, and scalability for large -data and query volumes. Impala is a modern, open-source MPP SQL engine -architected from the ground up for the Hadoop data processing -environment. At present, by virtue of its superior performance and rich -functionality, Impala has been comparable to many commercial MPP -database query engine. Mesa can satisfy the needs of many of our storage -requirements, however Mesa itself does not provide a SQL query engine; -Impala is a very good MPP SQL query engine, but the lack of a perfect -distributed storage engine. So in the end we chose the combination of -these two technologies. - -Learning from Mesa’s data model, we developed a distributed storage -engine. Unlike Mesa, this storage engine does not rely on any -distributed file system. Then we deeply integrate this storage engine -with Impala query engine. Query compiling, query execution coordination -and catalog management of storage engine are integrated to be frontend -daemon; query execution and data storage are integrated to be backend -daemon. With this integration, we implemented a single, full-featured, -high performance state the art of MPP database, as well as maintaining -the simplicity. - -2. System Overview ------------------- - -Doris’ implementation consists of two daemons: frontend (FE) and backend -(BE). The following figures gives the overview of architecture and -usage. - -.. figure:: https://raw.githubusercontent.com/apache/incubator-doris/master/docs/resources/images/palo_architecture.jpg - :alt: Doris Architecture - - Doris Architecture - -Frontend daemon consists of query coordinator and catalog manager. Query -coordinator is responsible for receiving user’s sql queries, compiling -queries and managing queries execution. Catalog manager is responsible -for managing metadata such as databases, tables, partitions, replicas -and etc. Several frontend daemons could be deployed to guarantee -fault-tolerance, and load balancing. - -Backend daemon stores the data and executes the query fragments. Many -backend daemons could also be deployed to provide scalability and -fault-tolerance. - -A typical Doris cluster generally composes of several frontend daemons -and dozens to hundreds of backend daemons. - -Clients can use MySQL-related tools to connect any frontend daemon to -submit SQL query. The frontend receives the query and compiles it into -query plans executable by the backends. Then frontend sends the query -plan fragments to backend. Backends will build a query execution DAG. -Data is fetched and pipelined into the DAG. The final result response is -sent to client via frontend. The distribution of query fragment -execution takes minimizing data movement and maximizing scan locality as -the main goal. Because Doris is designed to provide interactive -analysis, so the average execution time of queries is short. Considering -this, we adopt query re-execution to meet the fault tolerance of query -execution. - -A table is splitted into many tablets. Tablets are managed by backends. -The backend daemon could be configured to use multiple directories. Any -directory’s IO failure doesn’t influence the normal running of backend -daemon. Doris will recover and rebalance the whole cluster automatically -when necessary. - -3. Frontend ------------ - -In-memory catalog, multiple frontends, MySQL networking protocol, -consistency guarantee, and two-level table partitioning are the main -features of Doris’ frontend design. - -3.1 In-Memory Catalog -~~~~~~~~~~~~~~~~~~~~~ - -Traditional data warehouse always uses a RDBMS database to store their -catalog metadata. In order to produce query execution plan, frontend -needs to look up the catalog metadata. This kind of catalog storage may -be enough for low concurrent ad-hoc analysis queries. But for online -high concurrent queries, its performance is very bad,resulting in -increased response latency. For example, Hive metadata query latency is -sometimes up to tens of seconds or even minutes. In order to speedup the -metadata access, we adopt the in-memory catalog storage. - -.. figure:: ./resources/images/log_replication.jpg - :alt: log replication - - log replication - -In-memory catalog storage has three functional modules: real-time memory -data structures, memory checkpoints on local disk and an operation relay -log. When modifying catalog, the mutation operation is written into the -log file firstly. Then, the mutation operation is applied into the -memory data structures. Periodically, a thread does the checkpoint that -dumps memory data structure image into local disk. Checkpoint mechanism -enables the fast startup of frontend and reduces the disk storage -occupancy. Actually, in-memory catalog also simplifies the -implementation of multiple frontends. - -3.2 Multiple Frontends -~~~~~~~~~~~~~~~~~~~~~~ - -Many data warehouses only support single frontend-like node. There are -some systems supporting master and slave deploying. But for online data -serving, high availability is an essential feature. Further, the number -of queries per seconds may be very large, so high scalability is also -needed. In Doris, we provide the feature of multiple frontends using -replicated-state-machine technology. - -Frontends can be configured to three kinds of roles: leader, follower -and observer. Through a voting protocol, follower frontends firstly -elect a leader frontend. All the write requests of metadata are -forwarded to the leader, then the leader writes the operation into the -replicated log file. If the new log entry will be replicated to at least -quorum followers successfully, the leader commits the operation into -memory, and responses the write request. Followers always replay the -replicated logs to apply them into their memory metadata. If the leader -crashes, a new leader will be elected from the leftover followers. -Leader and follower mainly solve the problem of write availability and -partly solve the problem of read scalability. - -Usually one leader frontend and several follower frontends can meet most -applications’ write availability and read scalability requirements. For -very high concurrent reading, continuing to increase the number of -followers is not a good practice. Leader replicates log stream to -followers synchronously, so adding more followers will increases write -latency. Like Zookeeper,we have introduced a new type of frontend node -called observer that helps addressing this problem and further improving -metadata read scalability. Leader replicates log stream to observers -asynchronously. Observers don’t involve leader election. - -The replicated-state-machine is implemented based on BerkeleyDB java -version (BDB-JE). BDB-JE has achieved high availability by implementing -a Paxos-like consensus algorithm. We use BDB-JE to implement Doris’ log -replication and leader election. - -3.3 Consistency Guarantee -~~~~~~~~~~~~~~~~~~~~~~~~~ - -If a client process connects to the leader, it will see up-to-date -metadata, so that strong consistency semantics is guaranteed. If the -client connects to followers or observers, it will see metadata lagging -a little behind of the leader, but the monotonic consistency is -guaranteed. In most Doris’ use cases, monotonic consistency is accepted. - -If the client always connects to the same frontend, monotonic -consistency semantics is obviously guaranteed; however if the client -connects to other frontends due to failover, the semantics may be -violated. Doris provides a SYNC command to guarantee metadata monotonic -consistency semantics during failover. When failover happens, the client -can send a SYNC command to the new connected frontend, who will get the -latest operation log number from the leader. The SYNC command will not -return to client as long as local applied log number is still less than -fetched operation log number. This mechanism can guarantee the metadata -on the connected frontend is newer than the client have seen during its -last connection. - -3.4 MySQL Networking Protocol -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -MySQL compatible networking protocol is implemented in Doris’ frontend. -Firstly, SQL interface is preferred for engineers; Secondly, -compatibility with MySQL protocol makes the integrating with current -existing BI software, such as Tableau, easier; Lastly, rich MySQL client -libraries and tools reduce our development costs, but also reduces the -user’s using cost. - -Through the SQL interface, administrator can adjust system -configuration, add and remove frontend nodes or backend nodes, and -create new database for user; user can create tables, load data, and -submit SQL query. - -Online help document and Linux Proc-like mechanism are also supported in -SQL. Users can submit queries to get the help of related SQL statements -or show Doris’ internal running state. - -In frontend, a small response buffer is allocated to every MySQL -connection. The maximum size of this buffer is limited to 1MB. The -buffer is responsible for buffering the query response data. Only if the -response is finished or the buffer size reaches the 1MB,the response -data will begin to be sent to client. Through this small trick, frontend -can re-execution most of queries if errors occurred during query -execution. - -3.5 Two-Level Partitioning -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Like most of the distributed database system, data in Doris is -horizontally partitioned. However, a single-level partitioning rule -(hash partitioning or range partitioning) may not be a good solution to -all scenarios. For example, there have a user-based fact table that -stores rows of the form (date, userid, metric). Choosing only hash -partitioning by column userid may lead to uneven distribution of data, -when one user’s data is very large. If choosing range partitioning -according to column date, it will also lead to uneven distribution of -data due to the likely data explosion in a certain period of time. - -Therefore we support the two-level partitioning rule. The first level is -range partitioning. User can specify a column (usually the time series -column) range of values for the data partition. In one partition, the -user can also specify one or more columns and a number of buckets to do -the hash partitioning. User can combine with different partitioning -rules to better divide the data. Figure 4 gives an example of two-level -partitioning. - -Three benefits are gained by using the two-level partitioning mechanism. -Firstly, old and new data could be separated, and stored on different -storage mediums; Secondly, storage engine of backend can reduce the -consumption of IO and CPU for unnecessary data merging, because the data -in some partitions is no longer be updated; Lastly,every partition’s -buckets number can be different and adjusted according to the change of -data size. - -.. code:: sql - - -- Create partitions using CREATE TABLE -- - CREATE TABLE example_tbl ( - `date` DATE, - userid BIGINT, - metric BIGINT SUM - ) PARTITION BY RANGE (`date`) ( - PARTITION p201601 VALUES LESS THAN ("2016-02-01"), - PARTITION p201602 VALUES LESS THAN ("2016-03-01"), - PARTITION p201603 VALUES LESS THAN ("2016-04-01"), - PARTITION p201604 VALUES LESS THAN ("2016-05-01") - ) DISTRIBUTED BY HASH(userid) BUCKETS 32; - - -- Add partition using ALTER TABLE -- - ALTER TABLE example_tbl ADD PARTITION p201605 VALUES LESS THAN ("2016-06-01"); - -4. Backend ----------- - -4.1 Data Storage Model -~~~~~~~~~~~~~~~~~~~~~~ - -Doris combines Google Mesa’s data model and ORCFile / Parquet storage -technology. - -Data in Mesa is inherently multi-dimensional fact table. These facts in -table typically consist of two types of attributes: dimensional -attributes (which we call keys) and measure attributes (which we call -values). The table schema also specifies the aggregation function F: V -×V → V which is used to aggregate the values corresponding to the same -key. To achieve high update throughput, Mesa loads data in batch. Each -batch of data will be converted to a delta file. Mesa uses MVCC approach -to manage these delta files, and so to enforce update atomicity. Mesa -also supports creating materialized rollups, which contain a column -subset of schema to gain better aggregation effect. - -Mesa’s data model performs well in many interactive data service, but it -also has some drawbacks: 1. Users have difficulty in understanding key -and value space, as well as aggregation function, especially when they -rarely have such aggregation demand in analysis query scenarios. - -2. In order to ensure the aggregation semantic, count operation on a - single column must read all columns in key space, resulting in a - large number of additional read overheads. There is also unable to - push down the predicates on the value column to storage engine, which - also leads to additional read overheads. - -3. Essentially, it is still a key-value model. In order to aggregate the - values corresponding to the same key, all key columns must store in - order. When a table contains hundreds of columns, sorting cost - becomes the bottleneck of ETL process. - -To solve these problems, we introduce ORCFile / Parquet technology -widely used in the open source community, such as MapReduce + ORCFile, -SparkSQL + Parquet, mainly used for ad-hoc analysis of large amounts of -data with low concurrency. These data does not distinguish between key -and value. In addition, compared with the row-oriented database, -column-oriented organization is more efficient when an aggregate needs -to be computed over many rows but only for a small subset of all columns -of data, because reading that smaller subset of data can be faster than -reading all data. And columnar storage is also space-friendly due to the -high compression ratio of each column. Further, column support -block-level storage technology such as min/max index and bloom filter -index. Query executor can filter out a lot of blocks that do not meet -the predicate, to further improve the query performance. However, due to -the underlying storage does not require data order, query time -complexity is linear corresponding to the data volume. - -Like traditional databases, Doris stores structured data represented as -tables. Each table has a well-defined schema consisting of a finite -number of columns. We combine Mesa data model and ORCFile/Parquet -technology to develop a distributed analytical database. User can create -two types of table to meet different needs in interactive query -scenarios. - -In non-aggregation type of table, columns are not distinguished between -dimensions and metrics, but should specify the sort columns in order to -sort all rows. Doris will sort the table data according to the sort -columns without any aggregation. The following figure gives an example -of creating non-aggregation table. - -.. code:: sql - - -- Create non-aggregation table -- - CREATE TABLE example_tbl ( - `date` DATE, - id BIGINT, - country VARCHAR(32), - click BIGINT, - cost BIGINT - ) DUPLICATE KEY(`date`, id, country) - DISTRIBUTED BY HASH(id) BUCKETS 32; - -In aggregation data analysis case, we reference Mesa’s data model, and -distinguish columns between key and value, and specify the value columns -with aggregation method, such as SUM, REPLACE, etc. In the following -figure, we create an aggregation table like the non-aggregation table, -including two SUM aggregation columns (clicks, cost). Different from the -non-aggregation table, data in the table needs to be sorted on all key -columns for delta compaction and value aggregation. - -.. code:: sql - - -- Create aggregation table -- - CREATE TABLE example_tbl ( - `date` DATE, - id BIGINT, - country VARCHAR(32), - click BIGINT SUM, - cost BIGINT SUM - ) DISTRIBUTED BY HASH(id) BUCKETS 32; - -Rollup is a materialized view that contains a column subset of schema in -Doris. A table may contain multiple rollups with columns in different -order. According to sort key index and column covering of the rollups, -Doris can select the best rollup for different query. Because most -rollups only contain a few columns, the size of aggregated data is -typically much smaller and query performance can greatly be improved. -All the rollups in the same table are updated atomically. Because -rollups are materialized, users should make a trade-off between query -latency and storage space when using them. - -To achieve high update throughput, Doris only applies updates in batches -at the smallest frequency of every minute. Each update batch specifies -an increased version number and generates a delta data file, commits the -version when updates of quorum replicas are complete. You can query all -committed data using the committed version, and the uncommitted version -would not be used in query. All update versions are strictly be in -increasing order. If an update contains more than one table, the -versions of these tables are committed atomically. The MVCC mechanism -allows Doris to guarantee multiple table atomic updates and query -consistency. In addition, Doris uses compaction policies to merge delta -files to reduce delta number, also reduce the cost of delta merging -during query for higher performance. - -Doris’ data file is stored by column. The rows are stored in sorted -order by the sort columns in delta data files, and are organized into -row blocks, each block is compressed by type-specific columnar -encodings, such as run-length encoding for integer columns, then stored -into separate streams. In order to improve the performance of queries -that have a specific key, we also store a sparse sort key index file -corresponding to each delta data file. An index entry contains the short -key for the row block, which is a fixed size prefix of the first sort -columns for the row block, and the block id in the data file. Index -files are usually directly loaded into memory, as they are very small. -The algorithm for querying a specific key includes two steps. First, use -a binary search on the sort key index to find blocks that may contain -the specific key, and then perform a binary search on the compressed -blocks in the data files to find the desired key. We also store -block-level min/max index into separate index streams, and queries can -use this to filter undesired blocks. In addition to those basic columnar -features, we also offers an optional block-level bloom filter index for -queries with IN or EQUAL conditions to further filter undesired blocks. -Bloom filter index is stored in a separate stream, and is loaded on -demand. - -4.2 Data Loading -~~~~~~~~~~~~~~~~ - -Doris applies updates in batches. Three types of data loading are -supported: Hadoop-batch loading, loading ,mini-batch loading. - -1. Hadoop-batch loading. When a large amount of data volume needs to be - loaded into Doris, the hadoop-batch loading is recommended to achieve - high loading throughput. The data batches themselves are produced by - an external Hadoop system, typically at a frequency of every few - minutes. Unlike traditional data warehouses that use their own - computing resource to do the heavy data preparation, Doris could use - Hadoop to prepare the data (shuffle, sort and aggregate, etc.). By - using this approach, the most time-consuming computations are handed - over to Hadoop to complete. This will not only improve computational - efficiency, but also reduce the performance pressure of Doris cluster - and ensure the stability of the query service. The stability of the - online data services is the most important point. - -2. Loading. After deploying the fs-brokers, you can use Doris’ query - engine to import data. This type of loading is recommended for - incremental data loading. - -3. Mini-batch loading. When a small amount of data needs to be loaded - into Doris, the mini-batch loading is recommended to achieve low - loading latency. By using http interface, raw data is pushed into a - backend. Then the backend does the data preparing computing and - completes the final loading. Http tools could connect frontend or - backend. If frontend is connected, it will redirect the request - randomly to a backend. - -All the loading work is handled asynchronously. When load request is -submitted, a label needs to be provided. By using the load label, users -can submit show load request to get the loading status or submit cancel -load request to cancel the loading. If the status of loading task is -successful or in progress, its load label is not allowed to reuse again. -The label of failed task is allowed to be reused. - -4.3 Resource Isolation -~~~~~~~~~~~~~~~~~~~~~~ - -1. Multi-tenancy Isolation：Multiple virtual cluster can be created in - one pysical Doris cluster. Every backend node can deploy multiple - backend processes. Every backend process only belongs to one virtual - cluster. Virtual cluster is one tenancy. - -2. User Isolation: There are many users in one virtual cluster. You can - allocate the resource among different users and ensure that all - users’ tasks are executed under limited resource quota. - -3. Priority Isolation: There are three priorities isolation group for - one user. User could control resource allocated to different tasks - submitted by themselves, for example user’s query task and loading - tasks require different resource quota. - -4.4 Multi-Medium Storage -~~~~~~~~~~~~~~~~~~~~~~~~ - -Most machines in modern datacenter are equipped with both SSDs and HDDs. -SSD has good random read capability that is the ideal medium for query -that needs a large number of random read operations. However, SSD’s -capacity is small and is very expensive, we could not deploy it at a -large scale. HDD is cheap and has huge capacity that is suitable to -store large scale data but with high read latency. In OLAP scenario, we -find user usually submit a lot of queries to query the latest data (hot -data) and expect low latency. User occasionally executes query on -historical data (cold data). This kind of query usually needs to scan -large scale of data and is high latency. Multi-Medium Storage allows -users to manage the storage medium of the data to meet different query -scenarios and reduce the latency. For example, user could put latest -data on SSD and historical data which is not used frequently on HDD, -user will get low latency when querying latest data while get high -latency when query historical data which is normal because it needs scan -large scale data. - -In the following figure, user alters partition ‘p201601’ storage_medium -to SSD and storage_cooldown_time to ‘2016-07-01 00:00:00’. The setting -means data in this partition will be put on SSD and it will start to -migrate to HDD after the time of storage_cooldown_time. - -.. code:: sql - - ALTER TABLE example_tbl MODIFY PARTITION p201601 - SET ("storage_medium" = "SSD", "storage_cooldown_time" = "2016-07-01 00:00:00"); - -4.5 Vectorized Query Execution -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Runtime code generation using LLVM is one of the techniques employed -extensively by Impala to improve query execution times. Performance -could gains of 5X or more are typical for representative workloads. - -But, runtime code generation is not suitable for low latency query, -because the generation overhead costs about 100ms. Runtime code -generation is more suitable for large-scale ad-hoc query. To accelerate -the small queries (of course, big queries will also obtain benefits), we -introduced vectorized query execution into Doris. - -Vectorized query execution is a feature that greatly reduces the CPU -usage for typical query operations like scans, filters, aggregates, and -joins. A standard query execution system processes one row at a time. -This involves long code paths and significant metadata interpretation in -the inner loop of execution. Vectorized query execution streamlines -operations by processing a block of many rows at a time. Within the -block, each column is stored as a vector (an array of a primitive data -type). Simple operations like arithmetic and comparisons are done by -quickly iterating through the vectors in a tight loop, with no or very -few function calls or conditional branches inside the loop. These loops -compile in a streamlined way that uses relatively few instructions and -finishes each instruction in fewer clock cycles, on average, by -effectively using the processor pipeline and cache memory. - -The result of benchmark shows 2x~4x speedup in our typical queries. - -5. Backup and Recovery ----------------------- - -Data backup function is provided to enhance data security. The minimum -granularity of backup and recovery is partition. Users can develop -plugins to backup data to any specified remote storage. The backup data -can always be recovered to Doris at all time, to achieve the data -rollback purpose. - -Currently we only support full data backup data rather than incremental -backups for the following reasons: - -1. Remote storage system is beyond the control of the Doris system. We - cannot guarantee whether the data has been changed between two backup - operations. And data verification operations always come at a high - price. - -2. We support data backup on partition granularity. And majority of - applications are time series applications. By dividing data using - time column, it has been able to meet the needs of the vast majority - of incremental backup in chronological order. - -In addition to improving data security, the backup function also -provides a way to export the data. Data can be exported to other -downstream systems for further processing. - -.. toctree:: - :hidden: - - documentation/cn/index - documentation/en/index diff --git a/docs/zh-CN/README.md b/docs/zh-CN/README.md new file mode 100644 index 00000000000000..4910446048731a --- /dev/null +++ b/docs/zh-CN/README.md @@ -0,0 +1,8 @@ +--- +home: true +heroImage: /images/doris-logo.png +heroText: Apache Doris +tagline: 基于 MPP 的交互式 SQL 数据仓库，主要用于解决报表和多维分析。 +actionText: 快速上手 → +actionLink: /zh-CN/installing/compilation +--- diff --git a/docs/zh-CN/administrator-guide/alter-table/alter-table-bitmap-index.md b/docs/zh-CN/administrator-guide/alter-table/alter-table-bitmap-index.md new file mode 100644 index 00000000000000..0a768a99716e80 --- /dev/null +++ b/docs/zh-CN/administrator-guide/alter-table/alter-table-bitmap-index.md @@ -0,0 +1,86 @@ +--- +{ + "title": "Bitmap 索引", + "language": "zh-CN" +} +--- + + + +# Bitmap 索引 +用户可以通过创建bitmap index 加速查询 +本文档主要介绍如何创建 index 作业，以及创建 index 的一些注意事项和常见问题。 + +## 名词解释 +* bitmap index：位图索引，是一种快速数据结构，能够加快查询速度 + +## 原理介绍 +创建和删除本质上是一个 schema change 的作业，具体细节可以参照 [Schema Change](alter-table-schema-change)。 + +## 语法 +index 创建和修改相关语法有两种形式，一种集成与 alter table 语句中，另一种是使用单独的 +create/drop index 语法 +1. 创建索引 + + 创建索引的的语法可以参见 [CREATE INDEX](../../sql-reference/sql-statements/Data%20Definition/CREATE%20INDEX.html) + 或 [ALTER TABLE](../../sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE.html) 中bitmap 索引相关的操作, + 也可以通过在创建表时指定bitmap 索引，参见[CREATE TABLE](../../sql-reference/sql-statements/Data%20Definition/CREATE%20TABLE.html) + +2. 查看索引 + + 参照[SHOW INDEX](../../sql-reference/sql-statements/Administration/SHOW%20INDEX.html) + +3. 删除索引 + + 参照[DROP INDEX](../../sql-reference/sql-statements/Data%20Definition/DROP%20INDEX.html) + 或者 [ALTER TABLE](../../sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE.html) 中bitmap 索引相关的操作 + +## 创建作业 +参照 schema change 文档 [Scheam Change](alter-table-schema-change.html) + +## 查看作业 +参照 schema change 文档 [Scheam Change](alter-table-schema-change.html) + +## 取消作业 +参照 schema change 文档 [Scheam Change](alter-table-schema-change.html) + +## 注意事项 +* 目前索引仅支持 bitmap 类型的索引。 +* bitmap 索引仅在单列上创建。 +* bitmap 索引能够应用在 `Duplicate` 数据模型的所有列和 `Aggregate`, `Uniq` 模型的key列上。 +* bitmap 索引支持的数据类型如下: + * `TINYINT` + * `SMALLINT` + * `INT` + * `UNSIGNEDINT` + * `BIGINT` + * `CHAR` + * `VARCHAE` + * `DATE` + * `DATETIME` + * `LARGEINT` + * `DECIMAL` + * `BOOL` + +* bitmap索引仅在 segmentV2 下生效，需要在be的配置文件中增加如下配置 + + ``` + default_rowset_type=BETA + ``` diff --git a/docs/zh-CN/administrator-guide/alter-table/alter-table-rollup.md b/docs/zh-CN/administrator-guide/alter-table/alter-table-rollup.md new file mode 100644 index 00000000000000..7d571a2d8a6998 --- /dev/null +++ b/docs/zh-CN/administrator-guide/alter-table/alter-table-rollup.md @@ -0,0 +1,194 @@ +--- +{ + "title": "Rollup", + "language": "zh-CN" +} +--- + + + +# Rollup + +用户可以通过创建上卷表（Rollup）加速查询。关于 Rollup 的概念和使用方式可以参阅 [数据模型、ROLLUP 及前缀索引](../../getting-started/data-model-rollup.md) 和 [Rollup 与查询](../../getting-started/hit-the-rollup.md) 两篇文档。 + +本文档主要介绍如何创建 Rollup 作业，以及创建 Rollup 的一些注意事项和常见问题。 + +## 名词解释 + +* Base Table：基表。每一个表被创建时，都对应一个基表。基表存储了这个表的完整的数据。Rollup 通常基于基表中的数据创建（也可以通过其他 Rollup 创建）。 +* Index：物化索引。Rollup 或 Base Table 都被称为物化索引。 +* Transaction：事务。每一个导入任务都是一个事务，每个事务有一个唯一递增的 Transaction ID。 + +## 原理介绍 + +创建 Rollup 的基本过程，是通过 Base 表的数据，生成一份新的包含指定列的 Rollup 的数据。其中主要需要进行两部分数据转换，一是已存在的历史数据的转换，二是在 Rollup 执行过程中，新到达的导入数据的转换。 + +``` ++----------+ +| Load Job | ++----+-----+ + | + | Load job generates both base and rollup index data + | + | +------------------+ +---------------+ + | | Base Index | | Base Index | + +------> New Incoming Data| | History Data | + | +------------------+ +------+--------+ + | | + | | Convert history data + | | + | +------------------+ +------v--------+ + | | Rollup Index | | Rollup Index | + +------> New Incoming Data| | History Data | + +------------------+ +---------------+ +``` + +在开始转换历史数据之前，Doris 会获取一个最新的 Transaction ID。并等待这个 Transaction ID 之前的所有导入事务完成。这个 Transaction ID 成为分水岭。意思是，Doris 保证在分水岭之后的所有导入任务，都会同时为 Rollup Index 生成数据。这样当历史数据转换完成后，可以保证 Rollup 和 Base 表的数据是齐平的。 + +## 创建作业 + +创建 Rollup 的具体语法可以查看帮助 `HELP ALTER TABLE` 中 Rollup 部分的说明。 + +Rollup 的创建是一个异步过程，作业提交成功后，用户需要通过 `SHOW ALTER TABLE ROLLUP` 命令来查看作业进度。 + +## 查看作业 + +`SHOW ALTER TABLE ROLLUP` 可以查看当前正在执行或已经完成的 Rollup 作业。举例如下： + +``` + JobId: 20037 + TableName: tbl1 + CreateTime: 2019-08-06 15:38:49 + FinishedTime: N/A + BaseIndexName: tbl1 +RollupIndexName: r1 + RollupId: 20038 + TransactionId: 10034 + State: PENDING + Msg: + Progress: N/A + Timeout: 86400 +``` + +* JobId：每个 Rollup 作业的唯一 ID。 +* TableName：Rollup 对应的基表的表名。 +* CreateTime：作业创建时间。 +* FinishedTime：作业结束时间。如未结束，则显示 "N/A"。 +* BaseIndexName：Rollup 对应的源 Index 的名称。 +* RollupIndexName：Rollup 的名称。 +* RollupId：Rollup 的唯一 ID。 +* TransactionId：转换历史数据的分水岭 transaction ID。 +* State：作业所在阶段。 + * PENDING：作业在队列中等待被调度。 + * WAITING_TXN：等待分水岭 transaction ID 之前的导入任务完成。 + * RUNNING：历史数据转换中。 + * FINISHED：作业成功。 + * CANCELLED：作业失败。 +* Msg：如果作业失败，这里会显示失败信息。 +* Progress：作业进度。只有在 RUNNING 状态才会显示进度。进度是以 M/N 的形式显示。其中 N 为 Rollup 的总副本数。M 为已完成历史数据转换的副本数。 +* Timeout：作业超时时间。单位秒。 + +## 取消作业 + +在作业状态不为 FINISHED 或 CANCELLED 的情况下，可以通过以下命令取消 Rollup 作业： + +`CANCEL ALTER TABLE ROLLUP FROM tbl_name;` + +## 注意事项 + +* 一张表在同一时间只能有一个 Rollup 作业在运行。且一个作业中只能创建一个 Rollup。 + +* Rollup 操作不阻塞导入和查询操作。 + +* 如果 DELETE 操作，where 条件中的某个 Key 列在某个 Rollup 中不存在，则不允许该 DELETE。 + + 如果某个 Key 列在某一 Rollup 中不存在，则 DELETE 操作无法对该 Rollup 进行数据删除，从而无法保证 Rollup 表和 Base 表的数据一致性。 + +* Rollup 的列必须存在于 Base 表中。 + + Rollup 的列永远是 Base 表列的子集。不能出现 Base 表中不存在的列。 + +* 如果 Rollup 中包含 REPLACE 聚合类型的列，则该 Rollup 必须包含所有 Key 列。 + + 假设 Base 表结构如下： + + ```(k1 INT, k2 INT, v1 INT REPLACE, v2 INT SUM)``` + + 如果需要创建的 Rollup 包含 `v1` 列，则必须包含 `k1`, `k2` 列。否则系统无法决定 `v1` 列在 Rollup 中的取值。 + + 注意，Unique 数据模型表中的所有 Value 列都是 REPLACE 聚合类型。 + +* DUPLICATE 数据模型表的 Rollup，可以指定 Rollup 的 DUPLICATE KEY。 + + DUPLICATE 数据模型表中的 DUPLICATE KEY 其实就是排序列。Rollup 可以指定自己的排序列，但排序列必须是 Rollup 列顺序的前缀。如果不指定，则系统会检查 Rollup 是否包含了 Base 表的所有排序列，如果没有包含，则会报错。举例： + + Base 表结构：`(k1 INT, k2 INT, k3 INT) DUPLICATE KEY(k1, k2)` + + 则 Rollup 可以为：`(k2 INT, k1 INT) DUPLICATE KEY(k2)` + +* Rollup 不需要包含 Base 表的分区列或分桶列。 + +## 常见问题 + +* 一个表可以创建多少 Rollup + + 一个表能够创建的 Rollup 个数理论上没有限制，但是过多的 Rollup 会影响导入性能。因为导入时，会同时给所有 Rollup 产生数据。同时 Rollup 会占用物理存储空间。通常一个表的 Rollup 数量在 10 个以内比较合适。 + +* Rollup 创建的速度 + + 目前 Rollup 创建速度按照最差效率估计约为 10MB/s。保守起见，用户可以根据这个速率来设置作业的超时时间。 + +* 提交作业报错 `Table xxx is not stable. ...` + + Rollup 只有在表数据完整且非均衡状态下才可以开始。如果表的某些数据分片副本不完整，或者某些副本正在进行均衡操作，则提交会被拒绝。 + + 数据分片副本是否完整，可以通过以下命令查看： + + ```ADMIN SHOW REPLICA STATUS FROM tbl WHERE STATUS != "OK";``` + + 如果有返回结果，则说明有副本有问题。通常系统会自动修复这些问题，用户也可以通过以下命令优先修复这个表： + + ```ADMIN REPAIR TABLE tbl1;``` + + 用户可以通过以下命令查看是否有正在运行的均衡任务： + + ```SHOW PROC "/cluster_balance/pending_tablets";``` + + 可以等待均衡任务完成，或者通过以下命令临时禁止均衡操作： + + ```ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");``` + +## 相关配置 + +### FE 配置 + +* `alter_table_timeout_second`：作业默认超时时间，86400 秒。 + +### BE 配置 + +* `alter_tablet_worker_count`：在 BE 端用于执行历史数据转换的线程数。默认为 3。如果希望加快 Rollup 作业的速度，可以适当调大这个参数后重启 BE。但过多的转换线程可能会导致 IO 压力增加，影响其他操作。该线程和 Schema Change 作业共用。 + + + + + + + + diff --git a/docs/zh-CN/administrator-guide/alter-table/alter-table-schema-change.md b/docs/zh-CN/administrator-guide/alter-table/alter-table-schema-change.md new file mode 100644 index 00000000000000..6ab14cc7642c92 --- /dev/null +++ b/docs/zh-CN/administrator-guide/alter-table/alter-table-schema-change.md @@ -0,0 +1,249 @@ +--- +{ + "title": "Schema Change", + "language": "zh-CN" +} +--- + + + +# Schema Change + +用户可以通过 Schema Change 操作来修改已存在表的 Schema。目前 Doris 支持以下几种修改: + +* 增加、删除列 +* 修改列类型 +* 调整列顺序 +* 增加、修改 Bloom Filter +* 增加、删除 bitmap index + +本文档主要介绍如何创建 Schema Change 作业，以及进行 Schema Change 的一些注意事项和常见问题。 + +## 名词解释 + +* Base Table：基表。每一个表被创建时，都对应一个基表。 +* Rollup：基于基表或者其他 Rollup 创建出来的上卷表。 +* Index：物化索引。Rollup 或 Base Table 都被称为物化索引。 +* Transaction：事务。每一个导入任务都是一个事务，每个事务有一个唯一递增的 Transaction ID。 + +## 原理介绍 + +执行 Schema Change 的基本过程，是通过原 Index 的数据，生成一份新 Schema 的 Index 的数据。其中主要需要进行两部分数据转换，一是已存在的历史数据的转换，二是在 Schema Change 执行过程中，新到达的导入数据的转换。 + +``` ++----------+ +| Load Job | ++----+-----+ + | + | Load job generates both origin and new index data + | + | +------------------+ +---------------+ + | | Origin Index | | Origin Index | + +------> New Incoming Data| | History Data | + | +------------------+ +------+--------+ + | | + | | Convert history data + | | + | +------------------+ +------v--------+ + | | New Index | | New Index | + +------> New Incoming Data| | History Data | + +------------------+ +---------------+ +``` + +在开始转换历史数据之前，Doris 会获取一个最新的 Transaction ID。并等待这个 Transaction ID 之前的所有导入事务完成。这个 Transaction ID 成为分水岭。意思是，Doris 保证在分水岭之后的所有导入任务，都会同时为原 Index 和新 Index 生成数据。这样当历史数据转换完成后，可以保证新的 Index 中的数据是完整的。 + +## 创建作业 + +创建 Schema Change 的具体语法可以查看帮助 `HELP ALTER TABLE` 中 Schema Change 部分的说明。 + +Schema Change 的创建是一个异步过程，作业提交成功后，用户需要通过 `SHOW ALTER TABLE COLUMN` 命令来查看作业进度。 + +## 查看作业 + +`SHOW ALTER TABLE COLUMN` 可以查看当前正在执行或已经完成的 Schema Change 作业。当一次 Schema Change 作业涉及到多个 Index 时，该命令会显示多行，每行对应一个 Index。举例如下： + +``` + JobId: 20021 + TableName: tbl1 + CreateTime: 2019-08-05 23:03:13 + FinishTime: 2019-08-05 23:03:42 + IndexName: tbl1 + IndexId: 20022 +OriginIndexId: 20017 +SchemaVersion: 2:792557838 +TransactionId: 10023 + State: FINISHED + Msg: + Progress: N/A + Timeout: 86400 +``` + +* JobId：每个 Schema Change 作业的唯一 ID。 +* TableName：Schema Change 对应的基表的表名。 +* CreateTime：作业创建时间。 +* FinishedTime：作业结束时间。如未结束，则显示 "N/A"。 +* IndexName： 本次修改所涉及的某一个 Index 的名称。 +* IndexId：新的 Index 的唯一 ID。 +* OriginIndexId：旧的 Index 的唯一 ID。 +* SchemaVersion：以 M:N 的格式展示。其中 M 表示本次 Schema Change 变更的版本，N 表示对应的 Hash 值。每次 Schema Change，版本都会递增。 +* TransactionId：转换历史数据的分水岭 transaction ID。 +* State：作业所在阶段。 + * PENDING：作业在队列中等待被调度。 + * WAITING_TXN：等待分水岭 transaction ID 之前的导入任务完成。 + * RUNNING：历史数据转换中。 + * FINISHED：作业成功。 + * CANCELLED：作业失败。 +* Msg：如果作业失败，这里会显示失败信息。 +* Progress：作业进度。只有在 RUNNING 状态才会显示进度。进度是以 M/N 的形式显示。其中 N 为 Schema Change 涉及的总副本数。M 为已完成历史数据转换的副本数。 +* Timeout：作业超时时间。单位秒。 + +## 取消作业 + +在作业状态不为 FINISHED 或 CANCELLED 的情况下，可以通过以下命令取消 Schema Change 作业： + +`CANCEL ALTER TABLE COLUMN FROM tbl_name;` + +## 最佳实践 + +Schema Change 可以在一个作业中，对多个 Index 进行不同的修改。举例如下： + +源 Schema： + +``` ++-----------+-------+------+------+------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++-----------+-------+------+------+------+---------+-------+ +| tbl1 | k1 | INT | No | true | N/A | | +| | k2 | INT | No | true | N/A | | +| | k3 | INT | No | true | N/A | | +| | | | | | | | +| rollup2 | k2 | INT | No | true | N/A | | +| | | | | | | | +| rollup1 | k1 | INT | No | true | N/A | | +| | k2 | INT | No | true | N/A | | ++-----------+-------+------+------+------+---------+-------+ +``` + +可以通过以下命令给 rollup1 和 rollup2 都加入一列 k4，并且再给 rollup2 加入一列 k5： + +``` +ALTER TABLE tbl1 +ADD COLUMN k4 INT default "1" to rollup1, +ADD COLUMN k4 INT default "1" to rollup2, +ADD COLUMN k5 INT default "1" to rollup2; +``` + +完成后，Schema 变为： + +``` ++-----------+-------+------+------+------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++-----------+-------+------+------+------+---------+-------+ +| tbl1 | k1 | INT | No | true | N/A | | +| | k2 | INT | No | true | N/A | | +| | k3 | INT | No | true | N/A | | +| | k4 | INT | No | true | 1 | | +| | k5 | INT | No | true | 1 | | +| | | | | | | | +| rollup2 | k2 | INT | No | true | N/A | | +| | k4 | INT | No | true | 1 | | +| | k5 | INT | No | true | 1 | | +| | | | | | | | +| rollup1 | k1 | INT | No | true | N/A | | +| | k2 | INT | No | true | N/A | | +| | k4 | INT | No | true | 1 | | ++-----------+-------+------+------+------+---------+-------+ +``` + +可以看到，Base 表 tbl1 也自动加入了 k4, k5 列。即给任意 rollup 增加的列，都会自动加入到 Base 表中。 + +同时，不允许向 Rollup 中加入 Base 表已经存在的列。如果用户需要这样做，可以重新建立一个包含新增列的 Rollup，之后再删除原 Rollup。 + +## 注意事项 + +* 一张表在同一时间只能有一个 Schema Change 作业在运行。 + +* Schema Change 操作不阻塞导入和查询操作。 + +* 分区列和分桶列不能修改。 + +* 如果 Schema 中有 REPLACE 方式聚合的 value 列，则不允许删除 Key 列。 + + 如果删除 Key 列，Doris 无法决定 REPLACE 列的取值。 + + Unique 数据模型表的所有非 Key 列都是 REPLACE 聚合方式。 + +* 在新增聚合类型为 SUM 或者 REPLACE 的 value 列时，该列的默认值对历史数据没有含义。 + + 因为历史数据已经失去明细信息，所以默认值的取值并不能实际反映聚合后的取值。 + +* 当修改列类型时，除 Type 以外的字段都需要按原列上的信息补全。 + + 如修改列 `k1 INT SUM NULL DEFAULT "1"` 类型为 BIGINT，则需执行命令如下： + + ```ALTER TABLE tbl1 MODIFY COLUMN `k1` BIGINT SUM NULL DEFAULT "1";``` + + 注意，除新的列类型外，如聚合方式，Nullable 属性，以及默认值都要按照原信息补全。 + +* 不支持修改列名称、聚合类型、Nullable 属性、默认值以及列注释。 + +## 常见问题 + +* Schema Change 的执行速度 + + 目前 Schema Change 执行速度按照最差效率估计约为 10MB/s。保守起见，用户可以根据这个速率来设置作业的超时时间。 + +* 提交作业报错 `Table xxx is not stable. ...` + + Schema Change 只有在表数据完整且非均衡状态下才可以开始。如果表的某些数据分片副本不完整，或者某些副本正在进行均衡操作，则提交会被拒绝。 + + 数据分片副本是否完整，可以通过以下命令查看： + + ```ADMIN SHOW REPLICA STATUS FROM tbl WHERE STATUS != "OK";``` + + 如果有返回结果，则说明有副本有问题。通常系统会自动修复这些问题，用户也可以通过以下命令优先修复这个表： + + ```ADMIN REPAIR TABLE tbl1;``` + + 用户可以通过以下命令查看是否有正在运行的均衡任务： + + ```SHOW PROC "/cluster_balance/pending_tablets";``` + + 可以等待均衡任务完成，或者通过以下命令临时禁止均衡操作： + + ```ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");``` + +## 相关配置 + +### FE 配置 + +* `alter_table_timeout_second`：作业默认超时时间，86400 秒。 + +### BE 配置 + +* `alter_tablet_worker_count`：在 BE 端用于执行历史数据转换的线程数。默认为 3。如果希望加快 Schema Change 作业的速度，可以适当调大这个参数后重启 BE。但过多的转换线程可能会导致 IO 压力增加，影响其他操作。该线程和 Rollup 作业共用。 + + + + + + + + diff --git a/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md b/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md new file mode 100644 index 00000000000000..32b328f6a4e407 --- /dev/null +++ b/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md @@ -0,0 +1,251 @@ +--- +{ + "title": "临时分区", + "language": "zh-CN" +} +--- + + + +# 临时分区 + +在 0.12 版本中，Doris 支持了临时分区功能。 + +临时分区是归属于某一分区表的。只有分区表可以创建临时分区。 + +## 规则 + +* 临时分区的分区列和正式分区相同，且不可修改。 +* 一张表所有临时分区之间的分区范围不可重叠，但临时分区的范围和正式分区范围可以重叠。 +* 临时分区的分区名称不能和正式分区以及其他临时分区重复。 + +## 支持的操作 + +临时分区支持添加、删除、替换操作。 + +### 添加临时分区 + +可以通过 `ALTER TABLE ADD TEMPORARY PARTITION` 语句对一个表添加临时分区： + +``` +ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN("2020-02-01"); + +ALTER TABLE tbl2 ADD TEMPORARY PARTITION tp1 VALUES [("2020-01-01"), ("2020-02-01")); + +ALTER TABLE tbl1 ADD TEMPORARY PARTITION tp1 VALUES LESS THAN("2020-02-01") +("in_memory" = "true", "replication_num" = "1") +DISTRIBUTED BY HASH(k1) BUCKETS 5; +``` + +通过 `HELP ALTER TABLE;` 查看更多帮助和示例。 + +添加操作的一些说明： + +* 临时分区的添加和正式分区的添加操作相似。临时分区的分区范围独立于正式分区。 +* 临时分区可以独立指定一些属性。包括分桶数、副本数、是否是内存表、存储介质等信息。 + +### 删除临时分区 + +可以通过 `ALTER TABLE DROP TEMPORARY PARTITION` 语句删除一个表的临时分区： + +``` +ALTER TABLE tbl1 DROP TEMPORARY PARTITION tp1; +``` + +通过 `HELP ALTER TABLE;` 查看更多帮助和示例。 + +删除操作的一些说明： + +* 删除临时分区，不影响正式分区的数据。 + +### 替换分区 + +可以通过 `ALTER TABLE REPLACE PARTITION` 语句将一个表的正式分区替换为临时分区。 + +``` +ALTER TABLE tbl1 REPLACE PARTITION (p1) WITH TEMPORARY PARTITION (tp1); + +ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1, tp2, tp3); + +ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1, tp2) +PROPERTIES ( + "strict_range" = "false", + "use_temp_partition_name" = "true" +); +``` + +通过 `HELP ALTER TABLE;` 查看更多帮助和示例。 + +替换操作有两个特殊的可选参数： + +1. `strict_range` + + 默认为 true。当该参数为 true 时，表示要被替换的所有正式分区的范围并集需要和替换的临时分区的范围并集完全相同。当置为 false 时，只需要保证替换后，新的正式分区间的范围不重叠即可。下面举例说明： + + * 示例1 + + 待替换的分区 p1, p2, p3 的范围 (=> 并集)： + + ``` + [10, 20), [20, 30), [40, 50) => [10, 30), [40, 50) + ``` + + 替换分区 tp1, tp2 的范围(=> 并集)： + + ``` + [10, 30), [40, 45), [45, 50) => [10, 30), [40, 50) + ``` + + 范围并集相同，则可以使用 tp1 和 tp2 替换 p1, p2, p3。 + + * 示例2 + + 待替换的分区 p1 的范围 (=> 并集)： + + ``` + [10, 50) => [10, 50) + ``` + + 替换分区 tp1, tp2 的范围(=> 并集)： + + ``` + [10, 30), [40, 50) => [10, 30), [40, 50) + ``` + + 范围并集不相同，如果 `strict_range` 为 true，则不可以使用 tp1 和 tp2 替换 p1。如果为 false，且替换后的两个分区范围 `[10, 30), [40, 50)` 和其他正式分区不重叠，则可以替换。 + +2. `use_temp_partition_name` + + 默认为 false。当该参数为 false，并且待替换的分区和替换分区的个数相同时，则替换后的正式分区名称维持不变。如果为 true，则替换后，正式分区的名称为替换分区的名称。下面举例说明： + + * 示例1 + + ``` + ALTER TABLE tbl1 REPLACE PARTITION (p1) WITH TEMPORARY PARTITION (tp1); + ``` + + `use_temp_partition_name` 默认为 false，则在替换后，分区的名称依然为 p1，但是相关的数据和属性都替换为 tp1 的。 + + 如果 `use_temp_partition_name` 默认为 true，则在替换后，分区的名称为 tp1。p1 分区不再存在。 + + * 示例2 + + ``` + ALTER TABLE tbl1 REPLACE PARTITION (p1, p2) WITH TEMPORARY PARTITION (tp1); + ``` + + `use_temp_partition_name` 默认为 false，但因为待替换分区的个数和替换分区的个数不同，则该参数无效。替换后，分区名称为 tp1，p1 和 p2 不再存在。 + +替换操作的一些说明： + +* 分区替换成功后，被替换的分区将被删除且不可恢复。 + +## 临时分区的导入和查询 + +用户可以将数据导入到临时分区，也可以指定临时分区进行查询。 + +1. 导入临时分区 + + 根据导入方式的不同，指定导入临时分区的语法稍有差别。这里通过示例进行简单说明 + + ``` + INSERT INTO tbl TEMPORARY PARTITION(tp1, tp2, ...) SELECT .... + ``` + + ``` + curl --location-trusted -u root: -H "label:123" -H "temporary_partitions: tp1, tp2, ..." -T testData http://host:port/api/testDb/testTbl/_stream_load + ``` + + ``` + LOAD LABEL example_db.label1 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + TEMPORARY PARTITION (tp1, tp2, ...) + ... + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + ``` + + ``` + CREATE ROUTINE LOAD example_db.test1 ON example_tbl + COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), + TEMPORARY PARTITIONS(tp1, tp2, ...), + WHERE k1 > 100 + PROPERTIES + (...) + FROM KAFKA + (...); + ``` + +2. 查询临时分区 + + ``` + SELECT ... FROM + tbl1 TEMPORARY PARTITION(tp1, tp2, ...) + JOIN + tbl2 TEMPORARY PARTITION(tp1, tp2, ...) + ON ... + WHERE ...; + ``` + +## 和其他操作的关系 + +### DROP + +* 使用 Drop 操作直接删除数据库或表后，可以通过 Recover 命令恢复数据库或表（限定时间内），但临时分区不会被恢复。 +* 使用 Alter 命令删除正式分区后，可以通过 Recover 命令恢复分区（限定时间内）。操作正式分区和临时分区无关。 +* 使用 Alter 命令删除临时分区后，无法通过 Recover 命令恢复临时分区。 + +### TRUNCATE + +* 使用 Truncate 命令清空表，表的临时分区会被删除，且不可恢复。 +* 使用 Truncate 命令清空正式分区时，不影响临时分区。 +* 不可使用 Truncate 命令清空临时分区。 + +### ALTER + +* 当表存在临时分区时，无法使用 Alter 命令对表进行 Schema Change、Rollup 等变更操作。 +* 当表在进行变更操作时，无法对表添加临时分区。 + + +## 最佳实践 + +1. 原子的覆盖写操作 + + 某些情况下，用户希望能够重写某一分区的数据，但如果采用先删除再导入的方式进行，在中间会有一段时间无法查看数据。这是，用户可以先创建一个对应的临时分区，将新的数据导入到临时分区后，通过替换操作，原子的替换原有分区，以达到目的。 + +2. 修改分桶数 + + 某些情况下，用户在创建分区时使用了不合适的分桶数。则用户可以先创建一个对应分区范围的临时分区，并指定新的分桶数。然后通过 `INSERT INTO` 命令将正式分区的数据导入到临时分区中，通过替换操作，原子的替换原有分区，以达到目的。 + +3. 合并或分割分区 + + 某些情况下，用户希望对分区的范围进行修改，比如合并两个分区，或将一个大分区分割成多个小分区。则用户可以先建立对应合并或分割后范围的临时分区，然后通过 `INSERT INTO` 命令将正式分区的数据导入到临时分区中，通过替换操作，原子的替换原有分区，以达到目的。 + + + + + + + + + + diff --git a/docs/zh-CN/administrator-guide/backup-restore.md b/docs/zh-CN/administrator-guide/backup-restore.md new file mode 100644 index 00000000000000..2102e772dbc76f --- /dev/null +++ b/docs/zh-CN/administrator-guide/backup-restore.md @@ -0,0 +1,186 @@ +--- +{ + "title": "备份与恢复", + "language": "zh-CN" +} +--- + + + +# 备份与恢复 + +Doris 支持将当前数据以文件的形式，通过 broker 备份到远端存储系统中。之后可以通过 恢复 命令，从远端存储系统中将数据恢复到任意 Doris 集群。通过这个功能，Doris 可以支持将数据定期的进行快照备份。也可以通过这个功能，在不同集群间进行数据迁移。 + +该功能需要 Doris 版本 0.8.2+ + +使用该功能，需要部署对应远端存储的 broker。如 BOS、HDFS 等。可以通过 `SHOW BROKER;` 查看当前部署的 broker。 + +## 简要原理说明 + +### 备份（Backup） + +备份操作是将指定表或分区的数据，直接以 Doris 存储的文件的形式，上传到远端仓库中进行存储。当用户提交 Backup 请求后，系统内部会做如下操作： + +1. 快照及快照上传 + + 快照阶段会对指定的表或分区数据文件进行快照。之后，备份都是对快照进行操作。在快照之后，对表进行的更改、导入等操作都不再影响备份的结果。快照只是对当前数据文件产生一个硬链，耗时很少。快照完成后，会开始对这些快照文件进行逐一上传。快照上传由各个 Backend 并发完成。 + +2. 元数据准备及上传 + + 数据文件快照上传完成后，Frontend 会首先将对应元数据写成本地文件，然后通过 broker 将本地元数据文件上传到远端仓库。完成最终备份作业。 + +### 恢复（Restore） + +恢复操作需要指定一个远端仓库中已存在的备份，然后将这个备份的内容恢复到本地集群中。当用户提交 Restore 请求后，系统内部会做如下操作： + +1. 在本地创建对应的元数据 + + 这一步首先会在本地集群中，创建恢复对应的表分区等结构。创建完成后，该表可见，但是不可访问。 + +2. 本地snapshot + + 这一步是将上一步创建的表做一个快照。这其实是一个空快照（因为刚创建的表是没有数据的），其目的主要是在 Backend 上产生对应的快照目录，用于之后接收从远端仓库下载的快照文件。 + +3. 下载快照 + + 远端仓库中的快照文件，会被下载到对应的上一步生成的快照目录中。这一步由各个 Backend 并发完成。 + +4. 生效快照 + + 快照下载完成后，我们要将各个快照映射为当前本地表的元数据。然后重新加载这些快照，使之生效，完成最终的恢复作业。 + +## 最佳实践 + +### 备份 + +当前我们支持最小分区（Partition）粒度的全量备份（增量备份有可能在未来版本支持）。如果需要对数据进行定期备份，首先需要在建表时，合理的规划表的分区及分桶，比如按时间进行分区。然后在之后的运行过程中，按照分区粒度进行定期的数据备份。 + +### 数据迁移 + +用户可以先将数据备份到远端仓库，再通过远端仓库将数据恢复到另一个集群，完成数据迁移。因为数据备份是通过快照的形式完成的，所以，在备份作业的快照阶段之后的新的导入数据，是不会备份的。因此，在快照完成后，到恢复作业完成这期间，在原集群上导入的数据，都需要在新集群上同样导入一遍。 + +建议在迁移完成后，对新旧两个集群并行导入一段时间。完成数据和业务正确性校验后，再将业务迁移到新的集群。 + +## 重点说明 + +1. 备份恢复相关的操作目前只允许拥有 ADMIN 权限的用户执行。 +2. 一个 Database 内，只允许有一个正在执行的备份或恢复作业。 +3. 备份和恢复都支持最小分区（Partition）级别的操作，当表的数据量很大时，建议按分区分别执行，以降低失败重试的代价。 +4. 因为备份恢复操作，操作的都是实际的数据文件。所以当一个表的分片过多，或者一个分片有过多的小版本时，可能即使总数据量很小，依然需要备份或恢复很长时间。用户可以通过 `SHOW PARTITIONS FROM table_name;` 和 `SHOW TABLET FROM table_name;` 来查看各个分区的分片数量，以及各个分片的文件版本数量，来预估作业执行时间。文件数量对作业执行的时间影响非常大，所以建议在建表时，合理规划分区分桶，以避免过多的分片。 +5. 当通过 `SHOW BACKUP` 或者 `SHOW RESTORE` 命令查看作业状态时。有可能会在 `TaskErrMsg` 一列中看到错误信息。但只要 `State` 列不为 + `CANCELLED`，则说明作业依然在继续。这些 Task 有可能会重试成功。当然，有些 Task 错误，也会直接导致作业失败。 +6. 如果恢复作业是一次覆盖操作（指定恢复数据到已经存在的表或分区中），那么从恢复作业的 `COMMIT` 阶段开始，当前集群上被覆盖的数据有可能不能再被还原。此时如果恢复作业失败或被取消，有可能造成之前的数据已损坏且无法访问。这种情况下，只能通过再次执行恢复操作，并等待作业完成。因此，我们建议，如无必要，尽量不要使用覆盖的方式恢复数据，除非确认当前数据已不再使用。 + +## 相关命令 + +和备份恢复功能相关的命令如下。以下命令，都可以通过 mysql-client 连接 Doris 后，使用 `help cmd;` 的方式查看详细帮助。 + +1. CREATE REPOSITORY + + 创建一个远端仓库路径，用于备份或恢复。该命令需要借助 Broker 进程访问远端存储，不同的 Broker 需要提供不同的参数，具体请参阅 [Broker文档](broker.md) + +1. BACKUP + + 执行一次备份操作。 + +3. SHOW BACKUP + + 查看最近一次 backup 作业的执行情况，包括： + + * JobId：本次备份作业的 id。 + * SnapshotName：用户指定的本次备份作业的名称（Label）。 + * DbName：备份作业对应的 Database。 + * State：备份作业当前所在阶段： + * PENDING：作业初始状态。 + * SNAPSHOTING：正在进行快照操作。 + * UPLOAD_SNAPSHOT：快照结束，准备上传。 + * UPLOADING：正在上传快照。 + * SAVE_META：正在本地生成元数据文件。 + * UPLOAD_INFO：上传元数据文件和本次备份作业的信息。 + * FINISHED：备份完成。 + * CANCELLED：备份失败或被取消。 + * BackupObjs：本次备份涉及的表和分区的清单。 + * CreateTime：作业创建时间。 + * SnapshotFinishedTime：快照完成时间。 + * UploadFinishedTime：快照上传完成时间。 + * FinishedTime：本次作业完成时间。 + * UnfinishedTasks：在 `SNAPSHOTTING`，`UPLOADING` 等阶段，会有多个子任务在同时进行，这里展示的当前阶段，未完成的子任务的 task id。 + * TaskErrMsg：如果有子任务执行出错，这里会显示对应子任务的错误信息。 + * Status：用于记录在整个作业过程中，可能出现的一些状态信息。 + * Timeout：作业的超时时间，单位是秒。 + +4. SHOW SNAPSHOT + + 查看远端仓库中已存在的备份。 + + * Snapshot：备份时指定的该备份的名称（Label）。 + * Timestamp：备份的时间戳。 + * Status：该备份是否正常。 + + 如果在 `SHOW SNAPSHOT` 后指定了 where 子句，则可以显示更详细的备份信息。 + + * Database：备份时对应的 Database。 + * Details：展示了该备份完整的数据目录结构。 + +5. RESTORE + + 执行一次恢复操作。 + +6. SHOW RESTORE + + 查看最近一次 restore 作业的执行情况，包括： + + * JobId：本次恢复作业的 id。 + * Label：用户指定的仓库中备份的名称（Label）。 + * Timestamp：用户指定的仓库中备份的时间戳。 + * DbName：恢复作业对应的 Database。 + * State：恢复作业当前所在阶段： + * PENDING：作业初始状态。 + * SNAPSHOTING：正在进行本地新建表的快照操作。 + * DOWNLOAD：正在发送下载快照任务。 + * DOWNLOADING：快照正在下载。 + * COMMIT：准备生效已下载的快照。 + * COMMITTING：正在生效已下载的快照。 + * FINISHED：恢复完成。 + * CANCELLED：恢复失败或被取消。 + * AllowLoad：恢复期间是否允许导入。 + * ReplicationNum：恢复指定的副本数。 + * RestoreObjs：本次恢复涉及的表和分区的清单。 + * CreateTime：作业创建时间。 + * MetaPreparedTime：本地元数据生成完成时间。 + * SnapshotFinishedTime：本地快照完成时间。 + * DownloadFinishedTime：远端快照下载完成时间。 + * FinishedTime：本次作业完成时间。 + * UnfinishedTasks：在 `SNAPSHOTTING`，`DOWNLOADING`, `COMMITTING` 等阶段，会有多个子任务在同时进行，这里展示的当前阶段，未完成的子任务的 task id。 + * TaskErrMsg：如果有子任务执行出错，这里会显示对应子任务的错误信息。 + * Status：用于记录在整个作业过程中，可能出现的一些状态信息。 + * Timeout：作业的超时时间，单位是秒。 + +7. CANCEL BACKUP + + 取消当前正在执行的备份作业。 + +8. CANCEL RESTORE + + 取消当前正在执行的恢复作业。 + +9. DROP REPOSITORY + + 删除已创建的远端仓库。删除仓库，仅仅是删除该仓库在 Doris 中的映射，不会删除实际的仓库数据。 \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/broker.md b/docs/zh-CN/administrator-guide/broker.md new file mode 100644 index 00000000000000..6215a9d1d60da9 --- /dev/null +++ b/docs/zh-CN/administrator-guide/broker.md @@ -0,0 +1,287 @@ +--- +{ + "title": "Broker", + "language": "zh-CN" +} +--- + + + +# Broker + +Broker 是 Doris 集群中一种可选进程，主要用于支持 Doris 读写远端存储上的文件和目录，如 HDFS、BOS 和 AFS 等。 + +Broker 通过提供一个 RPC 服务端口来提供服务，是一个无状态的 Java 进程，负责为远端存储的读写操作封装一些类 POSIX 的文件操作，如 open，pread，pwrite 等等。除此之外，Broker 不记录任何其他信息，所以包括远端存储的连接信息、文件信息、权限信息等等，都需要通过参数在 RPC 调用中传递给 Broker 进程，才能使得 Broker 能够正确读写文件。 + +Broker 仅作为一个数据通路，并不参与任何计算，因此仅需占用较少的内存。通常一个 Doris 系统中会部署一个或多个 Broker 进程。并且相同类型的 Broker 会组成一个组，并设定一个 **名称（Broker name）**。 + +Broker 在 Doris 系统架构中的位置如下： + +``` ++----+ +----+ +| FE | | BE | ++-^--+ +--^-+ + | | + | | ++-v---------v-+ +| Broker | ++------^------+ + | + | ++------v------+ +|HDFS/BOS/AFS | ++-------------+ +``` + +本文档主要介绍 Broker 在访问不同远端存储时需要的参数，如连接信息、权限认证信息等等。 + +## 支持的存储系统 + +不同的 Broker 类型支持不同的存储系统。 + +1. 社区版 HDFS + + * 支持简单认证访问 + * 支持通过 kerberos 认证访问 + * 支持 HDFS HA 模式访问 + +2. 百度 HDFS/AFS（开源版本不支持） + + * 支持通过 ugi 简单认证访问 + +3. 百度对象存储 BOS（开源版本不支持） + + * 支持通过 AK/SK 认证访问 + +## 需要 Broker 的操作 + +1. Broker Load + + Broker Load 功能通过 Broker 进程读取远端存储上的文件数据并导入到 Doris 中。示例如下： + + ``` + LOAD LABEL example_db.label6 + ( + DATA INFILE("bos://my_bucket/input/file") + INTO TABLE `my_table` + ) + WITH BROKER "broker_name" + ( + "bos_endpoint" = "http://bj.bcebos.com", + "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "bos_secret_accesskey" = "yyyyyyyyyyyyyyyyyyyy" + ) + ``` + + 其中 `WITH BROKER` 以及之后的 Property Map 用于提供 Broker 相关信息。 + +2. 数据导出（Export） + + Export 功能通过 Broker 进程，将 Doris 中存储的数据以文本的格式导出到远端存储的文件中。示例如下： + + ``` + EXPORT TABLE testTbl + TO "hdfs://hdfs_host:port/a/b/c" + WITH BROKER "broker_name" + ( + "username" = "xxx", + "password" = "yyy" + ); + ``` + + 其中 `WITH BROKER` 以及之后的 Property Map 用于提供 Broker 相关信息。 + +3. 创建用于备份恢复的仓库（Create Repository） + + 当用户需要使用备份恢复功能时，需要先通过 `CREATE REPOSITORY` 命令创建一个 “仓库”，仓库元信息中记录了所使用的 Broker 以及相关信息。之后的备份恢复操作，会通过 Broker 将数据备份到这个仓库，或从这个仓库读取数据恢复到 Doris 中。示例如下： + + ``` + CREATE REPOSITORY `bos_repo` + WITH BROKER `broker_name` + ON LOCATION "bos://doris_backup" + PROPERTIES + ( + "bos_endpoint" = "http://gz.bcebos.com", + "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", + "bos_secret_accesskey" = "70999999999999de274d59eaa980a" + ); + ``` + + 其中 `WITH BROKER` 以及之后的 Property Map 用于提供 Broker 相关信息。 + + +## Broker 信息 + +Broker 的信息包括 **名称（Broker name）** 和 **认证信息** 两部分。通常的语法格式如下： + +``` +WITH BROKER "broker_name" +( + "username" = "xxx", + "password" = "yyy", + "other_prop" = "prop_value", + ... +); +``` + +### 名称 + +通常用户需要通过操作命令中的 `WITH BROKER "broker_name"` 子句来指定一个已经存在的 Broker Name。Broker Name 是用户在通过 `ALTER SYSTEM ADD BROKER` 命令添加 Broker 进程时指定的一个名称。一个名称通常对应一个或多个 Broker 进程。Doris 会根据名称选择可用的 Broker 进程。用户可以通过 `SHOW BROKER` 命令查看当前集群中已经存在的 Broker。 + +**注：Broker Name 只是一个用户自定义名称，不代表 Broker 的类型。** + +### 认证信息 + +不同的 Broker 类型，以及不同的访问方式需要提供不同的认证信息。认证信息通常在 `WITH BROKER "broker_name"` 之后的 Property Map 中以 Key-Value 的方式提供。 + +#### 社区版 HDFS + +1. 简单认证 + + 简单认证即 Hadoop 配置 `hadoop.security.authentication` 为 `simple`。 + + 使用系统用户访问 HDFS。或者在 Broker 启动的环境变量中添加：```HADOOP_USER_NAME```。 + + ``` + ( + "username" = "user", + "password" = "" + ); + ``` + + 密码置空即可。 + +2. Kerberos 认证 + + 该认证方式需提供以下信息： + + * `hadoop.security.authentication`：指定认证方式为 kerberos。 + * `kerberos_principal`：指定 kerberos 的 principal。 + * `kerberos_keytab`：指定 kerberos 的 keytab 文件路径。该文件必须为 Broker 进程所在服务器上的文件的绝对路径。并且可以被 Broker 进程访问。 + * `kerberos_keytab_content`：指定 kerberos 中 keytab 文件内容经过 base64 编码之后的内容。这个跟 `kerberos_keytab` 配置二选一即可。 + + 示例如下： + + ``` + ( + "hadoop.security.authentication" = "kerberos", + "kerberos_principal" = "doris@YOUR.COM", + "kerberos_keytab" = "/home/doris/my.keytab" + ) + ``` + ``` + ( + "hadoop.security.authentication" = "kerberos", + "kerberos_principal" = "doris@YOUR.COM", + "kerberos_keytab_content" = "ASDOWHDLAWIDJHWLDKSALDJSDIWALD" + ) + ``` + 如果采用Kerberos认证方式，则部署Broker进程的时候需要[krb5.conf](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html)文件， + krb5.conf文件包含Kerberos的配置信息，通常，您应该将krb5.conf文件安装在目录/etc中。您可以通过设置环境变量KRB5_CONFIG覆盖默认位置。 + krb5.conf文件的内容示例如下： + ``` + [libdefaults] + default_realm = DORIS.HADOOP + default_tkt_enctypes = des3-hmac-sha1 des-cbc-crc + default_tgs_enctypes = des3-hmac-sha1 des-cbc-crc + dns_lookup_kdc = true + dns_lookup_realm = false + + [realms] + DORIS.HADOOP = { + kdc = kerberos-doris.hadoop.service:7005 + } + ``` + +3. HDFS HA 模式 + + 这个配置用于访问以 HA 模式部署的 HDFS 集群。 + + * `dfs.nameservices`：指定 hdfs 服务的名字，自定义，如："dfs.nameservices" = "my_ha"。 + * `dfs.ha.namenodes.xxx`：自定义 namenode 的名字,多个名字以逗号分隔。其中 xxx 为 `dfs.nameservices` 中自定义的名字，如： "dfs.ha.namenodes.my_ha" = "my_nn"。 + * `dfs.namenode.rpc-address.xxx.nn`：指定 namenode 的rpc地址信息。其中 nn 表示 `dfs.ha.namenodes.xxx` 中配置的 namenode 的名字，如："dfs.namenode.rpc-address.my_ha.my_nn" = "host:port"。 + * `dfs.client.failover.proxy.provider`：指定 client 连接 namenode 的 provider，默认为：org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider。 + + 示例如下： + + ``` + ( + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ) + ``` + + HA 模式可以和前面两种认证方式组合，进行集群访问。如通过简单认证访问 HA HDFS： + + ``` + ( + "username"="user", + "password"="passwd", + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ) + ``` + 关于HDFS集群的配置可以写入hdfs-site.xml文件中，用户使用Broker进程读取HDFS集群的信息时，只需要填写集群的文件路径名和认证信息即可。 + +#### 百度对象存储 BOS + +**（开源版本不支持）** + +1. 通过 AK/SK 访问 + + * AK/SK：Access Key 和 Secret Key。在百度云安全认证中心可以查看用户的 AK/SK。 + * Region Endpoint：BOS 所在地区的 Endpoint： + + * 华北-北京：http://bj.bcebos.com + * 华北-保定：http://bd.bcebos.com + * 华南-广州：http://gz.bcebos.com + * 华东-苏州：http://sz.bcebos.com + + + 示例如下： + + ``` + ( + "bos_endpoint" = "http://bj.bcebos.com", + "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "bos_secret_accesskey" = "yyyyyyyyyyyyyyyyyyyyyyyyyy" + ) + ``` + +#### 百度 HDFS/AFS + +**（开源版本不支持）** + +百度 AFS 和 HDFS 仅支持使用 ugi 的简单认证访问。示例如下： + +``` +( + "username" = "user", + "password" = "passwd" +); +``` + +其中 user 和 passwd 为 Hadoop 的 UGI 配置。 diff --git a/docs/zh-CN/administrator-guide/colocation-join.md b/docs/zh-CN/administrator-guide/colocation-join.md new file mode 100644 index 00000000000000..c701efd7c7d8b6 --- /dev/null +++ b/docs/zh-CN/administrator-guide/colocation-join.md @@ -0,0 +1,448 @@ +--- +{ + "title": "Colocation Join", + "language": "zh-CN" +} +--- + + + +# Colocation Join + +Colocation Join 是在 Doris 0.9 版本中引入的新功能。旨在为某些 Join 查询提供本地性优化，来减少数据在节点间的传输耗时，加速查询。 + +最初的设计、实现和效果可以参阅 [ISSUE 245](https://github.com/apache/incubator-doris/issues/245)。 + +Colocation Join 功能经过一次改版，设计和使用方式和最初设计稍有不同。本文档主要介绍 Colocation Join 的原理、实现、使用方式和注意事项。 + +## 名词解释 + +* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 +* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 +* Colocation Group（CG）：一个 CG 中会包含一张及以上的 Table。在同一个 Group 内的 Table 有着相同的 Colocation Group Schema，并且有着相同的数据分片分布。 +* Colocation Group Schema（CGS）：用于描述一个 CG 中的 Table，和 Colocation 相关的通用 Schema 信息。包括分桶列类型，分桶数以及副本数等。 + +## 原理 + +Colocation Join 功能，是将一组拥有相同 CGS 的 Table 组成一个 CG。并保证这些 Table 对应的数据分片会落在同一个 BE 节点上。使得当 CG 内的表进行分桶列上的 Join 操作时，可以通过直接进行本地数据 Join，减少数据在节点间的传输耗时。 + +一个表的数据，最终会根据分桶列值 Hash、对桶数取模的后落在某一个分桶内。假设一个 Table 的分桶数为 8，则共有 `[0, 1, 2, 3, 4, 5, 6, 7]` 8 个分桶（Bucket），我们称这样一个序列为一个 `BucketsSequence`。每个 Bucket 内会有一个或多个数据分片（Tablet）。当表为单分区表时，一个 Bucket 内仅有一个 Tablet。如果是多分区表，则会有多个。 + +为了使得 Table 能够有相同的数据分布，同一 CG 内的 Table 必须保证以下属性相同： + +1. 分桶列和分桶数 + + 分桶列，即在建表语句中 `DISTRIBUTED BY HASH(col1, col2, ...)` 中指定的列。分桶列决定了一张表的数据通过哪些列的值进行 Hash 划分到不同的 Tablet 中。同一 CG 内的 Table 必须保证分桶列的类型和数量完全一致，并且桶数一致，才能保证多张表的数据分片能够一一对应的进行分布控制。 + +2. 副本数 + + 同一个 CG 内所有表的所有分区（Partition）的副本数必须一致。如果不一致，可能出现某一个 Tablet 的某一个副本，在同一个 BE 上没有其他的表分片的副本对应。 + +同一个 CG 内的表，分区的个数、范围以及分区列的类型不要求一致。 + +在固定了分桶列和分桶数后，同一个 CG 内的表会拥有相同的 BucketsSequnce。而副本数决定了每个分桶内的 Tablet 的多个副本，存放在哪些 BE 上。假设 BucketsSequnce 为 `[0, 1, 2, 3, 4, 5, 6, 7]`，BE 节点有 `[A, B, C, D]` 4个。则一个可能的数据分布如下： + +``` ++---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +| 0 | | 1 | | 2 | | 3 | | 4 | | 5 | | 6 | | 7 | ++---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +| A | | B | | C | | D | | A | | B | | C | | D | +| | | | | | | | | | | | | | | | +| B | | C | | D | | A | | B | | C | | D | | A | +| | | | | | | | | | | | | | | | +| C | | D | | A | | B | | C | | D | | A | | B | ++---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +``` + +CG 内所有表的数据都会按照上面的规则进行统一分布，这样就保证了，分桶列值相同的数据都在同一个 BE 节点上，可以进行本地数据 Join。 + +## 使用方式 + +### 建表 + +建表时，可以在 `PROPERTIES` 中指定属性 `"colocate_with" = "group_name"`，表示这个表是一个 Colocation Join 表，并且归属于一个指定的 Colocation Group。 + +示例： + +``` +CREATE TABLE tbl (k1 int, v1 int sum) +DISTRIBUTED BY HASH(k1) +BUCKETS 8 +PROPERTIES( + "colocate_with" = "group1" +); +``` + +如果指定的 Group 不存在，则 Doris 会自动创建一个只包含当前这张表的 Group。如果 Group 已存在，则 Doris 会检查当前表是否满足 Colocation Group Schema。如果满足，则会创建该表，并将该表加入 Group。同时，表会根据已存在的 Group 中的数据分布规则创建分片和副本。 +Group 归属于一个 Database，Group 的名字在一个 Database 内唯一。在内部存储是 Group 的全名为 `dbId_groupName`，但用户只感知 groupName。 + +### 删表 + +当 Group 中最后一张表彻底删除后（彻底删除是指从回收站中删除。通常，一张表通过 `DROP TABLE` 命令删除后，会在回收站默认停留一天的时间后，再删除），该 Group 也会被自动删除。 + +### 查看 Group + +以下命令可以查看集群内已存在的 Group 信息。 + +``` +SHOW PROC '/colocation_group'; + ++-------------+--------------+--------------+------------+----------------+----------+----------+ +| GroupId | GroupName | TableIds | BucketsNum | ReplicationNum | DistCols | IsStable | ++-------------+--------------+--------------+------------+----------------+----------+----------+ +| 10005.10008 | 10005_group1 | 10007, 10040 | 10 | 3 | int(11) | true | ++-------------+--------------+--------------+------------+----------------+----------+----------+ +``` + +* GroupId： 一个 Group 的全集群唯一标识，前半部分为 db id，后半部分为 group id。 +* GroupName： Group 的全名。 +* TabletIds： 该 Group 包含的 Table 的 id 列表。 +* BucketsNum： 分桶数。 +* ReplicationNum： 副本数。 +* DistCols： Distribution columns，即分桶列类型。 +* IsStable： 该 Group 是否稳定（稳定的定义，见 `Colocation 副本均衡和修复` 一节）。 + +通过以下命令可以进一步查看一个 Group 的数据分布情况： + +``` +SHOW PROC '/colocation_group/10005.10008'; + ++-------------+---------------------+ +| BucketIndex | BackendIds | ++-------------+---------------------+ +| 0 | 10004, 10002, 10001 | +| 1 | 10003, 10002, 10004 | +| 2 | 10002, 10004, 10001 | +| 3 | 10003, 10002, 10004 | +| 4 | 10002, 10004, 10003 | +| 5 | 10003, 10002, 10001 | +| 6 | 10003, 10004, 10001 | +| 7 | 10003, 10004, 10002 | ++-------------+---------------------+ +``` + +* BucketIndex： 分桶序列的下标。 +* BackendIds： 分桶中数据分片所在的 BE 节点 id 列表。 + +> 以上命令需要 AMDIN 权限。暂不支持普通用户查看。 + +### 修改表 Colocate Group 属性 + +可以对一个已经创建的表，修改其 Colocation Group 属性。示例： + +`ALTER TABLE tbl SET ("colocate_with" = "group2");` + +* 如果该表之前没有指定过 Group，则该命令检查 Schema，并将该表加入到该 Group（Group 不存在则会创建）。 +* 如果该表之前有指定其他 Group，则该命令会先将该表从原有 Group 中移除，并加入新 Group（Group 不存在则会创建）。 + +也可以通过以下命令，删除一个表的 Colocation 属性： + +`ALTER TABLE tbl SET ("colocate_with" = "");` + +### 其他相关操作 + +当对一个具有 Colocation 属性的表进行增加分区（ADD PARTITION）、修改副本数时，Doris 会检查修改是否会违反 Colocation Group Schema，如果违反则会拒绝。 + +## Colocation 副本均衡和修复 + +Colocation 表的副本分布需要遵循 Group 中指定的分布，所以在副本修复和均衡方面和普通分片有所区别。 + +Group 自身有一个 Stable 属性，当 Stable 为 true 时，表示当前 Group 内的表的所有分片没有正在进行变动，Colocation 特性可以正常使用。当 Stable 为 false 时（Unstable），表示当前 Group 内有部分表的分片正在做修复或迁移，此时，相关表的 Colocation Join 将退化为普通 Join。 + +### 副本修复 + +副本只能存储在指定的 BE 节点上。所以当某个 BE 不可用时（宕机、Decommission 等），需要寻找一个新的 BE 进行替换。Doris 会优先寻找负载最低的 BE 进行替换。替换后，该 Bucket 内的所有在旧 BE 上的数据分片都要做修复。迁移过程中，Group 被标记为 Unstable。 + +### 副本均衡 + +Doris 会尽力将 Colocation 表的分片均匀分布在所有 BE 节点上。对于普通表的副本均衡，是以单副本为粒度的，即单独为每一个副本寻找负载较低的 BE 节点即可。而 Colocation 表的均衡是 Bucket 级别的，即一个 Bucket 内的所有副本都会一起迁移。我们采用一个简单的均衡算法，即在不考虑副本实际大小，而只根据副本数量，将 BucketsSequnce 均匀的分布在所有 BE 上。具体算法可以参阅 `ColocateTableBalancer.java` 中的代码注释。 + +> 注1：当前的 Colocation 副本均衡和修复算法，对于异构部署的 Doris 集群效果可能不佳。所谓异构部署，即 BE 节点的磁盘容量、数量、磁盘类型（SSD 和 HDD）不一致。在异构部署情况下，可能出现小容量的 BE 节点和大容量的 BE 节点存储了相同的副本数量。 +> +> 注2：当一个 Group 处于 Unstable 状态时，其中的表的 Join 将退化为普通 Join。此时可能会极大降低集群的查询性能。如果不希望系统自动均衡，可以设置 FE 的配置项 `disable_colocate_balance` 来禁止自动均衡。然后在合适的时间打开即可。（具体参阅 `高级操作` 一节） + +## 查询 + +对 Colocation 表的查询方式和普通表一样，用户无需感知 Colocation 属性。如果 Colocation 表所在的 Group 处于 Unstable 状态，将自动退化为普通 Join。 + +举例说明： + +表1： + +``` +CREATE TABLE `tbl1` ( + `k1` date NOT NULL COMMENT "", + `k2` int(11) NOT NULL COMMENT "", + `v1` int(11) SUM NOT NULL COMMENT "" +) ENGINE=OLAP +AGGREGATE KEY(`k1`, `k2`) +PARTITION BY RANGE(`k1`) +( + PARTITION p1 VALUES LESS THAN ('2019-05-31'), + PARTITION p2 VALUES LESS THAN ('2019-06-30') +) +DISTRIBUTED BY HASH(`k2`) BUCKETS 8 +PROPERTIES ( + "colocate_with" = "group1" +); +``` + +表2： + +``` +CREATE TABLE `tbl2` ( + `k1` datetime NOT NULL COMMENT "", + `k2` int(11) NOT NULL COMMENT "", + `v1` double SUM NOT NULL COMMENT "" +) ENGINE=OLAP +AGGREGATE KEY(`k1`, `k2`) +DISTRIBUTED BY HASH(`k2`) BUCKETS 8 +PROPERTIES ( + "colocate_with" = "group1" +); +``` + +查看查询计划： + +``` +DESC SELECT * FROM tbl1 INNER JOIN tbl2 ON (tbl1.k2 = tbl2.k2); + ++----------------------------------------------------+ +| Explain String | ++----------------------------------------------------+ +| PLAN FRAGMENT 0 | +| OUTPUT EXPRS:`tbl1`.`k1` | | +| PARTITION: RANDOM | +| | +| RESULT SINK | +| | +| 2:HASH JOIN | +| | join op: INNER JOIN | +| | hash predicates: | +| | colocate: true | +| | `tbl1`.`k2` = `tbl2`.`k2` | +| | tuple ids: 0 1 | +| | | +| |----1:OlapScanNode | +| | TABLE: tbl2 | +| | PREAGGREGATION: OFF. Reason: null | +| | partitions=0/1 | +| | rollup: null | +| | buckets=0/0 | +| | cardinality=-1 | +| | avgRowSize=0.0 | +| | numNodes=0 | +| | tuple ids: 1 | +| | | +| 0:OlapScanNode | +| TABLE: tbl1 | +| PREAGGREGATION: OFF. Reason: No AggregateInfo | +| partitions=0/2 | +| rollup: null | +| buckets=0/0 | +| cardinality=-1 | +| avgRowSize=0.0 | +| numNodes=0 | +| tuple ids: 0 | ++----------------------------------------------------+ +``` +如果 Colocation Join 生效，则 Hash Join 节点会显示 `colocate: true`。 + +如果没有生效，则查询计划如下： + +``` ++----------------------------------------------------+ +| Explain String | ++----------------------------------------------------+ +| PLAN FRAGMENT 0 | +| OUTPUT EXPRS:`tbl1`.`k1` | | +| PARTITION: RANDOM | +| | +| RESULT SINK | +| | +| 2:HASH JOIN | +| | join op: INNER JOIN (BROADCAST) | +| | hash predicates: | +| | colocate: false, reason: group is not stable | +| | `tbl1`.`k2` = `tbl2`.`k2` | +| | tuple ids: 0 1 | +| | | +| |----3:EXCHANGE | +| | tuple ids: 1 | +| | | +| 0:OlapScanNode | +| TABLE: tbl1 | +| PREAGGREGATION: OFF. Reason: No AggregateInfo | +| partitions=0/2 | +| rollup: null | +| buckets=0/0 | +| cardinality=-1 | +| avgRowSize=0.0 | +| numNodes=0 | +| tuple ids: 0 | +| | +| PLAN FRAGMENT 1 | +| OUTPUT EXPRS: | +| PARTITION: RANDOM | +| | +| STREAM DATA SINK | +| EXCHANGE ID: 03 | +| UNPARTITIONED | +| | +| 1:OlapScanNode | +| TABLE: tbl2 | +| PREAGGREGATION: OFF. Reason: null | +| partitions=0/1 | +| rollup: null | +| buckets=0/0 | +| cardinality=-1 | +| avgRowSize=0.0 | +| numNodes=0 | +| tuple ids: 1 | ++----------------------------------------------------+ +``` + +HASH JOIN 节点会显示对应原因：`colocate: false, reason: group is not stable`。同时会有一个 EXCHANGE 节点生成。 + + +## 高级操作 + +### FE 配置项 + +* disable\_colocate\_relocate + + 是否关闭 Doris 的自动 Colocation 副本修复。默认为 false，即不关闭。该参数只影响 Colocation 表的副本修复，不影响普通表。 + +* disable\_colocate\_balance + + 是否关闭 Doris 的自动 Colocation 副本均衡。默认为 false，即不关闭。该参数只影响 Colocation 表的副本均衡，不影响普通表。 + +以上参数可以动态修改，设置方式请参阅 `HELP ADMIN SHOW CONFIG;` 和 `HELP ADMIN SET CONFIG;`。 + +* disable\_colocate\_join + + 是否关闭 Colocation Join 功能。在 0.10 及之前的版本，默认为 true，即关闭。在之后的某个版本中将默认为 false，即开启。 + +* use\_new\_tablet\_scheduler + + 在 0.10 及之前的版本中，新的副本调度逻辑与 Colocation Join 功能不兼容，所以在 0.10 及之前版本，如果 `disable_colocate_join = false`，则需设置 `use_new_tablet_scheduler = false`，即关闭新的副本调度器。之后的版本中，`use_new_tablet_scheduler` 将衡为 true。 + +### HTTP Restful API + +Doris 提供了几个和 Colocation Join 有关的 HTTP Restful API，用于查看和修改 Colocation Group。 + +该 API 实现在 FE 端，使用 `fe_host:fe_http_port` 进行访问。需要 ADMIN 权限。 + +1. 查看集群的全部 Colocation 信息 + + ``` + GET /api/colocate + + 返回以 Json 格式表示内部 Colocation 信息。 + + { + "colocate_meta": { + "groupName2Id": { + "g1": { + "dbId": 10005, + "grpId": 10008 + } + }, + "group2Tables": {}, + "table2Group": { + "10007": { + "dbId": 10005, + "grpId": 10008 + }, + "10040": { + "dbId": 10005, + "grpId": 10008 + } + }, + "group2Schema": { + "10005.10008": { + "groupId": { + "dbId": 10005, + "grpId": 10008 + }, + "distributionColTypes": [{ + "type": "INT", + "len": -1, + "isAssignedStrLenInColDefinition": false, + "precision": 0, + "scale": 0 + }], + "bucketsNum": 10, + "replicationNum": 2 + } + }, + "group2BackendsPerBucketSeq": { + "10005.10008": [ + [10004, 10002], + [10003, 10002], + [10002, 10004], + [10003, 10002], + [10002, 10004], + [10003, 10002], + [10003, 10004], + [10003, 10004], + [10003, 10004], + [10002, 10004] + ] + }, + "unstableGroups": [] + }, + "status": "OK" + } + ``` + +2. 将 Group 标记为 Stable 或 Unstable + + * 标记为 Stable + + ``` + POST /api/colocate/group_stable?db_id=10005&group_id=10008 + + 返回：200 + ``` + + * 标记为 Unstable + + ``` + DELETE /api/colocate/group_stable?db_id=10005&group_id=10008 + + 返回：200 + ``` + +3. 设置 Group 的数据分布 + + 该接口可以强制设置某一 Group 的数分布。 + + ``` + POST /api/colocate/bucketseq?db_id=10005&group_id= 10008 + + Body: + [[10004,10002],[10003,10002],[10002,10004],[10003,10002],[10002,10004],[10003,10002],[10003,10004],[10003,10004],[10003,10004],[10002,10004]] + + 返回 200 + ``` + 其中 Body 是以嵌套数组表示的 BucketsSequence 以及每个 Bucket 中分片分布所在 BE 的 id。 + + 注意，使用该命令，可能需要将 FE 的配置 `disable_colocate_relocate` 和 `disable_colocate_balance` 设为 true。即关闭系统自动的 Colocation 副本修复和均衡。否则可能在修改后，会被系统自动重置。 \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md new file mode 100644 index 00000000000000..6c273c5503e65c --- /dev/null +++ b/docs/zh-CN/administrator-guide/config/fe_config.md @@ -0,0 +1,41 @@ +--- +{ + "title": "基本配置", + "language": "zh-CN" +} +--- + + + +# 基本配置 + +## brpc_max_body_size + + 这个配置主要用来修改 brpc 的参数 max_body_size ，默认配置是 64M。一般发生在 multi distinct + 无 group by + 超过1T 数据量的情况下。尤其如果发现查询卡死，且 BE 出现类似 body_size is too large 的字样。 + + 由于这是一个 brpc 的配置，用户也可以在运行中直接修改该参数。通过访问 http://host:brpc_port/flags 修改。 + +## max_running_txn_num_per_db + + 这个配置主要是用来控制同一个 db 的并发导入个数的，默认配置是100。当导入的并发执行的个数超过这个配置的值的时候，同步执行的导入就会失败比如 stream load。异步执行的导入就会一直处在 pending 状态比如 broker load。 + + 一般来说不推荐更改这个并发数。如果当前导入并发超过这个值，则需要先检查是否单个导入任务过慢，或者小文件太多没有合并后导入的问题。 + + 报错信息比如：current running txns on db xxx is xx, larger than limit xx。就属于这类问题。 diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md new file mode 100644 index 00000000000000..4705fbc109fdbd --- /dev/null +++ b/docs/zh-CN/administrator-guide/dynamic-partition.md @@ -0,0 +1,208 @@ +--- +{ + "title": "动态分区", + "language": "zh-CN" +} +--- + + + +# 动态分区 + +动态分区是在 Doris 0.12 版本中引入的新功能。旨在对表级别的分区实现生命周期管理(TTL)，减少用户的使用负担。 + +最初的设计、实现和效果可以参阅 [ISSUE 2262](https://github.com/apache/incubator-doris/issues/2262)。 + +目前实现了动态添加分区及动态删除分区的功能。 + +## 名词解释 + +* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 +* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 + +## 原理 + +在某些使用场景下，用户会将表按照天进行分区划分，每天定时执行例行任务，这时需要使用方手动管理分区，否则可能由于使用方没有创建分区导致数据导入失败，这给使用方带来了额外的维护成本。 + +在实现方式上, FE会启动一个后台线程，根据fe.conf中`dynamic_partition_enable` 及 `dynamic_partition_check_interval_seconds`参数决定该线程是否启动以及该线程的调度频率。每次调度时，会在注册表中读取动态分区表的属性，并根据动态分区属性动态添加及删除分区。 + +建表时，在properties中指定dynamic_partition属性，FE首先对动态分区属性进行解析，校验输入参数的合法性，然后将对应的属性持久化到FE的元数据中，并将该表注册到动态分区列表中，后台线程会根据配置参数定期对动态分区列表进行扫描，读取表的动态分区属性，执行添加分区及删除分区的任务，每次的调度信息会保留在FE的内存中（重启后则丢失），可以通过`SHOW DYNAMIC PARTITION TABLES`查看调度任务是否成功，如果存在分区创建或删除失败，会将失败信息输出。 + +## 使用方式 + +### 动态分区属性参数说明: + +`dynamic_partition.enable`: 是否开启动态分区特性，可指定为 `TRUE` 或 `FALSE`。如果不填写，默认为 `TRUE`。 + + +`dynamic_partition.time_unit`: 动态分区调度的单位，可指定为 `DAY` `WEEK` `MONTH`，当指定为 `DAY` 时，动态创建的分区名后缀格式为`yyyyMMdd`，例如`20200325`。当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。 + +`dynamic_partition.start`: 动态分区的开始时间, 以当天为基准，超过该时间范围的分区将会被删除。如果不填写，则默认为`Integer.MIN_VALUE` 即 `-2147483648`。 + + +`dynamic_partition.end`: 动态分区的结束时间, 以当天为基准，会提前创建N个单位的分区范围。 + +`dynamic_partition.prefix`: 动态创建的分区名前缀。 + +`dynamic_partition.buckets`: 动态创建的分区所对应的分桶数量。 + +### 建表 + +建表时，可以在 `PROPERTIES` 中指定以下`dynamic_partition`属性，表示这个表是一个动态分区表。 + +示例： + +``` +CREATE TABLE example_db.dynamic_partition +( +k1 DATE, +k2 INT, +k3 SMALLINT, +v1 VARCHAR(2048), +v2 DATETIME DEFAULT "2014-02-04 15:36:00" +) +ENGINE=olap +DUPLICATE KEY(k1, k2, k3) +PARTITION BY RANGE (k1) +( +PARTITION p20200321 VALUES LESS THAN ("2020-03-22"), +PARTITION p20200322 VALUES LESS THAN ("2020-03-23"), +PARTITION p20200323 VALUES LESS THAN ("2020-03-24"), +PARTITION p20200324 VALUES LESS THAN ("2020-03-25") +) +DISTRIBUTED BY HASH(k2) BUCKETS 32 +PROPERTIES( +"storage_medium" = "SSD", +"dynamic_partition.enable" = "true", +"dynamic_partition.time_unit" = "DAY", +"dynamic_partition.start" = "-3", +"dynamic_partition.end" = "3", +"dynamic_partition.prefix" = "p", +"dynamic_partition.buckets" = "32" + ); +``` +创建一张动态分区表，指定开启动态分区特性，以当天为2020-03-25为例，在每次调度时，会删除分区上界小于 `2020-03-22` 的分区，为了避免删除非动态创建的分区，动态删除分区只会删除分区名符合动态创建分区规则的分区，例如分区名为a1, 则即使分区范围在待删除的分区范围内，也不会被删除。同时在调度时会提前创建今天以及以后3天（总共4天）的分区(若分区已存在则会忽略)，分区名根据指定前缀分别为`p20200325` `p20200326` `p20200327` `p20200328`,每个分区的分桶数量为32。同时会删除 `p20200321` 的分区，最终的分区范围如下: +``` +[types: [DATE]; keys: [2020-03-22]; ‥types: [DATE]; keys: [2020-03-23]; ) +[types: [DATE]; keys: [2020-03-23]; ‥types: [DATE]; keys: [2020-03-24]; ) +[types: [DATE]; keys: [2020-03-24]; ‥types: [DATE]; keys: [2020-03-25]; ) +[types: [DATE]; keys: [2020-03-25]; ‥types: [DATE]; keys: [2020-03-26]; ) +[types: [DATE]; keys: [2020-03-26]; ‥types: [DATE]; keys: [2020-03-27]; ) +[types: [DATE]; keys: [2020-03-27]; ‥types: [DATE]; keys: [2020-03-28]; ) +[types: [DATE]; keys: [2020-03-28]; ‥types: [DATE]; keys: [2020-03-29]; ) +``` + +### 开启动态分区功能 +1. 首先需要在fe.conf中设置`dynamic_partition_enable=true`，可以在集群启动时通过修改配置文件指定，或者通过MySQL连接后使用命令行 `ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true")`修改，也可以在运行时通过http接口动态修改,修改方法查看高级操作部分 + +2. 如果需要对0.12版本之前的表添加动态分区属性，则需要通过以下命令修改表的属性 +``` +ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32"); +``` + +### 停止动态分区功能 + +如果需要对集群中所有动态分区表停止动态分区功能，则需要在fe.conf中设置`dynamic_partition_enable=false` + +如果需要对指定表停止动态分区功能，则可以通过以下命令修改表的属性 +``` +ALTER TABLE dynamic_partition SET ("dynamic_partition.enable" = "false") +``` + +### 修改动态分区属性 + +通过如下命令可以修改动态分区的属性 +``` +ALTER TABLE dynamic_partition SET ("key" = "value") +``` + +### 查看动态分区表调度情况 + +通过以下命令可以进一步查看动态分区表的调度情况： + +``` +SHOW DYNAMIC PARTITION TABLES; + ++-------------------+--------+----------+-------+------+--------+---------+---------------------+---------------------+--------+------------------------+----------------------+ +| TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | ++-------------------+--------+----------+-------+------+--------+---------+---------------------+---------------------+--------+------------------------+----------------------+ +| dynamic_partition | true | DAY | -3 | 3 | p | 32 | 2020-03-12 17:25:47 | 2020-03-12 17:25:52 | NORMAL | N/A | N/A | ++-------------------+--------+----------+-------+------+--------+---------+---------------------+---------------------+--------+------------------------+----------------------+ +1 row in set (0.00 sec) + +``` + +* LastUpdateTime: 最后一次修改动态分区属性的时间 +* LastSchedulerTime: 最后一次执行动态分区调度的时间 +* State: 最后一次执行动态分区调度的状态 +* LastCreatePartitionMsg: 最后一次执行动态添加分区调度的错误信息 +* LastDropPartitionMsg: 最后一次执行动态删除分区调度的错误信息 + +## 高级操作 + +### FE 配置项 + +* dynamic\_partition\_enable + + 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。 + +* dynamic\_partition\_check\_interval\_seconds + + 动态分区线程的执行频率，默认为3600(1个小时)，即每1个小时进行一次调度 + +### HTTP Restful API + +Doris 提供了修改动态分区配置参数的 HTTP Restful API，用于运行时修改动态分区配置参数。 + +该 API 实现在 FE 端，使用 `fe_host:fe_http_port` 进行访问。需要 ADMIN 权限。 + +1. 将 dynamic_partition_enable 设置为 true 或 false + + * 标记为 true + + ``` + GET /api/_set_config?dynamic_partition_enable=true + + 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true + + 返回：200 + ``` + + * 标记为 false + + ``` + GET /api/_set_config?dynamic_partition_enable=false + + 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=false + + 返回：200 + ``` + +2. 设置 dynamic partition 的调度频率 + + * 设置调度时间为12小时调度一次 + + ``` + GET /api/_set_config?dynamic_partition_check_interval_seconds=432000 + + 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000 + + 返回：200 + ``` diff --git a/docs/documentation/cn/administrator-guide/export-manual.md b/docs/zh-CN/administrator-guide/export-manual.md similarity index 99% rename from docs/documentation/cn/administrator-guide/export-manual.md rename to docs/zh-CN/administrator-guide/export-manual.md index 7af73acfdf68bc..0c0845bba61a13 100644 --- a/docs/documentation/cn/administrator-guide/export-manual.md +++ b/docs/zh-CN/administrator-guide/export-manual.md @@ -1,3 +1,10 @@ +--- +{ + "title": "数据导出", + "language": "zh-CN" +} +--- + + +# CANCEL LABEL +## description + NAME: + cancel_label: cancel a transaction with label + + SYNOPSIS + curl -u user:passwd -XPOST http://host:port/api/{db}/{label}/_cancel + + DESCRIPTION + 该命令用于cancel一个指定Label对应的事务，事务在Prepare阶段能够被成功cancel + + RETURN VALUES + 执行完成后，会以Json格式返回这次导入的相关内容。当前包括一下字段 + Status: 是否成功cancel + Success: 成功cancel事务 + 其他: cancel失败 + Message: 具体的失败信息 + + ERRORS + +## example + + 1. cancel testDb, testLabel的作业 + curl -u root -XPOST http://host:port/api/testDb/testLabel/_cancel + +## keyword + CANCEL，LABEL + + + + + + diff --git a/docs/zh-CN/administrator-guide/http-actions/compaction-action.md b/docs/zh-CN/administrator-guide/http-actions/compaction-action.md new file mode 100644 index 00000000000000..ddf7d1f90be8ce --- /dev/null +++ b/docs/zh-CN/administrator-guide/http-actions/compaction-action.md @@ -0,0 +1,86 @@ +--- +{ + "title": "Compaction Action", + "language": "zh-CN" +} +--- + + + +# Compaction Action + +该 API 用于查看某个 BE 节点总体的 compaction 状态，或者指定 tablet 的 compaction 状态。也可以用于手动触发 Compaction。 + +## 查看 Compaction 状态 + +### 节点整体 compaction 状态 + +(TODO) + +### 指定 tablet 的 compaction 状态 + +``` +curl -X GET http://be_host:webserver_port/api/compaction/show?tablet_id=xxxx\&schema_hash=yyyy +``` + +若 tablet 不存在，返回 JSON 格式的错误： + +``` +{ + "status": "Fail", + "msg": "Tablet not found" +} +``` + +若 tablet 存在，则返回 JSON 格式的结果: + +``` +{ + "cumulative point": 50, + "last cumulative failure time": "2019-12-16 18:13:43.224", + "last base failure time": "2019-12-16 18:13:23.320", + "last cumu success time": "2019-12-16 18:12:15.110", + "last base success time": "2019-12-16 18:11:50.780", + "rowsets": [ + "[0-48] 10 DATA OVERLAPPING", + "[49-49] 2 DATA OVERLAPPING", + "[50-50] 0 DELETE NONOVERLAPPING", + "[51-51] 5 DATA OVERLAPPING" + ] +} +``` + +结果说明： + +* cumulative point：base 和 cumulative compaction 的版本分界线。在 point（不含）之前的版本由 base compaction 处理。point（含）之后的版本由 cumulative compaction 处理。 +* last cumulative failure time：上一次尝试 cumulative compaction 失败的时间。默认 10min 后才会再次尝试对该 tablet 做 cumulative compaction。 +* last base failure time：上一次尝试 base compaction 失败的时间。默认 10min 后才会再次尝试对该 tablet 做 base compaction。 +* rowsets：该 tablet 当前的 rowset 集合。如 [0-48] 表示 0-48 版本。第二位数字表示该版本中 segment 的数量。`DELETE` 表示 delete 版本。`DATA` 表示数据版本。`OVERLAPPING` 和 `NONOVERLAPPING` 表示segment数据是否重叠。 + +### 示例 + +``` +curl -X GET http://192.168.10.24:8040/api/compaction/show?tablet_id=10015\&schema_hash=1294206575 +``` + +## 手动触发 Compaction + +(TODO) + diff --git a/docs/zh-CN/administrator-guide/http-actions/fe-get-log-file.md b/docs/zh-CN/administrator-guide/http-actions/fe-get-log-file.md new file mode 100644 index 00000000000000..47ac508e172cc4 --- /dev/null +++ b/docs/zh-CN/administrator-guide/http-actions/fe-get-log-file.md @@ -0,0 +1,80 @@ +--- +{ + "title": "get\\_log\\_file", + "language": "zh-CN" +} +--- + + + +# get\_log\_file + +用户可以通过该 HTTP 接口获取 FE 的日志文件。 + +## 日志类型 + +支持获取以下类型的 FE 日志： + +1. fe.audit.log（审计日志） + + 审计日志记录了对应 FE 节点的所有请求语句已经请求的信息。审计日志的文件命名规则如下： + + ``` + fe.audit.log # 当前的最新日志 + fe.audit.log.20190603.1 # 对应日期的审计日志，当对应日期的日志大小超过 1GB 后，会生成序号后缀。序号越小的日志，内容越新。 + fe.audit.log.20190603.2 + fe.audit.log.20190602.1 + ... + ``` + +## 接口示例 + +1. 获取对应类型的日志文件列表 + + 示例： + + `curl -v -X HEAD -uuser:passwd http://fe_host:http_port/api/get_log_file?type=fe.audit.log` + + 返回结果： + + ``` + HTTP/1.1 200 OK + file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934} + content-type: text/html + connection: keep-alive + ``` + + 在返回的 header 中，`file_infos` 字段以 json 格式展示文件列表以及对应文件大小（单位字节） + +2. 下载日志文件 + + 示例： + + ``` + curl -X GET -uuser:passwd http://fe_host:http_port/api/get_log_file?type=fe.audit.log\&file=fe.audit.log.20190528.1 + ``` + + 返回结果： + + 以文件的形式下载指定的文件。 + +## 接口说明 + +该接口需要 admin 权限。 diff --git a/docs/zh-CN/administrator-guide/http-actions/get-label-state.md b/docs/zh-CN/administrator-guide/http-actions/get-label-state.md new file mode 100644 index 00000000000000..65f9210e94ed5b --- /dev/null +++ b/docs/zh-CN/administrator-guide/http-actions/get-label-state.md @@ -0,0 +1,59 @@ +--- +{ + "title": "GET LABEL STATE", + "language": "zh-CN" +} +--- + + + +# GET LABEL STATE +## description + NAME: + get_label_state: get label's state + + SYNOPSIS + curl -u user:passwd http://host:port/api/{db}/{label}/_state + + DESCRIPTION + 该命令用于查看一个Label对应的事务状态 + + RETURN VALUES + 执行完毕后，会以Json格式返回这次导入的相关内容。当前包括一下字段 + Label：本次导入的 label，如果没有指定，则为一个 uuid。 + Status：此命令是否成功执行，Success表示成功执行 + Message： 具体的执行信息 + State: 只有在Status为Success时才有意义 + UNKNOWN: 没有找到对应的Label + PREPARE: 对应的事务已经prepare，但尚未提交 + COMMITTED: 事务已经提交，不能被cancel + VISIBLE: 事务提交，并且数据可见，不能被cancel + ABORTED: 事务已经被ROLLBACK，导入已经失败。 + + ERRORS + +## example + + 1. 获得testDb, testLabel的状态 + curl -u root http://host:port/api/testDb/testLabel/_state + +## keyword + GET, LABEL, STATE + diff --git a/docs/zh-CN/administrator-guide/http-actions/restore-tablet.md b/docs/zh-CN/administrator-guide/http-actions/restore-tablet.md new file mode 100644 index 00000000000000..786578e30adee5 --- /dev/null +++ b/docs/zh-CN/administrator-guide/http-actions/restore-tablet.md @@ -0,0 +1,43 @@ +--- +{ + "title": "RESTORE TABLET", + "language": "zh-CN" +} +--- + + + +# RESTORE TABLET +## description + + 该功能用于恢复trash目录中被误删的tablet数据。 + + 说明：这个功能暂时只在be服务中提供一个http接口。如果要使用， + 需要向要进行数据恢复的那台be机器的http端口发送restore tablet api请求。api格式如下： + METHOD: POST + URI: http://be_host:be_http_port/api/restore_tablet?tablet_id=xxx&schema_hash=xxx + +## example + + curl -X POST "http://hostname:8088/api/restore_tablet?tablet_id=123456\&schema_hash=1111111" + +## keyword + + RESTORE,TABLET,RESTORE,TABLET diff --git a/docs/zh-CN/administrator-guide/load-data/broker-load-manual.md b/docs/zh-CN/administrator-guide/load-data/broker-load-manual.md new file mode 100644 index 00000000000000..666b0c165d8031 --- /dev/null +++ b/docs/zh-CN/administrator-guide/load-data/broker-load-manual.md @@ -0,0 +1,517 @@ +--- +{ + "title": "Broker Load", + "language": "zh-CN" +} +--- + + + +# Broker Load + +Broker load 是一个异步的导入方式，支持的数据源取决于 Broker 进程支持的数据源。 + +用户需要通过 MySQL 协议 创建 Broker load 导入，并通过查看导入命令检查导入结果。 + +## 适用场景 + +* 源数据在 Broker 可以访问的存储系统中，如 HDFS。 +* 数据量在 几十到百GB 级别。 + +## 名词解释 + +1. Frontend（FE）：Doris 系统的元数据和调度节点。在导入流程中主要负责导入 plan 生成和导入任务的调度工作。 +2. Backend（BE）：Doris 系统的计算和存储节点。在导入流程中主要负责数据的 ETL 和存储。 +3. Broker：Broker 为一个独立的无状态进程。封装了文件系统接口，提供 Doris 读取远端存储系统中文件的能力。 +4. Plan：导入执行计划，BE 会执行导入执行计划将数据导入到 Doris 系统中。 + +## 基本原理 + +用户在提交导入任务后，FE 会生成对应的 Plan 并根据目前 BE 的个数和文件的大小，将 Plan 分给 多个 BE 执行，每个 BE 执行一部分导入数据。 + +BE 在执行的过程中会从 Broker 拉取数据，在对数据 transform 之后将数据导入系统。所有 BE 均完成导入，由 FE 最终决定导入是否成功。 + +``` + + + | 1. user create broker load + v + +----+----+ + | | + | FE | + | | + +----+----+ + | + | 2. BE etl and load the data + +--------------------------+ + | | | ++---v---+ +--v----+ +---v---+ +| | | | | | +| BE | | BE | | BE | +| | | | | | ++---+-^-+ +---+-^-+ +--+-^--+ + | | | | | | + | | | | | | 3. pull data from broker ++---v-+-+ +---v-+-+ +--v-+--+ +| | | | | | +|Broker | |Broker | |Broker | +| | | | | | ++---+-^-+ +---+-^-+ +---+-^-+ + | | | | | | ++---v-+-----------v-+----------v-+-+ +| HDFS/BOS/AFS cluster | +| | ++----------------------------------+ + +``` + +## 基本操作 + +### 创建导入 + +Broker load 创建导入语句 + +语法： + +``` +LOAD LABEL db_name.label_name +(data_desc, ...) +WITH BROKER broker_name broker_properties +[PROPERTIES (key1=value1, ... )] + +* data_desc: + + DATA INFILE ('file_path', ...) + [NEGATIVE] + INTO TABLE tbl_name + [PARTITION (p1, p2)] + [COLUMNS TERMINATED BY separator ] + [(col1, ...)] + [SET (k1=f1(xx), k2=f2(xx))] + [WHERE predicate] + +* broker_properties: + + (key1=value1, ...) +``` +示例： + +``` +LOAD LABEL db1.label1 +( + DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file1") + INTO TABLE tbl1 + COLUMNS TERMINATED BY "," + (tmp_c1,tmp_c2) + SET + ( + id=tmp_c2, + name=tmp_c1 + ), + DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file2") + INTO TABLE tbl2 + COLUMNS TERMINATED BY "," + (col1, col2) + where col1 > 1 +) +WITH BROKER 'broker' +( + "username"="user", + "password"="pass" +) +PROPERTIES +( + "timeout" = "3600" +); + +``` + +创建导入的详细语法执行 ```HELP BROKER LOAD``` 查看语法帮助。这里主要介绍 Broker load 的创建导入语法中参数意义和注意事项。 + +#### Label + +导入任务的标识。每个导入任务，都有一个在单 database 内部唯一的 Label。Label 是用户在导入命令中自定义的名称。通过这个 Label，用户可以查看对应导入任务的执行情况。 + +Label 的另一个作用，是防止用户重复导入相同的数据。**强烈推荐用户同一批次数据使用相同的label。这样同一批次数据的重复请求只会被接受一次，保证了 At-Most-Once 语义** + +当 Label 对应的导入作业状态为 CANCELLED 时，可以再次使用该 Label 提交导入作业。 + +#### 数据描述类参数 + +数据描述类参数主要指的是 Broker load 创建导入语句中的属于 ```data_desc``` 部分的参数。每组 ```data_desc ``` 主要表述了本次导入涉及到的数据源地址，ETL 函数，目标表及分区等信息。 + +下面主要对数据描述类的部分参数详细解释： + ++ 多表导入 + + Broker load 支持一次导入任务涉及多张表，每个 Broker load 导入任务可在多个 ``` data_desc ``` 声明多张表来实现多表导入。每个单独的 ```data_desc``` 还可以指定属于该表的数据源地址。Broker load 保证了单次导入的多张表之间原子性成功或失败。 + ++ negative + + ```data_desc```中还可以设置数据取反导入。这个功能主要用于，当数据表中聚合列的类型都为 SUM 类型时。如果希望撤销某一批导入的数据。则可以通过 `negative` 参数导入同一批数据。Doris 会自动为这一批数据在聚合列上数据取反，以达到消除同一批数据的功能。 + ++ partition + + 在 ```data_desc``` 中可以指定待导入表的 partition 信息，如果待导入数据不属于指定的 partition 则不会被导入。同时，不在指定 Partition 的数据会被认为是错误数据。 + ++ set column mapping + + 在 ```data_desc``` 中的 SET 语句负责设置列函数变换，这里的列函数变换支持所有查询的等值表达式变换。如果原始数据的列和表中的列不一一对应，就需要用到这个属性。 + ++ where predicate + + 在 ```data_desc``` 中的 WHERE 语句中负责过滤已经完成 transform 的数据，被 filter 的数据不会进入容忍率的统计中。如果多个 data_desc 中声明了同一张表的多个条件的话，则会 merge 同一张表的多个条件，merge 策略是 AND 。 + +#### 导入作业参数 + +导入作业参数主要指的是 Broker load 创建导入语句中的属于 ```opt_properties```部分的参数。导入作业参数是作用于整个导入作业的。 + +下面主要对导入作业参数的部分参数详细解释： + ++ timeout + + 导入作业的超时时间(以秒为单位)，用户可以在 ```opt_properties``` 中自行设置每个导入的超时时间。导入任务在设定的 timeout 时间内未完成则会被系统取消，变成 CANCELLED。Broker load 的默认导入超时时间为4小时。 + + 通常情况下，用户不需要手动设置导入任务的超时时间。当在默认超时时间内无法完成导入时，可以手动设置任务的超时时间。 + + > 推荐超时时间 + > + > 总文件大小（MB） / 用户 Doris 集群最慢导入速度(MB/s) > timeout > （（总文件大小(MB) * 待导入的表及相关 Roll up 表的个数） / (10 * 导入并发数） ） + + > 导入并发数见文档最后的导入系统配置说明，公式中的 10 为目前的导入限速 10MB/s。 + + > 例如一个 1G 的待导入数据，待导入表包含3个 Rollup 表，当前的导入并发数为 3。则 timeout 的 最小值为 ```(1 * 1024 * 3 ) / (10 * 3) = 102 秒``` + + 由于每个 Doris 集群的机器环境不同且集群并发的查询任务也不同，所以用户 Doris 集群的最慢导入速度需要用户自己根据历史的导入任务速度进行推测。 + ++ max\_filter\_ratio + + 导入任务的最大容忍率，默认为0容忍，取值范围是0~1。当导入的错误率超过该值，则导入失败。 + + 如果用户希望忽略错误的行，可以通过设置这个参数大于 0，来保证导入可以成功。 + + 计算公式为： + + ``` max_filter_ratio = (dpp.abnorm.ALL / (dpp.abnorm.ALL + dpp.norm.ALL ) ) ``` + + ```dpp.abnorm.ALL``` 表示数据质量不合格的行数。如类型不匹配，列数不匹配，长度不匹配等等。 + + ```dpp.norm.ALL``` 指的是导入过程中正确数据的条数。可以通过 ```SHOW LOAD``` 命令查询导入任务的正确数据量。 + + 原始文件的行数 = `dpp.abnorm.ALL + dpp.norm.ALL` + ++ exec\_mem\_limit + + 导入内存限制。默认是 2GB。单位为字节。 + ++ strict\_mode + + Broker load 导入可以开启 strict mode 模式。开启方式为 ```properties ("strict_mode" = "true")``` 。默认的 strict mode 为关闭。 + + strict mode 模式的意思是：对于导入过程中的列类型转换进行严格过滤。严格过滤的策略如下： + + 1. 对于列类型转换来说，如果 strict mode 为true，则错误的数据将被 filter。这里的错误数据是指：原始数据并不为空值，在参与列类型转换后结果为空值的这一类数据。 + + 2. 对于导入的某列由函数变换生成时，strict mode 对其不产生影响。 + + 3. 对于导入的某列类型包含范围限制的，如果原始数据能正常通过类型转换，但无法通过范围限制的，strict mode 对其也不产生影响。例如：如果类型是 decimal(1,0), 原始数据为 10，则属于可以通过类型转换但不在列声明的范围内。这种数据 strict 对其不产生影响。 + +#### strict mode 与 source data 的导入关系 + +这里以列类型为 TinyInt 来举例 + +>注：当表中的列允许导入空值时 + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|---------| +|空值 | \N | N/A | true or false | NULL| +|not null | aaa or 2000 | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 | 1 | true or false | correct data| + +这里以列类型为 Decimal(1,0) 举例 + +>注：当表中的列允许导入空值时 + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|--------| +|空值 | \N | N/A | true or false | NULL| +|not null | aaa | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 or 10 | 1 | true or false | correct data| + +> 注意：10 虽然是一个超过范围的值，但是因为其类型符合 decimal的要求，所以 strict mode对其不产生影响。10 最后会在其他 ETL 处理流程中被过滤。但不会被 strict mode 过滤。 + +#### Broker 参数 + +Broker Load 需要借助 Broker 进程访问远端存储，不同的 Broker 需要提供不同的参数，具体请参阅 [Broker文档](../broker.md) + +### 查看导入 + +Broker load 导入方式由于是异步的，所以用户必须将创建导入的 Label 记录，并且在**查看导入命令中使用 Label 来查看导入结果**。查看导入命令在所有导入方式中是通用的，具体语法可执行 ```HELP SHOW LOAD``` 查看。 + +示例： + +``` +mysql> show load order by createtime desc limit 1\G +*************************** 1. row *************************** + JobId: 76391 + Label: label1 + State: FINISHED + Progress: ETL:N/A; LOAD:100% + Type: BROKER + EtlInfo: unselected.rows=4; dpp.abnorm.ALL=15; dpp.norm.ALL=28133376 + TaskInfo: cluster:N/A; timeout(s):10800; max_filter_ratio:5.0E-5 + ErrorMsg: N/A + CreateTime: 2019-07-27 11:46:42 + EtlStartTime: 2019-07-27 11:46:44 + EtlFinishTime: 2019-07-27 11:46:44 + LoadStartTime: 2019-07-27 11:46:44 +LoadFinishTime: 2019-07-27 11:50:16 + URL: http://192.168.1.1:8040/api/_load_error_log?file=__shard_4/error_log_insert_stmt_4bb00753932c491a-a6da6e2725415317_4bb00753932c491a_a6da6e2725415317 + JobDetails: {"Unfinished backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"ScannedRows":2390016,"TaskNumber":1,"All backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"FileNumber":1,"FileSize":1073741824} +``` + +下面主要介绍了查看导入命令返回结果集中参数意义： + ++ JobId + + 导入任务的唯一ID，每个导入任务的 JobId 都不同，由系统自动生成。与 Label 不同的是，JobId永远不会相同，而 Label 则可以在导入任务失败后被复用。 + ++ Label + + 导入任务的标识。 + ++ State + + 导入任务当前所处的阶段。在 Broker load 导入过程中主要会出现 PENDING 和 LOADING 这两个导入中的状态。如果 Broker load 处于 PENDING 状态，则说明当前导入任务正在等待被执行；LOADING 状态则表示正在执行中。 + + 导入任务的最终阶段有两个：CANCELLED 和 FINISHED，当 Load job 处于这两个阶段时，导入完成。其中 CANCELLED 为导入失败，FINISHED 为导入成功。 + ++ Progress + + 导入任务的进度描述。分为两种进度：ETL 和 LOAD，对应了导入流程的两个阶段 ETL 和 LOADING。目前 Broker load 由于只有 LOADING 阶段，所以 ETL 则会永远显示为 `N/A` + + LOAD 的进度范围为：0~100%。 + + ```LOAD 进度 = 当前完成导入的表个数 / 本次导入任务设计的总表个数 * 100%``` + + **如果所有导入表均完成导入，此时 LOAD 的进度为 99%** 导入进入到最后生效阶段，整个导入完成后，LOAD 的进度才会改为 100%。 + + 导入进度并不是线性的。所以如果一段时间内进度没有变化，并不代表导入没有在执行。 + ++ Type + + 导入任务的类型。Broker load 的 type 取值只有 BROKER。 + ++ EtlInfo + + 主要显示了导入的数据量指标 ```unselected.rows``` , ```dpp.norm.ALL``` 和 ```dpp.abnorm.ALL```。用户可以根据第一个数值判断 where 条件过滤了多少行，后两个指标验证当前导入任务的错误率是否超过 ```max_filter_ratio```。 + + 三个指标之和就是原始数据量的总行数。 + ++ TaskInfo + + 主要显示了当前导入任务参数，也就是创建 Broker load 导入任务时用户指定的导入任务参数，包括：`cluster`，`timeout` 和`max_filter_ratio`。 + ++ ErrorMsg + + 在导入任务状态为CANCELLED，会显示失败的原因，显示分两部分：type 和 msg，如果导入任务成功则显示 ```N/A```。 + + type的取值意义： + + ``` + USER_CANCEL: 用户取消的任务 + ETL_RUN_FAIL：在ETL阶段失败的导入任务 + ETL_QUALITY_UNSATISFIED：数据质量不合格，也就是错误数据率超过了 max_filter_ratio + LOAD_RUN_FAIL：在LOADING阶段失败的导入任务 + TIMEOUT：导入任务没在超时时间内完成 + UNKNOWN：未知的导入错误 + ``` + ++ CreateTime/EtlStartTime/EtlFinishTime/LoadStartTime/LoadFinishTime + + 这几个值分别代表导入创建的时间，ETL阶段开始的时间，ETL阶段完成的时间，Loading阶段开始的时间和整个导入任务完成的时间。 + + Broker load 导入由于没有 ETL 阶段，所以其 EtlStartTime, EtlFinishTime, LoadStartTime 被设置为同一个值。 + + 导入任务长时间停留在 CreateTime，而 LoadStartTime 为 N/A 则说明目前导入任务堆积严重。用户可减少导入提交的频率。 + + ``` + LoadFinishTime - CreateTime = 整个导入任务所消耗时间 + LoadFinishTime - LoadStartTime = 整个 Broker load 导入任务执行时间 = 整个导入任务所消耗时间 - 导入任务等待的时间 + ``` + ++ URL + + 导入任务的错误数据样例，访问 URL 地址既可获取本次导入的错误数据样例。当本次导入不存在错误数据时，URL 字段则为 N/A。 + ++ JobDetails + + 显示一些作业的详细运行状态。包括导入文件的个数、总大小（字节）、子任务个数、已处理的原始行数，运行子任务的 BE 节点 Id，未完成的 BE 节点 Id。 + + ``` + {"Unfinished backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"ScannedRows":2390016,"TaskNumber":1,"All backends":{"9c3441027ff948a0-8287923329a2b6a7":[10002]},"FileNumber":1,"FileSize":1073741824} + ``` + + 其中已处理的原始行数，每 5 秒更新一次。该行数仅用于展示当前的进度，不代表最终实际的处理行数。实际处理行数以 EtlInfo 中显示的为准。 + +### 取消导入 + +当 Broker load 作业状态不为 CANCELLED 或 FINISHED 时，可以被用户手动取消。取消时需要指定待取消导入任务的 Label 。取消导入命令语法可执行 ```HELP CANCEL LOAD```查看。 + +## 相关系统配置 + +### FE 配置 + +下面几个配置属于 Broker load 的系统级别配置，也就是作用于所有 Broker load 导入任务的配置。主要通过修改 ``` fe.conf```来调整配置值。 + ++ min\_bytes\_per\_broker\_scanner/max\_bytes\_per\_broker\_scanner/max\_broker\_concurrency + + 前两个配置限制了单个 BE 处理的数据量的最小和最大值。第三个配置限制了一个作业的最大的导入并发数。最小处理的数据量，最大并发数，源文件的大小和当前集群 BE 的个数 **共同决定了本次导入的并发数**。 + + ``` + 本次导入并发数 = Math.min(源文件大小/最小处理量，最大并发数，当前BE节点个数) + 本次导入单个BE的处理量 = 源文件大小/本次导入的并发数 + ``` + + 通常一个导入作业支持的最大数据量为 `max_bytes_per_broker_scanner * BE 节点数`。如果需要导入更大数据量，则需要适当调整 `max_bytes_per_broker_scanner` 参数的大小。 + + 默认配置： + + ``` + 参数名：min_bytes_per_broker_scanner， 默认 64MB，单位bytes。 + 参数名：max_broker_concurrency， 默认 10。 + 参数名：max_bytes_per_broker_scanner，默认 3G，单位bytes。 + ``` + +## 最佳实践 + +### 应用场景 + +使用 Broker load 最适合的场景就是原始数据在文件系统（HDFS，BOS，AFS）中的场景。其次，由于 Broker load 是单次导入中唯一的一种异步导入的方式，所以如果用户在导入大文件中，需要使用异步接入，也可以考虑使用 Broker load。 + +### 数据量 + +这里仅讨论单个 BE 的情况，如果用户集群有多个 BE 则下面标题中的数据量应该乘以 BE 个数来计算。比如：如果用户有3个 BE，则 3G 以下（包含）则应该乘以 3，也就是 9G 以下（包含）。 + ++ 3G 以下（包含） + + 用户可以直接提交 Broker load 创建导入请求。 + ++ 3G 以上 + + 由于单个导入 BE 最大的处理量为 3G，超过 3G 的待导入文件就需要通过调整 Broker load 的导入参数来实现大文件的导入。 + + 1. 根据当前 BE 的个数和原始文件的大小修改单个 BE 的最大扫描量和最大并发数。 + + ``` + 修改 fe.conf 中配置 + + max_broker_concurrency = BE 个数 + 当前导入任务单个 BE 处理的数据量 = 原始文件大小 / max_broker_concurrency + max_bytes_per_broker_scanner >= 当前导入任务单个 BE 处理的数据量 + + 比如一个 100G 的文件，集群的 BE 个数为 10 个 + max_broker_concurrency = 10 + max_bytes_per_broker_scanner >= 10G = 100G / 10 + + ``` + + 修改后，所有的 BE 会并发的处理导入任务，每个 BE 处理原始文件的一部分。 + + *注意：上述两个 FE 中的配置均为系统配置，也就是说其修改是作用于所有的 Broker load的任务的。* + + 2. 在创建导入的时候自定义当前导入任务的 timeout 时间 + + ``` + 当前导入任务单个 BE 处理的数据量 / 用户 Doris 集群最慢导入速度(MB/s) >= 当前导入任务的 timeout 时间 >= 当前导入任务单个 BE 处理的数据量 / 10M/s + + 比如一个 100G 的文件，集群的 BE 个数为 10个 + timeout >= 1000s = 10G / 10M/s + + ``` + + 3. 当用户发现第二步计算出的 timeout 时间超过系统默认的导入最大超时时间 4小时 + + 这时候不推荐用户将导入最大超时时间直接改大来解决问题。单个导入时间如果超过默认的导入最大超时时间4小时，最好是通过切分待导入文件并且分多次导入来解决问题。主要原因是：单次导入超过4小时的话，导入失败后重试的时间成本很高。 + + 可以通过如下公式计算出 Doris 集群期望最大导入文件数据量： + + ``` + 期望最大导入文件数据量 = 14400s * 10M/s * BE 个数 + 比如：集群的 BE 个数为 10个 + 期望最大导入文件数据量 = 14400s * 10M/s * 10 = 1440000M ≈ 1440G + + 注意：一般用户的环境可能达不到 10M/s 的速度，所以建议超过 500G 的文件都进行文件切分，再导入。 + + ``` + +### 完整例子 + +数据情况：用户数据在 HDFS 中，文件地址为 hdfs://abc.com:8888/store_sales, hdfs 的认证用户名为 root, 密码为 password, 数据量大小约为 30G，希望导入到数据库 bj_sales 的表 store_sales 中。 + +集群情况：集群的 BE 个数约为 3 个，Broker 名称均为 broker。 + ++ step1: 经过上述方法的计算，本次导入的单个 BE 导入量为 10G，则需要先修改 FE 的配置，将单个 BE 导入最大量修改为： + + ``` + max_bytes_per_broker_scanner = 10737418240 + + ``` + ++ step2: 经计算，本次导入的时间大约为 1000s，并未超过默认超时时间，可不配置导入自定义超时时间。 + ++ step3：创建导入语句 + + ``` + LOAD LABEL bj_sales.store_sales_broker_load_01 + ( + DATA INFILE("hdfs://abc.com:8888/store_sales") + INTO TABLE store_sales + ) + WITH BROKER 'broker' + ("username"="root", "password"="password"); + ``` + +## 常见问题 + +* 导入报错：`Scan bytes per broker scanner exceed limit:xxx` + + 请参照文档中最佳实践部分，修改 FE 配置项 `max_bytes_per_broker_scanner` 和 `max_broker_concurrency` + +* 导入报错：`failed to send batch` 或 `TabletWriter add batch with unknown id` + + 请参照 [导入手册](./load-manual.md) 中 **通用系统配置** 中 **BE 配置**，适当修改 `query_timeout` 和 `streaming_load_rpc_max_alive_time_sec`。 + +* 导入报错：`LOAD_RUN_FAIL; msg:Invalid Column Name:xxx` + + 如果是PARQUET或者ORC格式的数据,需要再文件头的列名与doris表中的列名一致，如 : + ``` + (tmp_c1,tmp_c2) + SET + ( + id=tmp_c2, + name=tmp_c1 + ) + ``` + 代表获取在parquet或orc中以(tmp_c1, tmp_c2)为列名的列，映射到doris表中的(id, name)列。如果没有设置set, 则以column中的列作为映射。 + + 注：如果使用某些hive版本直接生成的orc文件，orc文件中的表头并非hive meta数据，而是（_col0, _col1, _col2, ...）, 可能导致Invalid Column Name错误，那么则需要使用set进行映射 + diff --git a/docs/zh-CN/administrator-guide/load-data/delete-manual.md b/docs/zh-CN/administrator-guide/load-data/delete-manual.md new file mode 100644 index 00000000000000..fd123cc12626cf --- /dev/null +++ b/docs/zh-CN/administrator-guide/load-data/delete-manual.md @@ -0,0 +1,183 @@ +--- +{ + "title": "Delete", + "language": "zh-CN" +} +--- + + + +# Delete + +Delete不同于其他导入方式，它是一个同步过程。和Insert into相似，所有的Delete操作在Doris中是一个独立的导入作业，一般Delete语句需要指定表和分区以及删除的条件来筛选要删除的数据，并将会同时删除base表和rollup表的数据。 + +## 语法 + +主要的Delete语法如下： + +``` +DELETE FROM table_name [PARTITION partition_name] +WHERE +column_name1 op value[ AND column_name2 op value ...]; +``` + +示例1： + +``` +DELETE FROM my_table PARTITION p1 WHERE k1 = 3; +``` + +示例2: + +``` +DELETE FROM my_table PARTITION p1 WHERE k1 < 3 AND k2 = "abc"; +``` + +下面介绍删除语句中使用到的参数： + +* PARTITION + + Delete语句的目标分区，若未指定，则此表必须为单分区表，否则无法delete + +* WHERE + + Delete语句的条件语句，所有删除语句都必须指定WHERE语句 + +说明： + +1. `Where`语句中的op的类型可包括`=,>,<,>=,<=,!=`，目前暂时不支持 where key in (value1, value2, value3) 的方式选定范围，后续将加上此功能。 +2. `Where`语句中的列只能是`key`列 +3. 当选定的`key`列不存在某个rollup表内时，无法进行delete +4. 条件语句中各个条件只能是`and`关系，如希望达成`or`可将条件分别写入两个delete语句中 +5. 如果指定表为RANGE分区表，则必须指定 `PARTITION`。如果是单分区表，可以不指定。 +6. 不同于Insert into命令，delete不能手动指定`label`，有关label的概念可以查看[Insert Into文档] (./insert-into-manual.md) + +## 返回结果 + +Delete命令是一个SQL命令，返回结果是同步的，分为以下几种： + +1. 执行成功 + + 如果Delete顺利执行完成并可见，将返回下列结果，`Query OK`表示成功 + + ``` + mysql> delete from test_tbl PARTITION p1 where k1 = 1; + Query OK, 0 rows affected (0.04 sec) + {'label':'delete_e7830c72-eb14-4cb9-bbb6-eebd4511d251', 'status':'VISIBLE', 'txnId':'4005'} + ``` + +2. 提交成功，但未可见 + + Doris的事务提交分为两步：提交和发布版本，只有完成了发布版本步骤，结果才对用户是可见的。若已经提交成功了，那么就可以认为最终一定会发布成功，Doris会尝试在提交完后等待发布一段时间，如果超时后即使发布版本还未完成也会优先返回给用户，提示用户提交已经完成。若如果Delete已经提交并执行，但是仍未发布版本和可见，将返回下列结果 + + ``` + mysql> delete from test_tbl PARTITION p1 where k1 = 1; + Query OK, 0 rows affected (0.04 sec) + {'label':'delete_e7830c72-eb14-4cb9-bbb6-eebd4511d251', 'status':'VISIBLE', 'txnId':'4005', 'err':'delete job is committed but may be taking effect later' } + ``` + + 结果会同时返回一个json字符串： + + `affected rows`表示此次删除影响的行，由于Doris的删除目前是逻辑删除，因此对于这个值是恒为0。 + + `label`为自动生成的 label，是该导入作业的标识。每个导入作业，都有一个在单 database 内部唯一的 Label。 + + `status`表示数据删除是否可见，如果可见，显示`VISIBLE`，如果不可见,显示`COMMITTED`。 + + `txnId`为这个Delete job对应的事务id + + `err`字段会显示一些本次删除的详细信息 + +3. 提交失败，事务取消 + + 如果Delete语句没有提交成功，将会被Doris自动中止，返回下列结果 + + ``` + mysql> delete from test_tbl partition p1 where k1 > 80; + ERROR 1064 (HY000): errCode = 2, detailMessage = {错误原因} + ``` + + 示例： + + 比如说一个超时的删除，将会返回timeout时间和未完成的`(tablet=replica)` + + ``` + mysql> delete from test_tbl partition p1 where k1 > 80; + ERROR 1064 (HY000): errCode = 2, detailMessage = failed to delete replicas from job: 4005, Unfinished replicas:10000=60000, 10001=60000, 10002=60000 + ``` + + **综上，对于Delete操作返回结果的正确处理逻辑为：** + + 1. 如果返回结果为`ERROR 1064 (HY000)`，则表示删除失败 + + 2. 如果返回结果为`Query OK`，则表示删除执行成功 + + 1. 如果`status`为`COMMITTED`，表示数据仍不可见，用户可以稍等一段时间再用`show delete`命令查看结果 + 2. 如果`status`为`VISIBLE`，表示数据删除成功。 + +## 可配置项 + +### FE配置 + +**TIMEOUT配置** + +总体来说，Doris的删除作业的超时时间限制在30秒到5分钟时间内，具体时间可通过下面配置项调整 + +* tablet\_delete\_timeout\_second + + delete自身的超时时间是可受指定分区下tablet的数量弹性改变的，此项配置为平均一个tablet所贡献的timeout时间，默认值为2。 + + 假设此次删除所指定分区下有5个tablet，那么可提供给delete的timeout时间为10秒，由于低于最低超时时间30秒，因此最终超时时间为30秒。 + +* load\_straggler\_wait\_second + + 如果用户预估的数据量确实比较大，使得5分钟的上限不足时，用户可以通过此项调整timeout上限，默认值为300。 + + **TIMEOUT的具体计算规则为(秒)** + + `TIMEOUT = MIN(load_straggler_wait_second, MAX(30, tablet_delete_timeout_second * tablet_num))` + +* query_timeout + + 因为delete本身是一个SQL命令，因此删除语句也会受session限制，timeout还受Session中的`query_timeout`值影响，可以通过`SET query_timeout = xxx`来增加超时时间，单位是秒。 + +## 查看历史记录 + +1. 用户可以通过show delete语句查看历史上已执行完成的删除记录 + + 语法 + + ``` + SHOW DELETE [FROM db_name] + ``` + + 示例 + + ``` + mysql> show delete from test_db; + +-----------+---------------+---------------------+-----------------+----------+ + | TableName | PartitionName | CreateTime | DeleteCondition | State | + +-----------+---------------+---------------------+-----------------+----------+ + | empty_tbl | p3 | 2020-04-15 23:09:35 | k1 EQ "1" | FINISHED | + | test_tbl | p4 | 2020-04-15 23:09:53 | k1 GT "80" | FINISHED | + +-----------+---------------+---------------------+-----------------+----------+ + 2 rows in set (0.00 sec) + ``` + diff --git a/docs/zh-CN/administrator-guide/load-data/insert-into-manual.md b/docs/zh-CN/administrator-guide/load-data/insert-into-manual.md new file mode 100644 index 00000000000000..1a69e5a6a4294e --- /dev/null +++ b/docs/zh-CN/administrator-guide/load-data/insert-into-manual.md @@ -0,0 +1,289 @@ +--- +{ + "title": "Insert Into", + "language": "zh-CN" +} +--- + + + +# Insert Into + +Insert Into 语句的使用方式和 MySQL 等数据库中 Insert Into 语句的使用方式类似。但在 Doris 中，所有的数据写入都是一个独立的导入作业。所以这里将 Insert Into 也作为一种导入方式介绍。 + +主要的 Insert Into 命令包含以下两种； + +* INSERT INTO tbl SELECT ... +* INSERT INTO tbl (col1, col2, ...) VALUES (1, 2, ...), (1,3, ...); + +其中第二种命令仅用于 Demo，不要使用在测试或生产环境中。 + +## 基本操作 + +### 创建导入 + +Insert Into 命令需要通过 MySQL 协议提交，创建导入请求会同步返回导入结果。 + +语法： + +``` +INSERT INTO table_name [WITH LABEL label] [partition_info] [col_list] [query_stmt] [VALUES]; +``` + +示例： + +``` +INSERT INTO tbl2 WITH LABEL label1 SELECT * FROM tbl3; +INSERT INTO tbl1 VALUES ("qweasdzxcqweasdzxc"), ("a"); +``` + +**注意** + +当需要使用 `CTE(Common Table Expressions)` 作为 insert 操作中的查询部分时，必须指定 `WITH LABEL` 和 column list 部分。示例 + +``` +INSERT INTO tbl1 WITH LABEL label1 +WITH cte1 AS (SELECT * FROM tbl1), cte2 AS (SELECT * FROM tbl2) +SELECT k1 FROM cte1 JOIN cte2 WHERE cte1.k1 = 1; + + +INSERT INTO tbl1 (k1) +WITH cte1 AS (SELECT * FROM tbl1), cte2 AS (SELECT * FROM tbl2) +SELECT k1 FROM cte1 JOIN cte2 WHERE cte1.k1 = 1; +``` + +下面主要介绍创建导入语句中使用到的参数： + ++ partition\_info + + 导入表的目标分区，如果指定目标分区，则只会导入符合目标分区的数据。如果没有指定，则默认值为这张表的所有分区。 + ++ col\_list + + 导入表的目标列，可以以任意的顺序存在。如果没有指定目标列，那么默认值是这张表的所有列。如果待表中的某个列没有存在目标列中，那么这个列需要有默认值，否则 Insert Into 就会执行失败。 + + 如果查询语句的结果列类型与目标列的类型不一致，那么会调用隐式类型转化，如果不能够进行转化，那么 Insert Into 语句会报语法解析错误。 + ++ query\_stmt + + 通过一个查询语句，将查询语句的结果导入到 Doris 系统中的其他表。查询语句支持任意 Doris 支持的 SQL 查询语法。 + ++ VALUES + + 用户可以通过 VALUES 语法插入一条或者多条数据。 + + *注意：VALUES 方式仅适用于导入几条数据作为导入 DEMO 的情况，完全不适用于任何测试和生产环境。Doris 系统本身也不适合单条数据导入的场景。建议使用 INSERT INTO SELECT 的方式进行批量导入。* + +* WITH LABEL + + INSERT 操作作为一个导入任务，也可以指定一个 label。如果不指定，则系统会自动指定一个 UUID 作为 label。 + + 该功能需要 0.11+ 版本。 + + *注意：建议指定 Label 而不是由系统自动分配。如果由系统自动分配，但在 Insert Into 语句执行过程中，因网络错误导致连接断开等，则无法得知 Insert Into 是否成功。而如果指定 Label，则可以再次通过 Label 查看任务结果。* + +### 导入结果 + +Insert Into 本身就是一个 SQL 命令，其返回结果会根据执行结果的不同，分为以下几种： + +1. 结果集为空 + + 如果 insert 对应 select 语句的结果集为空，则返回如下： + + ``` + mysql> insert into tbl1 select * from empty_tbl; + Query OK, 0 rows affected (0.02 sec) + ``` + + `Query OK` 表示执行成功。`0 rows affected` 表示没有数据被导入。 + +2. 结果集不为空 + + 在结果集不为空的情况下。返回结果分为如下几种情况： + + 1. Insert 执行成功并可见： + + ``` + mysql> insert into tbl1 select * from tbl2; + Query OK, 4 rows affected (0.38 sec) + {'label':'insert_8510c568-9eda-4173-9e36-6adc7d35291c', 'status':'visible', 'txnId':'4005'} + + mysql> insert into tbl1 with label my_label1 select * from tbl2; + Query OK, 4 rows affected (0.38 sec) + {'label':'my_label1', 'status':'visible', 'txnId':'4005'} + + mysql> insert into tbl1 select * from tbl2; + Query OK, 2 rows affected, 2 warnings (0.31 sec) + {'label':'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status':'visible', 'txnId':'4005'} + + mysql> insert into tbl1 select * from tbl2; + Query OK, 2 rows affected, 2 warnings (0.31 sec) + {'label':'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status':'committed', 'txnId':'4005'} + ``` + + `Query OK` 表示执行成功。`4 rows affected` 表示总共有4行数据被导入。`2 warnings` 表示被过滤的行数。 + + 同时会返回一个 json 串： + + ``` + {'label':'my_label1', 'status':'visible', 'txnId':'4005'} + {'label':'insert_f0747f0e-7a35-46e2-affa-13a235f4020d', 'status':'committed', 'txnId':'4005'} + {'label':'my_label1', 'status':'visible', 'txnId':'4005', 'err':'some other error'} + ``` + + `label` 为用户指定的 label 或自动生成的 label。Label 是该 Insert Into 导入作业的标识。每个导入作业，都有一个在单 database 内部唯一的 Label。 + + `status` 表示导入数据是否可见。如果可见，显示 `visible`，如果不可见，显示 `committed`。 + + `txnId` 为这个 insert 对应的导入事务的 id。 + + `err` 字段会显示一些其他非预期错误。 + + 当需要查看被过滤的行时，用户可以通过如下语句 + + ``` + show load where label="xxx"; + ``` + + 返回结果中的 URL 可以用于查询错误的数据，具体见后面 **查看错误行** 小结。 + + **数据不可见是一个临时状态，这批数据最终是一定可见的** + + 可以通过如下语句查看这批数据的可见状态： + + ``` + show transaction where id=4005; + ``` + + 返回结果中的 `TransactionStatus` 列如果为 `visible`，则表述数据可见。 + + 2. Insert 执行失败 + + 执行失败表示没有任何数据被成功导入，并返回如下： + + ``` + mysql> insert into tbl1 select * from tbl2 where k1 = "a"; + ERROR 1064 (HY000): all partitions have no load data. url: http://10.74.167.16:8042/api/_load_error_log?file=__shard_2/error_log_insert_stmt_ba8bb9e158e4879-ae8de8507c0bf8a2_ba8bb9e158e4879_ae8de8507c0bf8a2 + ``` + + 其中 `ERROR 1064 (HY000): all partitions have no load data` 显示失败原因。后面的 url 可以用于查询错误的数据，具体见后面 **查看错误行** 小结。 + + +**综上，对于 insert 操作返回结果的正确处理逻辑应为：** + +1. 如果返回结果为 `ERROR 1064 (HY000)`，则表示导入失败。 +2. 如果返回结果为 `Query OK`，则表示执行成功。 + 1. 如果 `rows affected` 为 0，表示结果集为空，没有数据被导入。 + 2. 如果 `rows affected` 大于 0： + 1. 如果 `status` 为 `committed`，表示数据还不可见。需要通过 `show transaction` 语句查看状态直到 `visible` + 2. 如果 `status` 为 `visible`，表示数据导入成功。 + 3. 如果 `warnings` 大于 0，表示有数据被过滤，可以通过 `show load` 语句获取 url 查看被过滤的行。 + +## 相关系统配置 + +### FE 配置 + ++ timeout + + 导入任务的超时时间(以秒为单位)，导入任务在设定的 timeout 时间内未完成则会被系统取消，变成 CANCELLED。 + + 目前 Insert Into 并不支持自定义导入的 timeout 时间，所有 Insert Into 导入的超时时间是统一的，默认的 timeout 时间为1小时。如果导入的源文件无法再规定时间内完成导入，则需要调整 FE 的参数```insert_load_default_timeout_second```。 + + 同时 Insert Into 语句收到 Session 变量 `query_timeout` 的限制。可以通过 `SET query_timeout = xxx;` 来增加超时时间，单位是秒。 + +### Session 变量 + ++ enable\_insert\_strict + + Insert Into 导入本身不能控制导入可容忍的错误率。用户只能通过 `enable_insert_strict` 这个 Session 参数用来控制。 + + 当该参数设置为 false 时，表示至少有一条数据被正确导入，则返回成功。如果有失败数据，则还会返回一个 Label。 + + 当该参数设置为 true 时，表示如果有一条数据错误，则导入失败。 + + 默认为 false。可通过 `SET enable_insert_strict = true;` 来设置。 + ++ query\_timeout + + Insert Into 本身也是一个 SQL 命令，因此 Insert Into 语句也受到 Session 变量 `query_timeout` 的限制。可以通过 `SET query_timeout = xxx;` 来增加超时时间，单位是秒。 + +## 最佳实践 + +### 应用场景 +1. 用户希望仅导入几条假数据，验证一下 Doris 系统的功能。此时适合使用 INSERT INTO VALUS 的语法。 +2. 用户希望将已经在 Doris 表中的数据进行 ETL 转换并导入到一个新的 Doris 表中，此时适合使用 INSERT INTO SELECT 语法。 +3. 用户可以创建一种外部表，如 MySQL 外部表映射一张 MySQL 系统中的表。或者创建 Broker 外部表来映射 HDFS 上的数据文件。然后通过 INSERT INTO SELECT 语法将外部表中的数据导入到 Doris 表中存储。 + +### 数据量 +Insert Into 对数据量没有限制，大数据量导入也可以支持。但 Insert Into 有默认的超时时间，用户预估的导入数据量过大，就需要修改系统的 Insert Into 导入超时时间。 + +``` +导入数据量 = 36G 约≤ 3600s * 10M/s +其中 10M/s 是最大导入限速，用户需要根据当前集群情况计算出平均的导入速度来替换公式中的 10M/s +``` + +### 完整例子 + +用户有一张表 store\_sales 在数据库 sales 中，用户又创建了一张表叫 bj\_store\_sales 也在数据库 sales 中，用户希望将 store\_sales 中销售记录在 bj 的数据导入到这张新建的表 bj\_store\_sales 中。导入的数据量约为：10G。 + +``` +store_sales schema： +(id, total, user_id, sale_timestamp, region) + +bj_store_sales schema: +(id, total, user_id, sale_timestamp) + +``` + +集群情况：用户当前集群的平均导入速度约为 5M/s + ++ Step1: 判断是否要修改 Insert Into 的默认超时时间 + + ``` + 计算导入的大概时间 + 10G / 5M/s = 2000s + + 修改 FE 配置 + insert_load_default_timeout_second = 2000 + ``` + ++ Step2：创建导入任务 + + 由于用户是希望将一张表中的数据做 ETL 并导入到目标表中，所以应该使用 Insert into query\_stmt 方式导入。 + + ``` + INSERT INTO bj_store_sales WITH LABEL `label` SELECT id, total, user_id, sale_timestamp FROM store_sales where region = "bj"; + ``` + +## 常见问题 + +* 查看错误行 + + 由于 Insert Into 无法控制错误率，只能通过 `enable_insert_strict` 设置为完全容忍错误数据或完全忽略错误数据。因此如果 `enable_insert_strict` 设为 true，则 Insert Into 可能会失败。而如果 `enable_insert_strict` 设为 false，则可能出现仅导入了部分合格数据的情况。 + + 当返回结果中提供了 url 字段时，可以通过以下命令查看错误行： + + ```SHOW LOAD WARNINGS ON "url";``` + + 示例： + + ```SHOW LOAD WARNINGS ON "http://ip:port/api/_load_error_log?file=__shard_13/error_log_insert_stmt_d2cac0a0a16d482d-9041c949a4b71605_d2cac0a0a16d482d_9041c949a4b71605";``` + + 错误的原因通常如：源数据列长度超过目的数据列长度、列类型不匹配、分区不匹配、列顺序不匹配等等。 diff --git a/docs/zh-CN/administrator-guide/load-data/load-manual.md b/docs/zh-CN/administrator-guide/load-data/load-manual.md new file mode 100644 index 00000000000000..fe0de251f832ee --- /dev/null +++ b/docs/zh-CN/administrator-guide/load-data/load-manual.md @@ -0,0 +1,217 @@ +--- +{ + "title": "导入总览", + "language": "zh-CN" +} +--- + + + +# 导入总览 + +导入（Load）功能就是将用户的原始数据导入到 Doris 中。导入成功后，用户即可通过 Mysql 客户端查询数据。 + +Doris 支持多种导入方式。建议先完整阅读本文档，再根据所选择的导入方式，查看各自导入方式的详细文档。 + +## 基本概念 + +1. Frontend（FE）：Doris 系统的元数据和调度节点。在导入流程中主要负责导入规划生成和导入任务的调度工作。 +2. Backend（BE）：Doris 系统的计算和存储节点。在导入流程中主要负责数据的 ETL 和存储。 +3. Broker：Broker 为一个独立的无状态进程。封装了文件系统接口，提供 Doris 读取远端存储系统中文件的能力。 +4. 导入作业（Load job）：导入作业读取用户提交的源数据，转换或清洗后，将数据导入到 Doris 系统中。导入完成后，数据即可被用户查询到。 +5. Label：所有导入作业都有一个 Label。Label 在一个数据库内唯一，可由用户指定或系统自动生成，用于标识一个导入作业。相同的 Label 仅可用于一个成功的导入作业。 +6. MySQL 协议/HTTP 协议：Doris 提供两种访问协议接口。 MySQL 协议和 HTTP 协议。部分导入方式使用 MySQL 协议接口提交作业，部分导入方式使用 HTTP 协议接口提交作业。 + +## 导入方式 + +为适配不同的数据导入需求，Doris 系统提供了5种不同的导入方式。每种导入方式支持不同的数据源，存在不同的使用方式（异步，同步）。 + +所有导入方式都支持 csv 数据格式。其中 Broker load 还支持 parquet 和 orc 数据格式。 + +每个导入方式的说明请参阅单个导入方式的操作手册。 + +* Broker load + + 通过 Broker 进程访问并读取外部数据源（如 HDFS）导入到 Doris。用户通过 Mysql 协议提交导入作业后，异步执行。通过 `SHOW LOAD` 命令查看导入结果。 + +* Stream load + + 用户通过 HTTP 协议提交请求并携带原始数据创建导入。主要用于快速将本地文件或数据流中的数据导入到 Doris。导入命令同步返回导入结果。 + +* Insert + + 类似 MySQL 中的 Insert 语句，Doris 提供 `INSERT INTO tbl SELECT ...;` 的方式从 Doris 的表中读取数据并导入到另一张表。或者通过 `INSERT INTO tbl VALUES(...);` 插入单条数据。 + +* Multi load + + 用户通过 HTTP 协议提交多个导入作业。Multi Load 可以保证多个导入作业的原子生效。 + +* Routine load + + 用户通过 MySQL 协议提交例行导入作业，生成一个常驻线程，不间断的从数据源（如 Kafka）中读取数据并导入到 Doris 中。 + +## 基本原理 + +### 导入执行流程 + +``` ++---------+ +---------+ +----------+ +-----------+ +| | | | | | | | +| PENDING +----->+ ETL +----->+ LOADING +----->+ FINISHED | +| | | | | | | | ++---------+ +---+-----+ +----+-----+ +-----------+ + | | | + | | | + | | | + | | | +-----------+ + | | | | | + +---------------+-----------------+------------> CANCELLED | + | | + +-----------+ + +``` + +如上图，一个导入作业主要经过上面4个阶段。 + ++ PENDING（非必须）: 该阶段只有 Broker Load 才有。Broker Load 被用户提交后会短暂停留在这个阶段，直到被 FE 中的 Scheduler 调度。 其中 Scheduler 的调度间隔为5秒。 + ++ ETL（非必须）： 该阶段在版本 0.10.0(包含) 之前存在，主要是用于将原始数据按照用户声明的方式进行变换，并且过滤不满足条件的原始数据。在 0.10.0 后的版本，ETL 阶段不再存在，其中数据 transform 的工作被合并到 LOADING 阶段。 + ++ LOADING： 该阶段在版本 0.10.0（包含）之前主要用于将变换后的数据推到对应的 BE 存储中。在 0.10.0 后的版本，该阶段先对数据进行清洗和变换，然后将数据发送到 BE 存储中。当所有导入数据均完成导入后，进入等待生效过程，此时 Load job 依旧是 LOADING。 + ++ FINISHED： 在 Load job 涉及的所有数据均生效后，Load job 的状态变成 FINISHED。FINISHED 后导入的数据均可查询。 + ++ CANCELLED: 在作业 FINISH 的之前，作业都可能被取消并进入 CANCELLED 状态。如用户手动取消，或导入出现错误等。CANCELLED 也是 Load Job 的最终状态，不可被再次执行。 + +上述阶段，除了 PENDING 到 LOADING 阶段是 Scheduler 轮训调度的，其他阶段之前的转移都是回调机制实现。 + +### Label 和 原子性 + +Doris 对所有导入方式提供原子性保证。既保证同一个导入作业内的数据，原子生效。不会出现仅导入部分数据的情况。 + +同时，每一个导入作业都有一个由用户指定或者系统自动生成的 Label。Label 在一个 Database 内唯一。当一个 Label 对应的导入作业成功后，不可再重复使用该 Label 提交导入作业。如果 Label 对应的导入作业失败，则可以重复使用。 + +用户可以通过 Label 机制，来保证 Label 对应的数据最多被导入一次，即At-Most-Once 语义。 + +## 同步和异步 + +Doris 目前的导入方式分为两类，同步和异步。如果是外部程序接入 Doris 的导入功能，需要判断使用导入方式是哪类再确定接入逻辑。 + +### 同步 + +同步导入方式即用户创建导入任务，Doris 同步执行导入，执行完成后返回用户导入结果。用户可直接根据创建导入任务命令返回的结果同步判断导入是否成功。 + +同步类型的导入方式有: **Stream load**，**Insert**。 + +操作步骤： + +1. 用户（外部系统）创建导入任务。 +2. Doris 返回导入结果。 +3. 用户（外部系统）判断导入结果，如果失败可以再次提交导入任务。 + +*注意：如果用户使用的导入方式是同步返回的，且导入的数据量过大，则创建导入请求可能会花很长时间才能返回结果。* + +### 异步 +异步导入方式即用户创建导入任务后，Doris 直接返回创建成功。**创建成功不代表数据已经导入**。导入任务会被异步执行，用户在创建成功后，需要通过轮询的方式发送查看命令查看导入作业的状态。如果创建失败，则可以根据失败信息，判断是否需要再次创建。 + +异步类型的导入方式有：**Broker load**，**Multi load**。 + +操作步骤： + +1. 用户（外部系统）创建导入任务。 +2. Doris 返回导入创建结果。 +3. 用户（外部系统）判断导入创建结果，成功则进入4，失败回到重试创建导入，回到1。 +4. 用户（外部系统）轮询查看导入任务，直到状态变为 FINISHED 或 CANCELLED。 + +### 注意事项 +无论是异步还是同步的导入类型，都不应该在 Doris 返回导入失败或导入创建失败后，无休止的重试。**外部系统在有限次数重试并失败后，保留失败信息，大部分多次重试均失败问题都是使用方法问题或数据本身问题。** + +## 内存限制 + +用户可以通过设置参数来限制单个导入的内存使用，以防止导入占用过多的内存而导致系统OOM。 +不同导入方式限制内存的方式略有不同，可以参阅各自的导入手册查看。 + +一个导入作业通常会分布在多个 Backend 上执行，导入内存限制的是一个导入作业，在单个 Backend 上的内存使用，而不是在整个集群的内存使用。 + +同时，每个 Backend 会设置可用于导入的内存的总体上限。具体配置参阅下面的通用系统配置小节。这个配置限制了所有在该 Backend 上运行的导入任务的总体内存使用上限。 + +较小的内存限制可能会影响导入效率，因为导入流程可能会因为内存达到上限而频繁的将内存中的数据写回磁盘。而过大的内存限制可能导致当导入并发较高时，系统OOM。所以，需要根据需求，合理的设置导入的内存限制。 + +## 最佳实践 + +用户在接入 Doris 导入时，一般会采用程序接入的方式，这样可以保证数据被定期的导入到 Doris 中。下面主要说明了程序接入 Doris 的最佳实践。 + +1. 选择合适的导入方式：根据数据源所在位置选择导入方式。例如：如果原始数据存放在 HDFS 上，则使用 Broker load 导入。 +2. 确定导入方式的协议：如果选择了 Broker load 导入方式，则外部系统需要能使用 MySQL 协议定期提交和查看导入作业。 +3. 确定导入方式的类型：导入方式为同步或异步。比如 Broker load 为异步导入方式，则外部系统在提交创建导入后，必须调用查看导入命令，根据查看导入命令的结果来判断导入是否成功。 +4. 制定 Label 生成策略：Label 生成策略需满足，每一批次数据唯一且固定的原则。这样 Doris 就可以保证 At-Most-Once。 +5. 程序自身保证 At-Least-Once：外部系统需要保证自身的 At-Least-Once，这样就可以保证导入流程的 Exactly-Once。 + +## 通用系统配置 + +下面主要解释了几个所有导入方式均通用的系统级别的配置。 + +### FE 配置 + +以下配置属于 FE 的系统配置，可以通过修改 FE 的配置文件 ```fe.conf``` 来修改配置。 + ++ max\_load\_timeout\_second 和 min\_load\_timeout\_second + + 这两个配置含义为：最大的导入超时时间，最小的导入超时时间，以秒为单位。默认的最大超时时间为3天, 默认的最小超时时间为1秒。用户自定义的导入超时时间不可超过这个范围。该参数通用于所有的导入方式。 + ++ desired\_max\_waiting\_jobs + + 在等待队列中的导入任务个数最大值，默认为100。当在 FE 中处于 PENDING 状态（也就是等待执行的）导入个数超过该值，新的导入请求则会被拒绝。 + + 此配置仅对异步执行的导入有效，当异步执行的导入等待个数超过默认值，则后续的创建导入请求会被拒绝。 + ++ max\_running\_txn\_num\_per\_db + + 这个配置的含义是说，每个 Database 中正在运行的导入最大个数（不区分导入类型，统一计数）。当当前 Database 正在运行的导入个数超过最大值时，后续的导入不会被执行。如果是同步导入作业，则导入会被拒绝。如果是异步导入作业。则作业会在队列中等待。 + +### BE 配置 + +以下配置属于 BE 的系统配置，可以通过修改 BE 的配置文件 ```be.conf``` 来修改配置。 + ++ push\_write\_mbytes\_per\_sec + + BE 上单个 Tablet 的写入速度限制。默认是 10，即 10MB/s。通常 BE 对单个 Tablet 的最大写入速度，根据 Schema 以及系统的不同，大约在 10-30MB/s 之间。可以适当调整这个参数来控制导入速度。 + ++ write\_buffer\_size + + 导入数据在 BE 上会先写入一个 memtable，memtable 达到阈值后才会写回磁盘。默认大小是 100MB。过小的阈值可能导致 BE 上存在大量的小文件。可以适当提高这个阈值减少文件数量。但过大的阈值可能导致 RPC 超时，见下面的配置说明。 + ++ tablet\_writer\_rpc\_timeout\_sec + + 导入过程中，发送一个 Batch（1024行）的 RPC 超时时间。默认 600 秒。因为该 RPC 可能涉及多个 memtable 的写盘操作，所以可能会因为写盘导致 RPC 超时，可以适当调整这个超时时间来减少超时错误（如 `send batch fail` 错误）。同时，如果调大 `write_buffer_size` 配置，也需要适当调大这个参数。 + ++ streaming\_load\_rpc\_max\_alive\_time\_sec + + 在导入过程中，Doris 会为每一个 Tablet 开启一个 Writer，用于接收数据并写入。这个参数指定了 Writer 的等待超时时间。如果在这个时间内，Writer 没有收到任何数据，则 Writer 会被自动销毁。当系统处理速度较慢时，Writer 可能长时间接收不到下一批数据，导致导入报错：`TabletWriter add batch with unknown id`。此时可适当增大这个配置。默认为 600 秒。 + +* load\_process\_max\_memory\_limit\_bytes 和 load\_process\_max\_memory\_limit\_percent + + 这两个参数，限制了单个 Backend 上，可用于导入任务的内存上限。分别是最大内存和最大内存百分比。`load_process_max_memory_limit_percent` 默认为 80，表示对 Backend 总内存限制的百分比（总内存限制 `mem_limit` 默认为 80%，表示对物理内存的百分比）。即假设物理内存为 M，则默认导入内存限制为 M * 80% * 80%。 + + `load_process_max_memory_limit_bytes` 默认为 100GB。系统会在两个参数中取较小者，作为最终的 Backend 导入内存使用上限。 + ++ label\_keep\_max\_second + + 设置导入任务记录保留时间。已经完成的（ FINISHED or CANCELLED ）导入任务记录会保留在 Doris 系统中一段时间，时间由此参数决定。参数默认值时间为3天。该参数通用与所有类型的导入任务。 diff --git a/docs/zh-CN/administrator-guide/load-data/routine-load-manual.md b/docs/zh-CN/administrator-guide/load-data/routine-load-manual.md new file mode 100644 index 00000000000000..3925cd89b407af --- /dev/null +++ b/docs/zh-CN/administrator-guide/load-data/routine-load-manual.md @@ -0,0 +1,299 @@ +--- +{ + "title": "Routine Load", + "language": "zh-CN" +} +--- + + + +# Routine Load + +例行导入（Routine Load）功能为用户提供了一种自动从指定数据源进行数据导入的功能。 + +本文档主要介绍该功能的实现原理、使用方式以及最佳实践。 + +## 名词解释 + +* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 +* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 +* RoutineLoadJob：用户提交的一个例行导入作业。 +* JobScheduler：例行导入作业调度器，用于调度和拆分一个 RoutineLoadJob 为多个 Task。 +* Task：RoutineLoadJob 被 JobScheduler 根据规则拆分的子任务。 +* TaskScheduler：任务调度器。用于调度 Task 的执行。 + +## 原理 + +``` + +---------+ + | Client | + +----+----+ + | ++-----------------------------+ +| FE | | +| +-----------v------------+ | +| | | | +| | Routine Load Job | | +| | | | +| +---+--------+--------+--+ | +| | | | | +| +---v--+ +---v--+ +---v--+ | +| | task | | task | | task | | +| +--+---+ +---+--+ +---+--+ | +| | | | | ++-----------------------------+ + | | | + v v v + +---+--+ +--+---+ ++-----+ + | BE | | BE | | BE | + +------+ +------+ +------+ + +``` + +如上图，Client 向 FE 提交一个例行导入作业。 + +FE 通过 JobScheduler 将一个导入作业拆分成若干个 Task。每个 Task 负责导入指定的一部分数据。Task 被 TaskScheduler 分配到指定的 BE 上执行。 + +在 BE 上，一个 Task 被视为一个普通的导入任务，通过 Stream Load 的导入机制进行导入。导入完成后，向 FE 汇报。 + +FE 中的 JobScheduler 根据汇报结果，继续生成后续新的 Task，或者对失败的 Task 进行重试。 + +整个例行导入作业通过不断的产生新的 Task，来完成数据不间断的导入。 + +## Kafka 例行导入 + +当前我们仅支持从 Kafka 系统进行例行导入。该部分会详细介绍 Kafka 例行导入使用方式和最佳实践。 + +### 使用限制 + +1. 支持无认证的 Kafka 访问，以及通过 SSL 方式认证的 Kafka 集群。 +2. 支持的消息格式为 csv 文本格式。每一个 message 为一行，且行尾**不包含**换行符。 +3. 仅支持 Kafka 0.10.0.0(含) 以上版本。 + +### 创建例行导入任务 + +创建例行导入任务的的详细语法可以连接到 Doris 后，执行 `HELP ROUTINE LOAD;` 查看语法帮助。这里主要详细介绍，创建作业时的注意事项。 + +* columns_mapping + + `columns_mapping` 主要用于指定表结构和 message 中的列映射关系，以及一些列的转换。如果不指定，Doris 会默认 message 中的列和表结构的列按顺序一一对应。虽然在正常情况下，如果源数据正好一一对应，则不指定也可以进行正常的数据导入。但是我们依然强烈建议用户**显式的指定列映射关系**。这样当表结构发生变化（比如增加一个 nullable 的列），或者源文件发生变化（比如增加了一列）时，导入任务依然可以继续进行。否则，当发生上述变动后，因为列映射关系不再一一对应，导入将报错。 + + 在 `columns_mapping` 中我们同样可以使用一些内置函数进行列的转换。但需要注意函数参数对应的实际列类型。举例说明： + + 假设用户需要导入只包含 `k1` 一列的表，列类型为 `int`。并且需要将源文件中的 null 值转换为 0。该功能可以通过 `ifnull` 函数实现。正确是的使用方式如下： + + `COLUMNS (xx, k1=ifnull(xx, "3"))` + + 注意这里我们使用 `"3"` 而不是 `3`，虽然 `k1` 的类型为 `int`。因为对于导入任务来说，源数据中的列类型都为 `varchar`，所以这里 `xx` 虚拟列的类型也为 `varchar`。所以我们需要使用 `"3"` 来进行对应的匹配，否则 `ifnull` 函数无法找到参数为 `(varchar, int)` 的函数签名，将出现错误。 + + 再举例，假设用户需要导入只包含 `k1` 一列的表，列类型为 `int`。并且需要将源文件中的对应列进行处理：将负数转换为正数，而将正数乘以 100。这个功能可以通过 `case when` 函数实现，正确写法应如下： + + `COLUMNS (xx, case when xx < 0 than cast(-xx as varchar) else cast((xx + '100') as varchar) end)` + + 注意这里我们需要将 `case when` 中所有的参数都最终转换为 varchar，才能得到期望的结果。 + +* where_predicates + + `where_predicates` 中的的列的类型，已经是实际的列类型了，所以无需向 `columns_mapping` 那样强制的转换为 varchar 类型。按照实际的列类型书写即可。 + +* desired\_concurrent\_number + + `desired_concurrent_number` 用于指定一个例行作业期望的并发度。即一个作业，最多有多少 task 同时在执行。对于 Kafka 导入而言，当前的实际并发度计算如下： + + ``` + Min(partition num, desired_concurrent_number, alive_backend_num, Config.max_routine_load_task_concurrrent_num) + ``` + + 其中 `Config.max_routine_load_task_concurrrent_num` 是系统的一个默认的最大并发数限制。这是一个 FE 配置，可以通过改配置调整。默认为 5。 + + 其中 partition num 指订阅的 Kafka topic 的 partition 数量。`alive_backend_num` 是当前正常的 BE 节点数。 + +* max\_batch\_interval/max\_batch\_rows/max\_batch\_size + + 这三个参数用于控制单个任务的执行时间。其中任意一个阈值达到，则任务结束。其中 `max_batch_rows` 用于记录从 Kafka 中读取到的数据行数。`max_batch_size` 用于记录从 Kafka 中读取到的数据量，单位是字节。目前一个任务的消费速率大约为 5-10MB/s。 + + 那么假设一行数据 500B，用户希望每 100MB 或 10 秒为一个 task。100MB 的预期处理时间是 10-20 秒，对应的行数约为 200000 行。则一个合理的配置为： + + ``` + "max_batch_interval" = "10", + "max_batch_rows" = "200000", + "max_batch_size" = "104857600" + ``` + + 以上示例中的参数也是这些配置的默认参数。 + +* max\_error\_number + + `max_error_number` 用于控制错误率。在错误率过高的时候，作业会自动暂停。因为整个作业是面向数据流的，且由于数据流的无边界性，我们无法像其他导入任务一样，通过一个错误比例来计算错误率。因此这里提供了一种新的计算方式，来计算数据流中的错误比例。 + + 我们设定了一个采样窗口。窗口的大小为 `max_batch_rows * 10`。在一个采样窗口内，如果错误行数超过 `max_error_number`，则作业被暂停。如果没有超过，则下一个窗口重新开始计算错误行数。 + + 我们假设 `max_batch_rows` 为 200000，则窗口大小为 2000000。设 `max_error_number` 为 20000，即用户预期每 2000000 行的错误行为 20000。即错误率为 1%。但是因为不是每批次任务正好消费 200000 行，所以窗口的实际范围是 [2000000, 2200000]，即有 10% 的统计误差。 + + 错误行不包括通过 where 条件过滤掉的行。但是包括没有对应的 Doris 表中的分区的行。 + +* data\_source\_properties + + `data_source_properties` 中可以指定消费具体的 Kakfa partition。如果不指定，则默认消费所订阅的 topic 的所有 partition。 + + 注意，当显式的指定了 partition，则导入作业不会再动态的检测 Kafka partition 的变化。如果没有指定，则会根据 kafka partition 的变化，动态调整需要消费的 partition。 + +* strict\_mode + + Routine load 导入可以开启 strict mode 模式。开启方式为在 job\_properties 中增加 ```"strict_mode" = "true"``` 。默认的 strict mode 为关闭。 + + strict mode 模式的意思是：对于导入过程中的列类型转换进行严格过滤。严格过滤的策略如下： + + 1. 对于列类型转换来说，如果 strict mode 为true，则错误的数据将被 filter。这里的错误数据是指：原始数据并不为空值，在参与列类型转换后结果为空值的这一类数据。 + + 2. 对于导入的某列由函数变换生成时，strict mode 对其不产生影响。 + + 3. 对于导入的某列类型包含范围限制的，如果原始数据能正常通过类型转换，但无法通过范围限制的，strict mode 对其也不产生影响。例如：如果类型是 decimal(1,0), 原始数据为 10，则属于可以通过类型转换但不在列声明的范围内。这种数据 strict 对其不产生影响。 + +#### strict mode 与 source data 的导入关系 + +这里以列类型为 TinyInt 来举例 + +>注：当表中的列允许导入空值时 + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|---------| +|空值 | \N | N/A | true or false | NULL| +|not null | aaa or 2000 | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 | 1 | true or false | correct data| + +这里以列类型为 Decimal(1,0) 举例 + +>注：当表中的列允许导入空值时 + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|--------| +|空值 | \N | N/A | true or false | NULL| +|not null | aaa | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 or 10 | 1 | true or false | correct data| + +> 注意：10 虽然是一个超过范围的值，但是因为其类型符合 decimal的要求，所以 strict mode对其不产生影响。10 最后会在其他 ETL 处理流程中被过滤。但不会被 strict mode 过滤。 + +#### 访问 SSL 认证的 Kafka 集群 + +访问 SSL 认证的 Kafka 集群需要用户提供用于认证 Kafka Broker 公钥的证书文件（ca.pem）。如果 Kafka 集群同时开启了客户端认证，则还需提供客户端的公钥（client.pem）、密钥文件（client.key），以及密钥密码。这里所需的文件需要先通过 `CREAE FILE` 命令上传到 Doris 中，**并且 catalog 名称为 `kafka`**。`CREATE FILE` 命令的具体帮助可以参见 `HELP CREATE FILE;`。这里给出示例： + +1. 上传文件 + + ``` + CREATE FILE "ca.pem" PROPERTIES("url" = "https://example_url/kafka-key/ca.pem", "catalog" = "kafka"); + CREATE FILE "client.key" PROPERTIES("url" = "https://example_urlkafka-key/client.key", "catalog" = "kafka"); + CREATE FILE "client.pem" PROPERTIES("url" = "https://example_url/kafka-key/client.pem", "catalog" = "kafka"); + ``` + +2. 创建例行导入作业 + + ``` + CREATE ROUTINE LOAD db1.job1 on tbl1 + PROPERTIES + ( + "desired_concurrent_number"="1" + ) + FROM KAFKA + ( + "kafka_broker_list"= "broker1:9091,broker2:9091", + "kafka_topic" = "my_topic", + "property.security.protocol" = "ssl", + "property.ssl.ca.location" = "FILE:ca.pem", + "property.ssl.certificate.location" = "FILE:client.pem", + "property.ssl.key.location" = "FILE:client.key", + "property.ssl.key.password" = "abcdefg" + ); + ``` + +> Doris 通过 Kafka 的 C++ API `librdkafka` 来访问 Kafka 集群。`librdkafka` 所支持的参数可以参阅 +> +> `https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md` + + +### 查看导入作业状态 + +查看**作业**状态的具体命令和示例可以通过 `HELP SHOW ROUTINE LOAD;` 命令查看。 + +查看**任务**运行状态的具体命令和示例可以通过 `HELP SHOW ROUTINE LOAD TASK;` 命令查看。 + +只能查看当前正在运行中的任务，已结束和未开始的任务无法查看。 + +### 作业控制 + +用户可以通过 `STOP/PAUSE/RESUME` 三个命令来控制作业的停止，暂停和重启。可以通过 `HELP STOP ROUTINE LOAD;`, `HELP PAUSE ROUTINE LOAD;` 以及 `HELP RESUME ROUTINE LOAD;` 三个命令查看帮助和示例。 + +## 其他说明 + +1. 例行导入作业和 ALTER TABLE 操作的关系 + + * 例行导入不会阻塞 SCHEMA CHANGE 和 ROLLUP 操作。但是注意如果 SCHEMA CHANGE 完成后，列映射关系无法匹配，则会导致作业的错误数据激增，最终导致作业暂停。建议通过在例行导入作业中显式指定列映射关系，以及通过增加 Nullable 列或带 Default 值的列来减少这类问题。 + * 删除表的 Partition 可能会导致导入数据无法找到对应的 Partition，作业进入暂停。 + +2. 例行导入作业和其他导入作业的关系（LOAD, DELETE, INSERT） + + * 例行导入和其他 LOAD 作业以及 INSERT 操作没有冲突。 + * 当执行 DELETE 操作时，对应表分区不能有任何正在执行的导入任务。所以在执行 DELETE 操作前，可能需要先暂停例行导入作业，并等待已下发的 task 全部完成后，才可以执行 DELETE。 + +3. 例行导入作业和 DROP DATABASE/TABLE 操作的关系 + + 当例行导入对应的 database 或 table 被删除后，作业会自动 CANCEL。 + +4. kafka 类型的例行导入作业和 kafka topic 的关系 + + 当用户在创建例行导入声明的 `kafka_topic` 在kafka集群中不存在时。 + + * 如果用户 kafka 集群的 broker 设置了 `auto.create.topics.enable = true`，则 `kafka_topic` 会先被自动创建，自动创建的 partition 个数是由**用户方的kafka集群**中的 broker 配置 `num.partitions` 决定的。例行作业会正常的不断读取该 topic 的数据。 + * 如果用户 kafka 集群的 broker 设置了 `auto.create.topics.enable = false`, 则 topic 不会被自动创建，例行作业会在没有读取任何数据之前就被暂停，状态为 `PAUSED`。 + + 所以，如果用户希望当 kafka topic 不存在的时候，被例行作业自动创建的话，只需要将**用户方的kafka集群**中的 broker 设置 `auto.create.topics.enable = true` 即可。 + +## 相关参数 + +一些系统配置参数会影响例行导入的使用。 + +1. max\_routine\_load\_task\_concurrent\_num + + FE 配置项，默认为 5，可以运行时修改。该参数限制了一个例行导入作业最大的子任务并发数。建议维持默认值。设置过大，可能导致同时并发的任务数过多，占用集群资源。 + +2. max\_routine_load\_task\_num\_per\_be + + FE 配置项，默认为5，可以运行时修改。该参数限制了每个 BE 节点最多并发执行的子任务个数。建议维持默认值。如果设置过大，可能导致并发任务数过多，占用集群资源。 + +3. max\_routine\_load\_job\_num + + FE 配置项，默认为100，可以运行时修改。该参数限制的例行导入作业的总数，包括 NEED_SCHEDULED, RUNNING, PAUSE 这些状态。超过后，不能在提交新的作业。 + +4. max\_consumer\_num\_per\_group + + BE 配置项，默认为 3。该参数表示一个子任务中最多生成几个 consumer 进行数据消费。对于 Kafka 数据源，一个 consumer 可能消费一个或多个 kafka partition。假设一个任务需要消费 6 个 kafka partition，则会生成 3 个 consumer，每个 consumer 消费 2 个 partition。如果只有 2 个 partition，则只会生成 2 个 consumer，每个 consumer 消费 1 个 partition。 + +5. push\_write\_mbytes\_per\_sec + + BE 配置项。默认为 10，即 10MB/s。该参数为导入通用参数，不限于例行导入作业。该参数限制了导入数据写入磁盘的速度。对于 SSD 等高性能存储设备，可以适当增加这个限速。 + +6. max\_tolerable\_backend\_down\_num + FE 配置项，默认值是0。在满足某些条件下，Doris可PAUSED的任务重新调度，即变成RUNNING。该参数为0代表只有所有BE节点是alive状态才允许重新调度。 + +7. period\_of\_auto\_resume\_min + FE 配置项，默认是5分钟。Doris重新调度，只会在5分钟这个周期内，最多尝试3次. 如果3次都失败则锁定当前任务，后续不在进行调度。但可通过人为干预，进行手动恢复。 + diff --git a/docs/zh-CN/administrator-guide/load-data/stream-load-manual.md b/docs/zh-CN/administrator-guide/load-data/stream-load-manual.md new file mode 100644 index 00000000000000..345eb357a7764c --- /dev/null +++ b/docs/zh-CN/administrator-guide/load-data/stream-load-manual.md @@ -0,0 +1,337 @@ +--- +{ + "title": "Stream load", + "language": "zh-CN" +} +--- + + + +# Stream load + +Stream load 是一个同步的导入方式，用户通过发送 HTTP 协议发送请求将本地文件或数据流导入到 Doris 中。Stream load 同步执行导入并返回导入结果。用户可直接通过请求的返回体判断本次导入是否成功。 + +Stream load 主要适用于导入本地文件，或通过程序导入数据流中的数据。 + +## 基本原理 + +下图展示了 Stream load 的主要流程，省略了一些导入细节。 + +``` + ^ + + | | + | | 1A. User submit load to FE + | | + | +--v-----------+ + | | FE | +5. Return result to user | +--+-----------+ + | | + | | 2. Redirect to BE + | | + | +--v-----------+ + +---+Coordinator BE| 1B. User submit load to BE + +-+-----+----+-+ + | | | + +-----+ | +-----+ + | | | 3. Distrbute data + | | | + +-v-+ +-v-+ +-v-+ + |BE | |BE | |BE | + +---+ +---+ +---+ +``` + +Stream load 中，Doris 会选定一个节点作为 Coordinator 节点。该节点负责接数据并分发数据到其他数据节点。 + +用户通过 HTTP 协议提交导入命令。如果提交到 FE，则 FE 会通过 HTTP redirect 指令将请求转发给某一个 BE。用户也可以直接提交导入命令给某一指定 BE。 + +导入的最终结果由 Coordinator BE 返回给用户。 + +## 基本操作 +### 创建导入 + +Stream load 通过 HTTP 协议提交和传输数据。这里通过 `curl` 命令展示如何提交导入。 + +用户也可以通过其他 HTTP client 进行操作。 + +``` +curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_host:http_port/api/{db}/{table}/_stream_load + +Header 中支持属性见下面的 ‘导入任务参数’ 说明 +格式为: -H "key1:value1" +``` + +示例： + +``` +curl --location-trusted -u root -T date -H "label:123" http://abc.com:8030/api/test/date/_stream_load +``` +创建导入的详细语法帮助执行 ```HELP STREAM LOAD``` 查看, 下面主要介绍创建 Stream load 的部分参数意义。 + +#### 签名参数 + ++ user/passwd + + Stream load 由于创建导入的协议使用的是 HTTP 协议，通过 Basic access authentication 进行签名。Doris 系统会根据签名验证用户身份和导入权限。 + +#### 导入任务参数 + +Stream load 由于使用的是 HTTP 协议，所以所有导入任务有关的参数均设置在 Header 中。下面主要介绍了 Stream load 导入任务参数的部分参数意义。 + ++ label + + 导入任务的标识。每个导入任务，都有一个在单 database 内部唯一的 label。label 是用户在导入命令中自定义的名称。通过这个 label，用户可以查看对应导入任务的执行情况。 + + label 的另一个作用，是防止用户重复导入相同的数据。**强烈推荐用户同一批次数据使用相同的 label。这样同一批次数据的重复请求只会被接受一次，保证了 At-Most-Once** + + 当 label 对应的导入作业状态为 CANCELLED 时，该 label 可以再次被使用。 + ++ max\_filter\_ratio + + 导入任务的最大容忍率，默认为0容忍，取值范围是0~1。当导入的错误率超过该值，则导入失败。 + + 如果用户希望忽略错误的行，可以通过设置这个参数大于 0，来保证导入可以成功。 + + 计算公式为： + + ``` (dpp.abnorm.ALL / (dpp.abnorm.ALL + dpp.norm.ALL ) ) > max_filter_ratio ``` + + ```dpp.abnorm.ALL``` 表示数据质量不合格的行数。如类型不匹配，列数不匹配，长度不匹配等等。 + + ```dpp.norm.ALL``` 指的是导入过程中正确数据的条数。可以通过 ```SHOW LOAD``` 命令查询导入任务的正确数据量。 + + 原始文件的行数 = `dpp.abnorm.ALL + dpp.norm.ALL` + ++ where + + 导入任务指定的过滤条件。Stream load 支持对原始数据指定 where 语句进行过滤。被过滤的数据将不会被导入，也不会参与 filter ratio 的计算，但会被计入```num_rows_unselected```。 + ++ partition + + 待导入表的 Partition 信息，如果待导入数据不属于指定的 Partition 则不会被导入。这些数据将计入 ```dpp.abnorm.ALL ``` + ++ columns + + 待导入数据的函数变换配置，目前 Stream load 支持的函数变换方法包含列的顺序变化以及表达式变换，其中表达式变换的方法与查询语句的一致。 + + ``` + 列顺序变换例子：原始数据有两列，目前表也有两列（c1,c2）但是原始文件的第一列对应的是目标表的c2列, 而原始文件的第二列对应的是目标表的c1列，则写法如下： + columns: c2,c1 + + 表达式变换例子：原始文件有两列，目标表也有两列（c1,c2）但是原始文件的两列均需要经过函数变换才能对应目标表的两列，则写法如下： + columns: tmp_c1, tmp_c2, c1 = year(tmp_c1), c2 = month(tmp_c2) + 其中 tmp_*是一个占位符，代表的是原始文件中的两个原始列。 + ``` + ++ exec\_mem\_limit + + 导入内存限制。默认为 2GB，单位为字节。 + ++ strict\_mode + + Stream load 导入可以开启 strict mode 模式。开启方式为在 HEADER 中声明 ```strict_mode=true``` 。默认的 strict mode 为关闭。 + + strict mode 模式的意思是：对于导入过程中的列类型转换进行严格过滤。严格过滤的策略如下： + + 1. 对于列类型转换来说，如果 strict mode 为true，则错误的数据将被 filter。这里的错误数据是指：原始数据并不为空值，在参与列类型转换后结果为空值的这一类数据。 + + 2. 对于导入的某列由函数变换生成时，strict mode 对其不产生影响。 + + 3. 对于导入的某列类型包含范围限制的，如果原始数据能正常通过类型转换，但无法通过范围限制的，strict mode 对其也不产生影响。例如：如果类型是 decimal(1,0), 原始数据为 10，则属于可以通过类型转换但不在列声明的范围内。这种数据 strict 对其不产生影响。 + +#### strict mode 与 source data 的导入关系 + +这里以列类型为 TinyInt 来举例 + +>注：当表中的列允许导入空值时 + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|---------| +|空值 | \N | N/A | true or false | NULL| +|not null | aaa or 2000 | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 | 1 | true or false | correct data| + +这里以列类型为 Decimal(1,0) 举例 + +>注：当表中的列允许导入空值时 + +|source data | source data example | string to int | strict_mode | result| +|------------|---------------------|-----------------|--------------------|--------| +|空值 | \N | N/A | true or false | NULL| +|not null | aaa | NULL | true | invalid data(filtered)| +|not null | aaa | NULL | false | NULL| +|not null | 1 or 10 | 1 | true or false | correct data| + +> 注意：10 虽然是一个超过范围的值，但是因为其类型符合 decimal的要求，所以 strict mode对其不产生影响。10 最后会在其他 ETL 处理流程中被过滤。但不会被 strict mode 过滤。 + + +### 返回结果 + +由于 Stream load 是一种同步的导入方式，所以导入的结果会通过创建导入的返回值直接返回给用户。 + +示例： + +``` +{ + "TxnId": 1003, + "Label": "b6f3bc78-0d2c-45d9-9e4c-faa0a0149bee", + "Status": "Success", + "ExistingJobStatus": "FINISHED", // optional + "Message": "OK", + "NumberTotalRows": 1000000, + "NumberLoadedRows": 1000000, + "NumberFilteredRows": 1, + "NumberUnselectedRows": 0, + "LoadBytes": 40888898, + "LoadTimeMs": 2144, + "ErrorURL": "http://192.168.1.1:8042/api/_load_error_log?file=__shard_0/error_log_insert_stmt_db18266d4d9b4ee5-abb00ddd64bdf005_db18266d4d9b4ee5_abb00ddd64bdf005" +} +``` + +下面主要解释了 Stream load 导入结果参数： + ++ TxnId：导入的事务ID。用户可不感知。 + ++ Label：导入 Label。由用户指定或系统自动生成。 + ++ Status：导入完成状态。 + + "Success"：表示导入成功。 + + "Publish Timeout"：该状态也表示导入已经完成，只是数据可能会延迟可见，无需重试。 + + "Label Already Exists"：Label 重复，需更换 Label。 + + "Fail"：导入失败。 + ++ ExistingJobStatus：已存在的 Label 对应的导入作业的状态。 + + 这个字段只有在当 Status 为 "Label Already Exists" 是才会显示。用户可以通过这个状态，知晓已存在 Label 对应的导入作业的状态。"RUNNING" 表示作业还在执行，"FINISHED" 表示作业成功。 + ++ Message：导入错误信息。 + ++ NumberTotalRows：导入总处理的行数。 + ++ NumberLoadedRows：成功导入的行数。 + ++ NumberFilteredRows：数据质量不合格的行数。 + ++ NumberUnselectedRows：被 where 条件过滤的行数。 + ++ LoadBytes：导入的字节数。 + ++ LoadTimeMs：导入完成时间。单位毫秒。 + ++ ErrorURL：如果有数据质量问题，通过访问这个 URL 查看具体错误行。 + +> 注意：由于 Stream load 是同步的导入方式，所以并不会在 Doris 系统中记录导入信息，用户无法异步的通过查看导入命令看到 Stream load。使用时需监听创建导入请求的返回值获取导入结果。 + +### 取消导入 + +用户无法手动取消 Stream load，Stream load 在超时或者导入错误后会被系统自动取消。 + +## 相关系统配置 + +### FE 配置 + ++ stream\_load\_default\_timeout\_second + + 导入任务的超时时间(以秒为单位)，导入任务在设定的 timeout 时间内未完成则会被系统取消，变成 CANCELLED。 + + 默认的 timeout 时间为 600 秒。如果导入的源文件无法在规定时间内完成导入，用户可以在 stream load 请求中设置单独的超时时间。 + + 或者调整 FE 的参数```stream_load_default_timeout_second``` 来设置全局的默认超时时间。 + +### BE 配置 + ++ streaming\_load\_max\_mb + + Stream load 的最大导入大小，默认为 10G，单位是 MB。如果用户的原始文件超过这个值，则需要调整 BE 的参数 ```streaming_load_max_mb```。 + +## 最佳实践 + +### 应用场景 + +使用 Stream load 的最合适场景就是原始文件在内存中，或者在磁盘中。其次，由于 Stream load 是一种同步的导入方式，所以用户如果希望用同步方式获取导入结果，也可以使用这种导入。 + +### 数据量 + +由于 Stream load 的原理是由 BE 发起的导入并分发数据，建议的导入数据量在 1G 到 10G 之间。由于默认的最大 Stream load 导入数据量为 10G，所以如果要导入超过 10G 的文件需要修改 BE 的配置 ```streaming_load_max_mb``` + +``` +比如：待导入文件大小为15G +修改 BE 配置 streaming_load_max_mb 为 16000 即可。 +``` + +Stream load 的默认超时为 300秒，按照 Doris 目前最大的导入限速来看，约超过 3G 的文件就需要修改导入任务默认超时时间了。 + +``` +导入任务超时时间 = 导入数据量 / 10M/s （具体的平均导入速度需要用户根据自己的集群情况计算） +例如：导入一个 10G 的文件 +timeout = 1000s 等于 10G / 10M/s +``` + +### 完整例子 +数据情况： 数据在发送导入请求端的本地磁盘路径 /home/store_sales 中，导入的数据量约为 15G，希望导入到数据库 bj_sales 的表 store_sales 中。 + +集群情况：Stream load 的并发数不受集群大小影响。 + ++ step1: 导入文件大小是否超过默认的最大导入大小10G + + ``` + 修改 BE conf + streaming_load_max_mb = 16000 + ``` ++ step2: 计算大概的导入时间是否超过默认 timeout 值 + + ``` + 导入时间 ≈ 15000 / 10 = 1500s + 超过了默认的 timeout 时间，需要修改 FE 的配置 + stream_load_default_timeout_second = 1500 + ``` + ++ step3：创建导入任务 + + ``` + curl --location-trusted -u user:password -T /home/store_sales -H "label:abc" http://abc.com:8000/api/bj_sales/store_sales/_stream_load + ``` + +## 常见问题 + +* Label Already Exists + + Stream load 的 Label 重复排查步骤如下： + + 1. 是否和其他导入方式已经存在的导入 Label 冲突： + + 由于 Doris 系统中导入的 Label 不区分导入方式，所以存在其他导入方式使用了相同 Label 的问题。 + + 通过 ```SHOW LOAD WHERE LABEL = “xxx”```，其中 xxx 为重复的 Label 字符串，查看是否已经存在一个 FINISHED 导入的 Label 和用户申请创建的 Label 相同。 + + 2. 是否 Stream load 同一个作业被重复提交了 + + 由于 Stream load 是 HTTP 协议提交创建导入任务，一般各个语言的 HTTP Client 均会自带请求重试逻辑。Doris 系统在接受到第一个请求后，已经开始操作 Stream load，但是由于没有及时返回给 Client 端结果， Client 端会发生再次重试创建请求的情况。这时候 Doris 系统由于已经在操作第一个请求，所以第二个请求已经就会被报 Label Already Exists 的情况。 + + 排查上述可能的方法：使用 Label 搜索 FE Master 的日志，看是否存在同一个 Label 出现了两次 ```redirect load action to destination= ``` 的情况。如果有就说明，请求被 Client 端重复提交了。 + + 建议用户根据当前请求的数据量，计算出大致导入的时间，并根据导入超时时间改大 Client 端的请求超时时间，避免请求被 Client 端多次提交。 + + + diff --git a/docs/documentation/cn/administrator-guide/operation/disk-capacity.md b/docs/zh-CN/administrator-guide/operation/disk-capacity.md similarity index 98% rename from docs/documentation/cn/administrator-guide/operation/disk-capacity.md rename to docs/zh-CN/administrator-guide/operation/disk-capacity.md index 8c9746fd973639..e3e7f5ba508000 100644 --- a/docs/documentation/cn/administrator-guide/operation/disk-capacity.md +++ b/docs/zh-CN/administrator-guide/operation/disk-capacity.md @@ -1,3 +1,10 @@ +--- +{ + "title": "磁盘空间管理", + "language": "zh-CN" +} +--- + + +# 元数据运维 + +本文档主要介绍在实际生产环境中，如何对 Doris 的元数据进行管理。包括 FE 节点建议的部署方式、一些常用的操作方法、以及常见错误的解决方法。 + +在阅读本文当前，请先阅读 [Doris 元数据设计文档](../../internal/metadata-design.md) 了解 Doris 元数据的工作原理。 + +## 重要提示 + +* 当前元数据的设计是无法向后兼容的。即如果新版本有新增的元数据结构变动（可以查看 FE 代码中的 `FeMetaVersion.java` 文件中是否有新增的 VERSION），那么在升级到新版本后，通常是无法再回滚到旧版本的。所以，在升级 FE 之前，请务必按照 [升级文档](../../installing/upgrade.md) 中的操作，测试元数据兼容性。 + +## 元数据目录结构 + +我们假设在 fe.conf 中指定的 `meta_dir` 的路径为 `/path/to/palo-meta`。那么一个正常运行中的 Doris 集群，元数据的目录结构应该如下： + +``` +/path/to/palo-meta/ + |-- bdb/ + | |-- 00000000.jdb + | |-- je.config.csv + | |-- je.info.0 + | |-- je.info.0.lck + | |-- je.lck + | `-- je.stat.csv + `-- image/ + |-- ROLE + |-- VERSION + `-- image.xxxx +``` + +1. bdb 目录 + + 我们将 [bdbje](https://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html) 作为一个分布式的 kv 系统，存放元数据的 journal。这个 bdb 目录相当于 bdbje 的 “数据目录”。 + + 其中 `.jdb` 后缀的是 bdbje 的数据文件。这些数据文件会随着元数据 journal 的不断增多而越来越多。当 Doris 定期做完 image 后，旧的日志就会被删除。所以正常情况下，这些数据文件的总大小从几 MB 到几 GB 不等（取决于使用 Doris 的方式，如导入频率等）。当数据文件的总大小大于 10GB，则可能需要怀疑是否是因为 image 没有成功，或者分发 image 失败导致的历史 journal 一直无法删除。 + + `je.info.0` 是 bdbje 的运行日志。这个日志中的时间是 UTC+0 时区的。我们可能在后面的某个版本中修复这个问题。通过这个日志，也可以查看一些 bdbje 的运行情况。 + +2. image 目录 + + image 目录用于存放 Doris 定期生成的元数据镜像文件。通常情况下，你会看到有一个 `image.xxxxx` 的镜像文件。其中 `xxxxx` 是一个数字。这个数字表示该镜像包含 `xxxxx` 号之前的所有元数据 journal。而这个文件的生成时间（通过 `ls -al` 查看即可）通常就是镜像的生成时间。 + + 你也可能会看到一个 `image.ckpt` 文件。这是一个正在生成的元数据镜像。通过 `du -sh` 命令应该可以看到这个文件大小在不断变大，说明镜像内容正在写入这个文件。当镜像写完后，会自动重名为一个新的 `image.xxxxx` 并替换旧的 image 文件。 + + 只有角色为 Master 的 FE 才会主动定期生成 image 文件。每次生成完后，都会推送给其他非 Master 角色的 FE。当确认其他所有 FE 都收到这个 image 后，Master FE 会删除 bdbje 中旧的元数据 journal。所以，如果 image 生成失败，或者 image 推送给其他 FE 失败时，都会导致 bdbje 中的数据不断累积。 + + `ROLE` 文件记录了 FE 的类型（FOLLOWER 或 OBSERVER），是一个文本文件。 + + `VERSION` 文件记录了这个 Doris 集群的 cluster id，以及用于各个节点之间访问认证的 token，也是一个文本文件。 + + `ROLE` 文件和 `VERSION` 文件只可能同时存在，或同时不存在（如第一次启动时）。 + +## 基本操作 + +### 启动单节点 FE + +单节点 FE 是最基本的一种部署方式。一个完整的 Doris 集群，至少需要一个 FE 节点。当只有一个 FE 节点时，这个节点的类型为 Follower，角色为 Master。 + +1. 第一次启动 + + 1. 假设在 fe.conf 中指定的 `meta_dir` 的路径为 `/path/to/palo-meta`。 + 2. 确保 `/path/to/palo-meta` 已存在，权限正确，且目录为空。 + 3. 直接通过 `sh bin/start_fe.sh` 即可启动。 + 4. 启动后，你应该可以在 fe.log 中看到如下日志： + + * Palo FE starting... + * image does not exist: /path/to/palo-meta/image/image.0 + * transfer from INIT to UNKNOWN + * transfer from UNKNOWN to MASTER + * the very first time to open bdb, dbname is 1 + * start fencing, epoch number is 1 + * finish replay in xxx msec + * QE service start + * thrift server started + + 以上日志不一定严格按照这个顺序，但基本类似。 + + 5. 单节点 FE 的第一次启动通常不会遇到问题。如果你没有看到以上日志，一般来说是没有仔细按照文档步骤操作，请仔细阅读相关 wiki。 + +2. 重启 + + 1. 直接使用 `sh bin/start_fe.sh` 可以重新启动已经停止的 FE 节点。 + 2. 重启后，你应该可以在 fe.log 中看到如下日志： + + * Palo FE starting... + * finished to get cluster id: xxxx, role: FOLLOWER and node name: xxxx + * 如果重启前还没有 image 产生，则会看到： + * image does not exist: /path/to/palo-meta/image/image.0 + + * 如果重启前有 image 产生，则会看到： + * start load image from /path/to/palo-meta/image/image.xxx. is ckpt: false + * finished load image in xxx ms + + * transfer from INIT to UNKNOWN + * replayed journal id is xxxx, replay to journal id is yyyy + * transfer from UNKNOWN to MASTER + * finish replay in xxx msec + * master finish replay journal, can write now. + * begin to generate new image: image.xxxx + * start save image to /path/to/palo-meta/image/image.ckpt. is ckpt: true + * finished save image /path/to/palo-meta/image/image.ckpt in xxx ms. checksum is xxxx + * push image.xxx to other nodes. totally xx nodes, push successed xx nodes + * QE service start + * thrift server started + + 以上日志不一定严格按照这个顺序，但基本类似。 + +3. 常见问题 + + 对于单节点 FE 的部署，启停通常不会遇到什么问题。如果有问题，请先参照相关 wiki，仔细核对你的操作步骤。 + +### 添加 FE + +添加 FE 流程在 [部署和升级文档](https://github.com/apache/incubator-doris/wiki/Doris-Deploy-%26-Upgrade) 有详细介绍，不再赘述。这里主要说明一些注意事项，以及常见问题。 + +1. 注意事项 + + * 在添加新的 FE 之前，一定先确保当前的 Master FE 运行正常（连接是否正常，JVM 是否正常，image 生成是否正常，bdbje 数据目录是否过大等等） + * 第一次启动新的 FE，一定确保添加了 `-helper` 参数指向 Master FE。再次启动时可不用添加 `-helper`。（如果指定了 `-helper`，FE 会直接询问 helper 节点自己的角色，如果没有指定，FE会尝试从 `palo-meta/image/` 目录下的 `ROLE` 和 `VERSION` 文件中获取信息）。 + * 第一次启动新的 FE，一定确保这个 FE 的 `meta_dir` 已经创建、权限正确且为空。 + * 启动新的 FE，和执行 `ALTER SYSTEM ADD FOLLOWER/OBSERVER` 语句在元数据添加 FE，这两个操作的顺序没有先后要求。如果先启动了新的 FE，而没有执行语句，则新的 FE 日志中会一直滚动 `current node is not added to the group. please add it first.` 字样。当执行语句后，则会进入正常流程。 + * 请确保前一个 FE 添加成功后，再添加下一个 FE。 + * 建议直接连接到 MASTER FE 执行 `ALTER SYSTEM ADD FOLLOWER/OBSERVER` 语句。 + +2. 常见问题 + + 1. this node is DETACHED + + 当第一次启动一个待添加的 FE 时，如果 Master FE 上的 palo-meta/bdb 中的数据很大，则可能在待添加的 FE 日志中看到 `this node is DETACHED.` 字样。这时，bdbje 正在复制数据，你可以看到待添加的 FE 的 `bdb/` 目录正在变大。这个过程通常会在数分钟不等（取决于 bdbje 中的数据量）。之后，fe.log 中可能会有一些 bdbje 相关的错误堆栈信息。如果最终日志中显示 `QE service start` 和 `thrift server started`，则通常表示启动成功。可以通过 mysql-client 连接这个 FE 尝试操作。如果没有出现这些字样，则可能是 bdbje 复制日志超时等问题。这时，直接再次重启这个 FE，通常即可解决问题。 + + 2. 各种原因导致添加失败 + + * 如果添加的是 OBSERVER，因为 OBSERVER 类型的 FE 不参与元数据的多数写，理论上可以随意启停。因此，对于添加 OBSERVER 失败的情况。可以直接杀死 OBSERVER FE 的进程，清空 OBSERVER 的元数据目录后，重新进行一遍添加流程。 + + * 如果添加的是 FOLLOWER，因为 FOLLOWER 是参与元数据多数写的。所以有可能FOLLOWER 已经加入 bdbje 选举组内。如果这时只有两个 FOLLOWER 节点（包括 MASTER），那么停掉一个 FE，可能导致另一个 FE 也因无法进行多数写而退出。此时，我们应该先通过 `ALTER SYSTEM DROP FOLLOWER` 命令，从元数据中删除新添加的 FOLLOWER 节点，然后再杀死 FOLLOWER 进程，清空元数据，重新进行一遍添加流程。 + + +### 删除 FE + +通过 `ALTER SYSTEM DROP FOLLOWER/OBSERVER` 命令即可删除对应类型的 FE。以下有几点注意事项： + +* 对于 OBSERVER 类型的 FE，直接 DROP 即可，无风险。 + +* 对于 FOLLOWER 类型的 FE。首先，应保证在有奇数个 FOLLOWER 的情况下（3个或以上），开始删除操作。 + + 1. 如果删除非 MASTER 角色的 FE，建议连接到 MASTER FE，执行 DROP 命令，再杀死进程即可。 + 2. 如果要删除 MASTER FE，先确认有奇数个 FOLLOWER FE 并且运行正常。然后先杀死 MASTER FE 的进程。这时会有某一个 FE 被选举为 MASTER。在确认剩下的 FE 运行正常后，连接到新的 MASTER FE，执行 DROP 命令删除之前老的 MASTER FE 即可。 + +## 高级操作 + +### 故障恢复 + +FE 有可能因为某些原因出现无法启动 bdbje、FE 之间无法同步等问题。现象包括无法进行元数据写操作、没有 MASTER 等等。这时，我们需要手动操作来恢复 FE。手动恢复 FE 的大致原理，是先通过当前 `meta_dir` 中的元数据，启动一个新的 MASTER，然后再逐台添加其他 FE。请严格按照如下步骤操作： + +1. 首先，停止所有 FE 进程，同时停止一切业务访问。保证在元数据恢复期间，不会因为外部访问导致其他不可预期的问题。 + +2. 确认哪个 FE 节点的元数据是最新： + + * 首先，**务必先备份所有 FE 的 `meta_dir` 目录。** + * 通常情况下，Master FE 的元数据是最新的。可以查看 `meta_dir/image` 目录下，image.xxxx 文件的后缀，数字越大，则表示元数据越新。 + * 通常，通过比较所有 FOLLOWER FE 的 image 文件，找出最新的元数据即可。 + * 之后，我们要使用这个拥有最新元数据的 FE 节点，进行恢复。 + * 如果使用 OBSERVER 节点的元数据进行恢复会比较麻烦，建议尽量选择 FOLLOWER 节点。 + +3. 以下操作都在由第2步中选择出来的 FE 节点上进行。 + + 1. 如果该节点是一个 OBSERVER，先将 `meta_dir/image/ROLE` 文件中的 `role=OBSERVER` 改为 `role=FOLLOWER`。（从 OBSERVER 节点恢复会比较麻烦，先按这里的步骤操作，后面会有单独说明） + 2. 在 fe.conf 中添加配置：`metadata_failure_recovery=true`。 + 3. 执行 `sh bin/start_fe.sh` 启动这个 FE。 + 4. 如果正常，这个 FE 会以 MASTER 的角色启动，类似于前面 `启动单节点 FE` 一节中的描述。在 fe.log 应该会看到 `transfer from XXXX to MASTER` 等字样。 + 5. 启动完成后，先连接到这个 FE，执行一些查询导入，检查是否能够正常访问。如果不正常，有可能是操作有误，建议仔细阅读以上步骤，用之前备份的元数据再试一次。如果还是不行，问题可能就比较严重了。 + 6. 如果成功，通过 `show frontends;` 命令，应该可以看到之前所添加的所有 FE，并且当前 FE 是 master。 + 7. 将 fe.conf 中的 `metadata_failure_recovery=true` 配置项删除，或者设置为 `false`，然后重启这个 FE（**重要**）。 + + + > 如果你是从一个 OBSERVER 节点的元数据进行恢复的，那么完成如上步骤后，通过 `show frontends;` 语句你会发现，当前这个 FE 的角色为 OBSERVER，但是 `IsMaster` 显示为 `true`。这是因为，这里看到的 “OBSERVER” 是记录在 Doris 的元数据中的，而是否是 master，是记录在 bdbje 的元数据中的。因为我们是从一个 OBSERVER 节点恢复的，所以这里出现了不一致。请按如下步骤修复这个问题（这个问题我们会在之后的某个版本修复）： + + > 1. 先把除了这个 “OBSERVER” 以外的所有 FE 节点 DROP 掉。 + > 2. 通过 `ADD FOLLOWER` 命令，添加一个新的 FOLLOWER FE，假设在 hostA 上。 + > 3. 在 hostA 上启动一个全新的 FE，通过 `-helper` 的方式加入集群。 + > 4. 启动成功后，通过 `show frontends;` 语句，你应该能看到两个 FE，一个是之前的 OBSERVER，一个是新添加的 FOLLOWER，并且 OBSERVER 是 master。 + > 5. 确认这个新的 FOLLOWER 是可以正常工作之后，用这个新的 FOLLOWER 的元数据，重新执行一遍故障恢复操作。 + > 6. 以上这些步骤的目的，其实就是人为的制造出一个 FOLLOWER 节点的元数据，然后用这个元数据，重新开始故障恢复。这样就避免了从 OBSERVER 恢复元数据所遇到的不一致的问题。 + + > `metadata_failure_recovery=true` 的含义是，清空 "bdbje" 的元数据。这样 bdbje 就不会再联系之前的其他 FE 了，而作为一个独立的 FE 启动。这个参数只有在恢复启动时才需要设置为 true。恢复完成后，一定要设置为 false，否则一旦重启，bdbje 的元数据又会被清空，导致其他 FE 无法正常工作。 + +4. 第3步执行成功后，我们再通过 `ALTER SYSTEM DROP FOLLOWER/OBSERVER` 命令，将之前的其他的 FE 从元数据删除后，按加入新 FE 的方式，重新把这些 FE 添加一遍。 + +5. 如果以上操作正常，则恢复完毕。 + +### FE 类型变更 + +如果你需要将当前已有的 FOLLOWER/OBSERVER 类型的 FE，变更为 OBSERVER/FOLLOWER 类型，请先按照前面所述的方式删除 FE，再添加对应类型的 FE 即可 + +### FE 迁移 + +如果你需要将一个 FE 从当前节点迁移到另一个节点，分以下几种情况。 + +1. 非 MASTER 节点的 FOLLOWER，或者 OBSERVER 迁移 + + 直接添加新的 FOLLOWER/OBSERVER 成功后，删除旧的 FOLLOWER/OBSERVER 即可。 + +2. 单节点 MASTER 迁移 + + 当只有一个 FE 时，参考 `故障恢复` 一节。将 FE 的 palo-meta 目录拷贝到新节点上，按照 `故障恢复` 一节中，步骤3的方式启动新的 MASTER + +3. 一组 FOLLOWER 从一组节点迁移到另一组新的节点 + + 在新的节点上部署 FE，通过添加 FOLLOWER 的方式先加入新节点。再逐台 DROP 掉旧节点即可。在逐台 DROP 的过程中，MASTER 会自动选择在新的 FOLLOWER 节点上。 + +### 更换 FE 端口 + +FE 目前有以下几个端口 + +* edit_log_port：bdbje 的通信端口 +* http_port：http 端口，也用于推送 image +* rpc_port：FE 的 thrift server port +* query_port：Mysql 连接端口 + +1. edit_log_port + + 如果需要更换这个端口，则需要参照 `故障恢复` 一节中的操作，进行恢复。因为该端口已经被持久化到 bdbje 自己的元数据中（同时也记录在 Doris 自己的元数据中），需要通过设置 `metadata_failure_recovery=true` 来清空 bdbje 的元数据。 + +2. http_port + + 所有 FE 的 http_port 必须保持一致。所以如果要修改这个端口，则所有 FE 都需要修改并重启。修改这个端口，在多 FOLLOWER 部署的情况下会比较复杂（涉及到鸡生蛋蛋生鸡的问题...），所以不建议有这种操作。如果必须，直接按照 `故障恢复` 一节中的操作吧。 + +3. rpc_port + + 修改配置后，直接重启 FE 即可。Master FE 会通过心跳将新的端口告知 BE。只有 Master FE 的这个端口会被使用。但仍然建议所有 FE 的端口保持一致。 + +4. query_port + + 修改配置后，直接重启 FE 即可。这个只影响到 mysql 的连接目标。 + + +### 从 FE 内存中恢复元数据 + +在某些极端情况下，磁盘上 image 文件可能会损坏，但是内存中的元数据是完好的，此时我们可以先从内存中 dump 出元数据，再替换掉磁盘上的 image 文件，来恢复元数据，整个**不停查询服务**的操作步骤如下： +1. 集群停止所有 Load,Create,Alter 操作 +2. 执行以下命令，从 Master FE 内存中 dump 出元数据：(下面称为 image_mem) +``` +curl -u $root_user:$password http://$master_hostname:8030/dump +``` +3. 用 image_mem 文件替换掉 OBSERVER FE 节点上`meta_dir/image`目录下的 image 文件，重启 OBSERVER FE 节点， +验证 image_mem 文件的完整性和正确性（可以在 FE Web 页面查看 DB 和 Table 的元数据是否正常，查看fe.log 是否有异常，是否在正常 replayed journal） +4. 依次用 image_mem 文件替换掉 FOLLOWER FE 节点上`meta_dir/image`目录下的 image 文件，重启 FOLLOWER FE 节点， +确认元数据和查询服务都正常 +5. 用 image_mem 文件替换掉 Master FE 节点上`meta_dir/image`目录下的 image 文件，重启 Master FE 节点， +确认 FE Master 切换正常， Master FE 节点可以通过 checkpoint 正常生成新的 image 文件 +6. 集群恢复所有 Load,Create,Alter 操作 + +**注意：如果 Image 文件很大，整个操作过程耗时可能会很长，所以在此期间，要确保 Master FE 不会通过 checkpoint 生成新的 image 文件。 +当观察到 Master FE 节点上 `meta_dir/image`目录下的 `image.ckpt` 文件快和 `image.xxx` 文件一样大时，可以直接删除掉`image.ckpt` 文件。** + +## 最佳实践 + +FE 的部署推荐，在 [安装与部署文档](../../installing/install-deploy.md) 中有介绍，这里再做一些补充。 + +* **如果你并不十分了解 FE 元数据的运行逻辑，或者没有足够 FE 元数据的运维经验，我们强烈建议在实际使用中，只部署一个 FOLLOWER 类型的 FE 作为 MASTER，其余 FE 都是 OBSERVER，这样可以减少很多复杂的运维问题！** 不用过于担心 MASTER 单点故障导致无法进行元数据写操作。首先，如果你配置合理，FE 作为 java 进程很难挂掉。其次，如果 MASTER 磁盘损坏（概率非常低），我们也可以用 OBSERVER 上的元数据，通过 `故障恢复` 的方式手动恢复。 + +* FE 进程的 JVM 一定要保证足够的内存。我们**强烈建议** FE 的 JVM 内存至少在 10GB 以上，推荐 32GB 至 64GB。并且部署监控来监控 JVM 的内存使用情况。因为如果FE出现OOM，可能导致元数据写入失败，造成一些**无法恢复**的故障！ + +* FE 所在节点要有足够的磁盘空间，以防止元数据过大导致磁盘空间不足。同时 FE 日志也会占用十几G 的磁盘空间。 + +## 其他常见问题 + +1. fe.log 中一直滚动 `meta out of date. current time: xxx, synchronized time: xxx, has log: xxx, fe type: xxx` + + 这个通常是因为 FE 无法选举出 Master。比如配置了 3 个 FOLLOWER，但是只启动了一个 FOLLOWER，则这个 FOLLOWER 会出现这个问题。通常，只要把剩余的 FOLLOWER 启动起来就可以了。如果启动起来后，仍然没有解决问题，那么可能需要按照 `故障恢复` 一节中的方式，手动进行恢复。 + +2. `Clock delta: xxxx ms. between Feeder: xxxx and this Replica exceeds max permissible delta: xxxx ms.` + + bdbje 要求各个节点之间的时钟误差不能超过一定阈值。如果超过，节点会异常退出。我们默认设置的阈值为 5000 ms，由 FE 的参数 `max_bdbje_clock_delta_ms` 控制，可以酌情修改。但我们建议使用 ntp 等时钟同步方式保证 Doris 集群各主机的时钟同步。 + + +3. `image/` 目录下的镜像文件很久没有更新 + + Master FE 会默认每 50000 条元数据 journal，生成一个镜像文件。在一个频繁使用的集群中，通常每隔半天到几天的时间，就会生成一个新的 image 文件。如果你发现 image 文件已经很久没有更新了（比如超过一个星期），则可以顺序的按照如下方法，查看具体原因： + + 1. 在 Master FE 的 fe.log 中搜索 `memory is not enough to do checkpoint. Committed memroy xxxx Bytes, used memory xxxx Bytes.` 字样。如果找到，则说明当前 FE 的 JVM 内存不足以用于生成镜像（通常我们需要预留一半的 FE 内存用于 image 的生成）。那么需要增加 JVM 的内存并重启 FE 后，再观察。每次 Master FE 重启后，都会直接生成一个新的 image。也可用这种重启方式，主动地生成新的 image。注意，如果是多 FOLLOWER 部署，那么当你重启当前 Master FE 后，另一个 FOLLOWER FE 会变成 MASTER，则后续的 image 生成会由新的 Master 负责。因此，你可能需要修改所有 FOLLOWER FE 的 JVM 内存配置。 + + 2. 在 Master FE 的 fe.log 中搜索 `begin to generate new image: image.xxxx`。如果找到，则说明开始生成 image 了。检查这个线程的后续日志，如果出现 `checkpoint finished save image.xxxx`，则说明 image 写入成功。如果出现 `Exception when generate new image file`，则生成失败，需要查看具体的错误信息。 + + +4. `bdb/` 目录的大小非常大，达到几个G或更多 + + 如果在排除无法生成新的 image 的错误后，bdb 目录在一段时间内依然很大。则可能是因为 Master FE 推送 image 不成功。可以在 Master FE 的 fe.log 中搜索 `push image.xxxx to other nodes. totally xx nodes, push successed yy nodes`。如果 yy 比 xx 小，则说明有的 FE 没有被推送成功。可以在 fe.log 中查看到具体的错误 `Exception when pushing image file. url = xxx`。 + + 同时，你也可以在 FE 的配置文件中添加配置：`edit_log_roll_num=xxxx`。该参数设定了每多少条元数据 journal，做一次 image。默认是 50000。可以适当改小这个数字，使得 image 更加频繁，从而加速删除旧的 journal。 + +5. FOLLOWER FE 接连挂掉 + + 因为 Doris 的元数据采用多数写策略，即一条元数据 journal 必须至少写入多数个 FOLLOWER FE 后（比如 3 个 FOLLOWER，必须写成功 2 个），才算成功。而如果写入失败，FE 进程会主动退出。那么假设有 A、B、C 三个 FOLLOWER，C 先挂掉，然后 B 再挂掉，那么 A 也会跟着挂掉。所以如 `最佳实践` 一节中所述，如果你没有丰富的元数据运维经验，不建议部署多 FOLLOWER。 + +6. fe.log 中出现 `get exception when try to close previously opened bdb database. ignore it` + + 如果后面有 `ignore it` 字样，通常无需处理。如果你有兴趣，可以在 `BDBEnvironment.java` 搜索这个错误，查看相关注释说明。 + +7. 从 `show frontends;` 看，某个 FE 的 `Join` 列为 `true`，但是实际该 FE 不正常 + + 通过 `show frontends;` 查看到的 `Join` 信息。该列如果为 `true`，仅表示这个 FE **曾经加入过** 集群。并不能表示当前仍然正常的存在于集群中。如果为 `false`，则表示这个 FE **从未加入过** 集群。 + +8. 关于 FE 的配置 `master_sync_policy`, `replica_sync_policy` 和 `txn_rollback_limit` + + `master_sync_policy` 用于指定当 Leader FE 写元数据日志时，是否调用 fsync(), `replica_sync_policy` 用于指定当 FE HA 部署时，其他 Follower FE 在同步元数据时，是否调用 fsync()。在早期的 Doris 版本中，这两个参数默认是 `WRITE_NO_SYNC`，即都不调用 fsync()。在最新版本的 Doris 中，默认已修改为 `SYNC`，即都调用 fsync()。调用 fsync() 会显著降低元数据写盘的效率。在某些环境下，IOPS 可能降至几百，延迟增加到2-3ms（但对于 Doris 元数据操作依然够用）。因此我们建议以下配置： + + 1. 对于单 Follower FE 部署，`master_sync_policy` 设置为 `SYNC`，防止 FE 系统宕机导致元数据丢失。 + 2. 对于多 Follower FE 部署，可以将 `master_sync_policy` 和 `replica_sync_policy` 设为 `WRITE_NO_SYNC`，因为我们认为多个系统同时宕机的概率非常低。 + + 如果在单 Follower FE 部署中，`master_sync_policy` 设置为 `WRITE_NO_SYNC`，则可能出现 FE 系统宕机导致元数据丢失。这时如果有其他 Observer FE 尝试重启时，可能会报错： + + ``` + Node xxx must rollback xx total commits(numPassedDurableCommits of which were durable) to the earliest point indicated by transaction xxxx in order to rejoin the replication group, but the transaction rollback limit of xxx prohibits this. + ``` + + 意思有部分已经持久化的事务需要回滚，但条数超过上限。这里我们的默认上限是 100，可以通过设置 `txn_rollback_limit` 改变。该操作仅用于尝试正常启动 FE，但已丢失的元数据无法恢复。 diff --git a/docs/zh-CN/administrator-guide/operation/monitor-alert.md b/docs/zh-CN/administrator-guide/operation/monitor-alert.md new file mode 100644 index 00000000000000..c22225cc7ae081 --- /dev/null +++ b/docs/zh-CN/administrator-guide/operation/monitor-alert.md @@ -0,0 +1,309 @@ +--- +{ + "title": "监控和报警", + "language": "zh-CN" +} +--- + + + +# 监控和报警 + +本文档主要介绍 Doris 的监控项及如何采集、展示监控项。以及如何配置报警（TODO） + +[Dashborad 模板点击下载](https://grafana.com/dashboards/9734/revisions) + +> 注：0.9.0（不含）之前的版本请使用 revision 1。0.9.x 版本请使用 revision 2。0.10.x 版本请使用 revision 3。 + +Dashboard 模板会不定期更新。更新模板的方式见最后一小节。 + +欢迎提供更优的 dashboard。 + +## 组件 + +Doris 使用 [Prometheus](https://prometheus.io/) 和 [Grafana](https://grafana.com/) 进项监控项的采集和展示。 + +![](/images/dashboard_overview.png) + +1. Prometheus + + Prometheus 是一款开源的系统监控和报警套件。它可以通过 Pull 或 Push 采集被监控系统的监控项，存入自身的时序数据库中。并且通过丰富的多维数据查询语言，满足用户的不同数据展示需求。 + +2. Grafana + + Grafana 是一款开源的数据分析和展示平台。支持包括 Prometheus 在内的多个主流时序数据库源。通过对应的数据库查询语句，从数据源中获取展现数据。通过灵活可配置的 Dashboard，快速的将这些数据以图表的形式展示给用户。 + +> 注: 本文档仅提供一种使用 Prometheus 和 Grafana 进行 Doris 监控数据采集和展示的方式。原则上不开发、维护这些组件。更多关于这些组件的详细介绍，请移步对应官方文档进行查阅。 + +## 监控数据 + +Doris 的监控数据通过 Frontend 和 Backend 的 http 接口向外暴露。监控数据以 Key-Value 的文本形式对外展现。每个 Key 还可能有不同的 Label 加以区分。当用户搭建好 Doris 后，可以在浏览器，通过以下接口访问到节点的监控数据： + +* Frontend: `fe_host:fe_http_port/metrics` +* Backend: `be_host:be_web_server_port/metrics` +* Broker: 暂不提供 + +用户将看到如下监控项结果（示例为 FE 部分监控项）： + + ``` + # HELP jvm_heap_size_bytes jvm heap stat + # TYPE jvm_heap_size_bytes gauge + jvm_heap_size_bytes{type="max"} 41661235200 + jvm_heap_size_bytes{type="committed"} 19785285632 + jvm_heap_size_bytes{type="used"} 10113221064 + # HELP jvm_non_heap_size_bytes jvm non heap stat + # TYPE jvm_non_heap_size_bytes gauge + jvm_non_heap_size_bytes{type="committed"} 105295872 + jvm_non_heap_size_bytes{type="used"} 103184784 + # HELP jvm_young_size_bytes jvm young mem pool stat + # TYPE jvm_young_size_bytes gauge + jvm_young_size_bytes{type="used"} 6505306808 + jvm_young_size_bytes{type="peak_used"} 10308026368 + jvm_young_size_bytes{type="max"} 10308026368 + # HELP jvm_old_size_bytes jvm old mem pool stat + # TYPE jvm_old_size_bytes gauge + jvm_old_size_bytes{type="used"} 3522435544 + jvm_old_size_bytes{type="peak_used"} 6561017832 + jvm_old_size_bytes{type="max"} 30064771072 + # HELP jvm_direct_buffer_pool_size_bytes jvm direct buffer pool stat + # TYPE jvm_direct_buffer_pool_size_bytes gauge + jvm_direct_buffer_pool_size_bytes{type="count"} 91 + jvm_direct_buffer_pool_size_bytes{type="used"} 226135222 + jvm_direct_buffer_pool_size_bytes{type="capacity"} 226135221 + # HELP jvm_young_gc jvm young gc stat + # TYPE jvm_young_gc gauge + jvm_young_gc{type="count"} 2186 + jvm_young_gc{type="time"} 93650 + # HELP jvm_old_gc jvm old gc stat + # TYPE jvm_old_gc gauge + jvm_old_gc{type="count"} 21 + jvm_old_gc{type="time"} 58268 + # HELP jvm_thread jvm thread stat + # TYPE jvm_thread gauge + jvm_thread{type="count"} 767 + jvm_thread{type="peak_count"} 831 + ... + ``` + +这是一个以 [Promethus 格式](https://prometheus.io/docs/practices/naming/) 呈现的监控数据。我们以其中一个监控项为例进行说明： + +``` +# HELP jvm_heap_size_bytes jvm heap stat +# TYPE jvm_heap_size_bytes gauge +jvm_heap_size_bytes{type="max"} 41661235200 +jvm_heap_size_bytes{type="committed"} 19785285632 +jvm_heap_size_bytes{type="used"} 10113221064 +``` + +1. "#" 开头的行为注释行。其中 HELP 为该监控项的描述说明；TYPE 表示该监控项的数据类型，示例中为 Gauge，即标量数据。还有 Counter、Histogram 等数据类型。具体可见 [Prometheus 官方文档](https://prometheus.io/docs/practices/instrumentation/#counter-vs.-gauge,-summary-vs.-histogram) 。 +2. `jvm_heap_size_bytes` 即监控项的名称（Key）；`type="max"` 即为一个名为 `type` 的 Label，值为 `max`。一个监控项可以有多个 Label。 +3. 最后的数字，如 `41661235200`，即为监控数值。 + +## 监控架构 + +整个监控架构如下图所示： + +![](/images/monitor_arch.png) + +1. 黄色部分为 Prometheus 相关组件。Prometheus Server 为 Prometheus 的主进程，目前 Prometheus 通过 Pull 的方式访问 Doris 节点的监控接口，然后将时序数据存入时序数据库 TSDB 中（TSDB 包含在 Prometheus 进程中，无需单独部署）。Prometheus 也支持通过搭建 [Push Gateway](https://github.com/prometheus/pushgateway) 的方式，允许被监控系统将监控数据通过 Push 的方式推到 Push Gateway, 再由 Prometheus Server 通过 Pull 的方式从 Push Gateway 中获取数据。 +2. [Alert Manager](https://github.com/prometheus/alertmanager) 为 Prometheus 报警组件，需单独部署（暂不提供方案，可参照官方文档自行搭建）。通过 Alert Manager，用户可以配置报警策略，接收邮件、短信等报警。 +3. 绿色部分为 Grafana 相关组件。Grafana Server 为 Grafana 的主进程。启动后，用户可以通过 Web 页面对 Grafana 进行配置，包括数据源的设置、用户设置、Dashboard 绘制等。这里也是最终用户查看监控数据的地方。 + + +## 开始搭建 + +请在完成 Doris 的部署后，开始搭建监控系统。 + +### Prometheus + +1. 在 [Prometheus 官网](https://prometheus.io/download/) 下载最新版本的 Prometheus。这里我们以 2.3.2-linux-amd64 版本为例。 +2. 在准备运行监控服务的机器上，解压下载后的 tar 文件。 +3. 打开配置文件 promethues.yml。这里我们提供一个示例配置并加以说明（配置文件为 yml 格式，一定注意统一的缩进和空格）： + + 这里我们使用最简单的静态文件的方式进行监控配置。Prometheus 支持多种 [服务发现](https://prometheus.io/docs/prometheus/latest/configuration/configuration/) 方式，可以动态的感知节点的加入和删除。 + + ``` + # my global config + global: + scrape_interval: 15s # 全局的采集间隔，默认是 1m，这里设置为 15s + evaluation_interval: 15s # 全局的规则触发间隔，默认是 1m，这里设置 15s + + # Alertmanager configuration + alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + + # A scrape configuration containing exactly one endpoint to scrape: + # Here it's Prometheus itself. + scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'PALO_CLUSTER' # 每一个 Doris 集群，我们称为一个 job。这里可以给 job 取一个名字，作为 Doris 集群在监控系统中的名字。 + metrics_path: '/metrics' # 这里指定获取监控项的 restful api。配合下面的 targets 中的 host:port，Prometheus 最终会通过 host:port/metrics_path 来采集监控项。 + static_configs: # 这里开始分别配置 FE 和 BE 的目标地址。所有的 FE 和 BE 都分别写入各自的 group 中。 + - targets: ['fe_host1:8030', 'fe_host2:8030', 'fe_host3:8030'] + labels: + group: fe # 这里配置了 fe 的 group，该 group 中包含了 3 个 Frontends + + - targets: ['be_host1:8040', 'be_host2:8040', 'be_host3:8040'] + labels: + group: be # 这里配置了 be 的 group，该 group 中包含了 3 个 Backends + + - job_name: 'PALO_CLUSTER_2' # 我们可以在一个 Prometheus 中监控多个 Doris 集群，这里开始另一个 Doris 集群的配置。配置同上，以下略。 + metrics_path: '/metrics' + static_configs: + - targets: ['fe_host1:8030', 'fe_host2:8030', 'fe_host3:8030'] + labels: + group: fe + + - targets: ['be_host1:8040', 'be_host2:8040', 'be_host3:8040'] + labels: + group: be + + ``` + +4. 启动 Promethues + + 通过以下命令启动 Promethues： + + `nohup ./prometheus --web.listen-address="0.0.0.0:8181" &` + + 该命令将后台运行 Prometheus，并指定其 web 端口为 8181。启动后，即开始采集数据，并将数据存放在 data 目录中。 + +5. 停止 Promethues + + 目前没有发现正式的进程停止方式，直接 kill -9 即可。当然也可以将 Prometheus 设为一种 service，以 service 的方式启停。 + +6. 访问 Prometheus + + Prometheus 可以通过 web 页面进行简单的访问。通过浏览器打开 8181 端口，即可访问 Prometheus 的页面。点击导航栏中，`Status` -> `Targets`，可以看到所有分组 Job 的监控主机节点。正常情况下，所有节点都应为 `UP`，表示数据采集正常。点击某一个 `Endpoint`，即可看到当前的监控数值。如果节点状态不为 UP，可以先访问 Doris 的 metrics 接口（见前文）检查是否可以访问，或查询 Prometheus 相关文档尝试解决。 + +7. 至此，一个简单的 Prometheus 已经搭建、配置完毕。更多高级使用方式，请参阅 [官方文档](https://prometheus.io/docs/introduction/overview/) + +### Grafana + +1. 在 [Grafana 官网](https://grafana.com/grafana/download) 下载最新版本的 Grafana。这里我们以 5.2.1.linux-amd64 版本为例。 + +2. 在准备运行监控服务的机器上，解压下载后的 tar 文件。 + +3. 打开配置文件 conf/defaults.ini。这里我们仅列举需要改动的配置项，其余配置可使用默认。 + + ``` + # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) + data = data + + # Directory where grafana can store logs + logs = data/log + + # Protocol (http, https, socket) + protocal = http + + # The ip address to bind to, empty will bind to all interfaces + http_addr = + + # The http port to use + http_port = 8182 + ``` + +4. 启动 Grafana + + 通过以下命令启动 Grafana + + `nohup ./bin/grafana-server &` + + 该命令将后台运行 Grafana，访问端口为上面配置的 8182 + +5. 停止 Grafana + + 目前没有发现正式的进程停止方式，直接 kill -9 即可。当然也可以将 Grafana 设为一种 service，以 service 的方式启停。 + +6. 访问 Grafana + + 通过浏览器，打开 8182 端口，可以开始访问 Grafana 页面。默认用户名密码为 admin。 + +7. 配置 Grafana + + 初次登陆，需要根据提示设置数据源（data source）。我们这里的数据源，即上一步配置的 Prometheus。 + + 数据源配置的 Setting 页面说明如下： + + 1. Name: 数据源的名称，自定义，比如 doris_monitor_data_source + 2. Type: 选择 Prometheus + 3. URL: 填写 Prometheus 的 web 地址，如 http://host:8181 + 4. Access: 这里我们选择 Server 方式，即通过 Grafana 进程所在服务器，访问 Prometheus。 + 5. 其余选项默认即可。 + 6. 点击最下方 `Save & Test`，如果显示 `Data source is working`，即表示数据源可用。 + 7. 确认数据源可用后，点击左边导航栏的 + 号，开始添加 Dashboard。这里我们已经准备好了 Doris 的 Dashboard 模板（本文档开头）。下载完成后，点击上方的 `New dashboard`->`Import dashboard`->`Upload .json File`，将下载的 json 文件导入。 + 8. 导入后，可以命名 Dashboard，默认是 `Doris Overview`。同时，需要选择数据源，这里选择之前创建的 `doris_monitor_data_source` + 9. 点击 `Import`，即完成导入。之后，可以看到 Doris 的 Dashboard 展示。 + +8. 至此，一个简单的 Grafana 已经搭建、配置完毕。更多高级使用方式，请参阅 [官方文档](http://docs.grafana.org/) + + +## Dashboard 说明 + +这里我们简要介绍 Doris Dashboard。Dashboard 的内容可能会随版本升级，不断变化，本文档不保证是最新的 Dashboard 说明。 + +1. 顶栏 + + ![](/images/dashboard_navibar.png) + + * 左上角为 Dashboard 名称。 + * 右上角显示当前监控时间范围，可以下拉选择不同的时间范围，还可以指定定时刷新页面间隔。 + * cluster\_name: 即 Prometheus 配置文件中的各个 job\_name，代表一个 Doris 集群。选择不同的 cluster，下方的图表将展示对应集群的监控信息。 + * fe_master: 对应集群的 Master Frontend 节点。 + * fe_instance: 对应集群的所有 Frontend 节点。选择不同的 Frontend，下方的图表将展示对应 Frontend 的监控信息。 + * be_instance: 对应集群的所有 Backend 节点。选择不同的 Backend，下方的图表将展示对应 Backend 的监控信息。 + * interval: 有些图表展示了速率相关的监控项，这里可选择以多大间隔进行采样计算速率（注：15s 间隔可能导致一些图表无法显示）。 + +2. Row + + ![](/images/dashboard_row.png) + + Grafana 中，Row 的概念，即一组图表的集合。如上图中的 Overview、Cluster Overview 即两个不同的 Row。可以通过点击 Row，对 Row 进行折叠。当前 Dashboard 有如下 Rows（持续更新中）： + + 1. Overview: 所有 Doris 集群的汇总展示。 + 2. Cluster Overview: 选定集群的汇总展示。 + 3. Query Statistic: 选定集群的查询相关监控。 + 4. FE JVM: 选定 Frontend 的 JVM 监控。 + 5. BE: 选定集群的 Backends 的汇总展示。 + 6. BE Task: 选定集群的 Backends 任务信息的展示。 + +3. 图表 + + ![](/images/dashboard_panel.png) + + 一个典型的图标分为以下几部分： + + 1. 鼠标悬停左上角的 i 图标，可以查看该图表的说明。 + 2. 点击下方的图例，可以单独查看某一监控项。再次点击，则显示所有。 + 3. 在图表中拖拽可以选定时间范围。 + 4. 标题的 [] 中显示选定的集群名称。 + 5. 一些数值对应左边的Y轴，一些对应右边的，可以通过图例末尾的 `-right` 区分。 + 6. 点击图表名称->`Edit`，可以对图表进行编辑。 + +## Dashboard 更新 + +1. 点击 Grafana 左边栏的 `+`，点击 `Dashboard`。 +2. 点击左上角的 `New dashboard`，在点击右侧出现的 `Import dashboard`。 +3. 点击 `Upload .json File`，选择最新的模板文件。 +4. 选择数据源 +5. 点击 `Import(Overwrite)`，完成模板更新。 diff --git a/docs/zh-CN/administrator-guide/operation/multi-tenant.md b/docs/zh-CN/administrator-guide/operation/multi-tenant.md new file mode 100644 index 00000000000000..176d600d70a021 --- /dev/null +++ b/docs/zh-CN/administrator-guide/operation/multi-tenant.md @@ -0,0 +1,239 @@ +--- +{ + "title": "多租户(Experimental)", + "language": "zh-CN" +} +--- + + + +# 多租户(Experimental) + +该功能为实验性质，暂不建议在生产环境使用。 + +## 背景 +Doris 作为一款 PB 级别的在线报表与多维分析数据库，对外通过开放云提供云端的数据库服务，并且对于每个云上的客户都单独部署了一套物理集群。对内，一套物理集群部署了多个业务，对于隔离性要求比较高的业务单独搭建了集群。针对以上存在几点问题： + +- 部署多套物理集群维护代价大（升级、功能上线、bug修复）。 +- 一个用户的查询或者查询引起的bug经常会影响其他用户。 +- 实际生产环境单机只能部署一个BE进程。而多个BE可以更好的解决胖节点问题。并且对于join、聚合操作可以提供更高的并发度。 + +综合以上三点，Doris需要新的多租户方案，既能做到较好的资源隔离和故障隔离，同时也能减少维护的代价，满足公有云和私有云的需求。 + +## 设计原则 + +- 使用简单 +- 开发代价小 +- 方便现有集群的迁移 + +## 名词解释 + +- FE: Frontend，即 Doris 中用于元数据管理即查询规划的模块。 +- BE: Backend，即 Doris 中用于存储和查询数据的模块。 +- Master: FE 的一种角色。一个Doris集群只有一个Master，其他的FE为Observer或者Follower。 +- instance：一个 BE 进程即是一个 instance。 +- host：单个物理机 +- cluster：即一个集群，由多个instance组成。 +- 租户：一个cluster属于一个租户。cluster和租户之间是一对一关系。 +- database：一个用户创建的数据库 + +## 主要思路 + +- 一个host上部署多个BE的instance，在进程级别做资源隔离。 +- 多个instance形成一个cluster，一个cluster分配给一个业务独立的的租户。 +- FE增加cluster这一级并负责cluster的管理。 +- CPU,IO,内存等资源隔离采用cgroup。 + +## 设计方案 + +为了能够达到隔离的目的，引入了**虚拟cluster**的概念。 + +1. cluster表示一个虚拟的集群，由多个BE的instance组成。多个cluster共享FE。 +2. 一个host上可以启动多个instance。cluster创建时，选取任意指定数量的instance，组成一个cluster。 +3. 创建cluster的同时，会创建一个名为superuser的账户，隶属于该cluster。superuser可以对cluster进行管理、创建数据库、分配权限等。 +4. Doris启动后，会创建一个默认的cluster：default_cluster。如果用户不希望使用多cluster的功能，则会提供这个默认的cluster，并隐藏多cluster的其他操作细节。 + +具体架构如下图： +![](/images/multi_tenant_arch.png) + +## SQL 接口 + +- 登录 + + 默认集群登录名： user_name@default_cluster 或者 user_name + + 自定义集群登录名：user_name@cluster_name + + `mysqlclient -h host -P port -u user_name@cluster_name -p password` + +- 添加、删除、下线（decommission）以及取消下线BE + + `ALTER SYSTEM ADD BACKEND "host:port"` + `ALTER SYSTEM DROP BACKEND "host:port"` + `ALTER SYSTEM DECOMMISSION BACKEND "host:port"` + `CANCEL DECOMMISSION BACKEND "host:port"` + + 强烈建议使用 DECOMMISSION 而不是 DROP 来删除 BACKEND。DECOMMISSION 操作会首先将需要下线节点上的数据拷贝到集群内其他instance上。之后，才会真正下线。 + +- 创建集群，并指定superuser账户的密码 + + `CREATE CLUSTER cluster_name PROPERTIES ("instance_num" = "10") identified by "password"` + +- 进入一个集群 + + `ENTER cluster_name` + +- 集群扩容、缩容 + + `ALTER CLUSTER cluster_name PROPERTIES ("instance_num" = "10")` + + 当指定的实例个数多于cluster现有be的个数，则为扩容，如果少于则为缩容。 + +- 链接、迁移db + + `LINK DATABASE src_cluster_name.db_name dest_cluster_name.db_name` + + 软链一个cluster的db到另外一个cluster的db ，对于需要临时访问其他cluster的db却不需要进行实际数据迁移的用户可以采用这种方式。 + + `MIGRATE DATABASE src_cluster_name.db_name dest_cluster_name.db_name` + + 如果需要对db进行跨cluster的迁移，在链接之后，执行migrate对数据进行实际的迁移。 + + 迁移不影响当前两个db的查询、导入等操作，这是一个异步的操作，可以通过`SHOW MIGRATIONS`查看迁移的进度。 + +- 删除集群 + + `DROP CLUSTER cluster_name` + + 删除集群，要求先手动删除的集群内所有database。 + +- 其他 + + `SHOW CLUSTERS` + + 展示系统内已经创建的集群。只有root用户有该权限。 + + `SHOW BACKENDS` + + 查看集群内的BE instance。 + + `SHOW MIGRATIONS` + + 展示当前正在进行的db迁移任务。执行完db的迁移后可以通过此命令查看迁移的进度。 + +## 详细设计 + +1. 命名空间隔离 + + 为了引入多租户，需要对系统内的cluster之间的命名空间进行隔离。 + + Doris现有的元数据采用的是image + journal 的方式(元数据的设计见相关文档)。Doris会把涉及元数据的操作的记录为一个 journal （操作日志），然后定时的按照**图1**的方式写成image，加载的时候按照写入的顺序读即可。但是这样就带来一个问题已经写入的格式不容易修改，比如记录数据分布的元数据格式为：database+table+tablet+replica 嵌套，如果按照以往的方式要做cluster之间的命名空间隔离，则需要在database上增加一层cluster，内部元数据的层级变为:cluster+database+table+tablet+replica，如**图2**所示。但加一层带来的问题有： + + - 增加一层带来的元数据改动，不兼容，需要按照图2的方式cluster＋db＋table＋tablet＋replica层级写,这样就改变了以往的元数据组织方式，老版本的升级会比较麻烦，比较理想的方式是按照图3在现有元数据的格式下顺序写入cluster的元数据。 + + - 代码里所有用到db、user等，都需要加一层cluster，一工作量大改动的地方多，层级深，多数代码都获取db，现有功能几乎都要改一遍，并且需要在db的锁的基础上嵌套一层cluster的锁。 + + ![](/images/palo_meta.png) + + 综上这里采用了一种通过给db、user名加前缀的方式去隔离内部因为cluster之间db、user名字冲突的问题。 + + 如下，所有的sql输入涉及db名、user名的，都需要根据自己所在的cluster来拼写db、user的全名。 + + ![](/images/cluster_namaspace.png) + + 采用这种方式以上两个问题不再有。元数据的组织方式也比较简单。即采用**图3**每个cluster记录下属于自己cluster的db、user，以及节点即可。 + +2. BE 节点管理 + + 每个cluster都有属于自己的一组instance，可以通过`SHOW BACKENDS`查看，为了区分出instance属于哪个cluster以及使用情况，BE引入了多个状态： + + - free：当一个BE节点被加入系统内，此时be不属于任何cluster的时候处于空闲状态 + - using：当创建集群、或者扩容被选取到一个cluster内则处于使用中。 + - cluster decommission：如果执行缩容量，则正在执行缩容的be处于此状态。结束后，be状态变为free。 + - system decommission：be正在下线中。下线完成后，该be将会被永久删除。 + + 只有root用户可以通过`SHOW PROC "/backends"`中cluster这一项查看集群内所有be的是否被使用。为空则为空闲，否则为使用中。`SHOW BACKENDS`只能看到所在cluster的节点。以下是be节点状态变化的示意图。 + + ![](/images/backend_state.png) + +3. 创建集群 + + 只有root用户可以创建一个cluster，并指定任意数量的BE instance。 + + 支持在相同机器上选取多个instance。选择instance的大致原则是：尽可能选取不同机器上的be并且使所有机器上使用的be数尽可能均匀。 + + 对于使用来讲，每一个user、db都属于一个cluster（root除外）。为了创建user、db，首先需要进入一个cluster。在创建cluster的时候系统会默认生成这个cluster的管理员，即superuser账户。superuser具有在所属cluster内创建db、user，以及查看be节点数的权限。所有的非root用户登录必须指定一个cluster，即`user_name@cluster_name`。 + + 只有root用户可以通过`SHOW CLUSTER`查看系统内所有的cluster，并且可以通过@不同的集群名来进入不同的cluster。对于除了root之外的用户cluster都是不可见的。 + + 为了兼容老版本Doris内置了一个名字叫做default_cluster的集群，这个名字在创建集群的时候不能使用。 + + ![](/images/user_authority.png) + +4. 集群扩容 + + 集群扩容的流程同创建集群。会优先选取不在集群之外的host上的BE instance。选取的原则同创建集群。 + +5. 集群缩容、CLUSTER DECOMMISSION + + 用户可以通过设置 cluster 的 instance num 来进行集群缩容。 + + 集群的缩容会优先在BE instance 数量最多的 host 上选取 instance 进行下线。 + + 用户也可以直接使用 `ALTER CLUSTER DECOMMISSION BACKEND` 来指定BE，进行集群缩容。 + +![](/images/replica_recover.png) + +6. 建表 + + 为了保证高可用，每个分片的副本必需在不同的机器上。所以建表时，选择副本所在be的策略为在每个host上随机选取一个be。然后从这些be中随机选取所需副本数量的be。总体上做到每个机器上分片分布均匀。 + + 因此，假如需要创建一个3副本的分片，即使cluster包含3个或以上的instance，但是只有2个或以下的host，依然不能创建该分片。 + +7. 负载均衡 + + 负载均衡的粒度为cluster级别，cluster之间不做负载均衡。但是在计算负载是在host一级进行的，而一个host上可能存在多个不同cluster的BE instance。 cluster内，会通过每个host上所有分片数目、存储使用率计算负载，然后把负载高的机器上的分片往负载低的机器上拷贝（详见负载均衡相关文档）。 + +8. LINK DATABASE（软链） + + 多个集群之间可以通过软链的方式访问彼此的数据。链接的级别为不同cluster的db。 + + 通过在一个cluster内，添加需要访问的其他cluster的db的信息，来访问其他cluster中的db。 + + 当查询链接的db时，所使用的计算以及存储资源为源db所在cluster的资源。 + + 被软链的db不能在源cluster中删除。只有链接的db被删除后，才可以删除源db。而删除链接db，不会删除源db。 + +9. MIGRATE DATABASE + + db可以在cluster之间进行物理迁移。 + + 要迁移db，必须先链接db。执行迁移后数据会迁移到链接的db所在的cluster，并且执行迁移后源db被删除，链接断开。 + + 数据的迁移，复用了负载均衡以及副本恢复中，复制数据的流程（详见负载均衡相关文档）。具体实现上，在执行`MIRAGTE`命令后，Doris会在元数据中，将源db的所有副本所属的cluster，修改为目的cluster。 + + Doris会定期检查集群内机器之间是否均衡、副本是否齐全、是否有多余的副本。db的迁移即借用了这个流程，在检查副本齐全的时候同时检查副本所在的be是否属于该cluster，如果不属于，则记入要恢复的副本。并且副本多余要删除的时候会优先删除cluster外的副本，然后再按照现有的策略选择：宕机的be的副本->clone的副本->版本落后的副本->负载高的host上的副本，直到副本没有多余。 + +![](/images/cluster_link_and_migrate_db.png) + +10. BE的进程隔离 + +  为了实现be进程之间实际cpu、io以及内存的隔离，需要依赖于be的部署。部署的时候需要在外围配置cgroup，把要部署的be的进程都写入cgroup。如果要实现io的物理隔离各be配置的数据存放路径需要在不同磁盘上，这里不做过多的介绍。 + diff --git a/docs/zh-CN/administrator-guide/operation/tablet-meta-tool.md b/docs/zh-CN/administrator-guide/operation/tablet-meta-tool.md new file mode 100644 index 00000000000000..f1e03e805579d1 --- /dev/null +++ b/docs/zh-CN/administrator-guide/operation/tablet-meta-tool.md @@ -0,0 +1,114 @@ +--- +{ + "title": "Tablet 元数据管理工具", + "language": "zh-CN" +} +--- + + + +# Tablet 元数据管理工具 + +## 背景 + +在最新版本的代码中，我们在 BE 端引入了 RocksDB，用于存储 tablet 的元信息，以解决之前通过 header 文件的方式存储元信息，带来的各种功能和性能方面的问题。当前每一个数据目录（root\_path），都会有一个对应的 RocksDB 实例，其中以 key-value 的方式，存放对应 root\_path 上的所有 tablet 的元数据。 + +为了方便进行这些元数据的维护，我们提供了在线的 http 接口方式和离线的 meta\_tool 工具以完成相关的管理操作。 + +其中 http 接口仅用于在线的查看 tablet 的元数据，可以在 BE 进程运行的状态下使用。 + +而 meta\_tool 工具则仅用于离线的各类元数据管理操作，必须先停止BE进程后，才可使用。 + +meta\_tool 工具存放在 BE 的 lib/ 目录下。 + +## 操作 + +### 查看 Tablet Meta + +查看 Tablet Meta 信息可以分为在线方法和离线方法 + +#### 在线 + +访问 BE 的 http 接口，获取对应的 Tablet Meta 信息： + +api： + +`http://{host}:{port}/api/meta/header/{tablet_id}/{schema_hash}` + + +> host: BE 的 hostname +> +> port: BE 的 http 端口 +> +> tablet_id: tablet id +> +> schema_hash: tablet 的 schema hash + +举例： + +`http://be_host:8040/api/meta/header/14156/2458238340` + +最终查询成功的话，会将 Tablet Meta 以 json 形式返回。 + +#### 离线 + +基于 meta\_tool 工具获取某个盘上的 Tablet Meta。 + +命令： + +``` +./lib/meta_tool --root_path=/path/to/root_path --operation=get_meta --tablet_id=xxx --schema_hash=xxx +``` + +> root_path: 在 be.conf 中配置的对应的 root_path 路径。 + +结果也是按照 json 的格式展现 Tablet Meta。 + +### 加载 header + +加载 header 的功能是为了完成实现 tablet 人工迁移而提供的。该功能是基于 json 格式的 Tablet Meta 实现的，所以如果涉及 shard 字段、version 信息的更改，可以直接在 Tablet Meta 的 json 内容中更改。然后使用以下的命令进行加载。 + +命令： + +``` +./lib/meta_tool --operation=load_meta --root_path=/path/to/root_path --json_header_path=path +``` + +### 删除 header + +为了实现从某个 be 的某个盘中删除某个 tablet 的功能。 + +命令: + +``` +./lib/meta_tool --operation=delete_meta --root_path=/path/to/root_path --tablet_id=xxx --schema_hash=xxx +``` + +### 展示 pb 格式的 TabletMeta + +这个命令是为了查看旧的基于文件的管理的PB格式的 Tablet Meta，以 json 的格式展示 Tablet Meta。 + +命令： + +``` +./lib/meta_tool --operation=show_meta --root_path=/path/to/root_path --pb_header_path=path +``` + + diff --git a/docs/zh-CN/administrator-guide/operation/tablet-repair-and-balance.md b/docs/zh-CN/administrator-guide/operation/tablet-repair-and-balance.md new file mode 100644 index 00000000000000..87b3ddc4da9c80 --- /dev/null +++ b/docs/zh-CN/administrator-guide/operation/tablet-repair-and-balance.md @@ -0,0 +1,673 @@ +--- +{ + "title": "数据副本管理", + "language": "zh-CN" +} +--- + + + +# 数据副本管理 + +从 0.9.0 版本开始，Doris 引入了优化后的副本管理策略，同时支持了更为丰富的副本状态查看工具。本文档主要介绍 Doris 数据副本均衡、修复方面的调度策略，以及副本管理的运维方法。帮助用户更方便的掌握和管理集群中的副本状态。 + +> Colocation 属性的表的副本修复和均衡可以参阅 `docs/documentation/cn/administrator-guide/colocation-join.md` + +## 名词解释 + +1. Tablet：Doris 表的逻辑分片，一个表有多个分片。 +2. Replica：分片的副本，默认一个分片有3个副本。 +3. Healthy Replica：健康副本，副本所在 Backend 存活，且副本的版本完整。 +4. TabletChecker（TC）：是一个常驻的后台线程，用于定期扫描所有的 Tablet，检查这些 Tablet 的状态，并根据检查结果，决定是否将 tablet 发送给 TabletScheduler。 +5. TabletScheduler（TS）：是一个常驻的后台线程，用于处理由 TabletChecker 发来的需要修复的 Tablet。同时也会进行集群副本均衡的工作。 +6. TabletSchedCtx（TSC）：是一个 tablet 的封装。当 TC 选择一个 tablet 后，会将其封装为一个 TSC，发送给 TS。 +7. Storage Medium：存储介质。Doris 支持对分区粒度指定不同的存储介质，包括 SSD 和 HDD。副本调度策略也是针对不同的存储介质分别调度的。 + +``` + + +--------+ +-----------+ + | Meta | | Backends | + +---^----+ +------^----+ + | | | 3. Send clone tasks + 1. Check tablets | | | + +--------v------+ +-----------------+ + | TabletChecker +--------> TabletScheduler | + +---------------+ +-----------------+ + 2. Waiting to be scheduled + + +``` + +上图是一个简化的工作流程。 + + +## 副本状态 + +一个 Tablet 的多个副本，可能因为某些情况导致状态不一致。Doris 会尝试自动修复这些状态不一致的副本，让集群尽快从错误状态中恢复。 + +**一个 Replica 的健康状态有以下几种：** + +1. BAD + + 即副本损坏。包括但不限于磁盘故障、BUG等引起的副本不可恢复的损毁状态。 + +2. VERSION\_MISSING + + 版本缺失。Doris 中每一批次导入都对应一个数据版本。而一个副本的数据由多个连续的版本组成。而由于导入错误、延迟等原因，可能导致某些副本的数据版本不完整。 + +3. HEALTHY + + 健康副本。即数据正常的副本，并且副本所在的 BE 节点状态正常（心跳正常且不处于下线过程中） + +**一个 Tablet 的健康状态由其所有副本的状态决定，有以下几种：** + +1. REPLICA\_MISSING + + 副本缺失。即存活副本数小于期望副本数。 + +2. VERSION\_INCOMPLETE + + 存活副本数大于等于期望副本数，但其中健康副本数小于期望副本数。 + +3. REPLICA\_RELOCATING + + 拥有等于 replication num 的版本完整的存活副本数，但是部分副本所在的 BE 节点处于 unavailable 状态（比如 decommission 中） + +4. REPLICA\_MISSING\_IN\_CLUSTER + + 当使用多 cluster 方式时，健康副本数大于等于期望副本数，但在对应 cluster 内的副本数小于期望副本数。 + +5. REDUNDANT + + 副本冗余。健康副本都在对应 cluster 内，但数量大于期望副本数。或者有多余的 unavailable 副本。 + +6. FORCE\_REDUNDANT + + 这是一个特殊状态。只会出现在当期望副本数大于等于可用节点数时，并且 Tablet 处于副本缺失状态时出现。这种情况下，需要先删除一个副本，以保证有可用节点用于创建新副本。 + +7. COLOCATE\_MISMATCH + + 针对 Colocation 属性的表的分片状态。表示分片副本与 Colocation Group 的指定的分布不一致。 + +8. COLOCATE\_REDUNDANT + + 针对 Colocation 属性的表的分片状态。表示 Colocation 表的分片副本冗余。 + +9. HEALTHY + + 健康分片，即条件[1-8]都不满足。 + +## 副本修复 + +TabletChecker 作为常驻的后台进程，会定期检查所有分片的状态。对于非健康状态的分片，将会交给 TabletScheduler 进行调度和修复。修复的实际操作，都由 BE 上的 clone 任务完成。FE 只负责生成这些 clone 任务。 + +> 注1：副本修复的主要思想是先通过创建或补齐使得分片的副本数达到期望值，然后再删除多余的副本。 +> +> 注2：一个 clone 任务就是完成从一个指定远端 BE 拷贝指定数据到指定目的端 BE 的过程。 + +针对不同的状态，我们采用不同的修复方式： + +1. REPLICA\_MISSING/REPLICA\_RELOCATING + + 选择一个低负载的，可用的 BE 节点作为目的端。选择一个健康副本作为源端。clone 任务会从源端拷贝一个完整的副本到目的端。对于副本补齐，我们会直接选择一个可用的 BE 节点，而不考虑存储介质。 + +2. VERSION\_INCOMPLETE + + 选择一个相对完整的副本作为目的端。选择一个健康副本作为源端。clone 任务会从源端尝试拷贝缺失的版本到目的端的副本。 + +3. REPLICA\_MISSING\_IN\_CLUSTER + + 这种状态处理方式和 REPLICA\_MISSING 相同。 + +4. REDUNDANT + + 通常经过副本修复后，分片会有冗余的副本。我们选择一个冗余副本将其删除。冗余副本的选择遵从以下优先级： + 1. 副本所在 BE 已经下线 + 2. 副本已损坏 + 3. 副本所在 BE 失联或在下线中 + 4. 副本处于 CLONE 状态（该状态是 clone 任务执行过程中的一个中间状态） + 5. 副本有版本缺失 + 6. 副本所在 cluster 不正确 + 7. 副本所在 BE 节点负载高 + +5. FORCE\_REDUNDANT + + 不同于 REDUNDANT，因为此时虽然 Tablet 有副本缺失，但是因为已经没有额外的可用节点用于创建新的副本了。所以此时必须先删除一个副本，以腾出一个可用节点用于创建新的副本。 + 删除副本的顺序同 REDUNDANT。 + +6. COLOCATE\_MISMATCH + + 从 Colocation Group 中指定的副本分布 BE 节点中选择一个作为目的节点进行副本补齐。 + +7. COLOCATE\_REDUNDANT + + 删除一个非 Colocation Group 中指定的副本分布 BE 节点上的副本。 + +Doris 在选择副本节点时，不会将同一个 Tablet 的副本部署在同一个 host 的不同 BE 上。保证了即使同一个 host 上的所有 BE 都挂掉，也不会造成全部副本丢失。 + +### 调度优先级 + +TabletScheduler 里等待被调度的分片会根据状态不同，赋予不同的优先级。优先级高的分片将会被优先调度。目前有以下几种优先级。 + +1. VERY\_HIGH + + * REDUNDANT。对于有副本冗余的分片，我们优先处理。虽然逻辑上来讲，副本冗余的紧急程度最低，但是因为这种情况处理起来最快且可以快速释放资源（比如磁盘空间等），所以我们优先处理。 + * FORCE\_REDUNDANT。同上。 + +2. HIGH + + * REPLICA\_MISSING 且多数副本缺失（比如3副本丢失了2个） + * VERSION\_INCOMPLETE 且多数副本的版本缺失 + * COLOCATE\_MISMATCH 我们希望 Colocation 表相关的分片能够尽快修复完成。 + * COLOCATE\_REDUNDANT + +3. NORMAL + + * REPLICA\_MISSING 但多数存活（比如3副本丢失了1个） + * VERSION\_INCOMPLETE 但多数副本的版本完整 + * REPLICA\_RELOCATING 且多数副本需要 relocate（比如3副本有2个） + +4. LOW + + * REPLICA\_MISSING\_IN\_CLUSTER + * REPLICA\_RELOCATING 但多数副本 stable + +### 手动优先级 + +系统会自动判断调度优先级。但是有些时候，用户希望某些表或分区的分片能够更快的被修复。因此我们提供一个命令，用户可以指定某个表或分区的分片被优先修复： + +`ADMIN REPAIR TABLE tbl [PARTITION (p1, p2, ...)];` + +这个命令，告诉 TC，在扫描 Tablet 时，对需要优先修复的表或分区中的有问题的 Tablet，给予 VERY\_HIGH 的优先级。 + +> 注：这个命令只是一个 hint，并不能保证一定能修复成功，并且优先级也会随 TS 的调度而发生变化。并且当 Master FE 切换或重启后，这些信息都会丢失。 + +可以通过以下命令取消优先级： + +`ADMIN CANCEL REPAIR TABLE tbl [PARTITION (p1, p2, ...)];` + +### 优先级调度 + +优先级保证了损坏严重的分片能够优先被修复，提高系统可用性。但是如果高优先级的修复任务一直失败，则会导致低优先级的任务一直得不到调度。因此，我们会根据任务的运行状态，动态的调整任务的优先级，保证所有任务都有机会被调度到。 + +* 连续5次调度失败（如无法获取资源，无法找到合适的源端或目的端等），则优先级会被下调。 +* 持续 30 分钟未被调度，则上调优先级。 +* 同一 tablet 任务的优先级至少间隔 5 分钟才会被调整一次。 + +同时为了保证初始优先级的权重，我们规定，初始优先级为 VERY\_HIGH 的，最低被下调到 NORMAL。而初始优先级为 LOW 的，最多被上调为 HIGH。这里的优先级调整，也会调整用户手动设置的优先级。 + +## 副本均衡 + +Doris 会自动进行集群内的副本均衡。均衡的主要思想，是对某些分片，先在低负载的节点上创建一个副本，然后再删除这些分片在高负载节点上的副本。同时，因为不同存储介质的存在，在同一个集群内的不同 BE 节点上，可能存在一种或两种存储介质。我们要求存储介质为 A 的分片在均衡后，尽量依然存储在存储介质 A 中。所以我们根据存储介质，对集群的 BE 节点进行划分。然后针对不同的存储介质的 BE 节点集合，进行负载均衡调度。 + +同样，副本均衡会保证不会将同一个 Tablet 的副本部署在同一个 host 的 BE 上。 + +### BE 节点负载 + +我们用 ClusterLoadStatistics（CLS）表示一个 cluster 中各个 Backend 的负载均衡情况。TabletScheduler 根据这个统计值，来触发集群均衡。我们当前通过 **磁盘使用率** 和 **副本数量** 两个指标，为每个BE计算一个 loadScore，作为 BE 的负载分数。分数越高，表示该 BE 的负载越重。 + +磁盘使用率和副本数量各有一个权重系数，分别为 **capacityCoefficient** 和 **replicaNumCoefficient**，其 **和衡为1**。其中 capacityCoefficient 会根据实际磁盘使用率动态调整。当一个 BE 的总体磁盘使用率在 50% 以下，则 capacityCoefficient 值为 0.5，如果磁盘使用率在 75%（可通过 FE 配置项 `capacity_used_percent_high_water` 配置）以上，则值为 1。如果使用率介于 50% ~ 75% 之间，则该权重系数平滑增加，公式为： + +`capacityCoefficient= 2 * 磁盘使用率 - 0.5` + +该权重系数保证当磁盘使用率过高时，该 Backend 的负载分数会更高，以保证尽快降低这个 BE 的负载。 + +TabletScheduler 会每隔 1 分钟更新一次 CLS。 + +### 均衡策略 + +TabletScheduler 在每轮调度时，都会通过 LoadBalancer 来选择一定数目的健康分片作为 balance 的候选分片。在下一次调度时，会尝试根据这些候选分片，进行均衡调度。 + +## 资源控制 + +无论是副本修复还是均衡，都是通过副本在各个 BE 之间拷贝完成的。如果同一台 BE 同一时间执行过多的任务，则会带来不小的 IO 压力。因此，Doris 在调度时控制了每个节点上能够执行的任务数目。最小的资源控制单位是磁盘（即在 be.conf 中指定的一个数据路径）。我们默认为每块磁盘配置两个 slot 用于副本修复。一个 clone 任务会占用源端和目的端各一个 slot。如果 slot 数目为零，则不会再对这块磁盘分配任务。该 slot 个数可以通过 FE 的 `schedule_slot_num_per_path` 参数配置。 + +另外，我们默认为每块磁盘提供 2 个单独的 slot 用于均衡任务。目的是防止高负载的节点因为 slot 被修复任务占用，而无法通过均衡释放空间。 + +## 副本状态查看 + +副本状态查看主要是查看副本的状态，以及副本修复和均衡任务的运行状态。这些状态大部分都**仅存在于** Master FE 节点中。因此，以下命令需直连到 Master FE 执行。 + +### 副本状态 + +1. 全局状态检查 + + 通过 `SHOW PROC '/statistic';` 命令可以查看整个集群的副本状态。 + + ``` + +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ + | DbId | DbName | TableNum | PartitionNum | IndexNum | TabletNum | ReplicaNum | UnhealthyTabletNum | InconsistentTabletNum | + +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ + | 35153636 | default_cluster:DF_Newrisk | 3 | 3 | 3 | 96 | 288 | 0 | 0 | + | 48297972 | default_cluster:PaperData | 0 | 0 | 0 | 0 | 0 | 0 | 0 | + | 5909381 | default_cluster:UM_TEST | 7 | 7 | 10 | 320 | 960 | 1 | 0 | + | Total | 240 | 10 | 10 | 13 | 416 | 1248 | 1 | 0 | + +----------+-----------------------------+----------+--------------+----------+-----------+------------+--------------------+-----------------------+ + ``` + + 其中 `UnhealthyTabletNum` 列显示了对应的 Database 中，有多少 Tablet 处于非健康状态。`InconsistentTabletNum` 列显示了对应的 Database 中，有多少 Tablet 处于副本不一致的状态。最后一行 `Total` 行对整个集群进行了统计。正常情况下 `UnhealthyTabletNum` 和 `InconsistentTabletNum` 应为0。如果不为零，可以进一步查看具体有哪些 Tablet。如上图中，UM_TEST 数据库有 1 个 Tablet 状态不健康，则可以使用以下命令查看具体是哪一个 Tablet。 + + `SHOW PROC '/statistic/5909381';` + + 其中 `5909381` 为对应的 DbId。 + + ``` + +------------------+---------------------+ + | UnhealthyTablets | InconsistentTablets | + +------------------+---------------------+ + | [40467980] | [] | + +------------------+---------------------+ + ``` + + 上图会显示具体的不健康的 Tablet ID（40467980）。后面我们会介绍如何查看一个具体的 Tablet 的各个副本的状态。 + +2. 表（分区）级别状态检查 + + 用户可以通过以下命令查看指定表或分区的副本状态，并可以通过 WHERE 语句对状态进行过滤。如查看表 tbl1 中，分区 p1 和 p2 上状态为 NORMAL 的副本： + + `ADMIN SHOW REPLICA STATUS FROM tbl1 PARTITION (p1, p2) WHERE STATUS = "NORMAL";` + + ``` + +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ + | TabletId | ReplicaId | BackendId | Version | LastFailedVersion | LastSuccessVersion | CommittedVersion | SchemaHash | VersionNum | IsBad | State | Status | + +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ + | 29502429 | 29502432 | 10006 | 2 | -1 | 2 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502429 | 36885996 | 10002 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502429 | 48100551 | 10007 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502433 | 29502434 | 10001 | 2 | -1 | 2 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502433 | 44900737 | 10004 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | + | 29502433 | 48369135 | 10006 | 2 | -1 | -1 | 1 | -1 | 2 | false | NORMAL | OK | + +----------+-----------+-----------+---------+-------------------+--------------------+------------------+------------+------------+-------+--------+--------+ + ``` + + 这里会展示所有副本的状态。其中 `IsBad` 列为 `true` 则表示副本已经损坏。而 `Status` 列则会显示另外的其他状态。具体的状态说明，可以通过 `HELP ADMIN SHOW REPLICA STATUS;` 查看帮助。 + + `ADMIN SHOW REPLICA STATUS` 命令主要用于查看副本的健康状态。用户还可以通过以下命令查看指定表中副本的一些额外信息： + + `SHOW TABLET FROM tbl1;` + + ``` + +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ + | TabletId | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | DataSize | RowCount | State | LstConsistencyCheckTime | CheckVersion | CheckVersionHash | VersionCount | PathHash | + +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ + | 29502429 | 29502432 | 10006 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -5822326203532286804 | + | 29502429 | 36885996 | 10002 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -1441285706148429853 | + | 29502429 | 48100551 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -4784691547051455525 | + +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ + ``` + + 上图展示了包括副本大小、行数、版本数量、所在数据路径等一些额外的信息。 + + > 注：这里显示的 `State` 列的内容不代表副本的健康状态，而是副本处于某种任务下的状态，比如 CLONE、SCHEMA\_CHANGE、ROLLUP 等。 + + 此外，用户也可以通过以下命令，查看指定表或分区的副本分布情况，来检查副本分布是否均匀。 + + `ADMIN SHOW REPLICA DISTRIBUTION FROM tbl1;` + + ``` + +-----------+------------+-------+---------+ + | BackendId | ReplicaNum | Graph | Percent | + +-----------+------------+-------+---------+ + | 10000 | 7 | | 7.29 % | + | 10001 | 9 | | 9.38 % | + | 10002 | 7 | | 7.29 % | + | 10003 | 7 | | 7.29 % | + | 10004 | 9 | | 9.38 % | + | 10005 | 11 | > | 11.46 % | + | 10006 | 18 | > | 18.75 % | + | 10007 | 15 | > | 15.62 % | + | 10008 | 13 | > | 13.54 % | + +-----------+------------+-------+---------+ + ``` + + 这里分别展示了表 tbl1 的副本在各个 BE 节点上的个数、百分比，以及一个简单的图形化显示。 + +4. Tablet 级别状态检查 + + 当我们要定位到某个具体的 Tablet 时，可以使用如下命令来查看一个具体的 Tablet 的状态。如查看 ID 为 29502553 的 tablet： + + `SHOW TABLET 29502553;` + + ``` + +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ + | DbName | TableName | PartitionName | IndexName | DbId | TableId | PartitionId | IndexId | IsSync | DetailCmd | + +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ + | default_cluster:test | test | test | test | 29502391 | 29502428 | 29502427 | 29502428 | true | SHOW PROC '/dbs/29502391/29502428/partitions/29502427/29502428/29502553'; | + +------------------------+-----------+---------------+-----------+----------+----------+-------------+----------+--------+---------------------------------------------------------------------------+ + ``` + + 上图显示了这个 tablet 所对应的数据库、表、分区、上卷表等信息。用户可以复制 `DetailCmd` 命令中的命令继续执行： + + `SHOW PROC '/dbs/29502391/29502428/partitions/29502427/29502428/29502553';` + + ``` + +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ + | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | SchemaHash | DataSize | RowCount | State | IsBad | VersionCount | PathHash | + +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ + | 43734060 | 10004 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | -8566523878520798656 | + | 29502555 | 10002 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | 1885826196444191611 | + | 39279319 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | -1 | 784 | 0 | NORMAL | false | 2 | 1656508631294397870 | + +-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+ + ``` + + 上图显示了对应 Tablet 的所有副本情况。这里显示的内容和 `SHOW TABLET FROM tbl1;` 的内容相同。但这里可以清楚的知道，一个具体的 Tablet 的所有副本的状态。 + +### 副本调度任务 + +1. 查看等待被调度的任务 + + `SHOW PROC '/cluster_balance/pending_tablets';` + + ``` + +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ + | TabletId | Type | Status | State | OrigPrio | DynmPrio | SrcBe | SrcPath | DestBe | DestPath | Timeout | Create | LstSched | LstVisit | Finished | Rate | FailedSched | FailedRunning | LstAdjPrio | VisibleVer | VisibleVerHash | CmtVer | CmtVerHash | ErrMsg | + +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ + | 4203036 | REPAIR | REPLICA_MISSING | PENDING | HIGH | LOW | -1 | -1 | -1 | -1 | 0 | 2019-02-21 15:00:20 | 2019-02-24 11:18:41 | 2019-02-24 11:18:41 | N/A | N/A | 2 | 0 | 2019-02-21 15:00:43 | 1 | 0 | 2 | 0 | unable to find source replica | + +----------+--------+-----------------+---------+----------+----------+-------+---------+--------+----------+---------+---------------------+---------------------+---------------------+----------+------+-------------+---------------+---------------------+------------+---------------------+--------+---------------------+-------------------------------+ + ``` + + 各列的具体含义如下： + + * TabletId：等待调度的 Tablet 的 ID。一个调度任务只针对一个 Tablet + * Type：任务类型，可以是 REPAIR（修复） 或 BALANCE（均衡） + * Status：该 Tablet 当前的状态，如 REPLICA\_MISSING（副本缺失） + * State：该调度任务的状态，可能为 PENDING/RUNNING/FINISHED/CANCELLED/TIMEOUT/UNEXPECTED + * OrigPrio：初始的优先级 + * DynmPrio：当前动态调整后的优先级 + * SrcBe：源端 BE 节点的 ID + * SrcPath：源端 BE 节点的路径的 hash 值 + * DestBe：目的端 BE 节点的 ID + * DestPath：目的端 BE 节点的路径的 hash 值 + * Timeout：当任务被调度成功后，这里会显示任务的超时时间，单位秒 + * Create：任务被创建的时间 + * LstSched：上一次任务被调度的时间 + * LstVisit：上一次任务被访问的时间。这里“被访问”指包括被调度，任务执行汇报等和这个任务相关的被处理的时间点 + * Finished：任务结束时间 + * Rate：clone 任务的数据拷贝速率 + * FailedSched：任务调度失败的次数 + * FailedRunning：任务执行失败的次数 + * LstAdjPrio：上一次优先级调整的时间 + * CmtVer/CmtVerHash/VisibleVer/VisibleVerHash：用于执行 clone 任务的 version 信息 + * ErrMsg：任务被调度和运行过程中，出现的错误信息 + +2. 查看正在运行的任务 + + `SHOW PROC '/cluster_balance/running_tablets';` + + 其结果中各列的含义和 `pending_tablets` 相同。 + +3. 查看已结束任务 + + `SHOW PROC '/cluster_balance/history_tablets';` + + 我们默认只保留最近 1000 个完成的任务。其结果中各列的含义和 `pending_tablets` 相同。如果 `State` 列为 `FINISHED`，则说明任务正常完成。如果为其他，则可以根据 `ErrMsg` 列的错误信息查看具体原因。 + +## 集群负载及调度资源查看 + +1. 集群负载 + + 通过以下命令可以查看集群当前的负载情况： + + `SHOW PROC '/cluster_balance/cluster_load_stat';` + + 首先看到的是对不同存储介质的划分： + + ``` + +---------------+ + | StorageMedium | + +---------------+ + | HDD | + | SSD | + +---------------+ + ``` + + 点击某一种存储介质，可以看到包含该存储介质的 BE 节点的均衡状态： + + `SHOW PROC '/cluster_balance/cluster_load_stat/HDD';` + + ``` + +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ + | BeId | Cluster | Available | UsedCapacity | Capacity | UsedPercent | ReplicaNum | CapCoeff | ReplCoeff | Score | Class | + +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ + | 10003 | default_cluster | true | 3477875259079 | 19377459077121 | 17.948 | 493477 | 0.5 | 0.5 | 0.9284678149967587 | MID | + | 10002 | default_cluster | true | 3607326225443 | 19377459077121 | 18.616 | 496928 | 0.5 | 0.5 | 0.948660871419998 | MID | + | 10005 | default_cluster | true | 3523518578241 | 19377459077121 | 18.184 | 545331 | 0.5 | 0.5 | 0.9843539990641831 | MID | + | 10001 | default_cluster | true | 3535547090016 | 19377459077121 | 18.246 | 558067 | 0.5 | 0.5 | 0.9981869446537612 | MID | + | 10006 | default_cluster | true | 3636050364835 | 19377459077121 | 18.764 | 547543 | 0.5 | 0.5 | 1.0011489897614072 | MID | + | 10004 | default_cluster | true | 3506558163744 | 15501967261697 | 22.620 | 468957 | 0.5 | 0.5 | 1.0228319835582569 | MID | + | 10007 | default_cluster | true | 4036460478905 | 19377459077121 | 20.831 | 551645 | 0.5 | 0.5 | 1.057279369420761 | MID | + | 10000 | default_cluster | true | 4369719923760 | 19377459077121 | 22.551 | 547175 | 0.5 | 0.5 | 1.0964036415787461 | MID | + +----------+-----------------+-----------+---------------+----------------+-------------+------------+----------+-----------+--------------------+-------+ + ``` + + 其中一些列的含义如下： + + * Available：为 true 表示 BE 心跳正常，且没有处于下线中 + * UsedCapacity：字节，BE 上已使用的磁盘空间大小 + * Capacity：字节，BE 上总的磁盘空间大小 + * UsedPercent：百分比，BE 上的磁盘空间使用率 + * ReplicaNum：BE 上副本数量 + * CapCoeff/ReplCoeff：磁盘空间和副本数的权重系数 + * Score：负载分数。分数越高，负载越重 + * Class：根据负载情况分类，LOW/MID/HIGH。均衡调度会将高负载节点上的副本迁往低负载节点 + + 用户可以进一步查看某个 BE 上各个路径的使用率，比如 ID 为 10001 这个 BE： + + `SHOW PROC '/cluster_balance/cluster_load_stat/HDD/10001';` + + ``` + +------------------+------------------+---------------+---------------+---------+--------+----------------------+ + | RootPath | DataUsedCapacity | AvailCapacity | TotalCapacity | UsedPct | State | PathHash | + +------------------+------------------+---------------+---------------+---------+--------+----------------------+ + | /home/disk4/palo | 498.757 GB | 3.033 TB | 3.525 TB | 13.94 % | ONLINE | 4883406271918338267 | + | /home/disk3/palo | 704.200 GB | 2.832 TB | 3.525 TB | 19.65 % | ONLINE | -5467083960906519443 | + | /home/disk1/palo | 512.833 GB | 3.007 TB | 3.525 TB | 14.69 % | ONLINE | -7733211489989964053 | + | /home/disk2/palo | 881.955 GB | 2.656 TB | 3.525 TB | 24.65 % | ONLINE | 4870995507205544622 | + | /home/disk5/palo | 694.992 GB | 2.842 TB | 3.525 TB | 19.36 % | ONLINE | 1916696897889786739 | + +------------------+------------------+---------------+---------------+---------+--------+----------------------+ + ``` + + 这里显示了指定 BE 上，各个数据路径的磁盘使用率情况。 + +2. 调度资源 + + 用户可以通过以下命令，查看当前各个节点的 slot 使用情况： + + `SHOW PROC '/cluster_balance/working_slots';` + + ``` + +----------+----------------------+------------+------------+-------------+----------------------+ + | BeId | PathHash | AvailSlots | TotalSlots | BalanceSlot | AvgRate | + +----------+----------------------+------------+------------+-------------+----------------------+ + | 10000 | 8110346074333016794 | 2 | 2 | 2 | 2.459007474009069E7 | + | 10000 | -5617618290584731137 | 2 | 2 | 2 | 2.4730105014001578E7 | + | 10001 | 4883406271918338267 | 2 | 2 | 2 | 1.6711402709780257E7 | + | 10001 | -5467083960906519443 | 2 | 2 | 2 | 2.7540126380326536E7 | + | 10002 | 9137404661108133814 | 2 | 2 | 2 | 2.417217089806745E7 | + | 10002 | 1885826196444191611 | 2 | 2 | 2 | 1.6327378456676323E7 | + +----------+----------------------+------------+------------+-------------+----------------------+ + ``` + + 这里以数据路径为粒度，展示了当前 slot 的使用情况。其中 `AvgRate` 为历史统计的该路径上 clone 任务的拷贝速率，单位是字节/秒。 + +3. 优先修复查看 + + 以下命令，可以查看通过 `ADMIN REPAIR TABLE` 命令设置的优先修复的表或分区。 + + `SHOW PROC '/cluster_balance/priority_repair'；` + + 其中 `RemainingTimeMs` 表示，这些优先修复的内容，将在这个时间后，被自动移出优先修复队列。以防止优先修复一直失败导致资源被占用。 + +### 调度器统计状态查看 + +我们收集了 TabletChecker 和 TabletScheduler 在运行过程中的一些统计信息，可以通过以下命令查看： + +`SHOW PROC '/cluster_balance/sched_stat'；` + +``` ++---------------------------------------------------+-------------+ +| Item | Value | ++---------------------------------------------------+-------------+ +| num of tablet check round | 12041 | +| cost of tablet check(ms) | 7162342 | +| num of tablet checked in tablet checker | 18793506362 | +| num of unhealthy tablet checked in tablet checker | 7043900 | +| num of tablet being added to tablet scheduler | 1153 | +| num of tablet schedule round | 49538 | +| cost of tablet schedule(ms) | 49822 | +| num of tablet being scheduled | 4356200 | +| num of tablet being scheduled succeeded | 320 | +| num of tablet being scheduled failed | 4355594 | +| num of tablet being scheduled discard | 286 | +| num of tablet priority upgraded | 0 | +| num of tablet priority downgraded | 1096 | +| num of clone task | 230 | +| num of clone task succeeded | 228 | +| num of clone task failed | 2 | +| num of clone task timeout | 2 | +| num of replica missing error | 4354857 | +| num of replica version missing error | 967 | +| num of replica relocating | 0 | +| num of replica redundant error | 90 | +| num of replica missing in cluster error | 0 | +| num of balance scheduled | 0 | ++---------------------------------------------------+-------------+ +``` + +各行含义如下： + +* num of tablet check round：Tablet Checker 检查次数 +* cost of tablet check(ms)：Tablet Checker 检查总耗时 +* num of tablet checked in tablet checker：Tablet Checker 检查过的 tablet 数量 +* num of unhealthy tablet checked in tablet checker：Tablet Checker 检查过的不健康的 tablet 数量 +* num of tablet being added to tablet scheduler：被提交到 Tablet Scheduler 中的 tablet 数量 +* num of tablet schedule round：Tablet Scheduler 运行次数 +* cost of tablet schedule(ms)：Tablet Scheduler 运行总耗时 +* num of tablet being scheduled：被调度的 Tablet 总数量 +* num of tablet being scheduled succeeded：被成功调度的 Tablet 总数量 +* num of tablet being scheduled failed：调度失败的 Tablet 总数量 +* num of tablet being scheduled discard：调度失败且被抛弃的 Tablet 总数量 +* num of tablet priority upgraded：优先级上调次数 +* num of tablet priority downgraded：优先级下调次数 +* num of clone task：生成的 clone 任务数量 +* num of clone task succeeded：clone 任务成功的数量 +* num of clone task failed：clone 任务失败的数量 +* num of clone task timeout：clone 任务超时的数量 +* num of replica missing error：检查的状态为副本缺失的 tablet 的数量 +* num of replica version missing error：检查的状态为版本缺失的 tablet 的数量（该统计值包括了 num of replica relocating 和 num of replica missing in cluster error） +* num of replica relocating：检查的状态为 replica relocating 的 tablet 的数量 +* num of replica redundant error：检查的状态为副本冗余的 tablet 的数量 +* num of replica missing in cluster error：检查的状态为不在对应 cluster 的 tablet 的数量 +* num of balance scheduled：均衡调度的次数 + +> 注：以上状态都只是历史累加值。我们也在 FE 的日志中，定期打印了这些统计信息，其中括号内的数值表示自上次统计信息打印依赖，各个统计值的变化数量。 + +## 相关配置说明 + +### 可调整参数 + +以下可调整参数均为 fe.conf 中可配置参数。 + +* use\_new\_tablet\_scheduler + + * 说明：是否启用新的副本调度方式。新的副本调度方式即本文档介绍的副本调度方式。 + * 默认值：true + * 重要性：高 + +* tablet\_repair\_delay\_factor\_second + + * 说明：对于不同的调度优先级，我们会延迟不同的时间后开始修复。以防止因为例行重启、升级等过程中，产生大量不必要的副本修复任务。此参数为一个基准系数。对于 HIGH 优先级，延迟为 基准系数 * 1；对于 NORMAL 优先级，延迟为 基准系数 * 2；对于 LOW 优先级，延迟为 基准系数 * 3。即优先级越低，延迟等待时间越长。如果用户想尽快修复副本，可以适当降低该参数。 + * 默认值：60秒 + * 重要性：高 + +* schedule\_slot\_num\_per\_path + + * 说明：默认分配给每块磁盘用于副本修复的 slot 数目。该数目表示一块磁盘能同时运行的副本修复任务数。如果想以更快的速度修复副本，可以适当调高这个参数。单数值越高，可能对 IO 影响越大。 + * 默认值：2 + * 重要性：高 + +* balance\_load\_score\_threshold + + * 说明：集群均衡的阈值。默认为 0.1，即 10%。当一个 BE 节点的 load score，不高于或不低于平均 load score 的 10% 时，我们认为这个节点是均衡的。如果想让集群负载更加平均，可以适当调低这个参数。 + * 默认值：0.1 + * 重要性：中 + +* storage\_high\_watermark\_usage\_percent 和 storage\_min\_left\_capacity\_bytes + + * 说明：这两个参数，分别表示一个磁盘的最大空间使用率上限，以及最小的空间剩余下限。当一块磁盘的空间使用率大于上限，或者剩余空间小于下限时，该磁盘将不再作为均衡调度的目的地址。 + * 默认值：0.85 和 1048576000 （1GB） + * 重要性：中 + +* disable\_balance + + * 说明：控制是否关闭均衡功能。当副本处于均衡过程中时，有些功能，如 ALTER TABLE 等将会被禁止。而均衡可能持续很长时间。因此，如果用户希望尽快进行被禁止的操作。可以将该参数设为 true，以关闭均衡调度。 + * 默认值：true + * 重要性：中 + +### 不可调整参数 + +以下参数暂不支持修改，仅作说明。 + +* TabletChecker 调度间隔 + + TabletChecker 每20秒进行一次检查调度。 + +* TabletScheduler 调度间隔 + + TabletScheduler 每5秒进行一次调度 + +* TabletScheduler 每批次调度个数 + + TabletScheduler 每次调度最多 50 个 tablet。 + +* TabletScheduler 最大等待调度和运行中任务数 + + 最大等待调度任务数和运行中任务数为 2000。当超过 2000 后，TabletChecker 将不再产生新的调度任务给 TabletScheduler。 + +* TabletScheduler 最大均衡任务数 + + 最大均衡任务数为 500。当超过 500 后，将不再产生新的均衡任务。 + +* 每块磁盘用于均衡任务的 slot 数目 + + 每块磁盘用于均衡任务的 slot 数目为2。这个 slot 独立于用于副本修复的 slot。 + +* 集群均衡情况更新间隔 + + TabletScheduler 每隔 20 秒会重新计算一次集群的 load score。 + +* Clone 任务的最小和最大超时时间 + + 一个 clone 任务超时时间范围是 3min ~ 2hour。具体超时时间通过 tablet 的大小计算。计算公式为 (tablet size) / (5MB/s)。当一个 clone 任务运行失败 3 次后，该任务将终止。 + +* 动态优先级调整策略 + + 优先级最小调整间隔为 5min。当一个 tablet 调度失败5次后，会调低优先级。当一个 tablet 30min 未被调度时，会调高优先级。 + +## 相关问题 + +* 在某些情况下，默认的副本修复和均衡策略可能会导致网络被打满（多发生在千兆网卡，且每台 BE 的磁盘数量较多的情况下）。此时需要调整一些参数来减少同时进行的均衡和修复任务数。 + +* 目前针对 Colocate Table 的副本的均衡策略无法保证同一个 Tablet 的副本不会分布在同一个 host 的 BE 上。但 Colocate Table 的副本的修复策略会检测到这种分布错误并校正。但可能会出现，校正后，均衡策略再次认为副本不均衡而重新均衡。从而导致在两种状态间不停交替，无法使 Colocate Group 达成稳定。针对这种情况，我们建议在使用 Colocate 属性时，尽量保证集群是同构的，以减小副本分布在同一个 host 上的概率。 + + + + + diff --git a/docs/documentation/cn/administrator-guide/operation/tablet-restore-tool.md b/docs/zh-CN/administrator-guide/operation/tablet-restore-tool.md similarity index 97% rename from docs/documentation/cn/administrator-guide/operation/tablet-restore-tool.md rename to docs/zh-CN/administrator-guide/operation/tablet-restore-tool.md index 30a6caf723e5ab..5f1e1769119e93 100644 --- a/docs/documentation/cn/administrator-guide/operation/tablet-restore-tool.md +++ b/docs/zh-CN/administrator-guide/operation/tablet-restore-tool.md @@ -1,3 +1,10 @@ +--- +{ + "title": "BE Tablet数据恢复工具", + "language": "zh-CN" +} +--- + + +# 权限管理 + +Doris 新的权限管理系统参照了 Mysql 的权限管理机制，做到了表级别细粒度的权限控制，基于角色的权限访问控制，并且支持白名单机制。 + +## 名词解释 + +1. 用户标识 user_identity + + 在权限系统中，一个用户被识别为一个 User Identity（用户标识）。用户标识由两部分组成：username 和 userhost。其中 username 为用户名，由英文大小写组成。userhost 表示该用户链接来自的 IP。user_identity 以 username@'userhost' 的方式呈现，表示来自 userhost 的 username。 + + user_identity 的另一种表现方式为 username@['domain']，其中 domain 为域名，可以通过 DNS 或 BNS（百度名字服务）解析为一组 ip。最终表现为一组 username@'userhost'，所以后面我们统一使用 username@'userhost' 来表示。 + +2. 权限 Privilege + + 权限作用的对象是节点、数据库或表。不同的权限代表不同的操作许可。 + +3. 角色 Role + + Doris可以创建自定义命名的角色。角色可以被看做是一组权限的集合。新创建的用户可以被赋予某一角色，则自动被赋予该角色所拥有的权限。后续对角色的权限变更，也会体现在所有属于该角色的用户权限上。 + +4. 用户属性 user_property + + 用户属性直接附属于某一用户，而不是用户标识。即 cmy@'192.%' 和 cmy@['domain'] 都拥有同一组用户属性，该属性属于用户 cmy，而不是 cmy@'192.%' 或 cmy@['domain']。 + + 用户属性包括但不限于： 用户最大连接数、导入集群配置等等。 + +## 支持的操作 + +1. 创建用户：CREATE USER +2. 删除用户：DROP USER +3. 授权：GRANT +4. 撤权：REVOKE +5. 创建角色：CREATE ROLE +6. 删除角色：DROP ROLE +7. 查看当前用户权限：SHOW GRANTS +8. 查看所有用户权限：SHOW ALL GRANTS +9. 查看已创建的角色：SHOW ROLES +10. 查看用户属性：SHOW PROPERTY + +关于以上命令的详细帮助，可以通过 mysql 客户端连接 Doris 后，使用 help + command 获取帮助。如 `HELP CREATE USER`。 + +## 权限类型 + +Doris 目前支持以下几种权限 + +1. Node_priv + + 节点变更权限。包括 FE、BE、BROKER 节点的添加、删除、下线等操作。目前该权限只能授予 Root 用户。 + +2. Grant_priv + + 权限变更权限。允许执行包括授权、撤权、添加/删除/变更 用户/角色 等操作。 + +3. Select_priv + + 对数据库、表的只读权限。 + +4. Load_priv + + 对数据库、表的写权限。包括 Load、Insert、Delete 等。 + +5. Alter_priv + + 对数据库、表的更改权限。包括重命名 库/表、添加/删除/变更 列、添加/删除 分区等操作。 + +6. Create_priv + + 创建数据库、表、视图的权限。 + +7. Drop_priv + + 删除数据库、表、视图的权限。 + +## 权限层级 + +同时，根据权限适用范围的不同，我们将权限分为以下三个层级： + +1. GLOBAL LEVEL：全局权限。即通过 GRANT 语句授予的 `*.*` 上的权限。被授予的权限适用于任意数据库中的任意表。 +2. DATABASE LEVEL：数据库级权限。即通过 GRANT 语句授予的 `db.*` 上的权限。被授予的权限适用于指定数据库中的任意表。 +3. TABLE LEVEL：表级权限。即通过 GRANT 语句授予的 `db.tbl` 上的权限。被授予的权限适用于指定数据库中的指定表。 + + +## ADMIN/GRANT 权限说明 + +ADMIN\_PRIV 和 GRANT\_PRIV 权限同时拥有**授予权限**的权限，较为特殊。这里对和这两个权限相关的操作逐一说明。 + +1. CREATE USER + + * 拥有 ADMIN 权限，或任意层级的 GRANT 权限的用户可以创建新用户。 + +2. DROP USER + + * 只有 ADMIN 权限可以删除用户。 + +3. CREATE/DROP ROLE + + * 只有 ADMIN 权限可以创建角色。 + +4. GRANT/REVOKE + + * 拥有 ADMIN 权限，或者 GLOBAL 层级 GRANT 权限的用户，可以授予或撤销任意用户的权限。 + * 拥有 DATABASE 层级 GRANT 权限的用户，可以授予或撤销任意用户对指定数据库的权限。 + * 拥有 TABLE 层级 GRANT 权限的用户，可以授予或撤销任意用户对指定数据库中指定表的权限。 + +5. SET PASSWORD + + * 拥有 ADMIN 权限，或者 GLOBAL 层级 GRANT 权限的用户，可以设置任意用户的密码。 + * 普通用户可以设置自己对应的 UserIdentity 的密码。自己对应的 UserIdentity 可以通过 `SELECT CURRENT_USER();` 命令查看。 + * 拥有非 GLOBAL 层级 GRANT 权限的用户，不可以设置已存在用户的密码，仅能在创建用户时指定密码。 + + +## 一些说明 + +1. Doris 初始化时，会自动创建如下用户和角色： + + 1. operator 角色：该角色拥有 Node\_priv 和 Admin\_priv，即对Doris的所有权限。后续某个升级版本中，我们可能会将该角色的权限限制为 Node\_priv，即仅授予节点变更权限。以满足某些云上部署需求。 + + 2. admin 角色：该角色拥有 Admin\_priv，即除节点变更以外的所有权限。 + + 3. root@'%'：root 用户，允许从任意节点登陆，角色为 operator。 + + 4. admin@'%'：admin 用户，允许从任意节点登陆，角色为 admin。 + +2. 不支持删除或更改默认创建的角色或用户的权限。 + +3. operator 角色的用户有且只有一个。admin 角色的用户可以创建多个。 + +4. 一些可能产生冲突的操作说明 + + 1. 域名与ip冲突： + + 假设创建了如下用户： + + CREATE USER cmy@['domain']; + + 并且授权： + + GRANT SELECT_PRIV ON \*.\* TO cmy@['domain'] + + 该 domain 被解析为两个 ip：ip1 和 ip2 + + 假设之后，我们对 cmy@'ip1' 进行一次单独授权： + + GRANT ALTER_PRIV ON \*.\* TO cmy@'ip1'; + + 则 cmy@'ip1' 的权限会被修改为 SELECT\_PRIV, ALTER\_PRIV。并且当我们再次变更 cmy@['domain'] 的权限时，cmy@'ip1' 也不会跟随改变。 + + 2. 重复ip冲突： + + 假设创建了如下用户： + + CREATE USER cmy@'%' IDENTIFIED BY "12345"; + + CREATE USER cmy@'192.%' IDENTIFIED BY "abcde"; + + 在优先级上，'192.%' 优先于 '%'，因此，当用户 cmy 从 192.168.1.1 这台机器尝试使用密码 '12345' 登陆 Doris 会被拒绝。 + +5. 忘记密码 + + 如果忘记了密码无法登陆 Doris，可以在 Doris FE 节点所在机器，使用如下命令无密码登陆 Doris： + + `mysql-client -h 127.0.0.1 -P query_port -uroot` + + 登陆后，可以通过 SET PASSWORD 命令重置密码。 + +6. 任何用户都不能重置 root 用户的密码，除了 root 用户自己。 + +7. ADMIN\_PRIV 权限只能在 GLOBAL 层级授予或撤销。 + +8. 拥有 GLOBAL 层级 GRANT_PRIV 其实等同于拥有 ADMIN\_PRIV，因为该层级的 GRANT\_PRIV 有授予任意权限的权限，请谨慎使用。 + +9. `current_user()` 和 `user()` + + 用户可以通过 `SELECT current_user();` 和 `SELECT user();` 分别查看 `current_user` 和 `user`。其中 `current_user` 表示当前用户是以哪种身份通过认证系统的，而 `user` 则是用户当前实际的 `user_identity`。举例说明： + + 假设创建了 `user1@'192.%'` 这个用户，然后以为来自 192.168.10.1 的用户 user1 登陆了系统，则此时的 `current_user` 为 `user1@'192.%'`，而 `user` 为 `user1@'192.168.10.1'`。 + + 所有的权限都是赋予某一个 `current_user` 的，真实用户拥有对应的 `current_user` 的所有权限。 + +## 最佳实践 + +这里举例一些 Doris 权限系统的使用场景。 + +1. 场景一 + + Doris 集群的使用者分为管理员（Admin）、开发工程师（RD）和用户（Client）。其中管理员拥有整个集群的所有权限，主要负责集群的搭建、节点管理等。开发工程师负责业务建模，包括建库建表、数据的导入和修改等。用户访问不同的数据库和表来获取数据。 + + 在这种场景下，可以为管理员赋予 ADMIN 权限或 GRANT 权限。对 RD 赋予对任意或指定数据库表的 CREATE、DROP、ALTER、LOAD、SELECT 权限。对 Client 赋予对任意或指定数据库表 SELECT 权限。同时，也可以通过创建不同的角色，来简化对多个用户的授权操作。 + +2. 场景二 + + 一个集群内有多个业务，每个业务可能使用一个或多个数据。每个业务需要管理自己的用户。在这种场景下。管理员用户可以为每个数据库创建一个拥有 DATABASE 层级 GRANT 权限的用户。该用户仅可以对用户进行指定的数据库的授权。 + +3. 黑名单 + + Doris 本身不支持黑名单，只有白名单功能，但我们可以通过某些方式来模拟黑名单。假设先创建了名为 `user@'192.%'` 的用户，表示允许来自 `192.*` 的用户登录。此时如果想禁止来自 `192.168.10.1` 的用户登录。则可以再创建一个用户 `cmy@'192.168.10.1'` 的用户，并设置一个新的密码。因为 `192.168.10.1` 的优先级高于 `192.%`，所以来自 `192.168.10.1` 将不能再使用旧密码进行登录。 + + diff --git a/docs/documentation/cn/administrator-guide/segment-v2-usage.md b/docs/zh-CN/administrator-guide/segment-v2-usage.md similarity index 98% rename from docs/documentation/cn/administrator-guide/segment-v2-usage.md rename to docs/zh-CN/administrator-guide/segment-v2-usage.md index 09b6f0c56dd6d7..c3a15ccbd84ac4 100644 --- a/docs/documentation/cn/administrator-guide/segment-v2-usage.md +++ b/docs/zh-CN/administrator-guide/segment-v2-usage.md @@ -1,3 +1,10 @@ +--- +{ + "title": "Segment V2 升级手册", + "language": "zh-CN" +} +--- + + +# 文件管理器 + +Doris 中的一些功能需要使用一些用户自定义的文件。比如用于访问外部数据源的公钥、密钥文件、证书文件等等。文件管理器提供这样一个功能，能够让用户预先上传这些文件并保存在 Doris 系统中，然后可以在其他命令中引用或访问。 + +## 名词解释 + +* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 +* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 +* BDBJE：Oracle Berkeley DB Java Edition。FE 中用于持久化元数据的分布式嵌入式数据库。 +* SmallFileMgr：文件管理器。负责创建并维护用户的文件。 + +## 基本概念 + +文件是指用户创建并保存在 Doris 中的文件。 + +一个文件由 `数据库名称（database）`、`分类（catalog）` 和 `文件名（file_name）` 共同定位。同时每个文件也有一个全局唯一的 id（file_id），作为系统内的标识。 + +文件的创建和删除只能由拥有 `admin` 权限的用户进行操作。一个文件隶属于一个数据库。对某一数据库拥有访问权限（查询、导入、修改等等）的用户都可以使用该数据库下创建的文件。 + +## 具体操作 + +文件管理主要有三个命令：`CREATE FILE`，`SHOW FILE` 和 `DROP FILE`，分别为创建、查看和删除文件。这三个命令的具体语法可以通过连接到 Doris 后，执行 `HELP cmd;` 的方式查看帮助。 + +1. CREATE FILE + + 在创建文件的命令中，用户必须提供以下信息： + + * file_name：文件名。用户自定义，在一个 catalog 内唯一即可。 + * catalog：文件所属分类。用户自定义，在一个 database 内唯一即可。 + + > Doris 也有一些特殊的分类名称供特定的命令使用。 + + > 1. kafka + + > 当在例行导入命令中指定数据源为 Kafka，并且需要引用到文件时，Doris 会默认从 catalog 名为 "kafka" 的分类中查找文件。 + + * url：文件的下载地址。目前仅支持无认证的 http 下载地址。该下载地址仅用于在执行创建文件命令时，从这个地址下载文件。当文件成功创建并保存在 Doris 中后，该地址将不再被使用。 + * md5：可选项。文件的 MD5 值。如果用户提供该值，将在文件下载后进行 MD5 值的校验。校验失败则文件创建失败。 + + 文件创建成功后，文件相关的信息将持久化在 Doris 中。用户可以通过 `SHOW FILE` 命令查看已经创建成功的文件。 + +2. SHOW FILE + + 该命令可以查看已经创建成功的文件。具体操作见：`HELP SHOW FILE;` + +3. DROP FILE + + 该命令可以删除一个已经创建的文件。具体操作见：`HELP DROP FILE;` + +## 实现细节 + +### 创建和删除文件 + +当用户执行 `CREATE FILE` 命令后，FE 会从给定的 URL 下载文件。并将文件的内容以 Base64 编码的形式直接保存在 FE 的内存中。同时会将文件内容以及文件相关的元信息持久化在 BDBJE 中。所有被创建的文件，其元信息和文件内容都会常驻于 FE 的内存中。如果 FE 宕机重启，也会从 BDBJE 中加载元信息和文件内容到内存中。当文件被删除时，会直接从 FE 内存中删除相关信息，同时也从 BDBJE 中删除持久化的信息。 + +### 文件的使用 + +如果是 FE 端需要使用创建的文件，则 SmallFileMgr 会直接将 FE 内存中的数据保存为本地文件，存储在指定的目录中，并返回本地的文件路径供使用。 + +如果是 BE 端需要使用创建的文件，BE 会通过 FE 的 http 接口 `/api/get_small_file` 将文件内容下载到 BE 上指定的目录中，供使用。同时，BE 也会在内存中记录当前已经下载过的文件的信息。当 BE 请求一个文件时，会先查看本地文件是否存在并校验。如果校验通过，则直接返回本地文件路径。如果校验失败，则会删除本地文件，重新从 FE 下载。当 BE 重启时，会预先加载本地的文件到内存中。 + +## 使用限制 + +因为文件元信息和内容都存储于 FE 的内存中。所以默认仅支持上传大小在 1MB 以内的文件。并且总文件数量限制为 100 个。可以通过下一小节介绍的配置项进行修改。 + +## 相关配置 + +1. FE 配置 + + * `small_file_dir`：用于存放上传文件的路径，默认为 FE 运行目录的 `small_files/` 目录下。 + * `max_small_file_size_bytes`：单个文件大小限制，单位为字节。默认为 1MB。大于该配置的文件创建将会被拒绝。 + * `max_small_file_number`：一个 Doris 集群支持的总文件数量。默认为 100。当创建的文件数超过这个值后，后续的创建将会被拒绝。 + + > 如果需要上传更多文件或提高单个文件的大小限制，可以通过 `ADMIN SET CONFIG` 命令修改 `max_small_file_size_bytes` 和 `max_small_file_number` 参数。但文件数量和大小的增加，会导致 FE 内存使用量的增加。 + +2. BE 配置 + + * `small_file_dir`：用于存放从 FE 下载的文件的路径，默认为 BE 运行目录的 `lib/small_files/` 目录下。 diff --git a/docs/zh-CN/administrator-guide/sql-mode.md b/docs/zh-CN/administrator-guide/sql-mode.md new file mode 100644 index 00000000000000..7021677f8995d6 --- /dev/null +++ b/docs/zh-CN/administrator-guide/sql-mode.md @@ -0,0 +1,76 @@ +--- +{ + "title": "SQL MODE", + "language": "zh-CN" +} +--- + + + +# SQL MODE + +Doris新支持的sql mode参照了 Mysql 的sql mode管理机制，每个客户端都能设置自己的sql mode，拥有Admin权限的数据库管理员可以设置全局sql mode。 + +## sql mode 介绍 + +sql mode使用户能在不同风格的sql语法和数据校验严格度间做切换，使Doris对其他数据库有更好的兼容性。例如在一些数据库里，'||'符号是一个字符串连接符，但在Doris里却是与'or'等价的，这时用户只需要使用sql mode切换到自己想要的风格。每个客户端都能设置sql mode，并在当前对话中有效，只有拥有Admin权限的用户可以设置全局sql mode。 + +## 原理 + +sql mode用一个64位的Long型存储在SessionVariables中，这个地址的每一位都代表一个mode的开启/禁用(1表示开启，0表示禁用)状态，只要知道每一种mode具体是在哪一位，我们就可以通过位运算方便快速的对sql mode进行校验和操作。 + +每一次对sql mode的查询，都会对此Long型进行一次解析，变成用户可读的字符串形式，同理，用户发送给服务器的sql mode字符串，会被解析成能够存储在SessionVariables中的Long型。 + +已被设置好的全局sql mode会被持久化，因此对全局sql mode的操作总是只需一次，即使程序重启后仍可以恢复上一次的全局sql mode。 + +## 操作方式 + +1、设置sql mode + +``` +set global sql_mode = "DEFAULT" +set session sql_mode = "DEFAULT" +``` +>目前Doris的默认sql mode是DEFAULT（但马上会在后续修改中会改变）。 +>设置global sql mode需要Admin权限，并会影响所有在此后连接的客户端。 +>设置session sql mode只会影响当前对话客户端，默认为session方式。 + +2、查询sql mode + +``` +select @@global.sql_mode +select @@session.sql_mode +``` +>除了这种方式，你还可以通过下面方式返回所有session variables来查看当前sql mode + +``` +show global variables +show session variables +``` + +## 已支持mode + +1. `PIPES_AS_CONCAT` + + 在此模式下，'||'符号是一种字符串连接符号（同CONCAT()函数），而不是'OR'符号的同义词。(e.g., `'a'||'b' = 'ab'`, `1||0 = '10'`) + +## 复合mode + +（后续补充） \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/time-zone.md b/docs/zh-CN/administrator-guide/time-zone.md new file mode 100644 index 00000000000000..20e784f4d55fa7 --- /dev/null +++ b/docs/zh-CN/administrator-guide/time-zone.md @@ -0,0 +1,91 @@ +--- +{ + "title": "时区", + "language": "zh-CN" +} +--- + + + +# 时区 + +Doris 支持多时区设置 + +## 名词解释 + +* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 +* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 + +## 基本概念 + +Doris 内部存在多个时区相关参数 + +* system_time_zone : + 当服务器启动时，会根据机器设置时区自动设置，设置后不可修改。 + +* time_zone : + 服务器当前时区，区分session级别和global级别 + +## 具体操作 + +1. show variables like '%time_zone%' + + 查看当前时区相关配置 + +2. SET time_zone = 'Asia/Shanghai' + + 该命令可以设置session级别的时区，连接断开后失效 + +3. SET global time_zone = 'Asia/Shanghai' + + 该命令可以设置global级别的时区参数，fe会将参数持久化，连接断开后不失效 + +### 时区的影响 + +时区设置会影响对时区敏感的时间值的显示和存储。 + +包括NOW()或CURTIME()等时间函数显示的值，也包括show load, show backends中的时间值。 + +但不会影响 create table 中时间类型分区列的 less than 值，也不会影响存储为 date/datetime 类型的值的显示。 + +受时区影响的函数： + +* `FROM_UNIXTIME`：给定一个 UTC 时间戳，返回指定时区的日期时间：如 `FROM_UNIXTIME(0)`， 返回 CST 时区：`1970-01-01 08:00:00`。 +* `UNIX_TIMESTAMP`：给定一个指定时区日期时间，返回 UTC 时间戳：如 CST 时区 `UNIX_TIMESTAMP('1970-01-01 08:00:00')`，返回 `0`。 +* `CURTIME`：返回指定时区时间。 +* `NOW`：返指定地时区日期时间。 +* `CONVERT_TZ`：将一个日期时间从一个指定时区转换到另一个指定时区。 + +## 使用限制 + +时区值可以使用几种格式给出，不区分大小写: + +* 表示UTC偏移量的字符串，如'+10:00'或'-6:00' + +* 标准时区格式，如"Asia/Shanghai"、"America/Los_Angeles" + +* 不支持缩写时区格式，如"MET"、"CTT"。因为缩写时区在不同场景下存在歧义，不建议使用。 + +* 为了兼容Doris，支持CST缩写时区，内部会将CST转移为"Asia/Shanghai"的中国标准时区 + +## 时区格式列表 + +[List of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + diff --git a/docs/zh-CN/administrator-guide/variables.md b/docs/zh-CN/administrator-guide/variables.md new file mode 100644 index 00000000000000..d728a38ed542c9 --- /dev/null +++ b/docs/zh-CN/administrator-guide/variables.md @@ -0,0 +1,328 @@ +--- +{ + "title": "变量", + "language": "zh-CN" +} +--- + + + +# 变量 + +本文档主要介绍当前支持的变量（variables）。 + +Doris 中的变量参考 MySQL 中的变量设置。但部分变量仅用于兼容一些 MySQL 客户端协议，并不产生其在 MySQL 数据库中的实际意义。 + +## 变量设置与查看 + +### 查看 + +可以通过 `SHOW VARIABLES [LIKE 'xxx'];` 查看所有或指定的变量。如： + +``` +SHOW VARIABLES; +SHOW VARIABLES LIKE '%time_zone%'; +``` + +### 设置 + +部分变量可以设置全局生效或仅当前会话生效。设置全局生效后，后续新的会话连接中会沿用设置值。而设置仅当前会话生效，则变量仅对当前会话产生作用。 + +仅当前会话生效，通过 `SET var_name=xxx;` 语句来设置。如： + +``` +SET exec_mem_limit = 137438953472; +SET forward_to_master = true; +SET time_zone = "Asia/Shanghai"; +``` + +全局生效，通过 `SET GLOBALE var_name=xxx;` 设置。如： + +``` +SET GLOBAL exec_mem_limit = 137438953472 +``` + +> 注1：只有 ADMIN 用户可以设置变量的全局生效。 +> 注2：全局生效的变量不影响当前会话的变量值，仅影响新的会话中的变量。 + +支持全局生效的变量包括： + +* `time_zone` +* `wait_timeout` +* `sql_mode` +* `is_report_success` +* `query_timeout` +* `exec_mem_limit` +* `batch_size` +* `parallel_fragment_exec_instance_num` +* `parallel_exchange_instance_num` + +同时，变量设置也支持常量表达式。如： + +``` +SET exec_mem_limit = 10 * 1024 * 1024 * 1024; +SET forward_to_master = concat('tr', 'u', 'e'); +``` + +## 支持的变量 + +* `SQL_AUTO_IS_NULL` + + 用于兼容 JDBC 连接池 C3P0。 无实际作用。 + +* `auto_increment_increment` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `autocommit` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `batch_size` + + 用于指定在查询执行过程中，各个节点传输的单个数据包的行数。默认一个数据包的行数为 1024 行，即源端节点每产生 1024 行数据后，打包发给目的节点。 + + 较大的行数，会在扫描大数据量场景下提升查询的吞吐，但可能会在小查询场景下增加查询延迟。同时，也会增加查询的内存开销。建议设置范围 1024 至 4096。 + +* `character_set_client` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `character_set_connection` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `character_set_results` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `character_set_server` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `codegen_level` + + 用于设置 LLVM codegen 的等级。（当前未生效）。 + +* `collation_connection` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `collation_database` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `collation_server` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `disable_colocate_join` + + 控制是否启用 [Colocation Join](./colocation-join.md) 功能。默认为 false，表示启用该功能。true 表示禁用该功能。当该功能被禁用后，查询规划将不会尝试执行 Colocation Join。 + +* `disable_streaming_preaggregations` + + 控制是否开启流式预聚合。默认为 false，即开启。当前不可设置，且默认开启。 + +* `enable_insert_strict` + + 用于设置通过 INSERT 语句进行数据导入时，是否开启 `strict` 模式。默认为 false，即不开启 `strict` 模式。关于该模式的介绍，可以参阅 [这里](./load-data/insert-into-manual.md)。 + +* `enable_spilling` + + 用于设置是否开启大数据量落盘排序。默认为 false，即关闭该功能。当用户未指定 ORDER BY 子句的 LIMIT 条件，同时设置 `enable_spilling` 为 true 时，才会开启落盘排序。该功能启用后，会使用 BE 数据目录下 `doris-scratch/` 目录存放临时的落盘数据，并在查询结束后，清空临时数据。 + + 该功能主要用于使用有限的内存进行大数据量的排序操作。 + + 注意，该功能为实验性质，不保证稳定性，请谨慎开启。 + +* `exec_mem_limit` + + 用于设置单个查询的内存限制。默认为 2GB，单位为字节。 + + 该参数用于限制一个查询计划中，单个查询计划的实例所能使用的内存。一个查询计划可能有多个实例，一个 BE 节点可能执行一个或多个实例。所以该参数并不能准确限制一个查询在整个集群的内存使用，也不能准确限制一个查询在单一 BE 节点上的内存使用。具体需要根据生成的查询计划判断。 + + 通常只有在一些阻塞节点（如排序节点、聚合节点、Join 节点）上才会消耗较多的内存，而其他节点（如扫描节点）中，数据为流式通过，并不会占用较多的内存。 + + 当出现 `Memory Exceed Limit` 错误时，可以尝试指数级增加该参数，如 4G、8G、16G 等。 + +* `forward_to_master` + + 用户设置是否将一些命令转发到 Master FE 节点执行。默认为 false，即不转发。Doris 中存在多个 FE 节点，其中一个为 Master 节点。通常用户可以连接任意 FE 节点进行全功能操作。但部分信息查看指令，只有从 Master FE 节点才能获取详细信息。 + + 如 `SHOW BACKENDS;` 命令，如果不转发到 Master FE 节点，则仅能看到节点是否存活等一些基本信息，而转发到 Master FE 则可以获取包括节点启动时间、最后一次心跳时间等更详细的信息。 + + 当前受该参数影响的命令如下： + + 1. `SHOW FRONTENDS;` + + 转发到 Master 可以查看最后一次心跳信息。 + + 2. `SHOW BACKENDS;` + + 转发到 Master 可以查看启动时间、最后一次心跳信息、磁盘容量信息。 + + 3. `SHOW BROKER;` + + 转发到 Master 可以查看启动时间、最后一次心跳信息。 + + 4. `SHOW TABLET;`/`ADMIN SHOW REPLICA DISTRIBUTION;`/`ADMIN SHOW REPLICA STATUS;` + + 转发到 Master 可以查看 Master FE 元数据中存储的 tablet 信息。正常情况下，不同 FE 元数据中 tablet 信息应该是一致的。当出现问题时，可以通过这个方法比较当前 FE 和 Master FE 元数据的差异。 + + 5. `SHOW PROC;` + + 转发到 Master 可以查看 Master FE 元数据中存储的相关 PROC 的信息。主要用于元数据比对。 + +* `init_connect` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `interactive_timeout` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `is_report_success` + + 用于设置是否需要查看查询的 profile。默认为 false，即不需要 profile。 + + 默认情况下，只有在查询发生错误时，BE 才会发送 profile 给 FE，用于查看错误。正常结束的查询不会发送 profile。发送 profile 会产生一定的网络开销，对高并发查询场景不利。 + 当用户希望对一个查询的 profile 进行分析时，可以将这个变量设为 true 后，发送查询。查询结束后，可以通过在当前连接的 FE 的 web 页面查看到 profile： + + `fe_host:fe_http_port/query` + + 其中会显示最近100条，开启 `is_report_success` 的查询的 profile。 + +* `language` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `license` + + 显示 Doris 的 License。无其他作用。 + +* `load_mem_limit` + + 用于指定导入操作的内存限制。默认为 0，即表示不使用该变量，而采用 `exec_mem_limit` 作为导入操作的内存限制。 + + 这个变量仅用于 INSERT 操作。因为 INSERT 操作设计查询和导入两个部分，如果用户不设置此变量，则查询和导入操作各自的内存限制均为 `exec_mem_limit`。否则，INSERT 的查询部分内存限制为 `exec_mem_limit`，而导入部分限制为 `load_mem_limit`。 + + 其他导入方式，如 BROKER LOAD，STREAM LOAD 的内存限制依然使用 `exec_mem_limit`。 + +* `lower_case_table_names` + + 用于兼容 MySQL 客户端。不可设置。当前 Doris 中的表名默认为大小写敏感。 + +* `max_allowed_packet` + + 用于兼容 JDBC 连接池 C3P0。 无实际作用。 + +* `net_buffer_length` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `net_read_timeout` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `net_write_timeout` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `parallel_exchange_instance_num` + + 用于设置执行计划中，一个上层节点接收下层节点数据所使用的 exchange node 数量。默认为 -1，即表示 exchange node 数量等于下层节点执行实例的个数（默认行为）。当设置大于0，并且小于下层节点执行实例的个数，则 exchange node 数量等于设置值。 + + 在一个分布式的查询执行计划中，上层节点通常有一个或多个 exchange node 用于接收来自下层节点在不同 BE 上的执行实例的数据。通常 exchange node 数量等于下层节点执行实例数量。 + + 在一些聚合查询场景下，如果底层需要扫描的数据量较大，但聚合之后的数据量很小，则可以尝试修改此变量为一个较小的值，可以降低此类查询的资源开销。如在 DUPLICATE KEY 明细模型上进行聚合查询的场景。 + +* `parallel_fragment_exec_instance_num` + + 针对扫描节点，设置其在每个 BE 节点上，执行实例的个数。默认为 1。 + + 一个查询计划通常会产生一组 scan range，即需要扫描的数据范围。这些数据分布在多个 BE 节点上。一个 BE 节点会有一个或多个 scan range。默认情况下，每个 BE 节点的一组 scan range 只由一个执行实例处理。当机器资源比较充裕时，可以将增加该变量，让更多的执行实例同时处理一组 scan range，从而提升查询效率。 + + 而 scan 实例的数量决定了上层其他执行节点，如聚合节点，join 节点的数量。因此相当于增加了整个查询计划执行的并发度。修改该参数会对大查询效率提升有帮助，但较大数值会消耗更多的机器资源，如CPU、内存、磁盘IO。 + +* `query_cache_size` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `query_cache_type` + + 用于兼容 JDBC 连接池 C3P0。 无实际作用。 + +* `query_timeout` + + 用于设置查询超时。该变量会作用于当前连接中所有的查询语句，以及 INSERT 语句。默认为 5 分钟，单位为秒。 + +* `resource_group` + + 暂不使用。 + +* `sql_mode` + + 用于指定 SQL 模式，以适应某些 SQL 方言。关于 SQL 模式，可参阅 [这里](./sql-mode.md)。 + +* `sql_safe_updates` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `sql_select_limit` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `system_time_zone` + + 显示当前系统时区。不可更改。 + +* `time_zone` + + 用于设置当前会话的时区。时区会对某些时间函数的结果产生影响。关于时区，可以参阅 [这里](./time-zone.md)。 + +* `tx_isolation` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `version` + + 用于兼容 MySQL 客户端。无实际作用。 + +* `version_comment` + + 用于显示 Doris 的版本。不可更改。 + +* `wait_timeout` + + 用于设置空闲连接的连接时长。当一个空闲连接在该时长内与 Doris 没有任何交互，则 Doris 会主动断开这个链接。默认为 8 小时，单位为秒。 + +* `default_rowset_type` + + 用于设置计算节点存储引擎默认的存储格式。当前支持的存储格式包括：alpha/beta。 + +* `use_v2_rollup` + + 用于控制查询使用segment v2存储格式的rollup索引获取数据。该变量用于上线segment v2的时候，进行验证使用；其他情况，不建议使用。 + +* `rewrite_count_distinct_to_bitmap_hll` + + 是否将 bitmap 和 hll 类型的 count distinct 查询重写为 bitmap_union_count 和 hll_union_agg 。 + diff --git a/docs/zh-CN/community/gitter.md b/docs/zh-CN/community/gitter.md new file mode 100644 index 00000000000000..5a75982db53f56 --- /dev/null +++ b/docs/zh-CN/community/gitter.md @@ -0,0 +1,63 @@ +--- +{ + "title": "Gitter 使用指南", + "language": "zh-CN" +} +--- + + + +# Gitter 使用指南 + +## Gitter 简介 + +Gitter 是一款可支持 Markdown 的针对开发者的即时通讯软件，可无缝关联到 github，可在聊天中关联Github上的PR，可留存讨论的相关历史记录，可查询历史记录，支持中英文。 + +Doris 和很多其他的开源项目一样，可以使用 Gitter 作为技术交流和社区发展的即时通讯媒介，本文介绍如何使用 Gitter 参与到 Doris 的开源开发和社区发展中。 + +## 使用链接登录 + +在浏览器 输入 [https://gitter.im/apache-doris/Lobby](https://gitter.im/apache-doris/Lobby) ，会自动跳转到 Gitter 上 Doris 社区的聊天室界面。 + +点击下方的 `SIGN IN TO START TALKING` 进行登录，可支持两种登录方式，Github 账号或 Twitter 账号，笔者使用 Github 账号进行登录，如下： + +![](/images/login-gitter1.png) + +点击红圈处后，输入 Github 账号和密码既可登录进聊天室，从此开始进行技术或社区讨论： + +![](/images/login-gitter2.png) + +可以像使用微信一样，使用Gitter，并获得比微信更加令开发者和技术人员舒服的功能，如，直接提及某个Activity进行讨论，直接搜索历史聊天记录等。 + +已经加入聊天室的伙伴别忘记点击右上角的五角星进行收藏，这样会让本聊天室更容易被您自己找到。 + +更多 gitter 使用技巧，可以参考： + +[http://www.gitter.net.cn/book/gitter/roomsettings-1.html](http://www.gitter.net.cn/book/gitter/roomsettings-1.html) + +## 安装手机客户端 + +您可以下载 Gitter 的手机客户端，在手机上也随时随地的参与到技术讨论中，下载链接： + +[https://gitter.im/home](https://gitter.im/home) + +## 在 Gitter 上搜索并加入 Doris 社区聊天室 + +已经在使用 Gitter 的伙伴，直接登录 Gitter 后，搜索 `apache-doris` ,找到后可以加入聊天室，其他的功能使用同上一章节，这里不赘述。 diff --git a/docs/zh-CN/community/how-to-contribute.md b/docs/zh-CN/community/how-to-contribute.md new file mode 100644 index 00000000000000..a06bf2755ea10b --- /dev/null +++ b/docs/zh-CN/community/how-to-contribute.md @@ -0,0 +1,82 @@ +--- +{ + "title": "为 Doris 做贡献", + "language": "zh-CN" +} +--- + + + +# 为 Doris 做贡献 + +非常感谢您对 Doris 项目感兴趣，我们非常欢迎您对 Doris 项目的各种建议、意见（包括批评）、评论和贡献。 + +您对 Doris 的各种建议、意见、评论可以直接通过 GitHub 的 [Issues](https://github.com/apache/incubator-doris/issues/new/choose) 提出。 + +参与 Doris 项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表，进一步，有了足够的贡献后，您还可以有机会成为 Aapche 的 Commiter，拥有 Apache 邮箱，并被收录到 [Apache Commiter 列表中](http://people.apache.org/committer-index.html)。 + +任何问题，您都可以联系我们得到及时解答，联系方式包括微信、Gitter（GitHub提供的即时聊天工具）、邮件等等。 + +## 初次接触 + +初次来到 Doris 社区，您可以： + +* 关注 Doris [Github 代码库](https://github.com/apache/incubator-doris) +* 订阅我们的 [邮件列表](./subscribe-mail-list.md)； +* 加入 Doris 微信群(加微信号：morningman-cmy, 备注：加入Doris群) 随时提问； +* 进入 Doris 的 [Gitter](./gitter.md) 聊天室； + +通过以上方式及时了解 Doris 项目的开发动态并为您关注的话题发表意见。 + +## Doris 的代码和文档 + +正如您在 [GitHub] (https://github.com/apache/incubator-doris) 上看到的，Apache Doris (incubating) 的代码库主要包括三部分：Frontend (FE), Backend (BE) 和 Broker (为了支持 HDFS 等外部存储系统上的文件读取)。文档主要是 Doris 网站和 GitHub 上的 wiki，还有运行 Doris 的时候的在线帮助手册。这些组件的详细情况参见下表： + +| 组件名称 | 组件描述 | 相关语言 | +|--------|----------------------------|----------| +| [Frontend daemon (FE)](https://github.com/apache/incubator-doris)| 由“查询协调器”和“元数据管理器”组成 | Java| +| [Backend daemon (BE)](https://github.com/apache/incubator-doris) | 负责存储数据和执行查询片段 | C++| +| [Broker](https://github.com/apache/incubator-doris) | 读取 HDFS 数据到 Doris | Java | +| [Website](https://github.com/apache/incubator-doris-website) | Doris 网站 | Markdown | +| [GitHub Wiki](https://github.com/apache/incubator-doris/wiki) | Doris GitHub Wiki | Markdown | +| Doris 运行时 help 文档 | 运行 Doris 的时候的在线帮助手册 | Markdown | + +## 改进文档 + +文档是您了解 Apache Doris 的最主要的方式，也是我们最需要帮助的地方！ + +浏览文档，可以加深您对 Doris 的了解，也可以帮助您理解 Doris 的功能和技术细节，如果您发现文档有问题，请及时联系我们； + +如果您对改进文档的质量感兴趣，不论是修订一个页面的地址、更正一个链接、以及写一篇更优秀的入门文档，我们都非常欢迎！ + +我们的文档大多数是使用 markdown 格式编写的，您可以直接通过在 [GitHub] (https://github.com/apache/incubator-doris) 中的 `docs/` 中修改并提交文档变更。如果提交代码变更，可以参阅 [Pull Request](./pull-request.md)。 + +## 如果发现了一个 Bug 或问题 + +如果发现了一个 Bug 或问题，您可以直接通过 GitHub 的 [Issues](https://github.com/apache/incubator-doris/issues/new/choose) 提一个新的 Issue，我们会有人定期处理。 + +您也可以通过阅读分析代码自己修复（当然在这之前最好能和我们交流下，或许已经有人在修复同样的问题了），然后提交一个 [Pull Request](./pull-request.md)。 + +## 修改代码和提交PR（Pull Request） + +您可以下载代码，编译安装，部署运行试一试（可以参考[编译文档](../installing/compilation.md)），看看是否与您预想的一样工作。如果有问题，您可以直接联系我们，提 Issue 或者通过阅读和分析源代码自己修复。 + +无论是修复 Bug 还是增加 Feature，我们都非常欢迎。如果您希望给 Doris 提交代码，您需要从 GitHub 上 fork 代码库至您的项目空间下，为您提交的代码创建一个新的分支，添加源项目为upstream，并提交PR。 +提交PR的方式可以参考文档 [Pull Request](./pull-request.md)。 diff --git a/docs/zh-CN/community/members.md b/docs/zh-CN/community/members.md new file mode 100644 index 00000000000000..22a3772a77e4a0 --- /dev/null +++ b/docs/zh-CN/community/members.md @@ -0,0 +1,68 @@ +--- +{ + "title": "PMC Members & Committer", + "language": "zh-CN" +} +--- + + + +# PMC Members & Committer + +## Mentors (3) + +|Apache ID|Github Username |Public Name| +|--------|-----------|----------| +|wave |dave2wave |Dave Fisher | +|shaofengshi |shaofengshi| Shao Feng Shi | +|ningjiang |WillemJiang |Willem Ning Jiang| + +## PPMC (9) +(the listing below excludes mentors) + +|Apache ID|Github Username |Public Name| +|--------|-----------|----------| +|lingbin| lingbin |Bin Ling | +|lichaoyong |chaoyli |Chaoyong Li | +|zhaoc |imay |Chun Zhao | +|lide |lide-reed, doris-ci |De Li | +|chenhao |chenhao7253886 |Hao Chen | +|morningman |morningman |Mingyu Chen| +|maruyue |maruyue| Ruyue Ma | +|sijie |sijie |Sijie Guo | +|zshao |zshao |Zheng Shao| + +## Committers (13) + +|Apache ID|Github Username |Public Name| +|--------|-----------|----------| +|lingbin| lingbin |Bin Ling | +|lichaoyong |chaoyli |Chaoyong Li | +|zhaoc |imay |Chun Zhao | +|lide |lide-reed, doris-ci |De Li | +|chenhao |chenhao7253886 |Hao Chen | +|morningman |morningman |Mingyu Chen| +|maruyue |maruyue| Ruyue Ma | +|sijie |sijie |Sijie Guo | +|zshao |zshao |Zheng Shao| +|kangkaisen|kangkaisen|Kaisen Kang| +|lingmiao|EmmyMiao87|Ling Miao| +|gaodayue|gaodayue|Dayue Gao| +|liuhangyuan|HangyuanLiu|Hangyuan Liu| diff --git a/docs/zh-CN/community/pull-request.md b/docs/zh-CN/community/pull-request.md new file mode 100644 index 00000000000000..c1022afbfcd85d --- /dev/null +++ b/docs/zh-CN/community/pull-request.md @@ -0,0 +1,259 @@ +--- +{ + "title": "代码提交指南", + "language": "zh-CN" +} +--- + + + +# 代码提交指南 + +在 [Github](https://github.com/apache/incubator-doris) 上面可以很方便地提交 [Pull Request (PR)](https://help.github.com/articles/about-pull-requests/)，下面介绍 Doris 项目的 PR 方法。 + +### 1. Fork仓库 + +进入 apache/incubator-doris 的 [github 页面](https://github.com/apache/incubator-doris) ，点击右上角按钮 `Fork` 进行 Fork。 + +![Fork](/images/fork-repo.png) + +### 2. 配置git和提交修改 + +#### （1）将代码克隆到本地： + +``` +git clone https://github.com//incubator-doris.git +``` + +注意：请将 \ 替换为您的 github 名字。 + +clone 完成后，origin 会默认指向 github 上的远程 fork 地址。 + +#### （2）将 apache/incubator-doris 添加为本地仓库的远程分支 upstream： + +``` +cd incubator-doris +git remote add upstream https://github.com/apache/incubator-doris.git +``` + +#### （3）检查远程仓库设置： + +``` +git remote -v +origin https://github.com//incubator-doris.git (fetch) +origin https://github.com//incubator-doris.git (push) +upstream https://github.com/apache/incubator-doris.git (fetch) +upstream https://github.com/apache/incubator-doris.git (push) +``` + +#### （4）新建分支以便在分支上做修改： + +``` +git checkout -b +``` + +注意： \ 为您自定义的分支名字。 + +创建完成后可进行代码更改。 + +#### （5）提交代码到远程分支： + +``` +git commit -a -m "" +git push origin +``` + +更多 git 使用方法请访问：[git 使用](https://www.atlassian.com/git/tutorials/setting-up-a-repository)，这里不赘述。 + +### 3. 创建PR + +#### （1）新建 PR +在浏览器切换到自己的 github 页面，切换分支到提交的分支 \ ，点击 `New pull request` 按钮进行创建，如下图所示： + +![new PR](/images/new-pr.png) + +#### （2）准备分支 +这时候，会出现 `Create pull request` 按钮，如果没有请检查是否正确选择了分支，也可以点击 “compare across forks” 重新选择 repo 和分支。 + +![create PR](/images//create-pr.png) + +#### （3）填写 Commit Message +这里请填写 comment 的总结和详细内容，然后点击 `Create pull request` 进行创建。 + +关于如何写 Commit Message，下面列出了一些 Tips： + +* 请用英文 动词 + 宾语 的形式，动词不用过去式，语句用祈使句； +* 消息主题（Subject）和具体内容（Body）都要写，它们之间要有空行分隔（GitHub PR界面上分别填写即可）; +* 消息主题长度不要超过**50**个字符； +* 消息内容每行不要超过**72**个字符，超过的需要手动换行； +* 消息内容用于解释做了什么、为什么做以及怎么做的； +* 消息主题第一个字母要**大写**，句尾**不要**有句号； +* 消息内容中写明关联的issue(如果有)，例如 #233; + +更详细的内容请参考 + +![create PR](/images/create-pr2.png) + +#### （4）完成创建 +创建成功后，您可以看到 Doris 项目需要 review，您可以等待我们 review 和合入，您也可以直接联系我们。 + +![create PR](/images/create-pr3.png) + +至此，您的PR创建完成，更多关于 PR 请阅读 [collaborating-with-issues-and-pull-requests](https://help.github.com/categories/collaborating-with-issues-and-pull-requests/) 。 + +### 4. 冲突解决 + +提交PR时的代码冲突一般是由于多人编辑同一个文件引起的，解决冲突主要通过以下步骤即可： + +#### （1）切换至主分支 + +``` +git checkout master +``` + +#### （2）同步远端主分支至本地 + +``` +git pull upstream master +``` + +#### （3）切换回刚才的分支（假设分支名为fix） + +``` +git checkout fix +``` + +#### （4）进行rebase + +``` +git rebase -i master +``` + +此时会弹出修改记录的文件，一般直接保存即可。然后会提示哪些文件出现了冲突，此时可打开冲突文件对冲突部分进行修改，将提示的所有冲突文件的冲突都解决后，执行 + +``` +git add . +git rebase --continue +``` + +依此往复，直至屏幕出现类似 *rebase successful* 字样即可，此时您可以进行往提交PR的分支进行更新： + +``` +git push -f origin fix +``` + +### 5. 一个例子 + +#### （1）对于已经配置好 upstream 的本地分支 fetch 到最新代码 + +``` +$ git branch +* master + +$ git fetch upstream +remote: Counting objects: 195, done. +remote: Compressing objects: 100% (68/68), done. +remote: Total 141 (delta 75), reused 108 (delta 48) +Receiving objects: 100% (141/141), 58.28 KiB, done. +Resolving deltas: 100% (75/75), completed with 43 local objects. +From https://github.com/apache/incubator-doris + 9c36200..0c4edc2 master -> upstream/master +``` + +#### （2）进行rebase + +``` +$ git rebase upstream/master +First, rewinding head to replay your work on top of it... +Fast-forwarded master to upstream/master. +``` + +#### （3）检查看是否有别人提交未同步到自己 repo 的提交 + +``` +$ git status +# On branch master +# Your branch is ahead of 'origin/master' by 8 commits. +# +# Untracked files: +# (use "git add ..." to include in what will be committed) +# +# custom_env.sh +nothing added to commit but untracked files present (use "git add" to track) +``` + +#### （4）合并其他人提交的代码到自己的 repo + +``` +$ git push origin master +Counting objects: 195, done. +Delta compression using up to 32 threads. +Compressing objects: 100% (41/41), done. +Writing objects: 100% (141/141), 56.66 KiB, done. +Total 141 (delta 76), reused 140 (delta 75) +remote: Resolving deltas: 100% (76/76), completed with 44 local objects. +To https://lide-reed:fc35ff925bd8fd6629be3f6412bacee99d4e5f97@github.com/lide-reed/incubator-doris.git + 9c36200..0c4edc2 master -> master +``` + +#### （5）新建分支，准备开发 + +``` +$ git checkout -b my_branch +Switched to a new branch 'my_branch' + +$ git branch + master +* my_branch +``` + +#### （6）代码修改完成后，准备提交 + +``` +$ git add -u +``` + +#### （7）填写 message 并提交到本地的新建分支上 + +``` +$ git commit -m "Fix a typo" +[my_branch 55e0ba2] Fix a typo + 1 files changed, 2 insertions(+), 2 deletions(-) +``` + +#### （8）将分支推到 GitHub 远端自己的 repo 中 + +``` +$ git push origin my_branch +Counting objects: 11, done. +Delta compression using up to 32 threads. +Compressing objects: 100% (6/6), done. +Writing objects: 100% (6/6), 534 bytes, done. +Total 6 (delta 4), reused 0 (delta 0) +remote: Resolving deltas: 100% (4/4), completed with 4 local objects. +remote: +remote: Create a pull request for 'my_branch' on GitHub by visiting: +remote: https://github.com/lide-reed/incubator-doris/pull/new/my_branch +remote: +To https://lide-reed:fc35ff925bd8fd6629be3f6412bacee99d4e5f97@github.com/lide-reed/incubator-doris.git + * [new branch] my_branch -> my_branch +``` + +至此，就可以按照前面的流程进行创建 PR 了。 \ No newline at end of file diff --git a/docs/zh-CN/community/release-process.md b/docs/zh-CN/community/release-process.md new file mode 100644 index 00000000000000..b852374c595ffe --- /dev/null +++ b/docs/zh-CN/community/release-process.md @@ -0,0 +1,655 @@ +--- +{ + "title": "Apache Doris 发布流程", + "language": "zh-CN" +} +--- + + + +# Apache Doris 发布流程 + +Apache 的发布必须至少是 IPMC 成员，拥有 apache 邮箱的commiter，这个角色叫做 release manager。 + +发布的大致流程如下： + +1. 环境准备 +2. 发布准备 + 1. 社区发起 DISCUSS 并与社区交流具体发布计划 + 2. 创建分支用于发布 + 3. 清理 issue + 4. 将必要的 Patch 合并到发布的分支 +3. 社区发布投票流程 + 1. 将 tag 打包，签名并上传到[Apache Dev svn 仓库](https://dist.apache.org/repos/dist/dev/incubator/doris) + 2. 在 [Doris 社区](dev@doris.apache.org)发起投票 + 3. 投票通过后，在Doris社区发 Result 邮件 + 4. 在 [Incubator 社区](general@incubator.apache.org) 发起新一轮投票 + 5. 发 Result 邮件到 general@incubator.apache.org +4. 完成工作 + 1. 上传签名的软件包到 [Apache release repo](https://dist.apache.org/repos/dist/release/incubator/doris)，并生成相关链接 + 2. 准备 release note 并发 Announce 邮件到 general@incubator.apache.org + 3. 在 Doris 官网和 github 发布下载链接 + +## 准备环境 + +如果这是你第一次发布，那么你需要在你的环境中准备如下工具 + +1. release signing https://www.apache.org/dev/release-signing.html +2. gpg https://www.apache.org/dev/openpgp.html +3. svn https://www.apache.org/dev/openpgp.html + +### 准备gpg key + +Release manager 在发布前需要先生成自己的签名公钥，并上传到公钥服务器，之后就可以用这个公钥对准备发布的软件包进行签名。 +如果在[KEY](https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS)里已经存在了你的KEY，那么你可以跳过这个步骤了。 + +#### 签名软件 GnuPG 的安装配置 +##### GnuPG + +1991年，程序员 Phil Zimmermann 为了避开政府监视，开发了加密软件PGP。这个软件非常好用，迅速流传开来，成了许多程序员的必备工具。但是，它是商业软件，不能自由使用。所以，自由软件基金会决定，开发一个PGP的替代品，取名为GnuPG。这就是GPG的由来。 + +##### 安装配置 + +CentOS 安装命令： + +``` +yum install gnupg +``` +安装完成后，默认配置文件 gpg.conf 会放在 home 目录下。 + +``` +~/.gnupg/gpg.conf +``` + +如果不存在这个目录或文件，可以直接创建一个空文件。 +编辑gpg.conf, 修改或者增加 keyserver 配置： + +``` +keyserver hkp://keys.gnupg.net +``` + +Apache 签名推荐 SHA512， 可以通过配置 gpg 完成。 +编辑gpg.conf, 增加下面的三行： + +``` +personal-digest-preferences SHA512 +cert-digest-algo SHA512 +default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed +``` +#### 生成新的签名 + +##### 准备签名 + +推荐的生成新签名的设置： + +这里必须通过 SecureCRT 等终端直接登录用户账户，不能通过 su - user 或者 ssh 转，否则密码输入 box 会显示不出来而报错。 + +先看下 gpg 的 version 以及是否支持 SHA512. + +``` +$ gpg --version +gpg (GnuPG) 2.0.22 +libgcrypt 1.5.3 +Copyright (C) 2013 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. + +Home: ~/.gnupg +Supported algorithms: +Pubkey: RSA, ?, ?, ELG, DSA +Cipher: IDEA, 3DES, CAST5, BLOWFISH, AES, AES192, AES256, TWOFISH, + CAMELLIA128, CAMELLIA192, CAMELLIA256 +Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224 +Compression: Uncompressed, ZIP, ZLIB, BZIP2 +``` + +##### 生成新的签名 + +``` +$ gpg --gen-key +gpg (GnuPG) 2.0.22; Copyright (C) 2013 Free Software Foundation, Inc. +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. + +Please select what kind of key you want: + (1) RSA and RSA (default) + (2) DSA and Elgamal + (3) DSA (sign only) + (4) RSA (sign only) +Your selection? 1 +RSA keys may be between 1024 and 4096 bits long. +What keysize do you want? (2048) 4096 +Requested keysize is 4096 bits +Please specify how long the key should be valid. + 0 = key does not expire + = key expires in n days + w = key expires in n weeks + m = key expires in n months + y = key expires in n years +Key is valid for? (0) +Key does not expire at all +Is this correct? (y/N) y + +GnuPG needs to construct a user ID to identify your key. + +Real name: xxx +Name must be at least 5 characters long +Real name: xxx-yyy +Email address: xxx@apache.org +Comment: xxx's key +You selected this USER-ID: + "xxx-yyy (xxx's key) " + +Change (N)ame, (C)omment, (E)mail or (O)kay/(Q)uit? o +``` + +其中 Real name 需保持和 id.apache.org 中显示的 id 一致。 +Email address 为 apache 的邮箱。 + +##### 查看和输出 + +第一行显示公钥文件名（pubring.gpg），第二行显示公钥特征（4096位，Hash字符串和生成时间），第三行显示"用户ID"，第四行显示私钥特征。 + +``` +$ gpg --list-keys +/home/lide/.gnupg/pubring.gpg +----------------------------- +pub 4096R/33DBF2E0 2018-12-06 +uid xxx-yyy (xxx's key) +sub 4096R/0E8182E6 2018-12-06 +``` + +其中 xxx-yyy 就是用户ID。 + +``` +gpg --armor --output public-key.txt --export [用户ID] +``` + +``` +$ gpg --armor --output public-key.txt --export xxx-yyy +$ cat public-key.txt +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQINBFwJEQ0BEACwqLluHfjBqD/RWZ4uoYxNYHlIzZvbvxAlwS2mn53BirLIU/G3 +9opMWNplvmK+3+gNlRlFpiZ7EvHsF/YJOAP59HmI2Z... +``` + +#### 上传签名公钥 + +公钥服务器是网络上专门储存用户公钥的服务器。send-keys 参数可以将公钥上传到服务器。 + +``` +gpg --send-keys xxxx + +``` +其中 xxxx 为上一步 `--list-keys` 结果中 pub 后面的字符串，如上为：33DBF2E0 + +也可以通过[网站](https://keys.gnupg.net)上传上述 public-key.txt 的内容： + +上传成功之后，可以通过查询这个[网站](https://keys.gnupg.net)，输入 0x33DBF2E0 查询： + +该网站查询有延迟，可能需要等1个小时。 + +#### 生成 fingerprint 并上传到 apache 用户信息中 + +由于公钥服务器没有检查机制，任何人都可以用你的名义上传公钥，所以没有办法保证服务器上的公钥的可靠性。通常，你可以在网站上公布一个公钥指纹，让其他人核对下载到的公钥是否为真。 + +fingerprint参数生成公钥指纹： + +``` +gpg --fingerprint [用户ID] +``` + +``` +$ gpg --fingerprint xxx-yyy +pub 4096R/33DBF2E0 2018-12-06 + Key fingerprint = 07AA E690 B01D 1A4B 469B 0BEF 5E29 CE39 33DB F2E0 +uid xxx-yyy (xxx's key) +sub 4096R/0E8182E6 2018-12-06 +``` + +将上面的 fingerprint （即 07AA E690 B01D 1A4B 469B 0BEF 5E29 CE39 33DB F2E0）粘贴到自己的用户信息中： + +https://id.apache.org + +OpenPGP Public Key Primary Fingerprint: + +#### 生成 keys + +``` +svn co //dist.apache.org/repos/dist/dev/incubator/doris/ +# edit doris/KEY file +gpg --list-sigs [用户 ID] >> doris/KEYS +gpg --armor --export [用户 ID] >> doris/KEYS +svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Update KEYS" +``` + +## 准备发布 + +### 在社区发起 DISCUSS + +如果觉得已经修复了很多bug，开发了比较重要的 feature，任何 IPMC 成员都可以发起 DISCUSS 讨论发布新版本。 +可以发起一个标题为 [DISCUSS] x.y.z release 的邮件，在社区内部进行讨论，说明已经修复了哪些bug，开发了哪些 features。 +如果 DISCUSS 邮件得到大家支持就可以进行下一步。 + +### 准备分支 + +发布前需要先新建一个分支。例如： + +``` +$ git checkout -b branch-0.9 + +``` + +这个分支要进行比较充分的测试，使得功能可用，bug收敛，重要bug都得到修复。 +这个过程需要等待社区，看看是否有必要的patch需要在这个版本合入，如果有，需要把它 cherry-pick 到发布分支。 + +### 清理issue + +将属于这个版本的所有 issue 都过一遍，关闭已经完成的，如果没法完成的，推迟到更晚的版本。 + +### 合并必要的Patch + +在发布等待过程中，可能会有比较重要的Patch合入，如果社区有人说要有重要的Bug需要合入，那么 Release Manager 需要评估并将重要的Patch合入到发布分支中。 + +## 社区发布投票流程 + +### 打 tag + +当上述分支已经比较稳定后，就可以在此分支上打 tag。 +记得在创建 tag 时，修改 `gensrc/script/gen_build_version.sh` 中的 `build_version` 变量。如 `build_version="0.10.0-release"` + +例如： + +``` +$ git checkout branch-0.9 +$ git tag -a 0.9.0-rc01 -m "0.9.0 release candidate 01" +$ git push origin 0.9.0-rc01 +Counting objects: 1, done. +Writing objects: 100% (1/1), 165 bytes | 0 bytes/s, done. +Total 1 (delta 0), reused 0 (delta 0) +To git@github.com:apache/incubator-doris.git + * [new tag] 0.9.0-rc01 -> 0.9.0-rc01 + +$ git tag +``` + +### 打包、签名上传 + +如下步骤，也需要通过 SecureCRT 等终端直接登录用户账户，不能通过 su - user 或者 ssh 转，否则密码输入 box 会显示不出来而报错。 + +``` +$ git checkout 0.9.0-rc01 + +$ git archive --format=tar 0.9.0-rc01 --prefix=apache-doris-0.9.0-incubating-src/ | gzip > apache-doris-0.9.0-incubating-src.tar.gz + +$ gpg -u xxx@apache.org --armor --output apache-doris-0.9.0-incubating-src.tar.gz.asc --detach-sign apache-doris-0.9.0-incubating-src.tar.gz + +$ gpg --verify apache-doris-0.9.0-incubating-src.tar.gz.asc apache-doris-0.9.0-incubating-src.tar.gz + +$ sha512sum apache-doris-0.9.0-incubating-src.tar.gz > apache-doris-0.9.0-incubating-src.tar.gz.sha512 + +$ sha512sum --check apache-doris-0.9.0-incubating-src.tar.gz.sha512 +``` + +然后将打包的内容上传到svn仓库中，首先下载 svn 库： + +``` +svn co https://dist.apache.org/repos/dist/dev/incubator/doris/ +``` + +将之前得到的全部文件组织成以下svn路径 + +``` +./doris/ +|-- 0.11.0-rc1 +| |-- apache-doris-0.11.0-incubating-src.tar.gz +| |-- apache-doris-0.11.0-incubating-src.tar.gz.asc +| `-- apache-doris-0.11.0-incubating-src.tar.gz.sha512 +`-- KEYS +``` + +上传这些文件 + +``` +svn add 0.11.0-rc1 +svn commit -m "Add 0.11.0-rc1" +``` + +### 发社区投票邮件 + +[VOTE] Release Apache Doris 0.9.0-incubating-rc01 + +``` +Hi all, + +Please review and vote on Apache Doris 0.9.0-incubating-rc01 release. + +The release candidate has been tagged in GitHub as 0.9.0-rc01, available +here: +https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 + +Release Notes are here: +https://github.com/apache/incubator-doris/issues/1891 + +Thanks to everyone who has contributed to this release. + +The artifacts (source, signature and checksum) corresponding to this release +candidate can be found here: +https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc1/ + +This has been signed with PGP key 33DBF2E0, corresponding to +lide@apache.org. +KEYS file is available here: +https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS +It is also listed here: +https://people.apache.org/keys/committer/lide.asc + +To verify and build, you can refer to following wiki: +https://github.com/apache/incubator-doris/wiki/How-to-verify-Apache-Release +https://wiki.apache.org/incubator/IncubatorReleaseChecklist + +The vote will be open for at least 72 hours. +[ ] +1 Approve the release +[ ] +0 No opinion +[ ] -1 Do not release this package because ... + +Best Regards, +xxx + +---- +DISCLAIMER-WIP: +Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), +sponsored by the Apache Incubator. Incubation is required of all newly accepted projects +until a further review indicates that the infrastructure, communications, and decision +making process have stabilized in a manner consistent with other successful ASF projects. +While incubation status is not necessarily a reflection of the completeness or stability +of the code, it does indicate that the project has yet to be fully endorsed by the ASF. + +Some of the incubating project’s releases may not be fully compliant with ASF policy. For +example, releases may have incomplete or un-reviewed licensing conditions. What follows is +a list of known issues the project is currently aware of (note that this list, by definition, +is likely to be incomplete): + + * Releases may have incomplete licensing conditions + +If you are planning to incorporate this work into your product/project, please be aware that +you will need to conduct a thorough licensing review to determine the overall implications of +including this work. For the current status of this project through the Apache Incubator +visit: https://incubator.apache.org/projects/doris.html +``` + +### 投票通过后，发 Result 邮件 + +[Result][VOTE] Release Apache Doris 0.9.0-incubating-rc01 + +``` +Thanks to everyone, and this vote is now closed. + +It has passed with 4 +1 (binding) votes and no 0 or -1 votes. + +Binding: ++1 Zhao Chun ++1 xxx ++1 Li Chaoyong ++1 Mingyu Chen + +Best Regards, +xxx + +``` + +### 发邮件到 general@incubator.apache.org 进行投票 + +[VOTE] Release Apache Doris 0.9.0-incubating-rc01 + +``` +Hi all, + +Please review and vote on Apache Doris 0.9.0-incubating-rc01 release. + +Apache Doris is an MPP-based interactive SQL data warehousing for reporting and analysis. + +The Apache Doris community has voted on and approved this release: +https://lists.apache.org/thread.html/d70f7c8a8ae448bf6680a15914646005c6483564464cfa15f4ddc2fc@%3Cdev.doris.apache.org%3E + +The vote result email thread: +https://lists.apache.org/thread.html/64d229f0ba15d66adc83306bc8d7b7ccd5910ecb7e842718ce6a61da@%3Cdev.doris.apache.org%3E + +The release candidate has been tagged in GitHub as 0.9.0-rc01, available here: +https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 + +There is no CHANGE LOG file because this is the first release of Apache Doris. +Thanks to everyone who has contributed to this release, and there is a simple release notes can be found here: +https://github.com/apache/incubator-doris/issues/406 + +The artifacts (source, signature and checksum) corresponding to this release candidate can be found here: +https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc01/ + +This has been signed with PGP key 33DBF2E0, corresponding to lide@apache.org. +KEYS file is available here: +https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS +It is also listed here: +https://people.apache.org/keys/committer/lide.asc + +The vote will be open for at least 72 hours. +[ ] +1 Approve the release +[ ] +0 No opinion +[ ] -1 Do not release this package because ... + +To verify and build, you can refer to following instruction: + +Firstly, you must be install and start docker service, and then you could build Doris as following steps: + +Step1: Pull the docker image with Doris building environment +$ docker pull apachedoris/doris-dev:build-env +You can check it by listing images, its size is about 3.28GB. + +Step2: Run the Docker image +You can run image directly: +$ docker run -it apachedoris/doris-dev:build-env + +Step3: Download Doris source +Now you should in docker environment, and you can download Doris source package. +(If you have downloaded source and it is not in image, you can map its path to image in Step2.) +$ wget https://dist.apache.org/repos/dist/dev/incubator/doris/0.9/0.9.0-rc01/apache-doris-0.9.0.rc01-incubating-src.tar.gz + +Step4: Build Doris +Now you can decompress and enter Doris source path and build Doris. +$ tar zxvf apache-doris-0.9.0.rc01-incubating-src.tar.gz +$ cd apache-doris-0.9.0.rc01-incubating-src +$ sh build.sh + +Best Regards, +xxx + +---- +DISCLAIMER-WIP: +Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), +sponsored by the Apache Incubator. Incubation is required of all newly accepted projects +until a further review indicates that the infrastructure, communications, and decision +making process have stabilized in a manner consistent with other successful ASF projects. +While incubation status is not necessarily a reflection of the completeness or stability +of the code, it does indicate that the project has yet to be fully endorsed by the ASF. + +Some of the incubating project’s releases may not be fully compliant with ASF policy. For +example, releases may have incomplete or un-reviewed licensing conditions. What follows is +a list of known issues the project is currently aware of (note that this list, by definition, +is likely to be incomplete): + + * Releases may have incomplete licensing conditions + +If you are planning to incorporate this work into your product/project, please be aware that +you will need to conduct a thorough licensing review to determine the overall implications of +including this work. For the current status of this project through the Apache Incubator +visit: https://incubator.apache.org/projects/doris.html +``` + +邮件的 thread 连接可以在这里找到： + +`https://lists.apache.org/list.html?dev@doris.apache.org` + + +### 发 Result 邮件到 general@incubator.apache.org + +[RESULT][VOTE] Release Apache Doris 0.9.0-incubating-rc01 + + +``` +Hi, + +Thanks to everyone, and the vote for releasing Apache Doris 0.9.0-incubating-rc01 is now closed. + +It has passed with 4 +1 (binding) votes and no 0 or -1 votes. + +Binding: ++1 Willem Jiang ++1 Justin Mclean ++1 ShaoFeng Shi ++1 Makoto Yui + +The vote thread: +https://lists.apache.org/thread.html/da05fdd8d84e35de527f27200b5690d7811a1e97d419d1ea66562130@%3Cgeneral.incubator.apache.org%3E + +Best Regards, +xxx +``` + +## 完成发布流程 + +### 上传 package 到 release + +当正式发布投票成功后，先发[Result]邮件，然后就准备 release package。 +将之前在dev下发布的对应rc文件夹下的源码包、签名文件和hash文件拷贝到另一个目录 0.9.0-incubating，注意文件名字中不要rcxx (可以rename，但不要重新计算签名，hash可以重新计算，结果不会变) + +第一次发布的话 KEYS 文件也需要拷贝过来。然后add到svn release 下。 + +``` + +https://dist.apache.org/repos/dist/release/incubator/doris/0.9.0-incubating/ + +最终能在 apache 官网看到： +http://www.apache.org/dist/incubator/doris/0.9.0-incubating/ + +``` + +### 发 Announce 邮件到 general@incubator.apache.org + +Title: + +``` +[ANNOUNCE] Apache Doris (incubating) 0.9.0 Release +``` + +发送邮件组： + +``` +general@incubator.apache.org +dev@doris.apache.org +``` + +邮件正文： + +``` +Hi All, + +We are pleased to announce the release of Apache Doris 0.9.0-incubating. + +Apache Doris (incubating) is an MPP-based interactive SQL data warehousing for reporting and analysis. + +The release is available at: +http://doris.apache.org/downloads.html + +Thanks to everyone who has contributed to this release, and the release note can be found here: +https://github.com/apache/incubator-doris/releases + +Best Regards, + +On behalf of the Doris team, +xxx + +---- +DISCLAIMER-WIP: +Apache Doris is an effort undergoing incubation at The Apache Software Foundation (ASF), +sponsored by the Apache Incubator. Incubation is required of all newly accepted projects +until a further review indicates that the infrastructure, communications, and decision +making process have stabilized in a manner consistent with other successful ASF projects. +While incubation status is not necessarily a reflection of the completeness or stability +of the code, it does indicate that the project has yet to be fully endorsed by the ASF. + +Some of the incubating project’s releases may not be fully compliant with ASF policy. For +example, releases may have incomplete or un-reviewed licensing conditions. What follows is +a list of known issues the project is currently aware of (note that this list, by definition, +is likely to be incomplete): + + * Releases may have incomplete licensing conditions + +If you are planning to incorporate this work into your product/project, please be aware that +you will need to conduct a thorough licensing review to determine the overall implications of +including this work. For the current status of this project through the Apache Incubator +visit: https://incubator.apache.org/projects/doris.html +``` + +### 在 Doris 官网和 github 发布链接 + +#### 创建下载链接 + +下载链接： +http://www.apache.org/dyn/closer.cgi?filename=incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz&action=download + +wget --trust-server-names "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz" + +原始位置: +https://www.apache.org/dist/incubator/doris/0.9.0-incubating/ + +http://www.apache.org/dyn/closer.cgi/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz + +源码包（source package）: +http://www.apache.org/dyn/closer.cgi/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz + +ASC: +http://archive.apache.org/dist/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.asc + +sha512: +http://archive.apache.org/dist/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.sha512 + +KEYS: +http://archive.apache.org/dist/incubator/doris/KEYS + +refer to: + +#### 准备 release note + +需要修改如下两个地方： + +1、Github 的 release 页面 + +``` +https://github.com/apache/incubator-doris/releases/tag/0.9.0-rc01 +``` + +2、Doris 官网下载页面 + +``` +http://doris.apache.org/downloads.html +``` + diff --git a/docs/zh-CN/community/subscribe-mail-list.md b/docs/zh-CN/community/subscribe-mail-list.md new file mode 100644 index 00000000000000..13ca3f74fdbe8b --- /dev/null +++ b/docs/zh-CN/community/subscribe-mail-list.md @@ -0,0 +1,69 @@ +--- +{ + "title": "订阅邮件列表", + "language": "zh-CN" +} +--- + + + +# 订阅邮件列表 + +邮件列表（Mail List）是 Apache 社区最被认可的交流方式。一般来说，开源社区的提问与解答、技术讨论、事务决策等都通过邮件列表来承载。邮件列表异步、广播的特性，也非常适合开源社区的沟通交流。那么，如何订阅 Apache Doris (incubating) 的邮件列表呢？主要包括以下五个步骤。 + +## 1. 发送订阅邮件 + +打开自己的邮箱，新建邮件，向`dev-subscribe@doris.apache.org`发送一封邮件（邮件主题和内容任意） + +![step1](/images/subscribe-mail-list-step1.png) + +## 2. 接收来自 dev-help@doris.apache.org 的确认邮件 + +执行完第一步之后，您将收到一封来自`dev-help@doris.apache.org`的确认邮件，邮件内容如下图所示。（**如果长时间未能收到，请确认该邮件是否已被拦截，或已经被自动归入“订阅邮件”、“垃圾邮件”、“推广邮件”等文件夹**） + +![step2](/images/subscribe-mail-list-step2.png) + +## 3. 回复确认邮件 + +​针对上一步接收到的邮件， + +​**a.直接回复该邮件** + +​***或*** + +**b. 新建一封`收件人`为上一步中的`回复地址`的邮件** + +​均可，内容主题不限 + +![step3](/images/subscribe-mail-list-step3.png) + + +## 4. 接收欢迎邮件 + +​完成第三步之后，将会受到一封标题为**WELCOME to dev@doris.apache.org**的欢迎邮件。至此，订阅邮件列表的工作已经完成了，社区的动态都会通过邮件的方式通知您。 + +![step4](/images/subscribe-mail-list-step4.png) + + +## 5. 发起邮件讨论（可选） + +​成功订阅邮件列表后，若想发起讨论，直接往`dev@doris.apache.org`发送邮件即可。所有订阅了邮件列表的人都会收到邮件。 +​ +​ \ No newline at end of file diff --git a/docs/zh-CN/community/verify-apache-release.md b/docs/zh-CN/community/verify-apache-release.md new file mode 100644 index 00000000000000..379732888d4b89 --- /dev/null +++ b/docs/zh-CN/community/verify-apache-release.md @@ -0,0 +1,88 @@ +--- +{ + "title": "验证 Apache 发布版本", + "language": "zh-CN" +} +--- + + + +# 验证 Apache 发布版本 + +可以按照以下步骤对发布版本进行验证： + +1. [ ] 下载链接是否合法。 +2. [ ] 校验值和 PGP 签名是否合法。 +3. [ ] 是否包含 DISCLAIMER-WIP。 +4. [ ] 代码是否和当前发布版本相匹配。 +5. [ ] LICENSE 和 NOTICE 文件是否正确。 +6. [ ] 所有文件都携带必要的协议说明。 +7. [ ] 在源码包中不包含已经编译好的内容。 +8. [ ] 编译是否能够顺利执行。 + +## 1. 下载源码包、签名文件、校验值文件和 KEYS + +下载所有相关文件, 以 a.b.c-incubating 为示例: + +``` +wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz + +wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz.sha512 + +wget https://dist.apache.org/repos/dist/dev/incubator/doris/a.b.c-incubating/apache-doris-a.b.c-incubating-src.tar.gz.asc + +wget https://dist.apache.org/repos/dist/dev/incubator/doris/KEYS +``` + +## 2. 检查签名和校验值 + +推荐使用 GunPG，可以通过以下命令安装： + +``` +CentOS: yum install gnupg +Ubuntu: apt-get install gnupg +``` + +``` +gpg --import KEYS +gpg --verify apache-doris-a.b.c-incubating-src.tar.gz.asc apache-doris-a.b.c-incubating-src.tar.gz +sha512sum --check apache-doris-a.b.c-incubating-src.tar.gz.sha512 +``` + +## 3. 验证源码协议头 + +推荐使用 Apache RAT 验证源码协议，可以从一下连接下载： + +``` +wget http://mirrors.tuna.tsinghua.edu.cn/apache//creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz +tar zxvf apache-rat-0.12-bin.tar.gz +``` + +假设源码目录名称为 apache-doris-a.b.c-incubating-src，可以使用以下命令进行验证。 +这个命令会产生一个文件，其中列举了所有非 ASF 协议的文件。 + +``` +/usr/java/jdk/bin/java -jar apache-rat-0.12/apache-rat-0.12.jar -a -d apache-doris-a.b.c-incubating-src -E apache-doris-a.b.c-incubating-src/.rat-excudes +``` + +## 4. 验证编译 + +详细的编译步骤，请参阅 [编译文档](../installing/compilation.html) + diff --git a/docs/zh-CN/developer-guide/debug-tool.md b/docs/zh-CN/developer-guide/debug-tool.md new file mode 100644 index 00000000000000..c80f5082bb471b --- /dev/null +++ b/docs/zh-CN/developer-guide/debug-tool.md @@ -0,0 +1,273 @@ +--- +{ + "title": "调试工具", + "language": "zh-CN" +} +--- + + + +# 调试工具 + +在Doris的使用、开发过程中，经常会遇到需要对Doris进行调试的场景，这里介绍一些常用的调试工具。 + +## 环境准备 + +[pprof](https://github.com/google/pprof): 来自gperftools，用于将gperftools所产生的内容转化成便于人可以阅读的格式，比如pdf, svg, text等. + +[graphviz](http://www.graphviz.org/): 在没有这个库的时候pprof只可以转化为text格式，但这种方式不易查看。那么安装这个库后，pprof可以转化为svg、pdf等格式，对于调用关系则更加清晰明了。 + +[perf](https://perf.wiki.kernel.org/index.php/Main_Page): linux内核自带性能分析工具。[这里](http://www.brendangregg.com/perf.html)有一些perf的使用例子。 + +[FlameGraph](https://github.com/brendangregg/FlameGraph): 可视化工具，用于将perf的输出以火焰图的形式展示出来。 + +## 内存 + +对于内存的调试一般分为两个方面。一个是内存使用的总量是否合理，内存使用量过大一方面可能是由于系统存在内存泄露，另一方面可能是因为程序内存使用不当。其次就是是否存在内存越界、非法访问的问题，比如程序访问一个非法地址的内存，使用了未初始化内存等。对于内存方面的调试我们一般使用如下几种方式来进行问题追踪。 + +### 查看日志 + +当发现内存使用量过大的时候，我们可以先查看be.out日志，看看是否有大内存申请。由于Doris当前使用的TCMalloc管理内存，那么遇到大内存申请时，都会将申请的堆栈打印到be.out文件中，一般的表现形式如下： + +``` +tcmalloc: large alloc 1396277248 bytes == 0x3f3488000 @ 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed +``` + +这个表示在Doris BE在这个堆栈上尝试申请`1396277248 bytes`的内存。我们可以通过`addr2line`命令去把堆栈还原成我们能够看懂的信，具体的例子如下所示。 + +``` +$ addr2line -e lib/palo_be 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed + +/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1335 +/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1357 +/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.cpp:267 +/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.hpp:86 +/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:239 +/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:213 +thread.cpp:? +``` + +### HEAP PROFILE + +有时内存的申请并不是大内存的申请导致，而是通过小内存不断的堆积导致的。那么就没有办法通过查看日志定位到具体的申请信息，那么就需要通过其他方式来获得信息。 + +这个时候我们可以利用TCMalloc的[HEAPPROFILE](https://gperftools.github.io/gperftools/heapprofile.html)的功能。如果设置了HEAPPROFILE功能，那么我们可以获得进程整体的内存申请使用情况。使用方式是在启动Doris BE前设置`HEAPPROFILE`环境变量。比如： + +``` +export HEAPPROFILE=/tmp/doris_be.hprof +./bin/start_be.sh --daemon +``` + +这样，当满足HEAPPROFILE的dump条件时，就会将内存的整体使用情况写到指定路径的文件中。后续我们就可以通过使用`pprof`工具来对输出的内容进行分析。 + +``` +$ pprof --text lib/palo_be /tmp/doris_be.hprof.0012.heap | head -30 + +Using local file lib/palo_be. +Using local file /tmp/doris_be.hprof.0012.heap. +Total: 668.6 MB + 610.6 91.3% 91.3% 610.6 91.3% doris::SystemAllocator::allocate_via_malloc (inline) + 18.1 2.7% 94.0% 18.1 2.7% _objalloc_alloc + 5.6 0.8% 94.9% 63.4 9.5% doris::RowBatch::RowBatch + 5.1 0.8% 95.6% 7.1 1.1% butil::ResourcePool::add_block (inline) + 3.7 0.5% 96.2% 3.7 0.5% butil::iobuf::create_block (inline) + 3.4 0.5% 96.7% 3.4 0.5% butil::FlatMap::init + 3.2 0.5% 97.2% 5.2 0.8% butil::ObjectPool::add_block (inline) + 2.6 0.4% 97.6% 2.6 0.4% __gnu_cxx::new_allocator::allocate (inline) + 2.0 0.3% 97.9% 2.0 0.3% butil::ObjectPool::add_block_group (inline) + 2.0 0.3% 98.2% 2.0 0.3% butil::ResourcePool::add_block_group (inline) + 1.7 0.3% 98.4% 1.7 0.3% doris::SegmentReader::_load_index +``` + +上述文件各个列的内容： + +* 第一列：函数直接申请的内存大小，单位MB +* 第四列：函数以及函数所有调用的函数总共内存大小。 +* 第二列、第五列分别是第一列与第四列的的比例值。 +* 第三列是个第二列的累积值。 + +当然也可以生成调用关系图片，更加方便分析。比如下面的命令就能够生成SVG格式的调用关系图。 + +``` +pprof --svg lib/palo_be /tmp/doris_be.hprof.0012.heap > heap.svg +``` + +**注意：开启这个选项是要影响程序的执行性能的，请慎重对线上的实例开启** + +### pprof remote server + +HEAPPROFILE虽然能够获得全部的内存使用信息，但是也有比较受限的地方。1. 需要重启BE进行。2. 需要一直开启这个命令，导致对整个进程的性能造成影响。 + +对Doris BE也可以使用动态开启、关闭heap profile的方式来对进程进行内存申请分析。Doris内部支持了GPerftools的[远程server调试](https://gperftools.github.io/gperftools/pprof_remote_servers.html)。那么可以通过`pprof`直接对远程运行的Doris BE进行动态的HEAP PROFILE。比如我们可以通过以下命令来查看Doris的内存的使用增量 + +``` +$ pprof --text --seconds=60 http://be_host:be_webport/pprof/heap + +Total: 1296.4 MB + 484.9 37.4% 37.4% 484.9 37.4% doris::StorageByteBuffer::create + 272.2 21.0% 58.4% 273.3 21.1% doris::RowBlock::init + 157.5 12.1% 70.5% 157.5 12.1% doris::RowBatch::RowBatch + 90.7 7.0% 77.5% 90.7 7.0% doris::SystemAllocator::allocate_via_malloc + 66.6 5.1% 82.7% 66.6 5.1% doris::IntegerColumnReader::init + 47.9 3.7% 86.4% 47.9 3.7% __gnu_cxx::new_allocator::allocate + 20.8 1.6% 88.0% 35.4 2.7% doris::SegmentReader::_load_index + 12.7 1.0% 89.0% 12.7 1.0% doris::DecimalColumnReader::init + 12.7 1.0% 89.9% 12.7 1.0% doris::LargeIntColumnReader::init + 12.7 1.0% 90.9% 12.7 1.0% doris::StringColumnDirectReader::init + 12.3 0.9% 91.9% 12.3 0.9% std::__cxx11::basic_string::_M_mutate + 10.4 0.8% 92.7% 10.4 0.8% doris::VectorizedRowBatch::VectorizedRowBatch + 10.0 0.8% 93.4% 10.0 0.8% doris::PlainTextLineReader::PlainTextLineReader +``` + +这个命令的输出与HEAP PROFILE的输出及查看方式一样，这里就不再详细说明。这个命令只有在执行的过程中才会开启统计，相比HEAP PROFILE对于进程性能的影响有限。 + +### LSAN + +[LSAN](https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer)是一个地址检查工具，GCC已经集成。在我们编译代码的时候开启相应的编译选项，就能够开启这个功能。当程序发生可以确定的内存泄露时，会将泄露堆栈打印。Doris BE已经集成了这个工具，只需要在编译的时候使用如下的命令进行编译就能够生成带有内存泄露检测版本的BE二进制 + +``` +BUILD_TYPE=LSAN ./build.sh +``` + +当系统检测到内存泄露的时候，就会在be.out里面输出对应的信息。为了下面的演示，我们故意在代码中插入一段内存泄露代码。我们在`StorageEngine`的`open`函数中插入如下代码 + +``` + char* leak_buf = new char[1024]; + strcpy(leak_buf, "hello world"); + LOG(INFO) << leak_buf; +``` + +我们就在be.out中获得了如下的输出 + +``` +================================================================= +==24732==ERROR: LeakSanitizer: detected memory leaks + +Direct leak of 1024 byte(s) in 1 object(s) allocated from: + #0 0xd10586 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/lsan/lsan_interceptors.cc:164 + #1 0xe333a2 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104 + #2 0xd3cc96 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 + #3 0x7f573b5eebd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) + +SUMMARY: LeakSanitizer: 1024 byte(s) leaked in 1 allocation(s). +``` + +从上述的输出中，我们能看到有1024个字节被泄露了，并且打印出来了内存申请时的堆栈信息。 + +**注意：开启这个选项是要影响程序的执行性能的，请慎重对线上的实例开启** + +**注意：如果开启了LSAN开关的话，tcmalloc就会被自动关闭** + +### ASAN + +除了内存使用不合理、泄露以外。有的时候也会发生内存访问非法地址等错误。这个时候我们可以借助[ASAN](https://github.com/google/sanitizers/wiki/AddressSanitizer)来辅助我们找到问题的原因。与LSAN一样，ASAN也集成在了GCC中。Doris通过如下的方式进行编译就能够开启这个功能 + +``` +BUILD_TYPE=ASAN ./build.sh +``` + +执行编译生成的二进制文件，当检测工具发现有异常访问时，就会立即退出，并将非法访问的堆栈输出在be.out中。对于ASAN的输出与LSAN是一样的分析方法。这里我们也主动注入一个地址访问错误，来展示下具体的内容输出。我们仍然在`StorageEngine`的`open`函数中注入一段非法内存访问，具体的错误代码如下 + +``` + char* invalid_buf = new char[1024]; + for (int i = 0; i < 1025; ++i) { + invalid_buf[i] = i; + } + LOG(INFO) << invalid_buf; +``` + +然后我们就会在be.out中获得如下的输出 + +``` +================================================================= +==23284==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x61900008bf80 at pc 0x00000129f56a bp 0x7fff546eed90 sp 0x7fff546eed88 +WRITE of size 1 at 0x61900008bf80 thread T0 + #0 0x129f569 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 + #1 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 + #2 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) + #3 0xd30794 (/home/ssd0/zc/palo/doris/core/output3/be/lib/palo_be+0xd30794) + +0x61900008bf80 is located 0 bytes to the right of 1024-byte region [0x61900008bb80,0x61900008bf80) +allocated by thread T0 here: + #0 0xdeb040 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/asan/asan_new_delete.cc:82 + #1 0x129f50d in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104 + #2 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159 + #3 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4) + +SUMMARY: AddressSanitizer: heap-buffer-overflow /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) +``` + +从这段信息中该可以看到在`0x61900008bf80`这个地址我们尝试去写一个字节，但是这个地址是非法的。我们也可以看到 `[0x61900008bb80,0x61900008bf80)`这个地址的申请堆栈。 + +**注意：开启这个选项是要影响程序的执行性能的，请慎重对线上的实例开启** + +**注意：如果开启了ASAN开关的话，tcmalloc就会被自动关闭** + +另外，如果be.out中输出了堆栈信息，但是并没有函数符号，那么这个时候需要我们手动的处理下才能获得可读的堆栈信息。具体的处理方法需要借助一个脚本来解析ASAN的输出。这个时候我们需要使用[asan_symbolize](https://llvm.org/svn/llvm-project/compiler-rt/trunk/lib/asan/scripts/asan_symbolize.py)来帮忙解析下。具体的使用方式如下： + +``` +cat be.out | python asan_symbolize.py | c++filt +``` + +通过上述的命令，我们就能够获得可读的堆栈信息了。 + +## CPU + +当系统的CPU Idle很低的时候，说明系统的CPU已经成为了主要瓶颈，这个时候就需要分析一下当前的CPU使用情况。对于Doris的BE可以有如下两种方式来分析Doris的CPU瓶颈。 + +### pprof + +由于Doris内部已经集成了并兼容了GPerf的REST接口，那么用户可以通过`pprof`工具来分析远程的Doris BE。具体的使用方式如下： + +``` +pprof --svg --seconds=60 http://be_host:be_webport/pprof/profile > be.svg +``` + +这样就能够生成一张BE执行的CPU消耗图。 + +![CPU Pprof](/images/cpu-pprof-demo.png) + +### perf + flamegragh + +这个是相当通用的一种CPU分析方式，相比于`pprof`，这种方式必须要求能够登陆到分析对象的物理机上。但是相比于pprof只能定时采点，perf是能够通过不同的事件来完成堆栈信息采集的。具体的的使用方式如下： + +``` +perf record -g -p be_pid -- sleep 60 +``` + +这条命令会统计60秒钟BE的CPU运行情况，并且生成perf.data。对于perf.data的分析，可以通过perf的命令来进行分析 + +``` +perf report +``` + +分析得到如下的图片 + +![Perf Report](/images/perf-report-demo.png) + +来对生成的内容进行分析。当然也可以使用flamegragh完成可视化展示。 + +``` +perf script | ./FlameGraph/stackcollapse-perf.pl | ./FlameGraph/flamegraph.pl > be.svg +``` + +这样也会生成一张当时运行的CPU消耗图。 + +![CPU Flame](/images/cpu-flame-demo.svg) diff --git a/docs/zh-CN/developer-guide/format-code.md b/docs/zh-CN/developer-guide/format-code.md new file mode 100644 index 00000000000000..09521d7a73b683 --- /dev/null +++ b/docs/zh-CN/developer-guide/format-code.md @@ -0,0 +1,73 @@ +--- +{ + "title": "代码格式化", + "language": "zh-CN" +} +--- + + + +# 代码格式化 +为了自动格式化代码，推荐使用clang-format进行代码格式化。 + +## 代码风格定制 +Doris的代码风格在Google Style的基础上稍有改动，定制为.clang-format文件，位于Doris根目录。 + +目前，.clang-format配置文件适配clang-format-8.0.1以上的版本。 + +## 环境准备 +需要下载安装clang-format，也可使用IDE或Editor提供的clang-format插件，下面分别介绍。 + +### 下载安装clang-format +Ubuntu: `apt-get install clang-format` + +当前版本为10.0，也可指定旧版本，例如: `apt-get install clang-format-9` + +Centos 7: + +centos yum安装的clang-format版本过老，支持的StyleOption太少，建议源码编译10.0版本。 + +### clang-format插件 +Clion IDE可使用插件"ClangFormat"，`File->Setting->Plugins`搜索下载。但版本无法和 +clang-format程序的版本匹配，从支持的StyleOption上看，应该是低于clang-format-9.0。 + +## 使用方式 + +### 命令行运行 +`clang-format --style=file -i $File$` + +`--sytle=file`就会自动找到.clang-format文件，根据文件Option配置来格式化代码。 + +批量文件clang-format时，需注意过滤不应该格式化的文件。例如，只格式化*.h/*.cpp，并排除某些文件夹： + +`find . -type f -not \( -wholename ./env/* \) -regextype posix-egrep -regex + ".*\.(cpp|h)" | xargs clang-format -i -style=file` + +### 在IDE或Editor中使用clang-format +#### Clion +Clion如果使用插件，点击`Reformat Code`即可。 +#### VS Code +VS Code需安装扩展程序Clang-Format，但需要自行提供clang-format执行程序的位置。 + +``` +"clang-format.executable": "$clang-format path$", +"clang-format.style": "file" +``` +然后，点击`Format Document`即可。 \ No newline at end of file diff --git a/docs/zh-CN/downloads/downloads.md b/docs/zh-CN/downloads/downloads.md new file mode 100644 index 00000000000000..c7cfdd2b1f9d36 --- /dev/null +++ b/docs/zh-CN/downloads/downloads.md @@ -0,0 +1,21 @@ +--- +{ + "title": "下载", + "language": "zh-CN" +} +--- + +# 下载 + +您可以通过以下连接下载对应版本的 Doris 源码进行编译和部署。 + +| 版本 | 发布日期 | 从镜像网站下载 | +|---|---|---| +| 0.12.0 | 2020-04-24 | [Source](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.12.0-incubating/apache-doris-0.12.0-incubating-src.tar.gz.sha512)) | +| 0.11.0 | 2019-11-29 | [Source](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.11.0-incubating/apache-doris-0.11.0-incubating-src.tar.gz.sha512)) | +| 0.10.0 | 2019-07-02 | [Source](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.10.0-incubating/apache-doris-0.10.0-incubating-src.tar.gz.sha512)) | +| 0.9.0 | 2019-02-18 | [Source](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz) ([Signature](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.asc) [SHA512](https://downloads.apache.org/incubator/doris/0.9.0-incubating/apache-doris-0.9.0-incubating-src.tar.gz.sha512)) | + +关于如何校验下载文件，请参阅 [校验下载文件](../community/verify-apache-release.html)。 + +校验完成后，可以参阅 [编译文档](../installing/compilation.html) 以及 [安装与部署文档](../installing/install-deploy.html) 进行 Doris 的编译、安装与部署。 diff --git a/docs/zh-CN/extending-doris/audit-plugin.md b/docs/zh-CN/extending-doris/audit-plugin.md new file mode 100644 index 00000000000000..326cb69b7a6361 --- /dev/null +++ b/docs/zh-CN/extending-doris/audit-plugin.md @@ -0,0 +1,115 @@ +--- +{ + "title": "审计日志插件", + "language": "zh-CN" +} +--- + + + +# 审计日志插件 + +Doris 的审计日志插件是在 FE 的插件框架基础上开发的。是一个可选插件。用户可以在运行时安装或卸载这个插件。 + +该插件可以将 FE 的审计日志定期的导入到指定 Doris 集群中，以方便用户通过 SQL 对审计日志进行查看和分析。 + +## 编译、配置和部署 + +### 编译 + +在 Doris 代码目录下执行 `sh build_plugin.sh` 后，会在 `fe_plugins/output` 目录下得到 `auditloader.zip` 文件。 + +### 配置 + +解压 `auditloader.zip` 可以看到三个文件： + +``` +auditloader.jar +plugin.properties +plugin.conf +``` + +打开 `plugin.conf` 进行配置。配置项说明参见注释。 + +配置完成后，重新将三个文件打包为 `auditloader.zip`. + +### 部署 + +您可以将这个文件放置在一个 http 下载服务器上，或者拷贝到所有 FE 的指定目录下。这里我们使用后者。 + +### 安装 + +部署完成后，安装插件前，需要创建之前在 `plugin.conf` 中指定的审计数据库和表。其中建表语句如下： + +``` +create table doris_audit_tbl__ +( + query_id varchar(48) comment "Unique query id", + time datetime not null comment "Query start time", + client_ip varchar(32) comment "Client IP", + user varchar(64) comment "User name", + db varchar(96) comment "Database of this query", + state varchar(8) comment "Query result state. EOF, ERR, OK", + query_time bigint comment "Query execution time in millisecond", + scan_bytes bigint comment "Total scan bytes of this query", + scan_rows bigint comment "Total scan rows of this query", + return_rows bigint comment "Returned rows of this query", + stmt_id int comment "An incremental id of statement", + is_query tinyint comment "Is this statemt a query. 1 or 0", + frontend_ip varchar(32) comment "Frontend ip of executing this statement", + stmt varchar(2048) comment "The original statement, trimed if longer than 2048 bytes" +) +partition by range(time) () +distributed by hash(query_id) buckets 1 +properties( + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-30", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.enable" = "true", + "replication_num" = "1" +); +``` + +其中 `dynamic_partition` 属性根据自己的需要，选择审计日志安保留的天数。 + +之后，连接到 Doris 后使用 `INSTALL PLUGIN` 命令完成安装。安装成功后，可以通过 `SHOW PLUGINS` 看到已经安装的插件，并且状态为 `INSTALLED`。 + +完成后，插件会不断的以指定的时间间隔将审计日志插入到这个表中。 + + + + + + + + + + + + + + + + + + + diff --git a/docs/zh-CN/extending-doris/doris-on-es.md b/docs/zh-CN/extending-doris/doris-on-es.md new file mode 100644 index 00000000000000..afe5e4704d9b46 --- /dev/null +++ b/docs/zh-CN/extending-doris/doris-on-es.md @@ -0,0 +1,231 @@ +--- +{ + "title": "Doris On ES", + "language": "zh-CN" +} +--- + + + +# Doris On ES + +Doris-On-ES将Doris的分布式查询规划能力和ES(Elasticsearch)的全文检索能力相结合，提供更完善的OLAP分析场景解决方案： + + 1. ES中的多index分布式Join查询 + 2. Doris和ES中的表联合查询，更复杂的全文检索过滤 + 3. ES keyword类型字段的聚合查询：适用于index 频繁发生变化、单个分片文档数量千万级以上且该字段基数(cardinality)非常大 + +本文档主要介绍该功能的实现原理、使用方式等。 + +## 名词解释 + +* FE：Frontend，Doris 的前端节点。负责元数据管理和请求接入。 +* BE：Backend，Doris 的后端节点。负责查询执行和数据存储。 +* Elasticsearch(ES)：目前最流行的开源分布式搜索引擎。 +* DataNode：ES的数据存储与计算节点。 +* MasterNode：ES的Master节点，管理元数据、节点、数据分布等。 +* scroll：ES内置的数据集游标特性，用来对数据进行流式扫描和过滤。 + + +## 如何使用 + +### 创建外表 + +``` +CREATE EXTERNAL TABLE `es_table` ( + `id` bigint(20) COMMENT "", + `k1` bigint(20) COMMENT "", + `k2` datetime COMMENT "", + `k3` varchar(20) COMMENT "", + `k4` varchar(100) COMMENT "", + `k5` float COMMENT "" +) ENGINE=ELASTICSEARCH +PARTITION BY RANGE(`id`) +() +PROPERTIES ( +"host" = "http://192.168.0.1:8200,http://192.168.0.2:8200", +"user" = "root", +"password" = "root", +"index" = "tindex”, +"type" = "doc" +); +``` + +参数说明： + +参数 | 说明 +---|--- +host | ES集群连接地址，可指定一个或多个，Doris通过这个地址获取到ES版本号、index的shard分布信息 +user | 开启basic认证的ES集群的用户名，需要确保该用户有访问: /\_cluster/state/\_nodes/http等路径权限和对index的读权限 +password | 对应用户的密码信息 +index | Doris中的表对应的ES的index名字，可以是alias +type | 指定index的type，默认是_doc +transport | 内部保留，默认为http + +### 查询 + +#### 基本条件过滤 + +``` +select * from es_table where k1 > 1000 and k3 ='term' or k4 like 'fu*z_' +``` + +#### 扩展的esquery sql语法 +通过`esquery`函数将一些无法用sql表述的ES query如match、geoshape等下推给ES进行过滤处理，`esquery`的第一个列名参数用于关联`index`，第二个参数是ES的基本`Query DSL`的json表述，使用花括号`{}`包含，json的`root key`有且只能有一个，如match、geo_shape、bool等 + +match查询： + +``` +select * from es_table where esquery(k4, '{ + "match": { + "k4": "doris on elasticsearch" + } + }'); +``` +geo相关查询： + +``` +select * from es_table where esquery(k4, '{ + "geo_shape": { + "location": { + "shape": { + "type": "envelope", + "coordinates": [ + [ + 13, + 53 + ], + [ + 14, + 52 + ] + ] + }, + "relation": "within" + } + } + }'); +``` + +bool查询： + +``` +select * from es_table where esquery(k4, ' { + "bool": { + "must": [ + { + "terms": { + "k1": [ + 11, + 12 + ] + } + }, + { + "terms": { + "k2": [ + 100 + ] + } + } + ] + } + }'); +``` + + + +## 原理 + +``` ++----------------------------------------------+ +| | +| Doris +------------------+ | +| | FE +--------------+-------+ +| | | Request Shard Location +| +--+-------------+-+ | | +| ^ ^ | | +| | | | | +| +-------------------+ +------------------+ | | +| | | | | | | | | +| | +----------+----+ | | +--+-----------+ | | | +| | | BE | | | | BE | | | | +| | +---------------+ | | +--------------+ | | | ++----------------------------------------------+ | + | | | | | | | + | | | | | | | + | HTTP SCROLL | | HTTP SCROLL | | ++-----------+---------------------+------------+ | +| | v | | v | | | +| | +------+--------+ | | +------+-------+ | | | +| | | | | | | | | | | +| | | DataNode | | | | DataNode +<-----------+ +| | | | | | | | | | | +| | | +<--------------------------------+ +| | +---------------+ | | |--------------| | | | +| +-------------------+ +------------------+ | | +| Same Physical Node | | +| | | +| +-----------------------+ | | +| | | | | +| | MasterNode +<-----------------+ +| ES | | | +| +-----------------------+ | ++----------------------------------------------+ + + +``` + +1. 创建ES外表后，FE会请求建表指定的主机，获取所有节点的HTTP端口信息以及index的shard分布信息等，如果请求失败会顺序遍历host列表直至成功或完全失败 + +2. 查询时，会根据FE得到的一些节点信息和index的元数据信息，生成查询计划并发给对应的BE节点 + +3. BE节点会根据`就近原则`即优先请求本地部署的ES节点，BE通过`HTTP Scroll`方式流式的从ES index的每个分片中并发的获取数据 + +4. 计算完结果后，返回给client端 + +## Push-Down operations +`Doris On Elasticsearch`一个重要的功能就是过滤条件的下推: 过滤条件下推给ES，这样只有真正满足条件的数据才会被返回，能够显著的提高查询性能和降低Doris和Elasticsearch的CPU、memory、IO利用率 + +下面的操作符(Operators)会被优化成如下下推filters: + +| SQL syntax | ES 5.x+ syntax | +|-------|:---:| +| = | term query| +| in | terms query | +| > , < , >= , ⇐ | range | +| and | bool.filter | +| or | bool.should | +| not | bool.must_not | +| not in | bool.must_not + terms | +| esquery | ES Query DSL | + + +## 其他说明 + +1. ES的版本要求 + + ES主版本大于5，ES在2.x之前和5.x之后数据的扫描方式不同，目前支持5.x之后的 +2. 是否支持X-Pack认证的ES集群 + + 支持所有使用HTTP Basic认证方式的ES集群 +3. 一些查询比请求ES慢很多 + + 是，比如_count相关的query等，ES内部会直接读取满足条件的文档个数相关的元数据，不需要对真实的数据进行过滤 diff --git a/docs/zh-CN/extending-doris/plugin-development-manual.md b/docs/zh-CN/extending-doris/plugin-development-manual.md new file mode 100644 index 00000000000000..76f56ea1c7a62f --- /dev/null +++ b/docs/zh-CN/extending-doris/plugin-development-manual.md @@ -0,0 +1,298 @@ +--- +{ + "title": "插件开发手册", + "language": "zh-CN" +} +--- + + + +# 插件开发手册 + +## 介绍 + +Doris 支持动态加载插件。用户可以通过开发自己的插件来扩展Doris的功能。这个手册主要介绍如何开发、编译和部署 Frontend 端的插件。 + +`fe_plugins` 目录是 FE 插件的根模块。这个根模块统一管理插件所需的依赖。添加一个新的插件，相当于在这个根模块添加一个子模块。 + +## 插件组成 + +一个FE的插件可以使一个**zip压缩包**或者是一个**目录**。其内容至少包含两个文件：`plugin.properties` 和 `.jar` 文件。`plugin.properties`用于描述插件信息。 + +文件结构如下： + +``` +# plugin .zip +auditodemo.zip: + -plugin.properties + -auditdemo.jar + -xxx.config + -data/ + -test_data/ + +# plugin local directory +auditodemo/: + -plugin.properties + -auditdemo.jar + -xxx.config + -data/ + -test_data/ +``` + +`plugin.properties` 内容示例: + +``` +### required: +# +# the plugin name +name = audit_plugin_demo +# +# the plugin type +type = AUDIT +# +# simple summary of the plugin +description = just for test +# +# Doris's version, like: 0.11.0 +version = 0.11.0 + +### FE-Plugin optional: +# +# version of java the code is built against +# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4 +java.version = 1.8.31 +# +# the name of the class to load, fully-qualified. +classname = AuditPluginDemo + +### BE-Plugin optional: +# the name of the so to load +soName = example.so +``` + +## 编写一个插件 + +插件的开发环境依赖Doris的开发编译环境。所以请先确保Doris的编译开发环境运行正常。 + +### 创建一个模块 + +我们可以通过以下命令在 `fe_plugins` 目录创建一个子模块用户实现创建和创建工程。其中 `doris-fe-test` 为插件名称。 + +``` +mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false +``` + +这个命令会创建一个新的 maven 工程，并且自动向 `fe_plugins/pom.xml` 中添加一个子模块： + +``` +    ..... +    org.apache +    doris-fe-plugins +    pom +    1.0-SNAPSHOT +     +        auditdemo +        # new plugin module +        doris-fe-test +     +    ..... +``` + +新的工程目录结构如下： + +``` +-doris-fe-test/ +-pom.xml +-src/ + ---- main/java/org/apache/ + ------- App.java # mvn auto generate, ignore + ---- test/java/org/apache +``` + +接下来我们在 `main` 目录下添加一个 `assembly` 目录来存放 `plugin.properties` 和 `zip.xml`。最终的工程目录结构如下： + +``` +-doris-fe-test/ +-pom.xml +-src/ +---- main/ +------ assembly/ +-------- plugin.properties +-------- zip.xml +------ java/org/apache/ +--------App.java # mvn auto generate, ignore +---- test/java/org/apache +``` + +### 添加 zip.xml + +`zip.xml` 用于描述最终生成的 zip 压缩包中的文件内容。（如 .jar file, plugin.properties 等等） + +``` + +    plugin +     +        zip +     +     +    false +     +         +            target +             +                *.jar +             +            / +         + +         +            src/main/assembly +             +                plugin.properties +             +            / +         +     + +``` + +### 更新 pom.xml + +接下来我们需要更新子模块的 `pom.xml` 文件，添加 doris-fe 依赖： + +``` + + + + org.apache + doris-fe-plugins + 1.0-SNAPSHOT + + 4.0.0 + + auditloader + jar + + + + org.apache + doris-fe + + + + + ... + + + + + + auditloader + + + maven-assembly-plugin + 2.4.1 + + false + + src/main/assembly/zip.xml + + + + + make-assembly + package + + single + + + + + + + + +``` + +### 实现插件 + +之后我们就可以开始愉快的进行插件功能的开发啦。插件需要实现 `Plugin` 接口。具体可以参阅 Doris 自带的 `auditdemo` 插件示例代码。 + +### 编译 + +在编译插件之前，需要先执行 `sh build.sh --fe` 进行 Doris FE 代码的编译，并确保编译成功。 + +之后，执行 `sh build_plugin.sh` 编译所有插件。最终的产出会存放在 `fe_plugins/output` 目录中。 + +或者也可以执行 `sh build_plugin.sh --plugin your_plugin_name` 来仅编译指定的插件。 + +### 另一种开发方式 + +您可以直接通过修改自带的 `auditdemo` 插件示例代码进行开发。 + +## 部署 + +插件可以通过以下三种方式部署。 + +* 将 `.zip` 文件放在 Http 或 Https 服务器上。如：`http://xxx.xxxxxx.com/data/plugin.zip`, Doris 会下载这个文件。 +* 本地 `.zip` 文件。 如：`/home/work/data/plugin.zip`。需要在所有 FE 和 BE 节点部署。 +* 本地目录。如：`/home/work/data/plugin/`。这个相当于 `.zip` 文件解压后的目录。需要在所有 FE 和 BE 节点部署。 + +注意：需保证部署路径在整个插件生命周期内有效。 + +## 安装和卸载插件 + +通过如下命令安装和卸载插件。更多帮助请参阅 `HELP INSTALL PLUGIN;` `HELP IUNNSTALL PLUGIN;` `HELP SHOW PLUGINS;` + +``` +mysql> install plugin from "/home/users/seaven/auditdemo.zip"; +Query OK, 0 rows affected (0.09 sec) + +mysql> mysql> show plugins\G +*************************** 1. row *************************** + Name: auditloader + Type: AUDIT +Description: load audit log to olap load, and user can view the statistic of queries + Version: 0.12.0 +JavaVersion: 1.8.31 + ClassName: AuditLoaderPlugin + SoName: NULL + Sources: /home/cmy/git/doris/core/fe_plugins/output/auditloader.zip + Status: INSTALLED +*************************** 2. row *************************** + Name: AuditLogBuilder + Type: AUDIT +Description: builtin audit logger + Version: 0.12.0 +JavaVersion: 1.8.31 + ClassName: org.apache.doris.qe.AuditLogBuilder + SoName: NULL + Sources: Builtin + Status: INSTALLED +2 rows in set (0.00 sec) + +mysql> uninstall plugin auditloader; +Query OK, 0 rows affected (0.05 sec) + +mysql> show plugins; +Empty set (0.00 sec) +``` diff --git a/docs/zh-CN/extending-doris/user-defined-function.md b/docs/zh-CN/extending-doris/user-defined-function.md new file mode 100644 index 00000000000000..fdd2bcdfe61958 --- /dev/null +++ b/docs/zh-CN/extending-doris/user-defined-function.md @@ -0,0 +1,119 @@ +--- +{ + "title": "User Define Function", + "language": "zh-CN" +} +--- + + + +# User Define Function + +用户可以通过UDF机制来扩展Doris的能力。通过这篇文档，用户能够创建自己的UDF。 + +## 编写UDF函数 + +在使用UDF之前，用户需要先在Doris的UDF框架下，编写自己的UDF函数。在`be/src/udf_samples/udf_sample.h|cpp`文件中是一个简单的UDF Demo。 + +编写一个UDF函数需要以下几个步骤。 + +### 编写函数 + +创建对应的头文件、CPP文件，在CPP文件中实现你需要的逻辑。CPP文件中的实现函数格式与UDF的对应关系。 + +#### 非可变参数 + +对于非可变参数的UDF，那么两者之间的对应关系很直接。 +比如`INT MyADD(INT, INT)`的UDF就会对应`IntVal AddUdf(FunctionContext* context, const IntVal& arg1, const IntVal& arg2)`。 + +1. `AddUdf`可以为任意的名字，只要创建UDF的时候指定即可。 +2. 实现函数中的第一个参数永远是`FunctionContext*`。实现者可以通过这个结构体获得一些查询相关的内容，以及申请一些需要使用的内存。具体使用的接口可以参考`udf/udf.h`中的定义。 +3. 实现函数中从第二个参数开始需要与UDF的参数一一对应，比如`IntVal`对应`INT`类型。这部分的类型都要使用`const`引用。 +4. 返回参数与UDF的参数的类型要相对应。 + +#### 可变参数 + +对于可变参数，可以参见以下例子，UDF`String md5sum(String, ...)`对应的 +实现函数是`StringVal md5sumUdf(FunctionContext* ctx, int num_args, const StringVal* args)` + +1. `md5sumUdf`这个也是可以任意改变的，创建的时候指定即可。 +2. 第一个参数与非可变参数函数一样，传入的是一个`FunctionContext*`。 +3. 可变参数部分由两部分组成，首先会传入一个整数，说明后面还有几个参数。后面传入的是一个可变参数部分的数组。 + +#### 类型对应关系 + +|UDF Type|Argument Type| +|----|---------| +|TinyInt|TinyIntVal| +|SmallInt|SmallIntVal| +|Int|IntVal| +|BigInt|BigIntVal| +|LargeInt|LargeIntVal| +|Float|FloatVal| +|Double|DoubleVal| +|Date|DateTimeVal| +|Datetime|DateTimeVal| +|Char|StringVal| +|Varchar|StringVal| +|Decimal|DecimalVal| + +## 编译UDF函数 + +### 编译Doris + +在Doris根目录下执行`sh build.sh`就会在`output/udf/`生成对应`headers|libs` + +### 编写CMakeLists.txt + +基于上一步生成的`headers|libs`，用户可以使用`CMakeLists`等工具引入该依赖；在`CMakeLists`中，可以通过向`CMAKE_CXX_FLAGS`添加`-I|L`分别指定`headers|libs`路径；然后使用`add_library`添加动态库。例如，在`be/src/udf_samples/CMakeLists.txt`中，使用`add_library(udfsample SHARED udf_sample.cpp)` `target_link_libraries`(udfsample -static-libstdc++ -static-libgcc)增加了一个`udfsample`动态库。后面需要写上涉及的所有源文件（不包含头文件）。 + +### 执行编译 + +在该目录下创建一个`build`目录并在`build`下执行`cmake ../`生成`Makefile`，并执行`make`就会生成对应动态库。 + +## 创建UDF函数 + +通过上述的步骤后，你可以得到一个动态库。你需要将这个动态库放到一个能够通过HTTP协议访问到的位置。然后执行创建UDF函数在Doris系统内部创建一个UDF，你需要拥有AMDIN权限才能够完成这个操作。 + +``` +CREATE [AGGREGATE] FUNCTION + name ([argtype][,...]) + [RETURNS] rettype + PROPERTIES (["key"="value"][,...]) +``` +说明： + +1. PROPERTIES中`symbol`表示的是，执行入口函数的对应symbol，这个参数是必须设定。你可以通过`nm`命令来获得对应的symbol，比如`nm libudfsample.so | grep AddUdf`获得到的`_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_`就是对应的symbol。 +2. PROPERTIES中`object_file`表示的是从哪里能够下载到对应的动态库，这个参数是必须设定的。 +3. name: 一个function是要归属于某个DB的，name的形式为`dbName`.`funcName`。当`dbName`没有明确指定的时候，就是使用当前session所在的db作为`dbName`。 + +具体使用可以参见 `CREATE FUNCTION` 获取更详细信息。 + +## 使用UDF + +用户使用UDF/UDAF必须拥有对应数据库的 `SELECT` 权限。 + +UDF的使用与普通的函数方式一致，唯一的区别在于，内置函数的作用域是全局的，而UDF的作用域是DB内部。当链接session位于数据内部时，直接使用UDF名字会在当前DB内部查找对应的UDF。否则用户需要显示的指定UDF的数据库名字，例如`dbName`.`funcName`。 + + +## 删除UDF函数 + +当你不再需要UDF函数时，你可以通过下述命令来删除一个UDF函数, 可以参考 `DROP FUNCTION`。 + diff --git a/docs/zh-CN/getting-started/advance-usage.md b/docs/zh-CN/getting-started/advance-usage.md new file mode 100644 index 00000000000000..fc9f4104a94364 --- /dev/null +++ b/docs/zh-CN/getting-started/advance-usage.md @@ -0,0 +1,272 @@ +--- +{ + "title": "高级使用指南", + "language": "zh-CN" +} +--- + + + +# 高级使用指南 + +这里我们介绍 Doris 的一些高级特性。 + +## 1 表结构变更 + +使用 ALTER TABLE 命令可以修改表的 Schema，包括如下修改： + +* 增加列 +* 删除列 +* 修改列类型 +* 改变列顺序 + +以下举例说明。 + +原表 table1 的 Schema 如下: + +``` ++----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+-------------+------+-------+---------+-------+ +| siteid | int(11) | No | true | 10 | | +| citycode | smallint(6) | No | true | N/A | | +| username | varchar(32) | No | true | | | +| pv | bigint(20) | No | false | 0 | SUM | ++----------+-------------+------+-------+---------+-------+ +``` + +我们新增一列 uv，类型为 BIGINT，聚合类型为 SUM，默认值为 0: + +`ALTER TABLE table1 ADD COLUMN uv BIGINT SUM DEFAULT '0' after pv;` + +提交成功后，可以通过以下命令查看作业进度: + +`SHOW ALTER TABLE COLUMN;` + +当作业状态为 FINISHED，则表示作业完成。新的 Schema 已生效。 + +ALTER TABLE 完成之后, 可以通过 `DESC TABLE` 查看最新的 Schema。 + +``` +mysql> DESC table1; ++----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+-------------+------+-------+---------+-------+ +| siteid | int(11) | No | true | 10 | | +| citycode | smallint(6) | No | true | N/A | | +| username | varchar(32) | No | true | | | +| pv | bigint(20) | No | false | 0 | SUM | +| uv | bigint(20) | No | false | 0 | SUM | ++----------+-------------+------+-------+---------+-------+ +5 rows in set (0.00 sec) +``` + +可以使用以下命令取消当前正在执行的作业: + +`CANCEL ALTER TABLE COLUMN FROM table1` + +更多帮助，可以参阅 `HELP ALTER TABLE`。 + +## 2 Rollup + +Rollup 可以理解为 Table 的一个物化索引结构。**物化** 是因为其数据在物理上独立存储，而 **索引** 的意思是，Rollup可以调整列顺序以增加前缀索引的命中率，也可以减少key列以增加数据的聚合度。 + +以下举例说明。 + +原表table1的Schema如下: + +``` ++----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+-------------+------+-------+---------+-------+ +| siteid | int(11) | No | true | 10 | | +| citycode | smallint(6) | No | true | N/A | | +| username | varchar(32) | No | true | | | +| pv | bigint(20) | No | false | 0 | SUM | +| uv | bigint(20) | No | false | 0 | SUM | ++----------+-------------+------+-------+---------+-------+ +``` + +对于 table1 明细数据是 siteid, citycode, username 三者构成一组 key，从而对 pv 字段进行聚合；如果业务方经常有看城市 pv 总量的需求，可以建立一个只有 citycode, pv 的rollup。 + +`ALTER TABLE table1 ADD ROLLUP rollup_city(citycode, pv);` + +提交成功后，可以通过以下命令查看作业进度： + +`SHOW ALTER TABLE ROLLUP;` + +当作业状态为 FINISHED，则表示作业完成。 + +Rollup 建立完成之后可以使用 `DESC table1 ALL` 查看表的 Rollup 信息。 + +``` +mysql> desc table1 all; ++-------------+----------+-------------+------+-------+--------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++-------------+----------+-------------+------+-------+---------+-------+ +| table1 | siteid | int(11) | No | true | 10 | | +| | citycode | smallint(6) | No | true | N/A | | +| | username | varchar(32) | No | true | | | +| | pv | bigint(20) | No | false | 0 | SUM | +| | uv | bigint(20) | No | false | 0 | SUM | +| | | | | | | | +| rollup_city | citycode | smallint(6) | No | true | N/A | | +| | pv | bigint(20) | No | false | 0 | SUM | ++-------------+----------+-------------+------+-------+---------+-------+ +8 rows in set (0.01 sec) +``` + +可以使用以下命令取消当前正在执行的作业: + +`CANCEL ALTER TABLE ROLLUP FROM table1;` + +Rollup 建立之后，查询不需要指定 Rollup 进行查询。还是指定原有表进行查询即可。程序会自动判断是否应该使用 Rollup。是否命中 Rollup可以通过 `EXPLAIN your_sql;` 命令进行查看。 + +更多帮助，可以参阅 `HELP ALTER TABLE`。 + +## 2 数据表的查询 + +### 2.1 内存限制 + +为了防止用户的一个查询可能因为消耗内存过大。查询进行了内存控制，一个查询任务，在单个 BE 节点上默认使用不超过 2GB 内存。 + +用户在使用时，如果发现报 `Memory limit exceeded` 错误，一般是超过内存限制了。 + +遇到内存超限时，用户应该尽量通过优化自己的 sql 语句来解决。 + +如果确切发现2GB内存不能满足，可以手动设置内存参数。 + +显示查询内存限制: + +``` +mysql> SHOW VARIABLES LIKE "%mem_limit%"; ++---------------+------------+ +| Variable_name | Value | ++---------------+------------+ +| exec_mem_limit| 2147483648 | ++---------------+------------+ +1 row in set (0.00 sec) +``` + +`exec_mem_limit` 的单位是 byte，可以通过 `SET` 命令改变 `exec_mem_limit` 的值。如改为 8GB。 + +`SET exec_mem_limit = 8589934592;` + +``` +mysql> SHOW VARIABLES LIKE "%mem_limit%"; ++---------------+------------+ +| Variable_name | Value | ++---------------+------------+ +| exec_mem_limit| 8589934592 | ++---------------+------------+ +1 row in set (0.00 sec) +``` + +> * 以上该修改为 session 级别，仅在当前连接 session 内有效。断开重连则会变回默认值。 +> * 如果需要修改全局变量，可以这样设置：`SET GLOBAL exec_mem_limit = 8589934592;`。设置完成后，断开 session 重新登录，参数将永久生效。 + +### 2.2 查询超时 + +当前默认查询时间设置为最长为 300 秒，如果一个查询在 300 秒内没有完成，则查询会被 Doris 系统 cancel 掉。用户可以通过这个参数来定制自己应用的超时时间，实现类似 wait(timeout) 的阻塞方式。 + +查看当前超时设置: + +``` +mysql> SHOW VARIABLES LIKE "%query_timeout%"; ++---------------+-------+ +| Variable_name | Value | ++---------------+-------+ +| QUERY_TIMEOUT | 300 | ++---------------+-------+ +1 row in set (0.00 sec) +``` + +修改超时时间到1分钟: + +`SET query_timeout = 60;` + +> * 当前超时的检查间隔为 5 秒，所以小于 5 秒的超时不会太准确。 +> * 以上修改同样为 session 级别。可以通过 `SET GLOBAL` 修改全局有效。 + +### 2.3 Broadcast/Shuffle Join + +系统默认实现 Join 的方式，是将小表进行条件过滤后，将其广播到大表所在的各个节点上，形成一个内存 Hash 表，然后流式读出大表的数据进行Hash Join。但是如果当小表过滤后的数据量无法放入内存的话，此时 Join 将无法完成，通常的报错应该是首先造成内存超限。 + +如果遇到上述情况，建议使用 Shuffle Join 的方式，也被称作 Partitioned Join。即将小表和大表都按照 Join 的 key 进行 Hash，然后进行分布式的 Join。这个对内存的消耗就会分摊到集群的所有计算节点上。 + +使用 Broadcast Join（默认）: + +``` +mysql> select sum(table1.pv) from table1 join table2 where table1.siteid = 2; ++--------------------+ +| sum(`table1`.`pv`) | ++--------------------+ +| 10 | ++--------------------+ +1 row in set (0.20 sec) +``` + +使用 Broadcast Join（显式指定）: + +``` +mysql> select sum(table1.pv) from table1 join [broadcast] table2 where table1.siteid = 2; ++--------------------+ +| sum(`table1`.`pv`) | ++--------------------+ +| 10 | ++--------------------+ +1 row in set (0.20 sec) +``` + +使用 Shuffle Join: + +``` +mysql> select sum(table1.pv) from table1 join [shuffle] table2 where table1.siteid = 2; ++--------------------+ +| sum(`table1`.`pv`) | ++--------------------+ +| 10 | ++--------------------+ +1 row in set (0.15 sec) +``` + +### 2.4 查询重试和高可用 + +当部署多个 FE 节点时，用户可以在多个 FE 之上部署负载均衡层来实现 Doris 的高可用。 + +一下提供一些高可用的方案： + +**第一种** + +自己在应用层代码进行重试和负载均衡。比如发现一个连接挂掉，就自动在其他连接上进行重试。应用层代码重试需要应用自己配置多个doris前端节点地址。 + +**第二种** + +如果使用 mysql jdbc connector 来连接Doris，可以使用 jdbc 的自动重试机制: + +``` +jdbc:mysql://[host:port],[host:port].../[database][?propertyName1][=propertyValue1][&propertyName2][=propertyValue2]... +``` + +**第三种** + +应用可以连接到和应用部署到同一机器上的 MySQL Proxy，通过配置 MySQL Proxy 的 Failover 和 Load Balance 功能来达到目的。 + +`http://dev.mysql.com/doc/refman/5.6/en/mysql-proxy-using.html` diff --git a/docs/zh-CN/getting-started/basic-usage.md b/docs/zh-CN/getting-started/basic-usage.md new file mode 100644 index 00000000000000..c9ac3ea92d0fbb --- /dev/null +++ b/docs/zh-CN/getting-started/basic-usage.md @@ -0,0 +1,381 @@ +--- +{ + "title": "基础使用指南", + "language": "zh-CN" +} +--- + + + +# 基础使用指南 + +Doris 采用 MySQL 协议进行通信，用户可通过 MySQL client 或者 MySQL JDBC连接到 Doris 集群。选择 MySQL client 版本时建议采用5.1 之后的版本，因为 5.1 之前不能支持长度超过 16 个字符的用户名。本文以 MySQL client 为例，通过一个完整的流程向用户展示 Doris 的基本使用方法。 + +## 1 创建用户 + +### 1.1 Root 用户登录与密码修改 + +Doris 内置 root 和 admin 用户，密码默认都为空。启动完 Doris 程序之后，可以通过 root 或 admin 用户连接到 Doris 集群。 +使用下面命令即可登录 Doris： + +``` +mysql -h FE_HOST -P9030 -uroot +``` + +> `fe_host` 是任一 FE 节点的 ip 地址。`9030` 是 fe.conf 中的 query_port 配置。 + +登陆后，可以通过以下命令修改 root 密码 + +``` +SET PASSWORD FOR 'root' = PASSWORD('your_password'); +``` + +### 1.3 创建新用户 + +通过下面的命令创建一个普通用户。 + +``` +CREATE USER 'test' IDENTIFIED BY 'test_passwd'; +``` + +后续登录时就可以通过下列连接命令登录。 + +``` +mysql -h FE_HOST -P9030 -utest -ptest_passwd +``` + +> 新创建的普通用户默认没有任何权限。权限授予可以参考后面的权限授予。 + +## 2 数据表的创建与数据导入 + +### 2.1 创建数据库 + +初始可以通过 root 或 admin 用户创建数据库： + +`CREATE DATABASE example_db;` + +> 所有命令都可以使用 'HELP command;' 查看到详细的语法帮助。如：`HELP CREATE DATABASE;` + +> 如果不清楚命令的全名，可以使用 "help 命令某一字段" 进行模糊查询。如键入'HELP CREATE'，可以匹配到 `CREATE DATABASE`, `CREATE TABLE`, `CREATE USER` 等命令。 + +数据库创建完成之后，可以通过 `SHOW DATABASES;` 查看数据库信息。 + +``` +MySQL> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| example_db | +| information_schema | ++--------------------+ +2 rows in set (0.00 sec) +``` + +information_schema是为了兼容MySQL协议而存在，实际中信息可能不是很准确，所以关于具体数据库的信息建议通过直接查询相应数据库而获得。 + +### 2.2 账户授权 + +example_db 创建完成之后，可以通过 root/admin 账户将 example_db 读写权限授权给普通账户，如 test。授权之后采用 test 账户登录就可以操作 example_db 数据库了。 + +`GRANT ALL ON example_db TO test;` + +### 2.3 建表 + +使用 `CREATE TABLE` 命令建立一个表(Table)。更多详细参数可以查看: + +`HELP CREATE TABLE;` + +首先切换数据库: + +`USE example_db;` + +Doris支持支持单分区和复合分区两种建表方式。 + +在复合分区中： + +* 第一级称为 Partition，即分区。用户可以指定某一维度列作为分区列（当前只支持整型和时间类型的列），并指定每个分区的取值范围。 + +* 第二级称为 Distribution，即分桶。用户可以指定一个或多个维度列以及桶数对数据进行 HASH 分布。 + +以下场景推荐使用复合分区 + +* 有时间维度或类似带有有序值的维度，可以以这类维度列作为分区列。分区粒度可以根据导入频次、分区数据量等进行评估。 +* 历史数据删除需求：如有删除历史数据的需求（比如仅保留最近N 天的数据）。使用复合分区，可以通过删除历史分区来达到目的。也可以通过在指定分区内发送 DELETE 语句进行数据删除。 +* 解决数据倾斜问题：每个分区可以单独指定分桶数量。如按天分区，当每天的数据量差异很大时，可以通过指定分区的分桶数，合理划分不同分区的数据,分桶列建议选择区分度大的列。 + +用户也可以不使用复合分区，即使用单分区。则数据只做 HASH 分布。 + +下面以聚合模型为例，分别演示两种分区的建表语句。 + +#### 单分区 + +建立一个名字为 table1 的逻辑表。分桶列为 siteid，桶数为 10。 + +这个表的 schema 如下： + +* siteid：类型是INT（4字节）, 默认值为10 +* citycode：类型是SMALLINT（2字节） +* username：类型是VARCHAR, 最大长度为32, 默认值为空字符串 +* pv：类型是BIGINT（8字节）, 默认值是0; 这是一个指标列, Doris内部会对指标列做聚合操作, 这个列的聚合方法是求和（SUM） + +建表语句如下: +``` +CREATE TABLE table1 +( + siteid INT DEFAULT '10', + citycode SMALLINT, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' +) +AGGREGATE KEY(siteid, citycode, username) +DISTRIBUTED BY HASH(siteid) BUCKETS 10 +PROPERTIES("replication_num" = "1"); +``` + +#### 复合分区 + +建立一个名字为 table2 的逻辑表。 + +这个表的 schema 如下： + +* event_day：类型是DATE，无默认值 +* siteid：类型是INT（4字节）, 默认值为10 +* citycode：类型是SMALLINT（2字节） +* username：类型是VARCHAR, 最大长度为32, 默认值为空字符串 +* pv：类型是BIGINT（8字节）, 默认值是0; 这是一个指标列, Doris 内部会对指标列做聚合操作, 这个列的聚合方法是求和（SUM） + +我们使用 event_day 列作为分区列，建立3个分区: p201706, p201707, p201708 + +* p201706：范围为 [最小值, 2017-07-01) +* p201707：范围为 [2017-07-01, 2017-08-01) +* p201708：范围为 [2017-08-01, 2017-09-01) + +> 注意区间为左闭右开。 + +每个分区使用 siteid 进行哈希分桶，桶数为10 + +建表语句如下: +``` +CREATE TABLE table2 +( + event_day DATE, + siteid INT DEFAULT '10', + citycode SMALLINT, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' +) +AGGREGATE KEY(event_day, siteid, citycode, username) +PARTITION BY RANGE(event_day) +( + PARTITION p201706 VALUES LESS THAN ('2017-07-01'), + PARTITION p201707 VALUES LESS THAN ('2017-08-01'), + PARTITION p201708 VALUES LESS THAN ('2017-09-01') +) +DISTRIBUTED BY HASH(siteid) BUCKETS 10 +PROPERTIES("replication_num" = "1"); +``` + +表建完之后，可以查看 example_db 中表的信息: + +``` +MySQL> SHOW TABLES; ++----------------------+ +| Tables_in_example_db | ++----------------------+ +| table1 | +| table2 | ++----------------------+ +2 rows in set (0.01 sec) + +MySQL> DESC table1; ++----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+-------------+------+-------+---------+-------+ +| siteid | int(11) | Yes | true | 10 | | +| citycode | smallint(6) | Yes | true | N/A | | +| username | varchar(32) | Yes | true | | | +| pv | bigint(20) | Yes | false | 0 | SUM | ++----------+-------------+------+-------+---------+-------+ +4 rows in set (0.00 sec) + +MySQL> DESC table2; ++-----------+-------------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-----------+-------------+------+-------+---------+-------+ +| event_day | date | Yes | true | N/A | | +| siteid | int(11) | Yes | true | 10 | | +| citycode | smallint(6) | Yes | true | N/A | | +| username | varchar(32) | Yes | true | | | +| pv | bigint(20) | Yes | false | 0 | SUM | ++-----------+-------------+------+-------+---------+-------+ +5 rows in set (0.00 sec) +``` + +> 注意事项： +> +> 1. 上述表通过设置 replication_num 建的都是单副本的表，Doris建议用户采用默认的 3 副本设置，以保证高可用。 +> 2. 可以对复合分区表动态的增删分区。详见 `HELP ALTER TABLE` 中 Partition 相关部分。 +> 3. 数据导入可以导入指定的 Partition。详见 `HELP LOAD`。 +> 4. 可以动态修改表的 Schema。 +> 5. 可以对 Table 增加上卷表（Rollup）以提高查询性能，这部分可以参见高级使用指南关于 Rollup 的描述。 +> 6. 表的列的Null属性默认为true，会对查询性能有一定的影响。 + +### 2.4 导入数据 + +Doris 支持多种数据导入方式。具体可以参阅数据导入文档。这里我们使用流式导入和 Broker 导入做示例。 + +#### 流式导入 + +流式导入通过 HTTP 协议向 Doris 传输数据，可以不依赖其他系统或组件直接导入本地数据。详细语法帮助可以参阅 `HELP STREAM LOAD;`。 + +示例1：以 "table1_20170707" 为 Label，使用本地文件 table1_data 导入 table1 表。 + +``` +curl --location-trusted -u test:test -H "label:table1_20170707" -H "column_separator:," -T table1_data http://FE_HOST:8030/api/example_db/table1/_stream_load +``` + +> 1. FE_HOST 是任一 FE 所在节点 IP，8030 为 fe.conf 中的 http_port。 +> 2. 可以使用任一 BE 的 IP，以及 be.conf 中的 webserver_port 进行导入。如：`BE_HOST:8040` + +本地文件 `table1_data` 以 `,` 作为数据之间的分隔，具体内容如下： + +``` +1,1,jim,2 +2,1,grace,2 +3,2,tom,2 +4,3,bush,3 +5,3,helen,3 +``` + +示例2: 以 "table2_20170707" 为 Label，使用本地文件 table2_data 导入 table2 表。 + +``` +curl --location-trusted -u test:test -H "label:table2_20170707" -H "column_separator:|" -T table1_data http://127.0.0.1:8030/api/example_db/table2/_stream_load +``` + +本地文件 `table2_data` 以 `|` 作为数据之间的分隔，具体内容如下： + +``` +2017-07-03|1|1|jim|2 +2017-07-05|2|1|grace|2 +2017-07-12|3|2|tom|2 +2017-07-15|4|3|bush|3 +2017-07-12|5|3|helen|3 +``` + +> 注意事项： +> +> 1. 采用流式导入建议文件大小限制在 10GB 以内，过大的文件会导致失败重试代价变大。 +> 2. 每一批导入数据都需要取一个 Label，Label 最好是一个和一批数据有关的字符串，方便阅读和管理。Doris 基于 Label 保证在一个Database 内，同一批数据只可导入成功一次。失败任务的 Label 可以重用。 +> 3. 流式导入是同步命令。命令返回成功则表示数据已经导入，返回失败表示这批数据没有导入。 + +#### Broker 导入 + +Broker 导入通过部署的 Broker 进程，读取外部存储上的数据进行导入。更多帮助请参阅 `HELP BROKER LOAD;` + +示例：以 "table1_20170708" 为 Label，将 HDFS 上的文件导入 table1 表 + +``` +LOAD LABEL table1_20170708 +( + DATA INFILE("hdfs://your.namenode.host:port/dir/table1_data") + INTO TABLE table1 +) +WITH BROKER hdfs +( + "username"="hdfs_user", + "password"="hdfs_password" +) +PROPERTIES +( + "timeout"="3600", + "max_filter_ratio"="0.1" +); +``` + +Broker 导入是异步命令。以上命令执行成功只表示提交任务成功。导入是否成功需要通过 `SHOW LOAD;` 查看。如： + +`SHOW LOAD WHERE LABLE = "table1_20170708";` + +返回结果中，`State` 字段为 FINISHED 则表示导入成功。 + +关于 `SHOW LOAD` 的更多说明，可以参阅 `HELP SHOW LOAD;` + +异步的导入任务在结束前可以取消： + +`CANCEL LOAD WHERE LABEL = "table1_20170708";` + +## 3 数据的查询 + +### 3.1 简单查询 + +示例: + +``` +MySQL> SELECT * FROM table1 LIMIT 3; ++--------+----------+----------+------+ +| siteid | citycode | username | pv | ++--------+----------+----------+------+ +| 2 | 1 | 'grace' | 2 | +| 5 | 3 | 'helen' | 3 | +| 3 | 2 | 'tom' | 2 | ++--------+----------+----------+------+ +5 rows in set (0.01 sec) + +MySQL> SELECT * FROM table1 ORDER BY citycode; ++--------+----------+----------+------+ +| siteid | citycode | username | pv | ++--------+----------+----------+------+ +| 2 | 1 | 'grace' | 2 | +| 1 | 1 | 'jim' | 2 | +| 3 | 2 | 'tom' | 2 | +| 4 | 3 | 'bush' | 3 | +| 5 | 3 | 'helen' | 3 | ++--------+----------+----------+------+ +5 rows in set (0.01 sec) +``` + +### 3.3 Join 查询 + +示例: + +``` +MySQL> SELECT SUM(table1.pv) FROM table1 JOIN table2 WHERE table1.siteid = table2.siteid; ++--------------------+ +| sum(`table1`.`pv`) | ++--------------------+ +| 12 | ++--------------------+ +1 row in set (0.20 sec) +``` + +### 3.4 子查询 + +示例: + +``` +MySQL> SELECT SUM(pv) FROM table2 WHERE siteid IN (SELECT siteid FROM table1 WHERE siteid > 2); ++-----------+ +| sum(`pv`) | ++-----------+ +| 8 | ++-----------+ +1 row in set (0.13 sec) +``` diff --git a/docs/zh-CN/getting-started/best-practice.md b/docs/zh-CN/getting-started/best-practice.md new file mode 100644 index 00000000000000..c4e2a4fce7a295 --- /dev/null +++ b/docs/zh-CN/getting-started/best-practice.md @@ -0,0 +1,189 @@ +--- +{ + "title": "最佳实践", + "language": "zh-CN" +} +--- + + + +# 最佳实践 + +## 1 建表 + +### 1.1 数据模型选择 + +Doris 数据模型上目前分为三类: AGGREGATE KEY, UNIQUE KEY, DUPLICATE KEY。三种模型中数据都是按KEY进行排序。 + +1.1.1 AGGREGATE KEY + + AGGREGATE KEY相同时，新旧记录进行聚合，目前支持的聚合函数有SUM, MIN, MAX, REPLACE。 + + AGGREGATE KEY模型可以提前聚合数据, 适合报表和多维分析业务。 + + ``` + CREATE TABLE site_visit + ( + siteid INT, + city SMALLINT, + username VARCHAR(32), + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(siteid, city, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 10; + ``` + +1.1.2. UNIQUE KEY + + UNIQUE KEY 相同时，新记录覆盖旧记录。目前 UNIQUE KEY 实现上和 AGGREGATE KEY 的 REPLACE 聚合方法一样，二者本质上相同。适用于有更新需求的分析业务。 + + ``` + CREATE TABLE sales_order + ( + orderid BIGINT, + status TINYINT, + username VARCHAR(32), + amount BIGINT DEFAULT '0' + ) + UNIQUE KEY(orderid) + DISTRIBUTED BY HASH(orderid) BUCKETS 10; + ``` + +1.1.3. DUPLICATE KEY + + 只指定排序列，相同的行不会合并。适用于数据无需提前聚合的分析业务。 + + ``` + CREATE TABLE session_data + ( + visitorid SMALLINT, + sessionid BIGINT, + visittime DATETIME, + city CHAR(20), + province CHAR(20), + ip varchar(32), + brower CHAR(20), + url VARCHAR(1024) + ) + DUPLICATE KEY(visitorid, sessionid) + DISTRIBUTED BY HASH(sessionid, visitorid) BUCKETS 10; + ``` + +### 1.2 大宽表与 Star Schema + +业务方建表时, 为了和前端业务适配, 往往不对维度信息和指标信息加以区分, 而将 Schema 定义成大宽表。对于 Doris 而言, 这类大宽表往往性能不尽如人意: + +* Schema 中字段数比较多, 聚合模型中可能 key 列比较多, 导入过程中需要排序的列会增加。 +* 维度信息更新会反应到整张表中，而更新的频率直接影响查询的效率。 + +使用过程中，建议用户尽量使用 Star Schema 区分维度表和指标表。频繁更新的维度表也可以放在 MySQL 外部表中。而如果只有少量更新, 可以直接放在 Doris 中。在 Doris 中存储维度表时，可对维度表设置更多的副本，提升 Join 的性能。 + +### 1.3 分区和分桶 + +Doris 支持两级分区存储, 第一层为 RANGE 分区(partition), 第二层为 HASH 分桶(bucket)。 + +1.3.1. RANGE分区(partition) + + RANGE分区用于将数据划分成不同区间, 逻辑上可以理解为将原始表划分成了多个子表。业务上，多数用户会选择采用按时间进行partition, 让时间进行partition有以下好处： + + * 可区分冷热数据 + * 可用上Doris分级存储(SSD + SATA)的功能 + * 按分区删除数据时，更加迅速 + +1.3.2. HASH分桶(bucket) + + 根据hash值将数据划分成不同的 bucket。 + + * 建议采用区分度大的列做分桶, 避免出现数据倾斜 + * 为方便数据恢复, 建议单个 bucket 的 size 不要太大, 保持在 10GB 以内, 所以建表或增加 partition 时请合理考虑 bucket 数目, 其中不同 partition 可指定不同的 buckets 数。 + +### 1.4 稀疏索引和 Bloom Filter + +Doris对数据进行有序存储, 在数据有序的基础上为其建立稀疏索引,索引粒度为 block(1024行)。 + +稀疏索引选取 schema 中固定长度的前缀作为索引内容, 目前 Doris 选取 36 个字节的前缀作为索引。 + +* 建表时建议将查询中常见的过滤字段放在 Schema 的前面, 区分度越大，频次越高的查询字段越往前放。 +* 这其中有一个特殊的地方,就是 varchar 类型的字段。varchar 类型字段只能作为稀疏索引的最后一个字段。索引会在 varchar 处截断, 因此 varchar 如果出现在前面，可能索引的长度可能不足 36 个字节。具体可以参阅 [数据模型、ROLLUP 及前缀索引](./data-model-rollup.md)。 +* 除稀疏索引之外, Doris还提供bloomfilter索引, bloomfilter索引对区分度比较大的列过滤效果明显。 如果考虑到varchar不能放在稀疏索引中, 可以建立bloomfilter索引。 + +### 1.5 物化视图(rollup) + +Rollup 本质上可以理解为原始表(Base Table)的一个物化索引。建立 Rollup 时可只选取 Base Table 中的部分列作为 Schema。Schema 中的字段顺序也可与 Base Table 不同。 + +下列情形可以考虑建立 Rollup： + +1.5.1. Base Table 中数据聚合度不高。 + +这一般是因 Base Table 有区分度比较大的字段而导致。此时可以考虑选取部分列，建立 Rollup。 + +如对于 `site_visit` 表： + +``` +site_visit(siteid, city, username, pv) +``` + +siteid 可能导致数据聚合度不高，如果业务方经常根据城市统计pv需求，可以建立一个只有 city, pv 的 Rollup： + +``` +ALTER TABLE site_visit ADD ROLLUP rollup_city(city, pv); +``` + +1.5.2. Base Table 中的前缀索引无法命中 + +这一般是 Base Table 的建表方式无法覆盖所有的查询模式。此时可以考虑调整列顺序，建立 Rollup。 + +如对于 session_data 表： + +``` +session_data(visitorid, sessionid, visittime, city, province, ip, brower, url) +``` + +如果除了通过 visitorid 分析访问情况外，还有通过 brower, province 分析的情形，可以单独建立 Rollup。 + +``` +ALTER TABLE session_data ADD ROLLUP rollup_brower(brower,province,ip,url) DUPLICATE KEY(brower,province); +``` + +## 2 Schema Change + +Doris中目前进行 Schema Change 的方式有三种：Sorted Schema Change，Direct Schema Change, Linked Schema Change。 + +2.1. Sorted Schema Change + + 改变了列的排序方式，需对数据进行重新排序。例如删除排序列中的一列, 字段重排序。 + + ``` + ALTER TABLE site_visit DROP COLUMN city; + ``` + +2.2. Direct Schema Change: 无需重新排序，但是需要对数据做一次转换。例如修改列的类型，在稀疏索引中加一列等。 + + ``` + ALTER TABLE site_visit MODIFY COLUMN username varchar(64); + ``` + +2.3. Linked Schema Change: 无需转换数据，直接完成。例如加列操作。 + + ``` + ALTER TABLE site_visit ADD COLUMN click bigint SUM default '0'; + ``` + +建表时建议考虑好 Schema，这样在进行 Schema Change 时可以加快速度。 diff --git a/docs/zh-CN/getting-started/data-model-rollup.md b/docs/zh-CN/getting-started/data-model-rollup.md new file mode 100644 index 00000000000000..974d8975d57318 --- /dev/null +++ b/docs/zh-CN/getting-started/data-model-rollup.md @@ -0,0 +1,638 @@ +--- +{ + "title": "数据模型、ROLLUP 及前缀索引", + "language": "zh-CN" +} +--- + + + +# 数据模型、ROLLUP 及前缀索引 + +本文档主要从逻辑层面，描述 Doris 的数据模型、 ROLLUP 以及前缀索引的概念，以帮助用户更好的使用 Doris 应对不同的业务场景。 + +## 基本概念 + +在 Doris 中，数据以表（Table）的形式进行逻辑上的描述。 +一张表包括行（Row）和列（Column）。Row 即用户的一行数据。Column 用于描述一行数据中不同的字段。 + +Column 可以分为两大类：Key 和 Value。从业务角度看，Key 和 Value 可以分别对应维度列和指标列。 + +Doris 的数据模型主要分为3类: + +* Aggregate +* Uniq +* Duplicate + +下面我们分别介绍。 + +## Aggregate 模型 + +我们以实际的例子来说明什么是聚合模型，以及如何正确的使用聚合模型。 + +### 示例1：导入数据聚合 + +假设业务有如下数据表模式： + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +|user\_id|LARGEINT||用户id| +|date|DATE||数据灌入日期| +|city|VARCHAR(20)||用户所在城市| +|age|SMALLINT||用户年龄| +|sex|TINYINT||用户性别| +|last_visit_date|DATETIME|REPLACE|用户最后一次访问时间| +|cost|BIGINT|SUM|用户总消费| +|max\_dwell\_time|INT|MAX|用户最大停留时间| +|min\_dwell\_time|INT|MIN|用户最小停留时间| + +如果转换成建表语句则如下（省略建表语句中的 Partition 和 Distribution 信息） + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", +) +AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) +... /* 省略 Partition 和 Distribution 信息 */ +； +``` + +可以看到，这是一个典型的用户信息和访问行为的事实表。 +在一般星型模型中，用户信息和访问行为一般分别存放在维度表和事实表中。这里我们为了更加方便的解释 Doris 的数据模型，将两部分信息统一存放在一张表中。 + +表中的列按照是否设置了 `AggregationType`，分为 Key (维度列) 和 Value（指标列）。没有设置 `AggregationType` 的，如 `user_id`、`date`、`age` ... 等称为 **Key**，而设置了 `AggregationType` 的称为 **Value**。 + +当我们导入数据时，对于 Key 列相同的行和聚合成一行，而 Value 列会按照设置的 `AggregationType` 进行聚合。 `AggregationType` 目前有以下四种聚合方式： + +1. SUM：求和，多行的 Value 进行累加。 +2. REPLACE：替代，下一批数据中的 Value 会替换之前导入过的行中的 Value。 +3. MAX：保留最大值。 +4. MIN：保留最小值。 + +假设我们有以下导入数据（原始数据）： + +|user\_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +|10000|2017-10-01|北京|20|0|2017-10-01 06:00:00|20|10|10| +|10000|2017-10-01|北京|20|0|2017-10-01 07:00:00|15|2|2| +|10001|2017-10-01|北京|30|1|2017-10-01 17:05:45|2|22|22| +|10002|2017-10-02|上海|20|1|2017-10-02 12:59:12|200|5|5| +|10003|2017-10-02|广州|32|0|2017-10-02 11:20:00|30|11|11| +|10004|2017-10-01|深圳|35|0|2017-10-01 10:00:15|100|3|3| +|10004|2017-10-03|深圳|35|0|2017-10-03 10:20:22|11|6|6| + +我们假设这是一张记录用户访问某商品页面行为的表。我们以第一行数据为例，解释如下： + +|数据|说明| +|---|---| +|10000|用户id，每个用户唯一识别id| +|2017-10-01|数据入库时间，精确到日期| +|北京|用户所在城市| +|20|用户年龄| +|0|性别男（1 代表女性）| +|2017-10-01 06:00:00|用户本次访问该页面的时间，精确到秒| +|20|用户本次访问产生的消费| +|10|用户本次访问，驻留该页面的时间| +|10|用户本次访问，驻留该页面的时间（冗余）| + +那么当这批数据正确导入到 Doris 中后，Doris 中最终存储如下： + +|user\_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +|10000|2017-10-01|北京|20|0|2017-10-01 07:00:00|35|10|2| +|10001|2017-10-01|北京|30|1|2017-10-01 17:05:45|2|22|22| +|10002|2017-10-02|上海|20|1|2017-10-02 12:59:12|200|5|5| +|10003|2017-10-02|广州|32|0|2017-10-02 11:20:00|30|11|11| +|10004|2017-10-01|深圳|35|0|2017-10-01 10:00:15|100|3|3| +|10004|2017-10-03|深圳|35|0|2017-10-03 10:20:22|11|6|6| + +可以看到，用户 10000 只剩下了一行**聚合后**的数据。而其余用户的数据和原始数据保持一致。这里先解释下用户 10000 聚合后的数据： + +前5列没有变化，从第6列 `last_visit_date` 开始： + +* `2017-10-01 07:00:00`：因为 `last_visit_date` 列的聚合方式为 REPLACE，所以 `2017-10-01 07:00:00` 替换了 `2017-10-01 06:00:00` 保存了下来。 + > 注：在同一个导入批次中的数据，对于 REPLACE 这种聚合方式，替换顺序不做保证。如在这个例子中，最终保存下来的，也有可能是 `2017-10-01 06:00:00`。而对于不同导入批次中的数据，可以保证，后一批次的数据会替换前一批次。 + +* `35`：因为 `cost` 列的聚合类型为 SUM，所以由 20 + 15 累加获得 35。 +* `10`：因为 `max_dwell_time` 列的聚合类型为 MAX，所以 10 和 2 取最大值，获得 10。 +* `2`：因为 `min_dwell_time` 列的聚合类型为 MIN，所以 10 和 2 取最小值，获得 2。 + +经过聚合，Doris 中最终只会存储聚合后的数据。换句话说，即明细数据会丢失，用户不能够再查询到聚合前的明细数据了。 + +### 示例2：保留明细数据 + +接示例1，我们将表结构修改如下： + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +|user\_id|LARGEINT||用户id| +|date|DATE||数据灌入日期| +|timestamp|DATETIME||数据灌入时间，精确到秒| +|city|VARCHAR(20)||用户所在城市| +|age|SMALLINT||用户年龄| +|sex|TINYINT||用户性别| +|last\_visit\_date|DATETIME|REPLACE|用户最后一次访问时间| +|cost|BIGINT|SUM|用户总消费| +|max\_dwell\_time|INT|MAX|用户最大停留时间| +|min\_dwell\_time|INT|MIN|用户最小停留时间| + +即增加了一列 `timestamp`，记录精确到秒的数据灌入时间。 + +导入数据如下： + +|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---|---| +|10000|2017-10-01|2017-10-01 08:00:05|北京|20|0|2017-10-01 06:00:00|20|10|10| +|10000|2017-10-01|2017-10-01 09:00:05|北京|20|0|2017-10-01 07:00:00|15|2|2| +|10001|2017-10-01|2017-10-01 18:12:10|北京|30|1|2017-10-01 17:05:45|2|22|22| +|10002|2017-10-02|2017-10-02 13:10:00|上海|20|1|2017-10-02 12:59:12|200|5|5| +|10003|2017-10-02|2017-10-02 13:15:00|广州|32|0|2017-10-02 11:20:00|30|11|11| +|10004|2017-10-01|2017-10-01 12:12:48|深圳|35|0|2017-10-01 10:00:15|100|3|3| +|10004|2017-10-03|2017-10-03 12:38:20|深圳|35|0|2017-10-03 10:20:22|11|6|6| + +那么当这批数据正确导入到 Doris 中后，Doris 中最终存储如下： + +|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---|---| +|10000|2017-10-01|2017-10-01 08:00:05|北京|20|0|2017-10-01 06:00:00|20|10|10| +|10000|2017-10-01|2017-10-01 09:00:05|北京|20|0|2017-10-01 07:00:00|15|2|2| +|10001|2017-10-01|2017-10-01 18:12:10|北京|30|1|2017-10-01 17:05:45|2|22|22| +|10002|2017-10-02|2017-10-02 13:10:00|上海|20|1|2017-10-02 12:59:12|200|5|5| +|10003|2017-10-02|2017-10-02 13:15:00|广州|32|0|2017-10-02 11:20:00|30|11|11| +|10004|2017-10-01|2017-10-01 12:12:48|深圳|35|0|2017-10-01 10:00:15|100|3|3| +|10004|2017-10-03|2017-10-03 12:38:20|深圳|35|0|2017-10-03 10:20:22|11|6|6| + +我们可以看到，存储的数据，和导入数据完全一样，没有发生任何聚合。这是因为，这批数据中，因为加入了 `timestamp` 列，所有行的 Key 都**不完全相同**。也就是说，只要保证导入的数据中，每一行的 Key 都不完全相同，那么即使在聚合模型下，Doris 也可以保存完整的明细数据。 + +### 示例3：导入数据与已有数据聚合 + +接示例1。假设现在表中已有数据如下： + +|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +|10000|2017-10-01|北京|20|0|2017-10-01 07:00:00|35|10|2| +|10001|2017-10-01|北京|30|1|2017-10-01 17:05:45|2|22|22| +|10002|2017-10-02|上海|20|1|2017-10-02 12:59:12|200|5|5| +|10003|2017-10-02|广州|32|0|2017-10-02 11:20:00|30|11|11| +|10004|2017-10-01|深圳|35|0|2017-10-01 10:00:15|100|3|3| +|10004|2017-10-03|深圳|35|0|2017-10-03 10:20:22|11|6|6| + +我们再导入一批新的数据： + +|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +|10004|2017-10-03|深圳|35|0|2017-10-03 11:22:00|44|19|19| +|10005|2017-10-03|长沙|29|1|2017-10-03 18:11:02|3|1|1| + +那么当这批数据正确导入到 Doris 中后，Doris 中最终存储如下： + +|user_id|date|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---| +|10000|2017-10-01|北京|20|0|2017-10-01 07:00:00|35|10|2| +|10001|2017-10-01|北京|30|1|2017-10-01 17:05:45|2|22|22| +|10002|2017-10-02|上海|20|1|2017-10-02 12:59:12|200|5|5| +|10003|2017-10-02|广州|32|0|2017-10-02 11:20:00|30|11|11| +|10004|2017-10-01|深圳|35|0|2017-10-01 10:00:15|100|3|3| +|10004|2017-10-03|深圳|35|0|2017-10-03 11:22:00|55|19|6| +|10005|2017-10-03|长沙|29|1|2017-10-03 18:11:02|3|1|1| + +可以看到，用户 10004 的已有数据和新导入的数据发生了聚合。同时新增了 10005 用户的数据。 + +数据的聚合，在 Doris 中有如下三个阶段发生： + +1. 每一批次数据导入的 ETL 阶段。该阶段会在每一批次导入的数据内部进行聚合。 +2. 底层 BE 进行数据 Compaction 的阶段。该阶段，BE 会对已导入的不同批次的数据进行进一步的聚合。 +3. 数据查询阶段。在数据查询时，对于查询涉及到的数据，会进行对应的聚合。 + +数据在不同时间，可能聚合的程度不一致。比如一批数据刚导入时，可能还未与之前已存在的数据进行聚合。但是对于用户而言，用户**只能查询到**聚合后的数据。即不同的聚合程度对于用户查询而言是透明的。用户需始终认为数据以**最终的完成的聚合程度**存在，而**不应假设某些聚合还未发生**。（可参阅**聚合模型的局限性**一节获得更多详情。） + +## Uniq 模型 + +在某些多维分析场景下，用户更关注的是如何保证 Key 的唯一性，即如何获得 Primary Key 唯一性约束。因此，我们引入了 Uniq 的数据模型。该模型本质上是聚合模型的一个特例，也是一种简化的表结构表示方式。我们举例说明。 + +|ColumnName|Type|IsKey|Comment| +|---|---|---|---| +|user_id|BIGINT|Yes|用户id| +|username|VARCHAR(50)|Yes|用户昵称| +|city|VARCHAR(20)|No|用户所在城市| +|age|SMALLINT|No|用户年龄| +|sex|TINYINT|No|用户性别| +|phone|LARGEINT|No|用户电话| +|address|VARCHAR(500)|No|用户住址| +|register_time|DATETIME|No|用户注册时间| + +这是一个典型的用户基础信息表。这类数据没有聚合需求，只需保证主键唯一性。（这里的主键为 user_id + username）。那么我们的建表语句如下： + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `username` VARCHAR(50) NOT NULL COMMENT "用户昵称", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `phone` LARGEINT COMMENT "用户电话", + `address` VARCHAR(500) COMMENT "用户地址", + `register_time` DATETIME COMMENT "用户注册时间" +) +UNIQUE KEY(`user_id`, `user_name`) +... /* 省略 Partition 和 Distribution 信息 */ +； +``` + +而这个表结构，完全同等于以下使用聚合模型描述的表结构： + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +|user_id|BIGINT||用户id| +|username|VARCHAR(50)||用户昵称| +|city|VARCHAR(20)|REPLACE|用户所在城市| +|age|SMALLINT|REPLACE|用户年龄| +|sex|TINYINT|REPLACE|用户性别| +|phone|LARGEINT|REPLACE|用户电话| +|address|VARCHAR(500)|REPLACE|用户住址| +|register_time|DATETIME|REPLACE|用户注册时间| + +及建表语句： + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `username` VARCHAR(50) NOT NULL COMMENT "用户昵称", + `city` VARCHAR(20) REPLACE COMMENT "用户所在城市", + `age` SMALLINT REPLACE COMMENT "用户年龄", + `sex` TINYINT REPLACE COMMENT "用户性别", + `phone` LARGEINT REPLACE COMMENT "用户电话", + `address` VARCHAR(500) REPLACE COMMENT "用户地址", + `register_time` DATETIME REPLACE COMMENT "用户注册时间" +) +AGGREGATE KEY(`user_id`, `user_name`) +... /* 省略 Partition 和 Distribution 信息 */ +； +``` + +即 Uniq 模型完全可以用聚合模型中的 REPLACE 方式替代。其内部的实现方式和数据存储方式也完全一样。这里不再继续举例说明。 + +## Duplicate 模型 + +在某些多维分析场景下，数据既没有主键，也没有聚合需求。因此，我们引入 Duplicate 数据模型来满足这类需求。举例说明。 + +|ColumnName|Type|SortKey|Comment| +|---|---|---|---| +|timestamp|DATETIME|Yes|日志时间| +|type|INT|Yes|日志类型| +|error_code|INT|Yes|错误码| +|error_msg|VARCHAR(1024)|No|错误详细信息| +|op_id|BIGINT|No|负责人id| +|op_time|DATETIME|No|处理时间| + +建表语句如下： + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( + `timestamp` DATETIME NOT NULL COMMENT "日志时间", + `type` INT NOT NULL COMMENT "日志类型", + `error_code` INT COMMENT "错误码", + `error_msg` VARCHAR(1024) COMMENT "错误详细信息", + `op_id` BIGINT COMMENT "负责人id", + `op_time` DATETIME COMMENT "处理时间" +) +DUPLICATE KEY(`timestamp`, `type`) +... /* 省略 Partition 和 Distribution 信息 */ +； +``` + +这种数据模型区别于 Aggregate 和 Uniq 模型。数据完全按照导入文件中的数据进行存储，不会有任何聚合。即使两行数据完全相同，也都会保留。 +而在建表语句中指定的 DUPLICATE KEY，只是用来指明底层数据按照那些列进行排序。（更贴切的名称应该为 “Sorted Column”，这里取名 “DUPLICATE KEY” 只是用以明确表示所用的数据模型。关于 “Sorted Column”的更多解释，可以参阅**前缀索引**小节）。在 DUPLICATE KEY 的选择上，我们建议适当的选择前 2-4 列就可以。 + +这种数据模型适用于既没有聚合需求，又没有主键唯一性约束的原始数据的存储。更多使用场景，可参阅**聚合模型的局限性**小节。 + +## ROLLUP + +ROLLUP 在多维分析中是“上卷”的意思，即将数据按某种指定的粒度进行进一步聚合。 + +### 基本概念 + +在 Doris 中，我们将用户通过建表语句创建出来的表成为 Base 表（Base Table）。Base 表中保存着按用户建表语句指定的方式存储的基础数据。 + +在 Base 表之上，我们可以创建任意多个 ROLLUP 表。这些 ROLLUP 的数据是基于 Base 表产生的，并且在物理上是**独立存储**的。 + +ROLLUP 表的基本作用，在于在 Base 表的基础上，获得更粗粒度的聚合数据。 + +下面我们用示例详细说明在不同数据模型中的 ROLLUP 表及其作用。 + +#### Aggregate 和 Uniq 模型中的 ROLLUP + +因为 Uniq 只是 Aggregate 模型的一个特例，所以这里我们不加以区别。 + +1. 示例1：获得每个用户的总消费 + +接**Aggregate 模型**小节的**示例2**，Base 表结构如下： + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +|user_id|LARGEINT||用户id| +|date|DATE||数据灌入日期| +|timestamp|DATETIME||数据灌入时间，精确到秒| +|city|VARCHAR(20)||用户所在城市| +|age|SMALLINT||用户年龄| +|sex|TINYINT||用户性别| +|last_visit_date|DATETIME|REPLACE|用户最后一次访问时间| +|cost|BIGINT|SUM|用户总消费| +|max\_dwell\_time|INT|MAX|用户最大停留时间| +|min\_dwell\_time|INT|MIN|用户最小停留时间| + +存储的数据如下： + +|user_id|date|timestamp|city|age|sex|last\_visit\_date|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---|---|---|---|---|---| +|10000|2017-10-01|2017-10-01 08:00:05|北京|20|0|2017-10-01 06:00:00|20|10|10| +|10000|2017-10-01|2017-10-01 09:00:05|北京|20|0|2017-10-01 07:00:00|15|2|2| +|10001|2017-10-01|2017-10-01 18:12:10|北京|30|1|2017-10-01 17:05:45|2|22|22| +|10002|2017-10-02|2017-10-02 13:10:00|上海|20|1|2017-10-02 12:59:12|200|5|5| +|10003|2017-10-02|2017-10-02 13:15:00|广州|32|0|2017-10-02 11:20:00|30|11|11| +|10004|2017-10-01|2017-10-01 12:12:48|深圳|35|0|2017-10-01 10:00:15|100|3|3| +|10004|2017-10-03|2017-10-03 12:38:20|深圳|35|0|2017-10-03 10:20:22|11|6|6| + +在此基础上，我们创建一个 ROLLUP： + +|ColumnName| +|---| +|user_id| +|cost| + +该 ROLLUP 只包含两列：user_id 和 cost。则创建完成后，该 ROLLUP 中存储的数据如下： + +|user\_id|cost| +|---|---| +|10000|35| +|10001|2| +|10002|200| +|10003|30| +|10004|111| + +可以看到，ROLLUP 中仅保留了每个 user_id，在 cost 列上的 SUM 的结果。那么当我们进行如下查询时: + +`SELECT user_id, sum(cost) FROM table GROUP BY user_id;` + +Doris 会自动命中这个 ROLLUP 表，从而只需扫描极少的数据量，即可完成这次聚合查询。 + +2. 示例2：获得不同城市，不同年龄段用户的总消费、最长和最短页面驻留时间 + +紧接示例1。我们在 Base 表基础之上，再创建一个 ROLLUP： + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +|city|VARCHAR(20)||用户所在城市| +|age|SMALLINT||用户年龄| +|cost|BIGINT|SUM|用户总消费| +|max\_dwell\_time|INT|MAX|用户最大停留时间| +|min\_dwell\_time|INT|MIN|用户最小停留时间| + +则创建完成后，该 ROLLUP 中存储的数据如下： + +|city|age|cost|max\_dwell\_time|min\_dwell\_time| +|---|---|---|---|---| +|北京|20|0|30|10|2| +|北京|30|1|2|22|22| +|上海|20|1|200|5|5| +|广州|32|0|30|11|11| +|深圳|35|0|111|6|3| + +当我们进行如下这些查询时: + +* `SELECT city, age, sum(cost), max(max_dwell_time), min(min_dwell_time) FROM table GROUP BY city, age;` +* `SELECT city, sum(cost), max(max_dwell_time), min(min_dwell_time) FROM table GROUP BY city;` +* `SELECT city, age, sum(cost), min(min_dwell_time) FROM table GROUP BY city, age;` + +Doris 会自动命中这个 ROLLUP 表。 + +#### Duplicate 模型中的 ROLLUP + +因为 Duplicate 模型没有聚合的语意。所以该模型中的 ROLLUP，已经失去了“上卷”这一层含义。而仅仅是作为调整列顺序，以命中前缀索引的作用。我们将在接下来的小节中，详细介绍前缀索引，以及如何使用ROLLUP改变前缀索引，以获得更好的查询效率。 + +### 前缀索引与 ROLLUP + +#### 前缀索引 + +不同于传统的数据库设计，Doris 不支持在任意列上创建索引。Doris 这类 MPP 架构的 OLAP 数据库，通常都是通过提高并发，来处理大量数据的。 +本质上，Doris 的数据存储在类似 SSTable（Sorted String Table）的数据结构中。该结构是一种有序的数据结构，可以按照指定的列进行排序存储。在这种数据结构上，以排序列作为条件进行查找，会非常的高效。 + +在 Aggregate、Uniq 和 Duplicate 三种数据模型中。底层的数据存储，是按照各自建表语句中，AGGREGATE KEY、UNIQ KEY 和 DUPLICATE KEY 中指定的列进行排序存储的。 + +而前缀索引，即在排序的基础上，实现的一种根据给定前缀列，快速查询数据的索引方式。 + +我们将一行数据的前 **36 个字节** 作为这行数据的前缀索引。当遇到 VARCHAR 类型时，前缀索引会直接截断。我们举例说明： + +1. 以下表结构的前缀索引为 user_id(8Byte) + age(4Bytes) + message(prefix 24 Bytes)。 + +|ColumnName|Type| +|---|---| +|user_id|BIGINT| +|age|INT| +|message|VARCHAR(100)| +|max\_dwell\_time|DATETIME| +|min\_dwell\_time|DATETIME| + +2. 以下表结构的前缀索引为 user_name(20 Bytes)。即使没有达到 36 个字节，因为遇到 VARCHAR，所以直接截断，不再往后继续。 + +|ColumnName|Type| +|---|---| +|user_name|VARCHAR(20)| +|age|INT| +|message|VARCHAR(100)| +|max\_dwell\_time|DATETIME| +|min\_dwell\_time|DATETIME| + +当我们的查询条件，是**前缀索引的前缀**时，可以极大的加快查询速度。比如在第一个例子中，我们执行如下查询： + +`SELECT * FROM table WHERE user_id=1829239 and age=20；` + +该查询的效率会**远高于**如下查询： + +`SELECT * FROM table WHERE age=20；` + +所以在建表时，**正确的选择列顺序，能够极大地提高查询效率**。 + +#### ROLLUP 调整前缀索引 + +因为建表时已经指定了列顺序，所以一个表只有一种前缀索引。这对于使用其他不能命中前缀索引的列作为条件进行的查询来说，效率上可能无法满足需求。因此，我们可以通过创建 ROLLUP 来人为的调整列顺序。举例说明。 + +Base 表结构如下： + +|ColumnName|Type| +|---|---| +|user\_id|BIGINT| +|age|INT| +|message|VARCHAR(100)| +|max\_dwell\_time|DATETIME| +|min\_dwell\_time|DATETIME| + +我们可以在此基础上创建一个 ROLLUP 表： + +|ColumnName|Type| +|---|---| +|age|INT| +|user\_id|BIGINT| +|message|VARCHAR(100)| +|max\_dwell\_time|DATETIME| +|min\_dwell\_time|DATETIME| + +可以看到，ROLLUP 和 Base 表的列完全一样，只是将 user_id 和 age 的顺序调换了。那么当我们进行如下查询时： + +`SELECT * FROM table where age=20 and massage LIKE "%error%";` + +会优先选择 ROLLUP 表，因为 ROLLUP 的前缀索引匹配度更高。 + +### ROLLUP 的几点说明 + +* ROLLUP 最根本的作用是提高某些查询的查询效率（无论是通过聚合来减少数据量，还是修改列顺序以匹配前缀索引）。因此 ROLLUP 的含义已经超出了 “上卷” 的范围。这也是为什么我们在源代码中，将其命名为 Materized Index（物化索引）的原因。 +* ROLLUP 是附属于 Base 表的，可以看做是 Base 表的一种辅助数据结构。用户可以在 Base 表的基础上，创建或删除 ROLLUP，但是不能在查询中显式的指定查询某 ROLLUP。是否命中 ROLLUP 完全由 Doris 系统自动决定。 +* ROLLUP 的数据是独立物理存储的。因此，创建的 ROLLUP 越多，占用的磁盘空间也就越大。同时对导入速度也会有影响（导入的ETL阶段会自动产生所有 ROLLUP 的数据），但是不会降低查询效率（只会更好）。 +* ROLLUP 的数据更新与 Base 表示完全同步的。用户无需关心这个问题。 +* ROLLUP 中列的聚合方式，与 Base 表完全相同。在创建 ROLLUP 无需指定，也不能修改。 +* 查询能否命中 ROLLUP 的一个必要条件（非充分条件）是，查询所涉及的**所有列**（包括 select list 和 where 中的查询条件列等）都存在于该 ROLLUP 的列中。否则，查询只能命中 Base 表。 +* 某些类型的查询（如 count(*)）在任何条件下，都无法命中 ROLLUP。具体参见接下来的 **聚合模型的局限性** 一节。 +* 可以通过 `EXPLAIN your_sql;` 命令获得查询执行计划，在执行计划中，查看是否命中 ROLLUP。 +* 可以通过 `DESC tbl_name ALL;` 语句显示 Base 表和所有已创建完成的 ROLLUP。 + +在这篇文档中可以查看 [查询如何命中 Rollup](hit-the-rollup) + +## 聚合模型的局限性 + +这里我们针对 Aggregate 模型（包括 Uniq 模型），来介绍下聚合模型的局限性。 + +在聚合模型中，模型对外展现的，是**最终聚合后的**数据。也就是说，任何还未聚合的数据（比如说两个不同导入批次的数据），必须通过某种方式，以保证对外展示的一致性。我们举例说明。 + +假设表结构如下： + +|ColumnName|Type|AggregationType|Comment| +|---|---|---|---| +|user\_id|LARGEINT||用户id| +|date|DATE||数据灌入日期| +|cost|BIGINT|SUM|用户总消费| + +假设存储引擎中有如下两个已经导入完成的批次的数据： + +**batch 1** + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|50| +|10002|2017-11-21|39| + +**batch 2** + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|1| +|10001|2017-11-21|5| +|10003|2017-11-22|22| + +可以看到，用户 10001 分属在两个导入批次中的数据还没有聚合。但是为了保证用户只能查询到如下最终聚合后的数据： + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|51| +|10001|2017-11-21|5| +|10002|2017-11-21|39| +|10003|2017-11-22|22| + +我们在查询引擎中加入了聚合算子，来保证数据对外的一致性。 + +另外，在聚合列（Value）上，执行与聚合类型不一致的聚合类查询时，要注意语意。比如我们在如上示例中执行如下查询： + +`SELECT MIN(cost) FROM table;` + +得到的结果是 5，而不是 1。 + +同时，这种一致性保证，在某些查询中，会极大的降低查询效率。 + +我们以最基本的 count(*) 查询为例： + +`SELECT COUNT(*) FROM table;` + +在其他数据库中，这类查询都会很快的返回结果。因为在实现上，我们可以通过如“导入时对行进行计数，保存count的统计信息”，或者在查询时“仅扫描某一列数据，获得count值”的方式，只需很小的开销，即可获得查询结果。但是在 Doris 的聚合模型中，这种查询的开销**非常大**。 + +我们以刚才的数据为例： + +**batch 1** + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|50| +|10002|2017-11-21|39| + +**batch 2** + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|1| +|10001|2017-11-21|5| +|10003|2017-11-22|22| + +因为最终的聚合结果为： + +|user\_id|date|cost| +|---|---|---| +|10001|2017-11-20|51| +|10001|2017-11-21|5| +|10002|2017-11-21|39| +|10003|2017-11-22|22| + +所以，`select count(*) from table;` 的正确结果应该为 **4**。但如果我们只扫描 `user_id` 这一列，如果加上查询时聚合，最终得到的结果是 **3**（10001, 10002, 10003）。而如果不加查询时聚合，则得到的结果是 **5**（两批次一共5行数据）。可见这两个结果都是不对的。 + +为了得到正确的结果，我们必须同时读取 `user_id` 和 `date` 这两列的数据，**再加上查询时聚合**，才能返回 **4** 这个正确的结果。也就是说，在 count(\*) 查询中，Doris 必须扫描所有的 AGGREGATE KEY 列（这里就是 `user_id` 和 `date`），并且聚合后，才能得到语意正确的结果。当聚合列非常多时，count(\*) 查询需要扫描大量的数据。 + +因此，当业务上有频繁的 count(\*) 查询时，我们建议用户通过增加一个**值恒为 1 的，聚合类型为 SUM 的列来模拟 count(\*)**。如刚才的例子中的表结构，我们修改如下： + +|ColumnName|Type|AggreateType|Comment| +|---|---|---|---| +|user\_id|BIGINT||用户id| +|date|DATE||数据灌入日期| +|cost|BIGINT|SUM|用户总消费| +|count|BIGINT|SUM|用于计算count| + +增加一个 count 列，并且导入数据中，该列值**恒为 1**。则 `select count(*) from table;` 的结果等价于 `select sum(count) from table;`。而后者的查询效率将远高于前者。不过这种方式也有使用限制，就是用户需要自行保证，不会重复导入 AGGREGATE KEY 列都相同的行。否则，`select sum(count) from table;` 只能表述原始导入的行数，而不是 `select count(*) from table;` 的语义。 + +另一种方式，就是 **将如上的 `count` 列的聚合类型改为 REPLACE，且依然值恒为 1**。那么 `select sum(count) from table;` 和 `select count(*) from table;` 的结果将是一致的。并且这种方式，没有导入重复行的限制。 + +### Duplicate 模型 + +Duplicate 模型没有聚合模型的这个局限性。因为该模型不涉及聚合语意，在做 count(*) 查询时，任意选择一列查询，即可得到语意正确的结果。 + +## 数据模型的选择建议 + +因为数据模型在建表时就已经确定，且**无法修改**。所以，选择一个合适的数据模型**非常重要**。 + +1. Aggregate 模型可以通过预聚合，极大地降低聚合查询时所需扫描的数据量和查询的计算量，非常适合有固定模式的报表类查询场景。但是该模型对 count(*) 查询很不友好。同时因为固定了 Value 列上的聚合方式，在进行其他类型的聚合查询时，需要考虑语意正确性。 +2. Uniq 模型针对需要唯一主键约束的场景，可以保证主键唯一性约束。但是无法利用 ROLLUP 等预聚合带来的查询优势（因为本质是 REPLACE，没有 SUM 这种聚合方式）。 +3. Duplicate 适合任意维度的 Ad-hoc 查询。虽然同样无法利用预聚合的特性，但是不受聚合模型的约束，可以发挥列存模型的优势（只读取相关列，而不需要读取所有 Key 列）。 diff --git a/docs/zh-CN/getting-started/data-partition.md b/docs/zh-CN/getting-started/data-partition.md new file mode 100644 index 00000000000000..ef93749c143d19 --- /dev/null +++ b/docs/zh-CN/getting-started/data-partition.md @@ -0,0 +1,297 @@ +--- +{ + "title": "数据划分", + "language": "zh-CN" +} +--- + + + +# 数据划分 + +本文档主要介绍 Doris 的建表和数据划分，以及建表操作中可能遇到的问题和解决方法。 + +## 基本概念 + +在 Doris 中，数据都以表（Table）的形式进行逻辑上的描述。 + +### Row & Column + +一张表包括行（Row）和列（Column）。Row 即用户的一行数据。Column 用于描述一行数据中不同的字段。 + +Column 可以分为两大类：Key 和 Value。从业务角度看，Key 和 Value 可以分别对应维度列和指标列。从聚合模型的角度来说，Key 列相同的行，会聚合成一行。其中 Value 列的聚合方式由用户在建表时指定。关于更多聚合模型的介绍，可以参阅 [Doris 数据模型](./data-model-rollup.md)。 + +### Tablet & Partition + +在 Doris 的存储引擎中，用户数据被水平划分为若干个数据分片（Tablet，也称作数据分桶）。每个 Tablet 包含若干数据行。各个 Tablet 之间的数据没有交集，并且在物理上是独立存储的。 + +多个 Tablet 在逻辑上归属于不同的分区（Partition）。一个 Tablet 只属于一个 Partition。而一个 Partition 包含若干个 Tablet。因为 Tablet 在物理上是独立存储的，所以可以视为 Partition 在物理上也是独立。Tablet 是数据移动、复制等操作的最小物理存储单元。 + +若干个 Partition 组成一个 Table。Partition 可以视为是逻辑上最小的管理单元。数据的导入与删除，都可以或仅能针对一个 Partition 进行。 + +## 数据划分 + +我们以一个建表操作来说明 Doris 的数据划分。 + +Doris 的建表是一个同步命令，命令返回成功，即表示建表成功。 + +可以通过 `HELP CREATE TABLE;` 查看更多帮助。 + +本小节通过一个例子，来介绍 Doris 的建表方式。 + +``` +CREATE TABLE IF NOT EXISTS example_db.expamle_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" +) +ENGINE=olap +AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) +PARTITION BY RANGE(`date`) +( + PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), + PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), + PARTITION `p201703` VALUES LESS THAN ("2017-04-01") +) +DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 +PROPERTIES +( + "replication_num" = "3", + "storage_medium" = "SSD", + "storage_cooldown_time" = "2018-01-01 12:00:00" +); + +``` + +### 列定义 + +这里我们只以 AGGREGATE KEY 数据模型为例进行说明。更多数据模型参阅 [Doris 数据模型](./data-model-rollup.md)。 + +列的基本类型，可以通过在 mysql-client 中执行 `HELP CREATE TABLE;` 查看。 + +AGGREGATE KEY 数据模型中，所有没有指定聚合方式（SUM、REPLACE、MAX、MIN）的列视为 Key 列。而其余则为 Value 列。 + +定义列时，可参照如下建议： + +1. Key 列必须在所有 Value 列之前。 +2. 尽量选择整型类型。因为整型类型的计算和查找比较效率远高于字符串。 +3. 对于不同长度的整型类型的选择原则，遵循 **够用即可**。 +4. 对于 VARCHAR 和 STRING 类型的长度，遵循 **够用即可**。 +5. 所有列的总字节长度（包括 Key 和 Value）不能超过 100KB。 + +### 分区与分桶 + +Doris 支持两层的数据划分。第一层是 Partition，仅支持 Range 的划分方式。第二层是 Bucket（Tablet），仅支持 Hash 的划分方式。 + +也可以仅使用一层分区。使用一层分区时，只支持 Bucket 划分。 + +1. Partition + + * Partition 列可以指定一列或多列。分区类必须为 KEY 列。多列分区的使用方式在后面 **多列分区** 小结介绍。 + * 不论分区列是什么类型，在写分区值时，都需要加双引号。 + * 分区列通常为时间列，以方便的管理新旧数据。 + * 分区数量理论上没有上限。 + * 当不使用 Partition 建表时，系统会自动生成一个和表名同名的，全值范围的 Partition。该 Partition 对用户不可见，并且不可删改。 + * Partition 支持通过 `VALUES LESS THAN (...)` 仅指定上界，系统会将前一个分区的上界作为该分区的下界，生成一个左闭右开的区间。通过，也支持通过 `VALUES [...)` 指定同时指定上下界，生成一个左闭右开的区间。 + + * 通过 `VALUES [...)` 同时指定上下界比较容易理解。这里举例说明，当使用 `VALUES LESS THAN (...)` 语句进行分区的增删操作时，分区范围的变化情况： + + * 如上示例，当建表完成后，会自动生成如下3个分区： + + ``` + p201701: [MIN_VALUE, 2017-02-01) + p201702: [2017-02-01, 2017-03-01) + p201703: [2017-03-01, 2017-04-01) + ``` + + * 当我们增加一个分区 p201705 VALUES LESS THAN ("2017-06-01")，分区结果如下： + + ``` + p201701: [MIN_VALUE, 2017-02-01) + p201702: [2017-02-01, 2017-03-01) + p201703: [2017-03-01, 2017-04-01) + p201705: [2017-04-01, 2017-06-01) + ``` + + * 此时我们删除分区 p201703，则分区结果如下： + + ``` + p201701: [MIN_VALUE, 2017-02-01) + p201702: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) + ``` + + > 注意到 p201702 和 p201705 的分区范围并没有发生变化，而这两个分区之间，出现了一个空洞：[2017-03-01, 2017-04-01)。即如果导入的数据范围在这个空洞范围内，是无法导入的。 + + * 继续删除分区 p201702，分区结果如下： + + ``` + p201701: [MIN_VALUE, 2017-02-01) + p201705: [2017-04-01, 2017-06-01) + 空洞范围变为：[2017-02-01, 2017-04-01) + ``` + + * 现在增加一个分区 p201702new VALUES LESS THAN ("2017-03-01")，分区结果如下： + + ``` + p201701: [MIN_VALUE, 2017-02-01) + p201702new: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) + ``` + + > 可以看到空洞范围缩小为：[2017-03-01, 2017-04-01) + + * 现在删除分区 p201701，并添加分区 p201612 VALUES LESS THAN ("2017-01-01")，分区结果如下： + + ``` + p201612: [MIN_VALUE, 2017-01-01) + p201702new: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) + ``` + + > 即出现了一个新的空洞：[2017-01-01, 2017-02-01) + + 综上，分区的删除不会改变已存在分区的范围。删除分区可能出现空洞。通过 `VALUES LESS THAN` 语句增加分区时，分区的下界紧接上一个分区的上界。 + + 不可添加范围重叠的分区。 + +2. Bucket + + * 如果使用了 Partition，则 `DISTRIBUTED ...` 语句描述的是数据在**各个分区内**的划分规则。如果不使用 Partition，则描述的是对整个表的数据的划分规则。 + * 分桶列可以是多列，但必须为 Key 列。分桶列可以和 Partition 列相同或不同。 + * 分桶列的选择，是在 **查询吞吐** 和 **查询并发** 之间的一种权衡： + + 1. 如果选择多个分桶列，则数据分布更均匀。但如果查询条件不包含所有分桶列的等值条件的话，一个查询会扫描所有分桶。这样查询的吞吐会增加，但是单个查询的延迟也会增加。这个方式适合大吞吐低并发的查询场景。 + 2. 如果仅选择一个或少数分桶列，则点查询可以仅查询一个分桶。这种方式适合高并发的点查询场景。 + + * 分桶的数量理论上没有上限。 + +3. 关于 Partition 和 Bucket 的数量和数据量的建议。 + + * 一个表的 Tablet 总数量等于 (Partition num * Bucket num)。 + * 一个表的 Tablet 数量，在不考虑扩容的情况下，推荐略多于整个集群的磁盘数量。 + * 单个 Tablet 的数据量理论上没有上下界，但建议在 1G - 10G 的范围内。如果单个 Tablet 数据量过小，则数据的聚合效果不佳，且元数据管理压力大。如果数据量过大，则不利于副本的迁移、补齐，且会增加 Schema Change 或者 Rollup 操作失败重试的代价（这些操作失败重试的粒度是 Tablet）。 + * 当 Tablet 的数据量原则和数量原则冲突时，建议优先考虑数据量原则。 + * 在建表时，每个分区的 Bucket 数量统一指定。但是在动态增加分区时（`ADD PARTITION`），可以单独指定新分区的 Bucket 数量。可以利用这个功能方便的应对数据缩小或膨胀。 + * 一个 Partition 的 Bucket 数量一旦指定，不可更改。所以在确定 Bucket 数量时，需要预先考虑集群扩容的情况。比如当前只有 3 台 host，每台 host 有 1 块盘。如果 Bucket 的数量只设置为 3 或更小，那么后期即使再增加机器，也不能提高并发度。 + * 举一些例子：假设在有10台BE，每台BE一块磁盘的情况下。如果一个表总大小为 500MB，则可以考虑4-8个分片。5GB：8-16个。50GB：32个。500GB：建议分区，每个分区大小在 50GB 左右，每个分区16-32个分片。5TB：建议分区，每个分区大小在 50GB 左右，每个分区16-32个分片。 + + > 注：表的数据量可以通过 `show data` 命令查看，结果除以副本数，即表的数据量。 + +#### 多列分区 + +Doris 支持指定多列作为分区列，示例如下： + +``` +PARTITION BY RANGE(`date`, `id`) +( + PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), + PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), + PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") +) +``` + +在以上示例中，我们指定 `date`(DATE 类型) 和 `id`(INT 类型) 作为分区列。以上示例最终得到的分区如下： + +``` +* p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) +* p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) +* p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) +``` + +注意，最后一个分区用户缺省只指定了 `date` 列的分区值，所以 `id` 列的分区值会默认填充 `MIN_VALUE`。当用户插入数据时，分区列值会按照顺序依次比较，最终得到对应的分区。举例如下： + +``` +* 数据 --> 分区 +* 2017-01-01, 200 --> p201701_1000 +* 2017-01-01, 2000 --> p201701_1000 +* 2017-02-01, 100 --> p201701_1000 +* 2017-02-01, 2000 --> p201702_2000 +* 2017-02-15, 5000 --> p201702_2000 +* 2017-03-01, 2000 --> p201703_all +* 2017-03-10, 1 --> p201703_all +* 2017-04-01, 1000 --> 无法导入 +* 2017-05-01, 1000 --> 无法导入 +``` + +### PROPERTIES + +在建表语句的最后 PROPERTIES 中，可以指定以下两个参数： + +1. replication_num + + * 每个 Tablet 的副本数量。默认为3，建议保持默认即可。在建表语句中，所有 Partition 中的 Tablet 副本数量统一指定。而在增加新分区时，可以单独指定新分区中 Tablet 的副本数量。 + * 副本数量可以在运行时修改。强烈建议保持奇数。 + * 最大副本数量取决于集群中独立 IP 的数量（注意不是 BE 数量）。Doris 中副本分布的原则是，不允许同一个 Tablet 的副本分布在同一台物理机上，而识别物理机即通过 IP。所以，即使在同一台物理机上部署了 3 个或更多 BE 实例，如果这些 BE 的 IP 相同，则依然只能设置副本数为 1。 + * 对于一些小，并且更新不频繁的维度表，可以考虑设置更多的副本数。这样在 Join 查询时，可以有更大的概率进行本地数据 Join。 + +2. storage_medium & storage\_cooldown\_time + + * BE 的数据存储目录可以显式的指定为 SSD 或者 HDD（通过 .SSD 或者 .HDD 后缀区分）。建表时，可以统一指定所有 Partition 初始存储的介质。注意，后缀作用是显式指定磁盘介质，而不会检查是否与实际介质类型相符。 + * 默认初始存储介质可通过fe的配置文件 `fe.conf` 中指定 `default_storage_medium=xxx`，如果没有指定，则默认为 HDD。如果指定为 SSD，则数据初始存放在 SSD 上。 + * 如果没有指定 storage\_cooldown\_time，则默认 30 天后，数据会从 SSD 自动迁移到 HDD 上。如果指定了 storage\_cooldown\_time，则在到达 storage_cooldown_time 时间后，数据才会迁移。 + * 注意，当指定 storage_medium 时，该参数只是一个“尽力而为”的设置。即使集群内没有设置 SSD 存储介质，也不会报错，而是自动存储在可用的数据目录中。同样，如果 SSD 介质不可访问、空间不足，都可能导致数据初始直接存储在其他可用介质上。而数据到期迁移到 HDD 时，如果 HDD 介质不可访问、空间不足，也可能迁移失败（但是会不断尝试）。 + +### ENGINE + +本示例中，ENGINE 的类型是 olap，即默认的 ENGINE 类型。在 Doris 中，只有这个 ENGINE 类型是由 Doris 负责数据管理和存储的。其他 ENGINE 类型，如 mysql、broker、es 等等，本质上只是对外部其他数据库或系统中的表的映射，以保证 Doris 可以读取这些数据。而 Doris 本身并不创建、管理和存储任何非 olap ENGINE 类型的表和数据。 + +### 其他 + + `IF NOT EXISTS` 表示如果没有创建过该表，则创建。注意这里只判断表名是否存在，而不会判断新建表结构是否与已存在的表结构相同。所以如果存在一个同名但不同构的表，该命令也会返回成功，但并不代表已经创建了新的表和新的结构。 + +## 常见问题 + +### 建表操作常见问题 + +1. 如果在较长的建表语句中出现语法错误，可能会出现语法错误提示不全的现象。这里罗列可能的语法错误供手动纠错： + + * 语法结构错误。请仔细阅读 `HELP CREATE TABLE;`，检查相关语法结构。 + * 保留字。当用户自定义名称遇到保留字时，需要用反引号 `` 引起来。建议所有自定义名称使用这个符号引起来。 + * 中文字符或全角字符。非 utf8 编码的中文字符，或隐藏的全角字符（空格，标点等）会导致语法错误。建议使用带有显示不可见字符的文本编辑器进行检查。 + +2. `Failed to create partition [xxx] . Timeout` + + Doris 建表是按照 Partition 粒度依次创建的。当一个 Partition 创建失败时，可能会报这个错误。即使不使用 Partition，当建表出现问题时，也会报 `Failed to create partition`，因为如前文所述，Doris 会为没有指定 Partition 的表创建一个不可更改的默认的 Partition。 + + 当遇到这个错误是，通常是 BE 在创建数据分片时遇到了问题。可以参照以下步骤排查： + + 1. 在 fe.log 中，查找对应时间点的 `Failed to create partition` 日志。在该日志中，会出现一系列类似 `{10001-10010}` 字样的数字对。数字对的第一个数字表示 Backend ID，第二个数字表示 Tablet ID。如上这个数字对，表示 ID 为 10001 的 Backend 上，创建 ID 为 10010 的 Tablet 失败了。 + 2. 前往对应 Backend 的 be.INFO 日志，查找对应时间段内，tablet id 相关的日志，可以找到错误信息。 + 3. 以下罗列一些常见的 tablet 创建失败错误，包括但不限于： + * BE 没有收到相关 task，此时无法在 be.INFO 中找到 tablet id 相关日志。或者 BE 创建成功，但汇报失败。以上问题，请参阅 [部署与升级文档] 检查 FE 和 BE 的连通性。 + * 预分配内存失败。可能是表中一行的字节长度超过了 100KB。 + * `Too many open files`。打开的文件句柄数超过了 Linux 系统限制。需修改 Linux 系统的句柄数限制。 + + 如果创建数据分片时超时，也可以通过在 fe.conf 中设置 `tablet_create_timeout_second=xxx` 以及 `max_create_table_timeout_second=xxx` 来延长超时时间。其中 `tablet_create_timeout_second` 默认是1秒, `max_create_table_timeout_second` 默认是60秒，总体的超时时间为min(tablet_create_timeout_second * replication_num, max_create_table_timeout_second); + +3. 建表命令长时间不返回结果。 + + Doris 的建表命令是同步命令。该命令的超时时间目前设置的比较简单，即（tablet num * replication num）秒。如果创建较多的数据分片，并且其中有分片创建失败，则可能导致等待较长超时后，才会返回错误。 + + 正常情况下，建表语句会在几秒或十几秒内返回。如果超过一分钟，建议直接取消掉这个操作，前往 FE 或 BE 的日志查看相关错误。 diff --git a/docs/zh-CN/getting-started/hit-the-rollup.md b/docs/zh-CN/getting-started/hit-the-rollup.md new file mode 100644 index 00000000000000..6f3dd6eca63def --- /dev/null +++ b/docs/zh-CN/getting-started/hit-the-rollup.md @@ -0,0 +1,294 @@ +--- +{ + "title": "Rollup 与查询", + "language": "zh-CN" +} +--- + + + +# Rollup 与查询 + +在 Doris 里 Rollup 作为一份聚合物化视图，其在查询中可以起到两个作用： + +* 索引 +* 聚合数据（仅用于聚合模型，即aggregate key） + +但是为了命中 Rollup 需要满足一定的条件，并且可以通过执行计划中 ScanNdoe 节点的 PreAggregation 的值来判断是否可以命中 Rollup，以及 Rollup 字段来判断命中的是哪一张 Rollup 表。 + +## 名词解释 + +Base：基表。 + +Rollup：一般指基于 Base 表创建的 Rollup 表，但在一些场景包括 Base 以及 Rollup 表。 + +## 索引 + +前面的查询实践中已经介绍过 Doris 的前缀索引，即 Doris 会把 Base/Rollup 表中的前 36 个字节（有 varchar 类型则可能导致前缀索引不满 36 个字节，varchar 会截断前缀索引，并且最多使用 varchar 的 20 个字节）在底层存储引擎单独生成一份排序的稀疏索引数据(数据也是排序的，用索引定位，然后在数据中做二分查找)，然后在查询的时候会根据查询中的条件来匹配每个 Base/Rollup 的前缀索引，并且选择出匹配前缀索引最长的一个 Base/Rollup。 + +``` + -----> 从左到右匹配 ++----+----+----+----+----+----+ +| c1 | c2 | c3 | c4 | c5 |... | +``` + +如上图，取查询中 where 以及 on 上下推到 ScanNode 的条件，从前缀索引的第一列开始匹配，检查条件中是否有这些列，有则累计匹配的长度，直到匹配不上或者36字节结束（varchar类型的列只能匹配20个字节，并且会匹配不足36个字节截断前缀索引），然后选择出匹配长度最长的一个 Base/Rollup，下面举例说明，创建了一张Base表以及四张rollup： + +``` ++---------------+-------+--------------+------+-------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++---------------+-------+--------------+------+-------+---------+-------+ +| test | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup_index1 | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup_index2 | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k1 | TINYINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup_index3 | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup_index4 | k4 | BIGINT | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | ++---------------+-------+--------------+------+-------+---------+-------+ +``` + +三张表的前缀索引分别为 + +``` +Base(k1 ,k2, k3, k4, k5, k6, k7) + +rollup_index1(k9)，rollup_index2(k9) + +rollup_index3(k4, k5, k6, k1, k2, k3, k7) + +rollup_index4(k4, k6, k5, k1, k2, k3, k7) +``` + +能用的上前缀索引的列上的条件需要是 `=` `<` `>` `<=` `>=` `in` `between` 这些并且这些条件是并列的且关系使用 `and` 连接，对于`or`、`!=` 等这些不能命中，然后看以下查询： + + +`SELECT * FROM test WHERE k1 = 1 AND k2 > 3;` + + +有 k1 以及 k2 上的条件，检查只有 Base 的第一列含有条件里的 k1，所以匹配最长的前缀索引即 test，explain一下： + +``` +| 0:OlapScanNode +| TABLE: test +| PREAGGREGATION: OFF. Reason: No AggregateInfo +| PREDICATES: `k1` = 1, `k2` > 3 +| partitions=1/1 +| rollup: test +| buckets=1/10 +| cardinality=-1 +| avgRowSize=0.0 +| numNodes=0 +| tuple ids: 0 +``` + +再看以下查询： + +`SELECT * FROM test WHERE k4 = 1 AND k5 > 3;` + +有 k4 以及 k5 的条件，检查 rollup_index3、rollup_index4 的第一列含有 k4，但是 rollup_index3 的第二列含有k5，所以匹配的前缀索引最长。 + +``` +| 0:OlapScanNode +| TABLE: test +| PREAGGREGATION: OFF. Reason: No AggregateInfo +| PREDICATES: `k4` = 1, `k5` > 3 +| partitions=1/1 +| rollup: rollup_index3 +| buckets=10/10 +| cardinality=-1 +| avgRowSize=0.0 +| numNodes=0 +| tuple ids: 0 +``` + +现在我们尝试匹配含有 varchar 列上的条件，如下： + +`SELECT * FROM test WHERE k9 IN ("xxx", "yyyy") AND k1 = 10;` + +有 k9 以及 k1 两个条件，rollup_index1 以及 rollup_index2 的第一列都含有 k9，按理说这里选择这两个 rollup 都可以命中前缀索引并且效果是一样的随机选择一个即可（因为这里 varchar 刚好20个字节，前缀索引不足36个字节被截断），但是当前策略这里还会继续匹配 k1，因为 rollup_index1 的第二列为 k1，所以选择了 rollup_index1，其实后面的 k1 条件并不会起到加速的作用。(如果对于前缀索引外的条件需要其可以起到加速查询的目的，可以通过建立 Bloom Filter 过滤器加速。一般对于字符串类型建立即可，因为 Doris 针对列存在 Block 级别对于整形、日期已经有 Min/Max 索引) 以下是 explain 的结果。 + +``` +| 0:OlapScanNode +| TABLE: test +| PREAGGREGATION: OFF. Reason: No AggregateInfo +| PREDICATES: `k9` IN ('xxx', 'yyyy'), `k1` = 10 +| partitions=1/1 +| rollup: rollup_index1 +| buckets=1/10 +| cardinality=-1 +| avgRowSize=0.0 +| numNodes=0 +| tuple ids: 0 +``` + +最后看一个多张Rollup都可以命中的查询： + +`SELECT * FROM test WHERE k4 < 1000 AND k5 = 80 AND k6 >= 10000;` + +有 k4,k5,k6 三个条件，rollup_index3 以及 rollup_index4 的前3列分别含有这三列，所以两者匹配的前缀索引长度一致，选取两者都可以，当前默认的策略为选取了比较早创建的一张 rollup，这里为 rollup_index3。 + +``` +| 0:OlapScanNode +| TABLE: test +| PREAGGREGATION: OFF. Reason: No AggregateInfo +| PREDICATES: `k4` < 1000, `k5` = 80, `k6` >= 10000.0 +| partitions=1/1 +| rollup: rollup_index3 +| buckets=10/10 +| cardinality=-1 +| avgRowSize=0.0 +| numNodes=0 +| tuple ids: 0 +``` + +如果稍微修改上面的查询为： + +`SELECT * FROM test WHERE k4 < 1000 AND k5 = 80 OR k6 >= 10000;` + +则这里的查询不能命中前缀索引。（甚至 Doris 存储引擎内的任何 Min/Max,BloomFilter 索引都不能起作用) + +## 聚合数据 + +当然一般的聚合物化视图其聚合数据的功能是必不可少的，这类物化视图对于聚合类查询或报表类查询都有非常大的帮助，要命中聚合物化视图需要下面一些前提： + +1. 查询或者子查询中涉及的所有列都存在一张独立的 Rollup 中。 +2. 如果查询或者子查询中有 Join，则 Join 的类型需要是 Inner join。 + +以下是可以命中Rollup的一些聚合查询的种类， + +| 列类型 查询类型 | Sum | Distinct/Count Distinct | Min | Max | Ndv | +|--------------|-------|-------------------------|-------|-------|-------| +| Key | false | true | true | true | true | +| Value(Sum) | true | false | false | false | false | +|Value(Replace)| false | false | false | false | false | +| Value(Min) | false | false | true | false | false | +| Value(Max) | false | false | false | true | false | + +如果符合上述条件，则针对聚合模型在判断命中 Rollup 的时候会有两个阶段： + +1. 首先通过条件匹配出命中前缀索引索引最长的 Rollup 表，见上述索引策略。 +2. 然后比较 Rollup 的行数，选择最小的一张 Rollup。 + +如下 Base 表以及 Rollup： + +``` ++-------------+-------+--------------+------+-------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++-------------+-------+--------------+------+-------+---------+-------+ +| test_rollup | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k6 | CHAR(5) | Yes | true | N/A | | +| | k7 | DATE | Yes | true | N/A | | +| | k8 | DATETIME | Yes | true | N/A | | +| | k9 | VARCHAR(20) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup2 | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | +| | | | | | | | +| rollup1 | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | true | N/A | | +| | k3 | INT | Yes | true | N/A | | +| | k4 | BIGINT | Yes | true | N/A | | +| | k5 | DECIMAL(9,3) | Yes | true | N/A | | +| | k10 | DOUBLE | Yes | false | N/A | MAX | +| | k11 | FLOAT | Yes | false | N/A | SUM | ++-------------+-------+--------------+------+-------+---------+-------+ +``` + +看以下查询： + +`SELECT SUM(k11) FROM test_rollup WHERE k1 = 10 AND k2 > 200 AND k3 in (1,2,3);` + +首先判断查询是否可以命中聚合的 Rollup表，经过查上面的图是可以的，然后条件中含有 k1,k2,k3 三个条件，这三个条件 test_rollup、rollup1、rollup2 的前三列都含有，所以前缀索引长度一致，然后比较行数显然 rollup2 的聚合程度最高行数最少所以选取 rollup2。 + +``` +| 0:OlapScanNode | +| TABLE: test_rollup | +| PREAGGREGATION: ON | +| PREDICATES: `k1` = 10, `k2` > 200, `k3` IN (1, 2, 3) | +| partitions=1/1 | +| rollup: rollup2 | +| buckets=1/10 | +| cardinality=-1 | +| avgRowSize=0.0 | +| numNodes=0 | +| tuple ids: 0 | +``` diff --git a/docs/zh-CN/installing/compilation.md b/docs/zh-CN/installing/compilation.md new file mode 100644 index 00000000000000..b19ddd74b8171f --- /dev/null +++ b/docs/zh-CN/installing/compilation.md @@ -0,0 +1,108 @@ +--- +{ + "title": "编译", + "language": "zh-CN" +} +--- + + + +# 编译 + +本文档主要介绍如何通过源码编译 Doris。 + +## 使用 Docker 开发镜像编译（推荐） + +### 使用现成的镜像 + +1. 下载 Docker 镜像 + + `$ docker pull apachedoris/doris-dev:build-env` + + 检查镜像下载完成： + + ``` + $ docker images + REPOSITORY TAG IMAGE ID CREATED SIZE + apachedoris/doris-dev build-env f8bc5d4024e0 21 hours ago 3.28GB + ``` + +注: 针对不同的 Doris 版本，需要下载对应的镜像版本 + +| image version | commit id | release version | +|---|---|---| +| apachedoris/doris-dev:build-env | before [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.8.x, 0.9.x | +| apachedoris/doris-dev:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) | 0.10.x, 0.11.x | +| apachedoris/doris-dev:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) or later | 0.12.x or later + +2. 运行镜像 + + `$ docker run -it apachedoris/doris-dev:build-env` + + 如果你希望编译本地 Doris 源码，则可以挂载路径： + + ``` + $ docker run -it -v /your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/ apachedoris/doris-dev:build-env + ``` + +3. 下载源码 + + 启动镜像后，你应该已经处于容器内。可以通过以下命令下载 Doris 源码（已挂载本地源码目录则不用）： + + ``` + $ wget https://dist.apache.org/repos/dist/dev/incubator/doris/xxx.tar.gz + or + $ git clone https://github.com/apache/incubator-doris.git + ``` + +4. 编译 Doris + + ``` + $ sh build.sh + ``` + + 编译完成后，产出文件在 `output/` 目录中。 + +### 自行编译开发环境镜像 + +你也可以自己创建一个 Doris 开发环境镜像，具体可参阅 `docker/README.md` 文件。 + + +## 直接编译（CentOS/Ubuntu） + +你可以在自己的 linux 环境中直接尝试编译 Doris。 + +1. 系统依赖 + + `GCC 5.3.1+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+` + + 如果使用Ubuntu 16.04 及以上系统 可以执行以下命令来安装依赖 + + `sudo apt-get install build-essential openjdk-8-jdk maven cmake byacc flex automake libtool-bin bison binutils-dev libiberty-dev` + + 安装完成后，自行设置环境变量 `PATH`, `JAVA_HOME` 等。 + +2. 编译 Doris + + ``` + $ sh build.sh + ``` + + 编译完成后，产出文件在 `output/` 目录中。 diff --git a/docs/zh-CN/installing/install-deploy.md b/docs/zh-CN/installing/install-deploy.md new file mode 100644 index 00000000000000..60032aec1ba81d --- /dev/null +++ b/docs/zh-CN/installing/install-deploy.md @@ -0,0 +1,420 @@ +--- +{ + "title": "安装与部署", + "language": "zh-CN" +} +--- + + + +# 安装与部署 + +该文档主要介绍了部署 Doris 所需软硬件环境、建议的部署方式、集群扩容缩容，以及集群搭建到运行过程中的常见问题。 +在阅读本文档前，请先根据编译文档编译 Doris。 + +## 软硬件需求 + +### 概述 + +Doris 作为一款开源的 MPP 架构 OLAP 数据库，能够运行在绝大多数主流的商用服务器上。为了能够充分运用 MPP 架构的并发优势，以及 Doris 的高可用特性，我们建议 Doris 的部署遵循以下需求： + +#### Linux 操作系统版本需求 + +| Linux 系统 | 版本 | +|---|---| +| CentOS | 7.1 及以上 | +| Ubuntu | 16.04 及以上 | + +#### 软件需求 + +| 软件 | 版本 | +|---|---| +| Java | 1.8 及以上 | +| GCC | 4.8.2 及以上 | + +#### 开发测试环境 + +| 模块 | CPU | 内存 | 磁盘 | 网络 | 实例数量 | +|---|---|---|---|---|---| +| Frontend | 8核+ | 8GB+ | SSD 或 SATA，10GB+ * | 千兆网卡 | 1 | +| Backend | 8核+ | 16GB+ | SSD 或 SATA，50GB+ * | 千兆网卡 | 1-3 * | + +#### 生产环境 + +| 模块 | CPU | 内存 | 磁盘 | 网络 | 实例数量（最低要求） | +|---|---|---|---|---|---| +| Frontend | 16核+ | 64GB+ | SSD 或 RAID 卡，100GB+ * | 万兆网卡 | 1-5 * | +| Backend | 16核+ | 64GB+ | SSD 或 SATA，100G+ * | 万兆网卡 | 10-100 * | + +> 注1： +> 1. FE 的磁盘空间主要用于存储元数据，包括日志和 image。通常从几百 MB 到几个 GB 不等。 +> 2. BE 的磁盘空间主要用于存放用户数据，总磁盘空间按用户总数据量 * 3（3副本）计算，然后再预留额外 40% 的空间用作后台 compaction 以及一些中间数据的存放。 +> 3. 一台机器上可以部署多个 BE 实例，但是**只能部署一个 FE**。如果需要 3 副本数据，那么至少需要 3 台机器各部署一个 BE 实例（而不是1台机器部署3个BE实例）。**多个FE所在服务器的时钟必须保持一致（允许最多5秒的时钟偏差）** +> 4. 测试环境也可以仅适用一个 BE 进行测试。实际生产环境，BE 实例数量直接决定了整体查询延迟。 +> 5. 所有部署节点关闭 Swap。 + +> 注2：FE 节点的数量 +> 1. FE 角色分为 Follower 和 Observer，（Leader 为 Follower 组中选举出来的一种角色，以下统称 Follower，具体含义见 [元数据设计文档](../internal/metadata-design)）。 +> 2. FE 节点数据至少为1（1 个 Follower）。当部署 1 个 Follower 和 1 个 Observer 时，可以实现读高可用。当部署 3 个 Follower 时，可以实现读写高可用（HA）。 +> 3. Follower 的数量**必须**为奇数，Observer 数量随意。 +> 4. 根据以往经验，当集群可用性要求很高是（比如提供在线业务），可以部署 3 个 Follower 和 1-3 个 Observer。如果是离线业务，建议部署 1 个 Follower 和 1-3 个 Observer。 + +* **通常我们建议 10 ~ 100 台左右的机器，来充分发挥 Doris 的性能（其中 3 台部署 FE（HA），剩余的部署 BE）** +* **当然，Doris的性能与节点数量及配置正相关。在最少4台机器（一台 FE，三台 BE，其中一台 BE 混部一个 Observer FE 提供元数据备份），以及较低配置的情况下，依然可以平稳的运行 Doris。** +* **如果 FE 和 BE 混部，需注意资源竞争问题，并保证元数据目录和数据目录分属不同磁盘。** + +#### Broker 部署 + +Broker 是用于访问外部数据源（如 hdfs）的进程。通常，在每台机器上部署一个 broker 实例即可。 + +#### 网络需求 + +Doris 各个实例直接通过网络进行通讯。以下表格展示了所有需要的端口 + +| 实例名称 | 端口名称 | 默认端口 | 通讯方向 | 说明 | +|---|---|---|---| ---| +| BE | be_port | 9060 | FE --> BE | BE 上 thrift server 的端口，用于接收来自 FE 的请求 | +| BE | webserver_port | 8040 | BE <--> BE | BE 上的 http server 的端口 | +| BE | heartbeat\_service_port | 9050 | FE --> BE | BE 上心跳服务端口（thrift），用于接收来自 FE 的心跳 | +| BE | brpc\_port* | 8060 | FE<-->BE, BE <--> BE | BE 上的 brpc 端口，用于 BE 之间通讯 | +| FE | http_port * | 8030 | FE <--> FE，用户 |FE 上的 http server 端口 | +| FE | rpc_port | 9020 | BE --> FE, FE <--> FE | FE 上的 thrift server 端口 | +| FE | query_port | 9030 | 用户 | FE 上的 mysql server 端口 | +| FE | edit\_log_port | 9010 | FE <--> FE | FE 上的 bdbje 之间通信用的端口 | +| Broker | broker\_ipc_port | 8000 | FE --> Broker, BE --> Broker | Broker 上的 thrift server，用于接收请求 | + +> 注： +> 1. 当部署多个 FE 实例时，要保证 FE 的 http\_port 配置相同。 +> 2. 部署前请确保各个端口在应有方向上的访问权限。 + +#### IP 绑定 + +因为有多网卡的存在，或因为安装过 docker 等环境导致的虚拟网卡的存在，同一个主机可能存在多个不同的 ip。当前 Doris 并不能自动识别可用 IP。所以当遇到部署主机上有多个 IP 时，必须通过 priority\_networks 配置项来强制指定正确的 IP。 + +priority\_networks 是 FE 和 BE 都有的一个配置，配置项需写在 fe.conf 和 be.conf 中。该配置项用于在 FE 或 BE 启动时，告诉进程应该绑定哪个IP。示例如下： + +`priority_networks=10.1.3.0/24` + +这是一种 [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) 的表示方法。FE 或 BE 会根据这个配置项来寻找匹配的IP，作为自己的 localIP。 + +**注意**：当配置完 priority\_networks 并启动 FE 或 BE 后，只是保证了 FE 或 BE 自身的 IP 进行了正确的绑定。而在使用 ADD BACKEND 或 ADD FRONTEND 语句中，也需要指定和 priority\_networks 配置匹配的 IP，否则集群无法建立。举例： + +BE 的配置为：`priority_networks=10.1.3.0/24` + +但是在 ADD BACKEND 时使用的是：`ALTER SYSTEM ADD BACKEND "192.168.0.1:9050";` + +则 FE 和 BE 将无法正常通信。 + +这时，必须 DROP 掉这个添加错误的 BE，重新使用正确的 IP 执行 ADD BACKEND。 + +FE 同理。 + +BROKER 当前没有，也不需要 priority\_networks 这个选项。Broker 的服务默认绑定在 0.0.0.0 上。只需在 ADD BROKER 时，执行正确可访问的 BROKER IP 即可。 + +## 集群部署 + +### 手动部署 + +#### FE 部署 + +* 拷贝 FE 部署文件到指定节点 + + 将源码编译生成的 output 下的 fe 文件夹拷贝到 FE 的节点指定部署路径下。 + +* 配置 FE + + 1. 配置文件为 conf/fe.conf。其中注意：`meta_dir`：元数据存放位置。默认在 fe/doris-meta/ 下。需**手动创建**该目录。 + 2. fe.conf 中 JAVA_OPTS 默认 java 最大堆内存为 4GB，建议生产环境调整至 8G 以上。 + +* 启动FE + + `sh bin/start_fe.sh --daemon` + + FE进程启动进入后台执行。日志默认存放在 fe/log/ 目录下。如启动失败，可以通过查看 fe/log/fe.log 或者 fe/log/fe.out 查看错误信息。 + +* 如需部署多 FE，请参见 "FE 扩容和缩容" 章节 + +#### BE 部署 + +* 拷贝 BE 部署文件到所有要部署 BE 的节点 + + 将源码编译生成的 output 下的 be 文件夹拷贝到 BE 的节点的指定部署路径下。 + +* 修改所有 BE 的配置 + + 修改 be/conf/be.conf。主要是配置 `storage_root_path`：数据存放目录。默认在be/storage下，需要**手动创建**该目录。多个路径之间使用 `;` 分隔（最后一个目录后不要加 `;`）。 + +* 在 FE 中添加所有 BE 节点 + + BE 节点需要先在 FE 中添加，才可加入集群。可以使用 mysql-client 连接到 FE： + + `./mysql-client -h host -P port -uroot` + + 其中 host 为 FE 所在节点 ip；port 为 fe/conf/fe.conf 中的 query_port；默认使用 root 账户，无密码登录。 + + 登录后，执行以下命令来添加每一个 BE： + + `ALTER SYSTEM ADD BACKEND "host:port";` + + 如果使用多租户功能，则执行以下命令添加 BE: + + `ALTER SYSTEM ADD FREE BACKEND "host:port";` + + 其中 host 为 BE 所在节点 ip；port 为 be/conf/be.conf 中的 heartbeat_service_port。 + + 如果不添加 FREE 关键字，BE 默认进入自动生成的 cluster，添加了 FREE 关键字后新的 BE 不属于任何 cluster，这样创建新 cluster 的时候就可以从这些空闲的be中选取，详细见[多租户设计文档](../administrator-guide/operation/multi-tenant.md) + +* 启动 BE + + `sh bin/start_be.sh --daemon` + + BE 进程将启动并进入后台执行。日志默认存放在 be/log/ 目录下。如启动失败，可以通过查看 be/log/be.log 或者 be/log/be.out 查看错误信息。 + +* 查看BE状态 + + 使用 mysql-client 连接到 FE，并执行 `SHOW PROC '/backends';` 查看 BE 运行情况。如一切正常，`isAlive` 列应为 `true`。 + +#### （可选）FS_Broker 部署 + +Broker 以插件的形式，独立于 Doris 部署。如果需要从第三方存储系统导入数据，需要部署相应的 Broker，默认提供了读取 HDFS 和百度云 BOS 的 fs_broker。fs_broker 是无状态的，建议每一个 FE 和 BE 节点都部署一个 Broker。 + +* 拷贝源码 fs_broker 的 output 目录下的相应 Broker 目录到需要部署的所有节点上。建议和 BE 或者 FE 目录保持同级。 + +* 修改相应 Broker 配置 + + 在相应 broker/conf/ 目录下对应的配置文件中，可以修改相应配置。 + + * 启动 Broker + + `sh bin/start_broker.sh --daemon` 启动 Broker。 + +* 添加 Broker + + 要让 Doris 的 FE 和 BE 知道 Broker 在哪些节点上，通过 sql 命令添加 Broker 节点列表。 + + 使用 mysql-client 连接启动的 FE，执行以下命令： + + `ALTER SYSTEM ADD BROKER broker_name "host1:port1","host2:port2",...;` + + 其中 host 为 Broker 所在节点 ip；port 为 Broker 配置文件中的 broker\_ipc\_port。 + +* 查看 Broker 状态 + + 使用 mysql-client 连接任一已启动的 FE，执行以下命令查看 Broker 状态：`SHOW PROC "/brokers";` + +**注：在生产环境中，所有实例都应使用守护进程启动，以保证进程退出后，会被自动拉起，如 [Supervisor](http://supervisord.org/)。如需使用守护进程启动，在 0.9.0 及之前版本中，需要修改各个 start_xx.sh 脚本，去掉最后的 & 符号**。从 0.10.0 版本开始，直接调用 `sh start_xx.sh` 启动即可。也可参考 [这里](https://www.cnblogs.com/lenmom/p/9973401.html) + +## 扩容缩容 + +Doris 可以很方便的扩容和缩容 FE、BE、Broker 实例。 + +### FE 扩容和缩容 + +可以通过将 FE 扩容至 3 个一上节点来实现 FE 的高可用。 + +用户可以通过 mysql 客户端登陆 Master FE。通过: + +`SHOW PROC '/frontends';` + +来查看当前 FE 的节点情况。 + +也可以通过前端页面连接：```http://fe_hostname:fe_http_port/frontend``` 或者 ```http://fe_hostname:fe_http_port/system?path=//frontends``` 来查看 FE 节点的情况。 + +以上方式，都需要 Doris 的 root 用户权限。 + +FE 节点的扩容和缩容过程，不影响当前系统运行。 + +#### 增加 FE 节点 + +FE 分为 Leader，Follower 和 Observer 三种角色。 默认一个集群，只能有一个 Leader，可以有多个 Follower 和 Observer。其中 Leader 和 Follower 组成一个 Paxos 选择组，如果 Leader 宕机，则剩下的 Follower 会自动选出新的 Leader，保证写入高可用。Observer 同步 Leader 的数据，但是不参加选举。如果只部署一个 FE，则 FE 默认就是 Leader。 + +第一个启动的 FE 自动成为 Leader。在此基础上，可以添加若干 Follower 和 Observer。 + +添加 Follower 或 Observer。使用 mysql-client 连接到已启动的 FE，并执行： + +`ALTER SYSTEM ADD FOLLOWER "host:port";` + +或 + +`ALTER SYSTEM ADD OBSERVER "host:port";` + +其中 host 为 Follower 或 Observer 所在节点 ip，port 为其配置文件 fe.conf 中的 edit\_log\_port。 + +配置及启动 Follower 或 Observer。Follower 和 Observer 的配置同 Leader 的配置。第一次启动时，需执行以下命令： + +`./bin/start_fe.sh --helper host:port --daemon` + +其中 host 为 Leader 所在节点 ip, port 为 Leader 的配置文件 fe.conf 中的 edit_log_port。--helper 参数仅在 follower 和 observer 第一次启动时才需要。 + +查看 Follower 或 Observer 运行状态。使用 mysql-client 连接到任一已启动的 FE，并执行：SHOW PROC '/frontends'; 可以查看当前已加入集群的 FE 及其对应角色。 + +> FE 扩容注意事项： +> 1. Follower FE（包括 Leader）的数量必须为奇数，建议最多部署 3 个组成高可用（HA）模式即可。 +> 2. 当 FE 处于高可用部署时（1个 Leader，2个 Follower），我们建议通过增加 Observer FE 来扩展 FE 的读服务能力。当然也可以继续增加 Follower FE，但几乎是不必要的。 +> 3. 通常一个 FE 节点可以应对 10-20 台 BE 节点。建议总的 FE 节点数量在 10 个以下。而通常 3 个即可满足绝大部分需求。 + +#### 删除 FE 节点 + +使用以下命令删除对应的 FE 节点： + +```ALTER SYSTEM DROP FOLLOWER[OBSERVER] "fe_host:edit_log_port";``` + +> FE 缩容注意事项： +> 1. 删除 Follower FE 时，确保最终剩余的 Follower（包括 Leader）节点为奇数。 + +### BE 扩容和缩容 + +用户可以通过 mysql-client 登陆 Leader FE。通过: + +```SHOW PROC '/backends';``` + +来查看当前 BE 的节点情况。 + +也可以通过前端页面连接：```http://fe_hostname:fe_http_port/backend``` 或者 ```http://fe_hostname:fe_http_port/system?path=//backends``` 来查看 BE 节点的情况。 + +以上方式，都需要 Doris 的 root 用户权限。 + +BE 节点的扩容和缩容过程，不影响当前系统运行以及正在执行的任务，并且不会影响当前系统的性能。数据均衡会自动进行。根据集群现有数据量的大小，集群会在几个小时到1天不等的时间内，恢复到负载均衡的状态。集群负载情况，可以参见 [Tablet 负载均衡文档](../administrator-guide/operation/tablet-repair-and-balance.md)。 + +#### 增加 BE 节点 + +BE 节点的增加方式同 **BE 部署** 一节中的方式，通过 `ALTER SYSTEM ADD BACKEND` 命令增加 BE 节点。 + +> BE 扩容注意事项： +> 1. BE 扩容后，Doris 会自动根据负载情况，进行数据均衡，期间不影响使用。 + +#### 删除 BE 节点 + +删除 BE 节点有两种方式：DROP 和 DECOMMISSION + +DROP 语句如下： + +```ALTER SYSTEM DROP BACKEND "be_host:be_heartbeat_service_port";``` + +**注意：DROP BACKEND 会直接删除该 BE，并且其上的数据将不能再恢复！！！所以我们强烈不推荐使用 DROP BACKEND 这种方式删除 BE 节点。当你使用这个语句时，会有对应的防误操作提示。** + +DECOMMISSION 语句如下： + +```ALTER SYSTEM DECOMMISSION BACKEND "be_host:be_heartbeat_service_port";``` + +> DECOMMISSION 命令说明： +> 1. 该命令用于安全删除 BE 节点。命令下发后，Doris 会尝试将该 BE 上的数据向其他 BE 节点迁移，当所有数据都迁移完成后，Doris 会自动删除该节点。 +> 2. 该命令是一个异步操作。执行后，可以通过 ```SHOW PROC '/backends';``` 看到该 BE 节点的 isDecommission 状态为 true。表示该节点正在进行下线。 +> 3. 该命令**不一定执行成功**。比如剩余 BE 存储空间不足以容纳下线 BE 上的数据，或者剩余机器数量不满足最小副本数时，该命令都无法完成，并且 BE 会一直处于 isDecommission 为 true 的状态。 +> 4. DECOMMISSION 的进度，可以通过 ```SHOW PROC '/backends';``` 中的 TabletNum 查看，如果正在进行，TabletNum 将不断减少。 +> 5. 该操作可以通过: +> ```CANCEL DECOMMISSION BACKEND "be_host:be_heartbeat_service_port";``` +> 命令取消。取消后，该 BE 上的数据将维持当前剩余的数据量。后续 Doris 重新进行负载均衡 + +**对于多租户部署环境下，BE 节点的扩容和缩容，请参阅 [多租户设计文档](../administrator-guide/operation/multi-tenant.md)。** + +### Broker 扩容缩容 + +Broker 实例的数量没有硬性要求。通常每台物理机部署一个即可。Broker 的添加和删除可以通过以下命令完成： + +```ALTER SYSTEM ADD BROKER broker_name "broker_host:broker_ipc_port";``` +```ALTER SYSTEM DROP BROKER broker_name "broker_host:broker_ipc_port";``` +```ALTER SYSTEM DROP ALL BROKER broker_name;``` + +Broker 是无状态的进程，可以随意启停。当然，停止后，正在其上运行的作业会失败，重试即可。 + +## 常见问题 + +### 进程相关 + +1. 如何确定 FE 进程启动成功 + + FE 进程启动后，会首先加载元数据，根据 FE 角色的不同，在日志中会看到 ```transfer from UNKNOWN to MASTER/FOLLOWER/OBSERVER```。最终会看到 ```thrift server started``` 日志，并且可以通过 mysql 客户端连接到 FE，则表示 FE 启动成功。 + + 也可以通过如下连接查看是否启动成功： + `http://fe_host:fe_http_port/api/bootstrap` + + 如果返回： + `{"status":"OK","msg":"Success"}` + + 则表示启动成功，其余情况，则可能存在问题。 + + > 注：如果在 fe.log 中查看不到启动失败的信息，也许在 fe.out 中可以看到。 + +2. 如何确定 BE 进程启动成功 + + BE 进程启动后，如果之前有数据，则可能有数分钟不等的数据索引加载时间。 + + 如果是 BE 的第一次启动，或者该 BE 尚未加入任何集群，则 BE 日志会定期滚动 ```waiting to receive first heartbeat from frontend``` 字样。表示 BE 还未通过 FE 的心跳收到 Master 的地址，正在被动等待。这种错误日志，在 FE 中 ADD BACKEND 并发送心跳后，就会消失。如果在接到心跳后，又重复出现 ``````master client, get client from cache failed.host: , port: 0, code: 7`````` 字样，说明 FE 成功连接了 BE，但 BE 无法主动连接 FE。可能需要检查 BE 到 FE 的 rpc_port 的连通性。 + + 如果 BE 已经被加入集群，日志中应该每隔 5 秒滚动来自 FE 的心跳日志：```get heartbeat, host: xx.xx.xx.xx, port: 9020, cluster id: xxxxxx```，表示心跳正常。 + + 其次，日志中应该每隔 10 秒滚动 ```finish report task success. return code: 0``` 的字样，表示 BE 向 FE 的通信正常。 + + 同时，如果有数据查询，应该能看到不停滚动的日志，并且有 ```execute time is xxx``` 日志，表示 BE 启动成功，并且查询正常。 + + 也可以通过如下连接查看是否启动成功： + `http://be_host:be_http_port/api/health` + + 如果返回： + `{"status": "OK","msg": "To Be Added"}` + + 则表示启动成功，其余情况，则可能存在问题。 + + > 注：如果在 be.INFO 中查看不到启动失败的信息，也许在 be.out 中可以看到。 + +3. 搭建系统后，如何确定 FE、BE 连通性正常 + + 首先确认 FE 和 BE 进程都已经单独正常启动，并确认已经通过 `ADD BACKEND` 或者 `ADD FOLLOWER/OBSERVER` 语句添加了所有节点。 + + 如果心跳正常，BE 的日志中会显示 ```get heartbeat, host: xx.xx.xx.xx, port: 9020, cluster id: xxxxxx```。如果心跳失败，在 FE 的日志中会出现 ```backend[10001] got Exception: org.apache.thrift.transport.TTransportException``` 类似的字样，或者其他 thrift 通信异常日志，表示 FE 向 10001 这个 BE 的心跳失败。这里需要检查 FE 向 BE host 的心跳端口的连通性。 + + 如果 BE 向 FE 的通信正常，则 BE 日志中会显示 ```finish report task success. return code: 0``` 的字样。否则会出现 ```master client, get client from cache failed``` 的字样。这种情况下，需要检查 BE 向 FE 的 rpc_port 的连通性。 + +4. Doris 各节点认证机制 + + 除了 Master FE 以外，其余角色节点（Follower FE，Observer FE，Backend），都需要通过 `ALTER SYSTEM ADD` 语句先注册到集群，然后才能加入集群。 + + Master FE 在第一次启动时，会在 doris-meta/image/VERSION 文件中生成一个 cluster_id。 + + FE 在第一次加入集群时，会首先从 Master FE 获取这个文件。之后每次 FE 之间的重新连接（FE 重启），都会校验自身 cluster id 是否与已存在的其它 FE 的 cluster id 相同。如果不同，则该 FE 会自动退出。 + + BE 在第一次接收到 Master FE 的心跳时，会从心跳中获取到 cluster id，并记录到数据目录的 `cluster_id` 文件中。之后的每次心跳都会比对 FE 发来的 cluster id。如果 cluster id 不相等，则 BE 会拒绝响应 FE 的心跳。 + + 心跳中同时会包含 Master FE 的 ip。当 FE 切主时，新的 Master FE 会携带自身的 ip 发送心跳给 BE，BE 会更新自身保存的 Master FE 的 ip。 + + > **priority\_network** + > + > priority\_network 是 FE 和 BE 都有一个配置，其主要目的是在多网卡的情况下，协助 FE 或 BE 识别自身 ip 地址。priority\_network 采用 CIDR 表示法：[RFC 4632](https://tools.ietf.org/html/rfc4632) + > + > 当确认 FE 和 BE 连通性正常后，如果仍然出现建表 Timeout 的情况，并且 FE 的日志中有 `backend does not found. host: xxx.xxx.xxx.xxx` 字样的错误信息。则表示 Doris 自动识别的 IP 地址有问题，需要手动设置 priority\_network 参数。 + > + > 出现这个问题的主要原因是：当用户通过 `ADD BACKEND` 语句添加 BE 后，FE 会识别该语句中指定的是 hostname 还是 IP。如果是 hostname，则 FE 会自动将其转换为 IP 地址并存储到元数据中。当 BE 在汇报任务完成信息时，会携带自己的 IP 地址。而如果 FE 发现 BE 汇报的 IP 地址和元数据中不一致时，就会出现如上错误。 + > + > 这个错误的解决方法：1）分别在 FE 和 BE 设置 **priority\_network** 参数。通常 FE 和 BE 都处于一个网段，所以该参数设置为相同即可。2）在 `ADD BACKEND` 语句中直接填写 BE 正确的 IP 地址而不是 hostname，以避免 FE 获取到错误的 IP 地址。 + +5. BE 进程文件句柄数 + + BE进程文件句柄数，受min_file_descriptor_number/max_file_descriptor_number两个参数控制。 + + 如果不在[min_file_descriptor_number, max_file_descriptor_number]区间内，BE进程启动会出错，可以使用ulimit进行设置。 + + min_file_descriptor_number的默认值为65536。 + + max_file_descriptor_number的默认值为131072. + + 举例而言：ulimit -n 65536; 表示将文件句柄设成65536。 + + 启动BE进程之后，可以通过 cat /proc/$pid/limits 查看进程实际生效的句柄数 diff --git a/docs/zh-CN/installing/upgrade.md b/docs/zh-CN/installing/upgrade.md new file mode 100644 index 00000000000000..3315a06476d762 --- /dev/null +++ b/docs/zh-CN/installing/upgrade.md @@ -0,0 +1,63 @@ +--- +{ + "title": "集群升级", + "language": "zh-CN" +} +--- + + + +# 集群升级 + +Doris 可以通过滚动升级的方式，平滑进行升级。建议按照以下步骤进行安全升级。 + +> 注： +> 1. 以下方式均建立在高可用部署的情况下。即数据 3 副本，FE 高可用情况下。 + +## 测试 BE 升级正确性 + +1. 任意选择一个 BE 节点，部署最新的 palo_be 二进制文件。 +2. 重启 BE 节点，通过 BE 日志 be.INFO，查看是否启动成功。 +3. 如果启动失败，可以先排查原因。如果错误不可恢复，可以直接通过 DROP BACKEND 删除该 BE、清理数据后，使用上一个版本的 palo_be 重新启动 BE。然后重新 ADD BACKEND。（**该方法会导致丢失一个数据副本，请务必确保3副本完整的情况下，执行这个操作！！！**） + +## 测试 FE 元数据兼容性 + +0. **重要！!元数据兼容性异常很可能导致数据无法恢复！！** +1. 单独使用新版本部署一个测试用的 FE 进程（比如自己本地的开发机）。 +2. 修改测试用的 FE 的配置文件 fe.conf，将所有端口设置为**与线上不同**。 +3. 在 fe.conf 添加配置：cluster_id=123456 +4. 在 fe.conf 添加配置：metadata\_failure_recovery=true +5. 拷贝线上环境 Master FE 的元数据目录 palo-meta 到测试环境 +6. 将拷贝到测试环境中的 palo-meta/image/VERSION 文件中的 cluster_id 修改为 123456（即与第3步中相同） +7. 在测试环境中，运行 sh bin/start_fe.sh 启动 FE +8. 通过 FE 日志 fe.log 观察是否启动成功。 +9. 如果启动成功，运行 sh bin/stop_fe.sh 停止测试环境的 FE 进程。 +10. **以上 2-6 步的目的是防止测试环境的FE启动后，错误连接到线上环境中。** + +## 升级准备 + +1. 在完成数据正确性验证后，将 BE 和 FE 新版本的二进制文件分发到各自目录下。 +2. 通常小版本升级，BE 只需升级 palo_be；而 FE 只需升级 palo-fe.jar。如果是大版本升级，则可能需要升级其他文件（包括但不限于 bin/ lib/ 等等）如果你不清楚是否需要替换其他文件，建议全部替换。 + +## 滚动升级 + +1. 确认新版本的文件部署完成后。逐台重启 FE 和 BE 实例即可。 +2. 建议逐台重启 BE 后，再逐台重启 FE。因为通常 Doris 保证 FE 到 BE 的向后兼容性，即老版本的 FE 可以访问新版本的 BE。但可能不支持老版本的 BE 访问新版本的 FE。 +3. 建议确认前一个实例启动成功后，在重启下一个实例。实例启动成功的标识，请参阅安装部署文档。 diff --git a/docs/zh-CN/internal/doris_storage_optimization.md b/docs/zh-CN/internal/doris_storage_optimization.md new file mode 100644 index 00000000000000..3d6538844909ea --- /dev/null +++ b/docs/zh-CN/internal/doris_storage_optimization.md @@ -0,0 +1,232 @@ +--- +{ + "title": "Doris存储文件格式优化", + "language": "zh-CN" +} +--- + + + +# Doris存储文件格式优化 # + +## 文件格式 ## + +![](/images/segment_v2.png) +
图1. doris segment文件格式
+ +文件包括： +- 文件开始是8个字节的magic code，用于识别文件格式和版本 +- Data Region：用于存储各个列的数据信息，这里的数据是按需分page加载的 +- Index Region: doris中将各个列的index数据统一存储在Index Region，这里的数据会按照列粒度进行加载，所以跟列的数据信息分开存储 +- Footer信息 + - FileFooterPB:定义文件的元数据信息 + - 4个字节的footer pb内容的checksum + - 4个字节的FileFooterPB消息长度，用于读取FileFooterPB + - 8个字节的MAGIC CODE，之所以在末位存储，是方便不同的场景进行文件类型的识别 + +文件中的数据按照page的方式进行组织，page是编码和压缩的基本单位。现在的page类型包括以下几种: + +### DataPage ### + +DataPage分为两种：nullable和non-nullable的data page。 + +nullable的data page内容包括： +``` + + +----------------+ + | value count | + |----------------| + | first row id | + |----------------| + | bitmap length | + |----------------| + | null bitmap | + |----------------| + | data | + |----------------| + | checksum | + +----------------+ +``` + +non-nullable data page结构如下： + +``` + |----------------| + | value count | + |----------------| + | first row id | + |----------------| + | data | + |----------------| + | checksum | + +----------------+ +``` + +其中各个字段含义如下： + +- value count + - 表示page中的行数 +- first row id + - page中第一行的行号 +- bitmap length + - 表示接下来bitmap的字节数 +- null bitmap + - 表示null信息的bitmap +- data + - 存储经过encoding和compress之后的数据 + - 需要在数据的头部信息中写入：is_compressed + - 各种不同编码的data需要在头部信息写入一些字段信息，以实现数据的解析 + - TODO：添加各种encoding的header信息 +- checksum + - 存储page粒度的校验和，包括page的header和之后的实际数据 + + +### Bloom Filter Pages ### + +针对每个bloom filter列,会在page的粒度相应的生成一个bloom filter的page，保存在bloom filter pages区域 + +### Ordinal Index Page ### + +针对每个列，都会按照page粒度，建立行号的稀疏索引。内容为这个page的起始行的行号到这个block的指针（包括offset和length） + +### Short Key Index page ### + +我们会每隔N行（可配置）生成一个short key的稀疏索引，索引的内容为：short key->行号(ordinal) + +### Column的其他索引 ### + +该格式设计支持后续扩展其他的索引信息，比如bitmap索引，spatial索引等等，只需要将需要的数据写到现有的列数据后面，并且添加对应的元数据字段到FileFooterPB中 + +### 元数据定义 ### +FileFooterPB的定义为： + +``` +message ColumnPB { + optional uint32 column_id = 1; // 这里使用column id，不使用column name是因为计划支持修改列名 + optional string type = 2; // 列类型 + optional string aggregation = 3; // 是否聚合 + optional uint32 length = 4; // 长度 + optional bool is_key = 5; // 是否是主键列 + optional string default_value = 6; // 默认值 + optional uint32 precision = 9 [default = 27]; // 精度 + optional uint32 frac = 10 [default = 9]; + optional bool is_nullable = 11 [default=false]; // 是否有null + optional bool is_bf_column = 15 [default=false]; // 是否有bf词典 + optional bool is_bitmap_column = 16 [default=false]; // 是否有bitmap索引 +} + +// page偏移 +message PagePointerPB { + required uint64 offset; // page在文件中的偏移 + required uint32 length; // page的大小 +} + +message MetadataPairPB { + optional string key = 1; + optional bytes value = 2; +} + +message ColumnMetaPB { + optional ColumnMessage encoding; // 编码方式 + + optional PagePointerPB dict_page // 词典page + repeated PagePointerPB bloom_filter_pages; // bloom filter词典信息 + optional PagePointerPB ordinal_index_page; // 行号索引数据 + optional PagePointerPB page_zone_map_page; // page级别统计信息索引数据 + + optional PagePointerPB bitmap_index_page; // bitmap索引数据 + + optional uint64 data_footprint; // 列中索引的大小 + optional uint64 index_footprint; // 列中数据的大小 + optional uint64 raw_data_footprint; // 原始列数据大小 + + optional CompressKind compress_kind; // 列的压缩方式 + + optional ZoneMapPB column_zone_map; //文件级别的过滤条件 + repeated MetadataPairPB column_meta_datas; +} + +message FileFooterPB { + optional uint32 version = 2 [default = 1]; // 用于版本兼容和升级使用 + repeated ColumnPB schema = 5; // 列Schema + optional uint64 num_values = 4; // 文件中保存的行数 + optional uint64 index_footprint = 7; // 索引大小 + optional uint64 data_footprint = 8; // 数据大小 + optional uint64 raw_data_footprint = 8; // 原始数据大小 + + optional CompressKind compress_kind = 9 [default = COMPRESS_LZO]; // 压缩方式 + repeated ColumnMetaPB column_metas = 10; // 列元数据 + optional PagePointerPB key_index_page; // short key索引page +} + +``` + +## 读写逻辑 ## + +### 写入 ### + +大体的写入流程如下： +1. 写入magic +2. 根据schema信息，生成对应的ColumnWriter，每个ColumnWriter按照不同的类型，获取对应的encoding信息（可配置），根据encoding，生成对应的encoder +3. 调用encoder->add(value)进行数据写入，每个K行，生成一个short key index entry，并且，如果当前的page满足一定条件（大小超过1M或者行数为K），就生成一个新的page，缓存在内存中。 +4. 不断的循环步骤3，直到数据写入完成。将各个列的数据依序刷入文件中 +5. 生成FileFooterPB信息，写入文件中。 + +相关的问题： + +- short key的索引如何生成？ + - 现在还是按照每隔多少行生成一个short key的稀疏索引，保持每隔1024行生成一个short的稀疏索引,具体的内容是：short key -> ordinal + +- ordinal索引里面应该存什么？ + - 存储page的第一个ordinal到page pointer的映射信息 +- 不同encoding类型的page里存什么？ + - 词典压缩 + - plain + - rle + - bshuf + +### 读取 ### + +1. 读取文件的magic，判断文件类型和版本 +2. 读取FileFooterPB，进行checksum校验 +3. 按照需要的列，读取short key索引和对应列的数据ordinal索引信息 +4. 使用start key和end key，通过short key索引定位到要读取的行号，然后通过ordinal索引确定需要读取的row ranges, 同时需要通过统计信息、bitmap索引等过滤需要读取的row ranges +5. 然后按照row ranges通过ordinal索引读取行的数据 + +相关的问题： +1. 如何实现在page内部快速的定位到某一行？ + + page内部是的数据是经过encoding的，无法快速进行行级数据的定位。不同的encoding方式，在内部进行快速的行号定位的方案不一样，需要具体分析： + - 如果是rle编码的，需要通过解析rle的header进行skip，直到到达包含该行的那个rle块之后，再进行反解。 + - binary plain encoding：会在page的中存储offset信息，并且会在page header中指定offset信息的offset，读取的时候会先解析offset信息到数组中，这样子就可以通过各个行的offset数据信息快速的定位block某一行的数据 +2. 如何实现块的高效读取？可以考虑将相邻的块在读取的时候进行merge，一次性读取？ + 这个需要在读取的时候，判断block是否连续，如果连续，就一次性的读取 + +## 编码 ## + +现有的doris存储中，针对string类型的编码，采用plain encoding的方式，效率比较低。经过对比，发现在百度统计的场景下，数据会因为string类型的编码膨胀超过一倍。所以，计划引入基于词典的编码压缩。 + +## 压缩 ## + +实现可扩展的压缩框架，支持多种压缩算法，方便后续添加新的压缩算法，计划引入zstd压缩。 + +## TODO ## +1. 如何实现嵌套类型？如何在嵌套类型中进行行号定位？ +2. 如何优化现在的ScanRange拆分导致的下游bitmap、column statistic统计等进行多次？ diff --git a/docs/zh-CN/internal/grouping_sets_design.md b/docs/zh-CN/internal/grouping_sets_design.md new file mode 100644 index 00000000000000..0c19094cffc044 --- /dev/null +++ b/docs/zh-CN/internal/grouping_sets_design.md @@ -0,0 +1,517 @@ +--- +{ + "title": "GROUPING SETS 设计文档", + "language": "zh-CN" +} +--- + + + +# GROUPING SETS 设计文档 + +## 1. GROUPING SETS 相关背景知识 + +### 1.1 GROUPING SETS 子句 + +GROUP BY GROUPING SETS 是对 GROUP BY 子句的扩展，它能够在一个 GROUP BY 子句中一次实现多个集合的分组。其结果等价于将多个相应 GROUP BY 子句进行 UNION 操作。 + +特别地，一个空的子集意味着将所有的行聚集到一个分组。 +GROUP BY 子句是只含有一个元素的 GROUP BY GROUPING SETS 的特例。 + +例如，GROUPING SETS 语句： + +``` +SELECT k1, k2, SUM( k3 ) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k1), (k2), ( ) ); +``` + +其查询结果等价于： + +``` +SELECT k1, k2, SUM( k3 ) FROM t GROUP BY k1, k2 +UNION +SELECT k1, null, SUM( k3 ) FROM t GROUP BY k1 +UNION +SELECT null, k2, SUM( k3 ) FROM t GROUP BY k2 +UNION +SELECT null, null, SUM( k3 ) FROM t +``` + +下面是一个实际数据的例子： + +``` +mysql> SELECT * FROM t; ++------+------+------+ +| k1 | k2 | k3 | ++------+------+------+ +| a | A | 1 | +| a | A | 2 | +| a | B | 1 | +| a | B | 3 | +| b | A | 1 | +| b | A | 4 | +| b | B | 1 | +| b | B | 5 | ++------+------+------+ +8 rows in set (0.01 sec) + +mysql> SELECT k1, k2, SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); ++------+------+-----------+ +| k1 | k2 | sum(`k3`) | ++------+------+-----------+ +| b | B | 6 | +| a | B | 4 | +| a | A | 3 | +| b | A | 5 | +| NULL | B | 10 | +| NULL | A | 8 | +| a | NULL | 7 | +| b | NULL | 11 | +| NULL | NULL | 18 | ++------+------+-----------+ +9 rows in set (0.06 sec) +``` + +### 1.2 ROLLUP 子句 + +ROLLUP 是对 GROUPING SETS 的扩展。 + +``` +SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY ROLLUP(a,b,c) +``` + +这个 ROLLUP 等价于下面的 GROUPING SETS： + +``` +GROUPING SETS ( +(a,b,c), +( a, b ), +( a), +( ) +) +``` + +### 1.3 CUBE 子句 + +CUBE 也是对 GROUPING SETS 的扩展。 + +``` +CUBE ( e1, e2, e3, ... ) +``` + +其含义是 GROUPING SETS 后面列表中的所有子集。 + +例如，CUBE ( a, b, c ) 等价于下面的 GROUPING SETS： + +``` +GROUPING SETS ( +( a, b, c ), +( a, b ), +( a, c ), +( a ), +( b, c ), +( b ), +( c ), +( ) +) +``` + +### 1.4 GROUPING 和 GROUPING_ID 函数 +当我们没有统计某一列时，它的值显示为 NULL，这也可能是列本身就有 NULL 值，这就需要一种方法区分是没有统计还是值本来就是 NULL。为此引入 GROUPING 和 GROUPING_ID 函数。 +GROUPING(column:Column) 函数用于区分分组后的单个列是普通列和聚合列。如果是聚合列，则返回1，反之，则是0. GROUPING() 只能有一个参数列。 + +GROUPING_ID(column1, column2) 则根据指定的column 顺序，否则根据聚合的时候给的集合的元素顺序，计算出一个列列表的 bitmap 值，一个列如果是聚合列为0，否则为1. GROUPING_ID()函数返回位向量的十进制值。 +比如 [0 1 0] ->2 从下列第三个查询可以看到这种对应关系 + +例如，对于下面的表： + +``` +mysql> select * from t; ++------+------+------+ +| k1 | k2 | k3 | ++------+------+------+ +| a | A | 1 | +| a | A | 2 | +| a | B | 1 | +| a | B | 3 | +| b | A | 1 | +| b | A | 4 | +| b | B | 1 | +| b | B | 5 | ++------+------+------+ +``` + +grouping sets 的结果如下： + +``` +mysql> SELECT k1, k2, GROUPING(k1), GROUPING(k2), SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); ++------+------+----------------+----------------+-----------+ +| k1 | k2 | grouping(`k1`) | grouping(`k2`) | sum(`k3`) | ++------+------+----------------+----------------+-----------+ +| a | A | 0 | 0 | 3 | +| a | B | 0 | 0 | 4 | +| a | NULL | 0 | 1 | 7 | +| b | A | 0 | 0 | 5 | +| b | B | 0 | 0 | 6 | +| b | NULL | 0 | 1 | 11 | +| NULL | A | 1 | 0 | 8 | +| NULL | B | 1 | 0 | 10 | +| NULL | NULL | 1 | 1 | 18 | ++------+------+----------------+----------------+-----------+ +9 rows in set (0.02 sec) + +mysql> SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); ++------+------+-------------------------+-----------+ +| k1 | k2 | grouping_id(`k1`, `k2`) | sum(`k3`) | ++------+------+-------------------------+-----------+ +| a | A | 0 | 3 | +| a | B | 0 | 4 | +| a | NULL | 1 | 7 | +| b | A | 0 | 5 | +| b | B | 0 | 6 | +| b | NULL | 1 | 11 | +| NULL | A | 2 | 8 | +| NULL | B | 2 | 10 | +| NULL | NULL | 3 | 18 | ++------+------+-------------------------+-----------+ +9 rows in set (0.02 sec) + +mysql> SELECT k1, k2, grouping(k1), grouping(k2), GROUPING_ID(k1,k2), SUM(k4) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ) order by k1, k2; ++------+------+----------------+----------------+-------------------------+-----------+ +| k1 | k2 | grouping(`k1`) | grouping(`k2`) | grouping_id(`k1`, `k2`) | sum(`k4`) | ++------+------+----------------+----------------+-------------------------+-----------+ +| a | A | 0 | 0 | 0 | 3 | +| a | B | 0 | 0 | 0 | 4 | +| a | NULL | 0 | 1 | 1 | 7 | +| b | A | 0 | 0 | 0 | 5 | +| b | B | 0 | 0 | 0 | 6 | +| b | NULL | 0 | 1 | 1 | 11 | +| NULL | A | 1 | 0 | 2 | 8 | +| NULL | B | 1 | 0 | 2 | 10 | +| NULL | NULL | 1 | 1 | 3 | 18 | ++------+------+----------------+----------------+-------------------------+-----------+ +9 rows in set (0.02 sec) + +``` + +### 1.5 GROUPING SETS 的组合与嵌套 + +首先，一个 GROUP BY 子句本质上是一个 GROUPING SETS 的特例, 例如： + +``` + GROUP BY a +等同于 + GROUP BY GROUPING SETS((a)) +同样地， + GROUP BY a,b,c +等同于 + GROUP BY GROUPING SETS((a,b,c)) +``` + +同样的，CUBE 和 ROLLUP 也可以展开成 GROUPING SETS，因此 GROUP BY, CUBE, ROLLUP, GROUPING SETS 的各种组合和嵌套本质上就是 GROUPING SETS 的组合与嵌套。 + +对于 GROUPING SETS 的嵌套，语义上等价于将嵌套内的语句直接写到外面。（参考：），其中写道： + +``` +The CUBE and ROLLUP constructs can be used either directly in the GROUP BY clause, or nested inside a GROUPING SETS clause. If one GROUPING SETS clause is nested inside another, the effect is the same as if all the elements of the inner clause had been written directly in the outer clause. +``` + +对于多个 GROUPING SETS 的组合列表，很多数据库认为是叉乘（cross product）的关系。 + +例如： + +``` +GROUP BY a, CUBE (b, c), GROUPING SETS ((d), (e)) + +等同于： + +GROUP BY GROUPING SETS ( +(a, b, c, d), (a, b, c, e), +(a, b, d), (a, b, e), +(a, c, d), (a, c, e), +(a, d), (a, e) +) +``` + +对于 GROUPING SETS 的组合与嵌套，各个数据库支持不太一样。例如 snowflake 不支持任何的组合和嵌套。 +（） + +Oracle 既支持组合，也支持嵌套。 +（） + +Presto 支持组合，但不支持嵌套。 +（） + +## 2. 设计目标 + +从语法上支持 GROUPING SETS， ROLLUP 和 CUBE。实现上述所述的1.1, 1.2, 1.3 1.4. + +对于1.6 GROUPING SETS 的组合与嵌套 先不实现。 + +具体语法列出如下： + +### 2.1 GROUPING SETS 语法 + +``` +SELECT ... +FROM ... +[ ... ] +GROUP BY GROUPING SETS ( groupSet [ , groupSet [ , ... ] ] ) +[ ... ] + +groupSet ::= { ( expr [ , expr [ , ... ] ] )} + + +各种表达式，包括列名. + +``` + +### 2.2 ROLLUP 语法 + +``` +SELECT ... +FROM ... +[ ... ] +GROUP BY ROLLUP ( expr [ , expr [ , ... ] ] ) +[ ... ] + + +各种表达式，包括列名. + +``` + +### 2.3 CUBE 语法 + +``` +SELECT ... +FROM ... +[ ... ] +GROUP BY CUBE ( expr [ , expr [ , ... ] ] ) +[ ... ] + + +各种表达式，包括列名. + +``` + +## 3. 实现方案 + +### 3.1 整体思路 + +既然 GROUPING SET 子句逻辑上等价于多个相应 GROUP BY 子句的 UNION，可以通过扩展输入行(此输入行已经是通过下推条件过滤和投影后的), 在此基础上进行一个单一的 GROUP BY 操作来达到目的。 + +关键是怎样扩展输入行呢？下面举例说明： + +例如，对应下面的语句： + +``` +SELECT a, b FROM src GROUP BY a, b GROUPING SETS ((a, b), (a), (b), ()); + +``` + +假定 src 表的数据如下： + +``` +1, 2 +3, 4 + +``` + +根据 GROUPING SETS 子句给出的列表，可以将输入行扩展为下面的 8 行 （GROUPING SETS集合数 * 行数, 同时为每行生成对应的 全列的GROUPING_ID: 和其他grouping 函数的值 + +``` +1, 2 (GROUPING_ID: a, b -> 00->0) +1, null (GUPING_ID: a, null -> 01 -> 1) +null, 2 (GROUPING_ID: null, b -> 10 -> 2) +null, null (GROUPING_ID: null, null -> 11 -> 3) + +3, 4 (GROUPING_ID: a, b -> 00 -> 0) +3, null (GROUPING_ID: a, null -> 01 -> 1) +null, 4 (GROUPING_ID: null, b -> 10 -> 2) +null, null (GROUPING_ID: null, null -> 11 -> 3) + +``` + +然后，将上面的 8 行数据作为输入，对 a, b, GROUPING_ID 进行 GROUP BY 操作即可。 + +### 3.2 具体例子验证说明 + +假设有一个 t 表，包含如下列和数据： + +``` +mysql> select * from t; ++------+------+------+ +| k1 | k2 | k3 | ++------+------+------+ +| a | A | 1 | +| a | A | 2 | +| a | B | 1 | +| a | B | 3 | +| b | A | 1 | +| b | A | 4 | +| b | B | 1 | +| b | B | 5 | ++------+------+------+ +8 rows in set (0.01 sec) + +``` + +对于如下的查询： + +``` +SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ()); + +``` + +首先，对输入行进行扩展，每行数据扩展成 4 行 (GROUPING SETS子句的集合数目)，同时增加 GROUPING_ID() 列 ： + +例如 a, A, 1 扩展后变成下面的 4 行： + +``` ++------+------+------+-------------------------+ +| k1 | k2 | k3 | GROUPING_ID(`k1`, `k2`) | ++------+------+------+-------------------------+ +| a | A | 1 | 0 | +| a | NULL | 1 | 1 | +| NULL | A | 1 | 2 | +| NULL | NULL | 1 | 3 | ++------+------+------+-------------------------+ + +``` + +最终， 全部扩展后的输入行如下（总共 32 行）： + +``` ++------+------+------+-------------------------+ +| k1 | k2 | k3 | GROUPING_ID(`k1`, `k2`) | ++------+------+------+-------------------------+ +| a | A | 1 | 0 | +| a | A | 2 | 0 | +| a | B | 1 | 0 | +| a | B | 3 | 0 | +| b | A | 1 | 0 | +| b | A | 4 | 0 | +| b | B | 1 | 0 | +| b | B | 5 | 0 | +| a | NULL | 1 | 1 | +| a | NULL | 1 | 1 | +| a | NULL | 2 | 1 | +| a | NULL | 3 | 1 | +| b | NULL | 1 | 1 | +| b | NULL | 1 | 1 | +| b | NULL | 4 | 1 | +| b | NULL | 5 | 1 | +| NULL | A | 1 | 2 | +| NULL | A | 1 | 2 | +| NULL | A | 2 | 2 | +| NULL | A | 4 | 2 | +| NULL | B | 1 | 2 | +| NULL | B | 1 | 2 | +| NULL | B | 3 | 2 | +| NULL | B | 5 | 2 | +| NULL | NULL | 1 | 3 | +| NULL | NULL | 1 | 3 | +| NULL | NULL | 1 | 3 | +| NULL | NULL | 1 | 3 | +| NULL | NULL | 2 | 3 | +| NULL | NULL | 3 | 3 | +| NULL | NULL | 4 | 3 | +| NULL | NULL | 5 | 3 | ++------+------+------+-------------------------+ +32 rows in set. + +``` + +现在对k1, k2, GROUPING_ID(`k1`, `k2`) 进行 GROUP BY： + +``` ++------+------+-------------------------+-----------+ +| k1 | k2 | grouping_id(`k1`, `k2`) | sum(`k3`) | ++------+------+-------------------------+-----------+ +| a | A | 0 | 3 | +| a | B | 0 | 4 | +| a | NULL | 1 | 7 | +| b | A | 0 | 5 | +| b | B | 0 | 6 | +| b | NULL | 1 | 11 | +| NULL | A | 2 | 8 | +| NULL | B | 2 | 10 | +| NULL | NULL | 3 | 18 | ++------+------+-------------------------+-----------+ +9 rows in set (0.02 sec) + +``` + +可以看到，其结果与对 GROUPING SETS 子句后每个子集进行 GROUP BY 后再进行 UNION 的结果一致。 + +``` +select k1, k2, sum(k3) from t group by k1, k2 +UNION ALL +select NULL, k2, sum(k3) from t group by k2 +UNION ALL +select k1, NULL, sum(k3) from t group by k1 +UNION ALL +select NULL, NULL, sum(k3) from t; + ++------+------+-----------+ +| k1 | k2 | sum(`k3`) | ++------+------+-----------+ +| b | B | 6 | +| b | A | 5 | +| a | A | 3 | +| a | B | 4 | +| a | NULL | 7 | +| b | NULL | 11 | +| NULL | B | 10 | +| NULL | A | 8 | +| NULL | NULL | 18 | ++------+------+-----------+ +9 rows in set (0.06 sec) + +``` + +### 3.3 FE 规划阶段 + +#### 3.3.1 主要任务 + +1. 引入 GroupByClause 类，封装 Group By 相关信息，替换原有的 groupingExprs. +2. 增加 Grouping Sets, Cube 和 RollUp 的语法支持和语法检查、错误处理和错误信息； +3. 在 SelectStmt 类中增加 GroupByClause 成员； +4. 引入 GroupingFunctionCallExpr 类，封装grouping 和grouping_id 函数调用 +5. 引入 VirtualSlot 类，封装grouping，grouping_id 生成的虚拟列和实际列的对应关系 +6. 增加虚拟列 GROUPING_ID 和其他grouping，grouping_id 函数对应的虚拟列，并将此列加入到原有的 groupingExprs 表达式列表中； +7. 增加一个 PlanNode，考虑更通用的功能，命名为 RepeatNode。对于 GroupingSets 的聚合，在执行计划中插入 RepeatNode。 + +#### 3.3.2 Tuple + +在 GroupByClause 类中为了将 GROUPING_ID 加到 groupingExprs 表达式列表中，需要创建 virtual SlotRef, 相应的，需要对这个 slot 创建一个 tuple, 叫 GROUPING_ID Tuple。 + +对于 RepeatNode 这个执行计划，其输入是子节点的所有 tuple， 输出的 tuple 除了 repeat 子节点的数据外，还需要填写 GROUPING_ID 和其他grouping，grouping_id 对应的虚拟列，因此。 + + +### 3.4 BE 查询执行阶段 + +主要任务： + +1. 通过 RepeatNode 的执行类，增加扩展输入行的逻辑，其功能是在聚合之前将原有数据进行 repeat：对每行增加一列 GROUPING_ID， 然后按照 GroupingSets 中的集合数进行 repeat，并对对应列置为 null。根据grouping list设置新增虚拟列的值 +2. 实现 grouping_id() 和grouping() 函数。 + + + + diff --git a/docs/zh-CN/internal/metadata-design.md b/docs/zh-CN/internal/metadata-design.md new file mode 100644 index 00000000000000..56e441af90e12d --- /dev/null +++ b/docs/zh-CN/internal/metadata-design.md @@ -0,0 +1,126 @@ +--- +{ + "title": "元数据设计文档", + "language": "zh-CN" +} +--- + + + +# 元数据设计文档 + +## 名词解释 + +* FE：Frontend，即 Doris 的前端节点。主要负责接收和返回客户端请求、元数据以及集群管理、查询计划生成等工作。 +* BE：Backend，即 Doris 的后端节点。主要负责数据存储与管理、查询计划执行等工作。 +* bdbje：[Oracle Berkeley DB Java Edition](http://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html)。在 Doris 中，我们使用 bdbje 完成元数据操作日志的持久化、FE 高可用等功能。 + +## 整体架构 +![](/images/palo_architecture.jpg) + +如上图，Doris 的整体架构分为两层。多个 FE 组成第一层，提供 FE 的横向扩展和高可用。多个 BE 组成第二层，负责数据存储于管理。本文主要介绍 FE 这一层中，元数据的设计与实现方式。 + +1. FE 节点分为 follower 和 observer 两类。各个 FE 之间，通过 bdbje（[BerkeleyDB Java Edition](http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/overview/index-093405.html)）进行 leader 选举，数据同步等工作。 + +2. follower 节点通过选举，其中一个 follower 成为 leader 节点，负责元数据的写入操作。当 leader 节点宕机后，其他 follower 节点会重新选举出一个 leader，保证服务的高可用。 + +3. observer 节点仅从 leader 节点进行元数据同步，不参与选举。可以横向扩展以提供元数据的读服务的扩展性。 + +> 注：follower 和 observer 对应 bdbje 中的概念为 replica 和 observer。下文可能会同时使用两种名称。 + +## 元数据结构 + +Doris 的元数据是全内存的。每个 FE 内存中，都维护一个完整的元数据镜像。在百度内部，一个包含2500张表，100万个分片（300万副本）的集群，元数据在内存中仅占用约 2GB。（当然，查询所使用的中间对象、各种作业信息等内存开销，需要根据实际情况估算。但总体依然维持在一个较低的内存开销范围内。） + +同时，元数据在内存中整体采用树状的层级结构存储，并且通过添加辅助结构，能够快速访问各个层级的元数据信息。 + +下图是 Doris 元信息所存储的内容。 + +![](/images/metadata_contents.png) + +如上图，Doris 的元数据主要存储4类数据： + +1. 用户数据信息。包括数据库、表的 Schema、分片信息等。 +2. 各类作业信息。如导入作业，Clone 作业、SchemaChange 作业等。 +3. 用户及权限信息。 +4. 集群及节点信息。 + +## 数据流 + +![](/images/metadata_stream.png) + +元数据的数据流具体过程如下： + +1. 只有 leader FE 可以对元数据进行写操作。写操作在修改 leader 的内存后，会序列化为一条log，按照 key-value 的形式写入 bdbje。其中 key 为连续的整型，作为 log id，value 即为序列化后的操作日志。 + +2. 日志写入 bdbje 后，bdbje 会根据策略（写多数/全写），将日志复制到其他 non-leader 的 FE 节点。non-leader FE 节点通过对日志回放，修改自身的元数据内存镜像，完成与 leader 节点的元数据同步。 + +3. leader 节点的日志条数达到阈值后（默认 10w 条），会启动 checkpoint 线程。checkpoint 会读取已有的 image 文件，和其之后的日志，重新在内存中回放出一份新的元数据镜像副本。然后将该副本写入到磁盘，形成一个新的 image。之所以是重新生成一份镜像副本，而不是将已有镜像写成 image，主要是考虑写 image 加读锁期间，会阻塞写操作。所以每次 checkpoint 会占用双倍内存空间。 + +4. image 文件生成后，leader 节点会通知其他 non-leader 节点新的 image 已生成。non-leader 主动通过 http 拉取最新的 image 文件，来更换本地的旧文件。 + +5. bdbje 中的日志，在 image 做完后，会定期删除旧的日志。 + +## 实现细节 + +### 元数据目录 + +1. 元数据目录通过 FE 的配置项 `meta_dir` 指定。 + +2. `bdb/` 目录下为 bdbje 的数据存放目录。 + +3. `image/` 目录下为 image 文件的存放目录。 + + * `image.[logid]` 是最新的 image 文件。后缀 `logid` 表明 image 所包含的最后一条日志的 id。 + * `image.ckpt` 是正在写入的 image 文件，如果写入成功，会重命名为 `image.[logid]`，并替换掉旧的 image 文件。 + * `VERSION` 文件中记录着 `cluster_id`。`cluster_id` 唯一标识一个 Doris 集群。是在 leader 第一次启动时随机生成的一个 32 位整型。也可以通过 fe 配置项 `cluster_id` 来指定一个 cluster id。 + * `ROLE` 文件中记录的 FE 自身的角色。只有 `FOLLOWER` 和 `OBSERVER` 两种。其中 `FOLLOWER` 表示 FE 为一个可选举的节点。（注意：即使是 leader 节点，其角色也为 `FOLLOWER`） + +### 启动流程 + +1. FE 第一次启动，如果启动脚本不加任何参数，则会尝试以 leader 的身份启动。在 FE 启动日志中会最终看到 `transfer from UNKNOWN to MASTER`。 + +2. FE 第一次启动，如果启动脚本中指定了 `-helper` 参数，并且指向了正确的 leader FE 节点，那么该 FE 首先会通过 http 向 leader 节点询问自身的角色（即 ROLE）和 cluster_id。然后拉取最新的 image 文件。读取 image 文件，生成元数据镜像后，启动 bdbje，开始进行 bdbje 日志同步。同步完成后，开始回放 bdbje 中，image 文件之后的日志，完成最终的元数据镜像生成。 + + > 注1：使用 `-helper` 参数启动时，需要首先通过 mysql 命令，通过 leader 来添加该 FE，否则，启动时会报错。 + + > 注2：`-helper` 可以指向任何一个 follower 节点，即使它不是 leader。 + + > 注2：bdbje 在同步日志过程中，fe 日志会显示 `xxx detached`, 此时正在进行日志拉取，属于正常现象。 + +3. FE 非第一次启动，如果启动脚本不加任何参数，则会根据本地存储的 ROLE 信息，来确定自己的身份。同时根据本地 bdbje 中存储的集群信息，获取 leader 的信息。然后读取本地的 image 文件，以及 bdbje 中的日志，完成元数据镜像生成。（如果本地 ROLE 中记录的角色和 bdbje 中记录的不一致，则会报错。） + +4. FE 非第一次启动，且启动脚本中指定了 `-helper` 参数。则和第一次启动的流程一样，也会先去询问 leader 角色。但是会和自身存储的 ROLE 进行比较。如果不一致，则会报错。 + +#### 元数据读写与同步 + +1. 用户可以使用 mysql 连接任意一个 FE 节点进行元数据的读写访问。如果连接的是 non-leader 节点，则该节点会将写操作转发给 leader 节点。leader 写成功后，会返回一个 leader 当前最新的 log id。之后，non-leader 节点会等待自身回放的 log id 大于回传的 log id 后，才将命令成功的消息返回给客户端。这种方式保证了任意 FE 节点的 Read-Your-Write 语义。 + + > 注：一些非写操作，也会转发给 leader 执行。比如 `SHOW LOAD` 操作。因为这些命令通常需要读取一些作业的中间状态，而这些中间状态是不写 bdbje 的，因此 non-leader 节点的内存中，是没有这些中间状态的。（FE 之间的元数据同步完全依赖 bdbje 的日志回放，如果一个元数据修改操作不写 bdbje 日志，则在其他 non-leader 节点中是看不到该操作修改后的结果的。） + +2. leader 节点会启动一个 TimePrinter 线程。该线程会定期向 bdbje 中写入一个当前时间的 key-value 条目。其余 non-leader 节点通过回放这条日志，读取日志中记录的时间，和本地时间进行比较，如果发现和本地时间的落后大于指定的阈值（配置项：`meta_delay_toleration_second`。写入间隔为该配置项的一半），则该节点会处于**不可读**的状态。此机制解决了 non-leader 节点在长时间和 leader 失联后，仍然提供过期的元数据服务的问题。 + +3. 各个 FE 的元数据只保证最终一致性。正常情况下，不一致的窗口期仅为毫秒级。我们保证同一 session 中，元数据访问的单调一致性。但是如果同一 client 连接不同 FE，则可能出现元数据回退的现象。（但对于批量更新系统，该问题影响很小。） + +### 宕机恢复 + +1. leader 节点宕机后，其余 follower 会立即选举出一个新的 leader 节点提供服务。 +2. 当多数 follower 节点宕机时，元数据不可写入。当元数据处于不可写入状态下，如果这时发生写操作请求，目前的处理流程是 **FE 进程直接退出**。后续会优化这个逻辑，在不可写状态下，依然提供读服务。 +3. observer 节点宕机，不会影响任何其他节点的状态。也不会影响元数据在其他节点的读写。 diff --git a/docs/documentation/cn/internal/spark_load.md b/docs/zh-CN/internal/spark_load.md similarity index 99% rename from docs/documentation/cn/internal/spark_load.md rename to docs/zh-CN/internal/spark_load.md index 021b02b35c8b3e..654d415862ffe0 100644 --- a/docs/documentation/cn/internal/spark_load.md +++ b/docs/zh-CN/internal/spark_load.md @@ -1,3 +1,10 @@ +--- +{ + "title": "Doris支持spark导入设计文档", + "language": "zh-CN" +} +--- + + +# AVG +## description +### Syntax + +`AVG([DISTINCT] expr)` + + +用于返回选中字段的平均值 + +可选字段DISTINCT参数可以用来返回去重平均值 + +## example + +``` +mysql> SELECT datetime, AVG(cost_time) FROM log_statis group by datetime; ++---------------------+--------------------+ +| datetime | avg(`cost_time`) | ++---------------------+--------------------+ +| 2019-07-03 21:01:20 | 25.827794561933533 | ++---------------------+--------------------+ + +mysql> SELECT datetime, AVG(distinct cost_time) FROM log_statis group by datetime; ++---------------------+---------------------------+ +| datetime | avg(DISTINCT `cost_time`) | ++---------------------+---------------------------+ +| 2019-07-04 02:23:24 | 20.666666666666668 | ++---------------------+---------------------------+ + +``` +##keyword +AVG diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/bitmap.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/bitmap.md new file mode 100644 index 00000000000000..a9ba0e42985a45 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/bitmap.md @@ -0,0 +1,146 @@ +--- +{ + "title": "BITMAP", + "language": "zh-CN" +} +--- + + + +# BITMAP + +## Create table + +建表时需要使用聚合模型，数据类型是 bitmap , 聚合函数是 bitmap_union + +``` +CREATE TABLE `pv_bitmap` ( + `dt` int(11) NULL COMMENT "", + `page` varchar(10) NULL COMMENT "", + `user_id` bitmap BITMAP_UNION NULL COMMENT "" +) ENGINE=OLAP +AGGREGATE KEY(`dt`, `page`) +COMMENT "OLAP" +DISTRIBUTED BY HASH(`dt`) BUCKETS 2; +``` +注：当数据量很大时，最好为高频率的 bitmap_union 查询建立对应的 rollup 表 + +``` +ALTER TABLE pv_bitmap ADD ROLLUP pv (page, user_id); +``` + +## Data Load + +`TO_BITMAP(expr)` : 将 0 ~ 18446744073709551615 的 unsigned bigint 转为 bitmap + +`BITMAP_EMPTY()`: 生成空 bitmap 列，用于 insert 或导入的时填充默认值 + +`BITMAP_HASH(expr)`: 将任意类型的列通过 Hash 的方式转为 bitmap + +### Stream Load + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=to_bitmap(user_id)" http://host:8410/api/test/testDb/_stream_load +``` + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=bitmap_hash(user_id)" http://host:8410/api/test/testDb/_stream_load +``` + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=bitmap_empty()" http://host:8410/api/test/testDb/_stream_load +``` + +### Insert Into + +id2 的列类型是 bitmap +``` +insert into bitmap_table1 select id, id2 from bitmap_table2; +``` + +id2 的列类型是 bitmap +``` +INSERT INTO bitmap_table1 (id, id2) VALUES (1001, to_bitmap(1000)), (1001, to_bitmap(2000)); +``` + +id2 的列类型是 bitmap +``` +insert into bitmap_table1 select id, bitmap_union(id2) from bitmap_table2 group by id; +``` + +id2 的列类型是 int +``` +insert into bitmap_table1 select id, to_bitmap(id2) from table; +``` + +id2 的列类型是 String +``` +insert into bitmap_table1 select id, bitmap_hash(id_string) from table; +``` + + +## Data Query +### Syntax + + +`BITMAP_UNION(expr)` : 计算输入 Bitmap 的并集，返回新的bitmap + +`BITMAP_UNION_COUNT(expr)`: 计算输入 Bitmap 的并集，返回其基数，和 BITMAP_COUNT(BITMAP_UNION(expr)) 等价。目前推荐优先使用 BITMAP_UNION_COUNT ，其性能优于 BITMAP_COUNT(BITMAP_UNION(expr)) + +`BITMAP_UNION_INT(expr)` : 计算 TINYINT,SMALLINT 和 INT 类型的列中不同值的个数，返回值和 +COUNT(DISTINCT expr) 相同 + +`INTERSECT_COUNT(bitmap_column_to_count, filter_column, filter_values ...)` : 计算满足 +filter_column 过滤条件的多个 bitmap 的交集的基数值。 +bitmap_column_to_count 是 bitmap 类型的列，filter_column 是变化的维度列，filter_values 是维度取值列表 + + +### Example + +下面的 SQL 以上面的 pv_bitmap table 为例： + +计算 user_id 的去重值： + +``` +select bitmap_union_count(user_id) from pv_bitmap; + +select bitmap_count(bitmap_union(user_id)) from pv_bitmap; +``` + +计算 id 的去重值： + +``` +select bitmap_union_int(id) from pv_bitmap; +``` + +计算 user_id 的 留存: + +``` +select intersect_count(user_id, page, 'meituan') as meituan_uv, +intersect_count(user_id, page, 'waimai') as waimai_uv, +intersect_count(user_id, page, 'meituan', 'waimai') as retention //在 'meituan' 和 'waimai' 两个页面都出现的用户数 +from pv_bitmap +where page in ('meituan', 'waimai'); +``` + + +## keyword + +BITMAP,BITMAP_COUNT,BITMAP_EMPTY,BITMAP_UNION,BITMAP_UNION_INT,TO_BITMAP,BITMAP_UNION_COUNT,INTERSECT_COUNT \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/count.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/count.md new file mode 100755 index 00000000000000..7e126174af56fa --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/count.md @@ -0,0 +1,61 @@ +--- +{ + "title": "COUNT", + "language": "zh-CN" +} +--- + + + +# COUNT +## description +### Syntax + +`COUNT([DISTINCT] expr)` + + +用于返回满足要求的行的数目 + +## example + +``` +MySQL > select count(*) from log_statis group by datetime; ++----------+ +| count(*) | ++----------+ +| 28515903 | ++----------+ + +MySQL > select count(datetime) from log_statis group by datetime; ++-------------------+ +| count(`datetime`) | ++-------------------+ +| 28521682 | ++-------------------+ + +MySQL > select count(distinct datetime) from log_statis group by datetime; ++-------------------------------+ +| count(DISTINCT `datetime`) | ++-------------------------------+ +| 71045 | ++-------------------------------+ +``` +##keyword +COUNT diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md new file mode 100644 index 00000000000000..db1136ed37c90e --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/hll_union_agg.md @@ -0,0 +1,52 @@ +--- +{ + "title": "HLL_UNION_AGG", + "language": "zh-CN" +} +--- + + + +# HLL_UNION_AGG +## description +### Syntax + +`HLL_UNION_AGG(hll)` + + +HLL是基于HyperLogLog算法的工程实现，用于保存HyperLogLog计算过程的中间结果 + +它只能作为表的value列类型、通过聚合来不断的减少数据量，以此来实现加快查询的目的 + +基于它得到的是一个估算结果，误差大概在1%左右，hll列是通过其它列或者导入数据里面的数据生成的 + +导入的时候通过hll_hash函数来指定数据中哪一列用于生成hll列，它常用于替代count distinct，通过结合rollup在业务上用于快速计算uv等 + +## example +``` +MySQL > select HLL_UNION_AGG(uv_set) from test_uv;; ++-------------------------+ +| HLL_UNION_AGG(`uv_set`) | ++-------------------------+ +| 17721 | ++-------------------------+ +``` +##keyword +HLL_UNION_AGG,HLL,UNION,AGG diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/max.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/max.md new file mode 100755 index 00000000000000..5ba36fb9eb67e7 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/max.md @@ -0,0 +1,46 @@ +--- +{ + "title": "MAX", + "language": "zh-CN" +} +--- + + + +# MAX +## description +### Syntax + +`MAX(expr)` + + +返回expr表达式的最大值 + +## example +``` +MySQL > select max(scan_rows) from log_statis group by datetime; ++------------------+ +| max(`scan_rows`) | ++------------------+ +| 4671587 | ++------------------+ +``` +##keyword +MAX diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/min.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/min.md new file mode 100755 index 00000000000000..b34d62b5bbcd43 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/min.md @@ -0,0 +1,46 @@ +--- +{ + "title": "MIN", + "language": "zh-CN" +} +--- + + + +# MIN +## description +### Syntax + +`MIN(expr)` + + +返回expr表达式的最小值 + +## example +``` +MySQL > select min(scan_rows) from log_statis group by datetime; ++------------------+ +| min(`scan_rows`) | ++------------------+ +| 0 | ++------------------+ +``` +##keyword +MIN diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/ndv.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/ndv.md new file mode 100644 index 00000000000000..c2e857e98d1e3b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/ndv.md @@ -0,0 +1,48 @@ +--- +{ + "title": "NDV", + "language": "zh-CN" +} +--- + + + +# NDV +## description +### Syntax + +`NDV(expr)` + + +返回类似于 COUNT(DISTINCT col) 结果的近似值聚合函数。 + +它比 COUNT 和 DISTINCT 组合的速度更快，并使用固定大小的内存，因此对于高基数的列可以使用更少的内存。 + +## example +``` +MySQL > select ndv(query_id) from log_statis group by datetime; ++-----------------+ +| ndv(`query_id`) | ++-----------------+ +| 17721 | ++-----------------+ +``` +##keyword +NDV diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/percentile_approx.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/percentile_approx.md new file mode 100755 index 00000000000000..3f65f6f64f6079 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/percentile_approx.md @@ -0,0 +1,57 @@ +--- +{ + "title": "PERCENTILE_APPROX", + "language": "zh-CN" +} +--- + + + +# PERCENTILE_APPROX +## description +### Syntax + +`PERCENTILE_APPROX(expr, DOUBLE p[, DOUBLE compression])` + + +返回第p个百分位点的近似值，p的值介于0到1之间 + +compression参数是可选项，可设置范围是[2048, 10000]，值越大，精度越高，内存消耗越大，计算耗时越长。 +compression参数未指定或设置的值在[2048, 10000]范围外，以10000的默认值运行 + +该函数使用固定大小的内存，因此对于高基数的列可以使用更少的内存，可用于计算tp99等统计值 + +## example +``` +MySQL > select `table`, percentile_approx(cost_time,0.99) from log_statis group by `table`; ++---------------------+---------------------------+ +| table | percentile_approx(`cost_time`, 0.99) | ++----------+--------------------------------------+ +| test | 54.22 | ++----------+--------------------------------------+ + +MySQL > select `table`, percentile_approx(cost_time,0.99, 4096) from log_statis group by `table`; ++---------------------+---------------------------+ +| table | percentile_approx(`cost_time`, 0.99, 4096.0) | ++----------+--------------------------------------+ +| test | 54.21 | ++----------+--------------------------------------+ +##keyword +PERCENTILE_APPROX,PERCENTILE,APPROX diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/stddev.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/stddev.md new file mode 100755 index 00000000000000..524dd44f4d02dd --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/stddev.md @@ -0,0 +1,53 @@ +--- +{ + "title": "STDDEV,STDDEV_POP", + "language": "zh-CN" +} +--- + + + +# STDDEV,STDDEV_POP +## description +### Syntax + +`STDDEV(expr)` + + +返回expr表达式的标准差 + +## example +``` +MySQL > select stddev(scan_rows) from log_statis group by datetime; ++---------------------+ +| stddev(`scan_rows`) | ++---------------------+ +| 2.3736656687790934 | ++---------------------+ + +MySQL > select stddev_pop(scan_rows) from log_statis group by datetime; ++-------------------------+ +| stddev_pop(`scan_rows`) | ++-------------------------+ +| 2.3722760595994914 | ++-------------------------+ +``` +##keyword +STDDEV,STDDEV_POP,POP diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/stddev_samp.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/stddev_samp.md new file mode 100755 index 00000000000000..65a2232017265f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/stddev_samp.md @@ -0,0 +1,46 @@ +--- +{ + "title": "STDDEV_SAMP", + "language": "zh-CN" +} +--- + + + +# STDDEV_SAMP +## description +### Syntax + +`STDDEV_SAMP(expr)` + + +返回expr表达式的样本标准差 + +## example +``` +MySQL > select stddev_samp(scan_rows) from log_statis group by datetime; ++--------------------------+ +| stddev_samp(`scan_rows`) | ++--------------------------+ +| 2.372044195280762 | ++--------------------------+ +``` +##keyword +STDDEV_SAMP,STDDEV,SAMP diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/sum.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/sum.md new file mode 100755 index 00000000000000..8f6151957fea8c --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/sum.md @@ -0,0 +1,46 @@ +--- +{ + "title": "SUM", + "language": "zh-CN" +} +--- + + + +# SUM +## description +### Syntax + +`SUM(expr)` + + +用于返回选中字段所有值的和 + +## example +``` +MySQL > select sum(scan_rows) from log_statis group by datetime; ++------------------+ +| sum(`scan_rows`) | ++------------------+ +| 8217360135 | ++------------------+ +``` +##keyword +SUM diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/var_samp.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/var_samp.md new file mode 100755 index 00000000000000..74426414ff8cb6 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/var_samp.md @@ -0,0 +1,45 @@ +--- +{ + "title": "VAR_SAMP,VARIANCE_SAMP", + "language": "zh-CN" +} +--- + + + +# VAR_SAMP,VARIANCE_SAMP +## description +### Syntax + +`VAR_SAMP(expr)` + + +返回expr表达式的样本方差 + +## example +``` +MySQL > select var_samp(scan_rows) from log_statis group by datetime; ++-----------------------+ +| var_samp(`scan_rows`) | ++-----------------------+ +| 5.6227132145741789 | ++-----------------------+ +##keyword +VAR_SAMP,VARIANCE_SAMP,VAR,SAMP,VARIANCE diff --git a/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/variance.md b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/variance.md new file mode 100755 index 00000000000000..c97c227e911b93 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/aggregate-functions/variance.md @@ -0,0 +1,52 @@ +--- +{ + "title": "VARIANCE,VAR_POP,VARIANCE_POP", + "language": "zh-CN" +} +--- + + + +# VARIANCE,VAR_POP,VARIANCE_POP +## description +### Syntax + +`VARIANCE(expr)` + + +返回expr表达式的方差 + +## example +``` +MySQL > select variance(scan_rows) from log_statis group by datetime; ++-----------------------+ +| variance(`scan_rows`) | ++-----------------------+ +| 5.6183332881176211 | ++-----------------------+ + +MySQL > select var_pop(scan_rows) from log_statis group by datetime; ++----------------------+ +| var_pop(`scan_rows`) | ++----------------------+ +| 5.6230744719006163 | ++----------------------+ +##keyword +VARIANCE,VAR_POP,VARIANCE_POP,VAR,POP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_and.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_and.md new file mode 100644 index 00000000000000..3bb2b5c5888b86 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_and.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_and", + "language": "zh-CN" +} +--- + + + +# bitmap_and +## description +### Syntax + +`BITMAP BITMAP_AND(BITMAP lhs, BITMAP rhs)` + +计算两个输入bitmap的交集，返回新的bitmap. + +## example + +``` +mysql> select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(2))) cnt; ++------+ +| cnt | ++------+ +| 0 | ++------+ + +mysql> select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(1))) cnt; ++------+ +| cnt | ++------+ +| 1 | ++------+ +``` + +## keyword + + BITMAP_AND,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md new file mode 100644 index 00000000000000..1f3f67c928d706 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_contains.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_contains", + "language": "zh-CN" +} +--- + + + +# bitmap_contains +## description +### Syntax + +`B00LEAN BITMAP_CONTAINS(BITMAP bitmap, BIGINT input)` + +计算输入值是否在Bitmap列中，返回值是Boolean值. + +## example + +``` +mysql> select bitmap_contains(to_bitmap(1),2) cnt; ++------+ +| cnt | ++------+ +| 0 | ++------+ + +mysql> select bitmap_contains(to_bitmap(1),1) cnt; ++------+ +| cnt | ++------+ +| 1 | ++------+ +``` + +## keyword + + BITMAP_CONTAINS,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md new file mode 100644 index 00000000000000..e6d3d72707d0ad --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_empty.md @@ -0,0 +1,52 @@ +--- +{ + "title": "bitmap_empty", + "language": "zh-CN" +} +--- + + + +# bitmap_empty +## description +### Syntax + +`BITMAP BITMAP_EMPTY()` + +返回一个空bitmap。主要用于 insert 或 stream load 时填充默认值。例如 + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,v1,v2=bitmap_empty()" http://host:8410/api/test/testDb/_stream_load +``` + +## example + +``` +mysql> select bitmap_count(bitmap_empty()); ++------------------------------+ +| bitmap_count(bitmap_empty()) | ++------------------------------+ +| 0 | ++------------------------------+ +``` + +## keyword + + BITMAP_EMPTY,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md new file mode 100644 index 00000000000000..ba7c967ec9e19c --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md @@ -0,0 +1,63 @@ +--- +{ + "title": "bitmap_from_string", + "language": "zh-CN" +} +--- + + + +# bitmap_from_string + +## description +### Syntax + +`BITMAP BITMAP_FROM_STRING(VARCHAR input)` + +将一个字符串转化为一个BITAMP，字符串是由逗号分隔的一组UINT32数字组成. +比如"0, 1, 2"字符串会转化为一个Bitmap，其中的第0, 1, 2位被设置. +当输入字段不合法时，返回NULL + +## example + +``` +mysql> select bitmap_to_string(bitmap_empty()); ++----------------------------------+ +| bitmap_to_string(bitmap_empty()) | ++----------------------------------+ +| | ++----------------------------------+ + +mysql> select bitmap_to_string(bitmap_from_string("0, 1, 2")); ++-------------------------------------------------+ +| bitmap_to_string(bitmap_from_string('0, 1, 2')) | ++-------------------------------------------------+ +| 0,1,2 | ++-------------------------------------------------+ + +mysql> select bitmap_from_string("-1, 0, 1, 2"); ++-----------------------------------+ +| bitmap_from_string('-1, 0, 1, 2') | ++-----------------------------------+ +| NULL | ++-----------------------------------+ +``` + +## keyword + + BITMAP_FROM_STRING,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md new file mode 100644 index 00000000000000..13fc4217322241 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_has_any.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_has_any", + "language": "zh-CN" +} +--- + + + +# bitmap_has_any +## description +### Syntax + +`B00LEAN BITMAP_HAS_ANY(BITMAP lhs, BITMAP rhs)` + +计算两个Bitmap列是否存在相交元素，返回值是Boolean值. + +## example + +``` +mysql> select bitmap_has_any(to_bitmap(1),to_bitmap(2)) cnt; ++------+ +| cnt | ++------+ +| 0 | ++------+ + +mysql> select bitmap_has_any(to_bitmap(1),to_bitmap(1)) cnt; ++------+ +| cnt | ++------+ +| 1 | ++------+ +``` + +## keyword + + BITMAP_HAS_ANY,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md new file mode 100644 index 00000000000000..76eda65e913d9d --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_hash.md @@ -0,0 +1,52 @@ +--- +{ + "title": "bitmap_hash", + "language": "zh-CN" +} +--- + + + +# bitmap_hash +## description +### Syntax + +`BITMAP BITMAP_HASH(expr)` + +对任意类型的输入计算32位的哈希值，返回包含该哈希值的bitmap。主要用于stream load任务将非整型字段导入Doris表的bitmap字段。例如 + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,device_id, device_id=bitmap_hash(device_id)" http://host:8410/api/test/testDb/_stream_load +``` + +## example + +``` +mysql> select bitmap_count(bitmap_hash('hello')); ++------------------------------------+ +| bitmap_count(bitmap_hash('hello')) | ++------------------------------------+ +| 1 | ++------------------------------------+ +``` + +## keyword + + BITMAP_HASH,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_or.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_or.md new file mode 100644 index 00000000000000..0ba47431486ec7 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_or.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_or", + "language": "zh-CN" +} +--- + + + +# bitmap_or +## description +### Syntax + +`BITMAP BITMAP_OR(BITMAP lhs, BITMAP rhs)` + +计算两个输入bitmap的并集，返回新的bitmap. + +## example + +``` +mysql> select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt; ++------+ +| cnt | ++------+ +| 2 | ++------+ + +mysql> select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt; ++------+ +| cnt | ++------+ +| 1 | ++------+ +``` + +## keyword + + BITMAP_OR,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md new file mode 100644 index 00000000000000..ba447171f6ffe0 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md @@ -0,0 +1,69 @@ +--- +{ + "title": "bitmap_to_string", + "language": "zh-CN" +} +--- + + + +# bitmap_to_string + +## description +### Syntax + +`VARCHAR BITMAP_TO_STRING(BITMAP input)` + +将一个bitmap转化成一个逗号分隔的字符串，字符串中包含所有设置的BIT位。输入是null的话会返回null。 + +## example + +``` +mysql> select bitmap_to_string(null); ++------------------------+ +| bitmap_to_string(NULL) | ++------------------------+ +| NULL | ++------------------------+ + +mysql> select bitmap_to_string(bitmap_empty()); ++----------------------------------+ +| bitmap_to_string(bitmap_empty()) | ++----------------------------------+ +| | ++----------------------------------+ + +mysql> select bitmap_to_string(to_bitmap(1)); ++--------------------------------+ +| bitmap_to_string(to_bitmap(1)) | ++--------------------------------+ +| 1 | ++--------------------------------+ + +mysql> select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))); ++---------------------------------------------------------+ +| bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) | ++---------------------------------------------------------+ +| 1,2 | ++---------------------------------------------------------+ + +``` + +## keyword + + BITMAP_TO_STRING,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/to_bitmap.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/to_bitmap.md new file mode 100644 index 00000000000000..feae5d3fe93a39 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/to_bitmap.md @@ -0,0 +1,53 @@ +--- +{ + "title": "to_bitmap", + "language": "zh-CN" +} +--- + + + +# to_bitmap +## description +### Syntax + +`BITMAP TO_BITMAP(expr)` + +输入为取值在 0 ~ 18446744073709551615 区间的 unsigned bigint ，输出为包含该元素的bitmap。 +该函数主要用于stream load任务将整型字段导入Doris表的bitmap字段。例如 + +``` +cat data | curl --location-trusted -u user:passwd -T - -H "columns: dt,page,user_id, user_id=to_bitmap(user_id)" http://host:8410/api/test/testDb/_stream_load +``` + +## example + +``` +mysql> select bitmap_count(to_bitmap(10)); ++-----------------------------+ +| bitmap_count(to_bitmap(10)) | ++-----------------------------+ +| 1 | ++-----------------------------+ +``` + +## keyword + + TO_BITMAP,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/cast.md b/docs/zh-CN/sql-reference/sql-functions/cast.md new file mode 100644 index 00000000000000..40ca5bf60ab17b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/cast.md @@ -0,0 +1,82 @@ +--- +{ + "title": "CAST", + "language": "zh-CN" +} +--- + + + +# CAST +## description +### Syntax + +``` +cast (input as type) +``` + +### BIGINT type + +### Syntax(BIGINT) + +``` cast (input as BIGINT) ``` + + +将 input 转成 指定的 type + + +将当前列 input 转换为 BIGINT 类型 + +## example + +1. 转常量，或表中某列 + +``` +mysql> select cast (1 as BIGINT); ++-------------------+ +| CAST(1 AS BIGINT) | ++-------------------+ +| 1 | ++-------------------+ +``` + +2. 转导入的原始数据 + +``` +curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(tmp_k1 as BIGINT)" http://host:port/api/test/bigint/_stream_load +``` + +*注：在导入中，由于原始类型均为String，将值为浮点的原始数据做 cast的时候数据会被转换成 NULL ，比如 12.0 。Doris目前不会对原始数据做截断。* + +如果想强制将这种类型的原始数据 cast to int 的话。请看下面写法： + +``` +curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(cast(tmp_k1 as DOUBLE) as BIGINT)" http://host:port/api/test/bigint/_stream_load + +mysql> select cast(cast ("11.2" as double) as bigint); ++----------------------------------------+ +| CAST(CAST('11.2' AS DOUBLE) AS BIGINT) | ++----------------------------------------+ +| 11 | ++----------------------------------------+ +1 row in set (0.00 sec) +``` +##keyword +CAST diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/convert_tz.md similarity index 96% rename from docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md rename to docs/zh-CN/sql-reference/sql-functions/date-time-functions/convert_tz.md index 08cf7b74ab6dfc..b58783757f33f4 100644 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/convert_tz.md @@ -1,3 +1,10 @@ +--- +{ + "title": "convert_tz", + "language": "zh-CN" +} +--- + + +# curdate +## description +### Syntax + +`DATE CURDATE()` + +获取当前的日期，以DATE类型返回 + +## Examples + +``` +mysql> SELECT CURDATE(); ++------------+ +| CURDATE() | ++------------+ +| 2019-12-20 | ++------------+ + +mysql> SELECT CURDATE() + 0; ++---------------+ +| CURDATE() + 0 | ++---------------+ +| 20191220 | ++---------------+ +``` + +##keyword + + CURDATE diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/current_timestamp.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/current_timestamp.md new file mode 100644 index 00000000000000..1a1e99df3599f9 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/current_timestamp.md @@ -0,0 +1,49 @@ +--- +{ + "title": "current_timestamp", + "language": "zh-CN" +} +--- + + + +# current_timestamp +## description +### Syntax + +`DATETIME CURRENT_TIMESTAMP()` + + +获得当前的时间，以Datetime类型返回 + +## example + +``` +mysql> select current_timestamp(); ++---------------------+ +| current_timestamp() | ++---------------------+ +| 2019-05-27 15:59:33 | ++---------------------+ +``` + +## keyword + + CURRENT_TIMESTAMP,CURRENT,TIMESTAMP diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/curtime.md similarity index 93% rename from docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md rename to docs/zh-CN/sql-reference/sql-functions/date-time-functions/curtime.md index 3632abf2cf9716..19ed78811bcf3f 100644 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/curtime.md @@ -1,3 +1,10 @@ +--- +{ + "title": "curtime,current_time", + "language": "zh-CN" +} +--- + + +# date_add +## description +### Syntax + +`INT DATE_ADD(DATETIME date,INTERVAL expr type)` + + +向日期添加指定的时间间隔。 + +date 参数是合法的日期表达式。 + +expr 参数是您希望添加的时间间隔。 + +type 参数可以是下列值：YEAR, MONTH, DAY, HOUR, MINUTE, SECOND + +## example + +``` +mysql> select date_add('2010-11-30 23:59:59', INTERVAL 2 DAY); ++-------------------------------------------------+ +| date_add('2010-11-30 23:59:59', INTERVAL 2 DAY) | ++-------------------------------------------------+ +| 2010-12-02 23:59:59 | ++-------------------------------------------------+ +``` + +## keyword + + DATE_ADD,DATE,ADD diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/date_format.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/date_format.md new file mode 100644 index 00000000000000..b15929e0b9757f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/date_format.md @@ -0,0 +1,160 @@ +--- +{ + "title": "date_format", + "language": "zh-CN" +} +--- + + + +# date_format +## description +### Syntax + +`VARCHAR DATE_FORMAT(DATETIME date, VARCHAR format)` + + +将日期类型按照format的类型转化为字符串， +当前支持最大128字节的字符串，如果返回值长度超过128，则返回NULL + +date 参数是合法的日期。format 规定日期/时间的输出格式。 + +可以使用的格式有： + +%a | 缩写星期名 + +%b | 缩写月名 + +%c | 月，数值 + +%D | 带有英文前缀的月中的天 + +%d | 月的天，数值(00-31) + +%e | 月的天，数值(0-31) + +%f | 微秒 + +%H | 小时 (00-23) + +%h | 小时 (01-12) + +%I | 小时 (01-12) + +%i | 分钟，数值(00-59) + +%j | 年的天 (001-366) + +%k | 小时 (0-23) + +%l | 小时 (1-12) + +%M | 月名 + +%m | 月，数值(00-12) + +%p | AM 或 PM + +%r | 时间，12-小时（hh:mm:ss AM 或 PM） + +%S | 秒(00-59) + +%s | 秒(00-59) + +%T | 时间, 24-小时 (hh:mm:ss) + +%U | 周 (00-53) 星期日是一周的第一天 + +%u | 周 (00-53) 星期一是一周的第一天 + +%V | 周 (01-53) 星期日是一周的第一天，与 %X 使用 + +%v | 周 (01-53) 星期一是一周的第一天，与 %x 使用 + +%W | 星期名 + +%w | 周的天 （0=星期日, 6=星期六） + +%X | 年，其中的星期日是周的第一天，4 位，与 %V 使用 + +%x | 年，其中的星期一是周的第一天，4 位，与 %v 使用 + +%Y | 年，4 位 + +%y | 年，2 位 + +%% | 用于表示 % + +## example + +``` +mysql> select date_format('2009-10-04 22:23:00', '%W %M %Y'); ++------------------------------------------------+ +| date_format('2009-10-04 22:23:00', '%W %M %Y') | ++------------------------------------------------+ +| Sunday October 2009 | ++------------------------------------------------+ + +mysql> select date_format('2007-10-04 22:23:00', '%H:%i:%s'); ++------------------------------------------------+ +| date_format('2007-10-04 22:23:00', '%H:%i:%s') | ++------------------------------------------------+ +| 22:23:00 | ++------------------------------------------------+ + +mysql> select date_format('1900-10-04 22:23:00', '%D %y %a %d %m %b %j'); ++------------------------------------------------------------+ +| date_format('1900-10-04 22:23:00', '%D %y %a %d %m %b %j') | ++------------------------------------------------------------+ +| 4th 00 Thu 04 10 Oct 277 | ++------------------------------------------------------------+ + +mysql> select date_format('1997-10-04 22:23:00', '%H %k %I %r %T %S %w'); ++------------------------------------------------------------+ +| date_format('1997-10-04 22:23:00', '%H %k %I %r %T %S %w') | ++------------------------------------------------------------+ +| 22 22 10 10:23:00 PM 22:23:00 00 6 | ++------------------------------------------------------------+ + +mysql> select date_format('1999-01-01 00:00:00', '%X %V'); ++---------------------------------------------+ +| date_format('1999-01-01 00:00:00', '%X %V') | ++---------------------------------------------+ +| 1998 52 | ++---------------------------------------------+ + +mysql> select date_format('2006-06-01', '%d'); ++------------------------------------------+ +| date_format('2006-06-01 00:00:00', '%d') | ++------------------------------------------+ +| 01 | ++------------------------------------------+ + +mysql> select date_format('2006-06-01', '%%%d'); ++--------------------------------------------+ +| date_format('2006-06-01 00:00:00', '%%%d') | ++--------------------------------------------+ +| %01 | ++--------------------------------------------+ +``` + +## keyword + + DATE_FORMAT,DATE,FORMAT diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/date_sub.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/date_sub.md new file mode 100644 index 00000000000000..93c09311c1d441 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/date_sub.md @@ -0,0 +1,55 @@ +--- +{ + "title": "date_sub", + "language": "zh-CN" +} +--- + + + +# date_sub +## description +### Syntax + +`INT DATE_SUB(DATETIME date,INTERVAL expr type)` + + +从日期减去指定的时间间隔 + +date 参数是合法的日期表达式。 + +expr 参数是您希望添加的时间间隔。 + +type 参数可以是下列值：YEAR, MONTH, DAY, HOUR, MINUTE, SECOND + +## example + +``` +mysql> select date_sub('2010-11-30 23:59:59', INTERVAL 2 DAY); ++-------------------------------------------------+ +| date_sub('2010-11-30 23:59:59', INTERVAL 2 DAY) | ++-------------------------------------------------+ +| 2010-11-28 23:59:59 | ++-------------------------------------------------+ +``` + +##keyword + + DATE_SUB,DATE,SUB diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/datediff.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/datediff.md new file mode 100644 index 00000000000000..b60eee380544b0 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/datediff.md @@ -0,0 +1,58 @@ +--- +{ + "title": "datediff", + "language": "zh-CN" +} +--- + + + +# datediff +## description +### Syntax + +`DATETIME DATEDIFF(DATETIME expr1,DATETIME expr2)` + + +计算expr1 - expr2，结果精确到天。 + +expr1 和 expr2 参数是合法的日期或日期/时间表达式。 + +注释：只有值的日期部分参与计算。 + +## example + +``` +mysql> select datediff(CAST('2007-12-31 23:59:59' AS DATETIME), CAST('2007-12-30' AS DATETIME)); ++-----------------------------------------------------------------------------------+ +| datediff(CAST('2007-12-31 23:59:59' AS DATETIME), CAST('2007-12-30' AS DATETIME)) | ++-----------------------------------------------------------------------------------+ +| 1 | ++-----------------------------------------------------------------------------------+ + +mysql> select datediff(CAST('2010-11-30 23:59:59' AS DATETIME), CAST('2010-12-31' AS DATETIME)); ++-----------------------------------------------------------------------------------+ +| datediff(CAST('2010-11-30 23:59:59' AS DATETIME), CAST('2010-12-31' AS DATETIME)) | ++-----------------------------------------------------------------------------------+ +| -31 | ++-----------------------------------------------------------------------------------+ +``` +##keyword +DATEDIFF diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/day.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/day.md new file mode 100644 index 00000000000000..09145af7aedbbb --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/day.md @@ -0,0 +1,49 @@ +--- +{ + "title": "day", + "language": "zh-CN" +} +--- + + + +# day +## description +### Syntax + +`INT DAY(DATETIME date)` + + +获得日期中的天信息，返回值范围从1-31。 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select day('1987-01-31'); ++----------------------------+ +| day('1987-01-31 00:00:00') | ++----------------------------+ +| 31 | ++----------------------------+ +``` +##keyword +DAY diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayname.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayname.md new file mode 100644 index 00000000000000..043a8e437b190b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayname.md @@ -0,0 +1,50 @@ +--- +{ + "title": "dayname", + "language": "zh-CN" +} +--- + + + +# dayname +## description +### Syntax + +`VARCHAR DAYNAME(DATE)` + + +返回日期对应的日期名字 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select dayname('2007-02-03 00:00:00'); ++--------------------------------+ +| dayname('2007-02-03 00:00:00') | ++--------------------------------+ +| Saturday | ++--------------------------------+ +``` + +##keyword + DAYNAME diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofmonth.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofmonth.md new file mode 100644 index 00000000000000..551fdc60cf6a86 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofmonth.md @@ -0,0 +1,51 @@ +--- +{ + "title": "dayofmonth", + "language": "zh-CN" +} +--- + + + +# dayofmonth +## description +### Syntax + +`INT DAYOFMONTH(DATETIME date)` + + +获得日期中的天信息，返回值范围从1-31。 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select dayofmonth('1987-01-31'); ++-----------------------------------+ +| dayofmonth('1987-01-31 00:00:00') | ++-----------------------------------+ +| 31 | ++-----------------------------------+ +``` + +##keyword + + DAYOFMONTH diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofweek.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofweek.md new file mode 100644 index 00000000000000..6998d05de700b6 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofweek.md @@ -0,0 +1,58 @@ +--- +{ + "title": "dayofweek", + "language": "zh-CN" +} +--- + + + +# dayofweek +## description +### Syntax + +`INT dayofweek(DATETIME date)` + + +DAYOFWEEK函数返回日期的工作日索引值，即星期日为1，星期一为2，星期六为7 + +参数为Date或者Datetime类型或者可以cast为Date或者Datetime类型的数字 + +## example + +``` +mysql> select dayofweek('2019-06-25'); ++----------------------------------+ +| dayofweek('2019-06-25 00:00:00') | ++----------------------------------+ +| 3 | ++----------------------------------+ + +mysql> select dayofweek(cast(20190625 as date)); ++-----------------------------------+ +| dayofweek(CAST(20190625 AS DATE)) | ++-----------------------------------+ +| 3 | ++-----------------------------------+ +``` + +##keyword + + DAYOFWEEK diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofyear.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofyear.md new file mode 100644 index 00000000000000..cc3feee38273e3 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/dayofyear.md @@ -0,0 +1,51 @@ +--- +{ + "title": "dayofyear", + "language": "zh-CN" +} +--- + + + +# dayofyear +## description +### Syntax + +`INT DAYOFYEAR(DATETIME date)` + + +获得日期中对应当年中的哪一天。 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select dayofyear('2007-02-03 00:00:00'); ++----------------------------------+ +| dayofyear('2007-02-03 00:00:00') | ++----------------------------------+ +| 34 | ++----------------------------------+ +``` + +##keyword + + DAYOFYEAR diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/from_days.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/from_days.md new file mode 100644 index 00000000000000..218f2e8ecfdc49 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/from_days.md @@ -0,0 +1,49 @@ +--- +{ + "title": "from_days", + "language": "zh-CN" +} +--- + + + +# from_days +## description +### Syntax + +`DATE FROM_DAYS(INT N)` + + +通过距离0000-01-01日的天数计算出哪一天 + +## example + +``` +mysql> select from_days(730669); ++-------------------+ +| from_days(730669) | ++-------------------+ +| 2000-07-03 | ++-------------------+ +``` + +##keyword + + FROM_DAYS,FROM,DAYS diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/from_unixtime.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/from_unixtime.md new file mode 100644 index 00000000000000..37f38cbeaee1bd --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/from_unixtime.md @@ -0,0 +1,89 @@ +--- +{ + "title": "from_unixtime", + "language": "zh-CN" +} +--- + + + +# from_unixtime +## description +### Syntax + +`DATETIME FROM_UNIXTIME(INT unix_timestamp[, VARCHAR string_format])` + + +将 unix 时间戳转化为对应的 time 格式，返回的格式由 `string_format` 指定 + +默认为 yyyy-MM-dd HH:mm:ss ,也支持date_format中的format格式 + +传入的是整形，返回的是字符串类型 + +目前 `string_format` 支持格式： + + %Y：年。例：2014，1900 + %m：月。例：12，09 + %d：日。例：11，01 + %H：时。例：23，01，12 + %i：分。例：05，11 + %s：秒。例：59，01 + +其余 `string_format` 格式是非法的，返回NULL + +如果给定的时间戳小于 0 或大于 253402271999，则返回 NULL。即时间戳范围是： + +1970-01-01 00:00:00 ~ 9999-12-31 23:59:59 + +## example + +``` +mysql> select from_unixtime(1196440219); ++---------------------------+ +| from_unixtime(1196440219) | ++---------------------------+ +| 2007-12-01 00:30:19 | ++---------------------------+ + +mysql> select from_unixtime(1196440219, 'yyyy-MM-dd HH:mm:ss'); ++--------------------------------------------------+ +| from_unixtime(1196440219, 'yyyy-MM-dd HH:mm:ss') | ++--------------------------------------------------+ +| 2007-12-01 00:30:19 | ++--------------------------------------------------+ + +mysql> select from_unixtime(1196440219, '%Y-%m-%d'); ++-----------------------------------------+ +| from_unixtime(1196440219, '%Y-%m-%d') | ++-----------------------------------------+ +| 2007-12-01 | ++-----------------------------------------+ + +mysql> select from_unixtime(1196440219, '%Y-%m-%d %H:%i:%s'); ++--------------------------------------------------+ +| from_unixtime(1196440219, '%Y-%m-%d %H:%i:%s') | ++--------------------------------------------------+ +| 2007-12-01 00:30:19 | ++--------------------------------------------------+ +``` + +##keyword + + FROM_UNIXTIME,FROM,UNIXTIME diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/hour.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/hour.md new file mode 100644 index 00000000000000..388405be507904 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/hour.md @@ -0,0 +1,49 @@ +--- +{ + "title": "hour", + "language": "zh-CN" +} +--- + + + +# hour +## description +### Syntax + +`INT HOUR(DATETIME date)` + + +获得日期中的小时的信息，返回值范围从0-23。 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select hour('2018-12-31 23:59:59'); ++-----------------------------+ +| hour('2018-12-31 23:59:59') | ++-----------------------------+ +| 23 | ++-----------------------------+ +``` +##keyword +HOUR diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/minute.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/minute.md new file mode 100644 index 00000000000000..47d7878cc4ac37 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/minute.md @@ -0,0 +1,49 @@ +--- +{ + "title": "minute", + "language": "zh-CN" +} +--- + + + +# minute +## description +### Syntax + +`INT MINUTE(DATETIME date)` + + +获得日期中的分钟的信息，返回值范围从0-59。 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select minute('2018-12-31 23:59:59'); ++-----------------------------+ +| minute('2018-12-31 23:59:59') | ++-----------------------------+ +| 59 | ++-----------------------------+ +``` +##keyword +MINUTE diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/month.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/month.md new file mode 100644 index 00000000000000..5c4ce8b666d1ae --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/month.md @@ -0,0 +1,51 @@ +--- +{ + "title": "month", + "language": "zh-CN" +} +--- + + + +# month +## description +### Syntax + +`INT MONTH(DATETIME date)` + + +返回时间类型中的月份信息，范围是1, 12 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select month('1987-01-01'); ++-----------------------------+ +| month('1987-01-01 00:00:00') | ++-----------------------------+ +| 1 | ++-----------------------------+ +``` + +##keyword + + MONTH diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/monthname.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/monthname.md new file mode 100644 index 00000000000000..748d79fd5c97b8 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/monthname.md @@ -0,0 +1,51 @@ +--- +{ + "title": "monthname", + "language": "zh-CN" +} +--- + + + +# monthname +## description +### Syntax + +`VARCHAR MONTHNAME(DATE)` + + +返回日期对应的月份名字 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select monthname('2008-02-03 00:00:00'); ++----------------------------------+ +| monthname('2008-02-03 00:00:00') | ++----------------------------------+ +| February | ++----------------------------------+ +``` + +##keyword + + MONTHNAME diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/now.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/now.md new file mode 100644 index 00000000000000..fcf489e51124d6 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/now.md @@ -0,0 +1,49 @@ +--- +{ + "title": "now", + "language": "zh-CN" +} +--- + + + +# now +## description +### Syntax + +`DATETIME NOW()` + + +获得当前的时间，以Datetime类型返回 + +## example + +``` +mysql> select now(); ++---------------------+ +| now() | ++---------------------+ +| 2019-05-27 15:58:25 | ++---------------------+ +``` + +##keyword + + NOW diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/second.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/second.md new file mode 100644 index 00000000000000..1b9f3e61e8a422 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/second.md @@ -0,0 +1,49 @@ +--- +{ + "title": "second", + "language": "zh-CN" +} +--- + + + +# second +## description +### Syntax + +`INT SECOND(DATETIME date)` + + +获得日期中的秒的信息，返回值范围从0-59。 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select second('2018-12-31 23:59:59'); ++-----------------------------+ +| second('2018-12-31 23:59:59') | ++-----------------------------+ +| 59 | ++-----------------------------+ +``` +##keyword +SECOND diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/str_to_date.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/str_to_date.md new file mode 100644 index 00000000000000..a77cf54af28d4b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/str_to_date.md @@ -0,0 +1,65 @@ +--- +{ + "title": "str_to_date", + "language": "zh-CN" +} +--- + + + +# str_to_date +## description +### Syntax + +`DATETIME STR_TO_DATE(VARCHAR str, VARCHAR format)` + + +通过format指定的方式将str转化为DATE类型，如果转化结果不对返回NULL + +支持的format格式与date_format一致 + +## example + +``` +mysql> select str_to_date('2014-12-21 12:34:56', '%Y-%m-%d %H:%i:%s'); ++---------------------------------------------------------+ +| str_to_date('2014-12-21 12:34:56', '%Y-%m-%d %H:%i:%s') | ++---------------------------------------------------------+ +| 2014-12-21 12:34:56 | ++---------------------------------------------------------+ + +mysql> select str_to_date('2014-12-21 12:34%3A56', '%Y-%m-%d %H:%i%%3A%s'); ++--------------------------------------------------------------+ +| str_to_date('2014-12-21 12:34%3A56', '%Y-%m-%d %H:%i%%3A%s') | ++--------------------------------------------------------------+ +| 2014-12-21 12:34:56 | ++--------------------------------------------------------------+ + +mysql> select str_to_date('200442 Monday', '%X%V %W'); ++-----------------------------------------+ +| str_to_date('200442 Monday', '%X%V %W') | ++-----------------------------------------+ +| 2004-10-18 | ++-----------------------------------------+ +``` + +##keyword + + STR_TO_DATE,STR,TO,DATE diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timediff.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timediff.md new file mode 100644 index 00000000000000..5819dd6ea89ba2 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timediff.md @@ -0,0 +1,65 @@ +--- +{ + "title": "timediff", + "language": "zh-CN" +} +--- + + + +# timediff +## description +### Syntax + +`TIME TIMEDIFF(DATETIME expr1, DATETIME expr2)` + + +TIMEDIFF返回两个DATETIME之间的差值 + +TIMEDIFF函数返回表示为时间值的expr1 - expr2的结果，返回值为TIME类型 + +## example + +``` +mysql> SELECT TIMEDIFF(now(),utc_timestamp()); ++----------------------------------+ +| timediff(now(), utc_timestamp()) | ++----------------------------------+ +| 08:00:00 | ++----------------------------------+ + +mysql> SELECT TIMEDIFF('2019-07-11 16:59:30','2019-07-11 16:59:21'); ++--------------------------------------------------------+ +| timediff('2019-07-11 16:59:30', '2019-07-11 16:59:21') | ++--------------------------------------------------------+ +| 00:00:09 | ++--------------------------------------------------------+ + +mysql> SELECT TIMEDIFF('2019-01-01 00:00:00', NULL); ++---------------------------------------+ +| timediff('2019-01-01 00:00:00', NULL) | ++---------------------------------------+ +| NULL | ++---------------------------------------+ +``` + +##keyword + + TIMEDIFF diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timestampadd.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timestampadd.md new file mode 100644 index 00000000000000..7afa93bf5c9285 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timestampadd.md @@ -0,0 +1,59 @@ +--- +{ + "title": "timestampadd", + "language": "zh-CN" +} +--- + + + +# timestampadd +## description +### Syntax + +`DATETIME TIMESTAMPADD(unit, interval, DATETIME datetime_expr)` + + +将整数表达式间隔添加到日期或日期时间表达式datetime_expr中。 + +interval的单位由unit参数给出，它应该是下列值之一: + +SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, or YEAR。 + +## example + +``` + +mysql> SELECT TIMESTAMPADD(MINUTE,1,'2019-01-02'); ++------------------------------------------------+ +| timestampadd(MINUTE, 1, '2019-01-02 00:00:00') | ++------------------------------------------------+ +| 2019-01-02 00:01:00 | ++------------------------------------------------+ + +mysql> SELECT TIMESTAMPADD(WEEK,1,'2019-01-02'); ++----------------------------------------------+ +| timestampadd(WEEK, 1, '2019-01-02 00:00:00') | ++----------------------------------------------+ +| 2019-01-09 00:00:00 | ++----------------------------------------------+ +``` +##keyword +TIMESTAMPADD diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timestampdiff.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timestampdiff.md new file mode 100644 index 00000000000000..4c3b1a75598832 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/timestampdiff.md @@ -0,0 +1,67 @@ +--- +{ + "title": "timestampdiff", + "language": "zh-CN" +} +--- + + + +# timestampdiff +## description +### Syntax + +`INT TIMESTAMPDIFF(unit,DATETIME datetime_expr1, DATETIME datetime_expr2)` + +返回datetime_expr2−datetime_expr1，其中datetime_expr1和datetime_expr2是日期或日期时间表达式。 + +结果(整数)的单位由unit参数给出。interval的单位由unit参数给出，它应该是下列值之一: + +SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, or YEAR。 + +## example + +``` + +MySQL> SELECT TIMESTAMPDIFF(MONTH,'2003-02-01','2003-05-01'); ++--------------------------------------------------------------------+ +| timestampdiff(MONTH, '2003-02-01 00:00:00', '2003-05-01 00:00:00') | ++--------------------------------------------------------------------+ +| 3 | ++--------------------------------------------------------------------+ + +MySQL> SELECT TIMESTAMPDIFF(YEAR,'2002-05-01','2001-01-01'); ++-------------------------------------------------------------------+ +| timestampdiff(YEAR, '2002-05-01 00:00:00', '2001-01-01 00:00:00') | ++-------------------------------------------------------------------+ +| -1 | ++-------------------------------------------------------------------+ + + +MySQL> SELECT TIMESTAMPDIFF(MINUTE,'2003-02-01','2003-05-01 12:05:55'); ++---------------------------------------------------------------------+ +| timestampdiff(MINUTE, '2003-02-01 00:00:00', '2003-05-01 12:05:55') | ++---------------------------------------------------------------------+ +| 128885 | ++---------------------------------------------------------------------+ + +``` +##keyword +TIMESTAMPDIFF diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/to_days.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/to_days.md new file mode 100644 index 00000000000000..ea6b0af43839c5 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/to_days.md @@ -0,0 +1,51 @@ +--- +{ + "title": "to_days", + "language": "zh-CN" +} +--- + + + +# to_days +## description +### Syntax + +`INT TO_DAYS(DATETIME date)` + + +返回date距离0000-01-01的天数 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select to_days('2007-10-07'); ++-----------------------+ +| to_days('2007-10-07') | ++-----------------------+ +| 733321 | ++-----------------------+ +``` + +##keyword + + TO_DAYS,TO,DAYS diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/unix_timestamp.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/unix_timestamp.md new file mode 100644 index 00000000000000..fbb12cd096037f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/unix_timestamp.md @@ -0,0 +1,86 @@ +--- +{ + "title": "unix_timestamp", + "language": "zh-CN" +} +--- + + + +# unix_timestamp +## description +### Syntax + +`INT UNIX_TIMESTAMP(), UNIX_TIMESTAMP(DATETIME date), UNIX_TIMESTAMP(DATETIME date, STRING fmt),` + +将 Date 或者 Datetime 类型转化为 unix 时间戳。 + +如果没有参数，则是将当前的时间转化为时间戳。 + +参数需要是 Date 或者 Datetime 类型。 + +对于在 1970-01-01 00:00:00 之前或 2038-01-19 03:14:07 之后的时间，该函数将返回 0。 + +Format 的格式请参阅 `date_format` 函数的格式说明。 + +该函数受时区影响。 + +## example + +``` +mysql> select unix_timestamp(); ++------------------+ +| unix_timestamp() | ++------------------+ +| 1558589570 | ++------------------+ + +mysql> select unix_timestamp('2007-11-30 10:30:19'); ++---------------------------------------+ +| unix_timestamp('2007-11-30 10:30:19') | ++---------------------------------------+ +| 1196389819 | ++---------------------------------------+ + +mysql> select unix_timestamp('2007-11-30 10:30-19', '%Y-%m-%d %H:%i-%s'); ++---------------------------------------+ +| unix_timestamp('2007-11-30 10:30-19') | ++---------------------------------------+ +| 1196389819 | ++---------------------------------------+ + +mysql> select unix_timestamp('2007-11-30 10:30%3A19', '%Y-%m-%d %H:%i%%3A%s'); ++---------------------------------------+ +| unix_timestamp('2007-11-30 10:30%3A19') | ++---------------------------------------+ +| 1196389819 | ++---------------------------------------+ + +mysql> select unix_timestamp('1969-01-01 00:00:00'); ++---------------------------------------+ +| unix_timestamp('1969-01-01 00:00:00') | ++---------------------------------------+ +| 0 | ++---------------------------------------+ +``` + +##keyword + + UNIX_TIMESTAMP,UNIX,TIMESTAMP diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/utc_timestamp.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/utc_timestamp.md new file mode 100644 index 00000000000000..70d383fc836884 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/utc_timestamp.md @@ -0,0 +1,53 @@ +--- +{ + "title": "utc_timestamp", + "language": "zh-CN" +} +--- + + + +# utc_timestamp +## description +### Syntax + +`DATETIME UTC_TIMESTAMP()` + + +返回当前UTC日期和时间在 "YYYY-MM-DD HH:MM:SS" 或 + +"YYYYMMDDHHMMSS"格式的一个值 + +根据该函数是否用在字符串或数字语境中 + +## example + +``` +mysql> select utc_timestamp(),utc_timestamp() + 1; ++---------------------+---------------------+ +| utc_timestamp() | utc_timestamp() + 1 | ++---------------------+---------------------+ +| 2019-07-10 12:31:18 | 20190710123119 | ++---------------------+---------------------+ +``` + +##keyword + + UTC_TIMESTAMP,UTC,TIMESTAMP diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/workofyear.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/workofyear.md new file mode 100644 index 00000000000000..757c18ecd9739b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/workofyear.md @@ -0,0 +1,52 @@ +--- +{ + "title": "weekofyear", + "language": "zh-CN" +} +--- + + + +# weekofyear +## description +### Syntax + +`INT WEEKOFYEAR(DATETIME date)` + + + +获得一年中的第几周 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select weekofyear('2008-02-20 00:00:00'); ++-----------------------------------+ +| weekofyear('2008-02-20 00:00:00') | ++-----------------------------------+ +| 8 | ++-----------------------------------+ +``` + +##keyword + + WEEKOFYEAR diff --git a/docs/zh-CN/sql-reference/sql-functions/date-time-functions/year.md b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/year.md new file mode 100644 index 00000000000000..4bf28d4c64e396 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/date-time-functions/year.md @@ -0,0 +1,51 @@ +--- +{ + "title": "year", + "language": "zh-CN" +} +--- + + + +# year +## description +### Syntax + +`INT YEAR(DATETIME date)` + + +返回date类型的year部分，范围从1000-9999 + +参数为Date或者Datetime类型 + +## example + +``` +mysql> select year('1987-01-01'); ++-----------------------------+ +| year('1987-01-01 00:00:00') | ++-----------------------------+ +| 1987 | ++-----------------------------+ +``` + +##keyword + + YEAR diff --git a/docs/zh-CN/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md b/docs/zh-CN/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md new file mode 100644 index 00000000000000..f1f237255a0544 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md @@ -0,0 +1,61 @@ +--- +{ + "title": "murmur_hash3_32", + "language": "zh-CN" +} +--- + + + +# murmur_hash3_32 + +## description +### Syntax + +`INT MURMUR_HASH3_32(VARCHAR input, ...)` + +返回输入字符串的32位murmur3 hash值 + +## example + +``` +mysql> select murmur_hash3_32(null); ++-----------------------+ +| murmur_hash3_32(NULL) | ++-----------------------+ +| NULL | ++-----------------------+ + +mysql> select murmur_hash3_32("hello"); ++--------------------------+ +| murmur_hash3_32('hello') | ++--------------------------+ +| 1321743225 | ++--------------------------+ + +mysql> select murmur_hash3_32("hello", "world"); ++-----------------------------------+ +| murmur_hash3_32('hello', 'world') | ++-----------------------------------+ +| 984713481 | ++-----------------------------------+ +``` + +## keyword + + MURMUR_HASH3_32,HASH diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_astext.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_astext.md new file mode 100644 index 00000000000000..8bc47a5031ab71 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_astext.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_AsText`,`ST_AsWKT`", + "language": "zh-CN" +} +--- + + + +# `ST_AsText`,`ST_AsWKT` +## description +### Syntax + +`VARCHAR ST_AsText(GEOMETRY geo)` + + +将一个几何图形转化为WKT（Well Known Text）的表示形式 + +## example + +``` +mysql> SELECT ST_AsText(ST_Point(24.7, 56.7)); ++---------------------------------+ +| st_astext(st_point(24.7, 56.7)) | ++---------------------------------+ +| POINT (24.7 56.7) | ++---------------------------------+ +``` +##keyword +ST_ASTEXT,ST_ASWKT,ST,ASTEXT,ASWKT diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_circle.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_circle.md new file mode 100644 index 00000000000000..e084d930974ba6 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_circle.md @@ -0,0 +1,48 @@ +--- +{ + "title": "`ST_Circle`", + "language": "zh-CN" +} +--- + + + +# `ST_Circle` +## description +### Syntax + +`GEOMETRY ST_Circle(DOUBLE center_lng, DOUBLE center_lat, DOUBLE radius)` + + +将一个WKT（Well Known Text）转化为地球球面上的一个圆。其中`center_lng`表示的圆心的经度， +`center_lat`表示的是圆心的纬度，`radius`表示的是圆的半径，单位是米,最大支持9999999 + +## example + +``` +mysql> SELECT ST_AsText(ST_Circle(111, 64, 10000)); ++--------------------------------------------+ +| st_astext(st_circle(111.0, 64.0, 10000.0)) | ++--------------------------------------------+ +| CIRCLE ((111 64), 10000) | ++--------------------------------------------+ +``` +##keyword +ST_CIRCLE,ST,CIRCLE diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_contains.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_contains.md new file mode 100644 index 00000000000000..67671be2c200e8 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_contains.md @@ -0,0 +1,54 @@ +--- +{ + "title": "`ST_Contains`", + "language": "zh-CN" +} +--- + + + +# `ST_Contains` +## description +### Syntax + +`BOOL ST_Contains(GEOMETRY shape1, GEOMETRY shape2)` + + +判断几何图形shape1是否完全能够包含几何图形shape2 + +## example + +``` +mysql> SELECT ST_Contains(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"), ST_Point(5, 5)); ++----------------------------------------------------------------------------------------+ +| st_contains(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))'), st_point(5.0, 5.0)) | ++----------------------------------------------------------------------------------------+ +| 1 | ++----------------------------------------------------------------------------------------+ + +mysql> SELECT ST_Contains(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"), ST_Point(50, 50)); ++------------------------------------------------------------------------------------------+ +| st_contains(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))'), st_point(50.0, 50.0)) | ++------------------------------------------------------------------------------------------+ +| 0 | ++------------------------------------------------------------------------------------------+ +``` +##keyword +ST_CONTAINS,ST,CONTAINS diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md new file mode 100644 index 00000000000000..580d470e1c56f5 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_distance_sphere.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_Distance_Sphere`", + "language": "zh-CN" +} +--- + + + +# `ST_Distance_Sphere` +## description +### Syntax + +`DOUBLE ST_Distance_Sphere(DOUBLE x_lng, DOUBLE x_lat, DOUBLE y_lng, DOUBLE x_lat)` + + +计算地球两点之间的球面距离，单位为 米。传入的参数分别为X点的经度，X点的纬度，Y点的经度，Y点的纬度。 + +## example + +``` +mysql> select st_distance_sphere(116.35620117, 39.939093, 116.4274406433, 39.9020987219); ++----------------------------------------------------------------------------+ +| st_distance_sphere(116.35620117, 39.939093, 116.4274406433, 39.9020987219) | ++----------------------------------------------------------------------------+ +| 7336.9135549995917 | ++----------------------------------------------------------------------------+ +``` +##keyword +ST_DISTANCE_SPHERE,ST,DISTANCE,SPHERE diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md new file mode 100644 index 00000000000000..017e634232f587 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_geometryfromtext.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_GeometryFromText`,`ST_GeomFromText`", + "language": "zh-CN" +} +--- + + + +# `ST_GeometryFromText`,`ST_GeomFromText` +## description +### Syntax + +`GEOMETRY ST_GeometryFromText(VARCHAR wkt)` + + +将一个WKT（Well Known Text）转化为对应的内存的几何形式 + +## example + +``` +mysql> SELECT ST_AsText(ST_GeometryFromText("LINESTRING (1 1, 2 2)")); ++---------------------------------------------------------+ +| st_astext(st_geometryfromtext('LINESTRING (1 1, 2 2)')) | ++---------------------------------------------------------+ +| LINESTRING (1 1, 2 2) | ++---------------------------------------------------------+ +``` +##keyword +ST_GEOMETRYFROMTEXT,ST_GEOMFROMTEXT,ST,GEOMETRYFROMTEXT,GEOMFROMTEXT diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_linefromtext.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_linefromtext.md new file mode 100644 index 00000000000000..8af57c80db3d1c --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_linefromtext.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_LineFromText`,`ST_LineStringFromText`", + "language": "zh-CN" +} +--- + + + +# `ST_LineFromText`,`ST_LineStringFromText` +## description +### Syntax + +`GEOMETRY ST_LineFromText(VARCHAR wkt)` + + +将一个WKT（Well Known Text）转化为一个Line形式的内存表现形式 + +## example + +``` +mysql> SELECT ST_AsText(ST_LineFromText("LINESTRING (1 1, 2 2)")); ++---------------------------------------------------------+ +| st_astext(st_geometryfromtext('LINESTRING (1 1, 2 2)')) | ++---------------------------------------------------------+ +| LINESTRING (1 1, 2 2) | ++---------------------------------------------------------+ +``` +##keyword +ST_LINEFROMTEXT,ST_LINESTRINGFROMTEXT,ST,LINEFROMTEXT,LINESTRINGFROMTEXT diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_point.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_point.md new file mode 100644 index 00000000000000..74db364135772e --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_point.md @@ -0,0 +1,48 @@ +--- +{ + "title": "`ST_Point`", + "language": "zh-CN" +} +--- + + + +# `ST_Point` +## description +### Syntax + +`POINT ST_Point(DOUBLE x, DOUBLE y)` + + +通过给定的X坐标值，Y坐标值返回对应的Point。 +当前这个值只是在球面集合上有意义，X/Y对应的是经度/纬度(longitude/latitude);ps:直接select ST_Point()会卡主，慎重！！！ + +## example + +``` +mysql> SELECT ST_AsText(ST_Point(24.7, 56.7)); ++---------------------------------+ +| st_astext(st_point(24.7, 56.7)) | ++---------------------------------+ +| POINT (24.7 56.7) | ++---------------------------------+ +``` +##keyword +ST_POINT,ST,POINT diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_polygon.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_polygon.md new file mode 100644 index 00000000000000..1ef45b97ae517c --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_polygon.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_Polygon`,`ST_PolyFromText`,`ST_PolygonFromText`", + "language": "zh-CN" +} +--- + + + +# `ST_Polygon`,`ST_PolyFromText`,`ST_PolygonFromText` +## description +### Syntax + +`GEOMETRY ST_Polygon(VARCHAR wkt)` + + +将一个WKT（Well Known Text）转化为对应的多边形内存形式 + +## example + +``` +mysql> SELECT ST_AsText(ST_Polygon("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))")); ++------------------------------------------------------------------+ +| st_astext(st_polygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))')) | ++------------------------------------------------------------------+ +| POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0)) | ++------------------------------------------------------------------+ +``` +##keyword +ST_POLYGON,ST_POLYFROMTEXT,ST_POLYGONFROMTEXT,ST,POLYGON,POLYFROMTEXT,POLYGONFROMTEXT diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_x.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_x.md new file mode 100644 index 00000000000000..29af99d2c929e5 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_x.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_X`", + "language": "zh-CN" +} +--- + + + +# `ST_X` +## description +### Syntax + +`DOUBLE ST_X(POINT point)` + + +当point是一个合法的POINT类型时，返回对应的X坐标值 + +## example + +``` +mysql> SELECT ST_X(ST_Point(24.7, 56.7)); ++----------------------------+ +| st_x(st_point(24.7, 56.7)) | ++----------------------------+ +| 24.7 | ++----------------------------+ +``` +##keyword +ST_X,ST,X diff --git a/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_y.md b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_y.md new file mode 100644 index 00000000000000..d1bf468beeeae3 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/spatial-functions/st_y.md @@ -0,0 +1,47 @@ +--- +{ + "title": "`ST_Y`", + "language": "zh-CN" +} +--- + + + +# `ST_Y` +## description +### Syntax + +`DOUBLE ST_Y(POINT point)` + + +当point是一个合法的POINT类型时，返回对应的Y坐标值 + +## example + +``` +mysql> SELECT ST_Y(ST_Point(24.7, 56.7)); ++----------------------------+ +| st_y(st_point(24.7, 56.7)) | ++----------------------------+ +| 56.7 | ++----------------------------+ +``` +##keyword +ST_Y,ST,Y diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/ascii.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/ascii.md new file mode 100644 index 00000000000000..62a3ebf85d2c29 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/ascii.md @@ -0,0 +1,54 @@ +--- +{ + "title": "ascii", + "language": "zh-CN" +} +--- + + + +# ascii +## description +### Syntax + +`INT ascii(VARCHAR str)` + + +返回字符串第一个字符对应的 ascii 码 + +## example + +``` +mysql> select ascii('1'); ++------------+ +| ascii('1') | ++------------+ +| 49 | ++------------+ + +mysql> select ascii('234'); ++--------------+ +| ascii('234') | ++--------------+ +| 50 | ++--------------+ +``` +##keyword +ASCII diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/concat.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/concat.md new file mode 100644 index 00000000000000..225c96f80ac470 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/concat.md @@ -0,0 +1,61 @@ +--- +{ + "title": "concat", + "language": "zh-CN" +} +--- + + + +# concat +## description +### Syntax + +`VARCHAR concat(VARCHAR,...)` + + +将多个字符串连接起来, 如果参数中任意一个值是 NULL，那么返回的结果就是 NULL + +## example + +``` +mysql> select concat("a", "b"); ++------------------+ +| concat('a', 'b') | ++------------------+ +| ab | ++------------------+ + +mysql> select concat("a", "b", "c"); ++-----------------------+ +| concat('a', 'b', 'c') | ++-----------------------+ +| abc | ++-----------------------+ + +mysql> select concat("a", null, "c"); ++------------------------+ +| concat('a', NULL, 'c') | ++------------------------+ +| NULL | ++------------------------+ +``` +##keyword +CONCAT diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/concat_ws.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/concat_ws.md new file mode 100644 index 00000000000000..556600df9e769f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/concat_ws.md @@ -0,0 +1,63 @@ +--- +{ + "title": "concat_ws", + "language": "zh-CN" +} +--- + + + +# concat_ws +## description +### Syntax + +`VARCHAR concat_ws(VARCHAR sep, VARCHAR str,...)` + + +使用第一个参数 sep 作为连接符，将第二个参数以及后续所有参数拼接成一个字符串. +如果分隔符是 NULL，返回 NULL。 +`concat_ws`函数不会跳过空字符串，会跳过 NULL 值 + +## example + +``` +mysql> select concat_ws("or", "d", "is"); ++----------------------------+ +| concat_ws('or', 'd', 'is') | ++----------------------------+ +| doris | ++----------------------------+ + +mysql> select concat_ws(NULL, "d", "is"); ++----------------------------+ +| concat_ws(NULL, 'd', 'is') | ++----------------------------+ +| NULL | ++----------------------------+ + +mysql> select concat_ws("or", "d", NULL,"is"); ++---------------------------------+ +| concat_ws("or", "d", NULL,"is") | ++---------------------------------+ +| doris | ++---------------------------------+ +``` +##keyword +CONCAT_WS,CONCAT,WS diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/ends_with.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/ends_with.md new file mode 100644 index 00000000000000..f4faeb46a60eec --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/ends_with.md @@ -0,0 +1,53 @@ +--- +{ + "title": "ends_with", + "language": "zh-CN" +} +--- + + + +# ends_with +## description +### Syntax + +`BOOLEAN ENDS_WITH (VARCHAR str, VARCHAR suffix)` + +如果字符串以指定后缀结尾，返回true。否则，返回false。任意参数为NULL，返回NULL。 + +## example + +``` +mysql> select ends_with("Hello doris", "doris"); ++-----------------------------------+ +| ends_with('Hello doris', 'doris') | ++-----------------------------------+ +| 1 | ++-----------------------------------+ + +mysql> select ends_with("Hello doris", "Hello"); ++-----------------------------------+ +| ends_with('Hello doris', 'Hello') | ++-----------------------------------+ +| 0 | ++-----------------------------------+ +``` +##keyword +ENDS_WITH diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/find_in_set.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/find_in_set.md new file mode 100644 index 00000000000000..510f2d0b6a9994 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/find_in_set.md @@ -0,0 +1,47 @@ +--- +{ + "title": "find_in_set", + "language": "zh-CN" +} +--- + + + +# find_in_set +## description +### Syntax + +`INT find_in_set(VARCHAR str, VARCHAR strlist)` + + +返回 strlist 中第一次出现 str 的位置（从1开始计数）。strlist 是用逗号分隔的字符串。如果没有找到，返回0。任意参数为 NULL ，返回 NULL。 + +## example + +``` +mysql> select find_in_set("b", "a,b,c"); ++---------------------------+ +| find_in_set('b', 'a,b,c') | ++---------------------------+ +| 2 | ++---------------------------+ +``` +##keyword +FIND_IN_SET,FIND,IN,SET diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_double.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_double.md new file mode 100644 index 00000000000000..97c201a92f9f39 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_double.md @@ -0,0 +1,74 @@ +--- +{ + "title": "get_json_double", + "language": "zh-CN" +} +--- + + + +# get_json_double +## description +### Syntax + +`DOUBLE get_json_double(VARCHAR json_str, VARCHAR json_path) + + +解析并获取 json 字符串内指定路径的浮点型内容。 +其中 json_path 必须以 $ 符号作为开头，使用 . 作为路径分割符。如果路径中包含 . ，则可以使用双引号包围。 +使用 [ ] 表示数组下标，从 0 开始。 +path 的内容不能包含 ", [ 和 ]。 +如果 json_string 格式不对，或 json_path 格式不对，或无法找到匹配项，则返回 NULL。 + +## example + +1. 获取 key 为 "k1" 的 value + +``` +mysql> SELECT get_json_double('{"k1":1.3, "k2":"2"}', "$.k1"); ++-------------------------------------------------+ +| get_json_double('{"k1":1.3, "k2":"2"}', '$.k1') | ++-------------------------------------------------+ +| 1.3 | ++-------------------------------------------------+ +``` + +2. 获取 key 为 "my.key" 的数组中第二个元素 + +``` +mysql> SELECT get_json_double('{"k1":"v1", "my.key":[1.1, 2.2, 3.3]}', '$."my.key"[1]'); ++---------------------------------------------------------------------------+ +| get_json_double('{"k1":"v1", "my.key":[1.1, 2.2, 3.3]}', '$."my.key"[1]') | ++---------------------------------------------------------------------------+ +| 2.2 | ++---------------------------------------------------------------------------+ +``` + +3. 获取二级路径为 k1.key -> k2 的数组中，第一个元素 +``` +mysql> SELECT get_json_double('{"k1.key":{"k2":[1.1, 2.2]}}', '$."k1.key".k2[0]'); ++---------------------------------------------------------------------+ +| get_json_double('{"k1.key":{"k2":[1.1, 2.2]}}', '$."k1.key".k2[0]') | ++---------------------------------------------------------------------+ +| 1.1 | ++---------------------------------------------------------------------+ +``` +##keyword +GET_JSON_DOUBLE,GET,JSON,DOUBLE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_int.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_int.md new file mode 100644 index 00000000000000..c96bbe68f62b40 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_int.md @@ -0,0 +1,74 @@ +--- +{ + "title": "get_json_int", + "language": "zh-CN" +} +--- + + + +# get_json_int +## description +### Syntax + +`INT get_json_int(VARCHAR json_str, VARCHAR json_path) + + +解析并获取 json 字符串内指定路径的整型内容。 +其中 json_path 必须以 $ 符号作为开头，使用 . 作为路径分割符。如果路径中包含 . ，则可以使用双引号包围。 +使用 [ ] 表示数组下标，从 0 开始。 +path 的内容不能包含 ", [ 和 ]。 +如果 json_string 格式不对，或 json_path 格式不对，或无法找到匹配项，则返回 NULL。 + +## example + +1. 获取 key 为 "k1" 的 value + +``` +mysql> SELECT get_json_int('{"k1":1, "k2":"2"}', "$.k1"); ++--------------------------------------------+ +| get_json_int('{"k1":1, "k2":"2"}', '$.k1') | ++--------------------------------------------+ +| 1 | ++--------------------------------------------+ +``` + +2. 获取 key 为 "my.key" 的数组中第二个元素 + +``` +mysql> SELECT get_json_int('{"k1":"v1", "my.key":[1, 2, 3]}', '$."my.key"[1]'); ++------------------------------------------------------------------+ +| get_json_int('{"k1":"v1", "my.key":[1, 2, 3]}', '$."my.key"[1]') | ++------------------------------------------------------------------+ +| 2 | ++------------------------------------------------------------------+ +``` + +3. 获取二级路径为 k1.key -> k2 的数组中，第一个元素 +``` +mysql> SELECT get_json_int('{"k1.key":{"k2":[1, 2]}}', '$."k1.key".k2[0]'); ++--------------------------------------------------------------+ +| get_json_int('{"k1.key":{"k2":[1, 2]}}', '$."k1.key".k2[0]') | ++--------------------------------------------------------------+ +| 1 | ++--------------------------------------------------------------+ +``` +##keyword +GET_JSON_INT,GET,JSON,INT diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_string.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_string.md new file mode 100644 index 00000000000000..0f8b85f44f41cf --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/get_json_string.md @@ -0,0 +1,84 @@ +--- +{ + "title": "get_json_string", + "language": "zh-CN" +} +--- + + + +# get_json_string +## description +### Syntax + +`VARCHAR get_json_string(VARCHAR json_str, VARCHAR json_path) + + +解析并获取 json 字符串内指定路径的字符串内容。 +其中 json_path 必须以 $ 符号作为开头，使用 . 作为路径分割符。如果路径中包含 . ，则可以使用双引号包围。 +使用 [ ] 表示数组下标，从 0 开始。 +path 的内容不能包含 ", [ 和 ]。 +如果 json_string 格式不对，或 json_path 格式不对，或无法找到匹配项，则返回 NULL。 + +## example + +1. 获取 key 为 "k1" 的 value + +``` +mysql> SELECT get_json_string('{"k1":"v1", "k2":"v2"}', "$.k1"); ++---------------------------------------------------+ +| get_json_string('{"k1":"v1", "k2":"v2"}', '$.k1') | ++---------------------------------------------------+ +| v1 | ++---------------------------------------------------+ +``` + +2. 获取 key 为 "my.key" 的数组中第二个元素 + +``` +mysql> SELECT get_json_string('{"k1":"v1", "my.key":["e1", "e2", "e3"]}', '$."my.key"[1]'); ++------------------------------------------------------------------------------+ +| get_json_string('{"k1":"v1", "my.key":["e1", "e2", "e3"]}', '$."my.key"[1]') | ++------------------------------------------------------------------------------+ +| e2 | ++------------------------------------------------------------------------------+ +``` + +3. 获取二级路径为 k1.key -> k2 的数组中，第一个元素 +``` +mysql> SELECT get_json_string('{"k1.key":{"k2":["v1", "v2"]}}', '$."k1.key".k2[0]'); ++-----------------------------------------------------------------------+ +| get_json_string('{"k1.key":{"k2":["v1", "v2"]}}', '$."k1.key".k2[0]') | ++-----------------------------------------------------------------------+ +| v1 | ++-----------------------------------------------------------------------+ +``` + +4. 获取数组中，key 为 "k1" 的所有 value +``` +mysql> SELECT get_json_string('[{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}]', '$.k1'); ++---------------------------------------------------------------------------------+ +| get_json_string('[{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}]', '$.k1') | ++---------------------------------------------------------------------------------+ +| ["v1","v3","v4"] | ++---------------------------------------------------------------------------------+ +``` +##keyword +GET_JSON_STRING,GET,JSON,STRING diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/group_concat.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/group_concat.md new file mode 100644 index 00000000000000..73cb45d100c804 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/group_concat.md @@ -0,0 +1,63 @@ +--- +{ + "title": "group_concat", + "language": "zh-CN" +} +--- + + + +# group_concat +## description +### Syntax + +`VARCHAR group_concat(VARCHAR str[, VARCHAR sep])` + + +该函数是类似于 sum() 的聚合函数，group_concat 将结果集中的多行结果连接成一个字符串。第二个参数 sep 为字符串之间的连接符号，该参数可以省略。该函数通常需要和 group by 语句一起使用。 + +## example + +``` +mysql> select value from test; ++-------+ +| value | ++-------+ +| a | +| b | +| c | ++-------+ + +mysql> select group_concat(value) from test; ++-----------------------+ +| group_concat(`value`) | ++-----------------------+ +| a, b, c | ++-----------------------+ + +mysql> select group_concat(value, " ") from test; ++----------------------------+ +| group_concat(`value`, ' ') | ++----------------------------+ +| a b c | ++----------------------------+ +``` +##keyword +GROUP_CONCAT,GROUP,CONCAT diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/instr.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/instr.md new file mode 100644 index 00000000000000..e9c466ba7bd6c6 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/instr.md @@ -0,0 +1,54 @@ +--- +{ + "title": "instr", + "language": "zh-CN" +} +--- + + + +# instr +## description +### Syntax + +`INT instr(VARCHAR str, VARCHAR substr)` + + +返回 substr 在 str 中第一次出现的位置（从1开始计数）。如果 substr 不在 str 中出现，则返回0。 + +## example + +``` +mysql> select instr("abc", "b"); ++-------------------+ +| instr('abc', 'b') | ++-------------------+ +| 2 | ++-------------------+ + +mysql> select instr("abc", "d"); ++-------------------+ +| instr('abc', 'd') | ++-------------------+ +| 0 | ++-------------------+ +``` +##keyword +INSTR diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/lcase.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/lcase.md new file mode 100644 index 00000000000000..0133557156583c --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/lcase.md @@ -0,0 +1,37 @@ +--- +{ + "title": "lcase", + "language": "zh-CN" +} +--- + + + +# lcase +## description +### Syntax + +`INT lcase(VARCHAR str)` + + +与`lower`一致 + +##keyword +LCASE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/left.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/left.md new file mode 100644 index 00000000000000..99a9b83e330641 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/left.md @@ -0,0 +1,47 @@ +--- +{ + "title": "left", + "language": "zh-CN" +} +--- + + + +# left +## description +### Syntax + +`VARCHAR left(VARCHAR str)` + + +它返回具有指定长度的字符串的左边部分 + +## example + +``` +mysql> select left("Hello doris",5); ++------------------------+ +| left('Hello doris', 5) | ++------------------------+ +| Hello | ++------------------------+ +``` +##keyword +LEFT diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/length.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/length.md new file mode 100644 index 00000000000000..f86e3ee02b43ed --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/length.md @@ -0,0 +1,54 @@ +--- +{ + "title": "length", + "language": "zh-CN" +} +--- + + + +# length +## description +### Syntax + +`INT length(VARCHAR str)` + + +返回字符串的长度，对于多字节字符，返回的字符数。比如5个两字节宽度字，返回的长度是10。 + +## example + +``` +mysql> select length("abc"); ++---------------+ +| length('abc') | ++---------------+ +| 3 | ++---------------+ + +mysql> select length("中国"); ++------------------+ +| length('中国') | ++------------------+ +| 6 | ++------------------+ +``` +##keyword +LENGTH diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/locate.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/locate.md new file mode 100644 index 00000000000000..512c71bc88f682 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/locate.md @@ -0,0 +1,61 @@ +--- +{ + "title": "locate", + "language": "zh-CN" +} +--- + + + +# locate +## description +### Syntax + +`INT locate(VARCHAR substr, VARCHAR str[, INT pos])` + + +返回 substr 在 str 中出现的位置（从1开始计数）。如果指定第3个参数 pos，则从 str 以 pos 下标开始的字符串处开始查找 substr 出现的位置。如果没有找到，返回0 + +## example + +``` +mysql> SELECT LOCATE('bar', 'foobarbar'); ++----------------------------+ +| locate('bar', 'foobarbar') | ++----------------------------+ +| 4 | ++----------------------------+ + +mysql> SELECT LOCATE('xbar', 'foobar'); ++--------------------------+ +| locate('xbar', 'foobar') | ++--------------------------+ +| 0 | ++--------------------------+ + +mysql> SELECT LOCATE('bar', 'foobarbar', 5); ++-------------------------------+ +| locate('bar', 'foobarbar', 5) | ++-------------------------------+ +| 7 | ++-------------------------------+ +``` +##keyword +LOCATE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/lower.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/lower.md new file mode 100644 index 00000000000000..81a65d846ba78b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/lower.md @@ -0,0 +1,47 @@ +--- +{ + "title": "lower", + "language": "zh-CN" +} +--- + + + +# lower +## description +### Syntax + +`INT lower(VARCHAR str)` + + +将参数中所有的字符串都转换成小写 + +## example + +``` +mysql> SELECT lower("AbC123"); ++-----------------+ +| lower('AbC123') | ++-----------------+ +| abc123 | ++-----------------+ +``` +##keyword +LOWER diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/lpad.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/lpad.md new file mode 100644 index 00000000000000..62ea2cd03136d3 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/lpad.md @@ -0,0 +1,54 @@ +--- +{ + "title": "lpad", + "language": "zh-CN" +} +--- + + + +# lpad +## description +### Syntax + +`VARCHAR lpad(VARCHAR str, INT len, VARCHAR pad)` + + +返回 str 中长度为 len（从首字母开始算起）的字符串。如果 len 大于 str 的长度，则在 str 的前面不断补充 pad 字符，直到该字符串的长度达到 len 为止。如果 len 小于 str 的长度，该函数相当于截断 str 字符串，只返回长度为 len 的字符串。 + +## example + +``` +mysql> SELECT lpad("hi", 5, "xy"); ++---------------------+ +| lpad('hi', 5, 'xy') | ++---------------------+ +| xyxhi | ++---------------------+ + +mysql> SELECT lpad("hi", 1, "xy"); ++---------------------+ +| lpad('hi', 1, 'xy') | ++---------------------+ +| h | ++---------------------+ +``` +##keyword +LPAD diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/ltrim.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/ltrim.md new file mode 100644 index 00000000000000..129ec557328c09 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/ltrim.md @@ -0,0 +1,47 @@ +--- +{ + "title": "ltrim", + "language": "zh-CN" +} +--- + + + +# ltrim +## description +### Syntax + +`VARCHAR ltrim(VARCHAR str)` + + +将参数 str 中从开始部分连续出现的空格去掉 + +## example + +``` +mysql> SELECT ltrim(' ab d'); ++------------------+ +| ltrim(' ab d') | ++------------------+ +| ab d | ++------------------+ +``` +##keyword +LTRIM diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/money_format.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/money_format.md new file mode 100644 index 00000000000000..6e99ab2f2f087b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/money_format.md @@ -0,0 +1,61 @@ +--- +{ + "title": "money_format", + "language": "zh-CN" +} +--- + + + +# money_format +## description +### Syntax + +VARCHAR money_format(Number) + + +将数字按照货币格式输出，整数部分每隔3位用逗号分隔，小数部分保留2位 + +## example + +``` +mysql> select money_format(17014116); ++------------------------+ +| money_format(17014116) | ++------------------------+ +| 17,014,116.00 | ++------------------------+ + +mysql> select money_format(1123.456); ++------------------------+ +| money_format(1123.456) | ++------------------------+ +| 1,123.46 | ++------------------------+ + +mysql> select money_format(1123.4); ++----------------------+ +| money_format(1123.4) | ++----------------------+ +| 1,123.40 | ++----------------------+ +``` +##keyword +MONEY_FORMAT,MONEY,FORMAT diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/null_or_empty.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/null_or_empty.md new file mode 100644 index 00000000000000..05dd90360f2aaf --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/null_or_empty.md @@ -0,0 +1,60 @@ +--- +{ + "title": "null_or_empty", + "language": "zh-CN" +} +--- + + + +# null_or_empty +## description +### Syntax + +`BOOLEAN NULL_OR_EMPTY (VARCHAR str)` + +如果字符串为空字符串或者NULL，返回true。否则，返回false。 + +## example + +``` +MySQL [(none)]> select null_or_empty(null); ++---------------------+ +| null_or_empty(NULL) | ++---------------------+ +| 1 | ++---------------------+ + +MySQL [(none)]> select null_or_empty(""); ++-------------------+ +| null_or_empty('') | ++-------------------+ +| 1 | ++-------------------+ + +MySQL [(none)]> select null_or_empty("a"); ++--------------------+ +| null_or_empty('a') | ++--------------------+ +| 0 | ++--------------------+ +``` +##keyword +NULL_OR_EMPTY \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/regexp_extract.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/regexp_extract.md new file mode 100644 index 00000000000000..f15bbd21a17033 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/regexp_extract.md @@ -0,0 +1,54 @@ +--- +{ + "title": "regexp_extract", + "language": "zh-CN" +} +--- + + + +# regexp_extract +## description +### Syntax + +`VARCHAR regexp_extract(VARCHAR str, VARCHAR pattern, int pos)` + + +对字符串 str 进行正则匹配，抽取符合 pattern 的第 pos 个匹配部分。需要 pattern 完全匹配 str 中的某部分，这样才能返回 pattern 部分中需匹配部分。如果没有匹配，返回空字符串。 + +## example + +``` +mysql> SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 1); ++-------------------------------------------------------------+ +| regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 1) | ++-------------------------------------------------------------+ +| b | ++-------------------------------------------------------------+ + +mysql> SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 2); ++-------------------------------------------------------------+ +| regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', 2) | ++-------------------------------------------------------------+ +| d | ++-------------------------------------------------------------+ +``` +##keyword +REGEXP_EXTRACT,REGEXP,EXTRACT diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/regexp_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/regexp_replace.md new file mode 100644 index 00000000000000..ea0fa40b127193 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/regexp_replace.md @@ -0,0 +1,54 @@ +--- +{ + "title": "regexp_replace", + "language": "zh-CN" +} +--- + + + +# regexp_replace +## description +### Syntax + +`VARCHAR regexp_replace(VARCHAR str, VARCHAR pattern, VARCHAR repl) + + +对字符串 str 进行正则匹配, 将命中 pattern 的部分使用 repl 来进行替换 + +## example + +``` +mysql> SELECT regexp_replace('a b c', " ", "-"); ++-----------------------------------+ +| regexp_replace('a b c', ' ', '-') | ++-----------------------------------+ +| a-b-c | ++-----------------------------------+ + +mysql> SELECT regexp_replace('a b c','(b)','<\\1>'); ++----------------------------------------+ +| regexp_replace('a b c', '(b)', '<\1>') | ++----------------------------------------+ +| a c | ++----------------------------------------+ +``` +##keyword +REGEXP_REPLACE,REGEXP,REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/repeat.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/repeat.md new file mode 100644 index 00000000000000..9bd00851978aca --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/repeat.md @@ -0,0 +1,54 @@ +--- +{ + "title": "repeat", + "language": "zh-CN" +} +--- + + + +# repeat +## description +### Syntax + +`VARCHAR repeat(VARCHAR str, INT count) + + +将字符串 str 重复 count 次输出，count 小于1时返回空串，str，count 任一为NULL时，返回 NULL + +## example + +``` +mysql> SELECT repeat("a", 3); ++----------------+ +| repeat('a', 3) | ++----------------+ +| aaa | ++----------------+ + +mysql> SELECT repeat("a", -1); ++-----------------+ +| repeat('a', -1) | ++-----------------+ +| | ++-----------------+ +``` +##keyword +REPEAT, diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/right.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/right.md new file mode 100644 index 00000000000000..133c92b86cbb53 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/right.md @@ -0,0 +1,47 @@ +--- +{ + "title": "right", + "language": "zh-CN" +} +--- + + + +# right +## description +### Syntax + +`VARCHAR right(VARCHAR str)` + + +它返回具有指定长度的字符串的右边部分 + +## example + +``` +mysql> select right("Hello doris",5); ++-------------------------+ +| right('Hello doris', 5) | ++-------------------------+ +| doris | ++-------------------------+ +``` +##keyword +RIGHT diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/split_part.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/split_part.md new file mode 100644 index 00000000000000..e284b3a8168bef --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/split_part.md @@ -0,0 +1,69 @@ +--- +{ + "title": "split_part", + "language": "zh-CN" +} +--- + + + +# split_part +## description +### Syntax + +`VARCHAR split_part(VARCHAR content, VARCHAR delimiter, INT field)` + + +根据分割符拆分字符串, 返回指定的分割部分(从一开始计数)。 + +## example + +``` +mysql> select split_part("hello world", " ", 1); ++----------------------------------+ +| split_part('hello world', ' ', 1) | ++----------------------------------+ +| hello | ++----------------------------------+ + + +mysql> select split_part("hello world", " ", 2); ++----------------------------------+ +| split_part('hello world', ' ', 2) | ++----------------------------------+ +| world | ++----------------------------------+ + +mysql> select split_part("2019年7月8号", "月", 1); ++-----------------------------------------+ +| split_part('2019年7月8号', '月', 1) | ++-----------------------------------------+ +| 2019年7 | ++-----------------------------------------+ + +mysql> select split_part("abca", "a", 1); ++----------------------------+ +| split_part('abca', 'a', 1) | ++----------------------------+ +| | ++----------------------------+ +``` +##keyword +SPLIT_PART,SPLIT,PART diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/starts_with.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/starts_with.md new file mode 100644 index 00000000000000..dc914148643f6b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/starts_with.md @@ -0,0 +1,53 @@ +--- +{ + "title": "starts_with", + "language": "zh-CN" +} +--- + + + +# starts_with +## description +### Syntax + +`BOOLEAN STARTS_WITH (VARCHAR str, VARCHAR prefix)` + +如果字符串以指定前缀开头，返回true。否则，返回false。任意参数为NULL，返回NULL。 + +## example + +``` +MySQL [(none)]> select starts_with("hello world","hello"); ++-------------------------------------+ +| starts_with('hello world', 'hello') | ++-------------------------------------+ +| 1 | ++-------------------------------------+ + +MySQL [(none)]> select starts_with("hello world","world"); ++-------------------------------------+ +| starts_with('hello world', 'world') | ++-------------------------------------+ +| 0 | ++-------------------------------------+ +``` +##keyword +STARTS_WITH \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/strleft.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/strleft.md new file mode 100644 index 00000000000000..c4b5d32227eb90 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/strleft.md @@ -0,0 +1,47 @@ +--- +{ + "title": "strleft", + "language": "zh-CN" +} +--- + + + +# strleft +## description +### Syntax + +`VARCHAR strleft(VARCHAR str)` + + +它返回具有指定长度的字符串的左边部分 + +## example + +``` +mysql> select strleft("Hello doris",5); ++------------------------+ +| strleft('Hello doris', 5) | ++------------------------+ +| Hello | ++------------------------+ +``` +##keyword +STRLEFT diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/strright.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/strright.md new file mode 100644 index 00000000000000..b6e8657f8bf9d6 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/strright.md @@ -0,0 +1,47 @@ +--- +{ + "title": "strright", + "language": "zh-CN" +} +--- + + + +# strright +## description +### Syntax + +`VARCHAR strright(VARCHAR str)` + + +它返回具有指定长度的字符串的右边部分 + +## example + +``` +mysql> select strright("Hello doris",5); ++-------------------------+ +| strright('Hello doris', 5) | ++-------------------------+ +| doris | ++-------------------------+ +``` +##keyword +STRRIGHT diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/CREATE ROLE.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/CREATE ROLE.md new file mode 100644 index 00000000000000..6b430a203753ba --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/CREATE ROLE.md @@ -0,0 +1,45 @@ +--- +{ + "title": "CREATE ROLE", + "language": "zh-CN" +} +--- + + + +# CREATE ROLE +## description + 该语句用户创建一个角色 + + 语法： + CREATE ROLE role1; + + 该语句创建一个无权限的角色，可以后续通过 GRANT 命令赋予该角色权限。 + +## example + + 1. 创建一个角色 + + CREATE ROLE role1; + +## keyword + + CREATE, ROLE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/CREATE USER.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/CREATE USER.md new file mode 100644 index 00000000000000..2222e987efb1a9 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/CREATE USER.md @@ -0,0 +1,76 @@ +--- +{ + "title": "CREATE USER", + "language": "zh-CN" +} +--- + + + +# CREATE USER +## description + +Syntax: + + CREATE USER user_identity [IDENTIFIED BY 'password'] [DEFAULT ROLE 'role_name'] + + user_identity: + 'user_name'@'host' + +CREATE USER 命令用于创建一个 Doris 用户。在 Doris 中，一个 user_identity 唯一标识一个用户。user_identity 由两部分组成，user_name 和 host，其中 username 为用户名。host 标识用户端连接所在的主机地址。host 部分可以使用 % 进行模糊匹配。如果不指定 host，默认为 '%'，即表示该用户可以从任意 host 连接到 Doris。 + +host 部分也可指定为 domain，语法为：'user_name'@['domain']，即使用中括号包围，则 Doris 会认为这个是一个 domain，并尝试解析其 ip 地址。目前仅支持百度内部的 BNS 解析。 + +如果指定了角色（ROLE），则会自动将该角色所拥有的权限赋予新创建的这个用户。如果不指定，则该用户默认没有任何权限。指定的 ROLE 必须已经存在。 + +## example + +1. 创建一个无密码用户（不指定 host，则等价于 jack@'%'） + + CREATE USER 'jack'; + +2. 创建一个有密码用户，允许从 '172.10.1.10' 登陆 + + CREATE USER jack@'172.10.1.10' IDENTIFIED BY '123456'; + +3. 为了避免传递明文，用例2也可以使用下面的方式来创建 + + CREATE USER jack@'172.10.1.10' IDENTIFIED BY PASSWORD '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9'; + + 后面加密的内容可以通过PASSWORD()获得到,例如： + + SELECT PASSWORD('123456'); + +4. 创建一个允许从 '192.168' 子网登陆的用户，同时指定其角色为 example_role + + CREATE USER 'jack'@'192.168.%' DEFAULT ROLE 'example_role'; + +5. 创建一个允许从域名 'example_domain' 登陆的用户 + + CREATE USER 'jack'@['example_domain'] IDENTIFIED BY '12345'; + +6. 创建一个用户，并指定一个角色 + + CREATE USER 'jack'@'%' IDENTIFIED BY '12345' DEFAULT ROLE 'my_role'; + +## keyword + + CREATE, USER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/DROP ROLE.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/DROP ROLE.md new file mode 100644 index 00000000000000..ee0cc980bc8c97 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/DROP ROLE.md @@ -0,0 +1,44 @@ +--- +{ + "title": "DROP ROLE", + "language": "zh-CN" +} +--- + + + +# DROP ROLE +## description + 该语句用户删除一个角色 + + 语法： + DROP ROLE role1; + + 删除一个角色，不会影响之前属于该角色的用户的权限。仅相当于将该角色与用户解耦。用户已经从该角色中获取到的权限，不会改变。 + +## example + + 1. 删除一个角色 + + DROP ROLE role1; + +## keyword + DROP, ROLE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/DROP USER.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/DROP USER.md new file mode 100644 index 00000000000000..dbdebd164a2fe8 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/DROP USER.md @@ -0,0 +1,50 @@ +--- +{ + "title": "DROP USER", + "language": "zh-CN" +} +--- + + + +# DROP USER +## description + +Syntax: + + DROP USER 'user_identity' + + `user_identity`: + + user@'host' + user@['domain'] + + 删除指定的 user identitiy. + +## example + +1. 删除用户 jack@'192.%' + + DROP USER 'jack'@'192.%' + +## keyword + + DROP, USER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/GRANT.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/GRANT.md new file mode 100644 index 00000000000000..ef460589ef0242 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/GRANT.md @@ -0,0 +1,82 @@ +--- +{ + "title": "GRANT", + "language": "zh-CN" +} +--- + + + +# GRANT +## description + +GRANT 命令用于赋予指定用户或角色指定的权限。 + +Syntax: + + GRANT privilege_list ON db_name[.tbl_name] TO user_identity [ROLE role_name] + + +privilege_list 是需要赋予的权限列表，以逗号分隔。当前 Doris 支持如下权限： + + NODE_PRIV：集群节点操作权限，包括节点上下线等操作，只有 root 用户有该权限，不可赋予其他用户。 + ADMIN_PRIV：除 NODE_PRIV 以外的所有权限。 + GRANT_PRIV: 操作权限的权限。包括创建删除用户、角色，授权和撤权，设置密码等。 + SELECT_PRIV：对指定的库或表的读取权限 + LOAD_PRIV：对指定的库或表的导入权限 + ALTER_PRIV：对指定的库或表的schema变更权限 + CREATE_PRIV：对指定的库或表的创建权限 + DROP_PRIV：对指定的库或表的删除权限 + + 旧版权限中的 ALL 和 READ_WRITE 会被转换成：SELECT_PRIV,LOAD_PRIV,ALTER_PRIV,CREATE_PRIV,DROP_PRIV； + READ_ONLY 会被转换为 SELECT_PRIV。 + +db_name[.tbl_name] 支持以下三种形式： + + 1. *.* 权限可以应用于所有库及其中所有表 + 2. db.* 权限可以应用于指定库下的所有表 + 3. db.tbl 权限可以应用于指定库下的指定表 + + 这里指定的库或表可以是不存在的库和表。 + +user_identity： + + 这里的 user_identity 语法同 CREATE USER。且必须为使用 CREATE USER 创建过的 user_identity。user_identity 中的host可以是域名，如果是域名的话，权限的生效时间可能会有1分钟左右的延迟。 + + 也可以将权限赋予指定的 ROLE，如果指定的 ROLE 不存在，则会自动创建。 + +## example + + 1. 授予所有库和表的权限给用户 + + GRANT SELECT_PRIV ON *.* TO 'jack'@'%'; + + 2. 授予指定库表的权限给用户 + + GRANT SELECT_PRIV,ALTER_PRIV,LOAD_PRIV ON db1.tbl1 TO 'jack'@'192.8.%'; + + 3. 授予指定库表的权限给角色 + + GRANT LOAD_PRIV ON db1.* TO ROLE 'my_role'; + +## keyword + + GRANT + diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/REVOKE.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/REVOKE.md new file mode 100644 index 00000000000000..d016b3b5107dc4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/REVOKE.md @@ -0,0 +1,49 @@ +--- +{ + "title": "REVOKE", + "language": "zh-CN" +} +--- + + + +# REVOKE +## description + + REVOKE 命令用于撤销指定用户或角色指定的权限。 + Syntax： + REVOKE privilege_list ON db_name[.tbl_name] FROM user_identity [ROLE role_name] + + user_identity： + + 这里的 user_identity 语法同 CREATE USER。且必须为使用 CREATE USER 创建过的 user_identity。user_identity 中的host可以是域名，如果是域名的话，权限的撤销时间可能会有1分钟左右的延迟。 + + 也可以撤销指定的 ROLE 的权限，执行的 ROLE 必须存在。 + +## example + + 1. 撤销用户 jack 数据库 testDb 的权限 + + REVOKE SELECT_PRIV ON db1.* FROM 'jack'@'192.%'; + +## keyword + + REVOKE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PASSWORD.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PASSWORD.md new file mode 100644 index 00000000000000..63ae6158681e8e --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PASSWORD.md @@ -0,0 +1,56 @@ +--- +{ + "title": "SET PASSWORD", + "language": "zh-CN" +} +--- + + + +# SET PASSWORD +## description + +Syntax: + + SET PASSWORD [FOR user_identity] = + [PASSWORD('plain password')]|['hashed password'] + + SET PASSWORD 命令可以用于修改一个用户的登录密码。如果 [FOR user_identity] 字段不存在，那么修改当前用户的密码。 + + 注意这里的 user_identity 必须完全匹配在使用 CREATE USER 创建用户时指定的 user_identity，否则会报错用户不存在。如果不指定 user_identity，则当前用户为 'username'@'ip'，这个当前用户，可能无法匹配任何 user_identity。可以通过 SHOW GRANTS 查看当前用户。 + + PASSWORD() 方式输入的是明文密码; 而直接使用字符串，需要传递的是已加密的密码。 + 如果修改其他用户的密码，需要具有管理员权限。 + +## example + +1. 修改当前用户的密码 + + SET PASSWORD = PASSWORD('123456') + SET PASSWORD = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' + +2. 修改指定用户密码 + + SET PASSWORD FOR 'jack'@'192.%' = PASSWORD('123456') + SET PASSWORD FOR 'jack'@['domain'] = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' + +## keyword + SET, PASSWORD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PROPERTY.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PROPERTY.md new file mode 100644 index 00000000000000..41f294fa46651d --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PROPERTY.md @@ -0,0 +1,82 @@ +--- +{ + "title": "SET PROPERTY", + "language": "zh-CN" +} +--- + + + +# SET PROPERTY +## description + + Syntax: + + SET PROPERTY [FOR 'user'] 'key' = 'value' [, 'key' = 'value'] + + 设置用户的属性，包括分配给用户的资源、导入cluster等。这里设置的用户属性，是针对 user 的，而不是 user_identity。即假设通过 CREATE USER 语句创建了两个用户 'jack'@'%' 和 'jack'@'192.%'，则使用 SET PROPERTY 语句，只能针对 jack 这个用户，而不是 'jack'@'%' 或 'jack'@'192.%' + + 导入 cluster 仅适用于百度内部用户。 + + key: + + 超级用户权限: + max_user_connections: 最大连接数。 + resource.cpu_share: cpu资源分配。 + load_cluster.{cluster_name}.priority: 为指定的cluster分配优先级，可以为 HIGH 或 NORMAL + + 普通用户权限： + quota.normal: normal级别的资源分配。 + quota.high: high级别的资源分配。 + quota.low: low级别的资源分配。 + + load_cluster.{cluster_name}.hadoop_palo_path: palo使用的hadoop目录，需要存放etl程序及etl生成的中间数据供palo导入。导入完成后会自动清理中间数据，etl程序自动保留下次使用。 + load_cluster.{cluster_name}.hadoop_configs: hadoop的配置，其中fs.default.name、mapred.job.tracker、hadoop.job.ugi必须填写。 + load_cluster.{cluster_name}.hadoop_http_port: hadoop hdfs name node http端口。其中 hdfs 默认为8070，afs 默认 8010。 + default_load_cluster: 默认的导入cluster。 + +## example + + 1. 修改用户 jack 最大连接数为1000 + SET PROPERTY FOR 'jack' 'max_user_connections' = '1000'; + + 2. 修改用户 jack 的cpu_share为1000 + SET PROPERTY FOR 'jack' 'resource.cpu_share' = '1000'; + + 3. 修改 jack 用户的normal组的权重 + SET PROPERTY FOR 'jack' 'quota.normal' = '400'; + + 4. 为用户 jack 添加导入cluster + SET PROPERTY FOR 'jack' + 'load_cluster.{cluster_name}.hadoop_palo_path' = '/user/palo/palo_path', + 'load_cluster.{cluster_name}.hadoop_configs' = 'fs.default.name=hdfs://dpp.cluster.com:port;mapred.job.tracker=dpp.cluster.com:port;hadoop.job.ugi=user,password;mapred.job.queue.name=job_queue_name_in_hadoop;mapred.job.priority=HIGH;'; + + 5. 删除用户 jack 下的导入cluster。 + SET PROPERTY FOR 'jack' 'load_cluster.{cluster_name}' = ''; + + 6. 修改用户 jack 默认的导入cluster + SET PROPERTY FOR 'jack' 'default_load_cluster' = '{cluster_name}'; + + 7. 修改用户 jack 的集群优先级为 HIGH + SET PROPERTY FOR 'jack' 'load_cluster.{cluster_name}.priority' = 'HIGH'; + +## keyword + SET, PROPERTY + diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/SHOW GRANTS.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/SHOW GRANTS.md new file mode 100644 index 00000000000000..df9fa5ae7f23b8 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/SHOW GRANTS.md @@ -0,0 +1,57 @@ +--- +{ + "title": "SHOW GRANTS", + "language": "zh-CN" +} +--- + + + +# SHOW GRANTS +## description + + 该语句用于查看用户权限。 + + 语法： + SHOW [ALL] GRANTS [FOR user_identity]; + + 说明： + 1. SHOW ALL GRANTS 可以查看所有用户的权限。 + 2. 如果指定 user_identity，则查看该指定用户的权限。且该 user_identity 必须为通过 CREATE USER 命令创建的。 + 3. 如果不指定 user_identity，则查看当前用户的权限。 + + +## example + + 1. 查看所有用户权限信息 + + SHOW ALL GRANTS; + + 2. 查看指定 user 的权限 + + SHOW GRANTS FOR jack@'%'; + + 3. 查看当前用户的权限 + + SHOW GRANTS; + +## keyword + + SHOW, GRANTS diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/SHOW ROLES.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/SHOW ROLES.md new file mode 100644 index 00000000000000..ddf694812cd422 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/SHOW ROLES.md @@ -0,0 +1,42 @@ +--- +{ + "title": "SHOW ROLES", + "language": "zh-CN" +} +--- + + + +# SHOW ROLES +## description + 该语句用于展示所有已创建的角色信息，包括角色名称，包含的用户以及权限。 + + 语法： + SHOW ROLES; + +## example + + 1. 查看已创建的角色： + + SHOW ROLES; + +## keyword + SHOW,ROLES + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md new file mode 100644 index 00000000000000..f07cc70222cd6f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN CANCEL REPAIR.md @@ -0,0 +1,48 @@ +--- +{ + "title": "ADMIN CANCEL REPAIR", + "language": "zh-CN" +} +--- + + + +# ADMIN CANCEL REPAIR +## description + + 该语句用于取消以高优先级修复指定表或分区 + + 语法： + + ADMIN CANCEL REPAIR TABLE table_name[ PARTITION (p1,...)]; + + 说明： + + 1. 该语句仅表示系统不再以高优先级修复指定表或分区的分片副本。系统仍会以默认调度方式修复副本。 + +## example + + 1. 取消高优先级修复 + + ADMIN CANCEL REPAIR TABLE tbl PARTITION(p1); + +## keyword + ADMIN,CANCEL,REPAIR + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md new file mode 100644 index 00000000000000..b71eb0dc1e2816 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN CHECK TABLET.md @@ -0,0 +1,57 @@ +--- +{ + "title": "ADMIN CHECK TABLET", + "language": "zh-CN" +} +--- + + + +# ADMIN CHECK TABLET +## description + +该语句用于对一组 tablet 执行指定的检查操作 + +语法： + +``` +ADMIN CHECK TABLE (tablet_id1, tablet_id2, ...) +PROPERTIES("type" = "..."); +``` + +说明： + +1. 必须指定 tablet id 列表以及 PROPERTIES 中的 type 属性。 +2. 目前 type 仅支持： + + * consistency: 对tablet的副本数据一致性进行检查。该命令为异步命令，发送后，Doris 会开始执行对应 tablet 的一致性检查作业。最终的结果，将体现在 `SHOW PROC "/statistic";` 结果中的 InconsistentTabletNum 列。 + +## example + +1. 对指定的一组 tablet 进行副本数据一致性检查 + + ``` + ADMIN CHECK TABLET (10000, 10001) + PROPERTIES("type" = "consistency"); + ``` + +## keyword + + ADMIN,CHECK,TABLET diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN REPAIR.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN REPAIR.md new file mode 100644 index 00000000000000..851b0fa496a603 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN REPAIR.md @@ -0,0 +1,53 @@ +--- +{ + "title": "ADMIN REPAIR", + "language": "zh-CN" +} +--- + + + +# ADMIN REPAIR +## description + + 该语句用于尝试优先修复指定的表或分区 + + 语法： + + ADMIN REPAIR TABLE table_name[ PARTITION (p1,...)] + + 说明： + + 1. 该语句仅表示让系统尝试以高优先级修复指定表或分区的分片副本，并不保证能够修复成功。用户可以通过 ADMIN SHOW REPLICA STATUS 命令查看修复情况。 + 2. 默认的 timeout 是 14400 秒(4小时)。超时意味着系统将不再以高优先级修复指定表或分区的分片副本。需要重新使用该命令设置。 + +## example + + 1. 尝试修复指定表 + + ADMIN REPAIR TABLE tbl1; + + 2. 尝试修复指定分区 + + ADMIN REPAIR TABLE tbl1 PARTITION (p1, p2); + +## keyword + ADMIN,REPAIR + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md new file mode 100644 index 00000000000000..5341de98cf7de9 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SET CONFIG.md @@ -0,0 +1,44 @@ +--- +{ + "title": "ADMIN SET CONFIG", + "language": "zh-CN" +} +--- + + + +# ADMIN SET CONFIG +## description + + 该语句用于设置集群的配置项（当前仅支持设置FE的配置项）。 + 可设置的配置项，可以通过 AMDIN SHOW FRONTEND CONFIG; 命令查看。 + + 语法： + + ADMIN SET FRONTEND CONFIG ("key" = "value"); + +## example + + 1. 设置 'disable_balance' 为 true + + ADMIN SET FRONTEND CONFIG ("disable_balance" = "true"); + +## keyword + ADMIN,SET,CONFIG diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md new file mode 100644 index 00000000000000..71fd90fca420d1 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SET REPLICA STATUS.md @@ -0,0 +1,62 @@ +--- +{ + "title": "ADMIN SET REPLICA STATUS", + "language": "zh-CN" +} +--- + + + +# ADMIN SET REPLICA STATUS +## description + + 该语句用于设置指定副本的状态。 + 该命令目前仅用于手动将某些副本状态设置为 BAD 或 OK，从而使得系统能够自动修复这些副本。 + + 语法： + + ADMIN SET REPLICA STATUS + PROPERTIES ("key" = "value", ...); + + 目前支持如下属性： + "tablet_id"：必需。指定一个 Tablet Id. + "backend_id"：必需。指定 Backend Id. + "status"：必需。指定状态。当前仅支持 "bad" 或 "ok" + + 如果指定的副本不存在，或状态已经是 bad，则会被忽略。 + + 注意： + + 设置为 Bad 状态的副本可能立刻被删除，请谨慎操作。 + +## example + + 1. 设置 tablet 10003 在 BE 10001 上的副本状态为 bad。 + + ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "10003", "backend_id" = "10001", "status" = "bad"); + + 2. 设置 tablet 10003 在 BE 10001 上的副本状态为 ok。 + + ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "10003", "backend_id" = "10001", "status" = "ok"); + +## keyword + + ADMIN,SET,REPLICA,STATUS + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md new file mode 100644 index 00000000000000..f10c3c93ca53b5 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW CONFIG.md @@ -0,0 +1,53 @@ +--- +{ + "title": "ADMIN SHOW CONFIG", + "language": "zh-CN" +} +--- + + + +# ADMIN SHOW CONFIG +## description + + 该语句用于展示当前集群的配置（当前仅支持展示 FE 的配置项） + + 语法： + + ADMIN SHOW FRONTEND CONFIG; + + 说明： + + 结果中的各列含义如下： + 1. Key： 配置项名称 + 2. Value： 配置项值 + 3. Type： 配置项类型 + 4. IsMutable： 是否可以通过 ADMIN SET CONFIG 命令设置 + 5. MasterOnly： 是否仅适用于 Master FE + 6. Comment： 配置项说明 + +## example + + 1. 查看当前FE节点的配置 + + ADMIN SHOW FRONTEND CONFIG; + +## keyword + ADMIN,SHOW,CONFIG diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md new file mode 100644 index 00000000000000..6558981a7700a5 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA DISTRIBUTION.md @@ -0,0 +1,52 @@ +--- +{ + "title": "ADMIN SHOW REPLICA DISTRIBUTION", + "language": "zh-CN" +} +--- + + + +# ADMIN SHOW REPLICA DISTRIBUTION +## description + + 该语句用于展示一个表或分区副本分布状态 + + 语法： + + ADMIN SHOW REPLICA DISTRIBUTION FROM [db_name.]tbl_name [PARTITION (p1, ...)]; + + 说明： + + 结果中的 Graph 列以图形的形式展示副本分布比例 + +## example + + 1. 查看表的副本分布 + + ADMIN SHOW REPLICA DISTRIBUTION FROM tbl1; + + 2. 查看表的分区的副本分布 + + ADMIN SHOW REPLICA DISTRIBUTION FROM db1.tbl1 PARTITION(p1, p2); + +## keyword + ADMIN,SHOW,REPLICA,DISTRIBUTION + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md new file mode 100644 index 00000000000000..ccebcd6d5d0b35 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN SHOW REPLICA STATUS.md @@ -0,0 +1,65 @@ +--- +{ + "title": "ADMIN SHOW REPLICA STATUS", + "language": "zh-CN" +} +--- + + + +# ADMIN SHOW REPLICA STATUS +## description + + 该语句用于展示一个表或分区的副本状态信息 + + 语法： + + ADMIN SHOW REPLICA STATUS FROM [db_name.]tbl_name [PARTITION (p1, ...)] + [where_clause]; + + where_clause: + WHERE STATUS [!]= "replica_status" + + replica_status: + OK: replica 处于健康状态 + DEAD: replica 所在 Backend 不可用 + VERSION_ERROR: replica 数据版本有缺失 + SCHEMA_ERROR: replica 的 schema hash 不正确 + MISSING: replica 不存在 + +## example + + 1. 查看表全部的副本状态 + + ADMIN SHOW REPLICA STATUS FROM db1.tbl1; + + 2. 查看表某个分区状态为 VERSION_ERROR 的副本 + + ADMIN SHOW REPLICA STATUS FROM tbl1 PARTITION (p1, p2) + WHERE STATUS = "VERSION_ERROR"; + + 3. 查看表所有状态不健康的副本 + + ADMIN SHOW REPLICA STATUS FROM tbl1 + WHERE STATUS != "OK"; + +## keyword + ADMIN,SHOW,REPLICA,STATUS + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ALTER CLUSTER.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ALTER CLUSTER.md new file mode 100644 index 00000000000000..dc77914158d8e8 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ALTER CLUSTER.md @@ -0,0 +1,54 @@ +--- +{ + "title": "ALTER CLUSTER", + "language": "zh-CN" +} +--- + + + +# ALTER CLUSTER +## description + + 该语句用于更新逻辑集群。需要有管理员权限 + + 语法 + + ALTER CLUSTER cluster_name PROPERTIES ("key"="value", ...); + + 1. 缩容，扩容 （根据集群现有的be数目，大则为扩容，小则为缩容), 扩容为同步操作，缩容为异步操作，通过backend的状态可以得知是否缩容完成 + + PROERTIES ("instance_num" = "3") + + instance_num 逻辑集群节点树 + +## example + + 1. 缩容，减少含有3个be的逻辑集群test_cluster的be数为2 + + ALTER CLUSTER test_cluster PROPERTIES ("instance_num"="2"); + + 2. 扩容，增加含有3个be的逻辑集群test_cluster的be数为4 + + ALTER CLUSTER test_cluster PROPERTIES ("instance_num"="4"); + +## keyword + ALTER,CLUSTER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ALTER SYSTEM.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ALTER SYSTEM.md new file mode 100644 index 00000000000000..dcea763654984e --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ALTER SYSTEM.md @@ -0,0 +1,120 @@ +--- +{ + "title": "ALTER SYSTEM", + "language": "zh-CN" +} +--- + + + +# ALTER SYSTEM +## description + + 该语句用于操作一个系统内的节点。（仅管理员使用！） + 语法： + 1) 增加节点(不使用多租户功能则按照此方法添加) + ALTER SYSTEM ADD BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; + 2) 增加空闲节点(即添加不属于任何cluster的BACKEND) + ALTER SYSTEM ADD FREE BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; + 3) 增加节点到某个cluster + ALTER SYSTEM ADD BACKEND TO cluster_name "host:heartbeat_port"[,"host:heartbeat_port"...]; + 4) 删除节点 + ALTER SYSTEM DROP BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; + 5) 节点下线 + ALTER SYSTEM DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; + 6) 增加Broker + ALTER SYSTEM ADD BROKER broker_name "host:port"[,"host:port"...]; + 7) 减少Broker + ALTER SYSTEM DROP BROKER broker_name "host:port"[,"host:port"...]; + 8) 删除所有Broker + ALTER SYSTEM DROP ALL BROKER broker_name + 9) 设置一个 Load error hub，用于集中展示导入时的错误信息 + ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES ("key" = "value"[, ...]); + + 说明： + 1) host 可以是主机名或者ip地址 + 2) heartbeat_port 为该节点的心跳端口 + 3) 增加和删除节点为同步操作。这两种操作不考虑节点上已有的数据，节点直接从元数据中删除，请谨慎使用。 + 4) 节点下线操作用于安全下线节点。该操作为异步操作。如果成功，节点最终会从元数据中删除。如果失败，则不会完成下线。 + 5) 可以手动取消节点下线操作。详见 CANCEL DECOMMISSION + 6) Load error hub: + 当前支持两种类型的 Hub：Mysql 和 Broker。需在 PROPERTIES 中指定 "type" = "mysql" 或 "type" = "broker"。 + 如果需要删除当前的 load error hub，可以将 type 设为 null。 + 1) 当使用 Mysql 类型时，导入时产生的错误信息将会插入到指定的 mysql 库表中，之后可以通过 show load warnings 语句直接查看错误信息。 + + Mysql 类型的 Hub 需指定以下参数： + host：mysql host + port：mysql port + user：mysql user + password：mysql password + database：mysql database + table：mysql table + + 2) 当使用 Broker 类型时，导入时产生的错误信息会形成一个文件，通过 broker，写入到指定的远端存储系统中。须确保已经部署对应的 broker + Broker 类型的 Hub 需指定以下参数： + broker: broker 的名称 + path: 远端存储路径 + other properties: 其他访问远端存储所必须的信息，比如认证信息等。 + +## example + + 1. 增加一个节点 + ALTER SYSTEM ADD BACKEND "host:port"; + + 2. 增加一个空闲节点 + ALTER SYSTEM ADD FREE BACKEND "host:port"; + + 3. 删除两个节点 + ALTER SYSTEM DROP BACKEND "host1:port", "host2:port"; + + 4. 下线两个节点 + ALTER SYSTEM DECOMMISSION BACKEND "host1:port", "host2:port"; + + 5. 增加两个Hdfs Broker + ALTER SYSTEM ADD BROKER hdfs "host1:port", "host2:port"; + + 6. 添加一个 Mysql 类型的 load error hub + ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES + ("type"= "mysql", + "host" = "192.168.1.17" + "port" = "3306", + "user" = "my_name", + "password" = "my_passwd", + "database" = "doris_load", + "table" = "load_errors" + ); + + 7. 添加一个 Broker 类型的 load error hub + ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES + ("type"= "broker", + "name" = "bos", + "path" = "bos://backup-cmy/logs", + "bos_endpoint" = "http://gz.bcebos.com", + "bos_accesskey" = "069fc278xxxxxx24ddb522", + "bos_secret_accesskey"="700adb0c6xxxxxx74d59eaa980a" + ); + + 8. 删除当前的 load error hub + ALTER SYSTEM SET LOAD ERRORS HUB PROPERTIES + ("type"= "null"); + +## keyword + ALTER,SYSTEM,BACKEND,BROKER,FREE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md b/docs/zh-CN/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md new file mode 100644 index 00000000000000..ec5c7bfd1b4baf --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/CANCEL DECOMMISSION.md @@ -0,0 +1,41 @@ +--- +{ + "title": "CANCEL DECOMMISSION", + "language": "zh-CN" +} +--- + + + +# CANCEL DECOMMISSION +## description + + 该语句用于撤销一个节点下线操作。（仅管理员使用！） + 语法： + CANCEL DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; + +## example + + 1. 取消两个节点的下线操作： + CANCEL DECOMMISSION BACKEND "host1:port", "host2:port"; + +## keyword + CANCEL,DECOMMISSION,BACKEND + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/CREATE CLUSTER.md b/docs/zh-CN/sql-reference/sql-statements/Administration/CREATE CLUSTER.md new file mode 100644 index 00000000000000..daeaf197e9b40b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/CREATE CLUSTER.md @@ -0,0 +1,62 @@ +--- +{ + "title": "CREATE CLUSTER", + "language": "zh-CN" +} +--- + + + +# CREATE CLUSTER +## description + + 该语句用于新建逻辑集群 (cluster), 需要管理员权限。如果不使用多租户，直接创建一个名称为default_cluster的cluster。否则创建一个自定义名称的cluster。 + + 语法 + + CREATE CLUSTER [IF NOT EXISTS] cluster_name + + PROPERTIES ("key"="value", ...) + + IDENTIFIED BY 'password' + + 1. PROPERTIES + + 指定逻辑集群的属性 + + PROPERTIES ("instance_num" = "3") + + instance_num 逻辑集群节点树 + + 2. identified by ‘password' 每个逻辑集群含有一个superuser，创建逻辑集群时必须指定其密码 + +## example + + 1. 新建一个含有3个be节点逻辑集群 test_cluster, 并指定其superuser用户密码 + + CREATE CLUSTER test_cluster PROPERTIES("instance_num"="3") IDENTIFIED BY 'test'; + + 2. 新建一个含有3个be节点逻辑集群 default_cluster(不使用多租户), 并指定其superuser用户密码 + + CREATE CLUSTER default_cluster PROPERTIES("instance_num"="3") IDENTIFIED BY 'test'; + +## keyword + CREATE,CLUSTER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/CREATE FILE.md b/docs/zh-CN/sql-reference/sql-statements/Administration/CREATE FILE.md new file mode 100644 index 00000000000000..6c809abae036ca --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/CREATE FILE.md @@ -0,0 +1,77 @@ +--- +{ + "title": "CREATE FILE", + "language": "zh-CN" +} +--- + + + +# CREATE FILE +## description + + 该语句用于创建并上传一个文件到 Doris 集群。 + 该功能通常用于管理一些其他命令中需要使用到的文件，如证书、公钥私钥等等。 + + 该命令只用 amdin 权限用户可以执行。 + 某个文件都归属与某一个的 database。对 database 拥有访问权限的用户都可以使用该文件。 + + 单个文件大小限制为 1MB。 + 一个 Doris 集群最多上传 100 个文件。 + + 语法： + + CREATE FILE "file_name" [IN database] + [properties] + + 说明： + file_name: 自定义文件名。 + database: 文件归属于某一个 db，如果没有指定，则使用当前 session 的 db。 + properties 支持以下参数: + + url: 必须。指定一个文件的下载路径。当前仅支持无认证的 http 下载路径。命令执行成功后，文件将被保存在 doris 中，该 url 将不再需要。 + catalog: 必须。对文件的分类名，可以自定义。但在某些命令中，会查找指定 catalog 中的文件。比如例行导入中的，数据源为 kafka 时，会查找 catalog 名为 kafka 下的文件。 + md5: 可选。文件的 md5。如果指定，会在下载文件后进行校验。 + +## example + + 1. 创建文件 ca.pem ，分类为 kafka + + CREATE FILE "ca.pem" + PROPERTIES + ( + "url" = "https://test.bj.bcebos.com/kafka-key/ca.pem", + "catalog" = "kafka" + ); + + 2. 创建文件 client.key，分类为 my_catalog + + CREATE FILE "client.key" + IN my_database + PROPERTIES + ( + "url" = "https://test.bj.bcebos.com/kafka-key/client.key", + "catalog" = "my_catalog", + "md5" = "b5bb901bf10f99205b39a46ac3557dd9" + ); + +## keyword + CREATE,FILE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/DROP CLUSTER.md b/docs/zh-CN/sql-reference/sql-statements/Administration/DROP CLUSTER.md new file mode 100644 index 00000000000000..6b2875f996c47a --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/DROP CLUSTER.md @@ -0,0 +1,44 @@ +--- +{ + "title": "DROP CLUSTER", + "language": "zh-CN" +} +--- + + + +# DROP CLUSTER +## description + + 该语句用于删除逻辑集群，成功删除逻辑集群需要首先删除集群内的db，需要管理员权限 + + 语法 + + DROP CLUSTER [IF EXISTS] cluster_name + +## example + + 删除逻辑集群 test_cluster + + DROP CLUSTER test_cluster; + +## keyword + DROP,CLUSTER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/DROP FILE.md b/docs/zh-CN/sql-reference/sql-statements/Administration/DROP FILE.md new file mode 100644 index 00000000000000..4348832ea9a6c7 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/DROP FILE.md @@ -0,0 +1,51 @@ +--- +{ + "title": "DROP FILE", + "language": "zh-CN" +} +--- + + + +# DROP FILE +## description + + 该语句用于删除一个已上传的文件。 + + 语法： + + DROP FILE "file_name" [FROM database] + [properties] + + 说明： + file_name: 文件名。 + database: 文件归属的某一个 db，如果没有指定，则使用当前 session 的 db。 + properties 支持以下参数: + + catalog: 必须。文件所属分类。 + +## example + + 1. 删除文件 ca.pem + + DROP FILE "ca.pem" properties("catalog" = "kafka"); + +## keyword + DROP,FILE diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ENTER.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ENTER.md new file mode 100644 index 00000000000000..2af0c69e7d7808 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ENTER.md @@ -0,0 +1,44 @@ +--- +{ + "title": "ENTER", + "language": "zh-CN" +} +--- + + + +# ENTER +## description + + 该语句用于进入一个逻辑集群, 所有创建用户、创建数据库都需要在一个逻辑集群内执行，创建后并且隶属于这个逻 + + 辑集群，需要管理员权限 + + ENTER cluster_name + +## example + + 1. 进入逻辑集群test_cluster + + ENTER test_cluster; + +## keyword + ENTER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md b/docs/zh-CN/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md new file mode 100644 index 00000000000000..8dd5706d0a51f2 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md @@ -0,0 +1,57 @@ +--- +{ + "title": "INSTALL PLUGIN", + "language": "zh-CN" +} +--- + + + +# INSTALL PLUGIN +## description + + 该语句用于安装一个插件。 + + 语法 + + INSTALL PLUGIN FROM [source] + + source 支持三种类型： + + 1. 指向一个 zip 文件的绝对路径。 + 2. 指向一个插件目录的绝对路径。 + 3. 指向一个 http 或 https 协议的 zip 文件下载路径 + +## example + + 1. 安装一个本地 zip 文件插件： + + INSTALL PLUGIN FROM "/home/users/seaven/auditdemo.zip"; + + 2. 安装一个本地目录中的插件： + + INSTALL PLUGIN FROM "/home/users/seaven/auditdemo/"; + + 2. 下载并安装一个插件： + + INSTALL PLUGIN FROM "http://mywebsite.com/plugin.zip"; + +## keyword + INSTALL,PLUGIN diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/LINK DATABASE.md b/docs/zh-CN/sql-reference/sql-statements/Administration/LINK DATABASE.md new file mode 100644 index 00000000000000..d86e43a6828ab9 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/LINK DATABASE.md @@ -0,0 +1,51 @@ +--- +{ + "title": "LINK DATABASE", + "language": "zh-CN" +} +--- + + + +# LINK DATABASE +## description + + （已废弃！！！） + 该语句用户链接一个逻辑集群的数据库到另外一个逻辑集群, 一个数据库只允许同时被链接一次，删除链接的数据库 + + 并不会删除数据，并且被链接的数据库不能被删除, 需要管理员权限 + + 语法 + + LINK DATABASE src_cluster_name.src_db_name des_cluster_name.des_db_name + +## example + + 1. 链接test_clusterA中的test_db到test_clusterB,并命名为link_test_db + + LINK DATABASE test_clusterA.test_db test_clusterB.link_test_db; + + 2. 删除链接的数据库link_test_db + + DROP DATABASE link_test_db; + +## keyword + LINK,DATABASE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md b/docs/zh-CN/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md new file mode 100644 index 00000000000000..18fba71e8fdc11 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/MIGRATE DATABASE.md @@ -0,0 +1,47 @@ +--- +{ + "title": "MIGRATE DATABASE", + "language": "zh-CN" +} +--- + + + +# MIGRATE DATABASE +## description + + （已废弃！！！） + 该语句用于迁移一个逻辑集群的数据库到另外一个逻辑集群，执行此操作前数据库必须已经处于链接状态, 需要管理 + + 员权限 + + 语法 + + MIGRATE DATABASE src_cluster_name.src_db_name des_cluster_name.des_db_name + +## example + + 1. 迁移test_clusterA中的test_db到test_clusterB + + MIGRATE DATABASE test_clusterA.test_db test_clusterB.link_test_db; + +## keyword + MIGRATE,DATABASE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW BACKENDS.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW BACKENDS.md new file mode 100644 index 00000000000000..495c2edb193b4f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW BACKENDS.md @@ -0,0 +1,48 @@ +--- +{ + "title": "SHOW BACKENDS", + "language": "zh-CN" +} +--- + + + +# SHOW BACKENDS +## description + 该语句用于查看 cluster 内的 BE 节点 + 语法： + SHOW BACKENDS; + + 说明： + 1. LastStartTime 表示最近一次 BE 启动时间。 + 2. LastHeartbeat 表示最近一次心跳。 + 3. Alive 表示节点是否存活。 + 4. SystemDecommissioned 为 true 表示节点正在安全下线中。 + 5. ClusterDecommissioned 为 true 表示节点正在冲当前cluster中下线。 + 6. TabletNum 表示该节点上分片数量。 + 7. DataUsedCapacity 表示实际用户数据所占用的空间。 + 8. AvailCapacity 表示磁盘的可使用空间。 + 9. TotalCapacity 表示总磁盘空间。TotalCapacity = AvailCapacity + DataUsedCapacity + 其他非用户数据文件占用空间。 + 10. UsedPct 表示磁盘已使用量百分比。 + 11. ErrMsg 用于显示心跳失败时的错误信息。 + +## keyword + SHOW, BACKENDS + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW BROKER.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW BROKER.md new file mode 100644 index 00000000000000..8dd0ae640152e8 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW BROKER.md @@ -0,0 +1,41 @@ +--- +{ + "title": "SHOW BROKER", + "language": "zh-CN" +} +--- + + + +# SHOW BROKER +## description + 该语句用于查看当前存在的 broker + 语法： + SHOW BROKER; + + 说明： + 1. LastStartTime 表示最近一次 BE 启动时间。 + 2. LastHeartbeat 表示最近一次心跳。 + 3. Alive 表示节点是否存活。 + 4. ErrMsg 用于显示心跳失败时的错误信息。 + +## keyword + SHOW, BROKER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FILE.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FILE.md new file mode 100644 index 00000000000000..6787e80b9f75db --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FILE.md @@ -0,0 +1,53 @@ +--- +{ + "title": "SHOW FILE", + "language": "zh-CN" +} +--- + + + +# SHOW FILE +## description + + 该语句用于展示一个 database 内创建的文件 + + 语法： + + SHOW FILE [FROM database]; + + 说明： + + FileId: 文件ID，全局唯一 + DbName: 所属数据库名称 + Catalog: 自定义分类 + FileName: 文件名 + FileSize: 文件大小，单位字节 + MD5: 文件的 MD5 + +## example + + 1. 查看数据库 my_database 中已上传的文件 + + SHOW FILE FROM my_database; + +## keyword + SHOW,FILE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md new file mode 100644 index 00000000000000..db0077456ab699 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FRONTENDS.md @@ -0,0 +1,44 @@ +--- +{ + "title": "SHOW FRONTENDS", + "language": "zh-CN" +} +--- + + + +# SHOW FRONTENDS +## description + 该语句用于查看 FE 节点 + 语法： + SHOW FRONTENDS; + + 说明： + 1. name 表示该 FE 节点在 bdbje 中的名称。 + 2. Join 为 true 表示该节点曾经加入过集群。但不代表当前还在集群内（可能已失联） + 3. Alive 表示节点是否存活。 + 4. ReplayedJournalId 表示该节点当前已经回放的最大元数据日志id。 + 5. LastHeartbeat 是最近一次心跳。 + 6. IsHelper 表示该节点是否是 bdbje 中的 helper 节点。 + 7. ErrMsg 用于显示心跳失败时的错误信息。 + +## keyword + SHOW, FRONTENDS + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md new file mode 100644 index 00000000000000..12194c58e9a77f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW FULL COLUMNS.md @@ -0,0 +1,40 @@ +--- +{ + "title": "SHOW FULL COLUMNS", + "language": "zh-CN" +} +--- + + + +# SHOW FULL COLUMNS +## description + 该语句用于指定表的列信息 + 语法： + SHOW FULL COLUMNS FROM tbl; + +## example + 1. 查看指定表的列信息 + + SHOW FULL COLUMNS FROM tbl; + +## keyword + + SHOW,TABLE,STATUS diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW INDEX.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW INDEX.md new file mode 100644 index 00000000000000..81bba8c08d49af --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW INDEX.md @@ -0,0 +1,44 @@ +--- +{ + "title": "SHOW INDEX", + "language": "zh-CN" +} +--- + + + +# SHOW INDEX + +## description + + 该语句用于展示一个表中索引的相关信息，目前只支持bitmap 索引 + 语法： + SHOW INDEX[ES] FROM [db_name.]table_name [FROM database]; + 或者 + SHOW KEY[S] FROM [db_name.]table_name [FROM database]; + +## example + + 1. 展示指定 table_name 的下索引 + SHOW INDEX FROM example_db.table_name; + +## keyword + + SHOW,INDEX diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md new file mode 100644 index 00000000000000..a8b80affe9adf4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW MIGRATIONS.md @@ -0,0 +1,38 @@ +--- +{ + "title": "SHOW MIGRATIONS", + "language": "zh-CN" +} +--- + + + +# SHOW MIGRATIONS +## description + + 该语句用于查看数据库迁移的进度 + + 语法 + + SHOW MIGRATIONS + +## keyword + SHOW,MIGRATIONS + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW PLUGINS.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW PLUGINS.md new file mode 100644 index 00000000000000..f2e5c33dbf96d3 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW PLUGINS.md @@ -0,0 +1,45 @@ +--- +{ + "title": "SHOW PLUGINS", + "language": "zh-CN" +} +--- + + + +# SHOW PLUGINS +## description + + 该语句用于展示已安装的插件。 + + 语法 + + SHOW PLUGINS; + + 该命令会展示所有用户安装的和系统内置的插件。 + +## example + + 1. 展示已安装的插件： + + SHOW PLUGINS; + +## keyword + SHOW PLUGINS \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md new file mode 100644 index 00000000000000..8534d69f2161bb --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TABLE STATUS.md @@ -0,0 +1,48 @@ +--- +{ + "title": "SHOW TABLE STATUS", + "language": "zh-CN" +} +--- + + + +# SHOW TABLE STATUS +## description + 该语句用于查看 Table 的一些信息。 + 语法： + SHOW TABLE STATUS + [FROM db] [LIKE "pattern"] + + 说明： + 1. 该语句主要用于兼容 MySQL 语法，目前仅显示 Comment 等少量信息 + +## example + 1. 查看当前数据库下所有表的信息 + + SHOW TABLE STATUS; + + 2. 查看指定数据库下，名称包含 example 的表的信息 + + SHOW TABLE STATUS FROM db LIKE "%example%"; + +## keyword + + SHOW,TABLE,STATUS diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md b/docs/zh-CN/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md similarity index 94% rename from docs/documentation/cn/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md rename to docs/zh-CN/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md index 06abe10554ce4d..530126f179a92c 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md @@ -1,3 +1,10 @@ +--- +{ + "title": "UNINTALL PLUGIN", + "language": "zh-CN" +} +--- + + +# ALTER DATABASE +## description + 该语句用于设置指定数据库的属性。（仅管理员使用） + 语法： + 1) 设置数据库数据量配额，单位为B/K/KB/M/MB/G/GB/T/TB/P/PB + ALTER DATABASE db_name SET DATA QUOTA quota; + + 2) 重命名数据库 + ALTER DATABASE db_name RENAME new_db_name; + + 3) 设置数据库的副本数量配额 + ALTER DATABASE db_name SET REPLICA QUOTA quota; + + 说明： + 重命名数据库后，如需要，请使用 REVOKE 和 GRANT 命令修改相应的用户权限。 + 数据库的默认数据量配额为1024GB，默认副本数量配额为1073741824。 + +## example + 1. 设置指定数据库数据量配额 + ALTER DATABASE example_db SET DATA QUOTA 10995116277760; + 上述单位为字节,等价于 + ALTER DATABASE example_db SET DATA QUOTA 10T; + + ALTER DATABASE example_db SET DATA QUOTA 100G; + + ALTER DATABASE example_db SET DATA QUOTA 200M; + + 2. 将数据库 example_db 重命名为 example_db2 + ALTER DATABASE example_db RENAME example_db2; + + 3. 设定指定数据库副本数量配额 + ALTER DATABASE example_db SET REPLICA QUOTA 102400; + +## keyword + ALTER,DATABASE,RENAME + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/ALTER TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/ALTER TABLE.md new file mode 100644 index 00000000000000..d49c65f5a2e52c --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/ALTER TABLE.md @@ -0,0 +1,359 @@ +--- +{ + "title": "ALTER TABLE", + "language": "zh-CN" +} +--- + + + +# ALTER TABLE + +## description + + 该语句用于对已有的 table 进行修改。如果没有指定 rollup index，默认操作 base index。 + 该语句分为三种操作类型： schema change 、rollup 、partition + 这三种操作类型不能同时出现在一条 ALTER TABLE 语句中。 + 其中 schema change 和 rollup 是异步操作，任务提交成功则返回。之后可使用 SHOW ALTER 命令查看进度。 + partition 是同步操作，命令返回表示执行完毕。 + + 语法： + ALTER TABLE [database.]table + alter_clause1[, alter_clause2, ...]; + + alter_clause 分为 partition 、rollup、schema change、rename 和index五种。 + + partition 支持如下几种修改方式 + 1. 增加分区 + 语法： + ADD PARTITION [IF NOT EXISTS] partition_name + partition_desc ["key"="value"] + [DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]] + 注意： + 1) partition_desc 支持一下两种写法： + * VALUES LESS THAN [MAXVALUE|("value1", ...)] + * VALUES [("value1", ...), ("value1", ...)) + 1) 分区为左闭右开区间，如果用户仅指定右边界，系统会自动确定左边界 + 2) 如果没有指定分桶方式，则自动使用建表使用的分桶方式 + 3) 如指定分桶方式，只能修改分桶数，不可修改分桶方式或分桶列 + 4) ["key"="value"] 部分可以设置分区的一些属性，具体说明见 CREATE TABLE + + 2. 删除分区 + 语法： + DROP PARTITION [IF EXISTS] partition_name + 注意： + 1) 使用分区方式的表至少要保留一个分区。 + 2) 执行 DROP PARTITION 一段时间内，可以通过 RECOVER 语句恢复被删除的 partition。详见 RECOVER 语句 + + 3. 修改分区属性 + 语法： + MODIFY PARTITION partition_name SET ("key" = "value", ...) + 说明： + 1) 当前支持修改分区的下列属性： + - storage_medium + - storage_cooldown_time + - replication_num + — in_memory + 2) 对于单分区表，partition_name 同表名。 + + rollup 支持如下几种创建方式： + 1. 创建 rollup index + 语法： + ADD ROLLUP rollup_name (column_name1, column_name2, ...) + [FROM from_index_name] + [PROPERTIES ("key"="value", ...)] + + properties: 支持设置超时时间，默认超时时间为1天。 + 例子： + ADD ROLLUP r1(col1,col2) from r0 + 1.2 批量创建 rollup index + 语法： + ADD ROLLUP [rollup_name (column_name1, column_name2, ...) + [FROM from_index_name] + [PROPERTIES ("key"="value", ...)],...] + 例子： + ADD ROLLUP r1(col1,col2) from r0, r2(col3,col4) from r0 + 1.3 注意： + 1) 如果没有指定 from_index_name，则默认从 base index 创建 + 2) rollup 表中的列必须是 from_index 中已有的列 + 3) 在 properties 中，可以指定存储格式。具体请参阅 CREATE TABLE + + 2. 删除 rollup index + 语法： + DROP ROLLUP rollup_name [PROPERTIES ("key"="value", ...)] + 例子： + DROP ROLLUP r1 + 2.1 批量删除 rollup index + 语法：DROP ROLLUP [rollup_name [PROPERTIES ("key"="value", ...)],...] + 例子：DROP ROLLUP r1,r2 + 2.2 注意： + 1) 不能删除 base index + 2) 执行 DROP ROLLUP 一段时间内，可以通过 RECOVER 语句恢复被删除的 rollup index。详见 RECOVER 语句 + + + schema change 支持如下几种修改方式： + 1. 向指定 index 的指定位置添加一列 + 语法： + ADD COLUMN column_name column_type [KEY | agg_type] [DEFAULT "default_value"] + [AFTER column_name|FIRST] + [TO rollup_index_name] + [PROPERTIES ("key"="value", ...)] + 注意： + 1) 聚合模型如果增加 value 列，需要指定 agg_type + 2) 非聚合模型（如 DUPLICATE KEY）如果增加key列，需要指定KEY关键字 + 3) 不能在 rollup index 中增加 base index 中已经存在的列 + 如有需要，可以重新创建一个 rollup index） + + 2. 向指定 index 添加多列 + 语法： + ADD COLUMN (column_name1 column_type [KEY | agg_type] DEFAULT "default_value", ...) + [TO rollup_index_name] + [PROPERTIES ("key"="value", ...)] + 注意： + 1) 聚合模型如果增加 value 列，需要指定agg_type + 2) 非聚合模型如果增加key列，需要指定KEY关键字 + 3) 不能在 rollup index 中增加 base index 中已经存在的列 + （如有需要，可以重新创建一个 rollup index） + + 3. 从指定 index 中删除一列 + 语法： + DROP COLUMN column_name + [FROM rollup_index_name] + 注意： + 1) 不能删除分区列 + 2) 如果是从 base index 中删除列，则如果 rollup index 中包含该列，也会被删除 + + 4. 修改指定 index 的列类型以及列位置 + 语法： + MODIFY COLUMN column_name column_type [KEY | agg_type] [NULL | NOT NULL] [DEFAULT "default_value"] + [AFTER column_name|FIRST] + [FROM rollup_index_name] + [PROPERTIES ("key"="value", ...)] + 注意： + 1) 聚合模型如果修改 value 列，需要指定 agg_type + 2) 非聚合类型如果修改key列，需要指定KEY关键字 + 3) 只能修改列的类型，列的其他属性维持原样（即其他属性需在语句中按照原属性显式的写出，参见 example 8） + 4) 分区列不能做任何修改 + 5) 目前支持以下类型的转换（精度损失由用户保证） + TINYINT/SMALLINT/INT/BIGINT 转换成 TINYINT/SMALLINT/INT/BIGINT/DOUBLE。 + TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL 转换成 VARCHAR + LARGEINT 转换成 DOUBLE + VARCHAR 支持修改最大长度 + VARCHAR 转换成 TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE + VARCHAR 转换成 DATE (目前支持"%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d, "%y/%m/%d"六种格式化格式) + DATETIME 转换成 DATE(仅保留年-月-日信息, 例如: `2019-12-09 21:47:05` <--> `2019-12-09`) + DATE 转换成 DATETIME(时分秒自动补零， 例如: `2019-12-09` <--> `2019-12-09 00:00:00`) + FLOAT 转换成 DOUBLE + INT 转换成 DATE (如果INT类型数据不合法则转换失败，原始数据不变) + 6) 不支持从NULL转为NOT NULL + + 5. 对指定 index 的列进行重新排序 + 语法： + ORDER BY (column_name1, column_name2, ...) + [FROM rollup_index_name] + [PROPERTIES ("key"="value", ...)] + 注意： + 1) index 中的所有列都要写出来 + 2) value 列在 key 列之后 + + 6. 修改table的属性，目前支持修改bloom filter列, colocate_with 属性和dynamic_partition属性，replication_num和default.replication_num属性 + 语法： + PROPERTIES ("key"="value") + 注意： + 也可以合并到上面的schema change操作中来修改，见下面例子 + + + rename 支持对以下名称进行修改： + 1. 修改表名 + 语法： + RENAME new_table_name; + + 2. 修改 rollup index 名称 + 语法： + RENAME ROLLUP old_rollup_name new_rollup_name; + + 3. 修改 partition 名称 + 语法： + RENAME PARTITION old_partition_name new_partition_name; + bitmap index 支持如下几种修改方式 + 1. 创建bitmap 索引 + 语法： + ADD INDEX index_name (column [, ...],) [USING BITMAP] [COMMENT 'balabala']; + 注意： + 1. 目前仅支持bitmap 索引 + 1. BITMAP 索引仅在单列上创建 + 2. 删除索引 + 语法： + DROP INDEX index_name； + +## example + + [table] + 1. 修改表的默认副本数量, 新建分区副本数量默认使用此值 + ATLER TABLE example_db.my_table + SET ("default.replication_num" = "2"); + + 2. 修改单分区表的实际副本数量(只限单分区表) + ALTER TABLE example_db.my_table + SET ("replication_num" = "3"); + + [partition] + 1. 增加分区, 现有分区 [MIN, 2013-01-01)，增加分区 [2013-01-01, 2014-01-01)，使用默认分桶方式 + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES LESS THAN ("2014-01-01"); + + 2. 增加分区，使用新的分桶数 + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") + DISTRIBUTED BY HASH(k1) BUCKETS 20; + + 3. 增加分区，使用新的副本数 + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") + ("replication_num"="1"); + + 4. 修改分区副本数 + ALTER TABLE example_db.my_table + MODIFY PARTITION p1 SET("replication_num"="1"); + + 5. 删除分区 + ALTER TABLE example_db.my_table + DROP PARTITION p1; + + 6. 增加一个指定上下界的分区 + + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES [("2014-01-01"), ("2014-02-01")); + + [rollup] + 1. 创建 index: example_rollup_index，基于 base index（k1,k2,k3,v1,v2）。列式存储。 + ALTER TABLE example_db.my_table + ADD ROLLUP example_rollup_index(k1, k3, v1, v2) + PROPERTIES("storage_type"="column"); + + 2. 创建 index: example_rollup_index2，基于 example_rollup_index（k1,k3,v1,v2） + ALTER TABLE example_db.my_table + ADD ROLLUP example_rollup_index2 (k1, v1) + FROM example_rollup_index; + + 3. 创建 index: example_rollup_index3, 基于 base index (k1,k2,k3,v1), 自定义 rollup 超时时间一小时。 + ALTER TABLE example_db.my_table + ADD ROLLUP example_rollup_index(k1, k3, v1) + PROPERTIES("storage_type"="column", "timeout" = "3600"); + + 4. 删除 index: example_rollup_index2 + ALTER TABLE example_db.my_table + DROP ROLLUP example_rollup_index2; + + + + [schema change] + 1. 向 example_rollup_index 的 col1 后添加一个key列 new_col(非聚合模型) + ALTER TABLE example_db.my_table + ADD COLUMN new_col INT KEY DEFAULT "0" AFTER col1 + TO example_rollup_index; + + 2. 向example_rollup_index的col1后添加一个value列new_col(非聚合模型) + ALTER TABLE example_db.my_table + ADD COLUMN new_col INT DEFAULT "0" AFTER col1 + TO example_rollup_index; + + 3. 向example_rollup_index的col1后添加一个key列new_col(聚合模型) + ALTER TABLE example_db.my_table + ADD COLUMN new_col INT DEFAULT "0" AFTER col1 + TO example_rollup_index; + + 4. 向example_rollup_index的col1后添加一个value列new_col SUM聚合类型(聚合模型) + ALTER TABLE example_db.my_table + ADD COLUMN new_col INT SUM DEFAULT "0" AFTER col1 + TO example_rollup_index; + + 5. 向 example_rollup_index 添加多列(聚合模型) + ALTER TABLE example_db.my_table + ADD COLUMN (col1 INT DEFAULT "1", col2 FLOAT SUM DEFAULT "2.3") + TO example_rollup_index; + + 6. 从 example_rollup_index 删除一列 + ALTER TABLE example_db.my_table + DROP COLUMN col2 + FROM example_rollup_index; + + 7. 修改 base index 的 col1 列的类型为 BIGINT，并移动到 col2 列后面 + ALTER TABLE example_db.my_table + MODIFY COLUMN col1 BIGINT DEFAULT "1" AFTER col2; + + 8. 修改 base index 的 val1 列最大长度。原 val1 为 (val1 VARCHAR(32) REPLACE DEFAULT "abc") + ALTER TABLE example_db.my_table + MODIFY COLUMN val1 VARCHAR(64) REPLACE DEFAULT "abc"; + + 9. 重新排序 example_rollup_index 中的列（设原列顺序为：k1,k2,k3,v1,v2） + ALTER TABLE example_db.my_table + ORDER BY (k3,k1,k2,v2,v1) + FROM example_rollup_index; + + 10. 同时执行两种操作 + ALTER TABLE example_db.my_table + ADD COLUMN v2 INT MAX DEFAULT "0" AFTER k2 TO example_rollup_index, + ORDER BY (k3,k1,k2,v2,v1) FROM example_rollup_index; + + 11. 修改表的 bloom filter 列 + ALTER TABLE example_db.my_table SET ("bloom_filter_columns"="k1,k2,k3"); + + 也可以合并到上面的 schema change 操作中（注意多子句的语法有少许区别） + ALTER TABLE example_db.my_table + DROP COLUMN col2 + PROPERTIES ("bloom_filter_columns"="k1,k2,k3"); + + 12. 修改表的Colocate 属性 + + ALTER TABLE example_db.my_table set ("colocate_with" = "t1"); + + 13. 将表的分桶方式由 Random Distribution 改为 Hash Distribution + + ALTER TABLE example_db.my_table set ("distribution_type" = "hash"); + + 14. 修改表的动态分区属性(支持未添加动态分区属性的表添加动态分区属性) + ALTER TABLE example_db.my_table set ("dynamic_partition_enable" = "false"); + + 如果需要在未添加动态分区属性的表中添加动态分区属性，则需要指定所有的动态分区属性 + ALTER TABLE example_db.my_table set ("dynamic_partition.enable" = "true", dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32"); + 15. 修改表的 in_memory 属性 + + ALTER TABLE example_db.my_table set ("in_memory" = "true"); + + + [rename] + 1. 将名为 table1 的表修改为 table2 + ALTER TABLE table1 RENAME table2; + + 2. 将表 example_table 中名为 rollup1 的 rollup index 修改为 rollup2 + ALTER TABLE example_table RENAME ROLLUP rollup1 rollup2; + + 3. 将表 example_table 中名为 p1 的 partition 修改为 p2 + ALTER TABLE example_table RENAME PARTITION p1 p2; + [index] + 1. 在table1 上为siteid 创建bitmap 索引 + ALTER TABLE table1 ADD INDEX index_name (siteid) [USING BITMAP] COMMENT 'balabala'; + 2. 删除table1 上的siteid列的bitmap 索引 + ALTER TABLE table1 DROP INDEX index_name; + +## keyword + + ALTER,TABLE,ROLLUP,COLUMN,PARTITION,RENAME diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/ALTER VIEW.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/ALTER VIEW.md new file mode 100644 index 00000000000000..6969f1a6f78789 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/ALTER VIEW.md @@ -0,0 +1,52 @@ +--- +{ + "title": "ALTER VIEW", + "language": "zh-CN" +} +--- + + + +# ALTER VIEW +## description + 该语句用于修改一个view的定义 + 语法： + ALTER VIEW + [db_name.]view_name + (column1[ COMMENT "col comment"][, column2, ...]) + AS query_stmt + + 说明： + 1. 视图都是逻辑上的，其中的数据不会存储在物理介质上，在查询时视图将作为语句中的子查询，因此，修改视图的定义等价于修改query_stmt。 + 2. query_stmt 为任意支持的 SQL + +## example + 1、修改example_db上的视图example_view + + ALTER VIEW example_db.example_view + ( + c1 COMMENT "column 1", + c2 COMMENT "column 2", + c3 COMMENT "column 3" + ) + AS SELECT k1, k2, SUM(v1) FROM example_table + GROUP BY k1, k2 + + \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/BACKUP.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/BACKUP.md new file mode 100644 index 00000000000000..7233f9c5a479c8 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/BACKUP.md @@ -0,0 +1,65 @@ +--- +{ + "title": "BACKUP", + "language": "zh-CN" +} +--- + + + +# BACKUP +## description + 该语句用于备份指定数据库下的数据。该命令为异步操作。提交成功后，需通过 SHOW BACKUP 命令查看进度。仅支持备份 OLAP 类型的表。 + 语法： + BACKUP SNAPSHOT [db_name].{snapshot_name} + TO `repository_name` + ON ( + `table_name` [PARTITION (`p1`, ...)], + ... + ) + PROPERTIES ("key"="value", ...); + + 说明： + 1. 同一数据库下只能有一个正在执行的 BACKUP 或 RESTORE 任务。 + 2. ON 子句中标识需要备份的表和分区。如果不指定分区，则默认备份该表的所有分区。 + 3. PROPERTIES 目前支持以下属性： + "type" = "full"：表示这是一次全量更新（默认）。 + "timeout" = "3600"：任务超时时间，默认为一天。单位秒。 + +## example + + 1. 全量备份 example_db 下的表 example_tbl 到仓库 example_repo 中： + BACKUP SNAPSHOT example_db.snapshot_label1 + TO example_repo + ON (example_tbl) + PROPERTIES ("type" = "full"); + + 2. 全量备份 example_db 下，表 example_tbl 的 p1, p2 分区，以及表 example_tbl2 到仓库 example_repo 中： + BACKUP SNAPSHOT example_db.snapshot_label2 + TO example_repo + ON + ( + example_tbl PARTITION (p1,p2), + example_tbl2 + ); + +## keyword + BACKUP + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md new file mode 100644 index 00000000000000..843c643cc638f7 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL ALTER.md @@ -0,0 +1,69 @@ +--- +{ + "title": "CANCEL ALTER", + "language": "zh-CN" +} +--- + + + +# CANCEL ALTER +## description + 该语句用于撤销一个 ALTER 操作。 + 1. 撤销 ALTER TABLE COLUMN 操作 + 语法： + CANCEL ALTER TABLE COLUMN + FROM db_name.table_name + + 2. 撤销 ALTER TABLE ROLLUP 操作 + 语法： + CANCEL ALTER TABLE ROLLUP + FROM db_name.table_name + + 3. 根据job id批量撤销rollup操作 + 语法: + CANCEL ALTER TABLE ROLLUP + FROM db_name.table_name (jobid,...) + 注意： + 该命令为异步操作，具体是否执行成功需要使用`show alter table rollup`查看任务状态确认 + 4. 撤销 ALTER CLUSTER 操作 + 语法： + （待实现...） + + +## example + [CANCEL ALTER TABLE COLUMN] + 1. 撤销针对 my_table 的 ALTER COLUMN 操作。 + CANCEL ALTER TABLE COLUMN + FROM example_db.my_table; + + [CANCEL ALTER TABLE ROLLUP] + 1. 撤销 my_table 下的 ADD ROLLUP 操作。 + CANCEL ALTER TABLE ROLLUP + FROM example_db.my_table; + + [CANCEL ALTER TABLE ROLLUP] + 1. 根据job id撤销 my_table 下的 ADD ROLLUP 操作。 + CANCEL ALTER TABLE ROLLUP + FROM example_db.my_table (12801,12802); + +## keyword + CANCEL,ALTER,TABLE,COLUMN,ROLLUP + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md new file mode 100644 index 00000000000000..f9648cbab9a503 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL BACKUP.md @@ -0,0 +1,39 @@ +--- +{ + "title": "CANCEL BACKUP", + "language": "zh-CN" +} +--- + + + +# CANCEL BACKUP +## description + 该语句用于取消一个正在进行的 BACKUP 任务。 + 语法： + CANCEL BACKUP FROM db_name; + +## example + 1. 取消 example_db 下的 BACKUP 任务。 + CANCEL BACKUP FROM example_db; + +## keyword + CANCEL, BACKUP + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md new file mode 100644 index 00000000000000..5ae2052e3a4590 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CANCEL RESTORE.md @@ -0,0 +1,42 @@ +--- +{ + "title": "CANCEL RESTORE", + "language": "zh-CN" +} +--- + + + +# CANCEL RESTORE +## description + 该语句用于取消一个正在进行的 RESTORE 任务。 + 语法： + CANCEL RESTORE FROM db_name; + + 注意： + 当取消处于 COMMIT 或之后阶段的恢复左右时，可能导致被恢复的表无法访问。此时只能通过再次执行恢复作业进行数据恢复。 + +## example + 1. 取消 example_db 下的 RESTORE 任务。 + CANCEL RESTORE FROM example_db; + +## keyword + CANCEL, RESTORE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md new file mode 100644 index 00000000000000..48e2ef3e901385 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE DATABASE.md @@ -0,0 +1,39 @@ +--- +{ + "title": "CREATE DATABASE", + "language": "zh-CN" +} +--- + + + +# CREATE DATABASE +## description + 该语句用于新建数据库（database） + 语法： + CREATE DATABASE [IF NOT EXISTS] db_name; + +## example + 1. 新建数据库 db_test + CREATE DATABASE db_test; + +## keyword + CREATE,DATABASE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE INDEX.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE INDEX.md new file mode 100644 index 00000000000000..b65fc5895a615f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE INDEX.md @@ -0,0 +1,45 @@ +--- +{ + "title": "CREATE INDEX", + "language": "zh-CN" +} +--- + + + +# CREATE INDEX + +## description + + 该语句用于创建索引 + 语法： + CREATE INDEX index_name ON table_name (column [, ...],) [USING BITMAP] [COMMENT'balabala']; + 注意： + 1. 目前只支持bitmap 索引 + 2. BITMAP 索引仅在单列上创建 + +## example + + 1. 在table1 上为siteid 创建bitmap 索引 + CREATE INDEX index_name ON table1 (siteid) USING BITMAP COMMENT 'balabala'; + +## keyword + + CREATE,INDEX diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md new file mode 100644 index 00000000000000..a2a68c172157ed --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE MATERIALIZED VIEW.md @@ -0,0 +1,237 @@ +--- +{ + "title": "CREATE MATERIALIZED VIEW", + "language": "zh-CN" +} +--- + + + +# CREATE MATERIALIZED VIEW + +## description + + 该语句用于创建物化视图。 + +说明: + 异步语法，调用成功后仅表示创建物化视图的任务提交成功，用户需要先通过 ``` show alter table rollup ``` 来查看物化视图的创建进度。 + 在显示 FINISHED 后既可通过 ``` desc [table_name] all ``` 命令来查看物化视图的 schema 了。 + +语法: + + ``` + + CREATE MATERIALIZED VIEW [MV name] as [query] + [PROPERTIES ("key" = "value")] + + ``` + +1. MV name + + 物化视图的名称，必填项。 + + 相同表的物化视图名称不可重复。 + +2. query + + 用于构建物化视图的查询语句，查询语句的结果既物化视图的数据。目前支持的 query 格式为: + + ``` + + SELECT select_expr[, select_expr ...] + FROM [Base view name] + GROUP BY column_name[, column_name ...] + ORDER BY column_name[, column_name ...] + + 语法和查询语句语法一致。 + + ``` + + select_expr: 物化视图的 schema 中所有的列。 + + 仅支持不带表达式计算的单列，聚合列。 + + 其中聚合函数目前仅支持 SUM, MIN, MAX 三种，且聚合函数的参数只能是不带表达式计算的单列。 + + 至少包含一个单列。 + + 所有涉及到的列，均只能出现一次。 + + base view name: 物化视图的原始表名，必填项。 + + 必须是单表，且非子查询 + + group by: 物化视图的分组列，选填项。 + + 不填则数据不进行分组。 + + order by: 物化视图的排序列，选填项。 + + 排序列的声明顺序必须和 select_expr 中列声明顺序一致。 + + 如果不声明 order by，则根据规则自动补充排序列。 + 如果物化视图是聚合类型，则所有的分组列自动补充为排序列。 + 如果物化视图是非聚合类型，则前 36 个字节自动补充为排序列。如果自动补充的排序个数小于3个，则前三个作为排序列。 + + 如果 query 中包含分组列的话，则排序列必须和分组列一致。 + +3. properties + + 声明物化视图的一些配置，选填项。 + + ``` + + PROPERTIES ("key" = "value", "key" = "value" ...) + + ``` + + 以下几个配置，均可声明在此处： + + short_key: 排序列的个数。 + timeout: 物化视图构建的超时时间。 + +## example + +Base 表结构为 + +``` +mysql> desc duplicate_table; ++-------+--------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-------+--------+------+------+---------+-------+ +| k1 | INT | Yes | true | N/A | | +| k2 | INT | Yes | true | N/A | | +| k3 | BIGINT | Yes | true | N/A | | +| k4 | BIGINT | Yes | true | N/A | | ++-------+--------+------+------+---------+-------+ +``` + +1. 创建一个仅包含原始表 （k1, k2）列的物化视图 + + ``` + create materialized view k1_k2 as + select k1, k2 from duplicate_table; + ``` + + 物化视图的 schema 如下图，物化视图仅包含两列 k1, k2 且不带任何聚合 + + ``` + +-----------------+-------+--------+------+------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +-----------------+-------+--------+------+------+---------+-------+ + | k1_k2 | k1 | INT | Yes | true | N/A | | + | | k2 | INT | Yes | true | N/A | | + +-----------------+-------+--------+------+------+---------+-------+ + ``` + +2. 创建一个以 k2 为排序列的物化视图 + + ``` + create materialized view k2_order as + select k2, k1 from duplicate_table order by k2; + ``` + + 物化视图的 schema 如下图，物化视图仅包含两列 k2, k1，其中 k2 列为排序列，不带任何聚合。 + + ``` + +-----------------+-------+--------+------+-------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +-----------------+-------+--------+------+-------+---------+-------+ + | k2_order | k2 | INT | Yes | true | N/A | | + | | k1 | INT | Yes | false | N/A | NONE | + +-----------------+-------+--------+------+-------+---------+-------+ + ``` + +3. 创建一个以 k1, k2 分组，k3 列为 SUM 聚合的物化视图 + + ``` + create materialized view k1_k2_sumk3 as + select k1, k2, sum(k3) from duplicate_table group by k1, k2; + ``` + + 物化视图的 schema 如下图，物化视图包含两列 k1, k2，sum(k3) 其中 k1, k2 为分组列，sum(k3) 为根据 k1, k2 分组后的 k3 列的求和值。 + + 由于物化视图没有声明排序列，且物化视图带聚合数据，系统默认补充分组列 k1, k2 为排序列。 + + ``` + +-----------------+-------+--------+------+-------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +-----------------+-------+--------+------+-------+---------+-------+ + | k1_k2_sumk3 | k1 | INT | Yes | true | N/A | | + | | k2 | INT | Yes | true | N/A | | + | | k3 | BIGINT | Yes | false | N/A | SUM | + +-----------------+-------+--------+------+-------+---------+-------+ + ``` + +4. 创建一个去除重复行的物化视图 + + ``` + create materialized view deduplicate as + select k1, k2, k3, k4 from duplicate_table group by k1, k2, k3, k4; + ``` + + 物化视图 schema 如下图，物化视图包含 k1, k2, k3, k4列，且不存在重复行。 + + ``` + +-----------------+-------+--------+------+-------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +-----------------+-------+--------+------+-------+---------+-------+ + | deduplicate | k1 | INT | Yes | true | N/A | | + | | k2 | INT | Yes | true | N/A | | + | | k3 | BIGINT | Yes | true | N/A | | + | | k4 | BIGINT | Yes | true | N/A | | + +-----------------+-------+--------+------+-------+---------+-------+ + + ``` + +5. 创建一个不声明排序列的非聚合型物化视图 + + all_type_table 的 schema 如下： + + ``` + +-------+--------------+------+-------+---------+-------+ + | Field | Type | Null | Key | Default | Extra | + +-------+--------------+------+-------+---------+-------+ + | k1 | TINYINT | Yes | true | N/A | | + | k2 | SMALLINT | Yes | true | N/A | | + | k3 | INT | Yes | true | N/A | | + | k4 | BIGINT | Yes | true | N/A | | + | k5 | DECIMAL(9,0) | Yes | true | N/A | | + | k6 | DOUBLE | Yes | false | N/A | NONE | + | k7 | VARCHAR(20) | Yes | false | N/A | NONE | + +-------+--------------+------+-------+---------+-------+ + ``` + + 物化视图包含 k3, k4, k5, k6, k7 列，且不声明排序列，则创建语句如下： + + ``` + create materialized view mv_1 as + select k3, k4, k5, k6, k7 from all_type_table; + ``` + + 系统默认补充的排序列为 k3, k4, k5 三列。这三列类型的字节数之和为 4(INT) + 8(BIGINT) + 16(DECIMAL) = 28 < 36。所以补充的是这三列作为排序列。 + 物化视图的 schema 如下，可以看到其中 k3, k4, k5 列的 key 字段为 true，也就是排序列。k6, k7 列的 key 字段为 false，也就是非排序列。 + + ``` + +----------------+-------+--------------+------+-------+---------+-------+ + | IndexName | Field | Type | Null | Key | Default | Extra | + +----------------+-------+--------------+------+-------+---------+-------+ + | mv_1 | k3 | INT | Yes | true | N/A | | + | | k4 | BIGINT | Yes | true | N/A | | + | | k5 | DECIMAL(9,0) | Yes | true | N/A | | + | | k6 | DOUBLE | Yes | false | N/A | NONE | + | | k7 | VARCHAR(20) | Yes | false | N/A | NONE | + +----------------+-------+--------------+------+-------+---------+-------+ + ``` + + +## keyword + CREATE, MATERIALIZED, VIEW diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md new file mode 100644 index 00000000000000..b3839f87d351e2 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE REPOSITORY.md @@ -0,0 +1,76 @@ +--- +{ + "title": "CREATE REPOSITORY", + "language": "zh-CN" +} +--- + + + +# CREATE REPOSITORY +## description + 该语句用于创建仓库。仓库用于属于备份或恢复。仅 root 或 superuser 用户可以创建仓库。 + 语法： + CREATE [READ ONLY] REPOSITORY `repo_name` + WITH BROKER `broker_name` + ON LOCATION `repo_location` + PROPERTIES ("key"="value", ...); + + 说明： + 1. 仓库的创建，依赖于已存在的 broker + 2. 如果是只读仓库，则只能在仓库上进行恢复。如果不是，则可以进行备份和恢复操作。 + 3. 根据 broker 的不同类型，PROPERTIES 有所不同，具体见示例。 + +## example + 1. 创建名为 bos_repo 的仓库，依赖 BOS broker "bos_broker"，数据根目录为：bos://palo_backup + CREATE REPOSITORY `bos_repo` + WITH BROKER `bos_broker` + ON LOCATION "bos://palo_backup" + PROPERTIES + ( + "bos_endpoint" = "http://gz.bcebos.com", + "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", + "bos_secret_accesskey"="70999999999999de274d59eaa980a" + ); + + 2. 创建和示例 1 相同的仓库，但属性为只读： + CREATE READ ONLY REPOSITORY `bos_repo` + WITH BROKER `bos_broker` + ON LOCATION "bos://palo_backup" + PROPERTIES + ( + "bos_endpoint" = "http://gz.bcebos.com", + "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", + "bos_secret_accesskey"="70999999999999de274d59eaa980a" + ); + + 3. 创建名为 hdfs_repo 的仓库，依赖 Baidu hdfs broker "hdfs_broker"，数据根目录为：hdfs://hadoop-name-node:54310/path/to/repo/ + CREATE REPOSITORY `hdfs_repo` + WITH BROKER `hdfs_broker` + ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/" + PROPERTIES + ( + "username" = "user", + "password" = "password" + ); + +## keyword + CREATE REPOSITORY + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md new file mode 100644 index 00000000000000..3ded3b0025d37f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -0,0 +1,612 @@ +--- +{ + "title": "CREATE TABLE", + "language": "zh-CN" +} +--- + + + +# CREATE TABLE + +## description + +该语句用于创建 table。 +语法： + +``` + CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [database.]table_name + (column_definition1[, column_definition2, ...] + [, index_definition1[, ndex_definition12,]]) + [ENGINE = [olap|mysql|broker]] + [key_desc] + [COMMENT "table comment"]; + [partition_desc] + [distribution_desc] + [rollup_index] + [PROPERTIES ("key"="value", ...)] + [BROKER PROPERTIES ("key"="value", ...)] +``` + +1. column_definition + 语法： + `col_name col_type [agg_type] [NULL | NOT NULL] [DEFAULT "default_value"]` + + 说明： + col_name：列名称 + col_type：列类型 + + ``` + TINYINT（1字节） + 范围：-2^7 + 1 ~ 2^7 - 1 + SMALLINT（2字节） + 范围：-2^15 + 1 ~ 2^15 - 1 + INT（4字节） + 范围：-2^31 + 1 ~ 2^31 - 1 + BIGINT（8字节） + 范围：-2^63 + 1 ~ 2^63 - 1 + LARGEINT（16字节） + 范围：-2^127 + 1 ~ 2^127 - 1 + FLOAT（4字节） + 支持科学计数法 + DOUBLE（12字节） + 支持科学计数法 + DECIMAL[(precision, scale)] (16字节) + 保证精度的小数类型。默认是 DECIMAL(10, 0) + precision: 1 ~ 27 + scale: 0 ~ 9 + 其中整数部分为 1 ~ 18 + 不支持科学计数法 + DATE（3字节） + 范围：0000-01-01 ~ 9999-12-31 + DATETIME（8字节） + 范围：0000-01-01 00:00:00 ~ 9999-12-31 23:59:59 + CHAR[(length)] + 定长字符串。长度范围：1 ~ 255。默认为1 + VARCHAR[(length)] + 变长字符串。长度范围：1 ~ 65533 + HLL (1~16385个字节) + hll列类型，不需要指定长度和默认值、长度根据数据的聚合 + 程度系统内控制，并且HLL列只能通过配套的hll_union_agg、Hll_cardinality、hll_hash进行查询或使用 + BITMAP + bitmap列类型，不需要指定长度和默认值。表示整型的集合，元素最大支持到2^64 - 1 + ``` + + agg_type：聚合类型，如果不指定，则该列为 key 列。否则，该列为 value 列 + * SUM、MAX、MIN、REPLACE + * HLL_UNION(仅用于HLL列，为HLL独有的聚合方式)、 + * BITMAP_UNION(仅用于 BITMAP 列，为 BITMAP 独有的聚合方式)、 + * REPLACE_IF_NOT_NULL：这个聚合类型的含义是当且仅当新导入数据是非NULL值时会发生替换行为，如果新导入的数据是NULL，那么Doris仍然会保留原值。注意：如果用在建表时REPLACE_IF_NOT_NULL列指定了NOT NULL，那么Doris仍然会将其转化NULL，不会向用户报错。用户可以借助这个类型完成部分列导入的功能。 + * 该类型只对聚合模型(key_desc的type为AGGREGATE KEY)有用，其它模型不需要指这个。 + + 是否允许为NULL: 默认不允许为 NULL。NULL 值在导入数据中用 \N 来表示 + + 注意： + BITMAP_UNION聚合类型列在导入时的原始数据类型必须是TINYINT,SMALLINT,INT,BIGINT。 + +2. index_definition + 语法： + `INDEX index_name (col_name[, col_name, ...]) [USING BITMAP] COMMENT 'xxxxxx'` + 说明： + index_name：索引名称 + col_name：列名 + 注意： + 当前仅支持BITMAP索引， BITMAP索引仅支持应用于单列 + +3. ENGINE 类型 + 默认为 olap。可选 mysql, broker + 1) 如果是 mysql，则需要在 properties 提供以下信息： + +``` + PROPERTIES ( + "host" = "mysql_server_host", + "port" = "mysql_server_port", + "user" = "your_user_name", + "password" = "your_password", + "database" = "database_name", + "table" = "table_name" + ) +``` + + 注意： + "table" 条目中的 "table_name" 是 mysql 中的真实表名。 + 而 CREATE TABLE 语句中的 table_name 是该 mysql 表在 Palo 中的名字，可以不同。 + + 在 Palo 创建 mysql 表的目的是可以通过 Palo 访问 mysql 数据库。 + 而 Palo 本身并不维护、存储任何 mysql 数据。 + 1) 如果是 broker，表示表的访问需要通过指定的broker, 需要在 properties 提供以下信息： + ``` + PROPERTIES ( + "broker_name" = "broker_name", + "path" = "file_path1[,file_path2]", + "column_separator" = "value_separator" + "line_delimiter" = "value_delimiter" + ) + ``` + 另外还需要提供Broker需要的Property信息，通过BROKER PROPERTIES来传递，例如HDFS需要传入 + ``` + BROKER PROPERTIES( + "username" = "name", + "password" = "password" + ) + ``` + 这个根据不同的Broker类型，需要传入的内容也不相同 + 注意： + "path" 中如果有多个文件，用逗号[,]分割。如果文件名中包含逗号，那么使用 %2c 来替代。如果文件名中包含 %，使用 %25 代替 + 现在文件内容格式支持CSV，支持GZ，BZ2，LZ4，LZO(LZOP) 压缩格式。 + +1. key_desc + 语法： + `key_type(k1[,k2 ...])` + 说明： + 数据按照指定的key列进行排序，且根据不同的key_type具有不同特性。 + key_type支持一下类型： + AGGREGATE KEY:key列相同的记录，value列按照指定的聚合类型进行聚合， + 适合报表、多维分析等业务场景。 + UNIQUE KEY:key列相同的记录，value列按导入顺序进行覆盖， + 适合按key列进行增删改查的点查询业务。 + DUPLICATE KEY:key列相同的记录，同时存在于Palo中， + 适合存储明细数据或者数据无聚合特性的业务场景。 + 默认为DUPLICATE KEY，key列为列定义中前36个字节, 如果前36个字节的列数小于3，将使用前三列。 + 注意： + 除AGGREGATE KEY外，其他key_type在建表时，value列不需要指定聚合类型。 + +2. partition_desc + partition描述有两种使用方式 + 1) LESS THAN + 语法： + + ``` + PARTITION BY RANGE (k1, k2, ...) + ( + PARTITION partition_name1 VALUES LESS THAN MAXVALUE|("value1", "value2", ...), + PARTITION partition_name2 VALUES LESS THAN MAXVALUE|("value1", "value2", ...) + ... + ) + ``` + + 说明： + 使用指定的 key 列和指定的数值范围进行分区。 + 1) 分区名称仅支持字母开头，字母、数字和下划线组成 + 2) 目前仅支持以下类型的列作为 Range 分区列，且只能指定一个分区列 + TINYINT, SMALLINT, INT, BIGINT, LARGEINT, DATE, DATETIME + 3) 分区为左闭右开区间，首个分区的左边界为做最小值 + 4) NULL 值只会存放在包含最小值的分区中。当包含最小值的分区被删除后，NULL 值将无法导入。 + 5) 可以指定一列或多列作为分区列。如果分区值缺省，则会默认填充最小值。 + + 注意： + 1) 分区一般用于时间维度的数据管理 + 2) 有数据回溯需求的，可以考虑首个分区为空分区，以便后续增加分区 + + 2）Fixed Range + 语法： + ``` + PARTITION BY RANGE (k1, k2, k3, ...) + ( + PARTITION partition_name1 VALUES [("k1-lower1", "k2-lower1", "k3-lower1",...), ("k1-upper1", "k2-upper1", "k3-upper1", ...)), + PARTITION partition_name2 VALUES [("k1-lower1-2", "k2-lower1-2", ...), ("k1-upper1-2", MAXVALUE, )) + "k3-upper1-2", ... + ) + ``` + 说明： + 1）Fixed Range比LESS THAN相对灵活些，左右区间完全由用户自己确定 + 2）其他与LESS THAN保持同步 + +3. distribution_desc + 1) Hash 分桶 + 语法： + `DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]` + 说明： + 使用指定的 key 列进行哈希分桶。默认分区数为10 + + 建议:建议使用Hash分桶方式 + +4. PROPERTIES + 1) 如果 ENGINE 类型为 olap + 可以在 properties 设置该表数据的初始存储介质、存储到期时间和副本数。 + + ``` + PROPERTIES ( + "storage_medium" = "[SSD|HDD]", + ["storage_cooldown_time" = "yyyy-MM-dd HH:mm:ss"], + ["replication_num" = "3"] + ) + ``` + + storage_medium： 用于指定该分区的初始存储介质，可选择 SSD 或 HDD。默认初始存储介质可通过fe的配置文件 `fe.conf` 中指定 `default_storage_medium=xxx`，如果没有指定，则默认为 HDD。 + storage_cooldown_time： 当设置存储介质为 SSD 时，指定该分区在 SSD 上的存储到期时间。 + 默认存放 30 天。 + 格式为："yyyy-MM-dd HH:mm:ss" + replication_num: 指定分区的副本数。默认为 3 + + 当表为单分区表时，这些属性为表的属性。 + 当表为两级分区时，这些属性为附属于每一个分区。 + 如果希望不同分区有不同属性。可以通过 ADD PARTITION 或 MODIFY PARTITION 进行操作 + + 2 如果 Engine 类型为 olap, 可以指定某列使用 bloom filter 索引 + bloom filter 索引仅适用于查询条件为 in 和 equal 的情况，该列的值越分散效果越好 + 目前只支持以下情况的列:除了 TINYINT FLOAT DOUBLE 类型以外的 key 列及聚合方法为 REPLACE 的 value 列 + +``` + PROPERTIES ( + "bloom_filter_columns"="k1,k2,k3" + ) +``` + + 3) 如果希望使用 Colocate Join 特性，需要在 properties 中指定 + +``` + PROPERTIES ( + "colocate_with"="table1" + ) +``` + + 4) 如果希望使用动态分区特性，需要在properties 中指定 + +``` + PROPERTIES ( + "dynamic_partition.enable" = "true|false", + "dynamic_partition.time_unit" = "DAY|WEEK|MONTH", + "dynamic_partition.start" = "${integer_value}", + "dynamic_partitoin.end" = "${integer_value}", + "dynamic_partition.prefix" = "${string_value}", + "dynamic_partition.buckets" = "${integer_value} +``` + dynamic_partition.enable: 用于指定表级别的动态分区功能是否开启。默认为 true。 + dynamic_partition.time_unit: 用于指定动态添加分区的时间单位，可选择为DAY（天），WEEK(周)，MONTH（月） + dynamic_partition.start: 用于指定向前删除多少个分区。值必须小于0。默认为 Integer.MIN_VALUE。 + dynamic_partition.end: 用于指定提前创建的分区数量。值必须大于0。 + dynamic_partition.prefix: 用于指定创建的分区名前缀，例如分区名前缀为p，则自动创建分区名为p20200108 + dynamic_partition.buckets: 用于指定自动创建的分区分桶数量 + + 5) 建表时可以批量创建多个 Rollup + 语法： + ``` + ROLLUP (rollup_name (column_name1, column_name2, ...) + [FROM from_index_name] + [PROPERTIES ("key"="value", ...)],...) + ``` + + 6) 如果希望使用 内存表 特性，需要在 properties 中指定 + +``` + PROPERTIES ( + "in_memory"="true" + ) +``` + 当 in_memory 属性为 true 时，Doris会尽可能将该表的数据和索引Cache到BE 内存中 +## example + +1. 创建一个 olap 表，使用 HASH 分桶，使用列存，相同key的记录进行聚合 + + ``` + CREATE TABLE example_db.table_hash + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 CHAR(10) REPLACE, + v2 INT SUM + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + COMMENT "my first doris table" + DISTRIBUTED BY HASH(k1) BUCKETS 32 + PROPERTIES ("storage_type"="column"); + ``` + +2. 创建一个 olap 表，使用 Hash 分桶，使用列存，相同key的记录进行覆盖， + 设置初始存储介质和冷却时间 + + ``` + CREATE TABLE example_db.table_hash + ( + k1 BIGINT, + k2 LARGEINT, + v1 VARCHAR(2048) REPLACE, + v2 SMALLINT SUM DEFAULT "10" + ) + ENGINE=olap + UNIQUE KEY(k1, k2) + DISTRIBUTED BY HASH (k1, k2) BUCKETS 32 + PROPERTIES( + "storage_type"="column"， + "storage_medium" = "SSD", + "storage_cooldown_time" = "2015-06-04 00:00:00" + ); + ``` + +3. 创建一个 olap 表，使用 Range 分区，使用Hash分桶，默认使用列存， + 相同key的记录同时存在，设置初始存储介质和冷却时间 + + 1）LESS THAN + + ``` + CREATE TABLE example_db.table_range + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY RANGE (k1) + ( + PARTITION p1 VALUES LESS THAN ("2014-01-01"), + PARTITION p2 VALUES LESS THAN ("2014-06-01"), + PARTITION p3 VALUES LESS THAN ("2014-12-01") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", "storage_cooldown_time" = "2015-06-04 00:00:00" + ); + ``` + + 说明： + 这个语句会将数据划分成如下3个分区： + + ``` + ( { MIN }, {"2014-01-01"} ) + [ {"2014-01-01"}, {"2014-06-01"} ) + [ {"2014-06-01"}, {"2014-12-01"} ) + ``` + + 不在这些分区范围内的数据将视为非法数据被过滤 + + 2) Fixed Range + + ``` + CREATE TABLE table_range + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY RANGE (k1, k2, k3) + ( + PARTITION p1 VALUES [("2014-01-01", "10", "200"), ("2014-01-01", "20", "300")), + PARTITION p2 VALUES [("2014-06-01", "100", "200"), ("2014-07-01", "100", "300")) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD" + ); + ``` + +4. 创建一个 mysql 表 + +``` + CREATE TABLE example_db.table_mysql + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=mysql + PROPERTIES + ( + "host" = "127.0.0.1", + "port" = "8239", + "user" = "mysql_user", + "password" = "mysql_passwd", + "database" = "mysql_db_test", + "table" = "mysql_table_test" + ) +``` + +5. 创建一个数据文件存储在HDFS上的 broker 外部表, 数据使用 "|" 分割，"\n" 换行 + +``` + CREATE EXTERNAL TABLE example_db.table_broker ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=broker + PROPERTIES ( + "broker_name" = "hdfs", + "path" = "hdfs://hdfs_host:hdfs_port/data1,hdfs://hdfs_host:hdfs_port/data2,hdfs://hdfs_host:hdfs_port/data3%2c4", + "column_separator" = "|", + "line_delimiter" = "\n" + ) + BROKER PROPERTIES ( + "username" = "hdfs_user", + "password" = "hdfs_password" + ) +``` + +6. 创建一张含有HLL列的表 + +``` + CREATE TABLE example_db.example_table + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 HLL HLL_UNION, + v2 HLL HLL_UNION + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + DISTRIBUTED BY HASH(k1) BUCKETS 32 + PROPERTIES ("storage_type"="column"); +``` + +7. 创建一张含有BITMAP_UNION聚合类型的表（v1和v2列的原始数据类型必须是TINYINT,SMALLINT,INT） + +``` + CREATE TABLE example_db.example_table + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 BITMAP BITMAP_UNION, + v2 BITMAP BITMAP_UNION + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + DISTRIBUTED BY HASH(k1) BUCKETS 32 + PROPERTIES ("storage_type"="column"); +``` + +8. 创建两张支持Colocat Join的表t1 和t2 + +``` + CREATE TABLE `t1` ( + `id` int(11) COMMENT "", + `value` varchar(8) COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "colocate_with" = "t1" + ); + + CREATE TABLE `t2` ( + `id` int(11) COMMENT "", + `value` varchar(8) COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "colocate_with" = "t1" + ); +``` + +9. 创建一个数据文件存储在BOS上的 broker 外部表 + +``` + CREATE EXTERNAL TABLE example_db.table_broker ( + k1 DATE + ) + ENGINE=broker + PROPERTIES ( + "broker_name" = "bos", + "path" = "bos://my_bucket/input/file", + ) + BROKER PROPERTIES ( + "bos_endpoint" = "http://bj.bcebos.com", + "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "bos_secret_accesskey"="yyyyyyyyyyyyyyyyyyyy" + ) +``` + +10. 创建一个带有bitmap 索引的表 + +``` + CREATE TABLE example_db.table_hash + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 CHAR(10) REPLACE, + v2 INT SUM, + INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx' + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + COMMENT "my first doris table" + DISTRIBUTED BY HASH(k1) BUCKETS 32 + PROPERTIES ("storage_type"="column"); +``` + +11. 创建一个动态分区表(需要在FE配置中开启动态分区功能)，该表每天提前创建3天的分区，并删除3天前的分区。例如今天为`2020-01-08`，则会创建分区名为`p20200108`, `p20200109`, `p20200110`, `p20200111`的分区. 分区范围分别为: + +``` +[types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) +[types: [DATE]; keys: [2020-01-09]; ‥types: [DATE]; keys: [2020-01-10]; ) +[types: [DATE]; keys: [2020-01-10]; ‥types: [DATE]; keys: [2020-01-11]; ) +[types: [DATE]; keys: [2020-01-11]; ‥types: [DATE]; keys: [2020-01-12]; ) +``` + +``` + CREATE TABLE example_db.dynamic_partition + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY RANGE (k1) + ( + PARTITION p1 VALUES LESS THAN ("2014-01-01"), + PARTITION p2 VALUES LESS THAN ("2014-06-01"), + PARTITION p3 VALUES LESS THAN ("2014-12-01") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-3", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "32" + ); +``` + +12. Create a table with rollup index +``` + CREATE TABLE example_db.rolup_index_table + ( + event_day DATE, + siteid INT DEFAULT '10', + citycode SMALLINT, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(event_day, siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 10 + rollup ( + r1(event_day,siteid), + r2(event_day,citycode), + r3(event_day) + ) + PROPERTIES("replication_num" = "3"); + +13. 创建一个内存表 + +``` + CREATE TABLE example_db.table_hash + ( + k1 TINYINT, + k2 DECIMAL(10, 2) DEFAULT "10.5", + v1 CHAR(10) REPLACE, + v2 INT SUM, + INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx' + ) + ENGINE=olap + AGGREGATE KEY(k1, k2) + COMMENT "my first doris table" + DISTRIBUTED BY HASH(k1) BUCKETS 32 + PROPERTIES ("in_memory"="true"); +``` + +## keyword + + CREATE,TABLE diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE VIEW.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE VIEW.md new file mode 100644 index 00000000000000..85ae07fb8a3d5a --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE VIEW.md @@ -0,0 +1,64 @@ +--- +{ + "title": "CREATE VIEW", + "language": "zh-CN" +} +--- + + + +# CREATE VIEW +## description + 该语句用于创建一个逻辑视图 + 语法： + CREATE VIEW [IF NOT EXISTS] + [db_name.]view_name + (column1[ COMMENT "col comment"][, column2, ...]) + AS query_stmt + + 说明： + 1. 视图为逻辑视图，没有物理存储。所有在视图上的查询相当于在视图对应的子查询上进行。 + 2. query_stmt 为任意支持的 SQL + +## example + 1. 在 example_db 上创建视图 example_view + + CREATE VIEW example_db.example_view (k1, k2, k3, v1) + AS + SELECT c1 as k1, k2, k3, SUM(v1) FROM example_table + WHERE k1 = 20160112 GROUP BY k1,k2,k3; + + 2. 创建一个包含 comment 的 view + + CREATE VIEW example_db.example_view + ( + k1 COMMENT "first key", + k2 COMMENT "second key", + k3 COMMENT "third key", + v1 COMMENT "first value" + ) + COMMENT "my first view" + AS + SELECT c1 as k1, k2, k3, SUM(v1) FROM example_table + WHERE k1 = 20160112 GROUP BY k1,k2,k3; + +## keyword + CREATE,VIEW + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP DATABASE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP DATABASE.md new file mode 100644 index 00000000000000..48ed7524532088 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP DATABASE.md @@ -0,0 +1,42 @@ +--- +{ + "title": "DROP DATABASE", + "language": "zh-CN" +} +--- + + + +# DROP DATABASE +## description + 该语句用于删除数据库（database） + 语法： + DROP DATABASE [IF EXISTS] db_name; + + 说明： + 执行 DROP DATABASE 一段时间内，可以通过 RECOVER 语句恢复被删除的 database。详见 RECOVER 语句 + +## example + 1. 删除数据库 db_test + DROP DATABASE db_test; + +## keyword + DROP,DATABASE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP INDEX.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP INDEX.md new file mode 100644 index 00000000000000..45c1d1252cefef --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP INDEX.md @@ -0,0 +1,37 @@ +--- +{ + "title": "DROP INDEX", + "language": "zh-CN" +} +--- + + + +# DROP INDEX + +## description + + 该语句用于从一个表中删除指定名称的索引，目前仅支持bitmap 索引 + 语法： + DROP INDEX index_name ON [db_name.]table_name; + +## keyword + + DROP,INDEX diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md new file mode 100644 index 00000000000000..461e044b0b09c0 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP MATERIALIZED VIEW.md @@ -0,0 +1,108 @@ +--- +{ + "title": "DROP MATERIALIZED VIEW", + "language": "zh-CN" +} +--- + + + +# DROP MATERIALIZED VIEW + +## description + 该语句用于删除物化视图。同步语法 + +语法: + + ``` + DROP MATERIALIZED VIEW [IF EXISTS] mv_name ON table_name + ``` + +1. IF EXISTS + 如果物化视图不存在，不要抛出错误。如果不声明此关键字，物化视图不存在则报错。 + +2. mv_name + 待删除的物化视图的名称。必填项。 + +3. table_name + 待删除的物化视图所属的表名。必填项。 + +## example + +表结构为 + +``` +mysql> desc all_type_table all; ++----------------+-------+----------+------+-------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++----------------+-------+----------+------+-------+---------+-------+ +| all_type_table | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | false | N/A | NONE | +| | k3 | INT | Yes | false | N/A | NONE | +| | k4 | BIGINT | Yes | false | N/A | NONE | +| | k5 | LARGEINT | Yes | false | N/A | NONE | +| | k6 | FLOAT | Yes | false | N/A | NONE | +| | k7 | DOUBLE | Yes | false | N/A | NONE | +| | | | | | | | +| k1_sumk2 | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | false | N/A | SUM | ++----------------+-------+----------+------+-------+---------+-------+ +``` + +1. 删除表 all_type_table 的名为 k1_sumk2 的物化视图 + + ``` + drop materialized view k1_sumk2 on all_type_table; + ``` + 物化视图被删除后的表结构 + + ``` + +----------------+-------+----------+------+-------+---------+-------+ +| IndexName | Field | Type | Null | Key | Default | Extra | ++----------------+-------+----------+------+-------+---------+-------+ +| all_type_table | k1 | TINYINT | Yes | true | N/A | | +| | k2 | SMALLINT | Yes | false | N/A | NONE | +| | k3 | INT | Yes | false | N/A | NONE | +| | k4 | BIGINT | Yes | false | N/A | NONE | +| | k5 | LARGEINT | Yes | false | N/A | NONE | +| | k6 | FLOAT | Yes | false | N/A | NONE | +| | k7 | DOUBLE | Yes | false | N/A | NONE | ++----------------+-------+----------+------+-------+---------+-------+ + ``` + +2. 删除表 all_type_table 中一个不存在的物化视图 + + ``` + drop materialized view k1_k2 on all_type_table; + ERROR 1064 (HY000): errCode = 2, detailMessage = Materialized view [k1_k2] does not exist in table [all_type_table] + ``` + 删除请求直接报错 + +3. 删除表 all_type_table 中的物化视图 k1_k2，不存在不报错。 + + ``` + drop materialized view if exists k1_k2 on all_type_table; +Query OK, 0 rows affected (0.00 sec) + ``` + + 存在则删除，不存在则不报错。 + +## keyword + DROP, MATERIALIZED, VIEW diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md new file mode 100644 index 00000000000000..dce41aef9dc025 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP REPOSITORY.md @@ -0,0 +1,42 @@ +--- +{ + "title": "DROP REPOSITORY", + "language": "zh-CN" +} +--- + + + +# DROP REPOSITORY +## description + 该语句用于删除一个已创建的仓库。仅 root 或 superuser 用户可以删除仓库。 + 语法： + DROP REPOSITORY `repo_name`; + + 说明： + 1. 删除仓库，仅仅是删除该仓库在 Palo 中的映射，不会删除实际的仓库数据。删除后，可以再次通过指定相同的 broker 和 LOCATION 映射到该仓库。 + +## example + 1. 删除名为 bos_repo 的仓库： + DROP REPOSITORY `bos_repo`; + +## keyword + DROP REPOSITORY + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP TABLE.md new file mode 100644 index 00000000000000..dcb912393bbb86 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP TABLE.md @@ -0,0 +1,45 @@ +--- +{ + "title": "DROP TABLE", + "language": "zh-CN" +} +--- + + + +# DROP TABLE +## description + 该语句用于删除 table 。 + 语法： + DROP TABLE [IF EXISTS] [db_name.]table_name; + + 说明： + 执行 DROP TABLE 一段时间内，可以通过 RECOVER 语句恢复被删除的 table。详见 RECOVER 语句 + +## example + 1. 删除一个 table + DROP TABLE my_table; + + 2. 如果存在，删除指定 database 的 table + DROP TABLE IF EXISTS example_db.my_table; + +## keyword + DROP,TABLE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP VIEW.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP VIEW.md new file mode 100644 index 00000000000000..dedb001986e020 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/DROP VIEW.md @@ -0,0 +1,40 @@ +--- +{ + "title": "DROP VIEW", + "language": "zh-CN" +} +--- + + + +# DROP VIEW +## description + 该语句用于删除一个逻辑视图 VIEW + 语法： + DROP VIEW [IF EXISTS] + [db_name.]view_name; + +## example + 1. 如果存在，删除 example_db 上的视图 example_view + DROP VIEW IF EXISTS example_db.example_view; + +## keyword + DROP,VIEW + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/HLL.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/HLL.md new file mode 100644 index 00000000000000..dd1a108c4e4cf7 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/HLL.md @@ -0,0 +1,109 @@ +--- +{ + "title": "HLL", + "language": "zh-CN" +} +--- + + + +# HLL +## description + HLL是基于HyperLogLog算法的工程实现，用于保存HyperLogLog计算过程的中间结果，它只能作为表的value列类型 + 通过聚合来不断的减少数据量，以此来实现加快查询的目的，基于它到的是一个估算结果，误差大概在1%左右 + hll列是通过其它列或者导入数据里面的数据生成的，导入的时候通过hll_hash函数来指定数据中哪一列用于生成hll列 + 它常用于替代count distinct，通过结合rollup在业务上用于快速计算uv等 + + 相关函数: + + HLL_UNION_AGG(hll) + 此函数为聚合函数，用于计算满足条件的所有数据的基数估算。此函数还可用于分析函数，只支持默认窗口，不支持window从句。 + + HLL_RAW_AGG(hll) + 此函数为聚合函数，用于聚合hll类型字段，并且返回的还是hll类型。 + + HLL_CARDINALITY(hll) + 此函数用于计算单条hll列的基数估算 + + HLL_HASH(column_name) + 生成HLL列类型，用于insert或导入的时候，导入的使用见相关说明 + + EMPTY_HLL() + 生成空HLL列，用于insert或导入的时候补充默认值，导入的使用见相关说明 + +## example + 1. 首先创建一张含有hll列的表 + create table test( + dt date, + id int, + name char(10), + province char(10), + os char(1), + set1 hll hll_union, + set2 hll hll_union) + distributed by hash(id) buckets 32; + + 2. 导入数据，导入的方式见相关help curl + + a. 使用表中的列生成hll列 + curl --location-trusted -uname:password -T data -H "label:load_1" -H "columns:dt, id, name, province, os, set1=hll_hash(id), set2=hll_hash(name)" + http://host/api/test_db/test/_stream_load + b. 使用数据中的某一列生成hll列 + curl --location-trusted -uname:password -T data -H "label:load_1" -H "columns:dt, id, name, province, sex, cuid, os, set1=hll_hash(cuid), set2=hll_hash(os)" + http://host/api/test_db/test/_stream_load + + 3. 聚合数据，常用方式3种：（如果不聚合直接对base表查询，速度可能跟直接使用ndv速度差不多） + + a. 创建一个rollup，让hll列产生聚合， + alter table test add rollup test_rollup(dt, set1); + + b. 创建另外一张专门计算uv的表，然后insert数据） + + create table test_uv( + dt date, + uv_set hll hll_union) + distributed by hash(id) buckets 32; + + insert into test_uv select dt, set1 from test; + + c. 创建另外一张专门计算uv的表，然后insert并通过hll_hash根据test其它非hll列生成hll列 + + create table test_uv( + dt date, + id_set hll hll_union) + distributed by hash(id) buckets 32; + + insert into test_uv select dt, hll_hash(id) from test; + + 4. 查询，hll列不允许直接查询它的原始值，可以通过配套的函数进行查询 + + a. 求总uv + select HLL_UNION_AGG(uv_set) from test_uv; + + b. 求每一天的uv + select dt, HLL_CARDINALITY(uv_set) from test_uv; + + c. 求test表中set1的聚合值 + select dt, HLL_CARDINALITY(uv) from (select dt, HLL_RAW_AGG(set1) as uv from test group by dt) tmp; + select dt, HLL_UNION_AGG(set1) as uv from test group by dt; + +## keyword + HLL + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/RECOVER.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/RECOVER.md new file mode 100644 index 00000000000000..98cebc18be6791 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/RECOVER.md @@ -0,0 +1,54 @@ +--- +{ + "title": "RECOVER", + "language": "zh-CN" +} +--- + + + +# RECOVER +## description + 该语句用于恢复之前删除的 database、table 或者 partition + 语法： + 1) 恢复 database + RECOVER DATABASE db_name; + 2) 恢复 table + RECOVER TABLE [db_name.]table_name; + 3) 恢复 partition + RECOVER PARTITION partition_name FROM [db_name.]table_name; + + 说明： + 1. 该操作仅能恢复之前一段时间内删除的元信息。默认为 1 天。（可通过fe.conf中`catalog_trash_expire_second`参数配置） + 2. 如果删除元信息后新建立了同名同类型的元信息，则之前删除的元信息不能被恢复 + +## example + 1. 恢复名为 example_db 的 database + RECOVER DATABASE example_db; + + 2. 恢复名为 example_tbl 的 table + RECOVER TABLE example_db.example_tbl; + + 3. 恢复表 example_tbl 中名为 p1 的 partition + RECOVER PARTITION p1 FROM example_tbl; + +## keyword + RECOVER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/RESTORE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/RESTORE.md new file mode 100644 index 00000000000000..9871639e5ea2ea --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/RESTORE.md @@ -0,0 +1,78 @@ +--- +{ + "title": "RESTORE", + "language": "zh-CN" +} +--- + + + +# RESTORE +## description + 1. RESTORE + 该语句用于将之前通过 BACKUP 命令备份的数据，恢复到指定数据库下。该命令为异步操作。提交成功后，需通过 SHOW RESTORE 命令查看进度。仅支持恢复 OLAP 类型的表。 + 语法： + RESTORE SNAPSHOT [db_name].{snapshot_name} + FROM `repository_name` + ON ( + `table_name` [PARTITION (`p1`, ...)] [AS `tbl_alias`], + ... + ) + PROPERTIES ("key"="value", ...); + + 说明： + 1. 同一数据库下只能有一个正在执行的 BACKUP 或 RESTORE 任务。 + 2. ON 子句中标识需要恢复的表和分区。如果不指定分区，则默认恢复该表的所有分区。所指定的表和分区必须已存在于仓库备份中。 + 3. 可以通过 AS 语句将仓库中备份的表名恢复为新的表。但新表名不能已存在于数据库中。分区名称不能修改。 + 4. 可以将仓库中备份的表恢复替换数据库中已有的同名表，但须保证两张表的表结构完全一致。表结构包括：表名、列、分区、Rollup等等。 + 5. 可以指定恢复表的部分分区，系统会检查分区 Range 是否能够匹配。 + 6. PROPERTIES 目前支持以下属性： + "backup_timestamp" = "2018-05-04-16-45-08"：指定了恢复对应备份的哪个时间版本，必填。该信息可以通过 `SHOW SNAPSHOT ON repo;` 语句获得。 + "replication_num" = "3"：指定恢复的表或分区的副本数。默认为3。若恢复已存在的表或分区，则副本数必须和已存在表或分区的副本数相同。同时，必须有足够的 host 容纳多个副本。 + "timeout" = "3600"：任务超时时间，默认为一天。单位秒。 + "meta_version" = 40：使用指定的 meta_version 来读取之前备份的元数据。注意，该参数作为临时方案，仅用于恢复老版本 Doris 备份的数据。最新版本的备份数据中已经包含 meta version，无需再指定。 + +## example + 1. 从 example_repo 中恢复备份 snapshot_1 中的表 backup_tbl 到数据库 example_db1，时间版本为 "2018-05-04-16-45-08"。恢复为 1 个副本： + RESTORE SNAPSHOT example_db1.`snapshot_1` + FROM `example_repo` + ON ( `backup_tbl` ) + PROPERTIES + ( + "backup_timestamp"="2018-05-04-16-45-08", + "replication_num" = "1" + ); + + 2. 从 example_repo 中恢复备份 snapshot_2 中的表 backup_tbl 的分区 p1,p2，以及表 backup_tbl2 到数据库 example_db1，并重命名为 new_tbl，时间版本为 "2018-05-04-17-11-01"。默认恢复为 3 个副本： + RESTORE SNAPSHOT example_db1.`snapshot_2` + FROM `example_repo` + ON + ( + `backup_tbl` PARTITION (`p1`, `p2`), + `backup_tbl2` AS `new_tbl` + ) + PROPERTIES + ( + "backup_timestamp"="2018-05-04-17-11-01" + ); + +## keyword + RESTORE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md new file mode 100644 index 00000000000000..afb4919f6dba47 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/TRUNCATE TABLE.md @@ -0,0 +1,53 @@ +--- +{ + "title": "TRUNCATE TABLE", + "language": "zh-CN" +} +--- + + + +# TRUNCATE TABLE +## description + 该语句用于清空指定表和分区的数据 + 语法： + + TRUNCATE TABLE [db.]tbl[ PARTITION(p1, p2, ...)]; + + 说明： + 1. 该语句清空数据，但保留表或分区。 + 2. 不同于 DELETE，该语句只能整体清空指定的表或分区，不能添加过滤条件。 + 3. 不同于 DELETE，使用该方式清空数据不会对查询性能造成影响。 + 4. 该操作删除的数据不可恢复。 + 5. 使用该命令时，表状态需为 NORMAL，即不允许正在进行 SCHEMA CHANGE 等操作。 + +## example + + 1. 清空 example_db 下的表 tbl + + TRUNCATE TABLE example_db.tbl; + + 2. 清空表 tbl 的 p1 和 p2 分区 + + TRUNCATE TABLE tbl PARTITION(p1, p2); + +## keyword + TRUNCATE,TABLE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/create-function.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/create-function.md new file mode 100644 index 00000000000000..2bd2b7efae93c5 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/create-function.md @@ -0,0 +1,114 @@ +--- +{ + "title": "CREATE FUNCTION", + "language": "zh-CN" +} +--- + + + +# CREATE FUNCTION +## description +### Syntax + +``` +CREATE [AGGREGATE] FUNCTION function_name + (arg_type [, ...]) + RETURNS ret_type + [INTERMEDIATE inter_type] + [PROPERTIES ("key" = "value" [, ...]) ] +``` + +### Parameters + +> `AGGREGATE`: 如果有此项，表示的是创建的函数是一个聚合函数，否则创建的是一个标量函数。 +> +> `function_name`: 要创建函数的名字, 可以包含数据库的名字。比如：`db1.my_func`。 +> +> `arg_type`: 函数的参数类型，与建表时定义的类型一致。变长参数时可以使用`, ...`来表示，如果是变长类型，那么变长部分参数的类型与最后一个非变长参数类型一致。 +> +> `ret_type`: 函数返回类型。 +> +> `inter_type`: 用于表示聚合函数中间阶段的数据类型。 +> +> `properties`: 用于设定此函数相关属性，能够设置的属性包括 +> +> "object_file": 自定义函数动态库的URL路径，当前只支持 HTTP/HTTPS 协议，此路径需要在函数整个生命周期内保持有效。此选项为必选项 +> +> "symbol": 标量函数的函数签名，用于从动态库里面找到函数入口。此选项对于标量函数是必选项 +> +> "init_fn": 聚合函数的初始化函数签名。对于聚合函数是必选项 +> +> "update_fn": 聚合函数的更新函数签名。对于聚合函数是必选项 +> +> "merge_fn": 聚合函数的合并函数签名。对于聚合函数是必选项 +> +> "serialize_fn": 聚合函数的序列化函数签名。对于聚合函数是可选项，如果没有指定，那么将会使用默认的序列化函数 +> +> "finalize_fn": 聚合函数获取最后结果的函数签名。对于聚合函数是可选项，如果没有指定，将会使用默认的获取结果函数 +> +> "md5": 函数动态链接库的MD5值，用于校验下载的内容是否正确。此选项是可选项 +> +> "prepare_fn": 自定义函数的prepare函数的函数签名，用于从动态库里面找到prepare函数入口。此选项对于自定义函数是可选项 +> +> "close_fn": 自定义函数的close函数的函数签名，用于从动态库里面找到close函数入口。此选项对于自定义函数是可选项 + + +此语句创建一个自定义函数。执行此命令需要用户拥有 `ADMIN` 权限。 + +如果 `function_name` 中包含了数据库名字，那么这个自定义函数会创建在对应的数据库中，否则这个函数将会创建在当前会话所在的数据库。新函数的名字与参数不能够与当前命名空间中已存在的函数相同，否则会创建失败。但是只有名字相同，参数不同是能够创建成功的。 + +## example + +1. 创建一个自定义标量函数 + + ``` + CREATE FUNCTION my_add(INT, INT) RETURNS INT PROPERTIES ( + "symbol" = "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", + "object_file" = "http://host:port/libmyadd.so" + ); + ``` + +2. 创建一个有prepare/close函数的自定义标量函数 + + ``` + CREATE FUNCTION my_add(INT, INT) RETURNS INT PROPERTIES ( + "symbol" = "_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_", + "prepare_fn" = "_ZN9doris_udf14AddUdf_prepareEPNS_15FunctionContextENS0_18FunctionStateScopeE", + "close_fn" = "_ZN9doris_udf12AddUdf_closeEPNS_15FunctionContextENS0_18FunctionStateScopeE", + "object_file" = "http://host:port/libmyadd.so" + ); + ``` + +2. 创建一个自定义聚合函数 + + ``` + CREATE AGGREGATE FUNCTION my_count (BIGINT) RETURNS BIGINT PROPERTIES ( + "init_fn"="_ZN9doris_udf9CountInitEPNS_15FunctionContextEPNS_9BigIntValE", + "update_fn"="_ZN9doris_udf11CountUpdateEPNS_15FunctionContextERKNS_6IntValEPNS_9BigIntValE", + "merge_fn"="_ZN9doris_udf10CountMergeEPNS_15FunctionContextERKNS_9BigIntValEPS2_", + "finalize_fn"="_ZN9doris_udf13CountFinalizeEPNS_15FunctionContextERKNS_9BigIntValE", + "object_file"="http://host:port/libudasample.so" + ); + ``` + +## keyword + + CREATE,FUNCTION diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/drop-function.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/drop-function.md new file mode 100644 index 00000000000000..6fb66aed51bd2f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/drop-function.md @@ -0,0 +1,56 @@ +--- +{ + "title": "DROP FUNCTION", + "language": "zh-CN" +} +--- + + + +# DROP FUNCTION +## description +### Syntax + +``` +DROP FUNCTION function_name + (arg_type [, ...]) +``` + +### Parameters + +> `function_name`: 要删除函数的名字 +> +> `arg_type`: 要删除函数的参数列表 +> + + +删除一个自定义函数。函数的名字、参数类型完全一致才能够被删除 + +## example + +1. 删除掉一个函数 + +``` +DROP FUNCTION my_add(INT, INT) +``` + +## keyword + + DROP,FUNCTION diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/show-functions.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/show-functions.md new file mode 100644 index 00000000000000..b8808ba078f529 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/show-functions.md @@ -0,0 +1,79 @@ +--- +{ + "title": "SHOW FUNCTIONS", + "language": "zh-CN" +} +--- + + + +# SHOW FUNCTIONS +## description +### Syntax + +``` +SHOW [FULL] [BUILTIN] FUNCTIONS [IN|FROM db] [LIKE 'function_pattern'] +``` + +### Parameters + +>`full`:表示显示函数的详细信息 +>`builtin`:表示显示系统提供的函数 +>`db`: 要查询的数据库名字 +>`function_pattern`: 用来过滤函数名称的参数 + + +查看数据库下所有的自定义(系统提供)的函数。如果用户指定了数据库，那么查看对应数据库的，否则直接查询当前会话所在数据库 + +需要对这个数据库拥有 `SHOW` 权限 + +## example + +``` +mysql> show full functions in testDb\G +*************************** 1. row *************************** + Signature: my_add(INT,INT) + Return Type: INT + Function Type: Scalar +Intermediate Type: NULL + Properties: {"symbol":"_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_","object_file":"http://host:port/libudfsample.so","md5":"cfe7a362d10f3aaf6c49974ee0f1f878"} +*************************** 2. row *************************** + Signature: my_count(BIGINT) + Return Type: BIGINT + Function Type: Aggregate +Intermediate Type: NULL + Properties: {"object_file":"http://host:port/libudasample.so","finalize_fn":"_ZN9doris_udf13CountFinalizeEPNS_15FunctionContextERKNS_9BigIntValE","init_fn":"_ZN9doris_udf9CountInitEPNS_15FunctionContextEPNS_9BigIntValE","merge_fn":"_ZN9doris_udf10CountMergeEPNS_15FunctionContextERKNS_9BigIntValEPS2_","md5":"37d185f80f95569e2676da3d5b5b9d2f","update_fn":"_ZN9doris_udf11CountUpdateEPNS_15FunctionContextERKNS_6IntValEPNS_9BigIntValE"} + +2 rows in set (0.00 sec) +mysql> show builtin functions in testDb like 'year%'; ++---------------+ +| Function Name | ++---------------+ +| year | +| years_add | +| years_diff | +| years_sub | ++---------------+ +2 rows in set (0.00 sec) +``` + +## keyword + + SHOW,FUNCTIONS diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md new file mode 100644 index 00000000000000..1b4ed6ed17c25f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md @@ -0,0 +1,435 @@ +--- +{ + "title": "BROKER LOAD", + "language": "zh-CN" +} +--- + + + +# BROKER LOAD +## description + + Broker load 通过随 Doris 集群一同部署的 broker 进行，访问对应数据源的数据，进行数据导入。 + 可以通过 show broker 命令查看已经部署的 broker。 + 目前支持以下4种数据源： + + 1. Baidu HDFS：百度内部的 hdfs，仅限于百度内部使用。 + 2. Baidu AFS：百度内部的 afs，仅限于百度内部使用。 + 3. Baidu Object Storage(BOS)：百度对象存储。仅限百度内部用户、公有云用户或其他可以访问 BOS 的用户使用。 + 4. Apache HDFS：社区版本 hdfs。 + 5. Amazon S3：Amazon对象存储。 + +语法： + + LOAD LABEL load_label + ( + data_desc1[, data_desc2, ...] + ) + WITH BROKER broker_name + [broker_properties] + [opt_properties]; + + 1. load_label + + 当前导入批次的标签。在一个 database 内唯一。 + 语法： + [database_name.]your_label + + 2. data_desc + + 用于描述一批导入数据。 + 语法： + DATA INFILE + ( + "file_path1"[, file_path2, ...] + ) + [NEGATIVE] + INTO TABLE `table_name` + [PARTITION (p1, p2)] + [COLUMNS TERMINATED BY "column_separator"] + [FORMAT AS "file_type"] + [(column_list)] + [SET (k1 = func(k2))] + [WHERE predicate] + + 说明： + file_path: + + 文件路径，可以指定到一个文件，也可以用 * 通配符指定某个目录下的所有文件。通配符必须匹配到文件，而不能是目录。 + + PARTITION: + + 如果指定此参数，则只会导入指定的分区，导入分区以外的数据会被过滤掉。 + 如果不指定，默认导入table的所有分区。 + + NEGATIVE： + 如果指定此参数，则相当于导入一批“负”数据。用于抵消之前导入的同一批数据。 + 该参数仅适用于存在 value 列，并且 value 列的聚合类型仅为 SUM 的情况。 + + column_separator： + + 用于指定导入文件中的列分隔符。默认为 \t + 如果是不可见字符，则需要加\\x作为前缀，使用十六进制来表示分隔符。 + 如hive文件的分隔符\x01，指定为"\\x01" + + file_type： + + 用于指定导入文件的类型，例如：parquet、orc、csv。默认值通过文件后缀名判断。 + + column_list： + + 用于指定导入文件中的列和 table 中的列的对应关系。 + 当需要跳过导入文件中的某一列时，将该列指定为 table 中不存在的列名即可。 + 语法： + (col_name1, col_name2, ...) + + SET: + + 如果指定此参数，可以将源文件某一列按照函数进行转化，然后将转化后的结果导入到table中。语法为 `column_name` = expression。举几个例子帮助理解。 + 例1: 表中有3个列“c1, c2, c3", 源文件中前两列依次对应(c1,c2)，后两列之和对应c3；那么需要指定 columns (c1,c2,tmp_c3,tmp_c4) SET (c3=tmp_c3+tmp_c4); + 例2: 表中有3个列“year, month, day"三个列，源文件中只有一个时间列，为”2018-06-01 01:02:03“格式。 + 那么可以指定 columns(tmp_time) set (year = year(tmp_time), month=month(tmp_time), day=day(tmp_time)) 完成导入。 + + WHERE: + + 对做完 transform 的数据进行过滤，符合 where 条件的数据才能被导入。WHERE 语句中只可引用表中列名。 + 3. broker_name + + 所使用的 broker 名称，可以通过 show broker 命令查看。 + + 4. broker_properties + + 用于提供通过 broker 访问数据源的信息。不同的 broker，以及不同的访问方式，需要提供的信息不同。 + + 1. Baidu HDFS/AFS + + 访问百度内部的 hdfs/afs 目前仅支持简单认证，需提供： + username：hdfs 用户名 + password：hdfs 密码 + + 2. BOS + + 需提供： + bos_endpoint：BOS 的endpoint + bos_accesskey：公有云用户的 accesskey + bos_secret_accesskey：公有云用户的 secret_accesskey + + 3. Apache HDFS + + 社区版本的 hdfs，支持简单认证、kerberos 认证。以及支持 HA 配置。 + 简单认证： + hadoop.security.authentication = simple (默认) + username：hdfs 用户名 + password：hdfs 密码 + + kerberos 认证： + hadoop.security.authentication = kerberos + kerberos_principal：指定 kerberos 的 principal + kerberos_keytab：指定 kerberos 的 keytab 文件路径。该文件必须为 broker 进程所在服务器上的文件。 + kerberos_keytab_content：指定 kerberos 中 keytab 文件内容经过 base64 编码之后的内容。这个跟 kerberos_keytab 配置二选一就可以。 + + namenode HA： + 通过配置 namenode HA，可以在 namenode 切换时，自动识别到新的 namenode + dfs.nameservices: 指定 hdfs 服务的名字，自定义，如："dfs.nameservices" = "my_ha" + dfs.ha.namenodes.xxx：自定义 namenode 的名字,多个名字以逗号分隔。其中 xxx 为 dfs.nameservices 中自定义的名字，如 "dfs.ha.namenodes.my_ha" = "my_nn" + dfs.namenode.rpc-address.xxx.nn：指定 namenode 的rpc地址信息。其中 nn 表示 dfs.ha.namenodes.xxx 中配置的 namenode 的名字，如："dfs.namenode.rpc-address.my_ha.my_nn" = "host:port" + dfs.client.failover.proxy.provider：指定 client 连接 namenode 的 provider，默认为：org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + 4. Amazon S3 + + 需提供： + fs.s3a.access.key：AmazonS3的access key + fs.s3a.secret.key：AmazonS3的secret key + fs.s3a.endpoint：AmazonS3的endpoint + + 4. opt_properties + + 用于指定一些特殊参数。 + 语法： + [PROPERTIES ("key"="value", ...)] + + 可以指定如下参数： + timeout： 指定导入操作的超时时间。默认超时为4小时。单位秒。 + max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。 + exec_mem_limit： 导入内存限制。默认为 2GB。单位为字节。 + strict mode： 是否对数据进行严格限制。默认为 false。 + timezone: 指定某些受时区影响的函数的时区，如 strftime/alignment_timestamp/from_unixtime 等等，具体请查阅 [时区] 文档。如果不指定，则使用 "Asia/Shanghai" 时区。 + + 5. 导入数据格式样例 + + 整型类（TINYINT/SMALLINT/INT/BIGINT/LARGEINT）：1, 1000, 1234 + 浮点类（FLOAT/DOUBLE/DECIMAL）：1.1, 0.23, .356 + 日期类（DATE/DATETIME）：2017-10-03, 2017-06-13 12:34:03。 + （注：如果是其他日期格式，可以在导入命令中，使用 strftime 或者 time_format 函数进行转换） + 字符串类（CHAR/VARCHAR）："I am a student", "a" + NULL值：\N + +## example + + 1. 从 HDFS 导入一批数据，指定超时时间和过滤比例。使用铭文 my_hdfs_broker 的 broker。简单认证。 + + LOAD LABEL example_db.label1 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + ) + WITH BROKER my_hdfs_broker + ( + "username" = "hdfs_user", + "password" = "hdfs_passwd" + ) + PROPERTIES + ( + "timeout" = "3600", + "max_filter_ratio" = "0.1" + ); + + 其中 hdfs_host 为 namenode 的 host，hdfs_port 为 fs.defaultFS 端口（默认9000） + + 2. 从 AFS 一批数据，包含多个文件。导入不同的 table，指定分隔符，指定列对应关系。 + + LOAD LABEL example_db.label2 + ( + DATA INFILE("afs://afs_host:hdfs_port/user/palo/data/input/file1") + INTO TABLE `my_table_1` + COLUMNS TERMINATED BY "," + (k1, k3, k2, v1, v2), + DATA INFILE("afs://afs_host:hdfs_port/user/palo/data/input/file2") + INTO TABLE `my_table_2` + COLUMNS TERMINATED BY "\t" + (k1, k2, k3, v2, v1) + ) + WITH BROKER my_afs_broker + ( + "username" = "afs_user", + "password" = "afs_passwd" + ) + PROPERTIES + ( + "timeout" = "3600", + "max_filter_ratio" = "0.1" + ); + + + 3. 从 HDFS 导入一批数据，指定hive的默认分隔符\x01，并使用通配符*指定目录下的所有文件。 + 使用简单认证，同时配置 namenode HA + + LOAD LABEL example_db.label3 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/*") + INTO TABLE `my_table` + COLUMNS TERMINATED BY "\\x01" + ) + WITH BROKER my_hdfs_broker + ( + "username" = "hdfs_user", + "password" = "hdfs_passwd", + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "nn1_host:rpc_port", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "nn2_host:rpc_port", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ) + + 4. 从 HDFS 导入一批“负”数据。同时使用 kerberos 认证方式。提供 keytab 文件路径。 + + LOAD LABEL example_db.label4 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/old_file) + NEGATIVE + INTO TABLE `my_table` + COLUMNS TERMINATED BY "\t" + ) + WITH BROKER my_hdfs_broker + ( + "hadoop.security.authentication" = "kerberos", + "kerberos_principal"="doris@YOUR.COM", + "kerberos_keytab"="/home/palo/palo.keytab" + ) + + 5. 从 HDFS 导入一批数据，指定分区。同时使用 kerberos 认证方式。提供 base64 编码后的 keytab 文件内容。 + + LOAD LABEL example_db.label5 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (k1, k3, k2, v1, v2) + ) + WITH BROKER my_hdfs_broker + ( + "hadoop.security.authentication"="kerberos", + "kerberos_principal"="doris@YOUR.COM", + "kerberos_keytab_content"="BQIAAABEAAEACUJBSURVLkNPTQAEcGFsbw" + ) + + 6. 从 BOS 导入一批数据，指定分区, 并对导入文件的列做一些转化，如下： + 表结构为： + k1 varchar(20) + k2 int + + 假设数据文件只有一行数据： + + Adele,1,1 + + 数据文件中各列，对应导入语句中指定的各列： + k1,tmp_k2,tmp_k3 + + 转换如下： + + 1) k1: 不变换 + 2) k2：是 tmp_k2 和 tmp_k3 数据之和 + + LOAD LABEL example_db.label6 + ( + DATA INFILE("bos://my_bucket/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (k1, tmp_k2, tmp_k3) + SET ( + k2 = tmp_k2 + tmp_k3 + ) + ) + WITH BROKER my_bos_broker + ( + "bos_endpoint" = "http://bj.bcebos.com", + "bos_accesskey" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "bos_secret_accesskey"="yyyyyyyyyyyyyyyyyyyy" + ) + + 7. 导入数据到含有HLL列的表，可以是表中的列或者数据里面的列 + + 如果表中有三列分别是（id,v1,v2,v3）。其中v1和v2列是hll列。导入的源文件有3列。则（column_list）中声明第一列为id，第二三列为一个临时命名的k1,k2。 + 在SET中必须给表中的hll列特殊声明 hll_hash。表中的v1列等于原始数据中的hll_hash(k1)列, 表中的v3列在原始数据中并没有对应的值，使用empty_hll补充默认值。 + LOAD LABEL example_db.label7 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (id, k1, k2) + SET ( + v1 = hll_hash(k1), + v2 = hll_hash(k2), + v3 = empty_hll() + ) + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + + LOAD LABEL example_db.label8 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (k1, k2, tmp_k3, tmp_k4, v1, v2) + SET ( + v1 = hll_hash(tmp_k3), + v2 = hll_hash(tmp_k4) + ) + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + + 8. 导入Parquet文件中数据 指定FORMAT 为parquet， 默认是通过文件后缀判断 + + LOAD LABEL example_db.label9 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + FORMAT AS "parquet" + (k1, k2, k3) + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + + 9. 提取文件路径中的分区字段 + + 如果需要，则会根据表中定义的字段类型解析文件路径中的分区字段（partitioned fields），类似Spark中Partition Discovery的功能 + + LOAD LABEL example_db.label10 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/*/*") + INTO TABLE `my_table` + FORMAT AS "csv" + (k1, k2, k3) + COLUMNS FROM PATH AS (city, utc_date) + SET (uniq_id = md5sum(k1, city)) + ) + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + + hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing目录下包括如下文件： + + [hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/utc_date=2019-06-26/0000.csv, hdfs://hdfs_host:hdfs_port/user/palo/data/input/dir/city=beijing/utc_date=2019-06-26/0001.csv, ...] + + 则提取文件路径的中的city和utc_date字段 + + 10. 对待导入数据进行过滤，k1 值大于 k2 值的列才能被导入 + + LOAD LABEL example_db.label10 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + where k1 > k2 + ) + + 11. 从 AmazonS3 导入Parquet文件中数据，指定 FORMAT 为parquet，默认是通过文件后缀判断： + + LOAD LABEL example_db.label11 + ( + DATA INFILE("s3a://my_bucket/input/file") + INTO TABLE `my_table` + FORMAT AS "parquet" + (k1, k2, k3) + ) + WITH BROKER my_s3a_broker + ( + "fs.s3a.access.key" = "xxxxxxxxxxxxxxxxxxxxxxxxxx", + "fs.s3a.secret.key" = "yyyyyyyyyyyyyyyyyyyy", + "fs.s3a.endpoint" = "s3.amazonaws.com" + ) + + 12. 提取文件路径中的时间分区字段，并且时间包含 %3A (在 hdfs 路径中，不允许有 ':'，所有 ':' 会由 %3A 替换) + + 假设有如下文件： + + /user/data/data_time=2020-02-17 00%3A00%3A00/test.txt + /user/data/data_time=2020-02-18 00%3A00%3A00/test.txt + + 表结构为： + data_time DATETIME, + k2 INT, + k3 INT + + LOAD LABEL example_db.label12 + ( + DATA INFILE("hdfs://host:port/user/data/*/test.txt") + INTO TABLE `tbl12` + COLUMNS TERMINATED BY "," + (k2,k3) + COLUMNS FROM PATH AS (data_time) + SET (data_time=str_to_date(data_time, '%Y-%m-%d %H%%3A%i%%3A%s')) + ) + WITH BROKER "hdfs" ("username"="user", "password"="pass"); + +## keyword + + BROKER,LOAD diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md new file mode 100644 index 00000000000000..0ab78885e23ac6 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/CANCEL LOAD.md @@ -0,0 +1,46 @@ +--- +{ + "title": "CANCEL LOAD", + "language": "zh-CN" +} +--- + + + +# CANCEL LOAD +## description + + 该语句用于撤销指定 load label 的批次的导入作业。 + 这是一个异步操作，任务提交成功则返回。执行后可使用 SHOW LOAD 命令查看进度。 + 语法： + CANCEL LOAD + [FROM db_name] + WHERE LABEL = "load_label"; + +## example + + 1. 撤销数据库 example_db 上， label 为 example_db_test_load_label 的导入作业 + CANCEL LOAD + FROM example_db + WHERE LABEL = "example_db_test_load_label"; + +## keyword + CANCEL,LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/DELETE.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/DELETE.md new file mode 100644 index 00000000000000..8a3d800f6f365b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/DELETE.md @@ -0,0 +1,62 @@ +--- +{ + "title": "DELETE", + "language": "zh-CN" +} +--- + + + +# DELETE +## description + + 该语句用于按条件删除指定 table（base index） partition 中的数据。 + 该操作会同时删除和此 base index 相关的 rollup index 的数据。 + 语法： + DELETE FROM table_name [PARTITION partition_name] + WHERE + column_name1 op value[ AND column_name2 op value ...]; + + 说明： + 1) op 的可选类型包括：=, >, <, >=, <=, != + 2) 只能指定 key 列上的条件。 + 2) 当选定的 key 列不存在于某个 rollup 中时，无法进行 delete。 + 3) 条件之间只能是“与”的关系。 + 若希望达成“或”的关系，需要将条件分写在两个 DELETE 语句中。 + 4) 如果为RANGE分区表，则必须指定 PARTITION。如果是单分区表，可以不指定。 + + 注意： + 该语句可能会降低执行后一段时间内的查询效率。 + 影响程度取决于语句中指定的删除条件的数量。 + 指定的条件越多，影响越大。 + +## example + + 1. 删除 my_table partition p1 中 k1 列值为 3 的数据行 + DELETE FROM my_table PARTITION p1 + WHERE k1 = 3; + + 2. 删除 my_table partition p1 中 k1 列值大于等于 3 且 k2 列值为 "abc" 的数据行 + DELETE FROM my_table PARTITION p1 + WHERE k1 >= 3 AND k2 = "abc"; + +## keyword + DELETE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/EXPORT.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/EXPORT.md new file mode 100644 index 00000000000000..44814a47798b09 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/EXPORT.md @@ -0,0 +1,84 @@ +--- +{ + "title": "EXPORT", + "language": "zh-CN" +} +--- + + + +# EXPORT +## description + + 该语句用于将指定表的数据导出到指定位置。 + 该功能通过 broker 进程实现。对于不同的目的存储系统，需要部署不同的 broker。可以通过 SHOW BROKER 查看已部署的 broker。 + 这是一个异步操作，任务提交成功则返回。执行后可使用 SHOW EXPORT 命令查看进度。 + + 语法： + EXPORT TABLE table_name + [PARTITION (p1[,p2])] + TO export_path + [opt_properties] + broker; + + 1. table_name + 当前要导出的表的表名，目前支持engine为olap和mysql的表的导出。 + + 2. partition + 可以只导出指定表的某些指定分区 + + 3. export_path + 导出的路径，需为目录。目前不能导出到本地，需要导出到broker。 + + 4. opt_properties + 用于指定一些特殊参数。 + 语法： + [PROPERTIES ("key"="value", ...)] + + 可以指定如下参数： + column_separator: 指定导出的列分隔符，默认为\t。 + line_delimiter: 指定导出的行分隔符，默认为\n。 + exec_mem_limit: 导出在单个 BE 节点的内存使用上限，默认为 2GB，单位为字节。 + timeout：导入作业的超时时间，默认为1天，单位是秒。 + tablet_num_per_task：每个子任务能分配的最大 Tablet 数量。 + + 5. broker + 用于指定导出使用的broker + 语法： + WITH BROKER broker_name ("key"="value"[,...]) + 这里需要指定具体的broker name, 以及所需的broker属性 + + 对于不同存储系统对应的 broker，这里需要输入的参数不同。具体参数可以参阅：`help broker load` 中 broker 所需属性。 + +## example + + 1. 将 testTbl 表中的所有数据导出到 hdfs 上 + EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); + + 2. 将 testTbl 表中的分区p1,p2导出到 hdfs 上 + + EXPORT TABLE testTbl PARTITION (p1,p2) TO "hdfs://hdfs_host:port/a/b/c" WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); + 3. 将 testTbl 表中的所有数据导出到 hdfs 上，以","作为列分隔符 + + EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" PROPERTIES ("column_separator"=",") WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); + +## keyword + EXPORT + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/GROUP BY.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/GROUP BY.md new file mode 100644 index 00000000000000..30a96f1e1b82c1 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/GROUP BY.md @@ -0,0 +1,170 @@ +--- +{ + "title": "GROUP BY", + "language": "zh-CN" +} +--- + + + +# GROUP BY + +## description + + GROUP BY `GROUPING SETS` ｜ `CUBE` ｜ `ROLLUP` 是对 GROUP BY 子句的扩展，它能够在一个 GROUP BY 子句中实现多个集合的分组的聚合。其结果等价于将多个相应 GROUP BY 子句进行 UNION 操作。 + + GROUP BY 子句是只含有一个元素的 GROUP BY GROUPING SETS 的特例。 + 例如，GROUPING SETS 语句： + + ``` + SELECT a, b, SUM( c ) FROM tab1 GROUP BY GROUPING SETS ( (a, b), (a), (b), ( ) ); + ``` + + 其查询结果等价于： + + ``` + SELECT a, b, SUM( c ) FROM tab1 GROUP BY a, b + UNION + SELECT a, null, SUM( c ) FROM tab1 GROUP BY a + UNION + SELECT null, b, SUM( c ) FROM tab1 GROUP BY b + UNION + SELECT null, null, SUM( c ) FROM tab1 + ``` + + `GROUPING(expr)` 指示一个列是否为聚合列，如果是聚合列为0，否则为1 + + `GROUPING_ID(expr [ , expr [ , ... ] ])` 与GROUPING 类似， GROUPING_ID根据指定的column 顺序，计算出一个列列表的 bitmap 值，每一位为GROUPING的值. GROUPING_ID()函数返回位向量的十进制值。 + +### Syntax + + ``` + SELECT ... + FROM ... + [ ... ] + GROUP BY [ + , ... | + GROUPING SETS [, ...] ( groupSet [ , groupSet [ , ... ] ] ) | + ROLLUP(expr [ , expr [ , ... ] ]) | + expr [ , expr [ , ... ] ] WITH ROLLUP | + CUBE(expr [ , expr [ , ... ] ]) | + expr [ , expr [ , ... ] ] WITH CUBE + ] + [ ... ] + ``` + +### Parameters + + `groupSet` 表示 select list 中的列，别名或者表达式组成的集合 `groupSet ::= { ( expr [ , expr [ , ... ] ] )}` + + `expr` 表示 select list 中的列，别名或者表达式 + +### Note + + doris 支持类似PostgreSQL 语法, 语法实例如下 + + ``` + SELECT a, b, SUM( c ) FROM tab1 GROUP BY GROUPING SETS ( (a, b), (a), (b), ( ) ); + SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY ROLLUP(a,b,c) + SELECT a, b,c, SUM( d ) FROM tab1 GROUP BY CUBE(a,b,c) + ``` + + `ROLLUP(a,b,c)` 等价于如下`GROUPING SETS` 语句 + + ``` + GROUPING SETS ( + (a,b,c), + ( a, b ), + ( a), + ( ) + ) + ``` + + `CUBE ( a, b, c )` 等价于如下`GROUPING SETS` 语句 + + ``` + GROUPING SETS ( + ( a, b, c ), + ( a, b ), + ( a, c ), + ( a ), + ( b, c ), + ( b ), + ( c ), + ( ) + ) + ``` + +## example + + 下面是一个实际数据的例子 + + ``` + > SELECT * FROM t; + +------+------+------+ + | k1 | k2 | k3 | + +------+------+------+ + | a | A | 1 | + | a | A | 2 | + | a | B | 1 | + | a | B | 3 | + | b | A | 1 | + | b | A | 4 | + | b | B | 1 | + | b | B | 5 | + +------+------+------+ + 8 rows in set (0.01 sec) + + > SELECT k1, k2, SUM(k3) FROM t GROUP BY GROUPING SETS ( (k1, k2), (k2), (k1), ( ) ); + +------+------+-----------+ + | k1 | k2 | sum(`k3`) | + +------+------+-----------+ + | b | B | 6 | + | a | B | 4 | + | a | A | 3 | + | b | A | 5 | + | NULL | B | 10 | + | NULL | A | 8 | + | a | NULL | 7 | + | b | NULL | 11 | + | NULL | NULL | 18 | + +------+------+-----------+ + 9 rows in set (0.06 sec) + + > SELECT k1, k2, GROUPING_ID(k1,k2), SUM(k3) FROM t GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ()); + +------+------+---------------+----------------+ + | k1 | k2 | grouping_id(k1,k2) | sum(`k3`) | + +------+------+---------------+----------------+ + | a | A | 0 | 3 | + | a | B | 0 | 4 | + | a | NULL | 1 | 7 | + | b | A | 0 | 5 | + | b | B | 0 | 6 | + | b | NULL | 1 | 11 | + | NULL | A | 2 | 8 | + | NULL | B | 2 | 10 | + | NULL | NULL | 3 | 18 | + +------+------+---------------+----------------+ + 9 rows in set (0.02 sec) + ``` + +## keyword + + GROUP, GROUPING, GROUPING_ID, GROUPING_SETS, GROUPING SETS, CUBE, ROLLUP diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/LOAD.md new file mode 100644 index 00000000000000..469c8f02aee4d3 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/LOAD.md @@ -0,0 +1,293 @@ +--- +{ + "title": "LOAD", + "language": "zh-CN" +} +--- + + + +# LOAD +## description + + Palo 目前支持以下4种导入方式： + + 1. Hadoop Load：基于 MR 进行 ETL 的导入。 + 2. Broker Load：使用 broker 进行进行数据导入。 + 3. Mini Load：通过 http 协议上传文件进行批量数据导入。 + 4. Stream Load：通过 http 协议进行流式数据导入。 + + 本帮助主要描述第一种导入方式，即 Hadoop Load 相关帮助信息。其余导入方式可以使用以下命令查看帮助： + + !!!该导入方式可能在后续某个版本即不再支持，建议使用其他导入方式进行数据导入。!!! + + 1. help broker load; + 2. help mini load; + 3. help stream load; + + Hadoop Load 仅适用于百度内部环境。公有云、私有云以及开源环境无法使用这种导入方式。 + 该导入方式必须设置用于 ETL 的 Hadoop 计算队列，设置方式可以通过 help set property 命令查看帮助。 + + Stream load 暂时只支持百度内部用户使用。开源社区和公有云用户将在后续版本更新中支持。 + +语法： + + LOAD LABEL load_label + ( + data_desc1[, data_desc2, ...] + ) + [opt_properties]; + + 1. load_label + + 当前导入批次的标签。在一个 database 内唯一。 + 语法： + [database_name.]your_label + + 2. data_desc + + 用于描述一批导入数据。 + 语法： + DATA INFILE + ( + "file_path1"[, file_path2, ...] + ) + [NEGATIVE] + INTO TABLE `table_name` + [PARTITION (p1, p2)] + [COLUMNS TERMINATED BY "column_separator"] + [FORMAT AS "file_type"] + [(column_list)] + [COLUMNS FROM PATH AS (columns_from_path)] + [SET (k1 = func(k2))] + + 说明： + file_path: + + 文件路径，可以指定到一个文件，也可以用 * 通配符指定某个目录下的所有文件。通配符必须匹配到文件，而不能是目录。 + + PARTITION: + + 如果指定此参数，则只会导入指定的分区，导入分区以外的数据会被过滤掉。 + 如果不指定，默认导入table的所有分区。 + + NEGATIVE： + 如果指定此参数，则相当于导入一批“负”数据。用于抵消之前导入的同一批数据。 + 该参数仅适用于存在 value 列，并且 value 列的聚合类型仅为 SUM 的情况。 + + column_separator： + + 用于指定导入文件中的列分隔符。默认为 \t + 如果是不可见字符，则需要加\\x作为前缀，使用十六进制来表示分隔符。 + 如hive文件的分隔符\x01，指定为"\\x01" + + file_type： + + 用于指定导入文件的类型，例如：parquet、orc、csv。默认值通过文件后缀名判断。 + + column_list： + + 用于指定导入文件中的列和 table 中的列的对应关系。 + 当需要跳过导入文件中的某一列时，将该列指定为 table 中不存在的列名即可。 + 语法： + (col_name1, col_name2, ...) + + columns_from_path: + + 用于指定需要从文件路径中解析的字段。 + 语法： + (col_from_path_name1, col_from_path_name2, ...) + + SET: + + 如果指定此参数，可以将源文件某一列按照函数进行转化，然后将转化后的结果导入到table中。 + 目前支持的函数有： + + strftime(fmt, column) 日期转换函数 + fmt: 日期格式，形如%Y%m%d%H%M%S (年月日时分秒) + column: column_list中的列，即输入文件中的列。存储内容应为数字型的时间戳。 + 如果没有column_list，则按照palo表的列顺序默认输入文件的列。 + + time_format(output_fmt, input_fmt, column) 日期格式转化 + output_fmt: 转化后的日期格式，形如%Y%m%d%H%M%S (年月日时分秒) + input_fmt: 转化前column列的日期格式，形如%Y%m%d%H%M%S (年月日时分秒) + column: column_list中的列，即输入文件中的列。存储内容应为input_fmt格式的日期字符串。 + 如果没有column_list，则按照palo表的列顺序默认输入文件的列。 + + alignment_timestamp(precision, column) 将时间戳对齐到指定精度 + precision: year|month|day|hour + column: column_list中的列，即输入文件中的列。存储内容应为数字型的时间戳。 + 如果没有column_list，则按照palo表的列顺序默认输入文件的列。 + 注意：对齐精度为year、month的时候，只支持20050101~20191231范围内的时间戳。 + + default_value(value) 设置某一列导入的默认值 + 不指定则使用建表时列的默认值 + + md5sum(column1, column2, ...) 将指定的导入列的值求md5sum，返回32位16进制字符串 + + replace_value(old_value[, new_value]) 将导入文件中指定的old_value替换为new_value + new_value如不指定则使用建表时列的默认值 + + hll_hash(column) 用于将表或数据里面的某一列转化成HLL列的数据结构 + + 3. opt_properties + + 用于指定一些特殊参数。 + 语法： + [PROPERTIES ("key"="value", ...)] + + 可以指定如下参数： + cluster: 导入所使用的 Hadoop 计算队列。 + timeout： 指定导入操作的超时时间。默认超时为3天。单位秒。 + max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。 + load_delete_flag：指定该导入是否通过导入key列的方式删除数据，仅适用于UNIQUE KEY， + 导入时可不指定value列。默认为false。 + + 5. 导入数据格式样例 + + 整型类（TINYINT/SMALLINT/INT/BIGINT/LARGEINT）：1, 1000, 1234 + 浮点类（FLOAT/DOUBLE/DECIMAL）：1.1, 0.23, .356 + 日期类（DATE/DATETIME）：2017-10-03, 2017-06-13 12:34:03。 + （注：如果是其他日期格式，可以在导入命令中，使用 strftime 或者 time_format 函数进行转换） + 字符串类（CHAR/VARCHAR）："I am a student", "a" + NULL值：\N + +## example + + 1. 导入一批数据，指定超时时间和过滤比例。指定导入队列为 my_cluster。 + + LOAD LABEL example_db.label1 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + ) + PROPERTIES + ( + "cluster" = "my_cluster", + "timeout" = "3600", + "max_filter_ratio" = "0.1" + ); + + 其中 hdfs_host 为 namenode 的 host，hdfs_port 为 fs.defaultFS 端口（默认9000） + + 2. 导入一批数据，包含多个文件。导入不同的 table，指定分隔符，指定列对应关系 + + LOAD LABEL example_db.label2 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file1") + INTO TABLE `my_table_1` + COLUMNS TERMINATED BY "," + (k1, k3, k2, v1, v2), + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file2") + INTO TABLE `my_table_2` + COLUMNS TERMINATED BY "\t" + (k1, k2, k3, v2, v1) + ); + + 3. 导入一批数据，指定hive的默认分隔符\x01，并使用通配符*指定目录下的所有文件 + + LOAD LABEL example_db.label3 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/*") + NEGATIVE + INTO TABLE `my_table` + COLUMNS TERMINATED BY "\\x01" + ); + + 4. 导入一批“负”数据 + + LOAD LABEL example_db.label4 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/old_file) + NEGATIVE + INTO TABLE `my_table` + COLUMNS TERMINATED BY "\t" + ); + + 5. 导入一批数据，指定分区 + + LOAD LABEL example_db.label5 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (k1, k3, k2, v1, v2) + ); + + 6. 导入一批数据，指定分区, 并对导入文件的列做一些转化，如下： + 表结构为： + k1 datetime + k2 date + k3 bigint + k4 varchar(20) + k5 varchar(64) + k6 int + + 假设数据文件只有一行数据，5列，逗号分隔： + + 1537002087,2018-08-09 11:12:13,1537002087,-,1 + + 数据文件中各列，对应导入语句中指定的各列： + tmp_k1, tmp_k2, tmp_k3, k6, v1 + + 转换如下： + + 1) k1：将 tmp_k1 时间戳列转化为 datetime 类型的数据 + 2) k2：将 tmp_k2 datetime 类型的数据转化为 date 的数据 + 3) k3：将 tmp_k3 时间戳列转化为天级别时间戳 + 4) k4：指定导入默认值为1 + 5) k5：将 tmp_k1、tmp_k2、tmp_k3 列计算 md5 值 + 6) k6：将导入文件中的 - 值替换为 10 + + LOAD LABEL example_db.label6 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + (tmp_k1, tmp_k2, tmp_k3, k6, v1) + SET ( + k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1), + k2 = time_format("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", tmp_k2), + k3 = alignment_timestamp("day", tmp_k3), + k4 = default_value("1"), + k5 = md5sum(tmp_k1, tmp_k2, tmp_k3), + k6 = replace_value("-", "10") + ) + ); + + 7. 导入数据到含有HLL列的表，可以是表中的列或者数据里面的列 + + LOAD LABEL example_db.label7 + ( + DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") + INTO TABLE `my_table` + PARTITION (p1, p2) + COLUMNS TERMINATED BY "," + SET ( + v1 = hll_hash(k1), + v2 = hll_hash(k2) + ) + ); + +## keyword + LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md new file mode 100644 index 00000000000000..e2500aae7b1383 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/MINI LOAD.md @@ -0,0 +1,139 @@ +--- +{ + "title": "MINI LOAD", + "language": "zh-CN" +} +--- + + + +# MINI LOAD +## description + + MINI LOAD 和 STREAM LOAD 的导入实现方式完全一致。在导入功能支持上，MINI LOAD 的功能是 STREAM LOAD 的子集。 + 后续的导入新功能只会在 STREAM LOAD 中支持，MINI LOAD 将不再新增功能。建议改用 STREAM LOAD，具体使用方式请 HELP STREAM LOAD。 + + MINI LOAD 是 通过 http 协议完成的导入方式。用户可以不依赖 Hadoop，也无需通过 Mysql 客户端，即可完成导入。 + 用户通过 http 协议描述导入，数据在接受 http 请求的过程中被流式的导入 Doris , **导入作业完成后** 返回给用户导入的结果。 + + * 注：为兼容旧版本 mini load 使用习惯，用户依旧可以通过 'SHOW LOAD' 命令来查看导入结果。 + + 语法： + 导入： + + curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table}/_load?label=xxx + + 查看导入信息 + + curl -u user:passwd http://host:port/api/{db}/_load_info?label=xxx + + HTTP协议相关说明 + + 权限认证 当前 Doris 使用 http 的 Basic 方式权限认证。所以在导入的时候需要指定用户名密码 + 这种方式是明文传递密码的，暂不支持加密传输。 + + Expect Doris 需要发送过来的 http 请求带有 'Expect' 头部信息，内容为 '100-continue'。 + 为什么呢？因为我们需要将请求进行 redirect，那么必须在传输数据内容之前， + 这样可以避免造成数据的多次传输，从而提高效率。 + + Content-Length Doris 需要在发送请求时带有 'Content-Length' 这个头部信息。如果发送的内容比 + 'Content-Length' 要少，那么 Doris 认为传输出现问题，则提交此次任务失败。 + NOTE: 如果，发送的数据比 'Content-Length' 要多，那么 Doris 只读取 'Content-Length' + 长度的内容，并进行导入 + + + 参数说明： + + user: 用户如果是在default_cluster中的，user即为user_name。否则为user_name@cluster_name。 + + label: 用于指定这一批次导入的 label，用于后期进行作业查询等。 + 这个参数是必须传入的。 + + columns: 用于描述导入文件中对应的列名字。 + 如果不传入，那么认为文件中的列顺序与建表的顺序一致， + 指定的方式为逗号分隔，例如：columns=k1,k2,k3,k4 + + column_separator: 用于指定列与列之间的分隔符，默认的为'\t' + NOTE: 需要进行url编码，譬如 + 需要指定'\t'为分隔符，那么应该传入'column_separator=%09' + 需要指定'\x01'为分隔符，那么应该传入'column_separator=%01' + 需要指定','为分隔符，那么应该传入'column_separator=%2c' + + + max_filter_ratio: 用于指定允许过滤不规范数据的最大比例，默认是0，不允许过滤 + 自定义指定应该如下：'max_filter_ratio=0.2'，含义是允许20%的错误率 + + timeout: 指定 load 作业的超时时间，单位是秒。当load执行时间超过该阈值时，会自动取消。默认超时时间是 600 秒。 + 建议指定 timeout 时间小于 86400 秒。 + + hll: 用于指定数据里面和表里面的HLL列的对应关系，表中的列和数据里面指定的列 + （如果不指定columns，则数据列面的列也可以是表里面的其它非HLL列）通过","分割 + 指定多个hll列使用“:”分割，例如: 'hll1,cuid:hll2,device' + + strict_mode: 指定当前导入是否使用严格模式，默认为 true。严格模式下，非空原始数据在列类型转化后结果为 NULL 的会被过滤。 + 指定方式为 'strict_mode=false' + + NOTE: + 1. 此种导入方式当前是在一台机器上完成导入工作，因而不宜进行数据量较大的导入工作。 + 建议导入数据量不要超过 1 GB + + 2. 当前无法使用 `curl -T "{file1, file2}"` 这样的方式提交多个文件，因为curl是将其拆成多个 + 请求发送的，多个请求不能共用一个label号，所以无法使用 + + 3. mini load 的导入方式和 streaming 完全一致，都是在流式的完成导入后，同步的返回结果给用户。 + 后续查询虽可以查到 mini load 的信息，但不能对其进行操作，查询只为兼容旧的使用方式。 + + 4. 当使用 curl 命令行导入时，需要在 & 前加入 \ 转义，否则参数信息会丢失。 + +## example + + 1. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表（用户是defalut_cluster中的） + curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123 + + 2. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表（用户是test_cluster中的）。超时时间是 3600 秒 + curl --location-trusted -u root@test_cluster:root -T testData http://fe.host:port/api/testDb/testTbl/_load?label=123\&timeout=3600 + + 3. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率（用户是defalut_cluster中的） + curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 + + 4. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率，并且指定文件的列名（用户是defalut_cluster中的） + curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2\&columns=k1,k2,k3 + + 5. 使用streaming方式导入（用户是defalut_cluster中的） + seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_load?label=123 + + 6. 导入含有HLL列的表，可以是表中的列或者数据中的列用于生成HLL列（用户是defalut_cluster中的 + + curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 + \&columns=k1,k2,k3\&hll=hll_column1,k1:hll_column2,k2 + + curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&max_filter_ratio=0.2 + \&hll=hll_column1,tmp_k4:hll_column2,tmp_k5\&columns=k1,k2,k3,tmp_k4,tmp_k5 + + 7. 查看提交后的导入情况 + + curl -u root http://host:port/api/testDb/_load_info?label=123 + + 8. 指定非严格模式导入 + curl --location-trusted -u root -T testData http://host:port/api/testDb/testTbl/_load?label=123\&strict_mode=false + +## keyword + MINI, LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md new file mode 100644 index 00000000000000..526e7abb184bf4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/MULTI LOAD.md @@ -0,0 +1,108 @@ +--- +{ + "title": "MULTI LOAD", + "language": "zh-CN" +} +--- + + + +# MULTI LOAD +## description + + Syntax: + curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_start?label=xxx + curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table1}/_load?label=xxx\&sub_label=yyy + curl --location-trusted -u user:passwd -T data.file http://host:port/api/{db}/{table2}/_load?label=xxx\&sub_label=zzz + curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_commit?label=xxx + curl --location-trusted -u user:passwd -XPOST http://host:port/api/{db}/_multi_desc?label=xxx + + 'MULTI LOAD'在'MINI LOAD'的基础上，可以支持用户同时向多个表进行导入，具体的命令如上面所示 + '/api/{db}/_multi_start' 开始一个多表导入任务 + '/api/{db}/{table}/_load' 向一个导入任务添加一个要导入的表，与'MINI LOAD'的主要区别是，需要传入'sub_label'参数 + '/api/{db}/_multi_commit' 提交整个多表导入任务，后台开始进行处理 + '/api/{db}/_multi_abort' 放弃一个多表导入任务 + '/api/{db}/_multi_desc' 可以展示某个多表导入任务已经提交的作业数 + + HTTP协议相关说明 + 权限认证 当前 Doris 使用http的Basic方式权限认证。所以在导入的时候需要指定用户名密码 + 这种方式是明文传递密码的，鉴于我们当前都是内网环境。。。 + + Expect Doris 需要发送过来的http请求，需要有'Expect'头部信息，内容为'100-continue' + 为什么呢？因为我们需要将请求进行redirect，那么必须在传输数据内容之前， + 这样可以避免造成数据的多次传输，从而提高效率。 + + Content-Length Doris 需要在发送请求是带有'Content-Length'这个头部信息。如果发送的内容比 + 'Content-Length'要少，那么Palo认为传输出现问题，则提交此次任务失败。 + NOTE: 如果，发送的数据比'Content-Length'要多，那么 Doris 只读取'Content-Length' + 长度的内容，并进行导入 + + 参数说明： + user: 用户如果是在default_cluster中的，user即为user_name。否则为user_name@cluster_name。 + + label: 用于指定这一批次导入的label号，用于后期进行作业状态查询等。 + 这个参数是必须传入的。 + + sub_label: 用于指定一个多表导入任务内部的子版本号。对于多表导入的load， 这个参数是必须传入的。 + + columns: 用于描述导入文件中对应的列名字。 + 如果不传入，那么认为文件中的列顺序与建表的顺序一致， + 指定的方式为逗号分隔，例如：columns=k1,k2,k3,k4 + + column_separator: 用于指定列与列之间的分隔符，默认的为'\t' + NOTE: 需要进行url编码，譬如需要指定'\t'为分隔符， + 那么应该传入'column_separator=%09' + + max_filter_ratio: 用于指定允许过滤不规范数据的最大比例，默认是0，不允许过滤 + 自定义指定应该如下：'max_filter_ratio=0.2'，含义是允许20%的错误率 + 在'_multi_start'时传入有效果 + + NOTE: + 1. 此种导入方式当前是在一台机器上完成导入工作，因而不宜进行数据量较大的导入工作。 + 建议导入数据量不要超过1GB + + 2. 当前无法使用`curl -T "{file1, file2}"`这样的方式提交多个文件，因为curl是将其拆成多个 + 请求发送的，多个请求不能共用一个label号，所以无法使用 + + 3. 支持类似streaming的方式使用curl来向 Doris 中导入数据，但是，只有等这个streaming结束后 Doris + 才会发生真实的导入行为，这中方式数据量也不能过大。 + +## example + + 1. 将本地文件'testData1'中的数据导入到数据库'testDb'中'testTbl1'的表，并且 + 把'testData2'的数据导入到'testDb'中的表'testTbl2'(用户是defalut_cluster中的) + curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 + curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 + curl --location-trusted -u root -T testData2 http://host:port/api/testDb/testTbl2/_load?label=123\&sub_label=2 + curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_commit?label=123 + + 2. 多表导入中途放弃(用户是defalut_cluster中的) + curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 + curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 + curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_abort?label=123 + + 3. 多表导入查看已经提交多少内容(用户是defalut_cluster中的) + curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_start?label=123 + curl --location-trusted -u root -T testData1 http://host:port/api/testDb/testTbl1/_load?label=123\&sub_label=1 + curl --location-trusted -u root -XPOST http://host:port/api/testDb/_multi_desc?label=123 + +## keyword + MULTI, MINI, LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md new file mode 100644 index 00000000000000..85f8c866aea773 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/PAUSE ROUTINE LOAD.md @@ -0,0 +1,36 @@ +--- +{ + "title": "PAUSE ROUTINE LOAD", + "language": "zh-CN" +} +--- + + + +# PAUSE ROUTINE LOAD +## example + +1. 暂停名称为 test1 的例行导入作业。 + + PAUSE ROUTINE LOAD FOR test1; + +## keyword + PAUSE,ROUTINE,LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md new file mode 100644 index 00000000000000..9a3730b2ac2dc9 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/RESUME ROUTINE LOAD.md @@ -0,0 +1,36 @@ +--- +{ + "title": "RESUME ROUTINE LOAD", + "language": "zh-CN" +} +--- + + + +# RESUME ROUTINE LOAD +## example + +1. 恢复名称为 test1 的例行导入作业。 + + RESUME ROUTINE LOAD FOR test1; + +## keyword + RESUME,ROUTINE,LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md new file mode 100644 index 00000000000000..10d55c9b179014 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md @@ -0,0 +1,316 @@ +--- +{ + "title": "ROUTINE LOAD", + "language": "zh-CN" +} +--- + + + +# ROUTINE LOAD +## description + + 例行导入（Routine Load）功能，支持用户提交一个常驻的导入任务，通过不断的从指定的数据源读取数据，将数据导入到 Doris 中。 + 目前仅支持通过无认证或者 SSL 认证方式，从 Kakfa 导入文本格式（CSV）的数据。 + +语法： + + CREATE ROUTINE LOAD [db.]job_name ON tbl_name + [load_properties] + [job_properties] + FROM data_source + [data_source_properties] + + 1. [db.]job_name + + 导入作业的名称，在同一个 database 内，相同名称只能有一个 job 在运行。 + + 2. tbl_name + + 指定需要导入的表的名称。 + + 3. load_properties + + 用于描述导入数据。语法： + + [column_separator], + [columns_mapping], + [where_predicates], + [partitions] + + 1. column_separator: + + 指定列分隔符，如： + + COLUMNS TERMINATED BY "," + + 默认为：\t + + 2. columns_mapping: + + 指定源数据中列的映射关系，以及定义衍生列的生成方式。 + + 1. 映射列： + + 按顺序指定，源数据中各个列，对应目的表中的哪些列。对于希望跳过的列，可以指定一个不存在的列名。 + 假设目的表有三列 k1, k2, v1。源数据有4列，其中第1、2、4列分别对应 k2, k1, v1。则书写如下： + + COLUMNS (k2, k1, xxx, v1) + + 其中 xxx 为不存在的一列，用于跳过源数据中的第三列。 + + 2. 衍生列： + + 以 col_name = expr 的形式表示的列，我们称为衍生列。即支持通过 expr 计算得出目的表中对应列的值。 + 衍生列通常排列在映射列之后，虽然这不是强制的规定，但是 Doris 总是先解析映射列，再解析衍生列。 + 接上一个示例，假设目的表还有第4列 v2，v2 由 k1 和 k2 的和产生。则可以书写如下： + + COLUMNS (k2, k1, xxx, v1, v2 = k1 + k2); + + 3. where_predicates + + 用于指定过滤条件，以过滤掉不需要的列。过滤列可以是映射列或衍生列。 + 例如我们只希望导入 k1 大于 100 并且 k2 等于 1000 的列，则书写如下： + + WHERE k1 > 100 and k2 = 1000 + + 4. partitions + + 指定导入目的表的哪些 partition 中。如果不指定，则会自动导入到对应的 partition 中。 + 示例： + + PARTITION(p1, p2, p3) + + 4. job_properties + + 用于指定例行导入作业的通用参数。 + 语法： + + PROPERTIES ( + "key1" = "val1", + "key2" = "val2" + ) + + 目前我们支持以下参数： + + 1. desired_concurrent_number + + 期望的并发度。一个例行导入作业会被分成多个子任务执行。这个参数指定一个作业最多有多少任务可以同时执行。必须大于0。默认为3。 + 这个并发度并不是实际的并发度，实际的并发度，会通过集群的节点数、负载情况，以及数据源的情况综合考虑。 + 例： + + "desired_concurrent_number" = "3" + + 2. max_batch_interval/max_batch_rows/max_batch_size + + 这三个参数分别表示： + 1）每个子任务最大执行时间，单位是秒。范围为 5 到 60。默认为10。 + 2）每个子任务最多读取的行数。必须大于等于200000。默认是200000。 + 3）每个子任务最多读取的字节数。单位是字节，范围是 100MB 到 1GB。默认是 100MB。 + + 这三个参数，用于控制一个子任务的执行时间和处理量。当任意一个达到阈值，则任务结束。 + 例： + + "max_batch_interval" = "20", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200" + + 3. max_error_number + + 采样窗口内，允许的最大错误行数。必须大于等于0。默认是 0，即不允许有错误行。 + 采样窗口为 max_batch_rows * 10。即如果在采样窗口内，错误行数大于 max_error_number，则会导致例行作业被暂停，需要人工介入检查数据质量问题。 + 被 where 条件过滤掉的行不算错误行。 + + 4. strict_mode + + 是否开启严格模式，默认为开启。如果开启后，非空原始数据的列类型变换如果结果为 NULL，则会被过滤。指定方式为 "strict_mode" = "true" + + 5. timezone + + 指定导入作业所使用的时区。默认为使用 Session 的 timezone 参数。该参数会影响所有导入涉及的和时区有关的函数结果。 + + 5. data_source + + 数据源的类型。当前支持： + + KAFKA + + 6. data_source_properties + + 指定数据源相关的信息。 + 语法： + + ( + "key1" = "val1", + "key2" = "val2" + ) + + 1. KAFKA 数据源 + + 1. kafka_broker_list + + Kafka 的 broker 连接信息。格式为 ip:host。多个broker之间以逗号分隔。 + 示例： + + "kafka_broker_list" = "broker1:9092,broker2:9092" + + 2. kafka_topic + + 指定要订阅的 Kafka 的 topic。 + 示例： + + "kafka_topic" = "my_topic" + + 3. kafka_partitions/kafka_offsets + + 指定需要订阅的 kafka partition，以及对应的每个 partition 的起始 offset。 + + offset 可以指定从大于等于 0 的具体 offset，或者： + 1) OFFSET_BEGINNING: 从有数据的位置开始订阅。 + 2) OFFSET_END: 从末尾开始订阅。 + + 如果没有指定，则默认从 OFFSET_END 开始订阅 topic 下的所有 partition。 + 示例： + + "kafka_partitions" = "0,1,2,3", + "kafka_offsets" = "101,0,OFFSET_BEGINNING,OFFSET_END" + + 4. property + + 指定自定义kafka参数。 + 功能等同于kafka shell中 "--property" 参数。 + 当参数的 value 为一个文件时，需要在 value 前加上关键词："FILE:"。 + 关于如何创建文件，请参阅 "HELP CREATE FILE;" + 更多支持的自定义参数，请参阅 librdkafka 的官方 CONFIGURATION 文档中，client 端的配置项。 + + 示例: + "property.client.id" = "12345", + "property.ssl.ca.location" = "FILE:ca.pem" + + 1.使用 SSL 连接 Kafka 时，需要指定以下参数： + + "property.security.protocol" = "ssl", + "property.ssl.ca.location" = "FILE:ca.pem", + "property.ssl.certificate.location" = "FILE:client.pem", + "property.ssl.key.location" = "FILE:client.key", + "property.ssl.key.password" = "abcdefg" + + 其中： + "property.security.protocol" 和 "property.ssl.ca.location" 为必须，用于指明连接方式为 SSL，以及 CA 证书的位置。 + + 如果 Kafka server 端开启了 client 认证，则还需设置： + + "property.ssl.certificate.location" + "property.ssl.key.location" + "property.ssl.key.password" + + 分别用于指定 client 的 public key，private key 以及 private key 的密码。 + + + 2.指定kafka partition的默认起始offset + 如果没有指定kafka_partitions/kafka_offsets,默认消费所有分区,此时可以指定kafka_default_offsets指定起始 offset。默认为 OFFSET_END，即从末尾开始订阅。 + 值为 + 1) OFFSET_BEGINNING: 从有数据的位置开始订阅。 + 2) OFFSET_END: 从末尾开始订阅。 + 示例： + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + + 7. 导入数据格式样例 + + 整型类（TINYINT/SMALLINT/INT/BIGINT/LARGEINT）：1, 1000, 1234 + 浮点类（FLOAT/DOUBLE/DECIMAL）：1.1, 0.23, .356 + 日期类（DATE/DATETIME）：2017-10-03, 2017-06-13 12:34:03。 + 字符串类（CHAR/VARCHAR）（无引号）：I am a student, a + NULL值：\N + +## example + 1. 为 example_db 的 example_tbl 创建一个名为 test1 的 Kafka 例行导入任务。指定列分隔符和 group.id 和 client.id，并且自动默认消费所有分区，且从有数据的位置（OFFSET_BEGINNING）开始订阅 + + CREATE ROUTINE LOAD example_db.test1 ON example_tbl + COLUMNS TERMINATED BY ",", + COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100) + PROPERTIES + ( + "desired_concurrent_number"="3", + "max_batch_interval" = "20", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200", + "strict_mode" = "false" + ) + FROM KAFKA + ( + "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", + "kafka_topic" = "my_topic", + "property.group.id" = "xxx", + "property.client.id" = "xxx", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + + 2. 为 example_db 的 example_tbl 创建一个名为 test1 的 Kafka 例行导入任务。导入任务为严格模式。 + + CREATE ROUTINE LOAD example_db.test1 ON example_tbl + COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), + WHERE k1 > 100 and k2 like "%doris%" + PROPERTIES + ( + "desired_concurrent_number"="3", + "max_batch_interval" = "20", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200", + "strict_mode" = "false" + ) + FROM KAFKA + ( + "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", + "kafka_topic" = "my_topic", + "kafka_partitions" = "0,1,2,3", + "kafka_offsets" = "101,0,0,200" + ); + + 3. 通过 SSL 认证方式，从 Kafka 集群导入数据。同时设置 client.id 参数。导入任务为非严格模式，时区为 Africa/Abidjan + + CREATE ROUTINE LOAD example_db.test1 ON example_tbl + COLUMNS(k1, k2, k3, v1, v2, v3 = k1 * 100), + WHERE k1 > 100 and k2 like "%doris%" + PROPERTIES + ( + "desired_concurrent_number"="3", + "max_batch_interval" = "20", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200", + "strict_mode" = "false", + "timezone" = "Africa/Abidjan" + ) + FROM KAFKA + ( + "kafka_broker_list" = "broker1:9092,broker2:9092,broker3:9092", + "kafka_topic" = "my_topic", + "property.security.protocol" = "ssl", + "property.ssl.ca.location" = "FILE:ca.pem", + "property.ssl.certificate.location" = "FILE:client.pem", + "property.ssl.key.location" = "FILE:client.key", + "property.ssl.key.password" = "abcdefg", + "property.client.id" = "my_client_id" + ); + +## keyword + + CREATE,ROUTINE,LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md new file mode 100644 index 00000000000000..4e094eaf4a7741 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ALTER.md @@ -0,0 +1,55 @@ +--- +{ + "title": "SHOW ALTER", + "language": "zh-CN" +} +--- + + + +# SHOW ALTER +## description + 该语句用于展示当前正在进行的各类修改任务的执行情况 + 语法： + SHOW ALTER [CLUSTER | TABLE [COLUMN | ROLLUP] [FROM db_name]]; + + 说明： + TABLE COLUMN：展示修改列的 ALTER 任务 + 支持语法[WHERE TableName|CreateTime|FinishTime|State] [ORDER BY] [LIMIT] + TABLE ROLLUP：展示创建或删除 ROLLUP index 的任务 + 如果不指定 db_name，使用当前默认 db + CLUSTER: 展示集群操作相关任务情况（仅管理员使用！待实现...） + +## example + 1. 展示默认 db 的所有修改列的任务执行情况 + SHOW ALTER TABLE COLUMN; + + 2. 展示某个表最近一次修改列的任务执行情况 + SHOW ALTER TABLE COLUMN WHERE TableName = "table1" ORDER BY CreateTime DESC LIMIT 1; + + 3. 展示指定 db 的创建或删除 ROLLUP index 的任务执行情况 + SHOW ALTER TABLE ROLLUP FROM example_db; + + 4. 展示集群操作相关任务（仅管理员使用！待实现...） + SHOW ALTER CLUSTER; + +## keyword + SHOW,ALTER + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md new file mode 100644 index 00000000000000..f636b71512dbaf --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW BACKUP.md @@ -0,0 +1,63 @@ +--- +{ + "title": "SHOW BACKUP", + "language": "zh-CN" +} +--- + + + +# SHOW BACKUP +## description + 该语句用于查看 BACKUP 任务 + 语法： + SHOW BACKUP [FROM db_name] + + 说明： + 1. Palo 中仅保存最近一次 BACKUP 任务。 + 2. 各列含义如下： + JobId： 唯一作业id + SnapshotName： 备份的名称 + DbName： 所属数据库 + State： 当前阶段 + PENDING： 提交作业后的初始状态 + SNAPSHOTING： 执行快照中 + UPLOAD_SNAPSHOT：快照完成，准备上传 + UPLOADING： 快照上传中 + SAVE_META： 将作业元信息保存为本地文件 + UPLOAD_INFO： 上传作业元信息 + FINISHED： 作业成功 + CANCELLED： 作业失败 + BackupObjs： 备份的表和分区 + CreateTime： 任务提交时间 + SnapshotFinishedTime： 快照完成时间 + UploadFinishedTime： 快照上传完成时间 + FinishedTime： 作业结束时间 + UnfinishedTasks： 在 SNAPSHOTING 和 UPLOADING 阶段会显示还未完成的子任务id + Status： 如果作业失败，显示失败信息 + Timeout： 作业超时时间，单位秒 + +## example + 1. 查看 example_db 下最后一次 BACKUP 任务。 + SHOW BACKUP FROM example_db; + +## keyword + SHOW, BACKUP + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md new file mode 100644 index 00000000000000..494f980742581a --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATA.md @@ -0,0 +1,47 @@ +--- +{ + "title": "SHOW DATA", + "language": "zh-CN" +} +--- + + + +# SHOW DATA +## description + 该语句用于展示数据量和副本数量 + 语法： + SHOW DATA [FROM db_name[.table_name]]; + + 说明： + 1. 如果不指定 FROM 子句，使用展示当前 db 下细分到各个 table 的数据量和副本数量 + 2. 如果指定 FROM 子句，则展示 table 下细分到各个 index 的数据量和副本数量 + 3. 如果想查看各个 Partition 的大小，请参阅 help show partitions + +## example + 1. 展示默认 db 的各个 table 的数据量，副本数量，汇总数据量和汇总副本数量。 + SHOW DATA; + + 2. 展示指定 db 的下指定表的细分数据量和副本数量 + SHOW DATA FROM example_db.table_name; + +## keyword + SHOW,DATA + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md new file mode 100644 index 00000000000000..203bfda855abf5 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASES.md @@ -0,0 +1,35 @@ +--- +{ + "title": "SHOW DATABASES", + "language": "zh-CN" +} +--- + + + +# SHOW DATABASES +## description + 该语句用于展示当前可见的 db + 语法： + SHOW DATABASES; + +## keyword + SHOW,DATABASES + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md new file mode 100644 index 00000000000000..b6a653f0f349c9 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DELETE.md @@ -0,0 +1,39 @@ +--- +{ + "title": "SHOW DELETE", + "language": "zh-CN" +} +--- + + + +# SHOW DELETE +## description + 该语句用于展示已执行成功的历史 delete 任务 + 语法： + SHOW DELETE [FROM db_name] + +## example + 1. 展示数据库 database 的所有历史 delete 任务 + SHOW DELETE FROM database; + +## keyword + SHOW,DELETE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md new file mode 100644 index 00000000000000..eac7a2506002a2 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES.md @@ -0,0 +1,39 @@ +--- +{ + "title": "SHOW DYNAMIC PARTITION TABLES", + "language": "zh-CN" +} +--- + + + +# SHOW DYNAMIC PARTITION TABLES +## description + 该语句用于展示当前db下所有的动态分区表状态 + 语法： + SHOW DYNAMIC PARTITION TABLES [FROM db_name]; + +## example + 1. 展示数据库 database 的所有动态分区表状态 + SHOW DYNAMIC PARTITION TABLES FROM database; + +## keyword + SHOW,DYNAMIC,PARTITION,TABLES + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md new file mode 100644 index 00000000000000..f699d8a65b36f7 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW EXPORT.md @@ -0,0 +1,62 @@ +--- +{ + "title": "SHOW EXPORT", + "language": "zh-CN" +} +--- + + + +# SHOW EXPORT +## description + 该语句用于展示指定的导出任务的执行情况 + 语法： + SHOW EXPORT + [FROM db_name] + [ + WHERE + [EXPORT_JOB_ID = your_job_id] + [STATE = ["PENDING"|"EXPORTING"|"FINISHED"|"CANCELLED"]] + ] + [ORDER BY ...] + [LIMIT limit]; + + 说明： + 1) 如果不指定 db_name，使用当前默认db + 2) 如果指定了 STATE，则匹配 EXPORT 状态 + 3) 可以使用 ORDER BY 对任意列组合进行排序 + 4) 如果指定了 LIMIT，则显示 limit 条匹配记录。否则全部显示 + +## example + 1. 展示默认 db 的所有导出任务 + SHOW EXPORT; + + 2. 展示指定 db 的导出任务，按 StartTime 降序排序 + SHOW EXPORT FROM example_db ORDER BY StartTime DESC; + + 3. 展示指定 db 的导出任务，state 为 "exporting", 并按 StartTime 降序排序 + SHOW EXPORT FROM example_db WHERE STATE = "exporting" ORDER BY StartTime DESC; + + 4. 展示指定db，指定job_id的导出任务 + SHOW EXPORT FROM example_db WHERE EXPORT_JOB_ID = job_id; + +## keyword + SHOW,EXPORT + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md new file mode 100644 index 00000000000000..cbdced1ccbc15a --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW LOAD.md @@ -0,0 +1,75 @@ +--- +{ + "title": "SHOW LOAD", + "language": "zh-CN" +} +--- + + + +# SHOW LOAD +## description + 该语句用于展示指定的导入任务的执行情况 + 语法： + SHOW LOAD + [FROM db_name] + [ + WHERE + [LABEL [ = "your_label" | LIKE "label_matcher"]] + [STATE = ["PENDING"|"ETL"|"LOADING"|"FINISHED"|"CANCELLED"|]] + ] + [ORDER BY ...] + [LIMIT limit][OFFSET offset]; + + 说明： + 1) 如果不指定 db_name，使用当前默认db + 2) 如果使用 LABEL LIKE，则会匹配导入任务的 label 包含 label_matcher 的导入任务 + 3) 如果使用 LABEL = ，则精确匹配指定的 label + 4) 如果指定了 STATE，则匹配 LOAD 状态 + 5) 可以使用 ORDER BY 对任意列组合进行排序 + 6) 如果指定了 LIMIT，则显示 limit 条匹配记录。否则全部显示 + 7) 如果指定了 OFFSET，则从偏移量offset开始显示查询结果。默认情况下偏移量为0。 + 8) 如果是使用 broker/mini load，则 URL 列中的连接可以使用以下命令查看： + + SHOW LOAD WARNINGS ON 'url' + +## example + 1. 展示默认 db 的所有导入任务 + SHOW LOAD; + + 2. 展示指定 db 的导入任务，label 中包含字符串 "2014_01_02"，展示最老的10个 + SHOW LOAD FROM example_db WHERE LABEL LIKE "2014_01_02" LIMIT 10; + + 3. 展示指定 db 的导入任务，指定 label 为 "load_example_db_20140102" 并按 LoadStartTime 降序排序 + SHOW LOAD FROM example_db WHERE LABEL = "load_example_db_20140102" ORDER BY LoadStartTime DESC; + + 4. 展示指定 db 的导入任务，指定 label 为 "load_example_db_20140102" ，state 为 "loading", 并按 LoadStartTime 降序排序 + SHOW LOAD FROM example_db WHERE LABEL = "load_example_db_20140102" AND STATE = "loading" ORDER BY LoadStartTime DESC; + + 5. 展示指定 db 的导入任务 并按 LoadStartTime 降序排序,并从偏移量5开始显示10条查询结果 + SHOW LOAD FROM example_db ORDER BY LoadStartTime DESC limit 5,10; + SHOW LOAD FROM example_db ORDER BY LoadStartTime DESC limit 10 offset 5; + + 6. 小批量导入是查看导入状态的命令 + curl --location-trusted -u {user}:{passwd} http://{hostname}:{port}/api/{database}/_load_info?label={labelname} + +## keyword + SHOW,LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md new file mode 100644 index 00000000000000..96f65714d7e9ad --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PARTITIONS.md @@ -0,0 +1,45 @@ +--- +{ + "title": "SHOW PARTITIONS", + "language": "zh-CN" +} +--- + + + +# SHOW PARTITIONS +## description + 该语句用于展示分区信息 + 语法： + SHOW PARTITIONS FROM [db_name.]table_name [WHERE] [ORDER BY] [LIMIT]; + 说明: + 支持PartitionId,PartitionName,State,Buckets,ReplicationNum,LastConsistencyCheckTime等列的过滤 + +## example + 1.展示指定db下指定表的所有分区信息 + SHOW PARTITIONS FROM example_db.table_name; + + 2.展示指定db下指定表的指定分区的信息 + SHOW PARTITIONS FROM example_db.table_name WHERE PartitionName = "p1"; + + 3.展示指定db下指定表的最新分区的信息 + SHOW PARTITIONS FROM example_db.table_name ORDER BY PartitionId DESC LIMIT 1; +## keyword + SHOW,PARTITIONS diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md new file mode 100644 index 00000000000000..479e2b01fdf359 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PROPERTY.md @@ -0,0 +1,42 @@ +--- +{ + "title": "SHOW PROPERTY", + "language": "zh-CN" +} +--- + + + +# SHOW PROPERTY +## description + 该语句用于查看用户的属性 + 语法： + SHOW PROPERTY [FOR user] [LIKE key] + +## example + 1. 查看 jack 用户的属性 + SHOW PROPERTY FOR 'jack' + + 2. 查看 jack 用户导入cluster相关属性 + SHOW PROPERTY FOR 'jack' LIKE '%load_cluster%' + +## keyword + SHOW, PROPERTY + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md new file mode 100644 index 00000000000000..2d49ff89ff73a9 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW REPOSITORIES.md @@ -0,0 +1,49 @@ +--- +{ + "title": "SHOW REPOSITORIES", + "language": "zh-CN" +} +--- + + + +# SHOW REPOSITORIES +## description + 该语句用于查看当前已创建的仓库。 + 语法： + SHOW REPOSITORIES; + + 说明： + 1. 各列含义如下： + RepoId： 唯一的仓库ID + RepoName： 仓库名称 + CreateTime： 第一次创建该仓库的时间 + IsReadOnly： 是否为只读仓库 + Location： 仓库中用于备份数据的根目录 + Broker： 依赖的 Broker + ErrMsg： Palo 会定期检查仓库的连通性，如果出现问题，这里会显示错误信息 + +## example + 1. 查看已创建的仓库： + SHOW REPOSITORIES; + +## keyword + SHOW, REPOSITORY, REPOSITORIES + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md new file mode 100644 index 00000000000000..5be532f27c559a --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW RESTORE.md @@ -0,0 +1,67 @@ +--- +{ + "title": "SHOW RESTORE", + "language": "zh-CN" +} +--- + + + +# SHOW RESTORE +## description + 该语句用于查看 RESTORE 任务 + 语法： + SHOW RESTORE [FROM db_name] + + 说明： + 1. Palo 中仅保存最近一次 RESTORE 任务。 + 2. 各列含义如下： + JobId： 唯一作业id + Label： 要恢复的备份的名称 + Timestamp： 要恢复的备份的时间版本 + DbName： 所属数据库 + State： 当前阶段 + PENDING： 提交作业后的初始状态 + SNAPSHOTING： 执行快照中 + DOWNLOAD： 快照完成，准备下载仓库中的快照 + DOWNLOADING： 快照下载中 + COMMIT： 快照下载完成，准备生效 + COMMITING： 生效中 + FINISHED： 作业成功 + CANCELLED： 作业失败 + AllowLoad： 恢复时是否允许导入（当前不支持） + ReplicationNum： 指定恢复的副本数 + RestoreJobs： 要恢复的表和分区 + CreateTime： 任务提交时间 + MetaPreparedTime： 元数据准备完成时间 + SnapshotFinishedTime： 快照完成时间 + DownloadFinishedTime： 快照下载完成时间 + FinishedTime： 作业结束时间 + UnfinishedTasks： 在 SNAPSHOTING、DOWNLOADING 和 COMMITING 阶段会显示还未完成的子任务id + Status： 如果作业失败，显示失败信息 + Timeout： 作业超时时间，单位秒 + +## example + 1. 查看 example_db 下最近一次 RESTORE 任务。 + SHOW RESTORE FROM example_db; + +## keyword + SHOW, RESTORE + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md new file mode 100644 index 00000000000000..2e5c7623549b44 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD TASK.md @@ -0,0 +1,35 @@ +--- +{ + "title": "SHOW ROUTINE LOAD TASK", + "language": "zh-CN" +} +--- + + + +# SHOW ROUTINE LOAD TASK +## example + +1. 展示名为 test1 的例行导入任务的子任务信息。 + + SHOW ROUTINE LOAD TASK WHERE JobName = "test1"; + +## keyword + SHOW,ROUTINE,LOAD,TASK diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md new file mode 100644 index 00000000000000..43d6a2a323e741 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW ROUTINE LOAD.md @@ -0,0 +1,58 @@ +--- +{ + "title": "SHOW ROUTINE LOAD", + "language": "zh-CN" +} +--- + + + +# SHOW ROUTINE LOAD +## example + +1. 展示名称为 test1 的所有例行导入作业（包括已停止或取消的作业）。结果为一行或多行。 + + SHOW ALL ROUTINE LOAD FOR test1; + +2. 展示名称为 test1 的当前正在运行的例行导入作业 + + SHOW ROUTINE LOAD FOR test1; + +3. 显示 example_db 下，所有的例行导入作业（包括已停止或取消的作业）。结果为一行或多行。 + + use example_db; + SHOW ALL ROUTINE LOAD; + +4. 显示 example_db 下，所有正在运行的例行导入作业 + + use example_db; + SHOW ROUTINE LOAD; + +5. 显示 example_db 下，名称为 test1 的当前正在运行的例行导入作业 + + SHOW ROUTINE LOAD FOR example_db.test1; + +6. 显示 example_db 下，名称为 test1 的所有例行导入作业（包括已停止或取消的作业）。结果为一行或多行。 + + SHOW ALL ROUTINE LOAD FOR example_db.test1; + +## keyword + SHOW,ROUTINE,LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md new file mode 100644 index 00000000000000..c1075da902ea0b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW SNAPSHOT.md @@ -0,0 +1,57 @@ +--- +{ + "title": "SHOW SNAPSHOT", + "language": "zh-CN" +} +--- + + + +# SHOW SNAPSHOT +## description + 该语句用于查看仓库中已存在的备份。 + 语法： + SHOW SNAPSHOT ON `repo_name` + [WHERE SNAPSHOT = "snapshot" [AND TIMESTAMP = "backup_timestamp"]]; + + 说明： + 1. 各列含义如下： + Snapshot： 备份的名称 + Timestamp： 对应备份的时间版本 + Status： 如果备份正常，则显示 OK，否则显示错误信息 + + 2. 如果指定了 TIMESTAMP，则会额外显示如下信息： + Database： 备份数据原属的数据库名称 + Details： 以 Json 的形式，展示整个备份的数据目录及文件结构 + +## example + 1. 查看仓库 example_repo 中已有的备份： + SHOW SNAPSHOT ON example_repo; + + 2. 仅查看仓库 example_repo 中名称为 backup1 的备份： + SHOW SNAPSHOT ON example_repo WHERE SNAPSHOT = "backup1"; + + 2. 查看仓库 example_repo 中名称为 backup1 的备份，时间版本为 "2018-05-05-15-34-26" 的详细信息： + SHOW SNAPSHOT ON example_repo + WHERE SNAPSHOT = "backup1" AND TIMESTAMP = "2018-05-05-15-34-26"; + +## keyword + SHOW, SNAPSHOT + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md new file mode 100644 index 00000000000000..44e396b9df865f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TABLES.md @@ -0,0 +1,35 @@ +--- +{ + "title": "SHOW TABLES", + "language": "zh-CN" +} +--- + + + +# SHOW TABLES +## description + 该语句用于展示当前 db 下所有的 table + 语法： + SHOW TABLES; + +## keyword + SHOW,TABLES + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md new file mode 100644 index 00000000000000..fc6fb7768415b4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TABLET.md @@ -0,0 +1,67 @@ +--- +{ + "title": "SHOW TABLET", + "language": "zh-CN" +} +--- + + + +# SHOW TABLET +## description + 该语句用于显示 tablet 相关的信息（仅管理员使用） + 语法： + SHOW TABLET + [FROM [db_name.]table_name | tablet_id] [partiton(partition_name_1, partition_name_1)] + [where [version=1] [and backendid=10000] [and state="NORMAL|ROLLUP|CLONE|DECOMMISSION"]] + [order by order_column] + [limit [offset,]size] + + 现在show tablet命令支持按照按照以下字段进行过滤：partition, index name, version, backendid, + state，同时支持按照任意字段进行排序，并且提供limit限制返回条数。 + +## example + 1. 显示指定 db 的下指定表所有 tablet 信息 + SHOW TABLET FROM example_db.table_name; + + // 获取partition p1和p2的tablet信息 + SHOW TABLET FROM example_db.table_name partition(p1, p2); + + // 获取10个结果 + SHOW TABLET FROM example_db.table_name limit 10; + + // 从偏移5开始获取10个结果 + SHOW TABLET FROM example_db.table_name limit 5,10; + + // 按照backendid/version/state字段进行过滤 + SHOW TABLET FROM example_db.table_name where backendid=10000 and version=1 and state="NORMAL"; + + // 按照version字段进行排序 + SHOW TABLET FROM example_db.table_name where backendid=10000 order by version; + + // 获取index名字为t1_rollup的tablet相关信息 + SHOW TABLET FROM example_db.table_name where indexname="t1_rollup"; + + 2. 显示指定 tablet id 为 10000 的 tablet 的父层级 id 信息 + SHOW TABLET 10000; + +## keyword + SHOW,TABLET,LIMIT + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md new file mode 100644 index 00000000000000..ad9dabf55f356f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW TRANSACTION.md @@ -0,0 +1,87 @@ +--- +{ + "title": "SHOW TRANSACTION", + "language": "zh-CN" +} +--- + + + +# SHOW TRANSACTION +## description + +该语法用于查看指定 transaction id 的事务详情。 + +语法： + +``` +SHOW TRANSACTION +[FROM db_name] +WHERE id = transaction_id; +``` + +返回结果示例： + +``` + TransactionId: 4005 + Label: insert_8d807d5d-bcdd-46eb-be6d-3fa87aa4952d + Coordinator: FE: 10.74.167.16 + TransactionStatus: VISIBLE + LoadJobSourceType: INSERT_STREAMING + PrepareTime: 2020-01-09 14:59:07 + CommitTime: 2020-01-09 14:59:09 + FinishTime: 2020-01-09 14:59:09 + Reason: +ErrorReplicasCount: 0 + ListenerId: -1 + TimeoutMs: 300000 +``` + +* TransactionId：事务id +* Label：导入任务对应的 label +* Coordinator：负责事务协调的节点 +* TransactionStatus：事务状态 + * PREPARE：准备阶段 + * COMMITTED：事务成功，但数据不可见 + * VISIBLE：事务成功且数据可见 + * ABORTED：事务失败 +* LoadJobSourceType：导入任务的类型。 +* PrepareTime：事务开始时间 +* CommitTime：事务提交成功的时间 +* FinishTime：数据可见的时间 +* Reason：错误信息 +* ErrorReplicasCount：有错误的副本数 +* ListenerId：相关的导入作业的id +* TimeoutMs：事务超时时间，单位毫秒 + +## example + +1. 查看 id 为 4005 的事务： + + SHOW TRANSACTION WHERE ID=4005; + +2. 指定 db 中，查看 id 为 4005 的事务： + + SHOW TRANSACTION FROM db WHERE ID=4005; + +## keyword + + SHOW, TRANSACTION + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md new file mode 100644 index 00000000000000..12cdd58824c886 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STOP ROUTINE LOAD.md @@ -0,0 +1,36 @@ +--- +{ + "title": "STOP ROUTINE LOAD", + "language": "zh-CN" +} +--- + + + +# STOP ROUTINE LOAD +## example + +1. 停止名称为 test1 的例行导入作业。 + + STOP ROUTINE LOAD FOR test1; + +## keyword + STOP,ROUTINE,LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md new file mode 100644 index 00000000000000..01a306323f5b8d --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STREAM LOAD.md @@ -0,0 +1,133 @@ +--- +{ + "title": "STREAM LOAD", + "language": "zh-CN" +} +--- + + + +# STREAM LOAD +## description + NAME: + stream-load: load data to table in streaming + + SYNOPSIS + curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_host:http_port/api/{db}/{table}/_stream_load + + DESCRIPTION + 该语句用于向指定的 table 导入数据，与普通Load区别是，这种导入方式是同步导入。 + 这种导入方式仍然能够保证一批导入任务的原子性，要么全部数据导入成功，要么全部失败。 + 该操作会同时更新和此 base table 相关的 rollup table 的数据。 + 这是一个同步操作，整个数据导入工作完成后返回给用户导入结果。 + 当前支持HTTP chunked与非chunked上传两种方式，对于非chunked方式，必须要有Content-Length来标示上传内容长度，这样能够保证数据的完整性。 + 另外，用户最好设置Expect Header字段内容100-continue，这样可以在某些出错场景下避免不必要的数据传输。 + + OPTIONS + 用户可以通过HTTP的Header部分来传入导入参数 + + label: 一次导入的标签，相同标签的数据无法多次导入。用户可以通过指定Label的方式来避免一份数据重复导入的问题。 + 当前Palo内部保留30分钟内最近成功的label。 + + column_separator：用于指定导入文件中的列分隔符，默认为\t。如果是不可见字符，则需要加\x作为前缀，使用十六进制来表示分隔符。 + 如hive文件的分隔符\x01，需要指定为-H "column_separator:\x01" + + columns：用于指定导入文件中的列和 table 中的列的对应关系。如果源文件中的列正好对应表中的内容，那么是不需要指定这个字段的内容的。 + 如果源文件与表schema不对应，那么需要这个字段进行一些数据转换。这里有两种形式column，一种是直接对应导入文件中的字段，直接使用字段名表示； + 一种是衍生列，语法为 `column_name` = expression。举几个例子帮助理解。 + 例1: 表中有3个列“c1, c2, c3”，源文件中的三个列一次对应的是"c3,c2,c1"; 那么需要指定-H "columns: c3, c2, c1" + 例2: 表中有3个列“c1, c2, c3", 源文件中前三列依次对应，但是有多余1列；那么需要指定-H "columns: c1, c2, c3, xxx"; + 最后一个列随意指定个名称占位即可 + 例3: 表中有3个列“year, month, day"三个列，源文件中只有一个时间列，为”2018-06-01 01:02:03“格式； + 那么可以指定-H "columns: col, year = year(col), month=month(col), day=day(col)"完成导入 + + where: 用于抽取部分数据。用户如果有需要将不需要的数据过滤掉，那么可以通过设定这个选项来达到。 + 例1: 只导入大于k1列等于20180601的数据，那么可以在导入时候指定-H "where: k1 = 20180601" + + max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。数据不规范不包括通过 where 条件过滤掉的行。 + + partitions: 用于指定这次导入所设计的partition。如果用户能够确定数据对应的partition，推荐指定该项。不满足这些分区的数据将被过滤掉。 + 比如指定导入到p1, p2分区，-H "partitions: p1, p2" + + timeout: 指定导入的超时时间。单位秒。默认是 600 秒。可设置范围为 1 秒 ~ 259200 秒。 + + strict_mode: 用户指定此次导入是否开启严格模式，默认为开启。关闭方式为 -H "strict_mode: false"。 + + timezone: 指定本次导入所使用的时区。默认为东八区。该参数会影响所有导入涉及的和时区有关的函数结果。 + + exec_mem_limit: 导入内存限制。默认为 2GB。单位为字节。 + + RETURN VALUES + 导入完成后，会以Json格式返回这次导入的相关内容。当前包括一下字段 + Status: 导入最后的状态。 + Success：表示导入成功，数据已经可见； + Publish Timeout：表述导入作业已经成功Commit，但是由于某种原因并不能立即可见。用户可以视作已经成功不必重试导入 + Label Already Exists: 表明该Label已经被其他作业占用，可能是导入成功，也可能是正在导入。 + 用户需要通过get label state命令来确定后续的操作 + 其他：此次导入失败，用户可以指定Label重试此次作业 + Message: 导入状态详细的说明。失败时会返回具体的失败原因。 + NumberTotalRows: 从数据流中读取到的总行数 + NumberLoadedRows: 此次导入的数据行数，只有在Success时有效 + NumberFilteredRows: 此次导入过滤掉的行数，即数据质量不合格的行数 + NumberUnselectedRows: 此次导入，通过 where 条件被过滤掉的行数 + LoadBytes: 此次导入的源文件数据量大小 + LoadTimeMs: 此次导入所用的时间 + ErrorURL: 被过滤数据的具体内容，仅保留前1000条 + + ERRORS + 可以通过以下语句查看导入错误详细信息： + + SHOW LOAD WARNINGS ON 'url' + + 其中 url 为 ErrorURL 给出的 url。 + +## example + + 1. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重。指定超时时间为 100 秒 + curl --location-trusted -u root -H "label:123" -H "timeout:100" -T testData http://host:port/api/testDb/testTbl/_stream_load + + 2. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重, 并且只导入k1等于20180601的数据 + curl --location-trusted -u root -H "label:123" -H "where: k1=20180601" -T testData http://host:port/api/testDb/testTbl/_stream_load + + 3. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率（用户是defalut_cluster中的） + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -T testData http://host:port/api/testDb/testTbl/_stream_load + + 4. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率，并且指定文件的列名（用户是defalut_cluster中的） + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "columns: k2, k1, v1" -T testData http://host:port/api/testDb/testTbl/_stream_load + + 5. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表中的p1, p2分区, 允许20%的错误率。 + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "partitions: p1, p2" -T testData http://host:port/api/testDb/testTbl/_stream_load + + 6. 使用streaming方式导入（用户是defalut_cluster中的） + seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_stream_load + + 7. 导入含有HLL列的表，可以是表中的列或者数据中的列用于生成HLL列，也可使用hll_empty补充数据中没有的列 + curl --location-trusted -u root -H "columns: k1, k2, v1=hll_hash(k1), v2=hll_empty()" -T testData http://host:port/api/testDb/testTbl/_stream_load + + 8. 导入数据进行严格模式过滤，并设置时区为 Africa/Abidjan + curl --location-trusted -u root -H "strict_mode: true" -H "timezone: Africa/Abidjan" -T testData http://host:port/api/testDb/testTbl/_stream_load + + 9. 导入含有BITMAP列的表，可以是表中的列或者数据中的列用于生成BITMAP列，也可以使用bitmap_empty填充空的Bitmap + curl --location-trusted -u root -H "columns: k1, k2, v1=to_bitmap(k1), v2=bitmap_empty()" -T testData http://host:port/api/testDb/testTbl/_stream_load + + +## keyword + STREAM,LOAD + diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/insert.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/insert.md new file mode 100644 index 00000000000000..f19e1477b42c6c --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/insert.md @@ -0,0 +1,111 @@ +--- +{ + "title": "INSERT", + "language": "zh-CN" +} +--- + + + +# INSERT +## description +### Syntax + +``` +INSERT INTO table_name + [ PARTITION (p1, ...) ] + [ WITH LABEL label] + [ (column [, ...]) ] + [ [ hint [, ...] ] ] + { VALUES ( { expression | DEFAULT } [, ...] ) [, ...] | query } +``` + +### Parameters + +> tablet_name: 导入数据的目的表。可以是 `db_name.table_name` 形式 +> +> partitions: 指定待导入的分区，必须是 `table_name` 中存在的分区，多个分区名称用逗号分隔 +> +> label: 为 Insert 任务指定一个 label +> +> column_name: 指定的目的列，必须是 `table_name` 中存在的列 +> +> expression: 需要赋值给某个列的对应表达式 +> +> DEFAULT: 让对应列使用默认值 +> +> query: 一个普通查询，查询的结果会写入到目标中 +> +> hint: 用于指示 `INSERT` 执行行为的一些指示符。`streaming` 和 默认的非 `streaming` 方式均会使用同步方式完成 `INSERT` 语句执行 +> 非 `streaming` 方式在执行完成后会返回一个 label 方便用户通过 `SHOW LOAD` 查询导入的状态 + +### Note + +当前执行 `INSERT` 语句时，对于有不符合目标表格式的数据，默认的行为是过滤，比如字符串超长等。但是对于有要求数据不能够被过滤的业务场景，可以通过设置会话变量 `enable_insert_strict` 为 `true` 来确保当有数据被过滤掉的时候，`INSERT` 不会被执行成功。 + +## example + +`test` 表包含两个列`c1`, `c2`。 + +1. 向`test`表中导入一行数据 + +``` +INSERT INTO test VALUES (1, 2); +INSERT INTO test (c1, c2) VALUES (1, 2); +INSERT INTO test (c1, c2) VALUES (1, DEFAULT); +INSERT INTO test (c1) VALUES (1); +``` + +其中第一条、第二条语句是一样的效果。在不指定目标列时，使用表中的列顺序来作为默认的目标列。 +第三条、第四条语句表达的意思是一样的，使用`c2`列的默认值，来完成数据导入。 + +2. 向`test`表中一次性导入多行数据 + +``` +INSERT INTO test VALUES (1, 2), (3, 2 + 2); +INSERT INTO test (c1, c2) VALUES (1, 2), (3, 2 * 2); +INSERT INTO test (c1) VALUES (1), (3); +INSERT INTO test (c1, c2) VALUES (1, DEFAULT), (3, DEFAULT); +``` + +其中第一条、第二条语句效果一样，向`test`表中一次性导入两条数据 +第三条、第四条语句效果已知，使用`c2`列的默认值向`test`表中导入两条数据 + +3. 向 `test` 表中导入一个查询语句结果 + +``` +INSERT INTO test SELECT * FROM test2; +INSERT INTO test (c1, c2) SELECT * from test2; +``` + +4. 向 `test` 表中导入一个查询语句结果，并指定 partition 和 label + +``` +INSERT INTO test PARTITION(p1, p2) WITH LABEL `label1` SELECT * FROM test2; +INSERT INTO test WITH LABEL `label1` (c1, c2) SELECT * from test2; +``` + +异步的导入其实是，一个同步的导入封装成了异步。填写 streaming 和不填写的**执行效率是一样**的。 + +由于Doris之前的导入方式都是异步导入方式，为了兼容旧有的使用习惯，不加 streaming 的 `INSERT` 语句依旧会返回一个 label，用户需要通过`SHOW LOAD`命令查看此`label`导入作业的状态。 + +## keyword + + INSERT diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/BIGINT.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/BIGINT.md new file mode 100644 index 00000000000000..4af2a2ea2e2c5a --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/BIGINT.md @@ -0,0 +1,34 @@ +--- +{ + "title": "BIGINT", + "language": "zh-CN" +} +--- + + + +# BIGINT +## description + BIGINT + 8字节有符号整数，范围[-9223372036854775808, 9223372036854775807] + +## keyword + + BIGINT diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/BOOLEAN.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/BOOLEAN.md new file mode 100644 index 00000000000000..e3e3a724f8f80b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/BOOLEAN.md @@ -0,0 +1,34 @@ +--- +{ + "title": "BOOLEAN", + "language": "zh-CN" +} +--- + + + +# BOOLEAN +## description + BOOL, BOOLEAN + 与TINYINT一样，0代表false，1代表true + +## keyword + + BOOLEAN diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/CHAR.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/CHAR.md new file mode 100644 index 00000000000000..fa204f4c1bc27f --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/CHAR.md @@ -0,0 +1,34 @@ +--- +{ + "title": "CHAR", + "language": "zh-CN" +} +--- + + + +# CHAR +## description + CHAR(M) + 定长字符串，M代表的是定长字符串的长度。M的范围是1-255 + +## keyword + + CHAR diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/DATE.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/DATE.md new file mode 100644 index 00000000000000..e832cb1244dd1c --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/DATE.md @@ -0,0 +1,42 @@ +--- +{ + "title": "DATE", + "language": "zh-CN" +} +--- + + + +# DATE +## description + DATE函数 + Syntax: + DATE(expr) + 将输入的类型转化为DATE类型 + DATE类型 + 日期类型，目前的取值范围是['0000-01-01', '9999-12-31'], 默认的打印形式是'YYYY-MM-DD' + +## example + mysql> SELECT DATE('2003-12-31 01:02:03'); + -> '2003-12-31' + +## keyword + + DATE diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/DATETIME.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/DATETIME.md new file mode 100644 index 00000000000000..9872f5f20aa7e2 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/DATETIME.md @@ -0,0 +1,35 @@ +--- +{ + "title": "DATETIME", + "language": "zh-CN" +} +--- + + + +# DATETIME +## description + DATETIME + 日期时间类型，取值范围是['0000-01-01 00:00:00', '9999-12-31 23:59:59']. + 打印的形式是'YYYY-MM-DD HH:MM:SS' + +## keyword + + DATETIME diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/DECIMAL.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/DECIMAL.md new file mode 100644 index 00000000000000..570d2554354b79 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/DECIMAL.md @@ -0,0 +1,35 @@ +--- +{ + "title": "DECIMAL", + "language": "zh-CN" +} +--- + + + +# DECIMAL +## description + DECIMAL(M[,D]) + 高精度定点数，M代表一共有多少个有效数字(precision)，D代表小数点后最多有多少数字(scale) + M的范围是[1,27], D的范围[1, 9], 另外，M必须要大于等于D的取值。默认的D取值为0 + +## keyword + + DECIMAL diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/DOUBLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/DOUBLE.md new file mode 100644 index 00000000000000..d6e659b87495df --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/DOUBLE.md @@ -0,0 +1,34 @@ +--- +{ + "title": "DOUBLE", + "language": "zh-CN" +} +--- + + + +# DOUBLE +## description + DOUBLE + 8字节浮点数 + +## keyword + + DOUBLE diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/FLOAT.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/FLOAT.md new file mode 100644 index 00000000000000..454808d9065fa5 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/FLOAT.md @@ -0,0 +1,34 @@ +--- +{ + "title": "FLOAT", + "language": "zh-CN" +} +--- + + + +# FLOAT +## description + FLOAT + 4字节浮点数 + +## keyword + + FLOAT diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/HLL.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/HLL.md new file mode 100644 index 00000000000000..7357f4e1e3130a --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/HLL.md @@ -0,0 +1,36 @@ +--- +{ + "title": "HLL(HyperLogLog)", + "language": "zh-CN" +} +--- + + + +# HLL(HyperLogLog) +## description + VARCHAR(M) + 变长字符串，M代表的是变长字符串的长度。M的范围是1-16385 + 用户不需要指定长度和默认值。长度根据数据的聚合程度系统内控制 + 并且HLL列只能通过配套的hll_union_agg、hll_raw_agg、hll_cardinality、hll_hash进行查询或使用 + +## keyword + + HLL,HYPERLOGLOG diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/INT.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/INT.md new file mode 100644 index 00000000000000..e3943e5aad86a9 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/INT.md @@ -0,0 +1,34 @@ +--- +{ + "title": "INT", + "language": "zh-CN" +} +--- + + + +# INT +## description + INT + 4字节有符号整数，范围[-2147483648, 2147483647] + +## keyword + + INT diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/LARGEINT.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/LARGEINT.md similarity index 93% rename from docs/documentation/cn/sql-reference/sql-statements/Data Types/LARGEINT.md rename to docs/zh-CN/sql-reference/sql-statements/Data Types/LARGEINT.md index 84034b1426ff50..b682817cfa5f90 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/LARGEINT.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/LARGEINT.md @@ -1,3 +1,10 @@ +--- +{ + "title": "LARGEINT", + "language": "zh-CN" +} +--- + + +# SMALLINT +## description + SMALLINT + 2字节有符号整数，范围[-32768, 32767] + +## keyword + + SMALLINT diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/TINYINT.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/TINYINT.md new file mode 100644 index 00000000000000..39e18da9e06859 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/TINYINT.md @@ -0,0 +1,34 @@ +--- +{ + "title": "TINYINT", + "language": "zh-CN" +} +--- + + + +# TINYINT +## description + TINYINT + 1字节有符号整数，范围[-128, 127] + +## keyword + + TINYINT diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md new file mode 100644 index 00000000000000..59416788a3ec14 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md @@ -0,0 +1,34 @@ +--- +{ + "title": "VARCHAR", + "language": "zh-CN" +} +--- + + + +# VARCHAR +## description + VARCHAR(M) + 变长字符串，M代表的是变长字符串的长度。M的范围是1-65535 + +## keyword + + VARCHAR diff --git a/docs/documentation/cn/sql-reference/sql-statements/Utility/DESCRIBE.md b/docs/zh-CN/sql-reference/sql-statements/Utility/DESCRIBE.md similarity index 94% rename from docs/documentation/cn/sql-reference/sql-statements/Utility/DESCRIBE.md rename to docs/zh-CN/sql-reference/sql-statements/Utility/DESCRIBE.md index 99e704365fb7c1..95220250956db4 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Utility/DESCRIBE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Utility/DESCRIBE.md @@ -1,3 +1,10 @@ +--- +{ + "title": "DESCRIBE", + "language": "zh-CN" +} +--- +