diff --git a/be/src/io/cache/block/block_file_cache_profile.cpp b/be/src/io/cache/block/block_file_cache_profile.cpp index ef245be35059eb..345a53a48fa2bf 100644 --- a/be/src/io/cache/block/block_file_cache_profile.cpp +++ b/be/src/io/cache/block/block_file_cache_profile.cpp @@ -115,9 +115,9 @@ void FileCacheMetric::register_entity() { entity = DorisMetrics::instance()->metric_registry()->register_entity( std::string("cloud_file_cache"), {{"table_id", table_id_str}, {"partition_id", partition_id_str}}); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, num_io_bytes_read_total); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, num_io_bytes_read_from_cache); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, num_io_bytes_read_from_remote); + INT_COUNTER_METRIC_REGISTER(entity, num_io_bytes_read_total); + INT_COUNTER_METRIC_REGISTER(entity, num_io_bytes_read_from_cache); + INT_COUNTER_METRIC_REGISTER(entity, num_io_bytes_read_from_remote); entity->register_hook("cloud_file_cache", std::bind(&FileCacheMetric::update_table_metrics, this)); } diff --git a/be/src/io/cache/block/block_file_cache_profile.h b/be/src/io/cache/block/block_file_cache_profile.h index fcc9ea503ddf06..f78de2d179ad61 100644 --- a/be/src/io/cache/block/block_file_cache_profile.h +++ b/be/src/io/cache/block/block_file_cache_profile.h @@ -61,9 +61,9 @@ struct FileCacheMetric { int64_t table_id = -1; int64_t partition_id = -1; std::shared_ptr entity; - IntAtomicCounter* num_io_bytes_read_total = nullptr; - IntAtomicCounter* num_io_bytes_read_from_cache = nullptr; - IntAtomicCounter* num_io_bytes_read_from_remote = nullptr; + IntCounter* num_io_bytes_read_total = nullptr; + IntCounter* num_io_bytes_read_from_cache = nullptr; + IntCounter* num_io_bytes_read_from_remote = nullptr; }; struct FileCacheProfile { diff --git a/be/src/io/cache/block/block_lru_file_cache.cpp b/be/src/io/cache/block/block_lru_file_cache.cpp index 33c0e3474b721b..5b448503212dd4 100644 --- a/be/src/io/cache/block/block_lru_file_cache.cpp +++ b/be/src/io/cache/block/block_lru_file_cache.cpp @@ -49,6 +49,7 @@ #include "io/fs/local_file_system.h" #include "io/fs/path.h" #include "util/doris_metrics.h" +#include "util/metrics.h" #include "util/slice.h" #include "util/stopwatch.hpp" #include "vec/common/hex.h" @@ -101,9 +102,9 @@ LRUFileCache::LRUFileCache(const std::string& cache_base_path, "lru_file_cache", {{"path", _cache_base_path}}); _entity->register_hook(_cache_base_path, std::bind(&LRUFileCache::update_cache_metrics, this)); - INT_DOUBLE_METRIC_REGISTER(_entity, file_cache_hits_ratio); - INT_DOUBLE_METRIC_REGISTER(_entity, file_cache_hits_ratio_5m); - INT_DOUBLE_METRIC_REGISTER(_entity, file_cache_hits_ratio_1h); + DOUBLE_GAUGE_METRIC_REGISTER(_entity, file_cache_hits_ratio); + DOUBLE_GAUGE_METRIC_REGISTER(_entity, file_cache_hits_ratio_5m); + DOUBLE_GAUGE_METRIC_REGISTER(_entity, file_cache_hits_ratio_1h); INT_UGAUGE_METRIC_REGISTER(_entity, file_cache_removed_elements); INT_UGAUGE_METRIC_REGISTER(_entity, file_cache_index_queue_max_size); diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp index 741c2423915ede..fbaf51979fb0ca 100644 --- a/be/src/olap/lru_cache.cpp +++ b/be/src/olap/lru_cache.cpp @@ -544,10 +544,10 @@ ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t total_capacity, INT_GAUGE_METRIC_REGISTER(_entity, cache_capacity); INT_GAUGE_METRIC_REGISTER(_entity, cache_usage); INT_GAUGE_METRIC_REGISTER(_entity, cache_element_count); - INT_DOUBLE_METRIC_REGISTER(_entity, cache_usage_ratio); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_lookup_count); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_hit_count); - INT_DOUBLE_METRIC_REGISTER(_entity, cache_hit_ratio); + DOUBLE_GAUGE_METRIC_REGISTER(_entity, cache_usage_ratio); + INT_COUNTER_METRIC_REGISTER(_entity, cache_lookup_count); + INT_COUNTER_METRIC_REGISTER(_entity, cache_hit_count); + DOUBLE_GAUGE_METRIC_REGISTER(_entity, cache_hit_ratio); _hit_count_bvar.reset(new bvar::Adder("doris_cache", _name)); _hit_count_per_second.reset(new bvar::PerSecond>( diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h index 059020deab58f5..d031e531ae4938 100644 --- a/be/src/olap/lru_cache.h +++ b/be/src/olap/lru_cache.h @@ -436,8 +436,8 @@ class ShardedLRUCache : public Cache { IntGauge* cache_usage = nullptr; IntGauge* cache_element_count = nullptr; DoubleGauge* cache_usage_ratio = nullptr; - IntAtomicCounter* cache_lookup_count = nullptr; - IntAtomicCounter* cache_hit_count = nullptr; + IntCounter* cache_lookup_count = nullptr; + IntCounter* cache_hit_count = nullptr; DoubleGauge* cache_hit_ratio = nullptr; // bvars std::unique_ptr> _hit_count_bvar; diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index a88945ed6755c4..259b1f7108f1d0 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -168,6 +168,16 @@ Status EngineCloneTask::_do_clone() { auto duration = std::chrono::milliseconds(dp->param("duration", 10 * 1000)); std::this_thread::sleep_for(duration); }); + + DBUG_EXECUTE_IF("EngineCloneTask.failed_clone", { + LOG_WARNING("EngineCloneTask.failed_clone") + .tag("tablet_id", _clone_req.tablet_id) + .tag("replica_id", _clone_req.replica_id) + .tag("version", _clone_req.version); + return Status::InternalError( + "in debug point, EngineCloneTask.failed_clone tablet={}, replica={}, version={}", + _clone_req.tablet_id, _clone_req.replica_id, _clone_req.version); + }); Status status = Status::OK(); string src_file_path; TBackend src_host; diff --git a/be/src/util/core_local.cpp b/be/src/util/core_local.cpp deleted file mode 100644 index 1c4b1dd04715b4..00000000000000 --- a/be/src/util/core_local.cpp +++ /dev/null @@ -1,129 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/core_local.h" - -#include -#include -#include -#include - -#include "common/compiler_util.h" // IWYU pragma: keep -#include "common/logging.h" -#include "util/spinlock.h" -#include "util/sse_util.hpp" - -namespace doris { - -constexpr int BLOCK_SIZE = 4096; -struct alignas(CACHE_LINE_SIZE) CoreDataBlock { - void* at(size_t offset) { return data + offset; } - char data[BLOCK_SIZE]; - - static void* operator new(size_t nbytes) { - void* p = nullptr; - if (posix_memalign(&p, alignof(CoreDataBlock), nbytes) == 0) { - return p; - } - throw std::bad_alloc(); - } - - static void operator delete(void* p) { free(p); } -}; - -template -class CoreDataAllocatorImpl : public CoreDataAllocator { -public: - virtual ~CoreDataAllocatorImpl(); - void* get_or_create(size_t id) override { - size_t block_id = id / ELEMENTS_PER_BLOCK; - { - std::lock_guard l(_lock); - if (block_id >= _blocks.size()) { - _blocks.resize(block_id + 1); - } - } - CoreDataBlock* block = _blocks[block_id]; - if (block == nullptr) { - std::lock_guard l(_lock); - block = _blocks[block_id]; - if (block == nullptr) { - block = new CoreDataBlock(); - _blocks[block_id] = block; - } - } - size_t offset = (id % ELEMENTS_PER_BLOCK) * ELEMENT_BYTES; - return block->at(offset); - } - -private: - static constexpr int ELEMENTS_PER_BLOCK = BLOCK_SIZE / ELEMENT_BYTES; - SpinLock _lock; // lock to protect the modification of _blocks - std::vector _blocks; -}; - -template -CoreDataAllocatorImpl::~CoreDataAllocatorImpl() { - for (auto block : _blocks) { - delete block; - } -} - -CoreDataAllocatorFactory* CoreDataAllocatorFactory::instance() { - static CoreDataAllocatorFactory _s_instance; - return &_s_instance; -} - -CoreDataAllocator* CoreDataAllocatorFactory::get_allocator(size_t cpu_idx, size_t data_bytes) { - std::lock_guard l(_lock); - auto pair = std::make_pair(cpu_idx, data_bytes); - auto it = _allocators.find(pair); - if (it != std::end(_allocators)) { - return it->second; - } - CoreDataAllocator* allocator = nullptr; - switch (data_bytes) { - case 1: - allocator = new CoreDataAllocatorImpl<1>(); - break; - case 2: - allocator = new CoreDataAllocatorImpl<2>(); - break; - case 3: - case 4: - allocator = new CoreDataAllocatorImpl<4>(); - break; - case 5: - case 6: - case 7: - case 8: - allocator = new CoreDataAllocatorImpl<8>(); - break; - default: - DCHECK(false) << "don't support core local value for this size, size=" << data_bytes; - } - _allocators.emplace(pair, allocator); - return allocator; -} - -CoreDataAllocatorFactory::~CoreDataAllocatorFactory() { - for (auto& it : _allocators) { - delete it.second; - } -} - -} // namespace doris diff --git a/be/src/util/core_local.h b/be/src/util/core_local.h deleted file mode 100644 index 1610ae5a0bb046..00000000000000 --- a/be/src/util/core_local.h +++ /dev/null @@ -1,162 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "common/compiler_util.h" // IWYU pragma: keep - -namespace doris { - -class CoreDataAllocator { -public: - virtual ~CoreDataAllocator() {} - virtual void* get_or_create(size_t id) = 0; -}; - -class CoreDataAllocatorFactory { -public: - CoreDataAllocatorFactory() {} - ~CoreDataAllocatorFactory(); - CoreDataAllocator* get_allocator(size_t cpu_id, size_t data_bytes); - static CoreDataAllocatorFactory* instance(); - -private: - DISALLOW_COPY_AND_ASSIGN(CoreDataAllocatorFactory); - -private: - std::mutex _lock; - std::map, CoreDataAllocator*> _allocators; -}; - -template -class CoreLocalValueController { -public: - CoreLocalValueController() { - int num_cpus = static_cast(std::thread::hardware_concurrency()); - _size = 8; - while (_size < num_cpus) { - _size <<= 1; - } - _allocators.resize(_size, nullptr); - for (int i = 0; i < _size; ++i) { - _allocators[i] = CoreDataAllocatorFactory::instance()->get_allocator(i, sizeof(T)); - } - } - - ~CoreLocalValueController() {} - - int get_id() { - std::lock_guard l(_lock); - int id = 0; - if (_free_ids.empty()) { - id = _next_id++; - } else { - id = _free_ids.back(); - _free_ids.pop_back(); - } - return id; - } - void reclaim_id(int id) { - std::lock_guard l(_lock); - _free_ids.push_back(id); - } - size_t size() const { return _size; } - CoreDataAllocator* allocator(int i) const { return _allocators[i]; } - - static CoreLocalValueController* instance() { - static CoreLocalValueController _s_instance; - return &_s_instance; - } - -private: - DISALLOW_COPY_AND_ASSIGN(CoreLocalValueController); - -private: - std::mutex _lock; - int _next_id = 0; - std::deque _free_ids; - std::vector _allocators; - size_t _size; -}; - -template -class CoreLocalValue { -public: - CoreLocalValue(const T init_value = T()) { - CoreLocalValueController* controller = CoreLocalValueController::instance(); - _id = controller->get_id(); - _size = controller->size(); - _values.resize(_size, nullptr); - for (int i = 0; i < _size; ++i) { - void* ptr = controller->allocator(i)->get_or_create(_id); - _values[i] = new (ptr) T(init_value); - } - } - - ~CoreLocalValue() { - for (int i = 0; i < _size; ++i) { - _values[i]->~T(); - } - CoreLocalValueController::instance()->reclaim_id(_id); - } - - size_t size() const { return _size; } - T* access() const { -#ifdef __APPLE__ - size_t cpu_id = 0; -#else - size_t cpu_id = sched_getcpu(); -#endif - if (cpu_id >= _size) { - cpu_id &= _size - 1; - } - return access_at_core(cpu_id); - } - T* access_at_core(size_t core_idx) const { return _values[core_idx]; } - - inline void reset() { - for (int i = 0; i < _size; ++i) { - _values[i]->~T(); - } - _values.clear(); - _values.resize(_size, nullptr); - CoreLocalValueController* controller = CoreLocalValueController::instance(); - for (int i = 0; i < _size; ++i) { - void* ptr = controller->allocator(i)->get_or_create(_id); - _values[i] = new (ptr) T(); - } - } - -private: - int _id = -1; - size_t _size = 0; - std::vector _values; -}; - -} // namespace doris diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp index 4d68cc6e1f7f9a..65946b406b8acb 100644 --- a/be/src/util/doris_metrics.cpp +++ b/be/src/util/doris_metrics.cpp @@ -302,13 +302,13 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_GAUGE_METRIC_REGISTER(_server_metric_entity, local_file_open_writing); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, s3_file_open_writing); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_server_metric_entity, query_ctx_cnt); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_ctx_cnt); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_cnt); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_cnt); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_queued); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_running); - INT_ATOMIC_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_submit_failed); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_ctx_cnt); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_ctx_cnt); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_cnt); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_cnt); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_queued); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_running); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_submit_failed); } void DorisMetrics::initialize(bool init_system_metrics, const std::set& disk_devices, diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h index 6e27dc7344146d..e272ce12cb1a24 100644 --- a/be/src/util/doris_metrics.h +++ b/be/src/util/doris_metrics.h @@ -233,13 +233,13 @@ class DorisMetrics { UIntGauge* group_local_scan_thread_pool_queue_size = nullptr; UIntGauge* group_local_scan_thread_pool_thread_num = nullptr; - IntAtomicCounter* query_ctx_cnt = nullptr; - IntAtomicCounter* scanner_ctx_cnt = nullptr; - IntAtomicCounter* scanner_cnt = nullptr; - IntAtomicCounter* scanner_task_cnt = nullptr; - IntAtomicCounter* scanner_task_queued = nullptr; - IntAtomicCounter* scanner_task_submit_failed = nullptr; - IntAtomicCounter* scanner_task_running = nullptr; + IntCounter* query_ctx_cnt = nullptr; + IntCounter* scanner_ctx_cnt = nullptr; + IntCounter* scanner_cnt = nullptr; + IntCounter* scanner_task_cnt = nullptr; + IntCounter* scanner_task_queued = nullptr; + IntCounter* scanner_task_submit_failed = nullptr; + IntCounter* scanner_task_running = nullptr; static DorisMetrics* instance() { static DorisMetrics instance; diff --git a/be/src/util/metrics.h b/be/src/util/metrics.h index ac7e69a4ef8ab4..cb49884fefb60b 100644 --- a/be/src/util/metrics.h +++ b/be/src/util/metrics.h @@ -19,21 +19,17 @@ #include #include -#include -#include #include #include #include #include #include -#include #include #include #include #include -#include "util/core_local.h" #include "util/histogram.h" namespace doris { @@ -67,8 +63,8 @@ using Labels = std::unordered_map; class Metric { public: - Metric() {} - virtual ~Metric() {} + Metric() = default; + virtual ~Metric() = default; virtual std::string to_string() const = 0; virtual std::string to_prometheus(const std::string& display_name, const Labels& entity_labels, const Labels& metric_labels) const; @@ -83,7 +79,7 @@ template class AtomicMetric : public Metric { public: AtomicMetric() : _value(T()) {} - virtual ~AtomicMetric() {} + virtual ~AtomicMetric() = default; std::string to_string() const override { return std::to_string(value()); } @@ -101,81 +97,10 @@ class AtomicMetric : public Metric { std::atomic _value; }; -template -class LockSimpleMetric : public Metric { -public: - LockSimpleMetric() : _value(T()) {} - virtual ~LockSimpleMetric() {} - - std::string to_string() const override { return std::to_string(value()); } - - T value() const { - std::lock_guard l(_lock); - return _value; - } - - void increment(const T& delta) { - std::lock_guard l(this->_lock); - _value += delta; - } - - void set_value(const T& value) { - std::lock_guard l(this->_lock); - _value = value; - } - - rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { - return rj::Value(value()); - } - -protected: - // We use std::mutex instead of std::atomic is because atomic don't support - // double's fetch_add - // TODO(zc): If this is atomic is bottleneck, we change to thread local. - // performance: on Intel(R) Xeon(R) CPU E5-2450 int64_t - // original type: 2ns/op - // single thread std::mutex: 26ns/op - // multiple thread(8) std::mutex: 2500ns/op - mutable std::mutex _lock; - T _value; -}; - -template -class CoreLocalCounter : public Metric { -public: - CoreLocalCounter() {} - virtual ~CoreLocalCounter() {} - - std::string to_string() const override { - std::stringstream ss; - ss << value(); - return ss.str(); - } - - T value() const { - T sum = 0; - for (int i = 0; i < _value.size(); ++i) { - sum += *_value.access_at_core(i); - } - return sum; - } - - void increment(const T& delta) { __sync_fetch_and_add(_value.access(), delta); } - - void reset() { _value.reset(); } - - rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { - return rj::Value(value()); - } - -protected: - CoreLocalValue _value; -}; - class HistogramMetric : public Metric { public: - HistogramMetric() {} - virtual ~HistogramMetric() {} + HistogramMetric() = default; + virtual ~HistogramMetric() = default; HistogramMetric(const HistogramMetric&) = delete; HistogramMetric& operator=(const HistogramMetric&) = delete; @@ -208,41 +133,25 @@ class HistogramMetric : public Metric { template class AtomicCounter : public AtomicMetric { public: - AtomicCounter() {} - virtual ~AtomicCounter() {} + AtomicCounter() = default; + virtual ~AtomicCounter() = default; }; template class AtomicGauge : public AtomicMetric { public: AtomicGauge() : AtomicMetric() {} - virtual ~AtomicGauge() {} -}; - -template -class LockCounter : public LockSimpleMetric { -public: - LockCounter() : LockSimpleMetric() {} - virtual ~LockCounter() {} -}; - -// This can only used for trival type -template -class LockGauge : public LockSimpleMetric { -public: - LockGauge() : LockSimpleMetric() {} - virtual ~LockGauge() {} + virtual ~AtomicGauge() = default; }; -using IntCounter = CoreLocalCounter; -using IntAtomicCounter = AtomicCounter; -using UIntCounter = CoreLocalCounter; -using DoubleCounter = LockCounter; +using IntCounter = AtomicCounter; +using UIntCounter = AtomicCounter; +using DoubleCounter = AtomicCounter; using IntGauge = AtomicGauge; using UIntGauge = AtomicGauge; -using DoubleGauge = LockGauge; - +using DoubleGauge = AtomicGauge; using Labels = std::unordered_map; + struct MetricPrototype { public: MetricPrototype(MetricType type_, MetricUnit unit_, std::string name_, @@ -302,15 +211,12 @@ struct MetricPrototype { #define INT_GAUGE_METRIC_REGISTER(entity, metric) \ metric = (IntGauge*)(entity->register_metric(&METRIC_##metric)) -#define INT_DOUBLE_METRIC_REGISTER(entity, metric) \ +#define DOUBLE_GAUGE_METRIC_REGISTER(entity, metric) \ metric = (DoubleGauge*)(entity->register_metric(&METRIC_##metric)) #define INT_UGAUGE_METRIC_REGISTER(entity, metric) \ metric = (UIntGauge*)(entity->register_metric(&METRIC_##metric)) -#define INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, metric) \ - metric = (IntAtomicCounter*)(entity->register_metric(&METRIC_##metric)) - #define HISTOGRAM_METRIC_REGISTER(entity, metric) \ metric = (HistogramMetric*)(entity->register_metric(&METRIC_##metric)) @@ -338,8 +244,8 @@ enum class MetricEntityType { kServer, kTablet }; class MetricEntity { public: - MetricEntity(MetricEntityType type, const std::string& name, const Labels& labels) - : _type(type), _name(name), _labels(labels) {} + MetricEntity(MetricEntityType type, std::string name, Labels labels) + : _type(type), _name(std::move(name)), _labels(std::move(labels)) {} ~MetricEntity() { for (auto& metric : _metrics) { delete metric.second; @@ -401,7 +307,7 @@ using EntityMetricsByType = class MetricRegistry { public: - MetricRegistry(const std::string& name) : _name(name) {} + MetricRegistry(std::string name) : _name(std::move(name)) {} ~MetricRegistry(); std::shared_ptr register_entity( diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp index c1385b6244bf62..8203be6d6a73b7 100644 --- a/be/src/util/system_metrics.cpp +++ b/be/src/util/system_metrics.cpp @@ -55,16 +55,16 @@ DEFINE_CPU_COUNTER_METRIC(guest_nice); // /proc/stat: http://www.linuxhowtos.org/System/procstat.htm struct CpuMetrics { CpuMetrics(MetricEntity* ent) : entity(ent) { - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_user); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_nice); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_system); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_idle); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_iowait); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_irq); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_soft_irq); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_steal); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_guest); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, cpu_guest_nice); + INT_COUNTER_METRIC_REGISTER(entity, cpu_user); + INT_COUNTER_METRIC_REGISTER(entity, cpu_nice); + INT_COUNTER_METRIC_REGISTER(entity, cpu_system); + INT_COUNTER_METRIC_REGISTER(entity, cpu_idle); + INT_COUNTER_METRIC_REGISTER(entity, cpu_iowait); + INT_COUNTER_METRIC_REGISTER(entity, cpu_irq); + INT_COUNTER_METRIC_REGISTER(entity, cpu_soft_irq); + INT_COUNTER_METRIC_REGISTER(entity, cpu_steal); + INT_COUNTER_METRIC_REGISTER(entity, cpu_guest); + INT_COUNTER_METRIC_REGISTER(entity, cpu_guest_nice); metrics[0] = cpu_user; metrics[1] = cpu_nice; @@ -81,18 +81,18 @@ struct CpuMetrics { static constexpr int cpu_num_metrics = 10; MetricEntity* entity = nullptr; - IntAtomicCounter* cpu_user; - IntAtomicCounter* cpu_nice; - IntAtomicCounter* cpu_system; - IntAtomicCounter* cpu_idle; - IntAtomicCounter* cpu_iowait; - IntAtomicCounter* cpu_irq; - IntAtomicCounter* cpu_soft_irq; - IntAtomicCounter* cpu_steal; - IntAtomicCounter* cpu_guest; - IntAtomicCounter* cpu_guest_nice; - - IntAtomicCounter* metrics[cpu_num_metrics]; + IntCounter* cpu_user; + IntCounter* cpu_nice; + IntCounter* cpu_system; + IntCounter* cpu_idle; + IntCounter* cpu_iowait; + IntCounter* cpu_irq; + IntCounter* cpu_soft_irq; + IntCounter* cpu_steal; + IntCounter* cpu_guest; + IntCounter* cpu_guest_nice; + + IntCounter* metrics[cpu_num_metrics]; }; #define DEFINE_MEMORY_GAUGE_METRIC(metric, unit) \ @@ -201,25 +201,25 @@ DEFINE_DISK_COUNTER_METRIC(io_time_weigthed, MetricUnit::MILLISECONDS); struct DiskMetrics { DiskMetrics(MetricEntity* ent) : entity(ent) { - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, disk_reads_completed); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, disk_bytes_read); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, disk_read_time_ms); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, disk_writes_completed); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, disk_bytes_written); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, disk_write_time_ms); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, disk_io_time_ms); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, disk_io_time_weigthed); + INT_COUNTER_METRIC_REGISTER(entity, disk_reads_completed); + INT_COUNTER_METRIC_REGISTER(entity, disk_bytes_read); + INT_COUNTER_METRIC_REGISTER(entity, disk_read_time_ms); + INT_COUNTER_METRIC_REGISTER(entity, disk_writes_completed); + INT_COUNTER_METRIC_REGISTER(entity, disk_bytes_written); + INT_COUNTER_METRIC_REGISTER(entity, disk_write_time_ms); + INT_COUNTER_METRIC_REGISTER(entity, disk_io_time_ms); + INT_COUNTER_METRIC_REGISTER(entity, disk_io_time_weigthed); } MetricEntity* entity = nullptr; - IntAtomicCounter* disk_reads_completed; - IntAtomicCounter* disk_bytes_read; - IntAtomicCounter* disk_read_time_ms; - IntAtomicCounter* disk_writes_completed; - IntAtomicCounter* disk_bytes_written; - IntAtomicCounter* disk_write_time_ms; - IntAtomicCounter* disk_io_time_ms; - IntAtomicCounter* disk_io_time_weigthed; + IntCounter* disk_reads_completed; + IntCounter* disk_bytes_read; + IntCounter* disk_read_time_ms; + IntCounter* disk_writes_completed; + IntCounter* disk_bytes_written; + IntCounter* disk_write_time_ms; + IntCounter* disk_io_time_ms; + IntCounter* disk_io_time_weigthed; }; #define DEFINE_NETWORK_COUNTER_METRIC(metric, unit) \ @@ -231,17 +231,17 @@ DEFINE_NETWORK_COUNTER_METRIC(send_packets, MetricUnit::PACKETS); struct NetworkMetrics { NetworkMetrics(MetricEntity* ent) : entity(ent) { - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, network_receive_bytes); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, network_receive_packets); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, network_send_bytes); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, network_send_packets); + INT_COUNTER_METRIC_REGISTER(entity, network_receive_bytes); + INT_COUNTER_METRIC_REGISTER(entity, network_receive_packets); + INT_COUNTER_METRIC_REGISTER(entity, network_send_bytes); + INT_COUNTER_METRIC_REGISTER(entity, network_send_packets); } MetricEntity* entity = nullptr; - IntAtomicCounter* network_receive_bytes; - IntAtomicCounter* network_receive_packets; - IntAtomicCounter* network_send_bytes; - IntAtomicCounter* network_send_packets; + IntCounter* network_receive_bytes; + IntCounter* network_receive_packets; + IntCounter* network_send_bytes; + IntCounter* network_send_packets; }; #define DEFINE_SNMP_COUNTER_METRIC(metric, unit, desc) \ @@ -255,17 +255,17 @@ DEFINE_SNMP_COUNTER_METRIC(tcp_out_segs, MetricUnit::NOUNIT, "All send TCP packe // metrics read from /proc/net/snmp struct SnmpMetrics { SnmpMetrics(MetricEntity* ent) : entity(ent) { - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, snmp_tcp_in_errs); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, snmp_tcp_retrans_segs); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, snmp_tcp_in_segs); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, snmp_tcp_out_segs); + INT_COUNTER_METRIC_REGISTER(entity, snmp_tcp_in_errs); + INT_COUNTER_METRIC_REGISTER(entity, snmp_tcp_retrans_segs); + INT_COUNTER_METRIC_REGISTER(entity, snmp_tcp_in_segs); + INT_COUNTER_METRIC_REGISTER(entity, snmp_tcp_out_segs); } MetricEntity* entity = nullptr; - IntAtomicCounter* snmp_tcp_in_errs; - IntAtomicCounter* snmp_tcp_retrans_segs; - IntAtomicCounter* snmp_tcp_in_segs; - IntAtomicCounter* snmp_tcp_out_segs; + IntCounter* snmp_tcp_in_errs; + IntCounter* snmp_tcp_retrans_segs; + IntCounter* snmp_tcp_in_segs; + IntCounter* snmp_tcp_out_segs; }; #define DEFINE_FD_COUNTER_METRIC(metric, unit) \ @@ -293,9 +293,9 @@ DEFINE_LOAD_AVERAGE_DOUBLE_METRIC(15_minutes); struct LoadAverageMetrics { LoadAverageMetrics(MetricEntity* ent) : entity(ent) { - INT_DOUBLE_METRIC_REGISTER(entity, load_average_1_minutes); - INT_DOUBLE_METRIC_REGISTER(entity, load_average_5_minutes); - INT_DOUBLE_METRIC_REGISTER(entity, load_average_15_minutes); + DOUBLE_GAUGE_METRIC_REGISTER(entity, load_average_1_minutes); + DOUBLE_GAUGE_METRIC_REGISTER(entity, load_average_5_minutes); + DOUBLE_GAUGE_METRIC_REGISTER(entity, load_average_15_minutes); } MetricEntity* entity = nullptr; @@ -314,18 +314,18 @@ DEFINE_PROC_STAT_COUNTER_METRIC(procs_blocked); struct ProcMetrics { ProcMetrics(MetricEntity* ent) : entity(ent) { - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, proc_interrupt); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, proc_ctxt_switch); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, proc_procs_running); - INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, proc_procs_blocked); + INT_COUNTER_METRIC_REGISTER(entity, proc_interrupt); + INT_COUNTER_METRIC_REGISTER(entity, proc_ctxt_switch); + INT_COUNTER_METRIC_REGISTER(entity, proc_procs_running); + INT_COUNTER_METRIC_REGISTER(entity, proc_procs_blocked); } MetricEntity* entity = nullptr; - IntAtomicCounter* proc_interrupt; - IntAtomicCounter* proc_ctxt_switch; - IntAtomicCounter* proc_procs_running; - IntAtomicCounter* proc_procs_blocked; + IntCounter* proc_interrupt; + IntCounter* proc_ctxt_switch; + IntCounter* proc_procs_running; + IntCounter* proc_procs_blocked; }; DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(max_disk_io_util_percent, MetricUnit::PERCENT); diff --git a/be/src/vec/exprs/table_function/table_function_factory.cpp b/be/src/vec/exprs/table_function/table_function_factory.cpp index c72a897305a516..6fd6c6cf9ac978 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.cpp +++ b/be/src/vec/exprs/table_function/table_function_factory.cpp @@ -32,6 +32,7 @@ #include "vec/exprs/table_function/vexplode_map.h" #include "vec/exprs/table_function/vexplode_numbers.h" #include "vec/exprs/table_function/vexplode_split.h" +#include "vec/exprs/table_function/vposexplode.h" #include "vec/utils/util.hpp" namespace doris::vectorized { @@ -59,6 +60,7 @@ const std::unordered_map()}, {"explode_map", TableFunctionCreator {}}, {"explode_json_object", TableFunctionCreator {}}, + {"posexplode", TableFunctionCreator {}}, {"explode", TableFunctionCreator {}}}; Status TableFunctionFactory::get_fn(const std::string& fn_name_raw, ObjectPool* pool, diff --git a/be/src/vec/exprs/table_function/vposexplode.cpp b/be/src/vec/exprs/table_function/vposexplode.cpp new file mode 100644 index 00000000000000..20d04a219f831a --- /dev/null +++ b/be/src/vec/exprs/table_function/vposexplode.cpp @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exprs/table_function/vposexplode.h" + +#include + +#include +#include + +#include "common/status.h" +#include "vec/columns/column.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" + +namespace doris::vectorized { + +VPosExplodeTableFunction::VPosExplodeTableFunction() { + _fn_name = "posexplode"; +} + +Status VPosExplodeTableFunction::process_init(Block* block, RuntimeState* state) { + CHECK(_expr_context->root()->children().size() == 1) + << "VPosExplodeTableFunction only support 1 child but has " + << _expr_context->root()->children().size(); + + int value_column_idx = -1; + RETURN_IF_ERROR(_expr_context->root()->children()[0]->execute(_expr_context.get(), block, + &value_column_idx)); + + _collection_column = + block->get_by_position(value_column_idx).column->convert_to_full_column_if_const(); + + if (!extract_column_array_info(*_collection_column, _array_detail)) { + return Status::NotSupported("column type {} not supported now, only support array", + block->get_by_position(value_column_idx).column->get_name()); + } + if (is_column_nullable(*_collection_column)) { + _array_data_column = + assert_cast( + assert_cast(*_collection_column).get_nested_column()) + .get_data_ptr(); + } else { + _array_data_column = assert_cast(*_collection_column).get_data_ptr(); + } + return Status::OK(); +} + +void VPosExplodeTableFunction::process_row(size_t row_idx) { + DCHECK(row_idx < _collection_column->size()); + TableFunction::process_row(row_idx); + + if (!_array_detail.array_nullmap_data || !_array_detail.array_nullmap_data[row_idx]) { + _collection_offset = (*_array_detail.offsets_ptr)[row_idx - 1]; + _cur_size = (*_array_detail.offsets_ptr)[row_idx] - _collection_offset; + } +} + +void VPosExplodeTableFunction::process_close() { + _collection_column = nullptr; + _array_data_column = nullptr; + _array_detail.reset(); + _collection_offset = 0; +} + +void VPosExplodeTableFunction::get_same_many_values(MutableColumnPtr& column, int length) { + // now we only support array column explode to struct column + size_t pos = _collection_offset + _cur_offset; + // if current is empty array row, also append a default value + if (current_empty()) { + column->insert_many_defaults(length); + return; + } + ColumnStruct* ret = nullptr; + // this _is_nullable is whole output column's nullable + if (_is_nullable) { + ret = assert_cast( + assert_cast(column.get())->get_nested_column_ptr().get()); + assert_cast( + assert_cast(column.get())->get_null_map_column_ptr().get()) + ->insert_many_defaults(length); + } else if (column->is_column_struct()) { + ret = assert_cast(column.get()); + } else { + throw Exception(ErrorCode::INTERNAL_ERROR, + "only support array column explode to struct column"); + } + if (!ret || ret->tuple_size() != 2) { + throw Exception( + ErrorCode::INTERNAL_ERROR, + "only support array column explode to two column, but given: ", ret->tuple_size()); + } + auto& pose_column_nullable = assert_cast(ret->get_column(0)); + pose_column_nullable.get_null_map_column().insert_many_defaults(length); + assert_cast(pose_column_nullable.get_nested_column()) + .insert_many_vals(_cur_offset, length); + ret->get_column(1).insert_many_from(*_array_data_column, pos, length); +} + +int VPosExplodeTableFunction::get_value(MutableColumnPtr& column, int max_step) { + max_step = std::min(max_step, (int)(_cur_size - _cur_offset)); + size_t pos = _collection_offset + _cur_offset; + if (current_empty()) { + column->insert_default(); + max_step = 1; + } else { + ColumnStruct* struct_column = nullptr; + if (_is_nullable) { + auto* nullable_column = assert_cast(column.get()); + struct_column = + assert_cast(nullable_column->get_nested_column_ptr().get()); + auto* nullmap_column = + assert_cast(nullable_column->get_null_map_column_ptr().get()); + // here nullmap_column insert max_step many defaults as if array[row_idx] is NULL + // will be not update value, _cur_size = 0, means current_empty; + // so here could insert directly + nullmap_column->insert_many_defaults(max_step); + } else { + struct_column = assert_cast(column.get()); + } + if (!struct_column || struct_column->tuple_size() != 2) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "only support array column explode to two column, but given: ", + struct_column->tuple_size()); + } + auto& pose_column_nullable = assert_cast(struct_column->get_column(0)); + pose_column_nullable.get_null_map_column().insert_many_defaults(max_step); + assert_cast(pose_column_nullable.get_nested_column()) + .insert_range_of_integer(_cur_offset, _cur_offset + max_step); + struct_column->get_column(1).insert_range_from(*_array_data_column, pos, max_step); + } + forward(max_step); + return max_step; +} +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/table_function/vposexplode.h b/be/src/vec/exprs/table_function/vposexplode.h new file mode 100644 index 00000000000000..4e021fd58da918 --- /dev/null +++ b/be/src/vec/exprs/table_function/vposexplode.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "vec/columns/column_map.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_array.h" +#include "vec/exprs/table_function/table_function.h" +#include "vec/functions/array/function_array_utils.h" + +namespace doris::vectorized { + +class VPosExplodeTableFunction : public TableFunction { + ENABLE_FACTORY_CREATOR(VPosExplodeTableFunction); + +public: + VPosExplodeTableFunction(); + + ~VPosExplodeTableFunction() override = default; + + Status process_init(Block* block, RuntimeState* state) override; + void process_row(size_t row_idx) override; + void process_close() override; + void get_same_many_values(MutableColumnPtr& column, int length) override; + int get_value(MutableColumnPtr& column, int max_step) override; + +private: + ColumnPtr _collection_column; + ColumnPtr _array_data_column; + ColumnArrayExecutionData _array_detail; + size_t _collection_offset; // start offset of array[row_idx] +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/function_fake.cpp b/be/src/vec/functions/function_fake.cpp index c7edcf4df8f1b5..758ec9d0ebf2ae 100644 --- a/be/src/vec/functions/function_fake.cpp +++ b/be/src/vec/functions/function_fake.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -82,6 +83,25 @@ struct FunctionExplodeMap { static std::string get_error_msg() { return "Fake function do not support execute"; } }; +template +struct FunctionPoseExplode { + static DataTypePtr get_return_type_impl(const DataTypes& arguments) { + DCHECK(is_array(arguments[0])) << arguments[0]->get_name() << " not supported"; + DataTypes fieldTypes(2); + fieldTypes[0] = make_nullable(std::make_shared()); + fieldTypes[1] = + check_and_get_data_type(arguments[0].get())->get_nested_type(); + auto struct_type = std::make_shared(fieldTypes); + if constexpr (AlwaysNullable) { + return make_nullable(struct_type); + } else { + return arguments[0]->is_nullable() ? make_nullable(struct_type) : struct_type; + } + } + static DataTypes get_variadic_argument_types() { return {}; } + static std::string get_error_msg() { return "Fake function do not support execute"; } +}; + // explode json-object: expands json-object to struct with a pair of key and value in column string struct FunctionExplodeJsonObject { static DataTypePtr get_return_type_impl(const DataTypes& arguments) { @@ -137,6 +157,12 @@ void register_table_function_expand_outer_default(SimpleFunctionFactory& factory COMBINATOR_SUFFIX_OUTER); }; +template +void register_table_function_with_impl(SimpleFunctionFactory& factory, const std::string& name, + const std::string& suffix = "") { + factory.register_function>(name + suffix); +}; + void register_function_fake(SimpleFunctionFactory& factory) { register_function(factory, "esquery"); @@ -157,6 +183,9 @@ void register_function_fake(SimpleFunctionFactory& factory) { register_table_function_expand_outer_default( factory, "explode_json_array_double"); register_table_function_expand_outer_default(factory, "explode_bitmap"); + register_table_function_with_impl>(factory, "posexplode"); + register_table_function_with_impl>(factory, "posexplode", + COMBINATOR_SUFFIX_OUTER); } } // namespace doris::vectorized diff --git a/be/test/util/core_local_test.cpp b/be/test/util/core_local_test.cpp deleted file mode 100644 index ed87015b189e1c..00000000000000 --- a/be/test/util/core_local_test.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/core_local.h" - -#include -#include -#include -#include - -#include -#include - -#include "common/logging.h" -#include "gtest/gtest_pred_impl.h" -#include "testutil/test_util.h" -#include "util/stopwatch.hpp" - -namespace doris { - -// Fixture for testing class Decompressor -class CoreLocalTest : public ::testing::Test { -protected: - CoreLocalTest() {} - ~CoreLocalTest() {} -}; - -void updater(int64_t loop, CoreLocalValue* value, int64_t* used_ns) { - usleep(100); - MonotonicStopWatch stopwatch; - stopwatch.start(); - for (int i = 0; i < loop; ++i) { - __sync_fetch_and_add(value->access(), 1); - } - *used_ns = stopwatch.elapsed_time(); -} - -TEST_F(CoreLocalTest, CoreLocalValue) { - int64_t loop = LOOP_LESS_OR_MORE(1000, 1000000L); - CoreLocalValue value; - std::vector used_ns; - used_ns.resize(8); - std::vector workers; - for (int i = 0; i < 8; ++i) { - workers.emplace_back(updater, loop, &value, &used_ns[i]); - } - int64_t sum_ns = 0; - for (int i = 0; i < 8; ++i) { - workers[i].join(); - sum_ns += used_ns[i]; - } - int64_t sum = 0; - for (int i = 0; i < value.size(); ++i) { - sum += __sync_fetch_and_add(value.access_at_core(i), 0); - } - EXPECT_EQ(8 * loop, sum); - LOG(INFO) << "time:" << sum_ns / sum << "ns/op"; -} - -TEST_F(CoreLocalTest, CoreDataAllocator) { - CoreDataAllocatorFactory factory; - auto allocator1 = factory.get_allocator(1, 8); - auto ptr = allocator1->get_or_create(0); - EXPECT_TRUE(ptr != nullptr); - { - auto ptr2 = allocator1->get_or_create(0); - EXPECT_TRUE(ptr == ptr2); - } - { - auto ptr2 = allocator1->get_or_create(4096); - EXPECT_TRUE(ptr2 != nullptr); - } - { - auto allocator2 = factory.get_allocator(2, 8); - EXPECT_TRUE(allocator2 != allocator1); - } -} - -TEST_F(CoreLocalTest, CoreLocalValueController) { - CoreLocalValueController controller; - auto id = controller.get_id(); - EXPECT_EQ(0, id); - controller.reclaim_id(id); - id = controller.get_id(); - EXPECT_EQ(0, id); - id = controller.get_id(); - EXPECT_EQ(1, id); -} - -TEST_F(CoreLocalTest, CoreLocalValueNormal) { - CoreLocalValue value; - for (int i = 0; i < value.size(); ++i) { - EXPECT_EQ(0, *value.access_at_core(i)); - *value.access_at_core(i) += 1; - } - for (int i = 0; i < value.size(); ++i) { - EXPECT_EQ(1, *value.access_at_core(i)); - } - for (int i = 0; i < 10000; ++i) { - *value.access() += 1; - } - int64_t sum = 0; - for (int i = 0; i < value.size(); ++i) { - sum += *value.access_at_core(i); - } - EXPECT_EQ(10000 + value.size(), sum); -} -} // namespace doris diff --git a/be/test/util/doris_metrics_test.cpp b/be/test/util/doris_metrics_test.cpp index dcba57cb7e9ff2..6e9969b1210345 100644 --- a/be/test/util/doris_metrics_test.cpp +++ b/be/test/util/doris_metrics_test.cpp @@ -34,14 +34,14 @@ TEST_F(DorisMetricsTest, Normal) { auto server_entity = DorisMetrics::instance()->server_entity(); // check metric { - DorisMetrics::instance()->fragment_requests_total->reset(); + DorisMetrics::instance()->fragment_requests_total->set_value(0); DorisMetrics::instance()->fragment_requests_total->increment(12); auto metric = server_entity->get_metric("fragment_requests_total"); EXPECT_TRUE(metric != nullptr); EXPECT_STREQ("12", metric->to_string().c_str()); } { - DorisMetrics::instance()->fragment_request_duration_us->reset(); + DorisMetrics::instance()->fragment_request_duration_us->set_value(0); DorisMetrics::instance()->fragment_request_duration_us->increment(101); auto metric = server_entity->get_metric("fragment_request_duration_us"); EXPECT_TRUE(metric != nullptr); @@ -92,7 +92,7 @@ TEST_F(DorisMetricsTest, Normal) { } // engine request { - DorisMetrics::instance()->create_tablet_requests_total->reset(); + DorisMetrics::instance()->create_tablet_requests_total->set_value(0); DorisMetrics::instance()->create_tablet_requests_total->increment(15); auto metric = server_entity->get_metric("create_tablet_requests_total", "engine_requests_total"); @@ -100,7 +100,7 @@ TEST_F(DorisMetricsTest, Normal) { EXPECT_STREQ("15", metric->to_string().c_str()); } { - DorisMetrics::instance()->drop_tablet_requests_total->reset(); + DorisMetrics::instance()->drop_tablet_requests_total->set_value(0); DorisMetrics::instance()->drop_tablet_requests_total->increment(16); auto metric = server_entity->get_metric("drop_tablet_requests_total", "engine_requests_total"); @@ -129,7 +129,7 @@ TEST_F(DorisMetricsTest, Normal) { EXPECT_STREQ("20", metric->to_string().c_str()); } { - DorisMetrics::instance()->storage_migrate_requests_total->reset(); + DorisMetrics::instance()->storage_migrate_requests_total->set_value(0); DorisMetrics::instance()->storage_migrate_requests_total->increment(21); auto metric = server_entity->get_metric("storage_migrate_requests_total", "engine_requests_total"); diff --git a/be/test/util/metrics_test.cpp b/be/test/util/metrics_test.cpp index 305d17c47ca06f..4ed8cc89bae9cc 100644 --- a/be/test/util/metrics_test.cpp +++ b/be/test/util/metrics_test.cpp @@ -46,7 +46,7 @@ TEST_F(MetricsTest, Counter) { EXPECT_STREQ("100", counter.to_string().c_str()); } { - IntAtomicCounter counter; + IntCounter counter; EXPECT_EQ(0, counter.value()); counter.increment(100); EXPECT_EQ(100, counter.value()); @@ -97,9 +97,9 @@ TEST_F(MetricsTest, CounterPerf) { EXPECT_EQ(kLoopCount, sum); LOG(INFO) << "int64_t: elapsed: " << elapsed << "ns, ns/iter:" << elapsed / kLoopCount; } - // IntAtomicCounter + // IntCounter { - IntAtomicCounter counter; + IntCounter counter; MonotonicStopWatch watch; watch.start(); for (int i = 0; i < kLoopCount; ++i) { @@ -107,8 +107,7 @@ TEST_F(MetricsTest, CounterPerf) { } uint64_t elapsed = watch.elapsed_time(); EXPECT_EQ(kLoopCount, counter.value()); - LOG(INFO) << "IntAtomicCounter: elapsed: " << elapsed - << "ns, ns/iter:" << elapsed / kLoopCount; + LOG(INFO) << "IntCounter: elapsed: " << elapsed << "ns, ns/iter:" << elapsed / kLoopCount; } // IntCounter { @@ -139,19 +138,19 @@ TEST_F(MetricsTest, CounterPerf) { << "ns, ns/iter:" << used_time.load() / (8 * kThreadLoopCount); EXPECT_EQ(8 * kThreadLoopCount, mt_counter.value()); } - // multi-thread for IntAtomicCounter + // multi-thread for IntCounter { - IntAtomicCounter mt_counter; + IntCounter mt_counter; std::vector updaters; std::atomic used_time(0); for (int i = 0; i < 8; ++i) { - updaters.emplace_back(&mt_updater, kThreadLoopCount, &mt_counter, + updaters.emplace_back(&mt_updater, kThreadLoopCount, &mt_counter, &used_time); } for (int i = 0; i < 8; ++i) { updaters[i].join(); } - LOG(INFO) << "IntAtomicCounter multi-thread elapsed: " << used_time.load() + LOG(INFO) << "IntCounter multi-thread elapsed: " << used_time.load() << "ns, ns/iter:" << used_time.load() / (8 * kThreadLoopCount); EXPECT_EQ(8 * kThreadLoopCount, mt_counter.value()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java index 84894db95c5617..da8614cbf6d2b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java @@ -37,6 +37,8 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode; +import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import com.google.common.collect.ImmutableList; @@ -69,7 +71,9 @@ public class BuiltinTableGeneratingFunctions implements FunctionHelper { tableGenerating(ExplodeJsonArrayString.class, "explode_json_array_string"), tableGenerating(ExplodeJsonArrayStringOuter.class, "explode_json_array_string_outer"), tableGenerating(ExplodeJsonArrayJson.class, "explode_json_array_json"), - tableGenerating(ExplodeJsonArrayJsonOuter.class, "explode_json_array_json_outer") + tableGenerating(ExplodeJsonArrayJsonOuter.class, "explode_json_array_json_outer"), + tableGenerating(PosExplode.class, "posexplode"), + tableGenerating(PosExplodeOuter.class, "posexplode_outer") ); public static final BuiltinTableGeneratingFunctions INSTANCE = new BuiltinTableGeneratingFunctions(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index 5e29adbb6da817..d8f7bd3fb09e35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -62,6 +62,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; /* * TabletSchedCtx contains all information which is created during tablet scheduler processing. @@ -69,28 +70,6 @@ public class TabletSchedCtx implements Comparable { private static final Logger LOG = LogManager.getLogger(TabletSchedCtx.class); - /* - * SCHED_FAILED_COUNTER_THRESHOLD: - * threshold of times a tablet failed to be scheduled - * - * MIN_ADJUST_PRIORITY_INTERVAL_MS: - * min interval time of adjusting a tablet's priority - * - * MAX_NOT_BEING_SCHEDULED_INTERVAL_MS: - * max gap time of a tablet NOT being scheduled. - * - * These 3 params is for adjusting priority. - * If a tablet being scheduled failed for more than SCHED_FAILED_COUNTER_THRESHOLD times, its priority - * will be downgraded. And the interval between adjustment is larger than MIN_ADJUST_PRIORITY_INTERVAL_MS, - * to avoid being downgraded too soon. - * And if a tablet is not being scheduled longer than MAX_NOT_BEING_SCHEDULED_INTERVAL_MS, its priority - * will be upgraded, to avoid starvation. - * - */ - private static final int SCHED_FAILED_COUNTER_THRESHOLD = 5; - private static final long MIN_ADJUST_PRIORITY_INTERVAL_MS = 5 * 60 * 1000L; // 5 min - private static final long MAX_NOT_BEING_SCHEDULED_INTERVAL_MS = 30 * 60 * 1000L; // 30 min - /* * A clone task timeout is between Config.min_clone_task_timeout_sec and Config.max_clone_task_timeout_sec, * estimated by tablet size / MIN_CLONE_SPEED_MB_PER_SECOND. @@ -450,10 +429,6 @@ public void setSchedFailedCode(SubCode code) { schedFailedCode = code; } - public CloneTask getCloneTask() { - return cloneTask; - } - public long getCopySize() { return copySize; } @@ -932,12 +907,14 @@ public void releaseResource(TabletScheduler tabletScheduler, boolean reserveTabl } if (cloneTask != null) { AgentTaskQueue.removeTask(cloneTask.getBackendId(), TTaskType.CLONE, cloneTask.getSignature()); + cloneTask = null; // clear all CLONE replicas Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); if (db != null) { Table table = db.getTableNullable(tblId); - if (table != null && table.writeLockIfExist()) { + // try get table write lock, if failed TabletScheduler will try next time + if (table != null && table.tryWriteLockIfExist(Table.TRY_LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { try { List cloneReplicas = Lists.newArrayList(); tablet.getReplicas().stream().filter(r -> r.getState() == ReplicaState.CLONE).forEach(r -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java index 724d51c1791925..ba10e5edf862fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -104,9 +104,6 @@ public class TabletScheduler extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(TabletScheduler.class); - // handle at most BATCH_NUM tablets in one loop - private static final int MIN_BATCH_NUM = 50; - // the minimum interval of updating cluster statistics and priority of tablet info private static final long STAT_UPDATE_INTERVAL_MS = 20 * 1000; // 20s @@ -150,7 +147,7 @@ public enum AddResult { ADDED, // success to add ALREADY_IN, // already added, skip LIMIT_EXCEED, // number of pending tablets exceed the limit - REPLACE_ADDED, // succ to add, and envit a lowest task + REPLACE_ADDED, // succ to add, and evict a lowest task DISABLED // scheduler has been disabled. } @@ -285,7 +282,7 @@ public synchronized AddResult addTablet(TabletSchedCtx tablet, boolean force) { addResult = AddResult.REPLACE_ADDED; pendingTablets.pollLast(); finalizeTabletCtx(lowestPriorityTablet, TabletSchedCtx.State.CANCELLED, Status.UNRECOVERABLE, - "envit lower priority sched tablet because pending queue is full"); + "evict lower priority sched tablet because pending queue is full"); } if (!contains || tablet.getType() == TabletSchedCtx.Type.REPAIR) { @@ -1845,9 +1842,9 @@ public boolean finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request) tabletCtx.increaseFailedRunningCounter(); if (!tabletCtx.isExceedFailedRunningLimit()) { stat.counterCloneTaskFailed.incrementAndGet(); + tabletCtx.setState(TabletSchedCtx.State.PENDING); tabletCtx.releaseResource(this); tabletCtx.resetFailedSchedCounter(); - tabletCtx.setState(TabletSchedCtx.State.PENDING); addBackToPendingTablets(tabletCtx); return false; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index c9f3264f8bb978..da006c392e1781 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -58,13 +58,13 @@ import com.google.common.base.Stopwatch; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.sparkproject.guava.base.Throwables; import java.io.Closeable; import java.util.ArrayList; @@ -570,7 +570,7 @@ public synchronized void releasePlannerResources() { } } if (throwable != null) { - Throwables.propagateIfInstanceOf(throwable, RuntimeException.class); + Throwables.throwIfInstanceOf(throwable, RuntimeException.class); throw new IllegalStateException("Release resource failed", throwable); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplode.java new file mode 100644 index 00000000000000..16f8232606ff5f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplode.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * PosExplode(array('a','b','c')) generate two columns and three rows with: + * pose column: 0, 1, 2 + * value column: 'a', 'b', 'c' + */ +public class PosExplode extends TableGeneratingFunction implements UnaryExpression, PropagateNullable { + + /** + * constructor with 1 argument. + */ + public PosExplode(Expression arg) { + super("posexplode", arg); + } + + /** + * withChildren. + */ + @Override + public PosExplode withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new PosExplode(children.get(0)); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!(child().getDataType() instanceof ArrayType)) { + throw new AnalysisException("only support array type for posexplode function but got " + + child().getDataType()); + } + } + + @Override + public List getSignatures() { + return ImmutableList.of( + FunctionSignature.ret(new StructType(ImmutableList.of( + new StructField("pos", IntegerType.INSTANCE, true, ""), + new StructField("col", ((ArrayType) child().getDataType()).getItemType(), true, "")))) + .args(child().getDataType())); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPosExplode(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplodeOuter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplodeOuter.java new file mode 100644 index 00000000000000..6d181354f414bc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplodeOuter.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.IntegerType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * PosExplode(array('a','b','c')) generate two columns and three rows with: + * pose column: 0, 1, 2 + * value column: 'a', 'b', 'c' + */ +public class PosExplodeOuter extends TableGeneratingFunction implements UnaryExpression, AlwaysNullable { + + /** + * constructor with 1 argument. + */ + public PosExplodeOuter(Expression arg) { + super("posexplode_outer", arg); + } + + /** + * withChildren. + */ + @Override + public PosExplodeOuter withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new PosExplodeOuter(children.get(0)); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!(child().getDataType() instanceof ArrayType)) { + throw new AnalysisException("only support array type for posexplode_outer function but got " + + child().getDataType()); + } + } + + @Override + public List getSignatures() { + return ImmutableList.of( + FunctionSignature.ret(StructLiteral.constructStructType( + Lists.newArrayList(IntegerType.INSTANCE, + ((ArrayType) child().getDataType()).getItemType()))) + .args(child().getDataType())); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPosExplodeOuter(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java index f66d72f6409203..103078f8f7184c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java @@ -37,6 +37,8 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode; +import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; /** @@ -125,4 +127,11 @@ default R visitExplodeJsonArrayJsonOuter(ExplodeJsonArrayJsonOuter explodeJsonAr return visitTableGeneratingFunction(explodeJsonArrayJsonOuter, context); } + default R visitPosExplode(PosExplode posExplode, C context) { + return visitTableGeneratingFunction(posExplode, context); + } + + default R visitPosExplodeOuter(PosExplodeOuter posExplodeOuter, C context) { + return visitTableGeneratingFunction(posExplodeOuter, context); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletHealthTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletHealthTest.java index b22925e5d89270..320bff45229fba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletHealthTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletHealthTest.java @@ -40,12 +40,14 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.MinMaxPriorityQueue; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class TabletHealthTest extends TestWithFeService { @@ -78,6 +80,8 @@ protected void runBeforeAll() throws Exception { @Override protected void runBeforeEach() throws Exception { + // set back to default value + Config.max_scheduling_tablets = 2000; for (Table table : db.getTables()) { dropTable(table.getName(), true); } @@ -358,4 +362,52 @@ public void testColocateTabletHealth() throws Exception { dropTable(table.getName(), true); } + + @Test + public void testAddTabletNoDeadLock() throws Exception { + Config.max_scheduling_tablets = 1; + createTable("CREATE TABLE tbl3 (k INT) DISTRIBUTED BY HASH(k) BUCKETS 2" + + " PROPERTIES ('replication_num' = '3')"); + DebugPointUtil.addDebugPoint("MockedBackendFactory.handleCloneTablet.failed"); + OlapTable table = (OlapTable) db.getTableOrMetaException("tbl3"); + Partition partition = table.getPartitions().iterator().next(); + List tablets = partition.getMaterializedIndices(IndexExtState.ALL).iterator().next().getTablets(); + Assertions.assertEquals(2, tablets.size()); + + partition.updateVisibleVersion(10L); + tablets.forEach(tablet -> tablet.getReplicas().forEach(replica -> replica.updateVersion(10))); + + Tablet tabletA = tablets.get(0); + Tablet tabletB = tablets.get(1); + TabletScheduler scheduler = Env.getCurrentEnv().getTabletScheduler(); + tabletA.getReplicas().get(0).adminUpdateVersionInfo(8L, null, null, 0L); + checkTabletStatus(tabletA, TabletStatus.VERSION_INCOMPLETE, table, partition); + Env.getCurrentEnv().getTabletChecker().runAfterCatalogReady(); + Env.getCurrentEnv().getTabletScheduler().runAfterCatalogReady(); + Thread.sleep(1000); + MinMaxPriorityQueue queue = scheduler.getPendingTabletQueue(); + TabletSchedCtx tabletACtx = queue.peekFirst(); + Assertions.assertNotNull(tabletACtx); + tabletACtx.setLastVisitedTime(System.currentTimeMillis() + 3600 * 1000L); + tabletB.getReplicas().get(0).adminUpdateVersionInfo(8L, null, null, 0L); + checkTabletStatus(tabletB, TabletStatus.VERSION_INCOMPLETE, table, partition); + Thread thread = new Thread(() -> { + try { + Env.getCurrentEnv().getTabletChecker().runAfterCatalogReady(); + Env.getCurrentEnv().getTabletScheduler().runAfterCatalogReady(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + thread.start(); + Thread.sleep(1000); + Assertions.assertTrue(table.tryWriteLock(2, TimeUnit.SECONDS)); + table.writeUnlock(); + DebugPointUtil.clearDebugPoints(); + doRepair(); + Thread.sleep(1000); + doRepair(); + checkTabletIsHealth(tabletA, table, partition); + checkTabletIsHealth(tabletB, table, partition); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java index 3038dbab41d5a8..210408f80dc64f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java @@ -38,7 +38,12 @@ import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -46,6 +51,10 @@ import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.FloatType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -283,10 +292,163 @@ void testDoubleLiteral() { Expression rewrittenExpression = executor.rewrite(expression, context); Assertions.assertEquals(left.child(0).getDataType(), rewrittenExpression.child(1).getDataType()); Assertions.assertEquals(rewrittenExpression.child(0).getDataType(), rewrittenExpression.child(1).getDataType()); + + Expression tinyIntSlot = new SlotReference("a", TinyIntType.INSTANCE); + Expression smallIntSlot = new SlotReference("a", SmallIntType.INSTANCE); + Expression intSlot = new SlotReference("a", IntegerType.INSTANCE); + Expression bigIntSlot = new SlotReference("a", BigIntType.INSTANCE); + + // tiny int, literal not exceeds data type limit + assertRewrite(new EqualTo(new Cast(tinyIntSlot, FloatType.INSTANCE), new FloatLiteral(12.0f)), + new EqualTo(tinyIntSlot, new TinyIntLiteral((byte) 12))); + assertRewrite(new EqualTo(new Cast(tinyIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.0f)), + new EqualTo(tinyIntSlot, new TinyIntLiteral((byte) 12))); + assertRewrite(new EqualTo(new Cast(tinyIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + ExpressionUtils.falseOrNull(tinyIntSlot)); + assertRewrite(new NullSafeEqual(new Cast(tinyIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + BooleanLiteral.FALSE); + assertRewrite(new GreaterThan(new Cast(tinyIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new GreaterThan(tinyIntSlot, new TinyIntLiteral((byte) 12))); + assertRewrite(new GreaterThanEqual(new Cast(tinyIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new GreaterThanEqual(tinyIntSlot, new TinyIntLiteral((byte) 13))); + assertRewrite(new LessThan(new Cast(tinyIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new LessThan(tinyIntSlot, new TinyIntLiteral((byte) 13))); + assertRewrite(new LessThanEqual(new Cast(tinyIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new LessThanEqual(tinyIntSlot, new TinyIntLiteral((byte) 12))); + + // small int + assertRewrite(new EqualTo(new Cast(smallIntSlot, FloatType.INSTANCE), new FloatLiteral(12.0f)), + new EqualTo(smallIntSlot, new SmallIntLiteral((short) 12))); + assertRewrite(new EqualTo(new Cast(smallIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.0f)), + new EqualTo(smallIntSlot, new SmallIntLiteral((short) 12))); + assertRewrite(new EqualTo(new Cast(smallIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + ExpressionUtils.falseOrNull(smallIntSlot)); + assertRewrite(new NullSafeEqual(new Cast(smallIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + BooleanLiteral.FALSE); + assertRewrite(new GreaterThan(new Cast(smallIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new GreaterThan(smallIntSlot, new SmallIntLiteral((short) 12))); + assertRewrite(new GreaterThanEqual(new Cast(smallIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new GreaterThanEqual(smallIntSlot, new SmallIntLiteral((short) 13))); + assertRewrite(new LessThan(new Cast(smallIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new LessThan(smallIntSlot, new SmallIntLiteral((short) 13))); + assertRewrite(new LessThanEqual(new Cast(smallIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new LessThanEqual(smallIntSlot, new SmallIntLiteral((short) 12))); + + // int + assertRewrite(new EqualTo(new Cast(intSlot, FloatType.INSTANCE), new FloatLiteral(12.0f)), + new EqualTo(intSlot, new IntegerLiteral(12))); + assertRewrite(new EqualTo(new Cast(intSlot, DoubleType.INSTANCE), new DoubleLiteral(12.0f)), + new EqualTo(intSlot, new IntegerLiteral(12))); + assertRewrite(new EqualTo(new Cast(intSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + ExpressionUtils.falseOrNull(intSlot)); + assertRewrite(new NullSafeEqual(new Cast(intSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + BooleanLiteral.FALSE); + assertRewrite(new GreaterThan(new Cast(intSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new GreaterThan(intSlot, new IntegerLiteral(12))); + assertRewrite(new GreaterThanEqual(new Cast(intSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new GreaterThanEqual(intSlot, new IntegerLiteral(13))); + assertRewrite(new LessThan(new Cast(intSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new LessThan(intSlot, new IntegerLiteral(13))); + assertRewrite(new LessThanEqual(new Cast(intSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new LessThanEqual(intSlot, new IntegerLiteral(12))); + + // big int + assertRewrite(new EqualTo(new Cast(bigIntSlot, FloatType.INSTANCE), new FloatLiteral(12.0f)), + new EqualTo(bigIntSlot, new BigIntLiteral(12L))); + assertRewrite(new EqualTo(new Cast(bigIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.0f)), + new EqualTo(bigIntSlot, new BigIntLiteral(12L))); + assertRewrite(new EqualTo(new Cast(bigIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + ExpressionUtils.falseOrNull(bigIntSlot)); + assertRewrite(new NullSafeEqual(new Cast(bigIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + BooleanLiteral.FALSE); + assertRewrite(new GreaterThan(new Cast(bigIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new GreaterThan(bigIntSlot, new BigIntLiteral(12L))); + assertRewrite(new GreaterThanEqual(new Cast(bigIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new GreaterThanEqual(bigIntSlot, new BigIntLiteral(13L))); + assertRewrite(new LessThan(new Cast(bigIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new LessThan(bigIntSlot, new BigIntLiteral(13L))); + assertRewrite(new LessThanEqual(new Cast(bigIntSlot, DoubleType.INSTANCE), new DoubleLiteral(12.3f)), + new LessThanEqual(bigIntSlot, new BigIntLiteral(12L))); + } + + @Test + void testIntCmpDecimalV3Literal() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(SimplifyComparisonPredicate.INSTANCE) + )); + + Expression tinyIntSlot = new SlotReference("a", TinyIntType.INSTANCE); + Expression smallIntSlot = new SlotReference("a", SmallIntType.INSTANCE); + Expression intSlot = new SlotReference("a", IntegerType.INSTANCE); + Expression bigIntSlot = new SlotReference("a", BigIntType.INSTANCE); + + // tiny int, literal not exceeds data type limit + assertRewrite(new EqualTo(new Cast(tinyIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.0"))), + new EqualTo(tinyIntSlot, new TinyIntLiteral((byte) 12))); + assertRewrite(new EqualTo(new Cast(tinyIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + ExpressionUtils.falseOrNull(tinyIntSlot)); + assertRewrite(new NullSafeEqual(new Cast(tinyIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + BooleanLiteral.FALSE); + assertRewrite(new GreaterThan(new Cast(tinyIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new GreaterThan(tinyIntSlot, new TinyIntLiteral((byte) 12))); + assertRewrite(new GreaterThanEqual(new Cast(tinyIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new GreaterThanEqual(tinyIntSlot, new TinyIntLiteral((byte) 13))); + assertRewrite(new LessThan(new Cast(tinyIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new LessThan(tinyIntSlot, new TinyIntLiteral((byte) 13))); + assertRewrite(new LessThanEqual(new Cast(tinyIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new LessThanEqual(tinyIntSlot, new TinyIntLiteral((byte) 12))); + + // small int + assertRewrite(new EqualTo(new Cast(smallIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.0"))), + new EqualTo(smallIntSlot, new SmallIntLiteral((short) 12))); + assertRewrite(new EqualTo(new Cast(smallIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + ExpressionUtils.falseOrNull(smallIntSlot)); + assertRewrite(new NullSafeEqual(new Cast(smallIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + BooleanLiteral.FALSE); + assertRewrite(new GreaterThan(new Cast(smallIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new GreaterThan(smallIntSlot, new SmallIntLiteral((short) 12))); + assertRewrite(new GreaterThanEqual(new Cast(smallIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new GreaterThanEqual(smallIntSlot, new SmallIntLiteral((short) 13))); + assertRewrite(new LessThan(new Cast(smallIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new LessThan(smallIntSlot, new SmallIntLiteral((short) 13))); + assertRewrite(new LessThanEqual(new Cast(smallIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new LessThanEqual(smallIntSlot, new SmallIntLiteral((short) 12))); + + // int + assertRewrite(new EqualTo(new Cast(intSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.0"))), + new EqualTo(intSlot, new IntegerLiteral(12))); + assertRewrite(new EqualTo(new Cast(intSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + ExpressionUtils.falseOrNull(intSlot)); + assertRewrite(new NullSafeEqual(new Cast(intSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + BooleanLiteral.FALSE); + assertRewrite(new GreaterThan(new Cast(intSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new GreaterThan(intSlot, new IntegerLiteral(12))); + assertRewrite(new GreaterThanEqual(new Cast(intSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new GreaterThanEqual(intSlot, new IntegerLiteral(13))); + assertRewrite(new LessThan(new Cast(intSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new LessThan(intSlot, new IntegerLiteral(13))); + assertRewrite(new LessThanEqual(new Cast(intSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new LessThanEqual(intSlot, new IntegerLiteral(12))); + + // big int + assertRewrite(new EqualTo(new Cast(bigIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.0"))), + new EqualTo(bigIntSlot, new BigIntLiteral(12L))); + assertRewrite(new EqualTo(new Cast(bigIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + ExpressionUtils.falseOrNull(bigIntSlot)); + assertRewrite(new NullSafeEqual(new Cast(bigIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + BooleanLiteral.FALSE); + assertRewrite(new GreaterThan(new Cast(bigIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new GreaterThan(bigIntSlot, new BigIntLiteral(12L))); + assertRewrite(new GreaterThanEqual(new Cast(bigIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new GreaterThanEqual(bigIntSlot, new BigIntLiteral(13L))); + assertRewrite(new LessThan(new Cast(bigIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new LessThan(bigIntSlot, new BigIntLiteral(13L))); + assertRewrite(new LessThanEqual(new Cast(bigIntSlot, DecimalV3Type.createDecimalV3Type(3, 1)), new DecimalV3Literal(new BigDecimal("12.3"))), + new LessThanEqual(bigIntSlot, new BigIntLiteral(12L))); } @Test - void testDecimalV3Literal() { + void testDecimalCmpDecimalV3Literal() { executor = new ExpressionRuleExecutor(ImmutableList.of( bottomUp(SimplifyComparisonPredicate.INSTANCE) )); diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java index 93ce8b6766ba3e..4e2a9fa056b940 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java @@ -83,6 +83,7 @@ import org.apache.thrift.TException; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Random; import java.util.concurrent.BlockingQueue; @@ -293,6 +294,10 @@ private void handleCloneTablet(TAgentTaskRequest request, TFinishTaskRequest fin tabletInfo.setPathHash(pathHash); tabletInfo.setUsed(true); tabletInfos.add(tabletInfo); + if (DebugPointUtil.isEnable("MockedBackendFactory.handleCloneTablet.failed")) { + finishTaskRequest.setTaskStatus(new TStatus(TStatusCode.CANCELLED)); + finishTaskRequest.getTaskStatus().setErrorMsgs(Collections.singletonList("debug point set")); + } finishTaskRequest.setFinishTabletInfos(tabletInfos); } diff --git a/regression-test/data/nereids_p0/sql_functions/table_function/posexplode.out b/regression-test/data/nereids_p0/sql_functions/table_function/posexplode.out new file mode 100644 index 00000000000000..393e13a2b546a2 --- /dev/null +++ b/regression-test/data/nereids_p0/sql_functions/table_function/posexplode.out @@ -0,0 +1,166 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +0 zhangsan ["Chinese", "Math", "English"] +1 lisi ["null"] +2 wangwu ["88a", "90b", "96c"] +3 lisi2 [null] +4 amory \N + +-- !explode_sql -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N + +-- !explode_outer_sql -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N +4 amory \N \N \N + +-- !explode_sql_multi -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 1 Math +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 2 English +0 zhangsan ["Chinese", "Math", "English"] 1 Math 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math 1 Math +0 zhangsan ["Chinese", "Math", "English"] 1 Math 2 English +0 zhangsan ["Chinese", "Math", "English"] 2 English 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 2 English 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English 2 English +1 lisi ["null"] 0 null 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a 0 88a +2 wangwu ["88a", "90b", "96c"] 0 88a 1 90b +2 wangwu ["88a", "90b", "96c"] 0 88a 2 96c +2 wangwu ["88a", "90b", "96c"] 1 90b 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b 1 90b +2 wangwu ["88a", "90b", "96c"] 1 90b 2 96c +2 wangwu ["88a", "90b", "96c"] 2 96c 0 88a +2 wangwu ["88a", "90b", "96c"] 2 96c 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c 2 96c +3 lisi2 [null] 0 \N 0 \N + +-- !explode_sql_alias -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N + +-- !explode_outer_sql_alias -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N +4 amory \N \N \N + +-- !explode_sql_alias_multi -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 1 Math +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 2 English +0 zhangsan ["Chinese", "Math", "English"] 1 Math 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math 1 Math +0 zhangsan ["Chinese", "Math", "English"] 1 Math 2 English +0 zhangsan ["Chinese", "Math", "English"] 2 English 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 2 English 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English 2 English +1 lisi ["null"] 0 null 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a 0 88a +2 wangwu ["88a", "90b", "96c"] 0 88a 1 90b +2 wangwu ["88a", "90b", "96c"] 0 88a 2 96c +2 wangwu ["88a", "90b", "96c"] 1 90b 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b 1 90b +2 wangwu ["88a", "90b", "96c"] 1 90b 2 96c +2 wangwu ["88a", "90b", "96c"] 2 96c 0 88a +2 wangwu ["88a", "90b", "96c"] 2 96c 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c 2 96c +3 lisi2 [null] 0 \N 0 \N + +-- !sql -- +0 zhangsan ["Chinese", "Math", "English"] +1 lisi ["null"] +2 wangwu ["88a", "90b", "96c"] +3 lisi2 [null] +4 liuba [] + +-- !explode_sql_not -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N + +-- !explode_outer_sql_not -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N +4 liuba [] \N \N + +-- !explode_sql_alias_multi2 -- +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":2, "col":"English"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":2, "col":"English"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":2, "col":"English"} +1 lisi ["null"] {"pos":0, "col":"null"} {"pos":0, "col":"null"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":2, "col":"96c"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":2, "col":"96c"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":2, "col":"96c"} +3 lisi2 [null] {"pos":0, "col":null} {"pos":0, "col":null} + +-- !explode_sql_alias_multi3 -- +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":2, "col":"English"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":2, "col":"English"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":2, "col":"English"} +1 lisi ["null"] {"pos":0, "col":"null"} {"pos":0, "col":"null"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":2, "col":"96c"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":2, "col":"96c"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":2, "col":"96c"} +3 lisi2 [null] {"pos":0, "col":null} {"pos":0, "col":null} + diff --git a/regression-test/suites/account_p0/test_system_db.groovy b/regression-test/suites/account_p0/test_system_db.groovy new file mode 100644 index 00000000000000..11b9d6d492bf42 --- /dev/null +++ b/regression-test/suites/account_p0/test_system_db.groovy @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_system_db","p0,auth") { + String suiteName = "test_system_db" + String user = "${suiteName}_user" + String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + + sql """ + grant select_priv on __internal_schema.* to `${user}`; + """ + sql """ + grant select_priv on information_schema.* to `${user}`; + """ + sql """ + grant select_priv on mysql.* to `${user}`; + """ + sql """ + revoke select_priv on __internal_schema.* from `${user}`; + """ + sql """ + revoke select_priv on information_schema.* from `${user}`; + """ + sql """ + revoke select_priv on mysql.* from `${user}`; + """ + try_sql("DROP USER ${user}") +} diff --git a/regression-test/suites/account_p0/test_system_role.groovy b/regression-test/suites/account_p0/test_system_role.groovy new file mode 100644 index 00000000000000..64c0f122fa21dd --- /dev/null +++ b/regression-test/suites/account_p0/test_system_role.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_system_role","p0,auth") { + test { + sql """ + drop role operator; + """ + exception "Can not drop role" + } + + test { + sql """ + drop role `admin`; + """ + exception "Can not drop role" + } + + test { + sql """ + grant select_priv on *.*.* to role "operator"; + """ + exception "Can not grant" + } + test { + sql """ + grant select_priv on *.*.* to role "admin"; + """ + exception "Can not grant" + } + test { + sql """ + revoke Node_priv on *.*.* from role 'operator'; + """ + exception "Can not revoke" + } + + test { + sql """ + revoke Admin_priv on *.*.* from role 'admin'; + """ + exception "Can not revoke" + } + +} diff --git a/regression-test/suites/account_p0/test_system_user.groovy b/regression-test/suites/account_p0/test_system_user.groovy index 1805f1669ea570..5993e1d238b444 100644 --- a/regression-test/suites/account_p0/test_system_user.groovy +++ b/regression-test/suites/account_p0/test_system_user.groovy @@ -17,7 +17,7 @@ import org.junit.Assert; -suite("test_system_user") { +suite("test_system_user","p0,auth") { test { sql """ create user `root`; @@ -36,4 +36,30 @@ suite("test_system_user") { """ exception "system" } + test { + sql """ + revoke "operator" from root; + """ + exception "Can not revoke role" + } + test { + sql """ + revoke 'admin' from `admin`; + """ + exception "Unsupported operation" + } + + sql """ + grant select_priv on *.*.* to `root`; + """ + sql """ + revoke select_priv on *.*.* from `root`; + """ + sql """ + grant select_priv on *.*.* to `admin`; + """ + sql """ + revoke select_priv on *.*.* from `admin`; + """ + } diff --git a/regression-test/suites/auth_p0/test_catalogs_auth.groovy b/regression-test/suites/auth_p0/test_catalogs_auth.groovy new file mode 100644 index 00000000000000..96ebcef7cf81cb --- /dev/null +++ b/regression-test/suites/auth_p0/test_catalogs_auth.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_catalogs_auth","p0,auth") { + String suiteName = "test_catalogs_auth" + String catalogName = "${suiteName}_catalog" + String user = "${suiteName}_user" + String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + + sql """drop catalog if exists ${catalogName}""" + sql """CREATE CATALOG ${catalogName} PROPERTIES ( + "type"="es", + "hosts"="http://8.8.8.8:9200" + );""" + + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + } + + sql """grant select_priv on regression_test to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + def showRes = sql """show catalogs;""" + logger.info("showRes: " + showRes.toString()) + assertFalse(showRes.toString().contains("${catalogName}")) + + def tvfRes = sql """select * from catalogs();""" + logger.info("tvfRes: " + tvfRes.toString()) + assertFalse(tvfRes.toString().contains("${catalogName}")) + } + + sql """grant select_priv on ${catalogName}.*.* to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + def showRes = sql """show catalogs;""" + logger.info("showRes: " + showRes.toString()) + assertTrue(showRes.toString().contains("${catalogName}")) + + def tvfRes = sql """select * from catalogs();""" + logger.info("tvfRes: " + tvfRes.toString()) + assertTrue(tvfRes.toString().contains("${catalogName}")) + } + + try_sql("DROP USER ${user}") + sql """drop catalog if exists ${catalogName}""" +} diff --git a/regression-test/suites/auth_p0/test_mtmv_auth.groovy b/regression-test/suites/auth_p0/test_mtmv_auth.groovy new file mode 100644 index 00000000000000..52ecbebb70b268 --- /dev/null +++ b/regression-test/suites/auth_p0/test_mtmv_auth.groovy @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_mtmv_auth","p0,auth") { + String suiteName = "test_mtmv_auth" + String dbName = context.config.getDbNameByFile(context.file) + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + String user = "${suiteName}_user" + String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName};""" + sql """drop table if exists `${tableName}`""" + sql """ + CREATE TABLE `${tableName}` ( + `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', + `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"', + `num` SMALLINT NOT NULL COMMENT '\"数量\"' + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`, `date`, `num`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + select * from ${tableName}; + """ + + sql """refresh MATERIALIZED VIEW ${mvName} auto""" + waitingMTMVTaskFinishedByMvName(mvName) + + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + } + + sql """grant select_priv on regression_test to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + def mvsRes = sql """select * from mv_infos("database"="${dbName}");""" + logger.info("mvsRes: " + mvsRes.toString()) + assertFalse(mvsRes.toString().contains("${mvName}")) + + def jobsRes = sql """select * from jobs("type"="mv");""" + logger.info("jobsRes: " + jobsRes.toString()) + assertFalse(jobsRes.toString().contains("${mvName}")) + + def tasksRes = sql """select * from tasks("type"="mv");""" + logger.info("tasksRes: " + tasksRes.toString()) + assertFalse(tasksRes.toString().contains("${mvName}")) + + } + + sql """grant select_priv on ${dbName}.${mvName} to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + def mvsRes = sql """select * from mv_infos("database"="${dbName}");""" + logger.info("mvsRes: " + mvsRes.toString()) + assertTrue(mvsRes.toString().contains("${mvName}")) + + def jobsRes = sql """select * from jobs("type"="mv");""" + logger.info("jobsRes: " + jobsRes.toString()) + assertTrue(jobsRes.toString().contains("${mvName}")) + + def tasksRes = sql """select * from tasks("type"="mv");""" + logger.info("tasksRes: " + tasksRes.toString()) + assertTrue(tasksRes.toString().contains("${mvName}")) + } + + try_sql("DROP USER ${user}") + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName};""" + sql """drop table if exists `${tableName}`""" +} diff --git a/regression-test/suites/auth_p0/test_partition_values_tvf_auth.groovy b/regression-test/suites/auth_p0/test_partition_values_tvf_auth.groovy new file mode 100644 index 00000000000000..3f0ae7ea8d524c --- /dev/null +++ b/regression-test/suites/auth_p0/test_partition_values_tvf_auth.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_partition_values_tvf_auth","p0,auth") { + String suiteName = "test_partition_values_tvf_auth" + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return; + } + + for (String hivePrefix : ["hive3"]) { + String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp") + String extHiveHmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${hivePrefix}_test_external_catalog_hive_partition" + + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + String user = "${suiteName}_user" + String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + } + + sql """grant select_priv on regression_test to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + test { + sql """ + select * from partition_values("catalog" = "${catalog_name}", "database" = "multi_catalog", "table" = "orc_partitioned_columns") order by t_int, t_float; + """ + exception "denied" + } + } + sql """grant select_priv on ${catalog_name}.multi_catalog.orc_partitioned_columns to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """ + select * from partition_values("catalog" = "${catalog_name}", "database" = "multi_catalog", "table" = "orc_partitioned_columns") order by t_int, t_float; + """ + } + try_sql("DROP USER ${user}") + sql """drop catalog if exists ${catalog_name}""" + } +} + diff --git a/regression-test/suites/auth_p0/test_partitions_auth.groovy b/regression-test/suites/auth_p0/test_partitions_auth.groovy new file mode 100644 index 00000000000000..0b769f11567845 --- /dev/null +++ b/regression-test/suites/auth_p0/test_partitions_auth.groovy @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_partitions_auth","p0,auth") { + String suiteName = "test_partitions_auth" + String dbName = context.config.getDbNameByFile(context.file) + String tableName = "${suiteName}_table" + String user = "${suiteName}_user" + String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + + sql """drop table if exists `${tableName}`""" + sql """ + CREATE TABLE `${tableName}` ( + `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', + `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"', + `num` SMALLINT NOT NULL COMMENT '\"数量\"' + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`, `date`, `num`) + COMMENT 'OLAP' + PARTITION BY RANGE(`date`) + (PARTITION p201701_1000 VALUES [('0000-01-01'), ('2017-02-01')), + PARTITION p201702_2000 VALUES [('2017-02-01'), ('2017-03-01')), + PARTITION p201703_all VALUES [('2017-03-01'), ('2017-04-01'))) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + } + + sql """grant select_priv on regression_test to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + test { + sql """ + show partitions from ${dbName}.${tableName}; + """ + exception "denied" + } + test { + sql """ + select * from partitions('catalog'='internal',"database"="${dbName}","table"="${tableName}"); + """ + exception "denied" + } + } + + sql """grant select_priv on ${dbName}.${tableName} to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """ + show partitions from ${dbName}.${tableName}; + """ + sql """ + select * from partitions('catalog'='internal',"database"="${dbName}","table"="${tableName}"); + """ + } + + try_sql("DROP USER ${user}") + sql """drop table if exists `${tableName}`""" +} diff --git a/regression-test/suites/auth_p0/test_query_tvf_auth.groovy b/regression-test/suites/auth_p0/test_query_tvf_auth.groovy new file mode 100644 index 00000000000000..05c274077d9eb3 --- /dev/null +++ b/regression-test/suites/auth_p0/test_query_tvf_auth.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_jdbc_query_tvf","p0,auth") { + String suiteName = "test_jdbc_query_tvf" + String enabled = context.config.otherConfigs.get("enableJdbcTest") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar" + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String user = "test_jdbc_user"; + String pwd = '123456'; + String catalog_name = "${suiteName}_catalog" + String mysql_port = context.config.otherConfigs.get("mysql_57_port"); + + sql """drop catalog if exists ${catalog_name} """ + + sql """create catalog if not exists ${catalog_name} properties( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver" + );""" + + String dorisuser = "${suiteName}_user" + String dorispwd = 'C123_567p' + try_sql("DROP USER ${dorisuser}") + sql """CREATE USER '${dorisuser}' IDENTIFIED BY '${dorispwd}'""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${dorisuser}"""; + } + + sql """grant select_priv on regression_test to ${dorisuser}""" + + connect(user=dorisuser, password="${dorispwd}", url=context.config.jdbcUrl) { + test { + sql """ + select * from query('catalog' = '${catalog_name}', 'query' = 'select * from doris_test.all_types'); + """ + exception "denied" + } + } + sql """grant select_priv on ${catalog_name}.*.* to ${dorisuser}""" + connect(user=dorisuser, password="${dorispwd}", url=context.config.jdbcUrl) { + sql """ + select * from query('catalog' = '${catalog_name}', 'query' = 'select * from doris_test.all_types'); + """ + } + try_sql("DROP USER ${dorisuser}") + sql """drop catalog if exists ${catalog_name} """ + } +} + diff --git a/regression-test/suites/auth_p0/test_select_count_auth.groovy b/regression-test/suites/auth_p0/test_select_count_auth.groovy new file mode 100644 index 00000000000000..ccea1a4a580098 --- /dev/null +++ b/regression-test/suites/auth_p0/test_select_count_auth.groovy @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_select_count_auth","p0,auth") { + String suiteName = "test_select_count_auth" + String user = "${suiteName}_user" + String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + } + + sql """grant select_priv on regression_test to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + test { + sql """ + select count(*) from __internal_schema.audit_log; + """ + exception "denied" + } + test { + sql """ + select count(1) from __internal_schema.audit_log; + """ + exception "denied" + } + test { + sql """ + select count(query_id) from __internal_schema.audit_log; + """ + exception "denied" + } + } + + sql """grant select_priv(query_id) on __internal_schema.audit_log to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + test { + sql """ + select count(*) from __internal_schema.audit_log; + """ + exception "denied" + } + test { + sql """ + select count(1) from __internal_schema.audit_log; + """ + exception "denied" + } + sql """ + select count(query_id) from __internal_schema.audit_log; + """ + } + + sql """grant select_priv on __internal_schema.audit_log to ${user}""" + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """ + select count(*) from __internal_schema.audit_log; + """ + sql """ + select count(1) from __internal_schema.audit_log; + """ + sql """ + select count(query_id) from __internal_schema.audit_log; + """ + } + + try_sql("DROP USER ${user}") +} diff --git a/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy b/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy index 047b4a36fe2622..557eaf5b061d70 100644 --- a/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy +++ b/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy @@ -16,7 +16,7 @@ // under the License. suite("test_iceberg_meta", "p2,external,iceberg,external_remote,external_remote_iceberg") { - + String suiteName = "test_iceberg_meta" Boolean ignoreP2 = true; if (ignoreP2) { logger.info("disable p2 test"); @@ -54,5 +54,37 @@ suite("test_iceberg_meta", "p2,external,iceberg,external_remote,external_remote_ "query_type" = "snapshots") where snapshot_id = 7235593032487457798; """ + String user = "${suiteName}_user" + String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + } + + sql """grant select_priv on regression_test to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + test { + sql """ + select committed_at, snapshot_id, parent_id, operation from iceberg_meta( + "table" = "${iceberg_catalog_name}.${db}.multi_partition", + "query_type" = "snapshots"); + """ + exception "denied" + } + } + sql """grant select_priv on ${iceberg_catalog_name}.${db}.multi_partition to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """ + select committed_at, snapshot_id, parent_id, operation from iceberg_meta( + "table" = "${iceberg_catalog_name}.${db}.multi_partition", + "query_type" = "snapshots"); + """ + } + try_sql("DROP USER ${user}") } } diff --git a/regression-test/suites/nereids_p0/sql_functions/table_function/posexplode.groovy b/regression-test/suites/nereids_p0/sql_functions/table_function/posexplode.groovy new file mode 100644 index 00000000000000..8320af92f48ff5 --- /dev/null +++ b/regression-test/suites/nereids_p0/sql_functions/table_function/posexplode.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("posexplode") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + sql """ DROP TABLE IF EXISTS table_test """ + sql """ + CREATE TABLE IF NOT EXISTS `table_test`( + `id` INT NULL, + `name` TEXT NULL, + `score` array NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + // insert values + sql """ insert into table_test values (0, "zhangsan", ["Chinese","Math","English"]); """ + sql """ insert into table_test values (1, "lisi", ["null"]); """ + sql """ insert into table_test values (2, "wangwu", ["88a","90b","96c"]); """ + sql """ insert into table_test values (3, "lisi2", [null]); """ + sql """ insert into table_test values (4, "amory", NULL); """ + + qt_sql """ select * from table_test order by id; """ + order_qt_explode_sql """ select id,name,score, k,v from table_test lateral view posexplode(score) tmp as k,v order by id;""" + order_qt_explode_outer_sql """ select id,name,score, k,v from table_test lateral view posexplode_outer(score) tmp as k,v order by id; """ + + // multi lateral view + order_qt_explode_sql_multi """ select id,name,score, k,v,k1,v1 from table_test lateral view posexplode_outer(score) tmp as k,v lateral view posexplode(score) tmp2 as k1,v1 order by id;""" + + // test with alias + order_qt_explode_sql_alias """ select id,name,score, tmp.k, tmp.v from table_test lateral view posexplode(score) tmp as k,v order by id;""" + order_qt_explode_outer_sql_alias """ select id,name,score, tmp.k, tmp.v from table_test lateral view posexplode_outer(score) tmp as k,v order by id; """ + + order_qt_explode_sql_alias_multi """ select id,name,score, tmp.k, tmp.v, tmp2.k, tmp2.v from table_test lateral view posexplode_outer(score) tmp as k,v lateral view posexplode(score) tmp2 as k,v order by id;""" + + sql """ DROP TABLE IF EXISTS table_test_not """ + sql """ + CREATE TABLE IF NOT EXISTS `table_test_not`( + `id` INT NULL, + `name` TEXT NULL, + `score` array not NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + // insert values + sql """ insert into table_test_not values (0, "zhangsan", ["Chinese","Math","English"]); """ + sql """ insert into table_test_not values (1, "lisi", ["null"]); """ + sql """ insert into table_test_not values (2, "wangwu", ["88a","90b","96c"]); """ + sql """ insert into table_test_not values (3, "lisi2", [null]); """ + sql """ insert into table_test_not values (4, "liuba", []); """ + + qt_sql """ select * from table_test_not order by id; """ + order_qt_explode_sql_not """ select id,name,score, k,v from table_test_not lateral view posexplode(score) tmp as k,v order by id;""" + order_qt_explode_outer_sql_not """ select id,name,score, k,v from table_test_not lateral view posexplode_outer(score) tmp as k,v order by id; """ + order_qt_explode_sql_alias_multi2 """ select * from table_test_not lateral view posexplode(score) tmp as e1 lateral view posexplode(score) tmp2 as e2 order by id;""" + sql """ set batch_size = 1; """ + order_qt_explode_sql_alias_multi3 """ select * from table_test_not lateral view posexplode(score) tmp as e1 lateral view posexplode(score) tmp2 as e2 order by id;""" + +} diff --git a/regression-test/suites/nereids_rules_p0/expression/test_simplify_comparison_predicate.groovy b/regression-test/suites/nereids_rules_p0/expression/test_simplify_comparison_predicate.groovy new file mode 100644 index 00000000000000..af975aeeaa22e7 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/expression/test_simplify_comparison_predicate.groovy @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// TODO: date datetime comparison still has bug, need fix. +suite('test_simplify_comparison_predicate', 'nonConcurrent') { + def tbl = 'test_simplify_comparison_predicate_tbl' + def checkExplain = { expression, resExpression -> + def checker = { explainString, exception, startTime, endTime -> + assertNull(exception) + def foundOutputExprs = false + def succ = false + for (def line : explainString.split('\n')) { + if (foundOutputExprs) { + assertTrue(line.contains(resExpression), "'${line}' no contains '${resExpression}'") + succ = true + break + } + if (line.contains('OUTPUT EXPRS:')) { + foundOutputExprs = true + } + } + assertTrue(foundOutputExprs) + assertTrue(succ) + } + + explain { + sql "SELECT ${expression} FROM ${tbl}" + check checker + } + } + def testSimplify = { checkNullColumn, checkNotNullColumn, expression, resExpression -> + def types = [''] + def column = '' + if (expression.contains('{int_like_column}')) { + column = '{int_like_column}' + types = ['tinyint', 'smallint', 'int', 'bigint'] + } else if (expression.contains('{decimal_column}')) { + column = '{decimal_column}' + types = ['decimal_3_0', 'decimal_5_2'] + } else if (expression.contains('{date_column}')) { + column = '{date_column}' + types = ['date', 'datev1'] + } else if (expression.contains('{datetime_column}')) { + column = '{datetime_column}' + types = ['datetime_0', 'datetime_3', 'datetimev1'] + } + for (def type : types) { + if (type == '') { + checkExplain expression, resExpression + } else { + if (checkNullColumn) { + checkExplain expression.replace(column, "c_${type}_null"), resExpression.replace(column, "c_${type}_null") + } + if (checkNotNullColumn) { + checkExplain expression.replace(column, "c_${type}"), resExpression.replace(column, "c_${type}") + } + } + } + } + + setFeConfigTemporary([disable_datev1:false, disable_decimalv2:false]) { + sql """ + DROP TABLE IF EXISTS ${tbl} FORCE; + + CREATE TABLE ${tbl} ( + c_tinyint tinyint not null default 1, + c_tinyint_null tinyint, + c_smallint smallint not null default 1, + c_smallint_null smallint, + c_int int not null default 1, + c_int_null int, + c_bigint bigint not null default 1, + c_bigint_null bigint, + c_decimal_3_0 decimal(3, 0) not null default 1, + c_decimal_3_0_null decimal(3, 0), + c_decimal_5_2 decimal(5, 2) not null default 1, + c_decimal_5_2_null decimal(5, 2), + c_date date not null default '2000-01-01', + c_date_null date, + c_datev1 datev1 not null default '2000-01-01', + c_datev1_null datev1 null, + c_datetime_0 datetime(0) not null default '2000-01-01 00:00:00', + c_datetime_0_null datetime(0), + c_datetime_3 datetime(3) not null default '2000-01-01 00:00:00', + c_datetime_3_null datetime(3), + c_datetimev1 datetimev1 not null default '2000-01-01 00:00:00', + c_datetimev1_null datetimev1 + ) + PROPERTIES ('replication_num' = '1'); + + INSERT INTO ${tbl} VALUES(); + """ + + testSimplify true, true, '{int_like_column} = CAST(1.00 as DOUBLE)', '({int_like_column} = 1)' + testSimplify true, false, '{int_like_column} = CAST(1.01 as DOUBLE)', 'AND[{int_like_column} IS NULL,NULL]' + testSimplify false, true, '{int_like_column} = CAST(1.01 as DOUBLE)', 'FALSE' + testSimplify true, true, '{int_like_column} <=> CAST(1.01 as DOUBLE)', 'FALSE' + testSimplify true, true, '{int_like_column} > CAST(1.00 as DOUBLE)', '({int_like_column} > 1)' + testSimplify true, true, '{int_like_column} < CAST(1.00 as DOUBLE)', '({int_like_column} < 1)' + testSimplify true, true, '{int_like_column} > CAST(1.01 as DOUBLE)', '({int_like_column} > 1)' + testSimplify true, true, '{int_like_column} >= CAST(1.01 as DOUBLE)', '({int_like_column} >= 2)' + testSimplify true, true, '{int_like_column} <= CAST(1.01 as DOUBLE)', '({int_like_column} <= 1)' + testSimplify true, true, '{int_like_column} < CAST(1.01 as DOUBLE)', '({int_like_column} < 2)' + testSimplify true, true, '{int_like_column} = 1.00', '({int_like_column} = 1)' + testSimplify true, true, '{int_like_column} > 1.00', '({int_like_column} > 1)' + testSimplify true, true, '{int_like_column} < 1.00', '({int_like_column} < 1)' + testSimplify true, false, '{int_like_column} = 1.01', 'AND[{int_like_column} IS NULL,NULL]' + testSimplify false, true, '{int_like_column} = 1.01', 'FALSE' + testSimplify true, true, '{int_like_column} <=> 1.01', 'FALSE' + testSimplify true, true, '{int_like_column} > 1.01', '({int_like_column} > 1)' + testSimplify true, true, '{int_like_column} >= 1.01', '({int_like_column} >= 2)' + testSimplify true, true, '{int_like_column} <= 1.01', '({int_like_column} <= 1)' + testSimplify true, true, '{int_like_column} < 1.01', '({int_like_column} < 2)' + testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) = CAST(1.00 as DECIMAL(10, 5))', '(c_decimal_3_0_null = 1)' + testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) = CAST(1.1 as DECIMAL(10, 5))', 'AND[c_decimal_3_0_null IS NULL,NULL]' + testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) > CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_3_0_null > 1)' + testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) >= CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_3_0_null >= 2)' + testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) < CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_3_0_null < 2)' + testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) <= CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_3_0_null <= 1)' + testSimplify false, false, 'c_decimal_5_2_null = CAST(1.0 as DECIMAL(10, 5))', '(c_decimal_5_2_null = 1.00)' + testSimplify false, false, 'c_decimal_5_2_null = CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_5_2_null = 1.10)' + testSimplify false, false, 'c_decimal_5_2_null = CAST(1.12 as DECIMAL(10, 5))', '(c_decimal_5_2_null = 1.12)' + testSimplify false, false, 'c_decimal_5_2_null = CAST(1.123 as DECIMAL(10, 5))', 'AND[c_decimal_5_2_null IS NULL,NULL]' + testSimplify false, false, 'c_decimal_5_2 = CAST(1.123 as DECIMAL(10, 5))', 'FALSE' + testSimplify false, false, 'c_decimal_5_2_null > CAST(1.123 as DECIMAL(10, 5))', 'c_decimal_5_2_null > 1.12' + testSimplify false, false, 'c_decimal_5_2_null >= CAST(1.123 as DECIMAL(10, 5))', 'c_decimal_5_2_null >= 1.13' + testSimplify false, false, 'c_decimal_5_2_null <= CAST(1.123 as DECIMAL(10, 5))', 'c_decimal_5_2_null <= 1.12' + testSimplify false, false, 'c_decimal_5_2_null < CAST(1.123 as DECIMAL(10, 5))', 'c_decimal_5_2_null < 1.13' + testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) = '2000-01-01'", "(c_datetime_0 = '2000-01-01 00:00:00')" + testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) = '2000-01-01 00:00:00.1'", 'FALSE' + testSimplify false, false, "CAST(c_datetime_0_null AS DATETIME(5)) = '2000-01-01 00:00:00.1'", 'AND[c_datetime_0_null IS NULL,NULL]' + testSimplify false, false, "CAST(c_datetime_0_null AS DATETIME(5)) <=> '2000-01-01 00:00:00.1'", 'FALSE' + testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) >= '2000-01-01 00:00:00.1'", "(c_datetime_0 >= '2000-01-01 00:00:01')" + testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) > '2000-01-01 00:00:00.1'", "(c_datetime_0 > '2000-01-01 00:00:00')" + testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) <= '2000-01-01 00:00:00.1'", "(c_datetime_0 <= '2000-01-01 00:00:00')" + testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) < '2000-01-01 00:00:00.1'", "(c_datetime_0 < '2000-01-01 00:00:01')" + testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) = '2000-01-01'", "(c_datetime_3 = '2000-01-01 00:00:00.000')" + testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) = '2000-01-01 00:00:00.1234'", 'FALSE' + testSimplify false, false, "CAST(c_datetime_3_null AS DATETIME(5)) = '2000-01-01 00:00:00.1234'", 'AND[c_datetime_3_null IS NULL,NULL]' + testSimplify false, false, "CAST(c_datetime_3_null AS DATETIME(5)) <=> '2000-01-01 00:00:00.1234'", 'FALSE' + testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) >= '2000-01-01 00:00:00.1234'", "(c_datetime_3 >= '2000-01-01 00:00:00.124')" + testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) > '2000-01-01 00:00:00.1234'", "(c_datetime_3 > '2000-01-01 00:00:00.123')" + testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) <= '2000-01-01 00:00:00.1234'", "(c_datetime_3 <= '2000-01-01 00:00:00.123')" + testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) < '2000-01-01 00:00:00.1234'", "(c_datetime_3 < '2000-01-01 00:00:00.124')" + testSimplify false, false, "c_date = '2000-01-01 00:00:01'", 'FALSE' + testSimplify false, false, "CAST(c_date_null AS DATETIME(5)) = '2000-01-01 00:00:01'", 'AND[c_date_null IS NULL,NULL]' + testSimplify false, false, "CAST(c_date_null AS DATETIME(5)) <=> '2000-01-01 00:00:01'", 'FALSE' + testSimplify false, false, "CAST(c_date AS DATETIME(5)) > '2000-01-01 00:00:01'", "c_date > '2000-01-01'" + testSimplify false, false, "CAST(c_date AS DATETIME(5)) >= '2000-01-01 00:00:01'", "c_date >= '2000-01-02'" + testSimplify false, false, "CAST(c_date AS DATETIME(5)) <= '2000-01-01 00:00:01'", "c_date <= '2000-01-01'" + testSimplify false, false, "CAST(c_date AS DATETIME(5)) < '2000-01-01 00:00:01'", "c_date < '2000-01-02'" + + sql "DROP TABLE IF EXISTS ${tbl} FORCE" + } +}