diff --git a/.bazelrc b/.bazelrc index bd9e2fd766..7c1adc7ea9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -7,4 +7,20 @@ build:gcc7-later --cxxopt -faligned-new build --incompatible_blacklisted_protos_requires_proto_info=false build --copt=-fdiagnostics-color=always +build:sanitize-common --strip=never +build:sanitize-common --copt -O1 +build:sanitize-common --copt -g +build:sanitize-common --copt -fno-omit-frame-pointer + +build:asan --config=sanitize-common +build:asan --copt -fsanitize=address +build:asan --copt -DADDRESS_SANITIZER +build:asan --linkopt -fsanitize=address + +build:asan --config=sanitize-common +build:msan --copt -fsanitize=memory +build:msan --copt -fsanitize=undefined +build:msan --linkopt -fsanitize=address +build:msan --linkopt -fsanitize=undefined + run --copt=-fdiagnostics-color=always diff --git a/.clang-format b/.clang-format deleted file mode 100644 index e7f2f7fce4..0000000000 --- a/.clang-format +++ /dev/null @@ -1,147 +0,0 @@ ---- -Language: Cpp -AccessModifierOffset: -3 -AlignAfterOpenBracket: Align -AlignConsecutiveMacros: true -AlignConsecutiveAssignments: false -AlignConsecutiveBitFields: false -AlignConsecutiveDeclarations: false -AlignEscapedNewlines: Right -AlignOperands: Align -AlignTrailingComments: true -AllowAllArgumentsOnNextLine: true -AllowAllConstructorInitializersOnNextLine: true -AllowAllParametersOfDeclarationOnNextLine: true -AllowShortEnumsOnASingleLine: true -AllowShortBlocksOnASingleLine: Never -AllowShortCaseLabelsOnASingleLine: false -AllowShortFunctionsOnASingleLine: All -AllowShortLambdasOnASingleLine: All -AllowShortIfStatementsOnASingleLine: Never -AllowShortLoopsOnASingleLine: false -AlwaysBreakAfterDefinitionReturnType: None -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: false -AlwaysBreakTemplateDeclarations: MultiLine -BinPackArguments: true -BinPackParameters: true -BraceWrapping: - AfterCaseLabel: false - AfterClass: false - AfterControlStatement: Never - AfterEnum: false - AfterFunction: false - AfterNamespace: false - AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - AfterExternBlock: false - BeforeCatch: false - BeforeElse: false - BeforeLambdaBody: false - BeforeWhile: false - IndentBraces: false - SplitEmptyFunction: true - SplitEmptyRecord: true - SplitEmptyNamespace: true -BreakBeforeBinaryOperators: None -BreakBeforeBraces: Attach -BreakBeforeInheritanceComma: false -BreakInheritanceList: BeforeColon -BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: false -BreakConstructorInitializers: BeforeColon -BreakAfterJavaFieldAnnotations: false -BreakStringLiterals: true -ColumnLimit: 80 -CommentPragmas: '^ IWYU pragma:' -CompactNamespaces: false -ConstructorInitializerAllOnOneLineOrOnePerLine: false -ConstructorInitializerIndentWidth: 4 -ContinuationIndentWidth: 4 -Cpp11BracedListStyle: true -DeriveLineEnding: true -DerivePointerAlignment: false -DisableFormat: false -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: true -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH -IncludeBlocks: Preserve -IncludeCategories: - - Regex: '^"(llvm|llvm-c|clang|clang-c)/' - Priority: 2 - SortPriority: 0 - - Regex: '^(<|"(gtest|gmock|isl|json)/)' - Priority: 3 - SortPriority: 0 - - Regex: '.*' - Priority: 1 - SortPriority: 0 -IncludeIsMainRegex: '(Test)?$' -IncludeIsMainSourceRegex: '' -IndentCaseLabels: false -IndentCaseBlocks: false -IndentGotoLabels: true -IndentPPDirectives: None -IndentExternBlock: AfterExternBlock -IndentWidth: 4 -IndentWrappedFunctionNames: false -InsertTrailingCommas: None -JavaScriptQuotes: Leave -JavaScriptWrapImports: true -KeepEmptyLinesAtTheStartOfBlocks: true -MacroBlockBegin: '' -MacroBlockEnd: '' -MaxEmptyLinesToKeep: 2 -NamespaceIndentation: None -ObjCBinPackProtocolList: Auto -ObjCBlockIndentWidth: 2 -ObjCBreakBeforeNestedBlockParam: true -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: true -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 19 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyBreakTemplateDeclaration: 10 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 60 -PointerAlignment: Right -ReflowComments: true -SortIncludes: false -SortUsingDeclarations: true -SpaceAfterCStyleCast: false -SpaceAfterLogicalNot: false -SpaceAfterTemplateKeyword: true -SpaceBeforeAssignmentOperators: true -SpaceBeforeCpp11BracedList: false -SpaceBeforeCtorInitializerColon: true -SpaceBeforeInheritanceColon: true -SpaceBeforeParens: ControlStatements -SpaceBeforeRangeBasedForLoopColon: true -SpaceInEmptyBlock: false -SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 2 -SpacesInAngles: false -SpacesInConditionalStatement: false -SpacesInContainerLiterals: true -SpacesInCStyleCastParentheses: false -SpacesInParentheses: false -SpacesInSquareBrackets: false -SpaceBeforeSquareBrackets: false -Standard: Cpp11 -StatementMacros: - - Q_UNUSED - - QT_REQUIRE_VERSION -TabWidth: 4 -UseCRLF: false -UseTab: Never -WhitespaceSensitiveMacros: - - STRINGIZE - - PP_STRINGIZE - - BOOST_PP_STRINGIZE -... diff --git a/.clangd b/.clangd deleted file mode 100644 index efdc056fe2..0000000000 --- a/.clangd +++ /dev/null @@ -1,12 +0,0 @@ -Diagnostics: - ClangTidy: - Add: [performance-*, modernize-*, readability-*,bugprone-*,] - Remove: [modernize-use-trailing-return-type] - CheckOptions: - readability-identifier-naming.VariableCase: camelBack - UnusedIncludes: Strict - -InlayHints: - Enabled: No - ParameterNames: No - DeducedTypes: No diff --git a/.github/workflows/cppcheck.yml b/.github/workflows/cppcheck.yml new file mode 100644 index 0000000000..cc0c5e25b4 --- /dev/null +++ b/.github/workflows/cppcheck.yml @@ -0,0 +1,24 @@ +name: Static Checker + +on: pull_request + +jobs: + cppcheck: + name: CppCheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: chmorgan/cppcheck-action@v1.4 + with: + enable: all + std: c++11 + inconclusive: disable + output_file: ./cppcheck_report.txt + other_options: "-j4 --suppressions-list=util/cppcheck/cppcheck.suppressions --error-exitcode=1 -itest -icurvefs/test -inebd/test -inbd/test -icurvefs_python -icurvesnapshot_python -ithirdparties" + + - name: Show cppcheck report + if: failure() + run: | + cat ./cppcheck_report.txt + exit 1 diff --git a/.gitignore b/.gitignore index 1a66484ca7..353d3f019c 100755 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,9 @@ runlog/ !curve-snapshotcloneserver-nginx/app/lib !nebd/nebd-package/usr/bin +# build output +build + # mac .DS_Store @@ -166,3 +169,12 @@ test/integration/snapshotcloneserver/config/*.conf *.deb *.whl + +*.class +curvefs/sdk/java/target/ +curvefs/sdk/java/native/build +curvefs/sdk/libcurvefs/examples/bin +curvefs/sdk/output/ +hadoop-test/ +nnbench-test/ +dependency-reduced-pom.xml diff --git a/.obm.cfg b/.obm.cfg index 93bee71c49..1803905eb0 100644 --- a/.obm.cfg +++ b/.obm.cfg @@ -1,2 +1,2 @@ -container_name: curve-build-playground-master -container_image: opencurvedocker/curve-base:build-debian9 +container_name: curve-build-playground.master +container_image: opencurvedocker/curve-build:ubuntu22 diff --git a/CHANGELOG-2.7.md b/CHANGELOG-2.7.md new file mode 100644 index 0000000000..5f799aff4e --- /dev/null +++ b/CHANGELOG-2.7.md @@ -0,0 +1,5 @@ +# CHANGELOG of v2.7 + +[CHANGELOG-2.7](https://docs.opencurve.io/Release/release-notes-v2.7) + +> NOTE: All release notes will be published on our documents site from now on. diff --git a/Makefile b/Makefile index 7016723580..03c2187ae1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. -.PHONY: list build dep install image playground check test docker +.PHONY: list build dep ci-list ci-build ci-dep install image playground check test docker format stor?="" prefix?= "$(PWD)/projects" @@ -13,6 +13,14 @@ os?= "debian11" ci?=0 define help_msg +## build curvebs + make build stor=bs dep=1 + make dep stor=bs && make build stor=bs + +## build curvefs + make build stor=fs dep=1 + make dep stor=fs && make build stor=fs + ## list Usage: make list stor=bs/fs @@ -21,20 +29,25 @@ Examples: ## build Usage: - make build stor=bs/fs only=TARGET dep=0/1 release=0/1 os=OS + make build stor=bs/fs only=TARGET1,...,TARGETx dep=0/1 release=0/1 os=OS Examples: make build stor=bs only=//src/chunkserver:chunkserver - make build stor=bs only=src/* dep=0 - make build stor=fs only=test/* os=debian9 - make build stor=fs release=1 + make build stor=bs only=src/*,test/* dep=0 + make build stor=fs only=test/* os=debian11 + make build stor=fs release=1 +Note: + Extra build options can be specified using BUILD_OPTS environment variable, which will be passed to bazel build command. ## dep +## configure dependency(before build) Usage: make dep stor=bs/fs Examples: make dep stor=bs +## ci-list/build/dep +## use the same way above, but in the container ## install Usage: @@ -49,7 +62,21 @@ Examples: Usage: make image stor=bs/fs tag=TAG os=OS Examples: - make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian9 + make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian11 + + +## package +Usage: + make release=0/1 dep=0/1 os=OS +Examples: + make deb + make tar release=1 dep=1 os=debian11 + + +## playground +## create/run a container, changes outside will be mapped into the container +Usage/Example: + make playground endef export help_msg @@ -65,6 +92,9 @@ build: dep: @bash util/build.sh --stor=$(stor) --only="" --dep=1 +ci-list: + @bash util/build_in_image.sh --stor=$(stor) --list + ci-build: @bash util/build_in_image.sh --stor=$(stor) --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os) @@ -77,8 +107,11 @@ install: image: @bash util/image.sh $(stor) $(tag) $(os) +tar deb: + @RELEASE=$(release) DEP=$(dep) OS=$(os) bash util/package.sh $@ + playground: - @bash util/playground.sh + @bash util/playground.sh --version=$(version) check: @bash util/check.sh $(stor) @@ -88,3 +121,12 @@ test: docker: @bash util/docker.sh --os=$(os) --ci=$(ci) + +init-hadoop: + @bash util/init-hadoop.sh + +sdk: + @bash util/sdk.sh + +clean: + @bash util/clean.sh diff --git a/README.md b/README.md index 1405e0be98..1a988a16ab 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ **A sandbox project hosted by the CNCF Foundation** #### English | [简体中文](README_cn.md) -### 📄 [Documents](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [Official Website](https://www.opencurve.io/Curve/HOME) || 🏠 [Forum](https://ask.opencurve.io/t/topic/7) +### 📄 [Documents](https://docs.opencurve.io/) || 🌐 [Official Website](https://www.opencurve.io/Curve/HOME) || 🏠 [Forum](https://ask.opencurve.io/t/topic/7)
diff --git a/README_cn.md b/README_cn.md index 12d4140388..b722921fe3 100644 --- a/README_cn.md +++ b/README_cn.md @@ -9,7 +9,7 @@ **CNCF基金会的沙箱托管项目** #### [English](README.md) | 简体中文 -### 📄 [文档](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [官网](https://www.opencurve.io/Curve/HOME) || 🏠 [论坛](https://ask.opencurve.io/t/topic/7) +### 📄 [文档](https://docs.opencurve.io/) || 🌐 [官网](https://www.opencurve.io/Curve/HOME) || 🏠 [论坛](https://ask.opencurve.io/t/topic/7)
diff --git a/WORKSPACE b/WORKSPACE index 923fa56139..6efa122eea 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -35,6 +35,12 @@ git_repository( name = "com_github_baidu_braft", remote = "https://github.com/baidu/braft", commit = "d12de388c97998f5ccd5cb97ed0da728815ef438", + patches = [ + "//:thirdparties/braft/0001-fix-change-set_error-to-set_errorv.patch", + ], + patch_args = [ + "-p1" + ], ) bind( @@ -133,7 +139,16 @@ git_repository( name = "com_github_brpc_brpc", remote = "https://github.com/apache/incubator-brpc", commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0", - patches = ["//:thirdparties/brpc/brpc.patch","//:thirdparties/brpc/fix-gcc11.patch"], + patches = [ + "//:thirdparties/brpc/brpc.patch", + "//:thirdparties/brpc/fix-gcc11.patch", + "//:thirdparties/brpc/0001-bvar-warning-on-conflict-bvar-name.patch", + "//:thirdparties/brpc/0002-Support-fork-without-exec.patch", + "//:thirdparties/brpc/0003-Add-docs-on-fork-w-o-exec.patch", + "//:thirdparties/brpc/0004-not-register-pthread_atfork-in-child-process.patch", + "//:thirdparties/brpc/0005-Fix-LatencyRecorder-qps-not-accurate.patch", + "//:thirdparties/brpc/0006-fix-1973-1863.patch", + ], patch_args = ["-p1"], ) @@ -182,7 +197,6 @@ new_local_repository( path = "thirdparties/memcache/libmemcached-1.1.2", ) - http_archive( name = "aws", urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"], @@ -249,6 +263,21 @@ http_archive( build_file = "//:thirdparties/spdlog.BUILD", ) +# incbin +new_git_repository( + name = "incbin", + remote = "https://github.com/graphitemaster/incbin.git", + commit = "6e576cae5ab5810f25e2631f2e0b80cbe7dc8cbf", + build_file = "//:thirdparties/incbin.BUILD", +) + +# config +new_local_repository( + name = "config", + build_file = "//:thirdparties/config.BUILD", + path = "thirdparties/config", +) + # Bazel platform rules. http_archive( name = "platforms", @@ -264,6 +293,13 @@ new_local_repository( path = "thirdparties/rocksdb", ) +# jni +new_local_repository( + name = "jni", + build_file = "//:thirdparties/jni.BUILD", + path = "thirdparties", +) + # Hedron's Compile Commands Extractor for Bazel # https://github.com/hedronvision/bazel-compile-commands-extractor http_archive( @@ -272,11 +308,11 @@ http_archive( # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README). urls = [ - "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz", - "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz", + "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-3dddf205a1f5cde20faf2444c1757abe0564ff4c.tar.gz", + "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/3dddf205a1f5cde20faf2444c1757abe0564ff4c.tar.gz", ], - strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f", - # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." + strip_prefix = "bazel-compile-commands-extractor-3dddf205a1f5cde20faf2444c1757abe0564ff4c", + sha256 = "3cd0e49f0f4a6d406c1d74b53b7616f5e24f5fd319eafc1bf8eee6e14124d115", ) load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup") hedron_compile_commands_setup() diff --git a/conf/chunkserver.conf b/conf/chunkserver.conf index 0cfc27b544..19457b3c18 100644 --- a/conf/chunkserver.conf +++ b/conf/chunkserver.conf @@ -194,7 +194,7 @@ rconcurrentapply.queuedepth=1 # 是否开启从chunkfilepool获取chunk，一般是true chunkfilepool.enable_get_chunk_from_pool=true # chunkfilepool目录 -chunkfilepool.chunk_file_pool_dir=./0/ # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__ +chunkfilepool.chunk_file_pool_dir=./0/chunks # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__ # chunkfilepool meta文件路径 chunkfilepool.meta_path=./chunkfilepool.meta # __CURVEADM_TEMPLATE__ ${prefix}/data/chunkfilepool.meta __CURVEADM_TEMPLATE__ # chunkfilepool meta文件大小 @@ -207,6 +207,14 @@ chunkfilepool.clean.enable=true chunkfilepool.clean.bytes_per_write=4096 # The throttle iops for cleaning chunk (4KB/IO) chunkfilepool.clean.throttle_iops=500 +# Whether allocate filePool by percent of disk size. +chunkfilepool.allocated_by_percent=true +# Preallocate storage percent of total disk +chunkfilepool.allocate_percent=80 +# Preallocate storage size of chunkfilepool (None/KB/MB/GB/TB) +chunkfilepool.chunk_file_pool_size=1GB +# The thread num for format chunks +chunkfilepool.thread_num=1 # # WAL file pool @@ -229,6 +237,14 @@ walfilepool.metapage_size=4096 walfilepool.meta_file_size=4096 # WAL filepool get chunk最大重试次数 walfilepool.retry_times=5 +# Whether allocate filePool by percent of disk size. +walfilepool.allocated_by_percent=true +# Preallocate storage percent of total disk +walfilepool.allocate_percent=90 +# Preallocate storage size size of walfilepool (None/KB/MB/GB/TB) +walfilepool.wal_file_pool_size=0 +# The thread num for format chunks +walfilepool.thread_num=1 # # trash settings diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example index eb664c2fd6..443412215b 100644 --- a/conf/chunkserver.conf.example +++ b/conf/chunkserver.conf.example @@ -186,7 +186,7 @@ rconcurrentapply.queuedepth=1 # 是否开启从chunkfilepool获取chunk，一般是true chunkfilepool.enable_get_chunk_from_pool=true # chunkfilepool目录 -chunkfilepool.chunk_file_pool_dir=./0/ +chunkfilepool.chunk_file_pool_dir=./0/chunks # chunkfilepool meta文件路径 #chunkfilepool.meta_path=./chunkfilepool.meta # chunkfilepool meta文件大小 @@ -199,6 +199,14 @@ chunkfilepool.clean.enable=true chunkfilepool.clean.bytes_per_write=4096 # The throttle iops for cleaning chunk (4KB/IO) chunkfilepool.clean.throttle_iops=500 +# Whether allocate filePool by percent of disk size. +chunkfilepool.allocated_by_percent=true +# Preallocate storage percent of total disk +chunkfilepool.allocate_percent=80 +# Preallocate storage size of chunkfilepool (None/KB/MB/GB/TB) +chunkfilepool.chunk_file_pool_size=1GB +# The thread num for format chunks +chunkfilepool.thread_num=1 # # WAL file pool @@ -221,6 +229,14 @@ walfilepool.metapage_size=4096 walfilepool.meta_file_size=4096 # WAL filepool get chunk最大重试次数 walfilepool.retry_times=5 +# Whether allocate filePool by percent of disk size. +walfilepool.allocated_by_percent=true +# Preallocate storage percent of total disk +walfilepool.allocate_percent=10 +# Preallocate storage size size of walfilepool (None/KB/MB/GB/TB) +walfilepool.wal_file_pool_size=0 +# The thread num for format chunks +walfilepool.thread_num=1 # # trash settings diff --git a/conf/client.conf b/conf/client.conf index 144bf5ff47..bac0dc1108 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -110,10 +110,17 @@ chunkserver.serverStableThreshold=3 # 当一个请求重试次数超过这个值时，其超时时间一定进入指数退避 chunkserver.minRetryTimesForceTimeoutBackoff=5 +# ** 已废弃，不再使用，请使用 `chunkserver.slowRequestThresholdMS` ** +# ** dreprecated, use `chunkserver.slowRequestThresholdMS` instead ** # 当一个rpc重试超过次数maxRetryTimesBeforeConsiderSuspend的时候 # 记为悬挂IO，metric会报警 chunkserver.maxRetryTimesBeforeConsiderSuspend=20 +# 请求重试时间超过该阈值后，会标记为slow request +# specifies the retry duration threshold after which a request will be +# marked as a slow request. +chunkserver.slowRequestThresholdMS=45000 + # ################# 文件级别配置项 ############# # @@ -152,6 +159,14 @@ global.metricDummyServerStartPort=9000 # 是否关闭健康检查: true/关闭 false/不关闭 global.turnOffHealthCheck=true +# minimal open file limit +# if set value to 0, then will skip check and set open file limit +# NOTE: open file limit will affect how may sockets we can create, +# the number of sockets is related to the number of chunkserver and mds in the cluster, +# and during some exception handling processes, client will create additional sockets +# the SAFE value is 2 * (#chunkserver + #mds) +global.minOpenFileLimit=1024 + # ### throttle config # diff --git a/conf/cs_client.conf b/conf/cs_client.conf index 56825bd2df..09d567d8f7 100644 --- a/conf/cs_client.conf +++ b/conf/cs_client.conf @@ -144,6 +144,10 @@ global.logPath=/data/log/curve/ # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEA # global.metricDummyServerStartPort=9000 +# minimal open file limit +# if set value to 0, then will skip check and set open file limit +global.minOpenFileLimit=0 + # # session map文件，存储打开文件的filename到path的映射 # diff --git a/conf/mds.conf b/conf/mds.conf index 1e1a7eb273..cc8c661e0d 100644 --- a/conf/mds.conf +++ b/conf/mds.conf @@ -21,6 +21,10 @@ mds.etcd.retry.times=3 mds.etcd.dlock.timeoutMs=10000 # dlock lease timeout mds.etcd.dlock.ttlSec=10 +# etcd auth options +etcd.auth.enable=false +etcd.auth.username= +etcd.auth.password= # # segment分配量统计相关配置 diff --git a/conf/py_client.conf b/conf/py_client.conf index ebcd342adf..cb7999c5e4 100644 --- a/conf/py_client.conf +++ b/conf/py_client.conf @@ -123,6 +123,10 @@ global.fileMaxInFlightRPCNum=64 # 文件IO下发到底层chunkserver最大的分片KB global.fileIOSplitMaxSizeKB=64 +# minimal open file limit +# if set value to 0, then will skip check and set open file limit +global.minOpenFileLimit=0 + # ################# log相关配置 ############### # diff --git a/conf/snap_client.conf b/conf/snap_client.conf index a004cb4233..a643e44461 100644 --- a/conf/snap_client.conf +++ b/conf/snap_client.conf @@ -144,6 +144,10 @@ global.logPath=/data/log/curve/ # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEA # global.metricDummyServerStartPort=9000 +# minimal open file limit +# if set value to 0, then will skip check and set open file limit +global.minOpenFileLimit=0 + # # session map文件，存储打开文件的filename到path的映射 # diff --git a/conf/snapshot_clone_server.conf b/conf/snapshot_clone_server.conf index d4fdf2b64c..1c043686cd 100644 --- a/conf/snapshot_clone_server.conf +++ b/conf/snapshot_clone_server.conf @@ -86,6 +86,11 @@ etcd.retry.times=3 etcd.dlock.timeoutMs=10000 # dlock lease timeout etcd.dlock.ttlSec=10 +# etcd auth options +etcd.auth.enable=false +etcd.auth.username= +etcd.auth.password= + # # leader选举相关参数 diff --git a/curve-ansible/README.md b/curve-ansible/README.md index 4815a12153..1f52d2f43f 100644 --- a/curve-ansible/README.md +++ b/curve-ansible/README.md @@ -30,8 +30,8 @@ chunkserver_num=13 ### 3、包管理 curve-ansible同时提供了debian包和tar包的方式安装curve软件包。默认使用tar包方式安装，如果想用debian包的方式，需要在inventory或者命令行中指定install_with_deb=true。（debian的方式目前不支持格式化指定盘，只支持格式化全部ATA盘） -- debian包的方式下，需要使用mk-deb.sh打debian包，并将debian包上传到debian源中，使其能够用apt-get命令安装。 -- tar包的方式下，tar包可以是来自curve github release，也可以是自己执行mk-tar.sh脚本获取到的。 +- debian包的方式下，需要使用 make deb 打debian包，并将debian包上传到debian源中，使其能够用apt-get命令安装。 +- tar包的方式下，tar包可以是来自curve github release，也可以是自己执行 make tar 获取到的。 ## 三、使用命令 ### 1、集群部署 diff --git a/curve-sdk/DEBIAN/control b/curve-sdk/DEBIAN/control index 084bfcc85c..aef05f4a02 100644 --- a/curve-sdk/DEBIAN/control +++ b/curve-sdk/DEBIAN/control @@ -3,7 +3,7 @@ Section: Priority: optional Depends: libunwind8 Suggests: -Architecture:amd64 +Architecture: amd64 Installed-Size: Maintainer: curve-dev Provides: diff --git a/curvefs/conf/client.conf b/curvefs/conf/client.conf index 211423f69d..8ba0127ec2 100644 --- a/curvefs/conf/client.conf +++ b/curvefs/conf/client.conf @@ -85,6 +85,12 @@ fuseClient.getThreadPool=4 # it gurantee the consistent of file after rename, otherwise you should # disable it for performance. fuseClient.enableMultiMountPointRename=true + +# the rename transaction models are different between version 1 and version 2 +# the v2 version greatly improves the performance of rename, especially in concurrent scenarios. +# Node: v1 and v2 are incompatible and cannot be directly upgraded from a v1 cluster to v2. +fuseClient.txVersion=1 + # splice will bring higher performance in some cases # but there might be a kernel issue that will cause kernel panic when enabling it # see https://lore.kernel.org/all/CAAmZXrsGg2xsP1CK+cbuEMumtrqdvD-NKnWzhNcvn71RV3c1yw@mail.gmail.com/ @@ -92,8 +98,6 @@ fuseClient.enableMultiMountPointRename=true fuseClient.enableSplice=false # thread number of listDentry when get summary xattr fuseClient.listDentryThreads=10 -# disable xattr on one mountpoint can fast 'ls -l' -fuseClient.disableXattr=false # default data（s3ChunkInfo/volumeExtent） size in inode, if exceed will eliminate and try to get the merged one fuseClient.maxDataSize=1024 # default refresh data interval 30s @@ -128,17 +132,26 @@ fuseClient.throttle.burstReadIops=0 # the times that read burst Iops can continue, default 180s fuseClient.throttle.burstReadIopsSecs=180 +#### vfs (virtual filesystem) +#{ +vfs.userPermission.uid=0 +vfs.userPermission.gids=0 +vfs.userPermission.umask=0022 +vfs.entryCache.lruSize=2000000 +vfs.attrCache.lruSize=2000000 +#} + #### filesystem metadata # { -# fs.disableXattr: +# fs.disableXAttr: # if you want to get better metadata performance, -# you can mount fs with |fs.disableXattr| is true +# you can mount fs with |fs.disableXAttr| is true # # fs.lookupCache.negativeTimeoutSec: # entry which not found will be cached if |timeout| > 0 fs.cto=true fs.maxNameLength=255 -fs.disableXattr=false +fs.disableXAttr=true fs.accessLogging=true fs.kernelCache.attrTimeoutSec=3600 fs.kernelCache.dirAttrTimeoutSec=3600 @@ -204,6 +217,18 @@ s3.writeCacheMaxByte=838860800 s3.readCacheMaxByte=209715200 # file cache read thread num s3.readCacheThreads=5 + +# The data in the cache cluster download to local +s3.memClusterToLocal=true +# The data in the s3 storage download to local +s3.s3ToLocal=true +# read size bigger than this value will read until prefetch is finished +s3.bigIoSize=131072 +# retry times when read big io failed +s3.bigIoRetryTimes=100 +# retry interval when read big io failed +s3.bigIoRetryIntervalUs=100 + # http = 0, https = 1 s3.http_scheme=0 s3.verify_SSL=False @@ -217,7 +242,7 @@ s3.logLevel=4 s3.logPrefix=/data/logs/curvefs/aws_ # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__ s3.asyncThreadNum=500 # limit all inflight async requests' bytes, |0| means not limited -s3.maxAsyncRequestInflightBytes=104857600 +s3.maxAsyncRequestInflightBytes=1073741824 s3.chunkFlushThreads=5 # throttle s3.throttle.iopsTotalLimit=0 @@ -244,10 +269,19 @@ diskCache.forceFlush=true diskCache.trimCheckIntervalSec=5 # the interval of check to trim load file to s3 diskCache.asyncLoadPeriodMs=5 -# start trim file when disk cache use ratio is Greater than fullRatio, -# util less than safeRatio +# ok nearfull full +# |------------|-------------------|----------------------| +# 0 trimRatio*safeRatio safeRatio fullRatio +# +# 1. 0<=oknearfull does not clean up +# If the status is full or +# full->nearfull clean up diskCache.fullRatio=90 diskCache.safeRatio=70 +diskCache.trimRatio=50 diskCache.threads=5 # the max size disk cache can use diskCache.maxUsableSpaceBytes=107374182400 diff --git a/curvefs/conf/mds.conf b/curvefs/conf/mds.conf index 0c28d89946..41a5aae45f 100644 --- a/curvefs/conf/mds.conf +++ b/curvefs/conf/mds.conf @@ -35,6 +35,10 @@ etcd.dailtimeoutMs=5000 etcd.operation.timeoutMs=5000 # number of times a failed operation can be retried etcd.retry.times=3 +# etcd auth options +etcd.auth.enable=false +etcd.auth.username= +etcd.auth.password= # # leader election options diff --git a/curvefs/conf/metaserver.conf b/curvefs/conf/metaserver.conf index a16fd4d101..1140bef40d 100644 --- a/curvefs/conf/metaserver.conf +++ b/curvefs/conf/metaserver.conf @@ -258,11 +258,17 @@ storage.rocksdb.unordered_write_buffer_size=67108864 # for store inode which exclude its s3chunkinfo list (default: 3) storage.rocksdb.unordered_max_write_buffer_number=3 # rocksdb column family's write_buffer_size -# for store dentry and inode's s3chunkinfo list (unit: bytes, default: 128MB) +# for store dentry and inode's s3chunkinfo list (unit: bytes, default: 64MB) storage.rocksdb.ordered_write_buffer_size=67108864 # rocksdb column family's max_write_buffer_number # for store dentry and inode's s3chunkinfo list (default: 3) storage.rocksdb.ordered_max_write_buffer_number=3 +# rocksdb column family's write_buffer_size +# for store tx lock and write (unit: bytes, default: 64MB) +storage.rocksdb.tx_cf_write_buffer_size=67108864 +# rocksdb column family's max_write_buffer_number +# for store tx lock and write (default: 3) +storage.rocksdb.tx_cf_max_write_buffer_number=3 # The target number of write history bytes to hold in memory (default: 20MB) storage.rocksdb.max_write_buffer_size_to_maintain=20971520 # rocksdb memtable prefix bloom size ratio (size=write_buffer_size*memtable_prefix_bloom_size_ratio) @@ -286,6 +292,8 @@ storage.rocksdb.perf_sampling_ratio=0 # we will sending its with rpc streaming instead of # padding its into inode (default: 25000, about 25000 * 41 (byte) = 1MB) storage.s3_meta_inside_inode.limit_size=25000 +# TTL(millisecond) for tx lock +storage.tx_lock_ttl_ms=5000 # recycle options # metaserver scan recycle period, default 1h diff --git a/curvefs/conf/tools.conf b/curvefs/conf/tools.conf index d911b3e67b..e795a8c934 100644 --- a/curvefs/conf/tools.conf +++ b/curvefs/conf/tools.conf @@ -39,7 +39,7 @@ s3.useVirtualAddressing=false # if set 2 and other values mean hash prefix s3.objectPrefix=0 # statistic info in xattr, hardlink will not be supported when enable -enableSumInDir=true +enableSumInDir=false # fs recycle, if set 0, disable fs recycle, delete files directly, # if set not 0, enable fs recycle, delete files after a period of time diff --git a/curvefs/devops/util/tmpl.sh b/curvefs/devops/util/tmpl.sh index cb1e5ea181..ed69596098 100755 --- a/curvefs/devops/util/tmpl.sh +++ b/curvefs/devops/util/tmpl.sh @@ -3,7 +3,7 @@ # Usage: # tmpl.sh DSV SOURCE DESTINATION # Example: -# tmpl.sh = /usr/local/metaserver.conf /tmp/metaserver.con +# tmpl.sh = /usr/local/metaserver.conf /tmp/metaserver.conf g_dsv=$1 g_src=$2 diff --git a/curvefs/docker/debian11/Dockerfile b/curvefs/docker/debian11/Dockerfile index 9ea3ca5c1c..7a0fc90e61 100644 --- a/curvefs/docker/debian11/Dockerfile +++ b/curvefs/docker/debian11/Dockerfile @@ -1,6 +1,7 @@ FROM opencurvedocker/curve-base:debian11 -COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/ +COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 libetcdclient.so /usr/lib/ COPY curvefs /curvefs RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \ && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \ + && cp /curvefs/etcd/sbin/etcdctl /usr/bin/ \ && cp /curvefs/tools-v2/sbin/curve /usr/bin/ diff --git a/curvefs/docker/debian11/entrypoint.sh b/curvefs/docker/debian11/entrypoint.sh index 0ca397dace..35faec3cad 100755 --- a/curvefs/docker/debian11/entrypoint.sh +++ b/curvefs/docker/debian11/entrypoint.sh @@ -104,7 +104,10 @@ function prepare() { } function create_directory() { - chmod 700 "$g_prefix/data" + if [ "$g_role" != "monitor" ]; then + chmod 700 "$g_prefix/data" + fi + if [ "$g_role" == "etcd" ]; then mkdir -p "$g_prefix/data/wal" elif [ "$g_role" == "metaserver" ]; then diff --git a/curvefs/docker/openeuler/Dockerfile b/curvefs/docker/openeuler/Dockerfile new file mode 100644 index 0000000000..4aa04b8f56 --- /dev/null +++ b/curvefs/docker/openeuler/Dockerfile @@ -0,0 +1,9 @@ +FROM opencurvedocker/curve-base:openeuler +COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 libfuse3.so.3.10.5 libsnappy.so.1.1.9 libetcdclient.so /usr/lib/ +COPY curvefs /curvefs +RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \ + && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \ + && cp /curvefs/tools-v2/sbin/curve /usr/bin/ \ + && ln -s /usr/lib/libfuse3.so.3.10.5 /usr/lib/libfuse3.so.3 \ + && ln -s /usr/lib64/libsnappy.so.1.1.9 /usr/lib64/libsnappy.so.1 \ + && ldconfig diff --git a/curvefs/docker/openeuler/entrypoint.sh b/curvefs/docker/openeuler/entrypoint.sh new file mode 100644 index 0000000000..35faec3cad --- /dev/null +++ b/curvefs/docker/openeuler/entrypoint.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. + +############################ GLOBAL VARIABLES +g_role="" +g_args="" +g_prefix="" +g_binary="" +g_start_args="" +g_preexec="/curvefs/tools-v2/sbin/daemon" + +############################ BASIC FUNCTIONS +function msg() { + printf '%b' "$1" >&2 +} + +function success() { + msg "\33[32m[✔]\33[0m ${1}${2}" +} + +function die() { + msg "\33[31m[✘]\33[0m ${1}${2}" + exit 1 +} + +############################ FUNCTIONS +function usage () { + cat << _EOC_ +Usage: + entrypoint.sh --role=ROLE + entrypoint.sh --role=ROLE --args=ARGS + +Examples: + entrypoint.sh --role=etcd + entrypoint.sh --role=client --args="-o default_permissions" +_EOC_ +} + +function get_options() { + local long_opts="role:,args:,help" + local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"` + eval set -- "${args}" + while true + do + case "$1" in + -r|--role) + g_role=$2 + shift 2 + ;; + -a|--args) + g_args=$2 + shift 2 + ;; + -h) + usage + exit 1 + ;; + --) + shift + break + ;; + *) + exit 1 + ;; + esac + done +} + +function prepare() { + g_prefix="/curvefs/$g_role" + conf_path="$g_prefix/conf/$g_role.conf" + + case $g_role in + etcd) + g_binary="$g_prefix/sbin/etcd" + g_start_args="--config-file $conf_path" + ;; + mds) + g_binary="$g_prefix/sbin/curvefs-mds" + g_start_args="--confPath $conf_path" + ;; + metaserver) + g_binary="$g_prefix/sbin/curvefs-metaserver" + g_start_args="--confPath $conf_path" + ;; + client) + g_binary="$g_prefix/sbin/curve-fuse" + g_start_args="--confPath $conf_path" + ;; + monitor) + g_binary="python3" + g_start_args="target_json.py" + ;; + *) + usage + exit 1 + ;; + esac + + if [ "$g_args" != "" ]; then + g_start_args=$g_args + fi +} + +function create_directory() { + if [ "$g_role" != "monitor" ]; then + chmod 700 "$g_prefix/data" + fi + + if [ "$g_role" == "etcd" ]; then + mkdir -p "$g_prefix/data/wal" + elif [ "$g_role" == "metaserver" ]; then + mkdir -p "$g_prefix/data/storage" + elif [ "$g_role" == "client" ]; then + mkdir -p "$g_prefix/mnt" + fi +} + +function main() { + get_options "$@" + + prepare + create_directory + [[ $(command -v crontab) ]] && cron + [[ ! -z $g_preexec ]] && $g_preexec & + if [ $g_role == "etcd" ]; then + exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1 + elif [ $g_role == "monitor" ]; then + cd $g_prefix + exec $g_binary $g_start_args + else + exec $g_binary $g_start_args + fi + +} + +############################ MAIN() +main "$@" diff --git a/curvefs/monitor/grafana/provisioning/dashboards/client.json b/curvefs/monitor/grafana/provisioning/dashboards/client.json index a160f44f88..5303f00661 100644 --- a/curvefs/monitor/grafana/provisioning/dashboards/client.json +++ b/curvefs/monitor/grafana/provisioning/dashboards/client.json @@ -25,22 +25,19 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, + "id": 7, "links": [], "liveNow": false, "panels": [ { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 6, + "id": 236, "panels": [ { "aliasColors": {}, @@ -51,6 +48,13 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "fuse s3 client FuseOpRead latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -60,7 +64,7 @@ "y": 1 }, "hiddenSeries": false, - "id": 4, + "id": 237, "interval": "1s", "legend": { "alignAsTable": true, @@ -79,7 +83,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -93,40 +97,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "process_memory_resident{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "process_memory_resident {{instance}}", - "refId": "process_memory_resident" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "process_memory_virtual{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_memory_virtual {{instance}}", - "refId": "process_memory_virtual" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "process_memory_shared{instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_[[fs:regex]]_user_read_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "process_memory_shared {{instance}}", - "refId": "A" + "legendFormat": "", + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "process memory usage", + "title": "read latency", "tooltip": { "shared": true, "sort": 0, @@ -140,13 +123,13 @@ }, "yaxes": [ { - "$$hashKey": "object:63", - "format": "decbytes", + "$$hashKey": "object:212", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:64", + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -165,6 +148,13 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "fuse s3 client FuseOpWrite latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -174,7 +164,7 @@ "y": 1 }, "hiddenSeries": false, - "id": 2, + "id": 238, "interval": "1s", "legend": { "alignAsTable": true, @@ -182,7 +172,6 @@ "current": false, "max": true, "min": true, - "rightSide": false, "show": true, "total": false, "values": true @@ -194,7 +183,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -208,40 +197,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "process_cpu_usage{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "process_cpu_usage {{instance}}", - "refId": "process_cpu_usage" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "process_cpu_usage_system{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_cpu_usage_system {{instance}}", - "refId": "process_cpu_usage_system" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "process_cpu_usage_user{instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_[[fs:regex]]_user_write_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "process_cpu_usage_user {{instance}}", - "refId": "process_cpu_usage_user" + "legendFormat": "", + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "process cpu usage", + "title": "write latency", "tooltip": { "shared": true, "sort": 0, @@ -255,13 +223,13 @@ }, "yaxes": [ { - "$$hashKey": "object:495", - "format": "percentunit", + "$$hashKey": "object:212", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:496", + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -270,34 +238,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "process usage", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 85, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -307,6 +248,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "fuse s3 client FuseOpRead read attr latency", "fieldConfig": { "defaults": { "unit": "µs" @@ -319,10 +261,10 @@ "h": 8, "w": 12, "x": 0, - "y": 2 + "y": 9 }, "hiddenSeries": false, - "id": 107, + "id": 239, "interval": "1s", "legend": { "alignAsTable": true, @@ -341,7 +283,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -355,16 +297,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_read_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_fuse_s3_client_[[fs:regex]]_read_attr_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "read_latency", + "title": "read attr latency", "tooltip": { "shared": true, "sort": 0, @@ -378,11 +323,13 @@ }, "yaxes": [ { + "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -401,6 +348,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "fuse s3 client FuseOpRead write attr latency", "fieldConfig": { "defaults": { "unit": "µs" @@ -413,10 +361,10 @@ "h": 8, "w": 12, "x": 12, - "y": 2 + "y": 9 }, "hiddenSeries": false, - "id": 108, + "id": 240, "interval": "1s", "legend": { "alignAsTable": true, @@ -435,7 +383,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -449,16 +397,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_write_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_fuse_s3_client_[[fs:regex]]_write_attr_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "write_latency", + "title": "write attr latency", "tooltip": { "shared": true, "sort": 0, @@ -472,11 +423,13 @@ }, "yaxes": [ { + "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -495,6 +448,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "fuse s3 client FuseOpRead read data latency", "fieldConfig": { "defaults": { "unit": "µs" @@ -507,10 +461,10 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 17 }, "hiddenSeries": false, - "id": 87, + "id": 241, "interval": "1s", "legend": { "alignAsTable": true, @@ -529,7 +483,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -543,16 +497,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_mk_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_fuse_s3_client_[[fs:regex]]_read_data_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "mkdir_latency", + "title": "read data latency", "tooltip": { "shared": true, "sort": 0, @@ -566,11 +523,13 @@ }, "yaxes": [ { + "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -589,6 +548,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "fuse s3 client FuseOpRead write data latency", "fieldConfig": { "defaults": { "unit": "µs" @@ -601,10 +561,10 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 17 }, "hiddenSeries": false, - "id": 91, + "id": 242, "interval": "1s", "legend": { "alignAsTable": true, @@ -623,7 +583,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -637,16 +597,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_rm_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_fuse_s3_client_[[fs:regex]]_write_data_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "rmdir_latency", + "title": "write data latency", "tooltip": { "shared": true, "sort": 0, @@ -660,11 +623,13 @@ }, "yaxes": [ { + "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -673,7 +638,21 @@ "yaxis": { "align": false } - }, + } + ], + "title": "fuse s3 client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 208, + "panels": [ { "aliasColors": {}, "bars": false, @@ -685,7 +664,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -695,10 +674,10 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 41 }, "hiddenSeries": false, - "id": 88, + "id": 116, "interval": "1s", "legend": { "alignAsTable": true, @@ -717,7 +696,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -732,7 +711,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_create_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs.*_read_data_cache_num\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -740,7 +719,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "create_latency", + "title": "read_data_cache_num", "tooltip": { "shared": true, "sort": 0, @@ -754,11 +733,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -779,7 +760,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -789,10 +770,10 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 41 }, "hiddenSeries": false, - "id": 92, + "id": 117, "interval": "1s", "legend": { "alignAsTable": true, @@ -811,7 +792,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -826,7 +807,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_unlink_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs.*_write_data_cache_num\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -834,7 +815,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "unlink_latency", + "title": "write_data_cache_num", "tooltip": { "shared": true, "sort": 0, @@ -848,11 +829,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -873,7 +856,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -883,10 +866,10 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 49 }, "hiddenSeries": false, - "id": 89, + "id": 118, "interval": "1s", "legend": { "alignAsTable": true, @@ -905,7 +888,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -920,7 +903,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_open_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs.*_read_data_cache_byte\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -928,7 +911,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "open_latency", + "title": "read_data_cache_byte", "tooltip": { "shared": true, "sort": 0, @@ -942,11 +925,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -967,7 +952,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -977,10 +962,10 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 49 }, "hiddenSeries": false, - "id": 90, + "id": 119, "interval": "1s", "legend": { "alignAsTable": true, @@ -999,7 +984,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "9.4.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1014,7 +999,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_release_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs.*_write_data_cache_byte\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -1022,7 +1007,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "release_latency", + "title": "write_data_cache_byte", "tooltip": { "shared": true, "sort": 0, @@ -1036,11 +1021,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1049,7 +1036,21 @@ "yaxis": { "align": false } - }, + } + ], + "title": "memcache cache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 201, + "panels": [ { "aliasColors": {}, "bars": false, @@ -1061,7 +1062,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -1071,10 +1072,10 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 18 }, "hiddenSeries": false, - "id": 93, + "id": 202, "interval": "1s", "legend": { "alignAsTable": true, @@ -1093,7 +1094,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1107,16 +1108,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_open_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_kv_cache_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "opendir_latency", + "title": "read_kv_cache_qps", "tooltip": { "shared": true, "sort": 0, @@ -1130,11 +1133,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1155,7 +1160,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -1165,10 +1170,10 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 18 }, "hiddenSeries": false, - "id": 94, + "id": 203, "interval": "1s", "legend": { "alignAsTable": true, @@ -1187,7 +1192,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1201,16 +1206,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_release_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_kv_cache_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "releasedir_latency", + "title": "write_kv_cache_qps", "tooltip": { "shared": true, "sort": 0, @@ -1224,11 +1231,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1249,7 +1258,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "binBps" }, "overrides": [] }, @@ -1259,10 +1268,10 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 26 }, "hiddenSeries": false, - "id": 99, + "id": 204, "interval": "1s", "legend": { "alignAsTable": true, @@ -1281,7 +1290,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1295,16 +1304,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_read_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_kv_cache_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "readdir_latency", + "title": "read_kv_cache_bps", "tooltip": { "shared": true, "sort": 0, @@ -1318,11 +1329,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "binBps", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1343,7 +1356,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "binBps" }, "overrides": [] }, @@ -1353,10 +1366,10 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 26 }, "hiddenSeries": false, - "id": 100, + "id": 205, "interval": "1s", "legend": { "alignAsTable": true, @@ -1375,7 +1388,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1389,16 +1402,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_read_link_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_kv_cache_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "readlink_latency", + "title": "write_kv_cache_bps", "tooltip": { "shared": true, "sort": 0, @@ -1412,11 +1427,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "binBps", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1447,10 +1464,10 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 34 }, "hiddenSeries": false, - "id": 95, + "id": 206, "interval": "1s", "legend": { "alignAsTable": true, @@ -1469,7 +1486,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1483,16 +1500,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_lookup_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_kv_cache_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "lookup_latency", + "title": "read_kv_cache_latency", "tooltip": { "shared": true, "sort": 0, @@ -1506,11 +1526,13 @@ }, "yaxes": [ { + "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1541,10 +1563,10 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 34 }, "hiddenSeries": false, - "id": 96, + "id": 207, "interval": "1s", "legend": { "alignAsTable": true, @@ -1563,7 +1585,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1577,16 +1599,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_mk_nod_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_kv_cache_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "mknod_latency", + "title": "write_kv_cache_latency", "tooltip": { "shared": true, "sort": 0, @@ -1600,11 +1625,13 @@ }, "yaxes": [ { + "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1613,7 +1640,21 @@ "yaxis": { "align": false } - }, + } + ], + "title": "client kv cache r/w performance", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 194, + "panels": [ { "aliasColors": {}, "bars": false, @@ -1625,7 +1666,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -1635,10 +1676,10 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 19 }, "hiddenSeries": false, - "id": 97, + "id": 195, "interval": "1s", "legend": { "alignAsTable": true, @@ -1657,7 +1698,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1671,16 +1712,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_link_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_disk_cache_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "link_latency", + "title": "read_disk_cache_qps", "tooltip": { "shared": true, "sort": 0, @@ -1694,11 +1737,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1719,7 +1764,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -1729,10 +1774,10 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 19 }, "hiddenSeries": false, - "id": 98, + "id": 196, "interval": "1s", "legend": { "alignAsTable": true, @@ -1751,7 +1796,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1765,16 +1810,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_symlink_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_disk_cache_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "symlink_latency", + "title": "write_disk_cache_qps", "tooltip": { "shared": true, "sort": 0, @@ -1788,11 +1835,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1813,7 +1862,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "binBps" }, "overrides": [] }, @@ -1823,10 +1872,10 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 27 }, "hiddenSeries": false, - "id": 101, + "id": 197, "interval": "1s", "legend": { "alignAsTable": true, @@ -1845,7 +1894,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1859,16 +1908,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_get_attr_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_disk_cache_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "getattr_latency", + "title": "read_disk_cache_bps", "tooltip": { "shared": true, "sort": 0, @@ -1882,11 +1933,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "binBps", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -1907,7 +1960,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "binBps" }, "overrides": [] }, @@ -1917,10 +1970,10 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 27 }, "hiddenSeries": false, - "id": 102, + "id": 198, "interval": "1s", "legend": { "alignAsTable": true, @@ -1939,7 +1992,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1953,16 +2006,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_set_attr_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_disk_cache_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "setattr_latency", + "title": "write_disk_cache_bps", "tooltip": { "shared": true, "sort": 0, @@ -1976,11 +2031,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "binBps", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -2011,10 +2068,10 @@ "h": 8, "w": 12, "x": 0, - "y": 74 + "y": 35 }, "hiddenSeries": false, - "id": 103, + "id": 199, "interval": "1s", "legend": { "alignAsTable": true, @@ -2033,7 +2090,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2047,16 +2104,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_get_xattr_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_disk_cache_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "getxattr_latency", + "title": "read_disk_cache_latency", "tooltip": { "shared": true, "sort": 0, @@ -2070,11 +2130,13 @@ }, "yaxes": [ { + "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -2105,10 +2167,10 @@ "h": 8, "w": 12, "x": 12, - "y": 74 + "y": 35 }, "hiddenSeries": false, - "id": 104, + "id": 200, "interval": "1s", "legend": { "alignAsTable": true, @@ -2127,7 +2189,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2141,16 +2203,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_list_xattr_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_disk_cache_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "listxattr_latency", + "title": "write_disk_cache_latency", "tooltip": { "shared": true, "sort": 0, @@ -2164,11 +2229,13 @@ }, "yaxes": [ { + "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -2177,7 +2244,21 @@ "yaxis": { "align": false } - }, + } + ], + "title": "client diskcache performance", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 187, + "panels": [ { "aliasColors": {}, "bars": false, @@ -2189,7 +2270,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -2199,10 +2280,10 @@ "h": 8, "w": 12, "x": 0, - "y": 82 + "y": 20 }, "hiddenSeries": false, - "id": 105, + "id": 188, "interval": "1s", "legend": { "alignAsTable": true, @@ -2221,7 +2302,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2235,16 +2316,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_fsync_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_s3_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "fsync_latency", + "title": "read_s3_qps", "tooltip": { "shared": true, "sort": 0, @@ -2258,11 +2341,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -2283,7 +2368,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -2293,10 +2378,10 @@ "h": 8, "w": 12, "x": 12, - "y": 82 + "y": 20 }, "hiddenSeries": false, - "id": 106, + "id": 189, "interval": "1s", "legend": { "alignAsTable": true, @@ -2315,7 +2400,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2329,16 +2414,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_flush_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_s3_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "flush_latency", + "title": "write_s3_qps", "tooltip": { "shared": true, "sort": 0, @@ -2352,11 +2439,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "short", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -2377,7 +2466,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "binBps" }, "overrides": [] }, @@ -2387,10 +2476,10 @@ "h": 8, "w": 12, "x": 0, - "y": 90 + "y": 28 }, "hiddenSeries": false, - "id": 109, + "id": 190, "interval": "1s", "legend": { "alignAsTable": true, @@ -2409,7 +2498,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2423,16 +2512,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_rename_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_s3_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "rename_latency", + "title": "read_s3_bps", "tooltip": { "shared": true, "sort": 0, @@ -2446,11 +2537,13 @@ }, "yaxes": [ { - "format": "µs", + "$$hashKey": "object:212", + "format": "binBps", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -2459,34 +2552,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "op latency", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 176, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -2498,7 +2564,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "binBps" }, "overrides": [] }, @@ -2506,12 +2572,12 @@ "fillGradient": 0, "gridPos": { "h": 8, - "w": 24, - "x": 0, - "y": 43 + "w": 12, + "x": 12, + "y": 28 }, "hiddenSeries": false, - "id": 177, + "id": 191, "interval": "1s", "legend": { "alignAsTable": true, @@ -2530,7 +2596,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2544,16 +2610,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_client_op_[[op:regex]]_inflight_num\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_s3_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "op _inflight_num", + "title": "write_s3_bps", "tooltip": { "shared": true, "sort": 0, @@ -2567,11 +2635,13 @@ }, "yaxes": [ { - "format": "short", + "$$hashKey": "object:212", + "format": "binBps", "logBase": 1, "show": true }, { + "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -2580,34 +2650,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "op inflight number", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 47, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -2619,7 +2662,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -2629,10 +2672,10 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 36 }, "hiddenSeries": false, - "id": 48, + "id": 192, "interval": "1s", "legend": { "alignAsTable": true, @@ -2651,7 +2694,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2665,16 +2708,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_.*[[fs:regex]]_user_read_qps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_read_from_s3_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "read_qps", + "title": "read_s3_latency", "tooltip": { "shared": true, "sort": 0, @@ -2689,7 +2735,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, @@ -2715,7 +2761,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -2725,10 +2771,10 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 36 }, "hiddenSeries": false, - "id": 49, + "id": 193, "interval": "1s", "legend": { "alignAsTable": true, @@ -2747,7 +2793,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2761,16 +2807,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_.*[[fs:regex]]_user_write_qps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_write_to_s3_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "write_qps", + "title": "write_s3_latency", "tooltip": { "shared": true, "sort": 0, @@ -2785,7 +2834,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, @@ -2799,7 +2848,25 @@ "yaxis": { "align": false } - }, + } + ], + "title": "client s3 r/w performance", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 6, + "panels": [ { "aliasColors": {}, "bars": false, @@ -2809,22 +2876,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "binBps" - }, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 209 }, "hiddenSeries": false, - "id": 50, + "id": 4, "interval": "1s", "legend": { "alignAsTable": true, @@ -2843,7 +2904,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -2858,15 +2919,39 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_bps\", instance=~\"$instance\"}", + "expr": "process_memory_resident{instance=~\"$instance\"}", "interval": "", - "legendFormat": "", + "legendFormat": "process_memory_resident {{instance}}", + "refId": "process_memory_resident" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_memory_virtual{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_memory_virtual {{instance}}", + "refId": "process_memory_virtual" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_memory_shared{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_memory_shared {{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "read_bps", + "title": "process memory usage", "tooltip": { "shared": true, "sort": 0, @@ -2880,13 +2965,13 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "binBps", + "$$hashKey": "object:63", + "format": "decbytes", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", + "$$hashKey": "object:64", "format": "short", "logBase": 1, "show": true @@ -2905,22 +2990,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "binBps" - }, - "overrides": [] - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 52 + "y": 209 }, "hiddenSeries": false, - "id": 51, + "id": 2, "interval": "1s", "legend": { "alignAsTable": true, @@ -2928,6 +3007,7 @@ "current": false, "max": true, "min": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -2939,7 +3019,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -2954,15 +3034,39 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_bps\", instance=~\"$instance\"}", + "expr": "process_cpu_usage{instance=~\"$instance\"}", "interval": "", - "legendFormat": "", - "refId": "A" + "legendFormat": "process_cpu_usage {{instance}}", + "refId": "process_cpu_usage" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_cpu_usage_system{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_cpu_usage_system {{instance}}", + "refId": "process_cpu_usage_system" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_cpu_usage_user{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_cpu_usage_user {{instance}}", + "refId": "process_cpu_usage_user" } ], "thresholds": [], "timeRegions": [], - "title": "write_bps", + "title": "process cpu usage", "tooltip": { "shared": true, "sort": 0, @@ -2976,13 +3080,13 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "binBps", + "$$hashKey": "object:495", + "format": "percentunit", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", + "$$hashKey": "object:496", "format": "short", "logBase": 1, "show": true @@ -2991,7 +3095,34 @@ "yaxis": { "align": false } - }, + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "process usage", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 85, + "panels": [ { "aliasColors": {}, "bars": false, @@ -3003,7 +3134,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -3013,10 +3144,10 @@ "h": 8, "w": 12, "x": 0, - "y": 60 + "y": 15 }, "hiddenSeries": false, - "id": 52, + "id": 107, "interval": "1s", "legend": { "alignAsTable": true, @@ -3035,7 +3166,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3050,7 +3181,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_eps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_read_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -3058,7 +3189,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_eps", + "title": "read_latency", "tooltip": { "shared": true, "sort": 0, @@ -3072,13 +3203,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3097,16 +3226,22 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 60 + "y": 15 }, "hiddenSeries": false, - "id": 53, + "id": 108, "interval": "1s", "legend": { "alignAsTable": true, @@ -3125,7 +3260,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3140,7 +3275,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_adaptor_write_eps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_write_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -3148,7 +3283,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_eps", + "title": "write_latency", "tooltip": { "shared": true, "sort": 0, @@ -3162,13 +3297,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3189,7 +3322,7 @@ }, "fieldConfig": { "defaults": { - "unit": "reqps" + "unit": "µs" }, "overrides": [] }, @@ -3199,10 +3332,10 @@ "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 23 }, "hiddenSeries": false, - "id": 54, + "id": 87, "interval": "1s", "legend": { "alignAsTable": true, @@ -3221,7 +3354,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3236,7 +3369,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_rps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_mk_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -3244,7 +3377,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_rps", + "title": "mkdir_latency", "tooltip": { "shared": true, "sort": 0, @@ -3258,13 +3391,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "reqps", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3285,7 +3416,7 @@ }, "fieldConfig": { "defaults": { - "unit": "reqps" + "unit": "µs" }, "overrides": [] }, @@ -3295,10 +3426,10 @@ "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 23 }, "hiddenSeries": false, - "id": 43, + "id": 91, "interval": "1s", "legend": { "alignAsTable": true, @@ -3317,7 +3448,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3332,7 +3463,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_rps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_rm_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -3340,7 +3471,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_rps", + "title": "rmdir_latency", "tooltip": { "shared": true, "sort": 0, @@ -3354,13 +3485,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "reqps", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3381,7 +3510,7 @@ }, "fieldConfig": { "defaults": { - "unit": "bytes" + "unit": "µs" }, "overrides": [] }, @@ -3391,10 +3520,10 @@ "h": 8, "w": 12, "x": 0, - "y": 76 + "y": 31 }, "hiddenSeries": false, - "id": 56, + "id": 88, "interval": "1s", "legend": { "alignAsTable": true, @@ -3413,7 +3542,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3428,7 +3557,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_io_size\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_create_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -3436,7 +3565,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_io_size", + "title": "create_latency", "tooltip": { "shared": true, "sort": 0, @@ -3450,13 +3579,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "bytes", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3477,7 +3604,7 @@ }, "fieldConfig": { "defaults": { - "unit": "bytes" + "unit": "µs" }, "overrides": [] }, @@ -3487,10 +3614,10 @@ "h": 8, "w": 12, "x": 12, - "y": 76 + "y": 31 }, "hiddenSeries": false, - "id": 57, + "id": 92, "interval": "1s", "legend": { "alignAsTable": true, @@ -3509,7 +3636,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3524,7 +3651,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_io_size\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_unlink_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -3532,7 +3659,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_io_size", + "title": "unlink_latency", "tooltip": { "shared": true, "sort": 0, @@ -3546,13 +3673,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "bytes", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3583,10 +3708,10 @@ "h": 8, "w": 12, "x": 0, - "y": 84 + "y": 39 }, "hiddenSeries": false, - "id": 58, + "id": 89, "interval": "1s", "legend": { "alignAsTable": true, @@ -3605,7 +3730,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3620,16 +3745,15 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_lat_[[quantile:regex]]\", instance=~\"$instance\"}", - "hide": false, + "expr": "{__name__=~\"curvefs_client_op_open_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", - "refId": "B" + "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "read_latency", + "title": "open_latency", "tooltip": { "shared": true, "sort": 0, @@ -3643,13 +3767,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3680,10 +3802,10 @@ "h": 8, "w": 12, "x": 12, - "y": 84 + "y": 39 }, "hiddenSeries": false, - "id": 59, + "id": 90, "interval": "1s", "legend": { "alignAsTable": true, @@ -3702,7 +3824,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3717,16 +3839,15 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_lat_[[quantile:regex]]\", instance=~\"$instance\"}", - "hide": false, + "expr": "{__name__=~\"curvefs_client_op_release_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", - "refId": "B" + "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "write_latency", + "title": "release_latency", "tooltip": { "shared": true, "sort": 0, @@ -3740,13 +3861,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3755,34 +3874,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "client interface performance", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 8, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -3794,7 +3886,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -3804,10 +3896,10 @@ "h": 8, "w": 12, "x": 0, - "y": 45 + "y": 47 }, "hiddenSeries": false, - "id": 10, + "id": 93, "interval": "1s", "legend": { "alignAsTable": true, @@ -3826,7 +3918,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3841,7 +3933,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_qps\", job=\"client\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_open_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -3849,7 +3941,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_qps", + "title": "opendir_latency", "tooltip": { "shared": true, "sort": 0, @@ -3863,13 +3955,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3890,7 +3980,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -3900,10 +3990,10 @@ "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 47 }, "hiddenSeries": false, - "id": 11, + "id": 94, "interval": "1s", "legend": { "alignAsTable": true, @@ -3922,7 +4012,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3937,7 +4027,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_qps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_release_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -3945,7 +4035,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_qps", + "title": "releasedir_latency", "tooltip": { "shared": true, "sort": 0, @@ -3959,13 +4049,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -3986,7 +4074,7 @@ }, "fieldConfig": { "defaults": { - "unit": "binBps" + "unit": "µs" }, "overrides": [] }, @@ -3996,10 +4084,10 @@ "h": 8, "w": 12, "x": 0, - "y": 53 + "y": 55 }, "hiddenSeries": false, - "id": 12, + "id": 99, "interval": "1s", "legend": { "alignAsTable": true, @@ -4018,7 +4106,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4033,7 +4121,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_bps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_read_dir_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -4041,7 +4129,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_bps", + "title": "readdir_latency", "tooltip": { "shared": true, "sort": 0, @@ -4055,13 +4143,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "binBps", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4082,7 +4168,7 @@ }, "fieldConfig": { "defaults": { - "unit": "binBps" + "unit": "µs" }, "overrides": [] }, @@ -4092,10 +4178,10 @@ "h": 8, "w": 12, "x": 12, - "y": 53 + "y": 55 }, "hiddenSeries": false, - "id": 13, + "id": 100, "interval": "1s", "legend": { "alignAsTable": true, @@ -4114,7 +4200,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4129,7 +4215,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_bps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_read_link_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -4137,7 +4223,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_bps", + "title": "readlink_latency", "tooltip": { "shared": true, "sort": 0, @@ -4151,13 +4237,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "binBps", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4178,7 +4262,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -4188,10 +4272,10 @@ "h": 8, "w": 12, "x": 0, - "y": 61 + "y": 63 }, "hiddenSeries": false, - "id": 14, + "id": 95, "interval": "1s", "legend": { "alignAsTable": true, @@ -4210,7 +4294,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4225,7 +4309,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_eps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_lookup_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -4233,7 +4317,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_eps", + "title": "lookup_latency", "tooltip": { "shared": true, "sort": 0, @@ -4247,13 +4331,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4272,16 +4354,22 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 61 + "y": 63 }, "hiddenSeries": false, - "id": 15, + "id": 96, "interval": "1s", "legend": { "alignAsTable": true, @@ -4300,7 +4388,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4315,7 +4403,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_eps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_mk_nod_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -4323,7 +4411,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_eps", + "title": "mknod_latency", "tooltip": { "shared": true, "sort": 0, @@ -4337,13 +4425,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4364,7 +4450,7 @@ }, "fieldConfig": { "defaults": { - "unit": "reqps" + "unit": "µs" }, "overrides": [] }, @@ -4374,10 +4460,10 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 71 }, "hiddenSeries": false, - "id": 42, + "id": 97, "interval": "1s", "legend": { "alignAsTable": true, @@ -4396,7 +4482,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4411,7 +4497,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_rps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_link_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -4419,7 +4505,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_rps", + "title": "link_latency", "tooltip": { "shared": true, "sort": 0, @@ -4433,13 +4519,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "reqps", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4460,7 +4544,7 @@ }, "fieldConfig": { "defaults": { - "unit": "reqps" + "unit": "µs" }, "overrides": [] }, @@ -4470,10 +4554,10 @@ "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 71 }, "hiddenSeries": false, - "id": 55, + "id": 98, "interval": "1s", "legend": { "alignAsTable": true, @@ -4492,7 +4576,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4507,7 +4591,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_rps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_symlink_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -4515,7 +4599,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_rps", + "title": "symlink_latency", "tooltip": { "shared": true, "sort": 0, @@ -4529,13 +4613,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "reqps", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4556,7 +4638,7 @@ }, "fieldConfig": { "defaults": { - "unit": "bytes" + "unit": "µs" }, "overrides": [] }, @@ -4566,10 +4648,10 @@ "h": 8, "w": 12, "x": 0, - "y": 77 + "y": 79 }, "hiddenSeries": false, - "id": 60, + "id": 101, "interval": "1s", "legend": { "alignAsTable": true, @@ -4588,7 +4670,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4603,7 +4685,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_size\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_get_attr_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -4611,7 +4693,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_io_size", + "title": "getattr_latency", "tooltip": { "shared": true, "sort": 0, @@ -4625,13 +4707,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "bytes", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4652,7 +4732,7 @@ }, "fieldConfig": { "defaults": { - "unit": "bytes" + "unit": "µs" }, "overrides": [] }, @@ -4662,10 +4742,10 @@ "h": 8, "w": 12, "x": 12, - "y": 77 + "y": 79 }, "hiddenSeries": false, - "id": 61, + "id": 102, "interval": "1s", "legend": { "alignAsTable": true, @@ -4684,7 +4764,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4699,7 +4779,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_size\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_set_attr_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -4707,7 +4787,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_io_size", + "title": "setattr_latency", "tooltip": { "shared": true, "sort": 0, @@ -4721,13 +4801,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "bytes", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4758,10 +4836,10 @@ "h": 8, "w": 12, "x": 0, - "y": 85 + "y": 87 }, "hiddenSeries": false, - "id": 26, + "id": 103, "interval": "1s", "legend": { "alignAsTable": true, @@ -4780,7 +4858,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4795,16 +4873,15 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", - "hide": false, + "expr": "{__name__=~\"curvefs_client_op_get_xattr_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", - "refId": "B" + "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "read_latency", + "title": "getxattr_latency", "tooltip": { "shared": true, "sort": 0, @@ -4818,13 +4895,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4855,10 +4930,10 @@ "h": 8, "w": 12, "x": 12, - "y": 85 + "y": 87 }, "hiddenSeries": false, - "id": 27, + "id": 104, "interval": "1s", "legend": { "alignAsTable": true, @@ -4877,7 +4952,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4892,16 +4967,15 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", - "hide": false, + "expr": "{__name__=~\"curvefs_client_op_list_xattr_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", - "refId": "B" + "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "write_latency", + "title": "listxattr_latency", "tooltip": { "shared": true, "sort": 0, @@ -4915,13 +4989,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -4930,34 +5002,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "s3_adaptor r/w performance", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 72, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -4969,7 +5014,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -4979,10 +5024,10 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 95 }, "hiddenSeries": false, - "id": 73, + "id": 105, "interval": "1s", "legend": { "alignAsTable": true, @@ -5001,7 +5046,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5016,7 +5061,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_disk_cache_qps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_fsync_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -5024,7 +5069,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_diskcache_qps", + "title": "fsync_latency", "tooltip": { "shared": true, "sort": 0, @@ -5038,13 +5083,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -5065,7 +5108,7 @@ }, "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -5075,10 +5118,10 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 95 }, "hiddenSeries": false, - "id": 74, + "id": 106, "interval": "1s", "legend": { "alignAsTable": true, @@ -5097,7 +5140,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5112,7 +5155,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_disk_cache_qps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_flush_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -5120,7 +5163,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "write_diskcache_qps", + "title": "flush_latency", "tooltip": { "shared": true, "sort": 0, @@ -5134,13 +5177,11 @@ }, "yaxes": [ { - "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, { - "$$hashKey": "object:213", "format": "short", "logBase": 1, "show": true @@ -5161,7 +5202,7 @@ }, "fieldConfig": { "defaults": { - "unit": "binBps" + "unit": "µs" }, "overrides": [] }, @@ -5171,10 +5212,10 @@ "h": 8, "w": 12, "x": 0, - "y": 54 + "y": 103 }, "hiddenSeries": false, - "id": 75, + "id": 109, "interval": "1s", "legend": { "alignAsTable": true, @@ -5193,7 +5234,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5208,7 +5249,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_disk_cache_bps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_client_op_rename_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -5216,7 +5257,5469 @@ ], "thresholds": [], "timeRegions": [], - "title": "read_diskcache_bps", + "title": "rename_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "op latency", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 176, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 251 + }, + "hiddenSeries": false, + "id": 177, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_client_op_[[op:regex]]_inflight_num\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "op _inflight_num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "op inflight number", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 47, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 48, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_.*[[fs:regex]]_user_read_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 49, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_.*[[fs:regex]]_user_write_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 50, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 51, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 52, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_eps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_eps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 53, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_adaptor_write_eps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_eps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 54, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_rps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_rps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "reqps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "hiddenSeries": false, + "id": 43, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_rps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_rps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "reqps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "hiddenSeries": false, + "id": 56, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_io_size\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_io_size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 49 + }, + "hiddenSeries": false, + "id": 57, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_io_size\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_io_size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 58, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 59, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "client interface performance", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 8, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 10, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_qps\", job=\"client\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 11, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 65 + }, + "hiddenSeries": false, + "id": 12, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 65 + }, + "hiddenSeries": false, + "id": 13, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 73 + }, + "hiddenSeries": false, + "id": 14, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_eps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_eps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 73 + }, + "hiddenSeries": false, + "id": 15, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_eps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_eps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 81 + }, + "hiddenSeries": false, + "id": 42, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_rps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_rps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "reqps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 81 + }, + "hiddenSeries": false, + "id": 55, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_rps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_rps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "reqps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 89 + }, + "hiddenSeries": false, + "id": 60, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_size\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_io_size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 89 + }, + "hiddenSeries": false, + "id": 61, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_size\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_io_size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 97 + }, + "hiddenSeries": false, + "id": 26, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 97 + }, + "hiddenSeries": false, + "id": 27, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "s3_adaptor r/w performance", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 72, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 254 + }, + "hiddenSeries": false, + "id": 73, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_disk_cache_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_diskcache_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 254 + }, + "hiddenSeries": false, + "id": 74, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_disk_cache_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_diskcache_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 262 + }, + "hiddenSeries": false, + "id": 75, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_disk_cache_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_diskcache_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 262 + }, + "hiddenSeries": false, + "id": 76, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_disk_cache_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_diskcache_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 270 + }, + "hiddenSeries": false, + "id": 77, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_disk_cache_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_diskcache_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 270 + }, + "hiddenSeries": false, + "id": 78, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_disk_cache_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_diskcache_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "s3_adaptor r/w diskcache performance", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 69, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 113 + }, + "hiddenSeries": false, + "id": 62, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.0.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_s3_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_s3_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 113 + }, + "hiddenSeries": false, + "id": 63, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.0.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_s3_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_s3_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 121 + }, + "hiddenSeries": false, + "id": 66, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.0.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_s3_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_s3_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 121 + }, + "hiddenSeries": false, + "id": 65, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.0.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_s3_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_s3_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 129 + }, + "hiddenSeries": false, + "id": 64, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.0.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_s3_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read_s3_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 129 + }, + "hiddenSeries": false, + "id": 67, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.0.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_s3_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_s3_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "s3_adaptor r/w s3 performance", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 80, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 304 + }, + "hiddenSeries": false, + "id": 81, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_write_s3_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_s3_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 304 + }, + "hiddenSeries": false, + "id": 82, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_write_s3_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_s3_bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 312 + }, + "hiddenSeries": false, + "id": 83, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_write_s3_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write_s3_latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "diskcache r/w s3 performance", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 111, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 321 + }, + "hiddenSeries": false, + "id": 112, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*_file_manager_num\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "file_manager_num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 321 + }, + "hiddenSeries": false, + "id": 113, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs.*_chunk_manager_num\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "chunk_manager_num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "manager metric", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 121, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 122, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"diskcache_cache_count\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "diskcache_cache_count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "hiddenSeries": false, + "id": 123, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"diskcache_cache_bytes\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "diskcache_cache_bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "hiddenSeries": false, + "id": 124, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"icache_cache_count\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "icache_cache_count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 71 + }, + "hiddenSeries": false, + "id": 125, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"inode_s3_chunk_info_size\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "inode_s3_chunk_info_size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 79 + }, + "hiddenSeries": false, + "id": 126, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"dcache_cache_count\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "dcache_cache_count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "hiddenSeries": false, + "id": 127, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"dcache_cache_bytes\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "dcache_cache_bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "lru cache", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 115, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total cache disk capacity", + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 209, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_diskcache_totalbytes\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The size occupied by S3 objects cached in the cache disk.", + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 178, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_diskcache_usedbytes\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of objects in cache", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 210, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"diskcache_cache_count\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cache count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "diskcache usage in lru", + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "hiddenSeries": false, + "id": 217, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"diskcache_cache_bytes\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cache bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of cache hits", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 211, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"diskcache_cache_hit\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cache hit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Cache hit rate", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 213, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"diskcache_cache_hit\", instance=~\"$instance\"}", + "hide": true, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "hit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"diskcache_cache_miss\", instance=~\"$instance\"}", + "hide": true, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "miss" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$hit/($hit+$miss)", + "hide": false, + "refId": "A", + "type": "math" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cache hit rate", "tooltip": { "shared": true, "sort": 0, @@ -5231,7 +10734,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "binBps", + "format": "percentunit", "logBase": 1, "show": true }, @@ -5255,9 +10758,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Number of cache misses", "fieldConfig": { "defaults": { - "unit": "binBps" + "unit": "short" }, "overrides": [] }, @@ -5266,11 +10770,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 54 + "x": 0, + "y": 55 }, "hiddenSeries": false, - "id": 76, + "id": 212, "interval": "1s", "legend": { "alignAsTable": true, @@ -5289,7 +10793,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5303,16 +10807,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_disk_cache_bps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"diskcache_cache_miss\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "write_diskcache_bps", + "title": "cache miss", "tooltip": { "shared": true, "sort": 0, @@ -5327,7 +10833,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "binBps", + "format": "short", "logBase": 1, "show": true }, @@ -5351,9 +10857,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "This metric tracks the cache eviction throughput in terms of bandwidth. It measures how much data is being evicted from the cache disk per second, recorded in units like megabytes/second or gigabytes/second.", "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "binBps" }, "overrides": [] }, @@ -5362,11 +10869,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 62 + "x": 12, + "y": 55 }, "hiddenSeries": false, - "id": 77, + "id": 215, "interval": "1s", "legend": { "alignAsTable": true, @@ -5385,7 +10892,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5399,17 +10906,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_disk_cache_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", - "hide": false, + "expr": "{__name__=~\"curvefs_disk_cache_[[fs:regex]]_diskcache_trim_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", - "refId": "B" + "range": true, + "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "read_diskcache_latency", + "title": "trim bps", "tooltip": { "shared": true, "sort": 0, @@ -5424,7 +10932,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "µs", + "format": "binBps", "logBase": 1, "show": true }, @@ -5448,9 +10956,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "This metric measures the efficiency of evicting objects from the cache disk, recorded as the number of objects evicted from the cache per second", "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -5459,11 +10968,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 62 + "x": 0, + "y": 63 }, "hiddenSeries": false, - "id": 78, + "id": 214, "interval": "1s", "legend": { "alignAsTable": true, @@ -5482,7 +10991,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5496,17 +11005,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_disk_cache_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", - "hide": false, + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_diskcache_trim_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", - "refId": "B" + "range": true, + "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "write_diskcache_latency", + "title": "trim qps", "tooltip": { "shared": true, "sort": 0, @@ -5521,7 +11031,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "µs", + "format": "short", "logBase": 1, "show": true }, @@ -5535,637 +11045,128 @@ "yaxis": { "align": false } - } - ], - "targets": [ + }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "refId": "A" - } - ], - "title": "s3_adaptor r/w diskcache performance", - "type": "row" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 69, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "description": "This metric tracks the cache eviction latency. It measures how long it takes to evict objects from the cache disk, recorded in units like milliseconds or microseconds per eviction.", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] }, - "refId": "A" - } - ], - "title": "s3_adaptor r/w s3 performance", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 62, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.4.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 }, - "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_s3_qps\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "read_s3_qps", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:213", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 7 - }, - "hiddenSeries": false, - "id": 63, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.4.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "hiddenSeries": false, + "id": 216, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true }, - "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_s3_qps\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "write_s3_qps", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:213", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "binBps" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 66, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.4.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true }, - "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_s3_bps\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "read_s3_bps", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "binBps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:213", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "binBps" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "hiddenSeries": false, - "id": 65, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.4.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_diskcache_trim_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "trim latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_s3_bps\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "write_s3_bps", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "binBps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:213", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "µs" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 64, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.4.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_s3_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "read_s3_latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "µs", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:213", - "format": "short", - "logBase": 1, - "show": true + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "µs" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "hiddenSeries": false, - "id": 67, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.4.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "exemplar": true, - "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_s3_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "write_s3_latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "µs", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:213", - "format": "short", - "logBase": 1, - "show": true + "refId": "A" } ], - "yaxis": { - "align": false - } + "title": "diskcache cache", + "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 16 }, - "id": 80, + "id": 218, "panels": [ { "aliasColors": {}, @@ -6176,6 +11177,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Number of times using kvcache cache (set)", "fieldConfig": { "defaults": { "unit": "short" @@ -6188,10 +11190,10 @@ "h": 8, "w": 12, "x": 0, - "y": 96 + "y": 32 }, "hiddenSeries": false, - "id": 81, + "id": 219, "interval": "1s", "legend": { "alignAsTable": true, @@ -6210,7 +11212,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6224,16 +11226,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_write_s3_qps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_count\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "write_s3_qps", + "title": "set count", "tooltip": { "shared": true, "sort": 0, @@ -6272,9 +11276,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "kvcache cache hit rate", "fieldConfig": { "defaults": { - "unit": "binBps" + "unit": "percentunit" }, "overrides": [] }, @@ -6284,10 +11289,10 @@ "h": 8, "w": 12, "x": 12, - "y": 96 + "y": 32 }, "hiddenSeries": false, - "id": 82, + "id": 220, "interval": "1s", "legend": { "alignAsTable": true, @@ -6306,7 +11311,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6320,16 +11325,44 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_write_s3_bps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_hit\", instance=~\"$instance\"}", + "hide": true, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "hit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_miss\", instance=~\"$instance\"}", + "hide": true, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "miss" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$hit/($hit+$miss)", + "hide": false, + "refId": "hit rate", + "type": "math" } ], "thresholds": [], "timeRegions": [], - "title": "write_s3_bps", + "title": "hit rate", "tooltip": { "shared": true, "sort": 0, @@ -6344,7 +11377,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "binBps", + "format": "percentunit", "logBase": 1, "show": true }, @@ -6368,6 +11401,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Number of kvcache cache hits", "fieldConfig": { "defaults": { "unit": "short" @@ -6380,10 +11414,10 @@ "h": 8, "w": 12, "x": 0, - "y": 104 + "y": 40 }, "hiddenSeries": false, - "id": 83, + "id": 221, "interval": "1s", "legend": { "alignAsTable": true, @@ -6402,7 +11436,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6416,17 +11450,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_write_s3_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_hit\", instance=~\"$instance\"}", "hide": false, "interval": "", "legendFormat": "", - "refId": "B" + "range": true, + "refId": "hit" } ], "thresholds": [], "timeRegions": [], - "title": "write_s3_latency", + "title": "hit", "tooltip": { "shared": true, "sort": 0, @@ -6455,34 +11491,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "diskcache r/w s3 performance", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 111, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -6492,6 +11501,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Number of kvcache cache misses", "fieldConfig": { "defaults": { "unit": "short" @@ -6503,11 +11513,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 113 + "x": 12, + "y": 40 }, "hiddenSeries": false, - "id": 112, + "id": 222, "interval": "1s", "legend": { "alignAsTable": true, @@ -6526,7 +11536,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6540,16 +11550,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs.*_file_manager_num\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_miss\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "miss" } ], "thresholds": [], "timeRegions": [], - "title": "file_manager_num", + "title": "miss", "tooltip": { "shared": true, "sort": 0, @@ -6588,6 +11601,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "kvcache get qps", "fieldConfig": { "defaults": { "unit": "short" @@ -6599,11 +11613,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 113 + "x": 0, + "y": 48 }, "hiddenSeries": false, - "id": 113, + "id": 223, "interval": "1s", "legend": { "alignAsTable": true, @@ -6622,7 +11636,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6636,16 +11650,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs.*_chunk_manager_num\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_get_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "chunk_manager_num", + "title": "get qps", "tooltip": { "shared": true, "sort": 0, @@ -6674,34 +11690,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "manager metric", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 121, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -6711,6 +11700,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "kvcache set qps", "fieldConfig": { "defaults": { "unit": "short" @@ -6722,11 +11712,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 50 + "x": 12, + "y": 48 }, "hiddenSeries": false, - "id": 122, + "id": 226, "interval": "1s", "legend": { "alignAsTable": true, @@ -6745,7 +11735,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6759,16 +11749,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"diskcache_cache_count\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_set_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "diskcache_cache_count", + "title": "set qps", "tooltip": { "shared": true, "sort": 0, @@ -6807,9 +11799,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "kvcache get latency", "fieldConfig": { "defaults": { - "unit": "bytes" + "unit": "µs" }, "overrides": [] }, @@ -6818,11 +11811,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 50 + "x": 0, + "y": 56 }, "hiddenSeries": false, - "id": 123, + "id": 225, "interval": "1s", "legend": { "alignAsTable": true, @@ -6841,7 +11834,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6855,16 +11848,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"diskcache_cache_bytes\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_get_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "diskcache_cache_bytes", + "title": "get latency", "tooltip": { "shared": true, "sort": 0, @@ -6879,7 +11875,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "bytes", + "format": "µs", "logBase": 1, "show": true }, @@ -6903,9 +11899,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "kvcache set latency", "fieldConfig": { "defaults": { - "unit": "none" + "unit": "µs" }, "overrides": [] }, @@ -6914,11 +11911,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 58 + "x": 12, + "y": 56 }, "hiddenSeries": false, - "id": 124, + "id": 227, "interval": "1s", "legend": { "alignAsTable": true, @@ -6937,7 +11934,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6951,16 +11948,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"icache_cache_count\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_set_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "icache_cache_count", + "title": "set latency", "tooltip": { "shared": true, "sort": 0, @@ -6975,7 +11975,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "none", + "format": "µs", "logBase": 1, "show": true }, @@ -6999,9 +11999,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "kvcache get bps", "fieldConfig": { "defaults": { - "unit": "none" + "unit": "binBps" }, "overrides": [] }, @@ -7010,11 +12011,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 58 + "x": 0, + "y": 64 }, "hiddenSeries": false, - "id": 125, + "id": 224, "interval": "1s", "legend": { "alignAsTable": true, @@ -7033,7 +12034,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7047,16 +12048,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"inode_s3_chunk_info_size\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_get_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "inode_s3_chunk_info_size", + "title": "get bps", "tooltip": { "shared": true, "sort": 0, @@ -7071,7 +12074,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "none", + "format": "binBps", "logBase": 1, "show": true }, @@ -7095,9 +12098,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "kvcache set bps", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "binBps" }, "overrides": [] }, @@ -7106,11 +12110,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 66 + "x": 12, + "y": 64 }, "hiddenSeries": false, - "id": 126, + "id": 228, "interval": "1s", "legend": { "alignAsTable": true, @@ -7129,7 +12133,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7143,16 +12147,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"dcache_cache_count\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_set_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "dcache_cache_count", + "title": "set bps", "tooltip": { "shared": true, "sort": 0, @@ -7167,7 +12173,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "short", + "format": "binBps", "logBase": 1, "show": true }, @@ -7181,7 +12187,21 @@ "yaxis": { "align": false } - }, + } + ], + "title": "kvcache manager", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 229, + "panels": [ { "aliasColors": {}, "bars": false, @@ -7191,9 +12211,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "memcache client get latency", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -7202,11 +12223,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 66 + "x": 0, + "y": 33 }, "hiddenSeries": false, - "id": 127, + "id": 232, "interval": "1s", "legend": { "alignAsTable": true, @@ -7225,7 +12246,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7239,16 +12260,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"dcache_cache_bytes\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_get_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "dcache_cache_bytes", + "title": "get latency", "tooltip": { "shared": true, "sort": 0, @@ -7263,7 +12287,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, @@ -7277,34 +12301,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "lru cache", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 115, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -7314,6 +12311,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "memcache client set qps", "fieldConfig": { "defaults": { "unit": "short" @@ -7325,11 +12323,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 147 + "x": 12, + "y": 33 }, "hiddenSeries": false, - "id": 116, + "id": 231, "interval": "1s", "legend": { "alignAsTable": true, @@ -7348,7 +12346,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7362,16 +12360,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs.*_read_data_cache_num\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_set_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "read_data_cache_num", + "title": "set qps", "tooltip": { "shared": true, "sort": 0, @@ -7410,9 +12410,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "memcache client get bps", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "binBps" }, "overrides": [] }, @@ -7421,11 +12422,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 147 + "x": 0, + "y": 41 }, "hiddenSeries": false, - "id": 117, + "id": 234, "interval": "1s", "legend": { "alignAsTable": true, @@ -7444,7 +12445,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7458,16 +12459,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs.*_write_data_cache_num\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_get_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "write_data_cache_num", + "title": "get bps", "tooltip": { "shared": true, "sort": 0, @@ -7482,7 +12485,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "short", + "format": "binBps", "logBase": 1, "show": true }, @@ -7506,9 +12509,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "memcache client set latency", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, @@ -7517,11 +12521,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 155 + "x": 12, + "y": 41 }, "hiddenSeries": false, - "id": 118, + "id": 233, "interval": "1s", "legend": { "alignAsTable": true, @@ -7540,7 +12544,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7554,16 +12558,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs.*_read_data_cache_byte\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_set_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, "interval": "", "legendFormat": "", - "refId": "A" + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "read_data_cache_byte", + "title": "set latency", "tooltip": { "shared": true, "sort": 0, @@ -7578,7 +12585,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, @@ -7602,6 +12609,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "memcache get qps", "fieldConfig": { "defaults": { "unit": "short" @@ -7613,11 +12621,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 155 + "x": 0, + "y": 49 }, "hiddenSeries": false, - "id": 119, + "id": 230, "interval": "1s", "legend": { "alignAsTable": true, @@ -7636,7 +12644,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7650,16 +12658,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs.*_write_data_cache_byte\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_get_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "write_data_cache_byte", + "title": "get qps", "tooltip": { "shared": true, "sort": 0, @@ -7698,9 +12708,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "memcache client set bps", "fieldConfig": { "defaults": { - "unit": "bytes" + "unit": "binBps" }, "overrides": [] }, @@ -7709,11 +12720,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 163 + "x": 12, + "y": 49 }, "hiddenSeries": false, - "id": 178, + "id": 235, "interval": "1s", "legend": { "alignAsTable": true, @@ -7732,7 +12743,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7746,16 +12757,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_disk_cache_[[fs:regex]]_diskcache_usedbytes\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_set_bps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "diskcache_usedbytes", + "title": "set bps", "tooltip": { "shared": true, "sort": 0, @@ -7770,7 +12783,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "bytes", + "format": "binBps", "logBase": 1, "show": true }, @@ -7786,16 +12799,7 @@ } } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "diskcache cache num/byte", + "title": "memcache client", "type": "row" }, { @@ -7808,7 +12812,7 @@ "h": 1, "w": 24, "x": 0, - "y": 35 + "y": 18 }, "id": 32, "panels": [ @@ -7833,7 +12837,7 @@ "h": 6, "w": 12, "x": 0, - "y": 172 + "y": 138 }, "hiddenSeries": false, "id": 36, @@ -7855,7 +12859,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7929,7 +12933,7 @@ "h": 6, "w": 12, "x": 12, - "y": 172 + "y": 138 }, "hiddenSeries": false, "id": 128, @@ -7951,7 +12955,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -8025,7 +13029,7 @@ "h": 6, "w": 12, "x": 0, - "y": 178 + "y": 144 }, "hiddenSeries": false, "id": 129, @@ -8047,7 +13051,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -8121,7 +13125,7 @@ "h": 6, "w": 12, "x": 12, - "y": 178 + "y": 144 }, "hiddenSeries": false, "id": 130, @@ -8143,7 +13147,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -8217,7 +13221,7 @@ "h": 6, "w": 12, "x": 0, - "y": 184 + "y": 150 }, "hiddenSeries": false, "id": 131, @@ -8239,7 +13243,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -8313,7 +13317,7 @@ "h": 6, "w": 12, "x": 12, - "y": 184 + "y": 150 }, "hiddenSeries": false, "id": 132, @@ -8335,7 +13339,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -8409,7 +13413,7 @@ "h": 6, "w": 12, "x": 0, - "y": 190 + "y": 156 }, "hiddenSeries": false, "id": 133, @@ -8505,7 +13509,7 @@ "h": 6, "w": 12, "x": 12, - "y": 190 + "y": 156 }, "hiddenSeries": false, "id": 134, @@ -8601,7 +13605,7 @@ "h": 6, "w": 12, "x": 0, - "y": 196 + "y": 162 }, "hiddenSeries": false, "id": 135, @@ -8697,7 +13701,7 @@ "h": 6, "w": 12, "x": 12, - "y": 196 + "y": 162 }, "hiddenSeries": false, "id": 136, @@ -8793,7 +13797,7 @@ "h": 6, "w": 12, "x": 0, - "y": 202 + "y": 168 }, "hiddenSeries": false, "id": 137, @@ -8889,7 +13893,7 @@ "h": 6, "w": 12, "x": 12, - "y": 202 + "y": 168 }, "hiddenSeries": false, "id": 138, @@ -8985,7 +13989,7 @@ "h": 6, "w": 12, "x": 0, - "y": 208 + "y": 174 }, "hiddenSeries": false, "id": 139, @@ -9081,7 +14085,7 @@ "h": 6, "w": 12, "x": 12, - "y": 208 + "y": 174 }, "hiddenSeries": false, "id": 140, @@ -9176,7 +14180,7 @@ "h": 6, "w": 12, "x": 0, - "y": 214 + "y": 180 }, "hiddenSeries": false, "id": 141, @@ -9271,7 +14275,7 @@ "h": 6, "w": 12, "x": 12, - "y": 214 + "y": 180 }, "hiddenSeries": false, "id": 142, @@ -9367,7 +14371,7 @@ "h": 6, "w": 12, "x": 0, - "y": 220 + "y": 186 }, "hiddenSeries": false, "id": 143, @@ -9463,7 +14467,7 @@ "h": 6, "w": 12, "x": 12, - "y": 220 + "y": 186 }, "hiddenSeries": false, "id": 144, @@ -9559,7 +14563,7 @@ "h": 6, "w": 12, "x": 0, - "y": 226 + "y": 192 }, "hiddenSeries": false, "id": 145, @@ -9655,7 +14659,7 @@ "h": 6, "w": 12, "x": 12, - "y": 226 + "y": 192 }, "hiddenSeries": false, "id": 146, @@ -9751,7 +14755,7 @@ "h": 6, "w": 12, "x": 0, - "y": 232 + "y": 198 }, "hiddenSeries": false, "id": 147, @@ -9847,7 +14851,7 @@ "h": 6, "w": 12, "x": 12, - "y": 232 + "y": 198 }, "hiddenSeries": false, "id": 148, @@ -9943,7 +14947,7 @@ "h": 6, "w": 12, "x": 0, - "y": 238 + "y": 204 }, "hiddenSeries": false, "id": 149, @@ -10039,7 +15043,7 @@ "h": 6, "w": 12, "x": 12, - "y": 238 + "y": 204 }, "hiddenSeries": false, "id": 150, @@ -10137,7 +15141,7 @@ "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 19 }, "id": 34, "panels": [ @@ -10162,7 +15166,7 @@ "h": 6, "w": 12, "x": 0, - "y": 245 + "y": 66 }, "hiddenSeries": false, "id": 155, @@ -10184,7 +15188,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -10258,7 +15262,7 @@ "h": 6, "w": 12, "x": 12, - "y": 245 + "y": 66 }, "hiddenSeries": false, "id": 152, @@ -10280,7 +15284,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -10354,7 +15358,7 @@ "h": 6, "w": 12, "x": 0, - "y": 251 + "y": 72 }, "hiddenSeries": false, "id": 153, @@ -10376,7 +15380,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -10450,7 +15454,7 @@ "h": 6, "w": 12, "x": 12, - "y": 251 + "y": 72 }, "hiddenSeries": false, "id": 156, @@ -10472,7 +15476,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -10546,7 +15550,7 @@ "h": 6, "w": 12, "x": 0, - "y": 257 + "y": 78 }, "hiddenSeries": false, "id": 157, @@ -10568,7 +15572,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -10642,7 +15646,7 @@ "h": 6, "w": 12, "x": 12, - "y": 257 + "y": 78 }, "hiddenSeries": false, "id": 154, @@ -10664,7 +15668,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -10738,7 +15742,7 @@ "h": 6, "w": 12, "x": 0, - "y": 263 + "y": 84 }, "hiddenSeries": false, "id": 151, @@ -10834,7 +15838,7 @@ "h": 6, "w": 12, "x": 12, - "y": 263 + "y": 84 }, "hiddenSeries": false, "id": 158, @@ -10930,7 +15934,7 @@ "h": 6, "w": 12, "x": 0, - "y": 269 + "y": 90 }, "hiddenSeries": false, "id": 159, @@ -11026,7 +16030,7 @@ "h": 6, "w": 12, "x": 12, - "y": 269 + "y": 90 }, "hiddenSeries": false, "id": 160, @@ -11122,7 +16126,7 @@ "h": 6, "w": 12, "x": 0, - "y": 275 + "y": 96 }, "hiddenSeries": false, "id": 161, @@ -11218,7 +16222,7 @@ "h": 6, "w": 12, "x": 12, - "y": 275 + "y": 96 }, "hiddenSeries": false, "id": 162, @@ -11314,7 +16318,7 @@ "h": 6, "w": 12, "x": 0, - "y": 281 + "y": 102 }, "hiddenSeries": false, "id": 163, @@ -11410,7 +16414,7 @@ "h": 6, "w": 12, "x": 12, - "y": 281 + "y": 102 }, "hiddenSeries": false, "id": 164, @@ -11506,7 +16510,7 @@ "h": 6, "w": 12, "x": 0, - "y": 287 + "y": 108 }, "hiddenSeries": false, "id": 165, @@ -11602,7 +16606,7 @@ "h": 6, "w": 12, "x": 12, - "y": 287 + "y": 108 }, "hiddenSeries": false, "id": 166, @@ -11698,7 +16702,7 @@ "h": 6, "w": 12, "x": 0, - "y": 293 + "y": 114 }, "hiddenSeries": false, "id": 167, @@ -11794,7 +16798,7 @@ "h": 6, "w": 12, "x": 12, - "y": 293 + "y": 114 }, "hiddenSeries": false, "id": 168, @@ -11890,7 +16894,7 @@ "h": 6, "w": 12, "x": 0, - "y": 299 + "y": 120 }, "hiddenSeries": false, "id": 169, @@ -11986,7 +16990,7 @@ "h": 6, "w": 12, "x": 12, - "y": 299 + "y": 120 }, "hiddenSeries": false, "id": 170, @@ -12082,7 +17086,7 @@ "h": 6, "w": 12, "x": 0, - "y": 305 + "y": 126 }, "hiddenSeries": false, "id": 171, @@ -12178,7 +17182,7 @@ "h": 6, "w": 12, "x": 12, - "y": 305 + "y": 126 }, "hiddenSeries": false, "id": 172, @@ -12274,7 +17278,7 @@ "h": 6, "w": 12, "x": 0, - "y": 311 + "y": 132 }, "hiddenSeries": false, "id": 173, @@ -12370,7 +17374,7 @@ "h": 6, "w": 12, "x": 12, - "y": 311 + "y": 132 }, "hiddenSeries": false, "id": 174, @@ -12459,303 +17463,301 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 20 }, "id": 180, - "panels": [], - "title": "warmup", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 182, - "options": { - "legend": { - "calcs": [ - "max", - "min", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "builder", - "expr": "curvefs_warmup_s3_cached_bps", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "warmup_s3_bps", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 222 + }, + "id": 182, + "options": { + "legend": { + "calcs": [ + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "curvefs_warmup_s3_cached_bps", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "warmup_s3_bps", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 184, - "options": { - "legend": { - "calcs": [ - "max", - "min", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "builder", - "expr": "curvefs_warmup_s3_cached_qps", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "warmup_s3_qps", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } - ] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 222 + }, + "id": 184, + "options": { + "legend": { + "calcs": [ + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "unit": "bytes" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "curvefs_warmup_s3_cached_qps", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "warmup_s3_qps", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 186, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "builder", - "expr": "curvefs_warmup_s3_cache_size", - "legendFormat": "__auto", - "range": true, - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 230 + }, + "id": 186, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "curvefs_warmup_s3_cache_size", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "warmup_s3_size", + "type": "timeseries" } ], - "title": "warmup_s3_size", - "type": "timeseries" + "title": "warmup", + "type": "row" } ], "refresh": "5s", @@ -12828,10 +17830,10 @@ "current": { "selected": true, "text": [ - "latency_99" + "latency" ], "value": [ - "latency_99" + "latency" ] }, "datasource": { diff --git a/curvefs/monitor/grafana/provisioning/dashboards/cluster.json b/curvefs/monitor/grafana/provisioning/dashboards/cluster.json new file mode 100644 index 0000000000..2dcae0ec8e --- /dev/null +++ b/curvefs/monitor/grafana/provisioning/dashboards/cluster.json @@ -0,0 +1,1014 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Curvefs cluster", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 7, + "interval": "1s", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "{__name__=~\"fs_usage_info_fs_[[fsName:regex]]_used\", job=\"mds\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "fs used", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "cluster total used", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"fs_usage_info_fs_[[fsName:regex]]_used\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "cluster total used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 4, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "{__name__=~\"fs_usage_info_fs_[[fsName:regex]]_used\", job=\"mds\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "fs used", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 8, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"topology_fs_id_[[fsId:regex]]_.*inode_num\",job=\"mds\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "inode_num_by_type", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 13, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_fs_id_[0-9]*_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 10, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_.*_type_file_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster file inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 9, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_.*_type_directory_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster directory inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 12, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_.*_type_sym_link_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster sym_link inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 11, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_.*_type_s3_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster s3 inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "fs usage", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "{__name__=~\"fs_usage_info_fs_.*_used\",job=\"mds\"}", + "hide": 0, + "includeAll": true, + "label": "fsName", + "multi": false, + "name": "fsName", + "options": [], + "query": { + "query": "{__name__=~\"fs_usage_info_fs_.*_used\",job=\"mds\"}", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/fs_usage_info_fs_(.*)_used.*/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "{__name__=~\"topology_fs_id_[0-9]*_inode_num\",job=\"mds\"}", + "hide": 0, + "includeAll": true, + "label": "fsId", + "multi": false, + "name": "fsId", + "options": [], + "query": { + "query": "{__name__=~\"topology_fs_id_[0-9]*_inode_num\",job=\"mds\"}", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/topology_fs_id_([0-9]*)_inode_num.*/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "cluster", + "uid": "c57406c1-8262-4bf8-85c4-f4f65f336f96", + "version": 1, + "weekStart": "" +} diff --git a/curvefs/monitor/grafana/provisioning/dashboards/memcached.json b/curvefs/monitor/grafana/provisioning/dashboards/memcached.json new file mode 100644 index 0000000000..b1cc362ccf --- /dev/null +++ b/curvefs/monitor/grafana/provisioning/dashboards/memcached.json @@ -0,0 +1,1917 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Memcached dashboard for grafana 6.5.1 or above.\r\nDataSource: Prometheus\r\nCollector: Memcached official exporter : https://github.com/prometheus/memcached_exporter/releases \r\n", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11527, + "graphTooltip": 0, + "id": 15, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0.1 + }, + { + "color": "#299c46", + "value": 0.9 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.4", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "memcached_up{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Instance State", + "type": "stat" + }, + { + "aliasColors": { + "ratio": "#6ED0E0" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (memcached_commands_total{command=\"get\",job=\"$job\"}) / (sum (memcached_commands_total{command=\"get\",job=\"$job\"}) + sum (memcached_commands_total{command=\"set\",job=\"$job\"}))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Get", + "refId": "A", + "step": 5, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (memcached_commands_total{command=\"set\",job=\"$job\"}) / (sum (memcached_commands_total{command=\"get\",job=\"$job\"}) + sum (memcached_commands_total{command=\"set\",job=\"$job\"}))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Set", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Get & Set ratio", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 0 + }, + "height": "120px", + "hiddenSeries": false, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "miss", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (memcached_commands_total{status=\"hit\",command=\"get\",}) / sum (memcached_commands_total{command=\"get\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "hit", + "refId": "C", + "step": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (memcached_commands_total{status=\"miss\",command=\"get\",job=\"$job\"}) / sum (memcached_commands_total{command=\"get\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "miss", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Hit & Miss ratio", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "logBase": 1, + "max": "1", + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 35, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (memcached_items_evicted_total{job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 20 + } + ], + "title": "Evicts (total)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 37, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (memcached_items_reclaimed_total{job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 20 + } + ], + "title": "Reclaims (total)", + "type": "stat" + }, + { + "aliasColors": { + "evicts": "#890F02", + "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", + "reclaims": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 6 + }, + "height": "240px", + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "reclaims", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (irate (memcached_items_evicted_total{job=\"$job\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "evicts", + "refId": "A", + "step": 5, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (irate (memcached_items_reclaimed_total{job=\"$job\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "reclaims", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Evicts & Reclaims rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 6 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "memcached_current_connections{job=\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 6 + }, + "height": "240px", + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum (irate (memcached_commands_total{status=\"hit\",command=\"get\",job=\"$job\"}[5m])) / sum (irate (memcached_commands_total{command=\"get\",job=\"$job\"}[5m]\n))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Hit", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Hit rate", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "delta(memcached_read_bytes_total{job=\"$job\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 13 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "memcached_current_bytes{job=\"$job\", instance=~\"$instance\"}/memcached_limit_bytes{job=\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Used Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 13 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "memcached_current_items{job=\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Items in cache", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "delta(memcached_written_bytes_total{job=\"$job\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Write[1min]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 20 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "delta(memcached_read_bytes_total{job=\"$job\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read(1min)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 11, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum (delta(memcached_commands_total{job=\"$job\", status=\"hit\",command=\"get\"}[1m])) / sum (delta(memcached_commands_total{job=\"$job\",command=\"get\"}[1m])) * 100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "get ratio", + "range": true, + "refId": "A" + } + ], + "title": "Get Hit Ratio Command(1m)", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 120, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum (irate (memcached_commands_total{job=\"$job\", instance=~\"$instance\"}[5m])) by (command)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 4, + "target": "" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Commands", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(delta(memcached_commands_total{job=\"$job\", instance=~\"$instance\"}[30s]))/30", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "memcached_read_bytes_total{job=\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read Bytes Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 35 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "memcached_written_bytes_total{job=\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read Bytes Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "memcached", + "value": "memcached" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "job", + "options": [], + "query": "label_values(memcached_up, job)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "{__name__=~\"memcached_up\", job=\"memcached\"}", + "hide": 0, + "includeAll": true, + "label": "Addr", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "{__name__=~\"memcached_up\", job=\"memcached\"}", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/.*instance=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Memcached", + "uid": "AQxf3X-mk", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json b/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json index c21f1bf870..a4395d2af3 100644 --- a/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json +++ b/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -20,543 +23,1019 @@ }, "description": "Curvefs mds server", "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 5, - "iteration": 1660902423620, + "id": 8, "links": [], + "liveNow": false, "panels": [ { - "collapsed": false, - "datasource": null, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 6, - "panels": [], - "title": "process usage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 4, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "id": 70, + "panels": [ { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus" + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "exemplar": true, - "expr": "process_memory_resident{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "process_memory_resident {{hostname}}", - "refId": "process_memory_resident" - }, - { - "datasource": { - "type": "prometheus" - }, - "exemplar": true, - "expr": "process_memory_virtual{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_memory_virtual {{hostname}}", - "refId": "process_memory_virtual" - }, - { - "datasource": { - "type": "prometheus" + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] }, - "exemplar": true, - "expr": "process_memory_shared{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_memory_shared {{hostname}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "process memory usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:63", - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:64", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 2, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 }, - "exemplar": true, - "expr": "process_cpu_usage{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "process_cpu_usage {{hostname}}", - "refId": "process_cpu_usage" - }, - { - "datasource": { - "type": "prometheus" - }, - "exemplar": true, - "expr": "process_cpu_usage_system{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_cpu_usage_system {{hostname}}", - "refId": "process_cpu_usage_system" - }, - { - "datasource": { - "type": "prometheus" - }, - "exemplar": true, - "expr": "process_cpu_usage_user{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_cpu_usage_user {{hostname}}", - "refId": "process_cpu_usage_user" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "process cpu usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:495", - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:496", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 14, - "panels": [], - "title": "metaserver config", - "type": "row" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "hiddenSeries": false, + "id": 69, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - { - "color": "red", - "value": 80 - } - ] + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_s3_client_adaptor_delete_inode_bps\",job=\"metaserver\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "delete inode bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 16, - "interval": "1s", - "options": { - "showHeader": true - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "exemplar": true, - "expr": "{__name__=~\"curvefs_metaserver_config.*\", job=\"metaserver\", instance=~\"$instance\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "metaserver config", - "transformations": [ { - "id": "organize", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 71, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "excludeByName": { - "Time": true, - "Value": true, - "__name__": true, - "hostname": true, - "job": true + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_s3_client_adaptor_delete_inode_qps\",job=\"metaserver\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "delete inode qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "short", + "logBase": 1, + "show": true }, - "indexByName": {}, - "renameByName": {} + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false } - } - ], - "type": "table" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 30, - "panels": [], - "title": "service total performance", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "unit": "qps" }, - "overrides": [] + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 72, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_s3_client_adaptor_delete_inode_lat_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "delete inode latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 73, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_s3_client_adaptor_delete_inode_eps\",job=\"metaserver\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "delete inode eps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "s3 client adaptor", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 19 - }, - "hiddenSeries": false, - "id": 31, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "y": 1 }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "id": 6, + "panels": [ { - "exemplar": true, - "expr": "sum({__name__=~\".*curvefs_metaserver_meta_server_service.*_qps\",job=\"metaserver\", instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 4, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_memory_resident{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "process_memory_resident {{hostname}}", + "refId": "process_memory_resident" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_memory_virtual{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_memory_virtual {{hostname}}", + "refId": "process_memory_virtual" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_memory_shared{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_memory_shared {{hostname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "process memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "decbytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_cpu_usage{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "process_cpu_usage {{hostname}}", + "refId": "process_cpu_usage" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_cpu_usage_system{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_cpu_usage_system {{hostname}}", + "refId": "process_cpu_usage_system" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_cpu_usage_user{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_cpu_usage_user {{hostname}}", + "refId": "process_cpu_usage_user" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "process cpu usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:495", + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:496", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "total_qps", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:63", - "format": "qps", - "logBase": 1, - "show": true - }, + "targets": [ { - "$$hashKey": "object:64", - "format": "short", - "logBase": 1, - "show": true + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" } ], - "yaxis": { - "align": false - } + "title": "process usage", + "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "hiddenSeries": false, - "id": 32, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "h": 1, + "w": 24, + "x": 0, + "y": 2 }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "id": 14, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 16, + "interval": "1s", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_config.*\", job=\"metaserver\", instance=~\"$instance\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "metaserver config", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "hostname": true, + "job": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], "targets": [ { - "exemplar": true, - "expr": "sum({__name__=~\".*curvefs_metaserver_meta_server_service.*_error\",job=\"metaserver\", instance=~\"$instance\"}) by (instance)", - "interval": "", - "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "total_error_count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "title": "metaserver config", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 }, - "yaxes": [ + "id": 30, + "panels": [ { - "$$hashKey": "object:63", - "format": "none", - "logBase": 1, - "show": true + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "qps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 31, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum({__name__=~\".*curvefs_metaserver_meta_server_service.*_qps\",job=\"metaserver\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "total_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "qps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "$$hashKey": "object:64", - "format": "short", - "logBase": 1, - "show": true + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 32, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum({__name__=~\".*curvefs_metaserver_meta_server_service.*_error\",job=\"metaserver\", instance=~\"$instance\"}) by (instance)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "total_error_count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" } ], - "yaxis": { - "align": false - } + "title": "service total performance", + "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 4 }, "id": 8, "panels": [ @@ -565,7 +1044,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -578,7 +1060,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 21 }, "hiddenSeries": false, "id": 9, @@ -600,7 +1082,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -610,6 +1092,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -618,9 +1104,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "create_inode", "tooltip": { "shared": true, @@ -629,9 +1113,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -658,7 +1140,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -671,7 +1156,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 21 }, "hiddenSeries": false, "id": 10, @@ -693,7 +1178,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -703,6 +1188,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -711,9 +1200,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "create_dentry", "tooltip": { "shared": true, @@ -722,9 +1209,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -751,7 +1236,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -764,7 +1252,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 29 }, "hiddenSeries": false, "id": 17, @@ -786,7 +1274,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -796,6 +1284,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -804,9 +1296,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "delete_inode", "tooltip": { "shared": true, @@ -815,9 +1305,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -844,7 +1332,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -857,7 +1348,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 29 }, "hiddenSeries": false, "id": 18, @@ -879,7 +1370,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -889,6 +1380,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -897,9 +1392,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "delete_dentry", "tooltip": { "shared": true, @@ -908,9 +1401,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -937,7 +1428,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -950,7 +1444,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 37 }, "hiddenSeries": false, "id": 11, @@ -972,7 +1466,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -982,6 +1476,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -990,9 +1488,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "get_inode", "tooltip": { "shared": true, @@ -1001,9 +1497,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1030,7 +1524,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1043,7 +1540,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 37 }, "hiddenSeries": false, "id": 12, @@ -1065,7 +1562,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1075,6 +1572,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1083,9 +1584,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "get_dentry", "tooltip": { "shared": true, @@ -1094,9 +1593,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1123,7 +1620,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1136,7 +1636,7 @@ "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 45 }, "hiddenSeries": false, "id": 19, @@ -1158,7 +1658,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1168,6 +1668,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_batch_get_inode_attr\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1176,9 +1680,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "batch_get_inode_attr", "tooltip": { "shared": true, @@ -1187,9 +1689,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1216,7 +1716,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1229,7 +1732,7 @@ "h": 8, "w": 12, "x": 12, - "y": 52 + "y": 45 }, "hiddenSeries": false, "id": 20, @@ -1251,7 +1754,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1261,6 +1764,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_list_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1269,9 +1776,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "list_dentry", "tooltip": { "shared": true, @@ -1280,9 +1785,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1309,7 +1812,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1322,7 +1828,7 @@ "h": 8, "w": 12, "x": 0, - "y": 60 + "y": 53 }, "hiddenSeries": false, "id": 21, @@ -1344,7 +1850,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1354,6 +1860,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_update_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1362,9 +1872,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "update_inode", "tooltip": { "shared": true, @@ -1373,9 +1881,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1402,7 +1908,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1415,7 +1924,7 @@ "h": 8, "w": 12, "x": 12, - "y": 60 + "y": 53 }, "hiddenSeries": false, "id": 22, @@ -1437,7 +1946,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1447,6 +1956,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_batch_get_xattr\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1455,9 +1968,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "batch_get_xattr", "tooltip": { "shared": true, @@ -1466,9 +1977,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1495,7 +2004,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1508,7 +2020,7 @@ "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 61 }, "hiddenSeries": false, "id": 23, @@ -1530,7 +2042,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1540,6 +2052,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_prepare_rename_tx\", job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1548,9 +2064,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "prepare_rename_tx", "tooltip": { "shared": true, @@ -1559,9 +2073,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1588,7 +2100,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1601,7 +2116,7 @@ "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 61 }, "hiddenSeries": false, "id": 24, @@ -1623,7 +2138,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1633,6 +2148,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_or_modify_s3_chunk_info\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1641,9 +2160,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "get_or_modify_s3_chunk_info", "tooltip": { "shared": true, @@ -1652,9 +2169,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1681,7 +2196,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1694,7 +2212,7 @@ "h": 8, "w": 12, "x": 0, - "y": 76 + "y": 69 }, "hiddenSeries": false, "id": 25, @@ -1716,7 +2234,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1726,6 +2244,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_partition\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1734,9 +2256,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "create_partition", "tooltip": { "shared": true, @@ -1745,9 +2265,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1774,7 +2292,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1787,7 +2308,7 @@ "h": 8, "w": 12, "x": 12, - "y": 76 + "y": 69 }, "hiddenSeries": false, "id": 26, @@ -1809,7 +2330,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1819,6 +2340,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_partition\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1827,9 +2352,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "delete_partition", "tooltip": { "shared": true, @@ -1838,9 +2361,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1867,7 +2388,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1880,7 +2404,7 @@ "h": 8, "w": 12, "x": 0, - "y": 84 + "y": 77 }, "hiddenSeries": false, "id": 27, @@ -1902,7 +2426,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1912,6 +2436,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_volume_extent\",job=\"metaserver\", instance=~\"$instance\",quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1920,9 +2448,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "get_volume_extent", "tooltip": { "shared": true, @@ -1931,9 +2457,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1960,7 +2484,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1973,7 +2500,7 @@ "h": 8, "w": 12, "x": 12, - "y": 84 + "y": 77 }, "hiddenSeries": false, "id": 28, @@ -1995,7 +2522,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2005,6 +2532,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_update_volume_extent\",job=\"metaserver\", instance=~\"$instance\",quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -2013,9 +2544,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "update_volume_extent", "tooltip": { "shared": true, @@ -2024,9 +2553,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2049,17 +2576,29 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "service latency", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 5 }, "id": 34, "panels": [ @@ -2068,7 +2607,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "qps" @@ -2081,7 +2623,7 @@ "h": 8, "w": 8, "x": 0, - "y": 29 + "y": 13 }, "hiddenSeries": false, "id": 35, @@ -2103,7 +2645,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2113,6 +2655,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_qps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2121,9 +2667,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_qps", "tooltip": { "shared": true, @@ -2132,9 +2676,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2161,7 +2703,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "reqps" @@ -2174,7 +2719,7 @@ "h": 8, "w": 8, "x": 8, - "y": 29 + "y": 13 }, "hiddenSeries": false, "id": 36, @@ -2196,7 +2741,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2206,6 +2751,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_rps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2214,9 +2763,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_rps", "tooltip": { "shared": true, @@ -2225,9 +2772,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2254,7 +2799,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "eps" @@ -2267,7 +2815,7 @@ "h": 8, "w": 8, "x": 16, - "y": 29 + "y": 13 }, "hiddenSeries": false, "id": 37, @@ -2289,7 +2837,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2299,6 +2847,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_eps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2307,9 +2859,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_eps", "tooltip": { "shared": true, @@ -2318,9 +2868,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2347,7 +2895,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -2360,7 +2911,7 @@ "h": 8, "w": 8, "x": 0, - "y": 37 + "y": 21 }, "hiddenSeries": false, "id": 38, @@ -2382,7 +2933,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2392,6 +2943,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2400,9 +2955,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_apply_latency", "tooltip": { "shared": true, @@ -2411,9 +2964,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2440,7 +2991,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -2453,7 +3007,7 @@ "h": 8, "w": 8, "x": 8, - "y": 37 + "y": 21 }, "hiddenSeries": false, "id": 39, @@ -2475,7 +3029,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2485,6 +3039,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_execute_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2493,9 +3051,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_apply_execute_latency", "tooltip": { "shared": true, @@ -2504,9 +3060,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2533,7 +3087,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -2546,7 +3103,7 @@ "h": 8, "w": 8, "x": 16, - "y": 37 + "y": 21 }, "hiddenSeries": false, "id": 40, @@ -2568,7 +3125,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2578,6 +3135,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_wait_in_queue_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2586,9 +3147,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_apply_wait_in_queue_latency", "tooltip": { "shared": true, @@ -2597,9 +3156,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2622,17 +3179,29 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "raft op metric", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 6 }, "id": 42, "panels": [ @@ -2641,7 +3210,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -2654,7 +3226,7 @@ "h": 8, "w": 12, "x": 0, - "y": 30 + "y": 14 }, "hiddenSeries": false, "id": 43, @@ -2676,7 +3248,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2686,6 +3258,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"copyset_snapshot_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2694,9 +3270,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "copyset_snapshot_latency", "tooltip": { "shared": true, @@ -2705,9 +3279,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2734,7 +3306,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -2747,7 +3322,7 @@ "h": 8, "w": 12, "x": 12, - "y": 30 + "y": 14 }, "hiddenSeries": false, "id": 44, @@ -2769,7 +3344,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2779,6 +3354,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"copyset_snapshot_error_count\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2787,9 +3366,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "copyset_snapshot_error_count", "tooltip": { "shared": true, @@ -2798,9 +3375,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2827,7 +3402,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -2840,7 +3418,7 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 22 }, "hiddenSeries": false, "id": 45, @@ -2862,7 +3440,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2872,6 +3450,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"copyset_snapshot_flying_count\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2880,9 +3462,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "copyset_snapshot_flying_count", "tooltip": { "shared": true, @@ -2891,9 +3471,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2916,17 +3494,29 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "raft snapshot metric", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 7 }, "id": 47, "panels": [ @@ -2935,7 +3525,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -2948,7 +3541,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 15 }, "hiddenSeries": false, "id": 48, @@ -2970,7 +3563,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2980,6 +3573,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_flushing\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2988,9 +3585,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_flushing", "tooltip": { "shared": true, @@ -2999,9 +3594,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3028,7 +3621,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -3041,7 +3637,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 15 }, "hiddenSeries": false, "id": 49, @@ -3063,7 +3659,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3073,6 +3669,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_flush_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3081,9 +3681,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_flush_latency", "tooltip": { "shared": true, @@ -3092,9 +3690,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3121,7 +3717,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3134,7 +3733,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 23 }, "hiddenSeries": false, "id": 50, @@ -3156,7 +3755,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3166,6 +3765,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_compacting\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3174,9 +3777,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_compacting", "tooltip": { "shared": true, @@ -3185,9 +3786,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3214,7 +3813,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -3227,7 +3829,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 23 }, "hiddenSeries": false, "id": 51, @@ -3249,7 +3851,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3259,6 +3861,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_compaction_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3267,9 +3873,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_compaction_latency", "tooltip": { "shared": true, @@ -3278,9 +3882,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3307,7 +3909,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3320,7 +3925,7 @@ "h": 8, "w": 8, "x": 0, - "y": 47 + "y": 31 }, "hiddenSeries": false, "id": 52, @@ -3342,7 +3947,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3352,6 +3957,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_sealed_memtable\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3360,9 +3969,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_sealed_memtable", "tooltip": { "shared": true, @@ -3371,9 +3978,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3400,7 +4005,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3413,7 +4021,7 @@ "h": 8, "w": 8, "x": 8, - "y": 47 + "y": 31 }, "hiddenSeries": false, "id": 53, @@ -3435,7 +4043,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3445,6 +4053,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_delayed_write\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3453,9 +4065,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_delayed_write", "tooltip": { "shared": true, @@ -3464,9 +4074,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3493,7 +4101,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3506,7 +4117,7 @@ "h": 8, "w": 8, "x": 16, - "y": 47 + "y": 31 }, "hiddenSeries": false, "id": 54, @@ -3528,7 +4139,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3538,6 +4149,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_stopped_write\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3546,9 +4161,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_stopped_write", "tooltip": { "shared": true, @@ -3557,9 +4170,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3582,17 +4193,29 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "rocksdb metric", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 8 }, "id": 56, "panels": [ @@ -3601,7 +4224,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3614,7 +4240,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 16 }, "hiddenSeries": false, "id": 57, @@ -3636,7 +4262,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3646,6 +4272,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_apply_tasks_batch_avg_counter\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3653,6 +4283,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_apply_tasks_batch_counter_.*\",job=\"metaserver\", instance=~\"$instance\"}", "hide": false, @@ -3662,9 +4296,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_apply_tasks_batch_counter", "tooltip": { "shared": true, @@ -3673,9 +4305,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3702,7 +4332,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3715,7 +4348,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 16 }, "hiddenSeries": false, "id": 59, @@ -3737,7 +4370,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3747,6 +4380,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_send_entries_batch_avg_counter\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3754,6 +4391,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_send_entries_batch_counter_.*\",job=\"metaserver\", instance=~\"$instance\"}", "hide": false, @@ -3763,9 +4404,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_send_entries_batch_counter", "tooltip": { "shared": true, @@ -3774,9 +4413,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3803,7 +4440,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "qps" @@ -3816,7 +4456,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 24 }, "hiddenSeries": false, "id": 61, @@ -3838,7 +4478,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3848,6 +4488,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_send_entries_qps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3856,9 +4500,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_send_entries_qps", "tooltip": { "shared": true, @@ -3867,9 +4509,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3896,7 +4536,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -3909,7 +4552,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 24 }, "hiddenSeries": false, "id": 60, @@ -3931,7 +4574,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3941,6 +4584,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_send_entries_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3949,9 +4596,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_send_entries_latency", "tooltip": { "shared": true, @@ -3960,9 +4605,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3989,7 +4632,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "qps" @@ -4002,7 +4648,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 32 }, "hiddenSeries": false, "id": 62, @@ -4024,7 +4670,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4034,6 +4680,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*raft_service_append_entries_qps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4042,9 +4692,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_service_append_entries_qps", "tooltip": { "shared": true, @@ -4053,9 +4701,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4082,7 +4728,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -4095,7 +4744,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 32 }, "hiddenSeries": false, "id": 63, @@ -4117,7 +4766,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4127,6 +4776,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*raft_service_append_entries\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4135,9 +4788,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_service_append_entries_latency", "tooltip": { "shared": true, @@ -4146,9 +4797,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4175,7 +4824,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -4188,7 +4840,7 @@ "h": 8, "w": 12, "x": 0, - "y": 56 + "y": 40 }, "hiddenSeries": false, "id": 64, @@ -4210,7 +4862,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4220,6 +4872,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*raft_storage_append_entries_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4228,9 +4884,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_storage_append_entries_latency", "tooltip": { "shared": true, @@ -4239,9 +4893,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4268,7 +4920,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -4281,7 +4936,7 @@ "h": 8, "w": 12, "x": 12, - "y": 56 + "y": 40 }, "hiddenSeries": false, "id": 65, @@ -4303,7 +4958,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4313,6 +4968,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_storage_append_entries_normalized_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4321,9 +4980,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_storage_append_entries_normalized_latency", "tooltip": { "shared": true, @@ -4332,9 +4989,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4361,7 +5016,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -4374,7 +5032,7 @@ "h": 8, "w": 12, "x": 0, - "y": 64 + "y": 48 }, "hiddenSeries": false, "id": 66, @@ -4396,7 +5054,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4406,6 +5064,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_storage_flush_batch_avg_counter\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4413,6 +5075,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_storage_flush_batch_counter_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "hide": false, @@ -4422,9 +5088,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_storage_flush_batch_counter", "tooltip": { "shared": true, @@ -4433,9 +5097,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4462,7 +5124,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -4475,7 +5140,7 @@ "h": 8, "w": 12, "x": 12, - "y": 64 + "y": 48 }, "hiddenSeries": false, "id": 67, @@ -4497,7 +5162,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4507,6 +5172,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_num_log_entries\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4515,9 +5184,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_num_log_entries", "tooltip": { "shared": true, @@ -4526,9 +5193,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4555,7 +5220,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "qps" @@ -4568,7 +5236,7 @@ "h": 8, "w": 12, "x": 0, - "y": 72 + "y": 56 }, "hiddenSeries": false, "id": 68, @@ -4600,6 +5268,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*raft_service_install_snapshot_qps\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4608,9 +5280,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_service_install_snapshot_qps", "tooltip": { "shared": true, @@ -4619,9 +5289,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4644,20 +5312,28 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "raft builtin metric", "type": "row" } ], - "refresh": "", - "schemaVersion": 30, + "refresh": "5s", + "schemaVersion": 38, "style": "dark", "tags": [], "templating": { "list": [ { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -4665,10 +5341,11 @@ "$__all" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\"bthread_count\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "Addr", @@ -4686,9 +5363,8 @@ "type": "query" }, { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -4696,10 +5372,11 @@ "$__all" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\"op_apply_pool_.*_copyset.*create_inode_count\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "pool", @@ -4717,9 +5394,8 @@ "type": "query" }, { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -4727,10 +5403,11 @@ "$__all" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\".*copyset_.*_create_inode_count\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "copyset", @@ -4750,7 +5427,7 @@ { "allValue": "", "current": { - "selected": false, + "selected": true, "text": [ "latency" ], @@ -4758,10 +5435,11 @@ "latency" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\".*(latency_[0-9]*|latency)\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "latency_quantile", @@ -4779,9 +5457,8 @@ "type": "query" }, { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -4789,10 +5466,11 @@ "$__all" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\".*curvefs_metaserver_meta_server_service_.*_qps\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "op", @@ -4812,7 +5490,7 @@ { "allValue": "", "current": { - "selected": false, + "selected": true, "text": [ "1" ], @@ -4820,10 +5498,11 @@ "1" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\".*curvefs_metaserver_meta_server_service.*\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "service_quantile", @@ -4851,4 +5530,4 @@ "title": "metaserver", "uid": "d12ivuiVz", "version": 8 -} +} \ No newline at end of file diff --git a/curvefs/monitor/target_json.py b/curvefs/monitor/target_json.py index 3b36e2e617..1447219f84 100644 --- a/curvefs/monitor/target_json.py +++ b/curvefs/monitor/target_json.py @@ -11,6 +11,7 @@ CURVEFS_TOOL = "curvefs_tool" JSON_PATH = "/tmp/topology.json" +PLUGIN_PATH = "plugin" HOSTNAME_PORT_REGEX = r"[^\"\ ]\S*:\d+" IP_PORT_REGEX = r"[0-9]+(?:\.[0-9]+){3}:\d+" @@ -97,6 +98,19 @@ def unitValue(lables, targets): unit["targets"] = targets return unit +def loadPlugin(): + # load *.json file in plugin dir + # merge to one json + data = [] + if os.path.isdir(PLUGIN_PATH): + for filename in os.listdir(PLUGIN_PATH): + if filename.endswith('.json'): + with open(os.path.join(PLUGIN_PATH, filename)) as f: + plugin_data = json.load(f) + if len(plugin_data) == 0: + continue + data.append(unitValue(plugin_data["labels"], plugin_data["targets"])) + return data def refresh(): targets = [] @@ -113,6 +127,8 @@ def refresh(): # load client client = loadClient() targets.append(client) + plugin = loadPlugin() + targets += plugin with open(targetPath+'.new', 'w', 0o777) as fd: json.dump(targets, fd, indent=4) diff --git a/curvefs/proto/common.proto b/curvefs/proto/common.proto index d2772dbf46..25ba220f42 100644 --- a/curvefs/proto/common.proto +++ b/curvefs/proto/common.proto @@ -80,7 +80,7 @@ message PartitionInfo { // partition manage inodeid range [start, end] required uint64 start = 5; required uint64 end = 6; - required uint64 txId = 7; + optional uint64 txId = 7; optional uint64 nextId = 8; // status can change from READWRITE to READONLY, but can not chanage from READONLY to READWRITE // READWRITE/READONLY can change to DELETING, but DELETING can not change to READWRITE/READONLY @@ -92,6 +92,14 @@ message PartitionInfo { optional bool manageFlag = 13; // if a partition has recyclebin inode, set this flag true } +message AppliedIndex { + required int64 index = 1; +} + +message ItemCount { + required uint64 count = 1; +} + message Peer { optional uint64 id = 1; optional string address = 2; diff --git a/curvefs/proto/mds.proto b/curvefs/proto/mds.proto index eebec8ee34..f89c13e733 100644 --- a/curvefs/proto/mds.proto +++ b/curvefs/proto/mds.proto @@ -105,6 +105,10 @@ message FsInfo { optional uint32 recycleTimeHour =15; } +message FsUsage { + optional uint64 usedBytes = 1; +} + message GetFsInfoResponse { required FSStatusCode statusCode = 1; optional FsInfo fsInfo = 2; @@ -145,16 +149,14 @@ message UmountFsResponse { required FSStatusCode statusCode = 1; } -/* UpdateFsInfoRequest comment now message UpdateFsInfoRequest { required string fsName = 1; - // todo + optional uint64 capacity = 2; } + message UpdateFsInfoResponse { required FSStatusCode statusCode = 1; - optional FsInfo fsInfo = 2; } -*/ message DeleteFsRequest { required string fsName = 1; @@ -182,17 +184,26 @@ message ListClusterFsInfoResponse { repeated FsInfo fsInfo = 1; } +message FsDelta { + optional int64 bytes = 1; +} + message RefreshSessionRequest { repeated topology.PartitionTxId txIds = 1; // used for client timeout required string fsName = 2; required Mountpoint mountpoint = 3; + optional FsDelta fsDelta = 4; + optional string mdsAddrs = 5; } message RefreshSessionResponse { required FSStatusCode statusCode = 1; repeated topology.PartitionTxId latestTxIdList = 2; optional bool enableSumInDir = 3; + optional uint64 fsCapacity = 4; + optional uint64 fsUsedBytes = 5; + optional string mdsAddrsOverride = 6; } message DLockValue { @@ -225,6 +236,22 @@ message CommitTxResponse { required FSStatusCode statusCode = 1; } +message SetClientMdsAddrsOverrideRequest { + required string clientMdsAddrsOverride = 1; +} + +message SetClientMdsAddrsOverrideResponse { + required FSStatusCode statusCode = 1; +} + +message TsoRequest {} + +message TsoResponse { + required FSStatusCode statusCode = 1; + optional uint64 ts = 2; // transaction sequence number + optional uint64 timestamp = 3; +} + service MdsService { // fs interface rpc CreateFs(CreateFsRequest) returns (CreateFsResponse); @@ -232,7 +259,7 @@ service MdsService { rpc UmountFs(UmountFsRequest) returns (UmountFsResponse); // TODO(chengyi01): move to GetFssInfo rpc GetFsInfo(GetFsInfoRequest) returns (GetFsInfoResponse); - // rpc UpdateFsInfo(UpdateFsInfoRequest) returns (UpdateFsInfoResponse); + rpc UpdateFsInfo(UpdateFsInfoRequest) returns (UpdateFsInfoResponse); rpc DeleteFs(DeleteFsRequest) returns (DeleteFsResponse); rpc AllocateS3Chunk(AllocateS3ChunkRequest) returns (AllocateS3ChunkResponse); rpc ListClusterFsInfo (ListClusterFsInfoRequest) returns (ListClusterFsInfoResponse); @@ -240,6 +267,11 @@ service MdsService { rpc GetLatestTxId(GetLatestTxIdRequest) returns (GetLatestTxIdResponse); rpc CommitTx(CommitTxRequest) returns (CommitTxResponse); + rpc Tso(TsoRequest) returns (TsoResponse); + // client lease rpc RefreshSession(RefreshSessionRequest) returns (RefreshSessionResponse); + + // client mds addrs override, for mds migration + rpc SetClientMdsAddrsOverride(SetClientMdsAddrsOverrideRequest) returns (SetClientMdsAddrsOverrideResponse); } diff --git a/curvefs/proto/metaserver.proto b/curvefs/proto/metaserver.proto index 4340d638cc..d22246069e 100644 --- a/curvefs/proto/metaserver.proto +++ b/curvefs/proto/metaserver.proto @@ -49,6 +49,15 @@ enum MetaStatusCode { RPC_STREAM_ERROR = 25; INODE_S3_META_TOO_LARGE = 26; STORAGE_CLOSED = 27; + // tx v2 related + TX_FAILED = 28; + TX_WRITE_CONFLICT = 29; + TX_KEY_LOCKED = 30; + TX_COMMITTED = 31; + TX_ROLLBACKED = 32; + TX_TIMEOUT = 33; + TX_INPROGRESS = 34; + TX_MISMATCH = 35; } // dentry interface @@ -59,7 +68,7 @@ message GetDentryRequest { required uint32 fsId = 4; required uint64 parentInodeId = 5; required string name = 6; - required uint64 txId = 7; + optional uint64 txId = 7; optional uint64 appliedIndex = 8; } @@ -74,7 +83,8 @@ message Dentry { required uint64 inodeId = 2; required uint64 parentInodeId = 3; required string name = 4; - required uint64 txId = 5; + // reused txId as ts in tx v2 for compatibility in metaserver + optional uint64 txId = 5; optional uint32 flag = 6; optional FsFileType type = 7; optional uint64 txSequence = 8; @@ -88,6 +98,7 @@ message GetDentryResponse { required MetaStatusCode statusCode = 1; optional Dentry dentry = 2; optional uint64 appliedIndex = 3; + optional TxLock txLock = 4; } message ListDentryRequest { @@ -96,7 +107,7 @@ message ListDentryRequest { required uint32 partitionId = 3; required uint32 fsId = 4; required uint64 dirInodeId = 5; - required uint64 txId = 6; + optional uint64 txId = 6; optional string last = 7; // the name of last entry optional uint32 count = 8; // the number of entry required optional bool onlyDir = 9; @@ -107,6 +118,7 @@ message ListDentryResponse { required MetaStatusCode statusCode = 1; repeated Dentry dentrys = 2; optional uint64 appliedIndex = 3; + optional TxLock txLock = 4; } message CreateDentryRequest { @@ -114,11 +126,13 @@ message CreateDentryRequest { required uint32 copysetId = 2; required uint32 partitionId = 3; required Dentry dentry = 4; + optional Time create = 5; } message CreateDentryResponse { required MetaStatusCode statusCode = 1; optional uint64 appliedIndex = 2; + optional TxLock txLock = 3; } message DeleteDentryRequest { @@ -126,15 +140,17 @@ message DeleteDentryRequest { required uint32 copysetId = 2; required uint32 partitionId = 3; required uint32 fsId = 4; - required uint64 txId = 5; + optional uint64 txId = 5; required uint64 parentInodeId = 6; required string name = 7; optional FsFileType type = 8; + optional Time create = 9; } message DeleteDentryResponse { required MetaStatusCode statusCode = 1; optional uint64 appliedIndex = 2; + optional TxLock txLock = 3; } message PrepareRenameTxRequest { @@ -144,11 +160,99 @@ message PrepareRenameTxRequest { repeated Dentry dentrys = 4; } +message TransactionRequest { + enum TransactionType { + None = 0; + Rename = 1; + } + required TransactionType type = 1; + required string rawPayload = 2; +} + message PrepareRenameTxResponse { required MetaStatusCode statusCode = 1; optional uint64 appliedIndex = 2; } +message TxLock { + required string primaryKey = 1; + required uint64 startTs = 2; + required uint64 timestamp = 3; + optional uint32 index = 4; + optional int32 ttl = 5; +} + +enum TxWriteKind { + Commit = 1; + Rollback = 2; +} + +message TS { + required uint64 ts = 1; +} + +message TxWrite { + required uint64 startTs = 1; + required TxWriteKind kind = 2; +} + +message PrewriteRenameTxRequest { + required uint32 poolId = 1; + required uint32 copysetId = 2; + required uint32 partitionId = 3; + repeated Dentry dentrys = 4; + required TxLock txLock = 5; +} + +message PrewriteRenameTxResponse { + required MetaStatusCode statusCode = 1; + repeated Dentry dentrys = 2; + optional TxLock txLock = 3; + optional uint64 appliedIndex = 4; +} + +message CheckTxStatusRequest { + required uint32 poolId = 1; + required uint32 copysetId = 2; + required uint32 partitionId = 3; + required string primaryKey = 4; + required uint64 startTs = 5; + required uint64 curTimestamp = 6; +} + +message CheckTxStatusResponse { + required MetaStatusCode statusCode = 1; + optional uint64 appliedIndex = 2; +} + +message ResolveTxLockRequest { + required uint32 poolId = 1; + required uint32 copysetId = 2; + required uint32 partitionId = 3; + required Dentry dentry = 4; + required uint64 startTs = 5; + required uint64 commitTs = 6; +} + +message ResolveTxLockResponse { + required MetaStatusCode statusCode = 1; + optional uint64 appliedIndex = 2; +} + +message CommitTxRequest { + required uint32 poolId = 1; + required uint32 copysetId = 2; + required uint32 partitionId = 3; + repeated Dentry dentrys = 4; + required uint64 startTs = 5; + required uint64 commitTs = 6; +} + +message CommitTxResponse { + required MetaStatusCode statusCode = 1; + optional uint64 appliedIndex = 2; +} + // inode interface message GetInodeRequest { required uint32 poolId = 1; @@ -256,7 +360,7 @@ message Inode { optional uint64 rdev = 16; // field 17 is left for compatibility map s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index - optional uint32 dtime = 19; + optional uint64 dtime = 19; optional uint32 openmpcount = 20; // openmpcount mount points had the file open map xattr = 21; repeated uint64 parent = 22; @@ -528,6 +632,11 @@ service MetaServerService { rpc CreateDentry(CreateDentryRequest) returns (CreateDentryResponse); rpc DeleteDentry(DeleteDentryRequest) returns (DeleteDentryResponse); rpc PrepareRenameTx(PrepareRenameTxRequest) returns (PrepareRenameTxResponse); + // tx v2 + rpc PrewriteRenameTx(PrewriteRenameTxRequest) returns (PrewriteRenameTxResponse); + rpc CheckTxStatus(CheckTxStatusRequest) returns (CheckTxStatusResponse); + rpc ResolveTxLock(ResolveTxLockRequest) returns (ResolveTxLockResponse); + rpc CommitTx(CommitTxRequest) returns (CommitTxResponse); // inode interface rpc GetInode(GetInodeRequest) returns (GetInodeResponse); diff --git a/curvefs/proto/topology.proto b/curvefs/proto/topology.proto index c9d70682b5..f248655fca 100644 --- a/curvefs/proto/topology.proto +++ b/curvefs/proto/topology.proto @@ -61,7 +61,6 @@ message ClusterInfoData { required string clusterId = 1; // map partitionIndexs = 2; - } message PoolData { diff --git a/curvefs/sdk/.clang-format b/curvefs/sdk/.clang-format new file mode 100644 index 0000000000..47a38a93f2 --- /dev/null +++ b/curvefs/sdk/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: Never diff --git a/curvefs/sdk/README.md b/curvefs/sdk/README.md new file mode 100644 index 0000000000..6848362017 --- /dev/null +++ b/curvefs/sdk/README.md @@ -0,0 +1,18 @@ +Hadoop SDK +=== + +How to build +--- + +``` bash +$ git clone git@github.com:opencurve/curve.git +$ make playground +$ make ci-dep stor=fs +$ make sdk +``` + +It will generate a jar package after build success: + +``` +Build SDK success => /curve/curvefs/sdk/output/curvefs-hadoop-1.0-SNAPSHOT.jar +``` diff --git a/curvefs/sdk/java/native/BUILD b/curvefs/sdk/java/native/BUILD new file mode 100644 index 0000000000..13dcef4778 --- /dev/null +++ b/curvefs/sdk/java/native/BUILD @@ -0,0 +1,38 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_binary( + name = "curvefs_jni", + srcs = glob([ + "*.h", + "*.cpp", + ]), + visibility = ["//visibility:public"], + copts = CURVE_DEFAULT_COPTS, + linkopts = [ + "-Wl,-rpath=/tmp/libcurvefs,--disable-new-dtags", + "-L/usr/lib/x86_64-linux-gnu/", + "-lhashkit", + ], + deps = [ + "@com_google_absl//absl/cleanup", + "@jni//:copy_jni_hdr_lib", + "//curvefs/sdk/libcurvefs:curvefs_lib", + ], + linkshared = True, +) diff --git a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.cpp b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.cpp new file mode 100644 index 0000000000..0357d71873 --- /dev/null +++ b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.cpp @@ -0,0 +1,798 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-08 + * Author: Jingli Chen (Wine93) + */ + +#include +#include +#include +#include + +#include "absl/cleanup/cleanup.h" +#include "curvefs/sdk/libcurvefs/libcurvefs.h" +#include "curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.h" + +const char* statvfs_cls_name = + "io/opencurve/curve/fs/libfs/CurveFsMount$StatVfs"; +const char* stat_cls_name = + "io/opencurve/curve/fs/libfs/CurveFsMount$Stat"; +const char* file_cls_name = + "io/opencurve/curve/fs/libfs/CurveFsMount$File"; +const char* dirent_cls_name = + "io/opencurve/curve/fs/libfs/CurveFsMount$Dirent"; + +// Cached for class +static jclass statvfs_cls; +static jclass stat_cls; +static jclass file_cls; +static jclass dirent_cls; + +// Cached field IDs for io.opencurve.curve.fs.CurveFsMount.StatVfs +static jfieldID statvfs_bsize_fid; +static jfieldID statvfs_frsize_fid; +static jfieldID statvfs_blocks_fid; +static jfieldID statvfs_bavail_fid; +static jfieldID statvfs_files_fid; +static jfieldID statvfs_fsid_fid; +static jfieldID statvfs_namemax_fid; + +// Cached field IDs for io.opencurve.curve.fs.CurveFsMount.Stat +static jfieldID stat_mode_fid; +static jfieldID stat_uid_fid; +static jfieldID stat_gid_fid; +static jfieldID stat_size_fid; +static jfieldID stat_blksize_fid; +static jfieldID stat_blocks_fid; +static jfieldID stat_atime_fid; +static jfieldID stat_mtime_fid; +static jfieldID stat_isFile_fid; +static jfieldID stat_isDirectory_fid; +static jfieldID stat_isSymlink_fid; + +// Cached field IDs for io.opencurve.curve.fs.CurveFsMount.File +static jfieldID file_fd_fid; +static jfieldID file_length_fid; + +// Cached field IDs for io.opencurve.curve.fs.CurveFsMount.Dirent +static jfieldID dirent_name_fid; +static jfieldID dirent_stat_fid; + + +// Setup cached field IDs +static void setup_field_ids(JNIEnv* env) { +/* + * Get a fieldID from a class with a specific type + * + * clz: jclass + * field: field in clz + * type: integer, long, etc.. + * + * This macro assumes some naming convention that is used + * only in this file: + * + * GETFID(curvestat, mode, I) gets translated into + * curvestat_mode_fid = env->GetFieldID(curvestat_cls, "mode", "I"); + */ +#define GETFID(clz, field, type) do { \ + clz ## _ ## field ## _fid = env->GetFieldID(clz ## _cls, #field, #type); \ + if (!clz ## _ ## field ## _fid) \ + return; \ + } while (0) + + // Cache StatVfs fields + statvfs_cls = env->FindClass(statvfs_cls_name); + if (!statvfs_cls) { + return; + } + + GETFID(statvfs, bsize, J); + GETFID(statvfs, frsize, J); + GETFID(statvfs, blocks, J); + GETFID(statvfs, bavail, J); + GETFID(statvfs, files, J); + GETFID(statvfs, fsid, J); + GETFID(statvfs, namemax, J); + + // Cache Stat fields + stat_cls = env->FindClass(stat_cls_name); + if (!stat_cls) { + return; + } + + GETFID(stat, mode, I); + GETFID(stat, uid, I); + GETFID(stat, gid, I); + GETFID(stat, size, J); + GETFID(stat, blksize, J); + GETFID(stat, blocks, J); + GETFID(stat, atime, J); + GETFID(stat, mtime, J); + GETFID(stat, isFile, Z); + GETFID(stat, isDirectory, Z); + GETFID(stat, isSymlink, Z); + + // Cache File fields + file_cls = env->FindClass(file_cls_name); + if (!file_cls) { + return; + } + + GETFID(file, fd, I); + GETFID(file, length, J); + + // Cache Dirent fields + dirent_cls = env->FindClass(dirent_cls_name); + if (!dirent_cls) { + return; + } + + GETFID(dirent, name, Ljava/lang/String;); + GETFID(dirent, stat, Lio/opencurve/curve/fs/libfs/CurveFsMount$Stat;); + +#undef GETFID +} + +static void fill_statvfs(JNIEnv* env, jobject j_statvfs, struct statvfs* st) { + env->SetLongField(j_statvfs, statvfs_bsize_fid, st->f_bsize); + env->SetLongField(j_statvfs, statvfs_frsize_fid, st->f_frsize); + env->SetLongField(j_statvfs, statvfs_blocks_fid, st->f_blocks); + env->SetLongField(j_statvfs, statvfs_bavail_fid, st->f_bavail); + env->SetLongField(j_statvfs, statvfs_files_fid, st->f_files); + env->SetLongField(j_statvfs, statvfs_fsid_fid, st->f_fsid); + env->SetLongField(j_statvfs, statvfs_namemax_fid, st->f_namemax); +} + +static void fill_stat(JNIEnv* env, jobject j_stat, struct stat* stat) { + env->SetIntField(j_stat, stat_mode_fid, stat->st_mode); + env->SetIntField(j_stat, stat_uid_fid, stat->st_uid); + env->SetIntField(j_stat, stat_gid_fid, stat->st_gid); + env->SetLongField(j_stat, stat_size_fid, stat->st_size); + env->SetLongField(j_stat, stat_blksize_fid, stat->st_blksize); + env->SetLongField(j_stat, stat_blocks_fid, stat->st_blocks); + + // mtime + uint64_t time = stat->st_mtim.tv_sec; + time *= 1000; + time += stat->st_mtim.tv_nsec / 1000000; + env->SetLongField(j_stat, stat_mtime_fid, time); + + // atime + time = stat->st_atim.tv_sec; + time *= 1000; + time += stat->st_atim.tv_nsec / 1000000; + env->SetLongField(j_stat, stat_atime_fid, time); + + env->SetBooleanField(j_stat, stat_isFile_fid, + S_ISREG(stat->st_mode) ? JNI_TRUE : JNI_FALSE); + + env->SetBooleanField(j_stat, stat_isDirectory_fid, + S_ISDIR(stat->st_mode) ? JNI_TRUE : JNI_FALSE); + + env->SetBooleanField(j_stat, stat_isSymlink_fid, + S_ISLNK(stat->st_mode) ? JNI_TRUE : JNI_FALSE); +} + +static void fill_file(JNIEnv* env, jobject j_file, file_t* file) { + env->SetIntField(j_file, file_fd_fid, file->fd); + env->SetLongField(j_file, file_length_fid, file->length); +} + +static void fill_dirent(JNIEnv* env, jobject j_dirent, dirent_t* dirent) { + jstring j_name = env->NewStringUTF(dirent->name); + jobject j_stat = env->AllocObject(env->FindClass(stat_cls_name)); + fill_stat(env, j_stat, &dirent->stat); + + env->SetObjectField(j_dirent, dirent_name_fid, j_name); + env->SetObjectField(j_dirent, dirent_stat_fid, j_stat); + + env->DeleteLocalRef(j_name); + env->DeleteLocalRef(j_stat); +} + +// Map io_opencurve_curve_fs_libfs_CurveFsMount_O_* open flags to values in libc +static inline uint32_t fixup_open_flags(jint jflags) { + uint32_t flags = 0; + +#define FIXUP_OPEN_FLAG(name) \ + if (jflags & io_opencurve_curve_fs_libfs_CurveFsMount_##name) \ + flags |= name; + + FIXUP_OPEN_FLAG(O_RDONLY) + FIXUP_OPEN_FLAG(O_RDWR) + FIXUP_OPEN_FLAG(O_APPEND) + FIXUP_OPEN_FLAG(O_CREAT) + FIXUP_OPEN_FLAG(O_TRUNC) + FIXUP_OPEN_FLAG(O_EXCL) + FIXUP_OPEN_FLAG(O_WRONLY) + FIXUP_OPEN_FLAG(O_DIRECTORY) + +#undef FIXUP_OPEN_FLAG + + return flags; +} + +#define CURVEFS_SETATTR_MODE (1 << 0) +#define CURVEFS_SETATTR_UID (1 << 1) +#define CURVEFS_SETATTR_GID (1 << 2) +#define CURVEFS_SETATTR_SIZE (1 << 3) +#define CURVEFS_SETATTR_ATIME (1 << 4) +#define CURVEFS_SETATTR_MTIME (1 << 5) +#define CURVEFS_SETATTR_ATIME_NOW (1 << 7) +#define CURVEFS_SETATTR_MTIME_NOW (1 << 8) +#define CURVEFS_SETATTR_CTIME (1 << 10) + +// Map JAVA_SETATTR_* to values in curve lib +static inline int fixup_attr_mask(jint jmask) { + int mask = 0; + +#define FIXUP_ATTR_MASK(name) \ + if (jmask & io_opencurve_curve_fs_libfs_CurveFsMount_##name) \ + mask |= CURVEFS_##name; + + FIXUP_ATTR_MASK(SETATTR_MODE) + FIXUP_ATTR_MASK(SETATTR_UID) + FIXUP_ATTR_MASK(SETATTR_GID) + FIXUP_ATTR_MASK(SETATTR_MTIME) + FIXUP_ATTR_MASK(SETATTR_ATIME) + +#undef FIXUP_ATTR_MASK + return mask; +} + +// Exception throwing helper. Adapted from Apache Hadoop header +// org_apache_hadoop.h by adding the do {} while (0) construct. +#define THROW(env, exception_name, message) \ + do { \ + jclass ecls = env->FindClass(exception_name); \ + if (ecls) { \ + int ret = env->ThrowNew(ecls, message); \ + if (ret < 0) { \ + printf("(CurveFs) Fatal Error\n"); \ + } \ + env->DeleteLocalRef(ecls); \ + } \ + } while (0) + +static void handle_error(JNIEnv* env, int rc) { + switch (rc) { + case ENOENT: + THROW(env, "java/io/FileNotFoundException", ""); + return; + case EEXIST: + THROW(env, "org/apache/hadoop/fs/FileAlreadyExistsException", ""); + return; + case ENOTDIR: + THROW(env, "io/opencurve/curve/fs/libfs/CurveFsException$NotADirectoryException", ""); // NOLINT + return; + default: + break; + } + + THROW(env, "java/io/IOException", strerror(rc)); +} + +// nativeCurveFsCreate: curvefs_new +JNIEXPORT jlong +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsNew + (JNIEnv* env, jobject) { + setup_field_ids(env); + uintptr_t instance = curvefs_new(); + return reinterpret_cast(instance); +} + +// nativeCurveFsRelease: curvefs_delete +JNIEXPORT void +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsDelete + (JNIEnv* env, jobject, jlong j_instance) { + uintptr_t instance = static_cast(j_instance); + return curvefs_delete(instance); +} + +// nativeCurveFsConfSet: curvefs_conf_set +JNIEXPORT void +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsConfSet + (JNIEnv* env, jclass, jlong j_instance, jstring j_key, jstring j_value) { + uintptr_t instance = static_cast(j_instance); + const char* key = env->GetStringUTFChars(j_key, NULL); + const char* value = env->GetStringUTFChars(j_value, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_key, key); + env->ReleaseStringUTFChars(j_value, value); + }); + + return curvefs_conf_set(instance, key, value); +} + +// nativeCurveFsMount: curvefs_mount +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsMount + (JNIEnv* env, jclass, jlong j_instance, + jstring j_fsname, jstring j_mountpoint) { + uintptr_t instance = static_cast(j_instance); + const char* fsname = env->GetStringUTFChars(j_fsname, NULL); + const char* mountpoint = env->GetStringUTFChars(j_mountpoint, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_fsname, fsname); + env->ReleaseStringUTFChars(j_mountpoint, mountpoint); + }); + + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsUmount: curvefs_umount +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsUmount + (JNIEnv* env, jclass, jlong j_instance, + jstring j_fsname, jstring j_mountpoint) { + uintptr_t instance = static_cast(j_instance); + const char* fsname = env->GetStringUTFChars(j_fsname, NULL); + const char* mountpoint = env->GetStringUTFChars(j_mountpoint, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_fsname, fsname); + env->ReleaseStringUTFChars(j_mountpoint, mountpoint); + }); + + int rc = curvefs_umonut(instance, fsname, mountpoint); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsMkDirs: curvefs_mkdirs +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsMkDirs + (JNIEnv* env, jclass, jlong j_instance, jstring j_path, jint j_mode) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + uint16_t mode = static_cast(j_mode); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_mkdirs(instance, path, mode); + if (rc == EEXIST) { + rc = 0; + } else if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsRmDir: curvefs_rmdir +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRmDir + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_rmdir(instance, path); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsListDir: curvefs_opendir/curvefs_readdir/curvefs_closedir +JNIEXPORT jobjectArray +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsListDir + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + // curvefs_opendir + uint64_t fd; + auto rc = curvefs_opendir(instance, path, &fd); + if (rc != 0) { + handle_error(env, rc); + return NULL; + } + + // curvefs_readdir + std::vector dirents; + std::vector buffer(8192); + for ( ; ; ) { + ssize_t n = curvefs_readdir(instance, fd, buffer.data(), 8192); + if (n < 0) { + handle_error(env, rc); + return NULL; + } else if (n == 0) { + break; + } + + // TODO(Wine93): less memory copy + dirents.insert(dirents.end(), buffer.begin(), buffer.begin() + n); + } + + // closedir + rc = curvefs_closedir(instance, fd); + if (rc != 0) { + handle_error(env, rc); + return NULL; + } + + jobjectArray j_dirents = env->NewObjectArray(dirents.size(), + env->FindClass(dirent_cls_name), + NULL); + + for (int i = 0; i < dirents.size(); i++) { + // NOTE!!!: don't use static class + jobject j_dirent = env->AllocObject(env->FindClass(dirent_cls_name)); + fill_dirent(env, j_dirent, &dirents[i]); + env->SetObjectArrayElement(j_dirents, i, j_dirent); + env->DeleteLocalRef(j_dirent); + } + + return j_dirents; +} + +// nativeCurveFsOpen: curvefs_create +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsCreate + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jint j_mode, jobject j_file) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + uint16_t mode = static_cast(j_mode); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + file_t file; + int rc = curvefs_create(instance, path, mode, &file); + if (rc < 0) { + handle_error(env, rc); + } + + fill_file(env, j_file, &file); + return rc; +} + +// nativeCurveFsOpen: curvefs_open +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsOpen + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jint j_flags, jobject j_file) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + uint32_t flags = fixup_open_flags(j_flags); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + file_t file; + int rc = curvefs_open(instance, path, flags, &file); + if (rc < 0) { + handle_error(env, rc); + } + + fill_file(env, j_file, &file); + return rc; +} + +// nativeCurveFsLSeek: curvefs_lseek +JNIEXPORT jlong +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsLSeek + (JNIEnv* env, jclass, + jlong j_instance, jint j_fd, jlong j_offset, jint j_whence) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + uint64_t offset = static_cast(j_offset); + + int whence; + switch (j_whence) { + case io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_SET: + whence = SEEK_SET; + break; + case io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_CUR: + whence = SEEK_CUR; + break; + case io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_END: + whence = SEEK_END; + break; + default: + return -1; + } + + int rc = curvefs_lseek(instance, fd, offset, whence); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativieCurveFsRead: curvefs_read +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativieCurveFsRead + (JNIEnv* env, jclass, jlong j_instance, jint j_fd, + jlong offset, jbyteArray j_buffer, jlong j_size) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + jbyte* c_buffer = env->GetByteArrayElements(j_buffer, NULL); + char* buffer = reinterpret_cast(c_buffer); + size_t count = static_cast(j_size); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseByteArrayElements(j_buffer, c_buffer, 0); + }); + + ssize_t n = curvefs_read(instance, fd, buffer, count); + if (n < 0) { + handle_error(env, n); + } + return static_cast(n); +} + +// nativieCurveFsWrite: curvefs_write +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativieCurveFsWrite + (JNIEnv* env, jclass, jlong j_instance, jint j_fd, + jlong offset, jbyteArray j_buffer, jlong j_size) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + jbyte* c_buffer = env->GetByteArrayElements(j_buffer, NULL); + char* buffer = reinterpret_cast(c_buffer); + size_t count = static_cast(j_size); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseByteArrayElements(j_buffer, c_buffer, 0); + }); + + ssize_t n = curvefs_write(instance, fd, buffer, count); + if (n < 0) { + handle_error(env, n); + } + return static_cast(n); +} + +// nativeCurveFsFSync: curvefs_fsync +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsFSync + (JNIEnv* env, jclass, jlong j_instance, jint j_fd) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + + int rc = curvefs_fsync(instance, fd); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsClose: curvefs_close +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsClose + (JNIEnv* env, jclass, jlong j_instance, jint j_fd) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + + int rc = curvefs_close(instance, fd); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsUnlink: curvefs_unlink +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsUnlink + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_unlink(instance, path); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsStatFs: curvefs_statfs +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsStatFs + (JNIEnv* env, jclass, + jlong j_instance, jobject j_statvfs) { + uintptr_t instance = static_cast(j_instance); + + struct statvfs statvfs; + int rc = curvefs_statfs(instance, &statvfs); + if (rc != 0) { + handle_error(env, rc); + return rc; + } + + fill_statvfs(env, j_statvfs, &statvfs); + return rc; +} + +// nativeCurveFsLstat: curvefs_lstat +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsLStat + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jobject j_stat) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + // curvefs_lstat + struct stat stat; + auto rc = curvefs_lstat(instance, path, &stat); + if (rc != 0) { + handle_error(env, rc); + return rc; + } + + fill_stat(env, j_stat, &stat); + return rc; +} + +// nativeCurveFsFStat: curvefs_fstat +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsFStat + (JNIEnv* env, jclass, jlong j_instance, jint j_fd, jobject j_stat) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + + // curvefs_fstat + struct stat stat; + auto rc = curvefs_fstat(instance, fd, &stat); + if (rc != 0) { + handle_error(env, rc); + return rc; + } + + fill_stat(env, j_stat, &stat); + return rc; +} + +// nativeCurveFsSetAttr: curvefs_setattr +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsSetAttr + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jobject j_stat, jint j_mask) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + int to_set = fixup_attr_mask(j_mask); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + struct stat stat; + memset(&stat, 0, sizeof(stat)); + stat.st_mode = env->GetIntField(j_stat, stat_mode_fid); + stat.st_uid = env->GetIntField(j_stat, stat_uid_fid); + stat.st_gid = env->GetIntField(j_stat, stat_gid_fid); + uint64_t mtime_msec = env->GetLongField(j_stat, stat_mtime_fid); + uint64_t atime_msec = env->GetLongField(j_stat, stat_atime_fid); + stat.st_mtim.tv_sec = mtime_msec / 1000; + stat.st_mtim.tv_nsec = (mtime_msec % 1000) * 1000000; + stat.st_atim.tv_sec = atime_msec / 1000; + stat.st_atim.tv_nsec = (atime_msec % 1000) * 1000000; + + int rc = curvefs_setattr(instance, path, &stat, to_set); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsChmod: curvefs_chmod +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsChmod + (JNIEnv* env, jclass, jlong j_instance, jstring j_path, jint j_mode) { + uintptr_t instance = static_cast(j_instance); + uint16_t mode = static_cast(j_mode); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_chmod(instance, path, mode); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsChown: curvefs_chown +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsChown + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jint j_uid, jint j_gid) { + uintptr_t instance = static_cast(j_instance); + uint32_t uid = static_cast(j_uid); + uint32_t gid = static_cast(j_gid); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_chown(instance, path, uid, gid); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsRename: curvefs_rename +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRename + (JNIEnv* env, jclass, jlong j_instance, jstring j_src, jstring j_dst) { + uintptr_t instance = static_cast(j_instance); + const char* src = env->GetStringUTFChars(j_src, NULL); + const char* dst = env->GetStringUTFChars(j_dst, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_src, src); + env->ReleaseStringUTFChars(j_dst, dst); + }); + + int rc = curvefs_rename(instance, src, dst); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsRemove: curvefs_remove +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRemove + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_remove(instance, path); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsRemoveAll: curvefs_removeall +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRemoveAll + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_removeall(instance, path); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} diff --git a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.h b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.h new file mode 100644 index 0000000000..0e75e1b4a5 --- /dev/null +++ b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.h @@ -0,0 +1,245 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class io_opencurve_curve_fs_libfs_CurveFsMount */ + +#ifndef _Included_io_opencurve_curve_fs_libfs_CurveFsMount +#define _Included_io_opencurve_curve_fs_libfs_CurveFsMount +#ifdef __cplusplus +extern "C" { +#endif +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_RDONLY +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_RDONLY 1L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_RDWR +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_RDWR 2L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_APPEND +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_APPEND 4L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_CREAT +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_CREAT 8L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_TRUNC +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_TRUNC 16L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_EXCL +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_EXCL 32L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_WRONLY +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_WRONLY 64L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_DIRECTORY +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_DIRECTORY 128L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_SET +#define io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_SET 0L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_CUR +#define io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_CUR 1L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_END +#define io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_END 2L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_MODE +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_MODE 1L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_UID +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_UID 2L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_GID +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_GID 4L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_MTIME +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_MTIME 8L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_ATIME +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_ATIME 16L +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsNew + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsNew + (JNIEnv *, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsDelete + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsDelete + (JNIEnv *, jobject, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsConfSet + * Signature: (JLjava/lang/String;Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsConfSet + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsMount + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsMount + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsUmount + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsUmount + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsMkDirs + * Signature: (JLjava/lang/String;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsMkDirs + (JNIEnv *, jclass, jlong, jstring, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsRmDir + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRmDir + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsListDir + * Signature: (JLjava/lang/String;)[Lio/opencurve/curve/fs/libfs/CurveFsMount/Dirent; + */ +JNIEXPORT jobjectArray JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsListDir + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsCreate + * Signature: (JLjava/lang/String;ILio/opencurve/curve/fs/libfs/CurveFsMount/File;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsCreate + (JNIEnv *, jclass, jlong, jstring, jint, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsOpen + * Signature: (JLjava/lang/String;ILio/opencurve/curve/fs/libfs/CurveFsMount/File;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsOpen + (JNIEnv *, jclass, jlong, jstring, jint, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsLSeek + * Signature: (JIJI)J + */ +JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsLSeek + (JNIEnv *, jclass, jlong, jint, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativieCurveFsRead + * Signature: (JIJ[BJ)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativieCurveFsRead + (JNIEnv *, jclass, jlong, jint, jlong, jbyteArray, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativieCurveFsWrite + * Signature: (JIJ[BJ)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativieCurveFsWrite + (JNIEnv *, jclass, jlong, jint, jlong, jbyteArray, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsFSync + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsFSync + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsClose + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsClose + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsUnlink + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsUnlink + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsStatFs + * Signature: (JLio/opencurve/curve/fs/libfs/CurveFsMount/StatVfs;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsStatFs + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsLStat + * Signature: (JLjava/lang/String;Lio/opencurve/curve/fs/libfs/CurveFsMount/Stat;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsLStat + (JNIEnv *, jclass, jlong, jstring, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsFStat + * Signature: (JILio/opencurve/curve/fs/libfs/CurveFsMount/Stat;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsFStat + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsSetAttr + * Signature: (JLjava/lang/String;Lio/opencurve/curve/fs/libfs/CurveFsMount/Stat;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsSetAttr + (JNIEnv *, jclass, jlong, jstring, jobject, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsChmod + * Signature: (JLjava/lang/String;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsChmod + (JNIEnv *, jclass, jlong, jstring, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsChown + * Signature: (JLjava/lang/String;II)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsChown + (JNIEnv *, jclass, jlong, jstring, jint, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsRename + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRename + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsRemove + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRemove + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsRemoveAll + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRemoveAll + (JNIEnv *, jclass, jlong, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/curvefs/sdk/java/pom.xml b/curvefs/sdk/java/pom.xml new file mode 100644 index 0000000000..a3ed98aea9 --- /dev/null +++ b/curvefs/sdk/java/pom.xml @@ -0,0 +1,110 @@ + + + + 4.0.0 + + io.opencurve.curve.fs.hadoop + curvefs-hadoop + 1.0-SNAPSHOT + + + 8 + 8 + UTF-8 + 2.7.3 + 0.80.5 + 1.15.4 + + + + + ali-maven + http://maven.aliyun.com/nexus/content/groups/public + + + + + + + native/build + + + src/main/resources + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + + + + + + + + net.java.dev.jna + jna-platform + 4.0.0 + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + + org.apache.commons + commons-compress + 1.24.0 + + + + org.apache.flink + flink-hadoop-fs + ${flink.version} + provided + + + + org.apache.flink + flink-core + ${flink.version} + provided + + + + junit + junit + 4.13.1 + test + + + + org.apache.flink + flink-connector-files + ${flink.version} + provided + + + + org.apache.flink + flink-table-common + 1.15.4 + provided + + + + diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemFactory.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemFactory.java new file mode 100644 index 0000000000..2a79fcc1c4 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemFactory.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opencurve.curve.fs.flink; + +import io.opencurve.curve.fs.hadoop.CurveFileSystem; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.FileSystemFactory; +import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.net.URI; + +public class CurveFileSystemFactory implements FileSystemFactory { + private org.apache.hadoop.conf.Configuration conf = new Configuration(); + private static final String CURVE_FS_CONFIG_PREFIXES = "curvefs."; + private static final String FLINK_CONFIG_PREFIXES = "fs."; + public static String SCHEME = "curvefs"; + + @Override + public void configure(org.apache.flink.configuration.Configuration config) { + config.keySet() + .stream() + .filter(key -> key.startsWith(CURVE_FS_CONFIG_PREFIXES) || key.startsWith(FLINK_CONFIG_PREFIXES)) + .forEach(key -> conf.set(key, config.getString(key, ""))); + } + + @Override + public String getScheme() { + return SCHEME; + } + + @Override + public FileSystem create(URI uri) throws IOException { + CurveFileSystem fs = new CurveFileSystem(); + fs.initialize(uri, conf); + return new HadoopFileSystem(fs); + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemTableFactory.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemTableFactory.java new file mode 100644 index 0000000000..d065492109 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemTableFactory.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opencurve.curve.fs.flink; + +import org.apache.flink.connector.file.table.FileSystemTableFactory; + +public class CurveFileSystemTableFactory extends FileSystemTableFactory { + @Override + public String factoryIdentifier() { + return CurveFileSystemFactory.SCHEME; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFS.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFS.java new file mode 100644 index 0000000000..f15008a363 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFS.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.DelegateToFileSystem; +import io.opencurve.curve.fs.flink.CurveFileSystemFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * The CurveFS implementation of AbstractFileSystem. + * This impl delegates to the old FileSystem + */ +public class CurveFS extends DelegateToFileSystem { + /** + * This constructor has the signature needed by + * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}. + * + * @param theUri which must be that of localFs + * @param conf + * @throws IOException + * @throws URISyntaxException + */ + CurveFS(final URI theUri, final Configuration conf) throws IOException, + URISyntaxException { + super(theUri, new CurveFileSystem(conf), conf, CurveFileSystemFactory.SCHEME, true); + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java new file mode 100644 index 0000000000..8f1239afcf --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import java.net.URI; +import java.io.IOException; +import java.io.OutputStream; +import java.io.FileNotFoundException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Progressable; + +import io.opencurve.curve.fs.hadoop.permission.Permission; +import io.opencurve.curve.fs.libfs.CurveFsProto; +import io.opencurve.curve.fs.libfs.CurveFsMount; +import io.opencurve.curve.fs.libfs.CurveFsMount.StatVfs; +import io.opencurve.curve.fs.libfs.CurveFsMount.Stat; +import io.opencurve.curve.fs.libfs.CurveFsMount.File; +import io.opencurve.curve.fs.libfs.CurveFsMount.Dirent; +import io.opencurve.curve.fs.libfs.CurveFsException.NotADirectoryException; + +public class CurveFileSystem extends FileSystem { + private static final int O_RDONLY = CurveFsMount.O_RDONLY; + private static final int O_WRONLY = CurveFsMount.O_WRONLY; + private static final int O_APPEND = CurveFsMount.O_APPEND; + private static final int SETATTR_MTIME = CurveFsMount.SETATTR_MTIME; + private static final int SETATTR_ATIME = CurveFsMount.SETATTR_ATIME; + + private URI uri; + private Path workingDir; + private CurveFsProto curvefs = null; + private Permission permission = null; + + public CurveFileSystem() {} + + public CurveFileSystem(Configuration conf) { setConf(conf); } + + // e.g. /my/dir1 + private String makeAbsolute(Path path) { + return makeQualified(path).toUri().getPath(); + } + + /** + * Get the current working directory for the given FileSystem + * @return the directory pathname + */ + @Override + public Path getWorkingDirectory() { + return workingDir; + } + + /** + * Set the current working directory for the given FileSystem. All relative + * paths will be resolved relative to it. + * + * @param new_dir Path of new working directory + */ + @Override + public void setWorkingDirectory(Path new_dir) { + workingDir = fixRelativePart(new_dir); + checkPath(workingDir); + } + + /** Called after a new FileSystem instance is constructed. + * @param name a uri whose authority section names the host, port, etc. + * for this FileSystem + * @param conf the configuration + */ + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + super.initialize(uri, conf); + + if (curvefs == null) { + curvefs = new CurveFsMount(); + } + if (permission == null) { + permission = new Permission(); + } + + curvefs.initialize(uri, conf); + permission.initialize(conf); + setConf(conf); + this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); + this.workingDir = getHomeDirectory(); + } + + /** + * No more filesystem operations are needed. Will + * release any held locks. + */ + @Override + public void close() throws IOException { + super.close(); + curvefs.shutdown(); + } + + /** + * Return the protocol scheme for the FileSystem. + *

+ * This implementation throws an UnsupportedOperationException. + * + * @return the protocol scheme for the FileSystem. + */ + @Override + public String getScheme() { + return "hdfs"; + } + + /** Returns a URI whose scheme and authority identify this FileSystem.*/ + @Override + public URI getUri() { + return uri; + } + + /** + * Call {@link #mkdirs(Path, FsPermission)} with default permission. + */ + @Override + public boolean mkdirs(Path f) throws IOException { + FsPermission perms = FsPermission.getDirDefault().applyUMask(FsPermission.getUMask(getConf())); + return mkdirs(f, perms); + } + + /** + * Make the given file and all non-existent parents into + * directories. Has the semantics of Unix 'mkdir -p'. + * Existence of the directory hierarchy is not an error. + * @param f path to create + * @param permission to apply to f + */ + @Override + public boolean mkdirs(Path f, FsPermission perms) throws IOException { + try { + curvefs.mkdirs(makeAbsolute(f), perms.toShort()); + } catch (IOException e) { + return false; + } + return true; + } + + private FileStatus newFileStatus(Path f, Stat stat) { + return new FileStatus(stat.size, + stat.isDirectory, + 1, + stat.blksize, + stat.mtime, + stat.atime, + new FsPermission((short) stat.mode), + permission.getUsername(stat.uid), + permission.getGroupname(stat.gid), + makeQualified(f)); // e.g. curvefs://my/dir1 + } + + /** + * Return a file status object that represents the path. + * @param f The path we want information from + * @return a FileStatus object + * @throws FileNotFoundException when the path does not exist; + * IOException see specific implementation + */ + @Override + public FileStatus getFileStatus(Path f) throws IOException { + Stat stat = new Stat(); + curvefs.lstat(makeAbsolute(f), stat); + return newFileStatus(f, stat); + } + + /** + * List the statuses of the files/directories in the given path if the path is + * a directory. + * + * @param f given path + * @return the statuses of the files/directories in the given patch + * @throws FileNotFoundException when the path does not exist; + * IOException see specific implementation + */ + @Override + public FileStatus[] listStatus(Path f) throws IOException { + Dirent[] dirents; + try { + dirents = curvefs.listdir(makeAbsolute(f)); + } catch(NotADirectoryException e) { + return new FileStatus[]{ getFileStatus(f) }; + } + + FileStatus[] statuses = new FileStatus[dirents.length]; + for (int i = 0; i < dirents.length; i++) { + Path p = makeQualified(new Path(f, new String(dirents[i].name))); + statuses[i] = newFileStatus(p, dirents[i].stat); + } + return statuses; + } + + private FSDataInputStream createFsDataInputStream(File file, + int bufferSize) throws IOException { + CurveFsInputStream istream = + new CurveFsInputStream(getConf(), curvefs, file.fd, file.length, bufferSize); + return new FSDataInputStream(istream); + } + + private FSDataOutputStream createFsDataOutputStream(File file, + int bufferSize) throws IOException { + OutputStream ostream = + new CurveFsOutputStream(getConf(), curvefs, file.fd, bufferSize); + return new FSDataOutputStream(ostream, statistics); + } + + /** + * Opens an FSDataInputStream at the indicated Path. + * @param f the file name to open + * @param bufferSize the size of the buffer to be used. + */ + @Override + public FSDataInputStream open(Path f, int bufferSize) throws IOException { + File file = new File(); + curvefs.open(makeAbsolute(f), O_RDONLY, file); + return createFsDataInputStream(file, bufferSize); + } + + /** + * Create an FSDataOutputStream at the indicated Path with write-progress + * reporting. + * @param f the file name to open + * @param permission + * @param overwrite if a file with this name already exists, then if true, + * the file will be overwritten, and if false an error will be thrown. + * @param bufferSize the size of the buffer to be used. + * @param replication required block replication for the file. + * @param blockSize + * @param progress + * @throws IOException + * @see #setPermission(Path, FsPermission) + */ + @Override + public FSDataOutputStream create(Path f, + FsPermission permission, + boolean overwrite, + int bufferSize, + short replication, + long blockSize, + Progressable progress) throws IOException { + File file = new File(); + for ( ; ; ) { + try { + curvefs.create(makeAbsolute(f), permission.toShort(), file); + } catch(FileNotFoundException e) { // parent directorty not exist + Path parent = makeQualified(f).getParent(); + try { + mkdirs(parent, FsPermission.getDirDefault()); + } catch (FileAlreadyExistsException ignored) {} + continue; // create file again + } catch(FileAlreadyExistsException e) { + if (!overwrite || isDirectory(f)) { + throw new FileAlreadyExistsException("File already exists: " + f); + } + delete(f, false); + continue; // create file again + } + break; + } + return createFsDataOutputStream(file, bufferSize); + } + + /** + * Opens an FSDataOutputStream at the indicated Path with write-progress + * reporting. Same as create(), except fails if parent directory doesn't + * already exist. + * @param f the file name to open + * @param permission + * @param overwrite if a file with this name already exists, then if true, + * the file will be overwritten, and if false an error will be thrown. + * @param bufferSize the size of the buffer to be used. + * @param replication required block replication for the file. + * @param blockSize + * @param progress + * @throws IOException + * @see #setPermission(Path, FsPermission) + * @deprecated API only for 0.20-append + */ + @Deprecated + @Override + public FSDataOutputStream createNonRecursive(Path f, + FsPermission permission, + boolean overwrite, + int bufferSize, + short replication, + long blockSize, + Progressable progress) throws IOException { + File file = new File(); + for ( ; ; ) { + try { + curvefs.create(makeAbsolute(f), permission.toShort(), file); + } catch(FileAlreadyExistsException e) { + if (!overwrite || isDirectory(f)) { + throw new FileAlreadyExistsException("File already exists: " + f); + } + delete(f, false); + continue; // create file again + } + break; + } + return createFsDataOutputStream(file, bufferSize); + } + + /** + * Append to an existing file (optional operation). + * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null) + * @param f the existing file to be appended. + * @throws IOException + */ + @Override + public FSDataOutputStream append(Path f, + int bufferSize, + Progressable progress) throws IOException { + File file = new File(); + curvefs.open(makeAbsolute(f), O_WRONLY | O_APPEND, file); + return createFsDataOutputStream(file, bufferSize); + } + + /** + * Renames Path src to Path dst. Can take place on local fs + * or remote DFS. + * @param src path to be renamed + * @param dst new path after rename + * @throws IOException on failure + * @return true if rename is successful + */ + @Override + public boolean rename(Path src, Path dst) throws IOException { + try { + curvefs.rename(makeAbsolute(src), makeAbsolute(dst)); + } catch(FileNotFoundException e) { // src path not exist + return false; + } catch (FileAlreadyExistsException e) { + FileStatus status = getFileStatus(dst); + if (!status.isDirectory()) { + return false; + } + // FIXME: + } + + return true; + } + + /** Delete a file. + * + * @param f the path to delete. + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. + * @return true if delete is successful else false. + * @throws IOException + */ + @Override + public boolean delete(Path f, boolean recursive) throws IOException { + try { + if (recursive) { + curvefs.removeall(makeAbsolute(f)); + } else { + curvefs.remove(makeAbsolute(f)); + } + } catch (IOException e) { + return false; + } + return true; + } + + /** + * Returns a status object describing the use and capacity of the + * file system. If the file system has multiple partitions, the + * use and capacity of the partition pointed to by the specified + * path is reflected. + * @param p Path for which status should be obtained. null means + * the default partition. + * @return a FsStatus object + * @throws IOException + * see specific implementation + */ + @Override + public FsStatus getStatus(Path p) throws IOException { + StatVfs statvfs = new StatVfs(); + curvefs.statfs(makeAbsolute(p), statvfs); + return new FsStatus(statvfs.bsize * statvfs.blocks, // capacity + statvfs.bsize * (statvfs.blocks - statvfs.bavail), // used + statvfs.bsize * statvfs.bavail); // remaining + } + + /** + * Set permission of a path. + * @param p + * @param permission + */ + @Override + public void setPermission(Path p, FsPermission permission) throws IOException { + curvefs.chmod(makeAbsolute(p), permission.toShort()); + } + + /** + * Set owner of a path (i.e. a file or a directory). + * The parameters username and groupname cannot both be null. + * @param p The path + * @param username If it is null, the original username remains unchanged. + * @param groupname If it is null, the original groupname remains unchanged. + */ + @Override + public void setOwner(Path p, String username, String groupname) throws IOException { + Stat stat = new Stat(); + curvefs.lstat(makeAbsolute(p), stat); // TODO(Wine93): remove this operation + + int uid = stat.uid; + int gid = stat.gid; + if (username != null) { + uid = permission.getUid(username); + } + if (groupname != null) { + gid = permission.getGid(groupname); + } + curvefs.chown(makeAbsolute(p), uid, gid); + } + + /** + * Set access time of a file + * @param p The path + * @param mtime Set the modification time of this file. + * The number of milliseconds since Jan 1, 1970. + * A value of -1 means that this call should not set modification time. + * @param atime Set the access time of this file. + * The number of milliseconds since Jan 1, 1970. + * A value of -1 means that this call should not set access time. + */ + @Override + public void setTimes(Path p, long mtime, long atime) throws IOException { + Stat stat = new Stat(); + int mask = 0; + if (mtime != -1) { + stat.mtime = mtime; + mask |= SETATTR_MTIME; + } + if (atime != -1) { + stat.atime = atime; + mask |= SETATTR_ATIME; + } + curvefs.setattr(makeAbsolute(p), stat, mask); + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsInputStream.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsInputStream.java new file mode 100644 index 0000000000..eb40992bf7 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsInputStream.java @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSInputStream; +import io.opencurve.curve.fs.libfs.CurveFsMount; +import io.opencurve.curve.fs.libfs.CurveFsProto; + +import java.io.IOException; + +/** + *

+ * An {@link FSInputStream} for a CurveFileSystem and corresponding + * Curve instance. + */ +public class CurveFsInputStream extends FSInputStream { + private static final Log LOG = LogFactory.getLog(CurveFsInputStream.class); + private boolean closed; + + private int fileHandle; + + private long fileLength; + + private CurveFsProto curvefs; + + private byte[] buffer; + private int bufPos = 0; + private int bufValid = 0; + private long curvePos = 0; + + /** + * Create a new CurveInputStream. + * @param conf The system configuration. Unused. + * @param fh The filehandle provided by Curve to reference. + * @param flength The current length of the file. If the length changes + * you will need to close and re-open it to access the new data. + */ + public CurveFsInputStream(Configuration conf, CurveFsProto curve, + int fh, long flength, int bufferSize) { + fileLength = flength; + fileHandle = fh; + closed = false; + curvefs = curve; + buffer = new byte[1<<21]; + LOG.debug("CurveInputStream constructor: initializing stream with fh " + + fh + " and file length " + flength); + } + + /** Curve likes things to be closed before it shuts down, + * so closing the IOStream stuff voluntarily in a finalizer is good + */ + @Override + protected void finalize() throws Throwable { + try { + if (!closed) { + close(); + } + } finally { + super.finalize(); + } + } + + private synchronized boolean fillBuffer() throws IOException { + bufValid = curvefs.read(fileHandle, -1, buffer, buffer.length); + bufPos = 0; + if (bufValid < 0) { + int err = bufValid; + + bufValid = 0; + + curvefs.lseek(fileHandle, curvePos, CurveFsMount.SEEK_SET); + throw new IOException("Failed to fill read buffer! Error code:" + err); + } + curvePos += bufValid; + return (bufValid != 0); + } + + /* + * Get the current position of the stream. + */ + @Override + public synchronized long getPos() throws IOException { + return curvePos - bufValid + bufPos; + } + + /** + * Find the number of bytes remaining in the file. + */ + @Override + public synchronized int available() throws IOException { + if (closed) { + throw new IOException("file is closed"); + } + return (int) (fileLength - getPos()); + } + + @Override + public synchronized void seek(long targetPos) throws IOException { + LOG.trace("CurveInputStream.seek: Seeking to position " + targetPos + " on fd " + + fileHandle); + if (targetPos > fileLength) { + throw new IOException("CurveInputStream.seek: failed seek to position " + + targetPos + " on fd " + fileHandle + + ": Cannot seek after EOF " + fileLength); + } + long oldPos = curvePos; + + curvePos = curvefs.lseek(fileHandle, targetPos, CurveFsMount.SEEK_SET); + bufValid = 0; + bufPos = 0; + if (curvePos < 0) { + curvePos = oldPos; + throw new IOException("Curve failed to seek to new position!"); + } + } + + /** + * Failovers are handled by the Curve code at a very low level; + * if there are issues that can be solved by changing sources + * they'll be dealt with before anybody even tries to call this method! + * @return false. + */ + @Override + public synchronized boolean seekToNewSource(long targetPos) { + return false; + } + + /** + * Read a byte from the file. + * @return the next byte. + */ + @Override + public synchronized int read() throws IOException { + LOG.trace( + "CurveInputStream.read: Reading a single byte from fd " + fileHandle + + " by calling general read function"); + + byte result[] = new byte[1]; + + if (getPos() >= fileLength) { + return -1; + } + + if (-1 == read(result, 0, 1)) { + return -1; + } + + if (result[0] < 0) { + return 256 + (int) result[0]; + } else { + return result[0]; + } + } + + /** + * Read a specified number of bytes from the file into a byte[]. + * @param buf the byte array to read into. + * @param off the offset to start at in the file + * @param len the number of bytes to read + * @return 0 if successful, otherwise an error code. + * @throws IOException on bad input. + */ + @Override + public synchronized int read(byte buf[], int off, int len) throws IOException { + LOG.trace( + "CurveInputStream.read: Reading " + len + " bytes from fd " + fileHandle); + + if (closed) { + throw new IOException( + "CurveInputStream.read: cannot read " + len + " bytes from fd " + + fileHandle + ": stream closed"); + } + + // ensure we're not past the end of the file + if (getPos() >= fileLength) { + LOG.debug( + "CurveInputStream.read: cannot read " + len + " bytes from fd " + + fileHandle + ": current position is " + getPos() + + " and file length is " + fileLength); + + return -1; + } + + int totalRead = 0; + int initialLen = len; + int read; + + do { + read = Math.min(len, bufValid - bufPos); + try { + System.arraycopy(buffer, bufPos, buf, off, read); + } catch (IndexOutOfBoundsException ie) { + throw new IOException( + "CurveInputStream.read: Indices out of bounds:" + "read length is " + + len + ", buffer offset is " + off + ", and buffer size is " + + buf.length); + } catch (ArrayStoreException ae) { + throw new IOException( + "Uh-oh, CurveInputStream failed to do an array" + + "copy due to type mismatch..."); + } catch (NullPointerException ne) { + throw new IOException( + "CurveInputStream.read: cannot read " + len + "bytes from fd:" + + fileHandle + ": buf is null"); + } + bufPos += read; + len -= read; + off += read; + totalRead += read; + } while (len > 0 && fillBuffer()); + + LOG.trace( + "CurveInputStream.read: Reading " + initialLen + " bytes from fd " + + fileHandle + ": succeeded in reading " + totalRead + " bytes"); + return totalRead; + } + + /** + * Close the CurveInputStream and release the associated filehandle. + */ + @Override + public void close() throws IOException { + LOG.trace("CurveOutputStream.close:enter"); + if (!closed) { + curvefs.close(fileHandle); + closed = true; + LOG.trace("CurveOutputStream.close:exit"); + } + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsOutputStream.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsOutputStream.java new file mode 100644 index 0000000000..8f24b3c7ca --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsOutputStream.java @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import io.opencurve.curve.fs.libfs.CurveFsMount; +import io.opencurve.curve.fs.libfs.CurveFsProto; + +import java.io.IOException; +import java.io.OutputStream; + +/** + *

+ * An {@link OutputStream} for a CurveFileSystem and corresponding + * Curve instance. + * + * TODO: + * - When libcurvefs-jni supports ByteBuffer interface we can get rid of the + * use of the buffer here to reduce memory copies and just use buffers in + * libcurvefs. Currently it might be useful to reduce JNI crossings, but not + * much more. + */ +public class CurveFsOutputStream extends OutputStream { + private boolean closed; + private CurveFsProto curvefs; + private int fileHandle; + private byte[] buffer; + private int bufUsed = 0; + + /** + * Construct the CurveOutputStream. + * @param conf The FileSystem configuration. + * @param fh The Curve filehandle to connect to. + */ + public CurveFsOutputStream(Configuration conf, + CurveFsProto curve, + int fh, + int bufferSize) { + curvefs = curve; + fileHandle = fh; + closed = false; + buffer = new byte[1<<21]; + } + + /** + * Close the Curve file handle if close() wasn't explicitly called. + */ + protected void finalize() throws Throwable { + try { + if (!closed) { + close(); + } + } finally { + super.finalize(); + } + } + + /** + * Ensure that the stream is opened. + */ + private synchronized void checkOpen() throws IOException { + if (closed) { + throw new IOException("operation on closed stream (fd=" + fileHandle + ")"); + } + } + + /** + * Get the current position in the file. + * @return The file offset in bytes. + */ + public synchronized long getPos() throws IOException { + checkOpen(); + return curvefs.lseek(fileHandle, 0, CurveFsMount.SEEK_CUR); + } + + @Override + public synchronized void write(int b) throws IOException { + byte buf[] = new byte[1]; + buf[0] = (byte) b; + write(buf, 0, 1); + } + + @Override + public synchronized void write(byte buf[], int off, int len) throws IOException { + checkOpen(); + + while (len > 0) { + int remaining = Math.min(len, buffer.length - bufUsed); + System.arraycopy(buf, off, buffer, bufUsed, remaining); + + bufUsed += remaining; + off += remaining; + len -= remaining; + + if (buffer.length == bufUsed) { + flushBuffer(); + } + } + } + + /* + * Moves data from the buffer into libcurvefs. + */ + private synchronized void flushBuffer() throws IOException { + if (bufUsed == 0) { + return; + } + + while (bufUsed > 0) { + int ret = curvefs.write(fileHandle, -1, buffer, bufUsed); + if (ret < 0) { + throw new IOException("curve.write: ret=" + ret); + } + + if (ret == bufUsed) { + bufUsed = 0; + return; + } + + assert(ret > 0); + assert(ret < bufUsed); + + /* + * TODO: handle a partial write by shifting the remainder of the data in + * the buffer back to the beginning and retrying the write. It would + * probably be better to use a ByteBuffer 'view' here, and I believe + * using a ByteBuffer has some other performance benefits but we'll + * likely need to update the libcurvefs-jni implementation. + */ + int remaining = bufUsed - ret; + System.arraycopy(buffer, ret, buffer, 0, remaining); + bufUsed -= ret; + } + + assert(bufUsed == 0); + } + + @Override + public synchronized void flush() throws IOException { + checkOpen(); + flushBuffer(); // buffer -> libcurvefs + curvefs.fsync(fileHandle); // libcurvefs -> cluster + } + + @Override + public synchronized void close() throws IOException { + checkOpen(); + flush(); + curvefs.close(fileHandle); + closed = true; + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Group.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Group.java new file mode 100644 index 0000000000..d1605a863a --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Group.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: Xianfei Cao (caoxianfei1) + */ + +package io.opencurve.curve.fs.hadoop.permission; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; + +import java.util.HashMap; +import java.io.IOException; +import java.util.List; + +public class Group { + private HashMap groupnames; + private HashMap groupIDs; + private UserGroupInformation ugi; + + private static final String SUPER_GROUPNAME = "supergroup"; + private static final int SUPER_GID = 0; + + public Group() { + groupnames = new HashMap(); + groupIDs = new HashMap(); + } + + private int finger(String groupname) { + return Math.abs(groupname.hashCode()); + } + + private void addGroup(String groupname, int gid) { + groupnames.put(groupname, gid); + groupIDs.put(gid, groupname); + } + + private void loadGroupFromSystem() throws IOException { + List groups = Helper.getAllGroups(); + for (Entry group : groups) { + if (group.id == 0) { + group.id = finger(group.name); + } + addGroup(group.name, group.id); + } + addGroup(SUPER_GROUPNAME, SUPER_GID); + } + + private void loadGroupFromFile() throws IOException { + // implement it + } + + public void initialize(Path path) throws IOException { + this.ugi = UserGroupInformation.getCurrentUser(); + if (path != null) { + loadGroupFromFile(); + } else { + loadGroupFromSystem(); + } + } + + public int[] getCurrentGids() { + String[] groups = {"nogroup"}; + if (ugi.getGroupNames().length > 0) { + groups = ugi.getGroupNames(); + } + + int[] gids = new int[groups.length]; + for (int i = 0; i < groups.length; i++) { + gids[i] = getGid(groups[i]); + } + return gids; + } + + public int getGid(String groupname) { + Integer gid = groupnames.get(groupname); + if (null == gid) { + gid = finger(groupname); + addGroup(groupname, gid); + } + return gid; + } + + public String getGroupname(int gid) { + String groupname = groupIDs.get(gid); + if (null == groupname || groupname.isEmpty()) { + return String.valueOf(gid); + } + return groupname; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Helper.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Helper.java new file mode 100644 index 0000000000..fe505123aa --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Helper.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: Xianfei Cao (caoxianfei1) + */ + +package io.opencurve.curve.fs.hadoop.permission; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.Pointer; +import com.sun.jna.Structure; +import io.opencurve.curve.fs.hadoop.permission.Helper.CLibrary.passwd; +import io.opencurve.curve.fs.hadoop.permission.Helper.CLibrary.group; + +import java.util.Arrays; +import java.util.List; +import java.util.ArrayList; + +class Entry { + String name; + int id; + + public Entry(String name, int id) { + this.name = name; + this.id = id; + } +} + +public class Helper { + public interface CLibrary extends Library { + CLibrary INSTANCE = (CLibrary)Native.loadLibrary("c", CLibrary.class); + passwd getpwent(); + void endpwent(); + group getgrent(); + void endgrent(); + + public static class passwd extends Structure { + public String pw_name; + public String pw_passwd; + public int pw_uid; + public int pw_gid; + public String pw_gecos; + public String pw_dir; + public String pw_shell; + + protected List getFieldOrder() { + return Arrays.asList(new String[] { + "pw_name", "pw_passwd", "pw_uid", "pw_gid", "pw_gecos", "pw_dir", "pw_shell", + }); + } + } + + public static class group extends Structure { + public String gr_name; + public String gr_passwd; + public int gr_gid; + public Pointer gr_mem; + + protected List getFieldOrder() { + return Arrays.asList(new String[] { + "gr_name", "gr_passwd", "gr_gid", "gr_mem", + }); + } + } + } + + public static List getAllUsers() { + CLibrary lib = CLibrary.INSTANCE; + List users = new ArrayList(); + passwd usr = new passwd(); + while ((usr = lib.getpwent()) != null) { + users.add(new Entry(usr.pw_name, usr.pw_uid)); + } + lib.endpwent(); + return users; + } + + public static List getAllGroups() { + CLibrary lib = CLibrary.INSTANCE; + List groups = new ArrayList(); + group grp = new group(); + while ((grp = lib.getgrent()) != null) { + groups.add(new Entry(grp.gr_name, grp.gr_gid)); + } + lib.endgrent(); + return groups; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Permission.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Permission.java new file mode 100644 index 0000000000..82ce9c52e0 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Permission.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: Xianfei Cao (caoxianfei1) + */ + +package io.opencurve.curve.fs.hadoop.permission; + +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; + +public class Permission { + private static User user = null; + private static Group group = null; + + public Permission() { + user = new User(); + group = new Group(); + } + + public void initialize(Configuration conf) throws IOException { + user.initialize(null); + group.initialize(null); + } + + public int getCurrentUid() { + return user.getCurrentUid(); + } + + public int getUid(String username) { + return user.getUid(username); + } + + public String getUsername(int uid) { + return user.getUsername(uid); + } + + public int[] getCurrentGids() { + return group.getCurrentGids(); + } + + public int getGid(String groupname) { + return group.getGid(groupname); + } + + public String getGroupname(int gid) { + return group.getGroupname(gid); + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/User.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/User.java new file mode 100644 index 0000000000..7b892caf53 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/User.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: Xianfei Cao (caoxianfei1) + */ + +package io.opencurve.curve.fs.hadoop.permission; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; + +import java.util.HashMap; +import java.io.IOException; +import java.util.List; + +public class User { + private HashMap usernames; + private HashMap userIDs; + private UserGroupInformation ugi; + + private static final String SUPER_USERNAME = "hdfs"; + private static final int SUPER_UID = 0; + + public User() { + usernames = new HashMap(); + userIDs = new HashMap(); + } + + private int finger(String groupname) { + return Math.abs(groupname.hashCode()); + } + + private void addUser(String username, int uid) { + usernames.put(username, uid); + userIDs.put(uid, username); + } + + private void loadUserFromSystem() throws IOException { + List users = Helper.getAllUsers(); + for (Entry user : users) { + if (user.id == 0) { + user.id = finger(user.name); + } + addUser(user.name, user.id); + } + addUser(SUPER_USERNAME, SUPER_UID); + } + + private void loadUserFromFile() throws IOException { + // TODO: implement it + } + + public void initialize(Path path) throws IOException { + this.ugi = UserGroupInformation.getCurrentUser(); + if (path != null) { + loadUserFromFile(); + } else { + loadUserFromSystem(); + } + } + + public int getCurrentUid() { + String username = ugi.getShortUserName(); + return getUid(username); + } + + public int getUid(String username) { + Integer gid = usernames.get(username); + if (null == gid) { + gid = finger(username); + addUser(username, gid); + } + return gid; + } + + public String getUsername(int uid) { + String username = userIDs.get(uid); + if (null == username || username.isEmpty()) { + return String.valueOf(uid); + } + return username; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsException.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsException.java new file mode 100644 index 0000000000..175fd910eb --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsException.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-12-19 + * Author: Jingli Chen (Wine93) + */ + +package io.opencurve.curve.fs.libfs; + +import java.io.IOException; + +public class CurveFsException extends IOException { + public static class NotADirectoryException extends IOException { + public NotADirectoryException() { + super(); + } + + public NotADirectoryException(String s) { + super(s); + } + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsMount.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsMount.java new file mode 100644 index 0000000000..0695c69296 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsMount.java @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-07 + * Author: Jingli Chen (Wine93) + */ + +package io.opencurve.curve.fs.libfs; + +import java.net.URI; +import java.util.Map; +import java.util.UUID; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; + +public class CurveFsMount extends CurveFsProto { + // init + private native long nativeCurveFsNew(); + private native void nativeCurveFsDelete(long instancePtr); + private static native void nativeCurveFsConfSet(long instancePtr, String key, String value); + private static native int nativeCurveFsMount(long instancePtr, String fsname, String mountpoint); + private static native int nativeCurveFsUmount(long instancePtr, String fsname, String mountpoint); + // dir* + private static native int nativeCurveFsMkDirs(long instancePtr, String path, int mode); + private static native int nativeCurveFsRmDir(long instancePtr, String path); + private static native Dirent[] nativeCurveFsListDir(long instancePtr, String path); + // file* + private static native int nativeCurveFsCreate(long instancePtr, String path, int mode, File file); + private static native int nativeCurveFsOpen(long instancePtr, String path, int flags, File file); + private static native long nativeCurveFsLSeek(long instancePtr, int fd, long offset, int whence); + private static native int nativieCurveFsRead(long instancePtr, int fd, long offset, byte[] buffer, long size); + private static native int nativieCurveFsWrite(long instancePtr, int fd, long offset, byte[] buffer, long size); + private static native int nativeCurveFsFSync(long instancePtr, int fd); + private static native int nativeCurveFsClose(long instancePtr, int fd); + private static native int nativeCurveFsUnlink(long instancePtr, String path); + // others + private static native int nativeCurveFsStatFs(long instancePtr, StatVfs statvfs); + private static native int nativeCurveFsLStat(long instancePtr, String path, Stat stat); + private static native int nativeCurveFsFStat(long instancePtr, int fd, Stat stat); + private static native int nativeCurveFsSetAttr(long instancePtr, String path, Stat stat, int mask); + private static native int nativeCurveFsChmod(long instancePtr, String path, int mode); + private static native int nativeCurveFsChown(long instancePtr, String path, int uid, int gid); + private static native int nativeCurveFsRename(long instancePtr, String src, String dst); + private static native int nativeCurveFsRemove(long instancePtr, String path); + private static native int nativeCurveFsRemoveAll(long instancePtr, String path); + + public static class StatVfs { + public long bsize; + public long frsize; + public long blocks; + public long bavail; + public long files; + public long fsid; + public long namemax; + } + + public static class Stat { + public int mode; + public int uid; + public int gid; + public long size; + public long blksize; + public long blocks; + public long atime; + public long mtime; + + public boolean isFile; + public boolean isDirectory; + public boolean isSymlink; + } + + public static class File { + public int fd; + public long length; + }; + + public static class Dirent { + public String name; + public Stat stat; + }; + + // Flags for open(): must be synchronized with JNI if changed. + public static final int O_RDONLY = 1; + public static final int O_RDWR = 2; + public static final int O_APPEND = 4; + public static final int O_CREAT = 8; + public static final int O_TRUNC = 16; + public static final int O_EXCL = 32; + public static final int O_WRONLY = 64; + public static final int O_DIRECTORY = 128; + + // Whence flags for seek(): must be synchronized with JNI if changed. + public static final int SEEK_SET = 0; + public static final int SEEK_CUR = 1; + public static final int SEEK_END = 2; + + // Attribute flags for setattr(): must be synchronized with JNI if changed. + public static final int SETATTR_MODE = 1; + public static final int SETATTR_UID = 2; + public static final int SETATTR_GID = 4; + public static final int SETATTR_MTIME = 8; + public static final int SETATTR_ATIME = 16; + + private static final String PREFIX_KEY = "curvefs"; + public static final String REGEX_CURVEFS_CONF = "^" + PREFIX_KEY + "\\..*"; + public static final String KEY_FSNAME = "curvefs.name"; + + private static long instancePtr; + private static String fsname; + private static String mountpoint; + private static boolean initialized = false; + + static { + try { + CurveFsNativeLoader.getInstance().loadLibrary(); + } catch(Exception e) { + System.out.println("Load curvefs native library failed: " + e.getMessage()); + } + } + + public CurveFsMount() {} + + private void setConf(Configuration conf) throws IOException { + Map m = conf.getValByRegex(REGEX_CURVEFS_CONF); + for (Map.Entry entry : m.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.equals(KEY_FSNAME)) { + fsname = value; + continue; + } + nativeCurveFsConfSet(instancePtr, key.substring(PREFIX_KEY.length() + 1), value); + } + + if (null == fsname || fsname.isEmpty()) { + throw new IOException(KEY_FSNAME + " is not set"); + } + mountpoint = UUID.randomUUID().toString(); + } + + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + if (initialized) { + return; + } + instancePtr = nativeCurveFsNew(); + setConf(conf); + nativeCurveFsMount(instancePtr, fsname, mountpoint); + initialized = true; + } + + @Override + public void shutdown() throws IOException { + if (!initialized) { + return; + } + nativeCurveFsUmount(instancePtr, fsname, mountpoint); + nativeCurveFsDelete(instancePtr); + initialized = false; + } + + @Override + public void mkdirs(String path, int mode) throws IOException { + nativeCurveFsMkDirs(instancePtr, path, mode); + } + + @Override + public void rmdir(String path) throws IOException { + nativeCurveFsRmDir(instancePtr, path); + } + + @Override + public Dirent[] listdir(String path) throws IOException { + return nativeCurveFsListDir(instancePtr, path); + } + + @Override + public void create(String path, int mode, File file) throws IOException { + nativeCurveFsCreate(instancePtr, path, mode, file); + } + + public void open(String path, int flags, File file) throws IOException { + nativeCurveFsOpen(instancePtr, path, flags, file); + } + + @Override + public long lseek(int fd, long offset, int whence) throws IOException { + return nativeCurveFsLSeek(instancePtr, fd, offset, whence); + } + + @Override + public int read(int fd, long offset, byte[] buffer, long size) throws IOException { + return nativieCurveFsRead(instancePtr, fd, offset, buffer, size); + } + + @Override + public int write(int fd, long offset, byte[] buffer, long size) throws IOException { + return nativieCurveFsWrite(instancePtr, fd, offset, buffer, size); + } + + @Override + public void fsync(int fd) throws IOException { + nativeCurveFsFSync(instancePtr, fd); + } + + @Override + public void close(int fd) throws IOException { + nativeCurveFsClose(instancePtr, fd); + } + + @Override + public void unlink(String path) throws IOException { + nativeCurveFsUnlink(instancePtr, path); + } + + @Override + public void statfs(String path, StatVfs statvfs) throws IOException { + nativeCurveFsStatFs(instancePtr, statvfs); + } + + @Override + public void lstat(String path, Stat stat) throws IOException { + nativeCurveFsLStat(instancePtr, path, stat); + } + + @Override + public void fstat(int fd, Stat stat) throws IOException { + nativeCurveFsFStat(instancePtr, fd, stat); + } + + @Override + public void setattr(String path, Stat stat, int mask) throws IOException { + nativeCurveFsSetAttr(instancePtr, path, stat, mask); + } + + @Override + public void chmod(String path, int mode) throws IOException { + nativeCurveFsChmod(instancePtr, path, mode); + } + + @Override + public void chown(String path, int uid, int gid) throws IOException { + nativeCurveFsChown(instancePtr, path, uid, gid); + } + + @Override + public void rename(String src, String dst) throws IOException { + nativeCurveFsRename(instancePtr, src, dst); + } + + @Override + public void remove(String path) throws IOException { + nativeCurveFsRemove(instancePtr, path); + } + + @Override + public void removeall(String path) throws IOException { + nativeCurveFsRemoveAll(instancePtr, path); + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsNativeLoader.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsNativeLoader.java new file mode 100644 index 0000000000..95df921e51 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsNativeLoader.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Project: Curve + * Created Date: 2023-07-07 + * Author: Jingli Chen (Wine93) + */ +package io.opencurve.curve.fs.libfs; +import java.net.URL; +import java.net.URLConnection; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.utils.IOUtils; + +public class CurveFsNativeLoader { + boolean initialized = false; + private static final CurveFsNativeLoader instance = new CurveFsNativeLoader(); + + private static final String TMP_DIR = "/tmp"; + private static final String CURVEFS_LIBRARY_PATH = "/tmp/libcurvefs"; + private static final String RESOURCE_TAR_NAME = "libcurvefs.tar"; + private static final String JNI_LIBRARY_NAME = "libcurvefs_jni.so"; + + private CurveFsNativeLoader() {} + + public static CurveFsNativeLoader getInstance() { + return instance; + } + + public long getJarModifiedTime() throws IOException { + URL location = CurveFsNativeLoader.class.getProtectionDomain().getCodeSource().getLocation(); + URLConnection conn = location.openConnection(); + return conn.getLastModified(); + } + public void descompress(InputStream in, String dest) throws IOException { + File dir = new File(dest); + if (!dir.exists()) { + dir.mkdirs(); + } + ArchiveEntry entry; + TarArchiveInputStream reader = new TarArchiveInputStream(in); + while ((entry = reader.getNextTarEntry()) != null) { + if (entry.isDirectory()) { + continue; + } + String path = TMP_DIR + File.separator + entry.getName(); + File file = new File(path); + IOUtils.copy(reader, new FileOutputStream(file)); + } + reader.close(); + } + + public void loadJNILibrary() throws IOException { + File libFile = new File(CURVEFS_LIBRARY_PATH, JNI_LIBRARY_NAME); + System.load(libFile.getAbsolutePath()); + } + public synchronized void loadLibrary() throws IOException { + if (initialized) { + return; + } + long jarModifiedTime = getJarModifiedTime(); + File libDir = new File(CURVEFS_LIBRARY_PATH); + if (libDir.exists() && libDir.lastModified() == jarModifiedTime) { + loadJNILibrary(); + initialized = true; + return; + } + + InputStream reader = CurveFsNativeLoader.class.getResourceAsStream("/" + RESOURCE_TAR_NAME); + if (reader == null) { + throw new IOException("Cannot get resource " + RESOURCE_TAR_NAME + " from Jar file."); + } + descompress(reader, CURVEFS_LIBRARY_PATH); + reader.close(); + + libDir.setLastModified(jarModifiedTime); + loadJNILibrary(); + initialized = true; + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsProto.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsProto.java new file mode 100644 index 0000000000..35567aa3e1 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsProto.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-12-19 + * Author: Jingli Chen (Wine93) + */ + +package io.opencurve.curve.fs.libfs; + +import java.net.URI; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; + +import io.opencurve.curve.fs.libfs.CurveFsMount.StatVfs; +import io.opencurve.curve.fs.libfs.CurveFsMount.Stat; +import io.opencurve.curve.fs.libfs.CurveFsMount.File; +import io.opencurve.curve.fs.libfs.CurveFsMount.Dirent; + +public abstract class CurveFsProto { + public abstract void initialize(URI uri, Configuration conf) throws IOException; + public abstract void shutdown() throws IOException; + + public abstract void mkdirs(String path, int mode) throws IOException; + public abstract void rmdir(String path) throws IOException; + public abstract Dirent[] listdir(String path) throws IOException; + + public abstract void create(String path, int mode, File file) throws IOException; + public abstract void open(String path, int flags, File file) throws IOException; + public abstract long lseek(int fd, long offset, int whence) throws IOException; + public abstract int write(int fd, long offset, byte[] buffer, long size) throws IOException; + public abstract int read(int fd, long offset, byte[] buffer, long size) throws IOException; + public abstract void fsync(int fd) throws IOException; + public abstract void close(int fd) throws IOException; + public abstract void unlink(String path) throws IOException; + + public abstract void statfs(String path, StatVfs statvfs) throws IOException; + public abstract void lstat(String path, Stat stat) throws IOException; + public abstract void fstat(int fd, Stat stat) throws IOException; + public abstract void setattr(String path, Stat stat, int mask) throws IOException; + public abstract void chmod(String path, int mode) throws IOException; + public abstract void chown(String path, int uid, int gid) throws IOException; + public abstract void remove(String path) throws IOException; + public abstract void removeall(String path) throws IOException; + public abstract void rename(String src, String dst) throws IOException; +} diff --git a/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory b/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory new file mode 100644 index 0000000000..49f845958d --- /dev/null +++ b/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory @@ -0,0 +1 @@ +io.opencurve.curve.fs.flink.CurveFileSystemFactory diff --git a/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 0000000000..767cc341af --- /dev/null +++ b/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1 @@ +io.opencurve.curve.fs.flink.CurveFileSystemTableFactory diff --git a/curvefs/sdk/java/src/test/java/io/opencurve/curve/fs/others/TestOthers.java b/curvefs/sdk/java/src/test/java/io/opencurve/curve/fs/others/TestOthers.java new file mode 100644 index 0000000000..754e6cc797 --- /dev/null +++ b/curvefs/sdk/java/src/test/java/io/opencurve/curve/fs/others/TestOthers.java @@ -0,0 +1,15 @@ +package io.opencurve.curve.fs.others; + +import junit.framework.TestCase; +import io.opencurve.curve.fs.hadoop.CurveFileSystem; + +import java.io.IOException; +import java.net.URL; + +public class TestOthers extends TestCase { + public void testHelloWorld() throws IOException { + URL location = CurveFileSystem.class.getProtectionDomain().getCodeSource().getLocation(); + System.out.println("Hello World"); + System.out.println(location.getPath()); + } +} diff --git a/curvefs/sdk/libcurvefs/BUILD b/curvefs/sdk/libcurvefs/BUILD new file mode 100644 index 0000000000..44f0f8c703 --- /dev/null +++ b/curvefs/sdk/libcurvefs/BUILD @@ -0,0 +1,44 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_binary( + name = "curvefs", + srcs = [ + "libcurvefs.h", + "libcurvefs.cpp", + ], + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//curvefs/src/client/vfs:vfs", + ], + linkshared = True, +) + +cc_library( + name = "curvefs_lib", + srcs = [ + "libcurvefs.h", + "libcurvefs.cpp", + ], + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//curvefs/src/client/vfs:vfs", + ], +) diff --git a/curvefs/sdk/libcurvefs/README.md b/curvefs/sdk/libcurvefs/README.md new file mode 100644 index 0000000000..839fe28f20 --- /dev/null +++ b/curvefs/sdk/libcurvefs/README.md @@ -0,0 +1,29 @@ +libcurvefs +=== + +SDK C/C++ library for CurveFS. + +Example +=== + +```c +#include "libcurvefs.h" + +int instance = curvefs_create(); +curvefs_conf_set(instance, "s3.ak", "xxx") +curvefs_conf_set(instance, "s3.sk", "xxx") + +... + +int rc = curvefs_mount(instance, "fsname", "/); +if (rc != 0) { + // mount failed +} + +rc = curvefs_mkdir(instance_ptr, "/mydir") +if (rc != 0) { + // mkdir failed +} +``` + +See [examples](examples) for more examples. diff --git a/curvefs/sdk/libcurvefs/examples/Makefile b/curvefs/sdk/libcurvefs/examples/Makefile new file mode 100644 index 0000000000..676f6334d2 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/Makefile @@ -0,0 +1,25 @@ +# Copyright (C) 2024 Jingli Chen (Wine93), NetEase Inc. + +root= $(shell echo $${PWD%/curvefs/sdk/libcurvefs/examples}) +so_path?= $(root)/bazel-bin/curvefs/sdk/libcurvefs +hrd_opt?= -I/curve +build_opt?= -L$(so_path) +link_opt?= -Wl,-rpath=$(so_path) -Wall +flags?= $(hrd_opt) $(build_opt) $(link_opt) +#targets= mkdir rmdir ls touch read write unlink append rename stat statfs fstat chmod +targets= ls + +libcurvefs: + mkdir -p bin + bazel build \ + --config=gcc7-later \ + --compilation_mode=opt \ + --copt -g \ + --copt -DUNINSTALL_SIGSEGV=1 \ + --copt -DCLIENT_CONF_PATH="\"$(root)/curvefs/conf/client.conf\"" \ + //curvefs/sdk/libcurvefs:curvefs + +$(targets): libcurvefs + gcc ${flags} $@.c -o bin/$@ -lcurvefs + +all: $(targets) diff --git a/curvefs/sdk/libcurvefs/examples/append.c b/curvefs/sdk/libcurvefs/examples/append.c new file mode 100644 index 0000000000..79aaa3f41b --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/append.c @@ -0,0 +1,48 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 2); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_WRONLY, 0777); + if (fd != 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_lseek(instance, fd, 0, SEEK_END); + if (rc != 0) { + fprintf(stderr, "lseek failed: retcode = %d\n", rc); + return rc; + } + + ssize_t n = curvefs_write(instance, fd, argv[2], strlen(argv[2])); + if (n < 0) { + rc = n; + fprintf(stderr, "write failed: retcode = %d\n", rc); + return rc; + } else if (n != strlen(argv[2])) { + fprintf(stderr, "write failed: %zd != %zu\n", n, strlen(argv[2])); + return -1; + } + + rc = curvefs_close(instance, fd); + if (rc != 0) { + fprintf(stderr, "close failed: retcode = %d\n", rc); + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/chmod.c b/curvefs/sdk/libcurvefs/examples/chmod.c new file mode 100644 index 0000000000..f6da02ee77 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/chmod.c @@ -0,0 +1,24 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 2); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_chmod(instance, argv[1], atoi(argv[2])); + if (rc != 0) { + fprintf(stderr, "chmod failed: retcode = %d\n", rc); + return rc; + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/common.h b/curvefs/sdk/libcurvefs/examples/common.h new file mode 100644 index 0000000000..a80acd97b1 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/common.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CURVEFS_SDK_LIBCURVEFS_EXAMPLES_COMMON_H_ +#define CURVEFS_SDK_LIBCURVEFS_EXAMPLES_COMMON_H_ + +#include +#include +#include + +#include "curvefs/sdk/libcurvefs/libcurvefs.h" + +const char* KEY_FSNAME = "curvefs.name"; +const char* KEY_MDS_ADDRS = "mdsOpt.rpcRetryOpt.addrs"; + +char* +require_string(const char* name) { + char* value = getenv(name); + if (strlen(value) == 0) { + fprintf(stderr, "require %s\n", name); + exit(1); + } + return value; +} + +char* +get_filesystem_name() { + //return require_string(KEY_FSNAME); + return "perf02"; +} + +char* +get_mountpoint() { + return "/"; +} + +void +load_cfg_from_environ(uintptr_t instance) { + //curvefs_conf_set(instance, KEY_MDS_ADDRS, require_string(KEY_MDS_ADDRS)); + curvefs_conf_set(instance, KEY_MDS_ADDRS, "10.221.103.160:6700,10.221.103.160:6701,10.221.103.160:6702"); + curvefs_conf_set(instance, "client.common.logDir", "/tmp"); + curvefs_conf_set(instance, "fs.accessLogging", "true"); + curvefs_conf_set(instance, "client.loglevel", "6"); + curvefs_conf_set(instance, "diskCache.diskCacheType", "0"); +} + +void +exact_args(int argc, int number) { + if (--argc == number) { + return; + } + + fprintf(stderr, "requires exactly %d argument[s]\n", number); + exit(1); +} + +#endif // CURVEFS_SDK_LIBCURVEFS_EXAMPLES_COMMON_H_ diff --git a/curvefs/sdk/libcurvefs/examples/fstat.c b/curvefs/sdk/libcurvefs/examples/fstat.c new file mode 100644 index 0000000000..5745f19e05 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/fstat.c @@ -0,0 +1,36 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_WRONLY, 0777); + if (fd != 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + return rc; + } + + struct stat stat; + rc = curvefs_fstat(instance, fd, &stat); + if (rc != 0) { + fprintf(stderr, "fstat failed: retcode = %d\n", rc); + return rc; + } + + printf("ino=%d, size=%d\n", stat.st_ino, stat.st_size); + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/ls.c b/curvefs/sdk/libcurvefs/examples/ls.c new file mode 100644 index 0000000000..30ddb9764f --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/ls.c @@ -0,0 +1,51 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + // mount + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + // opendir + uint64_t fd; + rc = curvefs_opendir(instance, argv[1], &fd); + if (rc != 0) { + fprintf(stderr, "opendir failed: retcode = %d\n", rc); + return rc; + } + + // readdir + dirent_t dirents[8192]; + for ( ;; ) { + ssize_t n = curvefs_readdir(instance, fd, dirents, 8192); + if (n < 0) { + rc = n; + fprintf(stderr, "readdir failed: retcode = %d\n", rc); + break; + } else if (n == 0) { + break; + } + + for (int i = 0; i < n; i++) { + printf("%s: ino=%d size=%d\n", dirents[i].name, + dirents[i].stat.st_ino, + dirents[i].stat.st_size); + } + } + + rc = curvefs_closedir(instance, fd); + if (rc != 0) { + fprintf(stderr, "closedir failed: retcode = %d\n", rc); + } + return rc; +} diff --git a/curvefs/sdk/libcurvefs/examples/mkdir.c b/curvefs/sdk/libcurvefs/examples/mkdir.c new file mode 100644 index 0000000000..c2c5b4bce6 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/mkdir.c @@ -0,0 +1,25 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_mkdir(instance, argv[1], 0755); + if (rc != 0) { + fprintf(stderr, "mkdir failed: retcode = %d\n", rc); + return rc; + } + + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/read.c b/curvefs/sdk/libcurvefs/examples/read.c new file mode 100644 index 0000000000..3dcbd5d4ee --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/read.c @@ -0,0 +1,44 @@ +#include + +#include "common.h" + +#define MAX_BUFFER_SIZE 4096 + 5 + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_WRONLY, 0777); + if (fd != 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + return rc; + } + + char buffer[MAX_BUFFER_SIZE]; + for ( ;; ) { + ssize_t n = curvefs_read(instance, fd, buffer, sizeof(buffer)); + if (n < 0) { + rc = n; + fprintf(stderr, "read failed: retcode = %d\n", rc); + return rc; + } else if (n == 0) { + break; + } + + buffer[n] = '\0'; + printf("%s", buffer); + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/rename.c b/curvefs/sdk/libcurvefs/examples/rename.c new file mode 100644 index 0000000000..77f824531b --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/rename.c @@ -0,0 +1,24 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 2); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_rename(instance, argv[1], argv[2]); + if (rc != 0) { + fprintf(stderr, "rename failed: retcode = %d\n", rc); + return rc; + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/rmdir.c b/curvefs/sdk/libcurvefs/examples/rmdir.c new file mode 100644 index 0000000000..7ae25a80a7 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/rmdir.c @@ -0,0 +1,25 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_rmdir(instance, argv[1]); + if (rc != 0) { + fprintf(stderr, "rmdir failed: retcode = %d\n", rc); + return rc; + } + + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/stat.c b/curvefs/sdk/libcurvefs/examples/stat.c new file mode 100644 index 0000000000..34589bc6fa --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/stat.c @@ -0,0 +1,27 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + struct stat stat; + rc = curvefs_lstat(instance, argv[1], &stat); + if (rc != 0) { + fprintf(stderr, "stat failed: retcode = %d\n", rc); + return rc; + } + + printf("ino=%d, size=%d\n", stat.st_ino, stat.st_size); + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/statfs.c b/curvefs/sdk/libcurvefs/examples/statfs.c new file mode 100644 index 0000000000..81f0aee290 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/statfs.c @@ -0,0 +1,29 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 0); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + struct statvfs statvfs; + rc = curvefs_statfs(instance, &statvfs); + if (rc != 0) { + fprintf(stderr, "statvfs failed: retcode = %d\n", rc); + return rc; + } + + printf("fsid = %d\n", statvfs.f_fsid); + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/touch.c b/curvefs/sdk/libcurvefs/examples/touch.c new file mode 100644 index 0000000000..b3185eb6ec --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/touch.c @@ -0,0 +1,26 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_CREAT, 0644); + if (fd < 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + } + return rc; +} diff --git a/curvefs/sdk/libcurvefs/examples/unlink.c b/curvefs/sdk/libcurvefs/examples/unlink.c new file mode 100644 index 0000000000..e20fbe5636 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/unlink.c @@ -0,0 +1,25 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_unlink(instance, argv[1]); + if (rc != 0) { + fprintf(stderr, "unlink failed: retcode = %d\n", rc); + return rc; + } + + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/write.c b/curvefs/sdk/libcurvefs/examples/write.c new file mode 100644 index 0000000000..d3f85c26fb --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/write.c @@ -0,0 +1,42 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 2); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_WRONLY, 0777); + if (fd != 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + return rc; + } + + ssize_t n = curvefs_write(instance, fd, argv[2], strlen(argv[2])); + if (n < 0) { + rc = n; + fprintf(stderr, "write failed: retcode = %d\n", rc); + return rc; + } else if (n != strlen(argv[2])) { + fprintf(stderr, "write failed: %zd != %zu\n", n, strlen(argv[2])); + return -1; + } + + rc = curvefs_close(instance, fd); + if (rc != 0) { + fprintf(stderr, "close failed: retcode = %d\n", rc); + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/libcurvefs.cpp b/curvefs/sdk/libcurvefs/libcurvefs.cpp new file mode 100644 index 0000000000..550b4b6f66 --- /dev/null +++ b/curvefs/sdk/libcurvefs/libcurvefs.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-12 + * Author: Jingli Chen (Wine93) + */ + +#include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/vfs/meta.h" +#include "curvefs/sdk/libcurvefs/libcurvefs.h" + +using ::curvefs::client::filesystem::CURVEFS_ERROR; +using ::curvefs::client::filesystem::SysErr; +using ::curvefs::client::vfs::File; +using ::curvefs::client::vfs::Dirent; + +static curvefs_mount_t* get_instance(uintptr_t instance_ptr) { + return reinterpret_cast(instance_ptr); +} + +uintptr_t curvefs_new() { + auto mount = new curvefs_mount_t(); + mount->cfg = Configure::Default(); + return reinterpret_cast(mount); +} + +void curvefs_delete(uintptr_t instance_ptr) { + auto mount = get_instance(instance_ptr); + delete mount; + mount = nullptr; +} + +void curvefs_conf_set(uintptr_t instance_ptr, + const char* key, + const char* value) { + auto mount = get_instance(instance_ptr); + return mount->cfg->Set(key, value); +} + +int curvefs_mount(uintptr_t instance_ptr, + const char* fsname, + const char* mountpoint) { + auto mount = get_instance(instance_ptr); + mount->vfs = std::make_shared(mount->cfg); + auto rc = mount->vfs->Mount(fsname, mountpoint); + return SysErr(rc); +} + +int curvefs_umonut(uintptr_t instance_ptr, + const char* fsname, + const char* mountpoint) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Umount(fsname, mountpoint); + return SysErr(rc); +} + +int curvefs_mkdir(uintptr_t instance_ptr, const char* path, uint16_t mode) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->MkDir(path, mode); + return SysErr(rc); +} + +int curvefs_mkdirs(uintptr_t instance_ptr, const char* path, uint16_t mode) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->MkDirs(path, mode); + return SysErr(rc); +} + +int curvefs_rmdir(uintptr_t instance_ptr, const char* path) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->RmDir(path); + return SysErr(rc); +} + +int curvefs_opendir(uintptr_t instance_ptr, + const char* path, + uint64_t* fd) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->OpenDir(path, fd); + return SysErr(rc); +} + +ssize_t curvefs_readdir(uintptr_t instance_ptr, + uint64_t fd, + dirent_t dirents[], + size_t count) { + size_t nread = 0; + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->ReadDir( + fd, reinterpret_cast(dirents), count, &nread); + if (rc == CURVEFS_ERROR::OK) { + return nread; + } else if (rc == CURVEFS_ERROR::END_OF_FILE) { + return 0; + } + return SysErr(rc); +} + +int curvefs_closedir(uintptr_t instance_ptr, uint64_t fd) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->CloseDir(fd); + return SysErr(rc); +} + +int curvefs_create(uintptr_t instance_ptr, + const char* path, + uint16_t mode, + file_t* file) { + CURVEFS_ERROR rc; + auto mount = get_instance(instance_ptr); + rc = mount->vfs->Create(path, mode, reinterpret_cast(file)); + return SysErr(rc); +} + +int curvefs_open(uintptr_t instance_ptr, + const char* path, + uint32_t flags, + file_t* file) { + CURVEFS_ERROR rc; + auto mount = get_instance(instance_ptr); + rc = mount->vfs->Open(path, flags, reinterpret_cast(file)); + return SysErr(rc); +} + +int curvefs_lseek(uintptr_t instance_ptr, + int fd, + uint64_t offset, + int whence) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->LSeek(fd, offset, whence); + return SysErr(rc); +} + +ssize_t curvefs_read(uintptr_t instance_ptr, + int fd, + char* buffer, + size_t count) { + size_t nread = 0; + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Read(fd, buffer, count, &nread); + if (rc != CURVEFS_ERROR::OK) { + return SysErr(rc); + } + return nread; +} + +ssize_t curvefs_write(uintptr_t instance_ptr, + int fd, + char* buffer, + size_t count) { + size_t nwritten; + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Write(fd, buffer, count, &nwritten); + if (rc != CURVEFS_ERROR::OK) { + return SysErr(rc); + } + return nwritten; +} + +int curvefs_fsync(uintptr_t instance_ptr, int fd) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->FSync(fd); + return SysErr(rc); +} + +int curvefs_close(uintptr_t instance_ptr, int fd) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Close(fd); + return SysErr(rc); +} + +int curvefs_unlink(uintptr_t instance_ptr, const char* path) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Unlink(path); + return SysErr(rc); +} + +int curvefs_statfs(uintptr_t instance_ptr, + struct statvfs* statvfs) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->StatFs(statvfs); + return SysErr(rc); +} + +int curvefs_lstat(uintptr_t instance_ptr, const char* path, struct stat* stat) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->LStat(path, stat); + return SysErr(rc); +} + +int curvefs_fstat(uintptr_t instance_ptr, int fd, struct stat* stat) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->FStat(fd, stat); + return SysErr(rc); +} + +int curvefs_setattr(uintptr_t instance_ptr, + const char* path, + struct stat* stat, + int to_set) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->SetAttr(path, stat, to_set); + return SysErr(rc); +} + +int curvefs_chmod(uintptr_t instance_ptr, const char* path, uint16_t mode) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Chmod(path, mode); + return SysErr(rc); +} + +int curvefs_chown(uintptr_t instance_ptr, + const char* path, + uint32_t uid, + uint32_t gid) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Chown(path, uid, gid); + return SysErr(rc); +} + +int curvefs_rename(uintptr_t instance_ptr, + const char* oldpath, + const char* newpath) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Rename(oldpath, newpath); + return SysErr(rc); +} + +int curvefs_remove(uintptr_t instance_ptr, const char* path) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Remove(path); + return SysErr(rc); +} + +int curvefs_removeall(uintptr_t instance_ptr, const char* path) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->RemoveAll(path); + return SysErr(rc); +} diff --git a/curvefs/sdk/libcurvefs/libcurvefs.h b/curvefs/sdk/libcurvefs/libcurvefs.h new file mode 100644 index 0000000000..f78621aee7 --- /dev/null +++ b/curvefs/sdk/libcurvefs/libcurvefs.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-12 + * Author: Jingli Chen (Wine93) + */ + +#ifndef CURVEFS_SDK_LIBCURVEFS_LIBCURVEFS_H_ +#define CURVEFS_SDK_LIBCURVEFS_LIBCURVEFS_H_ + +#include +#include +#include + +#ifdef __cplusplus + +#include +#include +#include + +#include "curvefs/src/client/vfs/config.h" +#include "curvefs/src/client/vfs/vfs.h" + +using ::curvefs::client::vfs::Configure; +using ::curvefs::client::vfs::VFS; + +typedef struct { + std::shared_ptr cfg; + std::shared_ptr vfs; +} curvefs_mount_t; + +#endif // __cplusplus + +typedef struct { + uint64_t fd; + uint64_t length; +} file_t; + +typedef struct { + char name[256]; // TODO(Wine93): smaller buffer + struct stat stat; // sizeof(stat) = 144 +} dirent_t; + +#ifdef __cplusplus +extern "C" { +#endif + +uintptr_t curvefs_new(); + +void curvefs_delete(uintptr_t instance_ptr); + +// NOTE: instance_ptr is the pointer of curvefs_mount_t instance. +void curvefs_conf_set(uintptr_t instance_ptr, + const char* key, + const char* value); + +int curvefs_mount(uintptr_t instance_ptr, + const char* fsname, + const char* mountpoint); + +int curvefs_umonut(uintptr_t instance_ptr, + const char* fsname, + const char* mountpoint); + +// directory +int curvefs_mkdir(uintptr_t instance_ptr, const char* path, uint16_t mode); + +int curvefs_mkdirs(uintptr_t instance_ptr, const char* path, uint16_t mode); + +int curvefs_rmdir(uintptr_t instance_ptr, const char* path); + +int curvefs_opendir(uintptr_t instance_ptr, + const char* path, + uint64_t* fd); + +ssize_t curvefs_readdir(uintptr_t instance_ptr, + uint64_t fd, + dirent_t dirents[], + size_t count); + +int curvefs_closedir(uintptr_t instance_ptr, uint64_t fd); + +// file +int curvefs_create(uintptr_t instance_ptr, + const char* path, + uint16_t mode, + file_t* file); + +int curvefs_open(uintptr_t instance_ptr, + const char* path, + uint32_t flags, + file_t* file); + +int curvefs_lseek(uintptr_t instance_ptr, + int fd, + uint64_t offset, + int whence); + +ssize_t curvefs_read(uintptr_t instance_ptr, + int fd, + char* buffer, + size_t count); + +ssize_t curvefs_write(uintptr_t instance_ptr, + int fd, + char* buffer, + size_t count); + +int curvefs_fsync(uintptr_t instance_ptr, int fd); + +int curvefs_close(uintptr_t instance_ptr, int fd); + +int curvefs_unlink(uintptr_t instance_ptr, const char* path); + +// others +int curvefs_statfs(uintptr_t instance_ptr, struct statvfs* statvfs); + +int curvefs_lstat(uintptr_t instance_ptr, const char* path, struct stat* stat); + +int curvefs_fstat(uintptr_t instance_ptr, int fd, struct stat* stat); + +int curvefs_setattr(uintptr_t instance_ptr, + const char* path, + struct stat* stat, + int to_set); + +int curvefs_chmod(uintptr_t instance_ptr, const char* path, uint16_t mode); + +int curvefs_chown(uintptr_t instance_ptr, + const char* path, + uint32_t uid, + uint32_t gid); + +int curvefs_remove(uintptr_t instance_ptr, const char* path); + +int curvefs_removeall(uintptr_t instance_ptr, const char* path); + +int curvefs_rename(uintptr_t instance_ptr, + const char* oldpath, + const char* newpath); +#ifdef __cplusplus +} +#endif + +#endif // CURVEFS_SDK_LIBCURVEFS_LIBCURVEFS_H_ diff --git a/curvefs/src/client/BUILD b/curvefs/src/client/BUILD index b296af86f8..33a3b45089 100644 --- a/curvefs/src/client/BUILD +++ b/curvefs/src/client/BUILD @@ -32,6 +32,29 @@ cc_binary( ], ) +cc_library( + name = "filesystem", + srcs = glob([ + "filesystem/*.cpp", + "filesystem/*.h", + ]), + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + ":fuse_client_lib", + ], +) + +cc_library( + name = "filesystem_xattr", + hdrs = ["filesystem/xattr.h"], + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//external:bvar", + ], +) + cc_library( name = "fuse_client_lib", srcs = glob( @@ -64,6 +87,8 @@ cc_library( "//curvefs/src/common:curvefs_common", "//curvefs/src/client/lease:curvefs_lease", "//curvefs/src/client/kvclient:memcached_client_lib", + "//curvefs/src/client/logger:logger", + ":filesystem_xattr", "//external:brpc", "//external:gflags", "//external:glog", @@ -74,12 +99,14 @@ cc_library( "//curvefs/src/common:metric_utils", "//curvefs/src/common:dynamic_vlog", "//curvefs/src/common:threading", + "//curvefs/src/metaserver:metaserver_storage_conv", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/container:btree", "@com_google_googletest//:gtest_prod", "@spdlog//:spdlog", ], diff --git a/curvefs/src/client/client_operator.cpp b/curvefs/src/client/client_operator.cpp index 8d44a5150f..fc41375caf 100644 --- a/curvefs/src/client/client_operator.cpp +++ b/curvefs/src/client/client_operator.cpp @@ -20,18 +20,22 @@ * Author: Jingli Chen (Wine93) */ +#include "curvefs/src/client/client_operator.h" + #include -#include "src/common/uuid.h" -#include "curvefs/src/client/client_operator.h" #include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/metaserver/storage/converter.h" +#include "src/common/uuid.h" namespace curvefs { namespace client { using ::curve::common::UUIDGenerator; -using ::curvefs::metaserver::DentryFlag; -using ::curvefs::mds::topology::PartitionTxId; using ::curvefs::client::filesystem::ToFSError; +using ::curvefs::mds::topology::PartitionTxId; +using ::curvefs::metaserver::DentryFlag; +using ::curvefs::metaserver::TxLock; +using ::curvefs::metaserver::storage::Key4Dentry; #define LOG_ERROR(action, rc) \ LOG(ERROR) << action << " failed, retCode = " << rc \ @@ -60,6 +64,7 @@ RenameOperator::RenameOperator(uint32_t fsId, dstTxId_(0), oldInodeId_(0), oldInodeSize_(-1), + startTs_(0), dentryManager_(dentryManager), inodeManager_(inodeManager), metaClient_(metaClient), @@ -77,6 +82,7 @@ std::string RenameOperator::DebugString() { << ", srcPartitionId = " << srcPartitionId_ << ", dstPartitionId = " << dstPartitionId_ << ", srcTxId = " << srcTxId_ << ", dstTxId_ = " << dstTxId_ + << ", startTs = " << startTs_ << ", oldInodeId = " << oldInodeId_ << ", srcDentry = [" << srcDentry_.ShortDebugString() << "]" << ", dstDentry = [" << dstDentry_.ShortDebugString() << "]" @@ -153,7 +159,7 @@ CURVEFS_ERROR RenameOperator::CheckOverwrite() { if (rc == CURVEFS_ERROR::OK && !dentrys.empty()) { LOG(ERROR) << "The directory is not empty" << ", dentry = (" << dstDentry_.ShortDebugString() << ")"; - rc = CURVEFS_ERROR::NOTEMPTY; + rc = CURVEFS_ERROR::NOT_EMPTY; } return rc; @@ -170,7 +176,7 @@ CURVEFS_ERROR RenameOperator::Precheck() { } rc = dentryManager_->GetDentry(newParentId_, newname_, &dstDentry_); - if (rc == CURVEFS_ERROR::NOTEXIST) { + if (rc == CURVEFS_ERROR::NOT_EXIST) { return CURVEFS_ERROR::OK; } else if (rc == CURVEFS_ERROR::OK) { oldInodeId_ = dstDentry_.inodeid(); @@ -191,10 +197,9 @@ CURVEFS_ERROR RenameOperator::RecordOldInodeInfo() { oldInodeType_ = attr.type(); } else { LOG_ERROR("GetInode", rc); - return CURVEFS_ERROR::NOTEXIST; + return CURVEFS_ERROR::NOT_EXIST; } } - return CURVEFS_ERROR::OK; } @@ -204,7 +209,6 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx( if (rc != MetaStatusCode::OK) { LOG_ERROR("PrepareRenameTx", rc); } - return ToFSError(rc); } @@ -272,6 +276,118 @@ CURVEFS_ERROR RenameOperator::CommitTx() { return CURVEFS_ERROR::OK; } +CURVEFS_ERROR RenameOperator::PrewriteRenameTx( + const std::vector& dentrys, const TxLock& txLockIn) { + TxLock txLockOut; + uint32_t dcount = 0; + auto rc = metaClient_->PrewriteRenameTx(dentrys, txLockIn, &txLockOut); + while (rc == MetaStatusCode::TX_KEY_LOCKED) { + dcount += txLockOut.index(); + auto rt = dentryManager_->CheckAndResolveTx(dentrys[dcount], + txLockOut, txLockIn.timestamp(), txLockIn.startts()); + if (rt != MetaStatusCode::OK) { + LOG_ERROR("CheckAndResolveTx", rt); + return CURVEFS_ERROR::INTERNAL; + } + if (dcount < dentrys.size()) { + rc = metaClient_->PrewriteRenameTx(std::vector( + dentrys.begin() + dcount, dentrys.end()), + txLockIn, &txLockOut); + } else { + break; + } + } + if (rc != MetaStatusCode::OK) { + LOG_ERROR("PrewriteRenameTx", rc); + return CURVEFS_ERROR::INTERNAL; + } + return CURVEFS_ERROR::OK; +} + +CURVEFS_ERROR RenameOperator::PrewriteTx() { + uint64_t timestamp; + auto rc = mdsClient_->Tso(&startTs_, ×tamp); + if (rc != FSStatusCode::OK) { + LOG_ERROR("start Tso", rc); + return CURVEFS_ERROR::INTERNAL; + } + + dentry_ = Dentry(srcDentry_); + dentry_.set_flag(DentryFlag::DELETE_MARK_FLAG); + dentry_.set_type(srcDentry_.type()); + dentry_.set_txid(startTs_); + + newDentry_ = Dentry(srcDentry_); + newDentry_.set_parentinodeid(newParentId_); + newDentry_.set_name(newname_); + newDentry_.set_type(srcDentry_.type()); + newDentry_.set_txid(startTs_); + + Key4Dentry key4Dentry( + dentry_.fsid(), dentry_.parentinodeid(), dentry_.name()); + std::string primaryKey = key4Dentry.SerializeToString(); + + TxLock txLockIn; + txLockIn.set_primarykey(primaryKey); + txLockIn.set_startts(startTs_); + txLockIn.set_timestamp(timestamp); + + if (!metaClient_->GetPartitionId(dentry_.fsid(), dentry_.parentinodeid(), + &srcPartitionId_) || !metaClient_->GetPartitionId(newDentry_.fsid(), + newDentry_.parentinodeid(), &dstPartitionId_)) { + LOG_ERROR("GetPartitionId", rc); + return CURVEFS_ERROR::INTERNAL; + } + + // note: do not prewrite concurrently, the tx write table clear logic based primary key prewrite first // NOLINT + CURVEFS_ERROR rt = CURVEFS_ERROR::OK; + std::vector dentrys{dentry_}; + if (srcPartitionId_ == dstPartitionId_) { + dentrys.push_back(newDentry_); + rt = PrewriteRenameTx(dentrys, txLockIn); + } else { + rt = PrewriteRenameTx(dentrys, txLockIn); + if (rt == CURVEFS_ERROR::OK) { + dentrys[0] = newDentry_; + rt = PrewriteRenameTx(dentrys, txLockIn); + } + } + if (rt != CURVEFS_ERROR::OK) { + LOG_ERROR("PrepPrewriteTxareTx", rc); + return rt; + } + return CURVEFS_ERROR::OK; +} + +CURVEFS_ERROR RenameOperator::CommitTxV2() { + uint64_t commitTs; + uint64_t timestamp; + auto rc = mdsClient_->Tso(&commitTs, ×tamp); + if (rc != FSStatusCode::OK) { + LOG_ERROR("CommitTxV2 Tso", rc); + return CURVEFS_ERROR::INTERNAL; + } + + MetaStatusCode rt = MetaStatusCode::OK; + std::vector dentrys{dentry_}; + if (srcPartitionId_ == dstPartitionId_) { + dentrys.push_back(newDentry_); + rt = metaClient_->CommitTx(dentrys, startTs_, commitTs); + } else { + rt = metaClient_->CommitTx(dentrys, startTs_, commitTs); + if (rt == MetaStatusCode::OK) { + dentrys[0] = newDentry_; + // do not need check second key commit result + metaClient_->CommitTx(dentrys, startTs_, commitTs); + } + } + if (rt != MetaStatusCode::OK) { + LOG_ERROR("CommitTx", rt); + return CURVEFS_ERROR::INTERNAL; + } + return CURVEFS_ERROR::OK; +} + CURVEFS_ERROR RenameOperator::LinkInode(uint64_t inodeId, uint64_t parent) { std::shared_ptr inodeWrapper; auto rc = inodeManager_->GetInode(inodeId, inodeWrapper); @@ -413,8 +529,6 @@ CURVEFS_ERROR RenameOperator::UpdateInodeCtime() { LOG_ERROR("UpdateInodeCtime", rc); return rc; } - - LOG(INFO) << "UpdateInodeCtime inodeid = " << srcDentry_.inodeid(); return rc; } diff --git a/curvefs/src/client/client_operator.h b/curvefs/src/client/client_operator.h index 0f073a67c4..0408fd1181 100644 --- a/curvefs/src/client/client_operator.h +++ b/curvefs/src/client/client_operator.h @@ -27,13 +27,15 @@ #include #include -#include "curvefs/src/client/inode_cache_manager.h" -#include "curvefs/src/client/dentry_cache_manager.h" +#include "curvefs/src/client/inode_manager.h" +#include "curvefs/src/client/dentry_manager.h" #include "curvefs/src/client/rpcclient/mds_client.h" namespace curvefs { namespace client { +using ::curvefs::metaserver::MetaStatusCode; +using ::curvefs::metaserver::TxLock; using rpcclient::MdsClient; class RenameOperator { @@ -56,6 +58,8 @@ class RenameOperator { CURVEFS_ERROR LinkDestParentInode(); CURVEFS_ERROR PrepareTx(); CURVEFS_ERROR CommitTx(); + CURVEFS_ERROR PrewriteTx(); + CURVEFS_ERROR CommitTxV2(); CURVEFS_ERROR UnlinkSrcParentInode(); void UnlinkOldInode(); CURVEFS_ERROR UpdateInodeParent(); @@ -85,6 +89,9 @@ class RenameOperator { CURVEFS_ERROR PrepareRenameTx(const std::vector& dentrys); + CURVEFS_ERROR PrewriteRenameTx( + const std::vector& dentrys, const TxLock& txLockIn); + CURVEFS_ERROR LinkInode(uint64_t inodeId, uint64_t parent = 0); CURVEFS_ERROR UnLinkInode(uint64_t inodeId, uint64_t parent = 0); @@ -107,6 +114,7 @@ class RenameOperator { // if dest exist, record the size and type of file or empty dir int64_t oldInodeSize_; FsFileType oldInodeType_; + uint64_t startTs_; // tx sequence number Dentry srcDentry_; Dentry dstDentry_; Dentry dentry_; diff --git a/curvefs/src/client/common/common.cpp b/curvefs/src/client/common/common.cpp index 6567792c0c..1210287fc3 100644 --- a/curvefs/src/client/common/common.cpp +++ b/curvefs/src/client/common/common.cpp @@ -45,6 +45,18 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) { case MetaServerOpType::PrepareRenameTx: os << "PrepareRenameTx"; break; + case MetaServerOpType::PrewriteRenameTx: + os << "PrewriteRenameTx"; + break; + case MetaServerOpType::CheckTxStatus: + os << "CheckTxStatus"; + break; + case MetaServerOpType::ResolveTxLock: + os << "ResolveTxLock"; + break; + case MetaServerOpType::CommitTx: + os << "CommitTx"; + break; case MetaServerOpType::GetInode: os << "GetInode"; break; @@ -79,7 +91,7 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) { } const char kCurveFsWarmupOpAdd[] = "add"; -const char kCurveFsWarmupOpQuery[] = "query"; +const char kCurveFsWarmupOpCancel[] = "cancel"; const char kCurveFsWarmupTypeList[] = "list"; const char kCurveFsWarmupTypeSingle[] = "single"; @@ -87,9 +99,8 @@ WarmupOpType GetWarmupOpType(const std::string& op) { auto ret = WarmupOpType::kWarmupOpUnknown; if (op == kCurveFsWarmupOpAdd) { ret = WarmupOpType::kWarmupOpAdd; - } - if (op == kCurveFsWarmupOpQuery) { - ret = WarmupOpType::kWarmupOpQuery; + } else if (op == kCurveFsWarmupOpCancel) { + ret = WarmupOpType::kWarmupOpCancel; } return ret; } diff --git a/curvefs/src/client/common/common.h b/curvefs/src/client/common/common.h index b8b6cf9d7d..2a5a13d931 100644 --- a/curvefs/src/client/common/common.h +++ b/curvefs/src/client/common/common.h @@ -50,6 +50,10 @@ enum class MetaServerOpType { CreateDentry, DeleteDentry, PrepareRenameTx, + PrewriteRenameTx, + CheckTxStatus, + ResolveTxLock, + CommitTx, GetInode, BatchGetInodeAttr, BatchGetXAttr, @@ -65,22 +69,25 @@ enum class MetaServerOpType { std::ostream &operator<<(std::ostream &os, MetaServerOpType optype); -const uint32_t MAX_XATTR_NAME_LENGTH = 255; -const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024; - -const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op"; - - -constexpr int kWarmupOpNum = 4; +constexpr size_t kMinWarmupOpArgsNum = 1; +constexpr size_t kWarmupAddArgsNum = 6; +constexpr size_t kWarmupCancelArgsNum = 2; enum class WarmupOpType { kWarmupOpUnknown = 0, kWarmupOpAdd = 1, - kWarmupOpQuery = 2, + kWarmupOpCancel = 2, }; WarmupOpType GetWarmupOpType(const std::string& op); +constexpr int kWarmupOpType = 0; +constexpr int kWarmupDataType = 1; +constexpr int kEntryFilePathInClient = 2; +constexpr int kWarmupCacheStorageType = 3; +constexpr int kMountPointInCurvefs = 4; +constexpr int kRootPathInCurvefs = 5; + enum class WarmupType { kWarmupTypeUnknown = 0, kWarmupTypeList = 1, diff --git a/curvefs/src/client/common/config.cpp b/curvefs/src/client/common/config.cpp index bef20b5ab1..6125a22af9 100644 --- a/curvefs/src/client/common/config.cpp +++ b/curvefs/src/client/common/config.cpp @@ -39,6 +39,8 @@ namespace curvefs { namespace client { namespace common { DECLARE_bool(useFakeS3); +DEFINE_bool(fs_disableXattr, false, "disable xattr"); +DEFINE_validator(fs_disableXattr, [](const char*, bool value) { return true; }); } // namespace common } // namespace client } // namespace curvefs @@ -46,11 +48,13 @@ DECLARE_bool(useFakeS3); namespace curvefs { namespace client { namespace common { +static bool pass_bool(const char*, bool) { return true; } DEFINE_bool(enableCto, true, "acheieve cto consistency"); DEFINE_bool(useFakeS3, false, "Use fake s3 to inject more metadata for testing metaserver"); DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing"); DEFINE_bool(access_logging, true, "enable access log"); +DEFINE_validator(access_logging, &pass_bool); /** * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true @@ -101,6 +105,8 @@ DEFINE_uint64(fuseClientBurstReadIopsSecs, 180, "the times that Read burst iops can continue"); DEFINE_validator(fuseClientBurstReadIopsSecs, &pass_uint64); +DEFINE_int32(TxVersion, 1, "tx version"); + void InitMdsOption(Configuration *conf, MdsOption *mdsOpt) { conf->GetValueFatalIfFail("mdsOpt.mdsMaxRetryMS", &mdsOpt->mdsMaxRetryMS); conf->GetValueFatalIfFail("mdsOpt.rpcRetryOpt.maxRPCTimeoutMS", @@ -183,6 +189,8 @@ void InitDiskCacheOption(Configuration *conf, &diskCacheOption->fullRatio); conf->GetValueFatalIfFail("diskCache.safeRatio", &diskCacheOption->safeRatio); + conf->GetValueFatalIfFail("diskCache.trimRatio", + &diskCacheOption->trimRatio); conf->GetValueFatalIfFail("diskCache.maxUsableSpaceBytes", &diskCacheOption->maxUsableSpaceBytes); conf->GetValueFatalIfFail("diskCache.maxFileNums", @@ -223,6 +231,16 @@ void InitS3Option(Configuration *conf, S3Option *s3Opt) { &s3Opt->s3ClientAdaptorOpt.writeCacheMaxByte); conf->GetValueFatalIfFail("s3.readCacheMaxByte", &s3Opt->s3ClientAdaptorOpt.readCacheMaxByte); + conf->GetValueFatalIfFail("s3.memClusterToLocal", + &s3Opt->s3ClientAdaptorOpt.memClusterToLocal); + conf->GetValueFatalIfFail("s3.s3ToLocal", + &s3Opt->s3ClientAdaptorOpt.s3ToLocal); + conf->GetValueFatalIfFail("s3.bigIoSize", + &s3Opt->s3ClientAdaptorOpt.bigIoSize); + conf->GetValueFatalIfFail("s3.bigIoRetryTimes", + &s3Opt->s3ClientAdaptorOpt.bigIoRetryTimes); + conf->GetValueFatalIfFail("s3.bigIoRetryIntervalUs", + &s3Opt->s3ClientAdaptorOpt.bigIoRetryIntervalUs); conf->GetValueFatalIfFail("s3.readCacheThreads", &s3Opt->s3ClientAdaptorOpt.readCacheThreads); conf->GetValueFatalIfFail("s3.nearfullRatio", @@ -236,6 +254,7 @@ void InitS3Option(Configuration *conf, S3Option *s3Opt) { &s3Opt->s3ClientAdaptorOpt.readRetryIntervalMs); ::curve::common::InitS3AdaptorOptionExceptS3InfoOption(conf, &s3Opt->s3AdaptrOpt); + InitDiskCacheOption(conf, &s3Opt->s3ClientAdaptorOpt.diskCacheOpt); } @@ -297,10 +316,44 @@ void InitKVClientManagerOpt(Configuration *conf, &config->getThreadPooln); } +void GetGids( + Configuration* c, const std::string& key, std::vector* gids) { + std::string str; + std::vector ss; + c->GetValueFatalIfFail(key, &str); + curve::common::SplitString(str, ",", &ss); + uint32_t gid; + for (const auto& s : ss) { + LOG_IF(FATAL, !curve::common::StringToUl(s, &gid)) + << "Invalid `" << key << "`: <" << s << ">"; + gids->push_back(static_cast(gid)); + } +} + +void GetUmask(Configuration* c, const std::string& key, uint16_t* umask) { + std::string str; + c->GetValueFatalIfFail(key, &str); + *umask = stoi(str, 0, 8); +} + +void InitVFSOption(Configuration* c, VFSOption* option) { + { // vfs cache option + auto o = &option->vfsCacheOption; + c->GetValueFatalIfFail("vfs.entryCache.lruSize", &o->entryCacheLruSize); + c->GetValueFatalIfFail("vfs.attrCache.lruSize", &o->attrCacheLruSize); + } + { // user permission option + auto o = &option->userPermissionOption; + c->GetValueFatalIfFail("vfs.userPermission.uid", &o->uid); + GetGids(c, "vfs.userPermission.gids", &o->gids); + GetUmask(c, "vfs.userPermission.umask", &o->umask); + } +} + void InitFileSystemOption(Configuration* c, FileSystemOption* option) { c->GetValueFatalIfFail("fs.cto", &option->cto); c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto); - c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr); + c->GetValueFatalIfFail("fs.disableXAttr", &option->disableXAttr); c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength); c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging); { // kernel cache option @@ -366,6 +419,7 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) { InitLeaseOpt(conf, &clientOption->leaseOpt); InitRefreshDataOpt(conf, &clientOption->refreshDataOption); InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt); + InitVFSOption(conf, &clientOption->vfsOption); InitFileSystemOption(conf, &clientOption->fileSystemOption); conf->GetValueFatalIfFail("fuseClient.listDentryLimit", @@ -376,6 +430,8 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) { &clientOption->dummyServerStartPort); conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename", &clientOption->enableMultiMountPointRename); + conf->GetIntValue("fuseClient.txVersion", + &FLAGS_TxVersion); conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes", &clientOption->downloadMaxRetryTimes); conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum", diff --git a/curvefs/src/client/common/config.h b/curvefs/src/client/common/config.h index e53137ae45..464b1550c4 100644 --- a/curvefs/src/client/common/config.h +++ b/curvefs/src/client/common/config.h @@ -25,9 +25,10 @@ #include #include +#include -#include "curvefs/src/client/common/common.h" #include "curvefs/proto/common.pb.h" +#include "curvefs/src/client/common/common.h" #include "src/client/config_info.h" #include "src/common/configuration.h" #include "src/common/s3_adapter.h" @@ -89,9 +90,11 @@ struct DiskCacheOption { // async load interval uint64_t asyncLoadPeriodMs; // trim start if disk usage over fullRatio - uint64_t fullRatio; - // trim finish until disk usage below safeRatio - uint64_t safeRatio; + uint32_t fullRatio = 90; + // disk usage safeRatio + uint32_t safeRatio = 70; + // trim finish until disk usage < safeRatio*trimRatio/100 + uint32_t trimRatio = 50; // the max size disk cache can use uint64_t maxUsableSpaceBytes; // the max file nums can cache @@ -125,6 +128,11 @@ struct S3ClientAdaptorOption { uint32_t flushIntervalSec; uint64_t writeCacheMaxByte; uint64_t readCacheMaxByte; + bool memClusterToLocal; + bool s3ToLocal; + uint32_t bigIoSize; + uint32_t bigIoRetryTimes; + uint32_t bigIoRetryIntervalUs; uint32_t readCacheThreads; uint32_t nearfullRatio; uint32_t baseSleepUs; @@ -173,6 +181,24 @@ struct RefreshDataOption { uint32_t refreshDataIntervalSec = 30; }; +// { vfs option +struct UserPermissionOption { + uint32_t uid; + std::vector gids; + uint16_t umask; +}; + +struct VFSCacheOption { + uint32_t entryCacheLruSize; + uint32_t attrCacheLruSize; +}; + +struct VFSOption { + VFSCacheOption vfsCacheOption; + UserPermissionOption userPermissionOption; +}; +// } + // { filesystem option struct KernelCacheOption { uint32_t entryTimeoutSec; @@ -212,7 +238,7 @@ struct DeferSyncOption { struct FileSystemOption { bool cto; - bool disableXattr; + bool disableXAttr; uint32_t maxNameLength; uint32_t blockSize = 0x10000u; KernelCacheOption kernelCacheOption; @@ -237,6 +263,7 @@ struct FuseClientOption { LeaseOpt leaseOpt; RefreshDataOption refreshDataOption; KVClientManagerOpt kvClientManagerOpt; + VFSOption vfsOption; FileSystemOption fileSystemOption; uint32_t listDentryLimit; diff --git a/curvefs/src/client/curve_fuse_op.cpp b/curvefs/src/client/curve_fuse_op.cpp index c81cdd5a6d..cd2128d805 100644 --- a/curvefs/src/client/curve_fuse_op.cpp +++ b/curvefs/src/client/curve_fuse_op.cpp @@ -21,30 +21,34 @@ * Author: xuchaojie */ -#include -#include +#include "curvefs/src/client/curve_fuse_op.h" + #include +#include +#include +#include #include #include -#include "curvefs/src/client/curve_fuse_op.h" -#include "curvefs/src/client/fuse_client.h" -#include "curvefs/src/client/filesystem/error.h" -#include "curvefs/src/client/common/config.h" #include "curvefs/src/client/common/common.h" -#include "src/common/configuration.h" -#include "src/common/gflags_helper.h" -#include "curvefs/src/client/s3/client_s3_adaptor.h" -#include "curvefs/src/client/fuse_volume_client.h" +#include "curvefs/src/client/common/config.h" +#include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/filesystem/meta.h" +#include "curvefs/src/client/filesystem/xattr.h" +#include "curvefs/src/client/fuse_client.h" #include "curvefs/src/client/fuse_s3_client.h" -#include "curvefs/src/client/rpcclient/mds_client.h" -#include "curvefs/src/client/rpcclient/base_client.h" +#include "curvefs/src/client/fuse_volume_client.h" +#include "curvefs/src/client/logger/access_log.h" #include "curvefs/src/client/metric/client_metric.h" -#include "curvefs/src/common/metric_utils.h" -#include "curvefs/src/common/dynamic_vlog.h" +#include "curvefs/src/client/rpcclient/base_client.h" +#include "curvefs/src/client/rpcclient/mds_client.h" +#include "curvefs/src/client/s3/client_s3_adaptor.h" #include "curvefs/src/client/warmup/warmup_manager.h" -#include "curvefs/src/client/filesystem/meta.h" -#include "curvefs/src/client/filesystem/access_log.h" +#include "curvefs/src/common/dynamic_vlog.h" +#include "curvefs/src/common/metric_utils.h" +#include "src/common/configuration.h" +#include "src/common/gflags_helper.h" +#include "src/common/log_util.h" using ::curve::common::Configuration; using ::curvefs::client::CURVEFS_ERROR; @@ -52,21 +56,30 @@ using ::curvefs::client::FuseClient; using ::curvefs::client::FuseS3Client; using ::curvefs::client::FuseVolumeClient; using ::curvefs::client::common::FuseClientOption; -using ::curvefs::client::rpcclient::MdsClientImpl; -using ::curvefs::client::rpcclient::MDSBaseClient; -using ::curvefs::client::metric::ClientOpMetric; -using ::curvefs::common::LatencyUpdater; -using ::curvefs::client::metric::InflightGuard; -using ::curvefs::client::filesystem::EntryOut; +using ::curvefs::client::common::kEntryFilePathInClient; +using ::curvefs::client::common::kMountPointInCurvefs; +using ::curvefs::client::common::kRootPathInCurvefs; +using ::curvefs::client::common::kWarmupCacheStorageType; +using ::curvefs::client::common::kWarmupDataType; +using ::curvefs::client::common::kWarmupOpType; +using ::curvefs::client::common::WarmupStorageType; using ::curvefs::client::filesystem::AttrOut; +using ::curvefs::client::filesystem::EntryOut; using ::curvefs::client::filesystem::FileOut; -using ::curvefs::client::filesystem::AccessLogGuard; -using ::curvefs::client::filesystem::StrFormat; -using ::curvefs::client::filesystem::InitAccessLog; -using ::curvefs::client::filesystem::Logger; -using ::curvefs::client::filesystem::StrEntry; +using ::curvefs::client::filesystem::IsListWarmupXAttr; +using ::curvefs::client::filesystem::IsWarmupXAttr; using ::curvefs::client::filesystem::StrAttr; +using ::curvefs::client::filesystem::StrEntry; using ::curvefs::client::filesystem::StrMode; +using ::curvefs::client::logger::AccessLogGuard; +using ::curvefs::client::logger::InitAccessLog; +using ::curvefs::client::logger::StrFormat; +using ::curvefs::client::metric::ClientOpMetric; +using ::curvefs::client::metric::InflightGuard; +using ::curvefs::client::rpcclient::MDSBaseClient; +using ::curvefs::client::rpcclient::MdsClientImpl; +using ::curvefs::client::warmup::WarmupProgress; +using ::curvefs::common::LatencyUpdater; using ::curvefs::common::FLAGS_vlog_level; @@ -140,6 +153,7 @@ int InitLog(const char *confPath, const char *argv0) { FLAGS_vlog_level = FLAGS_v; // initialize logging module + curve::common::DisableLoggingToStdErr(); google::InitGoogleLogging(argv0); bool succ = InitAccessLog(FLAGS_log_dir); @@ -221,13 +235,15 @@ void UnInitFuseClient() { } int AddWarmupTask(curvefs::client::common::WarmupType type, fuse_ino_t key, - const std::string &path, - curvefs::client::common::WarmupStorageType storageType) { + const std::string& path, + curvefs::client::common::WarmupStorageType storageType, + const std::string& mount_point, const std::string& root) { int ret = 0; bool result = true; switch (type) { case curvefs::client::common::WarmupType::kWarmupTypeList: - result = g_ClientInstance->PutWarmFilelistTask(key, storageType); + result = g_ClientInstance->PutWarmFilelistTask(key, storageType, path, + mount_point, root); break; case curvefs::client::common::WarmupType::kWarmupTypeSingle: result = g_ClientInstance->PutWarmFileTask(key, path, storageType); @@ -243,8 +259,28 @@ int AddWarmupTask(curvefs::client::common::WarmupType type, fuse_ino_t key, return ret; } +int CancelWarmupTask(curvefs::client::common::WarmupType type, fuse_ino_t key) { + int ret = 0; + bool result = true; + switch (type) { + case curvefs::client::common::WarmupType::kWarmupTypeList: + case curvefs::client::common::WarmupType::kWarmupTypeSingle: + result = g_ClientInstance->RemoveWarmFileOrFilelistTask(key); + break; + default: + // not support cancel warmup type (warmup single file/dir or + // filelist) + LOG(ERROR) << "not support warmup type, only support single/list"; + ret = EOPNOTSUPP; + } + if (!result) { + ret = ERANGE; + } + return ret; +} + void QueryWarmupTask(fuse_ino_t key, std::string *data) { - curvefs::client::warmup::WarmupProgress progress; + WarmupProgress progress; bool ret = g_ClientInstance->GetWarmupProgress(key, &progress); if (!ret) { *data = "finished"; @@ -255,40 +291,115 @@ void QueryWarmupTask(fuse_ino_t key, std::string *data) { VLOG(9) << "Warmup [" << key << "]" << *data; } -int Warmup(fuse_ino_t key, const std::string& name, const std::string& value) { - // warmup +void ListWarmupTasks(std::string* data) { + WarmupProgress progress; + std::unordered_map filepath2progress; + + bool ret = g_ClientInstance->GetAllWarmupProgress(&filepath2progress); + + std::ostringstream filepath2warmupProgress; + + for (auto it = filepath2progress.begin(); it != filepath2progress.end(); + ++it) { + filepath2warmupProgress + << fmt::format("{}:{}/{};", it->first, it->second.GetFinished(), + it->second.GetTotal()); + VLOG(9) << fmt::format("Warmup [\"{}\"]: {}/{};", it->first, + it->second.GetFinished(), it->second.GetTotal()); + } + if (!ret) { + *data = "finished"; + } else { + *data = filepath2warmupProgress.str(); + } +} + +int Warmup(fuse_ino_t key, const char* name, const std::string& values) { if (g_ClientInstance->GetFsInfo()->fstype() != FSType::TYPE_S3) { LOG(ERROR) << "warmup only support s3"; return EOPNOTSUPP; } std::vector opTypePath; - curve::common::SplitString(value, "\n", &opTypePath); - if (opTypePath.size() != curvefs::client::common::kWarmupOpNum) { - LOG(ERROR) << name << " has invalid xattr value " << value; - return ERANGE; - } - auto storageType = - curvefs::client::common::GetWarmupStorageType(opTypePath[3]); - if (storageType == - curvefs::client::common::WarmupStorageType::kWarmupStorageTypeUnknown) { - LOG(ERROR) << name << " not support storage type: " << value; + curve::common::SplitString(values, "\n", &opTypePath); + + /* + * opTypePath[0]: warmupOpType: [add, cancel] (e.g., add) + * opTypePath[1]: warmupDataType: [single, list] (e.g., single) + * opTypePath[2]: entryFilePathInClient (e.g., /mnt/hello_world.txt) + * opTypePath[3]: storageType: [disk, mem] (e.g., disk) + * opTypePath[4]: mountPointInCurvefs: (e.g., /mnt) + * opTypePath[5]: rootPathInCurvefs: (e.g., /) + */ + + auto warmupOpType = opTypePath[kWarmupOpType]; + auto warmupDataType = opTypePath[kWarmupDataType]; + + if (opTypePath.size() < curvefs::client::common::kMinWarmupOpArgsNum) { + LOG(ERROR) << name + << " did not provide enough required xattr values (expected " + << curvefs::client::common::kMinWarmupOpArgsNum << " actual " + << opTypePath.size() << ") " << values; return ERANGE; } + int ret = 0; - switch (curvefs::client::common::GetWarmupOpType(opTypePath[0])) { - case curvefs::client::common::WarmupOpType::kWarmupOpAdd: - ret = - AddWarmupTask(curvefs::client::common::GetWarmupType(opTypePath[1]), - key, opTypePath[2], storageType); - if (ret != 0) { - LOG(ERROR) << name << " has invalid xattr value " << value; + + switch (curvefs::client::common::GetWarmupOpType(warmupOpType)) { + case curvefs::client::common::WarmupOpType::kWarmupOpAdd: { + if (opTypePath.size() != + curvefs::client::common::kWarmupAddArgsNum) { + LOG(ERROR) + << name + << " has an incorrect number of xattr values (expected " + << curvefs::client::common::kWarmupAddArgsNum << " actual " + << opTypePath.size() << ") " << values; + ret = ERANGE; + break; + } + auto entryFilePathInClient = opTypePath[kEntryFilePathInClient]; + + auto storageType = curvefs::client::common::GetWarmupStorageType( + opTypePath[kWarmupCacheStorageType]); + + if (storageType == WarmupStorageType::kWarmupStorageTypeUnknown) { + LOG(ERROR) << name << " not support storage type: " << values; + ret = ERANGE; + break; + } + auto mountPointInCurvefs = opTypePath[kMountPointInCurvefs]; + auto rootPathInCurvefs = opTypePath[kRootPathInCurvefs]; + + ret = AddWarmupTask( + curvefs::client::common::GetWarmupType(warmupDataType), key, + entryFilePathInClient, storageType, mountPointInCurvefs, + rootPathInCurvefs); + break; } - break; - default: - LOG(ERROR) << name << " has invalid xattr value " << value; - ret = ERANGE; + case curvefs::client::common::WarmupOpType::kWarmupOpCancel: { + if (opTypePath.size() != + curvefs::client::common::kWarmupCancelArgsNum) { + LOG(ERROR) + << name + << " has an incorrect number of xattr values (expected " + << curvefs::client::common::kWarmupCancelArgsNum + << " actual " << opTypePath.size() << ") " << values; + ret = ERANGE; + break; + } + ret = CancelWarmupTask( + curvefs::client::common::GetWarmupType(warmupDataType), key); + break; + } + default: { + ret = ERANGE; + } + } + + if (ret != 0) { + LOG(ERROR) << name << " has invalid xattr values " << values; } + return ret; } @@ -312,12 +423,6 @@ FuseClient* Client() { return g_ClientInstance; } -const char* warmupXAttr = ::curvefs::client::common::kCurveFsWarmupXAttr; - -bool IsWamupReq(const char* name) { - return strcmp(name, warmupXAttr) == 0; -} - void TriggerWarmup(fuse_req_t req, fuse_ino_t ino, const char* name, @@ -341,6 +446,17 @@ void QueryWarmup(fuse_req_t req, fuse_ino_t ino, size_t size) { return fs->ReplyBuffer(req, data.data(), data.length()); } +void ListWarmup(fuse_req_t req, size_t size) { + auto fs = Client()->GetFileSystem(); + + std::string data; + ListWarmupTasks(&data); + if (size == 0) { + return fs->ReplyXattr(req, data.length()); + } + return fs->ReplyBuffer(req, data.data(), data.length()); +} + void ReadThrottleAdd(size_t size) { Client()->Add(true, size); } void WriteThrottleAdd(size_t size) { Client()->Add(false, size); } @@ -803,7 +919,7 @@ void FuseOpSetXattr(fuse_req_t req, ino, name, size, flags, StrErr(rc)); }); - if (IsWamupReq(name)) { + if (IsWarmupXAttr(name)) { return TriggerWarmup(req, ino, name, value, size); } rc = client->FuseOpSetXattr(req, ino, name, value, size, flags); @@ -824,7 +940,9 @@ void FuseOpGetXattr(fuse_req_t req, ino, name, size, StrErr(rc), value.size()); }); - if (IsWamupReq(name)) { + if (IsListWarmupXAttr(name)) { + return ListWarmup(req, size); + } else if (IsWarmupXAttr(name)) { return QueryWarmup(req, ino, size); } @@ -889,5 +1007,5 @@ void FuseOpBmap(fuse_req_t req, auto client = Client(); auto fs = client->GetFileSystem(); - return fs->ReplyError(req, CURVEFS_ERROR::NOTSUPPORT); + return fs->ReplyError(req, CURVEFS_ERROR::NOT_SUPPORT); } diff --git a/curvefs/src/client/dentry_cache_manager.cpp b/curvefs/src/client/dentry_cache_manager.cpp deleted file mode 100644 index afb5e49eef..0000000000 --- a/curvefs/src/client/dentry_cache_manager.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Project: curve - * Created Date: Thur May 27 2021 - * Author: xuchaojie - */ -#include "curvefs/src/client/dentry_cache_manager.h" - -#include -#include -#include -#include -#include -#include - -using ::curvefs::metaserver::MetaStatusCode_Name; - -namespace curvefs { -namespace client { -namespace common { -DECLARE_bool(enableCto); -} // namespace common -} // namespace client -} // namespace curvefs - -namespace curvefs { -namespace client { - -using curve::common::WriteLockGuard; -using NameLockGuard = ::curve::common::GenericNameLockGuard; -using ::curvefs::client::filesystem::ToFSError; - -CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent, - const std::string &name, - Dentry *out) { - std::string key = GetDentryCacheKey(parent, name); - NameLockGuard lock(nameLock_, key); - - MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out); - if (ret != MetaStatusCode::OK) { - LOG_IF(ERROR, ret != MetaStatusCode::NOT_FOUND) - << "metaClient_ GetDentry failed, MetaStatusCode = " << ret - << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret) - << ", parent = " << parent << ", name = " << name; - return ToFSError(ret); - } - return CURVEFS_ERROR::OK; -} - -CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) { - std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name()); - NameLockGuard lock(nameLock_, key); - MetaStatusCode ret = metaClient_->CreateDentry(dentry); - if (ret != MetaStatusCode::OK) { - LOG(ERROR) << "metaClient_ CreateDentry failed, MetaStatusCode = " - << ret - << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret) - << ", parent = " << dentry.parentinodeid() - << ", name = " << dentry.name(); - return ToFSError(ret); - } - - return CURVEFS_ERROR::OK; -} - -CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent, - const std::string &name, - FsFileType type) { - std::string key = GetDentryCacheKey(parent, name); - NameLockGuard lock(nameLock_, key); - - MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type); - if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) { - LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret - << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret) - << ", parent = " << parent << ", name = " << name; - return ToFSError(ret); - } - return CURVEFS_ERROR::OK; -} - -CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent, - std::list *dentryList, - uint32_t limit, - bool onlyDir, - uint32_t nlink) { - dentryList->clear(); - // means no dir under this dir - if (onlyDir && nlink == 2) { - LOG(INFO) << "ListDentry parent = " << parent - << ", onlyDir = 1 and nlink = 2, return directly"; - return CURVEFS_ERROR::OK; - } - - MetaStatusCode ret = MetaStatusCode::OK; - bool perceed = true; - std::string last = ""; - do { - std::list part; - ret = metaClient_->ListDentry(fsId_, parent, last, limit, onlyDir, - &part); - VLOG(6) << "ListDentry fsId = " << fsId_ << ", parent = " << parent - << ", last = " << last << ", count = " << limit - << ", onlyDir = " << onlyDir - << ", ret = " << ret << ", part.size() = " << part.size(); - if (ret != MetaStatusCode::OK) { - LOG(ERROR) << "metaClient_ ListDentry failed" - << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret) - << ", parent = " << parent << ", last = " << last - << ", count = " << limit << ", onlyDir = " << onlyDir; - return ToFSError(ret); - } - - if (!onlyDir) { - if (part.size() < limit) { - perceed = false; - } - if (!part.empty()) { - last = part.back().name(); - dentryList->splice(dentryList->end(), part); - } - } else { - // means iterate over the range - if (part.empty()) { - perceed = false; - } else { - last = part.back().name(); - if (part.back().type() != FsFileType::TYPE_DIRECTORY) { - part.pop_back(); - } - dentryList->splice(dentryList->end(), part); - // means already get all the dir under this dir - if (nlink - dentryList->size() == 2) { - perceed = false; - } - } - } - } while (perceed); - - return CURVEFS_ERROR::OK; -} - -} // namespace client -} // namespace curvefs diff --git a/curvefs/src/client/dentry_manager.cpp b/curvefs/src/client/dentry_manager.cpp new file mode 100644 index 0000000000..9c5227ccfb --- /dev/null +++ b/curvefs/src/client/dentry_manager.cpp @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* + * Project: curve + * Created Date: Thur May 27 2021 + * Author: xuchaojie + */ +#include "curvefs/src/client/dentry_manager.h" + +#include +#include +#include +#include +#include +#include +#include "curvefs/src/metaserver/storage/converter.h" + +using ::curvefs::metaserver::MetaStatusCode_Name; +using ::curvefs::metaserver::storage::Key4Dentry; + +namespace curvefs { +namespace client { +namespace common { +DECLARE_bool(enableCto); +} // namespace common +} // namespace client +} // namespace curvefs + +namespace curvefs { +namespace client { + +using curve::common::WriteLockGuard; +using NameLockGuard = ::curve::common::GenericNameLockGuard; +using ::curvefs::client::filesystem::ToFSError; + +MetaStatusCode DentryCacheManagerImpl::CheckTxStatus( + const std::string primaryKey, uint64_t startTs, uint64_t curTimestamp) { + Key4Dentry key4Dentry; + if (!key4Dentry.ParseFromString(primaryKey)) { + LOG(ERROR) << "CheckTxStatus parse primary key failed, primaryKey = " + << primaryKey; + return MetaStatusCode::PARSE_FROM_STRING_FAILED; + } + return metaClient_->CheckTxStatus(key4Dentry.fsId, key4Dentry.parentInodeId, + primaryKey, startTs, curTimestamp); +} + +MetaStatusCode DentryCacheManagerImpl::ResolveTxLock(const Dentry &dentry, + uint64_t startTs, uint64_t commitTs) { + return metaClient_->ResolveTxLock(dentry, startTs, commitTs); +} + +MetaStatusCode DentryCacheManagerImpl::CheckAndResolveTx(const Dentry& dentry, + const TxLock& txLock, uint64_t timestamp, uint64_t commitTs) { + auto rt = CheckTxStatus(txLock.primarykey(), txLock.startts(), timestamp); + switch (rt) { + case MetaStatusCode::TX_COMMITTED: + return ResolveTxLock(dentry, txLock.startts(), commitTs); + case MetaStatusCode::TX_ROLLBACKED: + case MetaStatusCode::TX_TIMEOUT: + return ResolveTxLock(dentry, txLock.startts()); + default: + LOG(ERROR) << "CheckTxStatus unexpected rt = " + << MetaStatusCode_Name(rt); + return rt; + } + return MetaStatusCode::OK; +} + +CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent, + const std::string &name, + Dentry *out) { + std::string key = GetDentryCacheKey(parent, name); + NameLockGuard lock(nameLock_, key); + TxLock txLockOut; + MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out, + &txLockOut); + while (ret == MetaStatusCode::TX_KEY_LOCKED) { + uint64_t ts = 0; + uint64_t timestamp = 0; + if (mdsClient_->Tso(&ts, ×tamp) != FSStatusCode::OK) { + LOG(ERROR) << "GetDentry Tso failed, parent = " << parent + << ", name = " << name; + return CURVEFS_ERROR::INTERNAL; + } + Dentry dentry; + dentry.set_fsid(fsId_); + dentry.set_parentinodeid(parent); + dentry.set_name(name); + MetaStatusCode rc = CheckAndResolveTx(dentry, txLockOut, timestamp, ts); + if (rc != MetaStatusCode::OK) { + LOG(ERROR) << "GetDentry CheckAndResolveTx failed, rc = " + << MetaStatusCode_Name(rc) + << ", parent = " << parent << ", name = " << name; + return CURVEFS_ERROR::INTERNAL; + } + ret = metaClient_->GetDentry(fsId_, parent, name, out, &txLockOut); + } + + if (ret != MetaStatusCode::OK) { + LOG_IF(ERROR, ret != MetaStatusCode::NOT_FOUND) + << "metaClient_ GetDentry failed, MetaStatusCode = " << ret + << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret) + << ", parent = " << parent << ", name = " << name; + return ToFSError(ret); + } + return CURVEFS_ERROR::OK; +} + +CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) { + std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name()); + NameLockGuard lock(nameLock_, key); + TxLock txLockOut; + MetaStatusCode ret = metaClient_->CreateDentry(dentry, &txLockOut); + while (ret == MetaStatusCode::TX_KEY_LOCKED) { + uint64_t ts = 0; + uint64_t timestamp = 0; + if (mdsClient_->Tso(&ts, ×tamp) != FSStatusCode::OK) { + LOG(ERROR) << "CreateDentry Tso failed, dentry = " + << dentry.ShortDebugString(); + return CURVEFS_ERROR::INTERNAL; + } + MetaStatusCode rc = CheckAndResolveTx(dentry, txLockOut, timestamp, ts); + if (rc != MetaStatusCode::OK) { + LOG(ERROR) << "CreateDentry CheckAndResolveTx failed, rc = " + << MetaStatusCode_Name(rc) << ", dentry = " + << dentry.ShortDebugString(); + return CURVEFS_ERROR::INTERNAL; + } + ret = metaClient_->CreateDentry(dentry, &txLockOut); + } + if (ret != MetaStatusCode::OK) { + LOG(ERROR) << "metaClient_ CreateDentry failed" + << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret) + << ", parent = " << dentry.parentinodeid() + << ", name = " << dentry.name(); + return ToFSError(ret); + } + return CURVEFS_ERROR::OK; +} + +CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent, + const std::string &name, + FsFileType type) { + std::string key = GetDentryCacheKey(parent, name); + NameLockGuard lock(nameLock_, key); + + TxLock txLockOut; + MetaStatusCode ret = metaClient_->DeleteDentry( + fsId_, parent, name, type, &txLockOut); + while (ret == MetaStatusCode::TX_KEY_LOCKED) { + uint64_t ts = 0; + uint64_t timestamp = 0; + if (mdsClient_->Tso(&ts, ×tamp) != FSStatusCode::OK) { + LOG(ERROR) << "DeleteDentry Tso failed, parent = " << parent + << ", name = " << name; + return CURVEFS_ERROR::INTERNAL; + } + Dentry dentry; + dentry.set_fsid(fsId_); + dentry.set_parentinodeid(parent); + dentry.set_name(name); + MetaStatusCode rc = CheckAndResolveTx(dentry, txLockOut, timestamp, ts); + if (rc != MetaStatusCode::OK) { + LOG(ERROR) << "DeleteDentry CheckAndResolveTx failed, rc = " + << MetaStatusCode_Name(rc) << ", parent = " << parent + << ", name = " << name; + return CURVEFS_ERROR::INTERNAL; + } + ret = metaClient_->DeleteDentry(fsId_, parent, name, type, &txLockOut); + } + + if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) { + LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret + << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret) + << ", parent = " << parent << ", name = " << name; + return ToFSError(ret); + } + return CURVEFS_ERROR::OK; +} + +CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent, + std::list *dentryList, + uint32_t limit, + bool onlyDir, + uint32_t nlink) { + dentryList->clear(); + // means no dir under this dir + if (onlyDir && nlink == 2) { + LOG(INFO) << "ListDentry parent = " << parent + << ", onlyDir = 1 and nlink = 2, return directly"; + return CURVEFS_ERROR::OK; + } + + MetaStatusCode ret = MetaStatusCode::OK; + bool perceed = true; + std::string last = ""; + TxLock txLockOut; + do { + std::list part; + ret = metaClient_->ListDentry(fsId_, parent, last, limit, onlyDir, + &part, &txLockOut); + VLOG(6) << "ListDentry fsId = " << fsId_ << ", parent = " << parent + << ", last = " << last << ", count = " << limit + << ", onlyDir = " << onlyDir + << ", ret = " << ret << ", part.size() = " << part.size(); + if (ret == MetaStatusCode::TX_KEY_LOCKED) { + uint64_t ts = 0; + uint64_t timestamp = 0; + if (mdsClient_->Tso(&ts, ×tamp) != FSStatusCode::OK) { + LOG(ERROR) << "ListDentry Tso failed, parent = " << parent; + return CURVEFS_ERROR::INTERNAL; + } + Dentry dentry; + dentry.set_fsid(fsId_); + dentry.set_parentinodeid(parent); + if (part.empty()) { + LOG(ERROR) << "ListDentry tx key locked, but part is empty" + << ", parent = " << parent; + return CURVEFS_ERROR::INTERNAL; + } + dentry.set_name(part.back().name()); + part.pop_back(); + MetaStatusCode rc = CheckAndResolveTx( + dentry, txLockOut, timestamp, ts); + if (rc != MetaStatusCode::OK) { + LOG(ERROR) << "ListDentry CheckAndResolveTx failed, rc = " + << MetaStatusCode_Name(rc) + << ", parent = " << parent; + return CURVEFS_ERROR::INTERNAL; + } + } else if (ret != MetaStatusCode::OK) { + LOG(ERROR) << "metaClient_ ListDentry failed" + << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret) + << ", parent = " << parent << ", last = " << last + << ", count = " << limit << ", onlyDir = " << onlyDir; + return ToFSError(ret); + } + + if (!onlyDir) { + if (part.size() < limit && ret != MetaStatusCode::TX_KEY_LOCKED) { + perceed = false; + } + if (!part.empty()) { + last = part.back().name(); + dentryList->splice(dentryList->end(), part); + } + } else { + // means iterate over the range + if (part.empty() && ret != MetaStatusCode::TX_KEY_LOCKED) { + perceed = false; + } else { + if (!part.empty()) { + last = part.back().name(); + if (part.back().type() != FsFileType::TYPE_DIRECTORY) { + part.pop_back(); + } + dentryList->splice(dentryList->end(), part); + } + // means already get all the dir under this dir + if (nlink - dentryList->size() == 2) { + perceed = false; + } + } + } + } while (perceed); + + return CURVEFS_ERROR::OK; +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/dentry_cache_manager.h b/curvefs/src/client/dentry_manager.h similarity index 76% rename from curvefs/src/client/dentry_cache_manager.h rename to curvefs/src/client/dentry_manager.h index 84f9f20f53..0014a28349 100644 --- a/curvefs/src/client/dentry_cache_manager.h +++ b/curvefs/src/client/dentry_manager.h @@ -21,8 +21,8 @@ * Author: xuchaojie */ -#ifndef CURVEFS_SRC_CLIENT_DENTRY_CACHE_MANAGER_H_ -#define CURVEFS_SRC_CLIENT_DENTRY_CACHE_MANAGER_H_ +#ifndef CURVEFS_SRC_CLIENT_DENTRY_MANAGER_H_ +#define CURVEFS_SRC_CLIENT_DENTRY_MANAGER_H_ #include #include @@ -33,6 +33,7 @@ #include #include "curvefs/src/client/rpcclient/metaserver_client.h" +#include "curvefs/src/client/rpcclient/mds_client.h" #include "src/common/concurrent/concurrent.h" #include "src/common/concurrent/name_lock.h" #include "curvefs/src/client/filesystem/error.h" @@ -44,6 +45,7 @@ namespace client { using rpcclient::MetaServerClient; using rpcclient::MetaServerClientImpl; +using rpcclient::MdsClient; using ::curvefs::client::filesystem::CURVEFS_ERROR; static const char* kDentryKeyDelimiter = ":"; @@ -57,6 +59,8 @@ class DentryCacheManager { fsId_ = fsId; } + virtual void Init(std::shared_ptr mdsClient) = 0; + virtual CURVEFS_ERROR GetDentry(uint64_t parent, const std::string &name, Dentry *out) = 0; @@ -70,6 +74,9 @@ class DentryCacheManager { std::list *dentryList, uint32_t limit, bool onlyDir = false, uint32_t nlink = 0) = 0; + virtual MetaStatusCode CheckAndResolveTx(const Dentry& dentry, + const TxLock& txLock, uint64_t timestamp, uint64_t commitTs) = 0; + protected: uint32_t fsId_; }; @@ -83,6 +90,10 @@ class DentryCacheManagerImpl : public DentryCacheManager { const std::shared_ptr &metaClient) : metaClient_(metaClient) {} + void Init(std::shared_ptr mdsClient) override { + mdsClient_ = mdsClient; + } + CURVEFS_ERROR GetDentry(uint64_t parent, const std::string &name, Dentry *out) override; @@ -96,11 +107,22 @@ class DentryCacheManagerImpl : public DentryCacheManager { std::list *dentryList, uint32_t limit, bool dirOnly = false, uint32_t nlink = 0) override; + MetaStatusCode CheckAndResolveTx(const Dentry& dentry, const TxLock& txLock, + uint64_t timestamp, uint64_t commitTs) override; + std::string GetDentryCacheKey(uint64_t parent, const std::string &name) { return std::to_string(parent) + kDentryKeyDelimiter + name; } private: + MetaStatusCode CheckTxStatus(const std::string primaryKey, uint64_t startTs, + uint64_t curTimestamp); + + MetaStatusCode ResolveTxLock(const Dentry& dentry, uint64_t startTs, + uint64_t commitTs = 0); + + private: + std::shared_ptr mdsClient_; std::shared_ptr metaClient_; curve::common::GenericNameLock nameLock_; }; @@ -108,4 +130,4 @@ class DentryCacheManagerImpl : public DentryCacheManager { } // namespace client } // namespace curvefs -#endif // CURVEFS_SRC_CLIENT_DENTRY_CACHE_MANAGER_H_ +#endif // CURVEFS_SRC_CLIENT_DENTRY_MANAGER_H_ diff --git a/curvefs/src/client/filesystem/.clang-format b/curvefs/src/client/filesystem/.clang-format new file mode 100644 index 0000000000..a5660a57d8 --- /dev/null +++ b/curvefs/src/client/filesystem/.clang-format @@ -0,0 +1,26 @@ +# I don't like current clang-formtter's specified style and +# and i think my style is more intuitive than it :), so I use +# this file to disable format, let us give some code snippet to prove it: +# +# my: +# std::vector tests { +# { " abc", "abc" }, +# { "abc ", "abc" }, +# { " abc ", "abc" }, +# { " a bc ", "a bc" }, +# { "a b c", "a b c" }, +# { " ", "" }, +# +# +# clang-formatter: +# std::vector tests{ +# {" abc", "abc"}, {"abc ", "abc"}, {" abc ", "abc"}, +# {" a bc ", "a bc"}, {"a b c", "a b c"}, {" ", ""}, +# }; +# +# Maybe you have different opinions, but please have a little mercy for me. +# +# by Wine93, 2023-10-16 + +DisableFormat: true +SortIncludes: Never diff --git a/curvefs/src/client/filesystem/defer_sync.cpp b/curvefs/src/client/filesystem/defer_sync.cpp index a69cdeb3ba..3a91709f9d 100644 --- a/curvefs/src/client/filesystem/defer_sync.cpp +++ b/curvefs/src/client/filesystem/defer_sync.cpp @@ -30,14 +30,94 @@ namespace curvefs { namespace client { namespace filesystem { -DeferSync::DeferSync(DeferSyncOption option) - : option_(option), +using ::curve::common::LockGuard; +using ::curve::common::ReadLockGuard; +using ::curve::common::WriteLockGuard; +using ::curvefs::client::filesystem::AttrCtime; + +#define RETURN_FALSE_IF_CTO_ON() \ + do { \ + if (cto_) { \ + return false; \ + } \ + } while (0) + +DeferInodes::DeferInodes(bool cto) + : cto_(cto), + rwlock_(), + inodes_() {} + +bool DeferInodes::Add(const std::shared_ptr& inode) { + RETURN_FALSE_IF_CTO_ON(); + WriteLockGuard lk(rwlock_); + Ino ino = inode->GetInodeId(); + auto ret = inodes_.emplace(ino, inode); + auto iter = ret.first; + bool yes = ret.second; + if (!yes) { // already exists + iter->second = inode; + } + return true; +} + +bool DeferInodes::Get(Ino ino, std::shared_ptr* inode) { + RETURN_FALSE_IF_CTO_ON(); + ReadLockGuard lk(rwlock_); + auto iter = inodes_.find(ino); + if (iter == inodes_.end()) { + return false; + } + *inode = iter->second; + return true; +} + +bool DeferInodes::Remove(const std::shared_ptr& inode) { + RETURN_FALSE_IF_CTO_ON(); + WriteLockGuard lk(rwlock_); + InodeAttr attr; + inode->GetInodeAttrLocked(&attr); + auto iter = inodes_.find(attr.inodeid()); + if (iter == inodes_.end()) { + return false; + } + + InodeAttr defered; + iter->second->GetInodeAttrLocked(&defered); + if (AttrCtime(attr) < AttrCtime(defered)) { + // it means the old defered inode already replaced by the lastest one, + // so we can't remove it before it synced yet. + return false; + } + inodes_.erase(iter); + return true; +} + +size_t DeferInodes::Size() { + ReadLockGuard lk(rwlock_); + return inodes_.size(); +} + +SyncInodeClosure::SyncInodeClosure(const std::shared_ptr& inodes, + const std::shared_ptr& inode) + : inodes_(inodes), inode_(inode) {} + +void SyncInodeClosure::Run() { + std::unique_ptr self_guard(this); + MetaStatusCode rc = GetStatusCode(); + if (rc == MetaStatusCode::OK || rc == MetaStatusCode::NOT_FOUND) { + inodes_->Remove(inode_); + } +} + +DeferSync::DeferSync(bool cto, DeferSyncOption option) + : cto_(cto), + option_(option), mutex_(), running_(false), thread_(), sleeper_(), - inodes_() { -} + pending_(), + inodes_(std::make_shared(cto)) {} void DeferSync::Start() { if (!running_.exchange(true)) { @@ -55,20 +135,32 @@ void DeferSync::Stop() { } } +SyncInodeClosure* DeferSync::NewSyncInodeClosure( + const std::shared_ptr& inode) { + // NOTE: we only store the defer inodes in nocto scenario, + // which means we don't need to remove the inode from defer inodes + // even if the inode already synced done in cto scenario. + if (cto_) { + return nullptr; + } + return new SyncInodeClosure(inodes_, inode); +} + void DeferSync::SyncTask() { - std::vector> inodes; + std::vector> syncing; for ( ;; ) { bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay)); { LockGuard lk(mutex_); - inodes.swap(inodes_); + syncing.swap(pending_); } - for (const auto& inode : inodes) { + for (const auto& inode : syncing) { + auto closure = NewSyncInodeClosure(inode); UniqueLock lk(inode->GetUniqueLock()); - inode->Async(nullptr, true); + inode->Async(closure, true); } - inodes.clear(); + syncing.clear(); if (!running) { break; @@ -78,7 +170,12 @@ void DeferSync::SyncTask() { void DeferSync::Push(const std::shared_ptr& inode) { LockGuard lk(mutex_); - inodes_.emplace_back(inode); + pending_.emplace_back(inode); + inodes_->Add(inode); +} + +bool DeferSync::IsDefered(Ino ino, std::shared_ptr* inode) { + return inodes_->Get(ino, inode); } } // namespace filesystem diff --git a/curvefs/src/client/filesystem/defer_sync.h b/curvefs/src/client/filesystem/defer_sync.h index 0bd59bb9bc..17472ce11e 100644 --- a/curvefs/src/client/filesystem/defer_sync.h +++ b/curvefs/src/client/filesystem/defer_sync.h @@ -27,23 +27,56 @@ #include #include +#include "absl/container/btree_map.h" #include "src/common/interruptible_sleeper.h" #include "curvefs/src/client/common/config.h" +#include "curvefs/src/client/rpcclient/task_excutor.h" #include "curvefs/src/client/filesystem/meta.h" namespace curvefs { namespace client { namespace filesystem { -using ::curvefs::client::common::DeferSyncOption; - +using ::curve::common::RWLock; using ::curve::common::Mutex; -using ::curve::common::LockGuard; using ::curve::common::InterruptibleSleeper; +using ::curvefs::client::common::DeferSyncOption; +using ::curvefs::client::rpcclient::MetaServerClientDone; + +// NOTE: we only store the defer inodes in nocto scenario. +class DeferInodes { + public: + explicit DeferInodes(bool cto); + + bool Add(const std::shared_ptr& inode); + + bool Get(Ino ino, std::shared_ptr* inode); + + bool Remove(const std::shared_ptr& inode); + + size_t Size(); + + private: + bool cto_; + RWLock rwlock_; + absl::btree_map> inodes_; +}; + +class SyncInodeClosure : public MetaServerClientDone { + public: + explicit SyncInodeClosure(const std::shared_ptr& inodes, + const std::shared_ptr& inode); + + void Run() override; + + private: + std::shared_ptr inodes_; + std::shared_ptr inode_; +}; class DeferSync { public: - explicit DeferSync(DeferSyncOption option); + explicit DeferSync(bool cto, DeferSyncOption option); void Start(); @@ -51,16 +84,26 @@ class DeferSync { void Push(const std::shared_ptr& inode); + bool IsDefered(Ino ino, std::shared_ptr* inode); + private: + SyncInodeClosure* NewSyncInodeClosure( + const std::shared_ptr& inode); + void SyncTask(); private: + friend class SyncInodeClosure; + + private: + bool cto_; DeferSyncOption option_; Mutex mutex_; std::atomic running_; std::thread thread_; InterruptibleSleeper sleeper_; - std::vector> inodes_; + std::vector> pending_; + std::shared_ptr inodes_; }; } // namespace filesystem diff --git a/curvefs/src/client/filesystem/dir_cache.cpp b/curvefs/src/client/filesystem/dir_cache.cpp index 3beed645fe..b535d926e4 100644 --- a/curvefs/src/client/filesystem/dir_cache.cpp +++ b/curvefs/src/client/filesystem/dir_cache.cpp @@ -33,7 +33,7 @@ DirEntryList::DirEntryList() : rwlock_(), mtime_(), entries_(), - attrs_() {} + index_() {} size_t DirEntryList::Size() { ReadLockGuard lk(rwlock_); @@ -43,39 +43,41 @@ size_t DirEntryList::Size() { void DirEntryList::Add(const DirEntry& dirEntry) { WriteLockGuard lk(rwlock_); entries_.push_back(std::move(dirEntry)); - attrs_[dirEntry.ino] = &entries_.back(); + index_[dirEntry.ino] = entries_.size() - 1; } bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) { ReadLockGuard lk(rwlock_); - auto iter = attrs_.find(ino); - if (iter == attrs_.end()) { + auto iter = index_.find(ino); + if (iter == index_.end()) { return false; } - *dirEntry = *iter->second; + + *dirEntry = entries_[iter->second]; return true; } bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) { WriteLockGuard lk(rwlock_); - auto iter = attrs_.find(ino); - if (iter == attrs_.end()) { + auto iter = index_.find(ino); + if (iter == index_.end()) { return false; } - DirEntry* dirEntry = iter->second; + DirEntry* dirEntry = &entries_[iter->second]; dirEntry->attr = std::move(attr); return true; } bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) { WriteLockGuard lk(rwlock_); - auto iter = attrs_.find(ino); - if (iter == attrs_.end()) { + auto iter = index_.find(ino); + if (iter == index_.end()) { return false; } - InodeAttr* attr = &iter->second->attr; + DirEntry* dirEntry = &entries_[iter->second]; + InodeAttr* attr = &(dirEntry->attr); attr->set_length(open.length()); attr->set_mtime(open.mtime()); attr->set_mtime_ns(open.mtime_ns()); @@ -93,10 +95,21 @@ void DirEntryList::Iterate(IterateHandler handler) { } } +void DirEntryList::IterateRange(uint64_t offset, + uint64_t count, + IterateHandler handler) { + ReadLockGuard lk(rwlock_); + auto iter = entries_.begin() + offset; + while (count > 0 && iter != entries_.end()) { + handler(&(*iter++)); + count--; + } +} + void DirEntryList::Clear() { WriteLockGuard lk(rwlock_); entries_.clear(); - attrs_.clear(); + index_.clear(); } void DirEntryList::SetMtime(TimeSpec mtime) { @@ -118,6 +131,7 @@ DirCache::DirCache(DirCacheOption option) mq_->Subscribe([&](const std::shared_ptr& entries){ entries->Clear(); }); + metric_ = std::make_shared(); LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize; } @@ -135,14 +149,16 @@ void DirCache::Stop() { void DirCache::Delete(Ino parent, std::shared_ptr entries, bool evit) { - nentries_ -= entries->Size(); + size_t ndelete = entries->Size(); + nentries_ -= ndelete; + metric_->AddEntries(-static_cast(ndelete)); mq_->Publish(entries); // clear entries in background lru_->Remove(parent); VLOG(1) << "Delete directory cache (evit=" << evit << "): " << "parent = " << parent << ", mtime = " << entries->GetMtime() - << ", size = " << entries->Size() + << ", delete size = " << ndelete << ", nentries = " << nentries_; } @@ -160,13 +176,19 @@ void DirCache::Evit(size_t size) { void DirCache::Put(Ino parent, std::shared_ptr entries) { WriteLockGuard lk(rwlock_); + if (entries->Size() == 0) { // TODO(Wine93): cache it! + return; + } + Evit(entries->Size()); // it guarantee put entries success lru_->Put(parent, entries); - nentries_ += entries->Size(); + int64_t ninsert = entries->Size(); + nentries_ += ninsert; + metric_->AddEntries(static_cast(ninsert)); VLOG(1) << "Insert directory cache: parent = " << parent << ", mtime = " << entries->GetMtime() - << ", size = " << entries->Size() + << ", insert size = " << ninsert << ", nentries = " << nentries_; } diff --git a/curvefs/src/client/filesystem/dir_cache.h b/curvefs/src/client/filesystem/dir_cache.h index 6aa57fe5d5..2be4da8c8b 100644 --- a/curvefs/src/client/filesystem/dir_cache.h +++ b/curvefs/src/client/filesystem/dir_cache.h @@ -25,12 +25,15 @@ #include #include +#include #include +#include "absl/container/btree_map.h" #include "src/common/lru_cache.h" #include "src/common/concurrent/concurrent.h" #include "curvefs/src/client/common/config.h" #include "curvefs/src/client/filesystem/meta.h" +#include "curvefs/src/client/filesystem/metric.h" #include "curvefs/src/client/filesystem/message_queue.h" namespace curvefs { @@ -56,6 +59,8 @@ class DirEntryList { void Iterate(IterateHandler handler); + void IterateRange(uint64_t offset, uint64_t count, IterateHandler handler); + bool Get(Ino ino, DirEntry* dirEntry); bool UpdateAttr(Ino ino, const InodeAttr& attr); @@ -71,8 +76,8 @@ class DirEntryList { private: RWLock rwlock_; TimeSpec mtime_; - std::list entries_; - std::map attrs_; + std::vector entries_; + absl::btree_map index_; }; class DirCache { @@ -105,6 +110,7 @@ class DirCache { DirCacheOption option_; std::shared_ptr lru_; std::shared_ptr mq_; + std::shared_ptr metric_; }; } // namespace filesystem diff --git a/curvefs/src/client/filesystem/error.cpp b/curvefs/src/client/filesystem/error.cpp index f8a677497b..1a77f467bd 100644 --- a/curvefs/src/client/filesystem/error.cpp +++ b/curvefs/src/client/filesystem/error.cpp @@ -35,15 +35,15 @@ static const std::map> errors = { { CURVEFS_ERROR::INTERNAL, { EIO, "internal error" } }, { CURVEFS_ERROR::UNKNOWN, { -1, "unknown" } }, { CURVEFS_ERROR::EXISTS, { EEXIST, "inode or dentry already exist" } }, - { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } }, + { CURVEFS_ERROR::NOT_EXIST, { ENOENT, "inode or dentry not exist" } }, { CURVEFS_ERROR::NO_SPACE, { ENOSPC, "no space to alloc" } }, { CURVEFS_ERROR::BAD_FD, { EBADF, "bad file number" } }, - { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } }, - { CURVEFS_ERROR::NOPERMISSION, { EACCES, "permission denied" } }, - { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } }, - { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } }, - { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } }, - { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } }, + { CURVEFS_ERROR::INVALID_PARAM , { EINVAL , "invalid argument" } }, + { CURVEFS_ERROR::NO_PERMISSION, { EACCES, "permission denied" } }, + { CURVEFS_ERROR::NOT_EMPTY, { ENOTEMPTY, "directory not empty" } }, + { CURVEFS_ERROR::NO_FLUSH, { -1, "no flush" } }, + { CURVEFS_ERROR::NOT_SUPPORT, { EOPNOTSUPP, "operation not supported" } }, + { CURVEFS_ERROR::NAME_TOO_LONG, { ENAMETOOLONG, "file name too long" } }, { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } }, { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } }, { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } }, @@ -51,6 +51,8 @@ static const std::map> errors = { { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } }, { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } }, { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } }, + { CURVEFS_ERROR::END_OF_FILE, { EOF, "end of file" } }, + { CURVEFS_ERROR::NOT_A_DIRECTORY, { ENOTDIR, "not a directory" } }, }; std::string StrErr(CURVEFS_ERROR code) { @@ -86,8 +88,8 @@ std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) { CURVEFS_ERROR ToFSError(MetaStatusCode code) { static std::map errs = { { MetaStatusCode::OK, CURVEFS_ERROR::OK }, - { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST }, - { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM }, + { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOT_EXIST }, + { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALID_PARAM }, { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS }, { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS }, { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL }, diff --git a/curvefs/src/client/filesystem/error.h b/curvefs/src/client/filesystem/error.h index 1f837ca9d6..75efe38283 100644 --- a/curvefs/src/client/filesystem/error.h +++ b/curvefs/src/client/filesystem/error.h @@ -40,23 +40,25 @@ enum class CURVEFS_ERROR { INTERNAL = -1, UNKNOWN = -2, EXISTS = -3, - NOTEXIST = -4, + NOT_EXIST = -4, NO_SPACE = -5, BAD_FD = -6, - INVALIDPARAM = -7, - NOPERMISSION = -8, - NOTEMPTY = -9, - NOFLUSH = -10, - NOTSUPPORT = -11, - NAMETOOLONG = -12, + INVALID_PARAM = -7, + NO_PERMISSION = -8, + NOT_EMPTY = -9, + NO_FLUSH = -10, + NOT_SUPPORT = -10, + NAME_TOO_LONG = -12, MOUNT_POINT_EXIST = -13, MOUNT_FAILED = -14, OUT_OF_RANGE = -15, NODATA = -16, IO_ERROR = -17, - CACHETOOSMALL = -18, + CACHE_TOO_SMALL = -18, STALE = -19, NOSYS = -20, + END_OF_FILE = -21, + NOT_A_DIRECTORY = -22, }; std::string StrErr(CURVEFS_ERROR code); diff --git a/curvefs/src/client/filesystem/filesystem.cpp b/curvefs/src/client/filesystem/filesystem.cpp index 795c896a5b..4546ade2af 100644 --- a/curvefs/src/client/filesystem/filesystem.cpp +++ b/curvefs/src/client/filesystem/filesystem.cpp @@ -31,7 +31,8 @@ namespace filesystem { FileSystem::FileSystem(FileSystemOption option, ExternalMember member) : option_(option), member(member) { - deferSync_ = std::make_shared(option.deferSyncOption); + deferSync_ = std::make_shared(option.cto, + option.deferSyncOption); negative_ = std::make_shared(option.lookupCacheOption); dirCache_ = std::make_shared(option.dirCacheOption); openFiles_ = std::make_shared(option_.openFilesOption, @@ -234,29 +235,28 @@ FileSystemMember FileSystem::BorrowMember() { } // fuse request* -CURVEFS_ERROR FileSystem::Lookup(Request req, - Ino parent, +CURVEFS_ERROR FileSystem::Lookup(Ino parent, const std::string& name, EntryOut* entryOut) { if (name.size() > option_.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } bool yes = negative_->Get(parent, name); if (yes) { - return CURVEFS_ERROR::NOTEXIST; + return CURVEFS_ERROR::NOT_EXIST; } auto rc = rpc_->Lookup(parent, name, entryOut); if (rc == CURVEFS_ERROR::OK) { negative_->Delete(parent, name); - } else if (rc == CURVEFS_ERROR::NOTEXIST) { + } else if (rc == CURVEFS_ERROR::NOT_EXIST) { negative_->Put(parent, name); } return rc; } -CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) { +CURVEFS_ERROR FileSystem::GetAttr(Ino ino, AttrOut* attrOut) { InodeAttr attr; auto rc = rpc_->GetAttr(ino, &attr); if (rc == CURVEFS_ERROR::OK) { @@ -265,7 +265,7 @@ CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) { return rc; } -CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) { +CURVEFS_ERROR FileSystem::OpenDir(Ino ino, FileInfo* fi) { InodeAttr attr; CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr); if (rc != CURVEFS_ERROR::OK) { @@ -287,8 +287,7 @@ CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) { return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FileSystem::ReadDir(Request req, - Ino ino, +CURVEFS_ERROR FileSystem::ReadDir(Ino ino, FileInfo* fi, std::shared_ptr* entries) { bool yes = dirCache_->Get(ino, entries); @@ -306,17 +305,17 @@ CURVEFS_ERROR FileSystem::ReadDir(Request req, return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) { +CURVEFS_ERROR FileSystem::ReleaseDir(Ino ino, FileInfo* fi) { ReleaseHandler(fi->fh); return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) { +CURVEFS_ERROR FileSystem::Open(Ino ino, FileInfo* fi) { std::shared_ptr inode; bool yes = openFiles_->IsOpened(ino, &inode); if (yes) { openFiles_->Open(ino, inode); - // fi->keep_cache = 1; + // fi->keep_cache = 1; // FIXME(Wine93): let it works. return CURVEFS_ERROR::OK; } @@ -343,7 +342,7 @@ CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) { return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) { +CURVEFS_ERROR FileSystem::Release(Ino ino) { openFiles_->Close(ino); return CURVEFS_ERROR::OK; } diff --git a/curvefs/src/client/filesystem/filesystem.h b/curvefs/src/client/filesystem/filesystem.h index c2d48db708..eaa6cacc2a 100644 --- a/curvefs/src/client/filesystem/filesystem.h +++ b/curvefs/src/client/filesystem/filesystem.h @@ -67,25 +67,23 @@ class FileSystem { void Destory(); // fuse request - CURVEFS_ERROR Lookup(Request req, - Ino parent, + CURVEFS_ERROR Lookup(Ino parent, const std::string& name, EntryOut* entryOut); - CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut); + CURVEFS_ERROR GetAttr(Ino ino, AttrOut* attrOut); - CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi); + CURVEFS_ERROR OpenDir(Ino ino, FileInfo* fi); - CURVEFS_ERROR ReadDir(Request req, - Ino ino, + CURVEFS_ERROR ReadDir(Ino ino, FileInfo* fi, std::shared_ptr* entries); - CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi); + CURVEFS_ERROR ReleaseDir(Ino ino, FileInfo* fi); - CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi); + CURVEFS_ERROR Open(Ino ino, FileInfo* fi); - CURVEFS_ERROR Release(Request req, Ino ino); + CURVEFS_ERROR Release(Ino ino); // fuse reply: we control all replies to vfs layer in same entrance. void ReplyError(Request req, CURVEFS_ERROR code); @@ -129,15 +127,6 @@ class FileSystem { void ReleaseHandler(uint64_t fh); - // utility: others - FileSystemMember BorrowMember(); - - private: - FRIEND_TEST(FileSystemTest, Attr2Stat); - FRIEND_TEST(FileSystemTest, Entry2Param); - FRIEND_TEST(FileSystemTest, SetEntryTimeout); - FRIEND_TEST(FileSystemTest, SetAttrTimeout); - // utility: convert to system type. void Attr2Stat(InodeAttr* attr, struct stat* stat); @@ -148,6 +137,9 @@ class FileSystem { void SetAttrTimeout(AttrOut* attrOut); + // utility: others + FileSystemMember BorrowMember(); + private: FileSystemOption option_; ExternalMember member; diff --git a/curvefs/src/client/filesystem/lookup_cache.cpp b/curvefs/src/client/filesystem/lookup_cache.cpp index d5c473a114..7e49b0db1c 100644 --- a/curvefs/src/client/filesystem/lookup_cache.cpp +++ b/curvefs/src/client/filesystem/lookup_cache.cpp @@ -83,7 +83,7 @@ bool LookupCache::Put(Ino parent, const std::string& name) { if (yes) { entry.uses++; } else { - entry.uses = 0; + entry.uses = 1; } entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0); diff --git a/curvefs/src/client/filesystem/meta.cpp b/curvefs/src/client/filesystem/meta.cpp index 721c7c859b..4c835fd4d7 100644 --- a/curvefs/src/client/filesystem/meta.cpp +++ b/curvefs/src/client/filesystem/meta.cpp @@ -106,7 +106,6 @@ std::string Attr2Str(const InodeAttr& attr) { return ""; } - std::string smode; return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])", attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(), attr.uid(), attr.gid(), @@ -124,6 +123,10 @@ std::string StrAttr(AttrOut attrOut) { return Attr2Str(attrOut.attr); } +std::string StrAttr(InodeAttr attr) { + return Attr2Str(attr); +} + } // namespace filesystem } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/filesystem/meta.h b/curvefs/src/client/filesystem/meta.h index 1165a1fb4f..c10ec1b0d9 100644 --- a/curvefs/src/client/filesystem/meta.h +++ b/curvefs/src/client/filesystem/meta.h @@ -171,6 +171,8 @@ std::string StrEntry(EntryOut entryOut); std::string StrAttr(AttrOut attrOut); +std::string StrAttr(InodeAttr attr); + } // namespace filesystem } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/filesystem/metric.h b/curvefs/src/client/filesystem/metric.h new file mode 100644 index 0000000000..3627349eda --- /dev/null +++ b/curvefs/src/client/filesystem/metric.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-13 + * Author: Jingli Chen (Wine93) + */ + +#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_METRIC_H_ +#define CURVEFS_SRC_CLIENT_FILESYSTEM_METRIC_H_ + +#include + +#include + +namespace curvefs { +namespace client { +namespace filesystem { + +class DirCacheMetric { + public: + DirCacheMetric() = default; + + void AddEntries(int64_t n) { + metric_.nentries << n; + } + + private: + struct Metric { + Metric() : nentries("filesystem_dircache", "nentries") {} + bvar::Adder nentries; + }; + + Metric metric_; +}; + +class OpenfilesMetric { + public: + OpenfilesMetric() = default; + + void AddOpenfiles(int64_t n) { + metric_.nfiles << n; + } + + private: + struct Metric { + Metric() : nfiles("filesystem_openfiles", "nfiles") {} + bvar::Adder nfiles; + }; + + Metric metric_; +}; + +} // namespace filesystem +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_FILESYSTEM_METRIC_H_ diff --git a/curvefs/src/client/filesystem/openfile.cpp b/curvefs/src/client/filesystem/openfile.cpp index 49ba785834..944ac77785 100644 --- a/curvefs/src/client/filesystem/openfile.cpp +++ b/curvefs/src/client/filesystem/openfile.cpp @@ -31,8 +31,9 @@ OpenFiles::OpenFiles(OpenFilesOption option, std::shared_ptr deferSync) : rwlock_(), option_(option), - deferSync_(deferSync), - files_(std::make_shared(option.lruSize)) { + deferSync_(deferSync) { + files_ = std::make_shared(option.lruSize); + metric_ = std::make_shared(); LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize; } @@ -48,6 +49,7 @@ void OpenFiles::Delete(Ino ino, deferSync_->Push(file->inode); } files_->Remove(ino); + metric_->AddOpenfiles(-1); VLOG(1) << "Delete open file cache: ino = " << ino << ", refs = " << file->refs @@ -81,6 +83,7 @@ void OpenFiles::Open(Ino ino, std::shared_ptr inode) { file = std::make_shared(inode); file->refs++; files_->Put(ino, file); + metric_->AddOpenfiles(1); VLOG(1) << "Insert open file cache: ino = " << ino << ", refs = " << file->refs diff --git a/curvefs/src/client/filesystem/openfile.h b/curvefs/src/client/filesystem/openfile.h index 53ce0ae26e..7a88f8626f 100644 --- a/curvefs/src/client/filesystem/openfile.h +++ b/curvefs/src/client/filesystem/openfile.h @@ -30,6 +30,7 @@ #include "curvefs/src/client/filesystem/meta.h" #include "curvefs/src/client/filesystem/defer_sync.h" #include "curvefs/src/client/filesystem/dir_cache.h" +#include "curvefs/src/client/filesystem/metric.h" namespace curvefs { namespace client { @@ -77,6 +78,7 @@ class OpenFiles { OpenFilesOption option_; std::shared_ptr deferSync_; std::shared_ptr files_; + std::shared_ptr metric_; }; } // namespace filesystem diff --git a/curvefs/src/client/filesystem/package.h b/curvefs/src/client/filesystem/package.h index 47f9fccc36..8eb7a14b23 100644 --- a/curvefs/src/client/filesystem/package.h +++ b/curvefs/src/client/filesystem/package.h @@ -25,8 +25,8 @@ #include -#include "curvefs/src/client/dentry_cache_manager.h" -#include "curvefs/src/client/inode_cache_manager.h" +#include "curvefs/src/client/dentry_manager.h" +#include "curvefs/src/client/inode_manager.h" #include "curvefs/src/client/xattr_manager.h" namespace curvefs { diff --git a/curvefs/src/client/filesystem/rpc_client.cpp b/curvefs/src/client/filesystem/rpc_client.cpp index 4f5639d1f3..236c50687f 100644 --- a/curvefs/src/client/filesystem/rpc_client.cpp +++ b/curvefs/src/client/filesystem/rpc_client.cpp @@ -54,7 +54,7 @@ CURVEFS_ERROR RPCClient::Lookup(Ino parent, Dentry dentry; CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry); if (rc != CURVEFS_ERROR::OK) { - if (rc != CURVEFS_ERROR::NOTEXIST) { + if (rc != CURVEFS_ERROR::NOT_EXIST) { LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc << ", parent = " << parent << ", name = " << name; } @@ -80,6 +80,10 @@ CURVEFS_ERROR RPCClient::ReadDir(Ino ino, LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc << ", ino = " << ino; return rc; + } else if (dentries.size() == 0) { + VLOG(3) << "rpc(readdir::ListDentry) success and directory is empty" + << ", ino = " << ino; + return rc; } std::set inos; diff --git a/curvefs/src/client/filesystem/utils.cpp b/curvefs/src/client/filesystem/utils.cpp index 0a6b6c38e0..dd4e2a5c6c 100644 --- a/curvefs/src/client/filesystem/utils.cpp +++ b/curvefs/src/client/filesystem/utils.cpp @@ -40,7 +40,7 @@ bool IsVolmeFile(const InodeAttr& attr) { return attr.type() == FsFileType::TYPE_FILE; } -bool IsSymLink(const InodeAttr& attr) { +bool IsSymlink(const InodeAttr& attr) { return attr.type() == FsFileType::TYPE_SYM_LINK; } diff --git a/curvefs/src/client/filesystem/utils.h b/curvefs/src/client/filesystem/utils.h index 8f37104e1d..706fa3d271 100644 --- a/curvefs/src/client/filesystem/utils.h +++ b/curvefs/src/client/filesystem/utils.h @@ -42,7 +42,7 @@ bool IsS3File(const InodeAttr& attr); bool IsVolmeFile(const InodeAttr& attr); // symbol link -bool IsSymLink(const InodeAttr& attr); +bool IsSymlink(const InodeAttr& attr); struct TimeSpec AttrMtime(const InodeAttr& attr); diff --git a/curvefs/src/client/filesystem/xattr.h b/curvefs/src/client/filesystem/xattr.h new file mode 100644 index 0000000000..26bc3a6c2e --- /dev/null +++ b/curvefs/src/client/filesystem/xattr.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-19 + * Author: Jingli Chen (Wine93) + */ + +#include +#include +#include + +#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_XATTR_H_ +#define CURVEFS_SRC_CLIENT_FILESYSTEM_XATTR_H_ + +namespace curvefs { +namespace client { +namespace filesystem { + +const uint32_t MAX_XATTR_NAME_LENGTH = 255; +const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024; + +const char XATTR_DIR_FILES[] = "curve.dir.files"; +const char XATTR_DIR_SUBDIRS[] = "curve.dir.subdirs"; +const char XATTR_DIR_ENTRIES[] = "curve.dir.entries"; +const char XATTR_DIR_FBYTES[] = "curve.dir.fbytes"; +const char XATTR_DIR_RFILES[] = "curve.dir.rfiles"; +const char XATTR_DIR_RSUBDIRS[] = "curve.dir.rsubdirs"; +const char XATTR_DIR_RENTRIES[] = "curve.dir.rentries"; +const char XATTR_DIR_RFBYTES[] = "curve.dir.rfbytes"; +const char XATTR_DIR_PREFIX[] = "curve.dir"; +const char XATTR_WARMUP_OP[] = "curvefs.warmup.op"; +const char XATTR_WARMUP_OP_LIST[] = "curvefs.warmup.op.list"; + +inline bool IsSpecialXAttr(const std::string& key) { + static std::map xattrs { + { XATTR_DIR_FILES, true }, + { XATTR_DIR_SUBDIRS, true }, + { XATTR_DIR_ENTRIES, true }, + { XATTR_DIR_FBYTES, true }, + { XATTR_DIR_RFILES, true }, + { XATTR_DIR_RSUBDIRS, true }, + { XATTR_DIR_RENTRIES, true }, + { XATTR_DIR_RFBYTES, true }, + { XATTR_DIR_PREFIX, true }, + }; + return xattrs.find(key) != xattrs.end(); +} + +inline bool IsWarmupXAttr(const std::string& key) { + return key == XATTR_WARMUP_OP; +} + +inline bool IsListWarmupXAttr(const std::string& key) { + return key == XATTR_WARMUP_OP_LIST; +} + +} // namespace filesystem +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_FILESYSTEM_XATTR_H_ diff --git a/curvefs/src/client/fuse_client.cpp b/curvefs/src/client/fuse_client.cpp index b67d0faa66..f1e19721e1 100644 --- a/curvefs/src/client/fuse_client.cpp +++ b/curvefs/src/client/fuse_client.cpp @@ -23,42 +23,57 @@ #include "curvefs/src/client/fuse_client.h" -#include #include #include +#include #include -#include -#include #include +#include #include #include +#include +#include "absl/memory/memory.h" #include "curvefs/proto/mds.pb.h" +#include "curvefs/src/client/client_operator.h" #include "curvefs/src/client/common/common.h" #include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/filesystem/xattr.h" #include "curvefs/src/client/fuse_common.h" -#include "curvefs/src/client/client_operator.h" #include "curvefs/src/client/inode_wrapper.h" +#include "curvefs/src/client/rpcclient/fsdelta_updater.h" +#include "curvefs/src/client/rpcclient/fsquota_checker.h" #include "curvefs/src/client/warmup/warmup_manager.h" #include "curvefs/src/client/xattr_manager.h" #include "curvefs/src/common/define.h" -#include "src/common/net_common.h" -#include "src/common/dummyserver.h" #include "src/client/client_common.h" -#include "absl/memory/memory.h" +#include "src/common/dummyserver.h" +#include "src/common/net_common.h" #define PORT_LIMIT 65535 -using ::curvefs::common::S3Info; -using ::curvefs::common::Volume; -using ::curvefs::mds::topology::PartitionTxId; -using ::curvefs::mds::FSStatusCode_Name; -using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH; -using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH; -using ::curvefs::client::filesystem::ExternalMember; +using ::curvefs::client::common::FuseClientOption; using ::curvefs::client::filesystem::DirEntry; using ::curvefs::client::filesystem::DirEntryList; +using ::curvefs::client::filesystem::ExternalMember; using ::curvefs::client::filesystem::FileOut; +using ::curvefs::client::filesystem::IsSpecialXAttr; +using ::curvefs::client::filesystem::MAX_XATTR_NAME_LENGTH; +using ::curvefs::client::filesystem::MAX_XATTR_VALUE_LENGTH; +using ::curvefs::client::filesystem::XATTR_DIR_ENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_FILES; +using ::curvefs::client::filesystem::XATTR_DIR_PREFIX; +using ::curvefs::client::filesystem::XATTR_DIR_RENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_RFBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_RFILES; +using ::curvefs::client::filesystem::XATTR_DIR_RSUBDIRS; +using ::curvefs::client::filesystem::XATTR_DIR_SUBDIRS; +using ::curvefs::common::FSType; +using ::curvefs::common::S3Info; +using ::curvefs::common::Volume; +using ::curvefs::mds::FSStatusCode_Name; +using ::curvefs::mds::topology::PartitionTxId; #define RETURN_IF_UNSUCCESS(action) \ do { \ @@ -88,6 +103,9 @@ DECLARE_uint64(fuseClientBurstReadIopsSecs); DECLARE_uint64(fuseClientAvgReadBytes); DECLARE_uint64(fuseClientBurstReadBytes); DECLARE_uint64(fuseClientBurstReadBytesSecs); +DECLARE_bool(fs_disableXattr); + +DECLARE_int32(TxVersion); } // namespace common } // namespace client } // namespace curvefs @@ -142,6 +160,8 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) { metaCache, mdsClient_, &enableSumInDir_); + dentryManager_->Init(mdsClient_); + xattrManager_ = std::make_shared(inodeManager_, dentryManager_, option_.listDentryLimit, option_.listDentryThreads); @@ -186,6 +206,9 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) { warmupManager_->SetFsInfo(fsInfo_); } + FsDeltaUpdater::GetInstance().Init(); + FsQuotaChecker::GetInstance().Init(); + InitQosParam(); return CURVEFS_ERROR::OK; @@ -267,8 +290,8 @@ CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req, fuse_ino_t parent, const char* name, EntryOut* entryOut) { - CURVEFS_ERROR rc = fs_->Lookup(req, parent, name, entryOut); - if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOTEXIST) { + CURVEFS_ERROR rc = fs_->Lookup(parent, name, entryOut); + if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOT_EXIST) { LOG(ERROR) << "Lookup() failed, retCode = " << rc << ", parent = " << parent << ", name = " << name; } @@ -317,8 +340,8 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req, // update parent summary info const Inode *inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, - std::to_string(length)}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(length)}); for (const auto &it : inode->parent()) { auto tret = xattrManager_->UpdateParentInodeXattr( it, xattr, false); @@ -331,7 +354,7 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req, } inodeWrapper->GetInodeAttrLocked(&fileOut->attr); } else { - return CURVEFS_ERROR::NOPERMISSION; + return CURVEFS_ERROR::NO_PERMISSION; } } return CURVEFS_ERROR::OK; @@ -341,7 +364,7 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi, FileOut* fileOut) { - CURVEFS_ERROR rc = fs_->Open(req, ino, fi); + CURVEFS_ERROR rc = fs_->Open(ino, fi); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "open(" << ino << ") failed, retCode = " << rc; return rc; @@ -349,35 +372,6 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, return HandleOpenFlags(req, ino, fi, fileOut); } -CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink( - fuse_ino_t parent, FsFileType type, NlinkChange nlink) { - - std::shared_ptr parentInodeWrapper; - auto ret = inodeManager_->GetInode(parent, parentInodeWrapper); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "inodeManager get inode fail, ret = " << ret - << ", inodeid = " << parent; - return ret; - } - - { - curve::common::UniqueLock lk = parentInodeWrapper->GetUniqueLock(); - parentInodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime); - - if (FsFileType::TYPE_DIRECTORY == type) { - parentInodeWrapper->UpdateNlinkLocked(nlink); - } - - if (option_.fileSystemOption.deferSyncOption.deferDirMtime) { - inodeManager_->ShipToFlush(parentInodeWrapper); - } else { - return parentInodeWrapper->SyncAttr(); - } - } - - return CURVEFS_ERROR::OK; -} - CURVEFS_ERROR FuseClient::MakeNode( fuse_req_t req, fuse_ino_t parent, @@ -388,19 +382,19 @@ CURVEFS_ERROR FuseClient::MakeNode( bool internal, std::shared_ptr& inodeWrapper) { if (strlen(name) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } // check if node is recycle or under recycle if (!internal && strcmp(name, RECYCLENAME) == 0 && parent == ROOTINODEID) { LOG(WARNING) << "Can not make node " << RECYCLENAME << " under root dir."; - return CURVEFS_ERROR::NOPERMISSION; + return CURVEFS_ERROR::NO_PERMISSION; } if (!internal && parent == RECYCLEINODEID) { LOG(WARNING) << "Can not make node under recycle."; - return CURVEFS_ERROR::NOPERMISSION; + return CURVEFS_ERROR::NO_PERMISSION; } const struct fuse_ctx *ctx = fuse_req_ctx(req); @@ -411,6 +405,7 @@ CURVEFS_ERROR FuseClient::MakeNode( } else { param.length = 0; } + param.uid = ctx->uid; param.gid = ctx->gid; param.mode = mode; @@ -440,7 +435,6 @@ CURVEFS_ERROR FuseClient::MakeNode( if (type == FsFileType::TYPE_FILE || type == FsFileType::TYPE_S3) { dentry.set_flag(DentryFlag::TYPE_FILE_FLAG); } - ret = dentryManager_->CreateDentry(dentry); if (ret != CURVEFS_ERROR::OK) { LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret @@ -456,15 +450,6 @@ CURVEFS_ERROR FuseClient::MakeNode( return ret; } - ret = UpdateParentMCTimeAndNlink(parent, type, NlinkChange::kAddOne); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "UpdateParentMCTimeAndNlink failed" - << ", parent: " << parent - << ", name: " << name - << ", type: " << type; - return ret; - } - VLOG(6) << "dentryManager_ CreateDentry success" << ", parent = " << parent << ", name = " << name << ", mode = " << mode; @@ -472,14 +457,14 @@ CURVEFS_ERROR FuseClient::MakeNode( if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); if (type == FsFileType::TYPE_DIRECTORY) { - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); } else { - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { LOG(ERROR) << "UpdateParentInodeXattr failed," @@ -525,15 +510,6 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent, return ret; } - ret = UpdateParentMCTimeAndNlink(parent, type, NlinkChange::kSubOne); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "UpdateParentMCTimeAndNlink failed" - << ", parent: " << parent - << ", name: " << name - << ", type: " << type; - return ret; - } - std::shared_ptr inodeWrapper; ret = inodeManager_->GetInode(ino, inodeWrapper); if (ret != CURVEFS_ERROR::OK) { @@ -551,14 +527,14 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent, if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); if (FsFileType::TYPE_DIRECTORY == type) { - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); } else { - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, false); if (tret != CURVEFS_ERROR::OK) { LOG(WARNING) << "UpdateParentInodeXattr failed," @@ -585,7 +561,7 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, ManageInodeType manageType, EntryOut* entryOut) { if (strlen(name) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } InodeParam param; @@ -620,7 +596,6 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, if (type == FsFileType::TYPE_FILE || type == FsFileType::TYPE_S3) { dentry.set_flag(DentryFlag::TYPE_FILE_FLAG); } - ret = dentryManager_->CreateDentry(dentry); if (ret != CURVEFS_ERROR::OK) { LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret @@ -636,15 +611,6 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, return ret; } - ret = UpdateParentMCTimeAndNlink(parent, type, NlinkChange::kAddOne); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "UpdateParentMCTimeAndNlink failed" - << ", parent: " << parent - << ", name: " << name - << ", type: " << type; - return ret; - } - VLOG(6) << "dentryManager_ CreateDentry success" << ", parent = " << parent << ", name = " << name << ", mode = " << mode; @@ -652,14 +618,14 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); if (type == FsFileType::TYPE_DIRECTORY) { - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); } else { - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { LOG(ERROR) << "UpdateParentInodeXattr failed," @@ -674,12 +640,12 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, CURVEFS_ERROR FuseClient::GetOrCreateRecycleDir(fuse_req_t req, Dentry *out) { auto ret = dentryManager_->GetDentry(ROOTINODEID, RECYCLENAME, out); - if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOTEXIST) { + if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOT_EXIST) { LOG(ERROR) << "dentryManager_ GetDentry fail, ret = " << ret << ", inode = " << ROOTINODEID << ", name = " << RECYCLENAME; return ret; - } else if (ret == CURVEFS_ERROR::NOTEXIST) { + } else if (ret == CURVEFS_ERROR::NOT_EXIST) { LOG(INFO) << "recycle dir is not exist, create " << RECYCLENAME << ", parentid = " << ROOTINODEID; EntryOut entryOut; @@ -716,12 +682,12 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino, uint64_t recycleTimeDirIno; ret = dentryManager_->GetDentry(RECYCLEINODEID, recycleTimeDirName.c_str(), &dentry); - if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOTEXIST) { + if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOT_EXIST) { LOG(ERROR) << "dentryManager_ GetDentry fail, ret = " << ret << ", inode = " << RECYCLEINODEID << ", name = " << recycleTimeDirName; return ret; - } else if (ret == CURVEFS_ERROR::NOTEXIST) { + } else if (ret == CURVEFS_ERROR::NOT_EXIST) { std::shared_ptr inode; bool internal = true; ret = MakeNode(req, RECYCLEINODEID, recycleTimeDirName.c_str(), @@ -780,13 +746,13 @@ bool FuseClient::ShouldMoveToRecycle(fuse_ino_t parent) { CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent, const char *name, FsFileType type) { if (strlen(name) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } // check if node is recycle or recycle time dir if ((strcmp(name, RECYCLENAME) == 0 && parent == ROOTINODEID) || parent == RECYCLEINODEID) { - return CURVEFS_ERROR::NOPERMISSION; + return CURVEFS_ERROR::NO_PERMISSION; } Dentry dentry; @@ -811,10 +777,22 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent, } if (!dentryList.empty()) { LOG(ERROR) << "rmdir not empty"; - return CURVEFS_ERROR::NOTEMPTY; + return CURVEFS_ERROR::NOT_EMPTY; } } + int64_t deltaBytes = 0; + { + std::shared_ptr inodeWrapper; + ret = inodeManager_->GetInode(ino, inodeWrapper); + if (ret != CURVEFS_ERROR::OK) { + LOG(ERROR) << "inodeManager get inode fail, ret = " << ret + << ", inodeid = " << ino; + return ret; + } + deltaBytes = -1 * inodeWrapper->GetLength(); + } + // check if inode should move to recycle if (ShouldMoveToRecycle(parent)) { ret = MoveToRecycle(req, ino, parent, name, type); @@ -832,13 +810,15 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent, } } + FsDeltaUpdater::GetInstance().UpdateDeltaBytes(deltaBytes); + return CURVEFS_ERROR::OK; } CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - CURVEFS_ERROR rc = fs_->OpenDir(req, ino, fi); + CURVEFS_ERROR rc = fs_->OpenDir(ino, fi); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "opendir() failed, retCode = " << rc << ", ino = " << ino; @@ -858,7 +838,7 @@ CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req, DirBufferHead* buffer = handler->buffer; if (!handler->padding) { auto entries = std::make_shared(); - CURVEFS_ERROR rc = fs_->ReadDir(req, ino, fi, &entries); + CURVEFS_ERROR rc = fs_->ReadDir(ino, fi, &entries); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "readdir() failed, retCode = " << rc << ", ino = " << ino << ", fh = " << fi->fh; @@ -888,7 +868,7 @@ CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req, CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - CURVEFS_ERROR rc = fs_->ReleaseDir(req, ino, fi); + CURVEFS_ERROR rc = fs_->ReleaseDir(ino, fi); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "releasedir() failed, retCode = " << rc << ", ino = " << ino; @@ -907,7 +887,7 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent, // is only used in linux interface renameat(), not required by posix, // we can ignore it now if (flags != 0) { - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } uint64_t maxNameLength = option_.fileSystemOption.maxNameLength; @@ -916,41 +896,66 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent, << ", name len = " << strlen(name) << ", new name = " << newname << ", new name len = " << strlen(newname) << ", maxNameLength = " << maxNameLength; - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } - auto renameOp = - RenameOperator(fsInfo_->fsid(), fsInfo_->fsname(), - parent, name, newparent, newname, - dentryManager_, inodeManager_, metaClient_, mdsClient_, - option_.enableMultiMountPointRename); + auto renameOp = RenameOperator(fsInfo_->fsid(), fsInfo_->fsname(), parent, + name, newparent, newname, dentryManager_, inodeManager_, metaClient_, + mdsClient_, option_.enableMultiMountPointRename); - curve::common::LockGuard lg(renameMutex_); CURVEFS_ERROR rc = CURVEFS_ERROR::OK; - VLOG(3) << "FuseOpRename [start]: " << renameOp.DebugString(); - RETURN_IF_UNSUCCESS(GetTxId); - RETURN_IF_UNSUCCESS(Precheck); - RETURN_IF_UNSUCCESS(RecordOldInodeInfo); - // Do not move LinkDestParentInode behind CommitTx. - // If so, the nlink will be lost when the machine goes down - RETURN_IF_UNSUCCESS(LinkDestParentInode); - RETURN_IF_UNSUCCESS(PrepareTx); - RETURN_IF_UNSUCCESS(CommitTx); - VLOG(3) << "FuseOpRename [success]: " << renameOp.DebugString(); - // Do not check UnlinkSrcParentInode, beause rename is already success - renameOp.UnlinkSrcParentInode(); - renameOp.UnlinkOldInode(); - if (parent != newparent) { - renameOp.UpdateInodeParent(); - } - renameOp.UpdateInodeCtime(); - renameOp.UpdateCache(); + if (common::FLAGS_TxVersion == 1) { + curve::common::LockGuard lg(renameMutex_); + VLOG(3) << "FuseOpRename [start]: " << renameOp.DebugString(); + RETURN_IF_UNSUCCESS(GetTxId); + RETURN_IF_UNSUCCESS(Precheck); + RETURN_IF_UNSUCCESS(RecordOldInodeInfo); + // Do not move LinkDestParentInode behind CommitTx. + // If so, the nlink will be lost when the machine goes down + RETURN_IF_UNSUCCESS(LinkDestParentInode); + RETURN_IF_UNSUCCESS(PrepareTx); + RETURN_IF_UNSUCCESS(CommitTx); + VLOG(3) << "FuseOpRename [success]: " << renameOp.DebugString(); + // Do not check UnlinkSrcParentInode, beause rename is already success + renameOp.UnlinkSrcParentInode(); + renameOp.UnlinkOldInode(); + if (parent != newparent) { + renameOp.UpdateInodeParent(); + } + renameOp.UpdateInodeCtime(); + renameOp.UpdateCache(); - if (enableSumInDir_.load()) { - xattrManager_->UpdateParentXattrAfterRename( - parent, newparent, newname, &renameOp); - } + if (enableSumInDir_.load()) { + xattrManager_->UpdateParentXattrAfterRename( + parent, newparent, newname, &renameOp); + } + } else if (common::FLAGS_TxVersion == 2) { + VLOG(3) << "FuseOpRename [start]: " << renameOp.DebugString(); + RETURN_IF_UNSUCCESS(Precheck); + RETURN_IF_UNSUCCESS(RecordOldInodeInfo); + // Do not move LinkDestParentInode behind CommitTx. + // If so, the nlink will be lost when the machine goes down + RETURN_IF_UNSUCCESS(LinkDestParentInode); + RETURN_IF_UNSUCCESS(PrewriteTx); + RETURN_IF_UNSUCCESS(CommitTxV2); + VLOG(3) << "FuseOpRename [success]: " << renameOp.DebugString(); + // Do not check UnlinkSrcParentInode, beause rename is already success + renameOp.UnlinkSrcParentInode(); + renameOp.UnlinkOldInode(); + if (parent != newparent) { + renameOp.UpdateInodeParent(); + } + renameOp.UpdateInodeCtime(); + if (enableSumInDir_.load()) { + xattrManager_->UpdateParentXattrAfterRename( + parent, newparent, newname, &renameOp); + } + } else { + LOG(ERROR) << "FuseOpRename not support tx version: " + << common::FLAGS_TxVersion; + return CURVEFS_ERROR::NOT_SUPPORT; + } return rc; } @@ -958,7 +963,7 @@ CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi, AttrOut* attrOut) { - CURVEFS_ERROR rc = fs_->GetAttr(req, ino, attrOut); + CURVEFS_ERROR rc = fs_->GetAttr(ino, attrOut); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "getattr() fail, retCode = " << rc << ", ino = " << ino; @@ -1034,8 +1039,8 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, // update parent summary info const Inode* inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, - std::to_string(std::abs(changeSize))}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(std::abs(changeSize))}); bool direction = changeSize > 0; for (const auto &it : inode->parent()) { auto tret = xattrManager_->UpdateParentInodeXattr( @@ -1063,8 +1068,9 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, (void)req; VLOG(9) << "FuseOpGetXattr, ino: " << ino << ", name: " << name << ", size = " << size; - if (option_.fileSystemOption.disableXattr) { - return CURVEFS_ERROR::NOSYS; + + if (option_.fileSystemOption.disableXAttr && !IsSpecialXAttr(name)) { + return CURVEFS_ERROR::NODATA; } InodeAttr inodeAttr; @@ -1106,6 +1112,11 @@ CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino, VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name << ", size = " << size << ", strvalue: " << strvalue; + + if (option_.fileSystemOption.disableXAttr && !IsSpecialXAttr(name)) { + return CURVEFS_ERROR::NODATA; + } + if (strname.length() > MAX_XATTR_NAME_LENGTH || size > MAX_XATTR_VALUE_LENGTH) { LOG(ERROR) << "xattr length is too long, name = " << name @@ -1148,35 +1159,42 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino, // get xattr key for (const auto &it : inodeAttr.xattr()) { + if (option_.fileSystemOption.disableXAttr && + !IsSpecialXAttr(it.first)) { + continue; + } // +1 because, the format is key\0key\0 *realSize += it.first.length() + 1; } - // add summary xattr key if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) { - *realSize += strlen(XATTRRFILES) + 1; - *realSize += strlen(XATTRRSUBDIRS) + 1; - *realSize += strlen(XATTRRENTRIES) + 1; - *realSize += strlen(XATTRRFBYTES) + 1; + *realSize += strlen(XATTR_DIR_RFILES) + 1; + *realSize += strlen(XATTR_DIR_RSUBDIRS) + 1; + *realSize += strlen(XATTR_DIR_RENTRIES) + 1; + *realSize += strlen(XATTR_DIR_RFBYTES) + 1; } if (size == 0) { return CURVEFS_ERROR::OK; } else if (size >= *realSize) { for (const auto &it : inodeAttr.xattr()) { + if (option_.fileSystemOption.disableXAttr && + !IsSpecialXAttr(it.first)) { + continue; + } auto tsize = it.first.length() + 1; memcpy(value, it.first.c_str(), tsize); value += tsize; } if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) { - memcpy(value, XATTRRFILES, strlen(XATTRRFILES) + 1); - value += strlen(XATTRRFILES) + 1; - memcpy(value, XATTRRSUBDIRS, strlen(XATTRRSUBDIRS) + 1); - value += strlen(XATTRRSUBDIRS) + 1; - memcpy(value, XATTRRENTRIES, strlen(XATTRRENTRIES) + 1); - value += strlen(XATTRRENTRIES) + 1; - memcpy(value, XATTRRFBYTES, strlen(XATTRRFBYTES) + 1); - value += strlen(XATTRRFBYTES) + 1; + memcpy(value, XATTR_DIR_RFILES, strlen(XATTR_DIR_RFILES) + 1); + value += strlen(XATTR_DIR_RFILES) + 1; + memcpy(value, XATTR_DIR_RSUBDIRS, strlen(XATTR_DIR_RSUBDIRS) + 1); + value += strlen(XATTR_DIR_RSUBDIRS) + 1; + memcpy(value, XATTR_DIR_RENTRIES, strlen(XATTR_DIR_RENTRIES) + 1); + value += strlen(XATTR_DIR_RENTRIES) + 1; + memcpy(value, XATTR_DIR_RFBYTES, strlen(XATTR_DIR_RFBYTES) + 1); + value += strlen(XATTR_DIR_RFBYTES) + 1; } return CURVEFS_ERROR::OK; } @@ -1189,7 +1207,7 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char* name, EntryOut* entryOut) { if (strlen(name) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } const struct fuse_ctx *ctx = fuse_req_ctx(req); InodeParam param; @@ -1231,24 +1249,13 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, return ret; } - ret = UpdateParentMCTimeAndNlink(parent, FsFileType::TYPE_SYM_LINK, - NlinkChange::kAddOne); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "UpdateParentMCTimeAndNlink failed" - << ", link:" << link - << ", parent: " << parent - << ", name: " << name - << ", type: " << FsFileType::TYPE_SYM_LINK; - return ret; - } - if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { LOG(ERROR) << "UpdateParentInodeXattr failed," @@ -1268,7 +1275,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, FsFileType type, EntryOut* entryOut) { if (strlen(newname) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } std::shared_ptr inodeWrapper; CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper); @@ -1303,22 +1310,13 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, return ret; } - ret = UpdateParentMCTimeAndNlink(newparent, type, NlinkChange::kAddOne); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "UpdateParentMCTimeAndNlink failed" - << ", parent: " << newparent - << ", name: " << newname - << ", type: " << type; - return ret; - } - if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr( newparent, xattr, true); if (tret != CURVEFS_ERROR::OK) { @@ -1335,7 +1333,6 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino, std::string *linkStr) { (void)req; - VLOG(1) << "FuseOpReadLink, ino: " << ino << ", linkStr: " << linkStr; InodeAttr attr; CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &attr); if (ret != CURVEFS_ERROR::OK) { @@ -1344,13 +1341,14 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino, return ret; } *linkStr = attr.symlink(); + VLOG(1) << "FuseOpReadLink, ino: " << ino << ", linkStr: " << *linkStr; return CURVEFS_ERROR::OK; } CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) { - CURVEFS_ERROR rc = fs_->Release(req, ino); + CURVEFS_ERROR rc = fs_->Release(ino); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "release() failed, ino = " << ino; } diff --git a/curvefs/src/client/fuse_client.h b/curvefs/src/client/fuse_client.h index 3988b77866..aae53041cb 100644 --- a/curvefs/src/client/fuse_client.h +++ b/curvefs/src/client/fuse_client.h @@ -23,51 +23,55 @@ #ifndef CURVEFS_SRC_CLIENT_FUSE_CLIENT_H_ #define CURVEFS_SRC_CLIENT_FUSE_CLIENT_H_ -#include -#include #include +#include +#include +#include +#include #include #include #include -#include +#include #include #include -#include #include "curvefs/proto/common.pb.h" #include "curvefs/proto/mds.pb.h" +#include "curvefs/src/client/client_operator.h" +#include "curvefs/src/client/common/common.h" #include "curvefs/src/client/common/config.h" -#include "curvefs/src/client/dentry_cache_manager.h" +#include "curvefs/src/client/dentry_manager.h" #include "curvefs/src/client/dir_buffer.h" +#include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/filesystem/filesystem.h" +#include "curvefs/src/client/filesystem/meta.h" #include "curvefs/src/client/fuse_common.h" -#include "curvefs/src/client/inode_cache_manager.h" +#include "curvefs/src/client/inode_manager.h" +#include "curvefs/src/client/lease/lease_excutor.h" +#include "curvefs/src/client/metric/client_metric.h" #include "curvefs/src/client/rpcclient/mds_client.h" #include "curvefs/src/client/rpcclient/metaserver_client.h" #include "curvefs/src/client/s3/client_s3_adaptor.h" -#include "curvefs/src/common/fast_align.h" -#include "curvefs/src/client/metric/client_metric.h" -#include "src/common/concurrent/concurrent.h" +#include "curvefs/src/client/warmup/warmup_manager.h" +#include "curvefs/src/client/xattr_manager.h" #include "curvefs/src/common/define.h" +#include "curvefs/src/common/fast_align.h" #include "curvefs/src/common/s3util.h" -#include "curvefs/src/client/common/common.h" -#include "curvefs/src/client/client_operator.h" -#include "curvefs/src/client/lease/lease_excutor.h" -#include "curvefs/src/client/xattr_manager.h" -#include "curvefs/src/client/warmup/warmup_manager.h" +#include "src/common/concurrent/concurrent.h" +#include "src/common/configuration.h" #include "src/common/throttle.h" -#include "curvefs/src/client/filesystem/meta.h" -#include "curvefs/src/client/filesystem/filesystem.h" #define DirectIOAlignment 512 using ::curve::common::Atomic; using ::curve::common::InterruptibleSleeper; using ::curve::common::Thread; +using ::curvefs::client::filesystem::CURVEFS_ERROR; +using ::curvefs::client::metric::FSMetric; using ::curvefs::common::FSType; using ::curvefs::metaserver::DentryFlag; using ::curvefs::metaserver::ManageInodeType; -using ::curvefs::client::metric::FSMetric; namespace curvefs { namespace client { @@ -78,18 +82,22 @@ class WarmupManager; } using common::FuseClientOption; +using ::curve::common::Configuration; +using ::curvefs::client::filesystem::AttrOut; +using ::curvefs::client::filesystem::EntryOut; +using ::curvefs::client::filesystem::FileOut; +using ::curvefs::client::filesystem::FileSystem; using rpcclient::MDSBaseClient; using rpcclient::MdsClient; using rpcclient::MdsClientImpl; using rpcclient::MetaServerClient; using rpcclient::MetaServerClientImpl; -using ::curvefs::client::filesystem::FileSystem; -using ::curvefs::client::filesystem::EntryOut; -using ::curvefs::client::filesystem::AttrOut; -using ::curvefs::client::filesystem::FileOut; using curvefs::common::is_aligned; +using Filepath2WarmupProgressMap = + std::unordered_map; + const uint32_t kMaxHostNameLength = 255u; using mds::Mountpoint; @@ -290,13 +298,9 @@ class FuseClient { } } - std::shared_ptr GetFsInfo() { - return fsInfo_; - } + std::shared_ptr GetFsInfo() { return fsInfo_; } - std::shared_ptr GetFileSystem() { - return fs_; - } + virtual std::shared_ptr GetFileSystem() { return fs_; } virtual void FlushAll(); @@ -305,9 +309,13 @@ class FuseClient { enableSumInDir_ = enable; } - bool PutWarmFilelistTask(fuse_ino_t key, common::WarmupStorageType type) { + bool PutWarmFilelistTask(fuse_ino_t key, common::WarmupStorageType type, + const std::string& path, + const std::string& mount_point, + const std::string& root) { if (fsInfo_->fstype() == FSType::TYPE_S3) { - return warmupManager_->AddWarmupFilelist(key, type); + return warmupManager_->AddWarmupFilelist(key, type, path, + mount_point, root); } // only support s3 return true; } @@ -320,6 +328,13 @@ class FuseClient { return true; } + bool RemoveWarmFileOrFilelistTask(fuse_ino_t key) { + if (fsInfo_->fstype() == FSType::TYPE_S3) { + return warmupManager_->CancelWarmupFileOrFilelist(key); + } // only support s3 + return true; + } + bool GetWarmupProgress(fuse_ino_t key, warmup::WarmupProgress *progress) { if (fsInfo_->fstype() == FSType::TYPE_S3) { return warmupManager_->QueryWarmupProgress(key, progress); @@ -327,6 +342,13 @@ class FuseClient { return false; } + bool GetAllWarmupProgress(Filepath2WarmupProgressMap* filepath2progress) { + if (fsInfo_->fstype() == FSType::TYPE_S3) { + return warmupManager_->ListWarmupProgress(filepath2progress); + } + return false; + } + CURVEFS_ERROR SetMountStatus(const struct MountOption *mountOption); void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); } @@ -393,9 +415,6 @@ class FuseClient { private: virtual void FlushData() = 0; - CURVEFS_ERROR UpdateParentMCTimeAndNlink( - fuse_ino_t parent, FsFileType type, NlinkChange nlink); - std::string GenerateNewRecycleName(fuse_ino_t ino, fuse_ino_t parent, const char* name) { std::string newName(name); diff --git a/curvefs/src/client/fuse_common.h b/curvefs/src/client/fuse_common.h index 280d96aa16..d033afc6bc 100644 --- a/curvefs/src/client/fuse_common.h +++ b/curvefs/src/client/fuse_common.h @@ -26,8 +26,9 @@ #define FUSE_USE_VERSION 34 -#include +#include #include +#include #ifdef __cplusplus extern "C" { diff --git a/curvefs/src/client/fuse_s3_client.cpp b/curvefs/src/client/fuse_s3_client.cpp index e94fe31e2c..ab2524f76a 100644 --- a/curvefs/src/client/fuse_s3_client.cpp +++ b/curvefs/src/client/fuse_s3_client.cpp @@ -20,12 +20,16 @@ * Author: xuchaojie */ +#include "curvefs/src/client/fuse_s3_client.h" +#include #include #include -#include "curvefs/src/client/fuse_s3_client.h" +#include "curvefs/src/client/filesystem/xattr.h" #include "curvefs/src/client/kvclient/memcache_client.h" +#include "curvefs/src/client/rpcclient/fsdelta_updater.h" +#include "curvefs/src/client/rpcclient/fsquota_checker.h" namespace curvefs { namespace client { @@ -41,8 +45,9 @@ DECLARE_bool(supportKVcache); namespace curvefs { namespace client { -using curvefs::client::common::FLAGS_supportKVcache; using curvefs::client::common::FLAGS_enableCto; +using curvefs::client::common::FLAGS_supportKVcache; +using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; using curvefs::mds::topology::MemcacheClusterInfo; using curvefs::mds::topology::MemcacheServerInfo; @@ -74,7 +79,7 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) { LOG(ERROR) << "writeCacheMaxByte is too small" << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)" ", writeCacheMaxByte = " << writeCacheMaxByte; - return CURVEFS_ERROR::CACHETOOSMALL; + return CURVEFS_ERROR::CACHE_TOO_SMALL; } auto fsCacheManager = std::make_shared( @@ -100,6 +105,8 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) { inodeManager_, mdsClient_, fsCacheManager, nullptr, kvClientManager_, true); } + ioLatencyMetric_ = absl::make_unique( + fsInfo_->fsname()); return ret; } @@ -116,14 +123,14 @@ bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) { // init kvcache client auto memcacheClient = std::make_shared(); - if (!memcacheClient->Init(kvcachecluster)) { + if (!memcacheClient->Init(kvcachecluster, fsInfo_->fsname())) { LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache << ", but init memcache client fail"; return false; } kvClientManager_ = std::make_shared(); - if (!kvClientManager_->Init(opt, memcacheClient)) { + if (!kvClientManager_->Init(opt, memcacheClient, fsInfo_->fsname())) { LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache << ", but init kvClientManager fail"; return false; @@ -161,7 +168,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, if (fi->flags & O_DIRECT) { if (!(is_aligned(off, DirectIOAlignment) && is_aligned(size, DirectIOAlignment))) - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } uint64_t start = butil::cpuwide_time_us(); int wRet = s3Adaptor_->Write(ino, off, size, buf); @@ -169,14 +176,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, LOG(ERROR) << "s3Adaptor_ write failed, ret = " << wRet; return CURVEFS_ERROR::INTERNAL; } - - if (fsMetric_.get() != nullptr) { - fsMetric_->userWrite.bps.count << wRet; - fsMetric_->userWrite.qps.count << 1; - uint64_t duration = butil::cpuwide_time_us() - start; - fsMetric_->userWrite.latency << duration; - fsMetric_->userWriteIoSize.set_value(wRet); - } + uint64_t mid = butil::cpuwide_time_us(); + ioLatencyMetric_->writeDataLatency << mid - start; std::shared_ptr inodeWrapper; CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper); @@ -194,6 +195,9 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, if (inodeWrapper->GetLengthLocked() < off + *wSize) { changeSize = off + *wSize - inodeWrapper->GetLengthLocked(); inodeWrapper->SetLengthLocked(off + *wSize); + if (!FsQuotaChecker::GetInstance().QuotaBytesCheck(changeSize)) { + return CURVEFS_ERROR::NO_SPACE; + } } inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime); @@ -207,8 +211,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, if (enableSumInDir_ && changeSize != 0) { const Inode* inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, - std::to_string(changeSize)}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(changeSize)}); for (const auto &it : inode->parent()) { auto tret = xattrManager_->UpdateParentInodeXattr(it, xattr, true); if (tret != CURVEFS_ERROR::OK) { @@ -219,7 +223,18 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, } } + FsDeltaUpdater::GetInstance().UpdateDeltaBytes(changeSize); + inodeWrapper->GetInodeAttrLocked(&fileOut->attr); + + if (fsMetric_.get() != nullptr) { + fsMetric_->userWrite.bps.count << wRet; + fsMetric_->userWrite.qps.count << 1; + uint64_t duration = butil::cpuwide_time_us() - start; + fsMetric_->userWrite.latency << duration; + fsMetric_->userWriteIoSize.set_value(wRet); + } + ioLatencyMetric_->writeAttrLatency << butil::cpuwide_time_us() - mid; return ret; } @@ -232,7 +247,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, if (fi->flags & O_DIRECT) { if (!(is_aligned(off, DirectIOAlignment) && is_aligned(size, DirectIOAlignment))) - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } uint64_t start = butil::cpuwide_time_us(); @@ -243,6 +258,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, << ", inodeid = " << ino; return ret; } + uint64_t mid = butil::cpuwide_time_us(); + ioLatencyMetric_->readAttrLatency << mid - start; uint64_t fileSize = inodeWrapper->GetLength(); size_t len = 0; @@ -262,6 +279,11 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, return CURVEFS_ERROR::INTERNAL; } *rSize = rRet; + ioLatencyMetric_->readDataLatency << butil::cpuwide_time_us() - mid; + + ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock(); + inodeWrapper->UpdateTimestampLocked(kAccessTime); + inodeManager_->ShipToFlush(inodeWrapper); if (fsMetric_.get() != nullptr) { fsMetric_->userRead.bps.count << rRet; @@ -271,10 +293,6 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, fsMetric_->userReadIoSize.set_value(rRet); } - ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock(); - inodeWrapper->UpdateTimestampLocked(kAccessTime); - inodeManager_->ShipToFlush(inodeWrapper); - VLOG(9) << "read end, read size = " << *rSize; return ret; } diff --git a/curvefs/src/client/fuse_s3_client.h b/curvefs/src/client/fuse_s3_client.h index 68d7bf5d08..1b4b4a70b4 100644 --- a/curvefs/src/client/fuse_s3_client.h +++ b/curvefs/src/client/fuse_s3_client.h @@ -24,13 +24,14 @@ #ifndef CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_ #define CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_ +#include #include #include -#include -#include #include +#include #include "curvefs/src/client/fuse_client.h" +#include "curvefs/src/client/metric/client_metric.h" #include "curvefs/src/client/s3/client_s3_cache_manager.h" #include "curvefs/src/client/warmup/warmup_manager.h" #include "curvefs/src/volume/common.h" @@ -53,13 +54,17 @@ class FuseS3Client : public FuseClient { FuseS3Client() : FuseClient(), s3Adaptor_(std::make_shared()) { auto readFunc = [this](fuse_req_t req, fuse_ino_t ino, size_t size, - off_t off, struct fuse_file_info *fi, - char *buffer, size_t *rSize) { + off_t off, struct fuse_file_info* fi, + char* buffer, size_t* rSize) { return FuseOpRead(req, ino, size, off, fi, buffer, rSize); }; + auto readLinkFunc = [this](fuse_req_t req, fuse_ino_t ino, + std::string* linkStr) { + return FuseClient::FuseOpReadLink(req, ino, linkStr); + }; warmupManager_ = std::make_shared( metaClient_, inodeManager_, dentryManager_, fsInfo_, readFunc, - s3Adaptor_, nullptr); + readLinkFunc, nullptr, s3Adaptor_); } FuseS3Client(const std::shared_ptr &mdsClient, @@ -129,6 +134,7 @@ class FuseS3Client : public FuseClient { // s3 adaptor std::shared_ptr s3Adaptor_; std::shared_ptr kvClientManager_; + std::unique_ptr ioLatencyMetric_; static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB; }; diff --git a/curvefs/src/client/fuse_volume_client.cpp b/curvefs/src/client/fuse_volume_client.cpp index fc270a0282..26bcecdaa7 100644 --- a/curvefs/src/client/fuse_volume_client.cpp +++ b/curvefs/src/client/fuse_volume_client.cpp @@ -150,7 +150,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req, if (!(is_aligned(off, DirectIOAlignment) && is_aligned(size, DirectIOAlignment))) { fsMetric_->userWrite.eps.count << 1; - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } } @@ -208,7 +208,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req, is_aligned(size, DirectIOAlignment))) { fsMetric_->userRead.eps.count << 1; - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } } diff --git a/curvefs/src/client/inode_cache_manager.cpp b/curvefs/src/client/inode_manager.cpp similarity index 80% rename from curvefs/src/client/inode_cache_manager.cpp rename to curvefs/src/client/inode_manager.cpp index b71ac2ffa7..d3aa1c9fac 100644 --- a/curvefs/src/client/inode_cache_manager.cpp +++ b/curvefs/src/client/inode_manager.cpp @@ -21,7 +21,7 @@ * Author: xuchaojie */ -#include "curvefs/src/client/inode_cache_manager.h" +#include "curvefs/src/client/inode_manager.h" #include #include @@ -30,6 +30,8 @@ #include #include "curvefs/proto/metaserver.pb.h" #include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/filesystem/utils.h" +#include "curvefs/src/client/filesystem/defer_sync.h" #include "curvefs/src/client/inode_wrapper.h" using ::curvefs::metaserver::Inode; @@ -47,10 +49,73 @@ namespace curvefs { namespace client { using ::curvefs::client::filesystem::ToFSError; +using ::curvefs::client::filesystem::AttrCtime; using NameLockGuard = ::curve::common::GenericNameLockGuard; using curvefs::client::common::FLAGS_enableCto; +#define RETURN_IF_CTO_ON() \ + do { \ + if (cto_) { \ + return; \ + } \ + } while (0) + +DeferWatcher::DeferWatcher(bool cto, std::shared_ptr deferSync) + : cto_(cto), + deferSync_(deferSync), + deferAttrs_() {} + +void DeferWatcher::PreGetAttrs(const std::set& inos) { + RETURN_IF_CTO_ON(); + InodeAttr attr; + std::shared_ptr inode; + for (const auto& ino : inos) { + bool yes = deferSync_->IsDefered(ino, &inode); + if (!yes) { + continue; + } + inode->GetInodeAttr(&attr); + deferAttrs_.emplace(ino, attr); + } +} + +bool DeferWatcher::TryUpdate(InodeAttr* attr) { + Ino ino = attr->inodeid(); + auto iter = deferAttrs_.find(ino); + if (iter == deferAttrs_.end()) { + return false; + } + + auto& defered = iter->second; + if (AttrCtime(*attr) > AttrCtime(defered)) { + return false; + } + *attr = defered; + return true; +} + +void DeferWatcher::PostGetAttrs(std::list* attrs) { + RETURN_IF_CTO_ON(); + if (deferAttrs_.size() == 0) { + return; + } + for (auto& attr : *attrs) { + TryUpdate(&attr); + } +} + +void DeferWatcher::PostGetAttrs(std::map* attrs) { + RETURN_IF_CTO_ON(); + if (deferAttrs_.size() == 0) { + return; + } + for (auto& item : *attrs) { + auto& attr = item.second; + TryUpdate(&attr); + } +} + #define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING) \ MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \ if (ret != MetaStatusCode::OK) { \ @@ -76,6 +141,11 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId, return CURVEFS_ERROR::OK; } + bool cto = FLAGS_enableCto; + if (!cto && deferSync_->IsDefered(inodeId, &out)) { + return CURVEFS_ERROR::OK; + } + // get inode from metaserver Inode inode; bool streaming = false; @@ -97,6 +167,11 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId, std::set inodeIds; std::list attrs; inodeIds.emplace(inodeId); + + bool cto = FLAGS_enableCto; + auto watcher = std::make_shared(cto, deferSync_); + watcher->PreGetAttrs(inodeIds); + MetaStatusCode ret = metaClient_->BatchGetInodeAttr(fsId_, inodeIds, &attrs); if (MetaStatusCode::OK != ret) { @@ -113,6 +188,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId, return CURVEFS_ERROR::INTERNAL; } + watcher->PostGetAttrs(&attrs); *out = *attrs.begin(); return CURVEFS_ERROR::OK; } @@ -124,6 +200,10 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr( return CURVEFS_ERROR::OK; } + bool cto = FLAGS_enableCto; + auto watcher = std::make_shared(cto, deferSync_); + watcher->PreGetAttrs(*inodeIds); + MetaStatusCode ret = metaClient_->BatchGetInodeAttr(fsId_, *inodeIds, attrs); if (MetaStatusCode::OK != ret) { @@ -131,6 +211,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr( << ret << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret); } + watcher->PostGetAttrs(attrs); return ToFSError(ret); } @@ -144,10 +225,14 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync( return CURVEFS_ERROR::OK; } + bool cto = FLAGS_enableCto; + auto watcher = std::make_shared(cto, deferSync_); + watcher->PreGetAttrs(*inodeIds); + // split inodeIds by partitionId and batch limit std::vector> inodeGroups; if (!metaClient_->SplitRequestInodes(fsId_, *inodeIds, &inodeGroups)) { - return CURVEFS_ERROR::NOTEXIST; + return CURVEFS_ERROR::NOT_EXIST; } ::curve::common::Mutex mutex; @@ -168,6 +253,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync( // wait for all sudrequest finished cond->Wait(); + watcher->PostGetAttrs(attrs); return CURVEFS_ERROR::OK; } diff --git a/curvefs/src/client/inode_cache_manager.h b/curvefs/src/client/inode_manager.h similarity index 91% rename from curvefs/src/client/inode_cache_manager.h rename to curvefs/src/client/inode_manager.h index ce0ebb3d17..093b5e750d 100644 --- a/curvefs/src/client/inode_cache_manager.h +++ b/curvefs/src/client/inode_manager.h @@ -21,8 +21,8 @@ * Author: xuchaojie */ -#ifndef CURVEFS_SRC_CLIENT_INODE_CACHE_MANAGER_H_ -#define CURVEFS_SRC_CLIENT_INODE_CACHE_MANAGER_H_ +#ifndef CURVEFS_SRC_CLIENT_INODE_MANAGER_H_ +#define CURVEFS_SRC_CLIENT_INODE_MANAGER_H_ #include #include @@ -46,6 +46,7 @@ #include "curvefs/src/client/common/config.h" #include "curvefs/src/client/filesystem/openfile.h" #include "curvefs/src/client/filesystem/defer_sync.h" +#include "absl/container/btree_map.h" using ::curve::common::LRUCache; using ::curve::common::CacheMetrics; @@ -66,6 +67,7 @@ using rpcclient::BatchGetInodeAttrDone; using curve::common::CountDownEvent; using metric::S3ChunkInfoMetric; using common::RefreshDataOption; +using ::curvefs::client::filesystem::Ino; using ::curvefs::client::filesystem::OpenFiles; using ::curvefs::client::filesystem::DeferSync; @@ -116,6 +118,25 @@ class InodeCacheManager { uint32_t fsId_; }; +class DeferWatcher { + public: + DeferWatcher(bool cto, std::shared_ptr deferSync); + + void PreGetAttrs(const std::set& inos); + + void PostGetAttrs(std::list* attrs); + + void PostGetAttrs(std::map* attrs); + + private: + bool TryUpdate(InodeAttr* attr); + + private: + bool cto_; + std::shared_ptr deferSync_; + absl::btree_map deferAttrs_; +}; + class InodeCacheManagerImpl : public InodeCacheManager, public std::enable_shared_from_this { public: @@ -221,4 +242,4 @@ class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone { } // namespace client } // namespace curvefs -#endif // CURVEFS_SRC_CLIENT_INODE_CACHE_MANAGER_H_ +#endif // CURVEFS_SRC_CLIENT_INODE_MANAGER_H_ diff --git a/curvefs/src/client/kvclient/kvclient.h b/curvefs/src/client/kvclient/kvclient.h index 9d7b96d857..a96c733762 100644 --- a/curvefs/src/client/kvclient/kvclient.h +++ b/curvefs/src/client/kvclient/kvclient.h @@ -22,6 +22,8 @@ #ifndef CURVEFS_SRC_CLIENT_KVCLIENT_KVCLIENT_H_ #define CURVEFS_SRC_CLIENT_KVCLIENT_KVCLIENT_H_ +#include + #include namespace curvefs { @@ -49,8 +51,9 @@ class KVClient { virtual bool Set(const std::string &key, const char *value, const uint64_t value_len, std::string *errorlog) = 0; - virtual bool Get(const std::string &key, char *value, uint64_t offset, - uint64_t length, std::string *errorlog) = 0; + virtual bool Get(const std::string& key, char* value, uint64_t offset, + uint64_t length, std::string* errorlog, + uint64_t* actLength, memcached_return_t* retCod) = 0; }; } // namespace client diff --git a/curvefs/src/client/kvclient/kvclient_manager.cpp b/curvefs/src/client/kvclient/kvclient_manager.cpp index 1691af6119..9e677cc212 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.cpp +++ b/curvefs/src/client/kvclient/kvclient_manager.cpp @@ -21,27 +21,38 @@ */ #include "curvefs/src/client/kvclient/kvclient_manager.h" + #include + +#include "absl/memory/memory.h" +#include "curvefs/src/client/metric/client_metric.h" #include "src/client/client_metric.h" #include "src/common/concurrent/count_down_event.h" using curve::client::LatencyGuard; using curve::common::CountDownEvent; -using curvefs::client::metric::KVClientMetric; +using curvefs::client::metric::KVClientManagerMetric; namespace curvefs { namespace client { -#define ONRETURN(TYPE, RES) \ - if (RES) { \ - kvClientMetric_.kvClient##TYPE.qps.count << 1; \ - } else { \ - kvClientMetric_.kvClient##TYPE.eps.count << 1; \ - } \ +template +void OnReturn(Metric* metric, const TaskSharePtr task) { + task->timer.stop(); + if (task->res) { + curve::client::CollectMetrics(metric, task->length, + task->timer.u_elapsed()); + } else { + metric->eps.count << 1; + } + task->done(task); +} -bool KVClientManager::Init(const KVClientManagerOpt &config, - const std::shared_ptr &kvclient) { +bool KVClientManager::Init(const KVClientManagerOpt& config, + const std::shared_ptr& kvclient, + const std::string& fsName) { client_ = kvclient; + kvClientManagerMetric_ = absl::make_unique(fsName); return threadPool_.Start(config.setThreadPooln) == 0; } @@ -51,30 +62,75 @@ void KVClientManager::Uninit() { } void KVClientManager::Set(std::shared_ptr task) { + kvClientManagerMetric_->setQueueSize << 1; threadPool_.Enqueue([task, this]() { - LatencyGuard guard(&kvClientMetric_.kvClientSet.latency); - std::string error_log; - auto res = + task->res = client_->Set(task->key, task->value, task->length, &error_log); - ONRETURN(Set, res); - - task->done(task); + kvClientManagerMetric_->setQueueSize << -1; + if (task->res) { + kvClientManagerMetric_->count << 1; + } + OnReturn(&kvClientManagerMetric_->set, task); }); } +void UpdateHitMissMetric(memcached_return_t retCode, + KVClientManagerMetric* metric) { + // https://awesomized.github.io/libmemcached/libmemcached/memcached_return_t.html + switch (retCode) { + case MEMCACHED_SUCCESS: + metric->hit << 1; + break; + case MEMCACHED_DATA_DOES_NOT_EXIST: + // The data requested with the key given was not found. + case MEMCACHED_DATA_EXISTS: + // The data requested with the key given was not found. + case MEMCACHED_DELETED: + // The object requested by the key has been deleted. + case MEMCACHED_NOTFOUND: + // The object requested was not found. + metric->miss << 1; + break; + default: + break; + } +} + void KVClientManager::Get(std::shared_ptr task) { + kvClientManagerMetric_->getQueueSize << 1; threadPool_.Enqueue([task, this]() { - LatencyGuard guard(&kvClientMetric_.kvClientGet.latency); - std::string error_log; + memcached_return_t retCode; task->res = client_->Get(task->key, task->value, task->offset, - task->length, &error_log); - ONRETURN(Get, task->res); - - task->done(task); + task->valueLength, &error_log, &task->length, + &retCode); + kvClientManagerMetric_->getQueueSize << -1; + UpdateHitMissMetric(retCode, kvClientManagerMetric_.get()); + OnReturn(&kvClientManagerMetric_->get, task); }); } +void KVClientManager::Enqueue(std::shared_ptr context) { + auto task = [this, context]() { this->GetKvCache(context); }; + threadPool_.Enqueue(task); +} + +int KVClientManager::GetKvCache( + std::shared_ptr context) { + VLOG(9) << "GetKvCache start: " << context->key; + std::string error_log; + memcached_return_t retCode; + uint64_t actLength = 0; + context->retCode = + !client_->Get(context->key, context->buf, context->offset, context->len, + &error_log, &actLength, &retCode); + context->actualLen = actLength; + context->cb(nullptr, context); + VLOG(9) << "GetKvCache end: " << context->key << ", " << context->retCode + << ", " << context->actualLen; + return 0; +} + } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/kvclient/kvclient_manager.h b/curvefs/src/client/kvclient/kvclient_manager.h index 7214a64b6e..bad99193a8 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.h +++ b/curvefs/src/client/kvclient/kvclient_manager.h @@ -25,57 +25,107 @@ #include -#include +#include #include -#include #include +#include +#include #include "absl/strings/string_view.h" -#include "curvefs/src/client/kvclient/kvclient.h" #include "curvefs/src/client/common/config.h" +#include "curvefs/src/client/kvclient/kvclient.h" #include "curvefs/src/client/metric/client_metric.h" #include "src/common/concurrent/task_thread_pool.h" #include "src/common/s3_adapter.h" -using curvefs::client::metric::KVClientMetric; +using curvefs::client::metric::KVClientManagerMetric; namespace curvefs { namespace client { class KVClientManager; -class SetKVCacheTask; -class GetKVCacheTask; +struct SetKVCacheTask; +struct GetKVCacheTask; + +class GetKvCacheContext; +class SetKvCacheContext; + +using curve::common::GetObjectAsyncContext; using curve::common::TaskThreadPool; using curvefs::client::common::KVClientManagerOpt; -typedef std::function &)> - SetKVCacheDone; -typedef std::function &)> - GetKVCacheDone; +using SetKVCacheDone = + std::function&)>; +using GetKVCacheDone = + std::function&)>; struct SetKVCacheTask { std::string key; - const char *value; + const char* value; uint64_t length; + bool res; SetKVCacheDone done; - SetKVCacheTask() = default; - SetKVCacheTask( - const std::string &k, const char *val, const uint64_t len, - SetKVCacheDone done = [](const std::shared_ptr &) {}) - : key(k), value(val), length(len), done(std::move(done)) {} + butil::Timer timer; + + explicit SetKVCacheTask( + const std::string& k, const char* val, const uint64_t len, + SetKVCacheDone done = [](const std::shared_ptr&) {}) + : key(k), + value(val), + length(len), + res(false), + done(std::move(done)), + timer(butil::Timer::STARTED) {} }; struct GetKVCacheTask { - const std::string &key; - char *value; + const std::string& key; + char* value; uint64_t offset; - uint64_t length; + uint64_t valueLength; + uint64_t length; // actual length of value bool res; GetKVCacheDone done; - GetKVCacheTask(const std::string &k, char *v, uint64_t off, uint64_t len) - : key(k), value(v), offset(off), length(len), res(false) { - done = [](const std::shared_ptr &) {}; - } + butil::Timer timer; + + explicit GetKVCacheTask( + const std::string& k, char* v, uint64_t off, uint64_t len, + GetKVCacheDone done = [](const std::shared_ptr&) {}) + : key(k), + value(v), + offset(off), + valueLength(len), + length(0), + res(false), + done(std::move(done)), + timer(butil::Timer::STARTED) {} +}; + +using GetKvCacheCallBack = + std::function&)>; + +using SetKvCacheCallBack = + std::function&)>; + +struct KvCacheContext { + std::string key; + uint64_t inodeId; + uint64_t offset; + uint64_t length; + uint64_t chunkIndex; + uint64_t chunkPos; + uint64_t startTime; +}; + +struct GetKvCacheContext : KvCacheContext { + char* value; + bool res; + GetKvCacheCallBack cb; +}; + +struct SetKvCacheContext : KvCacheContext { + const char* value; + SetKvCacheCallBack cb; }; class KVClientManager { @@ -83,8 +133,9 @@ class KVClientManager { KVClientManager() = default; ~KVClientManager() { Uninit(); } - bool Init(const KVClientManagerOpt &config, - const std::shared_ptr &kvclient); + bool Init(const KVClientManagerOpt& config, + const std::shared_ptr& kvclient, + const std::string& fsName); /** * It will get a db client and set the key value asynchronusly. @@ -95,17 +146,22 @@ class KVClientManager { void Get(std::shared_ptr task); - KVClientMetric *GetClientMetricForTesting() { return &kvClientMetric_; } + KVClientManagerMetric* GetMetricForTesting() { + return kvClientManagerMetric_.get(); + } + + void Enqueue(std::shared_ptr context); private: void Uninit(); + int GetKvCache(std::shared_ptr context); private: TaskThreadPool threadPool_; std::shared_ptr client_; - KVClientMetric kvClientMetric_; + std::unique_ptr kvClientManagerMetric_; }; } // namespace client } // namespace curvefs -#endif // CURVEFS_SRC_CLIENT_KVCLIENT_KVCLIENT_MANAGER_H_ +#endif // CURVEFS_SRC_CLIENT_KVCLIENT_KVCLIENT_MANAGER_H_ diff --git a/curvefs/src/client/kvclient/memcache_client.h b/curvefs/src/client/kvclient/memcache_client.h index 82c11347ae..7ca3ea1248 100644 --- a/curvefs/src/client/kvclient/memcache_client.h +++ b/curvefs/src/client/kvclient/memcache_client.h @@ -27,10 +27,14 @@ #include #include +#include +#include #include -#include "curvefs/src/client/kvclient/kvclient.h" +#include "absl/memory/memory.h" #include "curvefs/proto/topology.pb.h" +#include "curvefs/src/client/kvclient/kvclient.h" +#include "curvefs/src/client/metric/client_metric.h" namespace curvefs { @@ -83,7 +87,9 @@ class MemCachedClient : public KVClient { explicit MemCachedClient(memcached_st *cli) : client_(cli) {} ~MemCachedClient() { UnInit(); } - bool Init(const MemcacheClusterInfo &kvcachecluster) { + bool Init(const MemcacheClusterInfo& kvcachecluster, + const std::string& fsName) { + metric_ = absl::make_unique(fsName); client_ = memcached(nullptr, 0); for (int i = 0; i < kvcachecluster.servers_size(); i++) { @@ -106,8 +112,9 @@ class MemCachedClient : public KVClient { } } - bool Set(const std::string &key, const char *value, - const uint64_t value_len, std::string *errorlog) override { + bool Set(const std::string& key, const char* value, + const uint64_t value_len, std::string* errorlog) override { + uint64_t start = butil::cpuwide_time_us(); if (nullptr == tcli) { tcli = memcached_clone(nullptr, client_); } @@ -115,17 +122,22 @@ class MemCachedClient : public KVClient { value_len, 0, 0); if (MEMCACHED_SUCCESS == res) { VLOG(9) << "Set key = " << key << " OK"; + curve::client::CollectMetrics(&metric_->set, value_len, + butil::cpuwide_time_us() - start); return true; } *errorlog = ResError(res); memcached_free(tcli); tcli = nullptr; LOG(ERROR) << "Set key = " << key << " error = " << *errorlog; + metric_->set.eps.count << 1; return false; } - bool Get(const std::string &key, char *value, uint64_t offset, - uint64_t length, std::string *errorlog) override { + bool Get(const std::string& key, char* value, uint64_t offset, + uint64_t length, std::string* errorlog, uint64_t* actLength, + memcached_return_t* retCode) override { + uint64_t start = butil::cpuwide_time_us(); if (nullptr == tcli) { // multi thread use a memcached_st* client is unsafe. // should clone it or use memcached_st_pool. @@ -136,23 +148,33 @@ class MemCachedClient : public KVClient { memcached_return_t ue; char *res = memcached_get(tcli, key.c_str(), key.length(), &value_length, &flags, &ue); + if (actLength != nullptr) { + (*actLength) = value_length; + } + if (retCode != nullptr) { + (*retCode) = ue; + } if (MEMCACHED_SUCCESS == ue && res != nullptr && value && value_length >= length) { VLOG(9) << "Get key = " << key << " OK"; memcpy(value, res + offset, length); free(res); + curve::client::CollectMetrics(&metric_->get, value_length, + butil::cpuwide_time_us() - start); return true; } *errorlog = ResError(ue); if (ue != MEMCACHED_NOTFOUND) { - LOG(ERROR) << "Get key = " << key << " error = " << *errorlog - << ", get_value_len = " << value_length - << ", expect_value_len = " << length; + LOG_EVERY_N(WARNING, 1000) << "Get key = " << key + << " error = " << *errorlog << ", get_value_len = " + << value_length << ", expect_value_len = " << length; + free(res); memcached_free(tcli); tcli = nullptr; } + metric_->get.eps.count << 1; return false; } @@ -198,7 +220,8 @@ class MemCachedClient : public KVClient { private: memcached_server_st *server_; - memcached_st *client_; + memcached_st* client_; + std::unique_ptr metric_; }; } // namespace client diff --git a/curvefs/src/client/lease/lease_excutor.cpp b/curvefs/src/client/lease/lease_excutor.cpp index 08dc400c88..13972db561 100644 --- a/curvefs/src/client/lease/lease_excutor.cpp +++ b/curvefs/src/client/lease/lease_excutor.cpp @@ -31,6 +31,10 @@ using curvefs::mds::topology::PartitionTxId; namespace curvefs { namespace client { +namespace common { +DECLARE_int32(TxVersion); +} // namespace common + LeaseExecutor::~LeaseExecutor() { if (task_) { task_->Stop(); @@ -71,26 +75,32 @@ void LeaseExecutor::Stop() { } bool LeaseExecutor::RefreshLease() { + // for tx v2 txIds and latestTxIdList will empty here // get partition txid list std::vector txIds; - metaCache_->GetAllTxIds(&txIds); - + if (common::FLAGS_TxVersion == 1) { + metaCache_->GetAllTxIds(&txIds); + } // refresh from mds std::vector latestTxIdList; - FSStatusCode ret = mdsCli_->RefreshSession(txIds, &latestTxIdList, - fsName_, mountpoint_, - enableSumInDir_); + std::string mdsAddrs = mdsCli_->GetMdsAddrs(); + std::string mdsAddrsOverride; + FSStatusCode ret = + mdsCli_->RefreshSession(txIds, &latestTxIdList, fsName_, mountpoint_, + enableSumInDir_, mdsAddrs, &mdsAddrsOverride); + if (ret != FSStatusCode::OK) { LOG(ERROR) << "LeaseExecutor refresh session fail, ret = " << ret << ", errorName = " << FSStatusCode_Name(ret); return true; } - // update to metacache std::for_each(latestTxIdList.begin(), latestTxIdList.end(), [&](const PartitionTxId &item) { metaCache_->SetTxId(item.partitionid(), item.txid()); }); + // update mds addrs + mdsCli_->SetMdsAddrs(mdsAddrsOverride); return true; } diff --git a/curvefs/src/client/logger/BUILD b/curvefs/src/client/logger/BUILD new file mode 100644 index 0000000000..e5a115c166 --- /dev/null +++ b/curvefs/src/client/logger/BUILD @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_library( + name = "logger", + srcs = glob(["*.cpp"]), + hdrs = glob(["*.h"]), + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//external:glog", + "@spdlog//:spdlog", + "@com_google_absl//absl/strings:str_format", + "//curvefs/src/common:dynamic_vlog", + "//curvefs/src/client/common:common", + ], +) diff --git a/curvefs/src/client/logger/access_log.cpp b/curvefs/src/client/logger/access_log.cpp new file mode 100644 index 0000000000..740c0e812c --- /dev/null +++ b/curvefs/src/client/logger/access_log.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-14 + * Author: Jingli Chen (Wine93) + */ + +#include "curvefs/src/client/logger/access_log.h" + +namespace curvefs { +namespace client { +namespace logger { + +std::shared_ptr Logger; +bool inited = false; + +bool InitAccessLog(const std::string& prefix) { + if (inited) { + return true; + } + + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + std::string name = StrFormat("access-%d-%d", now.tv_sec, now.tv_nsec); + + std::string filename = StrFormat("%s/access.%d.%d.log", + prefix, getpid(), now.tv_sec); + Logger = spdlog::daily_logger_mt(name, filename, 0, 0); + spdlog::flush_every(std::chrono::seconds(1)); + inited = true; + return true; +} + +} // namespace logger +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/filesystem/access_log.h b/curvefs/src/client/logger/access_log.h similarity index 75% rename from curvefs/src/client/filesystem/access_log.h rename to curvefs/src/client/logger/access_log.h index 1e2067c9b1..5c9e075087 100644 --- a/curvefs/src/client/filesystem/access_log.h +++ b/curvefs/src/client/logger/access_log.h @@ -20,6 +20,8 @@ * Author: Jingli Chen (Wine93) */ +// clang-format off + #include #include #include @@ -32,8 +34,8 @@ #include "absl/strings/str_format.h" #include "curvefs/src/client/common/config.h" -#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_ -#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_ +#ifndef CURVEFS_SRC_CLIENT_LOGGER_ACCESS_LOG_H_ +#define CURVEFS_SRC_CLIENT_LOGGER_ACCESS_LOG_H_ namespace curvefs { namespace client { @@ -42,24 +44,20 @@ namespace common { DECLARE_bool(access_logging); } -namespace filesystem { +namespace logger { using ::absl::StrFormat; using ::curvefs::client::common::FLAGS_access_logging; using MessageHandler = std::function; -static std::shared_ptr Logger; +extern std::shared_ptr Logger; +extern bool inited; -bool InitAccessLog(const std::string& prefix) { - std::string filename = StrFormat("%s/access.%d.log", prefix, getpid()); - Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0); - spdlog::flush_every(std::chrono::seconds(1)); - return true; -} +bool InitAccessLog(const std::string& prefix); struct AccessLogGuard { explicit AccessLogGuard(MessageHandler handler) - : enable(FLAGS_access_logging), + : enable(FLAGS_access_logging && inited), handler(handler) { if (!enable) { return; @@ -82,8 +80,8 @@ struct AccessLogGuard { butil::Timer timer; }; -} // namespace filesystem +} // namespace logger } // namespace client } // namespace curvefs -#endif // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_ +#endif // CURVEFS_SRC_CLIENT_LOGGER_ACCESS_LOG_H_ diff --git a/curvefs/src/client/logger/error_log.h b/curvefs/src/client/logger/error_log.h new file mode 100644 index 0000000000..a259ce755a --- /dev/null +++ b/curvefs/src/client/logger/error_log.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-03 + * Author: Jingli Chen (Wine93) + */ + +// clang-format off + +#include + +#include +#include +#include + +#include "curvefs/src/common/dynamic_vlog.h" + +#ifndef CURVEFS_SRC_CLIENT_LOGGER_ERROR_LOG_H_ +#define CURVEFS_SRC_CLIENT_LOGGER_ERROR_LOG_H_ + +namespace curvefs { +namespace client { +namespace logger { + +using ::curvefs::common::FLAGS_vlog_level; + +inline bool InitErrorLog(const std::string& prefix, + const std::string& name, + int32_t loglevel) { + FLAGS_log_dir = prefix; + FLAGS_v = loglevel; + FLAGS_vlog_level = loglevel; + FLAGS_minloglevel = 0; + FLAGS_logtostderr = 0; + FLAGS_alsologtostderr = 0; + google::InitGoogleLogging(name.c_str()); + return true; +} + +inline void ShutdownErrorLog() { google::ShutdownGoogleLogging(); } + +} // namespace logger +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_LOGGER_ERROR_LOG_H_ diff --git a/curvefs/src/client/metric/BUILD b/curvefs/src/client/metric/BUILD index 9fc0cdd625..d94ce1b16c 100644 --- a/curvefs/src/client/metric/BUILD +++ b/curvefs/src/client/metric/BUILD @@ -26,5 +26,6 @@ cc_library( "//external:gflags", "//external:bvar", "//src/client:curve_client", + "//src/common:curve_s3_adapter", ], ) diff --git a/curvefs/src/client/metric/client_metric.cpp b/curvefs/src/client/metric/client_metric.cpp index 0cd5e8d299..d6c2592ebb 100644 --- a/curvefs/src/client/metric/client_metric.cpp +++ b/curvefs/src/client/metric/client_metric.cpp @@ -22,10 +22,17 @@ #include "curvefs/src/client/metric/client_metric.h" +#include +#include + +#include "src/client/client_metric.h" + namespace curvefs { namespace client { namespace metric { +using curve::client::CollectMetrics; + const std::string MDSClientMetric::prefix = "curvefs_mds_client"; // NOLINT const std::string MetaServerClientMetric::prefix = "curvefs_metaserver_client"; // NOLINT const std::string ClientOpMetric::prefix = "curvefs_client"; // NOLINT @@ -33,11 +40,59 @@ const std::string S3MultiManagerMetric::prefix = "curvefs_client_manager"; // N const std::string FSMetric::prefix = "curvefs_client"; // NOLINT const std::string S3Metric::prefix = "curvefs_s3"; // NOLINT const std::string DiskCacheMetric::prefix = "curvefs_disk_cache"; // NOLINT -const std::string KVClientMetric::prefix = "curvefs_kvclient"; // NOLINT +const std::string KVClientManagerMetric::prefix = // NOLINT + "curvefs_kvclient_manager"; // NOLINT +const std::string MemcacheClientMetric::prefix = // NOLINT + "curvefs_memcache_client"; // NOLINT const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info"; // NOLINT -const std::string WarmupManagerS3Metric::prefix = "curvefs_warmup"; // NOLINT +const std::string WarmupManagerS3Metric::prefix = "curvefs_warmup"; // NOLINT +const std::string FuseS3ClientIOLatencyMetric::prefix = // NOLINT + "curvefs_fuse_s3_client"; // NOLINT + +void AsyncContextCollectMetrics( + std::shared_ptr s3Metric, + const std::shared_ptr& context) { + if (s3Metric.get() != nullptr) { + CollectMetrics(&s3Metric->adaptorWriteS3, context->bufferSize, + context->timer.u_elapsed()); + + switch (context->type) { + case curve::common::ContextType::Disk: + CollectMetrics(&s3Metric->writeToDiskCache, context->bufferSize, + context->timer.u_elapsed()); + break; + case curve::common::ContextType::S3: + CollectMetrics(&s3Metric->writeToS3, context->bufferSize, + context->timer.u_elapsed()); + break; + default: + break; + } + } +} + +void AsyncContextCollectMetrics( + std::shared_ptr s3Metric, + const std::shared_ptr& context) { + if (s3Metric.get() != nullptr) { + CollectMetrics(&s3Metric->adaptorReadS3, context->actualLen, + context->timer.u_elapsed()); + + switch (context->type) { + case curve::common::ContextType::Disk: + CollectMetrics(&s3Metric->readFromDiskCache, context->actualLen, + context->timer.u_elapsed()); + break; + case curve::common::ContextType::S3: + CollectMetrics(&s3Metric->readFromS3, context->actualLen, + context->timer.u_elapsed()); + break; + default: + break; + } + } +} } // namespace metric } // namespace client } // namespace curvefs - diff --git a/curvefs/src/client/metric/client_metric.h b/curvefs/src/client/metric/client_metric.h index 2787aa4d7f..f5140b896d 100644 --- a/curvefs/src/client/metric/client_metric.h +++ b/curvefs/src/client/metric/client_metric.h @@ -25,9 +25,14 @@ #define CURVEFS_SRC_CLIENT_METRIC_CLIENT_METRIC_H_ #include + +#include #include +#include #include + #include "src/client/client_metric.h" +#include "src/common/s3_adapter.h" using curve::client::InterfaceMetric; @@ -50,6 +55,7 @@ struct MDSClientMetric { InterfaceMetric refreshSession; InterfaceMetric getLatestTxId; InterfaceMetric commitTx; + InterfaceMetric tso; InterfaceMetric allocOrGetMemcacheCluster; MDSClientMetric() @@ -65,6 +71,7 @@ struct MDSClientMetric { refreshSession(prefix, "refreshSession"), getLatestTxId(prefix, "getLatestTxId"), commitTx(prefix, "commitTx"), + tso(prefix, "tso"), allocOrGetMemcacheCluster(prefix, "allocOrGetMemcacheCluster") {} }; @@ -88,6 +95,10 @@ struct MetaServerClientMetric { // tnx InterfaceMetric prepareRenameTx; + InterfaceMetric prewriteRenameTx; + InterfaceMetric checkTxStatus; + InterfaceMetric resolveTxLock; + InterfaceMetric commitTx; // volume extent InterfaceMetric updateVolumeExtent; @@ -95,9 +106,11 @@ struct MetaServerClientMetric { InterfaceMetric updateDeallocatableBlockGroup; MetaServerClientMetric() - : getDentry(prefix, "getDentry"), listDentry(prefix, "listDentry"), + : getDentry(prefix, "getDentry"), + listDentry(prefix, "listDentry"), createDentry(prefix, "createDentry"), - deleteDentry(prefix, "deleteDentry"), getInode(prefix, "getInode"), + deleteDentry(prefix, "deleteDentry"), + getInode(prefix, "getInode"), batchGetInodeAttr(prefix, "batchGetInodeAttr"), batchGetXattr(prefix, "batchGetXattr"), createInode(prefix, "createInode"), @@ -105,10 +118,14 @@ struct MetaServerClientMetric { deleteInode(prefix, "deleteInode"), appendS3ChunkInfo(prefix, "appendS3ChunkInfo"), prepareRenameTx(prefix, "prepareRenameTx"), + prewriteRenameTx(prefix, "prewriteRenameTx"), + checkTxStatus(prefix, "checkTxStatus"), + resolveTxLock(prefix, "resolveTxLock"), + commitTx(prefix, "commitTx"), updateVolumeExtent(prefix, "updateVolumeExtent"), getVolumeExtent(prefix, "getVolumeExtent"), - updateDeallocatableBlockGroup(prefix, - "updateDeallocatableBlockGroup") {} + updateDeallocatableBlockGroup( + prefix, "updateDeallocatableBlockGroup") {} }; struct InflightGuard { @@ -238,10 +255,22 @@ struct S3Metric { InterfaceMetric adaptorWriteDiskCache; InterfaceMetric adaptorReadS3; InterfaceMetric adaptorReadDiskCache; + // Write to the backend s3 + InterfaceMetric writeToS3; + // Read from backend s3 (excluding warmup) + InterfaceMetric readFromS3; + // write to the disk cache (excluding warmup) + InterfaceMetric writeToDiskCache; + // read from the disk cache (excluding warmup) + InterfaceMetric readFromDiskCache; + // write to kv cache (excluding warmup) + InterfaceMetric writeToKVCache; + // read from kv cache (excluding warmup) + InterfaceMetric readFromKVCache; bvar::Status readSize; bvar::Status writeSize; - explicit S3Metric(const std::string &name = "") + explicit S3Metric(const std::string& name = "") : fsName(!name.empty() ? name : prefix + curve::common::ToHexString(this)), adaptorWrite(prefix, fsName + "_adaptor_write"), @@ -250,31 +279,85 @@ struct S3Metric { adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"), adaptorReadS3(prefix, fsName + "_adaptor_read_s3"), adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"), + writeToS3(prefix, fsName + "_write_to_s3"), + readFromS3(prefix, fsName + "_read_from_s3"), + writeToDiskCache(prefix, fsName + "_write_to_disk_cache"), + readFromDiskCache(prefix, fsName + "_read_from_disk_cache"), + writeToKVCache(prefix, fsName + "_write_to_kv_cache"), + readFromKVCache(prefix, fsName + "_read_from_kv_cache"), readSize(prefix, fsName + "_adaptor_read_size", 0), writeSize(prefix, fsName + "_adaptor_write_size", 0) {} }; +template +uint64_t LoadAtomicValue(void* atomValue) { + std::atomic* bytes = reinterpret_cast*>(atomValue); + return static_cast(bytes->load()); +} + struct DiskCacheMetric { static const std::string prefix; std::string fsName; InterfaceMetric writeS3; - bvar::Status diskUsedBytes; + bvar::PassiveStatus usedBytes_; + bvar::PassiveStatus totalBytes_; + InterfaceMetric trim_; - explicit DiskCacheMetric(const std::string &name = "") + explicit DiskCacheMetric(const std::string& name = "", + std::atomic* usedBytes = nullptr, + std::atomic* totalBytes = nullptr) : fsName(!name.empty() ? name : prefix + curve::common::ToHexString(this)), writeS3(prefix, fsName + "_write_s3"), - diskUsedBytes(prefix, fsName + "_diskcache_usedbytes", 0) {} + usedBytes_(prefix, fsName + "_diskcache_usedbytes", + LoadAtomicValue, usedBytes), + totalBytes_(prefix, fsName + "_diskcache_totalbytes", + LoadAtomicValue, totalBytes), + trim_(prefix, fsName + "_diskcache_trim") {} }; -struct KVClientMetric { +struct KVClientManagerMetric { static const std::string prefix; - InterfaceMetric kvClientGet; - InterfaceMetric kvClientSet; - KVClientMetric() - : kvClientGet(prefix, "get"), kvClientSet(prefix, "set") {} + std::string fsName; + InterfaceMetric get; + InterfaceMetric set; + // kvcache count + bvar::Adder count; + // kvcache hit + bvar::Adder hit; + // kvcache miss + bvar::Adder miss; + // kvcache getQueueSize + bvar::Adder getQueueSize; + // kvcache setQueueSize + bvar::Adder setQueueSize; + + explicit KVClientManagerMetric(const std::string& name = "") + : fsName(!name.empty() ? name + : prefix + curve::common::ToHexString(this)), + get(prefix, fsName + "_get"), + set(prefix, fsName + "_set"), + count(prefix, fsName + "_count"), + hit(prefix, fsName + "_hit"), + miss(prefix, fsName + "_miss"), + getQueueSize(prefix, fsName + "_get_queue_size"), + setQueueSize(prefix, fsName + "_set_queue_size") {} +}; + +struct MemcacheClientMetric { + static const std::string prefix; + + std::string fsName; + InterfaceMetric get; + InterfaceMetric set; + + explicit MemcacheClientMetric(const std::string& name = "") + : fsName(!name.empty() ? name + : prefix + curve::common::ToHexString(this)), + get(prefix, fsName + "_get"), + set(prefix, fsName + "_set") {} }; struct S3ChunkInfoMetric { @@ -296,6 +379,33 @@ struct WarmupManagerS3Metric { warmupS3CacheSize(prefix, "s3_cache_size") {} }; +void AsyncContextCollectMetrics( + std::shared_ptr s3Metric, + const std::shared_ptr& context); + +void AsyncContextCollectMetrics( + std::shared_ptr s3Metric, + const std::shared_ptr& context); + +struct FuseS3ClientIOLatencyMetric { + static const std::string prefix; + + std::string fsName; + + bvar::LatencyRecorder readAttrLatency; + bvar::LatencyRecorder readDataLatency; + bvar::LatencyRecorder writeAttrLatency; + bvar::LatencyRecorder writeDataLatency; + + explicit FuseS3ClientIOLatencyMetric(const std::string& name = "") + : fsName(!name.empty() ? name + : prefix + curve::common::ToHexString(this)), + readAttrLatency(prefix, fsName + "_read_attr_latency"), + readDataLatency(prefix, fsName + "_read_data_latency"), + writeAttrLatency(prefix, fsName + "_write_attr_latency"), + writeDataLatency(prefix, fsName + "_write_data_latency") {} +}; + } // namespace metric } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/rpcclient/BUILD b/curvefs/src/client/rpcclient/BUILD index c0e6927f96..cd39cfa2f5 100644 --- a/curvefs/src/client/rpcclient/BUILD +++ b/curvefs/src/client/rpcclient/BUILD @@ -30,6 +30,7 @@ cc_library( "//curvefs/proto:space_cc_proto", "//curvefs/src/client/common", "//curvefs/src/client/metric:client_metric", + "//curvefs/src/client:filesystem_xattr", "//curvefs/src/common:curvefs_common", "//curvefs/src/common:metric_utils", "//external:brpc", @@ -38,5 +39,6 @@ cc_library( "//src/client:curve_client", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/strings", ], ) diff --git a/curvefs/src/client/rpcclient/base_client.cpp b/curvefs/src/client/rpcclient/base_client.cpp index ca504f5201..a2f75ae177 100644 --- a/curvefs/src/client/rpcclient/base_client.cpp +++ b/curvefs/src/client/rpcclient/base_client.cpp @@ -167,6 +167,12 @@ void MDSBaseClient::CommitTx(const CommitTxRequest& request, stub.CommitTx(cntl, &request, response, nullptr); } +void MDSBaseClient::Tso(const TsoRequest& request, TsoResponse* response, + brpc::Controller* cntl, brpc::Channel* channel) { + curvefs::mds::MdsService_Stub stub(channel); + stub.Tso(cntl, &request, response, nullptr); +} + // TODO(all): do we really need pass `fsId` all the time? // each curve-fuse process only mount one filesystem void MDSBaseClient::AllocateVolumeBlockGroup( diff --git a/curvefs/src/client/rpcclient/base_client.h b/curvefs/src/client/rpcclient/base_client.h index f43d2854ee..7f3bd8161f 100644 --- a/curvefs/src/client/rpcclient/base_client.h +++ b/curvefs/src/client/rpcclient/base_client.h @@ -57,11 +57,11 @@ using curvefs::metaserver::GetInodeResponse; using curvefs::metaserver::Inode; using curvefs::metaserver::ListDentryRequest; using curvefs::metaserver::ListDentryResponse; -using curvefs::metaserver::PrepareRenameTxRequest; -using curvefs::metaserver::PrepareRenameTxResponse; +using curvefs::metaserver::ManageInodeType; +using curvefs::metaserver::PrewriteRenameTxRequest; +using curvefs::metaserver::PrewriteRenameTxResponse; using curvefs::metaserver::UpdateInodeRequest; using curvefs::metaserver::UpdateInodeResponse; -using curvefs::metaserver::ManageInodeType; using curvefs::common::FSType; using curvefs::common::PartitionInfo; @@ -84,6 +84,8 @@ using curvefs::mds::CommitTxRequest; using curvefs::mds::CommitTxResponse; using curvefs::mds::RefreshSessionRequest; using curvefs::mds::RefreshSessionResponse; +using curvefs::mds::TsoRequest; +using curvefs::mds::TsoResponse; using curvefs::mds::UmountFsRequest; using curvefs::mds::UmountFsResponse; @@ -197,6 +199,9 @@ class MDSBaseClient { brpc::Controller* cntl, brpc::Channel* channel); + virtual void Tso(const TsoRequest& request, TsoResponse* response, + brpc::Controller* cntl, brpc::Channel* channel); + virtual void AllocateVolumeBlockGroup(uint32_t fsId, uint32_t count, const std::string& owner, diff --git a/curvefs/src/client/rpcclient/fsdelta_updater.cpp b/curvefs/src/client/rpcclient/fsdelta_updater.cpp new file mode 100644 index 0000000000..26cde8e071 --- /dev/null +++ b/curvefs/src/client/rpcclient/fsdelta_updater.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "curvefs/src/client/rpcclient/fsdelta_updater.h" + +namespace curvefs { +namespace client { + +void FsDeltaUpdater::UpdateDeltaBytes(int64_t deltaBytes) { + deltaBytes_.fetch_add(deltaBytes); +} + +int64_t FsDeltaUpdater::GetDeltaBytesAndReset() { + return deltaBytes_.exchange(0); +} + +int64_t FsDeltaUpdater::GetDeltaBytes() { return deltaBytes_.load(); } + +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/rpcclient/fsdelta_updater.h b/curvefs/src/client/rpcclient/fsdelta_updater.h new file mode 100644 index 0000000000..e449623702 --- /dev/null +++ b/curvefs/src/client/rpcclient/fsdelta_updater.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CURVEFS_SRC_CLIENT_RPCCLIENT_FSDELTA_UPDATER_H_ +#define CURVEFS_SRC_CLIENT_RPCCLIENT_FSDELTA_UPDATER_H_ + +#include + +namespace curvefs { +namespace client { + +class FsDeltaUpdater { + public: + static FsDeltaUpdater& GetInstance() { + static FsDeltaUpdater instance_; + return instance_; + } + + void Init() { deltaBytes_.store(0); } + + void UpdateDeltaBytes(int64_t deltaBytes); + + int64_t GetDeltaBytesAndReset(); + + int64_t GetDeltaBytes(); + + private: + uint32_t fsId_; + std::atomic deltaBytes_; +}; + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_RPCCLIENT_FSDELTA_UPDATER_H_ diff --git a/curvefs/src/client/rpcclient/fsquota_checker.cpp b/curvefs/src/client/rpcclient/fsquota_checker.cpp new file mode 100644 index 0000000000..7a619bc1c4 --- /dev/null +++ b/curvefs/src/client/rpcclient/fsquota_checker.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "curvefs/src/client/rpcclient/fsquota_checker.h" + +#include "curvefs/src/client/rpcclient/fsdelta_updater.h" + +namespace curvefs { + +namespace client { + +void FsQuotaChecker::Init() { + fsCapacityCache_.store(0); + fsUsedBytesCache_.store(0); +} + +bool FsQuotaChecker::QuotaBytesCheck(uint64_t incBytes) { + uint64_t capacity = fsCapacityCache_.load(); + uint64_t usedBytes = fsUsedBytesCache_.load(); + // quota disabled + if (capacity == 0) { + return true; + } + // need consider local delta + auto localDelta = FsDeltaUpdater::GetInstance().GetDeltaBytes(); + return capacity - usedBytes >= localDelta + incBytes; +} + +void FsQuotaChecker::UpdateQuotaCache(uint64_t capacity, uint64_t usedBytes) { + fsCapacityCache_.store(capacity); + fsUsedBytesCache_.store(usedBytes); +} + +} // namespace client + +} // namespace curvefs diff --git a/curvefs/src/client/rpcclient/fsquota_checker.h b/curvefs/src/client/rpcclient/fsquota_checker.h new file mode 100644 index 0000000000..40d2f1f70c --- /dev/null +++ b/curvefs/src/client/rpcclient/fsquota_checker.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CURVEFS_SRC_CLIENT_RPCCLIENT_FSQUOTA_CHECKER_H_ +#define CURVEFS_SRC_CLIENT_RPCCLIENT_FSQUOTA_CHECKER_H_ + +#include + +namespace curvefs { +namespace client { + +class FsQuotaChecker { + public: + static FsQuotaChecker& GetInstance() { + static FsQuotaChecker instance_; + return instance_; + } + + void Init(); + + bool QuotaBytesCheck(uint64_t incBytes); + + void UpdateQuotaCache(uint64_t capacity, uint64_t usedBytes); + + private: + std::atomic fsCapacityCache_; + std::atomic fsUsedBytesCache_; +}; + +} // namespace client + +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_RPCCLIENT_FSQUOTA_CHECKER_H_ diff --git a/curvefs/src/client/rpcclient/mds_client.cpp b/curvefs/src/client/rpcclient/mds_client.cpp index 91a6d5fb86..2f53aa3d7d 100644 --- a/curvefs/src/client/rpcclient/mds_client.cpp +++ b/curvefs/src/client/rpcclient/mds_client.cpp @@ -20,12 +20,16 @@ * Author: lixiaocui */ +#include "curvefs/src/client/rpcclient/mds_client.h" + #include #include #include +#include "absl/strings/str_join.h" #include "curvefs/proto/space.pb.h" -#include "curvefs/src/client/rpcclient/mds_client.h" +#include "curvefs/src/client/rpcclient/fsdelta_updater.h" +#include "curvefs/src/client/rpcclient/fsquota_checker.h" #include "curvefs/src/common/metric_utils.h" namespace curvefs { @@ -497,12 +501,11 @@ FSStatusCode MdsClientImpl::AllocS3ChunkId(uint32_t fsId, uint32_t idNum, return ReturnError(rpcexcutor_.DoRPCTask(task, mdsOpt_.mdsMaxRetryMS)); } -FSStatusCode -MdsClientImpl::RefreshSession(const std::vector &txIds, - std::vector *latestTxIdList, - const std::string& fsName, - const Mountpoint& mountpoint, - std::atomic* enableSumInDir) { +FSStatusCode MdsClientImpl::RefreshSession( + const std::vector &txIds, + std::vector *latestTxIdList, const std::string &fsName, + const Mountpoint &mountpoint, std::atomic *enableSumInDir, + const std::string &mdsAddrs, std::string *mdsAddrsOverride) { auto task = RPCTask { (void)addrindex; (void)rpctimeoutMS; @@ -513,6 +516,12 @@ MdsClientImpl::RefreshSession(const std::vector &txIds, *request.mutable_txids() = {txIds.begin(), txIds.end()}; request.set_fsname(fsName); *request.mutable_mountpoint() = mountpoint; + curvefs::mds::FsDelta fsDelta; + fsDelta.set_bytes( + FsDeltaUpdater::GetInstance().GetDeltaBytesAndReset()); + *request.mutable_fsdelta() = std::move(fsDelta); + request.set_mdsaddrs(mdsAddrs); + mdsbasecli_->RefreshSession(request, &response, cntl, channel); if (cntl->Failed()) { mdsClientMetric_.refreshSession.eps.count << 1; @@ -538,13 +547,35 @@ MdsClientImpl::RefreshSession(const std::vector &txIds, LOG(INFO) << "update enableSumInDir to " << response.enablesumindir(); } - + if (response.has_fscapacity() && response.has_fsusedbytes()) { + FsQuotaChecker::GetInstance().UpdateQuotaCache( + response.fscapacity(), response.fsusedbytes()); + } + if (response.has_mdsaddrsoverride()) { + *mdsAddrsOverride = response.mdsaddrsoverride(); + } return ret; }; return ReturnError(rpcexcutor_.DoRPCTask(task, mdsOpt_.mdsMaxRetryMS)); } +std::string MdsClientImpl::GetMdsAddrs() { + auto option = rpcexcutor_.GetOption(); + return absl::StrJoin(option.addrs, ","); +} + +void MdsClientImpl::SetMdsAddrs(const std::string &mdsAddrs) { + std::vector mdsAddrsVec = {}; + curve::common::SplitString(mdsAddrs, ",", &mdsAddrsVec); + if (!mdsAddrsVec.empty()) { + auto option = rpcexcutor_.GetOption(); + option.addrs = mdsAddrsVec; + rpcexcutor_.SetOption(option); + LOG(WARNING) << "update mdsAddrs to " << mdsAddrs; + } +} + FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request, GetLatestTxIdResponse* response) { auto task = RPCTask { @@ -672,6 +703,38 @@ FSStatusCode MdsClientImpl::CommitTxWithLock( return CommitTx(request); } +FSStatusCode MdsClientImpl::Tso(uint64_t* ts, uint64_t* timestamp) { + auto task = RPCTask { + (void)addrindex; + (void)rpctimeoutMS; + mdsClientMetric_.tso.qps.count << 1; + LatencyUpdater updater(&mdsClientMetric_.tso.latency); + TsoRequest request; + TsoResponse response; + mdsbasecli_->Tso(request, &response, cntl, channel); + if (cntl->Failed()) { + mdsClientMetric_.tso.eps.count << 1; + LOG(WARNING) << "Tso Failed, errorcode = " << cntl->ErrorCode() + << ", error content:" << cntl->ErrorText() + << ", log id = " << cntl->log_id(); + return -cntl->ErrorCode(); + } + + FSStatusCode ret = response.statuscode(); + if (ret != FSStatusCode::OK) { + LOG(ERROR) << "Tso: errcode = " << ret + << ", errmsg = " << FSStatusCode_Name(ret); + return ret; + } else { + *ts = response.ts(); + *timestamp = response.timestamp(); + } + return ret; + }; + // for rpc error or failed/timeout, we will retry until success + return ReturnError(rpcexcutor_.DoRPCTask(task, 0)); +} + FSStatusCode MdsClientImpl::ReturnError(int retcode) { // rpc error convert to FSStatusCode::RPC_ERROR if (retcode < 0) { diff --git a/curvefs/src/client/rpcclient/mds_client.h b/curvefs/src/client/rpcclient/mds_client.h index 3b7577c9e3..e74ec84c03 100644 --- a/curvefs/src/client/rpcclient/mds_client.h +++ b/curvefs/src/client/rpcclient/mds_client.h @@ -112,12 +112,15 @@ class MdsClient { virtual FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum, uint64_t *chunkId) = 0; - virtual FSStatusCode - RefreshSession(const std::vector &txIds, - std::vector *latestTxIdList, - const std::string& fsName, - const Mountpoint& mountpoint, - std::atomic* enableSumInDir) = 0; + virtual FSStatusCode RefreshSession( + const std::vector &txIds, + std::vector *latestTxIdList, const std::string &fsName, + const Mountpoint &mountpoint, std::atomic *enableSumInDir, + const std::string &mdsAddrs, std::string *mdsAddrsOverride) = 0; + + virtual std::string GetMdsAddrs() = 0; + + virtual void SetMdsAddrs(const std::string &mdsAddrs) = 0; virtual FSStatusCode GetLatestTxId(uint32_t fsId, std::vector* txIds) = 0; @@ -138,6 +141,8 @@ class MdsClient { const std::string& uuid, uint64_t sequence) = 0; + virtual FSStatusCode Tso(uint64_t* ts, uint64_t* timestamp) = 0; + // allocate block group virtual SpaceErrCode AllocateVolumeBlockGroup( uint32_t fsId, @@ -203,9 +208,15 @@ class MdsClientImpl : public MdsClient { FSStatusCode RefreshSession(const std::vector &txIds, std::vector *latestTxIdList, - const std::string& fsName, - const Mountpoint& mountpoint, - std::atomic* enableSumInDir) override; + const std::string &fsName, + const Mountpoint &mountpoint, + std::atomic *enableSumInDir, + const std::string &mdsAddrs, + std::string *mdsAddrsOverride) override; + + std::string GetMdsAddrs() override; + + void SetMdsAddrs(const std::string &mdsAddrs) override; FSStatusCode GetLatestTxId(uint32_t fsId, std::vector* txIds) override; @@ -225,6 +236,8 @@ class MdsClientImpl : public MdsClient { const std::string& uuid, uint64_t sequence) override; + FSStatusCode Tso(uint64_t* ts, uint64_t* timestamp) override; + // allocate block group SpaceErrCode AllocateVolumeBlockGroup( uint32_t fsId, @@ -233,11 +246,9 @@ class MdsClientImpl : public MdsClient { std::vector *groups) override; // acquire block group - SpaceErrCode AcquireVolumeBlockGroup( - uint32_t fsId, - uint64_t blockGroupOffset, - const std::string &owner, - curvefs::mds::space::BlockGroup *groups) override; + SpaceErrCode AcquireVolumeBlockGroup(uint32_t fsId, + uint64_t blockGroupOffset, const std::string& owner, + curvefs::mds::space::BlockGroup* groups) override; // release block group SpaceErrCode ReleaseVolumeBlockGroup( diff --git a/curvefs/src/client/rpcclient/metacache.cpp b/curvefs/src/client/rpcclient/metacache.cpp index d3bd4fd294..4ca48b4935 100644 --- a/curvefs/src/client/rpcclient/metacache.cpp +++ b/curvefs/src/client/rpcclient/metacache.cpp @@ -102,7 +102,7 @@ bool MetaCache::GetTarget(uint32_t fsID, uint64_t inodeID, } if (!GetCopysetIDwithInodeID(inodeID, &target->groupID, - &target->partitionID, &target->txId)) { + &target->partitionID, &target->txId)) { LOG(ERROR) << "{fsid:" << fsID << ", inodeid:" << inodeID << "} do not find partition"; return false; @@ -496,9 +496,7 @@ bool MetaCache::SelectPartition(CopysetTarget *target) { } bool MetaCache::GetCopysetIDwithInodeID(uint64_t inodeID, - CopysetGroupID *groupID, - PartitionID *partitionID, - uint64_t *txId) { + CopysetGroupID* groupID, PartitionID* partitionID, uint64_t *txId) { ReadLockGuard rl(rwlock4Partitions_); for (auto iter = partitionInfos_.begin(); iter != partitionInfos_.end(); ++iter) { diff --git a/curvefs/src/client/rpcclient/metacache.h b/curvefs/src/client/rpcclient/metacache.h index c4c5589761..a9a1505d86 100644 --- a/curvefs/src/client/rpcclient/metacache.h +++ b/curvefs/src/client/rpcclient/metacache.h @@ -54,8 +54,15 @@ using ::curvefs::common::PartitionStatus; namespace curvefs { namespace client { + +namespace common { + DECLARE_int32(TxVersion); +} // namespace common + namespace rpcclient { +using curvefs::client::common::FLAGS_TxVersion; + struct CopysetGroupID { LogicPoolID poolID = 0; CopysetID copysetID = 0; @@ -83,8 +90,9 @@ struct CopysetTarget { bool IsValid() const { return groupID.poolID != 0 && groupID.copysetID != 0 && - partitionID != 0 && txId != 0 && metaServerID != 0 && - endPoint.ip != butil::IP_ANY && endPoint.port != 0; + partitionID != 0 && metaServerID != 0 && + endPoint.ip != butil::IP_ANY && endPoint.port != 0 && + (FLAGS_TxVersion != 1 || txId != 0); } void Reset() { @@ -179,8 +187,8 @@ class MetaCache { bool SelectPartition(CopysetTarget *target); // get info from partitionMap or copysetMap - bool GetCopysetIDwithInodeID(uint64_t inodeID, CopysetGroupID *groupID, - PartitionID *patitionID, uint64_t *txId); + bool GetCopysetIDwithInodeID(uint64_t inodeID, CopysetGroupID* groupID, + PartitionID* patitionID, uint64_t *txId); bool GetCopysetInfowithCopySetID(const CopysetGroupID &groupID, CopysetInfo *targetInfo); diff --git a/curvefs/src/client/rpcclient/metaserver_client.cpp b/curvefs/src/client/rpcclient/metaserver_client.cpp index 0422436981..0f5d951134 100644 --- a/curvefs/src/client/rpcclient/metaserver_client.cpp +++ b/curvefs/src/client/rpcclient/metaserver_client.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include @@ -46,8 +45,12 @@ using curvefs::metaserver::BatchGetInodeAttrRequest; using curvefs::metaserver::BatchGetInodeAttrResponse; using curvefs::metaserver::BatchGetXAttrRequest; using curvefs::metaserver::BatchGetXAttrResponse; +using curvefs::metaserver::CheckTxStatusRequest; +using curvefs::metaserver::CheckTxStatusResponse; using curvefs::metaserver::GetOrModifyS3ChunkInfoRequest; using curvefs::metaserver::GetOrModifyS3ChunkInfoResponse; +using curvefs::metaserver::ResolveTxLockRequest; +using curvefs::metaserver::ResolveTxLockResponse; namespace curvefs { namespace client { @@ -57,6 +60,10 @@ using GetDentryExcutor = TaskExecutor; using ListDentryExcutor = TaskExecutor; using DeleteDentryExcutor = TaskExecutor; using PrepareRenameTxExcutor = TaskExecutor; +using PrewriteRenameTxExcutor = TaskExecutor; +using CheckTxStatusExcutor = TaskExecutor; +using ResolveTxLockExcutor = TaskExecutor; +using CommitTxExcutor = TaskExecutor; using DeleteInodeExcutor = TaskExecutor; using UpdateInodeExcutor = TaskExecutor; using GetInodeExcutor = TaskExecutor; @@ -120,9 +127,15 @@ void MetaServerClientImpl::SetTxId(uint32_t partitionId, uint64_t txId) { metaCache_->SetTxId(partitionId, txId); } +bool MetaServerClientImpl::GetPartitionId(uint32_t fsId, uint64_t inodeId, + PartitionID *partitionId) { + return metaCache_->GetPartitionIdByInodeId(fsId, inodeId, partitionId); +} + MetaStatusCode MetaServerClientImpl::GetDentry(uint32_t fsId, uint64_t inodeid, const std::string &name, - Dentry *out) { + Dentry *out, + TxLock* txLockOut) { auto task = RPCTask { (void)taskExecutorDone; metric_.getDentry.qps.count << 1; @@ -151,6 +164,9 @@ MetaStatusCode MetaServerClientImpl::GetDentry(uint32_t fsId, uint64_t inodeid, MetaStatusCode ret = response.statuscode(); if (ret != MetaStatusCode::OK) { + if (ret == MetaStatusCode::TX_KEY_LOCKED) { + *txLockOut = response.txlock(); + } LOG_IF(WARNING, ret != MetaStatusCode::NOT_FOUND) << "GetDentry: fsId = " << fsId << ", inodeid = " << inodeid << ", name = " << name << ", errcode = " << ret @@ -172,8 +188,7 @@ MetaStatusCode MetaServerClientImpl::GetDentry(uint32_t fsId, uint64_t inodeid, }; auto taskCtx = std::make_shared(MetaServerOpType::GetDentry, - task, fsId, inodeid, false, - opt_.enableRenameParallel); + task, fsId, inodeid, false, opt_.enableRenameParallel); GetDentryExcutor excutor(opt_, metaCache_, channelManager_, std::move(taskCtx)); return ConvertToMetaStatusCode(excutor.DoRPCTask()); @@ -182,7 +197,8 @@ MetaStatusCode MetaServerClientImpl::GetDentry(uint32_t fsId, uint64_t inodeid, MetaStatusCode MetaServerClientImpl::ListDentry(uint32_t fsId, uint64_t inodeid, const std::string &last, uint32_t count, bool onlyDir, - std::list *dentryList) { + std::list *dentryList, + TxLock* txLockOut) { auto task = RPCTask { (void)taskExecutorDone; metric_.listDentry.qps.count << 1; @@ -213,6 +229,9 @@ MetaStatusCode MetaServerClientImpl::ListDentry(uint32_t fsId, uint64_t inodeid, MetaStatusCode ret = response.statuscode(); if (ret != MetaStatusCode::OK) { + if (ret == MetaStatusCode::TX_KEY_LOCKED) { + *txLockOut = response.txlock(); + } LOG(WARNING) << "ListDentry: fsId = " << fsId << ", inodeid = " << inodeid << ", last = " << last << ", count = " << count << ", onlyDir = " << onlyDir @@ -230,14 +249,14 @@ MetaStatusCode MetaServerClientImpl::ListDentry(uint32_t fsId, uint64_t inodeid, }; auto taskCtx = std::make_shared(MetaServerOpType::ListDentry, - task, fsId, inodeid, false, - opt_.enableRenameParallel); + task, fsId, inodeid, false, opt_.enableRenameParallel); ListDentryExcutor excutor(opt_, metaCache_, channelManager_, std::move(taskCtx)); return ConvertToMetaStatusCode(excutor.DoRPCTask()); } -MetaStatusCode MetaServerClientImpl::CreateDentry(const Dentry &dentry) { +MetaStatusCode MetaServerClientImpl::CreateDentry( + const Dentry &dentry, TxLock* txLockOut) { auto task = RPCTask { (void)taskExecutorDone; metric_.createDentry.qps.count << 1; @@ -255,6 +274,7 @@ MetaStatusCode MetaServerClientImpl::CreateDentry(const Dentry &dentry) { d->set_txid(txId); d->set_type(dentry.type()); request.set_allocated_dentry(d); + SetCreateTime(request.mutable_create()); curvefs::metaserver::MetaServerService_Stub stub(channel); stub.CreateDentry(cntl, &request, &response, nullptr); @@ -274,7 +294,11 @@ MetaStatusCode MetaServerClientImpl::CreateDentry(const Dentry &dentry) { } MetaStatusCode ret = response.statuscode(); + if (ret != MetaStatusCode::OK) { + if (ret == MetaStatusCode::TX_KEY_LOCKED) { + *txLockOut = response.txlock(); + } LOG(WARNING) << "CreateDentry: dentry = " << dentry.DebugString() << ", errcode = " << ret << ", errmsg = " << MetaStatusCode_Name(ret); @@ -298,9 +322,8 @@ MetaStatusCode MetaServerClientImpl::CreateDentry(const Dentry &dentry) { } MetaStatusCode MetaServerClientImpl::DeleteDentry(uint32_t fsId, - uint64_t inodeid, - const std::string &name, - FsFileType type) { + uint64_t inodeid, const std::string &name, FsFileType type, + TxLock* txLockOut) { auto task = RPCTask { (void)taskExecutorDone; metric_.deleteDentry.qps.count << 1; @@ -315,7 +338,7 @@ MetaStatusCode MetaServerClientImpl::DeleteDentry(uint32_t fsId, request.set_name(name); request.set_txid(txId); request.set_type(type); - + SetCreateTime(request.mutable_create()); curvefs::metaserver::MetaServerService_Stub stub(channel); stub.DeleteDentry(cntl, &request, &response, nullptr); @@ -330,6 +353,9 @@ MetaStatusCode MetaServerClientImpl::DeleteDentry(uint32_t fsId, MetaStatusCode ret = response.statuscode(); if (ret != MetaStatusCode::OK) { + if (ret == MetaStatusCode::TX_KEY_LOCKED) { + *txLockOut = response.txlock(); + } LOG(WARNING) << "DeleteDentry: fsid = " << fsId << ", inodeid = " << inodeid << ", name = " << name << ", errcode = " << ret @@ -342,22 +368,21 @@ MetaStatusCode MetaServerClientImpl::DeleteDentry(uint32_t fsId, }; auto taskCtx = std::make_shared(MetaServerOpType::DeleteDentry, - task, fsId, inodeid, false, - opt_.enableRenameParallel); + task, fsId, inodeid, false, opt_.enableRenameParallel); DeleteDentryExcutor excutor(opt_, metaCache_, channelManager_, std::move(taskCtx)); return ConvertToMetaStatusCode(excutor.DoRPCTask()); } -MetaStatusCode -MetaServerClientImpl::PrepareRenameTx(const std::vector &dentrys) { +MetaStatusCode MetaServerClientImpl::PrepareRenameTx( + const std::vector &dentrys) { auto task = RPCTask { (void)txId; (void)taskExecutorDone; metric_.prepareRenameTx.qps.count << 1; LatencyUpdater updater(&metric_.prepareRenameTx.latency); - PrepareRenameTxRequest request; - PrepareRenameTxResponse response; + curvefs::metaserver::PrepareRenameTxRequest request; + curvefs::metaserver::PrepareRenameTxResponse response; request.set_poolid(poolID); request.set_copysetid(copysetID); request.set_partitionid(partitionID); @@ -395,6 +420,167 @@ MetaServerClientImpl::PrepareRenameTx(const std::vector &dentrys) { return ConvertToMetaStatusCode(excutor.DoRPCTask()); } +MetaStatusCode MetaServerClientImpl::PrewriteRenameTx( + const std::vector& dentrys, + const TxLock& txLockIn, TxLock* txLockOut) { + auto task = RPCTask { + (void)txId; + (void)taskExecutorDone; + metric_.prewriteRenameTx.qps.count << 1; + LatencyUpdater updater(&metric_.prewriteRenameTx.latency); + PrewriteRenameTxRequest request; + PrewriteRenameTxResponse response; + request.set_poolid(poolID); + request.set_copysetid(copysetID); + request.set_partitionid(partitionID); + *request.mutable_dentrys() = {dentrys.begin(), dentrys.end()}; + *request.mutable_txlock() = txLockIn; + + curvefs::metaserver::MetaServerService_Stub stub(channel); + stub.PrewriteRenameTx(cntl, &request, &response, nullptr); + + if (cntl->Failed()) { + metric_.prewriteRenameTx.eps.count << 1; + LOG(WARNING) << "PrewriteRenameTx Failed, errorcode = " + << cntl->ErrorCode() + << ", error content:" << cntl->ErrorText() + << ", request = " << request.DebugString() + << ", log id = " << cntl->log_id(); + return -cntl->ErrorCode(); + } + + MetaStatusCode ret = response.statuscode(); + if (ret != MetaStatusCode::OK) { + LOG(WARNING) << "PrewriteRenameTx: ret = " << ret + << ", errmsg = " << MetaStatusCode_Name(ret); + } else { + *txLockOut = response.txlock(); + } + + VLOG(6) << "PrewriteRenameTx done, request: " << request.DebugString() + << "response: " << response.DebugString(); + return ret; + }; + auto taskCtx = std::make_shared( + MetaServerOpType::PrewriteRenameTx, task, dentrys[0].fsid(), + dentrys[0].parentinodeid()); + PrewriteRenameTxExcutor excutor(opt_, metaCache_, channelManager_, + std::move(taskCtx)); + return ConvertToMetaStatusCode(excutor.DoRPCTask()); +} + +MetaStatusCode MetaServerClientImpl::CheckTxStatus(uint32_t fsId, + uint64_t inodeId, const std::string& primaryKey, uint64_t startTs, + uint64_t curTimestamp) { + auto task = RPCTask { + (void)txId; + (void)taskExecutorDone; + metric_.checkTxStatus.qps.count << 1; + LatencyUpdater updater(&metric_.checkTxStatus.latency); + CheckTxStatusRequest request; + CheckTxStatusResponse response; + request.set_poolid(poolID); + request.set_copysetid(copysetID); + request.set_partitionid(partitionID); + request.set_primarykey(primaryKey); + request.set_startts(startTs); + request.set_curtimestamp(curTimestamp); + + curvefs::metaserver::MetaServerService_Stub stub(channel); + stub.CheckTxStatus(cntl, &request, &response, nullptr); + + if (cntl->Failed()) { + metric_.checkTxStatus.eps.count << 1; + LOG(WARNING) << "CheckTxStatus failed" + << ", errorCode = " << cntl->ErrorCode() + << ", errorText = " << cntl->ErrorText() + << ", request = " << request.DebugString() + << ", logId = " << cntl->log_id(); + return -cntl->ErrorCode(); + } + + return response.statuscode(); + }; + auto taskCtx = std::make_shared( + MetaServerOpType::CheckTxStatus, task, fsId, inodeId); + CheckTxStatusExcutor excutor( + opt_, metaCache_, channelManager_, std::move(taskCtx)); + return ConvertToMetaStatusCode(excutor.DoRPCTask()); +} + +MetaStatusCode MetaServerClientImpl::ResolveTxLock(const Dentry& dentry, + uint64_t startTs, uint64_t commitTs) { + auto task = RPCTask { + (void)txId; + (void)taskExecutorDone; + metric_.resolveTxLock.qps.count << 1; + LatencyUpdater updater(&metric_.resolveTxLock.latency); + ResolveTxLockRequest request; + ResolveTxLockResponse response; + request.set_poolid(poolID); + request.set_copysetid(copysetID); + request.set_partitionid(partitionID); + *request.mutable_dentry() = dentry; + request.set_startts(startTs); + request.set_committs(commitTs); + + curvefs::metaserver::MetaServerService_Stub stub(channel); + stub.ResolveTxLock(cntl, &request, &response, nullptr); + + if (cntl->Failed()) { + metric_.resolveTxLock.eps.count << 1; + LOG(WARNING) << "ResolveTxLock failed" + << ", errorCode = " << cntl->ErrorCode() + << ", errorText = " << cntl->ErrorText() + << ", logId = " << cntl->log_id(); + return -cntl->ErrorCode(); + } + return response.statuscode(); + }; + auto taskCtx = + std::make_shared(MetaServerOpType::ResolveTxLock, task, + dentry.fsid(), dentry.parentinodeid()); + ResolveTxLockExcutor excutor( + opt_, metaCache_, channelManager_, std::move(taskCtx)); + return ConvertToMetaStatusCode(excutor.DoRPCTask()); +} + +MetaStatusCode MetaServerClientImpl::CommitTx( + const std::vector& dentrys, uint64_t startTs, uint64_t commitTs) { + auto task = RPCTask { + (void)txId; + (void)taskExecutorDone; + metric_.commitTx.qps.count << 1; + LatencyUpdater updater(&metric_.commitTx.latency); + curvefs::metaserver::CommitTxRequest request; + curvefs::metaserver::CommitTxResponse response; + request.set_poolid(poolID); + request.set_copysetid(copysetID); + request.set_partitionid(partitionID); + *request.mutable_dentrys() = {dentrys.begin(), dentrys.end()}; + request.set_startts(startTs); + request.set_committs(commitTs); + + curvefs::metaserver::MetaServerService_Stub stub(channel); + stub.CommitTx(cntl, &request, &response, nullptr); + + if (cntl->Failed()) { + metric_.commitTx.eps.count << 1; + LOG(WARNING) << "CommitTx failed" + << ", errorCode = " << cntl->ErrorCode() + << ", errorText = " << cntl->ErrorText() + << ", logId = " << cntl->log_id(); + return -cntl->ErrorCode(); + } + return response.statuscode(); + }; + auto taskCtx = std::make_shared(MetaServerOpType::CommitTx, + task, dentrys[0].fsid(), dentrys[0].parentinodeid()); + CommitTxExcutor excutor( + opt_, metaCache_, channelManager_, std::move(taskCtx)); + return ConvertToMetaStatusCode(excutor.DoRPCTask()); +} + MetaStatusCode MetaServerClientImpl::GetInode(uint32_t fsId, uint64_t inodeid, Inode *out, bool *streaming) { auto task = RPCTask { @@ -1141,12 +1327,7 @@ MetaStatusCode MetaServerClientImpl::CreateInode(const InodeParam ¶m, request.set_rdev(param.rdev); request.set_symlink(param.symlink); request.set_parent(param.parent); - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - Time *tm = new Time(); - tm->set_sec(now.tv_sec); - tm->set_nsec(now.tv_nsec); - request.set_allocated_create(tm); + SetCreateTime(request.mutable_create()); curvefs::metaserver::MetaServerService_Stub stub(channel); stub.CreateInode(cntl, &request, &response, nullptr); diff --git a/curvefs/src/client/rpcclient/metaserver_client.h b/curvefs/src/client/rpcclient/metaserver_client.h index cd2a233da3..ddce3c64ba 100644 --- a/curvefs/src/client/rpcclient/metaserver_client.h +++ b/curvefs/src/client/rpcclient/metaserver_client.h @@ -23,6 +23,8 @@ #ifndef CURVEFS_SRC_CLIENT_RPCCLIENT_METASERVER_CLIENT_H_ #define CURVEFS_SRC_CLIENT_RPCCLIENT_METASERVER_CLIENT_H_ +#include + #include #include #include @@ -43,17 +45,17 @@ #include "absl/types/optional.h" using ::curvefs::client::metric::MetaServerClientMetric; +using ::curvefs::common::StreamClient; +using ::curvefs::common::StreamStatus; +using ::curvefs::metaserver::DeallocatableBlockGroup; using ::curvefs::metaserver::Dentry; using ::curvefs::metaserver::FsFileType; using ::curvefs::metaserver::Inode; using ::curvefs::metaserver::InodeAttr; -using ::curvefs::metaserver::XAttr; -using ::curvefs::metaserver::MetaStatusCode; using ::curvefs::metaserver::S3ChunkInfoList; -using ::curvefs::metaserver::DeallocatableBlockGroup; -using ::curvefs::common::StreamStatus; -using ::curvefs::common::StreamClient; using ::curvefs::metaserver::Time; +using ::curvefs::metaserver::TxLock; +using ::curvefs::metaserver::XAttr; using DeallocatableBlockGroupMap = std::map; using S3ChunkInfoMap = google::protobuf::Map; @@ -70,6 +72,13 @@ struct DataIndices { absl::optional volumeExtents; }; +inline void SetCreateTime(Time* tm) { + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + tm->set_sec(now.tv_sec); + tm->set_nsec(now.tv_nsec); +} + class MetaServerClient { public: virtual ~MetaServerClient() = default; @@ -85,22 +94,36 @@ class MetaServerClient { virtual void SetTxId(uint32_t partitionId, uint64_t txId) = 0; virtual MetaStatusCode GetDentry(uint32_t fsId, uint64_t inodeid, - const std::string &name, Dentry *out) = 0; + const std::string &name, Dentry *out, TxLock* txLockOut) = 0; virtual MetaStatusCode ListDentry(uint32_t fsId, uint64_t inodeid, const std::string &last, uint32_t count, bool onlyDir, - std::list *dentryList) = 0; + std::list *dentryList, + TxLock* txLockOut) = 0; - virtual MetaStatusCode CreateDentry(const Dentry &dentry) = 0; + virtual MetaStatusCode CreateDentry( + const Dentry &dentry, TxLock* txLockOut) = 0; virtual MetaStatusCode DeleteDentry(uint32_t fsId, uint64_t inodeid, - const std::string &name, - FsFileType type) = 0; + const std::string &name, FsFileType type, TxLock* txLockOut) = 0; virtual MetaStatusCode PrepareRenameTx(const std::vector &dentrys) = 0; + virtual MetaStatusCode PrewriteRenameTx(const std::vector& dentrys, + const TxLock& txLockIn, TxLock* txLockOut) = 0; + + virtual MetaStatusCode CheckTxStatus(uint32_t fsId, uint64_t inodeId, + const std::string& primaryKey, uint64_t startTs, + uint64_t curTimestamp) = 0; + + virtual MetaStatusCode ResolveTxLock(const Dentry& dentry, + uint64_t startTs, uint64_t commitTs) = 0; + + virtual MetaStatusCode CommitTx(const std::vector& dentry, + uint64_t startTs, uint64_t commitTs) = 0; + virtual MetaStatusCode GetInode(uint32_t fsId, uint64_t inodeid, Inode *out, bool* streaming) = 0; @@ -176,6 +199,9 @@ class MetaServerClient { virtual MetaStatusCode UpdateDeallocatableBlockGroup(uint32_t fsId, uint64_t inodeId, DeallocatableBlockGroupMap *statistic) = 0; + + virtual bool GetPartitionId(uint32_t fsId, uint64_t inodeId, + PartitionID* partitionId) = 0; }; class MetaServerClientImpl : public MetaServerClient { @@ -193,21 +219,35 @@ class MetaServerClientImpl : public MetaServerClient { void SetTxId(uint32_t partitionId, uint64_t txId) override; MetaStatusCode GetDentry(uint32_t fsId, uint64_t inodeid, - const std::string &name, Dentry *out) override; + const std::string &name, Dentry *out, TxLock* txLockOut) override; MetaStatusCode ListDentry(uint32_t fsId, uint64_t inodeid, const std::string &last, uint32_t count, bool onlyDir, - std::list *dentryList) override; + std::list *dentryList, + TxLock* txLockOut) override; - MetaStatusCode CreateDentry(const Dentry &dentry) override; + MetaStatusCode CreateDentry( + const Dentry &dentry, TxLock* txLockOut) override; MetaStatusCode DeleteDentry(uint32_t fsId, uint64_t inodeid, - const std::string &name, - FsFileType type) override; + const std::string &name, FsFileType type, TxLock* txLockOut) override; MetaStatusCode PrepareRenameTx(const std::vector &dentrys) override; + MetaStatusCode PrewriteRenameTx(const std::vector& dentrys, + const TxLock& txLockIn, TxLock* txLockOut) override; + + MetaStatusCode CheckTxStatus(uint32_t fsId, uint64_t inodeId, + const std::string& primaryKey, uint64_t startTs, + uint64_t curTimestamp) override; + + MetaStatusCode ResolveTxLock(const Dentry& dentry, + uint64_t startTs, uint64_t commitTs) override; + + MetaStatusCode CommitTx(const std::vector& dentrys, + uint64_t startTs, uint64_t commitTs) override; + MetaStatusCode GetInode(uint32_t fsId, uint64_t inodeid, Inode *out, bool* streaming) override; @@ -285,6 +325,9 @@ class MetaServerClientImpl : public MetaServerClient { uint32_t fsId, uint64_t inodeId, DeallocatableBlockGroupMap *statistic) override; + bool GetPartitionId(uint32_t fsId, uint64_t inodeId, + PartitionID *partitionId) override; + private: MetaStatusCode UpdateInode(const UpdateInodeRequest &request, bool internal = false); @@ -308,6 +351,7 @@ class MetaServerClientImpl : public MetaServerClient { StreamClient streamClient_; MetaServerClientMetric metric_; }; + } // namespace rpcclient } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/rpcclient/task_excutor.cpp b/curvefs/src/client/rpcclient/task_excutor.cpp index 191cff6e08..6d5d939369 100644 --- a/curvefs/src/client/rpcclient/task_excutor.cpp +++ b/curvefs/src/client/rpcclient/task_excutor.cpp @@ -29,8 +29,8 @@ #include "curvefs/proto/metaserver.pb.h" #include "curvefs/src/common/define.h" -using ::curvefs::metaserver::MetaStatusCode; using ::curvefs::RECYCLEINODEID; +using ::curvefs::metaserver::MetaStatusCode; namespace curvefs { namespace client { @@ -160,6 +160,10 @@ bool TaskExecutor::OnReturn(int retCode) { needRetry = true; break; + case MetaStatusCode::TX_INPROGRESS: + needRetry = true; + break; + default: break; } @@ -233,9 +237,8 @@ int TaskExecutor::ExcuteTask(brpc::Channel *channel, task_->cntl_.Reset(); task_->cntl_.set_timeout_ms(task_->rpcTimeoutMs); return task_->rpctask(task_->target.groupID.poolID, - task_->target.groupID.copysetID, - task_->target.partitionID, task_->target.txId, - channel, &task_->cntl_, done); + task_->target.groupID.copysetID, task_->target.partitionID, + task_->target.txId, channel, &task_->cntl_, done); } void TaskExecutor::OnSuccess() {} diff --git a/curvefs/src/client/rpcclient/task_excutor.h b/curvefs/src/client/rpcclient/task_excutor.h index 18a69346e5..23df4d1938 100644 --- a/curvefs/src/client/rpcclient/task_excutor.h +++ b/curvefs/src/client/rpcclient/task_excutor.h @@ -45,13 +45,19 @@ using ::curvefs::client::common::ExcutorOpt; using ::curvefs::client::common::MetaserverID; using ::curvefs::client::common::MetaServerOpType; using ::curvefs::common::PartitionInfo; -using ::curvefs::metaserver::MetaStatusCode; -using ::google::protobuf::RepeatedPtrField; using ::curvefs::metaserver::Inode; using ::curvefs::metaserver::InodeAttr; +using ::curvefs::metaserver::MetaStatusCode; +using ::google::protobuf::RepeatedPtrField; +using ::curvefs::client::common::FLAGS_TxVersion; namespace curvefs { namespace client { + +namespace common { + DECLARE_int32(TxVersion); +} // namespace common + namespace rpcclient { class TaskExecutorDone; @@ -60,10 +66,9 @@ MetaStatusCode ConvertToMetaStatusCode(int retcode); class TaskContext { public: - using RpcFunc = std::function; + using RpcFunc = std::function; TaskContext() = default; TaskContext(MetaServerOpType type, @@ -77,7 +82,7 @@ class TaskContext { fsID(fsid), inodeID(inodeid), streaming(streaming), - refreshTxId(refreshTxId) {} + refreshTxId(FLAGS_TxVersion == 1 ? refreshTxId : false) {} std::string TaskContextStr() { std::ostringstream oss; diff --git a/curvefs/src/client/s3/client_s3_adaptor.cpp b/curvefs/src/client/s3/client_s3_adaptor.cpp index c640945311..cf3fa56796 100644 --- a/curvefs/src/client/s3/client_s3_adaptor.cpp +++ b/curvefs/src/client/s3/client_s3_adaptor.cpp @@ -20,19 +20,44 @@ * Author: huyao */ +#include "curvefs/src/client/s3/client_s3_adaptor.h" + #include #include + #include #include #include #include "absl/memory/memory.h" -#include "curvefs/src/client/s3/client_s3_adaptor.h" +#include "curvefs/src/client/rpcclient/fsdelta_updater.h" +#include "curvefs/src/client/rpcclient/fsquota_checker.h" #include "curvefs/src/common/s3util.h" namespace curvefs { - namespace client { + +/** + * use curl -L mdsIp:port/flags/memClusterToLocal?setvalue=true + * for dynamic parameter configuration + */ +static bool pass_uint32(const char*, uint32_t) { return true; } +static bool pass_bool(const char*, bool) { return true; } +DEFINE_bool(memClusterToLocal, true, + "The data in the cache cluster download to local"); +DEFINE_validator(memClusterToLocal, &pass_bool); +DEFINE_bool(s3ToLocal, true, "The data in the s3 storage download to local"); +DEFINE_validator(s3ToLocal, &pass_bool); +DEFINE_uint32( + bigIoSize, 131072, + "read size bigger than this value will read until prefetch is finished"); +DEFINE_validator(bigIoSize, &pass_uint32); +DEFINE_uint32(bigIoRetryTimes, 100, "retry times when read big io failed"); +DEFINE_validator(bigIoRetryTimes, &pass_uint32); +DEFINE_uint32(bigIoRetryIntervalUs, 100, + "retry interval when read big io failed"); +DEFINE_validator(bigIoRetryIntervalUs, &pass_uint32); + CURVEFS_ERROR S3ClientAdaptorImpl::Init( const S3ClientAdaptorOption &option, std::shared_ptr client, @@ -49,12 +74,17 @@ S3ClientAdaptorImpl::Init( LOG(ERROR) << "chunkSize:" << chunkSize_ << " is not integral multiple for the blockSize:" << blockSize_; - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } prefetchBlocks_ = option.prefetchBlocks; prefetchExecQueueNum_ = option.prefetchExecQueueNum; diskCacheType_ = option.diskCacheOpt.diskCacheType; memCacheNearfullRatio_ = option.nearfullRatio; + FLAGS_memClusterToLocal = option.memClusterToLocal; + FLAGS_s3ToLocal = option.s3ToLocal; + FLAGS_bigIoSize = option.bigIoSize; + FLAGS_bigIoRetryTimes = option.bigIoRetryTimes; + FLAGS_bigIoRetryIntervalUs = option.bigIoRetryIntervalUs; throttleBaseSleepUs_ = option.baseSleepUs; flushIntervalSec_ = option.flushIntervalSec; chunkFlushThreads_ = option.chunkFlushThreads; @@ -100,7 +130,13 @@ S3ClientAdaptorImpl::Init( << ", readCacheMaxByte: " << option.readCacheMaxByte << ", readCacheThreads: " << option.readCacheThreads << ", nearfullRatio: " << option.nearfullRatio - << ", baseSleepUs: " << option.baseSleepUs; + << ", baseSleepUs: " << option.baseSleepUs + << ", memClusterToLocal: " << FLAGS_memClusterToLocal + << ", s3ToLocal: " << FLAGS_s3ToLocal + << ", bigIoSize: " << FLAGS_bigIoSize + << ", bigIoRetryTimes: " << FLAGS_bigIoRetryTimes + << ", bigIoRetryIntervalUs: " << FLAGS_bigIoRetryIntervalUs; + // start chunk flush threads taskPool_.Start(chunkFlushThreads_); return CURVEFS_ERROR::OK; @@ -145,7 +181,8 @@ int S3ClientAdaptorImpl::Write(uint64_t inodeId, uint64_t offset, int ret = fileCacheManager->Write(offset, length, buf); fsCacheManager_->DataCacheByteDec(length); if (s3Metric_ != nullptr) { - CollectMetrics(&s3Metric_->adaptorWrite, ret, start); + curve::client::CollectMetrics(&s3Metric_->adaptorWrite, ret, + butil::cpuwide_time_us() - start); s3Metric_->writeSize.set_value(length); } VLOG(6) << "write end inodeId: " << inodeId << ", ret: " << ret; @@ -166,7 +203,8 @@ int S3ClientAdaptorImpl::Read(uint64_t inodeId, uint64_t offset, return ret; } if (s3Metric_.get() != nullptr) { - CollectMetrics(&s3Metric_->adaptorRead, ret, start); + curve::client::CollectMetrics(&s3Metric_->adaptorRead, ret, + butil::cpuwide_time_us() - start); s3Metric_->readSize.set_value(length); } VLOG(6) << "read end offset:" << offset << ", len:" << length @@ -179,6 +217,7 @@ CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(InodeWrapper *inodeWrapper, const auto *inode = inodeWrapper->GetInodeLocked(); uint64_t fileSize = inode->length(); + int64_t deltaBytes = 0; if (size < fileSize) { VLOG(6) << "Truncate size:" << size << " less than fileSize:" << fileSize; @@ -186,12 +225,15 @@ CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(InodeWrapper *inodeWrapper, fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inode->inodeid()); fileCacheManager->TruncateCache(size, fileSize); - return CURVEFS_ERROR::OK; + deltaBytes = -1 * (fileSize - size); } else if (size == fileSize) { - return CURVEFS_ERROR::OK; } else { VLOG(6) << "Truncate size:" << size << " more than fileSize" << fileSize; + deltaBytes = size - fileSize; + if (!FsQuotaChecker::GetInstance().QuotaBytesCheck(deltaBytes)) { + return CURVEFS_ERROR::NO_SPACE; + } uint64_t offset = fileSize; uint64_t len = size - fileSize; uint64_t index = offset / chunkSize_; @@ -241,8 +283,9 @@ CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(InodeWrapper *inodeWrapper, offset += n; chunkId++; } - return CURVEFS_ERROR::OK; } + FsDeltaUpdater::GetInstance().UpdateDeltaBytes(deltaBytes); + return CURVEFS_ERROR::OK; } void S3ClientAdaptorImpl::ReleaseCache(uint64_t inodeId) { @@ -344,17 +387,10 @@ void S3ClientAdaptorImpl::InitMetrics(const std::string &fsName) { fsName_ = fsName; s3Metric_ = std::make_shared(fsName); if (HasDiskCache()) { - diskCacheManagerImpl_->InitMetrics(fsName); + diskCacheManagerImpl_->InitMetrics(fsName, s3Metric_); } } -void S3ClientAdaptorImpl::CollectMetrics(InterfaceMetric *interface, int count, - uint64_t start) { - interface->bps.count << count; - interface->qps.count << 1; - interface->latency << (butil::cpuwide_time_us() - start); -} - CURVEFS_ERROR S3ClientAdaptorImpl::FlushAllCache(uint64_t inodeId) { VLOG(6) << "FlushAllCache, inodeId:" << inodeId; FileCacheManagerPtr fileCacheManager = diff --git a/curvefs/src/client/s3/client_s3_adaptor.h b/curvefs/src/client/s3/client_s3_adaptor.h index b8ff81dc82..9c9b249649 100644 --- a/curvefs/src/client/s3/client_s3_adaptor.h +++ b/curvefs/src/client/s3/client_s3_adaptor.h @@ -34,7 +34,7 @@ #include "curvefs/src/client/common/common.h" #include "curvefs/src/client/common/config.h" #include "curvefs/src/client/filesystem/error.h" -#include "curvefs/src/client/inode_cache_manager.h" +#include "curvefs/src/client/inode_manager.h" #include "curvefs/src/client/rpcclient/mds_client.h" #include "curvefs/src/client/s3/client_s3.h" #include "curvefs/src/client/s3/client_s3_cache_manager.h" @@ -91,9 +91,7 @@ class S3ClientAdaptor { virtual FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum, uint64_t *chunkId) = 0; virtual void SetFsId(uint32_t fsId) = 0; - virtual void InitMetrics(const std::string &fsName) = 0; - virtual void CollectMetrics(InterfaceMetric *interface, int count, - uint64_t start) = 0; + virtual void InitMetrics(const std::string& fsName) = 0; virtual std::shared_ptr GetDiskCacheManager() = 0; virtual std::shared_ptr GetS3Client() = 0; virtual uint64_t GetBlockSize() = 0; @@ -118,7 +116,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { public: S3ClientAdaptorImpl() {} virtual ~S3ClientAdaptorImpl() { - LOG(INFO) << "delete S3ClientAdaptorImpl"; + VLOG(3) << "delete S3ClientAdaptorImpl"; } /** * @brief Initailize s3 client @@ -145,12 +143,15 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { CURVEFS_ERROR FlushAllCache(uint64_t inodeId); CURVEFS_ERROR FsSync(); int Stop(); + uint64_t GetBlockSize() { return blockSize_; } + uint64_t GetChunkSize() { return chunkSize_; } + uint32_t GetObjectPrefix() { return objectPrefix_; } @@ -158,32 +159,43 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { std::shared_ptr GetFsCacheManager() { return fsCacheManager_; } + uint32_t GetFlushInterval() { return flushIntervalSec_; } + std::shared_ptr GetS3Client() { return client_; } + uint32_t GetPrefetchBlocks() { return prefetchBlocks_; } + uint32_t GetDiskCacheType() { return diskCacheType_; } + bool DisableDiskCache() { return diskCacheType_ == DiskCacheType::Disable; } + bool HasDiskCache() { return diskCacheType_ != DiskCacheType::Disable; } + bool IsReadCache() { return diskCacheType_ == DiskCacheType::OnlyRead; } + bool IsReadWriteCache() { return diskCacheType_ == DiskCacheType::ReadWrite; } + std::shared_ptr GetInodeCacheManager() { return inodeManager_; } + std::shared_ptr GetDiskCacheManager() { return diskCacheManagerImpl_; } + FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum, uint64_t *chunkId); void FsSyncSignal() { @@ -191,23 +203,28 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { VLOG(3) << "fs sync signal"; cond_.notify_one(); } + void FsSyncSignalAndDataCacheInc() { std::lock_guard lk(mtx_); fsCacheManager_->DataCacheNumInc(); VLOG(3) << "fs sync signal"; cond_.notify_one(); } + void SetFsId(uint32_t fsId) { fsId_ = fsId; } + uint32_t GetFsId() { return fsId_; } + uint32_t GetPageSize() { return pageSize_; } + void InitMetrics(const std::string &fsName); - void CollectMetrics(InterfaceMetric *interface, int count, uint64_t start); + void SetDiskCache(DiskCacheType type) { diskCacheType_ = type; } @@ -255,6 +272,11 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { uint32_t flushIntervalSec_; uint32_t chunkFlushThreads_; uint32_t memCacheNearfullRatio_; + bool memClusterToLocal_; + bool s3ToLocal_; + uint32_t bigIoSize_; + uint32_t bigIoRetryTimes_; + uint32_t bigIoRetryIntervalUs_; uint32_t throttleBaseSleepUs_; uint32_t maxReadRetryIntervalMs_; uint32_t readRetryIntervalMs_; diff --git a/curvefs/src/client/s3/client_s3_cache_manager.cpp b/curvefs/src/client/s3/client_s3_cache_manager.cpp index a8414ec2cc..5345827a85 100644 --- a/curvefs/src/client/s3/client_s3_cache_manager.cpp +++ b/curvefs/src/client/s3/client_s3_cache_manager.cpp @@ -24,17 +24,28 @@ #include #include + #include -#include "absl/synchronization/blocking_counter.h" -#include "absl/cleanup/cleanup.h" +#include "absl/cleanup/cleanup.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/blocking_counter.h" +#include "curvefs/src/client/kvclient/kvclient_manager.h" +#include "curvefs/src/client/metric/client_metric.h" #include "curvefs/src/client/s3/client_s3_adaptor.h" #include "curvefs/src/common/s3util.h" -#include "curvefs/src/client/metric/client_metric.h" -#include "curvefs/src/client/kvclient/kvclient_manager.h" +#include "src/common/s3_adapter.h" namespace curvefs { namespace client { + +DECLARE_bool(memClusterToLocal); +DECLARE_bool(s3ToLocal); +DECLARE_uint32(bigIoSize); +DECLARE_uint32(bigIoRetryTimes); +DECLARE_uint32(bigIoRetryIntervalUs); + namespace common { DECLARE_bool(enableCto); } // namespace common @@ -126,9 +137,9 @@ void FsCacheManager::ReleaseFileCacheManager(uint64_t inodeId) { bool FsCacheManager::Set(DataCachePtr dataCache, std::list::iterator *outIter) { std::lock_guard lk(lruMtx_); - VLOG(3) << "lru current byte:" << lruByte_ - << ",lru max byte:" << readCacheMaxByte_ - << ", dataCache len:" << dataCache->GetLen(); + VLOG(3) << "lru current byte: " << lruByte_ + << ", lru max byte: " << readCacheMaxByte_ + << ", dataCache len: " << dataCache->GetLen(); if (readCacheMaxByte_ == 0) { return false; } @@ -221,7 +232,7 @@ CURVEFS_ERROR FsCacheManager::FsSync(bool force) { g_s3MultiManagerMetric->fileManagerNum << -1; } } - } else if (ret == CURVEFS_ERROR::NOTEXIST) { + } else if (ret == CURVEFS_ERROR::NOT_EXIST) { iter->second->ReleaseCache(); WriteLockGuard writeLockGuard(rwLock_); auto iter1 = fileCacheManagerMap_.find(iter->first); @@ -435,7 +446,7 @@ int FileCacheManager::Read(uint64_t inodeId, uint64_t offset, uint64_t length, if (memCacheMissRequest.empty()) { return actualReadLen; } - + VLOG(6) << "memcache miss request size: " << memCacheMissRequest.size(); // 2. read from localcache and remote cluster std::shared_ptr inodeWrapper; @@ -475,15 +486,29 @@ int FileCacheManager::Read(uint64_t inodeId, uint64_t offset, uint64_t length, return actualReadLen; } -bool FileCacheManager::ReadKVRequestFromLocalCache(const std::string &name, - char *databuf, +bool FileCacheManager::ReadKVRequestFromLocalCache(const std::string& name, + char* databuf, uint64_t offset, uint64_t len) { uint64_t start = butil::cpuwide_time_us(); + if (!s3ClientAdaptor_->HasDiskCache()) { + return false; + } + if (!IsCachedInLocal(name) && len >= FLAGS_bigIoSize && + s3ClientAdaptor_->GetPrefetchBlocks()) { + int retry = 0; + do { + VLOG(6) << "wait for download object: " << name; + bthread_usleep(FLAGS_bigIoRetryIntervalUs); + if (++retry >= FLAGS_bigIoRetryTimes) { + LOG(WARNING) << "download object: " << name << " timeout"; + return false; + } + } while (!IsCachedInLocal(name)); + } - bool mayCached = s3ClientAdaptor_->HasDiskCache() && - s3ClientAdaptor_->GetDiskCacheManager()->IsCached(name); - if (!mayCached) { + if (!IsCachedInLocal(name)) { + VLOG(9) << "not cachd in disk, " << name; return false; } @@ -494,8 +519,12 @@ bool FileCacheManager::ReadKVRequestFromLocalCache(const std::string &name, } if (s3ClientAdaptor_->s3Metric_) { - s3ClientAdaptor_->CollectMetrics( - &s3ClientAdaptor_->s3Metric_->adaptorReadS3, len, start); + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->adaptorReadS3, len, + butil::cpuwide_time_us() - start); + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->readFromDiskCache, len, + butil::cpuwide_time_us() - start); } return true; } @@ -508,13 +537,19 @@ bool FileCacheManager::ReadKVRequestFromRemoteCache(const std::string &name, return false; } - auto task = std::make_shared(name, databuf, offset, length); CountDownEvent event(1); - task->done = [&](const std::shared_ptr &task) { - (void)task; + GetKVCacheDone cb = [&](const std::shared_ptr& task) { + if (task->res && s3ClientAdaptor_->s3Metric_ != nullptr) { + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->readFromKVCache, task->length, + task->timer.u_elapsed()); + } event.Signal(); return; }; + + auto task = + std::make_shared(name, databuf, offset, length, cb); kvClientManager_->Get(task); event.Wait(); @@ -533,8 +568,11 @@ bool FileCacheManager::ReadKVRequestFromS3(const std::string &name, } if (s3ClientAdaptor_->s3Metric_) { - s3ClientAdaptor_->CollectMetrics( - &s3ClientAdaptor_->s3Metric_->adaptorReadS3, length, start); + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->adaptorReadS3, length, + butil::cpuwide_time_us() - start); + curve::client::CollectMetrics(&s3ClientAdaptor_->s3Metric_->readFromS3, + length, butil::cpuwide_time_us() - start); } return true; @@ -579,8 +617,34 @@ void FileCacheManager::ProcessKVRequest(const S3ReadRequest &req, char *dataBuf, const uint32_t objectPrefix = s3ClientAdaptor_->GetObjectPrefix(); GetBlockLoc(req.offset, &chunkIndex, &chunkPos, &blockIndex, &blockPos); + std::string prefetchName = curvefs::common::s3util::GenObjName( + req.chunkId, blockIndex, req.compaction, req.fsId, req.inodeId, + objectPrefix); + bool waitDownloading = false; + // if obj is in downloading, wait for it. + while (true) { + { + curve::common::LockGuard lg(downloadMtx_); + if (downloadingObj_.find(prefetchName) != downloadingObj_.end()) { + VLOG(9) << "wait for obj is in downloading: " << prefetchName + << ", size: " << downloadingObj_.size() << ", " + << downloadingObj_.size(); + waitDownloading = true; + } else { + VLOG(9) << "obj is not in downloading: " << prefetchName + << ", size: " << downloadingObj_.size() << ", " + << downloadingObj_.size(); + break; + } + } + if (waitDownloading) { + bthread_usleep(FLAGS_bigIoRetryIntervalUs); + } + } + // prefetch - if (s3ClientAdaptor_->HasDiskCache()) { + if (s3ClientAdaptor_->HasDiskCache() && !waitDownloading && + !IsCachedInLocal(prefetchName)) { PrefetchForBlock(req, fileLen, blockSize, chunkSize, blockIndex); } @@ -658,19 +722,21 @@ void FileCacheManager::ProcessKVRequest(const S3ReadRequest &req, char *dataBuf, } } -void FileCacheManager::PrefetchForBlock(const S3ReadRequest &req, +void FileCacheManager::PrefetchForBlock(const S3ReadRequest& req, uint64_t fileLen, uint64_t blockSize, uint64_t chunkSize, uint64_t startBlockIndex) { uint32_t prefetchBlocks = s3ClientAdaptor_->GetPrefetchBlocks(); + if (prefetchBlocks == 0) { + return; + } uint32_t objectPrefix = s3ClientAdaptor_->GetObjectPrefix(); std::vector> prefetchObjs; - uint64_t blockIndex = startBlockIndex; for (uint32_t i = 0; i < prefetchBlocks; i++) { std::string name = curvefs::common::s3util::GenObjName( - req.chunkId, blockIndex, req.compaction, - req.fsId, req.inodeId, objectPrefix); + req.chunkId, blockIndex, req.compaction, req.fsId, req.inodeId, + objectPrefix); uint64_t maxReadLen = (blockIndex + 1) * blockSize; uint64_t needReadLen = maxReadLen > fileLen ? fileLen - blockIndex * blockSize : blockSize; @@ -683,18 +749,26 @@ void FileCacheManager::PrefetchForBlock(const S3ReadRequest &req, } } - PrefetchS3Objs(prefetchObjs); + // It is configurable whether to write to local cache or not + if (!kvClientManager_ && FLAGS_s3ToLocal) { + // get from s3 directly + PrefetchS3Objs(prefetchObjs, true); + } else if (FLAGS_memClusterToLocal) { + // get from memcached first, if failed, get from s3 + PrefetchS3Objs(prefetchObjs, false); + } } class AsyncPrefetchCallback { public: - AsyncPrefetchCallback(uint64_t inode, S3ClientAdaptorImpl *s3Client) - : inode_(inode), s3Client_(s3Client) {} + AsyncPrefetchCallback(uint64_t inode, S3ClientAdaptorImpl* s3Client, + bool fromS3) + : inode_(inode), s3Client_(s3Client), fromS3_(fromS3) {} - void operator()(const S3Adapter *, - const std::shared_ptr &context) { + void operator()(const S3Adapter*, + const std::shared_ptr& context) { VLOG(9) << "prefetch end: " << context->key << ", len " << context->len - << "actual len: " << context->actualLen; + << "actual len: " << context->actualLen << ", " << fromS3_; std::unique_ptr guard(context->buf); auto fileCache = s3Client_->GetFsCacheManager()->FindFileCacheManager(inode_); @@ -706,31 +780,55 @@ class AsyncPrefetchCallback { return; } - if (context->retCode != 0) { - LOG(WARNING) << "prefetch failed, key: " << context->key; + if (context->retCode != 0 && !fromS3_) { + VLOG(6) << "failed and then get from s3, key: " << context->key; + std::vector> prefetchObjs; + prefetchObjs.push_back(std::make_pair(context->key, context->len)); + fileCache->PrefetchS3Objs(prefetchObjs); + curve::common::LockGuard lg(fileCache->downloadMtx_); + fileCache->downloadingObj_.erase(context->key); + return; + } else if (context->retCode != 0 && fromS3_) { + curve::common::LockGuard lg(fileCache->downloadMtx_); + fileCache->downloadingObj_.erase(context->key); + LOG_EVERY_SECOND(INFO) << "prefetch failed, key: " << context->key; return; } + if (s3Client_->s3Metric_ != nullptr) { + // read from s3 + metric::AsyncContextCollectMetrics(s3Client_->s3Metric_, context); + } + uint64_t start = butil::cpuwide_time_us(); int ret = s3Client_->GetDiskCacheManager()->WriteReadDirect( context->key, context->buf, context->actualLen); if (ret < 0) { LOG_EVERY_SECOND(INFO) << "write read directly failed, key: " << context->key; } + if (s3Client_->s3Metric_ != nullptr) { + // prefetch to disk + curve::client::CollectMetrics( + &s3Client_->s3Metric_->writeToDiskCache, context->actualLen, + butil::cpuwide_time_us() - start); + } { curve::common::LockGuard lg(fileCache->downloadMtx_); fileCache->downloadingObj_.erase(context->key); } + VLOG(9) << "prefetch success: " << context->key; } private: const uint64_t inode_; S3ClientAdaptorImpl *s3Client_; + bool fromS3_; }; void FileCacheManager::PrefetchS3Objs( - const std::vector> &prefetchObjs) { - for (auto &obj : prefetchObjs) { + const std::vector>& prefetchObjs, + bool fromS3) { + for (auto& obj : prefetchObjs) { std::string name = obj.first; uint64_t readLen = obj.second; curve::common::LockGuard lg(downloadMtx_); @@ -739,30 +837,32 @@ void FileCacheManager::PrefetchS3Objs( << ", size: " << downloadingObj_.size(); continue; } - if (s3ClientAdaptor_->GetDiskCacheManager()->IsCached(name)) { + if (IsCachedInLocal(name)) { VLOG(9) << "downloading is exist in cache: " << name << ", size: " << downloadingObj_.size(); continue; } VLOG(9) << "download start: " << name - << ", size: " << downloadingObj_.size(); + << ", size: " << downloadingObj_.size() + << ", from s3: " << fromS3; downloadingObj_.emplace(name); - auto inode = inode_; - auto s3ClientAdaptor = s3ClientAdaptor_; - auto task = [name, inode, s3ClientAdaptor, readLen]() { - char *dataCacheS3 = new char[readLen]; - auto context = std::make_shared(); - context->key = name; - context->buf = dataCacheS3; - context->offset = 0; - context->len = readLen; - context->cb = AsyncPrefetchCallback{inode, s3ClientAdaptor}; - VLOG(9) << "prefetch start: " << context->key - << ", len: " << context->len; - s3ClientAdaptor->GetS3Client()->DownloadAsync(context); - }; - s3ClientAdaptor_->PushAsyncTask(task); + char* dataCacheS3 = new char[readLen]; + VLOG(9) << "prefetch start: " << name << ", len: " << readLen; + if (fromS3) { + auto context = std::make_shared( + name, dataCacheS3, 0, readLen, + AsyncPrefetchCallback{inode_, s3ClientAdaptor_, true}); + auto task = [this, context]() { + s3ClientAdaptor_->GetS3Client()->DownloadAsync(context); + }; + s3ClientAdaptor_->PushAsyncTask(task); + } else { + auto context = std::make_shared( + name, dataCacheS3, 0, readLen, + AsyncPrefetchCallback{inode_, s3ClientAdaptor_, false}); + kvClientManager_->Enqueue(context); + } } return; } @@ -1018,7 +1118,8 @@ void FileCacheManager::ReleaseCache() { } chunkCacheMap_.clear(); - g_s3MultiManagerMetric->chunkManagerNum << -1 * chunNum; + g_s3MultiManagerMetric->chunkManagerNum + << -1 * static_cast(chunNum); return; } @@ -1119,6 +1220,10 @@ CURVEFS_ERROR FileCacheManager::Flush(bool force, bool toS3) { return ret; } +bool FileCacheManager::IsCachedInLocal(const std::string name) { + return s3ClientAdaptor_->GetDiskCacheManager()->IsCached(name); +} + void ChunkCacheManager::ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen, char *dataBuf, uint64_t dataBufOffset, @@ -1276,7 +1381,7 @@ void ChunkCacheManager::ReadByReadCache(uint64_t chunkPos, uint64_t readLen, ReadLockGuard readLockGuard(rwLockRead_); VLOG(9) << "ReadByReadCache chunkPos:" << chunkPos << ",readLen:" << readLen - << ",dataBufOffset:" << dataBufOffset; + << ",dataBufOffset:" << dataBufOffset << "chunkIndex: " << index_; if (dataRCacheMap_.empty()) { VLOG(9) << "dataRCacheMap_ is empty"; ReadRequest request; @@ -1488,7 +1593,7 @@ DataCachePtr ChunkCacheManager::FindWriteableDataCache( } std::vector::iterator iterDel = waitDelVec.begin(); - for (; iterDel != waitDelVec.end(); iterDel++) { + for (; iterDel != waitDelVec.end(); ++iterDel) { auto iter = dataWCacheMap_.find(*iterDel); VLOG(9) << "delete data cache chunkPos:" << iter->second->GetChunkPos() @@ -1555,7 +1660,8 @@ void ChunkCacheManager::AddReadDataCache(DataCachePtr dataCache) { uint64_t actualLen = (*dcpIter)->GetActualLen(); if (s3ClientAdaptor_->GetFsCacheManager()->Delete(dcpIter)) { g_s3MultiManagerMetric->readDataCacheNum << -1; - g_s3MultiManagerMetric->readDataCacheByte << -1 * actualLen; + g_s3MultiManagerMetric->readDataCacheByte + << -1 * static_cast(actualLen); dataRCacheMap_.erase(iter); } } @@ -1648,7 +1754,8 @@ void ChunkCacheManager::TruncateReadCache(uint64_t chunkPos) { if ((dcChunkPos + dcLen) > chunkPos) { if (s3ClientAdaptor_->GetFsCacheManager()->Delete(rIter->second)) { g_s3MultiManagerMetric->readDataCacheNum << -1; - g_s3MultiManagerMetric->readDataCacheByte << -1 * dcActualLen; + g_s3MultiManagerMetric->readDataCacheByte + << -1 * static_cast(dcActualLen); dataRCacheMap_.erase(next(rIter).base()); } } else { @@ -1670,7 +1777,6 @@ void ChunkCacheManager::ReleaseWriteDataCache(const DataCachePtr &dataCache) { CURVEFS_ERROR ChunkCacheManager::Flush(uint64_t inodeId, bool force, bool toS3) { - std::map tmp; curve::common::LockGuard lg(flushMtx_); CURVEFS_ERROR ret = CURVEFS_ERROR::OK; // DataCachePtr dataCache; @@ -1702,7 +1808,7 @@ CURVEFS_ERROR ChunkCacheManager::Flush(uint64_t inodeId, bool force, assert(flushingDataCache_->IsDirty()); do { ret = flushingDataCache_->Flush(inodeId, toS3); - if (ret == CURVEFS_ERROR::NOTEXIST) { + if (ret == CURVEFS_ERROR::NOT_EXIST) { LOG(WARNING) << "dataCache flush failed. ret:" << ret << ",index:" << index_ << ",data chunkpos:" << flushingDataCache_->GetChunkPos(); @@ -2055,7 +2161,7 @@ void DataCache::Write(uint64_t chunkPos, uint64_t len, const char *data, } else { std::vector::const_iterator iter = mergeDataCacheVer.begin(); - for (; iter != mergeDataCacheVer.end(); iter++) { + for (; iter != mergeDataCacheVer.end(); ++iter) { /* ------ ------ DataCache --------------------- WriteData @@ -2105,7 +2211,7 @@ void DataCache::Write(uint64_t chunkPos, uint64_t len, const char *data, } else { std::vector::const_iterator iter = mergeDataCacheVer.begin(); - for (; iter != mergeDataCacheVer.end(); iter++) { + for (; iter != mergeDataCacheVer.end(); ++iter) { /* ------ ------ DataCache ---------------- WriteData @@ -2335,19 +2441,24 @@ CURVEFS_ERROR DataCache::PrepareFlushTasks( // generate flush to disk or s3 task std::string objectName = curvefs::common::s3util::GenObjName( *chunkId, blockIndex, 0, fsId, inodeId, objectPrefix); - auto context = std::make_shared(); - context->key = objectName; - context->buffer = data + (*writeOffset); - context->bufferSize = curentLen; - context->startTime = butil::cpuwide_time_us(); + auto context = std::make_shared( + objectName, data + (*writeOffset), curentLen); + // context->type and context->cb will set in FlushTaskExecute s3Tasks->emplace_back(context); // generate flush to kvcache task if (kvClientManager_) { - auto task = std::make_shared(); - task->key = objectName; - task->value = data + (*writeOffset); - task->length = curentLen; + SetKVCacheDone cb = + [this](const std::shared_ptr& setTask) { + if (setTask->res && + s3ClientAdaptor_->s3Metric_ != nullptr) { + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->writeToKVCache, + setTask->length, setTask->timer.u_elapsed()); + } + }; + auto task = std::make_shared( + objectName, data + (*writeOffset), curentLen, cb); kvCacheTasks->emplace_back(task); } @@ -2374,6 +2485,12 @@ CachePolicy DataCache::GetCachePolicy(bool toS3) { } } +/** + * @brief Decide whether to use diskcache or s3 based on the cache strategy +(note that kvcache is determined based on kvClientManager_). If the upload +fails, no matter the original type is diskcache or s3, use s3 to retry. + * + */ void DataCache::FlushTaskExecute( CachePolicy cachePolicy, const std::vector> &s3Tasks, @@ -2385,12 +2502,11 @@ void DataCache::FlushTaskExecute( CountDownEvent kvTaskEvent(kvPendingTaskCal); PutObjectAsyncCallBack s3cb = - [&](const std::shared_ptr &context) { - if (context->retCode == 0) { - if (s3ClientAdaptor_->s3Metric_.get() != nullptr) { - s3ClientAdaptor_->CollectMetrics( - &s3ClientAdaptor_->s3Metric_->adaptorWriteS3, - context->bufferSize, context->startTime); + [&](const std::shared_ptr& context) { + if (context->retCode >= 0) { + if (s3ClientAdaptor_->s3Metric_ != nullptr) { + metric::AsyncContextCollectMetrics( + s3ClientAdaptor_->s3Metric_, context); } if (CachePolicy::RCache == cachePolicy) { @@ -2407,11 +2523,18 @@ void DataCache::FlushTaskExecute( } LOG(WARNING) << "Put object failed, key: " << context->key; + // Retry using s3 no matter what the original was + context->type = curve::common::ContextType::S3; s3ClientAdaptor_->GetS3Client()->UploadAsync(context); }; SetKVCacheDone kvdone = [&](const std::shared_ptr &task) { kvTaskEvent.Signal(); + if (task->res && s3ClientAdaptor_->s3Metric_ != nullptr) { + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->writeToKVCache, task->length, + task->timer.u_elapsed()); + } return; }; @@ -2422,8 +2545,10 @@ void DataCache::FlushTaskExecute( [&](const std::shared_ptr &context) { context->cb = s3cb; if (CachePolicy::WRCache == cachePolicy) { + context->type = curve::common::ContextType::Disk; s3ClientAdaptor_->GetDiskCacheManager()->Enqueue(context); } else { + context->type = curve::common::ContextType::S3; s3ClientAdaptor_->GetS3Client()->UploadAsync(context); } }); @@ -2511,7 +2636,7 @@ void FsCacheManager::ReadCacheReleaseExecutor::Stop() { FsCacheManager::ReadCacheReleaseExecutor::~ReadCacheReleaseExecutor() { Stop(); t_.join(); - LOG(INFO) << "ReadCacheReleaseExecutor stopped"; + VLOG(3) << "ReadCacheReleaseExecutor stopped"; } void FsCacheManager::ReadCacheReleaseExecutor::Release( diff --git a/curvefs/src/client/s3/client_s3_cache_manager.h b/curvefs/src/client/s3/client_s3_cache_manager.h index d27aa308c5..df3d665ac4 100644 --- a/curvefs/src/client/s3/client_s3_cache_manager.h +++ b/curvefs/src/client/s3/client_s3_cache_manager.h @@ -27,19 +27,19 @@ #include #include #include +#include #include #include #include #include -#include #include "curvefs/proto/metaserver.pb.h" #include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/inode_wrapper.h" +#include "curvefs/src/client/kvclient/kvclient_manager.h" #include "curvefs/src/client/s3/client_s3.h" #include "src/common/concurrent/concurrent.h" #include "src/common/concurrent/task_thread_pool.h" -#include "curvefs/src/client/kvclient/kvclient_manager.h" -#include "curvefs/src/client/inode_wrapper.h" using curve::common::ReadLockGuard; using curve::common::RWLock; @@ -370,12 +370,13 @@ class FileCacheManager { void WriteChunk(uint64_t index, uint64_t chunkPos, uint64_t writeLen, const char *dataBuf); void GenerateS3Request(ReadRequest request, - const S3ChunkInfoList &s3ChunkInfoList, - char *dataBuf, std::vector *requests, + const S3ChunkInfoList& s3ChunkInfoList, + char* dataBuf, std::vector* requests, uint64_t fsId, uint64_t inodeId); void PrefetchS3Objs( - const std::vector> &prefetchObjs); + const std::vector>& prefetchObjs, + bool fromS3 = true); void HandleReadRequest(const ReadRequest &request, const S3ChunkInfo &s3ChunkInfo, @@ -403,9 +404,11 @@ class FileCacheManager { // miss read from memory read/write cache, need read from // kv(localdisk/remote cache/s3) - int GenerateKVRequest(const std::shared_ptr &inodeWrapper, - const std::vector &readRequest, - char *dataBuf, std::vector *kvRequest); + int GenerateKVRequest(const std::shared_ptr& inodeWrapper, + const std::vector& readRequest, + char* dataBuf, std::vector* kvRequest); + + bool IsCachedInLocal(const std::string name); enum class ReadStatus { OK = 0, diff --git a/curvefs/src/client/s3/disk_cache_base.cpp b/curvefs/src/client/s3/disk_cache_base.cpp index 3bfe3d8a73..1834a5f868 100644 --- a/curvefs/src/client/s3/disk_cache_base.cpp +++ b/curvefs/src/client/s3/disk_cache_base.cpp @@ -20,15 +20,19 @@ * Author: hzwuhongsong */ -#include -#include +#include "curvefs/src/client/s3/disk_cache_base.h" + +#include #include #include +#include +#include +#include #include -#include -#include +#include -#include "curvefs/src/client/s3/disk_cache_base.h" +#include +#include namespace curvefs { @@ -69,7 +73,7 @@ int DiskCacheBase::CreateIoDir(bool writreDir) { return 0; } -bool DiskCacheBase::IsFileExist(const std::string file) { +bool DiskCacheBase::IsFileExist(const std::string& file) { struct stat statFile; int ret; ret = posixWrapper_->stat(file.c_str(), &statFile); @@ -86,7 +90,7 @@ std::string DiskCacheBase::GetCacheIoFullDir() { return fullPath; } -int DiskCacheBase::CreateDir(const std::string dir) { +int DiskCacheBase::CreateDir(const std::string& dir) { size_t p = dir.find_last_of('/'); std::string dirPath = dir; if (p != -1ULL) { @@ -133,33 +137,46 @@ int DiskCacheBase::LoadAllCacheFile(std::set *cachedObj) { std::set *cacheObj) -> bool { DIR *dir; struct dirent *ent; - std::string fileName, nextdir; - if ((dir = posixWrapper_->opendir(path.c_str())) != NULL) { - while ((ent = posixWrapper_->readdir(dir)) != NULL) { - VLOG(9) << "LoadAllCacheFile obj, name = " << ent->d_name; - if (strncmp(ent->d_name, ".", 1) == 0 || - strncmp(ent->d_name, "..", 2) == 0) { - continue; - } else if (ent->d_type == 8) { + dir = posixWrapper_->opendir(path.c_str()); + if (dir == nullptr) { + LOG(ERROR) << "LoadAllCacheFile Opendir error, path =" << path; + return false; + } + while ((ent = posixWrapper_->readdir(dir)) != nullptr) { + VLOG(9) << "LoadAllCacheFile obj, name = " << ent->d_name; + if (strncmp(ent->d_name, ".", 1) == 0 || + strncmp(ent->d_name, "..", 2) == 0) { + continue; + } + std::string fileName, nextdir; + switch (GetFileType(path + '/' + ent->d_name)) { + case FileType::kNotExist: + // Generally, the path has been deleted or moved, + // and it is temporarily considered to be a file. + // In most cases it is from write to read + case FileType::kFile: fileName = std::string(ent->d_name); VLOG(9) << "LoadAllCacheFile obj, name = " << fileName; cacheObj->emplace(fileName); - } else { + break; + case FileType::kDir: nextdir = std::string(ent->d_name); nextdir = path + '/' + nextdir; if (!listDir(nextdir, cacheObj)) { return false; } - } - } - int ret = posixWrapper_->closedir(dir); - if (ret < 0) { - LOG(ERROR) << "close dir " << dir << ", error = " << errno; + break; + case FileType::kOther: + case FileType::kError: + default: + break; } - return ret >= 0; } - LOG(ERROR) << "LoadAllCacheFile Opendir error, path =" << path; - return false; + int ret = posixWrapper_->closedir(dir); + if (ret < 0) { + LOG(ERROR) << "close dir " << dir << ", error = " << errno; + } + return ret >= 0; }; ret = listDir(cachePath, cachedObj); if (!ret) { @@ -169,5 +186,26 @@ int DiskCacheBase::LoadAllCacheFile(std::set *cachedObj) { return 0; } +DiskCacheBase::FileType DiskCacheBase::GetFileType(const std::string& path) { + struct stat statbuf; + if (stat(path.c_str(), &statbuf) != 0) { + // stat path error or path not exist + // Maybe it was deleted + if (errno == ENOENT) { + return FileType::kNotExist; + } + LOG_EVERY_N(WARNING, 1000) + << "stat path error, path = " << path << ", errno = " << errno; + return FileType::kError; + } + if (S_ISREG(statbuf.st_mode)) { + return FileType::kFile; + } + if (S_ISDIR(statbuf.st_mode)) { + return FileType::kDir; + } + return FileType::kOther; +} + } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/s3/disk_cache_base.h b/curvefs/src/client/s3/disk_cache_base.h index 5d7d34417b..637154e15f 100644 --- a/curvefs/src/client/s3/disk_cache_base.h +++ b/curvefs/src/client/s3/disk_cache_base.h @@ -51,8 +51,28 @@ class DiskCacheBase { * @brief Create Read/Write Cache Dir. */ virtual int CreateIoDir(bool writreDir); - virtual bool IsFileExist(const std::string file); - virtual int CreateDir(const std::string name); + virtual bool IsFileExist(const std::string& file); + virtual int CreateDir(const std::string& name); + + enum FileType { + kError = -1, + kNotExist = 0, + kOther = 1, + kFile = 2, + kDir = 3, + }; + /** + * @brief + * + * @param path + * @return FileType + * kError = -1, + * kNotExist = 0, + * kOther = 1, + * kFile = 2, + * kDir = 3, + */ + virtual FileType GetFileType(const std::string& path); /** * @brief Get Read/Write Cache full Dir(include CacheDir_). */ diff --git a/curvefs/src/client/s3/disk_cache_manager.cpp b/curvefs/src/client/s3/disk_cache_manager.cpp index fb0dae3b1e..33a09e307d 100644 --- a/curvefs/src/client/s3/disk_cache_manager.cpp +++ b/curvefs/src/client/s3/disk_cache_manager.cpp @@ -19,15 +19,21 @@ * Created Date: 21-08-13 * Author: hzwuhongsong */ -#include + +#include "curvefs/src/client/s3/disk_cache_manager.h" + #include -#include +#include +#include + +#include #include -#include #include +#include +#include +#include "curvefs/src/client/metric/client_metric.h" #include "curvefs/src/client/s3/client_s3_adaptor.h" -#include "curvefs/src/client/s3/disk_cache_manager.h" #include "curvefs/src/common/s3util.h" namespace curvefs { @@ -38,6 +44,8 @@ namespace client { * use curl -L mdsIp:port/flags/avgFlushBytes?setvalue=true * for dynamic parameter configuration */ +static bool pass_uint32(const char *, uint32_t) { return true; } +static bool ValidateRatio(const char *, uint32_t ratio) { return ratio <= 100; } static bool pass_uint64(const char *, uint64_t) { return true; } DEFINE_uint64(avgFlushBytes, 83886080, "the write throttle bps of disk cache"); DEFINE_validator(avgFlushBytes, &pass_uint64); @@ -53,6 +61,19 @@ DEFINE_validator(avgReadFileBytes, &pass_uint64); DEFINE_uint64(avgReadFileIops, 0, "the read throttle iops of disk cache"); DEFINE_validator(avgReadFileIops, &pass_uint64); +DEFINE_uint32(diskNearFullRatio, 70, "the nearfull ratio of disk cache"); +DEFINE_validator(diskNearFullRatio, &ValidateRatio); +DEFINE_uint32(diskFullRatio, 90, "the nearfull ratio of disk cache"); +DEFINE_validator(diskFullRatio, &ValidateRatio); +DEFINE_uint32(diskTrimRatio, 50, "the trim ratio when disk nearfull"); +DEFINE_validator(diskTrimRatio, &ValidateRatio); +DEFINE_uint32(diskTrimCheckIntervalSec, 5, "trim check interval seconds"); +DEFINE_validator(diskTrimCheckIntervalSec, &pass_uint32); +DEFINE_uint64(diskMaxUsableSpaceBytes, 107374182400, "max space bytes can use"); +DEFINE_validator(diskMaxUsableSpaceBytes, &pass_uint64); +DEFINE_uint64(diskMaxFileNums, 1000000, "max file nums can owner"); +DEFINE_validator(diskMaxFileNums, &pass_uint64); + DiskCacheManager::DiskCacheManager(std::shared_ptr posixWrapper, std::shared_ptr cacheWrite, std::shared_ptr cacheRead) { @@ -62,10 +83,7 @@ DiskCacheManager::DiskCacheManager(std::shared_ptr posixWrapper, isRunning_ = false; usedBytes_ = 0; diskFsUsedRatio_ = 0; - fullRatio_ = 0; - safeRatio_ = 0; diskUsedInit_ = false; - maxUsableSpaceBytes_ = 0; objectPrefix_ = 0; // cannot limit the size, // because cache is been delete must after upload to s3 @@ -80,12 +98,13 @@ int DiskCacheManager::Init(std::shared_ptr client, client_ = client; option_ = option; - trimCheckIntervalSec_ = option.diskCacheOpt.trimCheckIntervalSec; - fullRatio_ = option.diskCacheOpt.fullRatio; - safeRatio_ = option.diskCacheOpt.safeRatio; + FLAGS_diskTrimCheckIntervalSec = option.diskCacheOpt.trimCheckIntervalSec; + FLAGS_diskFullRatio = option.diskCacheOpt.fullRatio; + FLAGS_diskNearFullRatio = option.diskCacheOpt.safeRatio; + FLAGS_diskTrimRatio = option.diskCacheOpt.trimRatio; cacheDir_ = option.diskCacheOpt.cacheDir; - maxUsableSpaceBytes_ = option.diskCacheOpt.maxUsableSpaceBytes; - maxFileNums_ = option.diskCacheOpt.maxFileNums; + FLAGS_diskMaxUsableSpaceBytes = option.diskCacheOpt.maxUsableSpaceBytes; + FLAGS_diskMaxFileNums = option.diskCacheOpt.maxFileNums; cmdTimeoutSec_ = option.diskCacheOpt.cmdTimeoutSec; objectPrefix_ = option.objectPrefix; cacheWrite_->Init(client_, posixWrapper_, cacheDir_, objectPrefix_, @@ -114,7 +133,7 @@ int DiskCacheManager::Init(std::shared_ptr client, // start trim thread TrimRun(); - SetDiskFsUsedRatio(); + UpdateDiskFsUsedRatio(); FLAGS_avgFlushIops = option_.diskCacheOpt.avgFlushIops; FLAGS_avgFlushBytes = option_.diskCacheOpt.avgFlushBytes; @@ -122,16 +141,16 @@ int DiskCacheManager::Init(std::shared_ptr client, FLAGS_burstSecs = option_.diskCacheOpt.burstSecs; FLAGS_avgReadFileIops = option_.diskCacheOpt.avgReadFileIops; FLAGS_avgReadFileBytes = option_.diskCacheOpt.avgReadFileBytes; - InitQosParam(); LOG(INFO) << "DiskCacheManager init success. " << ", cache dir is: " << cacheDir_ - << ", maxUsableSpaceBytes is: " << maxUsableSpaceBytes_ - << ", maxFileNums is: " << maxFileNums_ + << ", maxUsableSpaceBytes is: " << FLAGS_diskMaxUsableSpaceBytes + << ", maxFileNums is: " << FLAGS_diskMaxFileNums << ", cmdTimeoutSec is: " << cmdTimeoutSec_ - << ", safeRatio is: " << safeRatio_ - << ", fullRatio is: " << fullRatio_ + << ", safeRatio is: " << FLAGS_diskNearFullRatio + << ", fullRatio is: " << FLAGS_diskFullRatio + << ", trimRatio is: " << FLAGS_diskTrimRatio << ", disk used bytes: " << GetDiskUsedbytes(); return 0; } @@ -179,9 +198,6 @@ bool DiskCacheManager::IsCacheClean() { int DiskCacheManager::UmountDiskCache() { LOG(INFO) << "umount disk cache."; - if (diskInitThread_.joinable()) { - diskInitThread_.join(); - } TrimStop(); cacheWrite_->AsyncUploadStop(); LOG_IF(ERROR, !IsCacheClean()) << "umount disk cache error."; @@ -231,8 +247,9 @@ int DiskCacheManager::WriteDiskFile(const std::string fileName, const char *buf, // write throttle diskCacheThrottle_.Add(false, length); int ret = cacheWrite_->WriteDiskFile(fileName, buf, length, force); - if (ret > 0) - AddDiskUsedBytes(ret); + if (ret > 0) { + UpdateDiskUsedBytes(ret); + } return ret; } @@ -252,8 +269,9 @@ int DiskCacheManager::WriteReadDirect(const std::string fileName, // write hrottle diskCacheThrottle_.Add(false, length); int ret = cacheRead_->WriteDiskFile(fileName, buf, length); - if (ret > 0) - AddDiskUsedBytes(ret); + if (ret > 0) { + UpdateDiskUsedBytes(ret); + } return ret; } @@ -263,7 +281,7 @@ int DiskCacheManager::LinkWriteToRead(const std::string fileName, return cacheRead_->LinkWriteToRead(fileName, fullWriteDir, fullReadDir); } -int64_t DiskCacheManager::SetDiskFsUsedRatio() { +int64_t DiskCacheManager::UpdateDiskFsUsedRatio() { struct statfs stat; if (posixWrapper_->statfs(cacheDir_.c_str(), &stat) == -1) { LOG_EVERY_N(WARNING, 100) @@ -283,6 +301,7 @@ int64_t DiskCacheManager::SetDiskFsUsedRatio() { } int64_t usedPercent = 100 * usedBytes / (usedBytes + availableBytes) + 1; diskFsUsedRatio_.store(usedPercent); + totalBytes_.store(totalBytes); return usedPercent; } @@ -303,8 +322,6 @@ void DiskCacheManager::SetDiskInitUsedBytes() { return; } usedBytes_.fetch_add(usedBytes); - if (metric_.get() != nullptr) - metric_->diskUsedBytes.set_value(usedBytes_); diskUsedInit_.store(true); VLOG(9) << "cache disk used size is: " << result; return; @@ -313,53 +330,77 @@ void DiskCacheManager::SetDiskInitUsedBytes() { bool DiskCacheManager::IsDiskCacheFull() { int64_t ratio = diskFsUsedRatio_.load(); uint64_t usedBytes = GetDiskUsedbytes(); - if (ratio >= fullRatio_ || usedBytes >= maxUsableSpaceBytes_) { + if (ratio >= FLAGS_diskFullRatio || + usedBytes >= FLAGS_diskMaxUsableSpaceBytes || + IsExceedFileNums(kRatioLevel)) { VLOG(6) << "disk cache is full" - << ", ratio is: " << ratio << ", fullRatio is: " - << fullRatio_ << ", used bytes is: " << usedBytes; + << ", ratio is: " << ratio + << ", fullRatio is: " << FLAGS_diskFullRatio + << ", used bytes is: " << usedBytes; waitIntervalSec_.StopWait(); return true; } - if (!IsDiskCacheSafe()) { + if (!IsDiskCacheSafe(kRatioLevel)) { VLOG(6) << "wake up trim thread."; waitIntervalSec_.StopWait(); } return false; } -bool DiskCacheManager::IsDiskCacheSafe() { - if (IsExceedFileNums()) { +bool DiskCacheManager::IsDiskCacheSafe(uint32_t baseRatio) { + if (IsExceedFileNums(baseRatio)) { return false; } int64_t ratio = diskFsUsedRatio_.load(); uint64_t usedBytes = GetDiskUsedbytes(); - if ((usedBytes < (safeRatio_ * maxUsableSpaceBytes_ / 100)) - && (ratio < safeRatio_)) { + if ((usedBytes < (FLAGS_diskNearFullRatio * FLAGS_diskMaxUsableSpaceBytes / + kRatioLevel * baseRatio / kRatioLevel)) && + (ratio < FLAGS_diskNearFullRatio * baseRatio / kRatioLevel)) { VLOG(9) << "disk cache is safe" << ", usedBytes is: " << usedBytes - << ", use ratio is: " << ratio; + << ", use ratio is: " << ratio + << ", baseRatio is: " << baseRatio; return true; } - VLOG(6) << "disk cache is not safe" - << ", usedBytes is: " << usedBytes - << ", use ratio is: " << ratio; + VLOG_EVERY_N(6, 1000) << "disk cache is not safe" + << ", usedBytes is: " << usedBytes << ", limit is " + << FLAGS_diskNearFullRatio * FLAGS_diskMaxUsableSpaceBytes / + kRatioLevel * baseRatio / kRatioLevel + << ", use ratio is: " << ratio << ", limit is: " + << FLAGS_diskNearFullRatio * baseRatio / kRatioLevel; return false; } // TODO(wuhongsong): // See Also: https://github.com/opencurve/curve/issues/1534 -bool DiskCacheManager::IsExceedFileNums() { +bool DiskCacheManager::IsExceedFileNums(uint32_t baseRatio) { uint64_t fileNums = cachedObjName_->Size(); - if (fileNums >= maxFileNums_) { + if (fileNums >= FLAGS_diskMaxFileNums * baseRatio / kRatioLevel) { + VLOG_EVERY_N(9, 1000) << "disk cache file nums is exceed" + << ", fileNums is: " << fileNums + << ", maxFileNums is: " << FLAGS_diskMaxFileNums + << ", baseRatio is: " << baseRatio; return true; } return false; } + +// +// ok nearfull full +// |------------|-------------------|----------------------| +// 0 trimRatio*safeRatio safeRatio fullRatio +// +// 1. 0<=oknearfull does not clean up +// If the status is full or +// full->nearfull clean up void DiskCacheManager::TrimCache() { - const std::chrono::seconds sleepSec(trimCheckIntervalSec_); + const std::chrono::seconds sleepSec(FLAGS_diskTrimCheckIntervalSec); LOG(INFO) << "trim function start."; - waitIntervalSec_.Init(trimCheckIntervalSec_ * 1000); + waitIntervalSec_.Init(FLAGS_diskTrimCheckIntervalSec * 1000); // trim will start after get the disk size while (!IsDiskUsedInited()) { if (!isRunning_) { @@ -369,13 +410,15 @@ void DiskCacheManager::TrimCache() { } // 1. check cache disk usage every sleepSec seconds. // 2. if cache disk is full, - // then remove disk file until cache disk is lower than safeRatio_. + // then remove disk file until cache disk is lower than + // FLAGS_diskNearFullRatio*FLAGS_trimRatio/kRatioLevel. std::string cacheReadFullDir, cacheWriteFullDir, cacheReadFile, cacheWriteFile, cacheKey; cacheReadFullDir = GetCacheReadFullDir(); cacheWriteFullDir = GetCacheWriteFullDir(); while (true) { - SetDiskFsUsedRatio(); + UpdateDiskFsUsedRatio(); + waitIntervalSec_.Init(FLAGS_diskTrimCheckIntervalSec * 1000); waitIntervalSec_.WaitForNextExcution(); if (!isRunning_) { LOG(INFO) << "trim thread end."; @@ -383,56 +426,66 @@ void DiskCacheManager::TrimCache() { } VLOG(9) << "trim thread wake up."; InitQosParam(); - while (!IsDiskCacheSafe()) { - SetDiskFsUsedRatio(); - if (!cachedObjName_->GetBack(&cacheKey)) { - VLOG(9) << "obj is empty"; - break; + if (!IsDiskCacheSafe(kRatioLevel)) { + while (!IsDiskCacheSafe(FLAGS_diskTrimRatio)) { + if (!isRunning_) { + LOG(INFO) << "trim thread end."; + return; + } + UpdateDiskFsUsedRatio(); + if (!cachedObjName_->GetBack(&cacheKey)) { + VLOG_EVERY_N(9, 1000) << "obj is empty"; + break; + } + + uint64_t start = butil::cpuwide_time_us(); + VLOG(6) << "obj will be removed01: " << cacheKey; + cacheReadFile = cacheReadFullDir + "/" + + curvefs::common::s3util::GenPathByObjName( + cacheKey, objectPrefix_); + cacheWriteFile = cacheWriteFullDir + "/" + + curvefs::common::s3util::GenPathByObjName( + cacheKey, objectPrefix_); + struct stat statFile; + int ret = 0; + ret = posixWrapper_->stat(cacheWriteFile.c_str(), &statFile); + // if file has not been uploaded to S3, + // but remove the cache read file, + // then read will fail when do cache read, + // and then it cannot load the file from S3. + // so read is fail. + if (ret == 0) { + VLOG(1) << "do not remove this disk file" + << ", file has not been uploaded to S3." + << ", file is: " << cacheKey; + usleep(1000); + continue; + } + cachedObjName_->Remove(cacheKey); + struct stat statReadFile; + ret = posixWrapper_->stat(cacheReadFile.c_str(), &statReadFile); + if (ret != 0) { + VLOG(0) << "stat disk file error" + << ", file is: " << cacheKey; + continue; + } + // if remove disk file before delete cache, + // then read maybe fail. + const char *toDelFile; + toDelFile = cacheReadFile.c_str(); + ret = posixWrapper_->remove(toDelFile); + if (ret < 0) { + LOG(ERROR) + << "remove disk file error, file is: " << cacheKey + << "error is: " << errno; + continue; + } + curve::client::CollectMetrics(&metric_->trim_, + statReadFile.st_size, + butil::cpuwide_time_us() - start); + UpdateDiskUsedBytes(-statReadFile.st_size); + VLOG(6) << "remove disk file success, file is: " << cacheKey; } - - VLOG(6) << "obj will be removed01: " << cacheKey; - cacheReadFile = cacheReadFullDir + "/" + - curvefs::common::s3util::GenPathByObjName( - cacheKey, objectPrefix_); - cacheWriteFile = cacheWriteFullDir + "/" + - curvefs::common::s3util::GenPathByObjName( - cacheKey, objectPrefix_); - struct stat statFile; - int ret = 0; - ret = posixWrapper_->stat(cacheWriteFile.c_str(), &statFile); - // if file has not been uploaded to S3, - // but remove the cache read file, - // then read will fail when do cache read, - // and then it cannot load the file from S3. - // so read is fail. - if (ret == 0) { - VLOG(1) << "do not remove this disk file" - << ", file has not been uploaded to S3." - << ", file is: " << cacheKey; - usleep(1000); - continue; - } - cachedObjName_->Remove(cacheKey); - struct stat statReadFile; - ret = posixWrapper_->stat(cacheReadFile.c_str(), &statReadFile); - if (ret != 0) { - VLOG(0) << "stat disk file error" - << ", file is: " << cacheKey; - continue; - } - // if remove disk file before delete cache, - // then read maybe fail. - const char *toDelFile; - toDelFile = cacheReadFile.c_str(); - ret = posixWrapper_->remove(toDelFile); - if (ret < 0) { - LOG(ERROR) - << "remove disk file error, file is: " << cacheKey - << "error is: " << errno; - continue; - } - DecDiskUsedBytes(statReadFile.st_size); - VLOG(6) << "remove disk file success, file is: " << cacheKey; } } LOG(INFO) << "trim function end."; @@ -449,6 +502,9 @@ int DiskCacheManager::TrimRun() { } int DiskCacheManager::TrimStop() { + if (diskInitThread_.joinable()) { + diskInitThread_.join(); + } if (isRunning_.exchange(false)) { LOG(INFO) << "stop DiskCacheManager trim thread..."; isRunning_ = false; @@ -462,9 +518,11 @@ int DiskCacheManager::TrimStop() { return 0; } -void DiskCacheManager::InitMetrics(const std::string &fsName) { - metric_ = std::make_shared(fsName); - cacheWrite_->InitMetrics(metric_); +void DiskCacheManager::InitMetrics(const std::string& fsName, + std::shared_ptr s3Metric) { + metric_ = + std::make_shared(fsName, &usedBytes_, &totalBytes_); + cacheWrite_->InitMetrics(metric_, s3Metric); cacheRead_->InitMetrics(metric_); // this function move to here from init， // Otherwise, you can't get the original metric. @@ -472,6 +530,7 @@ void DiskCacheManager::InitMetrics(const std::string &fsName) { // so use a separate thread to do this. diskInitThread_ = curve::common::Thread( &DiskCacheManager::SetDiskInitUsedBytes, this); + s3Metric_ = s3Metric; } } // namespace client diff --git a/curvefs/src/client/s3/disk_cache_manager.h b/curvefs/src/client/s3/disk_cache_manager.h index 1c77fe6c83..b31c4e793f 100644 --- a/curvefs/src/client/s3/disk_cache_manager.h +++ b/curvefs/src/client/s3/disk_cache_manager.h @@ -23,24 +23,26 @@ #define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_ #include +#include #include +#include +#include #include #include -#include -#include +#include "curvefs/src/client/common/config.h" +#include "curvefs/src/client/metric/client_metric.h" +#include "curvefs/src/client/s3/client_s3.h" +#include "curvefs/src/client/s3/disk_cache_read.h" +#include "curvefs/src/client/s3/disk_cache_write.h" +#include "curvefs/src/common/utils.h" +#include "curvefs/src/common/wrap_posix.h" #include "src/common/concurrent/concurrent.h" #include "src/common/interruptible_sleeper.h" #include "src/common/lru_cache.h" #include "src/common/throttle.h" #include "src/common/wait_interval.h" -#include "curvefs/src/common/wrap_posix.h" -#include "curvefs/src/common/utils.h" -#include "curvefs/src/client/s3/client_s3.h" -#include "curvefs/src/client/s3/disk_cache_write.h" -#include "curvefs/src/client/s3/disk_cache_read.h" -#include "curvefs/src/client/common/config.h" namespace curvefs { namespace client { @@ -53,6 +55,8 @@ using curvefs::client::common::S3ClientAdaptorOption; using curvefs::common::PosixWrapper; using curvefs::common::SysUtils; +constexpr uint32_t kRatioLevel = 100; + class DiskCacheManager { public: DiskCacheManager(std::shared_ptr posixWrapper, @@ -92,13 +96,8 @@ class DiskCacheManager { int UploadAllCacheWriteFile(); int UploadWriteCacheByInode(const std::string &inode); int ClearReadCache(const std::list &files); - /** - * @brief get use ratio of cache disk - * @return the use ratio - */ - int64_t SetDiskFsUsedRatio(); virtual bool IsDiskCacheFull(); - bool IsDiskCacheSafe(); + /** * @brief: start trim thread. */ @@ -108,7 +107,8 @@ class DiskCacheManager { */ int TrimStop(); - void InitMetrics(const std::string &fsName); + void InitMetrics(const std::string& fsName, + std::shared_ptr s3Metric); /** * @brief: has got the origin used size or not. @@ -117,47 +117,55 @@ class DiskCacheManager { return diskUsedInit_.load(); } + std::shared_ptr GetS3Metric() { return s3Metric_; } + private: + FRIEND_TEST(TestDiskCacheManager, UpdateDiskFsUsedRatio); + FRIEND_TEST(TestDiskCacheManager, IsDiskCacheSafe); + FRIEND_TEST(TestDiskCacheManager, IsDiskCacheSafe_TrimRatio_Full); + FRIEND_TEST(TestDiskCacheManager, IsDiskCacheSafe_TrimRatio_Nearfull); + FRIEND_TEST(TestDiskCacheManager, IsDiskCacheSafe_TrimRatio_Ok); + /** - * @brief add the used bytes of disk cache. + * @brief get use ratio of cache disk + * @return the use ratio */ - void AddDiskUsedBytes(uint64_t length) { - usedBytes_.fetch_add(length); - if (metric_.get() != nullptr) - metric_->diskUsedBytes.set_value(usedBytes_/1024/1024); - VLOG(9) << "add disk used size is: " << length - << ", now is: " << usedBytes_.load(); - return; - } + int64_t UpdateDiskFsUsedRatio(); /** - * @brief dec the used bytes of disk cache. - * can not dec disk used bytes after file have been loaded, - * because there are link in read cache + * @brief: Determine whether the cache disk exceeds the safety line + * @param ratio: Adjust the safety line judged according to ratio, and the + * adjusted safety line is safety line*baseRatio/kRatioLevel. */ - void DecDiskUsedBytes(uint64_t length) { - usedBytes_.fetch_sub(length); + bool IsDiskCacheSafe(uint32_t baseRatio); + + // update disk used bytes by length + void UpdateDiskUsedBytes(int64_t length) { + usedBytes_.fetch_add(length); assert(usedBytes_ >= 0); - if (metric_.get() != nullptr) - metric_->diskUsedBytes.set_value(usedBytes_); - VLOG(9) << "dec disk used size is: " << length + VLOG(9) << "update disk used size is: " << length << ", now is: " << usedBytes_.load(); - return; } + void SetDiskInitUsedBytes(); uint64_t GetDiskUsedbytes() { return usedBytes_.load(); } + /** + * @brief qos param for diskcache + */ void InitQosParam(); + /** * @brief trim cache func. */ void TrimCache(); /** - * @brief whether the cache file is exceed maxFileNums_. + * @brief whether the cache file is exceed + * FLAGS_diskMaxFileNums*baseRatio/kRatioLevel. */ - bool IsExceedFileNums(); + bool IsExceedFileNums(uint32_t baseRatio); /** * @brief check whether cache dir does not exist or there is no cache file @@ -168,14 +176,12 @@ class DiskCacheManager { curve::common::Atomic isRunning_; curve::common::InterruptibleSleeper sleeper_; curve::common::WaitInterval waitIntervalSec_; - uint32_t trimCheckIntervalSec_; - uint32_t fullRatio_; - uint32_t safeRatio_; - uint64_t maxUsableSpaceBytes_; - uint64_t maxFileNums_; uint32_t objectPrefix_; - // used bytes of disk cache + // The size of the object cached to the local disk, + // rather than the size of the used capacity of the cache disk std::atomic usedBytes_; + // Cache disk size + std::atomic totalBytes_; // used ratio of the file system in disk cache std::atomic diskFsUsedRatio_; uint32_t cmdTimeoutSec_; @@ -188,6 +194,7 @@ class DiskCacheManager { std::shared_ptr client_; std::shared_ptr posixWrapper_; std::shared_ptr metric_; + std::shared_ptr s3Metric_; Throttle diskCacheThrottle_; diff --git a/curvefs/src/client/s3/disk_cache_manager_impl.cpp b/curvefs/src/client/s3/disk_cache_manager_impl.cpp index 209a2d4e97..df080f3c5c 100644 --- a/curvefs/src/client/s3/disk_cache_manager_impl.cpp +++ b/curvefs/src/client/s3/disk_cache_manager_impl.cpp @@ -19,15 +19,19 @@ * Created Date: 21-08-13 * Author: hzwuhongsong */ -#include +#include "curvefs/src/client/s3/disk_cache_manager_impl.h" + #include -#include +#include + +#include #include #include #include +#include #include "curvefs/src/client/s3/client_s3_adaptor.h" -#include "curvefs/src/client/s3/disk_cache_manager_impl.h" +#include "curvefs/src/client/s3/disk_cache_write.h" namespace curvefs { @@ -79,6 +83,11 @@ int DiskCacheManagerImpl::WriteReadDirectClosure( int ret = WriteReadDirect(context->key, context->buffer, context->bufferSize); VLOG(9) << "WriteReadClosure end, name: " << context->key; + context->retCode = ret; + context->timer.stop(); + if (context->cb != nullptr) { + context->cb(context); + } return ret; } @@ -89,6 +98,7 @@ int DiskCacheManagerImpl::WriteClosure( // set the returned value // it is need in CallBack context->retCode = ret; + context->timer.stop(); context->cb(context); VLOG(9) << "WriteClosure end, name: " << context->key; return 0; @@ -167,11 +177,15 @@ int DiskCacheManagerImpl::Read(const std::string name, char *buf, int ret = diskCacheManager_->ReadDiskFile(name, buf, offset, length); if (ret < static_cast(length)) { LOG(ERROR) << "read disk file error. readRet = " << ret; + int64_t start = butil::cpuwide_time_us(); ret = client_->Download(name, buf, offset, length); if (ret < 0) { LOG(ERROR) << "download object fail. object name = " << name; return ret; } + curve::client::CollectMetrics( + &diskCacheManager_->GetS3Metric()->readFromS3, length, + butil::cpuwide_time_us() - start); } VLOG(9) << "read success, read name = " << name; return ret; @@ -197,8 +211,9 @@ int DiskCacheManagerImpl::UmountDiskCache() { return 0; } -void DiskCacheManagerImpl::InitMetrics(std::string fsName) { - diskCacheManager_->InitMetrics(fsName); +void DiskCacheManagerImpl::InitMetrics(std::string fsName, + std::shared_ptr s3Metric) { + diskCacheManager_->InitMetrics(fsName, s3Metric); } int DiskCacheManagerImpl::UploadWriteCacheByInode(const std::string &inode) { diff --git a/curvefs/src/client/s3/disk_cache_manager_impl.h b/curvefs/src/client/s3/disk_cache_manager_impl.h index 7a691909ff..f6e02deeb1 100644 --- a/curvefs/src/client/s3/disk_cache_manager_impl.h +++ b/curvefs/src/client/s3/disk_cache_manager_impl.h @@ -50,8 +50,9 @@ struct DiskCacheOption { DiskCacheType diskCacheType; uint64_t trimCheckIntervalSec; uint64_t asyncLoadPeriodMs; - uint64_t fullRatio; - uint64_t safeRatio; + uint32_t fullRatio; + uint32_t safeRatio; + uint32_t trimRatio; std::string cacheDir; bool forceFlush; uint64_t maxUsableSpaceBytes; @@ -78,7 +79,7 @@ class DiskCacheManagerImpl { * @param[in] option config option * @return success: 0, fail : < 0 */ - int Init(const S3ClientAdaptorOption option); + virtual int Init(const S3ClientAdaptorOption option); /** * @brief Write obj * @param[in] name obj name @@ -86,13 +87,13 @@ class DiskCacheManagerImpl { * @param[in] length write length * @return success: write length, fail : < 0 */ - int Write(const std::string name, const char *buf, uint64_t length); + virtual int Write(const std::string name, const char* buf, uint64_t length); /** * @brief whether obj is cached in cached disk * @param[in] name obj name * @return cached: true, not cached : < 0 */ - bool IsCached(const std::string name); + virtual bool IsCached(const std::string name); /** * @brief read obj * @param[in] name obj name @@ -101,18 +102,18 @@ class DiskCacheManagerImpl { * @param[in] length read length * @return success: length, fail : < length */ - int Read(const std::string name, char *buf, uint64_t offset, - uint64_t length); + virtual int Read(const std::string name, char* buf, uint64_t offset, + uint64_t length); /** * @brief umount disk cache * @return success: 0, fail : < 0 */ - int UmountDiskCache(); + virtual int UmountDiskCache(); bool IsDiskCacheFull(); - int WriteReadDirect(const std::string fileName, const char *buf, - uint64_t length); - void InitMetrics(std::string fsName); + virtual int WriteReadDirect(const std::string fileName, const char* buf, + uint64_t length); + void InitMetrics(std::string fsName, std::shared_ptr s3Metric); virtual int UploadWriteCacheByInode(const std::string &inode); diff --git a/curvefs/src/client/s3/disk_cache_read.cpp b/curvefs/src/client/s3/disk_cache_read.cpp index 4f452a3181..27bee46776 100644 --- a/curvefs/src/client/s3/disk_cache_read.cpp +++ b/curvefs/src/client/s3/disk_cache_read.cpp @@ -85,7 +85,7 @@ int DiskCacheRead::LinkWriteToRead(const std::string fileName, const std::string fullWriteDir, const std::string fullReadDir) { VLOG(6) << "LinkWriteToRead start. name = " << fileName; - std::string fullReadPath, fullWritePath, dirPath; + std::string fullReadPath, fullWritePath; fullWritePath = fullWriteDir + "/" + fileName; fullReadPath = fullReadDir + "/" + fileName; int ret; diff --git a/curvefs/src/client/s3/disk_cache_write.cpp b/curvefs/src/client/s3/disk_cache_write.cpp index 149ea65fff..640c6bca0a 100644 --- a/curvefs/src/client/s3/disk_cache_write.cpp +++ b/curvefs/src/client/s3/disk_cache_write.cpp @@ -20,15 +20,18 @@ * Author: hzwuhongsong */ +#include "curvefs/src/client/s3/disk_cache_write.h" + +#include +#include #include #include #include -#include -#include #include -#include "curvefs/src/client/s3/disk_cache_write.h" + #include "curvefs/src/common/s3util.h" +#include "src/common/s3_adapter.h" namespace curvefs { @@ -136,14 +139,16 @@ int DiskCacheWrite::UploadFile(const std::string &name, } VLOG(9) << "async upload start, file = " << name; PutObjectAsyncCallBack cb = - [&, buffer, syncTask, name] - (const std::shared_ptr &context) { - if (context->retCode == 0) { + [&, buffer, syncTask, + name](const std::shared_ptr& context) { + if (context->retCode >= 0) { if (metric_ != nullptr) { - metric_->writeS3.bps.count << context->bufferSize; - metric_->writeS3.qps.count << 1; - metric_->writeS3.latency - << (butil::cpuwide_time_us() - context->startTime); + curve::client::CollectMetrics(&metric_->writeS3, + context->bufferSize, + context->timer.u_elapsed()); + } + if (s3Metric_ != nullptr) { + metric::AsyncContextCollectMetrics(s3Metric_, context); } RemoveFile(context->key); VLOG(9) << " PutObjectAsyncCallBack success, " @@ -161,12 +166,8 @@ int DiskCacheWrite::UploadFile(const std::string &name, LOG(WARNING) << "upload object failed: " << context->key; client_->UploadAsync(context); }; - auto context = std::make_shared(); - context->key = name; - context->buffer = buffer; - context->bufferSize = fileSize; - context->cb = cb; - context->startTime = butil::cpuwide_time_us(); + auto context = std::make_shared( + name, buffer, fileSize, cb, curve::common::ContextType::S3); client_->UploadAsync(context); VLOG(9) << "async upload end, file = " << name; return 0; @@ -175,7 +176,7 @@ int DiskCacheWrite::UploadFile(const std::string &name, void DiskCacheWrite::UploadFile(const std::list &toUpload, std::shared_ptr syncTask) { std::list::const_iterator iter; - for (iter = toUpload.begin(); iter != toUpload.end(); iter++) { + for (iter = toUpload.begin(); iter != toUpload.end(); ++iter) { UploadFile(*iter, syncTask); } } @@ -399,26 +400,30 @@ int DiskCacheWrite::UploadAllCacheWriteFile() { continue; } PutObjectAsyncCallBack cb = - [&, buffer](const std::shared_ptr &context) { - if (context->retCode == 0) { - if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) { - VLOG(3) << "pendingReq is over"; - cond.Signal(); + [&, buffer](const std::shared_ptr& context) { + if (context->retCode >= 0) { + if (s3Metric_ != nullptr) { + curve::client::CollectMetrics( + &metric_->writeS3, context->bufferSize, + context->timer.u_elapsed()); + metric::AsyncContextCollectMetrics(s3Metric_, context); + } + if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == + 1) { + VLOG(3) << "pendingReq is over"; + cond.Signal(); + } + VLOG(3) << "PutObjectAsyncCallBack success" + << ", file: " << context->key; + posixWrapper_->free(buffer); + return; } - VLOG(3) << "PutObjectAsyncCallBack success" - << ", file: " << context->key; - posixWrapper_->free(buffer); - return; - } - LOG(WARNING) << "upload object failed: " << context->key; - client_->UploadAsync(context); - }; - auto context = std::make_shared(); - context->key = curvefs::common::s3util::GenPathByObjName( - *iter, objectPrefix_); - context->buffer = buffer; - context->bufferSize = fileSize; - context->cb = cb; + LOG(WARNING) << "upload object failed: " << context->key; + client_->UploadAsync(context); + }; + auto context = std::make_shared( + curvefs::common::s3util::GenPathByObjName(*iter, objectPrefix_), + buffer, fileSize, cb, curve::common::ContextType::S3); client_->UploadAsync(context); } if (pendingReq.load(std::memory_order_seq_cst)) { @@ -443,7 +448,6 @@ int DiskCacheWrite::RemoveFile(const std::string fileName) { << ", errno = " << errno; return -1; } - cachedObjName_->MoveBack(fileName); VLOG(9) << "remove file success, file = " << fileName; return 0; } diff --git a/curvefs/src/client/s3/disk_cache_write.h b/curvefs/src/client/s3/disk_cache_write.h index d0da516171..8fb425118f 100644 --- a/curvefs/src/client/s3/disk_cache_write.h +++ b/curvefs/src/client/s3/disk_cache_write.h @@ -22,34 +22,35 @@ #ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_ #define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_ -#include #include #include +#include -#include -#include #include +#include #include +#include +#include "curvefs/src/client/common/config.h" +#include "curvefs/src/client/s3/client_s3.h" +#include "curvefs/src/client/s3/disk_cache_base.h" +#include "curvefs/src/client/s3/disk_cache_read.h" +#include "curvefs/src/common/utils.h" +#include "curvefs/src/common/wrap_posix.h" #include "src/common/concurrent/concurrent.h" #include "src/common/interruptible_sleeper.h" #include "src/common/lru_cache.h" #include "src/common/throttle.h" #include "src/common/wait_interval.h" -#include "curvefs/src/common/wrap_posix.h" -#include "curvefs/src/common/utils.h" -#include "curvefs/src/client/s3/client_s3.h" -#include "curvefs/src/client/s3/disk_cache_read.h" -#include "curvefs/src/client/common/config.h" -#include "curvefs/src/client/s3/disk_cache_base.h" namespace curvefs { namespace client { -using curvefs::common::PosixWrapper; using curve::common::InterruptibleSleeper; -using ::curve::common::SglLRUCache; using curve::common::PutObjectAsyncCallBack; +using ::curve::common::SglLRUCache; +using curvefs::client::metric::S3Metric; +using curvefs::common::PosixWrapper; class DiskCacheWrite : public DiskCacheBase { public: @@ -130,8 +131,10 @@ class DiskCacheWrite : public DiskCacheBase { */ virtual int AsyncUploadStop(); - virtual void InitMetrics(std::shared_ptr metric) { + virtual void InitMetrics(std::shared_ptr metric, + std::shared_ptr s3Metric) { metric_ = metric; + s3Metric_ = s3Metric; } /** @@ -160,6 +163,7 @@ class DiskCacheWrite : public DiskCacheBase { // file system operation encapsulation std::shared_ptr posixWrapper_; std::shared_ptr metric_; + std::shared_ptr s3Metric_; std::shared_ptr> cachedObjName_; }; diff --git a/curvefs/src/client/sdk_helper.cpp b/curvefs/src/client/sdk_helper.cpp new file mode 100644 index 0000000000..e75ae21978 --- /dev/null +++ b/curvefs/src/client/sdk_helper.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-08 + * Author: Jingli Chen (Wine93) + */ + +// clang-format off + +#include "curvefs/src/client/logger/access_log.h" +#include "curvefs/src/client/logger/error_log.h" +#include "src/common/gflags_helper.h" +#include "curvefs/src/client/sdk_helper.h" + +namespace curvefs { +namespace client { + +using ::curve::common::GflagsLoadValueFromConfIfCmdNotSet; +using ::curvefs::client::logger::InitAccessLog; +using ::curvefs::client::logger::InitErrorLog; +using ::curvefs::client::logger::ShutdownErrorLog; + +void SDKHelper::InitLog(Configuration* cfg) { + // prefix + std::string prefix; + cfg->GetStringValue("client.common.logDir", &prefix); + + // name + static std::string name = "/curvefs-sdk"; + + // log level + int32_t loglevel = 0; + curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy; + dummy.Load(cfg, "v", "client.loglevel", &loglevel); + + InitErrorLog(prefix, name, loglevel); + InitAccessLog(prefix); +} + +void SDKHelper::InitOption(Configuration* cfg, FuseClientOption* option) { + cfg->PrintConfig(); + InitFuseClientOption(cfg, option); +} + +FSType SDKHelper::Str2Type(const std::string& s) { + if (s == "s3") { + return FSType::TYPE_S3; + } else if (s == "volume") { + return FSType::TYPE_VOLUME; + } + return FSType(); +} + +std::string SDKHelper::Type2Str(FSType t) { + if (t == FSType::TYPE_S3) { + return "s3"; + } else if (t == FSType::TYPE_VOLUME) { + return "volume"; + } + return "unknown"; +} + +bool SDKHelper::GetFsInfoFromMDS(const MdsOption& option, + const std::string& fsname, + FsInfo* info) { + MdsClientImpl client; + MDSBaseClient base; + client.Init(option, &base); + auto rc = client.GetFsInfo(fsname, info); + if (rc != FSStatusCode::OK) { + LOG(ERROR) << "Get fsinfo from mds failed, fsname = " << fsname + << ", retCode = " << FSStatusCode_Name(rc); + return false; + } + return true; +} + +bool SDKHelper::CheckFsType(FSType real, FSType arg) { + if (real != arg) { + LOG(ERROR) << "The fstype obtained from the mds is " + << FSType_Name(real) << ", but user specified is " + << FSType_Name(arg); + return false; + } else if (real != FSType::TYPE_VOLUME && real != FSType::TYPE_S3) { + LOG(ERROR) << "The fstype obtained from the mds is " + << FSType_Name(real) << ", which is not supported"; + return false; + } + return true; +} + +MountOption SDKHelper::GetMountOption(const std::string& fsname, + const std::string& mountpoint) { + auto mountOption = MountOption(); + mountOption.mountPoint = mountpoint.c_str(); + mountOption.fsName = fsname.c_str(); + mountOption.fsType = Type2Str(FSType::TYPE_S3).c_str(); + return mountOption; +} + +CURVEFS_ERROR SDKHelper::CheckMountOption(std::shared_ptr client, + const MdsOption& mdsOption, + MountOption* mountOption) { + FsInfo info; + bool yes = GetFsInfoFromMDS(mdsOption, mountOption->fsName, &info) && + CheckFsType(info.fstype(), Str2Type(mountOption->fsType)); + if (!yes) { + return CURVEFS_ERROR::INTERNAL; + } + client->SetFsInfo(std::make_shared(info)); + return CURVEFS_ERROR::OK; +} + +CURVEFS_ERROR SDKHelper::Mount(std::shared_ptr client, + const std::string& fsname, + const std::string& mountpoint, + FuseClientOption option) { + auto mountOption = GetMountOption(fsname, mountpoint); + auto rc = CheckMountOption(client, option.mdsOpt, &mountOption); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + rc = client->Init(option); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + rc = client->Run(); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + rc = client->SetMountStatus(&mountOption); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + return client->FuseOpInit(nullptr, nullptr); +} + +CURVEFS_ERROR SDKHelper::Umount(std::shared_ptr client, + const std::string& fsname, + const std::string& mountpoint) { + auto mountOption = GetMountOption(fsname, mountpoint); + client->FuseOpDestroy(&mountOption); + client->Fini(); + client->UnInit(); + ShutdownErrorLog(); + return CURVEFS_ERROR::OK; +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/sdk_helper.h b/curvefs/src/client/sdk_helper.h new file mode 100644 index 0000000000..0a5b3c4fce --- /dev/null +++ b/curvefs/src/client/sdk_helper.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-08 + * Author: Jingli Chen (Wine93) + */ + +// clang-format off + +#ifndef CURVEFS_SRC_CLIENT_SDK_HELPER_H_ +#define CURVEFS_SRC_CLIENT_SDK_HELPER_H_ + +#include +#include + +#include "curvefs/proto/mds.pb.h" +#include "curvefs/proto/metaserver.pb.h" +#include "curvefs/src/client/common/config.h" +#include "curvefs/src/client/fuse_client.h" +#include "curvefs/src/client/fuse_common.h" +#include "curvefs/src/client/fuse_s3_client.h" +#include "curvefs/src/client/fuse_volume_client.h" +#include "src/common/configuration.h" + +namespace curvefs { +namespace client { + +using ::curve::common::Configuration; +using ::curvefs::client::common::MdsOption; +using ::curvefs::common::FSType; +using ::curvefs::mds::FsInfo; + +class SDKHelper { + public: + SDKHelper() = default; + + void InitLog(Configuration* cfg); + + void InitOption(Configuration* cfg, FuseClientOption* option); + + CURVEFS_ERROR Mount(std::shared_ptr client, + const std::string& fsname, + const std::string& mountpoint, + FuseClientOption option); + + CURVEFS_ERROR Umount(std::shared_ptr client, + const std::string& fsname, + const std::string& mountpoint); + + private: + FSType Str2Type(const std::string& s); + + std::string Type2Str(FSType t); + + bool GetFsInfoFromMDS(const MdsOption& option, + const std::string& fsname, + FsInfo* info); + + bool CheckFsType(FSType real, FSType arg); + + MountOption GetMountOption(const std::string& fsname, + const std::string& mountpoint); + + CURVEFS_ERROR CheckMountOption(std::shared_ptr client, + const MdsOption& mdsOption, + MountOption* mountOption); +}; + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_SDK_HELPER_H_ diff --git a/curvefs/src/client/vfs/.clang-format b/curvefs/src/client/vfs/.clang-format new file mode 100644 index 0000000000..a5660a57d8 --- /dev/null +++ b/curvefs/src/client/vfs/.clang-format @@ -0,0 +1,26 @@ +# I don't like current clang-formtter's specified style and +# and i think my style is more intuitive than it :), so I use +# this file to disable format, let us give some code snippet to prove it: +# +# my: +# std::vector tests { +# { " abc", "abc" }, +# { "abc ", "abc" }, +# { " abc ", "abc" }, +# { " a bc ", "a bc" }, +# { "a b c", "a b c" }, +# { " ", "" }, +# +# +# clang-formatter: +# std::vector tests{ +# {" abc", "abc"}, {"abc ", "abc"}, {" abc ", "abc"}, +# {" a bc ", "a bc"}, {"a b c", "a b c"}, {" ", ""}, +# }; +# +# Maybe you have different opinions, but please have a little mercy for me. +# +# by Wine93, 2023-10-16 + +DisableFormat: true +SortIncludes: Never diff --git a/curvefs/src/client/vfs/BUILD b/curvefs/src/client/vfs/BUILD new file mode 100644 index 0000000000..ba2926ff6f --- /dev/null +++ b/curvefs/src/client/vfs/BUILD @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_library( + name = "vfs", + srcs = glob(["*.cpp"]), + hdrs = glob(["*.h"]), + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "@incbin//:incbin", + "//src/common:curve_common", + "//curvefs/proto:metaserver_cc_proto", + "//curvefs/src/client:filesystem", + "//curvefs/src/client/logger:logger", + ], +) diff --git a/curvefs/src/client/vfs/README.md b/curvefs/src/client/vfs/README.md new file mode 100644 index 0000000000..b074365365 --- /dev/null +++ b/curvefs/src/client/vfs/README.md @@ -0,0 +1,21 @@ + +CurveFS VFS Design +=== + +Permission +--- + +`uid/gid` AND `euid/egid` AND `suid/sgid` + +```C++ +euid = uid +egid = gid + +if (suid != NULL) { + euid = suid; +} + +if (sgid != NULL) { + egid = sgid; +} +``` diff --git a/curvefs/src/client/vfs/cache.cpp b/curvefs/src/client/vfs/cache.cpp new file mode 100644 index 0000000000..c10fe93aff --- /dev/null +++ b/curvefs/src/client/vfs/cache.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-04 + * Author: Jingli Chen (Wine93) + */ + +#include + +#include "absl/strings/str_format.h" +#include "curvefs/src/client/filesystem/utils.h" +#include "curvefs/src/client/vfs/cache.h" + +namespace curvefs { +namespace client { +namespace vfs { + +#define RETURN_FALSE_IF_DISABLED() \ + do { \ + if (!enable_) { \ + return false; \ + } \ + } while (0) + +using ::curvefs::client::filesystem::Now; + +EntryCache::EntryCache(size_t lruSize) + : enable_(lruSize > 0), + lru_(std::make_shared(lruSize)) {} + +std::string EntryCache::EntryKey(Ino parent, const std::string& name) { + return absl::StrFormat("%lu:%s", parent, name); +} + +bool EntryCache::Get(Ino parent, const std::string& name, Ino* ino) { + RETURN_FALSE_IF_DISABLED(); + Entry value; + auto key = EntryKey(parent, name); + bool yes = lru_->Get(key, &value); + if (!yes) { + return false; + } else if (value.expire < Now()) { + return false; + } + *ino = value.ino; + return true; +} + +bool EntryCache::Put(Ino parent, + const std::string& name, + Ino ino, + uint64_t timeoutSec) { + RETURN_FALSE_IF_DISABLED(); + auto key = EntryKey(parent, name); + Entry value; + value.ino = ino; + value.expire = Now() + TimeSpec(timeoutSec, 0); + lru_->Put(key, value); + return true; +} + +bool EntryCache::Delete(Ino parent, const std::string& name) { + RETURN_FALSE_IF_DISABLED(); + auto key = EntryKey(parent, name); + lru_->Remove(key); + return true; +} + +size_t EntryCache::Size() { + return lru_->Size(); +} + +AttrCache::AttrCache(size_t lruSize) + : enable_(lruSize > 0), + lru_(std::make_shared(lruSize)) {} + +bool AttrCache::Get(Ino ino, InodeAttr* attr) { + RETURN_FALSE_IF_DISABLED(); + Attr value; + bool yes = lru_->Get(ino, &value); + if (!yes) { + return false; + } else if (value.expire < Now()) { + return false; + } + *attr = value.attr; + return true; +} + +bool AttrCache::Put(Ino ino, const InodeAttr& attr, uint64_t timeoutSec) { + RETURN_FALSE_IF_DISABLED(); + Attr value; + value.attr = std::move(attr); + value.expire = Now() + TimeSpec(timeoutSec, 0); + lru_->Put(ino, value); + return true; +} + +bool AttrCache::Delete(Ino ino) { + RETURN_FALSE_IF_DISABLED(); + lru_->Remove(ino); + return true; +} + +size_t AttrCache::Size() { + return lru_->Size(); +} + +} // namespace vfs +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/vfs/cache.h b/curvefs/src/client/vfs/cache.h new file mode 100644 index 0000000000..f1653444a8 --- /dev/null +++ b/curvefs/src/client/vfs/cache.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-04 + * Author: Jingli Chen (Wine93) + */ + +#ifndef CURVEFS_SRC_CLIENT_VFS_CACHE_H_ +#define CURVEFS_SRC_CLIENT_VFS_CACHE_H_ + +#include +#include + +#include "src/common/lru_cache.h" +#include "curvefs/src/client/vfs/meta.h" + +namespace curvefs { +namespace client { +namespace vfs { + +using ::curve::common::LRUCache; + +class EntryCache { + public: + struct Entry { + Ino ino; + TimeSpec expire; + }; + + // TODO(Wine93): is there a more effective type for entry cache? maybe + // absl::btree> is a choise, + // but it is a bit complex to implement lru evit strategy. + using LRUType = LRUCache; + + public: + EntryCache() = delete; + + explicit EntryCache(size_t lruSize); + + bool Get(Ino parent, const std::string& name, Ino* ino); + + bool Put(Ino parent, const std::string& name, Ino ino, uint64_t timeoutSec); + + bool Delete(Ino parent, const std::string& name); + + size_t Size(); + + private: + std::string EntryKey(Ino parent, const std::string& name); + + private: + bool enable_; + std::shared_ptr lru_; +}; + +class AttrCache { + public: + struct Attr { + InodeAttr attr; + TimeSpec expire; + }; + + using LRUType = LRUCache; + + public: + AttrCache() = delete; + + explicit AttrCache(size_t lruSize); + + bool Get(Ino ino, InodeAttr* attr); + + bool Put(Ino ino, const InodeAttr& attr, uint64_t timeoutSec); + + bool Delete(Ino ino); + + size_t Size(); + + private: + bool enable_; + std::shared_ptr lru_; +}; + +} // namespace vfs +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_VFS_CACHE_H_ diff --git a/curvefs/src/client/vfs/config.cpp b/curvefs/src/client/vfs/config.cpp new file mode 100644 index 0000000000..a75f01eb80 --- /dev/null +++ b/curvefs/src/client/vfs/config.cpp @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-11 + * Author: Jingli Chen (Wine93) + */ + +#define INCBIN_STYLE INCBIN_STYLE_SNAKE +#define INCBIN_PREFIX g_ + +#include + +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/strings/str_split.h" +#include "curvefs/src/client/vfs/utils.h" +#include "curvefs/src/client/vfs/config.h" + +/* Usage: INCBIN(<>, <>) + * + * Symbols defined by INCBIN + * ------------------------------------------ + * const unsigned char g_client_conf_data[] // g_<>_data + * const unsigned char* const g_client_conf_end; // g_<