From 703a91ea6999bc59bfc0224b68159c0de9a38928 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 20 Aug 2024 13:28:25 -0700 Subject: [PATCH] Added upgrade tests for hsync --- .../callbacks/1.5.0/callback.sh | 30 ++++++++ .../debug/ozone-debug-lease-recovery.robot | 19 +++-- .../dist/src/main/smoketest/freon/hsync.robot | 45 ++++++++++++ .../smoketest/hsync/upgrade-hsync-check.robot | 69 +++++++++++++++++++ .../src/main/smoketest/ozone-lib/freon.robot | 6 ++ 5 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.5.0/callback.sh create mode 100644 hadoop-ozone/dist/src/main/smoketest/freon/hsync.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/hsync/upgrade-hsync-check.robot diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.5.0/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.5.0/callback.sh new file mode 100644 index 000000000000..03843e8c6c82 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.5.0/callback.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source "$TEST_DIR"/testlib.sh + +with_this_version_pre_finalized() { + # New layout features were added in this version, so OM and SCM should be pre-finalized. + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/check-finalization.robot + # Test that HSync is disabled when pre-finalized. + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" --include pre-finalized-hsync-tests hsync/upgrade-hsync-check.robot +} + +with_this_version_finalized() { + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" debug/ozone-debug-lease-recovery.robot +} diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot index f867ee99f644..0a9c12834267 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot @@ -22,11 +22,20 @@ Test Timeout 5 minute Suite Setup Create volume bucket and put key *** Variables *** +${OMSERVICEID} ${VOLUME} lease-recovery-volume ${BUCKET} lease-recovery-bucket ${TESTFILE} testfile22 *** Keywords *** +Get OM serviceId + ${confKey} = Execute And Ignore Error ozone getconf confKey ozone.om.service.ids + ${result} = Evaluate "Configuration ozone.om.service.ids is missing" in """${confKey}""" + IF ${result} == ${True} + Set Suite Variable ${OMSERVICEID} om + ELSE + Set Suite Variable ${OMSERVICEID} ${confKey} + END Create volume bucket and put key Execute ozone sh volume create /${VOLUME} Execute ozone sh bucket create /${VOLUME}/${BUCKET} @@ -35,13 +44,15 @@ Create volume bucket and put key *** Test Cases *** Test ozone debug recover for o3fs - ${result} = Execute Lease recovery cli o3fs://${BUCKET}.${VOLUME}.om/${TESTFILE} + Get OM serviceId + ${result} = Execute Lease recovery cli o3fs://${BUCKET}.${VOLUME}.${OMSERVICEID}/${TESTFILE} Should Contain ${result} Lease recovery SUCCEEDED - ${result} = Execute Lease recovery cli o3fs://${BUCKET}.${VOLUME}.om/randomfile + ${result} = Execute Lease recovery cli o3fs://${BUCKET}.${VOLUME}.${OMSERVICEID}/randomfile Should Contain ${result} not found Test ozone debug recover for ofs - ${result} = Execute Lease recovery cli ofs://om/${VOLUME}/${BUCKET}/${TESTFILE} + Get OM serviceId + ${result} = Execute Lease recovery cli ofs://${OMSERVICEID}/${VOLUME}/${BUCKET}/${TESTFILE} Should Contain ${result} Lease recovery SUCCEEDED - ${result} = Execute Lease recovery cli ofs://om/${VOLUME}/${BUCKET}/randomfile + ${result} = Execute Lease recovery cli ofs://${OMSERVICEID}/${VOLUME}/${BUCKET}/randomfile Should Contain ${result} not found diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/hsync.robot b/hadoop-ozone/dist/src/main/smoketest/freon/hsync.robot new file mode 100644 index 000000000000..9905e8ef3e3b --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/freon/hsync.robot @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test HSync via freon CLI. +Library OperatingSystem +Library String +Library BuiltIn +Resource ../ozone-lib/freon.robot +Test Timeout 5 minutes +Suite Setup Get OM serviceId + +*** Variables *** +${OMSERVICEID} +${VOLUME} hsync-volume +${BUCKET} hsync-bucket + +*** Keywords *** +Get OM serviceId + ${confKey} = Execute And Ignore Error ozone getconf confKey ozone.om.service.ids + ${result} = Evaluate "Configuration ozone.om.service.ids is missing" in """${confKey}""" + IF ${result} == ${True} + Set Suite Variable ${OMSERVICEID} om + ELSE + Set Suite Variable ${OMSERVICEID} ${confKey} + END + +*** Test Cases *** +Generate key by HSYNC + Freon DFSG sync=HSYNC path=ofs://${OMSERVICEID}/${VOLUME}/${BUCKET} + +Generate key by HFLUSH + Freon DFSG sync=HFLUSH path=ofs://${OMSERVICEID}/${VOLUME}/${BUCKET} diff --git a/hadoop-ozone/dist/src/main/smoketest/hsync/upgrade-hsync-check.robot b/hadoop-ozone/dist/src/main/smoketest/hsync/upgrade-hsync-check.robot new file mode 100644 index 000000000000..4a5524b9c407 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/hsync/upgrade-hsync-check.robot @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test HSync during upgrade +Library OperatingSystem +Library String +Library BuiltIn +Resource ../commonlib.robot +Default Tags pre-finalized-hsync-tests +Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab + +*** Variables *** +${OMSERVICEID} +${VOLUME} +${BUCKET} +${KEY} + +*** Keywords *** +Get OM serviceId + ${confKey} = Execute And Ignore Error ozone getconf confKey ozone.om.service.ids + ${result} = Evaluate "Configuration ozone.om.service.ids is missing" in """${confKey}""" + IF ${result} == ${True} + Set Suite Variable ${OMSERVICEID} om + ELSE + Set Suite Variable ${OMSERVICEID} ${confKey} + END +Create volume for upgrade test + ${random} = Generate Random String 5 [LOWER] + ${volume} = Set Variable vol-${random} + ${result} = Execute ozone sh volume create /${volume} + Should not contain ${result} Failed + Set Suite Variable ${VOLUME} ${volume} + +Create bucket for upgrade test + ${random} = Generate Random String 5 [LOWER] + ${bucket} = Set Variable buc-${random} + ${result} = Execute ozone sh bucket create -l FILE_SYSTEM_OPTIMIZED /${volume}/${bucket} + Should not contain ${result} Failed + Set Suite Variable ${BUCKET} ${bucket} + +Create key for upgrade test + ${random} = Generate Random String 5 [LOWER] + ${key} = Set Variable key-${random} + ${result} = Execute ozone sh key put /${volume}/${bucket}/${key} /etc/hosts + Set Suite Variable ${KEY} ${key} + +*** Test Cases *** +Test HSync Prior To Finalization + Get OM serviceId + Create volume for upgrade test + Create bucket for upgrade test + Create key for upgrade test + ${result} = Execute and checkrc ozone debug recover --path=ofs://${OMSERVICEID}/${VOLUME}/${BUCKET}/${KEY} 255 + Should contain ${result} It belongs to the layout feature HBASE_SUPPORT, whose layout version is 7 + ${result} = Execute and checkrc ozone debug recover --path=o3fs://${BUCKET}.${VOLUME}.${OMSERVICEID}/${KEY} 255 + Should contain ${result} It belongs to the layout feature HBASE_SUPPORT, whose layout version is 7 diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot index 8d10cc81e900..3595255269e7 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot @@ -67,3 +67,9 @@ Freon OMBR [arguments] ${prefix}=ombg ${n}=1 ${threads}=1 ${args}=${EMPTY} ${result} = Execute ozone freon ombr ${OM_HA_PARAM} -t ${threads} -n${n} -p ${prefix} ${args} Should contain ${result} Successful executions: ${n} + +Freon DFSG + [arguments] ${prefix}=dfsg ${n}=1000 ${path}={EMPTY} ${threads}=1 ${sync}=HSYNC ${buffer}=1024 ${copy-buffer}=1024 ${size}=10240 ${size}=10240 ${args}=${EMPTY} + ${result} = Execute ozone freon dfsg ${OM_HA_PARAM} -n ${n} --sync ${sync} -s ${size} --path ${path} --buffer ${buffer} --copy-buffer ${copy-buffer} -p ${prefix} -t ${threads} ${args} + Should contain ${result} Successful executions: ${n} +