From 078cb0ab55804890e8fa691592dfd63a08470735 Mon Sep 17 00:00:00 2001 From: cliang li Date: Wed, 13 Sep 2023 07:08:42 -0700 Subject: [PATCH 1/3] Add mid-layer system tests for datamover put with large/empty file, nfs_cluster and client. --- dss_datamover/systemtests/conftest.py | 49 +++++++++++++- dss_datamover/systemtests/pytest_config.json | 7 +- dss_datamover/systemtests/test_client.py | 66 +++++++++++++++++++ dss_datamover/systemtests/test_datamover.py | 20 +++++- dss_datamover/systemtests/test_nfs_cluster.py | 64 ++++++++++++++++++ 5 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 dss_datamover/systemtests/test_client.py create mode 100644 dss_datamover/systemtests/test_nfs_cluster.py diff --git a/dss_datamover/systemtests/conftest.py b/dss_datamover/systemtests/conftest.py index 1a13b12..d9caa89 100644 --- a/dss_datamover/systemtests/conftest.py +++ b/dss_datamover/systemtests/conftest.py @@ -34,7 +34,8 @@ from utils import config from logger import MultiprocessingLogger -from master_application import Master +from master_application import Master, Client +from nfs_cluster import NFSCluster from multiprocessing import Queue, Value, Lock import json @@ -102,7 +103,9 @@ def get_multiprocessing_logger(tmpdir): logging_path = tmpdir logging_level = "INFO" - logger = MultiprocessingLogger(logger_queue, logger_lock, logger_status) + logger = MultiprocessingLogger(logger_queue, logger_status) + logger.logging_level = logging_level + logger.create_logger_handle() logger.config(logging_path, __file__, logging_level) logger.start() @@ -208,3 +211,45 @@ def reset_master_obj(get_master): # Get the directory prefix keys that are yet to be resumed for PUT operation obj.dir_prefixes_to_resume = list() obj.resume_flag = False + + +@pytest.fixture +def setup_large_file(get_pytest_configs): + path = get_pytest_configs["large_file_path"] + name = "large.file" + if not os.path.exists(path): + os.mkdir(path) + file = path + "/" + name + count = get_pytest_configs["large_file_size"] * 1024 * 1024 // 4 + os.system(f"dd if=/dev/zero of={file} bs=4k count={count}") + yield + shutil.rmtree(path) + + +@pytest.fixture +def setup_empty_file(get_pytest_configs): + path = get_pytest_configs["empty_file_path"] + name = "empty.file" + if not os.path.exists(path): + os.mkdir(path) + file = open(path + "/" + name, 'w') + file.close() + yield + shutil.rmtree(path) + + +@pytest.fixture +def get_nfs_cluster(get_system_config_dict, get_multiprocessing_logger): + get_system_config_dict["nfs"] = get_system_config_dict["fs_config"]["nfs"] + get_system_config_dict["server_as_prefix"] = get_system_config_dict["fs_config"]["server_as_prefix"] + nfs_cluster = NFSCluster(get_system_config_dict, "ansible", "ansible", get_multiprocessing_logger) + yield nfs_cluster + nfs_cluster.umount_all() + + +@pytest.fixture +def get_client(get_system_config_dict, get_multiprocessing_logger): + client_ip = get_system_config_dict["clients_hosts_or_ip_addresses"][0] + client = Client(0, client_ip, "PUT", get_multiprocessing_logger, get_system_config_dict, get_system_config_dict["fs_config"]["server_as_prefix"]) + yield client + client.stop(force_flag=True) diff --git a/dss_datamover/systemtests/pytest_config.json b/dss_datamover/systemtests/pytest_config.json index af8537f..19a52cf 100755 --- a/dss_datamover/systemtests/pytest_config.json +++ b/dss_datamover/systemtests/pytest_config.json @@ -3,5 +3,8 @@ "dest_path": "/tmp/xyz", "prefix": "/mnt/nfs_share/5gb/0_0/1_0/2_0/", "cache": ["/var/log/dss/prefix_index_data.json", "/var/log/dss/dm_resume_prefix_dir_keys.txt"], - "fs_config": {"server_as_prefix": "yes"} -} \ No newline at end of file + "fs_config": {"server_as_prefix": "yes"}, + "large_file_path": "/tmp/large_files", + "large_file_size": 10, + "empty_file_path": "/tmp/empty_files" +} diff --git a/dss_datamover/systemtests/test_client.py b/dss_datamover/systemtests/test_client.py new file mode 100644 index 0000000..7b36f40 --- /dev/null +++ b/dss_datamover/systemtests/test_client.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3 + +""" +# The Clear BSD License +# +# Copyright (c) 2023 Samsung Electronics Co., Ltd. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted (subject to the limitations in the disclaimer +# below) provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Samsung Electronics Co., Ltd. nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY +# THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT +# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +import os +import pytest + + +@pytest.mark.usefixtures( + "get_client", + "get_system_config_dict" +) +class TestClient: + """ + Used for client related system tests. + """ + + def test_client_put(self, get_client): + get_client.operation = "PUT" + get_client.make_client_app_command() + get_client.start() + + def test_client_put(self, get_client): + get_client.operation = "GET" + get_client.make_client_app_command() + get_client.start() + + def test_client_put(self, get_client): + get_client.operation = "LIST" + get_client.make_client_app_command() + get_client.start() + + def test_client_put(self, get_client): + get_client.operation = "DEL" + get_client.make_client_app_command() + get_client.start() diff --git a/dss_datamover/systemtests/test_datamover.py b/dss_datamover/systemtests/test_datamover.py index e06cee6..da87a9f 100755 --- a/dss_datamover/systemtests/test_datamover.py +++ b/dss_datamover/systemtests/test_datamover.py @@ -52,7 +52,9 @@ "clear_datamover_cache", "setup_data_dir", "generate_full_ip_prefix", - "reset_master_obj" + "reset_master_obj", + "setup_large_file", + "setup_empty_file" ) class TestDataMover: """ @@ -91,3 +93,19 @@ def test_delete_operation(self, get_master, reset_master_obj, setup_data_dir): get_master.start() process_del_operation(get_master) assert get_master.testcase_passed.value + + def test_put_large_file(self, get_master, reset_master_obj, setup_data_dir, setup_large_file, get_pytest_configs): + print("Testing PUT operation with large file") + get_master.operation = "PUT" + get_master.fs_config['nfs'] = {'127.0.0.1': get_pytest_configs['large_file_path']} + get_master.prefix = '127.0.0.1' + get_pytest_configs['large_file_path'] + '/' + get_master.start() + assert get_master.testcase_passed.value + + def test_put_empty_file(self, get_master, reset_master_obj, setup_data_dir, setup_empty_file, get_pytest_configs): + print("Testing PUT operation with empty file") + get_master.operation = "PUT" + get_master.fs_config['nfs'] = {'127.0.0.1': get_pytest_configs['empty_file_path']} + get_master.prefix = '127.0.0.1' + get_pytest_configs['empty_file_path'] + '/' + get_master.start() + assert get_master.testcase_passed.value diff --git a/dss_datamover/systemtests/test_nfs_cluster.py b/dss_datamover/systemtests/test_nfs_cluster.py new file mode 100644 index 0000000..9b17357 --- /dev/null +++ b/dss_datamover/systemtests/test_nfs_cluster.py @@ -0,0 +1,64 @@ +#!/usr/bin/python3 + +""" +# The Clear BSD License +# +# Copyright (c) 2023 Samsung Electronics Co., Ltd. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted (subject to the limitations in the disclaimer +# below) provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Samsung Electronics Co., Ltd. nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY +# THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT +# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +import os +import pytest + + +@pytest.mark.usefixtures( + "get_nfs_cluster", + "get_system_config_dict", + "generate_full_ip_prefix" +) +class TestNFSCluster: + """ + Used for NFS cluster related system tests. + """ + + def test_nfs_mount_all(self, get_nfs_cluster, get_system_config_dict): + get_nfs_cluster.mount_all() + for cluster_ip in get_system_config_dict["nfs"]: + for nfs_share in get_system_config_dict["nfs"][cluster_ip]: + nfs_share_mount = os.path.abspath("/" + cluster_ip + "/" + nfs_share) if \ + get_nfs_cluster.server_as_prefix else os.path.abspath("/" + nfs_share) + assert os.path.exists(nfs_share_mount) + + def test_nfs_mount_based_on_prefix(self, get_nfs_cluster, get_system_config_dict, generate_full_ip_prefix): + for ip, nfs_shares in get_system_config_dict["fs_config"]["nfs"].items(): + prefix = nfs_shares[0] + break + get_nfs_cluster.mount_based_on_prefix(prefix) + nfs_share_mount = os.path.abspath("/" + cluster_ip + "/" + prefix) if \ + get_nfs_cluster.server_as_prefix else os.path.abspath("/" + prefix) + assert os.path.exists(nfs_share_mount) From 7fe32d031a6dc4925581e76a97a4a33a6829d9d5 Mon Sep 17 00:00:00 2001 From: cliang li Date: Mon, 25 Sep 2023 02:41:23 -0700 Subject: [PATCH 2/3] Add verification for client tests and change 127.0.0.1 to localhost --- dss_datamover/systemtests/test_client.py | 10 +++++++--- dss_datamover/systemtests/test_datamover.py | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dss_datamover/systemtests/test_client.py b/dss_datamover/systemtests/test_client.py index 7b36f40..f0187b7 100644 --- a/dss_datamover/systemtests/test_client.py +++ b/dss_datamover/systemtests/test_client.py @@ -49,18 +49,22 @@ def test_client_put(self, get_client): get_client.operation = "PUT" get_client.make_client_app_command() get_client.start() + assert get_client.remote_client_status() - def test_client_put(self, get_client): + def test_client_get(self, get_client): get_client.operation = "GET" get_client.make_client_app_command() get_client.start() + assert get_client.remote_client_status() - def test_client_put(self, get_client): + def test_client_list(self, get_client): get_client.operation = "LIST" get_client.make_client_app_command() get_client.start() + assert get_client.remote_client_status() - def test_client_put(self, get_client): + def test_client_del(self, get_client): get_client.operation = "DEL" get_client.make_client_app_command() get_client.start() + assert get_client.remote_client_status() diff --git a/dss_datamover/systemtests/test_datamover.py b/dss_datamover/systemtests/test_datamover.py index da87a9f..7e5759a 100755 --- a/dss_datamover/systemtests/test_datamover.py +++ b/dss_datamover/systemtests/test_datamover.py @@ -97,15 +97,15 @@ def test_delete_operation(self, get_master, reset_master_obj, setup_data_dir): def test_put_large_file(self, get_master, reset_master_obj, setup_data_dir, setup_large_file, get_pytest_configs): print("Testing PUT operation with large file") get_master.operation = "PUT" - get_master.fs_config['nfs'] = {'127.0.0.1': get_pytest_configs['large_file_path']} - get_master.prefix = '127.0.0.1' + get_pytest_configs['large_file_path'] + '/' + get_master.fs_config['nfs'] = {'localhost': get_pytest_configs['large_file_path']} + get_master.prefix = 'localhost' + get_pytest_configs['large_file_path'] + '/' get_master.start() assert get_master.testcase_passed.value def test_put_empty_file(self, get_master, reset_master_obj, setup_data_dir, setup_empty_file, get_pytest_configs): print("Testing PUT operation with empty file") get_master.operation = "PUT" - get_master.fs_config['nfs'] = {'127.0.0.1': get_pytest_configs['empty_file_path']} - get_master.prefix = '127.0.0.1' + get_pytest_configs['empty_file_path'] + '/' + get_master.fs_config['nfs'] = {'localhost': get_pytest_configs['empty_file_path']} + get_master.prefix = 'localhost' + get_pytest_configs['empty_file_path'] + '/' get_master.start() assert get_master.testcase_passed.value From 26897f600933050c3f358227a9afc2bc26a3a703 Mon Sep 17 00:00:00 2001 From: cliang li Date: Mon, 25 Sep 2023 02:51:05 -0700 Subject: [PATCH 3/3] modify lint problems --- dss_datamover/systemtests/test_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dss_datamover/systemtests/test_client.py b/dss_datamover/systemtests/test_client.py index f0187b7..d0b6d06 100644 --- a/dss_datamover/systemtests/test_client.py +++ b/dss_datamover/systemtests/test_client.py @@ -49,22 +49,22 @@ def test_client_put(self, get_client): get_client.operation = "PUT" get_client.make_client_app_command() get_client.start() - assert get_client.remote_client_status() + assert get_client.remote_client_status() def test_client_get(self, get_client): get_client.operation = "GET" get_client.make_client_app_command() get_client.start() - assert get_client.remote_client_status() + assert get_client.remote_client_status() def test_client_list(self, get_client): get_client.operation = "LIST" get_client.make_client_app_command() get_client.start() - assert get_client.remote_client_status() + assert get_client.remote_client_status() def test_client_del(self, get_client): get_client.operation = "DEL" get_client.make_client_app_command() get_client.start() - assert get_client.remote_client_status() + assert get_client.remote_client_status()