Add mid-layer system tests for datamover put with large/empty file, n…

dss_datamover/systemtests/conftest.py

@@ -34,7 +34,8 @@
 
 from utils import config
 from logger import MultiprocessingLogger
-from master_application import Master
+from master_application import Master, Client
+from nfs_cluster import NFSCluster
 from multiprocessing import Queue, Value, Lock
 
 import json
@@ -102,7 +103,9 @@ def get_multiprocessing_logger(tmpdir):
     logging_path = tmpdir
     logging_level = "INFO"
 
-    logger = MultiprocessingLogger(logger_queue, logger_lock, logger_status)
+    logger = MultiprocessingLogger(logger_queue, logger_status)
+    logger.logging_level = logging_level
+    logger.create_logger_handle()
     logger.config(logging_path, __file__, logging_level)
     logger.start()
 
@@ -208,3 +211,45 @@ def reset_master_obj(get_master):
     # Get the directory prefix keys that are yet to be resumed for PUT operation
     obj.dir_prefixes_to_resume = list()
     obj.resume_flag = False
+
+
+@pytest.fixture
+def setup_large_file(get_pytest_configs):
+    path = get_pytest_configs["large_file_path"]
+    name = "large.file"
+    if not os.path.exists(path):
+        os.mkdir(path)
+    file = path + "/" + name
+    count = get_pytest_configs["large_file_size"] * 1024 * 1024 // 4
+    os.system(f"dd if=/dev/zero of={file} bs=4k count={count}")
+    yield
+    shutil.rmtree(path)
+
+
+@pytest.fixture
+def setup_empty_file(get_pytest_configs):
+    path = get_pytest_configs["empty_file_path"]
+    name = "empty.file"
+    if not os.path.exists(path):
+        os.mkdir(path)
+    file = open(path + "/" + name, 'w')
+    file.close()
+    yield
+    shutil.rmtree(path)
+
+
+@pytest.fixture
+def get_nfs_cluster(get_system_config_dict, get_multiprocessing_logger):
+    get_system_config_dict["nfs"] = get_system_config_dict["fs_config"]["nfs"]
+    get_system_config_dict["server_as_prefix"] = get_system_config_dict["fs_config"]["server_as_prefix"]
+    nfs_cluster = NFSCluster(get_system_config_dict, "ansible", "ansible", get_multiprocessing_logger)
+    yield nfs_cluster
+    nfs_cluster.umount_all()
+
+
+@pytest.fixture
+def get_client(get_system_config_dict, get_multiprocessing_logger):
+    client_ip = get_system_config_dict["clients_hosts_or_ip_addresses"][0]
+    client = Client(0, client_ip, "PUT", get_multiprocessing_logger, get_system_config_dict, get_system_config_dict["fs_config"]["server_as_prefix"])
+    yield client
+    client.stop(force_flag=True)

dss_datamover/systemtests/pytest_config.json

@@ -3,5 +3,8 @@
     "dest_path": "/tmp/xyz",
     "prefix": "/mnt/nfs_share/5gb/0_0/1_0/2_0/",
     "cache": ["/var/log/dss/prefix_index_data.json", "/var/log/dss/dm_resume_prefix_dir_keys.txt"],
-    "fs_config": {"server_as_prefix": "yes"}
-}
+    "fs_config": {"server_as_prefix": "yes"},
+    "large_file_path": "/tmp/large_files",
+    "large_file_size": 10,
+    "empty_file_path": "/tmp/empty_files"
+}

dss_datamover/systemtests/test_client.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+
+"""
+# The Clear BSD License
+#
+# Copyright (c) 2023 Samsung Electronics Co., Ltd.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted (subject to the limitations in the disclaimer
+# below) provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of Samsung Electronics Co., Ltd. nor the names of its
+#   contributors may be used to endorse or promote products derived from this
+#   software without specific prior written permission.
+# NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
+# THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
+# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+import os
+import pytest
+
+
+@pytest.mark.usefixtures(
+    "get_client",
+    "get_system_config_dict"
+)
+class TestClient:
+    """
+    Used for client related system tests.
+    """
+
+    def test_client_put(self, get_client):
+        get_client.operation = "PUT"
+        get_client.make_client_app_command()
+        get_client.start()
+        assert get_client.remote_client_status()
+
+    def test_client_get(self, get_client):
+        get_client.operation = "GET"
+        get_client.make_client_app_command()
+        get_client.start()
+        assert get_client.remote_client_status()
+
+    def test_client_list(self, get_client):
+        get_client.operation = "LIST"
+        get_client.make_client_app_command()
+        get_client.start()
+        assert get_client.remote_client_status()
+
+    def test_client_del(self, get_client):
+        get_client.operation = "DEL"
+        get_client.make_client_app_command()
+        get_client.start()
+        assert get_client.remote_client_status()

dss_datamover/systemtests/test_datamover.py

@@ -52,7 +52,9 @@
     "clear_datamover_cache",
     "setup_data_dir",
     "generate_full_ip_prefix",
-    "reset_master_obj"
+    "reset_master_obj",
+    "setup_large_file",
+    "setup_empty_file"
 )
 class TestDataMover:
     """
@@ -91,3 +93,19 @@ def test_delete_operation(self, get_master, reset_master_obj, setup_data_dir):
         get_master.start()
         process_del_operation(get_master)
         assert get_master.testcase_passed.value
+
+    def test_put_large_file(self, get_master, reset_master_obj, setup_data_dir, setup_large_file, get_pytest_configs):
+        print("Testing PUT operation with large file")
+        get_master.operation = "PUT"
+        get_master.fs_config['nfs'] = {'localhost': get_pytest_configs['large_file_path']}
+        get_master.prefix = 'localhost' + get_pytest_configs['large_file_path'] + '/'
+        get_master.start()
+        assert get_master.testcase_passed.value
+
+    def test_put_empty_file(self, get_master, reset_master_obj, setup_data_dir, setup_empty_file, get_pytest_configs):
+        print("Testing PUT operation with empty file")
+        get_master.operation = "PUT"
+        get_master.fs_config['nfs'] = {'localhost': get_pytest_configs['empty_file_path']}
+        get_master.prefix = 'localhost' + get_pytest_configs['empty_file_path'] + '/'
+        get_master.start()
+        assert get_master.testcase_passed.value

dss_datamover/systemtests/test_nfs_cluster.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python3
+
+"""
+# The Clear BSD License
+#
+# Copyright (c) 2023 Samsung Electronics Co., Ltd.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted (subject to the limitations in the disclaimer
+# below) provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of Samsung Electronics Co., Ltd. nor the names of its
+#   contributors may be used to endorse or promote products derived from this
+#   software without specific prior written permission.
+# NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
+# THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
+# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+import os
+import pytest
+
+
+@pytest.mark.usefixtures(
+    "get_nfs_cluster",
+    "get_system_config_dict",
+    "generate_full_ip_prefix"
+)
+class TestNFSCluster:
+    """
+    Used for NFS cluster related system tests.
+    """
+
+    def test_nfs_mount_all(self, get_nfs_cluster, get_system_config_dict):
+        get_nfs_cluster.mount_all()
+        for cluster_ip in get_system_config_dict["nfs"]:
+            for nfs_share in get_system_config_dict["nfs"][cluster_ip]:
+                nfs_share_mount = os.path.abspath("/" + cluster_ip + "/" + nfs_share) if \
+                    get_nfs_cluster.server_as_prefix else os.path.abspath("/" + nfs_share)
+                assert os.path.exists(nfs_share_mount)
+
+    def test_nfs_mount_based_on_prefix(self, get_nfs_cluster, get_system_config_dict, generate_full_ip_prefix):
+        for ip, nfs_shares in get_system_config_dict["fs_config"]["nfs"].items():
+            prefix = nfs_shares[0]
+            break
+        get_nfs_cluster.mount_based_on_prefix(prefix)
+        nfs_share_mount = os.path.abspath("/" + cluster_ip + "/" + prefix) if \
+            get_nfs_cluster.server_as_prefix else os.path.abspath("/" + prefix)
+        assert os.path.exists(nfs_share_mount)