Fix nkv_test_cli

README.md

@@ -506,7 +506,7 @@ This playbook will perform the following actions:
 * Search for errors in all MinIO logs across all [hosts] / [servers]
 * Search for errors in all target logs across all [targets] / [servers]
 
-#### playbooks/deploy_datamover.yml
+#### playbooks/deploy_client.yml
 
 Execute this playbook to deploy DSS Client, including datamover, client library, and their dependencies.
 Artifacts are deployed to hosts under the [clients] group.
@@ -539,7 +539,7 @@ This path can be changed by setting the `coredump_dir` var. see: /group_vars/all
 Execute this playbook to download artifacts from the dss-artifacts S3 bucket.
 
 By default, this playbook will download artifacts from the public AWS S3 dss-artifacts bucket (public HTTP URL).
-The bucket URL can be overridden with the public URL of any S3-compatible bucket (eg: MinIO, DSS).
+The bucket can be overridden with the public URL of any S3-compatible bucket (eg: MinIO, DSS)
 Additionally, the branch name can also be overridden.
 
 #### playbooks/format_redeploy_dss_software.yml
@@ -776,7 +776,7 @@ iperf can be tuned by configuring the following vars (default values shown):
 
 #### playbooks/test_nkv_test_cli.yml
 
-Perform a basic nkv_test_cli test and report observed throughput.
+By default, perform a basic nkv_test_cli test and report observed throughput.
 This playbook will execute a suite of nkv_test_cli tests in order:
 
 1. Put
@@ -787,17 +787,29 @@ This playbook will execute a suite of nkv_test_cli tests in order:
 
 Upon test completion, throughput is reported for PUT and GET.
 
+Optionally, this playbook can be used to execute a suite of regression test cases.
+This can be done by changing the value of `nkv_test_cli_test` from `smoke` to `suite`.
+The default set of test cases can be found in `roles/test_nkv_test_cli/vars/suite001.yml`.
+You can create additional test suites using this file as a template.
+You can specify your custom test suite by setting `nkv_test_cli_suite` to `your-test` (default: `suite001`).
+
 nkv_test_cli can be tuned by configuring the following vars (default values shown):
 
-| Var name                       | Default | Description                                                                 |
-| ------------------------------ | ------- | --------------------------------------------------------------------------- |
-| nkv_test_cli_keysize           | 60      | Key size in bytes. Max size = 255                                           |
-| nkv_test_cli_valsize           | 1048576 | Value size in bytes. Max size = 1048576                                     |
-| nkv_test_cli_threads           | 128     | Number of threads                                                           |
-| nkv_test_cli_objects           | 2000    | Number of objects for each thread (total objects = objects x threads)       |
-| nkv_test_cli_vm_objects        | 100     | Number of objects if host is a VM (default reduced due to lower throughput) |
-| nkv_test_cli_async_timeout     | 600     | Async timeout in seconds (increase for larger dataset, or slow throughput)  |
-| nkv_test_cli_async_retry_delay | 5       | Async retry delay in seconds                                                |
+| Var name                       | Default      | Description                                                                           |
+| ------------------------------ | ------------ | ------------------------------------------------------------------------------------- |
+| nkv_test_cli_port              | 1030         | Port used by nkv_test_cli to communicate with subsystem                               |
+| nkv_test_cli_prefix            | meta/ansible | KV prefix used to write object. Must beging with `meta/`                              |
+| nkv_test_cli_keysize           | 60           | Key size in bytes. Max size = 255                                                     |
+| nkv_test_cli_valsize           | 1048576      | Value size in bytes. Max size = 1048576                                               |
+| nkv_test_cli_threads           | 128          | Number of threads                                                                     |
+| nkv_test_cli_objects           | 2000         | Number of objects for each thread (total objects = objects x threads)                 |
+| nkv_test_cli_async_timeout     | 600          | Async timeout in seconds (increase for larger dataset, or slow throughput)            |
+| nkv_test_cli_async_retry_delay | 5            | Async retry delay in seconds                                                          |
+| nkv_test_cli_test              | smoke        | Run standard "smoke" test. Change to "suite" to run regression test suite             |
+| nkv_test_cli_suite             | suite001     | Name of test suite to run. Corresponds to suite vars in roles/test_nkv_test_cli/vars/ |
+| nkv_test_cli_integrity         | false        | Run nkv_test_cli in data integrity mode                                               |
+| nkv_test_cli_mixed_io          | false        | Run nkv_test_cli with "small meta io before doing a big io"                           |
+| nkv_test_cli_simulate_minio    | false        | Run nkv_test_cli with "IO pattern similar to MinIO"                                   |
 
 #### playbooks/test_ping.yml
 

group_vars/all.yml

@@ -225,13 +225,20 @@
 # iperf_duration: 10
 
 ### nkv_test_cli defaults
+# nkv_test_cli_port: 1030
+# nkv_test_cli_prefix: meta/ansible
 # nkv_test_cli_keysize: 60
 # nkv_test_cli_valsize: 1048576
 # nkv_test_cli_threads: 128
 # nkv_test_cli_objects: 2000
-# nkv_test_cli_vm_objects: 100
 # nkv_test_cli_async_timeout: 600
 # nkv_test_cli_async_retry_delay: 5
+# nkv_test_cli_test: smoke
+# nkv_test_cli_test: suite
+# nkv_test_cli_suite: suite001
+# nkv_test_cli_integrity: false
+# nkv_test_cli_mixed_io: false
+# nkv_test_cli_simulate_minio: false
 
 ### EPEL defaults
 # skip_epel: false

playbooks/test_nkv_test_cli.yml

@@ -33,7 +33,7 @@
 #
 # #### playbooks/test_nkv_test_cli.yml
 #
-# Perform a basic nkv_test_cli test and report observed throughput.
+# By default, perform a basic nkv_test_cli test and report observed throughput.
 # This playbook will execute a suite of nkv_test_cli tests in order:
 #
 # 1. Put
@@ -44,17 +44,29 @@
 #
 # Upon test completion, throughput is reported for PUT and GET.
 #
+# Optionally, this playbook can be used to execute a suite of regression test cases.
+# This can be done by changing the value of `nkv_test_cli_test` from `smoke` to `suite`.
+# The default set of test cases can be found in `roles/test_nkv_test_cli/vars/suite001.yml`.
+# You can create additional test suites using this file as a template.
+# You can specify your custom test suite by setting `nkv_test_cli_suite` to `your-test` (default: `suite001`).
+#
 # nkv_test_cli can be tuned by configuring the following vars (default values shown):
 #
-# | Var name                       | Default | Description                                                                 |
-# | ------------------------------ | ------- | --------------------------------------------------------------------------- |
-# | nkv_test_cli_keysize           | 60      | Key size in bytes. Max size = 255                                           |
-# | nkv_test_cli_valsize           | 1048576 | Value size in bytes. Max size = 1048576                                     |
-# | nkv_test_cli_threads           | 128     | Number of threads                                                           |
-# | nkv_test_cli_objects           | 2000    | Number of objects for each thread (total objects = objects x threads)       |
-# | nkv_test_cli_vm_objects        | 100     | Number of objects if host is a VM (default reduced due to lower throughput) |
-# | nkv_test_cli_async_timeout     | 600     | Async timeout in seconds (increase for larger dataset, or slow throughput)  |
-# | nkv_test_cli_async_retry_delay | 5       | Async retry delay in seconds                                                |
+# | Var name                       | Default      | Description                                                                           |
+# | ------------------------------ | ------------ | ------------------------------------------------------------------------------------- |
+# | nkv_test_cli_port              | 1030         | Port used by nkv_test_cli to communicate with subsystem                               |
+# | nkv_test_cli_prefix            | meta/ansible | KV prefix used to write object. Must beging with `meta/`                              |
+# | nkv_test_cli_keysize           | 60           | Key size in bytes. Max size = 255                                                     |
+# | nkv_test_cli_valsize           | 1048576      | Value size in bytes. Max size = 1048576                                               |
+# | nkv_test_cli_threads           | 128          | Number of threads                                                                     |
+# | nkv_test_cli_objects           | 2000         | Number of objects for each thread (total objects = objects x threads)                 |
+# | nkv_test_cli_async_timeout     | 600          | Async timeout in seconds (increase for larger dataset, or slow throughput)            |
+# | nkv_test_cli_async_retry_delay | 5            | Async retry delay in seconds                                                          |
+# | nkv_test_cli_test              | smoke        | Run standard "smoke" test. Change to "suite" to run regression test suite             |
+# | nkv_test_cli_suite             | suite001     | Name of test suite to run. Corresponds to suite vars in roles/test_nkv_test_cli/vars/ |
+# | nkv_test_cli_integrity         | false        | Run nkv_test_cli in data integrity mode                                               |
+# | nkv_test_cli_mixed_io          | false        | Run nkv_test_cli with "small meta io before doing a big io"                           |
+# | nkv_test_cli_simulate_minio    | false        | Run nkv_test_cli with "IO pattern similar to MinIO"                                   |
 
 - name: Validate ansible versions and dependencies
   hosts: localhost

roles/test_nkv_test_cli/defaults/main.yml

@@ -33,15 +33,24 @@
 dss_dir: /usr/dss
 nkv_sdk_dir: "{{ dss_dir }}/nkv-sdk"
 nkv_sdk_bin_dir: "{{ nkv_sdk_dir }}/bin"
+nkv_sdk_conf_dir: "{{ nkv_sdk_dir }}/conf"
+dss_log_dir: /var/log/dss
 
 ### nkv_test_cli defaults
+nkv_test_cli_port: 1030
+nkv_test_cli_prefix: meta/ansible
 nkv_test_cli_keysize: 60
 nkv_test_cli_valsize: 1048576
 nkv_test_cli_threads: 128
-nkv_test_cli_objects: 2000
-nkv_test_cli_vm_objects: 100
+nkv_test_cli_objects_phys: 2000
+nkv_test_cli_vm_objects_vm: 100
 nkv_test_cli_async_timeout: 600
 nkv_test_cli_async_retry_delay: 5
+nkv_test_cli_test: smoke
+nkv_test_cli_suite: suite001
+nkv_test_cli_integrity: false
+nkv_test_cli_mixed_io: false
+nkv_test_cli_simulate_minio: false
 
 ### VLAN defaults
 rocev2_vlans:

roles/test_nkv_test_cli/tasks/case.yml

@@ -0,0 +1,17 @@
+---
+
+- name: "Execute test step: {{ test_case.id }}: {{ test_case.description }}"
+  ansible.builtin.include_tasks: nkv_test_cli.yml
+  vars:
+    operation: "{{ lookup('vars', 'test_nkv_test_cli_' + step) }}"
+    nkv_test_cli_integrity: "{{ test_case.integrity | d(false) }}"
+    nkv_test_cli_mixed_io: "{{ test_case.mixed_io | d(false) }}"
+    nkv_test_cli_simulate_minio: "{{ test_case.simulate_minio | d(false) }}"
+    nkv_test_cli_keysize: "{{ test_case.keysize | d(suite_keysize) }}"
+    nkv_test_cli_valsize: "{{ test_case.valsize | d(suite_valsize) }}"
+    nkv_test_cli_objects: "{{ test_case.objects | d(suite_objects) }}"
+    nkv_test_cli_threads: "{{ test_case.threads | d(suite_threads) }}"
+    assert_num_keys: "{{ test_case.expected_list_keys | d(test_case.threads | d(suite_threads) | int * test_case.objects | d(suite_objects) | int) }}"
+  loop: "{{ test_case.steps }}"
+  loop_control:
+    loop_var: step

roles/test_nkv_test_cli/tasks/main.yml

@@ -34,56 +34,27 @@
     target_hostnames: "{{ (groups['servers'] | d([]) + groups['targets'] | d([])) | unique }}"
     host_hostnames: "{{ (groups['servers'] | d([]) + groups['hosts'] | d([])) | unique }}"
 
-- name: Get RoCEv2 IP lists and NUMA
-  ansible.builtin.include_tasks: get_ip_numa.yml
-  when: inventory_hostname in host_hostnames
-
-- name: Execute nkv_test_cli put
-  ansible.builtin.include_tasks: nkv_test_cli.yml
-  vars:
-    operation: "{{ test_nkv_test_cli_put }}"
-  when: inventory_hostname in host_hostnames
-
-- name: Set put_throughput
+- name: Set host_id var from inventory
   ansible.builtin.set_fact:
-    put_throughput: "{{ combined_throughput }}"
-  when: inventory_hostname in host_hostnames
+    host_id: "{{ ansible_loop.index }}"
+  when: >
+    item == inventory_hostname
+  loop: "{{ ansible_play_hosts }}"
+  loop_control:
+    extended: true
 
-- name: Start Compaction
-  ansible.builtin.include_role:
-    name: start_compaction
-  when:
-    - inventory_hostname in target_hostnames
-    - dss_target_mode is search('kv_block')
-
-- name: Execute nkv_test_cli get
-  ansible.builtin.include_tasks: nkv_test_cli.yml
-  vars:
-    operation: "{{ test_nkv_test_cli_get }}"
-  when: inventory_hostname in host_hostnames
-
-- name: Set get_throughput
-  ansible.builtin.set_fact:
-    get_throughput: "{{ combined_throughput }}"
-  when: inventory_hostname in host_hostnames
-
-- name: Execute nkv_test_cli delete
-  ansible.builtin.include_tasks: nkv_test_cli.yml
-  vars:
-    operation: "{{ test_nkv_test_cli_delete }}"
+- name: Get RoCEv2 IP lists and NUMA
+  ansible.builtin.include_tasks: get_ip_numa.yml
   when: inventory_hostname in host_hostnames
 
-- name: Start Compaction
-  ansible.builtin.include_role:
-    name: start_compaction
-  when:
-    - inventory_hostname in target_hostnames
-    - dss_target_mode is search('kv_block')
-
-- name: Print throughput
-  ansible.builtin.debug:
-    msg: |
-      Put throughput:    {{ '%0.2f' | format(put_throughput | float) }} GB/s
-      Get throughput:    {{ '%0.2f' | format(get_throughput | float) }} GB/s
-  run_once: true
-  when: inventory_hostname in host_hostnames
+- name: Create log dir
+  ansible.builtin.file:
+    path: "{{ dss_log_dir }}"
+    state: directory
+    mode: 0755
+    owner: "{{ ansible_effective_user_id }}"
+    group: "{{ ansible_effective_group_id }}"
+  become: true
+
+- name: Execute nkv_test_cli test
+  ansible.builtin.include_tasks: "{{ nkv_test_cli_test }}.yml"