Merge branch 'master' into daos_6923

Test-tag-hw-medium: pr,hw,medium,ib2 osa

Jenkinsfile

@@ -448,12 +448,27 @@ boolean skip_coverity() {
            skip_stage('build')
 }
 
+boolean skip_if_unstable() {
+    if (cachedCommitPragma(pragma: 'Allow-unstable-test') == 'true' ||
+        env.BRANCH_NAME == 'master' ||
+        env.BRANCH_NAME.startsWith("weekly-testing") ||
+        env.BRANCH_NAME.startsWith("release/")) {
+        return false
+    }
+
+    //Ok, it's a PR and the Allow pragma isn't set.  Skip if the build is
+    //unstable.
+
+    return currentBuild.currentResult == 'UNSTABLE'
+}
+
 boolean skip_testing_stage() {
     return  env.NO_CI_TESTING == 'true' ||
             (skip_stage('build') &&
              rpm_test_version() == '') ||
             doc_only_change() ||
-            skip_stage('test')
+            skip_stage('test') ||
+            skip_if_unstable()
 }
 
 boolean skip_unit_test() {
@@ -1361,6 +1376,48 @@ pipeline {
                         }
                     } // post
                 } // stage('Functional on Ubuntu 20.04')
+                stage('Test CentOS 7 RPMs') {
+                    when {
+                        beforeAgent true
+                        expression { ! skip_test_rpms_centos7() }
+                    }
+                    agent {
+                        label 'ci_vm1'
+                    }
+                    steps {
+                        testRpm inst_repos: daos_repos(),
+                                daos_pkg_version: daos_packages_version()
+                   }
+                } // stage('Test CentOS 7 RPMs')
+                stage('Scan CentOS 7 RPMs') {
+                    when {
+                        beforeAgent true
+                        expression { ! skip_scan_rpms_centos7() }
+                    }
+                    agent {
+                        label 'ci_vm1'
+                    }
+                    steps {
+                        scanRpms inst_repos: daos_repos(),
+                                 daos_pkg_version: daos_packages_version(),
+                                 inst_rpms: 'clamav clamav-devel',
+                                 test_script: 'ci/rpm/scan_daos.sh',
+                                 junit_files: 'maldetect.xml'
+                    }
+                    post {
+                        always {
+                            junit 'maldetect.xml'
+                        }
+                    }
+                } // stage('Scan CentOS 7 RPMs')
+            } // parallel
+        } // stage('Test')
+        stage('Test Hardware') {
+            when {
+                beforeAgent true
+                expression { ! skip_testing_stage() }
+            }
+            parallel {
                 stage('Functional_Hardware_Small') {
                     when {
                         beforeAgent true
@@ -1424,42 +1481,8 @@ pipeline {
                         }
                     }
                 } // stage('Functional_Hardware_Large')
-                stage('Test CentOS 7 RPMs') {
-                    when {
-                        beforeAgent true
-                        expression { ! skip_test_rpms_centos7() }
-                    }
-                    agent {
-                        label 'ci_vm1'
-                    }
-                    steps {
-                        testRpm inst_repos: daos_repos(),
-                                daos_pkg_version: daos_packages_version()
-                   }
-                } // stage('Test CentOS 7 RPMs')
-                stage('Scan CentOS 7 RPMs') {
-                    when {
-                        beforeAgent true
-                        expression { ! skip_scan_rpms_centos7() }
-                    }
-                    agent {
-                        label 'ci_vm1'
-                    }
-                    steps {
-                        scanRpms inst_repos: daos_repos(),
-                                 daos_pkg_version: daos_packages_version(),
-                                 inst_rpms: 'clamav clamav-devel',
-                                 test_script: 'ci/rpm/scan_daos.sh',
-                                 junit_files: 'maldetect.xml'
-                    }
-                    post {
-                        always {
-                            junit 'maldetect.xml'
-                        }
-                    }
-                } // stage('Scan CentOS 7 RPMs')
             } // parallel
-        } // stage('Test')
+        } // stage('Test Hardware')
         stage ('Test Report') {
             parallel {
                 stage('Bullseye Report') {

ci/provisioning/post_provision_config_nodes_EL_7.sh

@@ -38,9 +38,6 @@ group_repo_post() {
 }
 
 distro_custom() {
-    # shellcheck disable=SC2086
-    time dnf -y install python3
-
     if [ ! -e /usr/bin/pip3 ] &&
        [ -e /usr/bin/pip3.6 ]; then
         ln -s pip3.6 /usr/bin/pip3

src/bio/bio_xstream.c

@@ -28,9 +28,9 @@
 #define DAOS_BS_CLUSTER_SZ	(1ULL << 30)	/* 1GB */
 #define DAOS_BS_MD_PAGES	(1024 * 20)	/* 20k blobs per device */
 /* DMA buffer parameters */
-#define DAOS_DMA_CHUNK_MB	32		/* 32MB DMA chunks */
-#define DAOS_DMA_CHUNK_CNT_INIT	2		/* Per-xstream init chunks */
-#define DAOS_DMA_CHUNK_CNT_MAX	32		/* Per-xstream max chunks */
+#define DAOS_DMA_CHUNK_MB	2		/* 2MB DMA chunks */
+#define DAOS_DMA_CHUNK_CNT_INIT	128		/* Per-xstream init chunks */
+#define DAOS_DMA_CHUNK_CNT_MAX	512		/* Per-xstream max chunks */
 #define DAOS_NVME_MAX_CTRLRS	1024		/* Max read from nvme_conf */
 
 /* Max inflight blob IOs per io channel */

src/common/SConscript

@@ -51,7 +51,9 @@ def build_dts_library(env):
 
     denv.AppendUnique(CPPPATH=["../tests/suite"])
 
-    dts_lib = daos_build.library(denv, 'libdts', ['dts.c'], LIBS=libraries)
+    Import('dc_credit')
+    dts_lib = daos_build.library(denv, 'libdts', [dc_credit, 'dts.c'],
+                                 LIBS=libraries)
     denv.Install('$PREFIX/lib64/', dts_lib)
 
 def scons():
@@ -83,6 +85,12 @@ def scons():
     tlib_env = denv.Clone()
     tlib_env.AppendUnique(LIBS=['json-c'])
 
+    dc_credit = denv.SharedObject(['credit.c'])
+    Export('dc_credit')
+
+    cmd_parser = denv.SharedObject(['cmd_parser.c'])
+    Export('cmd_parser')
+
     prereqs.require(tlib_env, 'argobots')
 
     tests_lib_src = ['tests_lib.c', 'tests_dmg_helpers.c']

src/common/cmd_parser.c

@@ -0,0 +1,138 @@
+/**
+ * (C) Copyright 2021 Intel Corporation.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause-Patent
+ */
+
+#include <daos/common.h>
+#include <daos/object.h>
+#include <daos/cmd_parser.h>
+#include <daos.h>
+
+#define DTS_LINE_SIZE	1024
+
+/**
+ * Release a line buffer returned by readline
+ */
+void
+static freeline(char *line)
+{
+	D_FREE(line);
+}
+
+/**
+ * Read a command line from stdin, save trouble if we don't have libreadline
+ */
+static char *
+readline(const char *prompt)
+{
+	char	*line;
+	char	*cur;
+	bool	 eof;
+
+	D_ALLOC(line, DTS_LINE_SIZE);
+	if (!line)
+		return NULL;
+
+	if (prompt) {
+		fprintf(stdout, "%s", prompt);
+		fflush(stdout);
+	}
+
+	cur = line;
+	eof = false;
+	while (1) {
+		int	c;
+
+		c = fgetc(stdin);
+		if (c == EOF) {
+			if (ferror(stdin) || feof(stdin))
+				goto out_free;
+
+			eof = true;
+			break;
+		}
+
+		if (c == '\n')
+			break;
+
+		*cur++ = (char)c;
+		if (cur - line >= DTS_LINE_SIZE) {
+			fprintf(stderr, "line is too long\n");
+			goto out_free;
+		}
+	}
+	*cur = '\0';
+	if (eof && strlen(line) == 0)
+		goto out_free;
+
+	return line;
+ out_free:
+	freeline(line);
+	return NULL;
+}
+
+int
+cmd_parser(struct option *opts, const char *prompt,
+	   int (*cmd_func)(char opc, char *args))
+{
+	char	*line = NULL;
+	int	 rc;
+
+	for (rc = 0; rc == 0;) {
+		char	*args = NULL;
+		char	*cmd;
+		char	 opc;
+		int	 i;
+
+		if (line)
+			freeline(line);
+
+		line = readline(prompt);
+		if (!line)
+			break;
+
+		if (strlen(line) == 0)
+			continue; /* empty line */
+
+		cmd = daos_str_trimwhite(line);
+
+		for (i = 0, opc = 0;; i++) {
+			struct option *opt;
+
+			opt = &opts[i];
+			if (opt->name == NULL) {
+				opc = -1;
+				break;
+			}
+
+			if (strncasecmp(opt->name, cmd, strlen(opt->name)))
+				continue;
+
+			/* matched a command */
+			opc = (char)opt->val;
+			if (opt->has_arg) {
+				args = line + strlen(opt->name);
+				args = daos_str_trimwhite(args);
+			} else {
+				args = NULL;
+			}
+			break;
+		}
+
+		if (opc == -1) {
+			D_PRINT("Unknown command string %s, try \"help\"\n",
+				cmd);
+			continue;
+		}
+
+		rc = cmd_func(opc, args);
+		if (rc != 0)
+			break;
+	}
+
+	if (line)
+		freeline(line);
+
+	return rc;
+}