kdb cli: add support for external commands, with and without spec

doc/news/_preparation_next_release.md

@@ -299,6 +299,10 @@ This section keeps you up-to-date with the multi-language support provided by El
   - _NOTE_: The `path` argument for `kdb complete` is now required, so instead of `kdb complete` `kdb complete ""` has to be used. The
     reason for this is described in issue #4952.
 - <<TODO>>
+- add support for external commands, with and without spec _(@hannes99)_
+  - It is still possible to execute external binaries that are placed in a specific director, same as it worked in the old implementation.
+  - It is now possible to provide a specification to external commands to `kdb`, so `kdb` checks/parses arguments and then calls the
+    external program with them. [Add external command](../tutorials/external-commands.md)
 
 ### <<Tool>>
 

doc/tutorials/README.md

@@ -12,6 +12,7 @@ Read this first to get the basic concepts of Elektra.
 - [Cascading](cascading.md)
 - [Arrays](arrays.md)
 - [Mount Configuration Files](mount.md)
+- [External KDB commands](external-commands.md)
 
 ## Developers
 

doc/tutorials/external-commands.md

@@ -0,0 +1,121 @@
+# How-to: Add external commands
+
+This tutorial will describe how to provide `kdb` with the specification of external programs.
+So `kdb` can parse and check the provided options and arguments according to the provided specification.
+This allows you to have, for example, a shell script but its args are checked by `kdb` before running it.
+It is possible to either mount(`kdb mount`) the specification, or set the keys manually using `kdb set` and `kdb meta-set`.
+Both options will be described in the following.
+Here we'll define the specification for a simple script that removes files by moving them to a trash folder instead of directly deleting them.
+
+Since the arguments are already checked by `kdb`, the script knows these two are true
+
+1. The script can assume that if `$#` is `1`, `$1` has to be the file.
+2. And if `$#` is not `1` it hast to be `2`, with `$1` being the `-f` flag and `$2` being the filename.
+
+because the spec won't allow anything else.
+
+```bash
+#!/bin/bash
+
+if [ $# -eq 1 ]; then
+    # 1. was only called with the filename
+    mkdir -p $HOME/.trash
+
+    mv $1 $HOME/.trash
+    echo moved $1 to trash
+else
+    # 2. was called with -f flag and the filename
+    rm -f $2
+    echo deleted $2
+fi
+```
+
+Calling `kdb --help` will contain:
+
+```bash
+Usage: kdb [OPTION...] [COMMAND [...]]
+
+OPTIONS
+  --help                      Print this help message
+
+COMMANDS
+  ...
+  ...
+  trash                       Move a file to trash
+  ...
+  ...
+```
+
+and calling `kdb trash --help` will result in:
+
+```bash
+Usage: kdb trash [OPTION...] <file>
+
+OPTIONS
+  --help                      Print this help message
+  -f, --force                 Delete the file directly
+
+PARAMETERS
+  file                        the file that shall be deleted
+```
+
+For a reference of how the specification can look like [Command Line Options](command-line-options.md).
+
+## With `kdb mount`
+
+```ni
+[file]
+ meta:/description = the file that shall be deleted
+ meta:/args = indexed
+ meta:/args/index = 0
+
+[force]
+ meta:/description = Delete the file directly
+ meta:/opt = f
+ meta:/opt/long = force
+ meta:/opt/arg = none
+
+[]
+ meta:/command = trash
+ meta:/description = Move a file to trash
+ meta:/external = 1
+ meta:/bin = /path/to/trash.sh
+```
+
+The file then has to be mounted with
+
+```sh
+kdb mount /path/to/spec.ni spec:/sw/elektra/kdb/#0/current/trash mini
+```
+
+## Alternative to `kdb mount`
+
+This is the same as mounting the spec file.
+
+```bash
+kdb set spec:/sw/elektra/kdb/#0/current/trash ""
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash external 1
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash bin "/path/to/trash.sh"
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash command "trash"
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash description "Move a file to trash"
+kdb set spec:/sw/elektra/kdb/#0/current/trash/file ""
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash/file description "The file that should be moved to trash"
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash/file args indexed
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash/file args/index 0
+kdb set spec:/sw/elektra/kdb/#0/current/trash/force ""
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash/force description "Delete the file directly"
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash/force opt f
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash/force opt/long force
+kdb meta-set spec:/sw/elektra/kdb/#0/current/trash/force opt/arg none
+```
+
+> **_NOTE:_** Extra arguments are directly passed on to the external command. So it is possible to provide the external program with more
+> args than specified in the spec. Those are not check by `KDB`.
+
+So basically keys in `spec:/sw/elektra/kdb/#0/current/..` are considered external commands as long as the metakey `external` is set to 1
+and a metakey `bin`, that has the path to the binary, is set. Instead of mounting the spec file it is also possible to the set spec
+manually using `kdb`.
+
+`bin` should be an absolut path. If it is not, the binary will be search relative to where `kdb` is executed.
+
+External commands specified like this will appear in `kdb --help` and can be used with `kdb <command>` like any other command.

src/tools/kdb/external.c

@@ -0,0 +1,197 @@
+/**
+ * @file
+ *
+ * @brief Code to support external programs
+ *
+ * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
+ */
+
+#include <command.h>
+#include <external.h>
+#include <kdbease.h>
+
+#include <errno.h>
+#include <kdberrors.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#include <winsock2.h>
+#else
+#include <spawn.h>
+#include <sys/wait.h>
+#endif
+
+extern char ** environ;
+
+const char * getExternalBin (KeySet * binaries, const char * key)
+{
+	Key * tmp = keyNew ("/tmp", KEY_END);
+	keySetBaseName (tmp, key);
+	Key * resultKey = ksLookup (binaries, tmp, KDB_O_NONE);
+	keyDel (tmp);
+	if (resultKey == NULL)
+	{
+		return NULL;
+	}
+	return keyString (resultKey);
+}
+
+
+int tryLoadExternal (char * commandName, KeySet * binaries)
+{
+	char * execPathPtr = getenv ("KDB_EXEC_PATH");
+	bool found = false;
+	char path[PATH_MAX] = { 0 };
+	char * saveptr;
+	struct stat buf;
+
+	if (execPathPtr)
+	{
+		char * execPath = strdup (execPathPtr);
+		char * token = strtok_r (execPath, ":", &saveptr);
+		while (token != NULL && !found)
+		{
+			snprintf (path, sizeof (path), "%s/%s", token, commandName);
+			found = stat (path, &buf) != -1;
+			token = strtok_r (NULL, ":", &saveptr);
+		}
+		elektraFree (execPath);
+	}
+
+	if (!found)
+	{
+		snprintf (path, sizeof (path), "%s/%s", BUILTIN_EXEC_FOLDER, commandName);
+		found = stat (path, &buf) != -1;
+	}
+
+	if (found)
+	{
+		Key * tmp = keyNew ("/tmp", KEY_END);
+		keySetBaseName (tmp, commandName);
+		keySetString (tmp, path);
+		ksAppendKey (binaries, tmp);
+		return 0;
+	}
+	return 1;
+}
+
+int loadExternalSpec (KeySet * spec, KeySet * binaries, Key * errorKey)
+{
+	KDB * handle = kdbOpen (NULL, errorKey);
+	Key * baseKey = keyNew ("spec:" CLI_BASE_KEY, KEY_END);
+	KeySet * config = ksNew (0, KS_END);
+	if (kdbGet (handle, config, errorKey) == -1)
+	{
+		ELEKTRA_SET_VALIDATION_SEMANTIC_ERRORF (errorKey, "could not load '%s': %s", CLI_BASE_KEY, GET_ERR (baseKey));
+		keyDel (baseKey);
+		ksDel (config);
+		kdbClose (handle, errorKey);
+		return 1;
+	}
+	Key * cur = NULL;
+	KeySet * part = ksCut (config, baseKey);
+
+	for (elektraCursor it = 0; it < ksGetSize (part); ++it)
+	{
+		cur = ksAtCursor (part, it);
+		const Key * externalMeta = keyGetMeta (cur, "external");
+		const Key * externalBinary = keyGetMeta (cur, "bin");
+		const Key * externalCommandName = keyGetMeta (cur, "command");
+		bool isExternal = false;
+		if (externalCommandName != NULL && externalBinary != NULL && externalMeta != NULL &&
+		    elektraKeyToBoolean (externalMeta, &isExternal) && isExternal)
+		{ // add external spec and save path to binary
+			KeySet * externalCommandSpec = ksCut (part, cur);
+			Key * tmp = keyNew ("/tmp", KEY_END);
+			keySetBaseName (tmp, keyBaseName (cur));
+			keySetString (tmp, keyString (externalBinary));
+
+			ksAppendKey (binaries, tmp);
+			ksAppend (spec, externalCommandSpec);
+
+			ksDel (externalCommandSpec);
+		}
+	}
+	ksDel (part);
+	ksDel (config);
+	kdbClose (handle, errorKey);
+	keyDel (baseKey);
+	return 0;
+}
+
+int runExternal (const char * bin, char ** argv, Key * errorKey)
+{
+	// the external program should think it was called directly
+	argv[1] = (char *) bin;
+
+	int status = 0;
+
+#ifdef _WIN32
+	STARTUPINFO si;
+	PROCESS_INFORMATION pi;
+
+	ZeroMemory (&si, sizeof (si));
+	si.cb = sizeof (si);
+	ZeroMemory (&pi, sizeof (pi));
+
+	// Construct command line string
+	char cmdline[MAX_PATH] = "";
+	for (int i = 1; argv[i]; ++i)
+	{
+		strcat (cmdline, "\"");
+		strcat (cmdline, argv[i]);
+		strcat (cmdline, "\" ");
+	}
+
+	// Start the child process.
+	if (!CreateProcess (NULL,    // Module name
+			    cmdline, // Command line
+			    NULL,    // Process handle not inheritable
+			    NULL,    // Thread handle not inheritable
+			    FALSE,   // Set handle inheritance to FALSE
+			    0,	     // No creation flags
+			    NULL,    // Use parent's environment block
+			    NULL,    // Use parent's starting directory
+			    &si,     // Pointer to STARTUPINFO structure
+			    &pi)     // Pointer to PROCESS_INFORMATION structure
+	)
+	{
+		ELEKTRA_SET_RESOURCE_ERRORF (errorKey, "CreateProcess failed: %lu", GetLastError ());
+		return 1;
+	}
+
+	// Wait until child process exits.
+	WaitForSingleObject (pi.hProcess, INFINITE);
+
+	// Get exit code
+	DWORD exitCode;
+	GetExitCodeProcess (pi.hProcess, &exitCode);
+	status = (int) exitCode;
+
+	// Close process and thread handles.
+	CloseHandle (pi.hProcess);
+	CloseHandle (pi.hThread);
+#else
+	pid_t pid;
+
+	if (posix_spawn (&pid, bin, NULL, NULL, &(argv[1]), environ) != 0)
+	{
+		ELEKTRA_SET_RESOURCE_ERRORF (errorKey, "posix_spawn failed: %s", strerror (errno));
+		return 1;
+	}
+
+	if (waitpid (pid, &status, 0) < 0)
+	{
+		ELEKTRA_SET_RESOURCE_ERRORF (errorKey, "waitpid failed: %s", strerror (errno));
+		return 1;
+	}
+#endif
+
+	fflush (stdout);
+	return status;
+}

src/tools/kdb/external.h

@@ -0,0 +1,20 @@
+/**
+ * @file
+ *
+ * @brief Header for things needed for external programs
+ *
+ * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
+ */
+
+#ifndef ELEKTRA_KDB_EXTERNAL_H
+#define ELEKTRA_KDB_EXTERNAL_H
+
+#include <kdb.h>
+
+const char * getExternalBin (KeySet * binaries, const char * key);
+
+int runExternal (const char * bin, char ** argv, Key * errorKey);
+int loadExternalSpec (KeySet * spec, KeySet * binaries, Key * errorKey);
+int tryLoadExternal (char * commandName, KeySet * binaries);
+
+#endif // ELEKTRA_KDB_EXTERNAL_H