diff --git a/CMakeLists.txt b/CMakeLists.txt index 9276f4d8..de6ff7a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,6 +198,13 @@ add_executable(consumer ) target_link_libraries(consumer tcmu) +# As per `consumer`, but demonstrates how to use libtcmu without netlink +add_executable(consumer_no_netlink + scsi.c + consumer_no_netlink.c + ) + +target_link_libraries(consumer_no_netlink tcmu) if (with-zbc) # Stuff for building the file zbc handler add_library(handler_file_zbc diff --git a/consumer.c b/consumer.c index f5167d15..9fa1059e 100644 --- a/consumer.c +++ b/consumer.c @@ -130,6 +130,7 @@ int main(int argc, char **argv) struct pollfd pollfds[16]; int i; int ret; + bool use_netlink = true; if (tcmu_setup_log(LOG_DIR)) { fprintf(stderr, "Could not setup tcmu logger.\n"); @@ -139,7 +140,7 @@ int main(int argc, char **argv) /* If any TCMU devices that exist that match subtype, handler->added() will now be called from within tcmulib_initialize(). */ - tcmulib_ctx = tcmulib_initialize(&foo_handler, 1); + tcmulib_ctx = tcmulib_initialize(&foo_handler, 1, use_netlink); if (!tcmulib_ctx) { tcmu_err("tcmulib_initialize failed with %p\n", tcmulib_ctx); exit(1); diff --git a/consumer_no_netlink.c b/consumer_no_netlink.c new file mode 100644 index 00000000..8a6db475 --- /dev/null +++ b/consumer_no_netlink.c @@ -0,0 +1,299 @@ +/* + * This file is licensed to you under your choice of the GNU Lesser + * General Public License, version 2.1 or any later version (LGPLv2.1 or + * later), or the Apache License 2.0. + */ + +/* + * This example shows how one can use TCMU without using netlink. Only kernels + * which support per device disabling of netlink in target_core_user may use + * this feature. To check whether your kernel supports this feature try and + * create a TCMU backstore with `nl_reply_supported` set to "-1": + * + * $ mkdir -p /sys/kernal/config/target/core/user_1234/test + * $ echo "dev_size=1048576,dev_config=foo/bar,nl_reply_supported=-1" > /sys/kernel/config/target/core/user_1234/test/control + * $ echo 1 > /sys/kernel/config/target/core/user_1234/test/enable + * + * Using TCMU without netlink is useful for scenarios where they may be + * multiple applications using TCMU on one host. This sort of configuration is + * not possible when TCMU uses netlink because backstore device events are + * broadcasted over netlink to all TCMU subscribers. This mean one application + * can receive netlink events for devices which pertain to another application. + * If an app receives a netlink event for a device it does not know about libtcmu + * will return an error message and the kernel will fail the device action. + * + * The example creates a TCMU backing store (using the create_backstore.sh + * script) and then notifies libtcmu that the backstore has been created using + * tcmulib_notify_device_added. The example then sits in a loop processing + * commands. Note: unless you set up an HBA and LUN there won't be any + * commands to process. When the app recieves a termination signal it exits + * the processing loop. It then notifies libtcmu that the backstore is about to + * be removed via the tcmulib_notify_device_removed method. We then delete the + * backstore using the remove_backstore.sh script. + * + * This example must be ran in the the same directory as its accompanying + * bash scripts, i.e. the top level repo directory. + * + * NOTE: TCMU without netlink is not a panacea to the multi-TCMU problem. One + * still has to be careful to ensure that multiple apps don't try and create + * the same backstore (user_XXXX) at the same time. It may be necessary to + * synchronise/serialise other configs actions too. + * + * NOTE: this is a bare-bones example! It eschews any proper error handling, + * uses hardcoded values, and relies on bash scripts. It's intended to give + * a rough outline of how to use the feature and should not be used as the + * basis of any production code. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libtcmu.h" +#include "libtcmu_log.h" +#include "scsi.h" +#include "scsi_defs.h" +#include "target_core_user_local.h" + +#define LOG_DIR "/var/log" +#define CFG_STRING "tcm-user/1/test/foo/xxx" + +struct tcmu_device *tcmu_dev_array[128]; +size_t dev_array_len = 0; + +struct foo_state { + int fd; + uint64_t num_lbas; + uint32_t block_size; +}; + +static volatile int keep_running = 1; + +static void signal_handler(int) { + keep_running = 0; +} + +static bool run_command(char* buffer, size_t size, char* cmd) +{ + FILE *f = popen(cmd, "r"); + if (!f) { + tcmu_err("couldn't run command %s\n", cmd); + return false; + } + + fgets(buffer, size, f); + pclose(f); + return true; +} + +static bool create_backstore() +{ + int ret = system("./create_backstore.sh"); + return ret == 0; +} + +static bool find_uio_device(char** uio_name) +{ + char buffer[1024]; + if (!run_command(buffer, sizeof(buffer), "./find_uio_device.sh")) + return false; + + *uio_name = strdup(buffer); + return true; +} + +static void delete_backstore() +{ + system("./delete_backstore.sh"); +} + +static int foo_open(struct tcmu_device *dev) +{ + /* open the backing file */ + /* alloc private struct 'foo_state' */ + /* Save a ptr to it in dev->hm_private */ + + /* Add new device to our horrible fixed-length array */ + tcmu_dev_array[dev_array_len] = dev; + dev_array_len++; + + return 0; +} + +static void foo_close(struct tcmu_device *dev) +{ + /* not supported in this example */ +} + +static int foo_handle_cmd( + struct tcmu_device *dev, + uint8_t *cdb, + struct iovec *iovec, + size_t iov_cnt, + uint8_t *sense) +{ + struct foo_state *state = tcmu_dev_get_private(dev); + uint8_t cmd; + + cmd = cdb[0]; + + switch (cmd) { + case INQUIRY: + return tcmu_emulate_inquiry(dev, NULL, cdb, iovec, iov_cnt); + case TEST_UNIT_READY: + return tcmu_emulate_test_unit_ready(cdb, iovec, iov_cnt); + case SERVICE_ACTION_IN_16: + if (cdb[1] == READ_CAPACITY_16) + return tcmu_emulate_read_capacity_16(state->num_lbas, + state->block_size, + cdb, iovec, iov_cnt); + else + return TCMU_STS_NOT_HANDLED; + break; + case MODE_SENSE: + case MODE_SENSE_10: + return tcmu_emulate_mode_sense(dev, cdb, iovec, iov_cnt); + break; + case MODE_SELECT: + case MODE_SELECT_10: + return tcmu_emulate_mode_select(dev, cdb, iovec, iov_cnt); + break; + case READ_6: + case READ_10: + case READ_12: + case READ_16: + // A real "read" implementation goes here! + return TCMU_STS_RD_ERR; + + case WRITE_6: + case WRITE_10: + case WRITE_12: + case WRITE_16: + // A real "write" implemention goes here! + return TCMU_STS_OK; + + default: + tcmu_err("unknown command %x\n", cdb[0]); + return TCMU_STS_NOT_HANDLED; + } +} + +static struct tcmulib_handler foo_handler = { + .name = "Handler for foo devices (example code)", + .subtype = "foo", + .cfg_desc = "a description goes here", + + .added = foo_open, + .removed = foo_close, +}; + +int main(int argc, char **argv) +{ + struct tcmulib_context *tcmulib_ctx; + struct pollfd pollfds[16]; + int i; + int ret; + bool use_netlink = false; + char* uio_name = NULL; + + signal(SIGINT, signal_handler); + + if (tcmu_setup_log(LOG_DIR)) { + fprintf(stderr, "Could not setup tcmu logger.\n"); + exit(1); + } + + /* If any TCMU devices that exist that match subtype, + handler->added() will now be called from within + tcmulib_initialize(). */ + tcmulib_ctx = tcmulib_initialize(&foo_handler, 1, use_netlink); + if (!tcmulib_ctx) { + tcmu_err("tcmulib_initialize failed with %p\n", tcmulib_ctx); + exit(1); + } + + /* Create a TCMU backstore */ + if (!create_backstore()) { + tcmu_err("failed to create backstore\n"); + exit(1); + } + + /* Find the uio device */ + if (!find_uio_device(&uio_name)) { + tcmu_err("failed to find uio device for backstore\n"); + goto cleanup1; + } + + /* Notify libtcmu that we've created the device */ + ret = tcmulib_notify_device_added(tcmulib_ctx, uio_name, CFG_STRING); + if (ret != 0) { + tcmu_err("tcmulib_notify_device_added failed\n"); + goto cleanup2; + } + + while (keep_running) { + for (i = 0; i < dev_array_len; i++) { + pollfds[i].fd = tcmu_dev_get_fd(tcmu_dev_array[i]); + pollfds[i].events = POLLIN; + pollfds[i].revents = 0; + } + + ret = ppoll(pollfds, dev_array_len, NULL, NULL); + if (ret == -1 && keep_running) { + tcmu_err("ppoll() returned %d, exiting\n", ret); + exit(EXIT_FAILURE); + } + + /* Process any commands - in this demo binary there won't be any commands + to process unless you set up a HBA and LUN for the backstore */ + for (i = 0; i < dev_array_len; i++) { + if (pollfds[i].revents) { + struct tcmulib_cmd *cmd; + struct tcmu_device *dev = tcmu_dev_array[i]; + + tcmulib_processing_start(dev); + + while ((cmd = tcmulib_get_next_command(dev, 0)) != NULL) { + ret = foo_handle_cmd(dev, + cmd->cdb, + cmd->iovec, + cmd->iov_cnt, + cmd->sense_buf); + tcmulib_command_complete(dev, cmd, ret); + } + + tcmulib_processing_complete(dev); + } + } + } + + tcmu_info("main thread exiting\n"); + +cleanup2: + /* Notify the library that the device is about to be removed */ + tcmu_info("calling tcmulib_notify_device_removed\n"); + tcmulib_notify_device_removed(tcmulib_ctx, uio_name); + +cleanup1: + /* Delete the backstore */ + tcmu_info("removing backstore\n"); + delete_backstore(); + + free(uio_name); + + return 0; +} diff --git a/create_backstore.sh b/create_backstore.sh new file mode 100755 index 00000000..4b508a50 --- /dev/null +++ b/create_backstore.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +user_dir="user_1" +dev_config="foo/xxx" + +if [ -d /sys/kernel/config/target/core/$user_dir/test ]; then + echo "backstore already exists - remove it with delete_backstore.sh" + exit 1 +fi + +mkdir -p /sys/kernel/config/target/core/$user_dir/test +echo "dev_size=1048576,hw_max_sectors=256,hw_block_size=4096,dev_config=$dev_config,nl_reply_supported=-1" > /sys/kernel/config/target/core/$user_dir/test/control +echo "1" > /sys/kernel/config/target/core/$user_dir/test/enable +echo "created backstore" +exit 0 diff --git a/delete_backstore.sh b/delete_backstore.sh new file mode 100755 index 00000000..2932376c --- /dev/null +++ b/delete_backstore.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +user_dir="user_1" + +rmdir /sys/kernel/config/target/core/$user_dir/test +rmdir /sys/kernel/config/target/core/$user_dir diff --git a/find_uio_device.sh b/find_uio_device.sh new file mode 100755 index 00000000..f9878036 --- /dev/null +++ b/find_uio_device.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Finds the name of the uio device pertaining to the TCMU backstore +# "test/foo". The uio device - e.g. uio0 - is printed to stdout. + +for d in /sys/class/uio/*/name; +do + cfgstring=`cat $d` + if [[ $cfgstring == tcm-user/*/test/foo/xxx ]]; then + uio=`echo $d | grep -oP "uio\d"` + echo -n $uio + exit 0 + fi +done +exit 1 diff --git a/libtcmu.c b/libtcmu.c index 46a6fbcc..fcabc17e 100644 --- a/libtcmu.c +++ b/libtcmu.c @@ -767,7 +767,9 @@ static int open_devices(struct tcmulib_context *ctx) static void release_resources(struct tcmulib_context *ctx) { - teardown_netlink(ctx->nl_sock); + if (ctx->nl_sock) + teardown_netlink(ctx->nl_sock); + darray_free(ctx->handlers); darray_free(ctx->devices); free(ctx); @@ -775,7 +777,8 @@ static void release_resources(struct tcmulib_context *ctx) struct tcmulib_context *tcmulib_initialize( struct tcmulib_handler *handlers, - size_t handler_count) + size_t handler_count, + bool use_netlink) { struct tcmulib_context *ctx; int ret; @@ -785,10 +788,12 @@ struct tcmulib_context *tcmulib_initialize( if (!ctx) return NULL; - ctx->nl_sock = setup_netlink(ctx); - if (!ctx->nl_sock) { - free(ctx); - return NULL; + if (use_netlink) { + ctx->nl_sock = setup_netlink(ctx); + if (!ctx->nl_sock) { + free(ctx); + return NULL; + } } darray_init(ctx->handlers); @@ -817,12 +822,18 @@ void tcmulib_close(struct tcmulib_context *ctx) int tcmulib_get_master_fd(struct tcmulib_context *ctx) { - return nl_socket_get_fd(ctx->nl_sock); + if (ctx->nl_sock) + return nl_socket_get_fd(ctx->nl_sock); + else + return -1; } int tcmulib_master_fd_ready(struct tcmulib_context *ctx) { - return nl_recvmsgs_default(ctx->nl_sock); + if (ctx->nl_sock) + return nl_recvmsgs_default(ctx->nl_sock); + else + return -1; } void *tcmu_dev_get_private(struct tcmu_device *dev) @@ -1373,3 +1384,38 @@ void tcmulib_processing_complete(struct tcmu_device *dev) tcmu_err("failed to write device /dev/%s, %d\n", dev->dev_name, errno); } + +int tcmulib_notify_device_reconfig(struct tcmulib_context* ctx, char* dev_name, struct tcmulib_cfg_info* cfg) +{ + struct tcmu_device *dev; + int i, ret; + + dev = lookup_dev_by_name(ctx, dev_name, &i); + if (!dev) { + tcmu_err("Could not reconfigure device %s: not found.\n", + dev_name); + return -ENODEV; + } + + if (!dev->handler->reconfig) { + tcmu_dev_dbg(dev, "Reconfiguration is not supported with this device.\n"); + return -EOPNOTSUPP; + } + + ret = dev->handler->reconfig(dev, cfg); + if (ret < 0) { + tcmu_dev_dbg(dev, "Handler reconfig for %s failed with error %s.\n", + tcmulib_cfg_type_lookup[cfg->type], strerror(-ret)); + } + + return ret; +} + +int tcmulib_notify_device_added(struct tcmulib_context *ctx, char *dev_name, + char *cfgstring) { + return device_add(ctx, dev_name, cfgstring, false); +} + +void tcmulib_notify_device_removed(struct tcmulib_context *ctx, char *dev_name) { + device_remove(ctx, dev_name, false); +} diff --git a/libtcmu.h b/libtcmu.h index 8ac62329..bcc9482e 100644 --- a/libtcmu.h +++ b/libtcmu.h @@ -84,24 +84,64 @@ struct tcmulib_handler { /* Opaque (private) type */ struct tcmulib_context; -/* Claim subtypes you wish to handle. Returns libtcmu's master fd or -error.*/ +/* + * Claim subtypes you wish to handle. Returns a tcmulib_context or a NULL + * pointer on error. If you wish use libtcmu without netlink set use_netlink=false. + * In this mode you must: + * * create all backstores with netlink disabled (by setting nl_reply_supported=-1 + * at backstore create time) + * * manually notify the library of backstore creation, reconfiguration and removal + * events via the tcmu_notify* APIs. + */ struct tcmulib_context *tcmulib_initialize( struct tcmulib_handler *handlers, - size_t handler_count); + size_t handler_count, + bool use_netlink); /* Register to TCMU DBus service, for the claimed subtypes to be configurable * in targetcli. */ void tcmulib_register(struct tcmulib_context *ctx); -/* Gets the master file descriptor used by tcmulib. */ +/* Gets the master file descriptor used by tcmulib. If you called tcmulib_initialize + * with use_netlink=false then we aren't using netlink for device notifications and + * you shouldn't use this method. + */ int tcmulib_get_master_fd(struct tcmulib_context *ctx); /* * Call this when the master fd becomes ready, from your main thread. - * Handlers' callbacks may be called before it returns. + * Handlers' callbacks may be called before it returns. If you called + * tcmulib_initialize with use_netlink=false then we aren't using netlink + * for device notifications and you shouldn't use this method. */ int tcmulib_master_fd_ready(struct tcmulib_context *ctx); +/* + * Only applicable if tcmulib_initialize was called with use_netlink=false + * + * Notify the library that a TCMU backstore has been created. This + * function will open the backstore and call the appropriate handler. + */ +int tcmulib_notify_device_added(struct tcmulib_context *ctx, char *dev_name, + char *cfgstring); + +/* + * Only applicable if tcmulib_initialize was called with use_netlink=false + * + * Notify the library that the TCMU backstore is about to be removed. + * This function will close the backstore and call the removed handler. + * This method should be called before deleting the backstore. + */ +void tcmulib_notify_device_removed(struct tcmulib_context *ctx, char *dev_name); + +/* + * Only applicable if tcmulib_initialize was called with use_netlink=false + * + * Notify the library that the TCMU backstore should be reconfigured. + * This function will simply call the reconfig handler. + */ +int tcmulib_notify_device_reconfig(struct tcmulib_context* ctx, char* dev_name, struct tcmulib_cfg_info *cfg); + /* * When a device fd becomes ready, call this to get SCSI cmd info in * 'cmd' struct. libtcmu will allocate hm_cmd_size bytes for each cmd diff --git a/main.c b/main.c index 70b42335..97f6d57d 100644 --- a/main.c +++ b/main.c @@ -1380,7 +1380,7 @@ int main(int argc, char **argv) darray_append(handlers, tmp_handler); } - tcmulib_context = tcmulib_initialize(handlers.item, handlers.size); + tcmulib_context = tcmulib_initialize(handlers.item, handlers.size, true); if (!tcmulib_context) { tcmu_err("tcmulib_initialize failed\n"); goto err_free_handlers; diff --git a/tcmu-synthesizer.c b/tcmu-synthesizer.c index a9f7cb8e..2ed62afb 100644 --- a/tcmu-synthesizer.c +++ b/tcmu-synthesizer.c @@ -197,7 +197,7 @@ int main(int argc, char **argv) exit(1); } - ctx = tcmulib_initialize(&syn_handler, 1); + ctx = tcmulib_initialize(&syn_handler, 1, true); if (!ctx) { tcmu_err("tcmulib_initialize failed\n"); tcmu_destroy_log();