From 2d208a034b1eca35d036467e7bf8d9853b72d42d Mon Sep 17 00:00:00 2001 From: Rabit Date: Sun, 28 Mar 2021 12:46:33 -0700 Subject: [PATCH] Build on MacOS --- Makefile.am | 12 +++++-- configure.ac | 30 ++++++++++++++++ example/server.c | 11 ++++++ src/byte.c | 2 +- src/client.c | 14 ++++---- src/heap.c | 21 +++++++++++ src/raft.c | 1 + src/uv.c | 2 ++ src/uv_fs.c | 46 +++++++++++++++++++++++- src/uv_ip.h | 4 +++ src/uv_os.c | 40 +++++++++++++++++++-- src/uv_os.h | 4 +++ src/uv_snapshot.c | 1 - src/uv_writer.c | 92 +++++++++++++++++++++++++++++++++-------------- src/uv_writer.h | 4 +++ 15 files changed, 242 insertions(+), 42 deletions(-) diff --git a/Makefile.am b/Makefile.am index 93d62c9e3..560fb41a0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,7 +8,7 @@ raftinclude_HEADERS = lib_LTLIBRARIES = libraft.la libraft_la_CFLAGS = $(AM_CFLAGS) -fvisibility=hidden -libraft_la_LDFLAGS = -version-info 0:7:0 +libraft_la_LDFLAGS = -version-info 0:7:0 -no-undefined libraft_la_SOURCES = \ src/byte.c \ src/client.c \ @@ -33,10 +33,13 @@ libraft_la_SOURCES = \ src/snapshot.c \ src/start.c \ src/state.c \ - src/syscall.c \ src/tick.c \ src/tracing.c +if LINUX +libraft_la_SOURCES += src/syscall.c +endif + bin_PROGRAMS = check_PROGRAMS = \ @@ -154,7 +157,6 @@ libtest_la_SOURCES += \ test_unit_uv_SOURCES = \ src/err.c \ src/heap.c \ - src/syscall.c \ src/tracing.c \ src/uv_fs.c \ src/uv_os.c \ @@ -166,6 +168,10 @@ test_unit_uv_LDFLAGS = $(UV_LIBS) test_unit_uv_CFLAGS = $(AM_CFLAGS) -Wno-conversion test_unit_uv_LDADD = libtest.la +if LINUX +test_unit_uv_SOURCES += src/syscall.c +endif + # The integration/uv test is not linked to libraft, but built # directly against the libraft sources in order to test some # non-visible, non-API functions. diff --git a/configure.ac b/configure.ac index 4014b5ede..852f38b05 100644 --- a/configure.ac +++ b/configure.ac @@ -10,6 +10,21 @@ AC_USE_SYSTEM_EXTENSIONS # Defines _GNU_SOURCE and similar LT_INIT +AC_CANONICAL_HOST + +build_linux=no +build_windows=no +build_mac=no + +#Detect target OS +AS_CASE([$host_os],[linux*],[build_linux=yes]) +AS_CASE([$host_os],[mingw*],[build_windows=yes]) +AS_CASE([$host_os],[darwin*],[build_mac=yes]) + +AM_CONDITIONAL([LINUX], [test "x$build_linux" = "xyes"]) +AM_CONDITIONAL([WINDOWS], [test "x$build_windows" = "xyes"]) +AM_CONDITIONAL([MAC], [test "x$build_mac" = "xyes"]) + # The libuv raft_io implementation is built by default if libuv is found, unless # explicitly disabled. AC_ARG_ENABLE(uv, AS_HELP_STRING([--disable-uv], [do not build the libuv-based raft_io implementation])) @@ -125,5 +140,20 @@ CC_CHECK_FLAGS_APPEND([AM_LDFLAGS],[LDFLAGS],[ \ ]) AC_SUBST(AM_LDLAGS) +CC_CHECK_FLAGS_APPEND([AM_LDFLAGS],[LDFLAGS],[ \ + -z relro \ + -z now \ + -fstack-protector-strong \ + --param=ssp-buffer-size=4 \ +]) +AC_SUBST(AM_LDLAGS) + +case $host in + *mingw*) + # -D__USE_MINGW_ANSI_STDIO is deprecated it seems. + CC_CHECK_FLAGS_APPEND([AM_CFLAGS],[CFLAGS],[-D_POSIX]) + ;; +esac + AC_CONFIG_FILES([raft.pc Makefile]) AC_OUTPUT diff --git a/example/server.c b/example/server.c index 604c99487..b65ce3412 100644 --- a/example/server.c +++ b/example/server.c @@ -13,6 +13,11 @@ #define Logf(SERVER_ID, FORMAT, ...) \ printf("%d: " FORMAT "\n", SERVER_ID, __VA_ARGS__) +//apparently srandom isn't defined on mingw +#if !defined(srandom) +#define srandom srand +#endif + /******************************************************************** * * Sample application FSM that just increases a counter. @@ -166,8 +171,12 @@ static int ServerInit(struct Server *s, memset(s, 0, sizeof *s); /* Seed the random generator */ +#if defined(_WIN32) + srandom((unsigned)time(NULL)); +#else timespec_get(&now, TIME_UTC); srandom((unsigned)(now.tv_nsec ^ now.tv_sec)); +#endif s->loop = loop; @@ -385,8 +394,10 @@ int main(int argc, char *argv[]) dir = argv[1]; id = (unsigned)atoi(argv[2]); +#if !defined(_WIN32) /* Ignore SIGPIPE, see https://github.com/joyent/libuv/issues/1254 */ signal(SIGPIPE, SIG_IGN); +#endif /* Initialize the libuv loop. */ rv = uv_loop_init(&loop); diff --git a/src/byte.c b/src/byte.c index 3a856d48b..861942eed 100644 --- a/src/byte.c +++ b/src/byte.c @@ -98,7 +98,7 @@ A million repetitions of "a" #define PDP_ENDIAN 3412 /* LSB first in word, MSW first in long (pdp)*/ #if defined(vax) || defined(ns32000) || defined(sun386) || \ - defined(__i386__) || defined(MIPSEL) || defined(_MIPSEL) || \ + defined(__i386__) || defined(__x86_64__) || defined(MIPSEL) || defined(_MIPSEL) || \ defined(BIT_ZERO_ON_RIGHT) || defined(__alpha__) || defined(__alpha) #define BYTE_ORDER LITTLE_ENDIAN #endif diff --git a/src/client.c b/src/client.c index 986bc4b33..1b079cba8 100644 --- a/src/client.c +++ b/src/client.c @@ -198,14 +198,15 @@ int raft_add(struct raft *r, req->cb = cb; + assert(r->leader_state.change == NULL); + r->leader_state.change = req; + rv = clientChangeConfiguration(r, req, &configuration); if (rv != 0) { + r->leader_state.change = NULL; goto err_after_configuration_copy; } - assert(r->leader_state.change == NULL); - r->leader_state.change = req; - return 0; err_after_configuration_copy: @@ -352,14 +353,15 @@ int raft_remove(struct raft *r, req->cb = cb; + assert(r->leader_state.change == NULL); + r->leader_state.change = req; + rv = clientChangeConfiguration(r, req, &configuration); if (rv != 0) { + r->leader_state.change = NULL; goto err_after_configuration_copy; } - assert(r->leader_state.change == NULL); - r->leader_state.change = req; - return 0; err_after_configuration_copy: diff --git a/src/heap.c b/src/heap.c index 64b2a8665..ddc89cb8d 100644 --- a/src/heap.c +++ b/src/heap.c @@ -31,13 +31,34 @@ static void *defaultRealloc(void *data, void *ptr, size_t size) static void *defaultAlignedAlloc(void *data, size_t alignment, size_t size) { (void)data; +#ifdef _WIN32 + return _aligned_malloc(size, alignment); +#elif defined(__APPLE__) + void * p1; // original block + void ** p2; // aligned block + size_t offset = alignment + sizeof(void *) - 1; + if ((p1 = (void *)malloc(size + offset)) == NULL) + return NULL; + p2 = (void **)(((uintptr_t)(p1) + offset) & ~(alignment - 1)); + p2[-1] = p1; + return p2; +#else return aligned_alloc(alignment, size); +#endif } static void defaultAlignedFree(void *data, size_t alignment, void *ptr) { (void)alignment; +#ifdef _WIN32 + (void)ptr; + _aligned_free(data); +#elif defined(__APPLE__) + (void)data; + free(((void * *)ptr)[-1]); +#else defaultFree(data, ptr); +#endif } static struct raft_heap defaultHeap = { diff --git a/src/raft.c b/src/raft.c index 39aa55018..2f5aa37c1 100644 --- a/src/raft.c +++ b/src/raft.c @@ -157,6 +157,7 @@ int raft_bootstrap(struct raft *r, const struct raft_configuration *conf) rv = r->io->bootstrap(r->io, conf); if (rv != 0) { + ErrMsgTransfer(r->io->errmsg, r->errmsg, "io"); return rv; } diff --git a/src/uv.c b/src/uv.c index 792918f22..9693132bf 100644 --- a/src/uv.c +++ b/src/uv.c @@ -487,12 +487,14 @@ static int uvBootstrap(struct raft_io *io, /* Write the term */ rv = uvSetTerm(io, 1); if (rv != 0) { + ErrMsgPrintf(io->errmsg, "Unable to set UV term"); return rv; } /* Create the first closed segment file, containing just one entry. */ rv = uvSegmentCreateFirstClosed(uv, configuration); if (rv != 0) { + ErrMsgPrintf(io->errmsg, "Unable to create first closed segment"); return rv; } diff --git a/src/uv_fs.c b/src/uv_fs.c index b6ce117c9..00aa53d17 100644 --- a/src/uv_fs.c +++ b/src/uv_fs.c @@ -2,7 +2,9 @@ #include #include +#ifdef __linux__ #include +#endif #include #include "assert.h" @@ -55,6 +57,10 @@ int UvFsCheckDir(const char *dir, char *errmsg) int UvFsSyncDir(const char *dir, char *errmsg) { +#ifdef _WIN32 + // Windows doesn't really support sync on folders. + return 0; +#endif uv_file fd; int rv; rv = UvOsOpen(dir, UV_FS_O_RDONLY | UV_FS_O_DIRECTORY, 0, &fd); @@ -181,7 +187,9 @@ int UvFsAllocateFile(const char *dir, UvOsJoin(dir, filename, path); /* TODO: use RWF_DSYNC instead, if available. */ - flags |= O_DSYNC; +#if defined(UV_FS_O_DSYNC) + flags |= UV_FS_O_DSYNC; +#endif rv = uvFsOpenFile(dir, filename, flags, S_IRUSR | S_IWUSR, fd, errmsg); if (rv != 0) { @@ -204,6 +212,11 @@ int UvFsAllocateFile(const char *dir, } goto err_after_open; } + rv = UvOsFsync(*fd); + if (rv != 0) { + UvOsErrMsg(errmsg, "fsync", rv); + goto err_after_open; + } return 0; @@ -226,6 +239,9 @@ static int uvFsWriteFile(const char *dir, int rv; size_t size; unsigned i; +#ifdef _WIN32 + uv_buf_t convBufArr[n_bufs]; +#endif size = 0; for (i = 0; i < n_bufs; i++) { size += bufs[i].len; @@ -234,7 +250,18 @@ static int uvFsWriteFile(const char *dir, if (rv != 0) { goto err; } +#ifdef _WIN32 + // casting raft_buffer to uv_buf_t changes the value of the len field. + // uv_buf_t for Windows is ULONG, while for linux it's size_t. Copying + // the values and explicitly casting len to ULONG seems to work. + for (i = 0; i < n_bufs; i++) { + convBufArr[i].len = (unsigned long)bufs[i].len; + convBufArr[i].base = bufs[i].base; + } + rv = UvOsWrite(fd, convBufArr, n_bufs, 0); +#else rv = UvOsWrite(fd, (const uv_buf_t *)bufs, n_bufs, 0); +#endif if (rv != (int)(size)) { if (rv < 0) { UvOsErrMsg(errmsg, "write", rv); @@ -381,7 +408,14 @@ int UvFsMakeOrOverwriteFile(const char *dir, goto err; } +#ifdef WINDOWS + uv_buf_t convBufArr[1]; + convBufArr[0].len = (unsigned long)buf->len; + convBufArr[0].base = buf->base; + rv = UvOsWrite(fd, convBufArr, 1, 0); +#else rv = UvOsWrite(fd, (const uv_buf_t *)buf, 1, 0); +#endif if (rv != (int)(buf->len)) { if (rv < 0) { UvOsErrMsg(errmsg, "write", rv); @@ -593,6 +627,7 @@ int UvFsTruncateAndRenameFile(const char *dir, return RAFT_IOERR; } +#ifdef __linux__ /* Check if direct I/O is possible on the given fd. */ static int probeDirectIO(int fd, size_t *size, char *errmsg) { @@ -671,6 +706,7 @@ static int probeDirectIO(int fd, size_t *size, char *errmsg) *size = 0; return 0; } +#endif #if defined(RWF_NOWAIT) /* Check if fully non-blocking async I/O is possible on the given fd. */ @@ -763,6 +799,10 @@ int UvFsProbeCapabilities(const char *dir, bool *async, char *errmsg) { +#ifndef __linux__ + *direct = 0; +#endif + int fd; /* File descriptor of the probe file */ int rv; char ignored[RAFT_ERRMSG_BUF_SIZE]; @@ -777,11 +817,13 @@ int UvFsProbeCapabilities(const char *dir, } UvFsRemoveFile(dir, UV__FS_PROBE_FILE, ignored); +#ifdef __linux__ /* Check if we can use direct I/O. */ rv = probeDirectIO(fd, direct, errmsg); if (rv != 0) { goto err_after_file_open; } +#endif #if !defined(RWF_NOWAIT) /* We can't have fully async I/O, since io_submit might potentially block. @@ -806,8 +848,10 @@ int UvFsProbeCapabilities(const char *dir, close(fd); return 0; +#ifdef __linux__ err_after_file_open: close(fd); +#endif err: return rv; } diff --git a/src/uv_ip.h b/src/uv_ip.h index bada8b471..b718db1b2 100644 --- a/src/uv_ip.h +++ b/src/uv_ip.h @@ -3,7 +3,11 @@ #ifndef UV_IP_H_ #define UV_IP_H_ +#ifdef _WIN32 +#include +#else #include +#endif /* Split @address into @host and @port and populate @addr accordingly. */ int uvIpParse(const char *address, struct sockaddr_in *addr); diff --git a/src/uv_os.c b/src/uv_os.c index 0debca0ab..91a8f8105 100644 --- a/src/uv_os.c +++ b/src/uv_os.c @@ -5,16 +5,21 @@ #include #include #include -#include #include -#include -#include #include #include #include "assert.h" #include "err.h" + +#ifdef __linux__ +#include +#include #include "syscall.h" +#elif defined(__FreeBSD__) || defined(__APPLE__) +#include +#include +#endif /* Default permissions when creating a directory. */ #define DEFAULT_DIR_PERM 0700 @@ -37,6 +42,24 @@ int UvOsClose(uv_file fd) return uv_fs_close(NULL, &req, fd, NULL); } +#ifdef _WIN32 +int UvOsFallocate(uv_file fd, off_t offset, off_t len) +{ + // This is not really correct. A better implementation would probably + // be to open the file, seek to offset and write zeros. + int rv; + rv = ftruncate(fd, offset+len); + if (rv != 0) { + return rv; + } + rv = UvOsFsync(fd); + if (rv != 0) { + return rv; + } + return 0; +} +#else + /* Emulate fallocate(). Mostly taken from glibc's implementation. */ static int uvOsFallocateEmulation(int fd, off_t offset, off_t len) { @@ -70,6 +93,7 @@ static int uvOsFallocateEmulation(int fd, off_t offset, off_t len) int UvOsFallocate(uv_file fd, off_t offset, off_t len) { int rv; +#ifdef __linux__ rv = posix_fallocate(fd, offset, len); if (rv != 0) { /* From the manual page: @@ -80,6 +104,7 @@ int UvOsFallocate(uv_file fd, off_t offset, off_t len) if (rv != EOPNOTSUPP) { return -rv; } +#endif /* This might be a libc implementation (e.g. musl) that doesn't * implement a transparent fallback if fallocate() is not supported * by the underlying file system. */ @@ -87,9 +112,12 @@ int UvOsFallocate(uv_file fd, off_t offset, off_t len) if (rv != 0) { return -EOPNOTSUPP; } +#ifdef __linux__ } +#endif return 0; } +#endif int UvOsTruncate(uv_file fd, off_t offset) { @@ -147,10 +175,15 @@ void UvOsJoin(const char *dir, const char *filename, char *path) assert(UV__DIR_HAS_VALID_LEN(dir)); assert(UV__FILENAME_HAS_VALID_LEN(filename)); strcpy(path, dir); +#ifdef _WIN32 + strcat(path, "\\"); +#else strcat(path, "/"); +#endif strcat(path, filename); } +#ifdef __linux__ int UvOsIoSetup(unsigned nr, aio_context_t *ctxp) { int rv; @@ -225,3 +258,4 @@ int UvOsSetDirectIo(uv_file fd) } return 0; } +#endif diff --git a/src/uv_os.h b/src/uv_os.h index bceb91390..d54aa0151 100644 --- a/src/uv_os.h +++ b/src/uv_os.h @@ -4,7 +4,9 @@ #define UV_OS_H_ #include +#ifdef __linux__ #include +#endif #include #include #include @@ -105,6 +107,7 @@ int UvOsRename(const char *path1, const char *path2); void UvOsJoin(const char *dir, const char *filename, char *path); /* TODO: figure a portable abstraction. */ +#ifdef __linux__ int UvOsIoSetup(unsigned nr, aio_context_t *ctxp); int UvOsIoDestroy(aio_context_t ctx); int UvOsIoSubmit(aio_context_t ctx, long nr, struct iocb **iocbpp); @@ -115,6 +118,7 @@ int UvOsIoGetevents(aio_context_t ctx, struct timespec *timeout); int UvOsEventfd(unsigned int initval, int flags); int UvOsSetDirectIo(uv_file fd); +#endif /* Format an error message caused by a failed system call or stdlib function. */ #define UvOsErrMsg(ERRMSG, SYSCALL, ERRNUM) \ diff --git a/src/uv_snapshot.c b/src/uv_snapshot.c index 38486204d..5af015f44 100644 --- a/src/uv_snapshot.c +++ b/src/uv_snapshot.c @@ -1,6 +1,5 @@ #include #include -#include #include "array.h" #include "assert.h" diff --git a/src/uv_writer.c b/src/uv_writer.c index 0a597e1af..fa17b2262 100644 --- a/src/uv_writer.c +++ b/src/uv_writer.c @@ -7,6 +7,18 @@ #include "assert.h" #include "heap.h" +/* Return the total lengths of the given buffers. */ +static size_t lenOfBufs(const uv_buf_t bufs[], unsigned n) +{ + size_t len = 0; + unsigned i; + for (i = 0; i < n; i++) { + len += bufs[i].len; + } + return len; +} + +#ifdef __linux__ /* Copy the error message from the request object to the writer object. */ static void uvWriterReqTransferErrMsg(struct UvWriterReq *req) { @@ -204,6 +216,7 @@ static void uvWriterPollCb(uv_poll_t *poller, int status, int events) uvWriterReqFinish(req); } } +#endif /* __linux__ */ int UvWriterInit(struct UvWriter *w, struct uv_loop_s *loop, @@ -219,6 +232,13 @@ int UvWriterInit(struct UvWriter *w, w->data = data; w->loop = loop; w->fd = fd; + +#ifndef __linux__ + (void)direct; + (void)async; + (void)max_concurrent_writes; + (void)errmsg; +#else w->async = async; w->ctx = 0; w->events = NULL; @@ -306,9 +326,12 @@ int UvWriterInit(struct UvWriter *w, UvOsIoDestroy(w->ctx); err: assert(rv != 0); +#endif /* __linux__ */ + return rv; } +#ifdef __linux__ static void uvWriterCleanUpAndFireCloseCb(struct UvWriter *w) { assert(w->closing); @@ -363,20 +386,21 @@ static void uvWriterCheckCb(struct uv_check_s *check) } uv_close((struct uv_handle_s *)&w->check, uvWriterCheckCloseCb); } +#endif void UvWriterClose(struct UvWriter *w, UvWriterCloseCb cb) { - int rv; assert(!w->closing); w->closing = true; w->close_cb = cb; +#ifdef __linux__ /* We can close the event file descriptor right away, but we shouldn't close * the main file descriptor or destroy the AIO context since there might be * threadpool requests in flight. */ UvOsClose(w->event_fd); - rv = uv_poll_stop(&w->event_poller); + int rv = uv_poll_stop(&w->event_poller); assert(rv == 0); /* Can this ever fail? */ uv_close((struct uv_handle_s *)&w->event_poller, uvWriterPollerCloseCb); @@ -388,17 +412,10 @@ void UvWriterClose(struct UvWriter *w, UvWriterCloseCb cb) } else { uv_close((struct uv_handle_s *)&w->check, uvWriterCheckCloseCb); } -} - -/* Return the total lengths of the given buffers. */ -static size_t lenOfBufs(const uv_buf_t bufs[], unsigned n) -{ - size_t len = 0; - unsigned i; - for (i = 0; i < n; i++) { - len += bufs[i].len; - } - return len; +#else + uv_fs_close(w->loop, NULL, w->fd, NULL); //TODO: NULL for callback and request? + cb(w); +#endif /* __linux__ */ } int UvWriterSubmit(struct UvWriter *w, @@ -408,35 +425,55 @@ int UvWriterSubmit(struct UvWriter *w, size_t offset, UvWriterReqCb cb) { - int rv = 0; -#if defined(RWF_NOWAIT) - struct iocb *iocbs = &req->iocb; -#endif /* RWF_NOWAIT */ assert(!w->closing); - - /* TODO: at the moment we are not leveraging the support for concurrent - * writes, so ensure that we're getting write requests - * sequentially. */ - if (w->n_events == 1) { - assert(QUEUE_IS_EMPTY(&w->poll_queue)); - assert(QUEUE_IS_EMPTY(&w->work_queue)); - } - assert(w->fd >= 0); +#ifdef __linux__ assert(w->event_fd >= 0); assert(w->ctx != 0); assert(req != NULL); assert(bufs != NULL); assert(n > 0); +#endif req->writer = w; req->len = lenOfBufs(bufs, n); req->status = -1; req->work.data = NULL; req->cb = cb; - memset(&req->iocb, 0, sizeof req->iocb); memset(req->errmsg, 0, sizeof req->errmsg); +#ifndef __linux__ + uv_fs_t write_req; + + uv_fs_write(w->loop, &write_req, w->fd, bufs, n, (int64_t)offset, NULL); + + int result = (int)write_req.result; + // TODO: should probably check that write_req.result == n? + if (result == -1) { + req->status = RAFT_IOERR; + } else { + req->status = 0; + } + + cb(req, req->status); + + return req->status; +#else + int rv = 0; + memset(&req->iocb, 0, sizeof req->iocb); + +#if defined(RWF_NOWAIT) + struct iocb *iocbs = &req->iocb; +#endif /* RWF_NOWAIT */ + + /* TODO: at the moment we are not leveraging the support for concurrent + * writes, so ensure that we're getting write requests + * sequentially. */ + if (w->n_events == 1) { + assert(QUEUE_IS_EMPTY(&w->poll_queue)); + assert(QUEUE_IS_EMPTY(&w->work_queue)); + } + req->iocb.aio_fildes = (uint32_t)w->fd; req->iocb.aio_lio_opcode = IOCB_CMD_PWRITEV; req->iocb.aio_reqprio = 0; @@ -527,4 +564,5 @@ int UvWriterSubmit(struct UvWriter *w, err: assert(rv != 0); return rv; +#endif /* __linux__ */ } diff --git a/src/uv_writer.h b/src/uv_writer.h index bae95b755..d2e6d5f0c 100644 --- a/src/uv_writer.h +++ b/src/uv_writer.h @@ -22,10 +22,12 @@ struct UvWriter struct uv_loop_s *loop; /* Event loop */ uv_file fd; /* File handle */ bool async; /* Whether fully async I/O is supported */ +#ifdef __linux__ aio_context_t ctx; /* KAIO handle */ struct io_event *events; /* Array of KAIO response objects */ unsigned n_events; /* Length of the events array */ int event_fd; /* Poll'ed to check if write is finished */ +#endif struct uv_poll_s event_poller; /* Poll event_fd for completed poll requests */ struct uv_check_s check; /* Check for completed threadpool requests */ UvWriterCloseCb close_cb; /* Close callback */ @@ -61,7 +63,9 @@ struct UvWriterReq int status; /* Request result code */ struct uv_work_s work; /* To execute logic in the threadpool */ UvWriterReqCb cb; /* Callback to invoke upon request completion */ +#ifdef __linux__ struct iocb iocb; /* KAIO request (for writing) */ +#endif char errmsg[256]; /* Error description (for thread-safety) */ queue queue; /* Prev/next links in the inflight queue */ };