SlimRedis barebone (#158)

* skeleton redis server

* integration test

* process functions added, no processing done yet

* update how argument #s are handled; implement ping

* ping works

* use a single test config

* rename to slimredis; will build data structure on cuckoo for now

* fix ping with argument

CMakeLists.txt

@@ -34,6 +34,7 @@ option(HAVE_ASSERT_PANIC "assert_panic disabled by default" OFF)
 option(HAVE_LOGGING "logging enabled by default" ON)
 option(HAVE_STATS "stats enabled by default" ON)
 option(TARGET_PINGSERVER "build pingserver binary" ON)
+option(TARGET_SLIMREDIS "build slimredis binary" ON)
 option(TARGET_SLIMCACHE "build slimcache binary" ON)
 option(TARGET_TWEMCACHE "build twemcache binary" ON)
 option(COVERAGE "code coverage" OFF)

config/test.conf

@@ -0,0 +1 @@
+debug_log_level: 6

src/protocol/data/redis/cmd_hash.h

@@ -1,28 +1,23 @@
 #pragma once
 
-/*
- * Note: negative # of arguments means variable number of arguments:
- *   e.g. `-2' means at least two arguments. This notation is inherited from
- *   the original Redis server implementation.
- */
 
 /*          type                string          # of args */
 #define REQ_HASH(ACTION)                            \
-    ACTION( REQ_HDEL,           "hdel",         -3 )\
-    ACTION( REQ_HDELALL,        "hdelall",      2  )\
-    ACTION( REQ_HEXISTS,        "hexists",      3  )\
-    ACTION( REQ_HGET,           "hget",         3  )\
-    ACTION( REQ_HGETALL,        "hgetall",      2  )\
-    ACTION( REQ_HINCRBY,        "hincrby",      4  )\
-    ACTION( REQ_HINCRBYFLOAT,   "hincrbyfloat", 4  )\
-    ACTION( REQ_HKEYS,          "hkeys",        2  )\
-    ACTION( REQ_HLEN,           "hlen",         2  )\
-    ACTION( REQ_HMGET,          "hmget",        -3 )\
-    ACTION( REQ_HMSET,          "hmset",        -4 )\
-    ACTION( REQ_HSET,           "hset",         4  )\
-    ACTION( REQ_HSETNX,         "hsetnx",       4  )\
-    ACTION( REQ_HSTRLEN,        "hstrlen",      3  )\
-    ACTION( REQ_HVALS,          "hvals",        2  )\
-    ACTION( REQ_HSCAN,          "hscan",        -3 )
+    ACTION( REQ_HDEL,           "hdel",         3,  -1 )\
+    ACTION( REQ_HDELALL,        "hdelall",      2,   0 )\
+    ACTION( REQ_HEXISTS,        "hexists",      3,   0 )\
+    ACTION( REQ_HGET,           "hget",         3,   0 )\
+    ACTION( REQ_HGETALL,        "hgetall",      2,   0 )\
+    ACTION( REQ_HINCRBY,        "hincrby",      4,   0 )\
+    ACTION( REQ_HINCRBYFLOAT,   "hincrbyfloat", 4,   0 )\
+    ACTION( REQ_HKEYS,          "hkeys",        2,   0 )\
+    ACTION( REQ_HLEN,           "hlen",         2,   0 )\
+    ACTION( REQ_HMGET,          "hmget",        3,  -1 )\
+    ACTION( REQ_HMSET,          "hmset",        4,  -1 )\
+    ACTION( REQ_HSET,           "hset",         4,   0 )\
+    ACTION( REQ_HSETNX,         "hsetnx",       4,   0 )\
+    ACTION( REQ_HSTRLEN,        "hstrlen",      3,   0 )\
+    ACTION( REQ_HVALS,          "hvals",        2,   0 )\
+    ACTION( REQ_HSCAN,          "hscan",        3,   0 )
 
 /* "hlen KEY" == "*2\r\n$4\r\nhlen\r\n$3\r\nKEY\r\n" */

src/protocol/data/redis/cmd_misc.h

@@ -1,6 +1,7 @@
 #pragma once
 
-/*          type            string      # of args */
-#define REQ_MISC(ACTION)                    \
-    ACTION( REQ_PING,       "ping",     -1 )\
-    ACTION( REQ_QUIT,       "quit",     1  )
+/*          type            string      #arg#opt */
+#define REQ_MISC(ACTION)                        \
+    ACTION( REQ_FLUSHALL,   "flushall", 1,  0  )\
+    ACTION( REQ_PING,       "ping",     1,  1  )\
+    ACTION( REQ_QUIT,       "quit",     1,  0  )

src/protocol/data/redis/cmd_zset.h

@@ -1,25 +1,25 @@
 #pragma once
 
 /*          type                    string              # of args */
-#define REQ_ZSET(ACTION)                                    \
-    ACTION( REQ_ZADD,               "zadd",             -4 )\
-    ACTION( REQ_ZINCRBY,            "zincrby",          4  )\
-    ACTION( REQ_ZREM,               "zrem",             -3 )\
-    ACTION( REQ_ZREMRANGEBYSCORE,   "zremrangebyscore", 4  )\
-    ACTION( REQ_ZREMRANGEBYRANK,    "zremrangebyrank",  4  )\
-    ACTION( REQ_ZREMRANGEBYLEX,     "zremrangebylex",   4  )\
-    ACTION( REQ_ZUNIONSTORE,        "zunionstore",      -4 )\
-    ACTION( REQ_ZINTERSTORE,        "zinterstore",      -4 )\
-    ACTION( REQ_ZRANGE,             "zrange",           -4 )\
-    ACTION( REQ_ZRANGEBYSCORE,      "zrangebyscore",    -4 )\
-    ACTION( REQ_ZREVRANGEBYSCORE,   "zrevrangebyscore", -4 )\
-    ACTION( REQ_ZRANGEBYLEX,        "zrangebylex",      -4 )\
-    ACTION( REQ_ZREVRANGEBYLEX,     "zrevrangebylex",   -4 )\
-    ACTION( REQ_ZCOUNT,             "zcount",           4  )\
-    ACTION( REQ_ZLEXCOUNT,          "zlexcount",        4  )\
-    ACTION( REQ_ZREVRANGE,          "zrevrange",        -4 )\
-    ACTION( REQ_ZCARD,              "zcard",            2  )\
-    ACTION( REQ_ZSCORE,             "zscore",           3  )\
-    ACTION( REQ_ZRANK,              "zrank",            3  )\
-    ACTION( REQ_ZREVRANK,           "zrevrank",         3  )\
-    ACTION( REQ_ZSCAN,              "zscan",            -3 )
+#define REQ_ZSET(ACTION)                                        \
+    ACTION( REQ_ZADD,               "zadd",             4,  -1 )\
+    ACTION( REQ_ZINCRBY,            "zincrby",          4,  0  )\
+    ACTION( REQ_ZREM,               "zrem",             3,  -1 )\
+    ACTION( REQ_ZREMRANGEBYSCORE,   "zremrangebyscore", 4,  0  )\
+    ACTION( REQ_ZREMRANGEBYRANK,    "zremrangebyrank",  4,  0  )\
+    ACTION( REQ_ZREMRANGEBYLEX,     "zremrangebylex",   4,  0  )\
+    ACTION( REQ_ZUNIONSTORE,        "zunionstore",      4,  -1 )\
+    ACTION( REQ_ZINTERSTORE,        "zinterstore",      4,  -1 )\
+    ACTION( REQ_ZRANGE,             "zrange",           4,  -1 )\
+    ACTION( REQ_ZRANGEBYSCORE,      "zrangebyscore",    4,  -1 )\
+    ACTION( REQ_ZREVRANGEBYSCORE,   "zrevrangebyscore", 4,  -1 )\
+    ACTION( REQ_ZRANGEBYLEX,        "zrangebylex",      4,  -1 )\
+    ACTION( REQ_ZREVRANGEBYLEX,     "zrevrangebylex",   4,  -1 )\
+    ACTION( REQ_ZCOUNT,             "zcount",           4,  0  )\
+    ACTION( REQ_ZLEXCOUNT,          "zlexcount",        4,  0  )\
+    ACTION( REQ_ZREVRANGE,          "zrevrange",        4,  -1 )\
+    ACTION( REQ_ZCARD,              "zcard",            2,  0  )\
+    ACTION( REQ_ZSCORE,             "zscore",           3,  0  )\
+    ACTION( REQ_ZRANK,              "zrank",            3,  0  )\
+    ACTION( REQ_ZREVRANK,           "zrevrank",         3,  0  )\
+    ACTION( REQ_ZSCAN,              "zscan",            3,  -1 )

src/protocol/data/redis/compose.h

@@ -1,6 +1,7 @@
 #pragma once
 
-#include <buffer/cc_dbuf.h>
+#include "token.h"
+
 #include <cc_define.h>
 #include <cc_metric.h>
 
@@ -24,13 +25,6 @@ typedef struct {
     COMPOSE_RSP_METRIC(METRIC_DECLARE)
 } compose_rsp_metrics_st;
 
-typedef enum compose_rstatus {
-    COMPOSE_OK          = 0,
-    COMPOSE_EUNFIN      = -1,
-    COMPOSE_ENOMEM      = -2,
-    COMPOSE_EINVALID    = -3,
-    COMPOSE_EOTHER      = -4,
-} compose_rstatus_t;
 
 struct request;
 struct response;

src/protocol/data/redis/parse.c

@@ -72,9 +72,9 @@ _parse_cmd(struct request *req)
     /* check narg */
     cmd = command_table[type];
     narg = req->token->nelem;
-    if ((cmd.narg >= 0 && cmd.narg != narg) || narg + cmd.narg < 0) {
-        log_warn("wrong number of arguments for '%.*s': %d expected, %d given",
-                cmd.bstr.len, cmd.bstr.data, cmd.narg, narg);
+    if (narg < cmd.narg || narg > (cmd.narg + cmd.nopt)) {
+        log_warn("wrong # of arguments for '%.*s': %d+[%d] expected, %d given",
+                cmd.bstr.len, cmd.bstr.data, cmd.narg, cmd.nopt, narg);
         return PARSE_EINVALID;
     }
 
@@ -121,8 +121,8 @@ parse_req(struct request *req, struct buf *buf)
         }
         el = array_push(req->token);
         status = parse_element(el, buf);
+        log_verb("parse element returned status %d", status);
         if (status != PARSE_OK) {
-            log_verb("parse element returned status %d", status);
             request_reset(req);
             buf->rpos = old_rpos;
             return status;
@@ -131,6 +131,7 @@ parse_req(struct request *req, struct buf *buf)
     }
 
     status = _parse_cmd(req);
+    log_verb("parse command returned status %d", status);
     if (status != PARSE_OK) {
         buf->rpos = old_rpos;
         return status;

src/protocol/data/redis/parse.h

@@ -2,8 +2,8 @@
 
 #include "request.h"
 #include "response.h"
+#include "token.h"
 
-#include <buffer/cc_buf.h>
 #include <cc_define.h>
 #include <cc_metric.h>
 
@@ -33,14 +33,6 @@ typedef struct {
     PARSE_RSP_METRIC(METRIC_DECLARE)
 } parse_rsp_metrics_st;
 
-typedef enum parse_rstatus {
-    PARSE_OK        = 0,
-    PARSE_EUNFIN    = -1,
-    PARSE_EEMPTY    = -2,
-    PARSE_EOVERSIZE = -3,
-    PARSE_EINVALID  = -4,
-    PARSE_EOTHER    = -5,
-} parse_rstatus_t;
 
 void parse_setup(parse_req_metrics_st *req, parse_rsp_metrics_st *rsp);
 void parse_teardown(void);

src/protocol/data/redis/request.c

@@ -11,11 +11,13 @@
 static bool request_init = false;
 static request_metrics_st *request_metrics = NULL;
 
-struct command command_table[REQ_SENTINEL];
-#define CMD_INIT(_type, _str, _narg) \
-    { .type = _type, .bstr = { sizeof(_str) - 1, (_str) }, .narg = _narg },
+#define CMD_INIT(_type, _str, _narg, _nopt) {\
+    .type = _type,                           \
+    .bstr = { sizeof(_str) - 1, (_str) },    \
+    .narg = _narg,                           \
+    .nopt = _nopt },
 struct command command_table[REQ_SENTINEL] = {
-    { .type = REQ_UNKNOWN, .bstr = { 0, NULL }, .narg = 0 },
+    { .type = REQ_UNKNOWN, .bstr = { 0, NULL }, .narg = 0, .nopt = 0 },
     REQ_HASH(CMD_INIT)
     REQ_ZSET(CMD_INIT)
     REQ_MISC(CMD_INIT)
@@ -189,7 +191,13 @@ request_setup(request_options_st *options, request_metrics_st *metrics)
     request_metrics = metrics;
 
     if (options != NULL) {
+        int i;
         ntoken = option_uint(&options->request_ntoken);
+        for (i = 1; i < REQ_SENTINEL; i++) { /* update nopt based on ntoken */
+            if (command_table[i].nopt == -1) {
+                command_table[i].nopt = ntoken - command_table[i].narg;
+            }
+        }
         max = option_uint(&options->request_poolsize);
     }
     request_pool_create(max);

src/protocol/data/redis/request.h

@@ -3,6 +3,7 @@
 #include "cmd_hash.h"
 #include "cmd_misc.h"
 #include "cmd_zset.h"
+#include "token.h"
 
 #include <cc_array.h>
 #include <cc_define.h>
@@ -18,7 +19,7 @@
 
 /*          name                type                default         description */
 #define REQUEST_OPTION(ACTION)                                                          \
-    ACTION( request_ntoken,     OPTION_TYPE_UINT,   REQ_NTOKEN,     "# tokens in request")\
+    ACTION( request_ntoken,     OPTION_TYPE_UINT,   REQ_NTOKEN,     "# tokens in req"  )\
     ACTION( request_poolsize,   OPTION_TYPE_UINT,   REQ_POOLSIZE,   "request pool size")
 
 typedef struct {
@@ -38,7 +39,7 @@ typedef struct {
     REQUEST_METRIC(METRIC_DECLARE)
 } request_metrics_st;
 
-#define GET_TYPE(_type, _str, narg) _type,
+#define GET_TYPE(_type, _str, narg, nopt) _type,
 typedef enum cmd_type {
     REQ_UNKNOWN,
     REQ_HASH(GET_TYPE)
@@ -48,10 +49,19 @@ typedef enum cmd_type {
 } cmd_type_e;
 #undef GET_TYPE
 
+/*
+ * Note: though redis supports unboudned number of variables in some commands,
+ * implementation cannot operate with performance guarantee when this number
+ * gets too big. It also introduces uncertainty around resources. Therefore, we
+ * are limiting it to REQ_NTOKEN minus the # required args. For each command, if
+ * the # of optional arguments is declared as -1, (req_ntoken - narg) will be
+ * used to enforce argument limits.
+ */
 struct command {
     cmd_type_e      type;
     struct bstring  bstr;
-    int32_t         narg;
+    int32_t         narg; /* number of required arguments, including verb */
+    int32_t         nopt; /* number of optional arguments */
 };
 
 extern struct command command_table[REQ_SENTINEL];
@@ -65,7 +75,7 @@ struct request {
     bool                    cerror;   /* client error */
 
     cmd_type_e              type;
-    struct array            *token;    /* array elements are tokens */
+    struct array            *token;    /* member type: `struct element' */
 };
 
 void request_setup(request_options_st *options, request_metrics_st *metrics);

src/protocol/data/redis/response.c

@@ -24,6 +24,8 @@ response_reset(struct response *rsp)
     STAILQ_NEXT(rsp, next) = NULL;
     rsp->free = false;
 
+    rsp->serror = false;
+
     rsp->type = ELEM_UNKNOWN;
     rsp->nil = false;
     rsp->token->nelem = 0;

src/protocol/data/redis/response.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "token.h"
+
 #include <cc_array.h>
 #include <cc_bstring.h>
 #include <cc_define.h>
@@ -8,7 +10,7 @@
 #include <cc_queue.h>
 #include <cc_util.h>
 
-#define RSP_NTOKEN 255 /* # tokens in a command */
+#define RSP_NTOKEN 127 /* # tokens in a response */
 #define RSP_POOLSIZE 0
 
 /*          name                type                default         description */
@@ -39,6 +41,7 @@ typedef struct {
  * - a RSP_NUMERIC type that doesn't have a corresponding message body.
  */
 #define RSP_STR_OK "OK"
+#define RSP_PONG "pong"
 
 /*
  * NOTE(yao): we store fields as location in rbuf, this assumes the data will
@@ -50,9 +53,11 @@ struct response {
     STAILQ_ENTRY(response)  next;       /* allow response pooling/chaining */
     bool                    free;
 
-    int                     type;
-    bool                    nil;
-    struct array            *token;     /* array elements are tokens */
+    bool                    serror;     /* server error */
+
+    element_type_e          type;       /* only array can have >1 token */
+    bool                    nil;        /* null array or null bulk string */
+    struct array            *token;     /* member type: `struct element' */
 };
 
 void response_setup(response_options_st *options, response_metrics_st *metrics);

src/protocol/data/redis/token.c

@@ -3,6 +3,8 @@
 #include "request.h"
 #include "response.h"
 
+#include <buffer/cc_buf.h>
+#include <buffer/cc_dbuf.h>
 #include <cc_define.h>
 #include <cc_print.h>
 #include <cc_util.h>

src/protocol/data/redis/token.h

@@ -47,13 +47,27 @@
  * last element.
  */
 
-#include "parse.h"
-#include "compose.h"
-
 #include <buffer/cc_buf.h>
 #include <cc_bstring.h>
 #include <cc_util.h>
 
+typedef enum parse_rstatus {
+    PARSE_OK        = 0,
+    PARSE_EUNFIN    = -1,
+    PARSE_EEMPTY    = -2,
+    PARSE_EOVERSIZE = -3,
+    PARSE_EINVALID  = -4,
+    PARSE_EOTHER    = -5,
+} parse_rstatus_t;
+
+typedef enum compose_rstatus {
+    COMPOSE_OK          = 0,
+    COMPOSE_EUNFIN      = -1,
+    COMPOSE_ENOMEM      = -2,
+    COMPOSE_EINVALID    = -3,
+    COMPOSE_EOTHER      = -4,
+} compose_rstatus_t;
+
 /* array is not a basic element type */
 typedef enum element_type {
     ELEM_UNKNOWN    = 0,

src/server/CMakeLists.txt

@@ -2,6 +2,10 @@ if(TARGET_PINGSERVER)
     add_subdirectory(pingserver)
 endif()
 
+if(TARGET_REDIS)
+    add_subdirectory(slimredis)
+endif()
+
 if(TARGET_SLIMCACHE)
     add_subdirectory(slimcache)
 endif()