List based on ziplist (create/delete implemented in this diff) (#170)

* adding list interface

* List.create works

* adding List.delete impl

* revert some merge errors

src/data_structure/bitmap/bitset.c

@@ -4,22 +4,26 @@
 #include <string.h>
 
 
-#define DATA_OFFSET(_bs) ((uint8_t *)(_bs) + offsetof(struct bitset, data))
+#define DATA_POS(_bs) ((uint8_t *)(_bs) + offsetof(struct bitset, data))
 #define SEGMENT_OFFSET(_col) ((uint8_t)(_col) >> 5) /* uint32_t, 2^5 bits per segment */
 #define BIT_OFFSET(_col) ((uint8_t)(_col) & 0x1f)
 #define GET_SEGMENT(_bs, _col)  \
-        ((uint32_t *)DATA_OFFSET(_bs) + SEGMENT_OFFSET(_col))
+        ((uint32_t *)DATA_POS(_bs) + SEGMENT_OFFSET(_col))
 #define GET_COL(_v, _offset) (((uint32_t)(_v) >> (_offset)) & 1UL)
 
-void
+uint8_t
 bitset_init(struct bitset *bs, uint16_t ncol)
 {
-    uint8_t *d = (uint8_t *)DATA_OFFSET(bs);
+    uint8_t *d = (uint8_t *)DATA_POS(bs);
+    uint8_t sz;
 
     bs->size = (uint8_t)bit2long(ncol);
     bs->col_w = 1;
     bs->count = 0;
-    memset(d, 0, size2byte(bs->size));
+    sz = size2byte(bs->size);
+    memset(d, 0, sz);
+
+    return sz + sizeof(*bs);
 }
 
 uint8_t

src/data_structure/bitmap/bitset.h

@@ -23,7 +23,8 @@ struct bitset {
 #define size2bit(_sz) ((_sz) << 5)
 #define size2byte(_sz) ((_sz) << 2)
 
-void bitset_init(struct bitset *bs, uint16_t ncol);
+/* return size of the bitset after initialization, in bytes */
+uint8_t bitset_init(struct bitset *bs, uint16_t ncol);
 
 uint8_t bitset_get(struct bitset *bs, uint16_t col);
 /* Note: the interface is written as a generic set function with a val parameter

src/protocol/data/redis/cmd_list.h

@@ -0,0 +1,48 @@
+#pragma once
+
+/**
+ * create: create an empty list
+ * List.create KEY
+ *
+ * delete: delete a list or a particular value from list
+ * List.delete KEY [VALUE [COUNT]]
+ *
+ * trim: trimming a list
+ * List.trim KEY INDEX [COUNT]
+ *
+ * len: return number of entries in list
+ * List.len KEY
+ *
+ * find: find entry in list
+ * List.find KEY VALUE
+ *
+ * get: get entry/entries at an index
+ * List.get KEY [INDEX [COUNT]]
+ *
+ * insert: insert entry at an index
+ * List.insert KEY VALUE INDEX
+ *
+ * push: pushing entry/entries at the end
+ * List.push KEY VALUE [VALUE ...]
+ */
+
+
+/*          type                string          #arg    #opt */
+#define REQ_LIST(ACTION)                                    \
+    ACTION( REQ_LIST_CREATE,  "List.create",    2,      0  )\
+    ACTION( REQ_LIST_DELETE,  "List.delete",    2,      2  )\
+    ACTION( REQ_LIST_TRIM,    "List.trim",      4,      0  )\
+    ACTION( REQ_LIST_LEN,     "List.len",       2,      0  )\
+    ACTION( REQ_LIST_FIND,    "List.find",      3,      0  )\
+    ACTION( REQ_LIST_GET,     "List.get",       2,      2  )\
+    ACTION( REQ_LIST_INSERT,  "List.insert",    4,      0  )\
+    ACTION( REQ_LIST_PUSH,    "List.push",      3,      -1 )
+
+typedef enum list_elem {
+    LIST_VERB = 0,
+    LIST_KEY = 1,
+    LIST_VAL = 2,
+    LIST_IDX = 2,
+    LIST_VIDX = 3, /* when a value is also present */
+    LIST_CNT = 3,
+} list_elem_e;

src/protocol/data/redis/request.c

@@ -20,6 +20,7 @@ struct command command_table[REQ_SENTINEL] = {
     { .type = REQ_UNKNOWN, .bstr = { 0, NULL }, .narg = 0, .nopt = 0 },
     REQ_BITMAP(CMD_INIT)
     REQ_HASH(CMD_INIT)
+    REQ_LIST(CMD_INIT)
     REQ_ZSET(CMD_INIT)
     REQ_MISC(CMD_INIT)
 };

src/protocol/data/redis/request.h

@@ -2,6 +2,7 @@
 
 #include "cmd_bitmap.h"
 #include "cmd_hash.h"
+#include "cmd_list.h"
 #include "cmd_misc.h"
 #include "cmd_zset.h"
 #include "token.h"
@@ -45,6 +46,7 @@ typedef enum cmd_type {
     REQ_UNKNOWN,
     REQ_BITMAP(GET_TYPE)
     REQ_HASH(GET_TYPE)
+    REQ_LIST(GET_TYPE)
     REQ_ZSET(GET_TYPE)
     REQ_MISC(GET_TYPE)
     REQ_SENTINEL

src/server/redis/CMakeLists.txt

@@ -9,7 +9,7 @@ set(SOURCE
 
 set(MODULES
     core
-    ds_bitmap
+    ds_ziplist
     protocol_admin
     protocol_redis
     slab

src/server/redis/data/CMakeLists.txt

@@ -2,4 +2,5 @@ set(SOURCE
     ${SOURCE}
     ${CMAKE_CURRENT_SOURCE_DIR}/process.c
     ${CMAKE_CURRENT_SOURCE_DIR}/cmd_misc.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/cmd_list.c
     PARENT_SCOPE)

src/server/redis/data/cmd_list.c

@@ -0,0 +1,102 @@
+#include "process.h"
+
+#include "data_structure/ziplist/ziplist.h"
+#include "protocol/data/redis_include.h"
+#include "storage/slab/item.h"
+#include "storage/slab/slab.h"
+
+#include <cc_array.h>
+#include <cc_bstring.h>
+#include <cc_debug.h>
+
+
+static inline struct bstring *
+_get_key(struct request *req)
+{
+    struct element *key = (struct element *)array_get(req->token, LIST_KEY);
+
+    return &key->bstr;
+}
+
+static inline struct item *
+_add_key(struct response *rsp, struct bstring *key)
+{
+    struct element *reply = (struct element *)array_get(rsp->token, 0);
+    struct item *it;
+    item_rstatus_t istatus;
+
+    it = item_get(key);
+    if (it != NULL) {
+        rsp->type = reply->type = ELEM_ERR;
+        reply->bstr = str2bstr(RSP_EXIST);
+        INCR(process_metrics, list_create_exist);
+
+        return NULL;
+    } else {
+        /* TODO: figure out a TTL story here */
+        istatus = item_reserve(&it, key, NULL, ZIPLIST_HEADER_SIZE, 0, UINT32_MAX);
+        if (istatus != ITEM_OK) {
+            rsp->type = reply->type = ELEM_ERR;
+            reply->bstr = str2bstr(RSP_ERR_STORAGE);
+            INCR(process_metrics, list_create_ex);
+            INCR(process_metrics, process_ex);
+        } else {
+            INCR(process_metrics, list_create_stored);
+        }
+
+        return it;
+    }
+}
+
+
+void
+cmd_list_create(struct response *rsp, struct request *req, struct command *cmd)
+{
+    struct item *it;
+    struct bstring *key = _get_key(req);
+    struct element *reply = (struct element *)array_push(rsp->token);
+
+    INCR(process_metrics, list_create);
+
+    it = _add_key(rsp, key);
+    if (it == NULL) {
+        log_debug("command '%.*s' '%.*s' failed: cannot store", cmd->bstr.len,
+                cmd->bstr.data, key->len, key->data);
+        return;
+    }
+
+    /* initialize data structure */
+    ziplist_reset((ziplist_p)item_data(it));
+    it->vlen = ZIPLIST_HEADER_SIZE;
+
+    /* link into index */
+    item_insert(it, key);
+
+    rsp->type = reply->type = ELEM_STR;
+    reply->bstr = str2bstr(RSP_OK);
+
+    log_verb("command '%.*s' '%.*s' succeeded", cmd->bstr.len, cmd->bstr.data,
+            key->len, key->data);
+}
+
+void
+cmd_list_delete(struct response *rsp, struct request *req, struct command *cmd)
+{
+    struct bstring *key = _get_key(req);
+    struct element *reply = (struct element *)array_push(rsp->token);
+
+    INCR(process_metrics, list_delete);
+
+    if (item_delete(key)) {
+        reply->bstr = str2bstr(RSP_OK);
+        INCR(process_metrics, list_delete_deleted);
+        log_verb("command '%.*s' '%.*s' succeeded", cmd->bstr.len,
+                cmd->bstr.data, key->len, key->data);
+    } else {
+        reply->bstr = str2bstr(RSP_NOTFOUND);
+        INCR(process_metrics, list_delete_notfound);
+        log_verb("command '%.*s' '%.*s' completed as no-op, key not found",
+                cmd->bstr.len, cmd->bstr.data, key->len, key->data);
+    }
+
+}

src/server/redis/data/cmd_list.h

@@ -0,0 +1,25 @@
+#pragma once
+
+/*          name        type            description */
+#define PROCESS_LIST_METRIC(ACTION)                                             \
+    ACTION( list_create,            METRIC_COUNTER, "# list create requests"   )\
+    ACTION( list_create_exist,      METRIC_COUNTER, "# list already exist"     )\
+    ACTION( list_create_stored,     METRIC_COUNTER, "# list stored"            )\
+    ACTION( list_create_ex,         METRIC_COUNTER, "# list create exceptions" )\
+    ACTION( list_delete,            METRIC_COUNTER, "# list delete requests"   )\
+    ACTION( list_delete_deleted,    METRIC_COUNTER, "# list delete success"    )\
+    ACTION( list_delete_notfound,   METRIC_COUNTER, "# list delete success"    )\
+    ACTION( list_trim,              METRIC_COUNTER, "# list trim requests"     )\
+    ACTION( list_len,               METRIC_COUNTER, "# list length requests"   )\
+    ACTION( list_find,              METRIC_COUNTER, "# list find requests"     )\
+    ACTION( list_get,               METRIC_COUNTER, "# list get requests"      )\
+    ACTION( list_insert,            METRIC_COUNTER, "# list insert requests"   )\
+    ACTION( list_push,              METRIC_COUNTER, "# list push requests"     )
+
+struct request;
+struct response;
+struct command;
+
+/* cmd_* functions must be command_fn (process.c) compatible */
+void cmd_list_create(struct response *rsp, struct request *req, struct command *cmd);
+void cmd_list_delete(struct response *rsp, struct request *req, struct command *cmd);

src/server/redis/data/process.c

@@ -37,6 +37,8 @@ process_setup(process_options_st *options, process_metrics_st *metrics)
     }
 
     command_registry[REQ_PING] = cmd_ping;
+    command_registry[REQ_LIST_CREATE] = cmd_list_create;
+    command_registry[REQ_LIST_DELETE] = cmd_list_delete;
 
     process_init = true;
 }

src/server/redis/data/process.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "cmd_misc.h"
+#include "cmd_list.h"
 
 #include <buffer/cc_buf.h>
 #include <cc_metric.h>
@@ -25,6 +26,7 @@ typedef struct {
 
 typedef struct {
     PROCESS_METRIC(METRIC_DECLARE)
+    PROCESS_LIST_METRIC(METRIC_DECLARE)
     PROCESS_MISC_METRIC(METRIC_DECLARE)
 } process_metrics_st;
 

src/server/slimredis/data/cmd_bitmap.c

@@ -65,7 +65,6 @@ static inline struct item *
 _add_key(struct response *rsp, struct bstring *key)
 {
     struct element *reply = (struct element *)array_get(rsp->token, BITMAP_VERB);
-    struct val val = {VAL_TYPE_STR, {null_bstring}};
     struct item *it;
 
     it = cuckoo_get(key);
@@ -76,7 +75,7 @@ _add_key(struct response *rsp, struct bstring *key)
 
         return NULL;
     } else { /* cuckoo insert current won't fail as long as size is valid */
-        it = cuckoo_insert(key, &val, time_reltime(0));
+        it = cuckoo_insert(key, NULL, time_reltime(0));
         if (it == NULL) {
             rsp->type = reply->type = ELEM_ERR;
             reply->bstr = str2bstr(RSP_ERR_STORAGE);
@@ -117,7 +116,7 @@ cmd_bitmap_create(struct response *rsp, struct request *req, struct command *cmd
     }
 
     /* initialize data structure */
-    bitset_init((struct bitset *)ITEM_VAL_POS(it), (uint16_t)ncol);
+    it->vlen = bitset_init((struct bitset *)ITEM_VAL_POS(it), (uint16_t)ncol);
 
     rsp->type = reply->type = ELEM_STR;
     reply->bstr = str2bstr(RSP_OK);

src/storage/slab/item.c

@@ -204,8 +204,10 @@ _item_define(struct item *it, const struct bstring *key, const struct bstring
     it->olen = olen;
     cc_memcpy(item_key(it), key->data, key->len);
     it->klen = key->len;
-    cc_memcpy(item_data(it), val->data, val->len);
-    it->vlen = val->len;
+    if (val != NULL) {
+        cc_memcpy(item_data(it), val->data, val->len);
+    }
+    it->vlen = (val == NULL) ? 0 : val->len;
 }
 
 item_rstatus_t