[pre-segcache] preparations for segcache (#321)

* [pre-segcache] preparations for segcache. 1. add a new field in memcache response; 2. change the timer in pelikan_twemcache so that the default timer is memcached_time; 3. fix a bug in pelikan_twemcache where flush_at is not reset during tear and setup. 4. remove inline of _get_value in sarray.c as gcc 7 cannot inline it. 5. add segcache in calculator

* update time mode to time_memcache in global default time setting.

scripts/capacity/calculator.py

@@ -25,8 +25,10 @@
 SAFETY_BUF = 128  # in MB
 BASE_OVERHEAD = 10  # in MB
 KQPS = 30  # much lower than single-instance max, picked to scale to 10 jobs/host
-HASH_OVERHEAD = {'twemcache': 8, 'slimcache': 0}
-ITEM_OVERHEAD = {'twemcache': 40 + 8, 'slimcache': 6 + 8}  # ITEM_HDR_SIZE + CAS
+# segcache needs 8/7*8 byte per object for hash table, considering
+# hash bucket overflow, give it 12, this should be sufficient for hash table load 1
+HASH_OVERHEAD = {'twemcache': 8, 'slimcache': 0, 'segcache': 12}
+ITEM_OVERHEAD = {'twemcache': 40 + 8, 'slimcache': 6 + 8, 'segcache': 5}  # ITEM_HDR_SIZE + CAS
 KEYVAL_ALIGNMENT = 8  # in bytes
 NITEM_ALIGNMENT = 512  # so memory allocation is always 4K (page size) aligned
 
@@ -48,6 +50,11 @@ def hash_parameters(nkey, runnable):
 
 def calculate(args):
   """calculate job configuration according to requirements.
+     For segcache, returns a dict with:
+      cpu, ram, disk,
+      hash_power, seg_mem,
+      instance, host_limit, rack_limit,
+      memory_bound
      For twemcache, returns a dict with:
       cpu, ram, disk,
       hash_power, slab_mem,
@@ -110,7 +117,7 @@ def calculate(args):
 
   # recalculate hash parameters with the final job count
   nkey_per_shard = 1.0 * (sorted_ram[index] * GB - ram_fixed * MB - ram_conn * MB) / item_size
-  # only used by twemcache
+  # used by twemcache and segcache
   hash_power, ram_hash = hash_parameters(nkey_per_shard, args.runnable)
   slab_mem = sorted_ram[index] * GB / MB - ram_fixed - ram_conn - ram_hash
   # only used by slimcache
@@ -130,6 +137,9 @@ def calculate(args):
   if args.runnable == 'twemcache':
     ret['hash_power'] = hash_power
     ret['slab_mem'] = slab_mem
+  elif args.runnable == 'segcache':
+      ret['hash_power'] = hash_power
+      ret['seg_mem'] = slab_mem
   elif args.runnable == 'slimcache':
     ret['item_size'] = item_size
     ret['nitem'] = nitem
@@ -194,7 +204,7 @@ def slimcache_format_output(config):
 parser = argparse.ArgumentParser(
   formatter_class=argparse.RawDescriptionHelpFormatter,
   description=textwrap.dedent("""
-    This script calculates resource requirement of a pelikan cluster (twemcache or slimcache)
+    This script calculates resource requirement of a pelikan cluster (twemcache, segcache or slimcache)
     based on input. It has to be run from the top level directory of source.\n
 
     Optional arguments that probably should be overwritten:
@@ -222,8 +232,12 @@ def slimcache_format_output(config):
 
 if __name__ == "__main__":
   # add runnable as a positional option instead of subparser (as in aurora.py) to avoid import
-  parser.add_argument('runnable', choices=['twemcache', 'slimcache'], help='flavor of backend')
-  format_output = {'twemcache': twemcache_format_output, 'slimcache': slimcache_format_output}
+  parser.add_argument('runnable',
+                      choices=['twemcache', 'segcache', 'slimcache'],
+                      help='flavor of backend')
+  format_output = {'twemcache': twemcache_format_output,
+                   'segcache': twemcache_format_output,
+                   'slimcache': slimcache_format_output}
   args = parser.parse_args()
   print(format_input(args))
   config = calculate(args)

src/data_structure/sarray/sarray.c

@@ -32,7 +32,7 @@ _validate_range(uint32_t esize, uint64_t val)
     }
 }
 
-static inline uint64_t
+static uint64_t
 _get_value(char *p, uint32_t esize)
 {
     switch (esize) {

src/protocol/data/memcache/response.c

@@ -36,6 +36,7 @@ response_reset(struct response *rsp)
     rsp->vcas = 0;
     rsp->met = NULL;
     rsp->flag = 0;
+    rsp->item = NULL;
 
     rsp->cas = 0;
     rsp->num = 0;

src/protocol/data/memcache/response.h

@@ -86,6 +86,8 @@ struct response {
     uint64_t                vcas;       /* value for cas */
     struct metric           *met;       /* metric, for reporting stats */
 
+    void                    *item;      /* pointer to item, used by segcache */
+
     uint32_t                flag;
     uint32_t                vlen;
 

src/storage/slab/item.c

@@ -6,7 +6,7 @@
 #include <stdio.h>
 
 extern delta_time_i max_ttl;
-static proc_time_i flush_at = -1;
+proc_time_i flush_at = -1;
 
 static inline bool
 _item_expired(struct item *it)

src/storage/slab/slab.c

@@ -61,6 +61,8 @@ delta_time_i max_ttl = ITEM_MAX_TTL;
 static bool slab_init = false;
 slab_metrics_st *slab_metrics = NULL;
 
+extern proc_time_i flush_at;
+
 cc_declare_itt_function(,slab_malloc);
 cc_declare_itt_function(,slab_free);
 
@@ -620,6 +622,8 @@ slab_setup(slab_options_st *options, slab_metrics_st *metrics)
      cc_create_itt_malloc(slab_malloc);
      cc_create_itt_free(slab_free);
 
+    flush_at = -1;
+
     slab_init = true;
 
     return;

src/time/time.h

@@ -67,9 +67,9 @@ enum {
     TIME_SENTINEL = 3
 };
 
-/*          name          type                default       description */
+/*          name          type                default           description */
 #define TIME_OPTION(ACTION) \
-    ACTION( time_type,    OPTION_TYPE_UINT,   TIME_UNIX,    "Expiry timestamp mode" )
+    ACTION( time_type,    OPTION_TYPE_UINT,   TIME_MEMCACHE,    "Expiry timestamp mode" )
 
 typedef struct {
     TIME_OPTION(OPTION_DECLARE)