redis-6.0.8-dict的一些杂谈

数据结构

typedef struct dict {
    dictType *type; //函数指针
    void *privdata;
    /*ht属性是一个包含两个项的数组，数组中的每个项都是一个dictht哈希表，一般情况下， 字典只使用ht[0]哈希表，ht[1]哈希表只会在对ht[0]哈希表进行rehash时使用*/
    dictht ht[2];   //hash表
    /*它记录了rehash目前的进度，如果目前没有在进行rehash，那么它的值为-1*/
    long rehashidx; /* rehashing not in progress if rehashidx == -1 重新排列*/      //数据量特别大时一个数组槽位一个数组槽位的移动
    unsigned long iterators; /* number of iterators currently running */
} dict;

typedef struct dictht {
    dictEntry **table;      //哈希表，哈希冲突通过链表进行连接
    unsigned long size;         //哈希表大小
    unsigned long sizemask; //哈希表大小掩码，用于计算索引值。总是等于size-1
    unsigned long used;      //该哈希表已有节点的数量
} dictht;

typedef struct dictEntry {
    void *key;  //key值
    union {
        void *val;
        uint64_t u64;
        int64_t s64;
        double d;
    } v;
    struct dictEntry *next;
} dictEntry;

typedef struct redisObject {//value结构体
    unsigned type:4;    // 数据类型
    unsigned encoding:4;    // 数据编码
    unsigned lru:LRU_BITS; /* LRU time (relative to global lru_clock) or
                            * LFU data (least significant 8 bits frequency
                            * and most significant 16 bits access time). */ // LRU时钟
    int refcount;       //引用计数
    void *ptr;  // 指向数据的指针。当robj存储的数据可以使用long类型表示的时候，数据直接存储在ptr字段
    //当使用OBJ_ENCODING_EMBSTR编码时，ptr指向sdshdr8的flags字段之后
} robj;




typedef char *sds;

/* Note: sdshdr5 is never used, we just access the flags byte directly.
 * However is here to document the layout of type 5 SDS strings. */
struct __attribute__ ((__packed__)) sdshdr5 {
    unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
    char buf[];
};
struct __attribute__ ((__packed__)) sdshdr8 {
    uint8_t len; /* used */ //用户已用的内存
    uint8_t alloc; /* excluding the header and null terminator */  //申请的内存
    unsigned char flags; /* 3 lsb of type, 5 unused bits */ //标记SDS_TYPE_8
    char buf[];     //字符串数据区域
};
struct __attribute__ ((__packed__)) sdshdr16 {
    uint16_t len; /* used */
    uint16_t alloc; /* excluding the header and null terminator */
    unsigned char flags; /* 3 lsb of type, 5 unused bits */
    char buf[];
};

请求数据解析argc,argv

void readQueryFromClient(connection *conn) {
...
 nread = connRead(c->conn, c->querybuf+qblen, readlen);
 printf("c->querybuf_peak=%ld,qblen=%ld,nread=%d,c->querybuf+qblen=%s",c->querybuf_peak,qblen,nread,c->querybuf+qblen);
...
}

调用命令

127.0.0.1:6379> set abcd aa
OK

打印结果如下：

c->querybuf_peak=0,qblen=0,nread=31,c->querybuf+qblen=*3

set

abcd

aa

对c->querybuf中的数据解析出参数argc和argv
void processInputBuffer(client *c) {
...
if (c->reqtype == PROTO_REQ_MULTIBULK) {	//redis-cli命令请求的协议类型
            if (processMultibulkBuffer(c) != C_OK) break;   //解析read出来的参数
}
...
}
int processMultibulkBuffer(client *c) {
...
 serverAssertWithInfo(c,NULL,c->querybuf[c->qb_pos] == '*'); //*2表示后面有2个参数，*3表示后面有3个参数
 ok = string2ll(c->querybuf+1+c->qb_pos,newline-(c->querybuf+1+c->qb_pos),&ll);
...
 c->multibulklen = ll;//参数数量
 /* Setup argv array on client structure */
 if (c->argv) zfree(c->argv);
 c->argv = zmalloc(sizeof(robj*)*c->multibulklen);
...
while(c->multibulklen) {    //参数数量
...
             if (c->querybuf[c->qb_pos] != '$') {
                addReplyErrorFormat(c,
                    "Protocol error: expected '$', got '%c'",
                    c->querybuf[c->qb_pos]);
                setProtocolError("expected $ but got something else",c);
                return C_ERR;
            }

            ok = string2ll(c->querybuf+c->qb_pos+1,newline-(c->querybuf+c->qb_pos+1),&ll);
            ....
            c->bulklen = ll;    //参数长度 表示5个字节，表示7个字节
...
 c->argv[c->argc++] =createStringObject(c->querybuf+c->qb_pos,c->bulklen);	//加入到参数argv里面
 ...
	}
}

robj结构体与sdshdr8、sdshdr16、sdshdr32结构体

robj *createStringObject(const char *ptr, size_t len) {
    if (len <= OBJ_ENCODING_EMBSTR_SIZE_LIMIT)
        return createEmbeddedStringObject(ptr,len);
    else
        return createRawStringObject(ptr,len);
}

其中createEmbeddedStringObject为创建一整片内存，即robj结构体和ptr指针都在一片内存当中。createRawStringObject为创建两片内存，robj和ptr指向的内存为不同内存片。

robj *createEmbeddedStringObject(const char *ptr, size_t len) {
    robj *o = zmalloc(sizeof(robj)+sizeof(struct sdshdr8)+len+1);
    struct sdshdr8 *sh = (void*)(o+1);

    o->type = OBJ_STRING;
    o->encoding = OBJ_ENCODING_EMBSTR;
    o->ptr = sh+1;
    o->refcount = 1;
    if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
        o->lru = (LFUGetTimeInMinutes()<<8) | LFU_INIT_VAL;
    } else {
        o->lru = LRU_CLOCK();
    }

    sh->len = len;
    sh->alloc = len;
    sh->flags = SDS_TYPE_8;
    if (ptr == SDS_NOINIT)
        sh->buf[len] = ';'else
    if ( )ptrmemcpy {
        (,sh->buf,ptr)len;[
        sh->buf]len= ';' }else
    memset ( {
        ,0sh->buf,+1len);}return
    ;
    } o*
createRawStringObject

robj (constchar* , size_tptr) return lencreateObject {
    ( ,sdsnewlenOBJ_STRING( ,)ptr)len;}#
define

sds类型内存模型如下：

其中关键的宏定义和函数定义如下：

SDS_TYPE_50 #  define
SDS_TYPE_81 #  define
SDS_TYPE_162 # define
SDS_TYPE_323 # define
SDS_TYPE_644 # define
SDS_TYPE_MASK7 # define
SDS_TYPE_BITS3 # define
SDS_HDR_VAR( ,)Tstructssdshdr ## *=T (sh void *)(()-s(sizeof(structsdshdr## ))T);#define
SDS_HDR( ,)T(s( structsdshdr## *)T (()-s(sizeof(structsdshdr## ))T))#define
SDS_TYPE_5_LEN( )(f( ))fstatic>>SDS_TYPE_BITSinline

size_t sdslen ( const)//已使用的内存 sds sunsigned {      char
    = [ flags - s1];switch(
    &)flagscaseSDS_TYPE_MASK: {
        return SDS_TYPE_5SDS_TYPE_5_LEN
            ( );flagscase:
        return SDS_TYPE_8SDS_HDR
            ( 8,);scase->len:
        return SDS_TYPE_16SDS_HDR
            ( 16,);scase->len:
        return SDS_TYPE_32SDS_HDR
            ( 32,);scase->len:
        return SDS_TYPE_64SDS_HDR
            ( 64,);s}->lenreturn
    0
    ; }static
inline

size_t sdsavail ( const)//剩余内存 sds sunsigned {    char
    = [ flags - s1];switch(
    &)flagscaseSDS_TYPE_MASK: {
        return SDS_TYPE_50 {
            ; }case
        :
        SDS_HDR_VAR SDS_TYPE_8( {
            8,);sreturn-
            ; sh->alloc } sh->lencase
        :
        SDS_HDR_VAR SDS_TYPE_16( {
            16,);sreturn-
            ; sh->alloc } sh->lencase
        :
        SDS_HDR_VAR SDS_TYPE_32( {
            32,);sreturn-
            ; sh->alloc } sh->lencase
        :
        SDS_HDR_VAR SDS_TYPE_64( {
            64,);sreturn-
            ; sh->alloc } sh->len}
        return
    0
    ; }int
dictResize

哈希表dict的扩容和缩容

( *)dict //哈希表缩容dunsigned long
{
    ; if minimal(

    ! ||dictIsRehashingdict_can_resize ( ))dreturn; = DICT_ERR[
    minimal 0 d->ht].;ifused(
    < )minimal = DICT_HT_INITIAL_SIZE;
        minimal return DICT_HT_INITIAL_SIZEdictExpand
    ( ,)d; minimal}int
dictExpand

( *,dict unsigneddlong ) //哈希表进行扩容 size/* the size is invalid if it is smaller than the number of
     * elements already inside the hash table */ if
{
    (
    dictIsRehashing ()||d[ 0 d->ht].)returnused > size;
        ; DICT_ERR/* the new hash table */

    dictht n/*
    1.如果执行的是扩展 *** 作，那么ht[1]的大小为第一个大于等于ht[0].used*2的2^n。      例如ht[0].used为5，ht[0].used*2=10，第一个大于的2^n次方为2^4即16。
    2.如果执行的是收缩 *** 作，那么ht[1]的大小为第一个大于等于ht[0].used的2^n。        例如ht[0].used为5，ht[0].used=5，第一个大于的2^n次方为2^3即8。
    */ unsigned
    long
    = _dictNextPower realsize ( );size/* Rehashing to the same table size is not useful. */if  

    (
    == [realsize 0 d->ht].)returnsize; /* Allocate the new hash table and initialize all pointers to NULL */ DICT_ERR.

    =
    n;size . realsize=
    n-sizemask 1 realsize;.=
    nzcalloctable ( *sizeofrealsize(*)dictEntry);.=
    n0used ; /* Is this the first initialization? If so it's not really a rehashing
     * we just set the first hash table so that it can accept keys. */if

    (
    [ 0d->ht].==NULLtable ) [0 {
        d->ht]=; return n;
        } DICT_OK/* Prepare a second hash table for incremental rehashing */
    [

    1
    d->ht]=; = n0
    d->rehashidx ; return;
    } DICT_OKint
dictRehash

其中哈希表dict包含两个哈希表项在通过key进行查找时ht[0]和ht[1]都会进行查找匹配表项中的key值。在int dictRehash(dict *d, int n)函数中每次移动哈希表中的一个n个槽位，从从ht[0]哈希表上重新哈希到ht[1].table[h]位置。
当ht[0]中已使用节点数为0时，将ht[1]赋值给ht[0]即d->ht[0] = d->ht[1];随后ht[1]进行复位，为下次扩容缩容做好准备。

( *,dict intd) //挪动数据，n表示几个槽位   扩容缩容时同时通过key进行find时哈希表1和哈希表2同时进行查找 nint {    =
    * empty_visits 10 n;/* Max number of empty buckets to visit. 最大访问空桶数*/if (
    ! dictIsRehashing())dreturn0 ; while(

    --&&n[ 0 d->ht].!=0used ) *, {
        dictEntry *de; /* Note that rehashidx can't overflow as we are sure there are more
         * elements because ht[0].used != 0 */nextdeassert

        (
        [0d->ht].(unsignedsize > long) );d->rehashidxwhile(
        [0d->ht].[]table==d->rehashidxNULL ) ++; {
            d->rehashidxif(
            -- ==0empty_visits ) return1 ; }=
        [
        de 0 d->ht].[]table;d->rehashidx/* Move all the keys in this bucket from the old to the new hash HT */while
        (
        )uint64_tde; {
            = h;

            nextde /* Get the index in the new hash table */ de->next=
            dictHashKey
            h ( ,)d& de->key[ 1 d->ht].;=sizemask[
            de->next 1 d->ht].[]table;h[1
            d->ht].[]table=h; //de节点从ht[0]哈希表上重新哈希到ht[1].table[h]位置 de[ 0
            d->ht].--;used[1
            d->ht].++;used=;
            de } nextde[
        0
        d->ht].[]table=d->rehashidxNULL ; ++;
        d->rehashidx}/* Check if we already rehashed the whole table... */
    if

    (
    [ 0d->ht].==0used ) zfree( {
        [0d->ht].);table[0
        d->ht]=[ 1 d->ht];_dictReset(
        &[1d->ht]);//哈希表1复位=      -
        d->rehashidx 1 ;return0
        ; }/* More to rehash... */
    return

    1
    ; }