acl/lib_acl/samples/zdb/zdb_test.cpp
2017-07-10 15:26:12 +08:00

791 lines
19 KiB
C++

#include "lib_acl.h"
#ifndef WIN32
#include <algorithm>
#include <vector>
#endif
#include "db/zdb.h"
#include "zdb_test.h"
#include "md5.h"
#pragma pack(4)
typedef struct {
int n;
char s[11];
} DUMMY;
#pragma pack(8)
#ifndef STORE_PATH
#define STORE_PATH(s) ACL_FHANDLE_PATH(&(s)->fhandle)
#endif
struct RANGE {
int from;
int to;
};
#ifdef WIN32
#define snprintf _snprintf
#endif
/****************************************************************************/
#ifdef WIN32
static const char *var_md5_key = "hello world!";
#endif
static const char *var_str = "hi!";
/* 模拟取得一个随机的整形数组 */
#ifndef WIN32
static int *random_get(int max, int mod)
{
int *a = (int*) acl_mymalloc(sizeof(int) * max);
int n = 1;
if (mod > 1) {
max /= mod;
mod--;
} else
mod = 0;
for (int i = 0; i < max; i++)
a[i] = i + mod;
printf("shuffle: %d\n", n++);
std::random_shuffle(a, a + max);
printf("shuffle: %d\n", n++);
std::random_shuffle(a, a + max);
printf("shuffle: %d\n", n++);
std::random_shuffle(a, a + max);
printf("shuffle: %d\n", n++);
std::random_shuffle(a, a + max);
return (a);
}
#else
static int *random_get(int max, int mod)
{
int i, j, k, *a, key, off = 0;
char buf[64], key_buf[16];
int m = 0;
/* 先分配一个大数组 */
a = (int*) acl_mymalloc(sizeof(int) * max);
/* 初始化该大数组 */
memset(a, -1, sizeof(int) * max);
printf("begin: max: %d\n", max);
for (i = 0; i < max; i++) {
snprintf(buf, sizeof(buf), "key: %d", i);
/* 先 mda5 */
MD5Key(buf, strlen(buf), var_md5_key, strlen(var_md5_key),
key_buf, sizeof(key_buf));
/* 再取模 */
key = acl_hash_crc32(key_buf, sizeof(key_buf)) % max;
if (a[key] == -1) {
a[key] = i;
continue;
}
acl_assert(a[key] != i);
// printf("key: %d, %d\n", i, a[key]);
m++;
/* 说明该 key 的数组下标已经被占用,需要顺序找一个位置即可 */
for (j = off; j < max; j++) {
if (a[j] == -1) {
a[j] = i;
for (k = j + 1; k < max; k++) {
if (a[k] == -1) {
off = k;
break;
}
}
break;
}
}
acl_assert(j != max);
}
printf("end, m: %d\n", m);
return (a);
}
#endif
static void random_free(int *a)
{
acl_myfree(a);
}
/* 随机批量写 */
static int bench_random_add(ZDB *db, int max, int mod)
{
DUMMY dummy;
int i, ret, *a;
time_t begin;
ACL_SAFE_STRNCPY(dummy.s, var_str, sizeof(dummy.s));
a = random_get(max, mod);
if (mod > 1)
max /= mod;
time(&begin);
printf("---range_random_add: >> max: %d, ", max);
ACL_METER_TIME(" ");
for (i = 0; i < max; i++) {
dummy.n = i;
ret = zdb_update(db, a[i], NULL, &dummy, sizeof(dummy));
if (ret < 0) {
printf("zdb_udpate error, i: %d, key: %d\n", i, a[i]);
random_free(a);
return (-1);
}
if (i > 0 && i % 100000 == 0) {
printf("---%s: >> current: i: %d, key: %d, ",
__FUNCTION__, i, a[i]);
ACL_METER_TIME(" ");
}
}
printf("---range_add: >> end: current: %d, max: %d, ", i, max);
ACL_METER_TIME(" ");
printf("================= update ok, time: %ld ===============\n",
(long) (time(NULL) - begin));
random_free(a);
return (max);
}
/* 随机批量读 */
static int bench_random_get(ZDB *db, int max, int mod)
{
DUMMY *dummy;
int i, *a;
size_t size;
ZDB_BLK *blk;
ZDB_BLK_OFF blk_off;
time_t begin;
char buf[64];
a = random_get(max, mod);
if (mod > 1)
max /= mod;
time(&begin);
printf("---range_random_get: >> begin: max: %d, ", max);
ACL_METER_TIME(" ");
for (i = 0; i < max; i++) {
snprintf(buf, sizeof(buf), "key: %d", i);
blk = zdb_lookup(db, a[i], &size, &blk_off);
if (blk == NULL) {
printf("zdb_lookup error, i: %d\n", i);
random_free(a);
return (-1);
}
dummy = (DUMMY*) zdb_blk_data(blk);
if (i > 0 && i % 100000 == 0) {
printf("---%s: >> current: i: %d, n: %d,"
" key: %d, blk_off: %lld, ", __FUNCTION__,
i, dummy->n, a[i], blk_off.offset);
ACL_METER_TIME(" ");
}
zdb_blk_free(blk);
}
printf("---range_get: >> end: current: %d, max: %d, ", i, max);
ACL_METER_TIME(" ");
printf("================= lookup ok, time: %ld ===============\n",
(long) (time(NULL) - begin));
random_free(a);
return (max);
}
/****************************************************************************/
/* 顺序批量写 */
static int bench_add(ZDB *db, int max)
{
time_t begin;
int i, ret, delta = 100000;
DUMMY dummy;
if (max <= 100)
delta = 0;
time(&begin);
ACL_SAFE_STRNCPY(dummy.s, var_str, sizeof(dummy.s));
ACL_METER_TIME("----begin---");
for (i = 0; i < max; i++) {
dummy.n = i;
ret = zdb_update(db, i, NULL, &dummy, sizeof(dummy));
if (ret < 0) {
printf("zdb_udpate error, i: %d\n", i);
return (-1);
}
if (delta > 0 && i % delta == 0) {
printf("---%s: i: %d, ", __FUNCTION__, i);
ACL_METER_TIME("-----");
} else if (delta == 0)
printf("---%s: i: %d\n", __FUNCTION__, i);
}
printf("---%s: max: %d, ", __FUNCTION__, max);
ACL_METER_TIME("----end---");
printf("================= add ok, time: %ld ===============\n",
(long) (time(NULL) - begin));
return (max);
}
/* 顺序批量修改 */
static int bench_update(ZDB *db, int max, int num)
{
time_t begin;
int i, ret, j, delta = 100000;
DUMMY *dummy;
if (max <= 100)
delta = 0;
time(&begin);
dummy = (DUMMY*) acl_mycalloc(num, sizeof(DUMMY));
for (i = 0; i < num; i++) {
snprintf(dummy[i].s, sizeof(dummy[i].s), "%d", (int) i);
}
ACL_METER_TIME("----begin---");
for (i = 0; i < max; i++) {
for (j = 0; j < num; j++)
dummy[j].n = 0;
ret = zdb_update(db, i, NULL, dummy, num * sizeof(DUMMY));
if (ret < 0) {
printf("zdb_udpate error, i: %d\n", i);
return (-1);
}
if (delta > 0 && i % delta == 0) {
printf("---%s: i: %d, ", __FUNCTION__, i);
ACL_METER_TIME("-----");
} else if (delta == 0)
printf("---%s: i: %d\n", __FUNCTION__, i);
}
printf("---%s: max: %d, ", __FUNCTION__, max);
ACL_METER_TIME("----end---");
printf("================= update ok, time: %ld ===============\n",
(long) (time(NULL) - begin));
acl_myfree(dummy);
return (max);
}
/* 顺序批量读 */
static int bench_get(ZDB *db, int max)
{
ZDB_BLK *blk;
DUMMY *dummy;
time_t begin;
size_t size;
int i, delta = 100000;
ZDB_BLK_OFF blk_off;
if (max <= 100)
delta = 0;
printf(">> max: %d, delta: %d\n", max, delta);
time(&begin);
ACL_METER_TIME("----begin---");
//for (i = 0; i < max; i++) {
for (i = max - 1; i >= 0; i--) {
blk = zdb_lookup(db, i, &size, &blk_off);
if (blk == NULL) {
printf("zdb_lookup error, i: %d\n", i);
return (-1);
}
dummy = (DUMMY*) zdb_blk_data(blk);
if (delta == 0)
printf("---%s: i: %d, n: %d, %s, size: %d,"
" blk_off: %lld, \n", __FUNCTION__,
i, dummy[0].n, dummy[0].s,
(int) size, blk_off.offset);
else if (delta > 0 && i % delta == 0) {
printf("---%s: i: %d, n: %d, %s, size: %d,"
" blk_off: %lld, ", __FUNCTION__,
i, dummy[0].n, dummy[0].s,
(int) size, blk_off.offset);
ACL_METER_TIME("");
}
zdb_blk_free(blk);
}
printf("---%s: max: %d, ", __FUNCTION__, max);
ACL_METER_TIME("----end---");
printf("================= lookup ok, time: %ld ===============\n",
(long) (time(NULL) - begin));
return (max);
}
/****************************************************************************/
static int zdb_test1(ZDB *db)
{
DUMMY dummy[2], *pdummy;
ZDB_BLK *blk;
size_t size;
zdb_key_t key = 10000;
int ret, i;
dummy[0].n = (int) key;
ACL_SAFE_STRNCPY(dummy[0].s, var_str, sizeof(dummy[0].s));
dummy[1].n = (int) key + 1;
ACL_SAFE_STRNCPY(dummy[1].s, var_str, sizeof(dummy[1].s));
printf(">>> first update, key: %lld, ", key);
ret = zdb_update(db, key, NULL, &dummy, sizeof(dummy) / 2);
printf("zdb_update: return %d\n\n", ret);
printf(">>> second update, key: %lld, ", key);
ret = zdb_update(db, key, NULL, &dummy, sizeof(dummy) / 2);
blk = zdb_lookup(db, key, &size, NULL);
if (blk) {
printf("zdb_lookup: key(%lld), size: %u\n", key, (unsigned) size);
pdummy = (DUMMY*) zdb_blk_data(blk);
for (i = 0; i < (int) (size / sizeof(DUMMY)); i++) {
printf("zdb_lookup: n: %d, s: %s\n", pdummy->n, pdummy->s);
pdummy++;
}
zdb_blk_free(blk);
} else
printf("zdb_lookup: key(%lld), return null\n", key);
printf(">>> third update, key: %lld\n", key);
ret = zdb_update(db, key, NULL, &dummy, sizeof(dummy) / 2);
return (ret);
}
static int zdb_test2(ZDB *db)
{
DUMMY dummy1[1], dummy2[2], dummy3[3], *pdummy;
ZDB_BLK *blk;
int key = 100, ret;
size_t size, i;
dummy1[0].n = key;
ACL_SAFE_STRNCPY(dummy1[0].s, "1: hello world!", sizeof(dummy1[0].s));
dummy2[0].n = key;
ACL_SAFE_STRNCPY(dummy2[0].s, "2: hello world!", sizeof(dummy2[0].s));
dummy2[1].n = key;
ACL_SAFE_STRNCPY(dummy2[1].s, "2: hello world!", sizeof(dummy2[1].s));
dummy3[0].n = key;
ACL_SAFE_STRNCPY(dummy3[0].s, "3: hello world!", sizeof(dummy3[0].s));
dummy3[1].n = key;
ACL_SAFE_STRNCPY(dummy3[1].s, "3: hello world!", sizeof(dummy3[1].s));
dummy3[2].n = key;
ACL_SAFE_STRNCPY(dummy3[2].s, "3: hello world!", sizeof(dummy3[2].s));
ret = zdb_update(db, key, NULL, &dummy1, sizeof(dummy1));
if (ret < 0) {
printf("zdb_update: set dummy1 error\n");
return (-1);
}
blk = zdb_lookup(db, key, &size, NULL);
if (blk == NULL) {
printf("zdb_lookup: get dummy1 error\n");
return (-1);
}
pdummy = (DUMMY*) zdb_blk_data(blk);
for (i = 0; i < sizeof(dummy1) / sizeof(DUMMY); i++) {
printf("dummy1: key: %d, n: %d, s: %s\n", key, pdummy->n, pdummy->s);
pdummy++;
}
zdb_blk_free(blk);
ret = zdb_update(db, key, NULL, &dummy2, sizeof(dummy2));
if (ret < 0) {
printf("zdb_update: set dummy2 error\n");
return (-1);
}
blk = zdb_lookup(db, key, &size, NULL);
if (blk == NULL) {
printf("zdb_lookup: get dummy2 error\n");
return (-1);
}
pdummy = (DUMMY*) zdb_blk_data(blk);
for (i = 0; i < sizeof(dummy2) / sizeof(DUMMY); i++) {
printf("dummy2: key: %d, n: %d, s: %s\n", key, pdummy->n, pdummy->s);
pdummy++;
}
zdb_blk_free(blk);
ret = zdb_update(db, key, NULL, &dummy3, sizeof(dummy3));
if (ret < 0) {
printf("zdb_update: set dummy3 error\n");
return (-1);
}
blk = zdb_lookup(db, key, &size, NULL);
if (blk == NULL) {
printf("zdb_lookup: get dummy3 error\n");
return (-1);
}
pdummy = (DUMMY*) zdb_blk_data(blk);
for (i = 0; i < sizeof(dummy3) / sizeof(DUMMY); i++) {
printf("dummy3: key: %d, n: %d, s: %s\n", key, pdummy->n, pdummy->s);
pdummy++;
}
zdb_blk_free(blk);
return (3);
}
static int key_walk_fn(ZDB_KEY_STORE *store)
{
const char *myname = "key_walk_fn";
ACL_ITER iter;
ZDB_BLK_OFF *blk_off;
int i = 0;
printf(">>> acl_foreach for %s now:\n", STORE_PATH((ZDB_STORE*) store));
acl_foreach(iter, (ZDB_STORE*) store) {
blk_off = (ZDB_BLK_OFF*) iter.data;
if (i > 0 && (i % 100000) == 0) {
printf("%s: i:%d, key: %d, blk_off: %lld, inode: %d, ",
myname, i, iter.i, blk_off->offset, blk_off->inode);
ACL_METER_TIME("-");
}
i++;
}
return (i);
}
static int test_zdb_key_walk(ZDB *db)
{
const char *myname = "test_zdb_key_walk";
int ret;
time_t begin;
time(&begin);
ret = zdb_key_walk(db, key_walk_fn);
printf(">>>%s, acl_foreach for key, total: %d, time: %ld\n",
myname, ret, (long) (time(NULL) - begin));
return (0);
}
static int dat_walk_fn(ZDB_DAT_STORE *store)
{
const char *myname = "dat_walk_fn";
int i = 0;
ZDB_BLK *blk;
DUMMY *dummy;
ACL_ITER iter;
printf(">>>%s, acl_foreach for %s now:\n", myname, STORE_PATH((ZDB_STORE*) store));
acl_foreach(iter, (ZDB_STORE*) store) {
blk = (ZDB_BLK*) iter.data;
dummy = (DUMMY*) zdb_blk_data(blk);
if (i > 0 && (i % 100000) == 0) {
printf("%s: i:%d,key:%lld,data:%s,inext:%d,",
myname, i, blk->hdr.key, dummy->s, iter.i);
ACL_METER_TIME("-");
} else if (store->hdr.count <= 100) {
printf("%s: i:%d, key:%lld, data:%s, inext:%d\n",
myname, i, blk->hdr.key, dummy->s, iter.i);
}
i++;
}
return (i);
}
static int test_zdb_dat_walk(ZDB *db)
{
const char *myname = "test_zdb_dat_walk";
time_t begin;
int ret;
time(&begin);
ret = zdb_dat_walk(db, dat_walk_fn);
printf(">>>%s, acl_foreach for dat, total: %d, time: %ld\n",
myname, ret, (long) (time(NULL) - begin));
return (ret);
}
static int dat_stat_fn(ZDB_DAT_STORE *store)
{
const char *myname = "dat_stat_fn";
ZDB_DAT_HDR hdr;
memcpy(&hdr, &store->hdr, sizeof(ZDB_DAT_HDR));
printf("%s(%d): %s's hdr status\n",
myname, __LINE__, STORE_PATH((ZDB_STORE*) store));
printf("%s(%d):\tlimit: %lld, size: %lld, used count: %lld\n",
myname, __LINE__, hdr.limit, hdr.size, hdr.count);
printf("%s(%d):\tnstep: %d, blk_hdr_dlen: %d, blk_dlen: %d,"
" blk_count: %d\n", myname, __LINE__, hdr.nstep,
hdr.blk_hdr_dlen, hdr.blk_dlen, hdr.blk_count);
printf("%s(%d):\tihead_idle: %d, ihead_busy: %d\n",
myname, __LINE__, hdr.ihead_idle, hdr.ihead_busy);
return (0);
}
static int test_zdb_dat_stat(ZDB *db)
{
return (zdb_dat_walk(db, dat_stat_fn));
}
static int key_stat_fn(ZDB_KEY_STORE *store)
{
const char *myname = "key_stat_fn";
ZDB_KEY_HDR key_hdr;
memcpy(&key_hdr, &store->hdr, sizeof(ZDB_KEY_HDR));
printf(">> %s: %s's status:\n", myname, STORE_PATH((ZDB_STORE*) store));
printf(">>\tkey_limit: %lld, key_count: %lld, key_begin: %lld\n",
key_hdr.key_limit, key_hdr.key_count, key_hdr.key_begin);
return (0);
}
static int test_zdb_key_stat(ZDB *db)
{
return (zdb_key_walk(db, key_stat_fn));
}
static int dat_check_fn(ZDB_DAT_STORE *store)
{
return (zdb_dat_check(store, NULL));
}
static int test_zdb_dat_check(ZDB *db)
{
return (zdb_dat_walk(db, dat_check_fn));
}
static int key_check_fn(ZDB_KEY_STORE *store)
{
return (zdb_key_check(store, NULL));
}
static int test_zdb_key_check(ZDB *db)
{
return (zdb_key_walk(db, key_check_fn));
}
void zdb_test_main(const char *cmd)
{
const char *myname = "zdb_test";
ZDB *db;
const char *dbname = "test", *key_path = "./var";
/*ZDB_FLAG_CACHE_DAT | ZDB_FLAG_CACHE_KEY | ZDB_FLAG_SLICE_KEY | ZDB_FLAG_SLICE_DAT;*/
unsigned int oflags = 0;
ZDB_CFG zdb_cfg;
zdb_key_t key_begin = 0, key_limit = 1000000, dat_limit = 10000000, blk_dlen = sizeof(DUMMY);
int ret, max = 10000000, use_random = 0, walk_all = 0, dat_nstep = 1000, mod = 1;
int cache_key_limit = 0, cache_key_timeout = 0;
int cmd_add = 0, cmd_get = 0, cmd_update = 0, update_num = 2;
int cmd_stat = 0, cmd_check = 0;
ACL_ARGV *argv = acl_argv_split(cmd, ":");
ACL_ITER iter;
char *name, *value, *ptr;
char usage[] = "help:stat:check:link:random:walk:add:get:update={d}"
":max={d}:key_limit={d}:dat_limit={d}:nstep={d}:mod={d}:cache_key_limit={d}"
":cache_key_timeout={d}";
acl_make_dirs(key_path, 0700);
acl_foreach(iter, argv) {
name = ptr = (char*) iter.data;
if (strcasecmp(name, "help") == 0) {
printf("usage: %s\n", usage);
acl_argv_free(argv);
return;
} else if (strcasecmp(name, "stat") == 0) {
cmd_stat = 1;
continue;
} else if (strcasecmp(name, "check") == 0) {
cmd_check = 1;
continue;
} else if (strcasecmp(name, "link") == 0) {
oflags |= ZDB_FLAG_LINK_BUSY;
continue;
} else if (strcasecmp(name, "random") == 0) {
use_random = 1;
continue;
} else if (strcasecmp(name, "walk") == 0) {
walk_all = 1;
continue;
} else if (strcasecmp(name, "add") == 0) {
cmd_add = 1;
continue;
} else if (strcasecmp(name, "get") == 0) {
cmd_get = 1;
continue;
}
name = acl_mystrtok(&ptr, "=");
if (name == NULL)
continue;
value = acl_mystrtok(&ptr, "=");
if (value == NULL)
continue;
if (strcasecmp(name, "max") == 0) {
ret = atoi(value);
if (ret > 0)
max = ret;
} else if (strcasecmp(name, "key_limit") == 0) {
ret = atoi(value);
if (ret > 0)
key_limit = ret;
} else if (strcasecmp(name, "dat_limit") == 0) {
ret = atoi(value);
if (ret > 0)
dat_limit = ret;
} else if (strcasecmp(name, "update") == 0) {
ret = atoi(value);
if (ret > 0) {
update_num = ret;
cmd_update = 1;
}
} else if (strcasecmp(name, "nstep") == 0) {
ret = atoi(value);
if (ret > 0)
dat_nstep = ret;
} else if (strcasecmp(name, "mod") == 0) {
ret = atoi(value);
if (ret > 0)
mod = ret;
} else if (strcasecmp(name, "cache_key_limit") == 0) {
ret = atoi(value);
if (ret > 0)
cache_key_limit = ret;
} else if (strcasecmp(name, "cache_key_timeout") == 0) {
ret = atoi(value);
if (ret > 0)
cache_key_timeout = ret;
}
}
acl_argv_free(argv);
memset(&zdb_cfg, 0, sizeof(zdb_cfg));
/* 初始化 ZDB 存储引擎 */
zdb_init();
/* 开始配置 ZDB 配置对象 */
zdb_cfg.key_path = key_path;
zdb_cfg.key_begin = key_begin;
zdb_cfg.key_limit = key_limit;
zdb_cfg.dat_limit = dat_limit;
zdb_cfg.blk_dlen = (int) blk_dlen;
zdb_cfg.dat_nstep = dat_nstep; /* 值存储文件每次增加的数据块个数 */
if (cache_key_limit > 0 && cache_key_timeout > 0) {
zdb_cfg.key_cache_max = cache_key_limit * 2;
zdb_cfg.key_cache_timeout = cache_key_timeout;
zdb_cfg.key_wback_max = cache_key_limit;
oflags |= ZDB_FLAG_CACHE_KEY | ZDB_FLAG_SLICE_KEY;
printf("===========open key\n");
} else {
zdb_cfg.key_cache_max = 0;
zdb_cfg.key_cache_timeout = 0;
zdb_cfg.key_wback_max = 0;
}
zdb_cfg.dat_cache_max = 0;
zdb_cfg.dat_cache_timeout = 0;
zdb_cfg.dat_wback_max = 0;
/* 打开一个 ZDB 数据库对象 */
db = zdb_open(dbname, oflags, &zdb_cfg);
if (db == NULL)
acl_msg_fatal("%s: zdb open error(%s)", myname, acl_last_serror());
printf("DUMMY.sizeof: %d\n", (int) sizeof(DUMMY));
if (0) {
ret = zdb_test1(db);
if (ret > 0)
ret = zdb_test2(db);
} else
ret = 1;
if (!use_random) {
if (ret > 0 && cmd_add)
ret = bench_add(db, max);
if (ret > 0 && cmd_get)
ret = bench_get(db, max);
if (ret > 0 && cmd_update)
ret = bench_update(db, max, update_num);
} else {
if (ret > 0 && cmd_add)
ret = bench_random_add(db, max, mod);
if (ret > 0 && cmd_get)
ret = bench_random_get(db, max, mod);
}
if (walk_all) {
if (ret > 0)
ret = test_zdb_dat_walk(db);
if (ret >= 0)
ret = test_zdb_key_walk(db);
}
if (cmd_stat) {
if (ret >= 0)
ret = test_zdb_dat_stat(db);
if (ret >= 0)
ret = test_zdb_key_stat(db);
}
if (cmd_check) {
if (ret >= 0)
ret = test_zdb_dat_check(db);
if (ret >= 0)
ret = test_zdb_key_check(db);
}
zdb_close(db);
zdb_end();
}