修复了C++库中连接池及HTTP模块中的BUG

C++库中连接池管理存在BUG;C++库的 HTTP 模块存在BUG;C库中的 master 服务器框架允许以非 ROOT
身份启动;C++库中增加了目录扫描类:scan_dir
This commit is contained in:
zsxxsz 2014-05-22 21:33:41 +08:00
parent ad62afa90b
commit 1c565c875b
22 changed files with 194 additions and 69 deletions

View File

@ -33,3 +33,5 @@ queue_directory = {install_path}/var
pid_file = {install_path}/var/pid/acl_master.pid
# 是否扫描并行 {install_path}/conf/service/ 目录下的子目录服务配置文件0 -- 1 --
scan_subdir = 0
# 是否允许自动切换用户运行身份
# limit_privilege = 0

View File

@ -1,6 +1,10 @@
修改历史列表:
------------------------------------------------------------------------
443) 2014.5.20
443.1) feature: acl_master 允许以非 root 用户身份运行,需要在其配置文件(main.cf)中
的 limit_privilege = 0
442) 2014.5.15
442.1) feature: acl_scan_dir.c 增加子函数 acl_scan_dir_next_name 及丰富了一些接口的功能

View File

@ -22,8 +22,11 @@
# include <errno.h>
*/
# ifndef c_pathdelim_chr
# define c_pathdelim_chr '/'
# ifndef PATH_SEP_C
# define PATH_SEP_C '/'
# endif
# ifndef PATH_SEP_S
# define PATH_SEP_S "/"
# endif
/*

View File

@ -65,11 +65,14 @@
# ifdef ACL_BCB_COMPILER
# pragma hdrstop
# endif
# define _USE_FAST_MACRO
# define _USE_HTABLE_SEARCH
# define _USE_FAST_MACRO
# define _USE_HTABLE_SEARCH
# ifndef c_pathdelim_chr
# define c_pathdelim_chr '\\'
# ifndef PATH_SEP_C
# define PATH_SEP_C '\\'
# endif
# ifndef PATH_SEP_S
# define PATH_SEP_S "\\"
# endif
# undef ACL_HAS_PTHREAD

View File

@ -70,11 +70,12 @@ static void master_listen_sock(ACL_MASTER_SERV *serv)
service_type = ACL_VSTREAM_TYPE_LISTEN_INET;
break;
case ACL_MASTER_SERV_TYPE_UNIX:
acl_set_eugid(acl_var_master_owner_uid,
acl_var_master_owner_gid);
if (acl_var_master_limit_privilege)
acl_set_eugid(acl_var_master_owner_uid,
acl_var_master_owner_gid);
serv->listen_fds[i] = acl_unix_listen(
addr->addr, qlen, ACL_NON_BLOCKING);
if (acl_var_master_set_ugid)
if (acl_var_master_limit_privilege)
acl_set_ugid(getuid(), getgid());
service_type = ACL_VSTREAM_TYPE_LISTEN_UNIX;
@ -178,7 +179,8 @@ static void master_listen_unix(ACL_MASTER_SERV *serv)
qlen = 128;
}
acl_set_eugid(acl_var_master_owner_uid, acl_var_master_owner_gid);
if (acl_var_master_limit_privilege)
acl_set_eugid(acl_var_master_owner_uid, acl_var_master_owner_gid);
serv->listen_fds[0] = acl_unix_listen(serv->name, qlen, ACL_NON_BLOCKING);
if (serv->listen_fds[0] == ACL_SOCKET_INVALID)
acl_msg_fatal("%s(%d)->%s: listen on addr(%s) error(%s)",
@ -188,7 +190,7 @@ static void master_listen_unix(ACL_MASTER_SERV *serv)
O_RDONLY, acl_var_master_buf_size,
acl_var_master_rw_timeout, ACL_VSTREAM_TYPE_LISTEN_UNIX);
acl_close_on_exec(serv->listen_fds[0], ACL_CLOSE_ON_EXEC);
if (acl_var_master_set_ugid)
if (acl_var_master_limit_privilege)
acl_set_ugid(getuid(), getgid());
acl_msg_info("%s(%d), %s: listen on domain socket: %s, qlen: %d",
__FILE__, __LINE__, myname, serv->name, qlen);
@ -198,7 +200,8 @@ static void master_listen_fifo(ACL_MASTER_SERV *serv)
{
const char *myname = "master_listen_fifo";
acl_set_eugid(acl_var_master_owner_uid, acl_var_master_owner_gid);
if (acl_var_master_limit_privilege)
acl_set_eugid(acl_var_master_owner_uid, acl_var_master_owner_gid);
serv->listen_fds[0] = acl_fifo_listen(serv->name,
0622, ACL_NON_BLOCKING);
if (serv->listen_fds[0] == ACL_SOCKET_INVALID)
@ -209,7 +212,7 @@ static void master_listen_fifo(ACL_MASTER_SERV *serv)
O_RDONLY, acl_var_master_buf_size,
acl_var_master_rw_timeout, ACL_VSTREAM_TYPE_LISTEN);
acl_close_on_exec(serv->listen_fds[0], ACL_CLOSE_ON_EXEC);
if (acl_var_master_set_ugid)
if (acl_var_master_limit_privilege)
acl_set_ugid(getuid(), getgid());
acl_msg_info("%s(%d), %s: listen on fifo socket: %s",
__FILE__, __LINE__, myname, serv->name);

View File

@ -99,11 +99,12 @@ static void master_wakeup_timer_event(int type acl_unused,
* module that is paranoid about open() calls.
*/
case ACL_MASTER_SERV_TYPE_FIFO:
acl_set_eugid(acl_var_master_owner_uid,
acl_var_master_owner_gid);
if (acl_var_master_limit_privilege)
acl_set_eugid(acl_var_master_owner_uid,
acl_var_master_owner_gid);
status = acl_fifo_trigger(acl_var_master_global_event,
serv->name, &wakeup, sizeof(wakeup), BRIEFLY);
if (acl_var_master_set_ugid)
if (acl_var_master_limit_privilege)
acl_set_ugid(getuid(), getgid());
break;
default:

View File

@ -72,11 +72,11 @@ static ACL_CONFIG_STR_TABLE __conf_str_tab[] = {
};
int acl_var_master_scan_subdir;
int acl_var_master_set_ugid;
int acl_var_master_limit_privilege;
static ACL_CONFIG_BOOL_TABLE __conf_bool_tab[] = {
{ ACL_VAR_MASTER_SCAN_SUBDIR, ACL_DEF_MASTER_SCAN_SUBDIR, &acl_var_master_scan_subdir },
{ ACL_VAR_MASTER_SET_UGID, ACL_DEF_MASTER_SET_UGID, &acl_var_master_set_ugid },
{ ACL_VAR_MASTER_LIMIT_PRIVILEGE, ACL_DEF_MASTER_LIMIT_PRIVILEGE, &acl_var_master_limit_privilege },
{ 0, 0, 0 },
};
#endif /* ACL_UNIX */

View File

@ -114,9 +114,9 @@ extern int acl_var_master_rw_timeout;
#define ACL_DEF_MASTER_SCAN_SUBDIR 0
extern int acl_var_master_scan_subdir;
#define ACL_VAR_MASTER_SET_UGID "set_ugid"
#define ACL_DEF_MASTER_SET_UGID 1
extern int acl_var_master_set_ugid;
#define ACL_VAR_MASTER_LIMIT_PRIVILEGE "limit_privilege"
#define ACL_DEF_MASTER_LIMIT_PRIVILEGE 0
extern int acl_var_master_limit_privilege;
extern pid_t acl_var_master_pid;

View File

@ -184,6 +184,7 @@ int acl_make_dirs(const char *path, int perms)
if (error != ERROR_ALREADY_EXISTS)
break;
ret = 0;
/* Race condition? */
if ((ret = stat(saved_path, &st)) < 0)
break;
@ -192,7 +193,8 @@ int acl_make_dirs(const char *path, int perms)
ret = -1;
break;
}
}
} else
ret = 0;
}
if (saved_ch != 0)
*cp = saved_ch;

View File

@ -201,7 +201,7 @@ int acl_scan_dir_push(ACL_SCAN_DIR *scan, const char *path)
info = (ACL_SCAN_INFO *) acl_mymalloc(sizeof(*info));
if (scan->current) {
info->path = acl_concatenate(ACL_SCAN_DIR_PATH(scan),
"/", path, (char *) 0);
PATH_SEP_S, path, (char *) 0);
} else {
info->path = acl_mystrdup(path);
}
@ -290,8 +290,8 @@ const char *acl_scan_dir_next_file(ACL_SCAN_DIR *scan)
return (NULL);
continue;
}
snprintf(pathbuf, sizeof(pathbuf), "%s/%s",
ACL_SCAN_DIR_PATH(scan), name);
snprintf(pathbuf, sizeof(pathbuf), "%s%c%s",
ACL_SCAN_DIR_PATH(scan), PATH_SEP_C, name);
if (acl_stat(pathbuf, &sbuf) < 0) {
char tbuf[256];
acl_msg_error("%s(%d), %s: stat file(%s) error(%s)",
@ -333,8 +333,8 @@ const char *acl_scan_dir_next_dir(ACL_SCAN_DIR *scan)
return (NULL);
continue;
}
snprintf(pathbuf, sizeof(pathbuf), "%s/%s",
ACL_SCAN_DIR_PATH(scan), name);
snprintf(pathbuf, sizeof(pathbuf), "%s%c%s",
ACL_SCAN_DIR_PATH(scan), PATH_SEP_C, name);
if (acl_stat(pathbuf, &sbuf) < 0) {
char tbuf[256];
acl_msg_error("%s(%d), %s: stat file(%s) error(%s)",
@ -372,8 +372,8 @@ const char *acl_scan_dir_next_name(ACL_SCAN_DIR *scan, int *is_file)
return (NULL);
continue;
}
snprintf(pathbuf, sizeof(pathbuf), "%s/%s",
ACL_SCAN_DIR_PATH(scan), name);
snprintf(pathbuf, sizeof(pathbuf), "%s%c%s",
ACL_SCAN_DIR_PATH(scan), PATH_SEP_C, name);
if (acl_stat(pathbuf, &sbuf) < 0) {
char tbuf[256];
acl_msg_error("%s(%d), %s: stat file(%s) error(%s)",
@ -420,8 +420,8 @@ acl_int64 acl_scan_dir_size2(ACL_SCAN_DIR *scan, int *nfile, int *ndir)
break;
continue;
}
snprintf(pathbuf, sizeof(pathbuf), "%s/%s",
ACL_SCAN_DIR_PATH(scan), name);
snprintf(pathbuf, sizeof(pathbuf), "%s%c%s",
ACL_SCAN_DIR_PATH(scan), PATH_SEP_C, name);
if (acl_stat(pathbuf, &sbuf) < 0) {
char tbuf[256];
acl_msg_error("%s(%d), %s: stat file(%s) error(%s)",
@ -509,8 +509,8 @@ acl_int64 acl_scan_dir_rm2(ACL_SCAN_DIR *scan, int *ndir, int *nfile)
continue;
}
snprintf(path, sizeof(path), "%s/%s",
ACL_SCAN_DIR_PATH(scan), name);
snprintf(path, sizeof(path), "%s%c%s",
ACL_SCAN_DIR_PATH(scan), PATH_SEP_C, name);
if (acl_stat(path, &sbuf) < 0) {
char tbuf[256];

View File

@ -6,11 +6,18 @@ extern "C" {
#endif
#ifdef WIN32
#include <direct.h>
#define __S_ISTYPE(mode, mask) (((mode) & _S_IFMT) == (mask))
#ifndef S_ISDIR
# define S_ISDIR(mode) __S_ISTYPE((mode), _S_IFDIR)
#endif
#ifndef S_ISREG
# define S_ISREG(mode) __S_ISTYPE((mode), _S_IFREG)
#endif
#endif
#ifdef __cplusplus

View File

@ -274,8 +274,8 @@ static char *path_str_strip(const char *psrc, char *pbuf, int sizeb)
n = sizeb;
while (*ptr_src && n > 0) {
if (*ptr_src == c_pathdelim_chr
&& *(ptr_src + 1) == c_pathdelim_chr)
if (*ptr_src == PATH_SEP_C
&& *(ptr_src + 1) == PATH_SEP_C)
; /* skip any useless '/'(in unix) or '\\'(in windows) */
else {
*ptr_obj++ = *ptr_src;
@ -325,10 +325,10 @@ int acl_dir_correct(const char *psrc_dir, char *pbuf, int sizeb)
/* 为了保证最后一个字符肯定为 '/'(unix) or '\\'(windows), 需做如下处理 */
if (*(ptr - 1) != c_pathdelim_chr) {
if (*(ptr - 1) != PATH_SEP_C) {
if (ptr >= pbuf + sizeb) /* 说明所给的内存空间不够 */
return(-1);
*ptr++ = c_pathdelim_chr;
*ptr++ = PATH_SEP_C;
*ptr = 0;
}
return(0);
@ -345,7 +345,7 @@ int acl_dir_getpath(const char *pathname, char *pbuf, int bsize)
n = acl_file_path_correct(pathname, pbuf, bsize);
if (n < 0)
return (-1);
ptr = strrchr(pbuf, c_pathdelim_chr);
ptr = strrchr(pbuf, PATH_SEP_C);
if (ptr != NULL)
*ptr = 0;
if (ptr == pbuf) { /* such as "/tmp.txt", I'll left "/" */

View File

@ -1,7 +1,18 @@
修改历史列表:
------------------------------------------------------------------------
236) 2014.5.22
236.1) bufix: http_client 类的构造函数 http_client() 中没有对 buf_ 赋 NULL结果
导致在 http_client::reset 中对 buf_ 进行 reset 操作时造成了非法指针引用
235) 2014.5.20
235.1) bugfix: connect_manager 连接池集群管理类当某个连接池有问题时,不能自动将失败的
连接池进行恢复,最后可能会导致所有连接池不可用
235.2) feature: connect_manager 增加了设置了用于连接池自动恢复的时来设置恢复时间间隔
的方法set_retry_inter
234) 2014.5.15
234.1) feature: 增加了 stdlib/scan_dir 类,该类用于磁盘目录扫描
234.2) sample: 新增示例 samples/dircopy 用来拷贝目录结构,内部使用了 scan_dir 类
233) 2014.5.13
233.1) feature: string/http_client/http_request 类增加了按行读数据的方法

View File

@ -37,14 +37,20 @@ public:
int default_count);
/**
*
*
*
* @param addr {const char*} (ip:port)
* @param count {int}
* @return {connect_pool&}
*/
connect_pool& set(const char* addr, int count);
/**
*
* @param n {int} <= 0
* @return {void}
*/
void set_retry_inter(int n);
/**
*
*
@ -165,6 +171,7 @@ private:
size_t service_idx_; // 下一个要访问的的下标值
locker lock_; // 访问 pools_ 时的互斥锁
int stat_inter_; // 统计访问量的定时器间隔
int retry_inter_; // 连接池失败后重试的时间间隔
connect_monitor* monitor_; // 后台检测线程句柄
// 设置除缺省服务之外的服务器集群

View File

@ -35,7 +35,7 @@ public:
*
* @param retry_inter {int} ()
* <= 0
* 10
* 1
* @return {connect_pool&}
*/
connect_pool& set_retry_inter(int retry_inter);
@ -79,13 +79,11 @@ public:
void set_alive(bool ok /* true | false */);
/**
*
* @return {bool}
*
*
* @return {bool} true
*/
bool aliving(void) const
{
return alive_;
}
bool aliving();
/**
*

View File

@ -50,7 +50,7 @@ all:
@(cd thread_pool; make)
@(cd thread_client; make)
@(cd http_request_manager; make)
@(cd scan_dir; make)
@(cd dircopy; make)
clean:
@(cd string; make clean)
@ -104,4 +104,4 @@ clean:
@(cd thread_pool; make clean)
@(cd thread_client; make clean)
@(cd http_request_manager; make clean)
@(cd scan_dir; make clean)
@(cd dircopy; make clean)

View File

@ -8,6 +8,21 @@
#endif // WIN32
#include "acl_cpp/lib_acl.hpp"
#ifdef WIN32
#define SEP '\\'
#else
#define SEP '/'
#endif
// 去年路径前的 "./" 或 ".\",因为在 WIN32 下
#define SKIP(ptr) do \
{ \
if (*ptr == '.' && *(ptr + 1) == '/') \
ptr += 2; \
else if (*ptr == '.' && *(ptr + 1) == '\\') \
ptr += 2; \
} while (0)
static bool copy_file(acl::ifstream& in, const acl::string& to_path,
const acl::string& to_filepath, int* ncopied)
{
@ -54,15 +69,16 @@ static bool copy_file(acl::ifstream& in, const acl::string& to_path,
ret = in.read(buf, sizeof(buf), false);
if (ret == -1)
{
if (nread != length)
{
logger_error("read from file: %s error: %s, nread: %lld, "
"length: %lld", in.file_path(),
acl::last_serror(), nread, length);
return false;
}
break;
if (nread == length)
break;
logger_error("read from file: %s error: %s, "
"nread: %lld, length: %lld",
in.file_path(), acl::last_serror(),
nread, length);
return false;
}
nread += ret;
if (out.write(buf, ret) == -1)
@ -89,8 +105,17 @@ static bool cmp_copy(acl::scan_dir& scan, const char* name,
return false;
}
SKIP(rpath);
SKIP(name);
// printf(">>rpath: %s\r\n", rpath);
// printf(">>name: %s\r\n", name);
acl::string from_filepath;
from_filepath.format("%s/%s", rpath, name);
if (*rpath == 0)
from_filepath << name;
else
from_filepath << rpath << SEP << name;
acl::ifstream from_fp;
if (from_fp.open_read(from_filepath.c_str()) == false)
@ -102,8 +127,8 @@ static bool cmp_copy(acl::scan_dir& scan, const char* name,
acl::string to_pathbuf;
acl::string to_filepath;
to_pathbuf.format("%s/%s", to_path.c_str(), rpath);
to_filepath.format("%s/%s/%s", to_path.c_str(), rpath, name);
to_pathbuf << to_path << SEP << rpath;
to_filepath << to_path << SEP << rpath << SEP << name;
//printf("from_filepath: %s, to_filepath: %s\r\n",
// from_fp.file_path(), to_filepath.c_str());
@ -174,8 +199,11 @@ static bool check_dir(acl::scan_dir& scan, const char* to, int* ncopied)
return false;
}
SKIP(rpath);
acl::string to_path;
to_path.format("%s/%s", to, rpath);
to_path << to << SEP << rpath;
// printf(">>to_path: %s, to: %s\r\n", to_path.c_str(), to);
if (access(to_path.c_str(), 0) == 0)
return true;
@ -188,7 +216,11 @@ static bool check_dir(acl::scan_dir& scan, const char* to, int* ncopied)
return true;
}
else
{
logger_error("make dirs(%s) error: %s",
to_path.c_str(), acl::last_serror());
return false;
}
}
}
@ -207,14 +239,22 @@ static void do_copy(const acl::string& from, const acl::string& to)
int nfiles = 0, ndirs = 0, nfiles_copied = 0, ndirs_copied = 0;
while ((name = scan.next(false, &is_file)) != NULL)
{
SKIP(name);
if (is_file)
{
if (cmp_copy(scan, name, to, &nfiles_copied) == false)
{
printf(">>cm_copy failed, name: %s\r\n", name);
break;
}
nfiles++;
}
else if (check_dir(scan, to, &ndirs_copied) == false)
{
printf(">>check_dir failed, name: %s\r\n", name);
break;
}
else
ndirs++;
@ -283,7 +323,7 @@ int main(int argc, char* argv[])
}
char path[256];
if (getcwd(path, sizeof(path)) < 0)
if (getcwd(path, sizeof(path)) == NULL)
{
logger_error("getcwd error: %s", path);
return 1;

View File

@ -13,6 +13,7 @@ connect_manager::connect_manager()
: default_pool_(NULL)
, service_idx_(0)
, stat_inter_(1)
, retry_inter_(1)
, monitor_(NULL)
{
}
@ -58,6 +59,22 @@ static int check_addr(const char* addr, string& buf, int default_count)
return conn_max;
}
void connect_manager::set_retry_inter(int n)
{
if (n == retry_inter_)
return;
lock_.lock();
retry_inter_ = n;
std::vector<connect_pool*>::iterator it = pools_.begin();
for (; it != pools_.end(); ++it)
(*it)->set_retry_inter(retry_inter_);
lock_.unlock();
}
void connect_manager::init(const char* default_addr,
const char* addr_list, int count)
{
@ -129,6 +146,8 @@ connect_pool& connect_manager::set(const char* addr, int count)
}
connect_pool* pool = create_pool(key, count, pools_.size() - 1);
if (retry_inter_ > 0)
pool->set_retry_inter(retry_inter_);
pools_.push_back(pool);
lock_.unlock();

View File

@ -21,7 +21,7 @@ connect_pool::connect_pool(const char* addr, int max, size_t idx /* = 0 */)
, current_used_(0)
, last_(0)
{
retry_inter_ = 10;
retry_inter_ = 1;
if (max_ < 1)
max_ = 10;
@ -44,10 +44,34 @@ connect_pool& connect_pool::set_idle_ttl(time_t ttl)
connect_pool& connect_pool::set_retry_inter(int retry_inter)
{
lock_.lock();
retry_inter_ = retry_inter;
lock_.unlock();
return *this;
}
bool connect_pool::aliving()
{
// XXX虽然此处未加锁但也应该不会有问题因为下面的 peek() 过程会再次
// 对 alive_ 加锁,以防止多线程操作时的冲突
if (alive_)
return true;
time_t now = time(NULL);
lock_.lock();
if (now - last_dead_ >= retry_inter_)
{
alive_ = true;
lock_.unlock();
return true;
}
lock_.unlock();
return false;
}
connect_client* connect_pool::peek()
{
lock_.lock();

View File

@ -23,6 +23,7 @@ http_client::http_client(void)
, body_finish_(false)
, disconnected_(true)
, chunked_transfer_(false)
, buf_(NULL)
{
}

View File

@ -78,7 +78,7 @@ const char* scan_dir::next_file(bool full /* = false */)
if (file_buf_ == NULL)
file_buf_ = NEW string(256);
file_buf_->format("%s/%s", path, file);
file_buf_->format("%s%c%s", path, PATH_SEP_C, file);
return file_buf_->c_str();
}
@ -100,7 +100,7 @@ const char* scan_dir::next_dir(bool full /* = false */)
if (file_buf_ == NULL)
file_buf_ = NEW string(256);
file_buf_->format("%s/%s", path, dir);
file_buf_->format("%s%c%s", path, PATH_SEP_C, dir);
return file_buf_->c_str();
}
@ -125,7 +125,7 @@ const char* scan_dir::next(bool full /* = false */, bool* is_file /* = NULL */)
if (file_buf_ == NULL)
file_buf_ = NEW string(256);
file_buf_->format("%s/%s", path, name);
file_buf_->format("%s%c%s", path, PATH_SEP_C, name);
return file_buf_->c_str();
}
@ -179,7 +179,7 @@ const char* scan_dir::curr_path(bool full /* = false */)
if (path_buf_ == NULL)
path_buf_ = NEW string(256);
path_buf_->format("%s/%s", buf, rpath);
path_buf_->format("%s%c%s", buf, PATH_SEP_C, rpath);
return path_buf_->c_str();
}
@ -200,7 +200,7 @@ const char* scan_dir::curr_file(bool full /* = false */)
if (file_buf_ == NULL)
file_buf_ = NEW string(256);
file_buf_->format("%s/%s", path, ptr);
file_buf_->format("%s%c%s", path, PATH_SEP_C, ptr);
return file_buf_->c_str();
}

File diff suppressed because one or more lines are too long