From 5309c65d2afc696262474d3fb87ac4ed6f4d8729 Mon Sep 17 00:00:00 2001
From: wangbin579 <wangbin579@gmail.com>
Date: Tue, 5 Jun 2018 11:09:36 +0800
Subject: [PATCH 1/8] Fix dup event problems

---
 src/chassis-event.c         | 2 --
 src/chassis-mainloop.c      | 2 ++
 src/chassis-mainloop.h      | 1 +
 src/chassis-options-utils.c | 2 +-
 src/mysql-proxy-cli.c       | 2 +-
 src/network-backend.c       | 4 ++--
 src/network-mysqld.c        | 5 ++---
 src/plugin-common.c         | 1 +
 8 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/chassis-event.c b/src/chassis-event.c
index d18b952..dd9b825 100644
--- a/src/chassis-event.c
+++ b/src/chassis-event.c
@@ -56,9 +56,7 @@ void
 chassis_event_add_with_timeout(chassis *chas, struct event *ev, struct timeval *tv)
 {
     event_base_set(chas->event_base, ev);
-#if NETWORK_DEBUG_TRACE_EVENT
     CHECK_PENDING_EVENT(ev);
-#endif
     event_add(ev, tv);
     g_debug("%s:event add ev:%p", G_STRLOC, ev);
 }
diff --git a/src/chassis-mainloop.c b/src/chassis-mainloop.c
index d45b64b..0154c6f 100644
--- a/src/chassis-mainloop.c
+++ b/src/chassis-mainloop.c
@@ -329,6 +329,7 @@ chassis_mainloop(void *_chas)
         }
     }
 
+#ifndef SIMPLE_PARSER
     chas->dist_tran_id = g_random_int_range(0, 100000000);
     int srv_id = g_random_int_range(0, 10000);
     if (chas->proxy_address) {
@@ -338,6 +339,7 @@ chassis_mainloop(void *_chas)
     }
     g_message("Initial dist_tran_id:%llu", chas->dist_tran_id);
     g_message("dist_tran_prefix:%s", chas->dist_tran_prefix);
+#endif
 
     /*
      * drop root privileges if requested
diff --git a/src/chassis-mainloop.h b/src/chassis-mainloop.h
index 2d4be32..8f840ff 100644
--- a/src/chassis-mainloop.h
+++ b/src/chassis-mainloop.h
@@ -55,6 +55,7 @@ typedef struct chassis chassis;
 #define MAX_QUERY_TIME 1000
 #define MAX_WAIT_TIME 1024
 #define MAX_DIST_TRAN_PREFIX 32
+#define DEFAULT_LIVE_TIME 7200
 
 #define MAX_ALLOWED_PACKET_CEIL    (1 * GB)
 #define MAX_ALLOWED_PACKET_DEFAULT (32 * MB)
diff --git a/src/chassis-options-utils.c b/src/chassis-options-utils.c
index 059f4a7..d46142b 100644
--- a/src/chassis-options-utils.c
+++ b/src/chassis-options-utils.c
@@ -532,7 +532,7 @@ show_max_alive_time(gpointer param) {
         return g_strdup_printf("%d (s)", srv->max_alive_time);
     }
     if(CAN_SAVE_OPTS_PROPERTY(opt_type)) {
-        if(srv->max_alive_time == 7200) {
+        if(srv->max_alive_time == DEFAULT_LIVE_TIME) {
             return NULL;
         }
         return g_strdup_printf("%d", srv->max_alive_time);
diff --git a/src/mysql-proxy-cli.c b/src/mysql-proxy-cli.c
index c9ca17a..84011a1 100644
--- a/src/mysql-proxy-cli.c
+++ b/src/mysql-proxy-cli.c
@@ -166,7 +166,7 @@ chassis_frontend_new(void)
 
     frontend->default_pool_size = 100;
     frontend->max_resp_len = 10 * 1024 * 1024;  /* 10M */
-    frontend->max_alive_time = 7200;
+    frontend->max_alive_time = DEFAULT_LIVE_TIME;
     frontend->merged_output_size = 8192;
     frontend->max_header_size = 65536;
     frontend->config_port = 3306;
diff --git a/src/network-backend.c b/src/network-backend.c
index 092fca6..512a5d2 100644
--- a/src/network-backend.c
+++ b/src/network-backend.c
@@ -31,7 +31,7 @@
 #include "cetus-users.h"
 
 const char *backend_state_t_str[] = {
-    "unkown",
+    "unknown",
     "online",
     "down",
     "maintaining",
@@ -39,7 +39,7 @@ const char *backend_state_t_str[] = {
 };
 
 const char *backend_type_t_str[] = {
-    "unkown",
+    "unknown",
     "read/write",
     "readonly"
 };
diff --git a/src/network-mysqld.c b/src/network-mysqld.c
index bf42df3..d6798e7 100644
--- a/src/network-mysqld.c
+++ b/src/network-mysqld.c
@@ -3304,6 +3304,7 @@ process_service_unavailable(network_mysqld_con *con)
         for (i = 0; i < con->servers->len; i++) {
             server_session_t *ss = g_ptr_array_index(con->servers, i);
             if (ss->fresh) {
+                CHECK_PENDING_EVENT(&(ss->server->event));
                 network_pool_add_idle_conn(ss->backend->pool, con->srv, ss->server);
                 ss->backend->connected_clients--;
                 g_message("%s: connected_clients sub:%d, %d ndx for con:%p", G_STRLOC,
@@ -4570,12 +4571,10 @@ process_self_read_auth_result(server_connection_state_t *con)
         network_mysqld_queue_reset(con->server);
         network_queue_clear(con->server->recv_queue);
         con->server->is_multi_stmt_set = con->is_multi_stmt_set;
-#if NETWORK_DEBUG_TRACE_EVENT
-        CHECK_PENDING_EVENT(&(con->server->event));
-#endif
         if (con->srv->is_back_compressed) {
             con->server->do_compress = 1;
         }
+        CHECK_PENDING_EVENT(&(con->server->event));
         network_pool_add_idle_conn(con->pool, con->srv, con->server);
         con->server = NULL;     /* tell _self_con_free we succeed */
         network_mysqld_self_con_free(con);
diff --git a/src/plugin-common.c b/src/plugin-common.c
index 9fc7f5c..9a1cc7a 100644
--- a/src/plugin-common.c
+++ b/src/plugin-common.c
@@ -584,6 +584,7 @@ proxy_put_shard_conn_to_pool(network_mysqld_con *con)
 
             int alive_time = con->srv->current_time - server->create_time;
             if (alive_time > con->srv->max_alive_time) {
+                g_debug("%s: reach max_alive_time", G_STRLOC);
                 is_put_to_pool_allowed = 0;
             }
 

From 541bc27a921711ca4fb87312ca873203d7a3f7a1 Mon Sep 17 00:00:00 2001
From: wangbin579 <wangbin579@gmail.com>
Date: Tue, 5 Jun 2018 11:21:43 +0800
Subject: [PATCH 2/8] Don't create conn for backend when meeting errors

---
 src/network-mysqld.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/network-mysqld.c b/src/network-mysqld.c
index d6798e7..e146b9d 100644
--- a/src/network-mysqld.c
+++ b/src/network-mysqld.c
@@ -4881,6 +4881,8 @@ network_connection_pool_create_conns(chassis *srv)
                           G_STRLOC, i, scs->server, scs);
 
                 scs->backend->connected_clients++;
+                int create_err = 0;
+
                 switch (network_socket_connect(scs->server)) {
                 case NETWORK_SOCKET_ERROR_RETRY:{
                     scs->state = ST_ASYNC_CONN;
@@ -4899,6 +4901,7 @@ network_connection_pool_create_conns(chassis *srv)
                     g_message("%s: set backend conn:%p read handshake", G_STRLOC, scs);
                     break;
                 default:
+                    create_err = 1;
                     scs->backend->connected_clients--;
                     network_mysqld_self_con_free(scs);
                     if (scs->srv->disable_threads) {
@@ -4912,6 +4915,10 @@ network_connection_pool_create_conns(chassis *srv)
                     }
                     break;
                 }
+
+                if (create_err) {
+                    break;
+                }
             }
         }
     }

From 8c8a7db44c7806ba58bee7ec85a596237a356661 Mon Sep 17 00:00:00 2001
From: wangbin579 <wangbin579@gmail.com>
Date: Tue, 5 Jun 2018 13:49:51 +0800
Subject: [PATCH 3/8] Add debug log for tracing create conn problems

---
 src/network-mysqld.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/network-mysqld.c b/src/network-mysqld.c
index e146b9d..411ae2e 100644
--- a/src/network-mysqld.c
+++ b/src/network-mysqld.c
@@ -4642,6 +4642,9 @@ network_mysqld_self_con_handle(int event_fd, short events, void *user_data)
                 return;
             }
 
+            g_debug("%s: connection between %s and %s",
+                G_STRLOC, con->client->src->name->str, con->server ? con->server->dst->name->str : "(server)");
+
             switch (proxy_self_read_handshake(srv, con)) {
             case RET_SUCCESS:
                 break;

From f0120d155116a109b3a363cd61216e691ef957b9 Mon Sep 17 00:00:00 2001
From: wangbin579 <wangbin579@gmail.com>
Date: Tue, 5 Jun 2018 13:51:27 +0800
Subject: [PATCH 4/8] Modify debug log for tracing create conn problems

---
 src/network-mysqld.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/network-mysqld.c b/src/network-mysqld.c
index 411ae2e..5405d05 100644
--- a/src/network-mysqld.c
+++ b/src/network-mysqld.c
@@ -4642,8 +4642,8 @@ network_mysqld_self_con_handle(int event_fd, short events, void *user_data)
                 return;
             }
 
-            g_debug("%s: connection between %s and %s",
-                G_STRLOC, con->client->src->name->str, con->server ? con->server->dst->name->str : "(server)");
+            g_debug("%s: connection %s and %s",
+                G_STRLOC, con->server->src->name->str, con->server->dst->name->str);
 
             switch (proxy_self_read_handshake(srv, con)) {
             case RET_SUCCESS:

From 67f36cbc38cbb7136396b22773e583a166ffc362 Mon Sep 17 00:00:00 2001
From: wangbin579 <wangbin579@gmail.com>
Date: Tue, 5 Jun 2018 14:48:53 +0800
Subject: [PATCH 5/8] Modify debug log for tracing create conn problems

---
 src/network-mysqld.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/network-mysqld.c b/src/network-mysqld.c
index 5405d05..b6a2a8f 100644
--- a/src/network-mysqld.c
+++ b/src/network-mysqld.c
@@ -4461,7 +4461,9 @@ chassis_event_add_with_timeout(srv, &(sock->event), timeout);
 static int
 process_self_event(server_connection_state_t *con, int events, int event_fd)
 {
+    g_debug("%s:events:%d, ev:%p, state:%d", G_STRLOC, events, (&con->server->event), con->state);
     if (events == EV_READ) {
+        g_debug("%s:EV_READ, ev:%p, state:%d", G_STRLOC, (&con->server->event), con->state);
         int b = -1;
         if (ioctl(con->server->fd, FIONREAD, &b)) {
             g_warning("ioctl(%d, FIONREAD, ...) failed: %s", event_fd, strerror(errno));
@@ -4478,7 +4480,7 @@ process_self_event(server_connection_state_t *con, int events, int event_fd)
             }
         }
     } else if (events == EV_TIMEOUT) {
-        g_debug("%s:timeout, ev:%p", G_STRLOC, (&con->server->event));
+        g_debug("%s:timeout, ev:%p, state:%d", G_STRLOC, (&con->server->event), con->state);
         if (con->state == ST_ASYNC_CONN) {
             g_message("%s: self conn timeout, state:%d, con:%p, server:%p", G_STRLOC, con->state, con, con->server);
             con->state = ST_ASYNC_ERROR;
@@ -4638,13 +4640,14 @@ network_mysqld_self_con_handle(int event_fd, short events, void *user_data)
         case ST_ASYNC_READ_HANDSHAKE:
             g_assert(events == 0 || event_fd == con->server->fd);
 
+            g_debug("%s: ST_ASYNC_READ_HANDSHAKE for con:%p, connection %s and %s",
+                G_STRLOC, con, con->server->src->name->str, con->server->dst->name->str);
+
+
             if (!process_self_server_read(con)) {
                 return;
             }
 
-            g_debug("%s: connection %s and %s",
-                G_STRLOC, con->server->src->name->str, con->server->dst->name->str);
-
             switch (proxy_self_read_handshake(srv, con)) {
             case RET_SUCCESS:
                 break;

From c7018f15482fcbe03aa4d0bab4ac36937b9b4961 Mon Sep 17 00:00:00 2001
From: wangbin579 <wangbin579@gmail.com>
Date: Tue, 5 Jun 2018 15:19:11 +0800
Subject: [PATCH 6/8] Add read timeout for create conn

---
 src/network-mysqld.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/network-mysqld.c b/src/network-mysqld.c
index b6a2a8f..f7bfd6f 100644
--- a/src/network-mysqld.c
+++ b/src/network-mysqld.c
@@ -4507,8 +4507,13 @@ process_self_server_read(server_connection_state_t *con)
     case NETWORK_SOCKET_SUCCESS:
         break;
     case NETWORK_SOCKET_WAIT_FOR_EVENT:{
+        struct timeval timeout;
+        timeout.tv_sec = 3;
+        timeout.tv_usec = 0;
+        g_debug("%s: set timeout:%d for new conn:%p", G_STRLOC,
+                (int)timeout.tv_sec, con);
         /* call us again when you have a event */
-        ASYNC_WAIT_FOR_EVENT(con->server, EV_READ, NULL, con);
+        ASYNC_WAIT_FOR_EVENT(con->server, EV_READ, &timeout, con);
         return 0;
     }
     case NETWORK_SOCKET_ERROR:

From 3abb518aaa9c0fe26e3be40d4a01be29e9ced8de Mon Sep 17 00:00:00 2001
From: wangbin579 <wangbin579@gmail.com>
Date: Tue, 5 Jun 2018 15:39:59 +0800
Subject: [PATCH 7/8] Limit wait times for create conn

---
 src/chassis-mainloop.h | 1 +
 src/network-mysqld.c   | 5 +++++
 src/network-mysqld.h   | 1 +
 3 files changed, 7 insertions(+)

diff --git a/src/chassis-mainloop.h b/src/chassis-mainloop.h
index 8f840ff..be93cef 100644
--- a/src/chassis-mainloop.h
+++ b/src/chassis-mainloop.h
@@ -54,6 +54,7 @@ typedef struct chassis chassis;
 #define MAX_SERVER_NUM 64
 #define MAX_QUERY_TIME 1000
 #define MAX_WAIT_TIME 1024
+#define MAX_TRY_NUM 6
 #define MAX_DIST_TRAN_PREFIX 32
 #define DEFAULT_LIVE_TIME 7200
 
diff --git a/src/network-mysqld.c b/src/network-mysqld.c
index f7bfd6f..0363fcc 100644
--- a/src/network-mysqld.c
+++ b/src/network-mysqld.c
@@ -4507,6 +4507,11 @@ process_self_server_read(server_connection_state_t *con)
     case NETWORK_SOCKET_SUCCESS:
         break;
     case NETWORK_SOCKET_WAIT_FOR_EVENT:{
+        con->retry_cnt++;
+        if (con->retry_cnt >= MAX_TRY_NUM) {
+            con->state = ST_ASYNC_ERROR;
+            break;
+        }
         struct timeval timeout;
         timeout.tv_sec = 3;
         timeout.tv_usec = 0;
diff --git a/src/network-mysqld.h b/src/network-mysqld.h
index 60afd14..7ea8698 100644
--- a/src/network-mysqld.h
+++ b/src/network-mysqld.h
@@ -340,6 +340,7 @@ struct server_connection_state_t {
     chassis *srv;
     network_connection_pool *pool;
     unsigned int is_multi_stmt_set:1;
+    unsigned int retry_cnt:4;
     guint8 charset_code;
 };
 

From 6cb57c8f8e302fc054d3ba81c18fe7741d560aca Mon Sep 17 00:00:00 2001
From: wangbin579 <wangbin579@gmail.com>
Date: Tue, 5 Jun 2018 15:46:46 +0800
Subject: [PATCH 8/8] Fix problems for create conn

---
 src/network-mysqld.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/network-mysqld.c b/src/network-mysqld.c
index 0363fcc..025bfc1 100644
--- a/src/network-mysqld.c
+++ b/src/network-mysqld.c
@@ -4658,6 +4658,10 @@ network_mysqld_self_con_handle(int event_fd, short events, void *user_data)
                 return;
             }
 
+            if (con->state == ST_ASYNC_ERROR) {
+                break;
+            }
+
             switch (proxy_self_read_handshake(srv, con)) {
             case RET_SUCCESS:
                 break;