Show stack trace info when rpc fail (#12290)

Signed-off-by: dragondriver <jiquan.long@zilliz.com>

internal/distributed/datacoord/client/client.go

@@ -202,7 +202,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 		return ret, nil
 	}
 	if err == context.Canceled || err == context.DeadlineExceeded {
-		return nil, err
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 
 	log.Debug("DataCoord Client grpc error", zap.Error(err))
@@ -210,6 +210,10 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 	c.resetConnection()
 
 	ret, err = caller()
+	if err != nil {
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
+	}
+
 	return ret, err
 }
 

internal/distributed/datanode/client/client.go

@@ -183,7 +183,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 		return ret, nil
 	}
 	if err == context.Canceled || err == context.DeadlineExceeded {
-		return nil, err
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 
 	log.Debug("DataNode Client grpc error", zap.Error(err))
@@ -191,8 +191,8 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 	c.resetConnection()
 
 	ret, err = caller()
-	if err == nil {
+	if err != nil {
-		return ret, nil
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	return ret, err
 }

internal/distributed/indexcoord/client/client.go

@@ -189,7 +189,7 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 		return ret, nil
 	}
 	if err == context.Canceled || err == context.DeadlineExceeded {
-		return nil, err
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 
 	log.Debug("IndexCoord Client grpc error", zap.Error(err))
@@ -197,8 +197,8 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 	c.resetConnection()
 
 	ret, err = caller()
-	if err == nil {
+	if err != nil {
-		return ret, nil
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	return ret, err
 }

internal/distributed/indexnode/client/client.go

@@ -179,15 +179,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 		return ret, nil
 	}
 	if err == context.Canceled || err == context.DeadlineExceeded {
-		return nil, err
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	log.Debug("IndexNode Client grpc error", zap.Error(err))
 
 	c.resetConnection()
 
 	ret, err = caller()
-	if err == nil {
+	if err != nil {
-		return ret, nil
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	return ret, err
 }

internal/distributed/proxy/client/client.go

@@ -180,15 +180,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 		return ret, nil
 	}
 	if err == context.Canceled || err == context.DeadlineExceeded {
-		return nil, err
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	log.Debug("Proxy Client grpc error", zap.Error(err))
 
 	c.resetConnection()
 
 	ret, err = caller()
-	if err == nil {
+	if err != nil {
-		return ret, nil
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	return ret, err
 }

internal/distributed/querycoord/client/client.go

@@ -204,15 +204,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 		return ret, nil
 	}
 	if err == context.Canceled || err == context.DeadlineExceeded {
-		return nil, err
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	log.Debug("QueryCoord Client grpc error", zap.Error(err))
 
 	c.resetConnection()
 
 	ret, err = caller()
-	if err == nil {
+	if err != nil {
-		return ret, nil
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	return ret, err
 }

internal/distributed/querynode/client/client.go

@@ -171,15 +171,15 @@ func (c *Client) recall(caller func() (interface{}, error)) (interface{}, error)
 		return ret, nil
 	}
 	if err == context.Canceled || err == context.DeadlineExceeded {
-		return nil, err
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	log.Debug("QueryNode Client grpc error", zap.Error(err))
 
 	c.resetConnection()
 
 	ret, err = caller()
-	if err == nil {
+	if err != nil {
-		return ret, nil
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	return ret, err
 }

internal/distributed/rootcoord/client/client.go

@@ -232,15 +232,15 @@ func (c *GrpcClient) recall(caller func() (interface{}, error)) (interface{}, er
 		return ret, nil
 	}
 	if err == context.Canceled || err == context.DeadlineExceeded {
-		return nil, err
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	log.Debug("RootCoord Client grpc error", zap.Error(err))
 
 	c.resetConnection()
 
 	ret, err = caller()
-	if err == nil {
+	if err != nil {
-		return ret, nil
+		return nil, fmt.Errorf("err: %s\n, %s", err.Error(), trace.StackTrace())
 	}
 	return ret, err
 }

internal/util/trace/stack_trace.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+package trace
+
+import (
+	"fmt"
+	"runtime"
+)
+
+const numFuncsInStack = 10
+
+// StackTraceMsg returns the stack information, which numFuncs means how many functions do you want to show in the stack
+// information.
+func StackTraceMsg(numFuncs uint) string {
+	pc := make([]uintptr, numFuncs)
+	n := runtime.Callers(0, pc)
+	frames := runtime.CallersFrames(pc[:n])
+
+	ret := ""
+
+	for {
+		frame, more := frames.Next()
+		ret += fmt.Sprintf("%s:%d %s\n", frame.File, frame.Line, frame.Function)
+		if !more {
+			break
+		}
+	}
+
+	return ret
+}
+
+// StackTrace returns the stack trace information.
+func StackTrace() string {
+	return StackTraceMsg(numFuncsInStack)
+}

internal/util/trace/stack_trace_test.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+package trace
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestStackTraceMsg(t *testing.T) {
+	fmt.Println(StackTraceMsg(1))
+	fmt.Println(StackTraceMsg(5))
+	fmt.Println(StackTraceMsg(10))
+
+	func() {
+		fmt.Println(StackTraceMsg(10))
+	}()
+
+	func() {
+		func() {
+			fmt.Println(StackTraceMsg(10))
+		}()
+	}()
+}
+
+func TestStackTrace(t *testing.T) {
+	fmt.Println(StackTrace())
+
+	func() {
+		fmt.Println(StackTrace())
+	}()
+
+	func() {
+		func() {
+			fmt.Println(StackTrace())
+		}()
+	}()
+}