test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

enhance: add retry termination log (#37894)

Browse Source 
/kind improvement

Signed-off-by: SimFG <bang.fu@zilliz.com>
This commit is contained in:
SimFG 2024-11-22 11:40:33 +08:00 committed by GitHub
parent bc131a911c
commit cef55eb093
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 27 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
pkg/util/retry/retry.go
View File

@ -47,19 +47,34 @@ func Do(ctx context.Context, fn func() error, opts ...Option) error {
} }
if !IsRecoverable(err) { if !IsRecoverable(err) {
if errors.IsAny(err, context.Canceled, context.DeadlineExceeded) && lastErr != nil { isContextErr := errors.IsAny(err, context.Canceled, context.DeadlineExceeded)
log.Warn("retry func failed, not be recoverable",
zap.Uint("retried", i),
zap.Uint("attempt", c.attempts),
zap.Bool("isContextErr", isContextErr),
)
if isContextErr && lastErr != nil {
return lastErr return lastErr
} }
return err return err
} }
if c.isRetryErr != nil && !c.isRetryErr(err) { if c.isRetryErr != nil && !c.isRetryErr(err) {
log.Warn("retry func failed, not be retryable",
zap.Uint("retried", i),
zap.Uint("attempt", c.attempts),
)
return err return err
} }
deadline, ok := ctx.Deadline() deadline, ok := ctx.Deadline()
if ok && time.Until(deadline) < c.sleep { if ok && time.Until(deadline) < c.sleep {
// to avoid sleep until ctx done isContextErr := errors.IsAny(err, context.Canceled, context.DeadlineExceeded)
if errors.IsAny(err, context.Canceled, context.DeadlineExceeded) && lastErr != nil { log.Warn("retry func failed, deadline",
zap.Uint("retried", i),
zap.Uint("attempt", c.attempts),
zap.Bool("isContextErr", isContextErr),
)
if isContextErr && lastErr != nil {
return lastErr return lastErr
} }
return err return err
@ -70,6 +85,10 @@ func Do(ctx context.Context, fn func() error, opts ...Option) error {
select { select {
case <-time.After(c.sleep): case <-time.After(c.sleep):
case <-ctx.Done(): case <-ctx.Done():
log.Warn("retry func failed, ctx done",
zap.Uint("retried", i),
zap.Uint("attempt", c.attempts),
)
return lastErr return lastErr
} }
@ -81,6 +100,11 @@ func Do(ctx context.Context, fn func() error, opts ...Option) error {
return nil return nil
} }
} }
if lastErr != nil {
log.Warn("retry func failed, reach max retry",
zap.Uint("attempt", c.attempts),
)
}
return lastErr return lastErr
} }