storage: Add S3 retry time (#906)

pingcap · Mar 23, 2021 · 8f80b8e · 8f80b8e
1 parent 0b223bc
commit 8f80b8e
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 3 deletions.
diff --git a/pkg/backup/client.go b/pkg/backup/client.go
@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
 	"sync"
 	"time"
 
@@ -216,6 +217,20 @@ func (bc *Client) SaveBackupMeta(ctx context.Context, backupMeta *backuppb.Backu
 	log.Debug("backup meta", zap.Reflect("meta", backupMeta))
 	backendURL := storage.FormatBackendURL(bc.backend)
 	log.Info("save backup meta", zap.Stringer("path", &backendURL), zap.Int("size", len(backupMetaData)))
+	failpoint.Inject("s3-outage-during-writing-file", func(v failpoint.Value) {
+		log.Info("failpoint s3-outage-during-writing-file injected, " +
+			"process will sleep for 3s and notify the shell to kill s3 service.")
+		if sigFile, ok := v.(string); ok {
+			file, err := os.Create(sigFile)
+			if err != nil {
+				log.Warn("failed to find shell to notify, skipping notify", zap.Error(err))
+			}
+			if file != nil {
+				file.Close()
+			}
+		}
+		time.Sleep(3 * time.Second)
+	})
 	return bc.storage.WriteFile(ctx, utils.MetaFile, backupMetaData)
 }
 

diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go
@@ -13,10 +13,13 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
@@ -39,7 +42,7 @@ const (
 	s3ProviderOption     = "s3.provider"
 	notFound             = "NotFound"
 	// number of retries to make of operations.
-	maxRetries = 6
+	maxRetries = 7
 	// max number of retries when meets error
 	maxErrorRetries = 3
 
@@ -233,9 +236,9 @@ func NewS3Storage( // revive:disable-line:flag-parameter
 func newS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (*S3Storage, error) {
 	qs := *backend
 	awsConfig := aws.NewConfig().
-		WithMaxRetries(maxRetries).
 		WithS3ForcePathStyle(qs.ForcePathStyle).
 		WithRegion(qs.Region)
+	request.WithRetryer(awsConfig, defaultS3Retryer())
 	if qs.Endpoint != "" {
 		awsConfig.WithEndpoint(qs.Endpoint)
 	}
@@ -671,3 +674,26 @@ func (rs *S3Storage) Create(ctx context.Context, name string) (ExternalFileWrite
 	uploaderWriter := newBufferedWriter(uploader, hardcodedS3ChunkSize, NoCompression)
 	return uploaderWriter, nil
 }
+
+// retryerWithLog wrappes the client.DefaultRetryer, and logging when retry triggered.
+type retryerWithLog struct {
+	client.DefaultRetryer
+}
+
+func (rl retryerWithLog) RetryRules(r *request.Request) time.Duration {
+	backoffTime := rl.DefaultRetryer.RetryRules(r)
+	if backoffTime > 0 {
+		log.Warn("failed to request s3, retrying", zap.Error(r.Error), zap.Duration("backoff", backoffTime))
+	}
+	return backoffTime
+}
+
+func defaultS3Retryer() request.Retryer {
+	return retryerWithLog{
+		DefaultRetryer: client.DefaultRetryer{
+			NumMaxRetries:    maxRetries,
+			MinRetryDelay:    1 * time.Second,
+			MinThrottleDelay: 2 * time.Second,
+		},
+	}
+}
diff --git a/tests/br_s3/run.sh b/tests/br_s3/run.sh
@@ -25,8 +25,11 @@ export MINIO_BROWSER=off
 export AWS_ACCESS_KEY_ID=$MINIO_ACCESS_KEY
 export AWS_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
 export S3_ENDPOINT=127.0.0.1:24927
+
 rm -rf "$TEST_DIR/$DB"
 mkdir -p "$TEST_DIR/$DB"
+sig_file="$TEST_DIR/sig_file_$RANDOM"
+rm -f "$sig_file"
 
 s3_pid=""
 start_s3() {
@@ -43,6 +46,12 @@ start_s3() {
     done
 }
 
+wait_sig() {
+    until [ -e "$sig_file" ]; do
+        sleep 1
+    done
+}
+
 start_s3
 echo "started s3 with pid = $s3_pid"
 bin/mc config --config-dir "$TEST_DIR/$TEST_NAME" \
@@ -67,7 +76,8 @@ for p in $(seq 2); do
   BACKUP_LOG="backup.log"
   rm -f $BACKUP_LOG
   unset BR_LOG_TO_TERM
-  ( run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT$S3_KEY" \
+  ( GO_FAILPOINTS="github.com/pingcap/br/pkg/backup/s3-outage-during-writing-file=1*return(\"$sig_file\")" \
+      run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT$S3_KEY" \
       --ratelimit 1 \
       --log-file $BACKUP_LOG || \
       ( cat $BACKUP_LOG && BR_LOG_TO_TERM=1 && exit 1 ) ) &
@@ -77,6 +87,10 @@ for p in $(seq 2); do
   kill -9 $s3_pid
   sleep 15
   start_s3
+  wait_sig
+  kill -9 $s3_pid
+  sleep 15
+  start_s3
   wait $br_pid
 
   cat $BACKUP_LOG