chore(gensupport): add retry invocation headers (#1577)

Adds invocation ID and attempt count to x-goog-api-client header
for simple and resumable uploads.

Note that this will also add this information to chunk upload
requests for individual chunks for BQ and other APIs that do uploads
using gensupport as well. Hopefully this isn't an issue.

Similar to googleapis/google-cloud-go#6013

go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	cloud.google.com/go/compute v1.7.0
 	github.com/google/go-cmp v0.5.8
+	github.com/google/uuid v1.1.2
 	github.com/googleapis/enterprise-certificate-proxy v0.1.0
 	github.com/googleapis/gax-go/v2 v2.4.0
 	go.opencensus.io v0.23.0

go.sum

@@ -160,6 +160,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw=

internal/gensupport/resumable.go

@@ -10,8 +10,12 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/api/internal"
 )
 
 // ResumableUpload is used by the generated APIs to provide resumable uploads.
@@ -38,6 +42,11 @@ type ResumableUpload struct {
 	// ChunkRetryDeadline configures the per-chunk deadline after which no further
 	// retries should happen.
 	ChunkRetryDeadline time.Duration
+
+	// Track current request invocation ID and attempt count for retry metric
+	// headers.
+	invocationID string
+	attempts     int
 }
 
 // Progress returns the number of bytes uploaded at this point.
@@ -72,6 +81,10 @@ func (rx *ResumableUpload) doUploadRequest(ctx context.Context, data io.Reader,
 	req.Header.Set("Content-Type", rx.MediaType)
 	req.Header.Set("User-Agent", rx.UserAgent)
 
+	baseXGoogHeader := "gl-go/" + GoVersion() + " gdcl/" + internal.Version
+	invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", rx.invocationID, rx.attempts)
+	req.Header.Set("X-Goog-Api-Client", strings.Join([]string{baseXGoogHeader, invocationHeader}, " "))
+
 	// Google's upload endpoint uses status code 308 for a
 	// different purpose than the "308 Permanent Redirect"
 	// since-standardized in RFC 7238. Because of the conflict in
@@ -178,9 +191,11 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err
 	for {
 		var pause time.Duration
 
-		// Each chunk gets its own initialized-at-zero backoff.
+		// Each chunk gets its own initialized-at-zero backoff and invocation ID.
 		bo := rx.Retry.backoff()
 		quitAfter := time.After(retryDeadline)
+		rx.attempts = 1
+		rx.invocationID = uuid.New().String()
 
 		// Retry loop for a single chunk.
 		for {
@@ -223,6 +238,7 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err
 				break
 			}
 
+			rx.attempts++
 			pause = bo.Pause()
 			if resp != nil && resp.Body != nil {
 				resp.Body.Close()

internal/gensupport/send.go

@@ -8,9 +8,12 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/googleapis/gax-go/v2"
 )
 
@@ -71,6 +74,9 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r
 
 	var resp *http.Response
 	var err error
+	attempts := 1
+	invocationID := uuid.New().String()
+	baseXGoogHeader := req.Header.Get("X-Goog-Api-Client")
 
 	// Loop to retry the request, up to the context deadline.
 	var pause time.Duration
@@ -109,6 +115,9 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r
 			}
 			return resp, err
 		}
+		invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", invocationID, attempts)
+		xGoogHeader := strings.Join([]string{invocationHeader, baseXGoogHeader}, " ")
+		req.Header.Set("X-Goog-Api-Client", xGoogHeader)
 
 		resp, err = client.Do(req.WithContext(ctx))
 
@@ -123,6 +132,7 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r
 		if req.GetBody == nil || !errorFunc(status, err) {
 			break
 		}
+		attempts++
 		var errBody error
 		req.Body, errBody = req.GetBody()
 		if errBody != nil {