Fix: do not fail TaskRun for concurrent modification errors

This commit fixes the behaviour that a concurrent modification error
when stopping sidecar will fail the TaskRun, which could cause
successful Tasks to fail even though it could succeed after retrying.

/kind bug

pkg/reconciler/taskrun/taskrun.go

@@ -280,6 +280,11 @@ func (c *Reconciler) stopSidecars(ctx context.Context, tr *v1.TaskRun) error {
 		// it has probably evicted. We can return the error, but we consider it a permanent one.
 		return controller.NewPermanentError(err)
 	} else if err != nil {
+		// It is admissible for Pods to fail with concurrentModification errors
+		// when stopping sideCars. The reconciler can retry on this.
+		if isConcurrentModificationError(err) {
+			return controller.NewRequeueAfter(time.Second)
+		}
 		logger.Errorf("Error stopping sidecars for TaskRun %q: %v", tr.Name, err)
 		tr.Status.MarkResourceFailed(v1.TaskRunReasonStopSidecarFailed, err)
 	}
@@ -1014,6 +1019,28 @@ func isResourceQuotaConflictError(err error) bool {
 	return k8ErrStatus.Details != nil && k8ErrStatus.Details.Kind == "resourcequotas"
 }
 
+const (
+	// TODO(#7466) Currently this appears as a local constant due to upstream dependencies bump blocker.
+	// This shall reference to k8s.io/apiserver/pkg/registry/generic/registry.OptimisticLockErrorMsg
+	// once #7464 is unblocked.
+	optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
+)
+
+// isConcurrentModificationError determines whether it is a concurrent
+// modification  error depending on its error type and error message.
+func isConcurrentModificationError(err error) bool {
+	if !k8serrors.IsConflict(err) {
+		return false
+	}
+
+	se, ok := err.(*k8serrors.StatusError)
+	if !ok {
+		return false
+	}
+
+	return strings.Contains(se.Error(), optimisticLockErrorMsg)
+}
+
 // retryTaskRun archives taskRun.Status to taskRun.Status.RetriesStatus, and set
 // taskRun status to Unknown with Reason v1.TaskRunReasonToBeRetried.
 func retryTaskRun(tr *v1.TaskRun, message string) {