From d2597301f1b6873dbf5a0d04b58eae632de3c7dc Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Tue, 12 Mar 2024 11:37:58 -0400
Subject: [PATCH 01/22] Ensure that binary logs for PITR are restored to a
 shared location

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/mysqlctld.go    | 3 ++-
 pkg/operator/vttablet/pod.go          | 5 +++++
 pkg/operator/vttablet/vtbackup_pod.go | 9 +++++++--
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/pkg/operator/vttablet/mysqlctld.go b/pkg/operator/vttablet/mysqlctld.go
index 442aaf6d..4860aa20 100644
--- a/pkg/operator/vttablet/mysqlctld.go
+++ b/pkg/operator/vttablet/mysqlctld.go
@@ -29,6 +29,7 @@ const (
 	vtRootInitScript = `set -ex
 mkdir -p /mnt/vt/bin
 cp --no-clobber /vt/bin/mysqlctld /mnt/vt/bin/
+cp --no-clobber $(command -v mysqlbinlog) /mnt/vt/bin/ || true
 mkdir -p /mnt/vt/config
 if [[ -d /vt/config/mycnf ]]; then
   cp --no-clobber -R /vt/config/mycnf /mnt/vt/config/
@@ -68,7 +69,7 @@ func init() {
 
 		securityContext := &corev1.SecurityContext{}
 		if planetscalev2.DefaultVitessRunAsUser >= 0 {
-			securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser)
+			securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser)
 		}
 
 		// Use an init container to copy only the files we need from the Vitess image.
diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go
index 9dcc25f1..37b5f555 100644
--- a/pkg/operator/vttablet/pod.go
+++ b/pkg/operator/vttablet/pod.go
@@ -168,6 +168,11 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 	var mysqldExporterContainer *corev1.Container
 
 	mysqlctldAllFlags := mysqlctldFlags.Get(spec)
+	// Ensure that binary logs are restored to/from a location that all containers
+	// in the pod can access if no location was explicitly provided.
+	if _, ok := mysqlctldAllFlags["builtinbackup-incremental-restore-path"]; !ok {
+		mysqlctldAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
+	}
 	mysql.UpdateMySQLServerVersion(mysqlctldAllFlags, spec.Images.Mysqld.Image())
 
 	if spec.Mysqld != nil {
diff --git a/pkg/operator/vttablet/vtbackup_pod.go b/pkg/operator/vttablet/vtbackup_pod.go
index 5823cdb1..1f5a5ec9 100644
--- a/pkg/operator/vttablet/vtbackup_pod.go
+++ b/pkg/operator/vttablet/vtbackup_pod.go
@@ -122,11 +122,11 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri
 
 	podSecurityContext := &corev1.PodSecurityContext{}
 	if planetscalev2.DefaultVitessFSGroup >= 0 {
-		podSecurityContext.FSGroup = pointer.Int64Ptr(planetscalev2.DefaultVitessFSGroup)
+		podSecurityContext.FSGroup = pointer.Int64(planetscalev2.DefaultVitessFSGroup)
 	}
 	securityContext := &corev1.SecurityContext{}
 	if planetscalev2.DefaultVitessRunAsUser >= 0 {
-		securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser)
+		securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser)
 	}
 
 	var containerResources corev1.ResourceRequirements
@@ -134,6 +134,11 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri
 	update.ResourceRequirements(&containerResources, &tabletSpec.Mysqld.Resources)
 
 	vtbackupAllFlags := vtbackupFlags.Get(backupSpec)
+	// Ensure that binary logs are restored to/from a location that all containers
+	// in the pod can access if no location was explicitly provided.
+	if _, ok := vtbackupAllFlags["builtinbackup-incremental-restore-path"]; !ok {
+		vtbackupAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
+	}
 	mysql.UpdateMySQLServerVersion(vtbackupAllFlags, mysqldImage)
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{

From e2e5e8b1e749b6f550ee72e77dce2f8ef326408d Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Fri, 15 Mar 2024 23:59:36 -0400
Subject: [PATCH 02/22] Add ability to specify flags for mysqlctld

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 docs/api/index.html                           | 43 +++++++++++++++++++
 pkg/apis/planetscale/v2/vitessshard_types.go  | 13 ++++++
 .../planetscale/v2/zz_generated.deepcopy.go   | 23 ++++++++++
 3 files changed, 79 insertions(+)

diff --git a/docs/api/index.html b/docs/api/index.html
index cbf2b048..2163c9b9 100644
--- a/docs/api/index.html
+++ b/docs/api/index.html
@@ -1410,6 +1410,49 @@ <h3 id="planetscale.com/v2.LockserverStatus">LockserverStatus
 </tr>
 </tbody>
 </table>
+<h3 id="planetscale.com/v2.MysqlctldSpec">MysqlctldSpec
+</h3>
+<p>
+<p>MysqlctldSpec configures the local mysqlctld gRPC server within a tablet.</p>
+</p>
+<table class="table table-striped">
+<thead class="thead-dark">
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>resources</code></br>
+<em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements
+</a>
+</em>
+</td>
+<td>
+<p>Resources specify the compute resources to allocate for just the MySQL Control Daemon.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>extraFlags</code></br>
+<em>
+map[string]string
+</em>
+</td>
+<td>
+<p>ExtraFlags can optionally be used to override default flags set by the
+operator, or pass additional flags to mysqlctld. All entries must be
+key-value string pairs of the form &ldquo;flag&rdquo;: &ldquo;value&rdquo;. The flag name should
+not have any prefix (just &ldquo;flag&rdquo;, not &ldquo;-flag&rdquo;). To set a boolean flag,
+set the string value to either &ldquo;true&rdquo; or &ldquo;false&rdquo;.</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="planetscale.com/v2.MysqldExporterSpec">MysqldExporterSpec
 </h3>
 <p>
diff --git a/pkg/apis/planetscale/v2/vitessshard_types.go b/pkg/apis/planetscale/v2/vitessshard_types.go
index 615e423f..c243484c 100644
--- a/pkg/apis/planetscale/v2/vitessshard_types.go
+++ b/pkg/apis/planetscale/v2/vitessshard_types.go
@@ -315,6 +315,19 @@ type MysqldSpec struct {
 	ConfigOverrides string `json:"configOverrides,omitempty"`
 }
 
+// MysqlctldSpec configures the local mysqlctld gRPC server within a tablet.
+type MysqlctldSpec struct {
+	// Resources specify the compute resources to allocate for just the MySQL Control Daemon.
+	Resources corev1.ResourceRequirements `json:"resources"`
+
+	// ExtraFlags can optionally be used to override default flags set by the
+	// operator, or pass additional flags to mysqlctld. All entries must be
+	// key-value string pairs of the form "flag": "value". The flag name should
+	// not have any prefix (just "flag", not "-flag"). To set a boolean flag,
+	// set the string value to either "true" or "false".
+	ExtraFlags map[string]string `json:"extraFlags,omitempty"`
+}
+
 // MysqldExporterSpec configures the local MySQL exporter within a tablet.
 type MysqldExporterSpec struct {
 	// Resources specify the compute resources to allocate for just the MySQL Exporter.
diff --git a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go
index 4243ff17..e30236a6 100644
--- a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go
+++ b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go
@@ -390,6 +390,29 @@ func (in *LockserverStatus) DeepCopy() *LockserverStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MysqlctldSpec) DeepCopyInto(out *MysqlctldSpec) {
+	*out = *in
+	in.Resources.DeepCopyInto(&out.Resources)
+	if in.ExtraFlags != nil {
+		in, out := &in.ExtraFlags, &out.ExtraFlags
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MysqlctldSpec.
+func (in *MysqlctldSpec) DeepCopy() *MysqlctldSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(MysqlctldSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MysqldExporterSpec) DeepCopyInto(out *MysqldExporterSpec) {
 	*out = *in

From 10a95247108c9a5a2d065642f38cccb45651d4a9 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Tue, 26 Mar 2024 13:35:11 -0400
Subject: [PATCH 03/22] It's a tablet flag not a mysqlctld one

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/pod.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go
index 37b5f555..de4de832 100644
--- a/pkg/operator/vttablet/pod.go
+++ b/pkg/operator/vttablet/pod.go
@@ -170,8 +170,8 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 	mysqlctldAllFlags := mysqlctldFlags.Get(spec)
 	// Ensure that binary logs are restored to/from a location that all containers
 	// in the pod can access if no location was explicitly provided.
-	if _, ok := mysqlctldAllFlags["builtinbackup-incremental-restore-path"]; !ok {
-		mysqlctldAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
+	if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok {
+		vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
 	}
 	mysql.UpdateMySQLServerVersion(mysqlctldAllFlags, spec.Images.Mysqld.Image())
 

From 48ba49f9ba1a86e56166a1f8c55c516d1e929c80 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Tue, 26 Mar 2024 20:25:42 -0400
Subject: [PATCH 04/22] Move the flag handling code

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/pod.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go
index de4de832..4f41f9e9 100644
--- a/pkg/operator/vttablet/pod.go
+++ b/pkg/operator/vttablet/pod.go
@@ -87,6 +87,11 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 		key = strings.TrimLeft(key, "-")
 		vttabletAllFlags[key] = value
 	}
+	// Ensure that binary logs are restored to/from a location that all containers
+	// in the pod can access if no location was explicitly provided.
+	if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok {
+		vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
+	}
 	mysql.UpdateMySQLServerVersion(vttabletAllFlags, spec.Images.Mysqld.Image())
 
 	// Compute all operator-generated env vars first.
@@ -168,11 +173,6 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 	var mysqldExporterContainer *corev1.Container
 
 	mysqlctldAllFlags := mysqlctldFlags.Get(spec)
-	// Ensure that binary logs are restored to/from a location that all containers
-	// in the pod can access if no location was explicitly provided.
-	if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok {
-		vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
-	}
 	mysql.UpdateMySQLServerVersion(mysqlctldAllFlags, spec.Images.Mysqld.Image())
 
 	if spec.Mysqld != nil {

From 411993c0eed5e72587ad7975749e74d50141c40c Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 27 Mar 2024 11:30:36 -0400
Subject: [PATCH 05/22] Only set flag on 2.12+

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/pod.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go
index 4f41f9e9..17ea992e 100644
--- a/pkg/operator/vttablet/pod.go
+++ b/pkg/operator/vttablet/pod.go
@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
 	"planetscale.dev/vitess-operator/pkg/operator/mysql"
+	"planetscale.dev/vitess-operator/version"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/topo/topoproto"
@@ -90,7 +91,14 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 	// Ensure that binary logs are restored to/from a location that all containers
 	// in the pod can access if no location was explicitly provided.
 	if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok {
-		vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
+		parts := strings.Split(version.Version, ".")
+		if len(parts) > 1 {
+			major, _ := strconv.Atoi(parts[1])
+			minor, _ := strconv.Atoi(parts[2])
+			if major >= 2 && minor >= 12 {
+				vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
+			}
+		}
 	}
 	mysql.UpdateMySQLServerVersion(vttabletAllFlags, spec.Images.Mysqld.Image())
 
@@ -113,7 +121,7 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 
 	securityContext := &corev1.SecurityContext{}
 	if planetscalev2.DefaultVitessRunAsUser >= 0 {
-		securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser)
+		securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser)
 	}
 
 	vttabletLifecycle := &spec.Vttablet.Lifecycle
@@ -320,13 +328,13 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 		obj.Spec.SecurityContext = &corev1.PodSecurityContext{}
 	}
 	if planetscalev2.DefaultVitessFSGroup >= 0 {
-		obj.Spec.SecurityContext.FSGroup = pointer.Int64Ptr(planetscalev2.DefaultVitessFSGroup)
+		obj.Spec.SecurityContext.FSGroup = pointer.Int64(planetscalev2.DefaultVitessFSGroup)
 	}
 
 	if spec.Vttablet.TerminationGracePeriodSeconds != nil {
 		obj.Spec.TerminationGracePeriodSeconds = spec.Vttablet.TerminationGracePeriodSeconds
 	} else {
-		obj.Spec.TerminationGracePeriodSeconds = pointer.Int64Ptr(defaultTerminationGracePeriodSeconds)
+		obj.Spec.TerminationGracePeriodSeconds = pointer.Int64(defaultTerminationGracePeriodSeconds)
 	}
 
 	// In both the case of the user injecting their own affinity and the default, we

From 79bf28de627fbe62655a334cfd82643a39cb7954 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 27 Mar 2024 12:39:15 -0400
Subject: [PATCH 06/22] Add PITR restore to backup test

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 test/endtoend/backup_restore_test.sh |  1 +
 test/endtoend/utils.sh               | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/test/endtoend/backup_restore_test.sh b/test/endtoend/backup_restore_test.sh
index 1654c35b..dfc0c8b0 100755
--- a/test/endtoend/backup_restore_test.sh
+++ b/test/endtoend/backup_restore_test.sh
@@ -107,6 +107,7 @@ verifyVtGateVersion "20.0.0"
 checkSemiSyncSetup
 takeBackup "commerce/-"
 verifyListBackupsOutput
+restoreBackup "$(vtctldclient GetTablets --keyspace commerce --tablet-type replica --shard '-' | head -1 | awk '{print $1}')"
 takedownShard
 resurrectShard
 checkSemiSyncSetup
diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index 259e0abd..51c93189 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -104,6 +104,20 @@ function takeBackup() {
   exit 1
 }
 
+# restoreBackup:
+# $1: tablet alias for which the backup needs to be restored
+function restoreBackup() {
+  tabletAlias=$1
+
+  # Issue the PITR restore command to vtctldclient.
+  vtctldclient RestoreFromBackup --restore-to-timestamp $(date --rfc-3339=seconds) "${tabletAlias}"
+
+  if [[ $? -ne 0 ]]; then
+    echo "Restore failed"
+    exit 1
+  fi
+}
+
 function verifyListBackupsOutput() {
   backupCount=$(kubectl get vtb --no-headers | wc -l)
   for i in {1..600} ; do

From 8277844ec355699619fa6805898af3b7574d3a64 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 27 Mar 2024 13:00:48 -0400
Subject: [PATCH 07/22] Use recent vtctldclient

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 tools/get-e2e-test-deps.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/get-e2e-test-deps.sh b/tools/get-e2e-test-deps.sh
index 2501650a..528b0911 100755
--- a/tools/get-e2e-test-deps.sh
+++ b/tools/get-e2e-test-deps.sh
@@ -40,8 +40,9 @@ fi
 if ! command -v vtctldclient &> /dev/null
 then
   echo "Downloading vtctldclient..."
-  version=16.0.1
-  file=vitess-${version}-d1ba625.tar.gz
+  version="19.0.1"
+  hash="3a43ab8"
+  file="vitess-${version}-${hash}.tar.gz"
   wget https://github.com/vitessio/vitess/releases/download/v${version}/${file}
   tar -xzf ${file}
   cd ${file/.tar.gz/}

From 9d2b978eaf61864f788d1134efccb908b02d8fa0 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 27 Mar 2024 13:15:26 -0400
Subject: [PATCH 08/22] Use different date command

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 test/endtoend/utils.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index 51c93189..f60cff27 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -108,9 +108,13 @@ function takeBackup() {
 # $1: tablet alias for which the backup needs to be restored
 function restoreBackup() {
   tabletAlias=$1
+  if [[ -z "$tabletAlias" ]]; then
+    echo "Tablet alias not provided as restore target"
+    exit 1
+  fi
 
   # Issue the PITR restore command to vtctldclient.
-  vtctldclient RestoreFromBackup --restore-to-timestamp $(date --rfc-3339=seconds) "${tabletAlias}"
+  vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%dT%H:%M:%S") "${tabletAlias}"
 
   if [[ $? -ne 0 ]]; then
     echo "Restore failed"

From db311ed6ba015b82f27ca9697ab1f3411b1f8af7 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 27 Mar 2024 13:44:58 -0400
Subject: [PATCH 09/22] Use different date format

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 test/endtoend/utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index f60cff27..c1b1bb9a 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -114,7 +114,7 @@ function restoreBackup() {
   fi
 
   # Issue the PITR restore command to vtctldclient.
-  vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%dT%H:%M:%S") "${tabletAlias}"
+  vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%d.%H%M%S") "${tabletAlias}"
 
   if [[ $? -ne 0 ]]; then
     echo "Restore failed"

From 63abf801f23cdbfb749d683a66202fd980ba46e5 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 27 Mar 2024 16:48:06 -0400
Subject: [PATCH 10/22] Test incremental backup and restore

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/pod.go |  8 +++--
 test/endtoend/utils.sh       | 60 +++++++++++++++++++++++++++++-------
 2 files changed, 54 insertions(+), 14 deletions(-)

diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go
index 17ea992e..932b1b08 100644
--- a/pkg/operator/vttablet/pod.go
+++ b/pkg/operator/vttablet/pod.go
@@ -91,10 +91,12 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 	// Ensure that binary logs are restored to/from a location that all containers
 	// in the pod can access if no location was explicitly provided.
 	if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok {
+		// This flag only exists in 2.12.0 and later as it includes vitess
+		// 20.0 as a dependency.
 		parts := strings.Split(version.Version, ".")
-		if len(parts) > 1 {
-			major, _ := strconv.Atoi(parts[1])
-			minor, _ := strconv.Atoi(parts[2])
+		if len(parts) > 0 {
+			major, _ := strconv.Atoi(parts[0])
+			minor, _ := strconv.Atoi(parts[1])
 			if major >= 2 && minor >= 12 {
 				vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
 			}
diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index c1b1bb9a..5883efc5 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -83,24 +83,47 @@ function removeBackupFiles() {
 
 # takeBackup:
 # $1: keyspace-shard for which the backup needs to be taken
+declare INCREMENTAL_RESTORE_TIMESTAMP=""
 function takeBackup() {
   keyspaceShard=$1
   initialBackupCount=$(kubectl get vtb --no-headers | wc -l)
   finalBackupCount=$((initialBackupCount+1))
 
-  # issue the backupShard command to vtctldclient
-  vtctldclient BackupShard "$keyspaceShard"
+  # Issue the BackupShard command to vtctldclient.
+  vtctldclient BackupShard "${keyspaceShard}"
 
   for i in {1..600} ; do
     out=$(kubectl get vtb --no-headers | wc -l)
-    echo "$out" | grep "$finalBackupCount" > /dev/null 2>&1
-    if [ $? -eq 0 ]; then
-      echo "Backup created"
+    if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then
+      echo "Full backup created"
+      break
+    fi
+    sleep 3
+  done
+
+  sleep 10
+
+  # Now perform an incremental backup.
+  insertWithRetry
+  INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
+  sleep 1
+  insertWithRetry
+
+  sleep 10
+
+  vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}"
+  let finalBackupCount=${finalBackupCount}+1
+
+  for i in {1..600} ; do
+    out=$(kubectl get vtb --no-headers | wc -l)
+    if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then
+      echo "Incremental backup created"
       return 0
     fi
     sleep 3
   done
-  echo -e "ERROR: Backup not created - $out. $backupCount backups expected."
+
+  echo -e "ERROR: Backups not created - ${out}. ${backupCount} backups expected."
   exit 1
 }
 
@@ -108,18 +131,33 @@ function takeBackup() {
 # $1: tablet alias for which the backup needs to be restored
 function restoreBackup() {
   tabletAlias=$1
-  if [[ -z "$tabletAlias" ]]; then
+  if [[ -z "${tabletAlias}" ]]; then
     echo "Tablet alias not provided as restore target"
     exit 1
   fi
 
   # Issue the PITR restore command to vtctldclient.
-  vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%d.%H%M%S") "${tabletAlias}"
-
-  if [[ $? -ne 0 ]]; then
-    echo "Restore failed"
+  # This should restore the last full backup, followed by applying the
+  # binary logs to reach the desired timestamp.
+  if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then
+    echo "ERROR: failed to perform incremental restore"
     exit 1
   fi
+
+  cell="${tabletAlias%-*}"
+  uid="${tabletAlias##*-}"
+
+  for i in {1..600} ; do
+    out=$(kubectl get pods --no-headers -l "planetscale.com/cell=${cell},planetscale.com/tablet-uid=${uid}" | grep "Running" | wc -l)
+    if echo "$out" | grep -c "1" >/dev/null; then
+      echo "Tablet ${tabletAlias} restore complete"
+      return 0
+    fi
+    sleep 3
+  done
+
+  echo -e "ERROR: restored tablet ${tabletAlias} did not become healthy after the restore."
+  exit 1
 }
 
 function verifyListBackupsOutput() {

From 425b7255de2d035286e8dd7bd94518561f25b543 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 27 Mar 2024 23:26:48 -0400
Subject: [PATCH 11/22] WiP

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/pod.go | 18 ++++++++++--------
 test/endtoend/utils.sh       |  7 ++++++-
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go
index 932b1b08..f3e2fa01 100644
--- a/pkg/operator/vttablet/pod.go
+++ b/pkg/operator/vttablet/pod.go
@@ -26,7 +26,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
 	"planetscale.dev/vitess-operator/pkg/operator/mysql"
-	"planetscale.dev/vitess-operator/version"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/topo/topoproto"
@@ -93,14 +92,17 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 	if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok {
 		// This flag only exists in 2.12.0 and later as it includes vitess
 		// 20.0 as a dependency.
-		parts := strings.Split(version.Version, ".")
-		if len(parts) > 0 {
-			major, _ := strconv.Atoi(parts[0])
-			minor, _ := strconv.Atoi(parts[1])
-			if major >= 2 && minor >= 12 {
-				vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
+		/*
+			parts := strings.Split(version.Version, ".")
+			if len(parts) > 0 {
+				major, _ := strconv.Atoi(parts[0])
+				minor, _ := strconv.Atoi(parts[1])
+				if major >= 2 && minor >= 12 {
+					vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
+				}
 			}
-		}
+		*/
+		vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
 	}
 	mysql.UpdateMySQLServerVersion(vttabletAllFlags, spec.Images.Mysqld.Image())
 
diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index 5883efc5..a70100e7 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -103,17 +103,22 @@ function takeBackup() {
 
   sleep 10
 
+  mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed"
+
   # Now perform an incremental backup.
   insertWithRetry
   INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
   sleep 1
-  insertWithRetry
+
+  mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed"
 
   sleep 10
 
   vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}"
   let finalBackupCount=${finalBackupCount}+1
 
+  mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed"
+
   for i in {1..600} ; do
     out=$(kubectl get vtb --no-headers | wc -l)
     if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then

From 75c701d8da4670b53fce7c5b9c27a01d3f0f3240 Mon Sep 17 00:00:00 2001
From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
Date: Thu, 28 Mar 2024 08:06:58 +0200
Subject: [PATCH 12/22] attempt to fix tests

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
---
 test/endtoend/utils.sh | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index a70100e7..7ec9913b 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -101,24 +101,16 @@ function takeBackup() {
     sleep 3
   done
 
-  sleep 10
-
-  mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed"
-
   # Now perform an incremental backup.
   insertWithRetry
+  sleep 2
   INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
-  sleep 1
-
-  mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed"
-
-  sleep 10
+  sleep 2
+  insertWithRetry
 
   vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}"
   let finalBackupCount=${finalBackupCount}+1
 
-  mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed"
-
   for i in {1..600} ; do
     out=$(kubectl get vtb --no-headers | wc -l)
     if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then

From f09d7be7a0d4a6ae67c5a6b96c3b15fbeec01ace Mon Sep 17 00:00:00 2001
From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
Date: Thu, 28 Mar 2024 08:26:21 +0200
Subject: [PATCH 13/22] more debug output

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
---
 test/endtoend/utils.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index 7ec9913b..f9dbf27e 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -102,11 +102,18 @@ function takeBackup() {
   done
 
   # Now perform an incremental backup.
+  echo "Checking @@gtid_exeuted just after full backup:"
+  mysql -e "select @@gtid_executed;"
   insertWithRetry
+  echo "Checking @@gtid_exeuted after some writes and before taking timestamp:"
+  mysql -e "select @@gtid_executed;"
   sleep 2
   INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
+  echo "Timestamp is $INCREMENTAL_RESTORE_TIMESTAMP"
   sleep 2
   insertWithRetry
+  echo "Checking @@gtid_exeuted jbefore taking incremental backup:"
+  mysql -e "select @@gtid_executed;"
 
   vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}"
   let finalBackupCount=${finalBackupCount}+1
@@ -136,6 +143,7 @@ function restoreBackup() {
   # Issue the PITR restore command to vtctldclient.
   # This should restore the last full backup, followed by applying the
   # binary logs to reach the desired timestamp.
+  echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}"
   if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then
     echo "ERROR: failed to perform incremental restore"
     exit 1

From 77161354e7c35e700153c537e49e4af9f3f5b732 Mon Sep 17 00:00:00 2001
From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
Date: Thu, 28 Mar 2024 08:41:35 +0200
Subject: [PATCH 14/22] more debug info

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
---
 test/endtoend/utils.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index f9dbf27e..6af58a26 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -143,6 +143,7 @@ function restoreBackup() {
   # Issue the PITR restore command to vtctldclient.
   # This should restore the last full backup, followed by applying the
   # binary logs to reach the desired timestamp.
+  find "$VTDATAROOT" -name MANIFEST | xargs cat
   echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}"
   if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then
     echo "ERROR: failed to perform incremental restore"

From 34523599f1222718c0f6d4478640c10c9574934f Mon Sep 17 00:00:00 2001
From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
Date: Thu, 28 Mar 2024 12:36:02 +0200
Subject: [PATCH 15/22] list backups as debug step

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
---
 test/endtoend/utils.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index 6af58a26..386eec43 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -143,7 +143,9 @@ function restoreBackup() {
   # Issue the PITR restore command to vtctldclient.
   # This should restore the last full backup, followed by applying the
   # binary logs to reach the desired timestamp.
-  find "$VTDATAROOT" -name MANIFEST | xargs cat
+  echo "Listing backups"
+  vtctldclient ListBackups "$keyspaceShard"
+  echo "End listing backups"
   echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}"
   if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then
     echo "ERROR: failed to perform incremental restore"

From b1a0cb35be505258ed24c6444cad4b1b701bdec0 Mon Sep 17 00:00:00 2001
From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
Date: Thu, 28 Mar 2024 12:47:16 +0200
Subject: [PATCH 16/22] GetBackups

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
---
 test/endtoend/utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index 386eec43..5c74d0f2 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -144,7 +144,7 @@ function restoreBackup() {
   # This should restore the last full backup, followed by applying the
   # binary logs to reach the desired timestamp.
   echo "Listing backups"
-  vtctldclient ListBackups "$keyspaceShard"
+  vtctldclient GetBackups "$keyspaceShard"
   echo "End listing backups"
   echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}"
   if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then

From 919119b286ec18429fe3113c83fde2bf969b49ab Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Thu, 28 Mar 2024 13:04:48 -0400
Subject: [PATCH 17/22] Get test working locally

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 test/endtoend/utils.sh | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index 5c74d0f2..9888cd9c 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -83,7 +83,7 @@ function removeBackupFiles() {
 
 # takeBackup:
 # $1: keyspace-shard for which the backup needs to be taken
-declare INCREMENTAL_RESTORE_TIMESTAMP=""
+declare INCREMENTAL_RESTORE_POS=""
 function takeBackup() {
   keyspaceShard=$1
   initialBackupCount=$(kubectl get vtb --no-headers | wc -l)
@@ -102,18 +102,13 @@ function takeBackup() {
   done
 
   # Now perform an incremental backup.
-  echo "Checking @@gtid_exeuted just after full backup:"
-  mysql -e "select @@gtid_executed;"
   insertWithRetry
-  echo "Checking @@gtid_exeuted after some writes and before taking timestamp:"
-  mysql -e "select @@gtid_executed;"
+  pos=$(mysql -sN -e "select @@global.gtid_executed")
+  INCREMENTAL_RESTORE_POS="MySQL56/${pos//\\n/}"
   sleep 2
-  INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
-  echo "Timestamp is $INCREMENTAL_RESTORE_TIMESTAMP"
+  echo "Backup position is $INCREMENTAL_RESTORE_POS"
   sleep 2
   insertWithRetry
-  echo "Checking @@gtid_exeuted jbefore taking incremental backup:"
-  mysql -e "select @@gtid_executed;"
 
   vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}"
   let finalBackupCount=${finalBackupCount}+1
@@ -146,8 +141,8 @@ function restoreBackup() {
   echo "Listing backups"
   vtctldclient GetBackups "$keyspaceShard"
   echo "End listing backups"
-  echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}"
-  if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then
+  echo "Restoring tablet ${tabletAlias} to position ${INCREMENTAL_RESTORE_POS}"
+  if ! vtctldclient RestoreFromBackup --restore-to-pos "${INCREMENTAL_RESTORE_POS}" "${tabletAlias}"; then
     echo "ERROR: failed to perform incremental restore"
     exit 1
   fi

From 49b933298063bfc5974389338cae79eb19ec73e7 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 10 Apr 2024 12:17:31 -0400
Subject: [PATCH 18/22] Ignore vtdataroot path created when running local tests

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 76cbd661..f77e5e70 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 build/_output
 build/_test
 /tools/_bin/
+vtdataroot
 
 # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode
 ### Emacs ###

From 2aeb36b8bcdade520950f8dd2ecf31004c351ac6 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Wed, 10 Apr 2024 14:11:53 -0400
Subject: [PATCH 19/22] Fix backup test; use percona image

This includes mysqld (of course) and mysqlbinlog

But it does NOT include xtrabackup

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 test/endtoend/operator/101_initial_cluster_backup.yaml | 2 +-
 tools/get-e2e-test-deps.sh                             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/endtoend/operator/101_initial_cluster_backup.yaml b/test/endtoend/operator/101_initial_cluster_backup.yaml
index cb9f3c80..28deded4 100644
--- a/test/endtoend/operator/101_initial_cluster_backup.yaml
+++ b/test/endtoend/operator/101_initial_cluster_backup.yaml
@@ -21,7 +21,7 @@ spec:
     vtorc: vitess/lite:latest
     vtbackup: vitess/lite:latest
     mysqld:
-      mysql56Compatible: vitess/lite:latest
+      mysql80Compatible: percona/percona-server:8.0.34
     mysqldExporter: prom/mysqld-exporter:v0.11.0
   cells:
   - name: zone1
diff --git a/tools/get-e2e-test-deps.sh b/tools/get-e2e-test-deps.sh
index 528b0911..5805c7a5 100755
--- a/tools/get-e2e-test-deps.sh
+++ b/tools/get-e2e-test-deps.sh
@@ -40,7 +40,7 @@ fi
 if ! command -v vtctldclient &> /dev/null
 then
   echo "Downloading vtctldclient..."
-  version="19.0.1"
+  version="19.0.3"
   hash="3a43ab8"
   file="vitess-${version}-${hash}.tar.gz"
   wget https://github.com/vitessio/vitess/releases/download/v${version}/${file}

From 2f7768ab255717645fb9b629147d55f64d63bc74 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Tue, 30 Apr 2024 22:04:15 -0400
Subject: [PATCH 20/22] Use /vt/bin for mysqlbinlog and add it to PATH

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/env_vars.go  | 7 ++++++-
 pkg/operator/vttablet/mysqlctld.go | 2 +-
 test/endtoend/utils.sh             | 2 ++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/pkg/operator/vttablet/env_vars.go b/pkg/operator/vttablet/env_vars.go
index 3e3189c5..34c1b6da 100644
--- a/pkg/operator/vttablet/env_vars.go
+++ b/pkg/operator/vttablet/env_vars.go
@@ -18,10 +18,11 @@ package vttablet
 
 import (
 	"fmt"
+	"os"
 	"strings"
 
-	"planetscale.dev/vitess-operator/pkg/operator/lazy"
 	corev1 "k8s.io/api/core/v1"
+	"planetscale.dev/vitess-operator/pkg/operator/lazy"
 )
 
 func init() {
@@ -40,6 +41,10 @@ func init() {
 				Name:  "EXTRA_MY_CNF",
 				Value: strings.Join(extraMyCnf.Get(spec), ":"),
 			},
+			// The mysqlbinlog binary is NOT in the vendor mysql container images
+			// so we copy it to /mnt/vt/bin when initializing the mysqld container
+			// so that mysqlctl can find it.
+			{Name: "PATH", Value: fmt.Sprintf("%s:%s", os.Getenv("PATH"), vtBinPath)},
 		}
 	})
 
diff --git a/pkg/operator/vttablet/mysqlctld.go b/pkg/operator/vttablet/mysqlctld.go
index 4860aa20..d204ca06 100644
--- a/pkg/operator/vttablet/mysqlctld.go
+++ b/pkg/operator/vttablet/mysqlctld.go
@@ -29,7 +29,7 @@ const (
 	vtRootInitScript = `set -ex
 mkdir -p /mnt/vt/bin
 cp --no-clobber /vt/bin/mysqlctld /mnt/vt/bin/
-cp --no-clobber $(command -v mysqlbinlog) /mnt/vt/bin/ || true
+cp --no-clobber /usr/bin/mysqlbinlog /mnt/vt/bin/
 mkdir -p /mnt/vt/config
 if [[ -d /vt/config/mycnf ]]; then
   cp --no-clobber -R /vt/config/mycnf /mnt/vt/config/
diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh
index 731bb9d9..88710b58 100644
--- a/test/endtoend/utils.sh
+++ b/test/endtoend/utils.sh
@@ -279,7 +279,9 @@ function waitForKeyspaceToBeServing() {
   nb_of_replica=$3
   for i in {1..600} ; do
     out=$(mysql --table --execute="show vitess_tablets")
+    echo "Serving output: ${out}"
     numtablets=$(echo "$out" | grep -E "$ks(.*)$shard(.*)PRIMARY(.*)SERVING|$ks(.*)$shard(.*)REPLICA(.*)SERVING" | wc -l)
+    echo "Number of serving tablets: ${numtablets}"
     if [[ $numtablets -ge $((nb_of_replica+1)) ]]; then
       echo "Shard $ks/$shard is serving"
       return

From ce28c33207d95a0e3f0843108eb155dd64f3bd3d Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Fri, 12 Jul 2024 14:17:48 -0400
Subject: [PATCH 21/22] Add mysqlbinlog to $VTROOT/bin in backup pods

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/vtbackup_pod.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/operator/vttablet/vtbackup_pod.go b/pkg/operator/vttablet/vtbackup_pod.go
index 1f5a5ec9..4bcb3007 100644
--- a/pkg/operator/vttablet/vtbackup_pod.go
+++ b/pkg/operator/vttablet/vtbackup_pod.go
@@ -35,6 +35,7 @@ const (
 	vtbackupInitScript = `set -ex
 mkdir -p /mnt/vt/bin
 cp --no-clobber /vt/bin/vtbackup /mnt/vt/bin/
+cp --no-clobber /usr/bin/mysqlbinlog /mnt/vt/bin/
 mkdir -p /mnt/vt/config
 if [[ -d /vt/config/mycnf ]]; then
   cp --no-clobber -R /vt/config/mycnf /mnt/vt/config/

From 5e7df2b58f45d9097a64706ac50d498353c29937 Mon Sep 17 00:00:00 2001
From: Matt Lord <mattalord@gmail.com>
Date: Fri, 12 Jul 2024 14:54:50 -0400
Subject: [PATCH 22/22] Testing...

Signed-off-by: Matt Lord <mattalord@gmail.com>
---
 pkg/operator/vttablet/vtbackup_pod.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pkg/operator/vttablet/vtbackup_pod.go b/pkg/operator/vttablet/vtbackup_pod.go
index 4bcb3007..8e8b3a1e 100644
--- a/pkg/operator/vttablet/vtbackup_pod.go
+++ b/pkg/operator/vttablet/vtbackup_pod.go
@@ -116,6 +116,12 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri
 			MountPath: sslCertsPath,
 			SubPath:   "certs",
 		},
+		{
+			Name:      vtRootVolumeName,
+			ReadOnly:  true,
+			MountPath: vtBinPath,
+			SubPath:   "bin",
+		},
 	}
 	volumeMounts = append(volumeMounts, mysqldVolumeMounts.Get(tabletSpec)...)
 	volumeMounts = append(volumeMounts, tabletVolumeMounts.Get(tabletSpec)...)