From 0c0377d2c11f40f5278beeab476f7c781cd6a561 Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 21:32:25 +0200 Subject: [PATCH] [release-17.0] Flakes: empty vtdataroot before starting a new vreplication e2e test (#13803) (#13822) Signed-off-by: Rohit Nayak Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com> --- go/test/endtoend/vreplication/cluster_test.go | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/go/test/endtoend/vreplication/cluster_test.go b/go/test/endtoend/vreplication/cluster_test.go index 12539b778de..1b5581c01e7 100644 --- a/go/test/endtoend/vreplication/cluster_test.go +++ b/go/test/endtoend/vreplication/cluster_test.go @@ -320,6 +320,9 @@ func init() { func NewVitessCluster(t *testing.T, name string, cellNames []string, clusterConfig *ClusterConfig) *VitessCluster { vc := &VitessCluster{Name: name, Cells: make(map[string]*Cell), ClusterConfig: clusterConfig} require.NotNil(t, vc) + + vc.CleanupDataroot(t, true) + topo := cluster.TopoProcessInstance(vc.ClusterConfig.topoPort, vc.ClusterConfig.topoPort+1, vc.ClusterConfig.hostname, "etcd2", "global") require.NotNil(t, topo) @@ -355,6 +358,26 @@ func NewVitessCluster(t *testing.T, name string, cellNames []string, clusterConf return vc } +// CleanupDataroot deletes the vtdataroot directory. Since we run multiple tests sequentially in a single CI test shard, +// we can run out of disk space due to all the leftover artifacts from previous tests. +func (vc *VitessCluster) CleanupDataroot(t *testing.T, recreate bool) { + // This is always set to "true" on GitHub Actions runners: + // https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + ci, ok := os.LookupEnv("CI") + if !ok || strings.ToLower(ci) != "true" { + // Leave the directory in place to support local debugging. + return + } + dir := vc.ClusterConfig.vtdataroot + log.Infof("Deleting vtdataroot %s", dir) + err := os.RemoveAll(dir) + require.NoError(t, err) + if recreate { + err = os.Mkdir(dir, 0700) + require.NoError(t, err) + } +} + // AddKeyspace creates a keyspace with specified shard keys and number of replica/read-only tablets. // You can pass optional key value pairs (opts) if you want conditional behavior. func (vc *VitessCluster) AddKeyspace(t *testing.T, cells []*Cell, ksName string, shards string, vschema string, schema string, numReplicas int, numRdonly int, tabletIDBase int, opts map[string]string) (*Keyspace, error) { @@ -613,7 +636,7 @@ func (vc *VitessCluster) AddCell(t testing.TB, name string) (*Cell, error) { return cell, nil } -func (vc *VitessCluster) teardown(t testing.TB) { +func (vc *VitessCluster) teardown() { for _, cell := range vc.Cells { for _, vtgate := range cell.Vtgates { if err := vtgate.TearDown(); err != nil { @@ -676,13 +699,13 @@ func (vc *VitessCluster) teardown(t testing.TB) { } // TearDown brings down a cluster, deleting processes, removing topo keys -func (vc *VitessCluster) TearDown(t testing.TB) { +func (vc *VitessCluster) TearDown(t *testing.T) { if debugMode { return } done := make(chan bool) go func() { - vc.teardown(t) + vc.teardown() done <- true }() select { @@ -693,6 +716,7 @@ func (vc *VitessCluster) TearDown(t testing.TB) { } // some processes seem to hang around for a bit time.Sleep(5 * time.Second) + vc.CleanupDataroot(t, false) } func (vc *VitessCluster) getVttabletsInKeyspace(t *testing.T, cell *Cell, ksName string, tabletType string) map[string]*cluster.VttabletProcess {