feat: Shard xcuitest suite by testList (#989)

* feat: Shard by items in testListFile

* Add new concurrency option to schema

* Better name

* add test for testList concurrency

* Rename test table attributes for clarity

* Update description of shard for xcuitest

api/saucectl.schema.json

@@ -2233,10 +2233,11 @@
                   "$ref": "#/allOf/1/then/properties/suites/items/properties/smartRetry"
                 },
                 "shard": {
-                  "description": "When shard is configured as concurrency, saucectl automatically splits the tests by concurrency so that they can easily run in parallel.",
+                  "description": "When sharding is configured, saucectl automatically splits the tests (e.g. by testList or concurrency) so that they can easily run in parallel.",
                   "enum": [
                     "",
-                    "concurrency"
+                    "concurrency",
+                    "testList"
                   ]
                 },
                 "testListFile": {

api/v1alpha/framework/xcuitest.schema.json

@@ -244,10 +244,11 @@
             "$ref": "../subschema/common.schema.json#/definitions/smartRetry"
           },
           "shard": {
-            "description": "When shard is configured as concurrency, saucectl automatically splits the tests by concurrency so that they can easily run in parallel.",
+            "description": "When sharding is configured, saucectl automatically splits the tests (e.g. by testList or concurrency) so that they can easily run in parallel.",
             "enum": [
               "",
-              "concurrency"
+              "concurrency",
+              "testList"
             ]
           },
           "testListFile": {

internal/xcuitest/config.go

@@ -309,22 +309,28 @@ func SortByHistory(suites []Suite, history insights.JobHistory) []Suite {
 func ShardSuites(p *Project) error {
 	var suites []Suite
 	for _, s := range p.Suites {
-		if s.Shard != "concurrency" {
+		if s.Shard == "concurrency" {
+			shardedSuites, err := shardByConcurrency(s, p.Sauce.Concurrency)
+			if err != nil {
+				return fmt.Errorf("failed to get tests from testListFile(%q): %v", s.TestListFile, err)
+			}
+			suites = append(suites, shardedSuites...)
+		} else if s.Shard == "testList" {
+			shardedSuites, err := shardByTestList(s)
+			if err != nil {
+				return fmt.Errorf("failed to get tests from testListFile(%q): %v", s.TestListFile, err)
+			}
+			suites = append(suites, shardedSuites...)
+		} else {
 			suites = append(suites, s)
-			continue
-		}
-		shardedSuites, err := getShardedSuites(s, p.Sauce.Concurrency)
-		if err != nil {
-			return fmt.Errorf("failed to get tests from testListFile(%q): %v", s.TestListFile, err)
 		}
-		suites = append(suites, shardedSuites...)
 	}
 	p.Suites = suites
 
 	return nil
 }
 
-func getShardedSuites(suite Suite, ccy int) ([]Suite, error) {
+func shardByConcurrency(suite Suite, ccy int) ([]Suite, error) {
 	readFile, err := os.Open(suite.TestListFile)
 	if err != nil {
 		return nil, err
@@ -356,6 +362,36 @@ func getShardedSuites(suite Suite, ccy int) ([]Suite, error) {
 	return suites, nil
 }
 
+func shardByTestList(suite Suite) ([]Suite, error) {
+	readFile, err := os.Open(suite.TestListFile)
+	if err != nil {
+		return nil, err
+	}
+	defer readFile.Close()
+
+	fileScanner := bufio.NewScanner(readFile)
+	fileScanner.Split(bufio.ScanLines)
+	var tests []string
+	for fileScanner.Scan() {
+		text := strings.TrimSpace(fileScanner.Text())
+		if text == "" {
+			continue
+		}
+		tests = append(tests, text)
+	}
+	if len(tests) == 0 {
+		return nil, errors.New("empty file")
+	}
+	var suites []Suite
+	for _, t := range tests {
+		currSuite := suite
+		currSuite.Name = fmt.Sprintf("%s - %s", suite.Name, t)
+		currSuite.TestOptions.Class = []string{t}
+		suites = append(suites, currSuite)
+	}
+	return suites, nil
+}
+
 func GetShardTypes(suites []Suite) []string {
 	var set = map[string]bool{}
 	for _, s := range suites {

internal/xcuitest/config_test.go

@@ -534,12 +534,12 @@ func TestXCUITest_SortByHistory(t *testing.T) {
 
 func TestXCUITest_ShardSuites(t *testing.T) {
 	testCases := []struct {
-		name          string
-		project       Project
-		content       string
-		configEnabled bool
-		expSuites     []Suite
-		expErr        bool
+		name              string
+		project           Project
+		testListContent   string
+		needsTestListFile bool
+		expSuites         []Suite
+		expErr            bool
 	}{
 		{
 			name: "should keep original test options when sharding is disabled",
@@ -575,8 +575,8 @@ func TestXCUITest_ShardSuites(t *testing.T) {
 					},
 				},
 			},
-			content:       "test1\ntest2\n",
-			configEnabled: true,
+			testListContent:   "test1\ntest2\n",
+			needsTestListFile: true,
 			expSuites: []Suite{
 				{
 					Name: "sharding test - 1/2",
@@ -592,6 +592,36 @@ func TestXCUITest_ShardSuites(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "shards suite by testList",
+			project: Project{
+				Sauce: config.SauceConfig{
+					Concurrency: 1,
+				},
+				Suites: []Suite{
+					{
+						Name:  "sharding test",
+						Shard: "testList",
+					},
+				},
+			},
+			testListContent:   "test1\ntest2\n",
+			needsTestListFile: true,
+			expSuites: []Suite{
+				{
+					Name: "sharding test - test1",
+					TestOptions: TestOptions{
+						Class: []string{"test1"},
+					},
+				},
+				{
+					Name: "sharding test - test2",
+					TestOptions: TestOptions{
+						Class: []string{"test2"},
+					},
+				},
+			},
+		},
 		{
 			name: "should ignore empty lines and spaces in testListFile when sharding is enabled",
 			project: Project{
@@ -605,8 +635,8 @@ func TestXCUITest_ShardSuites(t *testing.T) {
 					},
 				},
 			},
-			content:       "   test1\t\n\ntest2\t\n\n",
-			configEnabled: true,
+			testListContent:   "   test1\t\n\ntest2\t\n\n",
+			needsTestListFile: true,
 			expSuites: []Suite{
 				{
 					Name: "sharding test - 1/2",
@@ -638,7 +668,7 @@ func TestXCUITest_ShardSuites(t *testing.T) {
 					},
 				},
 			},
-			configEnabled: false,
+			needsTestListFile: false,
 			expSuites: []Suite{
 				{
 					Name: "sharding test",
@@ -665,8 +695,8 @@ func TestXCUITest_ShardSuites(t *testing.T) {
 					},
 				},
 			},
-			configEnabled: true,
-			content:       "",
+			needsTestListFile: true,
+			testListContent:   "",
 			expSuites: []Suite{
 				{
 					Name: "sharding test",
@@ -682,17 +712,19 @@ func TestXCUITest_ShardSuites(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			var testListFile string
-			if tc.configEnabled {
-				testListFile = createTestListFile(t, tc.content)
+			if tc.needsTestListFile {
+				testListFile = createTestListFile(t, tc.testListContent)
 				tc.project.Suites[0].TestListFile = testListFile
 			}
 			err := ShardSuites(&tc.project)
 			if err != nil {
 				assert.True(t, tc.expErr)
 			}
 			for i, s := range tc.project.Suites {
-				assert.True(t, cmp.Equal(s.TestOptions, tc.expSuites[i].TestOptions))
-				assert.True(t, cmp.Equal(s.Name, tc.expSuites[i].Name))
+				if diff := cmp.Diff(tc.expSuites[i].TestOptions, s.TestOptions); diff != "" {
+					t.Errorf("shard by testList error (-want +got): %s", diff)
+				}
+				assert.Equal(t, s.Name, tc.expSuites[i].Name)
 			}
 		})
 	}