Add make target for doc automation tests; create a separate folder fo…

…r doc automation tests; add documentation for helper functions

Makefile

@@ -211,4 +211,9 @@ test-integration: test-integration-no-cluster test-integration-cluster
 
 .PHONY: test-e2e
 test-e2e:
-	$(RUN_GINKGO) $(GINKGO_FLAGS) --junit-report="test-e2e.xml"  tests/e2escenarios
+	$(RUN_GINKGO) $(GINKGO_FLAGS) --junit-report="test-e2e.xml"  tests/e2escenarios
+
+
+.PHONY: test-doc-automation
+test-doc-automation:
+	$(RUN_GINKGO) $(GINKGO_FLAGS) --junit-report="test-doc-automation.xml"  tests/documentation

tests/integration/documentation/doc_command_reference_init_test.go → tests/documentation/doc_command_reference_init_test.go

@@ -1,4 +1,4 @@
-package integration
+package docautomation
 
 import (
 	"fmt"
@@ -29,7 +29,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 	Context("Interactive Mode", func() {
 		It("Empty directory", func() {
 			args := []string{"odo", "init"}
-			out, err := helper.RunInteractive(args, nil, func(ctx helper.InteractiveContext) {
+			out, err := helper.RunInteractive(args, []string{"ODO_LOG_LEVEL=0"}, func(ctx helper.InteractiveContext) {
 				helper.ExpectString(ctx, "Select language")
 				helper.SendLine(ctx, "Java")
 
@@ -48,7 +48,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 				helper.ExpectString(ctx, "Your new component 'my-java-maven-app' is ready in the current directory")
 			})
 			Expect(err).To(BeNil())
-			got := fmt.Sprintf(outputStringFormat, " init", helper.CleanStringOfSpinner(out))
+			got := fmt.Sprintf(outputStringFormat, " init", helper.StripSpinner(out))
 			got = helper.ReplaceTimeInString(got, "[1s]")
 			want := helper.GetMDXContent(filepath.Join(commonPath, "interactive_mode_empty_directory_output.mdx"))
 			want = helper.ReplaceTimeInString(got, "[1s]")
@@ -63,7 +63,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 
 			It("Directory with sources", func() {
 				args := []string{"odo", "init"}
-				out, err := helper.RunInteractive(args, nil, func(ctx helper.InteractiveContext) {
+				out, err := helper.RunInteractive(args, []string{"ODO_LOG_LEVEL=0"}, func(ctx helper.InteractiveContext) {
 					helper.ExpectString(ctx, "Is this correct?")
 					helper.SendLine(ctx, "")
 
@@ -79,7 +79,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 				})
 				Expect(err).To(BeNil())
 				got := helper.StripAnsi(out)
-				got = fmt.Sprintf(outputStringFormat, "init", helper.CleanStringOfSpinner(got))
+				got = fmt.Sprintf(outputStringFormat, "init", helper.StripSpinner(got))
 				got = strings.Replace(got, "? Is this correct? (Y/n) ", "", 1)
 				got = strings.Replace(got, "? Is this correct? Yes", "? Is this correct? Yes\n✓  Downloading devfile \"nodejs:2.1.1\" from registry \"DefaultDevfileRegistry\" [3s]", 1)
 				got = strings.Replace(got, "\n? Select container for which you want to change configuration?  [Use arrows to move, type to filter]\nruntime\n> NONE - configuration is correct\n", "", 1)
@@ -95,7 +95,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 		It("Fetch Devfile of a specific version", func() {
 			args := []string{"init", "--devfile", "go", "--name", "my-go-app", "--devfile-version", "2.0.0"}
 			out := helper.Cmd("odo", args...).ShouldPass().Out()
-			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.CleanStringOfSpinner(out))
+			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(out))
 			got = helper.ReplaceTimeInString(got, "[3s]")
 			want := helper.GetMDXContent(filepath.Join(commonPath, "versioned_devfile_output.mdx"))
 			diff := cmp.Diff(want, got)
@@ -105,7 +105,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 		It("Fetch Devfile of the latest version", func() {
 			args := []string{"init", "--devfile", "go", "--name", "my-go-app", "--devfile-version", "latest"}
 			out := helper.Cmd("odo", args...).ShouldPass().Out()
-			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.CleanStringOfSpinner(out))
+			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(out))
 			got = helper.ReplaceTimeInString(got, "[4s]")
 			want := helper.GetMDXContent(filepath.Join(commonPath, "latest_versioned_devfile_output.mdx"))
 			diff := cmp.Diff(want, got)
@@ -115,7 +115,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 		It("Fetch Devfile from a URL", func() {
 			args := []string{"init", "--devfile-path", "https://registry.devfile.io/devfiles/nodejs-angular", "--name", "my-nodejs-app", "--starter", "nodejs-angular-starter"}
 			out := helper.Cmd("odo", args...).ShouldPass().Out()
-			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.CleanStringOfSpinner(out))
+			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(out))
 			got = helper.ReplaceTimeInString(got, "[4s]")
 			want := helper.GetMDXContent(filepath.Join(commonPath, "devfile_from_url_output.mdx"))
 			want = helper.ReplaceTimeInString(want, "[4s]")
@@ -153,7 +153,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 					args := []string{"preference", "view"}
 					out := helper.Cmd("odo", args...).ShouldPass().Out()
 					got := helper.StripAnsi(out)
-					got = fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.CleanStringOfSpinner(got))
+					got = fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(got))
 					got = removePreferenceKeys(got)
 					want := helper.GetMDXContent(filepath.Join(commonPath, "registry_output.mdx"))
 					diff := cmp.Diff(want, got)
@@ -168,7 +168,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 					By("checking for the init output", func() {
 						args := []string{"init", "--name", "my-spring-app", "--devfile", "java-springboot", "--devfile-registry", "DefaultDevfileRegistry", "--starter", "springbootproject"}
 						out := helper.Cmd("odo", args...).ShouldPass().Out()
-						got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.CleanStringOfSpinner(out))
+						got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(out))
 						got = helper.ReplaceTimeInString(got, "[4s]")
 						want := helper.GetMDXContent(filepath.Join(commonPath, "devfile_from_specific_registry_output.mdx"))
 						want = helper.ReplaceTimeInString(got, "[4s]")
@@ -185,7 +185,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 						args := []string{"registry", "--devfile", "nodejs-react"}
 						out := helper.Cmd("odo", args...).ShouldPass().Out()
 						got := helper.StripAnsi(out)
-						got = fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.CleanStringOfSpinner(got))
+						got = fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(got))
 						want := helper.GetMDXContent(filepath.Join(commonPath, "registry_list_output.mdx"))
 						diff := cmp.Diff(want, got)
 						Expect(diff).To(BeEmpty())
@@ -194,7 +194,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 					By("checking for the init output", func() {
 						args := []string{"init", "--devfile", "nodejs-react", "--name", "my-nr-app"}
 						out := helper.Cmd("odo", args...).ShouldPass().Out()
-						got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.CleanStringOfSpinner(out))
+						got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(out))
 						got = helper.ReplaceTimeInString(got, "[3s]")
 						want := helper.GetMDXContent(filepath.Join(commonPath, "devfile_from_any_registry_output.mdx"))
 						diff := cmp.Diff(want, got)
@@ -208,7 +208,7 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 		It("Fetch Devfile from a URL", func() {
 			args := []string{"init", "--devfile-path", "https://registry.devfile.io/devfiles/nodejs-angular", "--name", "my-nodejs-app", "--starter", "nodejs-angular-starter"}
 			out := helper.Cmd("odo", args...).ShouldPass().Out()
-			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.CleanStringOfSpinner(out))
+			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(out))
 			got = helper.ReplaceTimeInString(got, "[1s]")
 			want := helper.GetMDXContent(filepath.Join(commonPath, "devfile_from_url_output.mdx"))
 			want = helper.ReplaceTimeInString(got, "[1s]")

tests/documentation/documentation_suite_test.go

@@ -0,0 +1,14 @@
+//go:build !race
+// +build !race
+
+package docautomation
+
+import (
+	"testing"
+
+	"github.com/redhat-developer/odo/tests/helper"
+)
+
+func TestDocAutomation(t *testing.T) {
+	helper.RunTestSpecs(t, "Doc Automation Suite")
+}

tests/helper/helper_documentation.go

@@ -15,16 +15,19 @@ import (
 const (
 	timePatternInOdo = `(\[[0-9smh]+\])` // e.g. [4s], [1m], [3ms]
 	// Credit: https://github.com/acarl005/stripansi/blob/master/stripansi.go
-	ansi                 = "[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))"
+	ansiPattern          = "[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))"
 	unicodeSpinnerFrames = "◓◐◑◒"
 )
 
+// ReplaceTimeInString replaces the time taken to download a Devfile or a starter project for an odo command with a custom value;
+// this function is helpful because the time value is variable and replacing it with the value in mdx content helps in comparing.
 func ReplaceTimeInString(docString string, timeString string) string {
 	reg := regexp.MustCompile(timePatternInOdo)
 	return reg.ReplaceAllString(docString, timeString)
 }
 
-func CleanStringOfSpinner(docString string) (returnString string) {
+// StripSpinner strips the cmd out string of spaces, spinner statements and spinner frames
+func StripSpinner(docString string) (returnString string) {
 	for _, line := range strings.Split(docString, "\n") {
 		// trim any special character present in the line
 		line = strings.TrimFunc(line, unicode.IsSpace)
@@ -37,6 +40,8 @@ func CleanStringOfSpinner(docString string) (returnString string) {
 	return
 }
 
+// GetMDXContent reads the content of MDX files, strips it of extra spaces and returns the string
+// it strips the extra space for an easy comparison
 func GetMDXContent(filePath string) (mdxContent string) {
 	// filename of this file
 	_, filename, _, _ := runtime.Caller(0)
@@ -59,8 +64,12 @@ func GetMDXContent(filePath string) (mdxContent string) {
 	return
 }
 
+// StripAnsi strips the cmd out of ansi values used for fomatting(underline, colored line, etc.) the cmd out;
+// this function should be called before StripSpinner for better results
+// and is essential because mdx content does not support ansi
+// The regex used by this function is copied from https://github.com/acarl005/stripansi/
 func StripAnsi(docString string) (returnString string) {
-	reg, err := regexp.Compile(ansi)
+	reg, err := regexp.Compile(ansiPattern)
 	Expect(err).To(BeNil())
 	returnString = reg.ReplaceAllString(docString, "")
 	return