From a41c5abea1a4d17ec34db06d5938b67f17f3a5f2 Mon Sep 17 00:00:00 2001
From: Michael Dawson <michael_dawson@ca.ibm.com>
Date: Wed, 11 May 2016 17:54:42 -0400
Subject: [PATCH 1/4] test: add --repeat option to tools/test.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I often want to run a test many times to see if a failure
can be recreated and I believe this is a common
use case.  We even have this job in the CI
https://ci.nodejs.org/job/node-stress-single-test/configure
but often you want to run it on a specific machine.

This patch adds the --repeat option so that
you can repeat the selected set of tests a
number of times. Given existing options
in test.py this will allow you to run
one or more tests for the number of
repeats specified. For example:

tools/test.py -j8 --repeat 1000 parallel/test-process-exec-argv

runs the test-process-exec-argv test 1000 times,
running 8 copies in parallel

tools/test.py --repeat 2

would run the entire test suite twice.


---
 tools/test.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tools/test.py b/tools/test.py
index 68141ee1ecb58f..a71a1ed0d73e70 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -721,13 +721,14 @@ def GetConfiguration(self, context):
   def GetBuildRequirements(self, path, context):
     return self.GetConfiguration(context).GetBuildRequirements()
 
-  def AddTestsToList(self, result, current_path, path, context, arch, mode):
+  def AddTestsToList(self, result, current_path, path, context,
+                     arch, mode, repeat):
     for v in VARIANT_FLAGS:
       tests = self.GetConfiguration(context).ListTests(current_path, path,
                                                        arch, mode)
       for t in tests: t.variant_flags = v
-      result += tests
-
+      for rep in range(0, repeat):
+        result += tests
 
   def GetTestStatus(self, context, sections, defs):
     self.GetConfiguration(context).GetTestStatus(sections, defs)
@@ -747,14 +748,16 @@ def GetBuildRequirements(self, path, context):
         result += test.GetBuildRequirements(rest, context)
     return result
 
-  def ListTests(self, current_path, path, context, arch, mode):
+  def ListTests(self, current_path, path, context,
+                arch, mode, repeat):
     (name, rest) = CarCdr(path)
     result = [ ]
     for test in self.tests:
       test_name = test.GetName()
       if not name or name.match(test_name):
         full_path = current_path + [test_name]
-        test.AddTestsToList(result, full_path, path, context, arch, mode)
+        test.AddTestsToList(result, full_path, path, context,
+                            arch, mode, repeat)
     result.sort(cmp=lambda a, b: cmp(a.GetName(), b.GetName()))
     return result
 
@@ -1324,6 +1327,9 @@ def BuildOptions():
       default="")
   result.add_option('--temp-dir',
       help='Optional path to change directory used for tests', default=False)
+  result.add_option('--repeat',
+      help='how many times to repeat each test',
+      default=1, type="int")
   return result
 
 
@@ -1534,7 +1540,7 @@ def Main():
           'system': utils.GuessOS(),
           'arch': vmArch,
         }
-        test_list = root.ListTests([], path, context, arch, mode)
+        test_list = root.ListTests([], path, context, arch, mode, options.repeat)
         unclassified_tests += test_list
         (cases, unused_rules, all_outcomes) = (
             config.ClassifyTests(test_list, env))

From abb3e63236ea6cebb5015d989d256563c9342855 Mon Sep 17 00:00:00 2001
From: Michael Dawson <michael_dawson@ca.ibm.com>
Date: Thu, 12 May 2016 09:23:30 -0400
Subject: [PATCH 2/4] address first round of comments

---
 tools/test.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/tools/test.py b/tools/test.py
index a71a1ed0d73e70..cf2d94e6d631db 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -721,13 +721,12 @@ def GetConfiguration(self, context):
   def GetBuildRequirements(self, path, context):
     return self.GetConfiguration(context).GetBuildRequirements()
 
-  def AddTestsToList(self, result, current_path, path, context,
-                     arch, mode, repeat):
+  def AddTestsToList(self, result, current_path, path, context, arch, mode):
     for v in VARIANT_FLAGS:
       tests = self.GetConfiguration(context).ListTests(current_path, path,
                                                        arch, mode)
       for t in tests: t.variant_flags = v
-      for rep in range(0, repeat):
+      for rep in range(0, context.repeat):
         result += tests
 
   def GetTestStatus(self, context, sections, defs):
@@ -748,16 +747,14 @@ def GetBuildRequirements(self, path, context):
         result += test.GetBuildRequirements(rest, context)
     return result
 
-  def ListTests(self, current_path, path, context,
-                arch, mode, repeat):
+  def ListTests(self, current_path, path, context, arch, mode):
     (name, rest) = CarCdr(path)
     result = [ ]
     for test in self.tests:
       test_name = test.GetName()
       if not name or name.match(test_name):
         full_path = current_path + [test_name]
-        test.AddTestsToList(result, full_path, path, context,
-                            arch, mode, repeat)
+        test.AddTestsToList(result, full_path, path, context, arch, mode)
     result.sort(cmp=lambda a, b: cmp(a.GetName(), b.GetName()))
     return result
 
@@ -783,7 +780,7 @@ def GetTestStatus(self, context, sections, defs):
 class Context(object):
 
   def __init__(self, workspace, buildspace, verbose, vm, args, expect_fail,
-               timeout, processor, suppress_dialogs, store_unexpected_output):
+               timeout, processor, suppress_dialogs, store_unexpected_output, repeat):
     self.workspace = workspace
     self.buildspace = buildspace
     self.verbose = verbose
@@ -794,6 +791,7 @@ def __init__(self, workspace, buildspace, verbose, vm, args, expect_fail,
     self.processor = processor
     self.suppress_dialogs = suppress_dialogs
     self.store_unexpected_output = store_unexpected_output
+    self.repeat = repeat
 
   def GetVm(self, arch, mode):
     if arch == 'none':
@@ -1495,7 +1493,8 @@ def Main():
                     options.timeout,
                     processor,
                     options.suppress_dialogs,
-                    options.store_unexpected_output)
+                    options.store_unexpected_output,
+                    options.repeat)
   # First build the required targets
   if not options.no_build:
     reqs = [ ]
@@ -1540,7 +1539,7 @@ def Main():
           'system': utils.GuessOS(),
           'arch': vmArch,
         }
-        test_list = root.ListTests([], path, context, arch, mode, options.repeat)
+        test_list = root.ListTests([], path, context, arch, mode)
         unclassified_tests += test_list
         (cases, unused_rules, all_outcomes) = (
             config.ClassifyTests(test_list, env))

From f36f59d15b179f7245902ea439d8fa134721f2e3 Mon Sep 17 00:00:00 2001
From: Michael Dawson <michael_dawson@ca.ibm.com>
Date: Thu, 12 May 2016 09:31:25 -0400
Subject: [PATCH 3/4] address rest of comments

---
 tools/test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tools/test.py b/tools/test.py
index cf2d94e6d631db..4646377fe82cd2 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -726,8 +726,7 @@ def AddTestsToList(self, result, current_path, path, context, arch, mode):
       tests = self.GetConfiguration(context).ListTests(current_path, path,
                                                        arch, mode)
       for t in tests: t.variant_flags = v
-      for rep in range(0, context.repeat):
-        result += tests
+      result += tests * context.repeat
 
   def GetTestStatus(self, context, sections, defs):
     self.GetConfiguration(context).GetTestStatus(sections, defs)

From efad8e9ddc01507625747a6e66ac122590e30f47 Mon Sep 17 00:00:00 2001
From: Michael Dawson <michael_dawson@ca.ibm.com>
Date: Thu, 12 May 2016 12:56:48 -0400
Subject: [PATCH 4/4] address another comment

---
 tools/test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/test.py b/tools/test.py
index 4646377fe82cd2..6784f1d21839f5 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -779,7 +779,8 @@ def GetTestStatus(self, context, sections, defs):
 class Context(object):
 
   def __init__(self, workspace, buildspace, verbose, vm, args, expect_fail,
-               timeout, processor, suppress_dialogs, store_unexpected_output, repeat):
+               timeout, processor, suppress_dialogs,
+               store_unexpected_output, repeat):
     self.workspace = workspace
     self.buildspace = buildspace
     self.verbose = verbose