Merge branch 'master' into increase_gke_nodes

src/clusterfuzz/_internal/bot/minimizer/minimizer.py

@@ -183,6 +183,7 @@ def _deadline_exceeded(self, cleanup_function=None, soft_check=False):
     """Check to see if we have exceeded the deadline for execution."""
     if self.minimizer.deadline and time.time() > self.minimizer.deadline:
       if soft_check:
+        logs.warning(f'Minimization deadline exceeded. soft_check={soft_check}')
         return True
 
       # If we are here, we have exceeded the deadline on a hard check. Clean up.
@@ -192,6 +193,7 @@ def _deadline_exceeded(self, cleanup_function=None, soft_check=False):
       if self.minimizer.cleanup_function:
         self.minimizer.cleanup_function()
 
+      logs.warning('Minimization deadline exceeded.')
       # Raise an exception if this is not a soft deadline check.
       raise errors.MinimizationDeadlineExceededError(self)
 
@@ -558,6 +560,7 @@ def minimize(self, data):
       # minimized test case is stored with it so that we can recover the work
       # that had been done up to that point.
       testcase = error.testcase
+      logs.warning('Minimization Deadline Exceeded.')
     except errors.TokenizationFailureError:
       logs.info('Tokenized data did not match original data. Defaulting to line'
                 'minimization.')

src/clusterfuzz/_internal/bot/tasks/utasks/minimize_task.py

@@ -445,7 +445,10 @@ def utask_main(uworker_input: uworker_msg_pb2.Input):  # pylint: disable=no-memb
         ' should have been detected in preprocess.')
     return None
 
-  max_threads = utils.maximum_parallel_processes_allowed()
+  # TODO(alhijazi): re-install multithreaded runs
+  # max_threads = utils.maximum_parallel_processes_allowed()
+  # Temporarily run minimization single threaded.
+  max_threads = 1
 
   # Prepare the test case runner.
   crash_retries = environment.get_value('CRASH_RETRIES')
@@ -1748,8 +1751,10 @@ def do_html_minimization(test_function, get_temp_file, data, deadline, threads,
       delete_temp_files=delete_temp_files,
       progress_report_function=logs.info)
   try:
+    logs.info('Launching html minimization.')
     return current_minimizer.minimize(data)
   except minimizer_errors.AntlrDecodeError:
+    logs.info('Launching line minimization.')
     return do_line_minimization(test_function, get_temp_file, data, deadline,
                                 threads, cleanup_interval, delete_temp_files)
 

src/clusterfuzz/_internal/config/local_config.py

@@ -34,6 +34,7 @@
 MONITORING_REGIONS_PATH = 'monitoring.regions'
 PROJECT_PATH = 'project'
 SWARMING_PATH = 'swarming.swarming'
+EXTERNAL_TESTCASE_UPLOADER_PATH = 'external_testcase_reader.config'
 
 
 def _load_yaml_file(yaml_file_path):
@@ -259,3 +260,10 @@ class SwarmingConfig(Config):
 
   def __init__(self):
     super().__init__(SWARMING_PATH)
+
+
+class ExternalTestcaseReaderConfig(Config):
+  """External testcase reader config."""
+
+  def __init__(self):
+    super().__init__(EXTERNAL_TESTCASE_UPLOADER_PATH)

src/clusterfuzz/_internal/cron/external_testcase_reader.py

@@ -22,6 +22,7 @@
 from appengine.libs import form
 from appengine.libs import gcs
 from appengine.libs import helpers
+from clusterfuzz._internal.config import local_config
 from clusterfuzz._internal.issue_management.google_issue_tracker import \
     issue_tracker
 
@@ -32,12 +33,11 @@
 ISSUETRACKER_WONTFIX_STATE = 'NOT_REPRODUCIBLE'
 
 
-def get_vrp_uploaders():
+def get_vrp_uploaders(config):
   """Checks whether the given reporter has permission to upload."""
-  # TODO(pgrace) Add this to a YAML file.
   storage_client = storage.Client()
-  bucket = storage_client.bucket('clusterfuzz-vrp-uploaders')
-  blob = bucket.blob('vrp-uploaders')
+  bucket = storage_client.bucket(config.get('vrp-uploaders-bucket'))
+  blob = bucket.blob(config.get('vrp-uploaders-blob'))
   members = blob.download_as_string().decode('utf-8').splitlines()[0].split(',')
   return members
 
@@ -151,7 +151,7 @@ def submit_testcase(issue_id, file, filename, filetype, cmds):
       'https://clusterfuzz.com/upload-testcase/upload', data=data, timeout=10)
 
 
-def handle_testcases(tracker):
+def handle_testcases(tracker, config):
   """Fetches and submits testcases from bugs or closes unnecssary bugs."""
   # TODO(pgrace) remove ID filter once done testing.
   issues = tracker.find_issues_with_filters(
@@ -163,7 +163,7 @@ def handle_testcases(tracker):
     return
 
   # TODO(pgrace) Cache in redis.
-  vrp_uploaders = get_vrp_uploaders()
+  vrp_uploaders = get_vrp_uploaders(config)
 
   # TODO(pgrace) Implement rudimentary rate limiting.
 
@@ -199,7 +199,7 @@ def handle_testcases(tracker):
 def main():
   tracker = issue_tracker.IssueTracker('chromium', None,
                                        {'default_component_id': 1363614})
-  handle_testcases(tracker)
+  handle_testcases(tracker, local_config.ExternalTestcaseReaderConfig())
 
 
 if __name__ == '__main__':

src/clusterfuzz/_internal/tests/appengine/handlers/cron/external_testcase_reader_test.py

@@ -30,6 +30,10 @@
     },
     'etag': 'TXpjek9Ea3pNekV4TFRZd01USTNOalk0TFRjNE9URTROVFl4TlE9PQ=='
 }
+BASIC_CONFIG = {
+    'vrp-uploaders-bucket': 'bucket-name',
+    'vrp-uploaders-blob': 'blob-name'
+}
 
 
 @mock.patch.object(
@@ -52,7 +56,7 @@ def test_handle_testcases(self, mock_close_issue_if_invalid,
     basic_issue.reporter.return_value = 'test-reporter@gmail.com'
     mock_it.find_issues_with_filters.return_value = [basic_issue]
 
-    external_testcase_reader.handle_testcases(mock_it)
+    external_testcase_reader.handle_testcases(mock_it, BASIC_CONFIG)
 
     mock_close_issue_if_invalid.assert_called_once()
     mock_it.get_attachment.assert_called_once()
@@ -67,7 +71,7 @@ def test_handle_testcases_invalid(self, mock_close_issue_if_invalid,
     basic_issue.reporter.return_value = 'test-reporter@gmail.com'
     mock_it.find_issues_with_filters.return_value = [basic_issue]
 
-    external_testcase_reader.handle_testcases(mock_it)
+    external_testcase_reader.handle_testcases(mock_it, BASIC_CONFIG)
 
     mock_close_issue_if_invalid.assert_called_once()
     mock_it.get_attachment.assert_not_called()
@@ -86,7 +90,7 @@ def test_handle_testcases_not_reproducible(
     basic_issue.reporter.return_value = 'test-reporter@gmail.com'
     mock_it.find_issues_with_filters.return_value = [basic_issue]
 
-    external_testcase_reader.handle_testcases(mock_it)
+    external_testcase_reader.handle_testcases(mock_it, BASIC_CONFIG)
 
     mock_close_issue_if_invalid.assert_not_called()
     mock_it.get_attachment.assert_not_called()
@@ -98,7 +102,7 @@ def test_handle_testcases_no_issues(self, mock_close_issue_if_invalid,
     mock_it = mock.create_autospec(issue_tracker.IssueTracker)
     mock_it.find_issues_with_filters.return_value = []
 
-    external_testcase_reader.handle_testcases(mock_it)
+    external_testcase_reader.handle_testcases(mock_it, BASIC_CONFIG)
 
     mock_close_issue_if_invalid.assert_not_called()
     mock_it.get_attachment.assert_not_called()
@@ -221,9 +225,8 @@ def test_get_vrp_uploaders(self):
       mock_blob.download_as_string.return_value = "test-user@google.com,test-user2@chromium.org".encode(
           'utf-8')
 
-      actual = external_testcase_reader.get_vrp_uploaders()
-      mock_storage.return_value.bucket.assert_called_once_with(
-          'clusterfuzz-vrp-uploaders')
-      mock_bucket.blob.assert_called_once_with('vrp-uploaders')
+      actual = external_testcase_reader.get_vrp_uploaders(BASIC_CONFIG)
+      mock_storage.return_value.bucket.assert_called_once_with('bucket-name')
+      mock_bucket.blob.assert_called_once_with('blob-name')
       self.assertEqual(actual,
                        ['test-user@google.com', 'test-user2@chromium.org'])