Add daily slack notifier for nightlies (#1042)

* Update log_reports to send to slack
huggingface · Feb 6, 2023 · 2b981c0 · 2b981c0
1 parent a60640d
commit 2b981c0
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 10 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -8,12 +8,15 @@ on:
 env:
   RUN_SLOW: "yes"
   IS_GITHUB_CI: "1"
+  SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
+
 
 jobs:
   run_all_tests_single_gpu:
     runs-on: [self-hosted, docker-gpu, multi-gpu]
     env:
       CUDA_VISIBLE_DEVICES: "0"
+      TEST_TYPE: "single_gpu"
     container:
       image: huggingface/accelerate-gpu:latest
       options: --gpus all --shm-size "16gb"
@@ -43,12 +46,14 @@ jobs:
       - name: Generate Report
         if: always()
         run: |
+          pip install slack_sdk
           python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
 
   run_all_tests_multi_gpu:
     runs-on: [self-hosted, docker-gpu, multi-gpu]
     env:
       CUDA_VISIBLE_DEVICES: "0,1"
+      TEST_TYPE: "multi_gpu"
     container:
       image: huggingface/accelerate-gpu:latest
       options: --gpus all --shm-size "16gb"
@@ -85,4 +90,5 @@ jobs:
       - name: Generate Report
         if: always()
         run: |
+          pip install slack_sdk
           python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
diff --git a/utils/log_reports.py b/utils/log_reports.py
@@ -1,5 +1,6 @@
-import json
+import json, os
 from pathlib import Path
+from datetime import date
 
 failed = []
 passed = []
@@ -19,15 +20,32 @@
                     if line.get("outcome", "") == "failed":
                         section_num_failed += 1
                         failed.append([test, duration, log.name.split('_')[0]])
+                        total_num_failed += 1
                     else:
                         passed.append([test, duration, log.name.split('_')[0]])
-    group_info.append([str(log), section_num_failed])
+    group_info.append([str(log), section_num_failed, failed])
+    failed = []
+message = ""
+if total_num_failed > 0:
+    for name, num_failed, failed_tests in group_info:
+        if num_failed > 0:
+            if len(num_failed) == 1:
+                message += f"*{name}: {num_failed} failed test*\n"
+            else:
+                message += f"*{name}: {num_failed} failed tests*\n"
+            failed_table = '| Test Location | Test Class | Test Name | PyTorch Version |\n|---|---|---|---|\n| '
+            for test in failed_tests:
+                failed_table += ' | '.join(test[0].split("::"))
+            failed_table += f" | {test[2]} |"
+            message += failed_table
+    print(f'### {message}')
+else:
+    message = "No failed tests! 🤗"
+    print(f'## {message}')
 
-if len(failed) > 0:
-    result = "## Failed Tests:\n"
-    failed_table = '| Test Location | Test Class | Test Name | PyTorch Version |\n|---|---|---|---|\n| '
-    for test in failed:
-        failed_table += ' | '.join(test[0].split("::"))
-    failed_table += f" | {test[2]} |"
-    result += failed_table
-    print(result)
+if os.environ.get("TEST_TYPE", "") != "":
+    from slack_sdk import WebClient
+    message = f'*Nightly {os.environ.get("TEST_TYPE")} test results for {date.today()}:*\n{message}'
+
+    client = WebClient(token=os.environ['SLACK_API_TOKEN'])
+    client.chat_postMessage(channel='#accelerate-ci-daily', text=message)