diff --git a/.github/workflows/check_backports.yml b/.github/workflows/check_backports.yml
index 487b9b86c82..6d4da7265f2 100644
--- a/.github/workflows/check_backports.yml
+++ b/.github/workflows/check_backports.yml
@@ -8,11 +8,11 @@ on:
 jobs:
   check_patchback_comment:
     if: >
-      github.event.issue.pull_request.merged == true
+      github.event.issue.pull_request.merged == true &&
+      contains(github.event.comment.body, '💔 cherry-picking failed')
     runs-on: ubuntu-latest
 
     steps:
       - name: Add Label for Backport Failure
         run: |
-          echo ${{ github.event.issue_comment.comment }}
           gh pr edit ${{ github.event.issue.number }} -R ${{ github.repository }} --add-label 'backport-failed'