diff --git a/.buildbot.sh b/.buildbot.sh
index 8fa94df..c1d0a6d 100644
--- a/.buildbot.sh
+++ b/.buildbot.sh
@@ -2,10 +2,17 @@
 
 set -e
 
+# We only want checks to really run when a github merge queue pushes a branch.
+if [ "${EVENT}" = "pull_request" ]; then
+    exit 0
+elif [ "${EVENT}" != "push" ]; then
+    echo "Unknown event. Failing CI"
+fi
+
 for i in `seq 30`; do
     echo $i
 done
 
 echo hi 1
 
-exit 0
+exit 1