diff --git a/src/config.rs b/src/config.rs
index 0c6f5f2e1..cf73536b5 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -78,6 +78,10 @@ pub(crate) struct RelabelConfig {
 #[derive(PartialEq, Eq, Debug, serde::Deserialize)]
 pub(crate) struct PrioritizeConfig {
     pub(crate) label: String,
+    #[serde(default)]
+    pub(crate) prioritize_on: Vec<String>,
+    #[serde(default)]
+    pub(crate) priority_labels: String,
     pub(crate) zulip_stream: u64,
 }
 
diff --git a/src/handlers/prioritize.rs b/src/handlers/prioritize.rs
index 3c311e830..d843ced53 100644
--- a/src/handlers/prioritize.rs
+++ b/src/handlers/prioritize.rs
@@ -38,6 +38,17 @@ impl Handler for PrioritizeHandler {
                     if e.label.as_ref().expect("label").name == config.label {
                         // We need to take the exact same action in this case.
                         return Ok(Some(Prioritize::Start));
+                    } else {
+                        if let Ok(glob) = glob::Pattern::new(&config.priority_labels) {
+                            let issue_labels = event.issue().unwrap().labels();
+                            let label_name = &e.label.as_ref().expect("label").name;
+
+                            if issue_labels.iter().all(|l| !glob.matches(&l.name))
+                                && config.prioritize_on.iter().any(|l| l == label_name)
+                            {
+                                return Ok(Some(Prioritize::Label));
+                            }
+                        }
                     }
                 }
             }