diff --git a/lib/node-labels.js b/lib/node-labels.js
index b1a128c0..ec0305e1 100644
--- a/lib/node-labels.js
+++ b/lib/node-labels.js
@@ -25,8 +25,9 @@ function resolveLabels (filepathsChanged) {
 }
 
 function matchExclusiveSubSystem (filepathsChanged) {
+  const isExclusive = filepathsChanged.every(matchesAnExclusiveLabel)
   const labels = matchSubSystemsByRegex(exclusiveLabelsMap, filepathsChanged)
-  return labels.length === 1 ? labels : []
+  return (isExclusive && labels.length === 1) ? labels : []
 }
 
 function matchAllSubSystem (filepathsChanged) {
@@ -75,4 +76,8 @@ function withoutUndefinedValues (label) {
   return label !== undefined
 }
 
+function matchesAnExclusiveLabel (filepath) {
+  return mappedSubSystemForFile(exclusiveLabelsMap, filepath) !== undefined
+}
+
 exports.resolveLabels = resolveLabels
diff --git a/test/node-labels.test.js b/test/node-labels.test.js
index ce8ab1a9..5b220265 100644
--- a/test/node-labels.test.js
+++ b/test/node-labels.test.js
@@ -27,6 +27,20 @@ tap.test('no labels: when ./test/ and ./doc/ files has been changed', (t) => {
   t.end()
 })
 
+// This ensures older mislabelling issues doesn't happen again
+// https://github.com/nodejs/node/pull/6432
+// https://github.com/nodejs/node/pull/6448
+tap.test('no labels: when ./test/ and ./lib/ files has been changed', (t) => {
+  const labels = nodeLabels.resolveLabels([
+    'lib/assert.js',
+    'test/parallel/test-assert.js'
+  ])
+
+  t.same(labels, [])
+
+  t.end()
+})
+
 tap.test('label: "doc" when only ./doc/ files has been changed', (t) => {
   const labels = nodeLabels.resolveLabels([
     'doc/api/fs.md',