diff --git a/docs/labels.md b/docs/labels.md
index e3a2b7d1465..8a75986a547 100644
--- a/docs/labels.md
+++ b/docs/labels.md
@@ -89,8 +89,6 @@ Tekton automatically adds labels to Tekton entities as described in the followin
 	</tbody>
 </table>
 
-**Note:** For backward compatibility, `TaskRuns` that reference a `ClusterTask` also get the `tekton.dev/task` label.
-
 ## Usage examples
 
 Below are some examples of using labels:
diff --git a/pkg/reconciler/taskrun/taskrun.go b/pkg/reconciler/taskrun/taskrun.go
index 17f580411d2..ba7d8d72ee6 100644
--- a/pkg/reconciler/taskrun/taskrun.go
+++ b/pkg/reconciler/taskrun/taskrun.go
@@ -293,9 +293,10 @@ func (c *Reconciler) prepare(ctx context.Context, tr *v1beta1.TaskRun) (*v1beta1
 		tr.ObjectMeta.Labels[key] = value
 	}
 	if tr.Spec.TaskRef != nil {
-		tr.ObjectMeta.Labels[pipeline.GroupName+pipeline.TaskLabelKey] = taskMeta.Name
 		if tr.Spec.TaskRef.Kind == "ClusterTask" {
 			tr.ObjectMeta.Labels[pipeline.GroupName+pipeline.ClusterTaskLabelKey] = taskMeta.Name
+		} else {
+			tr.ObjectMeta.Labels[pipeline.GroupName+pipeline.TaskLabelKey] = taskMeta.Name
 		}
 	}
 
diff --git a/pkg/reconciler/taskrun/taskrun_test.go b/pkg/reconciler/taskrun/taskrun_test.go
index 54e96461215..2c812894031 100644
--- a/pkg/reconciler/taskrun/taskrun_test.go
+++ b/pkg/reconciler/taskrun/taskrun_test.go
@@ -1304,7 +1304,6 @@ func TestReconcile(t *testing.T) {
 		wantPod: tb.Pod("test-taskrun-with-cluster-task-pod-abcde",
 			tb.PodNamespace("foo"),
 			tb.PodAnnotation(podconvert.ReleaseAnnotation, version.PipelineVersion),
-			tb.PodLabel(taskNameLabelKey, "test-cluster-task"),
 			tb.PodLabel(clusterTaskNameLabelKey, "test-cluster-task"),
 			tb.PodLabel(taskRunNameLabelKey, "test-taskrun-with-cluster-task"),
 			tb.PodLabel("app.kubernetes.io/managed-by", "tekton-pipelines"),