diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 55fa0e87..6862e135 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -19,8 +19,6 @@ jobs:
       - name: Setup Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
       - name: Restore Go cache
         uses: actions/cache@v1
         with:
@@ -92,6 +90,27 @@ jobs:
           make dev-deploy IMG=test/kustomize-controller:latest
           kubectl -n kustomize-system rollout status deploy/source-controller --timeout=1m
           kubectl -n kustomize-system rollout status deploy/kustomize-controller --timeout=1m
+      - name: Run tests for removing kubectl managed fields
+        run: |
+          kubectl create ns managed-fields
+          kustomize build github.com/stefanprodan/podinfo//kustomize?ref=6.0.0 > /tmp/podinfo.yaml
+          kubectl -n managed-fields apply -f /tmp/podinfo.yaml
+          kubectl -n managed-fields apply -f ./config/testdata/managed-fields
+          kubectl -n managed-fields wait kustomization/podinfo --for=condition=ready --timeout=4m
+          OUTDATA=$(kubectl -n managed-fields get deploy podinfo --show-managed-fields -oyaml)
+          if echo "$OUTDATA" | grep -q "kubectl";then
+            echo "kubectl client-side manager not removed"
+            exit 1
+          fi
+          kubectl -n managed-fields apply --server-side --force-conflicts -f /tmp/podinfo.yaml
+          kubectl -n managed-fields annotate --overwrite kustomization/podinfo reconcile.fluxcd.io/requestedAt="$(date +%s)"
+          kubectl -n managed-fields wait kustomization/podinfo --for=condition=ready --timeout=4m
+          OUTDATA=$(kubectl -n managed-fields get deploy podinfo --show-managed-fields -oyaml)
+          if echo "$OUTDATA" | grep -q "kubectl";then
+            echo "kubectl server-side manager not removed"
+            exit 1
+          fi
+          kubectl delete ns managed-fields
       - name: Run overlays tests
         run: |
           kubectl -n kustomize-system apply -k ./config/testdata/overlays
diff --git a/config/testdata/managed-fields/podinfo.yaml b/config/testdata/managed-fields/podinfo.yaml
new file mode 100644
index 00000000..1f065fc0
--- /dev/null
+++ b/config/testdata/managed-fields/podinfo.yaml
@@ -0,0 +1,23 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: podinfo
+spec:
+  interval: 15m
+  path: "./kustomize/"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: podinfo
+  timeout: 1m
+  targetNamespace: managed-fields
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: podinfo
+spec:
+  interval: 5m
+  url: https://github.com/stefanprodan/podinfo
+  ref:
+    semver: "6.0.0"
diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go
index 0d8478ca..0aa1bc21 100644
--- a/controllers/kustomization_controller.go
+++ b/controllers/kustomization_controller.go
@@ -32,6 +32,7 @@ import (
 
 	securejoin "github.com/cyphar/filepath-securejoin"
 	"github.com/hashicorp/go-retryablehttp"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -153,7 +154,7 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	// Add our finalizer if it does not exist
 	if !controllerutil.ContainsFinalizer(&kustomization, kustomizev1.KustomizationFinalizer) {
 		controllerutil.AddFinalizer(&kustomization, kustomizev1.KustomizationFinalizer)
-		if err := r.Update(ctx, &kustomization); err != nil {
+		if err := r.Update(ctx, &kustomization, client.FieldOwner(r.ControllerName)); err != nil {
 			log.Error(err, "unable to register finalizer")
 			return ctrl.Result{}, err
 		}
@@ -681,6 +682,36 @@ func (r *KustomizationReconciler) apply(ctx context.Context, manager *ssa.Resour
 	applyOpts.Exclusions = map[string]string{
 		fmt.Sprintf("%s/reconcile", kustomizev1.GroupVersion.Group): kustomizev1.DisabledValue,
 	}
+	applyOpts.Cleanup = ssa.ApplyCleanupOptions{
+		Annotations: []string{
+			// remove the kubectl annotation
+			corev1.LastAppliedConfigAnnotation,
+			// remove deprecated fluxcd.io annotations
+			"kustomize.toolkit.fluxcd.io/checksum",
+			"fluxcd.io/sync-checksum",
+		},
+		Labels: []string{
+			// remove deprecated fluxcd.io labels
+			"fluxcd.io/sync-gc-mark",
+		},
+		FieldManagers: []ssa.FiledManager{
+			{
+				// to undo changes made with 'kubectl apply --server-side --force-conflicts'
+				Name:          "kubectl",
+				OperationType: metav1.ManagedFieldsOperationApply,
+			},
+			{
+				// to undo changes made with 'kubectl apply'
+				Name:          "kubectl",
+				OperationType: metav1.ManagedFieldsOperationUpdate,
+			},
+			{
+				// to undo changes made with 'kubectl apply'
+				Name:          "before-first-apply",
+				OperationType: metav1.ManagedFieldsOperationUpdate,
+			},
+		},
+	}
 
 	// contains only CRDs and Namespaces
 	var stageOne []*unstructured.Unstructured
@@ -896,7 +927,7 @@ func (r *KustomizationReconciler) finalize(ctx context.Context, kustomization ku
 
 	// Remove our finalizer from the list and update it
 	controllerutil.RemoveFinalizer(&kustomization, kustomizev1.KustomizationFinalizer)
-	if err := r.Update(ctx, &kustomization); err != nil {
+	if err := r.Update(ctx, &kustomization, client.FieldOwner(r.ControllerName)); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -994,5 +1025,5 @@ func (r *KustomizationReconciler) patchStatus(ctx context.Context, req ctrl.Requ
 	patch := client.MergeFrom(kustomization.DeepCopy())
 	kustomization.Status = newStatus
 
-	return r.Status().Patch(ctx, &kustomization, patch)
+	return r.Status().Patch(ctx, &kustomization, patch, client.FieldOwner(r.ControllerName))
 }
diff --git a/go.mod b/go.mod
index e79a1111..d9d34d2c 100644
--- a/go.mod
+++ b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/fluxcd/pkg/apis/kustomize v0.3.1
 	github.com/fluxcd/pkg/apis/meta v0.10.2
 	github.com/fluxcd/pkg/runtime v0.12.3
-	github.com/fluxcd/pkg/ssa v0.8.0
+	github.com/fluxcd/pkg/ssa v0.9.1-0.20220110192134-be14616f58e1
 	github.com/fluxcd/pkg/testserver v0.2.0
 	github.com/fluxcd/pkg/untar v0.1.0
 	github.com/fluxcd/source-controller/api v0.20.1
diff --git a/go.sum b/go.sum
index 3482eb9d..469916d0 100644
--- a/go.sum
+++ b/go.sum
@@ -249,8 +249,8 @@ github.com/fluxcd/pkg/apis/meta v0.10.2 h1:pnDBBEvfs4HaKiVAYgz+e/AQ8dLvcgmVfSeBr
 github.com/fluxcd/pkg/apis/meta v0.10.2/go.mod h1:KQ2er9xa6koy7uoPMZjIjNudB5p4tXs+w0GO6fRcy7I=
 github.com/fluxcd/pkg/runtime v0.12.3 h1:h21AZ3YG5MAP7DxFF9hfKrP+vFzys2L7CkUbPFjbP/0=
 github.com/fluxcd/pkg/runtime v0.12.3/go.mod h1:imJ2xYy/d4PbSinX2IefmZk+iS2c1P5fY0js8mCE4SM=
-github.com/fluxcd/pkg/ssa v0.8.0 h1:f3fNpKFPncCoWMDvxnTqX+8LAAMb3ZXc1N41mzw54k8=
-github.com/fluxcd/pkg/ssa v0.8.0/go.mod h1:3brodT9mai+iKz4nizqZUESITGMoMr4CCdt5MdfyTXw=
+github.com/fluxcd/pkg/ssa v0.9.1-0.20220110192134-be14616f58e1 h1:30b/fC92OJZac/rTRkV2QJxxAV5BdjDI2MMl95B7VU4=
+github.com/fluxcd/pkg/ssa v0.9.1-0.20220110192134-be14616f58e1/go.mod h1:3brodT9mai+iKz4nizqZUESITGMoMr4CCdt5MdfyTXw=
 github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNKQ0Krt4=
 github.com/fluxcd/pkg/testserver v0.2.0/go.mod h1:bgjjydkXsZTeFzjz9Cr4heGANr41uTB1Aj1Q5qzuYVk=
 github.com/fluxcd/pkg/untar v0.1.0 h1:k97V/xV5hFrAkIkVPuv5AVhyxh1ZzzAKba/lbDfGo6o=