diff --git a/vertical-pod-autoscaler/pkg/admission-controller/Dockerfile b/vertical-pod-autoscaler/pkg/admission-controller/Dockerfile
index 51da41ff2c99..a8f95eb8ec11 100644
--- a/vertical-pod-autoscaler/pkg/admission-controller/Dockerfile
+++ b/vertical-pod-autoscaler/pkg/admission-controller/Dockerfile
@@ -12,10 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+FROM --platform=$BUILDPLATFORM golang:1.21.5 as builder
+
+ENV GOPATH /gopath/
+ENV PATH $GOPATH/bin:$PATH
+
+COPY . /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
+WORKDIR /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
+
+ARG TARGETOS TARGETARCH
+
+RUN CGO_ENABLED=0 LD_FLAGS=-s GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/admission-controller -mod vendor -o admission-controller-$TARGETARCH
+
 FROM gcr.io/distroless/static:latest
 MAINTAINER Tomasz Kulczynski "tkulczynski@google.com"
-ARG ARCH
-copy admission-controller-$ARCH /admission-controller
+
+ARG TARGETARCH
+
+COPY --from=builder /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/admission-controller-$TARGETARCH /admission-controller
 
 ENTRYPOINT ["/admission-controller"]
 CMD ["--v=4", "--stderrthreshold=info"]
diff --git a/vertical-pod-autoscaler/pkg/admission-controller/Makefile b/vertical-pod-autoscaler/pkg/admission-controller/Makefile
index b540b3fba2dc..c0b945e15d68 100644
--- a/vertical-pod-autoscaler/pkg/admission-controller/Makefile
+++ b/vertical-pod-autoscaler/pkg/admission-controller/Makefile
@@ -19,13 +19,6 @@ build: clean
 build-binary: clean
 	$(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT}
 
-.PHONY: build-binary-with-vendor
-build-binary-with-vendor: $(addprefix build-binary-with-vendor-,$(ALL_ARCHITECTURES)) clean
-
-.PHONY: build-binary-with-vendor-*
-build-binary-with-vendor-%:
-	$(ENVVAR) GOARCH=$* GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT}-$*
-
 test-unit: clean build
 	$(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS)
 
@@ -42,13 +35,10 @@ ifndef TAG
 	ERR = $(error TAG is undefined)
 	$(ERR)
 endif
-	docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* .
+	docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* -f ./Dockerfile ../../
 
 .PHONY: docker-push
-docker-push: $(addprefix sub-push-,$(ALL_ARCHITECTURES)) push-multi-arch;
-
-.PHONY: sub-push-*
-sub-push-%: docker-build-% do-push-% ;
+docker-push: $(addprefix do-push-,$(ALL_ARCHITECTURES)) push-multi-arch;
 
 .PHONY: do-push-*
 do-push-%:
@@ -68,20 +58,13 @@ push-multi-arch:
 	@for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done
 	docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG)
 
-docker-builder:
-	docker build -t vpa-autoscaling-builder ../../builder
-
-.PHONY: build-in-docker
-build-in-docker: $(addprefix build-in-docker-,$(ALL_ARCHITECTURES))
-
-.PHONY: build-in-docker-*
-build-in-docker-%: clean docker-builder
+.PHONY: show-git-info
+show-git-info:
 	echo '=============== local git status ==============='
 	git status
 	echo '=============== last commit ==============='
 	git log -1
 	echo '=============== bulding from the above ==============='
-	docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor-$* -C pkg/admission-controller'
 
 .PHONY: create-buildx-builder
 create-buildx-builder:
@@ -92,7 +75,7 @@ remove-buildx-builder:
 	docker buildx rm ${BUILDER}
 
 .PHONY: release
-release: build-in-docker create-buildx-builder docker-build remove-buildx-builder docker-push
+release: show-git-info create-buildx-builder docker-build remove-buildx-builder docker-push
 	@echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed"
 
 clean: $(addprefix clean-,$(ALL_ARCHITECTURES))
diff --git a/vertical-pod-autoscaler/pkg/recommender/Dockerfile b/vertical-pod-autoscaler/pkg/recommender/Dockerfile
index 6e5d172824b8..84f14f5017d5 100644
--- a/vertical-pod-autoscaler/pkg/recommender/Dockerfile
+++ b/vertical-pod-autoscaler/pkg/recommender/Dockerfile
@@ -12,11 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+FROM --platform=$BUILDPLATFORM golang:1.21.5 as builder
+
+ENV GOPATH /gopath/
+ENV PATH $GOPATH/bin:$PATH
+
+COPY . /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
+WORKDIR /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
+
+ARG TARGETOS TARGETARCH
+
+RUN CGO_ENABLED=0 LD_FLAGS=-s GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/recommender -mod vendor -o recommender-$TARGETARCH
+
 FROM gcr.io/distroless/static:latest
 MAINTAINER Krzysztof Grygiel "kgrygiel@google.com"
 
-ARG ARCH
-COPY recommender-$ARCH /recommender
+ARG TARGETARCH
+
+COPY --from=builder /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/recommender-$TARGETARCH /recommender
 
 ENTRYPOINT ["/recommender"]
 CMD ["--v=4", "--stderrthreshold=info", "--prometheus-address=http://prometheus.monitoring.svc"]
diff --git a/vertical-pod-autoscaler/pkg/recommender/Makefile b/vertical-pod-autoscaler/pkg/recommender/Makefile
index eb3834d8a033..8bd5fcd0c377 100644
--- a/vertical-pod-autoscaler/pkg/recommender/Makefile
+++ b/vertical-pod-autoscaler/pkg/recommender/Makefile
@@ -21,13 +21,6 @@ build: clean
 build-binary: clean
 	$(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT}
 
-.PHONY: build-binary-with-vendor
-build-binary-with-vendor: $(addprefix build-binary-with-vendor-,$(ALL_ARCHITECTURES)) clean
-
-.PHONY: build-binary-with-vendor-*
-build-binary-with-vendor-%:
-	$(ENVVAR) GOARCH=$* GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT}-$*
-
 test-unit: clean build
 	$(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS)
 
@@ -35,7 +28,7 @@ test-unit: clean build
 docker-build: $(addprefix docker-build-,$(ALL_ARCHITECTURES))
 
 .PHONY: docker-build-*
-docker-build-%: 
+docker-build-%:
 ifndef REGISTRY
 	ERR = $(error REGISTRY is undefined)
 	$(ERR)
@@ -44,13 +37,10 @@ ifndef TAG
 	ERR = $(error TAG is undefined)
 	$(ERR)
 endif
-	docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* .
+	docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* -f ./Dockerfile ../../
 
 .PHONY: docker-push
-docker-push: $(addprefix sub-push-,$(ALL_ARCHITECTURES)) push-multi-arch;
-
-.PHONY: sub-push-*
-sub-push-%: docker-build-% do-push-% ;
+docker-push: $(addprefix do-push-,$(ALL_ARCHITECTURES)) push-multi-arch;
 
 .PHONY: do-push-*
 do-push-%:
@@ -66,24 +56,17 @@ endif
 
 .PHONY: push-multi-arch
 push-multi-arch:
-	docker manifest create $(INSECURE) --amend $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(shell echo $(ALL_ARCHITECTURES) | sed -e "s~[^ ]*~$(REGISTRY)/${FULL_COMPONENT}\-&:$(TAG)~g")
+	docker manifest create --amend $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(shell echo $(ALL_ARCHITECTURES) | sed -e "s~[^ ]*~$(REGISTRY)/${FULL_COMPONENT}\-&:$(TAG)~g")
 	@for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done
-	docker manifest push $(INSECURE) --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG)
-
-docker-builder:
-	docker build -t vpa-autoscaling-builder ../../builder
-
-.PHONY: build-in-docker
-build-in-docker: $(addprefix build-in-docker-,$(ALL_ARCHITECTURES))
+	docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG)
 
-.PHONY: build-in-docker-*
-build-in-docker-%: clean docker-builder
+.PHONY: show-git-info
+show-git-info:
 	echo '=============== local git status ==============='
 	git status
 	echo '=============== last commit ==============='
 	git log -1
 	echo '=============== bulding from the above ==============='
-	docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor-$* -C pkg/recommender'
 
 .PHONY: create-buildx-builder
 create-buildx-builder:
@@ -93,7 +76,8 @@ create-buildx-builder:
 remove-buildx-builder:
 	docker buildx rm ${BUILDER}
 
-release: build-in-docker create-buildx-builder docker-build remove-buildx-builder docker-push
+.PHONY: release
+release: show-git-info create-buildx-builder docker-build remove-buildx-builder docker-push
 	@echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed"
 
 clean: $(addprefix clean-,$(ALL_ARCHITECTURES))
diff --git a/vertical-pod-autoscaler/pkg/updater/Dockerfile b/vertical-pod-autoscaler/pkg/updater/Dockerfile
index 631584bc5e04..319781044c89 100644
--- a/vertical-pod-autoscaler/pkg/updater/Dockerfile
+++ b/vertical-pod-autoscaler/pkg/updater/Dockerfile
@@ -12,12 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+FROM --platform=$BUILDPLATFORM golang:1.21.5 as builder
+
+ENV GOPATH /gopath/
+ENV PATH $GOPATH/bin:$PATH
+
+COPY . /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
+WORKDIR /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
+
+ARG TARGETOS TARGETARCH
+
+RUN CGO_ENABLED=0 LD_FLAGS=-s GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/updater -mod vendor -o updater-$TARGETARCH
 
 FROM gcr.io/distroless/static:latest
 MAINTAINER Marcin Wielgus "mwielgus@google.com"
 
-ARG ARCH
-COPY updater-$ARCH /updater
+ARG TARGETARCH
+
+COPY --from=builder /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/updater-$TARGETARCH /updater
 
 ENTRYPOINT ["/updater"]
 CMD ["--v=4", "--stderrthreshold=info"]
diff --git a/vertical-pod-autoscaler/pkg/updater/Makefile b/vertical-pod-autoscaler/pkg/updater/Makefile
index 8f7a4a35d2bd..4d4e00b1b9ef 100644
--- a/vertical-pod-autoscaler/pkg/updater/Makefile
+++ b/vertical-pod-autoscaler/pkg/updater/Makefile
@@ -19,13 +19,6 @@ build: clean
 build-binary: clean
 	$(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT}
 
-.PHONY: build-binary-with-vendor
-build-binary-with-vendor: $(addprefix build-binary-with-vendor-,$(ALL_ARCHITECTURES)) clean
-
-.PHONY: build-binary-with-vendor-*
-build-binary-with-vendor-%:
-	$(ENVVAR) GOARCH=$* GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT}-$*
-
 test-unit: clean build
 	$(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS)
 
@@ -42,13 +35,10 @@ ifndef TAG
 	ERR = $(error TAG is undefined)
 	$(ERR)
 endif
-	docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* .
+	docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* -f ./Dockerfile ../../
 
 .PHONY: docker-push
-docker-push: $(addprefix sub-push-,$(ALL_ARCHITECTURES)) push-multi-arch;
-
-.PHONY: sub-push-*
-sub-push-%: docker-build-% do-push-% ;
+docker-push: $(addprefix do-push-,$(ALL_ARCHITECTURES)) push-multi-arch;
 
 .PHONY: do-push-*
 do-push-%:
@@ -68,20 +58,13 @@ push-multi-arch:
 	@for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done
 	docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG)
 
-docker-builder:
-	docker build -t vpa-autoscaling-builder ../../builder
-
-.PHONY: build-in-docker
-build-in-docker: $(addprefix build-in-docker-,$(ALL_ARCHITECTURES))
-
-.PHONY: build-in-docker-*
-build-in-docker-%: clean docker-builder
+.PHONY: show-git-info
+show-git-info:
 	echo '=============== local git status ==============='
 	git status
 	echo '=============== last commit ==============='
 	git log -1
 	echo '=============== bulding from the above ==============='
-	docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor-$* -C pkg/updater'
 
 .PHONY: create-buildx-builder
 create-buildx-builder:
@@ -92,7 +75,7 @@ remove-buildx-builder:
 	docker buildx rm ${BUILDER}
 
 .PHONY: release
-release: build-in-docker create-buildx-builder docker-build remove-buildx-builder docker-push
+release: show-git-info create-buildx-builder docker-build remove-buildx-builder docker-push
 	@echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed"
 
 clean: $(addprefix clean-,$(ALL_ARCHITECTURES))