diff --git a/vertical-pod-autoscaler/hack/vpa-process-yamls.sh b/vertical-pod-autoscaler/hack/vpa-process-yamls.sh
index 72033d904ac5..41e19ba192bc 100755
--- a/vertical-pod-autoscaler/hack/vpa-process-yamls.sh
+++ b/vertical-pod-autoscaler/hack/vpa-process-yamls.sh
@@ -74,6 +74,10 @@ for i in $COMPONENTS; do
   if [[ ${ACTION} == print ]]; then
     ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh $(script_path $i)
   else
-    ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh $(script_path $i) | kubectl ${ACTION} -f - || true
+    EXTRA_FLAGS=""
+    if [[ ${ACTION} == delete ]]; then
+      EXTRA_FLAGS+=" --ignore-not-found"
+    fi
+    ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh $(script_path $i) | kubectl ${ACTION} ${EXTRA_FLAGS} -f - || true
   fi
 done
diff --git a/vertical-pod-autoscaler/pkg/admission-controller/delete-webhook.sh b/vertical-pod-autoscaler/pkg/admission-controller/delete-webhook.sh
index 9df625d47540..1a1e2c233816 100644
--- a/vertical-pod-autoscaler/pkg/admission-controller/delete-webhook.sh
+++ b/vertical-pod-autoscaler/pkg/admission-controller/delete-webhook.sh
@@ -19,5 +19,4 @@ set -e
 
 echo "Unregistering VPA admission controller webhook"
 
-kubectl delete -n kube-system mutatingwebhookconfiguration.v1.admissionregistration.k8s.io vpa-webhook-config
-
+kubectl delete -n kube-system mutatingwebhookconfiguration.v1.admissionregistration.k8s.io vpa-webhook-config --ignore-not-found
diff --git a/vertical-pod-autoscaler/pkg/admission-controller/rmcerts.sh b/vertical-pod-autoscaler/pkg/admission-controller/rmcerts.sh
index 5f1e3f8813b2..8703612b4a7d 100755
--- a/vertical-pod-autoscaler/pkg/admission-controller/rmcerts.sh
+++ b/vertical-pod-autoscaler/pkg/admission-controller/rmcerts.sh
@@ -20,5 +20,5 @@
 set -e
 
 echo "Deleting VPA Admission Controller certs."
-kubectl delete secret --namespace=kube-system vpa-tls-certs
-kubectl delete secret --namespace=kube-system --ignore-not-found=true vpa-e2e-certs
+kubectl delete secret --namespace=kube-system vpa-tls-certs --ignore-not-found
+kubectl delete secret --namespace=kube-system vpa-e2e-certs --ignore-not-found