diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/secretmanager/GcpSecretManagerBootstrapConfiguration.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/secretmanager/GcpSecretManagerBootstrapConfiguration.java
index ea03df298b..4d0c1e5d5b 100644
--- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/secretmanager/GcpSecretManagerBootstrapConfiguration.java
+++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/secretmanager/GcpSecretManagerBootstrapConfiguration.java
@@ -27,7 +27,6 @@
 import com.google.cloud.spring.core.UserAgentHeaderProvider;
 import com.google.cloud.spring.secretmanager.SecretManagerPropertySourceLocator;
 import com.google.cloud.spring.secretmanager.SecretManagerTemplate;
-import com.google.protobuf.ByteString;
 
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -35,7 +34,6 @@
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.core.convert.converter.Converter;
 import org.springframework.core.env.ConfigurableEnvironment;
 
 /**
@@ -61,23 +59,6 @@ public GcpSecretManagerBootstrapConfiguration(
 		this.gcpProjectIdProvider = properties.getProjectId() != null
 				? properties::getProjectId
 				: new DefaultGcpProjectIdProvider();
-
-		// Registers {@link ByteString} type converters to convert to String and byte[].
-		configurableEnvironment.getConversionService().addConverter(
-				new Converter<ByteString, String>() {
-					@Override
-					public String convert(ByteString source) {
-						return source.toStringUtf8();
-					}
-				});
-
-		configurableEnvironment.getConversionService().addConverter(
-				new Converter<ByteString, byte[]>() {
-					@Override
-					public byte[] convert(ByteString source) {
-						return source.toByteArray();
-					}
-				});
 	}
 
 	@Bean
diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/secretmanager/GcpSecretManagerEnvironmentPostProcessor.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/secretmanager/GcpSecretManagerEnvironmentPostProcessor.java
new file mode 100644
index 0000000000..cdf802597a
--- /dev/null
+++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/secretmanager/GcpSecretManagerEnvironmentPostProcessor.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2017-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spring.autoconfigure.secretmanager;
+
+import com.google.protobuf.ByteString;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.env.EnvironmentPostProcessor;
+import org.springframework.core.convert.converter.Converter;
+import org.springframework.core.env.ConfigurableEnvironment;
+
+/**
+ * Registers converters used by Spring Cloud GCP Secret Manager.
+ */
+public class GcpSecretManagerEnvironmentPostProcessor implements EnvironmentPostProcessor {
+
+	@Override
+	public void postProcessEnvironment(ConfigurableEnvironment environment, SpringApplication application) {
+		boolean isSecretManagerEnabled =
+				Boolean.parseBoolean(
+						environment.getProperty("spring.cloud.gcp.secretmanager.enabled", "true"));
+
+		if (isSecretManagerEnabled) {
+			// Registers {@link ByteString} type converters to convert to String and byte[].
+			environment.getConversionService().addConverter(
+					new Converter<ByteString, String>() {
+						@Override
+						public String convert(ByteString source) {
+							return source.toStringUtf8();
+						}
+					});
+
+			environment.getConversionService().addConverter(
+					new Converter<ByteString, byte[]>() {
+						@Override
+						public byte[] convert(ByteString source) {
+							return source.toByteArray();
+						}
+					});
+		}
+	}
+}
diff --git a/spring-cloud-gcp-autoconfigure/src/main/resources/META-INF/spring.factories b/spring-cloud-gcp-autoconfigure/src/main/resources/META-INF/spring.factories
index 941834db67..106229519f 100644
--- a/spring-cloud-gcp-autoconfigure/src/main/resources/META-INF/spring.factories
+++ b/spring-cloud-gcp-autoconfigure/src/main/resources/META-INF/spring.factories
@@ -31,4 +31,5 @@ com.google.cloud.spring.autoconfigure.config.GcpConfigBootstrapConfiguration,\
 com.google.cloud.spring.autoconfigure.secretmanager.GcpSecretManagerBootstrapConfiguration
 
 org.springframework.boot.env.EnvironmentPostProcessor=\
-com.google.cloud.spring.autoconfigure.sql.CloudSqlEnvironmentPostProcessor
\ No newline at end of file
+com.google.cloud.spring.autoconfigure.sql.CloudSqlEnvironmentPostProcessor,\
+com.google.cloud.spring.autoconfigure.secretmanager.GcpSecretManagerEnvironmentPostProcessor
diff --git a/spring-cloud-gcp-samples/pom.xml b/spring-cloud-gcp-samples/pom.xml
index 81a15ed2d8..e8093a3480 100644
--- a/spring-cloud-gcp-samples/pom.xml
+++ b/spring-cloud-gcp-samples/pom.xml
@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>org.springframework.boot</groupId>
 		<artifactId>spring-boot-starter-parent</artifactId>
-		<version>2.4.9</version>
+		<version>2.5.3</version>
 		<relativePath/>
 	</parent>