diff --git a/components/esp_hw_support/include/esp_private/rtc_clk.h b/components/esp_hw_support/include/esp_private/rtc_clk.h
index 52610fa0d1ce..4480780217f4 100644
--- a/components/esp_hw_support/include/esp_private/rtc_clk.h
+++ b/components/esp_hw_support/include/esp_private/rtc_clk.h
@@ -18,11 +18,12 @@ extern "C" {
  * @brief Switch CPU clock source to XTAL, and let cpu frequency equal to main XTAL frequency.
  *
  * This function does not disable CPU's source PLL. If the PLL requires to be disabled to save power, please call
- * `rtc_clk_cpu_freq_set_xtal` instead. It does one extra check (if necessary) to see whether can disable the
- * corresponding PLL after switching the CPU clock source to XTAL.
+ * `rtc_clk_cpu_freq_set_xtal` instead. It will always disable the corresponding PLL after switching the CPU clock
+ * source to XTAL (except for S2).
  *
- * Currently, this function should only be called in `esp_restart_noos` and `esp_restart_noos_dig` to switch the CPU
- * clock source back to XTAL (by default) before reset.
+ * Currently, this function is only called in `esp_restart_noos` and `esp_restart_noos_dig` to switch the CPU
+ * clock source back to XTAL (by default) before reset, and in `esp_sleep_start` to switch CPU clock source to XTAL
+ * before entering sleep for PMU supported chips.
  */
 void rtc_clk_cpu_set_to_default_config(void);
 
@@ -32,12 +33,6 @@ void rtc_clk_cpu_set_to_default_config(void);
  * Currently, this function is only used for tracking whether USB Serial/JTAG is using the 48MHz PHY clock
  *
  * Note: Calling this function only helps to not disable the BBPLL clock in `rtc_clk_cpu_freq_set_config`.
- *       For light and deep sleep, whether to disable the BBPLL in the internal call to `rtc_clk_cpu_freq_set_xtal`
- *       varies for targets.
- *       On ESP32C3/S3, USB CDC device can not function properly during sleep due to the lack of APB clock. Therefore.
- *       `rtc_clk_cpu_freq_set_xtal` will always disable BBPLL, no matter whether BBPLL has any consumer.
- *       On ESP32C6/H2, USB CDC device can maintain the minimum connection with the host during sleep, so
- *       `rtc_clk_cpu_freq_set_xtal` will check for BBPLL consumers, and keep BBPLL if USB Serial/JTAG is in use.
  */
 void rtc_clk_bbpll_add_consumer(void);
 
diff --git a/components/esp_hw_support/port/esp32c5/include/soc/rtc.h b/components/esp_hw_support/port/esp32c5/include/soc/rtc.h
index 92a2c4f5c255..a11d201f20cf 100644
--- a/components/esp_hw_support/port/esp32c5/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32c5/include/soc/rtc.h
@@ -307,8 +307,9 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32C5, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note This function always disables BBPLL after switching the CPU clock source to XTAL for power saving purpose.
+ * If this is unwanted, please use rtc_clk_cpu_freq_set_config. It helps to check whether USB Serial JTAG is in use,
+ * if so, then BBPLL will not be turned off.
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32c5/pmu_param.c b/components/esp_hw_support/port/esp32c5/pmu_param.c
index e676f8c478df..6f44a1b4e17d 100644
--- a/components/esp_hw_support/port/esp32c5/pmu_param.c
+++ b/components/esp_hw_support/port/esp32c5/pmu_param.c
@@ -291,18 +291,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2active_backup_en      = 0, \
         .hp_modem2active_backup_en      = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_GDMA)      | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_MODEM_RETENTION_CONFIG_DEFAULT() {   \
@@ -314,17 +303,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2modem_backup_mode     = PMU_HP_RETENTION_REGDMA_CONFIG(0, 1), \
         .hp_sleep2modem_backup_en       = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_SLEEP_RETENTION_CONFIG_DEFAULT() {   \
@@ -341,17 +320,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_modem2sleep_backup_en       = 0, \
         .hp_active2sleep_backup_en      = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 const pmu_hp_system_retention_param_t * pmu_hp_system_retention_param_default(pmu_hp_mode_t mode)
diff --git a/components/esp_hw_support/port/esp32c5/private_include/pmu_param.h b/components/esp_hw_support/port/esp32c5/private_include/pmu_param.h
index 8e9909749b81..bbcc0c8e70f9 100644
--- a/components/esp_hw_support/port/esp32c5/private_include/pmu_param.h
+++ b/components/esp_hw_support/port/esp32c5/private_include/pmu_param.h
@@ -87,7 +87,7 @@ const pmu_hp_system_analog_param_t* pmu_hp_system_analog_param_default(pmu_hp_mo
 
 typedef struct {
     pmu_hp_backup_reg_t     retention;
-    uint32_t                backup_clk;
+    uint32_t                backup_clk; // icg_func
 } pmu_hp_system_retention_param_t;
 
 const pmu_hp_system_retention_param_t* pmu_hp_system_retention_param_default(pmu_hp_mode_t mode);
diff --git a/components/esp_hw_support/port/esp32c5/rtc_clk.c b/components/esp_hw_support/port/esp32c5/rtc_clk.c
index e3d1052c4b07..23a3941c9d2b 100644
--- a/components/esp_hw_support/port/esp32c5/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c5/rtc_clk.c
@@ -383,10 +383,7 @@ void rtc_clk_cpu_freq_set_config_fast(const rtc_cpu_freq_config_t *config)
 void rtc_clk_cpu_freq_set_xtal(void)
 {
     rtc_clk_cpu_set_to_default_config();
-    // We don't turn off the bbpll if some consumers depend on bbpll
-    if (!s_bbpll_digi_consumers_ref_count) {
-        rtc_clk_bbpll_disable();
-    }
+    rtc_clk_bbpll_disable();
 }
 
 void rtc_clk_cpu_set_to_default_config(void)
@@ -394,6 +391,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
+    s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
diff --git a/components/esp_hw_support/port/esp32c6/include/soc/rtc.h b/components/esp_hw_support/port/esp32c6/include/soc/rtc.h
index 8d508d5df39b..3880b325e5b7 100644
--- a/components/esp_hw_support/port/esp32c6/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32c6/include/soc/rtc.h
@@ -338,8 +338,9 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32C6, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note This function always disables BBPLL after switching the CPU clock source to XTAL for power saving purpose.
+ * If this is unwanted, please use rtc_clk_cpu_freq_set_config. It helps to check whether USB Serial JTAG is in use,
+ * if so, then BBPLL will not be turned off.
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32c6/pmu_param.c b/components/esp_hw_support/port/esp32c6/pmu_param.c
index 044c300688e5..23acfc344727 100644
--- a/components/esp_hw_support/port/esp32c6/pmu_param.c
+++ b/components/esp_hw_support/port/esp32c6/pmu_param.c
@@ -291,18 +291,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2active_backup_en      = 0, \
         .hp_modem2active_backup_en      = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_GDMA)      | \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_MODEM_RETENTION_CONFIG_DEFAULT() {   \
@@ -314,17 +303,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2modem_backup_mode     = PMU_HP_RETENTION_REGDMA_CONFIG(0, 1), \
         .hp_sleep2modem_backup_en       = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_SLEEP_RETENTION_CONFIG_DEFAULT() {   \
@@ -341,18 +320,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_modem2sleep_backup_en       = 0, \
         .hp_active2sleep_backup_en      = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_GDMA)      | \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 const pmu_hp_system_retention_param_t * pmu_hp_system_retention_param_default(pmu_hp_mode_t mode)
diff --git a/components/esp_hw_support/port/esp32c6/private_include/pmu_param.h b/components/esp_hw_support/port/esp32c6/private_include/pmu_param.h
index fdef78b3300b..155c3e810f4a 100644
--- a/components/esp_hw_support/port/esp32c6/private_include/pmu_param.h
+++ b/components/esp_hw_support/port/esp32c6/private_include/pmu_param.h
@@ -92,7 +92,7 @@ const pmu_hp_system_analog_param_t* pmu_hp_system_analog_param_default(pmu_hp_mo
 
 typedef struct {
     pmu_hp_backup_reg_t     retention;
-    uint32_t                backup_clk;
+    uint32_t                backup_clk; // icg_func
 } pmu_hp_system_retention_param_t;
 
 const pmu_hp_system_retention_param_t* pmu_hp_system_retention_param_default(pmu_hp_mode_t mode);
diff --git a/components/esp_hw_support/port/esp32c6/rtc_clk.c b/components/esp_hw_support/port/esp32c6/rtc_clk.c
index 521e2fd03f54..c36c48d85934 100644
--- a/components/esp_hw_support/port/esp32c6/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c6/rtc_clk.c
@@ -333,10 +333,7 @@ void rtc_clk_cpu_freq_set_config_fast(const rtc_cpu_freq_config_t *config)
 void rtc_clk_cpu_freq_set_xtal(void)
 {
     rtc_clk_cpu_set_to_default_config();
-    // We don't turn off the bbpll if some consumers depend on bbpll
-    if (!s_bbpll_digi_consumers_ref_count) {
-        rtc_clk_bbpll_disable();
-    }
+    rtc_clk_bbpll_disable();
 }
 
 void rtc_clk_cpu_set_to_default_config(void)
@@ -344,6 +341,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
+    s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
diff --git a/components/esp_hw_support/port/esp32c61/include/soc/rtc.h b/components/esp_hw_support/port/esp32c61/include/soc/rtc.h
index 12c15bc101d4..b04132733988 100644
--- a/components/esp_hw_support/port/esp32c61/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32c61/include/soc/rtc.h
@@ -307,8 +307,9 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32C61, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note This function always disables BBPLL after switching the CPU clock source to XTAL for power saving purpose.
+ * If this is unwanted, please use rtc_clk_cpu_freq_set_config. It helps to check whether USB Serial JTAG is in use,
+ * if so, then BBPLL will not be turned off.
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32c61/pmu_param.c b/components/esp_hw_support/port/esp32c61/pmu_param.c
index 37105bef5381..4883b9067723 100644
--- a/components/esp_hw_support/port/esp32c61/pmu_param.c
+++ b/components/esp_hw_support/port/esp32c61/pmu_param.c
@@ -290,18 +290,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2active_backup_en      = 0, \
         .hp_modem2active_backup_en      = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_GDMA)      | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_MODEM_RETENTION_CONFIG_DEFAULT() {   \
@@ -313,17 +302,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2modem_backup_mode     = PMU_HP_RETENTION_REGDMA_CONFIG(0, 1), \
         .hp_sleep2modem_backup_en       = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_SLEEP_RETENTION_CONFIG_DEFAULT() {   \
@@ -340,17 +319,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_modem2sleep_backup_en       = 0, \
         .hp_active2sleep_backup_en      = 0, \
     }, \
-    .backup_clk = (                       \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)    | \
-        BIT(PMU_ICG_FUNC_ENA_TG0)       | \
-        BIT(PMU_ICG_FUNC_ENA_TG1)       | \
-        BIT(PMU_ICG_FUNC_ENA_HPBUS)     | \
-        BIT(PMU_ICG_FUNC_ENA_MSPI)      | \
-        BIT(PMU_ICG_FUNC_ENA_IOMUX)     | \
-        BIT(PMU_ICG_FUNC_ENA_SPI2)      | \
-        BIT(PMU_ICG_FUNC_ENA_UART0)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSTIMER)    \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 const pmu_hp_system_retention_param_t * pmu_hp_system_retention_param_default(pmu_hp_mode_t mode)
diff --git a/components/esp_hw_support/port/esp32c61/private_include/pmu_param.h b/components/esp_hw_support/port/esp32c61/private_include/pmu_param.h
index 80aaac61ee96..4a7516a4f932 100644
--- a/components/esp_hw_support/port/esp32c61/private_include/pmu_param.h
+++ b/components/esp_hw_support/port/esp32c61/private_include/pmu_param.h
@@ -90,7 +90,7 @@ const pmu_hp_system_analog_param_t* pmu_hp_system_analog_param_default(pmu_hp_mo
 
 typedef struct {
     pmu_hp_backup_reg_t     retention;
-    uint32_t                backup_clk;
+    uint32_t                backup_clk; // icg_func
 } pmu_hp_system_retention_param_t;
 
 const pmu_hp_system_retention_param_t* pmu_hp_system_retention_param_default(pmu_hp_mode_t mode);
diff --git a/components/esp_hw_support/port/esp32c61/rtc_clk.c b/components/esp_hw_support/port/esp32c61/rtc_clk.c
index e46cdbe1c480..5c32d173585a 100644
--- a/components/esp_hw_support/port/esp32c61/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c61/rtc_clk.c
@@ -327,10 +327,7 @@ void rtc_clk_cpu_freq_set_config_fast(const rtc_cpu_freq_config_t *config)
 void rtc_clk_cpu_freq_set_xtal(void)
 {
     rtc_clk_cpu_set_to_default_config();
-    // We don't turn off the bbpll if some consumers depend on bbpll
-    if (!s_bbpll_digi_consumers_ref_count) {
-        rtc_clk_bbpll_disable();
-    }
+    rtc_clk_bbpll_disable();
 }
 
 void rtc_clk_cpu_set_to_default_config(void)
@@ -338,6 +335,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
+    s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
diff --git a/components/esp_hw_support/port/esp32h2/include/soc/rtc.h b/components/esp_hw_support/port/esp32h2/include/soc/rtc.h
index dfd1f96f0197..fa820b48d82b 100644
--- a/components/esp_hw_support/port/esp32h2/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32h2/include/soc/rtc.h
@@ -354,8 +354,9 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32H2, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note This function always disables BBPLL after switching the CPU clock source to XTAL for power saving purpose.
+ * If this is unwanted, please use rtc_clk_cpu_freq_set_config. It helps to check whether USB Serial JTAG is in use,
+ * if so, then BBPLL will not be turned off.
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32h2/pmu_param.c b/components/esp_hw_support/port/esp32h2/pmu_param.c
index 036792a6faae..56f679a91451 100644
--- a/components/esp_hw_support/port/esp32h2/pmu_param.c
+++ b/components/esp_hw_support/port/esp32h2/pmu_param.c
@@ -290,18 +290,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2active_backup_en      = 0, \
         .hp_modem2active_backup_en      = 0, \
     }, \
-    .backup_clk = (                             \
-        BIT(PMU_ICG_FUNC_ENA_GDMA)              \
-        | BIT(PMU_ICG_FUNC_ENA_REGDMA)          \
-        | BIT(PMU_ICG_FUNC_ENA_TG0)             \
-        | BIT(PMU_ICG_FUNC_ENA_HPBUS)           \
-        | BIT(PMU_ICG_FUNC_ENA_MSPI)            \
-        | BIT(PMU_ICG_FUNC_ENA_IOMUX)           \
-        | BIT(PMU_ICG_FUNC_ENA_SPI2)            \
-        | BIT(PMU_ICG_FUNC_ENA_SEC)             \
-        | BIT(PMU_ICG_FUNC_ENA_PWM)             \
-        | BIT(PMU_ICG_FUNC_ENA_SYSTIMER)        \
-        | BIT(PMU_ICG_FUNC_ENA_UART0)),         \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_MODEM_RETENTION_CONFIG_DEFAULT() {   \
@@ -313,16 +302,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2modem_backup_mode     = PMU_HP_RETENTION_REGDMA_CONFIG(0, 1), \
         .hp_sleep2modem_backup_en       = 0, \
     }, \
-    .backup_clk = (BIT(PMU_ICG_FUNC_ENA_REGDMA) \
-        | BIT(PMU_ICG_FUNC_ENA_TG0)             \
-        | BIT(PMU_ICG_FUNC_ENA_HPBUS)           \
-        | BIT(PMU_ICG_FUNC_ENA_MSPI)            \
-        | BIT(PMU_ICG_FUNC_ENA_IOMUX)           \
-        | BIT(PMU_ICG_FUNC_ENA_SPI2)            \
-        | BIT(PMU_ICG_FUNC_ENA_SEC)            \
-        | BIT(PMU_ICG_FUNC_ENA_PWM)            \
-        | BIT(PMU_ICG_FUNC_ENA_SYSTIMER)        \
-        | BIT(PMU_ICG_FUNC_ENA_UART0)),         \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_SLEEP_RETENTION_CONFIG_DEFAULT() {   \
@@ -339,18 +319,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_modem2sleep_backup_en       = 0, \
         .hp_active2sleep_backup_en      = 0, \
     }, \
-    .backup_clk = (                             \
-        BIT(PMU_ICG_FUNC_ENA_GDMA)              \
-        | BIT(PMU_ICG_FUNC_ENA_REGDMA)          \
-        | BIT(PMU_ICG_FUNC_ENA_TG0)             \
-        | BIT(PMU_ICG_FUNC_ENA_HPBUS)           \
-        | BIT(PMU_ICG_FUNC_ENA_MSPI)            \
-        | BIT(PMU_ICG_FUNC_ENA_IOMUX)           \
-        | BIT(PMU_ICG_FUNC_ENA_SPI2)            \
-        | BIT(PMU_ICG_FUNC_ENA_SEC)             \
-        | BIT(PMU_ICG_FUNC_ENA_PWM)             \
-        | BIT(PMU_ICG_FUNC_ENA_SYSTIMER)        \
-        | BIT(PMU_ICG_FUNC_ENA_UART0)),         \
+    .backup_clk = 0xffffffff,   \
 }
 
 const pmu_hp_system_retention_param_t * pmu_hp_system_retention_param_default(pmu_hp_mode_t mode)
diff --git a/components/esp_hw_support/port/esp32h2/private_include/pmu_param.h b/components/esp_hw_support/port/esp32h2/private_include/pmu_param.h
index 58fbf97ff48b..bb78a1e7759f 100644
--- a/components/esp_hw_support/port/esp32h2/private_include/pmu_param.h
+++ b/components/esp_hw_support/port/esp32h2/private_include/pmu_param.h
@@ -85,7 +85,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
 
 typedef struct {
     pmu_hp_backup_reg_t     retention;
-    uint32_t                backup_clk;
+    uint32_t                backup_clk; // icg_func
 } pmu_hp_system_retention_param_t;
 
 const pmu_hp_system_retention_param_t * pmu_hp_system_retention_param_default(pmu_hp_mode_t mode);
@@ -407,7 +407,7 @@ typedef struct {
 
 typedef struct pmu_sleep_machine_constant {
     struct {
-        uint16_t    min_slp_time_us;            /* Mininum sleep protection time (unit: microsecond) */
+        uint16_t    min_slp_time_us;            /* Minimum sleep protection time (unit: microsecond) */
         uint8_t     reserved0;
         uint16_t    reserved1;
         uint16_t    analog_wait_time_us;        /* LP LDO power up wait time (unit: microsecond) */
@@ -418,7 +418,7 @@ typedef struct pmu_sleep_machine_constant {
         uint16_t    power_up_wait_time_us;      /* (unit: microsecond) */
     } lp;
     struct {
-        uint16_t    min_slp_time_us;            /* Mininum sleep protection time (unit: microsecond) */
+        uint16_t    min_slp_time_us;            /* Minimum sleep protection time (unit: microsecond) */
         uint16_t    analog_wait_time_us;        /* HP LDO power up wait time (unit: microsecond) */
         uint16_t    power_supply_wait_time_us;  /* (unit: microsecond) */
         uint16_t    power_up_wait_time_us;      /* (unit: microsecond) */
diff --git a/components/esp_hw_support/port/esp32h2/rtc_clk.c b/components/esp_hw_support/port/esp32h2/rtc_clk.c
index 0c6664fdce84..7a8a82ef5948 100644
--- a/components/esp_hw_support/port/esp32h2/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32h2/rtc_clk.c
@@ -393,10 +393,7 @@ void rtc_clk_cpu_freq_set_config_fast(const rtc_cpu_freq_config_t *config)
 void rtc_clk_cpu_freq_set_xtal(void)
 {
     rtc_clk_cpu_set_to_default_config();
-    // We don't turn off the bbpll if some consumers only depends on bbpll
-    if (!s_bbpll_digi_consumers_ref_count) {
-        rtc_clk_bbpll_disable();
-    }
+    rtc_clk_bbpll_disable();
 }
 
 void rtc_clk_cpu_set_to_default_config(void)
@@ -404,6 +401,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
+    s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 soc_xtal_freq_t rtc_clk_xtal_freq_get(void)
diff --git a/components/esp_hw_support/port/esp32p4/include/soc/rtc.h b/components/esp_hw_support/port/esp32p4/include/soc/rtc.h
index 183f8a79fe60..43acaa253faf 100644
--- a/components/esp_hw_support/port/esp32p4/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32p4/include/soc/rtc.h
@@ -350,8 +350,8 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32C6, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note On ESP32P4, this function always disables CPLL after switching the CPU clock source to XTAL,
+ * since there is no peripheral relies on CPLL clock (except Flash/PSRAM if their clock source selects CPLL).
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32p4/pmu_param.c b/components/esp_hw_support/port/esp32p4/pmu_param.c
index 0905a1268a4a..c1c8896f37ab 100644
--- a/components/esp_hw_support/port/esp32p4/pmu_param.c
+++ b/components/esp_hw_support/port/esp32p4/pmu_param.c
@@ -208,18 +208,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_sleep2active_backup_en      = 0, \
         .hp_modem2active_backup_en      = 0, \
     }, \
-    .backup_clk = ( \
-        BIT(PMU_ICG_FUNC_ENA_L2MEM_MEM)     | \
-        BIT(PMU_ICG_FUNC_ENA_L2MEM_SYS)     | \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)        | \
-        BIT(PMU_ICG_FUNC_ENA_HP_CLKRST)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSREG_APB)    | \
-        BIT(PMU_ICG_FUNC_ENA_ICM_CPU)       | \
-        BIT(PMU_ICG_FUNC_ENA_ICM_APB)       | \
-        BIT(PMU_ICG_FUNC_ENA_ICM_SYS)       | \
-        BIT(PMU_ICG_FUNC_ENA_ICM_MEM)       | \
-        BIT(PMU_ICG_FUNC_ENA_INTRMTX_APB) \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 #define PMU_HP_SLEEP_RETENTION_CONFIG_DEFAULT() {   \
@@ -236,18 +225,7 @@ const pmu_hp_system_analog_param_t * pmu_hp_system_analog_param_default(pmu_hp_m
         .hp_modem2sleep_backup_en       = 0, \
         .hp_active2sleep_backup_en      = 0, \
     }, \
-    .backup_clk = ( \
-        BIT(PMU_ICG_FUNC_ENA_L2MEM_MEM)     | \
-        BIT(PMU_ICG_FUNC_ENA_L2MEM_SYS)     | \
-        BIT(PMU_ICG_FUNC_ENA_REGDMA)        | \
-        BIT(PMU_ICG_FUNC_ENA_HP_CLKRST)     | \
-        BIT(PMU_ICG_FUNC_ENA_SYSREG_APB)    | \
-        BIT(PMU_ICG_FUNC_ENA_ICM_CPU)       | \
-        BIT(PMU_ICG_FUNC_ENA_ICM_APB)       | \
-        BIT(PMU_ICG_FUNC_ENA_ICM_SYS)       | \
-        BIT(PMU_ICG_FUNC_ENA_ICM_MEM)       | \
-        BIT(PMU_ICG_FUNC_ENA_INTRMTX_APB) \
-    ) \
+    .backup_clk = 0xffffffff,   \
 }
 
 const pmu_hp_system_retention_param_t * pmu_hp_system_retention_param_default(pmu_hp_mode_t mode)
diff --git a/components/esp_hw_support/port/esp32p4/private_include/pmu_param.h b/components/esp_hw_support/port/esp32p4/private_include/pmu_param.h
index 9c8be63ac555..37e67fc48318 100644
--- a/components/esp_hw_support/port/esp32p4/private_include/pmu_param.h
+++ b/components/esp_hw_support/port/esp32p4/private_include/pmu_param.h
@@ -86,7 +86,7 @@ const pmu_hp_system_analog_param_t* pmu_hp_system_analog_param_default(pmu_hp_mo
 
 typedef struct {
     pmu_hp_backup_reg_t     retention;
-    uint32_t                backup_clk;
+    uint32_t                backup_clk; // icg_func
 } pmu_hp_system_retention_param_t;
 
 const pmu_hp_system_retention_param_t* pmu_hp_system_retention_param_default(pmu_hp_mode_t mode);
diff --git a/components/esp_hw_support/port/esp32p4/rtc_clk.c b/components/esp_hw_support/port/esp32p4/rtc_clk.c
index aa3bfd967505..f9e19ce08c44 100644
--- a/components/esp_hw_support/port/esp32p4/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32p4/rtc_clk.c
@@ -426,6 +426,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1, true);
+    s_cur_cpll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 soc_xtal_freq_t rtc_clk_xtal_freq_get(void)
diff --git a/components/esp_hw_support/sleep_modes.c b/components/esp_hw_support/sleep_modes.c
index 22201a21ced5..8a3e27640eef 100644
--- a/components/esp_hw_support/sleep_modes.c
+++ b/components/esp_hw_support/sleep_modes.c
@@ -810,7 +810,16 @@ static esp_err_t IRAM_ATTR esp_sleep_start(uint32_t pd_flags, esp_sleep_mode_t m
     // Save current frequency and switch to XTAL
     rtc_cpu_freq_config_t cpu_freq_config;
     rtc_clk_cpu_freq_get_config(&cpu_freq_config);
+#if SOC_PMU_SUPPORTED
+    // For PMU supported chips, CPU's PLL power can be turned off by PMU, so no need to disable the PLL at here.
+    // Leaving PLL on at this stage also helps USJ keep connection and retention operation (if they rely on this PLL).
+    rtc_clk_cpu_set_to_default_config();
+#else
+    // For earlier chips, there is no PMU module that can turn off the CPU's PLL, so it has to be disabled at here to save the power consumption.
+    // Though ESP32C3/S3 has USB CDC device, it can not function properly during sleep due to the lack of APB clock (before C6, USJ relies on APB clock to work).
+    // Therefore, we will always disable CPU's PLL (i.e. BBPLL).
     rtc_clk_cpu_freq_set_xtal();
+#endif
 
 #if SOC_PM_SUPPORT_EXT0_WAKEUP
     // Configure pins for external wakeup