diff --git a/arch/arm/mach-imx/imx8/cpu.c b/arch/arm/mach-imx/imx8/cpu.c
index c511e909fd..63de88fd69 100644
--- a/arch/arm/mach-imx/imx8/cpu.c
+++ b/arch/arm/mach-imx/imx8/cpu.c
@@ -185,6 +185,14 @@ int arch_cpu_init(void)
 
 	gd->arch.ipc_channel_handle = ipcHndl;
 
+/* Dual bootloader feature will require CAAM access, but JR0 and JR1 will be
+ * assigned to seco for imx8, use JR3 instead.
+ */
+#if defined(CONFIG_SPL_BUILD) && defined(CONFIG_DUAL_BOOTLOADER)
+	sc_pm_set_resource_power_mode(ipcHndl, SC_R_CAAM_JR3, SC_PM_PW_MODE_ON);
+	sc_pm_set_resource_power_mode(ipcHndl, SC_R_CAAM_JR3_OUT, SC_PM_PW_MODE_ON);
+#endif
+
 	if (IS_ENABLED(CONFIG_XEN))
 		return 0;
 
@@ -203,14 +211,6 @@ int arch_cpu_init(void)
 		return 0;
 #endif
 
-/* Dual bootloader feature will require CAAM access, but JR0 and JR1 will be
- * assigned to seco for imx8, use JR3 instead.
- */
-#if defined(CONFIG_SPL_BUILD) && defined(CONFIG_DUAL_BOOTLOADER)
-	sc_pm_set_resource_power_mode(ipcHndl, SC_R_CAAM_JR3, SC_PM_PW_MODE_ON);
-	sc_pm_set_resource_power_mode(ipcHndl, SC_R_CAAM_JR3_OUT, SC_PM_PW_MODE_ON);
-#endif
-
 	return 0;
 }