diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 0ae371f..a9169dd 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -419,9 +419,9 @@ ENTRY(pax_enter_kernel_user)
 	sub phys_base(%rip),%rbx
 
 #ifdef CONFIG_PARAVIRT
-	pushq %rdi
 	cmpl $0, pv_info+PARAVIRT_enabled
 	jz 1f
+	pushq %rdi
 	i = 0
 	.rept USER_PGD_PTRS
 	mov i*8(%rbx),%rsi
@@ -430,6 +430,7 @@ ENTRY(pax_enter_kernel_user)
 	call PARA_INDIRECT(pv_mmu_ops+PV_MMU_set_pgd_batched)
 	i = i + 1
 	.endr
+	popq %rdi
 	jmp 2f
 1:
 #endif
@@ -441,7 +442,7 @@ ENTRY(pax_enter_kernel_user)
 	.endr
 
 #ifdef CONFIG_PARAVIRT
-2:	popq %rdi
+2:
 #endif
 	SET_RDI_INTO_CR3
 
@@ -492,7 +493,6 @@ ENTRY(pax_exit_kernel_user)
 	call PARA_INDIRECT(pv_mmu_ops+PV_MMU_set_pgd_batched)
 	i = i + 1
 	.endr
-	PV_RESTORE_REGS(CLBR_RDI)
 	jmp 2f
 1:
 #endif
@@ -507,6 +507,10 @@ ENTRY(pax_exit_kernel_user)
 2:
 #endif
 
+#ifdef CONFIG_PARAVIRT
+	PV_RESTORE_REGS(CLBR_RDI)
+#endif
+
 	popq %rbx
 	popq %rdi
 	pax_force_retaddr