1315 lines
45 KiB
ArmAsm
1315 lines
45 KiB
ArmAsm
/* $OpenBSD: vector.S,v 1.94 2023/07/31 04:01:07 guenther Exp $ */
|
|
/* $NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $ */
|
|
|
|
/*
|
|
* Copyright (c) 2001 Wasabi Systems, Inc.
|
|
* All rights reserved.
|
|
*
|
|
* Written by Frank van der Linden for Wasabi Systems, Inc.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* 3. All advertising materials mentioning features or use of this software
|
|
* must display the following acknowledgement:
|
|
* This product includes software developed for the NetBSD Project by
|
|
* Wasabi Systems, Inc.
|
|
* 4. The name of Wasabi Systems, Inc. may not be used to endorse
|
|
* or promote products derived from this software without specific prior
|
|
* written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
|
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
|
|
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
* POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
/*-
|
|
* Copyright (c) 1998 The NetBSD Foundation, Inc.
|
|
* All rights reserved.
|
|
*
|
|
* This code is derived from software contributed to The NetBSD Foundation
|
|
* by Charles M. Hannum.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
|
|
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
|
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
|
|
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
* POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <machine/param.h>
|
|
#include <machine/i8259.h>
|
|
#include <machine/i82093reg.h>
|
|
#include <machine/i82489reg.h>
|
|
#include <machine/asm.h>
|
|
#include <machine/frameasm.h>
|
|
#include <machine/segments.h>
|
|
#include <machine/trap.h>
|
|
#include <machine/intr.h>
|
|
#include <machine/psl.h>
|
|
#include <machine/codepatch.h>
|
|
#include <machine/specialreg.h>
|
|
|
|
#include "ioapic.h"
|
|
#include "lapic.h"
|
|
#include "assym.h"
|
|
#include "xen.h"
|
|
#include "hyperv.h"
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* Trap and fault vector routines
|
|
*
|
|
* On exit from the kernel to user mode, we always need to check for ASTs. In
|
|
* addition, we need to do this atomically; otherwise an interrupt may occur
|
|
* which causes an AST, but it won't get processed until the next kernel entry
|
|
* (possibly the next clock tick). Thus, we disable interrupt before checking,
|
|
* and only enable them again on the final `iret' or before calling the AST
|
|
* handler.
|
|
*/
|
|
|
|
/*****************************************************************************/
|
|
|
|
#define TRAP(a) pushq $(a) ; jmp alltraps
|
|
#define ZTRAP(a) pushq $0 ; TRAP(a)
|
|
|
|
IDTVEC(trap00)
|
|
ZTRAP(T_DIVIDE)
|
|
IDTVEC(trap01)
|
|
ZTRAP(T_TRCTRAP)
|
|
|
|
/*
|
|
* NMIs can happen at any time, so there's no simple way to tell
|
|
* which GS.base is in place at the time of the interrupt. Instead,
|
|
* borrow a couple ideas from FreeBSD and put the CPU's kernel
|
|
* GS.base in the memory right above the stack, storing the current
|
|
* one in a pair of callee-saved registers (%r12/13). We save the
|
|
* current %cr3 in a callee-saved register too (%r15).
|
|
* Note: we don't unblock interrupts because a nested normal interrupt
|
|
* would also reenable NMIs.
|
|
*/
|
|
IDTVEC(trap02)
|
|
pushq $0
|
|
pushq $T_NMI
|
|
calltrap_specstk: # special stack path
|
|
TRAP_ENTRY_KERN
|
|
INTR_CLEAR_GPRS
|
|
movl $MSR_GSBASE,%ecx # save current GS.base...
|
|
rdmsr
|
|
movq %rax,%r12 # ...in %r12 and %r13
|
|
movq %rdx,%r13
|
|
movq FRAMESIZE(%rsp),%rax # get kernel GS.base
|
|
movq %rax,%rdx
|
|
shrq $32,%rdx
|
|
wrmsr # switch to it
|
|
movq %cr3,%r15 # save current %cr3 in %r15
|
|
movq CPUVAR(KERN_CR3),%rax # switch to kernel page tables
|
|
testq %rax,%rax
|
|
jz INTRENTRY_LABEL(calltrap_specstk)
|
|
movq %rax,%cr3
|
|
jmp INTRENTRY_LABEL(calltrap_specstk)
|
|
.text
|
|
.globl INTRENTRY_LABEL(calltrap_specstk)
|
|
INTRENTRY_LABEL(calltrap_specstk):
|
|
lfence # block speculation through jz above
|
|
cld
|
|
SMAP_CLAC
|
|
movq %rsp,%rdi
|
|
call kerntrap
|
|
movl $MSR_GSBASE,%ecx # restore GS.base
|
|
movq %r12,%rax
|
|
movq %r13,%rdx
|
|
wrmsr
|
|
call pku_xonly
|
|
popq %rdi
|
|
popq %rsi
|
|
popq %rdx
|
|
popq %rcx
|
|
popq %r8
|
|
popq %r9
|
|
popq %r10
|
|
popq %r11
|
|
popq %r12
|
|
popq %r13
|
|
popq %r14
|
|
jmp calltrap_specstk_tramp
|
|
KUENTRY(calltrap_specstk_tramp)
|
|
movq %r15,%cr3 # restore %cr3
|
|
popq %r15
|
|
addq $8,%rsp # ignore tf_err
|
|
popq %rbx
|
|
popq %rax
|
|
addq $8,%rsp # ignore tf_trapno
|
|
popq %rbp
|
|
iretq
|
|
|
|
IDTVEC(trap03)
|
|
#if defined(GPROF) || !defined(DDBPROF)
|
|
ZTRAP(T_BPTFLT)
|
|
#else /* !defined(GPROF) && defined(DDBPROF) */
|
|
pushq $0
|
|
pushq $T_BPTFLT
|
|
testb $SEL_RPL,24(%rsp)
|
|
je INTRENTRY_LABEL(trap03)
|
|
jmp alltraps
|
|
.text
|
|
.global INTRENTRY_LABEL(trap03)
|
|
INTRENTRY_LABEL(trap03):
|
|
FENCE_NO_SAFE_SMAP
|
|
TRAP_ENTRY_KERN
|
|
sti
|
|
cld
|
|
SMAP_CLAC
|
|
leaq dt_prov_kprobe, %rdi
|
|
movq %rsp, %rsi
|
|
call dt_prov_kprobe_hook
|
|
cmpl $0, %eax
|
|
je .Lreal_kern_trap
|
|
|
|
cli
|
|
movq TF_RDI(%rsp),%rdi
|
|
movq TF_RSI(%rsp),%rsi
|
|
movq TF_R8(%rsp),%r8
|
|
movq TF_R9(%rsp),%r9
|
|
movq TF_R10(%rsp),%r10
|
|
movq TF_R12(%rsp),%r12
|
|
movq TF_R13(%rsp),%r13
|
|
movq TF_R14(%rsp),%r14
|
|
movq TF_R15(%rsp),%r15
|
|
movq TF_RBP(%rsp),%rbp
|
|
movq TF_RBX(%rsp),%rbx
|
|
movq TF_RDX(%rsp),%rdx
|
|
movq TF_RCX(%rsp),%rcx
|
|
movq TF_R11(%rsp),%r11
|
|
/* %rax restored below, after being used to shift the stack */
|
|
|
|
cmpl $2, %eax
|
|
je .Lemulate_ret
|
|
|
|
.Lemulate_push_rbp:
|
|
|
|
/*
|
|
* We are returning from a probe trap so we need to fix the
|
|
* stack layout and emulate the patched instruction.
|
|
* Reserve enough room to emulate "pushq %rbp".
|
|
*/
|
|
subq $16, %rsp
|
|
|
|
movq (TF_RAX + 16)(%rsp), %rax
|
|
movq %rax, TF_RAX(%rsp)
|
|
|
|
/* Shift hardware-saved registers. */
|
|
movq (TF_RIP + 16)(%rsp), %rax
|
|
movq %rax, TF_RIP(%rsp)
|
|
movq (TF_CS + 16)(%rsp), %rax
|
|
movq %rax, TF_CS(%rsp)
|
|
movq (TF_RFLAGS + 16)(%rsp), %rax
|
|
movq %rax, TF_RFLAGS(%rsp)
|
|
movq (TF_RSP + 16)(%rsp), %rax
|
|
movq %rax, TF_RSP(%rsp)
|
|
movq (TF_SS + 16)(%rsp), %rax
|
|
movq %rax, TF_SS(%rsp)
|
|
|
|
/* Pull 8 bytes off the stack and store %rbp in the expected location.*/
|
|
movq TF_RSP(%rsp), %rax
|
|
subq $8, %rax
|
|
movq %rax, TF_RSP(%rsp)
|
|
movq %rbp, (%rax)
|
|
|
|
/* Finally restore %rax */
|
|
movq (TF_RAX + 16)(%rsp),%rax
|
|
jmp .ret_int3
|
|
|
|
.Lemulate_ret:
|
|
|
|
/* Store a new return address in %rip */
|
|
movq TF_RSP(%rsp), %rax
|
|
movq (%rax), %rax
|
|
movq %rax, TF_RIP(%rsp)
|
|
addq $8, TF_RSP(%rsp)
|
|
|
|
/* Finally restore %rax */
|
|
movq (TF_RAX)(%rsp),%rax
|
|
|
|
.ret_int3:
|
|
addq $TF_RIP,%rsp
|
|
iretq
|
|
#endif /* !defined(GPROF) && defined(DDBPROF) */
|
|
|
|
IDTVEC(trap04)
|
|
ZTRAP(T_OFLOW) # impossible: INTO instruction invalid in amd64
|
|
IDTVEC(trap05)
|
|
ZTRAP(T_BOUND) # impossible: BOUND instruction invalid in amd64
|
|
IDTVEC(trap06)
|
|
ZTRAP(T_PRIVINFLT)
|
|
IDTVEC(trap07)
|
|
ZTRAP(T_DNA) # impossible: we don't do lazy FPU
|
|
IDTVEC(trap08)
|
|
pushq $T_DOUBLEFLT
|
|
jmp calltrap_specstk
|
|
IDTVEC(trap09)
|
|
ZTRAP(T_FPOPFLT) # impossible: not generated on amd64
|
|
IDTVEC(trap0a)
|
|
TRAP(T_TSSFLT)
|
|
IDTVEC(trap0b)
|
|
TRAP(T_SEGNPFLT)
|
|
IDTVEC(trap0c)
|
|
TRAP(T_STKFLT)
|
|
|
|
/*
|
|
* The #GP (general protection fault) handler has a few weird cases
|
|
* to handle:
|
|
* - trapping in iretq to userspace and
|
|
* - trapping in xrstor in the kernel.
|
|
* - trapping when invalid MSRs are read in rdmsr_safe
|
|
* We detect these by examining the %rip in the iretq_frame.
|
|
* Handling them is done by updating %rip in the iretq_frame to point
|
|
* to a stub handler of some sort and then iretq'ing to it. For the
|
|
* iretq fault we resume in a stub which acts like we got a fresh #GP.
|
|
* For the xrstor fault we resume to a stub which returns an error to
|
|
* the routine that requested the xrstor.
|
|
*/
|
|
IDTVEC(trap0d)
|
|
pushq %rdx
|
|
pushq %rcx
|
|
movq 24(%rsp),%rdx /* over %r[cd]x and err to %rip */
|
|
leaq doreti_iret(%rip),%rcx
|
|
cmpq %rcx,%rdx
|
|
je .Lhandle_doreti
|
|
leaq xrstor_fault(%rip),%rcx
|
|
cmpq %rcx,%rdx
|
|
je .Lhandle_xrstor
|
|
leaq xsetbv_fault(%rip),%rcx
|
|
cmpq %rcx,%rdx
|
|
je .Lhandle_xsetbv
|
|
leaq rdmsr_safe_fault(%rip),%rcx
|
|
cmpq %rcx,%rdx
|
|
je .Lhandle_rdmsr_safe
|
|
popq %rcx
|
|
popq %rdx
|
|
TRAP(T_PROTFLT)
|
|
|
|
.Lhandle_rdmsr_safe:
|
|
/* rdmsr faulted; just resume in rdmsr_resume */
|
|
leaq rdmsr_resume(%rip),%rcx
|
|
jmp 1f
|
|
|
|
.Lhandle_xrstor:
|
|
/* xrstor faulted; just resume in xrstor_resume */
|
|
leaq xrstor_resume(%rip),%rcx
|
|
jmp 1f
|
|
|
|
.Lhandle_xsetbv:
|
|
/* xsetbv faulted; just resume in xsetbv_resume */
|
|
leaq xsetbv_resume(%rip),%rcx
|
|
jmp 1f
|
|
|
|
.Lhandle_doreti:
|
|
/* iretq faulted; resume in a stub that acts like we got a #GP */
|
|
leaq .Lhandle_doreti_resume(%rip),%rcx
|
|
1: lfence /* block speculation through conditionals above */
|
|
movq %rcx,24(%rsp) /* over %r[cd]x and err to %rip */
|
|
popq %rcx
|
|
popq %rdx
|
|
addq $8,%rsp /* pop the err code */
|
|
jmp doreti_iret
|
|
.Lhandle_doreti_resume:
|
|
ZTRAP(T_PROTFLT)
|
|
|
|
IDTVEC(trap0e)
|
|
TRAP(T_PAGEFLT)
|
|
IDTVEC(intrspurious)
|
|
IDTVEC_ALIAS(trap0f, intrspurious)
|
|
iretq
|
|
IDTVEC(trap10)
|
|
ZTRAP(T_ARITHTRAP)
|
|
IDTVEC(trap11)
|
|
TRAP(T_ALIGNFLT)
|
|
IDTVEC(trap12)
|
|
ZTRAP(T_MCA)
|
|
IDTVEC(trap13)
|
|
ZTRAP(T_XMM)
|
|
IDTVEC(trap14)
|
|
ZTRAP(T_VE)
|
|
IDTVEC(trap15)
|
|
TRAP(T_CP)
|
|
IDTVEC(trap1f)
|
|
IDTVEC_ALIAS(trap16, trap1f)
|
|
IDTVEC_ALIAS(trap17, trap1f)
|
|
IDTVEC_ALIAS(trap18, trap1f)
|
|
IDTVEC_ALIAS(trap19, trap1f)
|
|
IDTVEC_ALIAS(trap1a, trap1f)
|
|
IDTVEC_ALIAS(trap1b, trap1f)
|
|
IDTVEC_ALIAS(trap1c, trap1f)
|
|
IDTVEC_ALIAS(trap1d, trap1f)
|
|
IDTVEC_ALIAS(trap1e, trap1f)
|
|
/* 22 - 31 reserved for future exp */
|
|
ZTRAP(T_RESERVED)
|
|
|
|
.section .rodata
|
|
.globl Xexceptions
|
|
.type Xexceptions,@object
|
|
Xexceptions:
|
|
.quad Xtrap00, Xtrap01, Xtrap02, Xtrap03
|
|
.quad Xtrap04, Xtrap05, Xtrap06, Xtrap07
|
|
.quad Xtrap08, Xtrap09, Xtrap0a, Xtrap0b
|
|
.quad Xtrap0c, Xtrap0d, Xtrap0e, Xtrap0f
|
|
.quad Xtrap10, Xtrap11, Xtrap12, Xtrap13
|
|
.quad Xtrap14, Xtrap15, Xtrap16, Xtrap17
|
|
.quad Xtrap18, Xtrap19, Xtrap1a, Xtrap1b
|
|
.quad Xtrap1c, Xtrap1d, Xtrap1e, Xtrap1f
|
|
END(Xexceptions)
|
|
|
|
/*
|
|
* All traps go through here. Figure out whether we're
|
|
* a) coming from usermode and need the Meltdown mitigation before
|
|
* jumping to user trap handling followed by AST and
|
|
* return-to-userspace handling, or
|
|
* b) coming from supervisor mode and can directly jump to kernel
|
|
* trap handling before returning sans AST or other handling.
|
|
*/
|
|
KUTEXT_PAGE_START
|
|
.align NBPG, 0xcc
|
|
/*
|
|
* This is the Meltdown alltraps page, which is mapped into
|
|
* the U-K page tables at the same location as alltraps
|
|
* below. For this, the Meltdown case, we must be coming from
|
|
* userspace so we skip the SEL_RPL test and go directly to
|
|
* the swapgs+use-scratch+change-cr3 sequence. Switching to
|
|
* the kernel page tables (thank you, Intel) will make us
|
|
* continue at the "TRAP_ENTRY_USER" after alltraps below.
|
|
* In case the CPU speculates past the mov to cr3,
|
|
* we put a retpoline-style pause-lfence-jmp-to-pause loop.
|
|
*/
|
|
Xalltraps:
|
|
swapgs
|
|
movq %rax,CPUVAR(SCRATCH)
|
|
movq CPUVAR(KERN_CR3),%rax
|
|
.byte 0x66, 0x90 /* space for FENCE_SWAPGS_MIS_TAKEN below */
|
|
movq %rax,%cr3
|
|
0: pause
|
|
lfence
|
|
jmp 0b
|
|
KUTEXT_PAGE_END
|
|
|
|
KTEXT_PAGE_START
|
|
.align NBPG, 0xcc
|
|
GENTRY(alltraps)
|
|
CODEPATCH_START
|
|
testb $SEL_RPL,24(%rsp)
|
|
je alltraps_kern
|
|
swapgs
|
|
CODEPATCH_END(CPTAG_MELTDOWN_ALLTRAPS)
|
|
FENCE_SWAPGS_MIS_TAKEN
|
|
movq %rax,CPUVAR(SCRATCH)
|
|
.space (0b - Xalltraps) - (. - alltraps), 0x90
|
|
|
|
/*
|
|
* Traps from userspace
|
|
*/
|
|
TRAP_ENTRY_USER
|
|
sti
|
|
cld
|
|
SMAP_CLAC
|
|
.globl recall_trap
|
|
recall_trap:
|
|
movq %rsp, %rdi
|
|
call usertrap
|
|
cli
|
|
jmp intr_user_exit
|
|
END(alltraps)
|
|
|
|
/*
|
|
* Traps from supervisor mode (kernel)
|
|
* If we're not mitigating Meltdown, then there's a conditional branch
|
|
* above and we may need a fence to mitigate CVE-2019-1125. If we're
|
|
* doing Meltdown mitigation there's just an unconditional branch and
|
|
* can skip the fence.
|
|
*/
|
|
_ALIGN_TRAPS
|
|
GENTRY(alltraps_kern)
|
|
FENCE_NO_SAFE_SMAP
|
|
GENTRY(alltraps_kern_meltdown)
|
|
TRAP_ENTRY_KERN
|
|
sti
|
|
cld
|
|
SMAP_CLAC
|
|
.Lreal_kern_trap:
|
|
#ifdef DIAGNOSTIC
|
|
movl CPUVAR(ILEVEL),%ebx
|
|
#endif /* DIAGNOSTIC */
|
|
movq %rsp, %rdi
|
|
call kerntrap
|
|
2: cli
|
|
#ifndef DIAGNOSTIC
|
|
INTRFASTEXIT
|
|
#else /* DIAGNOSTIC */
|
|
cmpl CPUVAR(ILEVEL),%ebx
|
|
jne 3f
|
|
INTRFASTEXIT
|
|
3: sti
|
|
leaq spl_lowered(%rip),%rdi
|
|
movl CPUVAR(ILEVEL),%esi
|
|
movl %ebx,%edx
|
|
xorq %rax,%rax
|
|
call printf
|
|
#ifdef DDB
|
|
int $3
|
|
#endif /* DDB */
|
|
movl %ebx,CPUVAR(ILEVEL)
|
|
jmp 2b
|
|
|
|
.pushsection .rodata
|
|
.type spl_lowered,@object
|
|
spl_lowered:
|
|
.asciz "WARNING: SPL NOT LOWERED ON TRAP EXIT %x %x\n"
|
|
END(spl_lowered)
|
|
.popsection
|
|
#endif /* DIAGNOSTIC */
|
|
END(alltraps_kern)
|
|
END(alltraps_kern_meltdown)
|
|
KTEXT_PAGE_END
|
|
|
|
|
|
/*
|
|
* Macros for interrupt entry, call to handler, and exit.
|
|
*
|
|
* XXX
|
|
* The interrupt frame is set up to look like a trap frame. This may be a
|
|
* waste. The only handler which needs a frame is the clock handler, and it
|
|
* only needs a few bits. Xdoreti() needs a trap frame for handling ASTs, but
|
|
* it could easily convert the frame on demand.
|
|
*
|
|
* The direct costs of setting up a trap frame are two pushq's (error code and
|
|
* trap number), an addl to get rid of these, and pushing and popping the
|
|
* callee-saved registers %ebx, %ebp, and %r1[2-5] twice.
|
|
*
|
|
* If the interrupt frame is made more flexible, INTR can push %eax first and
|
|
* decide the ipending case with less overhead
|
|
*/
|
|
|
|
/* XXX See comment in locore.s */
|
|
#define XINTR(name,num) Xintr_##name##num
|
|
|
|
KUENTRY(x2apic_eoi)
|
|
pushq %rax
|
|
pushq %rcx
|
|
pushq %rdx
|
|
mov $MSR_X2APIC_EOI,%ecx
|
|
mov $0,%eax
|
|
mov $0,%edx
|
|
wrmsr
|
|
popq %rdx
|
|
popq %rcx
|
|
popq %rax
|
|
ret
|
|
lfence
|
|
END(x2apic_eoi)
|
|
|
|
#if NLAPIC > 0
|
|
#ifdef MULTIPROCESSOR
|
|
KIDTVEC(recurse_lapic_ipi)
|
|
INTR_RECURSE
|
|
jmp 1f
|
|
END(Xrecurse_lapic_ipi)
|
|
IDTVEC(intr_lapic_ipi)
|
|
INTRENTRY(intr_lapic_ipi)
|
|
CODEPATCH_START
|
|
movl $0,local_apic+LAPIC_EOI
|
|
CODEPATCH_END(CPTAG_EOI)
|
|
movl CPUVAR(ILEVEL),%ebx
|
|
cmpl $IPL_IPI,%ebx
|
|
jae 2f
|
|
END(INTRENTRY_LABEL(intr_lapic_ipi))
|
|
KIDTVEC_FALLTHROUGH(resume_lapic_ipi)
|
|
endbr64
|
|
1:
|
|
incl CPUVAR(IDEPTH)
|
|
movl $IPL_IPI,CPUVAR(ILEVEL)
|
|
sti
|
|
cld
|
|
SMAP_CLAC
|
|
movq %rbx,IF_PPL(%rsp)
|
|
call x86_ipi_handler
|
|
jmp Xdoreti
|
|
2:
|
|
movq $(1 << LIR_IPI),%rax
|
|
orq %rax,CPUVAR(IPENDING)
|
|
INTRFASTEXIT
|
|
END(Xresume_lapic_ipi)
|
|
|
|
/*
|
|
* "Fast" IPI handlers. These are the IPIs which are handled without
|
|
* unblocking interrupts, so no need for 'recurse' or 'resume' entry points
|
|
*/
|
|
/* invalidate the entire TLB, no PCIDs version */
|
|
IDTVEC(ipi_invltlb)
|
|
pushq %rax
|
|
|
|
ioapic_asm_ack()
|
|
|
|
movq %cr3, %rax
|
|
movq %rax, %cr3
|
|
|
|
lock
|
|
decq tlb_shoot_wait
|
|
|
|
popq %rax
|
|
iretq
|
|
END(Xipi_invltlb)
|
|
|
|
/* invalidate a single page, no PCIDs version */
|
|
IDTVEC(ipi_invlpg)
|
|
pushq %rax
|
|
|
|
ioapic_asm_ack()
|
|
|
|
movq tlb_shoot_addr1, %rax
|
|
invlpg (%rax)
|
|
|
|
lock
|
|
decq tlb_shoot_wait
|
|
|
|
popq %rax
|
|
iretq
|
|
END(Xipi_invlpg)
|
|
|
|
/* invalidate a range of pages, no PCIDs version */
|
|
IDTVEC(ipi_invlrange)
|
|
pushq %rax
|
|
pushq %rdx
|
|
|
|
ioapic_asm_ack()
|
|
|
|
movq tlb_shoot_addr1, %rax
|
|
movq tlb_shoot_addr2, %rdx
|
|
1: invlpg (%rax)
|
|
addq $PAGE_SIZE, %rax
|
|
cmpq %rdx, %rax
|
|
jb 1b
|
|
|
|
lock
|
|
decq tlb_shoot_wait
|
|
|
|
popq %rdx
|
|
popq %rax
|
|
iretq
|
|
END(Xipi_invlrange)
|
|
|
|
/*
|
|
* Invalidate the userspace PCIDs.
|
|
*/
|
|
IDTVEC(ipi_invltlb_pcid)
|
|
pushq %rax
|
|
|
|
ioapic_asm_ack()
|
|
|
|
/* set the type */
|
|
movl $INVPCID_PCID,%eax
|
|
|
|
/* finish getting space for the INVPCID descriptor */
|
|
#if INVPCID_PCID == PCID_PROC
|
|
pushq %rax
|
|
#else
|
|
pushq $PCID_PROC
|
|
#endif
|
|
|
|
invpcid (%rsp),%rax
|
|
|
|
/* bump the pcid in the descriptor and invpcid again */
|
|
movl $PCID_PROC_INTEL,(%rsp)
|
|
invpcid (%rsp),%rax
|
|
|
|
lock
|
|
decq tlb_shoot_wait
|
|
|
|
/* restore the stack */
|
|
popq %rax
|
|
popq %rax
|
|
iretq
|
|
END(Xipi_invltlb_pcid)
|
|
|
|
/*
|
|
* Invalidate a VA in two PCIDs. Kernel VAs are present in PCIDs 0 and 1,
|
|
* while userspace VAs are present in PCIDs 1 and 2.
|
|
*/
|
|
IDTVEC(ipi_invlpg_pcid)
|
|
pushq %rax
|
|
|
|
ioapic_asm_ack()
|
|
|
|
/* space for the INVPCID descriptor */
|
|
subq $16,%rsp
|
|
|
|
/* set the PCID in the descriptor */
|
|
movl tlb_shoot_first_pcid,%eax
|
|
movq %rax,(%rsp)
|
|
|
|
/* set the address in the descriptor */
|
|
movq tlb_shoot_addr1,%rax
|
|
movq %rax,8(%rsp)
|
|
|
|
/* set the type to zero, and invpcid */
|
|
xorl %eax,%eax
|
|
invpcid (%rsp),%rax
|
|
|
|
/* bump the pcid in the descriptor and invpcid again */
|
|
addl $1,(%rsp)
|
|
invpcid (%rsp),%rax
|
|
|
|
lock
|
|
decq tlb_shoot_wait
|
|
|
|
/* restore the stack */
|
|
addq $16,%rsp
|
|
popq %rax
|
|
iretq
|
|
END(Xipi_invlpg_pcid)
|
|
|
|
/*
|
|
* Invalidate a range of VA in two PCIDs. Kernel VAs are present in
|
|
* PCIDs 0 and 1, while userspace VAs are present in PCIDs 1 and 2.
|
|
*/
|
|
IDTVEC(ipi_invlrange_pcid)
|
|
pushq %rax
|
|
pushq %rdx
|
|
pushq %rcx
|
|
|
|
ioapic_asm_ack()
|
|
|
|
/* space for the INVPCID descriptor */
|
|
subq $16,%rsp
|
|
|
|
/* set the PCID in the descriptor */
|
|
movl tlb_shoot_first_pcid,%eax
|
|
movq %rax,(%rsp)
|
|
|
|
/* set up for the loop: load the limit and set the type to zero */
|
|
movq tlb_shoot_addr2,%rdx
|
|
xorl %ecx,%ecx
|
|
|
|
/* set the address in the descriptor and loop the invalidate */
|
|
movq tlb_shoot_addr1,%rax
|
|
1: movq %rax,8(%rsp)
|
|
invpcid (%rsp),%rcx
|
|
addl $1,(%rsp)
|
|
invpcid (%rsp),%rcx
|
|
subl $1,(%rsp)
|
|
addq $PAGE_SIZE,%rax
|
|
cmpq %rdx,%rax
|
|
jb 1b
|
|
|
|
lock
|
|
decq tlb_shoot_wait
|
|
|
|
/* restore the stack */
|
|
addq $16,%rsp
|
|
popq %rcx
|
|
popq %rdx
|
|
popq %rax
|
|
iretq
|
|
END(Xipi_invlrange_pcid)
|
|
|
|
#endif /* MULTIPROCESSOR */
|
|
|
|
/*
|
|
* Interrupt from the local APIC timer.
|
|
*/
|
|
KIDTVEC(recurse_lapic_ltimer)
|
|
INTR_RECURSE
|
|
jmp 1f
|
|
END(Xrecurse_lapic_ltimer)
|
|
IDTVEC(intr_lapic_ltimer)
|
|
INTRENTRY(intr_lapic_ltimer)
|
|
CODEPATCH_START
|
|
movl $0,local_apic+LAPIC_EOI
|
|
CODEPATCH_END(CPTAG_EOI)
|
|
movl CPUVAR(ILEVEL),%ebx
|
|
cmpl $IPL_CLOCK,%ebx
|
|
jae 2f
|
|
END(INTRENTRY_LABEL(intr_lapic_ltimer))
|
|
KIDTVEC_FALLTHROUGH(resume_lapic_ltimer)
|
|
endbr64
|
|
1:
|
|
incl CPUVAR(IDEPTH)
|
|
movl $IPL_CLOCK,CPUVAR(ILEVEL)
|
|
sti
|
|
cld
|
|
SMAP_CLAC
|
|
movq %rbx,IF_PPL(%rsp)
|
|
xorq %rdi,%rdi
|
|
call lapic_clockintr
|
|
jmp Xdoreti
|
|
2:
|
|
movq $(1 << LIR_TIMER),%rax
|
|
orq %rax,CPUVAR(IPENDING)
|
|
INTRFASTEXIT
|
|
END(Xresume_lapic_ltimer)
|
|
|
|
#if NXEN > 0
|
|
/*
|
|
* Xen event channel upcall interrupt handler.
|
|
* Only used when the hypervisor supports direct vector callbacks.
|
|
*/
|
|
KIDTVEC(recurse_xen_upcall)
|
|
INTR_RECURSE
|
|
jmp 1f
|
|
END(Xrecurse_xen_upcall)
|
|
IDTVEC(intr_xen_upcall)
|
|
INTRENTRY(intr_xen_upcall)
|
|
call xen_intr_ack
|
|
movl CPUVAR(ILEVEL),%ebx
|
|
cmpl $IPL_NET,%ebx
|
|
jae 2f
|
|
END(INTRENTRY_LABEL(intr_xen_upcall))
|
|
KIDTVEC_FALLTHROUGH(resume_xen_upcall)
|
|
endbr64
|
|
1:
|
|
incl CPUVAR(IDEPTH)
|
|
movl $IPL_NET,CPUVAR(ILEVEL)
|
|
sti
|
|
cld
|
|
SMAP_CLAC
|
|
movq %rbx,IF_PPL(%rsp)
|
|
call xen_intr
|
|
jmp Xdoreti
|
|
2:
|
|
movq $(1 << LIR_XEN),%rax
|
|
orq %rax,CPUVAR(IPENDING)
|
|
INTRFASTEXIT
|
|
END(Xresume_xen_upcall)
|
|
#endif /* NXEN > 0 */
|
|
|
|
#if NHYPERV > 0
|
|
/*
|
|
* Hyperv event channel upcall interrupt handler.
|
|
* Only used when the hypervisor supports direct vector callbacks.
|
|
*/
|
|
KIDTVEC(recurse_hyperv_upcall)
|
|
INTR_RECURSE
|
|
jmp 1f
|
|
END(Xrecurse_hyperv_upcall)
|
|
IDTVEC(intr_hyperv_upcall)
|
|
INTRENTRY(intr_hyperv_upcall)
|
|
movl CPUVAR(ILEVEL),%ebx
|
|
cmpl $IPL_NET,%ebx
|
|
jae 2f
|
|
END(INTRENTRY_LABEL(intr_hyperv_upcall))
|
|
KIDTVEC_FALLTHROUGH(resume_hyperv_upcall)
|
|
endbr64
|
|
1:
|
|
incl CPUVAR(IDEPTH)
|
|
movl $IPL_NET,CPUVAR(ILEVEL)
|
|
sti
|
|
cld
|
|
SMAP_CLAC
|
|
movq %rbx,IF_PPL(%rsp)
|
|
call hv_intr
|
|
jmp Xdoreti
|
|
2:
|
|
movq $(1 << LIR_HYPERV),%rax
|
|
orq %rax,CPUVAR(IPENDING)
|
|
INTRFASTEXIT
|
|
END(Xresume_hyperv_upcall)
|
|
#endif /* NHYPERV > 0 */
|
|
#endif /* NLAPIC > 0 */
|
|
|
|
#define voidop(num)
|
|
|
|
|
|
/*
|
|
* This macro defines the generic stub code. Its arguments modify it
|
|
* for specific PICs.
|
|
*/
|
|
|
|
#define INTRSTUB(name, num, early_ack, late_ack, mask, unmask, level_mask) \
|
|
KIDTVEC(recurse_##name##num) ;\
|
|
INTR_RECURSE ;\
|
|
END(Xrecurse_##name##num) ;\
|
|
KIDTVEC_FALLTHROUGH(resume_##name##num) \
|
|
endbr64 ;\
|
|
movq $IREENT_MAGIC,TF_ERR(%rsp) ;\
|
|
movl %ebx,%r13d ;\
|
|
movq CPUVAR(ISOURCES) + (num) * 8, %r14 ;\
|
|
movl IS_MAXLEVEL(%r14),%ebx ;\
|
|
jmp 1f ;\
|
|
END(Xresume_##name##num) ;\
|
|
IDTVEC(intr_##name##num) ;\
|
|
INTRENTRY(intr_##name##num) ;\
|
|
movq CPUVAR(ISOURCES) + (num) * 8, %r14 ;\
|
|
mask(num) /* mask it in hardware */ ;\
|
|
early_ack(num) /* and allow other intrs */ ;\
|
|
incl uvmexp+V_INTR /* statistical info */ ;\
|
|
testq %r14,%r14 ;\
|
|
jz 9f /* stray */ ;\
|
|
movl IS_MAXLEVEL(%r14),%ebx ;\
|
|
movl CPUVAR(ILEVEL),%r13d ;\
|
|
cmpl %ebx,%r13d ;\
|
|
jae 10f /* currently masked; hold it */ ;\
|
|
1: \
|
|
movq %r13,IF_PPL(%rsp) ;\
|
|
movl %ebx,CPUVAR(ILEVEL) ;\
|
|
sti ;\
|
|
cld ;\
|
|
SMAP_CLAC ;\
|
|
incl CPUVAR(IDEPTH) ;\
|
|
movq IS_HANDLERS(%r14),%rbx ;\
|
|
6: /* loop, walking chain of handlers */ \
|
|
movl IH_LEVEL(%rbx),%r12d ;\
|
|
cmpl %r13d,%r12d ;\
|
|
jle 7f ;\
|
|
movl %r12d,CPUVAR(ILEVEL) ;\
|
|
movq %rbx, %rsi ;\
|
|
movq %rsp, %rdi ;\
|
|
call intr_handler /* call it */ ;\
|
|
orl %eax,%eax /* should it be counted? */ ;\
|
|
jz 4f /* no, skip it */ ;\
|
|
incq IH_COUNT(%rbx) /* count the intrs */ ;\
|
|
cmpl $0,intr_shared_edge ;\
|
|
jne 4f /* if no shared edges ... */ ;\
|
|
orl %eax,%eax /* 1 means stop trying */ ;\
|
|
jns 5f ;\
|
|
4: movq IH_NEXT(%rbx),%rbx /* next handler in chain */ ;\
|
|
testq %rbx,%rbx ;\
|
|
jnz 6b ;\
|
|
5: /* successfully handled */ \
|
|
cli ;\
|
|
unmask(num) /* unmask it in hardware */ ;\
|
|
late_ack(num) ;\
|
|
sti ;\
|
|
jmp Xdoreti /* lower spl and do ASTs */ ;\
|
|
7: /* current IPL > handler's ih_level */ \
|
|
cli ;\
|
|
movq $(1 << num),%rax ;\
|
|
orq %rax,CPUVAR(IPENDING) ;\
|
|
level_mask(num) ;\
|
|
late_ack(num) ;\
|
|
sti ;\
|
|
jmp Xdoreti /* lower spl and do ASTs */ ;\
|
|
10: /* currently masked */ \
|
|
cli ;\
|
|
movq $(1 << num),%rax ;\
|
|
orq %rax,CPUVAR(IPENDING) ;\
|
|
level_mask(num) ;\
|
|
late_ack(num) ;\
|
|
INTRFASTEXIT ;\
|
|
9: /* spurious interrupt */ \
|
|
unmask(num) ;\
|
|
late_ack(num) ;\
|
|
testb $SEL_RPL,TF_CS(%rsp) ;\
|
|
jnz intr_user_exit ;\
|
|
INTRFASTEXIT ;\
|
|
END(INTRENTRY_LABEL(intr_##name##num))
|
|
|
|
#define ICUADDR IO_ICU1
|
|
|
|
INTRSTUB(legacy,0,i8259_asm_ack1,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,1,i8259_asm_ack1,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,2,i8259_asm_ack1,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,3,i8259_asm_ack1,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,4,i8259_asm_ack1,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,5,i8259_asm_ack1,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,6,i8259_asm_ack1,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,7,i8259_asm_ack1,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
#undef ICUADDR
|
|
#define ICUADDR IO_ICU2
|
|
|
|
INTRSTUB(legacy,8,i8259_asm_ack2,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,9,i8259_asm_ack2,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,10,i8259_asm_ack2,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,11,i8259_asm_ack2,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,12,i8259_asm_ack2,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,13,i8259_asm_ack2,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,14,i8259_asm_ack2,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
INTRSTUB(legacy,15,i8259_asm_ack2,voidop,i8259_asm_mask,i8259_asm_unmask,
|
|
voidop)
|
|
|
|
#if NIOAPIC > 0
|
|
|
|
INTRSTUB(ioapic_edge,0,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,1,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,2,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,3,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,4,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,5,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,6,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,7,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,8,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,9,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,10,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,11,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,12,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,13,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,14,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,15,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,16,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,17,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,18,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,19,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,20,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,21,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,22,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,23,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,24,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,25,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,26,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,27,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,28,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,29,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,30,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,31,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,32,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,33,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,34,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,35,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,36,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,37,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,38,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,39,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,40,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,41,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,42,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,43,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,44,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,45,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,46,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,47,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,48,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,49,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,50,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,51,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,52,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,53,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,54,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,55,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,56,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,57,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,58,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,59,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,60,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,61,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,62,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
INTRSTUB(ioapic_edge,63,voidop,ioapic_asm_ack,voidop,voidop,voidop)
|
|
|
|
INTRSTUB(ioapic_level,0,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,1,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,2,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,3,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,4,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,5,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,6,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,7,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,8,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,9,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,10,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,11,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,12,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,13,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,14,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,15,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,16,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,17,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,18,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,19,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,20,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,21,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,22,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,23,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,24,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,25,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,26,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,27,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,28,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,29,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,30,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,31,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,32,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,33,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,34,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,35,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,36,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,37,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,38,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,39,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,40,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,41,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,42,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,43,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,44,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,45,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,46,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,47,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,48,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,49,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,50,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,51,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,52,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,53,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,54,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,55,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,56,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,57,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,58,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,59,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,60,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,61,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,62,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
INTRSTUB(ioapic_level,63,voidop,ioapic_asm_ack,voidop,ioapic_unmask,ioapic_mask)
|
|
|
|
#endif
|
|
|
|
.section .rodata
|
|
|
|
.globl i8259_stubs
|
|
.type i8259_stubs,@object
|
|
i8259_stubs:
|
|
.quad Xintr_legacy0, Xrecurse_legacy0, Xresume_legacy0
|
|
.quad Xintr_legacy1, Xrecurse_legacy1, Xresume_legacy1
|
|
.quad Xintr_legacy2, Xrecurse_legacy2, Xresume_legacy2
|
|
.quad Xintr_legacy3, Xrecurse_legacy3, Xresume_legacy3
|
|
.quad Xintr_legacy4, Xrecurse_legacy4, Xresume_legacy4
|
|
.quad Xintr_legacy5, Xrecurse_legacy5, Xresume_legacy5
|
|
.quad Xintr_legacy6, Xrecurse_legacy6, Xresume_legacy6
|
|
.quad Xintr_legacy7, Xrecurse_legacy7, Xresume_legacy7
|
|
.quad Xintr_legacy8, Xrecurse_legacy8, Xresume_legacy8
|
|
.quad Xintr_legacy9, Xrecurse_legacy9, Xresume_legacy9
|
|
.quad Xintr_legacy10, Xrecurse_legacy10, Xresume_legacy10
|
|
.quad Xintr_legacy11, Xrecurse_legacy11, Xresume_legacy11
|
|
.quad Xintr_legacy12, Xrecurse_legacy12, Xresume_legacy12
|
|
.quad Xintr_legacy13, Xrecurse_legacy13, Xresume_legacy13
|
|
.quad Xintr_legacy14, Xrecurse_legacy14, Xresume_legacy14
|
|
.quad Xintr_legacy15, Xrecurse_legacy15, Xresume_legacy15
|
|
END(i8259_stubs)
|
|
|
|
#if NIOAPIC > 0
|
|
.globl ioapic_edge_stubs
|
|
.type ioapic_edge_stubs,@object
|
|
ioapic_edge_stubs:
|
|
.quad Xintr_ioapic_edge0, Xrecurse_ioapic_edge0, Xresume_ioapic_edge0
|
|
.quad Xintr_ioapic_edge1, Xrecurse_ioapic_edge1, Xresume_ioapic_edge1
|
|
.quad Xintr_ioapic_edge2, Xrecurse_ioapic_edge2, Xresume_ioapic_edge2
|
|
.quad Xintr_ioapic_edge3, Xrecurse_ioapic_edge3, Xresume_ioapic_edge3
|
|
.quad Xintr_ioapic_edge4, Xrecurse_ioapic_edge4, Xresume_ioapic_edge4
|
|
.quad Xintr_ioapic_edge5, Xrecurse_ioapic_edge5, Xresume_ioapic_edge5
|
|
.quad Xintr_ioapic_edge6, Xrecurse_ioapic_edge6, Xresume_ioapic_edge6
|
|
.quad Xintr_ioapic_edge7, Xrecurse_ioapic_edge7, Xresume_ioapic_edge7
|
|
.quad Xintr_ioapic_edge8, Xrecurse_ioapic_edge8, Xresume_ioapic_edge8
|
|
.quad Xintr_ioapic_edge9, Xrecurse_ioapic_edge9, Xresume_ioapic_edge9
|
|
.quad Xintr_ioapic_edge10, Xrecurse_ioapic_edge10, Xresume_ioapic_edge10
|
|
.quad Xintr_ioapic_edge11, Xrecurse_ioapic_edge11, Xresume_ioapic_edge11
|
|
.quad Xintr_ioapic_edge12, Xrecurse_ioapic_edge12, Xresume_ioapic_edge12
|
|
.quad Xintr_ioapic_edge13, Xrecurse_ioapic_edge13, Xresume_ioapic_edge13
|
|
.quad Xintr_ioapic_edge14, Xrecurse_ioapic_edge14, Xresume_ioapic_edge14
|
|
.quad Xintr_ioapic_edge15, Xrecurse_ioapic_edge15, Xresume_ioapic_edge15
|
|
.quad Xintr_ioapic_edge16, Xrecurse_ioapic_edge16, Xresume_ioapic_edge16
|
|
.quad Xintr_ioapic_edge17, Xrecurse_ioapic_edge17, Xresume_ioapic_edge17
|
|
.quad Xintr_ioapic_edge18, Xrecurse_ioapic_edge18, Xresume_ioapic_edge18
|
|
.quad Xintr_ioapic_edge19, Xrecurse_ioapic_edge19, Xresume_ioapic_edge19
|
|
.quad Xintr_ioapic_edge20, Xrecurse_ioapic_edge20, Xresume_ioapic_edge20
|
|
.quad Xintr_ioapic_edge21, Xrecurse_ioapic_edge21, Xresume_ioapic_edge21
|
|
.quad Xintr_ioapic_edge22, Xrecurse_ioapic_edge22, Xresume_ioapic_edge22
|
|
.quad Xintr_ioapic_edge23, Xrecurse_ioapic_edge23, Xresume_ioapic_edge23
|
|
.quad Xintr_ioapic_edge24, Xrecurse_ioapic_edge24, Xresume_ioapic_edge24
|
|
.quad Xintr_ioapic_edge25, Xrecurse_ioapic_edge25, Xresume_ioapic_edge25
|
|
.quad Xintr_ioapic_edge26, Xrecurse_ioapic_edge26, Xresume_ioapic_edge26
|
|
.quad Xintr_ioapic_edge27, Xrecurse_ioapic_edge27, Xresume_ioapic_edge27
|
|
.quad Xintr_ioapic_edge28, Xrecurse_ioapic_edge28, Xresume_ioapic_edge28
|
|
.quad Xintr_ioapic_edge29, Xrecurse_ioapic_edge29, Xresume_ioapic_edge29
|
|
.quad Xintr_ioapic_edge30, Xrecurse_ioapic_edge30, Xresume_ioapic_edge30
|
|
.quad Xintr_ioapic_edge31, Xrecurse_ioapic_edge31, Xresume_ioapic_edge31
|
|
.quad Xintr_ioapic_edge32, Xrecurse_ioapic_edge32, Xresume_ioapic_edge32
|
|
.quad Xintr_ioapic_edge33, Xrecurse_ioapic_edge33, Xresume_ioapic_edge33
|
|
.quad Xintr_ioapic_edge34, Xrecurse_ioapic_edge34, Xresume_ioapic_edge34
|
|
.quad Xintr_ioapic_edge35, Xrecurse_ioapic_edge35, Xresume_ioapic_edge35
|
|
.quad Xintr_ioapic_edge36, Xrecurse_ioapic_edge36, Xresume_ioapic_edge36
|
|
.quad Xintr_ioapic_edge37, Xrecurse_ioapic_edge37, Xresume_ioapic_edge37
|
|
.quad Xintr_ioapic_edge38, Xrecurse_ioapic_edge38, Xresume_ioapic_edge38
|
|
.quad Xintr_ioapic_edge39, Xrecurse_ioapic_edge39, Xresume_ioapic_edge39
|
|
.quad Xintr_ioapic_edge40, Xrecurse_ioapic_edge40, Xresume_ioapic_edge40
|
|
.quad Xintr_ioapic_edge41, Xrecurse_ioapic_edge41, Xresume_ioapic_edge41
|
|
.quad Xintr_ioapic_edge42, Xrecurse_ioapic_edge42, Xresume_ioapic_edge42
|
|
.quad Xintr_ioapic_edge43, Xrecurse_ioapic_edge43, Xresume_ioapic_edge43
|
|
.quad Xintr_ioapic_edge44, Xrecurse_ioapic_edge44, Xresume_ioapic_edge44
|
|
.quad Xintr_ioapic_edge45, Xrecurse_ioapic_edge45, Xresume_ioapic_edge45
|
|
.quad Xintr_ioapic_edge46, Xrecurse_ioapic_edge46, Xresume_ioapic_edge46
|
|
.quad Xintr_ioapic_edge47, Xrecurse_ioapic_edge47, Xresume_ioapic_edge47
|
|
.quad Xintr_ioapic_edge48, Xrecurse_ioapic_edge48, Xresume_ioapic_edge48
|
|
.quad Xintr_ioapic_edge49, Xrecurse_ioapic_edge49, Xresume_ioapic_edge49
|
|
.quad Xintr_ioapic_edge50, Xrecurse_ioapic_edge50, Xresume_ioapic_edge50
|
|
.quad Xintr_ioapic_edge51, Xrecurse_ioapic_edge51, Xresume_ioapic_edge51
|
|
.quad Xintr_ioapic_edge52, Xrecurse_ioapic_edge52, Xresume_ioapic_edge52
|
|
.quad Xintr_ioapic_edge53, Xrecurse_ioapic_edge53, Xresume_ioapic_edge53
|
|
.quad Xintr_ioapic_edge54, Xrecurse_ioapic_edge54, Xresume_ioapic_edge54
|
|
.quad Xintr_ioapic_edge55, Xrecurse_ioapic_edge55, Xresume_ioapic_edge55
|
|
.quad Xintr_ioapic_edge56, Xrecurse_ioapic_edge56, Xresume_ioapic_edge56
|
|
.quad Xintr_ioapic_edge57, Xrecurse_ioapic_edge57, Xresume_ioapic_edge57
|
|
.quad Xintr_ioapic_edge58, Xrecurse_ioapic_edge58, Xresume_ioapic_edge58
|
|
.quad Xintr_ioapic_edge59, Xrecurse_ioapic_edge59, Xresume_ioapic_edge59
|
|
.quad Xintr_ioapic_edge60, Xrecurse_ioapic_edge60, Xresume_ioapic_edge60
|
|
.quad Xintr_ioapic_edge61, Xrecurse_ioapic_edge61, Xresume_ioapic_edge61
|
|
.quad Xintr_ioapic_edge62, Xrecurse_ioapic_edge62, Xresume_ioapic_edge62
|
|
.quad Xintr_ioapic_edge63, Xrecurse_ioapic_edge63, Xresume_ioapic_edge63
|
|
END(ioapic_edge_stubs)
|
|
|
|
.globl ioapic_level_stubs
|
|
.type ioapic_level_stubs,@object
|
|
ioapic_level_stubs:
|
|
.quad Xintr_ioapic_level0, Xrecurse_ioapic_level0, Xresume_ioapic_level0
|
|
.quad Xintr_ioapic_level1, Xrecurse_ioapic_level1, Xresume_ioapic_level1
|
|
.quad Xintr_ioapic_level2, Xrecurse_ioapic_level2, Xresume_ioapic_level2
|
|
.quad Xintr_ioapic_level3, Xrecurse_ioapic_level3, Xresume_ioapic_level3
|
|
.quad Xintr_ioapic_level4, Xrecurse_ioapic_level4, Xresume_ioapic_level4
|
|
.quad Xintr_ioapic_level5, Xrecurse_ioapic_level5, Xresume_ioapic_level5
|
|
.quad Xintr_ioapic_level6, Xrecurse_ioapic_level6, Xresume_ioapic_level6
|
|
.quad Xintr_ioapic_level7, Xrecurse_ioapic_level7, Xresume_ioapic_level7
|
|
.quad Xintr_ioapic_level8, Xrecurse_ioapic_level8, Xresume_ioapic_level8
|
|
.quad Xintr_ioapic_level9, Xrecurse_ioapic_level9, Xresume_ioapic_level9
|
|
.quad Xintr_ioapic_level10, Xrecurse_ioapic_level10, Xresume_ioapic_level10
|
|
.quad Xintr_ioapic_level11, Xrecurse_ioapic_level11, Xresume_ioapic_level11
|
|
.quad Xintr_ioapic_level12, Xrecurse_ioapic_level12, Xresume_ioapic_level12
|
|
.quad Xintr_ioapic_level13, Xrecurse_ioapic_level13, Xresume_ioapic_level13
|
|
.quad Xintr_ioapic_level14, Xrecurse_ioapic_level14, Xresume_ioapic_level14
|
|
.quad Xintr_ioapic_level15, Xrecurse_ioapic_level15, Xresume_ioapic_level15
|
|
.quad Xintr_ioapic_level16, Xrecurse_ioapic_level16, Xresume_ioapic_level16
|
|
.quad Xintr_ioapic_level17, Xrecurse_ioapic_level17, Xresume_ioapic_level17
|
|
.quad Xintr_ioapic_level18, Xrecurse_ioapic_level18, Xresume_ioapic_level18
|
|
.quad Xintr_ioapic_level19, Xrecurse_ioapic_level19, Xresume_ioapic_level19
|
|
.quad Xintr_ioapic_level20, Xrecurse_ioapic_level20, Xresume_ioapic_level20
|
|
.quad Xintr_ioapic_level21, Xrecurse_ioapic_level21, Xresume_ioapic_level21
|
|
.quad Xintr_ioapic_level22, Xrecurse_ioapic_level22, Xresume_ioapic_level22
|
|
.quad Xintr_ioapic_level23, Xrecurse_ioapic_level23, Xresume_ioapic_level23
|
|
.quad Xintr_ioapic_level24, Xrecurse_ioapic_level24, Xresume_ioapic_level24
|
|
.quad Xintr_ioapic_level25, Xrecurse_ioapic_level25, Xresume_ioapic_level25
|
|
.quad Xintr_ioapic_level26, Xrecurse_ioapic_level26, Xresume_ioapic_level26
|
|
.quad Xintr_ioapic_level27, Xrecurse_ioapic_level27, Xresume_ioapic_level27
|
|
.quad Xintr_ioapic_level28, Xrecurse_ioapic_level28, Xresume_ioapic_level28
|
|
.quad Xintr_ioapic_level29, Xrecurse_ioapic_level29, Xresume_ioapic_level29
|
|
.quad Xintr_ioapic_level30, Xrecurse_ioapic_level30, Xresume_ioapic_level30
|
|
.quad Xintr_ioapic_level31, Xrecurse_ioapic_level31, Xresume_ioapic_level31
|
|
.quad Xintr_ioapic_level32, Xrecurse_ioapic_level32, Xresume_ioapic_level32
|
|
.quad Xintr_ioapic_level33, Xrecurse_ioapic_level33, Xresume_ioapic_level33
|
|
.quad Xintr_ioapic_level34, Xrecurse_ioapic_level34, Xresume_ioapic_level34
|
|
.quad Xintr_ioapic_level35, Xrecurse_ioapic_level35, Xresume_ioapic_level35
|
|
.quad Xintr_ioapic_level36, Xrecurse_ioapic_level36, Xresume_ioapic_level36
|
|
.quad Xintr_ioapic_level37, Xrecurse_ioapic_level37, Xresume_ioapic_level37
|
|
.quad Xintr_ioapic_level38, Xrecurse_ioapic_level38, Xresume_ioapic_level38
|
|
.quad Xintr_ioapic_level39, Xrecurse_ioapic_level39, Xresume_ioapic_level39
|
|
.quad Xintr_ioapic_level40, Xrecurse_ioapic_level40, Xresume_ioapic_level40
|
|
.quad Xintr_ioapic_level41, Xrecurse_ioapic_level41, Xresume_ioapic_level41
|
|
.quad Xintr_ioapic_level42, Xrecurse_ioapic_level42, Xresume_ioapic_level42
|
|
.quad Xintr_ioapic_level43, Xrecurse_ioapic_level43, Xresume_ioapic_level43
|
|
.quad Xintr_ioapic_level44, Xrecurse_ioapic_level44, Xresume_ioapic_level44
|
|
.quad Xintr_ioapic_level45, Xrecurse_ioapic_level45, Xresume_ioapic_level45
|
|
.quad Xintr_ioapic_level46, Xrecurse_ioapic_level46, Xresume_ioapic_level46
|
|
.quad Xintr_ioapic_level47, Xrecurse_ioapic_level47, Xresume_ioapic_level47
|
|
.quad Xintr_ioapic_level48, Xrecurse_ioapic_level48, Xresume_ioapic_level48
|
|
.quad Xintr_ioapic_level49, Xrecurse_ioapic_level49, Xresume_ioapic_level49
|
|
.quad Xintr_ioapic_level50, Xrecurse_ioapic_level50, Xresume_ioapic_level50
|
|
.quad Xintr_ioapic_level51, Xrecurse_ioapic_level51, Xresume_ioapic_level51
|
|
.quad Xintr_ioapic_level52, Xrecurse_ioapic_level52, Xresume_ioapic_level52
|
|
.quad Xintr_ioapic_level53, Xrecurse_ioapic_level53, Xresume_ioapic_level53
|
|
.quad Xintr_ioapic_level54, Xrecurse_ioapic_level54, Xresume_ioapic_level54
|
|
.quad Xintr_ioapic_level55, Xrecurse_ioapic_level55, Xresume_ioapic_level55
|
|
.quad Xintr_ioapic_level56, Xrecurse_ioapic_level56, Xresume_ioapic_level56
|
|
.quad Xintr_ioapic_level57, Xrecurse_ioapic_level57, Xresume_ioapic_level57
|
|
.quad Xintr_ioapic_level58, Xrecurse_ioapic_level58, Xresume_ioapic_level58
|
|
.quad Xintr_ioapic_level59, Xrecurse_ioapic_level59, Xresume_ioapic_level59
|
|
.quad Xintr_ioapic_level60, Xrecurse_ioapic_level60, Xresume_ioapic_level60
|
|
.quad Xintr_ioapic_level61, Xrecurse_ioapic_level61, Xresume_ioapic_level61
|
|
.quad Xintr_ioapic_level62, Xrecurse_ioapic_level62, Xresume_ioapic_level62
|
|
.quad Xintr_ioapic_level63, Xrecurse_ioapic_level63, Xresume_ioapic_level63
|
|
END(ioapic_level_stubs)
|
|
#endif
|
|
|
|
/*
|
|
* Soft interrupt handlers
|
|
*/
|
|
NENTRY(retpoline_r13)
|
|
CODEPATCH_START
|
|
JMP_RETPOLINE(r13)
|
|
CODEPATCH_END(CPTAG_RETPOLINE_R13)
|
|
END(retpoline_r13)
|
|
|
|
KIDTVEC(softtty)
|
|
endbr64
|
|
movl $IPL_SOFTTTY, CPUVAR(ILEVEL)
|
|
sti
|
|
incl CPUVAR(IDEPTH)
|
|
movl $X86_SOFTINTR_SOFTTTY,%edi
|
|
call softintr_dispatch
|
|
decl CPUVAR(IDEPTH)
|
|
CODEPATCH_START
|
|
jmp retpoline_r13
|
|
CODEPATCH_END(CPTAG_RETPOLINE_R13)
|
|
END(Xsofttty)
|
|
|
|
KIDTVEC(softnet)
|
|
endbr64
|
|
movl $IPL_SOFTNET, CPUVAR(ILEVEL)
|
|
sti
|
|
incl CPUVAR(IDEPTH)
|
|
movl $X86_SOFTINTR_SOFTNET,%edi
|
|
call softintr_dispatch
|
|
decl CPUVAR(IDEPTH)
|
|
CODEPATCH_START
|
|
jmp retpoline_r13
|
|
CODEPATCH_END(CPTAG_RETPOLINE_R13)
|
|
END(Xsoftnet)
|
|
|
|
KIDTVEC(softclock)
|
|
endbr64
|
|
movl $IPL_SOFTCLOCK, CPUVAR(ILEVEL)
|
|
sti
|
|
incl CPUVAR(IDEPTH)
|
|
movl $X86_SOFTINTR_SOFTCLOCK,%edi
|
|
call softintr_dispatch
|
|
decl CPUVAR(IDEPTH)
|
|
CODEPATCH_START
|
|
jmp retpoline_r13
|
|
CODEPATCH_END(CPTAG_RETPOLINE_R13)
|
|
END(Xsoftclock)
|