diff --git a/bsp/qemu-virt64-riscv/SConstruct b/bsp/qemu-virt64-riscv/SConstruct index 3387d0e1490..ae0e3375e11 100644 --- a/bsp/qemu-virt64-riscv/SConstruct +++ b/bsp/qemu-virt64-riscv/SConstruct @@ -38,5 +38,25 @@ if GetDepend('__STACKSIZE__'): stack_size = GetDepend('__STACKSIZE__') stack_lds.write('__STACKSIZE__ = %d;\n' % stack_size) stack_lds.close() +# Obtain the number of harts from rtconfig.h and write +# it into link_cpus.lds for the linker script +try: + with open('rtconfig.h', 'r') as f: + rtconfig_content = f.readlines() +except FileNotFoundError: + cpus_nr = 1 +else: + cpus_nr = 1 # default value + for line in rtconfig_content: + line = line.strip() + if line.startswith('#define') and 'RT_CPUS_NR' in line: + parts = line.split() + if len(parts) >= 3 and parts[2].isdigit(): + cpus_nr = int(parts[2]) + break + +with open('link_cpus.lds', 'w') as cpus_lds: + cpus_lds.write(f'RT_CPUS_NR = {cpus_nr};\n') + # make a building DoBuilding(TARGET, objs) diff --git a/bsp/qemu-virt64-riscv/driver/board.c b/bsp/qemu-virt64-riscv/driver/board.c index c5116aad0c5..c0df2d4e59e 100644 --- a/bsp/qemu-virt64-riscv/driver/board.c +++ b/bsp/qemu-virt64-riscv/driver/board.c @@ -89,6 +89,11 @@ void rt_hw_board_init(void) rt_hw_tick_init(); +#ifdef RT_USING_SMP + /* ipi init */ + rt_hw_ipi_init(); +#endif /* RT_USING_SMP */ + #ifdef RT_USING_COMPONENTS_INIT rt_components_board_init(); #endif diff --git a/bsp/qemu-virt64-riscv/link.lds b/bsp/qemu-virt64-riscv/link.lds index a76fed4fa30..52010cdf1dc 100644 --- a/bsp/qemu-virt64-riscv/link.lds +++ b/bsp/qemu-virt64-riscv/link.lds @@ -9,6 +9,7 @@ */ INCLUDE "link_stacksize.lds" +INCLUDE "link_cpus.lds" OUTPUT_ARCH( "riscv" ) @@ -121,12 +122,9 @@ SECTIONS { . = ALIGN(64); __stack_start__ = .; - - . += __STACKSIZE__; - __stack_cpu0 = .; - - . += __STACKSIZE__; - __stack_cpu1 = .; + /* Dynamically allocate stack areas according to RT_CPUS_NR */ + . += (__STACKSIZE__ * RT_CPUS_NR); + __stack_end__ = .; } > SRAM .sbss : diff --git a/bsp/qemu-virt64-riscv/link_cpus.lds b/bsp/qemu-virt64-riscv/link_cpus.lds new file mode 100644 index 00000000000..2659b2befb4 --- /dev/null +++ b/bsp/qemu-virt64-riscv/link_cpus.lds @@ -0,0 +1 @@ +RT_CPUS_NR = 8; diff --git a/bsp/qemu-virt64-riscv/qemu-dbg.sh b/bsp/qemu-virt64-riscv/qemu-dbg.sh index a7958ef8e88..69f62e7f6fb 100755 --- a/bsp/qemu-virt64-riscv/qemu-dbg.sh +++ b/bsp/qemu-virt64-riscv/qemu-dbg.sh @@ -1,4 +1,16 @@ -qemu-system-riscv64 -nographic -machine virt -m 256M -kernel rtthread.bin -s -S \ +QEMU_CMD="qemu-system-riscv64 -nographic -machine virt -m 256M -kernel rtthread.bin -s -S" + +if grep -q "#define RT_USING_SMP" ./rtconfig.h 2>/dev/null; then + hart_num=$(grep "RT_CPUS_NR = [0-9]*;" ./link_cpus.lds | awk -F'[=;]' '{gsub(/ /, "", $2); print $2}') + if [ -z "$hart_num" ]; then + hart_num=1 + fi + QEMU_CMD="$QEMU_CMD -smp $hart_num" +fi + +QEMU_CMD="$QEMU_CMD \ -drive if=none,file=sd.bin,format=raw,id=blk0 -device virtio-blk-device,drive=blk0,bus=virtio-mmio-bus.0 \ -netdev user,id=tap0 -device virtio-net-device,netdev=tap0,bus=virtio-mmio-bus.1 \ --device virtio-serial-device -chardev socket,host=127.0.0.1,port=4321,server=on,wait=off,telnet=on,id=console0 -device virtserialport,chardev=console0 +-device virtio-serial-device -chardev socket,host=127.0.0.1,port=4321,server=on,wait=off,telnet=on,id=console0 -device virtserialport,chardev=console0" + +eval $QEMU_CMD \ No newline at end of file diff --git a/bsp/qemu-virt64-riscv/run.sh b/bsp/qemu-virt64-riscv/run.sh index dd53c95f612..e723369fc7d 100755 --- a/bsp/qemu-virt64-riscv/run.sh +++ b/bsp/qemu-virt64-riscv/run.sh @@ -24,7 +24,19 @@ if [ ! -f $path_image ]; then exit fi -qemu-system-riscv64 -nographic -machine virt -m 256M -kernel rtthread.bin \ +QEMU_CMD="qemu-system-riscv64 -nographic -machine virt -m 256M -kernel rtthread.bin" + +if grep -q "#define RT_USING_SMP" ./rtconfig.h 2>/dev/null; then + hart_num=$(grep "RT_CPUS_NR = [0-9]*;" ./link_cpus.lds | awk -F'[=;]' '{gsub(/ /, "", $2); print $2}') + if [ -z "$hart_num" ]; then + hart_num=1 + fi + QEMU_CMD="$QEMU_CMD -smp $hart_num" +fi + +QEMU_CMD="$QEMU_CMD \ -drive if=none,file=$path_image,format=raw,id=blk0 -device virtio-blk-device,drive=blk0,bus=virtio-mmio-bus.0 \ -netdev user,id=tap0 -device virtio-net-device,netdev=tap0,bus=virtio-mmio-bus.1 \ --device virtio-serial-device -chardev socket,host=127.0.0.1,port=4321,server=on,wait=off,telnet=on,id=console0 -device virtserialport,chardev=console0 +-device virtio-serial-device -chardev socket,host=127.0.0.1,port=4321,server=on,wait=off,telnet=on,id=console0 -device virtserialport,chardev=console0" + +eval $QEMU_CMD \ No newline at end of file diff --git a/libcpu/risc-v/common64/atomic_riscv.c b/libcpu/risc-v/common64/atomic_riscv.c new file mode 100644 index 00000000000..08af84bf5f5 --- /dev/null +++ b/libcpu/risc-v/common64/atomic_riscv.c @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-03-14 WangShun first version + */ + +#include + +rt_atomic_t rt_hw_atomic_exchange(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile("amoswap.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoswap.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_add(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile("amoadd.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoadd.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_sub(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; + val = -val; +#if __riscv_xlen == 32 + asm volatile("amoadd.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoadd.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_xor(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile("amoxor.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoxor.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_and(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile("amoand.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoand.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_or(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile("amoor.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoor.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_load(volatile rt_atomic_t *ptr) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile("amoxor.w %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoxor.d %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); +#endif + return result; +} + +void rt_hw_atomic_store(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile("amoswap.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoswap.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif +} + +rt_atomic_t rt_hw_atomic_flag_test_and_set(volatile rt_atomic_t *ptr) +{ + rt_atomic_t result = 0; + rt_atomic_t temp = 1; +#if __riscv_xlen == 32 + asm volatile("amoor.w %0, %1, (%2)" : "=r"(result) : "r"(temp), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoor.d %0, %1, (%2)" : "=r"(result) : "r"(temp), "r"(ptr) : "memory"); +#endif + return result; +} + +void rt_hw_atomic_flag_clear(volatile rt_atomic_t *ptr) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile("amoand.w %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile("amoand.d %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); +#endif +} + +rt_atomic_t rt_hw_atomic_compare_exchange_strong(volatile rt_atomic_t *ptr, rt_atomic_t *old, rt_atomic_t desired) +{ + rt_atomic_t tmp = *old; + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile( + " fence iorw, ow\n" + "1: lr.w.aq %[result], (%[ptr])\n" + " bne %[result], %[tmp], 2f\n" + " sc.w.rl %[tmp], %[desired], (%[ptr])\n" + " bnez %[tmp], 1b\n" + " li %[result], 1\n" + " j 3f\n" + " 2:sw %[result], (%[old])\n" + " li %[result], 0\n" + " 3:\n" + : [result] "+r"(result), [tmp] "+r"(tmp), [ptr] "+r"(ptr) + : [desired] "r"(desired), [old] "r"(old) + : "memory"); +#elif __riscv_xlen == 64 + asm volatile( + " fence iorw, ow\n" + "1: lr.d.aq %[result], (%[ptr])\n" + " bne %[result], %[tmp], 2f\n" + " sc.d.rl %[tmp], %[desired], (%[ptr])\n" + " bnez %[tmp], 1b\n" + " li %[result], 1\n" + " j 3f\n" + " 2:sd %[result], (%[old])\n" + " li %[result], 0\n" + " 3:\n" + : [result] "+r"(result), [tmp] "+r"(tmp), [ptr] "+r"(ptr) + : [desired] "r"(desired), [old] "r"(old) + : "memory"); +#endif + return result; +} diff --git a/libcpu/risc-v/common64/context_gcc.S b/libcpu/risc-v/common64/context_gcc.S index ed216716b09..6d667362983 100644 --- a/libcpu/risc-v/common64/context_gcc.S +++ b/libcpu/risc-v/common64/context_gcc.S @@ -69,14 +69,27 @@ .endm /* + * #ifdef RT_USING_SMP + * void rt_hw_context_switch_to(rt_ubase_t to, stuct rt_thread *to_thread); + * #else * void rt_hw_context_switch_to(rt_ubase_t to); - * - * a0 --> to SP pointer + * #endif + * a0 --> to + * a1 --> to_thread */ .globl rt_hw_context_switch_to rt_hw_context_switch_to: LOAD sp, (a0) +#ifdef RT_USING_SMP + /* + * Pass the previous CPU lock status to + * rt_cpus_lock_status_restore for restoration + */ + mv a0, a1 + call rt_cpus_lock_status_restore +#endif + call rt_thread_self mv s1, a0 @@ -88,10 +101,15 @@ rt_hw_context_switch_to: sret /* + * #ifdef RT_USING_SMP + * void rt_hw_context_switch(rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread); + * #else * void rt_hw_context_switch(rt_ubase_t from, rt_ubase_t to); + * #endif * * a0 --> from SP pointer * a1 --> to SP pointer + * a2 --> to_thread * * It should only be used on local interrupt disable */ @@ -103,6 +121,15 @@ rt_hw_context_switch: // restore to thread SP LOAD sp, (a1) +#ifdef RT_USING_SMP + /* + * Pass the previous CPU lock status to + * rt_cpus_lock_status_restore for restoration + */ + mv a0, a2 + call rt_cpus_lock_status_restore +#endif /*RT_USING_SMP*/ + // restore Address Space call rt_thread_self mv s1, a0 diff --git a/libcpu/risc-v/common64/cpuport.c b/libcpu/risc-v/common64/cpuport.c index 76ae7d38271..d6a5c40c42c 100644 --- a/libcpu/risc-v/common64/cpuport.c +++ b/libcpu/risc-v/common64/cpuport.c @@ -19,15 +19,15 @@ #include #ifdef ARCH_RISCV_FPU - #define K_SSTATUS_DEFAULT_BASE (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM | SSTATUS_FS) +#define K_SSTATUS_DEFAULT_BASE (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM | SSTATUS_FS) #else - #define K_SSTATUS_DEFAULT_BASE (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM) +#define K_SSTATUS_DEFAULT_BASE (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM) #endif #ifdef ARCH_RISCV_VECTOR - #define K_SSTATUS_DEFAULT (K_SSTATUS_DEFAULT_BASE | SSTATUS_VS) +#define K_SSTATUS_DEFAULT (K_SSTATUS_DEFAULT_BASE | SSTATUS_VS) #else - #define K_SSTATUS_DEFAULT K_SSTATUS_DEFAULT_BASE +#define K_SSTATUS_DEFAULT K_SSTATUS_DEFAULT_BASE #endif #ifdef RT_USING_SMART #include @@ -51,8 +51,7 @@ volatile rt_ubase_t rt_thread_switch_interrupt_flag = 0; void *_rt_hw_stack_init(rt_ubase_t *sp, rt_ubase_t ra, rt_ubase_t sstatus) { - rt_hw_switch_frame_t frame = (rt_hw_switch_frame_t) - ((rt_ubase_t)sp - sizeof(struct rt_hw_switch_frame)); + rt_hw_switch_frame_t frame = (rt_hw_switch_frame_t)((rt_ubase_t)sp - sizeof(struct rt_hw_switch_frame)); rt_memset(frame, 0, sizeof(struct rt_hw_switch_frame)); @@ -64,7 +63,14 @@ void *_rt_hw_stack_init(rt_ubase_t *sp, rt_ubase_t ra, rt_ubase_t sstatus) int rt_hw_cpu_id(void) { +#ifndef RT_USING_SMP return 0; +#else + /* Currently, the hartid is stored in the satp register. */ + rt_ubase_t hart_id; + asm volatile("csrr %0, satp" : "=r"(hart_id)); + return hart_id; +#endif /* RT_USING_SMP */ } /** @@ -117,6 +123,18 @@ void rt_hw_context_switch_interrupt(rt_ubase_t from, rt_ubase_t to, rt_thread_t return; } +#else +void rt_hw_context_switch_interrupt(void *context, rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread) +{ + /* Perform architecture-specific context switch. This call will + * restore the target thread context and should not return when a + * switch is performed. The caller (scheduler) invoked this function + * in a context where local IRQs are disabled. */ + rt_uint32_t level; + level = rt_hw_local_irq_disable(); + rt_hw_context_switch((rt_ubase_t)from, (rt_ubase_t)to, to_thread); + rt_hw_local_irq_enable(level); +} #endif /* end of RT_USING_SMP */ /** shutdown CPU */ @@ -137,3 +155,48 @@ void rt_hw_set_process_id(int pid) { // TODO } + +#ifdef RT_USING_SMP +extern void _start(void); +extern int boot_hartid; +/* Boot secondary harts using the SBI HSM hart_start call. */ +void rt_hw_secondary_cpu_up(void) +{ + rt_uint64_t entry_pa; + int hart, ret; + + /* translate kernel virtual _start to physical address. + * TODO: Virtual-to-physical translation is not needed here + * because &_start is already a physical address on this platform. + */ + entry_pa = (rt_uint64_t)&_start; + + for (hart = 0; hart < RT_CPUS_NR; hart++) + { + if (hart == boot_hartid) + continue; + + ret = sbi_hsm_hart_start((unsigned long)hart, + (unsigned long)entry_pa, + 0UL); + if (ret) + { + rt_kprintf("sbi_hsm_hart_start failed for hart %d: %d\n", hart, ret); + } + } +} + +void secondary_cpu_entry(void) +{ + /* The PLIC peripheral interrupts are currently handled by the boot_hart. */ + /* Enable the Supervisor-Timer bit in SIE */ + rt_hw_tick_init(); + + /* ipi init */ + rt_hw_ipi_init(); + + rt_hw_spin_lock(&_cpus_lock); + /* invoke system scheduler start for secondary CPU */ + rt_system_scheduler_start(); +} +#endif /* RT_USING_SMP */ diff --git a/libcpu/risc-v/common64/interrupt_gcc.S b/libcpu/risc-v/common64/interrupt_gcc.S index 015900a16dd..5b1f2866688 100644 --- a/libcpu/risc-v/common64/interrupt_gcc.S +++ b/libcpu/risc-v/common64/interrupt_gcc.S @@ -60,10 +60,17 @@ _handle_interrupt_and_exception: call handle_trap _interrupt_exit: +#ifndef RT_USING_SMP la s0, rt_thread_switch_interrupt_flag lw s2, 0(s0) beqz s2, _resume_execution sw zero, 0(s0) +#else + mv a0, sp + call rt_scheduler_do_irq_switch + // if failed, jump to __resume_execution + j _resume_execution +#endif /* RT_USING_SMP */ _context_switch: la t0, rt_interrupt_from_thread @@ -88,6 +95,7 @@ _resume_kernel: csrw sscratch, zero sret +#ifndef RT_USING_SMP .global rt_hw_interrupt_enable rt_hw_interrupt_enable: csrs sstatus, a0 /* restore to old csr */ @@ -97,3 +105,18 @@ rt_hw_interrupt_enable: rt_hw_interrupt_disable: csrrci a0, sstatus, 2 /* clear SIE */ jr ra +#else +.global rt_hw_local_irq_disable +rt_hw_local_irq_disable: + csrrci a0, sstatus, 2 + jr ra + +.global rt_hw_local_irq_enable +rt_hw_local_irq_enable: + csrs sstatus, a0 + jr ra + +.global rt_hw_secondary_cpu_idle_exec +rt_hw_secondary_cpu_idle_exec: + jr ra +#endif /* RT_USING_SMP */ \ No newline at end of file diff --git a/libcpu/risc-v/common64/startup_gcc.S b/libcpu/risc-v/common64/startup_gcc.S index 184b48a1aeb..2a4153778b1 100644 --- a/libcpu/risc-v/common64/startup_gcc.S +++ b/libcpu/risc-v/common64/startup_gcc.S @@ -32,9 +32,19 @@ _start: 1: /* save hartid */ la t0, boot_hartid /* global varible rt_boot_hartid */ +#ifdef RT_USING_SMP + lw t2, (t0) + li t3, 0xdeadbeef + li t4, 0xffffffff + and t2, t2, t4 /* Extract the lower 32 bits. */ + bne t2, t3, system_init /* If the current value is 0xdeadbeef, skip the boot_hartid assignment operation. */ +#endif mv t1, a0 /* get hartid in S-mode frome a0 register */ sw t1, (t0) /* store t1 register low 4 bits in memory address which is stored in t0 */ +#ifdef RT_USING_SMP +system_init: +#endif /* clear Interrupt Registers */ csrw sie, 0 csrw sip, 0 @@ -51,7 +61,10 @@ _start: li x7, 0 li x8, 0 li x9, 0 +#ifndef RT_USING_SMP + /* In the SMP architecture, a0 will be used again later */ li x10,0 +#endif li x11,0 li x12,0 li x13,0 @@ -85,10 +98,29 @@ _start: la gp, __global_pointer$ .option pop +#ifndef RT_USING_SMP /* removed SMP support here */ la sp, __stack_start__ li t0, __STACKSIZE__ add sp, sp, t0 +#else + csrw satp, a0 /* Currently, the hartid is stored in the satp register. */ + /* Initialize the sp pointer according to different hartids. */ + mv t0, a0 + /* calculate stack offset: hartid * __STACKSIZE__ */ + li t1, __STACKSIZE__ + mul t0, t0, t1 /* t0 = hartid * __STACKSIZE__ */ + + /* set stack pointer */ + la sp, __stack_start__ + add sp, sp, t0 /* sp = __stack_start__ + hartid * __STACKSIZE__ */ + add sp, sp, t1 /* sp += __STACKSIZE__ (point to stack top) */ + + mv t0, a0 + lw t1, boot_hartid + bne t0, t1, secondary_cpu_entry + li x10,0 /* Clear the a0 register. */ +#endif /* RT_USING_SMP */ /** * sscratch is always zero on kernel mode diff --git a/libcpu/risc-v/common64/trap.c b/libcpu/risc-v/common64/trap.c index 1b79b73950c..4cfc9d82804 100644 --- a/libcpu/risc-v/common64/trap.c +++ b/libcpu/risc-v/common64/trap.c @@ -76,44 +76,44 @@ void dump_regs(struct rt_hw_stack_frame *regs) rt_kprintf("\tCurrent Page Table(Physical) = %p\n", __MASKVALUE(satp_v, __MASK(44)) << PAGE_OFFSET_BIT); rt_kprintf("\tCurrent ASID = %p\n", __MASKVALUE(satp_v >> 44, __MASK(16)) - << PAGE_OFFSET_BIT); + << PAGE_OFFSET_BIT); const char *mode_str = "Unknown Address Translation/Protection Mode"; switch (__MASKVALUE(satp_v >> 60, __MASK(4))) { - case 0: - mode_str = "No Address Translation/Protection Mode"; - break; + case 0: + mode_str = "No Address Translation/Protection Mode"; + break; - case 8: - mode_str = "Page-based 39-bit Virtual Addressing Mode"; - break; + case 8: + mode_str = "Page-based 39-bit Virtual Addressing Mode"; + break; - case 9: - mode_str = "Page-based 48-bit Virtual Addressing Mode"; - break; + case 9: + mode_str = "Page-based 48-bit Virtual Addressing Mode"; + break; } rt_kprintf("\tMode = %s\n", mode_str); rt_kprintf("-----------------Dump OK---------------------\n"); } -static const char *Exception_Name[] = {"Instruction Address Misaligned", - "Instruction Access Fault", - "Illegal Instruction", - "Breakpoint", - "Load Address Misaligned", - "Load Access Fault", - "Store/AMO Address Misaligned", - "Store/AMO Access Fault", - "Environment call from U-mode", - "Environment call from S-mode", - "Reserved-10", - "Reserved-11", - "Instruction Page Fault", - "Load Page Fault", - "Reserved-14", - "Store/AMO Page Fault"}; +static const char *Exception_Name[] = { "Instruction Address Misaligned", + "Instruction Access Fault", + "Illegal Instruction", + "Breakpoint", + "Load Address Misaligned", + "Load Access Fault", + "Store/AMO Address Misaligned", + "Store/AMO Access Fault", + "Environment call from U-mode", + "Environment call from S-mode", + "Reserved-10", + "Reserved-11", + "Instruction Page Fault", + "Load Page Fault", + "Reserved-14", + "Store/AMO Page Fault" }; static const char *Interrupt_Name[] = { "User Software Interrupt", @@ -135,7 +135,16 @@ static volatile int nested = 0; #define ENTER_TRAP nested += 1 #define EXIT_TRAP nested -= 1 #define CHECK_NESTED_PANIC(cause, tval, epc, eframe) \ - if (nested != 1) handle_nested_trap_panic(cause, tval, epc, eframe) + if (nested != 1) \ + handle_nested_trap_panic(cause, tval, epc, eframe) +#else +/* Add trap nesting detection under the SMP architecture. */ +static volatile int nested[RT_CPUS_NR] = { 0 }; +#define ENTER_TRAP nested[rt_hw_cpu_id()] += 1 +#define EXIT_TRAP nested[rt_hw_cpu_id()] -= 1 +#define CHECK_NESTED_PANIC(cause, tval, epc, eframe) \ + if (nested[rt_hw_cpu_id()] != 1) \ + handle_nested_trap_panic(cause, tval, epc, eframe) #endif /* RT_USING_SMP */ static const char *get_exception_msg(int id) @@ -165,44 +174,44 @@ void handle_user(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, enum rt_mm_fault_type fault_type; switch (id) { - case EP_LOAD_PAGE_FAULT: - fault_op = MM_FAULT_OP_READ; - fault_type = MM_FAULT_TYPE_GENERIC_MMU; - break; - case EP_LOAD_ACCESS_FAULT: - fault_op = MM_FAULT_OP_READ; - fault_type = MM_FAULT_TYPE_BUS_ERROR; - break; - case EP_LOAD_ADDRESS_MISALIGNED: - fault_op = MM_FAULT_OP_READ; - fault_type = MM_FAULT_TYPE_BUS_ERROR; - break; - case EP_STORE_PAGE_FAULT: - fault_op = MM_FAULT_OP_WRITE; - fault_type = MM_FAULT_TYPE_GENERIC_MMU; - break; - case EP_STORE_ACCESS_FAULT: - fault_op = MM_FAULT_OP_WRITE; - fault_type = MM_FAULT_TYPE_BUS_ERROR; - break; - case EP_STORE_ADDRESS_MISALIGNED: - fault_op = MM_FAULT_OP_WRITE; - fault_type = MM_FAULT_TYPE_BUS_ERROR; - break; - case EP_INSTRUCTION_PAGE_FAULT: - fault_op = MM_FAULT_OP_EXECUTE; - fault_type = MM_FAULT_TYPE_GENERIC_MMU; - break; - case EP_INSTRUCTION_ACCESS_FAULT: - fault_op = MM_FAULT_OP_EXECUTE; - fault_type = MM_FAULT_TYPE_BUS_ERROR; - break; - case EP_INSTRUCTION_ADDRESS_MISALIGNED: - fault_op = MM_FAULT_OP_EXECUTE; - fault_type = MM_FAULT_TYPE_BUS_ERROR; - break; - default: - fault_op = 0; + case EP_LOAD_PAGE_FAULT: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; + break; + case EP_LOAD_ACCESS_FAULT: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EP_LOAD_ADDRESS_MISALIGNED: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EP_STORE_PAGE_FAULT: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; + break; + case EP_STORE_ACCESS_FAULT: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EP_STORE_ADDRESS_MISALIGNED: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EP_INSTRUCTION_PAGE_FAULT: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; + break; + case EP_INSTRUCTION_ACCESS_FAULT: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EP_INSTRUCTION_ADDRESS_MISALIGNED: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + default: + fault_op = 0; } if (fault_op) @@ -228,7 +237,7 @@ void handle_user(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, dump_regs(sp); rt_thread_t cur_thr = rt_thread_self(); - struct rt_hw_backtrace_frame frame = {.fp = sp->s0_fp, .pc = sepc}; + struct rt_hw_backtrace_frame frame = { .fp = sp->s0_fp, .pc = sepc }; rt_kprintf("fp = %p\n", frame.fp); lwp_backtrace_frame(cur_thr, &frame); @@ -260,12 +269,12 @@ static int illegal_inst_recoverable(rt_ubase_t stval, switch (opcode) { - case 0x57: // V - case 0x27: // scalar FLOAT - case 0x07: - case 0x73: // CSR - flag = 1; - break; + case 0x57: // V + case 0x27: // scalar FLOAT + case 0x07: + case 0x73: // CSR + flag = 1; + break; } if (flag) @@ -314,6 +323,15 @@ void handle_trap(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, tick_isr(); rt_interrupt_leave(); } +#ifdef RT_USING_SMP + else if ((SCAUSE_INTERRUPT | SCAUSE_S_SOFTWARE_INTR) == scause) + { + /* supervisor software interrupt for ipi */ + rt_interrupt_enter(); + rt_hw_ipi_handler(); + rt_interrupt_leave(); + } +#endif /* RT_USING_SMP */ else { if (SCAUSE_INTERRUPT & scause) @@ -364,7 +382,7 @@ void handle_trap(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, rt_kprintf("current thread: %s\n", cur_thr->parent.name); rt_kprintf("--------------Backtrace--------------\n"); - struct rt_hw_backtrace_frame frame = {.fp = sp->s0_fp, .pc = sepc}; + struct rt_hw_backtrace_frame frame = { .fp = sp->s0_fp, .pc = sepc }; #ifdef RT_USING_SMART if (!(sp->sstatus & 0x100)) diff --git a/libcpu/risc-v/virt64/interrupt.c b/libcpu/risc-v/virt64/interrupt.c index f627210dac2..4147c33c85e 100644 --- a/libcpu/risc-v/virt64/interrupt.c +++ b/libcpu/risc-v/virt64/interrupt.c @@ -16,6 +16,10 @@ #include "interrupt.h" struct rt_irq_desc irq_desc[MAX_HANDLERS]; +#ifdef RT_USING_SMP +struct rt_irq_desc ipi_desc[RT_MAX_IPI]; +uint8_t ipi_vectors[RT_CPUS_NR] = { 0 }; +#endif static rt_isr_handler_t rt_hw_interrupt_handle(rt_uint32_t vector, void *param) { @@ -53,11 +57,11 @@ void rt_hw_interrupt_umask(int vector) * @param old_handler the old interrupt service routine */ rt_isr_handler_t rt_hw_interrupt_install(int vector, rt_isr_handler_t handler, - void *param, const char *name) + void *param, const char *name) { rt_isr_handler_t old_handler = RT_NULL; - if(vector < MAX_HANDLERS) + if (vector < MAX_HANDLERS) { old_handler = irq_desc[vector].handler; if (handler != RT_NULL) @@ -92,3 +96,129 @@ void rt_hw_interrupt_init() plic_set_threshold(0); } + +#ifdef RT_USING_SMP +void rt_hw_interrupt_set_priority(int vector, unsigned int priority) +{ + plic_set_priority(vector, priority); +} + +unsigned int rt_hw_interrupt_get_priority(int vector) +{ + return (*(uint32_t *)PLIC_PRIORITY(vector)); +} + +rt_bool_t rt_hw_interrupt_is_disabled(void) +{ + /* Determine the interrupt enable state */ + rt_ubase_t sstatus; + __asm__ volatile("csrr %0, sstatus" : "=r"(sstatus)); + return (sstatus & SSTATUS_SIE) == 0; +} + +void rt_hw_spin_lock_init(rt_hw_spinlock_t *_lock) +{ + _lock->slock = 0; +} + +void rt_hw_spin_lock(rt_hw_spinlock_t *lock) +{ + /* Use ticket lock implemented on top of the 32/64-bit atomic AMO ops. + * The combined word layout (slock) maps two uint16_t fields: + * low 16 bits: owner + * high 16 bits: next (ticket allocator) + * We atomically increment the "next" field by (1 << 16) and use the + * returned old value to compute our ticket. Then wait until owner == ticket. + */ + rt_atomic_t prev; + rt_atomic_t ticket; + rt_atomic_t owner; + + /* Allocate a ticket by adding (1 << 16) to slock, prev holds previous value */ + prev = rt_hw_atomic_add((volatile rt_atomic_t *)&lock->slock, (rt_atomic_t)(1UL << 16)); + ticket = (prev >> 16) & 0xffffUL; + + /* Wait until owner equals our ticket */ + for (;;) + { + owner = rt_hw_atomic_load((volatile rt_atomic_t *)&lock->slock) & 0xffffUL; + if (owner == ticket) + break; + /* TODO: low-power wait for interrupt while spinning */ + } + + /* Ensure all following memory accesses are ordered after acquiring the lock */ + __asm__ volatile("fence rw, rw" ::: "memory"); +} + +void rt_hw_spin_unlock(rt_hw_spinlock_t *lock) +{ + /* Ensure memory operations before unlock are visible before owner increment */ + __asm__ volatile("fence rw, rw" ::: "memory"); + + /* Increment owner (low 16 bits) to hand over lock to next ticket */ + rt_hw_atomic_add((volatile rt_atomic_t *)&lock->slock, (rt_atomic_t)1); + + // TODO: IPI interrupt to wake up other harts waiting for the lock + + /* Make the increment visible to other harts */ + __asm__ volatile("fence rw, rw" ::: "memory"); +} + +void rt_hw_ipi_send(int ipi_vector, unsigned int cpu_mask) +{ + int cpuid = __builtin_ctz(cpu_mask); // get the bit position of the lowest set bit + ipi_vectors[cpuid] |= (uint8_t)ipi_vector; + sbi_send_ipi((const unsigned long *)&cpu_mask); +} + +void rt_hw_ipi_init(void) +{ + int idx = 0, cpuid = rt_cpu_get_id(); + ipi_vectors[cpuid] = 0; + /* init exceptions table */ + for (idx = 0; idx < RT_MAX_IPI; idx++) + { + ipi_desc[idx].handler = RT_NULL; + ipi_desc[idx].param = RT_NULL; +#ifdef RT_USING_INTERRUPT_INFO + rt_snprintf(ipi_desc[idx].name, RT_NAME_MAX - 1, "default"); + ipi_desc[idx].counter = 0; +#endif + } + set_csr(sie, SIP_SSIP); +} + +void rt_hw_ipi_handler_install(int ipi_vector, rt_isr_handler_t ipi_isr_handler) +{ + if (ipi_vector < RT_MAX_IPI) + { + if (ipi_isr_handler != RT_NULL) + { + ipi_desc[ipi_vector].handler = (rt_isr_handler_t)ipi_isr_handler; + ipi_desc[ipi_vector].param = RT_NULL; + } + } +} + +void rt_hw_ipi_handler(void) +{ + rt_uint32_t ipi_vector; + + ipi_vector = ipi_vectors[rt_cpu_get_id()]; + while (ipi_vector) + { + int bitpos = __builtin_ctz(ipi_vector); + ipi_vector &= ~(1 << bitpos); + if (bitpos < RT_MAX_IPI && ipi_desc[bitpos].handler != RT_NULL) + { + /* call the irq service routine */ + ipi_desc[bitpos].handler(bitpos, ipi_desc[bitpos].param); + } + } + ipi_vectors[rt_cpu_get_id()] = 0; + + // clear software interrupt pending bit + clear_csr(sip, SIP_SSIP); +} +#endif /* RT_USING_SMP */