mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-11-21 07:15:49 +00:00
Code Issues Releases Activity

riscv: Convert local interrupt controller to a newbus PIC

Browse Source 
Currently the local interrupt controller implementation is based on
pre-INTRNG arm/arm64 code, using hand-rolled event code rather than
INTRNG. This then interacts weirdly with the PLIC, and other future
interrupt controllers like the APLIC and IMSICs in the upcoming AIA
specification, since they become the root PIC despite not being the
logical root. Instead, use a real newbus device for it and register
it as the root PIC.

This also adapts the IPI code to make use of the newly-added INTRNG
generic IPI handling framework, adding a new sbi_ipi as the PIC. In
future there will be alternative devices for sending IPIs that will
register with higher priorities, such as the proposed AIA IMSIC and
ACLINT SSWI.

Reviewed by:	mhorne
MFC after:	1 month
Differential Revision:	https://reviews.freebsd.org/D35901
This commit is contained in:
Jessica Clarke 2024-01-24 23:49:54 +00:00
parent c55272fdf8
commit 6ec8bf9f3d
12 changed files with 732 additions and 394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sys/conf/files.riscv
View File

@ -43,7 +43,7 @@ riscv/riscv/elf_machdep.c standard
riscv/riscv/exception.S standard
riscv/riscv/exec_machdep.c standard
riscv/riscv/gdb_machdep.c optional gdb
riscv/riscv/intr_machdep.c standard
riscv/riscv/intc.c standard
riscv/riscv/identcpu.c standard
riscv/riscv/locore.S standard no-obj
riscv/riscv/machdep.c standard
@ -58,6 +58,7 @@ riscv/riscv/ptrace_machdep.c standard
riscv/riscv/riscv_console.c optional rcons
riscv/riscv/riscv_syscon.c optional syscon riscv_syscon fdt
riscv/riscv/sbi.c standard
riscv/riscv/sbi_ipi.c optional smp
riscv/riscv/stack_machdep.c optional ddb | stack
riscv/riscv/support.S standard
riscv/riscv/swtch.S standard

24
sys/riscv/include/intr.h
View File

@ -35,32 +35,11 @@
#ifndef _MACHINE_INTR_MACHDEP_H_
#define _MACHINE_INTR_MACHDEP_H_
#define RISCV_NIRQ 1024
#ifndef NIRQ
#define NIRQ RISCV_NIRQ
#define NIRQ 1024
#endif
#ifdef INTRNG
#include <sys/intr.h>
#endif
struct trapframe;
int riscv_teardown_intr(void *);
int riscv_setup_intr(const char *, driver_filter_t *, driver_intr_t *,
void *, int, int, void **);
void riscv_cpu_intr(struct trapframe *);
typedef unsigned long * riscv_intrcnt_t;
riscv_intrcnt_t riscv_intrcnt_create(const char *);
void riscv_intrcnt_setname(riscv_intrcnt_t, const char *);
#ifdef SMP
void riscv_setup_ipihandler(driver_filter_t *);
void riscv_unmask_ipi(void);
#endif
enum {
IRQ_SOFTWARE_USER,
@ -75,7 +54,6 @@ enum {
IRQ_EXTERNAL_SUPERVISOR,
IRQ_EXTERNAL_HYPERVISOR,
IRQ_EXTERNAL_MACHINE,
INTC_NIRQS
};
#endif /* !_MACHINE_INTR_MACHDEP_H_ */

17
sys/riscv/include/smp.h
View File

@ -37,14 +37,15 @@
#include <machine/pcb.h>
#define IPI_AST (1 << 0)
#define IPI_PREEMPT (1 << 1)
#define IPI_RENDEZVOUS (1 << 2)
#define IPI_STOP (1 << 3)
#define IPI_STOP_HARD (1 << 4)
#define IPI_HARDCLOCK (1 << 5)
#define INTR_IPI_COUNT 1
enum {
IPI_AST,
IPI_PREEMPT,
IPI_RENDEZVOUS,
IPI_STOP,
IPI_STOP_HARD,
IPI_HARDCLOCK,
INTR_IPI_COUNT
};
void ipi_all_but_self(u_int ipi);
void ipi_cpu(int cpu, u_int ipi);

311
sys/riscv/riscv/intc.c Normal file
View File

@ -0,0 +1,311 @@
/*-
* Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
* Copyright (c) 2021 Jessica Clarke <jrtc27@FreeBSD.org>
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/proc.h>
#include <sys/cpuset.h>
#include <sys/interrupt.h>
#include <sys/smp.h>
#include <machine/bus.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/frame.h>
#include <machine/intr.h>
#include <dev/fdt/simplebus.h>
#include <dev/ofw/openfirm.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#include "pic_if.h"
#define INTC_NIRQS 16
struct intc_irqsrc {
struct intr_irqsrc isrc;
u_int irq;
};
struct intc_softc {
device_t dev;
struct intc_irqsrc isrcs[INTC_NIRQS];
};
static int intc_intr(void *arg);
static phandle_t
intc_ofw_find(device_t dev, uint32_t hartid)
{
phandle_t node;
pcell_t reg;
node = OF_finddevice("/cpus");
if (node == -1) {
device_printf(dev, "Can't find cpus node\n");
return ((phandle_t)-1);
}
for (node = OF_child(node); node != 0; node = OF_peer(node)) {
if (!ofw_bus_node_status_okay(node))
continue;
if (!ofw_bus_node_is_compatible(node, "riscv"))
continue;
if (OF_searchencprop(node, "reg", &reg, sizeof(reg)) == -1)
continue;
if (reg == hartid)
break;
}
if (node == 0) {
device_printf(dev, "Can't find boot cpu node\n");
return ((phandle_t)-1);
}
for (node = OF_child(node); node != 0; node = OF_peer(node)) {
if (!ofw_bus_node_status_okay(node))
continue;
if (ofw_bus_node_is_compatible(node, "riscv,cpu-intc"))
break;
}
if (node == 0) {
device_printf(dev,
"Can't find boot cpu local interrupt controller\n");
return ((phandle_t)-1);
}
return (node);
}
static void
intc_identify(driver_t *driver, device_t parent)
{
device_t dev;
phandle_t node;
if (device_find_child(parent, "intc", -1) != NULL)
return;
node = intc_ofw_find(parent, PCPU_GET(hart));
if (node == -1)
return;
dev = simplebus_add_device(parent, node, 0, "intc", -1, NULL);
if (dev == NULL)
device_printf(parent, "Can't add intc child\n");
}
static int
intc_probe(device_t dev)
{
device_set_desc(dev, "RISC-V Local Interrupt Controller");
return (BUS_PROBE_NOWILDCARD);
}
static int
intc_attach(device_t dev)
{
struct intc_irqsrc *isrcs;
struct intc_softc *sc;
struct intr_pic *pic;
const char *name;
phandle_t xref;
u_int flags;
int i, error;
sc = device_get_softc(dev);
sc->dev = dev;
name = device_get_nameunit(dev);
xref = OF_xref_from_node(ofw_bus_get_node(dev));
isrcs = sc->isrcs;
for (i = 0; i < INTC_NIRQS; i++) {
isrcs[i].irq = i;
flags = i == IRQ_SOFTWARE_SUPERVISOR
? INTR_ISRCF_IPI : INTR_ISRCF_PPI;
error = intr_isrc_register(&isrcs[i].isrc, sc->dev, flags,
"%s,%u", name, i);
if (error != 0) {
device_printf(dev, "Can't register interrupt %d\n", i);
return (error);
}
}
pic = intr_pic_register(sc->dev, xref);
if (pic == NULL)
return (ENXIO);
return (intr_pic_claim_root(sc->dev, xref, intc_intr, sc));
}
static void
intc_disable_intr(device_t dev, struct intr_irqsrc *isrc)
{
u_int irq;
irq = ((struct intc_irqsrc *)isrc)->irq;
if (irq >= INTC_NIRQS)
panic("%s: Unsupported IRQ %u", __func__, irq);
csr_clear(sie, 1ul << irq);
}
static void
intc_enable_intr(device_t dev, struct intr_irqsrc *isrc)
{
u_int irq;
irq = ((struct intc_irqsrc *)isrc)->irq;
if (irq >= INTC_NIRQS)
panic("%s: Unsupported IRQ %u", __func__, irq);
csr_set(sie, 1ul << irq);
}
static int
intc_map_intr(device_t dev, struct intr_map_data *data,
struct intr_irqsrc **isrcp)
{
struct intr_map_data_fdt *daf;
struct intc_softc *sc;
sc = device_get_softc(dev);
if (data->type != INTR_MAP_DATA_FDT)
return (ENOTSUP);
daf = (struct intr_map_data_fdt *)data;
if (daf->ncells != 1 || daf->cells[0] >= INTC_NIRQS)
return (EINVAL);
*isrcp = &sc->isrcs[daf->cells[0]].isrc;
return (0);
}
static int
intc_setup_intr(device_t dev, struct intr_irqsrc *isrc,
struct resource *res, struct intr_map_data *data)
{
if (isrc->isrc_flags & INTR_ISRCF_PPI)
CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
return (0);
}
#ifdef SMP
static void
intc_init_secondary(device_t dev)
{
struct intc_softc *sc;
struct intr_irqsrc *isrc;
u_int cpu, irq;
sc = device_get_softc(dev);
cpu = PCPU_GET(cpuid);
/* Unmask attached interrupts */
for (irq = 0; irq < INTC_NIRQS; irq++) {
isrc = &sc->isrcs[irq].isrc;
if (intr_isrc_init_on_cpu(isrc, cpu))
intc_enable_intr(dev, isrc);
}
}
#endif
static int
intc_intr(void *arg)
{
struct trapframe *frame;
struct intc_softc *sc;
uint64_t active_irq;
struct intc_irqsrc *src;
sc = arg;
frame = curthread->td_intr_frame;
KASSERT((frame->tf_scause & SCAUSE_INTR) != 0,
("%s: not an interrupt frame", __func__));
active_irq = frame->tf_scause & SCAUSE_CODE;
if (active_irq >= INTC_NIRQS)
return (FILTER_HANDLED);
src = &sc->isrcs[active_irq];
if (intr_isrc_dispatch(&src->isrc, frame) != 0) {
intc_disable_intr(sc->dev, &src->isrc);
device_printf(sc->dev, "Stray irq %lu disabled\n",
active_irq);
}
return (FILTER_HANDLED);
}
static device_method_t intc_methods[] = {
/* Device interface */
DEVMETHOD(device_identify, intc_identify),
DEVMETHOD(device_probe, intc_probe),
DEVMETHOD(device_attach, intc_attach),
/* Interrupt controller interface */
DEVMETHOD(pic_disable_intr, intc_disable_intr),
DEVMETHOD(pic_enable_intr, intc_enable_intr),
DEVMETHOD(pic_map_intr, intc_map_intr),
DEVMETHOD(pic_setup_intr, intc_setup_intr),
#ifdef SMP
DEVMETHOD(pic_init_secondary, intc_init_secondary),
#endif
DEVMETHOD_END
};
DEFINE_CLASS_0(intc, intc_driver, intc_methods, sizeof(struct intc_softc));
EARLY_DRIVER_MODULE(intc, ofwbus, intc_driver, 0, 0,
BUS_PASS_INTERRUPT + BUS_PASS_ORDER_FIRST);

274
sys/riscv/riscv/intr_machdep.c
View File

@ -1,274 +0,0 @@
/*-
* Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/module.h>
#include <sys/cpuset.h>
#include <sys/interrupt.h>
#include <sys/smp.h>
#include <machine/bus.h>
#include <machine/clock.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/frame.h>
#include <machine/intr.h>
#include <machine/sbi.h>
#include <dev/ofw/openfirm.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#ifdef SMP
#include <machine/smp.h>
#endif
void intr_irq_handler(struct trapframe *tf);
struct intc_irqsrc {
struct intr_irqsrc isrc;
u_int irq;
};
struct intc_irqsrc isrcs[INTC_NIRQS];
static void
riscv_mask_irq(void *source)
{
int irq;
irq = (int)(uintptr_t)source;
switch (irq) {
case IRQ_TIMER_SUPERVISOR:
csr_clear(sie, SIE_STIE);
break;
case IRQ_SOFTWARE_USER:
csr_clear(sie, SIE_USIE);
break;
case IRQ_SOFTWARE_SUPERVISOR:
csr_clear(sie, SIE_SSIE);
break;
default:
panic("Unknown irq %d\n", irq);
}
}
static void
riscv_unmask_irq(void *source)
{
int irq;
irq = (int)(uintptr_t)source;
switch (irq) {
case IRQ_TIMER_SUPERVISOR:
csr_set(sie, SIE_STIE);
break;
case IRQ_SOFTWARE_USER:
csr_set(sie, SIE_USIE);
break;
case IRQ_SOFTWARE_SUPERVISOR:
csr_set(sie, SIE_SSIE);
break;
default:
panic("Unknown irq %d\n", irq);
}
}
int
riscv_setup_intr(const char *name, driver_filter_t *filt,
void (*handler)(void*), void *arg, int irq, int flags, void **cookiep)
{
struct intr_irqsrc *isrc;
int error;
if (irq < 0 || irq >= INTC_NIRQS)
panic("%s: unknown intr %d", __func__, irq);
isrc = &isrcs[irq].isrc;
if (isrc->isrc_event == NULL) {
error = intr_event_create(&isrc->isrc_event, isrc, 0, irq,
riscv_mask_irq, riscv_unmask_irq, NULL, NULL, "int%d", irq);
if (error)
return (error);
riscv_unmask_irq((void*)(uintptr_t)irq);
}
error = intr_event_add_handler(isrc->isrc_event, name,
filt, handler, arg, intr_priority(flags), flags, cookiep);
if (error) {
printf("Failed to setup intr: %d\n", irq);
return (error);
}
return (0);
}
int
riscv_teardown_intr(void *ih)
{
/* TODO */
return (0);
}
void
riscv_cpu_intr(struct trapframe *frame)
{
struct intr_irqsrc *isrc;
int active_irq;
KASSERT((frame->tf_scause & SCAUSE_INTR) != 0,
("riscv_cpu_intr: wrong frame passed"));
active_irq = frame->tf_scause & SCAUSE_CODE;
CTR3(KTR_TRAP, "%s: irq=%d, umode=%d", __func__, active_irq,
TRAPF_USERMODE(frame));
switch (active_irq) {
case IRQ_SOFTWARE_USER:
case IRQ_SOFTWARE_SUPERVISOR:
case IRQ_TIMER_SUPERVISOR:
critical_enter();
isrc = &isrcs[active_irq].isrc;
if (intr_isrc_dispatch(isrc, frame) != 0)
printf("stray interrupt %d\n", active_irq);
critical_exit();
break;
case IRQ_EXTERNAL_SUPERVISOR:
intr_irq_handler(frame);
break;
}
}
#ifdef SMP
void
riscv_setup_ipihandler(driver_filter_t *filt)
{
riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE_SUPERVISOR,
INTR_TYPE_MISC, NULL);
}
void
riscv_unmask_ipi(void)
{
csr_set(sie, SIE_SSIE);
}
/* Sending IPI */
static void
ipi_send(struct pcpu *pc, int ipi)
{
u_long mask;
CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, pc->pc_cpuid, ipi);
atomic_set_32(&pc->pc_pending_ipis, ipi);
mask = (1 << pc->pc_hart);
sbi_send_ipi(&mask);
CTR1(KTR_SMP, "%s: sent", __func__);
}
void
ipi_all_but_self(u_int ipi)
{
cpuset_t other_cpus;
other_cpus = all_cpus;
CPU_CLR(PCPU_GET(cpuid), &other_cpus);
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
ipi_selected(other_cpus, ipi);
}
void
ipi_cpu(int cpu, u_int ipi)
{
cpuset_t cpus;
CPU_ZERO(&cpus);
CPU_SET(cpu, &cpus);
ipi_send(cpuid_to_pcpu[cpu], ipi);
}
void
ipi_selected(cpuset_t cpus, u_int ipi)
{
struct pcpu *pc;
u_long mask;
CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi);
mask = 0;
STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
if (CPU_ISSET(pc->pc_cpuid, &cpus)) {
CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc,
ipi);
atomic_set_32(&pc->pc_pending_ipis, ipi);
mask |= (1 << pc->pc_hart);
}
}
sbi_send_ipi(&mask);
}
#endif
/* Interrupt machdep initialization routine. */
static void
intc_init(void *dummy __unused)
{
int error;
int i;
for (i = 0; i < INTC_NIRQS; i++) {
isrcs[i].irq = i;
error = intr_isrc_register(&isrcs[i].isrc, NULL,
0, "intc,%u", i);
if (error != 0)
printf("Can't register interrupt %d\n", i);
}
}
SYSINIT(intc_init, SI_SUB_INTR, SI_ORDER_MIDDLE, intc_init, NULL);

156
sys/riscv/riscv/mp_machdep.c
View File

@ -83,7 +83,11 @@ static device_identify_t riscv64_cpu_identify;
static device_probe_t riscv64_cpu_probe;
static device_attach_t riscv64_cpu_attach;
static int ipi_handler(void *);
static void ipi_ast(void *);
static void ipi_hardclock(void *);
static void ipi_preempt(void *);
static void ipi_rendezvous(void *);
static void ipi_stop(void *);
extern uint32_t boot_hart;
extern cpuset_t all_harts;
@ -191,8 +195,13 @@ release_aps(void *dummy __unused)
if (mp_ncpus == 1)
return;
/* Setup the IPI handler */
riscv_setup_ipihandler(ipi_handler);
/* Setup the IPI handlers */
intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL);
intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
atomic_store_rel_int(&aps_ready, 1);
@ -244,17 +253,14 @@ init_secondary(uint64_t hart)
pcpup->pc_curthread = pcpup->pc_idlethread;
schedinit_ap();
/* Enable software interrupts */
riscv_unmask_ipi();
/* Setup and enable interrupts */
intr_pic_init_secondary();
#ifndef EARLY_AP_STARTUP
/* Start per-CPU event timers. */
cpu_initclocks_ap();
#endif
/* Enable external (PLIC) interrupts */
csr_set(sie, SIE_SEIE);
/* Activate this hart in the kernel pmap. */
CPU_SET_ATOMIC(hart, &kernel_pmap->pm_active);
@ -308,74 +314,59 @@ smp_after_idle_runnable(void *arg __unused)
SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
smp_after_idle_runnable, NULL);
static int
ipi_handler(void *arg)
static void
ipi_ast(void *dummy __unused)
{
u_int ipi_bitmap;
u_int cpu, ipi;
int bit;
CTR0(KTR_SMP, "IPI_AST");
}
csr_clear(sip, SIP_SSIP);
static void
ipi_preempt(void *dummy __unused)
{
CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
sched_preempt(curthread);
}
static void
ipi_rendezvous(void *dummy __unused)
{
CTR0(KTR_SMP, "IPI_RENDEZVOUS");
smp_rendezvous_action();
}
static void
ipi_stop(void *dummy __unused)
{
u_int cpu;
CTR0(KTR_SMP, "IPI_STOP");
cpu = PCPU_GET(cpuid);
savectx(&stoppcbs[cpu]);
mb();
/* Indicate we are stopped */
CPU_SET_ATOMIC(cpu, &stopped_cpus);
ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis));
if (ipi_bitmap == 0)
return (FILTER_HANDLED);
/* Wait for restart */
while (!CPU_ISSET(cpu, &started_cpus))
cpu_spinwait();
while ((bit = ffs(ipi_bitmap))) {
bit = (bit - 1);
ipi = (1 << bit);
ipi_bitmap &= ~ipi;
CPU_CLR_ATOMIC(cpu, &started_cpus);
CPU_CLR_ATOMIC(cpu, &stopped_cpus);
CTR0(KTR_SMP, "IPI_STOP (restart)");
mb();
/*
* The kernel debugger might have set a breakpoint,
* so flush the instruction cache.
*/
fence_i();
}
switch (ipi) {
case IPI_AST:
CTR0(KTR_SMP, "IPI_AST");
break;
case IPI_PREEMPT:
CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
sched_preempt(curthread);
break;
case IPI_RENDEZVOUS:
CTR0(KTR_SMP, "IPI_RENDEZVOUS");
smp_rendezvous_action();
break;
case IPI_STOP:
case IPI_STOP_HARD:
CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD");
savectx(&stoppcbs[cpu]);
/* Indicate we are stopped */
CPU_SET_ATOMIC(cpu, &stopped_cpus);
/* Wait for restart */
while (!CPU_ISSET(cpu, &started_cpus))
cpu_spinwait();
CPU_CLR_ATOMIC(cpu, &started_cpus);
CPU_CLR_ATOMIC(cpu, &stopped_cpus);
CTR0(KTR_SMP, "IPI_STOP (restart)");
/*
* The kernel debugger might have set a breakpoint,
* so flush the instruction cache.
*/
fence_i();
break;
case IPI_HARDCLOCK:
CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
hardclockintr();
break;
default:
panic("Unknown IPI %#0x on cpu %d", ipi, curcpu);
}
}
return (FILTER_HANDLED);
static void
ipi_hardclock(void *dummy __unused)
{
CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
hardclockintr();
}
struct cpu_group *
@ -574,3 +565,34 @@ cpu_mp_setmaxid(void)
}
}
}
void
ipi_all_but_self(u_int ipi)
{
cpuset_t other_cpus;
other_cpus = all_cpus;
CPU_CLR(PCPU_GET(cpuid), &other_cpus);
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
intr_ipi_send(other_cpus, ipi);
}
void
ipi_cpu(int cpu, u_int ipi)
{
cpuset_t cpus;
CPU_ZERO(&cpus);
CPU_SET(cpu, &cpus);
CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi);
intr_ipi_send(cpus, ipi);
}
void
ipi_selected(cpuset_t cpus, u_int ipi)
{
CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi);
intr_ipi_send(cpus, ipi);
}

49
sys/riscv/riscv/plic.c
View File

@ -91,16 +91,18 @@ struct plic_context {
struct plic_softc {
device_t dev;
struct resource * intc_res;
struct resource *mem_res;
struct resource *irq_res;
void *ih;
struct plic_irqsrc isrcs[PLIC_MAX_IRQS];
struct plic_context contexts[MAXCPU];
int ndev;
};
#define RD4(sc, reg) \
bus_read_4(sc->intc_res, (reg))
bus_read_4(sc->mem_res, (reg))
#define WR4(sc, reg, val) \
bus_write_4(sc->intc_res, (reg), (val))
bus_write_4(sc->mem_res, (reg), (val))
static u_int plic_irq_cpu;
@ -276,9 +278,9 @@ plic_attach(device_t dev)
/* Request memory resources */
rid = 0;
sc->intc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->intc_res == NULL) {
if (sc->mem_res == NULL) {
device_printf(dev,
"Error: could not allocate memory resources\n");
return (ENXIO);
@ -302,7 +304,7 @@ plic_attach(device_t dev)
*
* This is tricky for a few reasons. The PLIC divides the interrupt
* enable, threshold, and claim bits by "context", where each context
* routes to a Core-Local Interrupt Controller (CLIC).
* routes to a core's local interrupt controller.
*
* The tricky part is that the PLIC spec imposes no restrictions on how
* these contexts are laid out. So for example, there is no guarantee
@ -316,10 +318,15 @@ plic_attach(device_t dev)
* entries that are not for supervisor external interrupts.
*
* 2. Walk up the device tree to find the corresponding CPU, and grab
* it's hart ID.
* its hart ID.
*
* 3. Convert the hart to a cpuid, and calculate the register offsets
* based on the context number.
*
* 4. Save the index for the boot hart's S-mode external interrupt in
* order to allocate and setup the corresponding resource, since the
* local interrupt controller newbus device is associated with that
* specific node.
*/
nintr = OF_getencprop_alloc_multi(node, "interrupts-extended",
sizeof(uint32_t), (void **)&cells);
@ -329,12 +336,16 @@ plic_attach(device_t dev)
}
/* interrupts-extended is a list of phandles and interrupt types. */
rid = -1;
for (i = 0, context = 0; i < nintr; i += 2, context++) {
/* Skip M-mode external interrupts */
if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR)
continue;
/* Get the hart ID from the CLIC's phandle. */
/*
* Get the hart ID from the core's interrupt controller
* phandle.
*/
hart = plic_get_hartid(dev, OF_node_from_xref(cells[i]));
if (hart < 0) {
OF_prop_free(cells);
@ -349,6 +360,9 @@ plic_attach(device_t dev)
return (ENXIO);
}
if (cpu == 0)
rid = i / 2;
/* Set the enable and context register offsets for the CPU. */
sc->contexts[cpu].enable_offset = PLIC_ENABLE_BASE +
context * PLIC_ENABLE_STRIDE;
@ -357,6 +371,20 @@ plic_attach(device_t dev)
}
OF_prop_free(cells);
if (rid == -1) {
device_printf(dev,
"Could not find local interrupt controller\n");
return (ENXIO);
}
sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_ACTIVE);
if (sc->irq_res == NULL) {
device_printf(dev,
"Error: could not allocate IRQ resources\n");
return (ENXIO);
}
/* Set the threshold for each CPU to accept all priorities. */
CPU_FOREACH(cpu)
WR4(sc, PLIC_THRESHOLD(sc, cpu), 0);
@ -366,9 +394,8 @@ plic_attach(device_t dev)
if (pic == NULL)
return (ENXIO);
csr_set(sie, SIE_SEIE);
return (intr_pic_claim_root(sc->dev, xref, plic_intr, sc));
return (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK | INTR_MPSAFE,
plic_intr, NULL, sc, &sc->ih));
}
static void

2
sys/riscv/riscv/riscv_console.c
View File

@ -243,8 +243,6 @@ rcons_attach(device_t dev)
sc = device_get_softc(dev);
sc->dev = dev;
csr_set(sie, SIE_SSIE);
bus_generic_attach(sc->dev);
return (0);

43
sys/riscv/riscv/sbi.c
View File

@ -31,6 +31,7 @@
#include <sys/bus.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/reboot.h>
@ -45,6 +46,10 @@ struct sbi_softc {
device_t dev;
};
struct sbi_devinfo {
struct resource_list rl;
};
static struct sbi_softc *sbi_softc = NULL;
static u_long sbi_spec_version;
@ -347,6 +352,10 @@ static int
sbi_attach(device_t dev)
{
struct sbi_softc *sc;
#ifdef SMP
device_t child;
struct sbi_devinfo *di;
#endif
if (sbi_softc != NULL)
return (ENXIO);
@ -358,15 +367,49 @@ sbi_attach(device_t dev)
EVENTHANDLER_REGISTER(shutdown_final, sbi_shutdown_final, NULL,
SHUTDOWN_PRI_LAST);
#ifdef SMP
di = malloc(sizeof(*di), M_DEVBUF, M_WAITOK | M_ZERO);
resource_list_init(&di->rl);
child = device_add_child(dev, "sbi_ipi", -1);
if (child == NULL) {
device_printf(dev, "Could not add sbi_ipi child\n");
return (ENXIO);
}
device_set_ivars(child, di);
#endif
return (0);
}
static struct resource_list *
sbi_get_resource_list(device_t bus, device_t child)
{
struct sbi_devinfo *di;
di = device_get_ivars(child);
KASSERT(di != NULL, ("%s: No devinfo", __func__));
return (&di->rl);
}
static device_method_t sbi_methods[] = {
/* Device interface */
DEVMETHOD(device_identify, sbi_identify),
DEVMETHOD(device_probe, sbi_probe),
DEVMETHOD(device_attach, sbi_attach),
/* Bus interface */
DEVMETHOD(bus_alloc_resource, bus_generic_rl_alloc_resource),
DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource),
DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
DEVMETHOD(bus_get_resource_list, sbi_get_resource_list),
DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
DEVMETHOD_END
};

208
sys/riscv/riscv/sbi_ipi.c Normal file
View File

@ -0,0 +1,208 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Jessica Clarke <jrtc27@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/cpuset.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/rman.h>
#include <machine/intr.h>
#include <machine/sbi.h>
#include <machine/smp.h>
#include <dev/ofw/openfirm.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#include "pic_if.h"
struct sbi_ipi_softc {
device_t dev;
struct resource *irq_res;
void *ih;
struct intr_irqsrc isrc;
uint32_t pending_ipis[MAXCPU];
};
static void
sbi_ipi_pic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus,
u_int ipi)
{
struct sbi_ipi_softc *sc;
struct pcpu *pc;
u_long mask;
u_int cpu;
sc = device_get_softc(dev);
KASSERT(isrc == &sc->isrc, ("%s: not the IPI isrc", __func__));
KASSERT(ipi < INTR_IPI_COUNT,
("%s: not a valid IPI: %u", __func__, ipi));
mask = 0;
STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
cpu = pc->pc_cpuid;
if (CPU_ISSET(cpu, &cpus)) {
atomic_set_32(&sc->pending_ipis[cpu], 1u << ipi);
mask |= (1ul << pc->pc_hart);
}
}
sbi_send_ipi(&mask);
}
static int
sbi_ipi_pic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp)
{
struct sbi_ipi_softc *sc;
sc = device_get_softc(dev);
KASSERT(ipi < INTR_IPI_COUNT,
("%s: not a valid IPI: %u", __func__, ipi));
*isrcp = &sc->isrc;
return (0);
}
static int
sbi_ipi_intr(void *arg)
{
struct sbi_ipi_softc *sc;
uint32_t ipi_bitmap;
u_int cpu, ipi;
int bit;
sc = arg;
csr_clear(sip, SIP_SSIP);
cpu = PCPU_GET(cpuid);
mb();
ipi_bitmap = atomic_readandclear_32(&sc->pending_ipis[cpu]);
if (ipi_bitmap == 0)
return (FILTER_HANDLED);
mb();
while ((bit = ffs(ipi_bitmap))) {
ipi = (bit - 1);
ipi_bitmap &= ~(1u << ipi);
intr_ipi_dispatch(ipi);
}
return (FILTER_HANDLED);
}
static int
sbi_ipi_probe(device_t dev)
{
device_set_desc(dev, "RISC-V SBI Inter-Processor Interrupts");
return (BUS_PROBE_NOWILDCARD);
}
static int
sbi_ipi_attach(device_t dev)
{
struct sbi_ipi_softc *sc;
const char *name;
int irq, rid, error;
phandle_t iparent;
pcell_t cell;
sc = device_get_softc(dev);
sc->dev = dev;
memset(sc->pending_ipis, 0, sizeof(sc->pending_ipis));
name = device_get_nameunit(dev);
error = intr_isrc_register(&sc->isrc, sc->dev, INTR_ISRCF_IPI,
"%s,ipi", name);
if (error != 0) {
device_printf(dev, "Can't register interrupt: %d\n", error);
return (ENXIO);
}
iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev));
cell = IRQ_SOFTWARE_SUPERVISOR;
irq = ofw_bus_map_intr(dev, iparent, 1, &cell);
error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1);
if (error != 0) {
device_printf(dev, "Unable to register IRQ resource\n");
return (ENXIO);
}
rid = 0;
sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_ACTIVE | RF_SHAREABLE);
if (sc->irq_res == NULL) {
device_printf(dev, "Unable to alloc IRQ resource\n");
return (ENXIO);
}
error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK,
sbi_ipi_intr, NULL, sc, &sc->ih);
if (error != 0) {
device_printf(dev, "Unable to setup IRQ resource\n");
return (ENXIO);
}
/* TODO: Define a set of priorities once other IPI sources exist */
error = intr_ipi_pic_register(dev, 0);
if (error != 0) {
device_printf(dev, "Can't register as IPI source: %d\n", error);
return (ENXIO);
}
return (0);
}
static device_method_t sbi_ipi_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, sbi_ipi_probe),
DEVMETHOD(device_attach, sbi_ipi_attach),
/* Interrupt controller interface */
DEVMETHOD(pic_ipi_send, sbi_ipi_pic_ipi_send),
DEVMETHOD(pic_ipi_setup, sbi_ipi_pic_ipi_setup),
DEVMETHOD_END
};
DEFINE_CLASS_0(sbi_ipi, sbi_ipi_driver, sbi_ipi_methods,
sizeof(struct sbi_ipi_softc));
/* Local interrupt controller attaches during BUS_PASS_ORDER_FIRST */
EARLY_DRIVER_MODULE(sbi_ipi, sbi, sbi_ipi_driver, 0, 0,
BUS_PASS_INTERRUPT + BUS_PASS_ORDER_EARLY);

33
sys/riscv/riscv/timer.c
View File

@ -43,6 +43,7 @@
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/rman.h>
#include <sys/timeet.h>
#include <sys/timetc.h>
#include <sys/vdso.h>
@ -53,9 +54,11 @@
#include <machine/md_var.h>
#include <machine/sbi.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/openfirm.h>
struct riscv_timer_softc {
struct resource *irq_res;
void *ih;
uint32_t clkfreq;
struct eventtimer et;
@ -116,7 +119,6 @@ riscv_timer_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
if (first != 0) {
counts = ((uint32_t)et->et_frequency * first) >> 32;
set_timecmp(get_timecount() + counts);
csr_set(sie, SIE_STIE);
return (0);
}
@ -188,7 +190,9 @@ static int
riscv_timer_attach(device_t dev)
{
struct riscv_timer_softc *sc;
int error;
int irq, rid, error;
phandle_t iparent;
pcell_t cell;
sc = device_get_softc(dev);
if (riscv_timer_sc != NULL)
@ -204,11 +208,28 @@ riscv_timer_attach(device_t dev)
riscv_timer_sc = sc;
iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev));
cell = IRQ_TIMER_SUPERVISOR;
irq = ofw_bus_map_intr(dev, iparent, 1, &cell);
error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1);
if (error != 0) {
device_printf(dev, "Unable to register IRQ resource\n");
return (ENXIO);
}
rid = 0;
sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_ACTIVE);
if (sc->irq_res == NULL) {
device_printf(dev, "Unable to alloc IRQ resource\n");
return (ENXIO);
}
/* Setup IRQs handler */
error = riscv_setup_intr(device_get_nameunit(dev), riscv_timer_intr,
NULL, sc, IRQ_TIMER_SUPERVISOR, INTR_TYPE_CLK, &sc->ih);
if (error) {
device_printf(dev, "Unable to alloc int resource.\n");
error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK,
riscv_timer_intr, NULL, sc, &sc->ih);
if (error != 0) {
device_printf(dev, "Unable to setup IRQ resource\n");
return (ENXIO);
}

6
sys/riscv/riscv/trap.c
View File

@ -74,6 +74,8 @@
#include <ddb/db_sym.h>
#endif
void intr_irq_handler(struct trapframe *tf);
int (*dtrace_invop_jump_addr)(struct trapframe *);
/* Called from exception.S */
@ -317,7 +319,7 @@ do_trap_supervisor(struct trapframe *frame)
exception = frame->tf_scause & SCAUSE_CODE;
if ((frame->tf_scause & SCAUSE_INTR) != 0) {
/* Interrupt */
riscv_cpu_intr(frame);
intr_irq_handler(frame);
return;
}
@ -398,7 +400,7 @@ do_trap_user(struct trapframe *frame)
exception = frame->tf_scause & SCAUSE_CODE;
if ((frame->tf_scause & SCAUSE_INTR) != 0) {
/* Interrupt */
riscv_cpu_intr(frame);
intr_irq_handler(frame);
return;
}
intr_enable();