mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-11-28 08:02:54 +00:00
Code Issues Releases Activity

intrng: Extract arm/arm64 IPI->PIC glue code

Browse Source 
The arm and arm64 implementations of dispatching IPIs via PIC_IPI_SEND
are almost identical, and entirely MI with the lone exception of a
single store barrier on arm64 (that is likely either redundant or needed
on arm too). Thus, de-duplicate this code by moving it to INTRNG as a
generic IPI glue framework. The ipi_* functions remain declared in MD
smp.h headers and implemented in MD code, but are trivial wrappers
around intr_ipi_send that could be made MI, at least for INTRNG ports,
at a later date.

Note that, whilst both arm and arm64 had an ii_send member in intr_ipi
to abstract over how to send interrupts,, they were always ultimately
using PIC_IPI_SEND, and so this complexity has been removed. A follow-up
commit will re-introduce the same flexibility by instead allowing a
device other than the root PIC to be registered as the IPI sender.

As part of this, strengthen a MAXCPU assertion that was missed in commit
2f0b059eea ("intrng: switch from MAXCPU to mp_ncpus") (which itself is
mis-titled).

Reviewed by:	mmel, mhorne
MFC after:	1 month
Differential Revision:	https://reviews.freebsd.org/D35898
This commit is contained in:
Jessica Clarke 2024-01-24 23:49:53 +00:00
parent e06afdb285
commit fae8755f16
7 changed files with 178 additions and 390 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
sys/arm/arm/machdep_intr.c
View File

@ -29,39 +29,10 @@
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/syslog.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/proc.h>
#include <sys/bus.h>
#include <sys/interrupt.h>
#include <sys/conf.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/smp.h>
#include <machine/atomic.h>
#include <machine/bus.h>
#include <machine/intr.h>
#include <machine/cpu.h>
#include <machine/smp.h>
#include "pic_if.h"
#ifdef SMP
#define INTR_IPI_NAMELEN (MAXCOMLEN + 1)
struct intr_ipi {
intr_ipi_handler_t * ii_handler;
void * ii_handler_arg;
intr_ipi_send_t * ii_send;
void * ii_send_arg;
char ii_name[INTR_IPI_NAMELEN];
u_long * ii_count;
};
static struct intr_ipi ipi_sources[INTR_IPI_COUNT];
#endif
#include <machine/cpufunc.h>
#include <machine/intr.h>
/*
* arm_irq_memory_barrier()
@ -125,96 +96,3 @@ arm_irq_memory_barrier(uintptr_t irq)
dsb();
cpu_l2cache_drain_writebuf();
}
#ifdef SMP
static inline struct intr_ipi *
intr_ipi_lookup(u_int ipi)
{
if (ipi >= INTR_IPI_COUNT)
panic("%s: no such IPI %u", __func__, ipi);
return (&ipi_sources[ipi]);
}
void
intr_ipi_dispatch(u_int ipi)
{
struct intr_ipi *ii;
ii = intr_ipi_lookup(ipi);
if (ii->ii_count == NULL)
panic("%s: not setup IPI %u", __func__, ipi);
intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid));
ii->ii_handler(ii->ii_handler_arg);
}
void
intr_ipi_send(cpuset_t cpus, u_int ipi)
{
struct intr_ipi *ii;
ii = intr_ipi_lookup(ipi);
if (ii->ii_count == NULL)
panic("%s: not setup IPI %u", __func__, ipi);
ii->ii_send(ii->ii_send_arg, cpus, ipi);
}
void
intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
void *h_arg, intr_ipi_send_t *send, void *s_arg)
{
struct intr_ipi *ii;
ii = intr_ipi_lookup(ipi);
KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi));
KASSERT(send != NULL, ("%s: ipi %u no sender", __func__, ipi));
KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi));
ii->ii_handler = hand;
ii->ii_handler_arg = h_arg;
ii->ii_send = send;
ii->ii_send_arg = s_arg;
strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN);
ii->ii_count = intr_ipi_setup_counters(name);
}
/*
* Send IPI thru interrupt controller.
*/
static void
pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi)
{
KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi);
}
/*
* Setup IPI handler on interrupt controller.
*
* Not SMP coherent.
*/
int
intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
void *arg)
{
int error;
struct intr_irqsrc *isrc;
KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc);
if (error != 0)
return (error);
isrc->isrc_handlers++;
intr_ipi_setup(ipi, name, hand, arg, pic_ipi_send, isrc);
PIC_ENABLE_INTR(intr_irq_root_dev, isrc);
return (0);
}
#endif

10
sys/arm/arm/mp_machdep.c
View File

@ -300,11 +300,11 @@ release_aps(void *dummy __unused)
if (mp_ncpus == 1)
return;
intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
atomic_store_rel_int(&aps_ready, 1);
/* Wake the other threads up */

13
sys/arm/include/intr.h
View File

@ -49,19 +49,6 @@
#include <sys/intr.h>
#ifdef SMP
typedef void intr_ipi_send_t(void *, cpuset_t, u_int);
typedef void intr_ipi_handler_t(void *);
void intr_ipi_dispatch(u_int);
void intr_ipi_send(cpuset_t, u_int);
void intr_ipi_setup(u_int, const char *, intr_ipi_handler_t *, void *,
intr_ipi_send_t *, void *);
int intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *, void *);
#endif
void arm_irq_memory_barrier(uintptr_t);
#endif /* _MACHINE_INTR_H */

204
sys/arm64/arm64/mp_machdep.c
View File

@ -77,8 +77,6 @@
#include <dev/psci/psci.h>
#include "pic_if.h"
#define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE)
#define MP_QUIRK_CPULIST 0x01 /* The list of cpus may be wrong, */
@ -98,25 +96,6 @@ static struct {
};
#endif
typedef void intr_ipi_send_t(void *, cpuset_t, u_int);
typedef void intr_ipi_handler_t(void *);
#define INTR_IPI_NAMELEN (MAXCOMLEN + 1)
struct intr_ipi {
intr_ipi_handler_t * ii_handler;
void * ii_handler_arg;
intr_ipi_send_t * ii_send;
void * ii_send_arg;
char ii_name[INTR_IPI_NAMELEN];
u_long * ii_count;
};
static struct intr_ipi ipi_sources[INTR_IPI_COUNT];
static struct intr_ipi *intr_ipi_lookup(u_int);
static void intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *,
void *);
static void ipi_ast(void *);
static void ipi_hardclock(void *);
static void ipi_preempt(void *);
@ -165,12 +144,12 @@ release_aps(void *dummy __unused)
if (mp_ncpus == 1)
return;
intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
intr_pic_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL);
intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL);
intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
atomic_store_rel_int(&aps_ready, 1);
/* Wake up the other CPUs */
@ -315,71 +294,6 @@ smp_after_idle_runnable(void *arg __unused)
SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
smp_after_idle_runnable, NULL);
/*
* Send IPI thru interrupt controller.
*/
static void
pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi)
{
KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
/*
* Ensure that this CPU's stores will be visible to IPI
* recipients before starting to send the interrupts.
*/
dsb(ishst);
PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi);
}
/*
* Setup IPI handler on interrupt controller.
*
* Not SMP coherent.
*/
static void
intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
void *arg)
{
struct intr_irqsrc *isrc;
struct intr_ipi *ii;
int error;
KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi));
error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc);
if (error != 0)
return;
isrc->isrc_handlers++;
ii = intr_ipi_lookup(ipi);
KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi));
ii->ii_handler = hand;
ii->ii_handler_arg = arg;
ii->ii_send = pic_ipi_send;
ii->ii_send_arg = isrc;
strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN);
ii->ii_count = intr_ipi_setup_counters(name);
PIC_ENABLE_INTR(intr_irq_root_dev, isrc);
}
static void
intr_ipi_send(cpuset_t cpus, u_int ipi)
{
struct intr_ipi *ii;
ii = intr_ipi_lookup(ipi);
if (ii->ii_count == NULL)
panic("%s: not setup IPI %u", __func__, ipi);
ii->ii_send(ii->ii_send_arg, cpus, ipi);
}
static void
ipi_ast(void *dummy __unused)
{
@ -888,112 +802,6 @@ cpu_mp_setmaxid(void)
}
}
/*
* Lookup IPI source.
*/
static struct intr_ipi *
intr_ipi_lookup(u_int ipi)
{
if (ipi >= INTR_IPI_COUNT)
panic("%s: no such IPI %u", __func__, ipi);
return (&ipi_sources[ipi]);
}
/*
* interrupt controller dispatch function for IPIs. It should
* be called straight from the interrupt controller, when associated
* interrupt source is learned. Or from anybody who has an interrupt
* source mapped.
*/
void
intr_ipi_dispatch(u_int ipi)
{
struct intr_ipi *ii;
ii = intr_ipi_lookup(ipi);
if (ii->ii_count == NULL)
panic("%s: not setup IPI %u", __func__, ipi);
intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid));
ii->ii_handler(ii->ii_handler_arg);
}
#ifdef notyet
/*
* Map IPI into interrupt controller.
*
* Not SMP coherent.
*/
static int
ipi_map(struct intr_irqsrc *isrc, u_int ipi)
{
boolean_t is_percpu;
int error;
if (ipi >= INTR_IPI_COUNT)
panic("%s: no such IPI %u", __func__, ipi);
KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
isrc->isrc_type = INTR_ISRCT_NAMESPACE;
isrc->isrc_nspc_type = INTR_IRQ_NSPC_IPI;
isrc->isrc_nspc_num = ipi_next_num;
error = PIC_REGISTER(intr_irq_root_dev, isrc, &is_percpu);
if (error == 0) {
isrc->isrc_dev = intr_irq_root_dev;
ipi_next_num++;
}
return (error);
}
/*
* Setup IPI handler to interrupt source.
*
* Note that there could be more ways how to send and receive IPIs
* on a platform like fast interrupts for example. In that case,
* one can call this function with ASIF_NOALLOC flag set and then
* call intr_ipi_dispatch() when appropriate.
*
* Not SMP coherent.
*/
int
intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter,
void *arg, u_int flags)
{
struct intr_irqsrc *isrc;
int error;
if (filter == NULL)
return(EINVAL);
isrc = intr_ipi_lookup(ipi);
if (isrc->isrc_ipifilter != NULL)
return (EEXIST);
if ((flags & AISHF_NOALLOC) == 0) {
error = ipi_map(isrc, ipi);
if (error != 0)
return (error);
}
isrc->isrc_ipifilter = filter;
isrc->isrc_arg = arg;
isrc->isrc_handlers = 1;
isrc->isrc_count = intr_ipi_setup_counters(name);
isrc->isrc_index = 0; /* it should not be used in IPI case */
if (isrc->isrc_dev != NULL) {
PIC_ENABLE_INTR(isrc->isrc_dev, isrc);
PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc);
}
return (0);
}
#endif
/* Sending IPI */
void
ipi_all_but_self(u_int ipi)

4
sys/arm64/include/intr.h
View File

@ -42,10 +42,6 @@ arm_irq_memory_barrier(uintptr_t irq)
{
}
#ifdef SMP
void intr_ipi_dispatch(u_int);
#endif
#ifdef DEV_ACPI
#define ACPI_INTR_XREF 1
#define ACPI_MSI_XREF 2

188
sys/kern/subr_intr.c
View File

@ -2,6 +2,11 @@
* Copyright (c) 2015-2016 Svatopluk Kraus
* Copyright (c) 2015-2016 Michal Meloun
* All rights reserved.
* Copyright (c) 2015-2016 The FreeBSD Foundation
* Copyright (c) 2021 Jessica Clarke <jrtc27@FreeBSD.org>
*
* Portions of this software were developed by Andrew Turner under
* sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -124,6 +129,18 @@ struct intr_pic {
SLIST_HEAD(, intr_pic_child) pic_children;
};
#ifdef SMP
#define INTR_IPI_NAMELEN (MAXCOMLEN + 1)
struct intr_ipi {
intr_ipi_handler_t *ii_handler;
void *ii_handler_arg;
struct intr_irqsrc *ii_isrc;
char ii_name[INTR_IPI_NAMELEN];
u_long *ii_count;
};
#endif
static struct mtx pic_list_lock;
static SLIST_HEAD(, intr_pic) pic_list;
@ -140,6 +157,8 @@ static bool irq_assign_cpu = true;
#else
static bool irq_assign_cpu = false;
#endif
static struct intr_ipi ipi_sources[INTR_IPI_COUNT];
#endif
u_int intr_nirq = NIRQ;
@ -298,39 +317,6 @@ isrc_release_counters(struct intr_irqsrc *isrc)
bit_nclear(intrcnt_bitmap, idx, idx + 1);
}
#ifdef SMP
/*
* Virtualization for interrupt source IPI counters setup.
*/
u_long *
intr_ipi_setup_counters(const char *name)
{
u_int index, i;
char str[INTRNAME_LEN];
mtx_lock(&isrc_table_lock);
/*
* We should never have a problem finding mp_maxid + 1 contiguous
* counters, in practice. Interrupts will be allocated sequentially
* during boot, so the array should fill from low to high index. Once
* reserved, the IPI counters will never be released. Similarly, we
* will not need to allocate more IPIs once the system is running.
*/
bit_ffc_area(intrcnt_bitmap, nintrcnt, mp_maxid + 1, &index);
if (index == -1)
panic("Failed to allocate %d counters. Array exhausted?",
mp_maxid + 1);
bit_nset(intrcnt_bitmap, index, index + mp_maxid);
for (i = 0; i < mp_maxid + 1; i++) {
snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name);
intrcnt_setname(str, index + i);
}
mtx_unlock(&isrc_table_lock);
return (&intrcnt[index]);
}
#endif
/*
* Main interrupt dispatch handler. It's called straight
* from the assembler, where CPU interrupt is served.
@ -1774,3 +1760,139 @@ intr_map_init(void *dummy __unused)
M_INTRNG, M_WAITOK | M_ZERO);
}
SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL);
#ifdef SMP
/* Virtualization for interrupt source IPI counter increment. */
static inline void
intr_ipi_increment_count(u_long *counter, u_int cpu)
{
KASSERT(cpu < mp_maxid + 1, ("%s: too big cpu %u", __func__, cpu));
counter[cpu]++;
}
/*
* Virtualization for interrupt source IPI counters setup.
*/
static u_long *
intr_ipi_setup_counters(const char *name)
{
u_int index, i;
char str[INTRNAME_LEN];
mtx_lock(&isrc_table_lock);
/*
* We should never have a problem finding mp_maxid + 1 contiguous
* counters, in practice. Interrupts will be allocated sequentially
* during boot, so the array should fill from low to high index. Once
* reserved, the IPI counters will never be released. Similarly, we
* will not need to allocate more IPIs once the system is running.
*/
bit_ffc_area(intrcnt_bitmap, nintrcnt, mp_maxid + 1, &index);
if (index == -1)
panic("Failed to allocate %d counters. Array exhausted?",
mp_maxid + 1);
bit_nset(intrcnt_bitmap, index, index + mp_maxid);
for (i = 0; i < mp_maxid + 1; i++) {
snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name);
intrcnt_setname(str, index + i);
}
mtx_unlock(&isrc_table_lock);
return (&intrcnt[index]);
}
/*
* Lookup IPI source.
*/
static struct intr_ipi *
intr_ipi_lookup(u_int ipi)
{
if (ipi >= INTR_IPI_COUNT)
panic("%s: no such IPI %u", __func__, ipi);
return (&ipi_sources[ipi]);
}
/*
* Setup IPI handler on interrupt controller.
*
* Not SMP coherent.
*/
void
intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
void *arg)
{
struct intr_irqsrc *isrc;
struct intr_ipi *ii;
int error;
KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi));
error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc);
if (error != 0)
return;
isrc->isrc_handlers++;
ii = intr_ipi_lookup(ipi);
KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi));
ii->ii_handler = hand;
ii->ii_handler_arg = arg;
ii->ii_isrc = isrc;
strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN);
ii->ii_count = intr_ipi_setup_counters(name);
PIC_ENABLE_INTR(intr_irq_root_dev, isrc);
}
void
intr_ipi_send(cpuset_t cpus, u_int ipi)
{
struct intr_ipi *ii;
KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
ii = intr_ipi_lookup(ipi);
if (ii->ii_count == NULL)
panic("%s: not setup IPI %u", __func__, ipi);
/*
* XXX: Surely needed on other architectures too? Either way should be
* some kind of MI hook defined in an MD header, or the responsibility
* of the MD caller if not widespread.
*/
#ifdef __aarch64__
/*
* Ensure that this CPU's stores will be visible to IPI
* recipients before starting to send the interrupts.
*/
dsb(ishst);
#endif
PIC_IPI_SEND(intr_irq_root_dev, ii->ii_isrc, cpus, ipi);
}
/*
* interrupt controller dispatch function for IPIs. It should
* be called straight from the interrupt controller, when associated
* interrupt source is learned. Or from anybody who has an interrupt
* source mapped.
*/
void
intr_ipi_dispatch(u_int ipi)
{
struct intr_ipi *ii;
ii = intr_ipi_lookup(ipi);
if (ii->ii_count == NULL)
panic("%s: not setup IPI %u", __func__, ipi);
intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid));
ii->ii_handler(ii->ii_handler_arg);
}
#endif

23
sys/sys/intr.h
View File

@ -148,21 +148,18 @@ int intr_release_msix(device_t, device_t, intptr_t, int);
int intr_bind_irq(device_t, struct resource *, int);
void intr_pic_init_secondary(void);
/* Virtualization for interrupt source IPI counter increment. */
static inline void
intr_ipi_increment_count(u_long *counter, u_int cpu)
{
KASSERT(cpu < MAXCPU, ("%s: too big cpu %u", __func__, cpu));
counter[cpu]++;
}
/* Virtualization for interrupt source IPI counters setup. */
u_long * intr_ipi_setup_counters(const char *name);
#endif
extern u_int intr_nirq; /* number of IRQs on intrng platforms */
/* Intr interface for IPIs. */
#ifdef SMP
typedef void intr_ipi_handler_t(void *);
void intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
void *arg);
void intr_ipi_send(cpuset_t cpus, u_int ipi);
void intr_ipi_dispatch(u_int ipi);
#endif
#endif /* _SYS_INTR_H */