1
0
mirror of https://git.FreeBSD.org/src.git synced 2024-11-27 08:00:11 +00:00
freebsd/sys/powerpc/booke/spe.c
2020-09-01 21:20:08 +00:00

689 lines
16 KiB
C

/*-
* Copyright (C) 1996 Wolfgang Solfrank.
* Copyright (C) 1996 TooLs GmbH.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by TooLs GmbH.
* 4. The name of TooLs GmbH may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $NetBSD: fpu.c,v 1.5 2001/07/22 11:29:46 wiz Exp $
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/limits.h>
#include <machine/altivec.h>
#include <machine/fpu.h>
#include <machine/ieeefp.h>
#include <machine/pcb.h>
#include <machine/psl.h>
#include <powerpc/fpu/fpu_arith.h>
#include <powerpc/fpu/fpu_emu.h>
#include <powerpc/fpu/fpu_extern.h>
void spe_handle_fpdata(struct trapframe *);
void spe_handle_fpround(struct trapframe *);
static int spe_emu_instr(uint32_t, struct fpemu *, struct fpn **, uint32_t *);
static void
save_vec_int(struct thread *td)
{
int msr;
struct pcb *pcb;
pcb = td->td_pcb;
/*
* Temporarily re-enable the vector unit during the save
*/
msr = mfmsr();
mtmsr(msr | PSL_VEC);
/*
* Save the vector registers and SPEFSCR to the PCB
*/
#define EVSTDW(n) __asm ("evstdw %1,0(%0)" \
:: "b"(pcb->pcb_vec.vr[n]), "n"(n));
EVSTDW(0); EVSTDW(1); EVSTDW(2); EVSTDW(3);
EVSTDW(4); EVSTDW(5); EVSTDW(6); EVSTDW(7);
EVSTDW(8); EVSTDW(9); EVSTDW(10); EVSTDW(11);
EVSTDW(12); EVSTDW(13); EVSTDW(14); EVSTDW(15);
EVSTDW(16); EVSTDW(17); EVSTDW(18); EVSTDW(19);
EVSTDW(20); EVSTDW(21); EVSTDW(22); EVSTDW(23);
EVSTDW(24); EVSTDW(25); EVSTDW(26); EVSTDW(27);
EVSTDW(28); EVSTDW(29); EVSTDW(30); EVSTDW(31);
#undef EVSTDW
__asm ( "evxor 0,0,0\n"
"evmwumiaa 0,0,0\n"
"evstdd 0,0(%0)" :: "b"(&pcb->pcb_vec.spare[0]));
pcb->pcb_vec.vscr = mfspr(SPR_SPEFSCR);
/*
* Disable vector unit again
*/
isync();
mtmsr(msr);
}
void
enable_vec(struct thread *td)
{
int msr;
struct pcb *pcb;
struct trapframe *tf;
pcb = td->td_pcb;
tf = trapframe(td);
/*
* Save the thread's SPE CPU number, and set the CPU's current
* vector thread
*/
td->td_pcb->pcb_veccpu = PCPU_GET(cpuid);
PCPU_SET(vecthread, td);
/*
* Enable the vector unit for when the thread returns from the
* exception. If this is the first time the unit has been used by
* the thread, initialise the vector registers and VSCR to 0, and
* set the flag to indicate that the vector unit is in use.
*/
tf->srr1 |= PSL_VEC;
if (!(pcb->pcb_flags & PCB_VEC)) {
memset(&pcb->pcb_vec, 0, sizeof pcb->pcb_vec);
pcb->pcb_flags |= PCB_VEC;
pcb->pcb_vec.vscr = mfspr(SPR_SPEFSCR);
}
/*
* Temporarily enable the vector unit so the registers
* can be restored.
*/
msr = mfmsr();
mtmsr(msr | PSL_VEC);
/* Restore SPEFSCR and ACC. Use %r0 as the scratch for ACC. */
mtspr(SPR_SPEFSCR, pcb->pcb_vec.vscr);
__asm __volatile("isync;evldd 0, 0(%0); evmra 0,0\n"
:: "b"(&pcb->pcb_vec.spare[0]));
/*
* The lower half of each register will be restored on trap return. Use
* %r0 as a scratch register, and restore it last.
*/
#define EVLDW(n) __asm __volatile("evldw 0, 0(%0); evmergehilo "#n",0,"#n \
:: "b"(&pcb->pcb_vec.vr[n]));
EVLDW(1); EVLDW(2); EVLDW(3); EVLDW(4);
EVLDW(5); EVLDW(6); EVLDW(7); EVLDW(8);
EVLDW(9); EVLDW(10); EVLDW(11); EVLDW(12);
EVLDW(13); EVLDW(14); EVLDW(15); EVLDW(16);
EVLDW(17); EVLDW(18); EVLDW(19); EVLDW(20);
EVLDW(21); EVLDW(22); EVLDW(23); EVLDW(24);
EVLDW(25); EVLDW(26); EVLDW(27); EVLDW(28);
EVLDW(29); EVLDW(30); EVLDW(31); EVLDW(0);
#undef EVLDW
isync();
mtmsr(msr);
}
void
save_vec(struct thread *td)
{
struct pcb *pcb;
pcb = td->td_pcb;
save_vec_int(td);
/*
* Clear the current vec thread and pcb's CPU id
* XXX should this be left clear to allow lazy save/restore ?
*/
pcb->pcb_veccpu = INT_MAX;
PCPU_SET(vecthread, NULL);
}
/*
* Save SPE state without dropping ownership. This will only save state if
* the current vector-thread is `td'. This is used for taking core dumps, so
* don't leak kernel information; overwrite the low words of each vector with
* their real value, taken from the thread's trap frame, unconditionally.
*/
void
save_vec_nodrop(struct thread *td)
{
struct pcb *pcb;
int i;
if (td == PCPU_GET(vecthread))
save_vec_int(td);
pcb = td->td_pcb;
for (i = 0; i < 32; i++) {
pcb->pcb_vec.vr[i][1] =
td->td_frame ? td->td_frame->fixreg[i] : 0;
}
}
#define SPE_INST_MASK 0x31f
#define EADD 0x200
#define ESUB 0x201
#define EABS 0x204
#define ENABS 0x205
#define ENEG 0x206
#define EMUL 0x208
#define EDIV 0x209
#define ECMPGT 0x20c
#define ECMPLT 0x20d
#define ECMPEQ 0x20e
#define ECFUI 0x210
#define ECFSI 0x211
#define ECTUI 0x214
#define ECTSI 0x215
#define ECTUF 0x216
#define ECTSF 0x217
#define ECTUIZ 0x218
#define ECTSIZ 0x21a
#define SPE 0x4
#define SPFP 0x6
#define DPFP 0x7
#define SPE_OPC 4
#define OPC_SHIFT 26
#define EVFSADD 0x280
#define EVFSSUB 0x281
#define EVFSABS 0x284
#define EVFSNABS 0x285
#define EVFSNEG 0x286
#define EVFSMUL 0x288
#define EVFSDIV 0x289
#define EVFSCMPGT 0x28c
#define EVFSCMPLT 0x28d
#define EVFSCMPEQ 0x28e
#define EVFSCFUI 0x290
#define EVFSCFSI 0x291
#define EVFSCTUI 0x294
#define EVFSCTSI 0x295
#define EVFSCTUF 0x296
#define EVFSCTSF 0x297
#define EVFSCTUIZ 0x298
#define EVFSCTSIZ 0x29a
#define EFSADD 0x2c0
#define EFSSUB 0x2c1
#define EFSABS 0x2c4
#define EFSNABS 0x2c5
#define EFSNEG 0x2c6
#define EFSMUL 0x2c8
#define EFSDIV 0x2c9
#define EFSCMPGT 0x2cc
#define EFSCMPLT 0x2cd
#define EFSCMPEQ 0x2ce
#define EFSCFD 0x2cf
#define EFSCFUI 0x2d0
#define EFSCFSI 0x2d1
#define EFSCTUI 0x2d4
#define EFSCTSI 0x2d5
#define EFSCTUF 0x2d6
#define EFSCTSF 0x2d7
#define EFSCTUIZ 0x2d8
#define EFSCTSIZ 0x2da
#define EFDADD 0x2e0
#define EFDSUB 0x2e1
#define EFDABS 0x2e4
#define EFDNABS 0x2e5
#define EFDNEG 0x2e6
#define EFDMUL 0x2e8
#define EFDDIV 0x2e9
#define EFDCMPGT 0x2ec
#define EFDCMPLT 0x2ed
#define EFDCMPEQ 0x2ee
#define EFDCFS 0x2ef
#define EFDCFUI 0x2f0
#define EFDCFSI 0x2f1
#define EFDCTUI 0x2f4
#define EFDCTSI 0x2f5
#define EFDCTUF 0x2f6
#define EFDCTSF 0x2f7
#define EFDCTUIZ 0x2f8
#define EFDCTSIZ 0x2fa
enum {
NONE,
SINGLE,
DOUBLE,
VECTOR,
};
static uint32_t fpscr_to_spefscr(uint32_t fpscr)
{
uint32_t spefscr;
spefscr = 0;
if (fpscr & FPSCR_VX)
spefscr |= SPEFSCR_FINV;
if (fpscr & FPSCR_OX)
spefscr |= SPEFSCR_FOVF;
if (fpscr & FPSCR_UX)
spefscr |= SPEFSCR_FUNF;
if (fpscr & FPSCR_ZX)
spefscr |= SPEFSCR_FDBZ;
if (fpscr & FPSCR_XX)
spefscr |= SPEFSCR_FX;
return (spefscr);
}
/* Sign is 0 for unsigned, 1 for signed. */
static int
spe_to_int(struct fpemu *fpemu, struct fpn *fpn, uint32_t *val, int sign)
{
uint32_t res[2];
res[0] = fpu_ftox(fpemu, fpn, res);
if (res[0] != UINT_MAX && res[0] != 0)
fpemu->fe_cx |= FPSCR_OX;
else if (sign == 0 && res[0] != 0)
fpemu->fe_cx |= FPSCR_UX;
else
*val = res[1];
return (0);
}
/* Masked instruction */
/*
* For compare instructions, returns 1 if success, 0 if not. For all others,
* returns -1, or -2 if no result needs recorded.
*/
static int
spe_emu_instr(uint32_t instr, struct fpemu *fpemu,
struct fpn **result, uint32_t *iresult)
{
switch (instr & SPE_INST_MASK) {
case EABS:
case ENABS:
case ENEG:
/* Taken care of elsewhere. */
break;
case ECTUIZ:
fpemu->fe_cx &= ~FPSCR_RN;
fpemu->fe_cx |= FP_RZ;
case ECTUI:
spe_to_int(fpemu, &fpemu->fe_f2, iresult, 0);
return (-1);
case ECTSIZ:
fpemu->fe_cx &= ~FPSCR_RN;
fpemu->fe_cx |= FP_RZ;
case ECTSI:
spe_to_int(fpemu, &fpemu->fe_f2, iresult, 1);
return (-1);
case EADD:
*result = fpu_add(fpemu);
break;
case ESUB:
*result = fpu_sub(fpemu);
break;
case EMUL:
*result = fpu_mul(fpemu);
break;
case EDIV:
*result = fpu_div(fpemu);
break;
case ECMPGT:
fpu_compare(fpemu, 0);
if (fpemu->fe_cx & FPSCR_FG)
return (1);
return (0);
case ECMPLT:
fpu_compare(fpemu, 0);
if (fpemu->fe_cx & FPSCR_FL)
return (1);
return (0);
case ECMPEQ:
fpu_compare(fpemu, 0);
if (fpemu->fe_cx & FPSCR_FE)
return (1);
return (0);
default:
printf("Unknown instruction %x\n", instr);
}
return (-1);
}
static int
spe_explode(struct fpemu *fe, struct fpn *fp, uint32_t type,
uint32_t hi, uint32_t lo)
{
uint32_t s;
fp->fp_sign = hi >> 31;
fp->fp_sticky = 0;
switch (type) {
case SINGLE:
s = fpu_stof(fp, hi);
break;
case DOUBLE:
s = fpu_dtof(fp, hi, lo);
break;
}
if (s == FPC_QNAN && (fp->fp_mant[0] & FP_QUIETBIT) == 0) {
/*
* Input is a signalling NaN. All operations that return
* an input NaN operand put it through a ``NaN conversion'',
* which basically just means ``turn on the quiet bit''.
* We do this here so that all NaNs internally look quiet
* (we can tell signalling ones by their class).
*/
fp->fp_mant[0] |= FP_QUIETBIT;
fe->fe_cx = FPSCR_VXSNAN; /* assert invalid operand */
s = FPC_SNAN;
}
fp->fp_class = s;
return (0);
}
/*
* Save the high word of a 64-bit GPR for manipulation in the exception handler.
*/
static uint32_t
spe_save_reg_high(int reg)
{
uint32_t vec[2];
#define EVSTDW(n) case n: __asm __volatile ("evstdw %1,0(%0)" \
:: "b"(vec), "n"(n) : "memory"); break;
switch (reg) {
EVSTDW(0); EVSTDW(1); EVSTDW(2); EVSTDW(3);
EVSTDW(4); EVSTDW(5); EVSTDW(6); EVSTDW(7);
EVSTDW(8); EVSTDW(9); EVSTDW(10); EVSTDW(11);
EVSTDW(12); EVSTDW(13); EVSTDW(14); EVSTDW(15);
EVSTDW(16); EVSTDW(17); EVSTDW(18); EVSTDW(19);
EVSTDW(20); EVSTDW(21); EVSTDW(22); EVSTDW(23);
EVSTDW(24); EVSTDW(25); EVSTDW(26); EVSTDW(27);
EVSTDW(28); EVSTDW(29); EVSTDW(30); EVSTDW(31);
}
#undef EVSTDW
return (vec[0]);
}
/*
* Load the given value into the high word of the requested register.
*/
static void
spe_load_reg_high(int reg, uint32_t val)
{
#define EVLDW(n) case n: __asm __volatile("evmergelo "#n",%0,"#n \
:: "r"(val)); break;
switch (reg) {
EVLDW(1); EVLDW(2); EVLDW(3); EVLDW(4);
EVLDW(5); EVLDW(6); EVLDW(7); EVLDW(8);
EVLDW(9); EVLDW(10); EVLDW(11); EVLDW(12);
EVLDW(13); EVLDW(14); EVLDW(15); EVLDW(16);
EVLDW(17); EVLDW(18); EVLDW(19); EVLDW(20);
EVLDW(21); EVLDW(22); EVLDW(23); EVLDW(24);
EVLDW(25); EVLDW(26); EVLDW(27); EVLDW(28);
EVLDW(29); EVLDW(30); EVLDW(31); EVLDW(0);
}
#undef EVLDW
}
void
spe_handle_fpdata(struct trapframe *frame)
{
struct fpemu fpemu;
struct fpn *result;
uint32_t instr, instr_sec_op;
uint32_t cr_shift, ra, rb, rd, src;
uint32_t high, low, res, tmp; /* For vector operations. */
uint32_t spefscr = 0;
uint32_t ftod_res[2];
int width; /* Single, Double, Vector, Integer */
int err;
uint32_t msr;
err = fueword32((void *)frame->srr0, &instr);
if (err != 0)
return;
/* Fault. */;
if ((instr >> OPC_SHIFT) != SPE_OPC)
return;
msr = mfmsr();
/*
* 'cr' field is the upper 3 bits of rd. Magically, since a) rd is 5
* bits, b) each 'cr' field is 4 bits, and c) Only the 'GT' bit is
* modified for most compare operations, the full value of rd can be
* used as a shift value.
*/
rd = (instr >> 21) & 0x1f;
ra = (instr >> 16) & 0x1f;
rb = (instr >> 11) & 0x1f;
src = (instr >> 5) & 0x7;
cr_shift = 28 - (rd & 0x1f);
instr_sec_op = (instr & 0x7ff);
memset(&fpemu, 0, sizeof(fpemu));
width = NONE;
switch (src) {
case SPE:
mtmsr(msr | PSL_VEC);
switch (instr_sec_op) {
case EVFSABS:
high = spe_save_reg_high(ra) & ~(1U << 31);
frame->fixreg[rd] = frame->fixreg[ra] & ~(1U << 31);
spe_load_reg_high(rd, high);
break;
case EVFSNABS:
high = spe_save_reg_high(ra) | (1U << 31);
frame->fixreg[rd] = frame->fixreg[ra] | (1U << 31);
spe_load_reg_high(rd, high);
break;
case EVFSNEG:
high = spe_save_reg_high(ra) ^ (1U << 31);
frame->fixreg[rd] = frame->fixreg[ra] ^ (1U << 31);
spe_load_reg_high(rd, high);
break;
default:
/* High word */
spe_explode(&fpemu, &fpemu.fe_f1, SINGLE,
spe_save_reg_high(ra), 0);
spe_explode(&fpemu, &fpemu.fe_f2, SINGLE,
spe_save_reg_high(rb), 0);
high = spe_emu_instr(instr_sec_op, &fpemu, &result,
&tmp);
if (high < 0)
spe_load_reg_high(rd, tmp);
spefscr = fpscr_to_spefscr(fpemu.fe_cx) << 16;
/* Clear the fpemu to start over on the lower bits. */
memset(&fpemu, 0, sizeof(fpemu));
/* Now low word */
spe_explode(&fpemu, &fpemu.fe_f1, SINGLE,
frame->fixreg[ra], 0);
spe_explode(&fpemu, &fpemu.fe_f2, SINGLE,
frame->fixreg[rb], 0);
spefscr |= fpscr_to_spefscr(fpemu.fe_cx);
low = spe_emu_instr(instr_sec_op, &fpemu, &result,
&frame->fixreg[rd]);
if (instr_sec_op == EVFSCMPEQ ||
instr_sec_op == EVFSCMPGT ||
instr_sec_op == EVFSCMPLT) {
res = (high << 3) | (low << 2) |
((high | low) << 1) | (high & low);
width = NONE;
} else
width = VECTOR;
break;
}
goto end;
case SPFP:
switch (instr_sec_op) {
case EFSABS:
frame->fixreg[rd] = frame->fixreg[ra] & ~(1U << 31);
break;
case EFSNABS:
frame->fixreg[rd] = frame->fixreg[ra] | (1U << 31);
break;
case EFSNEG:
frame->fixreg[rd] = frame->fixreg[ra] ^ (1U << 31);
break;
case EFSCFD:
mtmsr(msr | PSL_VEC);
spe_explode(&fpemu, &fpemu.fe_f3, DOUBLE,
spe_save_reg_high(rb), frame->fixreg[rb]);
result = &fpemu.fe_f3;
width = SINGLE;
break;
default:
spe_explode(&fpemu, &fpemu.fe_f1, SINGLE,
frame->fixreg[ra], 0);
spe_explode(&fpemu, &fpemu.fe_f2, SINGLE,
frame->fixreg[rb], 0);
width = SINGLE;
}
break;
case DPFP:
mtmsr(msr | PSL_VEC);
switch (instr_sec_op) {
case EFDABS:
high = spe_save_reg_high(ra) & ~(1U << 31);
frame->fixreg[rd] = frame->fixreg[ra];
spe_load_reg_high(rd, high);
break;
case EFDNABS:
high = spe_save_reg_high(ra) | (1U << 31);
frame->fixreg[rd] = frame->fixreg[ra];
spe_load_reg_high(rd, high);
break;
case EFDNEG:
high = spe_save_reg_high(ra) ^ (1U << 31);
frame->fixreg[rd] = frame->fixreg[ra];
spe_load_reg_high(rd, high);
break;
case EFDCFS:
spe_explode(&fpemu, &fpemu.fe_f3, SINGLE,
frame->fixreg[rb], 0);
result = &fpemu.fe_f3;
width = DOUBLE;
break;
default:
spe_explode(&fpemu, &fpemu.fe_f1, DOUBLE,
spe_save_reg_high(ra), frame->fixreg[ra]);
spe_explode(&fpemu, &fpemu.fe_f2, DOUBLE,
spe_save_reg_high(rb), frame->fixreg[rb]);
width = DOUBLE;
}
break;
}
switch (instr_sec_op) {
case EFDCFS:
case EFSCFD:
/* Already handled. */
break;
default:
res = spe_emu_instr(instr_sec_op, &fpemu, &result,
&frame->fixreg[rd]);
if (res != -1)
res <<= 2;
break;
}
switch (instr_sec_op & SPE_INST_MASK) {
case ECMPEQ:
case ECMPGT:
case ECMPLT:
frame->cr &= ~(0xf << cr_shift);
frame->cr |= (res << cr_shift);
break;
case ECTUI:
case ECTUIZ:
case ECTSI:
case ECTSIZ:
break;
default:
switch (width) {
case NONE:
case VECTOR:
break;
case SINGLE:
frame->fixreg[rd] = fpu_ftos(&fpemu, result);
break;
case DOUBLE:
spe_load_reg_high(rd, fpu_ftod(&fpemu, result, ftod_res));
frame->fixreg[rd] = ftod_res[1];
break;
default:
panic("Unknown storage width %d", width);
break;
}
}
end:
spefscr |= (mfspr(SPR_SPEFSCR) & ~SPEFSCR_FINVS);
mtspr(SPR_SPEFSCR, spefscr);
frame->srr0 += 4;
mtmsr(msr);
return;
}
void
spe_handle_fpround(struct trapframe *frame)
{
/*
* Punt fpround exceptions for now. This leaves the truncated result in
* the register. We'll deal with overflow/underflow later.
*/
return;
}