mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-12-16 10:20:30 +00:00
Code Issues Releases Activity

- update firmware to 5.0

Browse Source 
- add support for T3C
- add DDP support (zero-copy receive)
- fix TOE transmit of large requests
- fix shutdown so that sockets don't remain in CLOSING state indefinitely
- register listeners when an interface is brought up after tom is loaded
- fix setting of multicast filter
- enable link at device attach
- exit tick handler if shutdown is in progress
- add helper for logging TCB
- add sysctls for dumping transmit queues

- note that TOE wxill not be MFC'd until after 7.0 has been finalized

MFC after: 3 days
This commit is contained in:
Kip Macy 2008-02-23 01:06:17 +00:00
parent 60e15db992
commit 8e10660f12
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=176472
46 changed files with 5929 additions and 4083 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
sys/dev/cxgb/common/cxgb_ael1002.c
View File

@ -36,6 +36,9 @@ __FBSDID("$FreeBSD$");
#include <dev/cxgb/cxgb_include.h>
#endif
#undef msleep
#define msleep t3_os_sleep
enum {
AEL100X_TX_DISABLE = 9,
AEL100X_TX_CONFIG1 = 0xc002,
@ -52,9 +55,9 @@ static void ael100x_txon(struct cphy *phy)
{
int tx_on_gpio = phy->addr == 0 ? F_GPIO7_OUT_VAL : F_GPIO2_OUT_VAL;
t3_os_sleep(100);
msleep(100);
t3_set_reg_field(phy->adapter, A_T3DBG_GPIO_EN, 0, tx_on_gpio);
t3_os_sleep(30);
msleep(30);
}
static int ael1002_power_down(struct cphy *phy, int enable)
@ -115,7 +118,6 @@ static int ael100x_get_link_status(struct cphy *phy, int *link_ok,
#ifdef C99_NOT_SUPPORTED
static struct cphy_ops ael1002_ops = {
NULL,
ael1002_reset,
ael1002_intr_noop,
ael1002_intr_noop,
@ -141,11 +143,14 @@ static struct cphy_ops ael1002_ops = {
};
#endif
void t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
int t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
{
cphy_init(phy, adapter, phy_addr, &ael1002_ops, mdio_ops);
cphy_init(phy, adapter, phy_addr, &ael1002_ops, mdio_ops,
SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE,
"10GBASE-XR");
ael100x_txon(phy);
return 0;
}
static int ael1006_reset(struct cphy *phy, int wait)
@ -188,7 +193,6 @@ static int ael1006_power_down(struct cphy *phy, int enable)
#ifdef C99_NOT_SUPPORTED
static struct cphy_ops ael1006_ops = {
NULL,
ael1006_reset,
ael1006_intr_enable,
ael1006_intr_disable,
@ -214,16 +218,18 @@ static struct cphy_ops ael1006_ops = {
};
#endif
void t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
int t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
{
cphy_init(phy, adapter, phy_addr, &ael1006_ops, mdio_ops);
cphy_init(phy, adapter, phy_addr, &ael1006_ops, mdio_ops,
SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE,
"10GBASE-SR");
ael100x_txon(phy);
return 0;
}
#ifdef C99_NOT_SUPPORTED
static struct cphy_ops qt2045_ops = {
NULL,
ael1006_reset,
ael1006_intr_enable,
ael1006_intr_disable,
@ -249,12 +255,14 @@ static struct cphy_ops qt2045_ops = {
};
#endif
void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
int t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
{
unsigned int stat;
cphy_init(phy, adapter, phy_addr, &qt2045_ops, mdio_ops);
cphy_init(phy, adapter, phy_addr, &qt2045_ops, mdio_ops,
SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_TP,
"10GBASE-CX4");
/*
* Some cards where the PHY is supposed to be at address 0 actually
@ -263,6 +271,7 @@ void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
if (!phy_addr && !mdio_read(phy, MDIO_DEV_PMA_PMD, MII_BMSR, &stat) &&
stat == 0xffff)
phy->addr = 1;
return 0;
}
static int xaui_direct_reset(struct cphy *phy, int wait)
@ -300,7 +309,6 @@ static int xaui_direct_power_down(struct cphy *phy, int enable)
#ifdef C99_NOT_SUPPORTED
static struct cphy_ops xaui_direct_ops = {
NULL,
xaui_direct_reset,
ael1002_intr_noop,
ael1002_intr_noop,
@ -326,8 +334,11 @@ static struct cphy_ops xaui_direct_ops = {
};
#endif
void t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
int t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
{
cphy_init(phy, adapter, phy_addr, &xaui_direct_ops, mdio_ops);
cphy_init(phy, adapter, phy_addr, &xaui_direct_ops, mdio_ops,
SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_TP,
"10GBASE-CX4");
return 0;
}

40
sys/dev/cxgb/common/cxgb_common.h
View File

@ -98,8 +98,8 @@ enum {
(((x) >> S_TP_VERSION_MICRO) & M_TP_VERSION_MICRO)
enum {
FW_VERSION_MAJOR = 4,
FW_VERSION_MINOR = 7,
FW_VERSION_MAJOR = 5,
FW_VERSION_MINOR = 0,
FW_VERSION_MICRO = 0
};
@ -157,10 +157,10 @@ struct adapter_info {
};
struct port_type_info {
void (*phy_prep)(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *ops);
unsigned int caps;
const char *desc;
int (*phy_prep)(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *ops);
};
struct mc5_stats {
@ -508,7 +508,6 @@ enum {
/* PHY operations */
struct cphy_ops {
void (*destroy)(struct cphy *phy);
int (*reset)(struct cphy *phy, int wait);
int (*intr_enable)(struct cphy *phy);
@ -530,7 +529,9 @@ struct cphy_ops {
/* A PHY instance */
struct cphy {
int addr; /* PHY address */
unsigned int caps; /* PHY capabilities */
adapter_t *adapter; /* associated adapter */
const char *desc; /* PHY description */
unsigned long fifo_errors; /* FIFO over/under-flows */
const struct cphy_ops *ops; /* PHY operations */
int (*mdio_read)(adapter_t *adapter, int phy_addr, int mmd_addr,
@ -555,10 +556,13 @@ static inline int mdio_write(struct cphy *phy, int mmd, int reg,
/* Convenience initializer */
static inline void cphy_init(struct cphy *phy, adapter_t *adapter,
int phy_addr, struct cphy_ops *phy_ops,
const struct mdio_ops *mdio_ops)
const struct mdio_ops *mdio_ops, unsigned int caps,
const char *desc)
{
phy->adapter = adapter;
phy->addr = phy_addr;
phy->caps = caps;
phy->desc = desc;
phy->ops = phy_ops;
if (mdio_ops) {
phy->mdio_read = mdio_ops->read;
@ -667,11 +671,12 @@ int t3_seeprom_wp(adapter_t *adapter, int enable);
int t3_read_flash(adapter_t *adapter, unsigned int addr, unsigned int nwords,
u32 *data, int byte_oriented);
int t3_get_tp_version(adapter_t *adapter, u32 *vers);
int t3_check_tpsram_version(adapter_t *adapter);
int t3_check_tpsram_version(adapter_t *adapter, int *must_load);
int t3_check_tpsram(adapter_t *adapter, const u8 *tp_ram, unsigned int size);
int t3_load_fw(adapter_t *adapter, const const u8 *fw_data, unsigned int size);
int t3_load_boot(adapter_t *adapter, u8 *boot_data, unsigned int size);
int t3_get_fw_version(adapter_t *adapter, u32 *vers);
int t3_check_fw_version(adapter_t *adapter);
int t3_check_fw_version(adapter_t *adapter, int *must_load);
int t3_init_hw(adapter_t *adapter, u32 fw_params);
void mac_prep(struct cmac *mac, adapter_t *adapter, int index);
void early_hw_init(adapter_t *adapter, const struct adapter_info *ai);
@ -769,18 +774,21 @@ int t3_vsc7323_set_mtu(adapter_t *adap, unsigned int mtu, int port);
int t3_vsc7323_set_addr(adapter_t *adap, u8 addr[6], int port);
int t3_vsc7323_enable(adapter_t *adap, int port, int which);
int t3_vsc7323_disable(adapter_t *adap, int port, int which);
int t3_phy_advertise_fiber(struct cphy *phy, unsigned int advert);
const struct mac_stats *t3_vsc7323_update_stats(struct cmac *mac);
void t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
int t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops);
void t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
int t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops);
void t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
int t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops);
void t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
int t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops);
void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
int t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops);
void t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
int t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops);
#endif /* __CHELSIO_COMMON_H */

4
sys/dev/cxgb/common/cxgb_ctl_defs.h
View File

@ -125,8 +125,8 @@ struct rdma_info {
unsigned int rqt_top; /* RQT last entry address */
unsigned int udbell_len; /* user doorbell region length */
unsigned long udbell_physbase; /* user doorbell physical start addr */
void volatile *kdb_addr; /* kernel doorbell register address */
struct pci_dev *pdev; /* associated PCI device */
void *kdb_addr; /* kernel doorbell register address */
struct device *pdev; /* associated PCI device */
};
/*

2
sys/dev/cxgb/common/cxgb_firmware_exports.h
View File

@ -74,6 +74,8 @@ POSSIBILITY OF SUCH DAMAGE.
#define FW_WROPCODE_MNGT 0x1D
#define FW_MNGTOPCODE_PKTSCHED_SET 0x00
#define FW_MNGTOPCODE_WRC_SET 0x01
#define FW_MNGTOPCODE_TUNNEL_CR_FLUSH 0x02
/* Maximum size of a WR sent from the host, limited by the SGE.
*

19
sys/dev/cxgb/common/cxgb_mc5.c
View File

@ -384,7 +384,7 @@ int t3_mc5_init(struct mc5 *mc5, unsigned int nservers, unsigned int nfilters,
return err;
}
/*
/**
* read_mc5_range - dump a part of the memory managed by MC5
* @mc5: the MC5 handle
* @start: the start address for the dump
@ -425,8 +425,11 @@ int t3_read_mc5_range(const struct mc5 *mc5, unsigned int start,
#define MC5_INT_FATAL (F_PARITYERR | F_REQQPARERR | F_DISPQPARERR)
/*
* MC5 interrupt handler
/**
* t3_mc5_intr_handler - MC5 interrupt handler
* @mc5: the MC5 handle
*
* The MC5 interrupt handler.
*/
void t3_mc5_intr_handler(struct mc5 *mc5)
{
@ -462,6 +465,16 @@ void t3_mc5_intr_handler(struct mc5 *mc5)
t3_write_reg(adap, A_MC5_DB_INT_CAUSE, cause);
}
/**
* t3_mc5_prep - initialize the SW state for MC5
* @adapter: the adapter
* @mc5: the MC5 handle
* @mode: whether the TCAM will be in 72- or 144-bit mode
*
* Initialize the SW state associated with MC5. Among other things
* this determines the size of the attached TCAM.
*/
void __devinit t3_mc5_prep(adapter_t *adapter, struct mc5 *mc5, int mode)
{
#define K * 1024

35
sys/dev/cxgb/common/cxgb_mv88e1xxx.c
View File

@ -221,6 +221,16 @@ static int mv88e1xxx_get_link_status(struct cphy *cphy, int *link_ok,
return 0;
}
static int mv88e1xxx_set_speed_duplex(struct cphy *phy, int speed, int duplex)
{
int err = t3_set_phy_speed_duplex(phy, speed, duplex);
/* PHY needs reset for new settings to take effect */
if (!err)
err = mv88e1xxx_reset(phy, 0);
return err;
}
static int mv88e1xxx_downshift_set(struct cphy *cphy, int downshift_enable)
{
/*
@ -258,7 +268,6 @@ static int mv88e1xxx_intr_handler(struct cphy *cphy)
#ifdef C99_NOT_SUPPORTED
static struct cphy_ops mv88e1xxx_ops = {
NULL,
mv88e1xxx_reset,
mv88e1xxx_intr_enable,
mv88e1xxx_intr_disable,
@ -268,7 +277,7 @@ static struct cphy_ops mv88e1xxx_ops = {
mv88e1xxx_autoneg_restart,
t3_phy_advertise,
mv88e1xxx_set_loopback,
t3_set_phy_speed_duplex,
mv88e1xxx_set_speed_duplex,
mv88e1xxx_get_link_status,
mv88e1xxx_power_down,
};
@ -283,20 +292,28 @@ static struct cphy_ops mv88e1xxx_ops = {
.autoneg_restart = mv88e1xxx_autoneg_restart,
.advertise = t3_phy_advertise,
.set_loopback = mv88e1xxx_set_loopback,
.set_speed_duplex = t3_set_phy_speed_duplex,
.set_speed_duplex = mv88e1xxx_set_speed_duplex,
.get_link_status = mv88e1xxx_get_link_status,
.power_down = mv88e1xxx_power_down,
};
#endif
void t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
int t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
{
cphy_init(phy, adapter, phy_addr, &mv88e1xxx_ops, mdio_ops);
int err;
cphy_init(phy, adapter, phy_addr, &mv88e1xxx_ops, mdio_ops,
SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full |
SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII |
SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T");
/* Configure copper PHY transmitter as class A to reduce EMI. */
mdio_write(phy, 0, MV88E1XXX_EXTENDED_ADDR, 0xb);
mdio_write(phy, 0, MV88E1XXX_EXTENDED_DATA, 0x8004);
mv88e1xxx_downshift_set(phy, 1); /* Enable downshift */
err = mdio_write(phy, 0, MV88E1XXX_EXTENDED_ADDR, 0xb);
if (!err)
err = mdio_write(phy, 0, MV88E1XXX_EXTENDED_DATA, 0x8004);
if (!err)
err = mv88e1xxx_downshift_set(phy, 1); /* Enable downshift */
return err;
}

1675
sys/dev/cxgb/common/cxgb_regs.h
View File

File diff suppressed because it is too large Load Diff

10
sys/dev/cxgb/common/cxgb_t3_cpl.h
View File

@ -173,8 +173,9 @@ enum { /* TCP congestion control algorithms */
enum { /* RSS hash type */
RSS_HASH_NONE = 0,
RSS_HASH_2_TUPLE = 1 << 0,
RSS_HASH_4_TUPLE = 1 << 1
RSS_HASH_2_TUPLE = 1,
RSS_HASH_4_TUPLE = 2,
RSS_HASH_TCPV6 = 3
};
union opcode_tid {
@ -1097,6 +1098,11 @@ struct cpl_rx_data_ddp {
#define V_DDP_OFFSET(x) ((x) << S_DDP_OFFSET)
#define G_DDP_OFFSET(x) (((x) >> S_DDP_OFFSET) & M_DDP_OFFSET)
#define S_DDP_DACK_MODE 22
#define M_DDP_DACK_MODE 0x3
#define V_DDP_DACK_MODE(x) ((x) << S_DDP_DACK_MODE)
#define G_DDP_DACK_MODE(x) (((x) >> S_DDP_DACK_MODE) & M_DDP_DACK_MODE)
#define S_DDP_URG 24
#define V_DDP_URG(x) ((x) << S_DDP_URG)
#define F_DDP_URG V_DDP_URG(1U)

373
sys/dev/cxgb/common/cxgb_t3_hw.c
View File

@ -403,6 +403,29 @@ int t3_phy_advertise(struct cphy *phy, unsigned int advert)
return mdio_write(phy, 0, MII_ADVERTISE, val);
}
/**
* t3_phy_advertise_fiber - set fiber PHY advertisement register
* @phy: the PHY to operate on
* @advert: bitmap of capabilities the PHY should advertise
*
* Sets a fiber PHY's advertisement register to advertise the
* requested capabilities.
*/
int t3_phy_advertise_fiber(struct cphy *phy, unsigned int advert)
{
unsigned int val = 0;
if (advert & ADVERTISED_1000baseT_Half)
val |= ADVERTISE_1000XHALF;
if (advert & ADVERTISED_1000baseT_Full)
val |= ADVERTISE_1000XFULL;
if (advert & ADVERTISED_Pause)
val |= ADVERTISE_1000XPAUSE;
if (advert & ADVERTISED_Asym_Pause)
val |= ADVERTISE_1000XPSE_ASYM;
return mdio_write(phy, 0, MII_ADVERTISE, val);
}
/**
* t3_set_phy_speed_duplex - force PHY speed and duplex
* @phy: the PHY to operate on
@ -451,8 +474,8 @@ static struct adapter_info t3_adap_info[] = {
&mi1_mdio_ops, "Chelsio T302" },
{ 1, 0, 0, 0, 0,
F_GPIO1_OEN | F_GPIO6_OEN | F_GPIO7_OEN | F_GPIO10_OEN |
F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, 0,
SUPPORTED_10000baseT_Full | SUPPORTED_AUI,
F_GPIO11_OEN | F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL,
0, SUPPORTED_10000baseT_Full | SUPPORTED_AUI,
&mi1_mdio_ext_ops, "Chelsio T310" },
{ 1, 1, 0, 0, 0,
F_GPIO1_OEN | F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO5_OEN | F_GPIO6_OEN |
@ -476,31 +499,20 @@ const struct adapter_info *t3_get_adapter_info(unsigned int id)
return id < ARRAY_SIZE(t3_adap_info) ? &t3_adap_info[id] : NULL;
}
#define CAPS_1G (SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full | \
SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII)
#define CAPS_10G (SUPPORTED_10000baseT_Full | SUPPORTED_AUI)
static struct port_type_info port_types[] = {
{ NULL },
{ t3_ael1002_phy_prep, CAPS_10G | SUPPORTED_FIBRE,
"10GBASE-XR" },
{ t3_vsc8211_phy_prep, CAPS_1G | SUPPORTED_TP | SUPPORTED_IRQ,
"10/100/1000BASE-T" },
{ t3_mv88e1xxx_phy_prep, CAPS_1G | SUPPORTED_TP | SUPPORTED_IRQ,
"10/100/1000BASE-T" },
{ t3_xaui_direct_phy_prep, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
{ NULL, CAPS_10G, "10GBASE-KX4" },
{ t3_qt2045_phy_prep, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
{ t3_ael1006_phy_prep, CAPS_10G | SUPPORTED_FIBRE,
"10GBASE-SR" },
{ NULL, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
{ t3_ael1002_phy_prep },
{ t3_vsc8211_phy_prep },
{ t3_mv88e1xxx_phy_prep },
{ t3_xaui_direct_phy_prep },
{ NULL },
{ t3_qt2045_phy_prep },
{ t3_ael1006_phy_prep },
{ NULL },
};
#undef CAPS_1G
#undef CAPS_10G
#define VPD_ENTRY(name, len) \
u8 name##_kword[2]; u8 name##_len; char name##_data[len]
u8 name##_kword[2]; u8 name##_len; u8 name##_data[len]
/*
* Partial EEPROM Vital Product Data structure. Includes only the ID and
@ -678,6 +690,15 @@ static int get_vpd_params(adapter_t *adapter, struct vpd_params *p)
return 0;
}
/* BIOS boot header */
typedef struct boot_header_s {
u8 signature[2]; /* signature */
u8 length; /* image length (include header) */
u8 offset[4]; /* initialization vector */
u8 reserved[19]; /* reserved */
u8 exheader[2]; /* offset to expansion header */
} boot_header_t;
/* serial flash and firmware constants */
enum {
SF_ATTEMPTS = 5, /* max retries for SF1 operations */
@ -694,7 +715,14 @@ enum {
FW_FLASH_BOOT_ADDR = 0x70000, /* start address of FW in flash */
FW_VERS_ADDR = 0x77ffc, /* flash address holding FW version */
FW_MIN_SIZE = 8 /* at least version and csum */
FW_MIN_SIZE = 8, /* at least version and csum */
FW_MAX_SIZE = FW_VERS_ADDR - FW_FLASH_BOOT_ADDR,
BOOT_FLASH_BOOT_ADDR = 0x0,/* start address of boot image in flash */
BOOT_SIGNATURE = 0xaa55, /* signature of BIOS boot ROM */
BOOT_SIZE_INC = 512, /* image size measured in 512B chunks */
BOOT_MIN_SIZE = sizeof(boot_header_t), /* at least basic header */
BOOT_MAX_SIZE = 0xff*BOOT_SIZE_INC /* 1 byte * length increment */
};
/**
@ -817,16 +845,21 @@ int t3_read_flash(adapter_t *adapter, unsigned int addr, unsigned int nwords,
* @addr: the start address to write
* @n: length of data to write
* @data: the data to write
* @byte_oriented: whether to store data as bytes or as words
*
* Writes up to a page of data (256 bytes) to the serial flash starting
* at the given address.
* If @byte_oriented is set the write data is stored as a 32-bit
* big-endian array, otherwise in the processor's native endianess.
*
*/
static int t3_write_flash(adapter_t *adapter, unsigned int addr,
unsigned int n, const u8 *data)
unsigned int n, const u8 *data,
int byte_oriented)
{
int ret;
u32 buf[64];
unsigned int i, c, left, val, offset = addr & 0xff;
unsigned int c, left, val, offset = addr & 0xff;
if (addr + n > SF_SIZE || offset + n > 256)
return -EINVAL;
@ -839,8 +872,10 @@ static int t3_write_flash(adapter_t *adapter, unsigned int addr,
for (left = n; left; left -= c) {
c = min(left, 4U);
for (val = 0, i = 0; i < c; ++i)
val = (val << 8) + *data++;
val = *(const u32*)data;
data += c;
if (byte_oriented)
val = htonl(val);
ret = sf1_write(adapter, c, c != left, val);
if (ret)
@ -850,7 +885,8 @@ static int t3_write_flash(adapter_t *adapter, unsigned int addr,
return ret;
/* Read the page to verify the write succeeded */
ret = t3_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1);
ret = t3_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf,
byte_oriented);
if (ret)
return ret;
@ -887,16 +923,18 @@ int t3_get_tp_version(adapter_t *adapter, u32 *vers)
* @adapter: the adapter
*
*/
int t3_check_tpsram_version(adapter_t *adapter)
int t3_check_tpsram_version(adapter_t *adapter, int *must_load)
{
int ret;
u32 vers;
unsigned int major, minor;
/* Get version loaded in SRAM */
t3_write_reg(adapter, A_TP_EMBED_OP_FIELD0, 0);
ret = t3_wait_op_done(adapter, A_TP_EMBED_OP_FIELD0,
1, 1, 5, 1);
if (adapter->params.rev == T3_REV_A)
return 0;
*must_load = 1;
ret = t3_get_tp_version(adapter, &vers);
if (ret)
return ret;
@ -908,9 +946,16 @@ int t3_check_tpsram_version(adapter_t *adapter)
if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
return 0;
CH_WARN(adapter, "found wrong TP version (%u.%u), "
"driver needs version %d.%d\n", major, minor,
TP_VERSION_MAJOR, TP_VERSION_MINOR);
if (major != TP_VERSION_MAJOR)
CH_ERR(adapter, "found wrong TP version (%u.%u), "
"driver needs version %d.%d\n", major, minor,
TP_VERSION_MAJOR, TP_VERSION_MINOR);
else {
*must_load = 0;
CH_ERR(adapter, "found wrong TP version (%u.%u), "
"driver compiled for version %d.%d\n", major, minor,
TP_VERSION_MAJOR, TP_VERSION_MINOR);
}
return -EINVAL;
}
@ -966,12 +1011,13 @@ int t3_get_fw_version(adapter_t *adapter, u32 *vers)
* Checks if an adapter's FW is compatible with the driver. Returns 0
* if the versions are compatible, a negative error otherwise.
*/
int t3_check_fw_version(adapter_t *adapter)
int t3_check_fw_version(adapter_t *adapter, int *must_load)
{
int ret;
u32 vers;
unsigned int type, major, minor;
*must_load = 1;
ret = t3_get_fw_version(adapter, &vers);
if (ret)
return ret;
@ -984,9 +1030,21 @@ int t3_check_fw_version(adapter_t *adapter)
minor == FW_VERSION_MINOR)
return 0;
CH_WARN(adapter, "found wrong FW version (%u.%u), "
"driver needs version %d.%d\n", major, minor,
FW_VERSION_MAJOR, FW_VERSION_MINOR);
if (major != FW_VERSION_MAJOR)
CH_ERR(adapter, "found wrong FW version(%u.%u), "
"driver needs version %u.%u\n", major, minor,
FW_VERSION_MAJOR, FW_VERSION_MINOR);
else if ((int)minor < FW_VERSION_MINOR) {
*must_load = 0;
CH_WARN(adapter, "found old FW minor version(%u.%u), "
"driver compiled for version %u.%u\n", major, minor,
FW_VERSION_MAJOR, FW_VERSION_MINOR);
} else {
CH_WARN(adapter, "found newer FW version(%u.%u), "
"driver compiled for version %u.%u\n", major, minor,
FW_VERSION_MAJOR, FW_VERSION_MINOR);
return 0;
}
return -EINVAL;
}
@ -1033,7 +1091,7 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
if ((size & 3) || size < FW_MIN_SIZE)
return -EINVAL;
if (size > FW_VERS_ADDR + 8 - FW_FLASH_BOOT_ADDR)
if (size - 8 > FW_MAX_SIZE)
return -EFBIG;
for (csum = 0, i = 0; i < size / sizeof(csum); i++)
@ -1052,7 +1110,7 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
for (addr = FW_FLASH_BOOT_ADDR; size; ) {
unsigned int chunk_size = min(size, 256U);
ret = t3_write_flash(adapter, addr, chunk_size, fw_data);
ret = t3_write_flash(adapter, addr, chunk_size, fw_data, 1);
if (ret)
goto out;
@ -1061,13 +1119,71 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
size -= chunk_size;
}
ret = t3_write_flash(adapter, FW_VERS_ADDR, 4, fw_data);
ret = t3_write_flash(adapter, FW_VERS_ADDR, 4, fw_data, 1);
out:
if (ret)
CH_ERR(adapter, "firmware download failed, error %d\n", ret);
return ret;
}
/*
* t3_load_boot - download boot flash
* @adapter: the adapter
* @boot_data: the boot image to write
* @size: image size
*
* Write the supplied boot image to the card's serial flash.
* The boot image has the following sections: a 28-byte header and the
* boot image.
*/
int t3_load_boot(adapter_t *adapter, u8 *boot_data, unsigned int size)
{
boot_header_t *header = (boot_header_t *)boot_data;
int ret;
unsigned int addr;
unsigned int boot_sector = BOOT_FLASH_BOOT_ADDR >> 16;
unsigned int boot_end = (BOOT_FLASH_BOOT_ADDR + size - 1) >> 16;
/*
* Perform some primitive sanity testing to avoid accidentally
* writing garbage over the boot sectors. We ought to check for
* more but it's not worth it for now ...
*/
if (size < BOOT_MIN_SIZE || size > BOOT_MAX_SIZE) {
CH_ERR(adapter, "boot image too small/large\n");
return -EFBIG;
}
if (le16_to_cpu(*(u16*)header->signature) != BOOT_SIGNATURE) {
CH_ERR(adapter, "boot image missing signature\n");
return -EINVAL;
}
if (header->length * BOOT_SIZE_INC != size) {
CH_ERR(adapter, "boot image header length != image length\n");
return -EINVAL;
}
ret = t3_flash_erase_sectors(adapter, boot_sector, boot_end);
if (ret)
goto out;
for (addr = BOOT_FLASH_BOOT_ADDR; size; ) {
unsigned int chunk_size = min(size, 256U);
ret = t3_write_flash(adapter, addr, chunk_size, boot_data, 0);
if (ret)
goto out;
addr += chunk_size;
boot_data += chunk_size;
size -= chunk_size;
}
out:
if (ret)
CH_ERR(adapter, "boot image download failed, error %d\n", ret);
return ret;
}
#define CIM_CTL_BASE 0x2000
/**
@ -1175,7 +1291,6 @@ int t3_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc)
fc);
/* Also disables autoneg */
phy->ops->set_speed_duplex(phy, lc->speed, lc->duplex);
phy->ops->reset(phy, 0);
} else
phy->ops->autoneg_enable(phy);
} else {
@ -1248,7 +1363,13 @@ static int t3_handle_intr_status(adapter_t *adapter, unsigned int reg,
return fatal;
}
#define SGE_INTR_MASK (F_RSPQDISABLED)
#define SGE_INTR_MASK (F_RSPQDISABLED | \
F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR | \
F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
F_HIRCQPARITYERROR)
#define MC5_INTR_MASK (F_PARITYERR | F_ACTRGNFULL | F_UNKNOWNCMD | \
F_REQQPARERR | F_DISPQPARERR | F_DELACTEMPTY | \
F_NFASRCHFAIL)
@ -1265,16 +1386,23 @@ static int t3_handle_intr_status(adapter_t *adapter, unsigned int reg,
#define PCIE_INTR_MASK (F_UNXSPLCPLERRR | F_UNXSPLCPLERRC | F_PCIE_PIOPARERR |\
F_PCIE_WFPARERR | F_PCIE_RFPARERR | F_PCIE_CFPARERR | \
/* V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR) | */ \
V_BISTERR(M_BISTERR) | F_PEXERR)
#define ULPRX_INTR_MASK F_PARERR
#define ULPTX_INTR_MASK 0
#define CPLSW_INTR_MASK (F_TP_FRAMING_ERROR | \
F_RETRYBUFPARERR | F_RETRYLUTPARERR | F_RXPARERR | \
F_TXPARERR | V_BISTERR(M_BISTERR))
#define ULPRX_INTR_MASK (F_PARERRDATA | F_PARERRPCMD | F_ARBPF1PERR | \
F_ARBPF0PERR | F_ARBFPERR | F_PCMDMUXPERR | \
F_DATASELFRAMEERR1 | F_DATASELFRAMEERR0)
#define ULPTX_INTR_MASK 0xfc
#define CPLSW_INTR_MASK (F_CIM_OP_MAP_PERR | F_TP_FRAMING_ERROR | \
F_SGE_FRAMING_ERROR | F_CIM_FRAMING_ERROR | \
F_ZERO_SWITCH_ERROR)
#define CIM_INTR_MASK (F_BLKWRPLINT | F_BLKRDPLINT | F_BLKWRCTLINT | \
F_BLKRDCTLINT | F_BLKWRFLASHINT | F_BLKRDFLASHINT | \
F_SGLWRFLASHINT | F_WRBLKFLASHINT | F_BLKWRBOOTINT | \
F_FLASHRANGEINT | F_SDRAMRANGEINT | F_RSVDSPACEINT)
F_FLASHRANGEINT | F_SDRAMRANGEINT | F_RSVDSPACEINT | \
F_DRAMPARERR | F_ICACHEPARERR | F_DCACHEPARERR | \
F_OBQSGEPARERR | F_OBQULPHIPARERR | F_OBQULPLOPARERR | \
F_IBQSGELOPARERR | F_IBQSGEHIPARERR | F_IBQULPPARERR | \
F_IBQTPPARERR | F_ITAGPARERR | F_DTAGPARERR)
#define PMTX_INTR_MASK (F_ZERO_C_CMD_ERROR | ICSPI_FRM_ERR | OESPI_FRM_ERR | \
V_ICSPI_PAR_ERROR(M_ICSPI_PAR_ERROR) | \
V_OESPI_PAR_ERROR(M_OESPI_PAR_ERROR))
@ -1343,6 +1471,10 @@ static void pcie_intr_handler(adapter_t *adapter)
{ F_PCIE_CFPARERR, "PCI command FIFO parity error", -1, 1 },
{ V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR),
"PCI MSI-X table/PBA parity error", -1, 1 },
{ F_RETRYBUFPARERR, "PCI retry buffer parity error", -1, 1 },
{ F_RETRYLUTPARERR, "PCI retry LUT parity error", -1, 1 },
{ F_RXPARERR, "PCI Rx parity error", -1, 1 },
{ F_TXPARERR, "PCI Tx parity error", -1, 1 },
{ V_BISTERR(M_BISTERR), "PCI BIST error", -1, 1 },
{ 0 }
};
@ -1367,9 +1499,16 @@ static void tp_intr_handler(adapter_t *adapter)
{ 0x2000000, "TP out of Tx pages", -1, 1 },
{ 0 }
};
static struct intr_info tp_intr_info_t3c[] = {
{ 0x1fffffff, "TP parity error", -1, 1 },
{ F_FLMRXFLSTEMPTY, "TP out of Rx pages", -1, 1 },
{ F_FLMTXFLSTEMPTY, "TP out of Tx pages", -1, 1 },
{ 0 }
};
if (t3_handle_intr_status(adapter, A_TP_INT_CAUSE, 0xffffffff,
tp_intr_info, NULL))
adapter->params.rev < T3_REV_C ?
tp_intr_info : tp_intr_info_t3c, NULL))
t3_fatal_err(adapter);
}
@ -1391,10 +1530,22 @@ static void cim_intr_handler(adapter_t *adapter)
{ F_BLKWRCTLINT, "CIM block write to CTL space", -1, 1 },
{ F_BLKRDPLINT, "CIM block read from PL space", -1, 1 },
{ F_BLKWRPLINT, "CIM block write to PL space", -1, 1 },
{ F_DRAMPARERR, "CIM DRAM parity error", -1, 1 },
{ F_ICACHEPARERR, "CIM icache parity error", -1, 1 },
{ F_DCACHEPARERR, "CIM dcache parity error", -1, 1 },
{ F_OBQSGEPARERR, "CIM OBQ SGE parity error", -1, 1 },
{ F_OBQULPHIPARERR, "CIM OBQ ULPHI parity error", -1, 1 },
{ F_OBQULPLOPARERR, "CIM OBQ ULPLO parity error", -1, 1 },
{ F_IBQSGELOPARERR, "CIM IBQ SGELO parity error", -1, 1 },
{ F_IBQSGEHIPARERR, "CIM IBQ SGEHI parity error", -1, 1 },
{ F_IBQULPPARERR, "CIM IBQ ULP parity error", -1, 1 },
{ F_IBQTPPARERR, "CIM IBQ TP parity error", -1, 1 },
{ F_ITAGPARERR, "CIM itag parity error", -1, 1 },
{ F_DTAGPARERR, "CIM dtag parity error", -1, 1 },
{ 0 }
};
if (t3_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, 0xffffffff,
if (t3_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, CIM_INTR_MASK,
cim_intr_info, NULL))
t3_fatal_err(adapter);
}
@ -1405,7 +1556,14 @@ static void cim_intr_handler(adapter_t *adapter)
static void ulprx_intr_handler(adapter_t *adapter)
{
static struct intr_info ulprx_intr_info[] = {
{ F_PARERR, "ULP RX parity error", -1, 1 },
{ F_PARERRDATA, "ULP RX data parity error", -1, 1 },
{ F_PARERRPCMD, "ULP RX command parity error", -1, 1 },
{ F_ARBPF1PERR, "ULP RX ArbPF1 parity error", -1, 1 },
{ F_ARBPF0PERR, "ULP RX ArbPF0 parity error", -1, 1 },
{ F_ARBFPERR, "ULP RX ArbF parity error", -1, 1 },
{ F_PCMDMUXPERR, "ULP RX PCMDMUX parity error", -1, 1 },
{ F_DATASELFRAMEERR1, "ULP RX frame error", -1, 1 },
{ F_DATASELFRAMEERR0, "ULP RX frame error", -1, 1 },
{ 0 }
};
@ -1424,6 +1582,7 @@ static void ulptx_intr_handler(adapter_t *adapter)
STAT_ULP_CH0_PBL_OOB, 0 },
{ F_PBL_BOUND_ERR_CH1, "ULP TX channel 1 PBL out of bounds",
STAT_ULP_CH1_PBL_OOB, 0 },
{ 0xfc, "ULP TX parity error", -1, 1 },
{ 0 }
};
@ -1498,7 +1657,8 @@ static void pmrx_intr_handler(adapter_t *adapter)
static void cplsw_intr_handler(adapter_t *adapter)
{
static struct intr_info cplsw_intr_info[] = {
// { F_CIM_OVFL_ERROR, "CPL switch CIM overflow", -1, 1 },
{ F_CIM_OP_MAP_PERR, "CPL switch CIM parity error", -1, 1 },
{ F_CIM_OVFL_ERROR, "CPL switch CIM overflow", -1, 1 },
{ F_TP_FRAMING_ERROR, "CPL switch TP framing error", -1, 1 },
{ F_SGE_FRAMING_ERROR, "CPL switch SGE framing error", -1, 1 },
{ F_CIM_FRAMING_ERROR, "CPL switch CIM framing error", -1, 1 },
@ -1632,7 +1792,7 @@ int t3_phy_intr_handler(adapter_t *adapter)
mask = gpi - (gpi & (gpi - 1));
gpi -= mask;
if (!(p->port_type->caps & SUPPORTED_IRQ))
if (!(p->phy.caps & SUPPORTED_IRQ))
continue;
if (cause & mask) {
@ -1728,7 +1888,6 @@ void t3_intr_enable(adapter_t *adapter)
MC7_INTR_MASK },
{ A_MC5_DB_INT_ENABLE, MC5_INTR_MASK },
{ A_ULPRX_INT_ENABLE, ULPRX_INTR_MASK },
{ A_TP_INT_ENABLE, 0x3bfffff },
{ A_PM1_TX_INT_ENABLE, PMTX_INTR_MASK },
{ A_PM1_RX_INT_ENABLE, PMRX_INTR_MASK },
{ A_CIM_HOST_INT_ENABLE, CIM_INTR_MASK },
@ -1738,6 +1897,8 @@ void t3_intr_enable(adapter_t *adapter)
adapter->slow_intr_mask = PL_INTR_MASK;
t3_write_regs(adapter, intr_en_avp, ARRAY_SIZE(intr_en_avp), 0);
t3_write_reg(adapter, A_TP_INT_ENABLE,
adapter->params.rev >= T3_REV_C ? 0x2bfffff : 0x3bfffff);
if (adapter->params.rev > 0) {
t3_write_reg(adapter, A_CPL_INTR_ENABLE,
@ -1889,6 +2050,15 @@ static int t3_sge_write_context(adapter_t *adapter, unsigned int id,
0, SG_CONTEXT_CMD_ATTEMPTS, 1);
}
static int clear_sge_ctxt(adapter_t *adap, unsigned int id, unsigned int type)
{
t3_write_reg(adap, A_SG_CONTEXT_DATA0, 0);
t3_write_reg(adap, A_SG_CONTEXT_DATA1, 0);
t3_write_reg(adap, A_SG_CONTEXT_DATA2, 0);
t3_write_reg(adap, A_SG_CONTEXT_DATA3, 0);
return t3_sge_write_context(adap, id, type);
}
/**
* t3_sge_init_ecntxt - initialize an SGE egress context
* @adapter: the adapter to configure
@ -2389,20 +2559,6 @@ static void tp_wr_bits_indirect(adapter_t *adap, unsigned int addr,
t3_write_reg(adap, A_TP_PIO_DATA, val);
}
/**
* t3_enable_filters - enable the HW filters
* @adap: the adapter
*
* Enables the HW filters for NIC traffic.
*/
void t3_enable_filters(adapter_t *adap)
{
t3_set_reg_field(adap, A_TP_IN_CONFIG, F_NICMODE, 0);
t3_set_reg_field(adap, A_MC5_DB_CONFIG, 0, F_FILTEREN);
t3_set_reg_field(adap, A_TP_GLOBAL_CONFIG, 0, V_FIVETUPLELOOKUP(3));
tp_wr_bits_indirect(adap, A_TP_INGRESS_CONFIG, 0, F_LOOKUPEVERYPKT);
}
/**
* pm_num_pages - calculate the number of pages of the payload memory
* @mem_size: the size of the payload memory
@ -2508,7 +2664,7 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
V_AUTOSTATE2(1) | V_AUTOSTATE1(0) |
V_BYTETHRESHOLD(16384) | V_MSSTHRESHOLD(2) |
F_AUTOCAREFUL | F_AUTOENABLE | V_DACK_MODE(1));
t3_set_reg_field(adap, A_TP_IN_CONFIG, F_IPV6ENABLE | F_NICMODE,
t3_set_reg_field(adap, A_TP_IN_CONFIG, F_RXFBARBPRIO | F_TXFBARBPRIO,
F_IPV6ENABLE | F_NICMODE);
t3_write_reg(adap, A_TP_TX_RESOURCE_LIMIT, 0x18141814);
t3_write_reg(adap, A_TP_PARA_REG4, 0x5050105);
@ -2519,7 +2675,9 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
F_ENABLEEPCMDAFULL,
F_ENABLEOCSPIFULL |F_TXDEFERENABLE | F_HEARBEATDACK |
F_TXCONGESTIONMODE | F_RXCONGESTIONMODE);
t3_set_reg_field(adap, A_TP_PC_CONFIG2, F_CHDRAFULL, 0);
t3_set_reg_field(adap, A_TP_PC_CONFIG2, F_CHDRAFULL,
F_ENABLEIPV6RSS | F_ENABLENONOFDTNLSYN |
F_ENABLEARPMISS | F_DISBLEDAPARBIT0);
t3_write_reg(adap, A_TP_PROXY_FLOW_CNTL, 1080);
t3_write_reg(adap, A_TP_PROXY_FLOW_CNTL, 1000);
@ -2534,6 +2692,11 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
} else
t3_set_reg_field(adap, A_TP_PARA_REG3, 0, F_TXPACEFIXED);
if (adap->params.rev == T3_REV_C)
t3_set_reg_field(adap, A_TP_PC_CONFIG,
V_TABLELATENCYDELTA(M_TABLELATENCYDELTA),
V_TABLELATENCYDELTA(4));
t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT1, 0);
t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0, 0);
t3_write_reg(adap, A_TP_MOD_CHANNEL_WEIGHT, 0);
@ -2972,7 +3135,7 @@ int t3_config_sched(adapter_t *adap, unsigned int kbps, int sched)
if (bpt > 0 && bpt <= 255) {
v = bpt * tps;
delta = v >= kbps ? v - kbps : kbps - v;
if (delta <= mindelta) {
if (delta < mindelta) {
mindelta = delta;
selected_cpt = cpt;
selected_bpt = bpt;
@ -3383,7 +3546,8 @@ static void config_pcie(adapter_t *adap)
V_REPLAYLMT(rpllmt));
t3_write_reg(adap, A_PCIE_PEX_ERR, 0xffffffff);
t3_set_reg_field(adap, A_PCIE_CFG, F_PCIE_CLIDECEN, F_PCIE_CLIDECEN);
t3_set_reg_field(adap, A_PCIE_CFG, 0,
F_PCIE_DMASTOPEN | F_PCIE_CLIDECEN);
}
/**
@ -3401,7 +3565,7 @@ static void config_pcie(adapter_t *adap)
*/
int t3_init_hw(adapter_t *adapter, u32 fw_params)
{
int err = -EIO, attempts = 100;
int err = -EIO, attempts, i;
const struct vpd_params *vpd = &adapter->params.vpd;
if (adapter->params.rev > 0)
@ -3422,6 +3586,10 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
adapter->params.mc5.nfilters,
adapter->params.mc5.nroutes))
goto out_err;
for (i = 0; i < 32; i++)
if (clear_sge_ctxt(adapter, i, F_CQ))
goto out_err;
}
if (tp_init(adapter, &adapter->params.tp))
@ -3438,7 +3606,12 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
if (is_pcie(adapter))
config_pcie(adapter);
else
t3_set_reg_field(adapter, A_PCIX_CFG, 0, F_CLIDECEN);
t3_set_reg_field(adapter, A_PCIX_CFG, 0,
F_DMASTOPEN | F_CLIDECEN);
if (adapter->params.rev == T3_REV_C)
t3_set_reg_field(adapter, A_ULPTX_CONFIG, 0,
F_CFG_CQE_SOP_MASK);
t3_write_reg(adapter, A_PM1_RX_CFG, 0xffffffff);
t3_write_reg(adapter, A_PM1_RX_MODE, 0);
@ -3451,6 +3624,7 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
V_BOOTADDR(FW_FLASH_BOOT_ADDR >> 2));
(void) t3_read_reg(adapter, A_CIM_BOOT_CFG); /* flush */
attempts = 100;
do { /* wait for uP to initialize */
msleep(20);
} while (t3_read_reg(adapter, A_CIM_HOST_ACC_DATA) && --attempts);
@ -3601,6 +3775,7 @@ void early_hw_init(adapter_t *adapter, const struct adapter_info *ai)
t3_write_reg(adapter, A_T3DBG_GPIO_EN,
ai->gpio_out | F_GPIO0_OEN | F_GPIO0_OUT_VAL);
t3_write_reg(adapter, A_MC5_DB_SERVER_INDEX, 0);
t3_write_reg(adapter, A_SG_OCO_BASE, V_BASE1(0xfff));
if (adapter->params.rev == 0 || !uses_xaui(adapter))
val |= F_ENRGMII;
@ -3651,6 +3826,36 @@ static int t3_reset_adapter(adapter_t *adapter)
return 0;
}
static int __devinit init_parity(adapter_t *adap)
{
int i, err, addr;
if (t3_read_reg(adap, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY)
return -EBUSY;
for (err = i = 0; !err && i < 16; i++)
err = clear_sge_ctxt(adap, i, F_EGRESS);
for (i = 0xfff0; !err && i <= 0xffff; i++)
err = clear_sge_ctxt(adap, i, F_EGRESS);
for (i = 0; !err && i < SGE_QSETS; i++)
err = clear_sge_ctxt(adap, i, F_RESPONSEQ);
if (err)
return err;
t3_write_reg(adap, A_CIM_IBQ_DBG_DATA, 0);
for (i = 0; i < 4; i++)
for (addr = 0; addr <= M_IBQDBGADDR; addr++) {
t3_write_reg(adap, A_CIM_IBQ_DBG_CFG, F_IBQDBGEN |
F_IBQDBGWR | V_IBQDBGQID(i) |
V_IBQDBGADDR(addr));
err = t3_wait_op_done(adap, A_CIM_IBQ_DBG_CFG,
F_IBQDBGBUSY, 0, 2, 1);
if (err)
return err;
}
return 0;
}
/**
* t3_prep_adapter - prepare SW and HW for operation
* @adapter: the adapter
@ -3732,6 +3937,9 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
}
early_hw_init(adapter, ai);
ret = init_parity(adapter);
if (ret)
return ret;
if (adapter->params.nports > 2 &&
(ret = t3_vsc7323_init(adapter, adapter->params.nports)))
@ -3739,14 +3947,17 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
for_each_port(adapter, i) {
u8 hw_addr[6];
const struct port_type_info *pti;
struct port_info *p = adap2pinfo(adapter, i);
while (!adapter->params.vpd.port_type[j])
++j;
p->port_type = &port_types[adapter->params.vpd.port_type[j]];
p->port_type->phy_prep(&p->phy, adapter, ai->phy_base_addr + j,
ai->mdio_ops);
pti = &port_types[adapter->params.vpd.port_type[j]];
ret = pti->phy_prep(&p->phy, adapter, ai->phy_base_addr + j,
ai->mdio_ops);
if (ret)
return ret;
mac_prep(&p->mac, adapter, j);
++j;
@ -3759,9 +3970,9 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
hw_addr[5] = adapter->params.vpd.eth_base[5] + i;
t3_os_set_hw_addr(adapter, i, hw_addr);
init_link_config(&p->link_config, p->port_type->caps);
init_link_config(&p->link_config, p->phy.caps);
p->phy.ops->power_down(&p->phy, 1);
if (!(p->port_type->caps & SUPPORTED_IRQ))
if (!(p->phy.caps & SUPPORTED_IRQ))
adapter->params.linkpoll_period = 10;
}

7
sys/dev/cxgb/common/cxgb_tcb.h
View File

@ -668,7 +668,10 @@ POSSIBILITY OF SUCH DAMAGE.
#define S_TF_DDP_BUF1_FLUSH 28
#define V_TF_DDP_BUF1_FLUSH(x) ((x) << S_TF_DDP_BUF1_FLUSH)
#define S_TF_DDP_PSH_NO_INVALIDATE 29
#define V_TF_DDP_PSH_NO_INVALIDATE(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE)
#define S_TF_DDP_PSH_NO_INVALIDATE0 29
#define V_TF_DDP_PSH_NO_INVALIDATE0(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE0)
#define S_TF_DDP_PSH_NO_INVALIDATE1 30
#define V_TF_DDP_PSH_NO_INVALIDATE1(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE1)
#endif /* _TCB_DEFS_H */

2
sys/dev/cxgb/common/cxgb_version.h
View File

@ -37,5 +37,5 @@ POSSIBILITY OF SUCH DAMAGE.
#define __CHELSIO_VERSION_H
#define DRV_DESC "Chelsio T3 Network Driver"
#define DRV_NAME "cxgb"
#define DRV_VERSION "1.0.086"
#define DRV_VERSION "1.0.129a"
#endif

176
sys/dev/cxgb/common/cxgb_vsc8211.c
View File

@ -36,11 +36,17 @@ __FBSDID("$FreeBSD$");
#include <dev/cxgb/cxgb_include.h>
#endif
#undef msleep
#define msleep t3_os_sleep
/* VSC8211 PHY specific registers. */
enum {
VSC8211_SIGDET_CTRL = 19,
VSC8211_EXT_CTRL = 23,
VSC8211_INTR_ENABLE = 25,
VSC8211_INTR_STATUS = 26,
VSC8211_AUX_CTRL_STAT = 28,
VSC8211_EXT_PAGE_AXS = 31,
};
enum {
@ -55,11 +61,19 @@ enum {
VSC_INTR_SYMBOL_ERR = 1 << 8, /* symbol error */
VSC_INTR_NEG_DONE = 1 << 10, /* autoneg done */
VSC_INTR_NEG_ERR = 1 << 11, /* autoneg error */
VSC_INTR_DPLX_CHG = 1 << 12, /* duplex change */
VSC_INTR_LINK_CHG = 1 << 13, /* link change */
VSC_INTR_SPD_CHG = 1 << 14, /* speed change */
VSC_INTR_ENABLE = 1 << 15, /* interrupt enable */
};
enum {
VSC_CTRL_CLAUSE37_VIEW = 1 << 4, /* Switch to Clause 37 view */
VSC_CTRL_MEDIA_MODE_HI = 0xf000 /* High part of media mode select */
};
#define CFG_CHG_INTR_MASK (VSC_INTR_LINK_CHG | VSC_INTR_NEG_ERR | \
VSC_INTR_DPLX_CHG | VSC_INTR_SPD_CHG | \
VSC_INTR_NEG_DONE)
#define INTR_MASK (CFG_CHG_INTR_MASK | VSC_INTR_TX_FIFO | VSC_INTR_RX_FIFO | \
VSC_INTR_ENABLE)
@ -189,6 +203,98 @@ static int vsc8211_get_link_status(struct cphy *cphy, int *link_ok,
return 0;
}
static int vsc8211_get_link_status_fiber(struct cphy *cphy, int *link_ok,
int *speed, int *duplex, int *fc)
{
unsigned int bmcr, status, lpa, adv;
int err, sp = -1, dplx = -1, pause = 0;
err = mdio_read(cphy, 0, MII_BMCR, &bmcr);
if (!err)
err = mdio_read(cphy, 0, MII_BMSR, &status);
if (err)
return err;
if (link_ok) {
/*
* BMSR_LSTATUS is latch-low, so if it is 0 we need to read it
* once more to get the current link state.
*/
if (!(status & BMSR_LSTATUS))
err = mdio_read(cphy, 0, MII_BMSR, &status);
if (err)
return err;
*link_ok = (status & BMSR_LSTATUS) != 0;
}
if (!(bmcr & BMCR_ANENABLE)) {
dplx = (bmcr & BMCR_FULLDPLX) ? DUPLEX_FULL : DUPLEX_HALF;
if (bmcr & BMCR_SPEED1000)
sp = SPEED_1000;
else if (bmcr & BMCR_SPEED100)
sp = SPEED_100;
else
sp = SPEED_10;
} else if (status & BMSR_ANEGCOMPLETE) {
err = mdio_read(cphy, 0, MII_LPA, &lpa);
if (!err)
err = mdio_read(cphy, 0, MII_ADVERTISE, &adv);
if (err)
return err;
if (adv & lpa & ADVERTISE_1000XFULL) {
dplx = DUPLEX_FULL;
sp = SPEED_1000;
} else if (adv & lpa & ADVERTISE_1000XHALF) {
dplx = DUPLEX_HALF;
sp = SPEED_1000;
}
if (fc && dplx == DUPLEX_FULL) {
if (lpa & adv & ADVERTISE_1000XPAUSE)
pause = PAUSE_RX | PAUSE_TX;
else if ((lpa & ADVERTISE_1000XPAUSE) &&
(adv & lpa & ADVERTISE_1000XPSE_ASYM))
pause = PAUSE_TX;
else if ((lpa & ADVERTISE_1000XPSE_ASYM) &&
(adv & ADVERTISE_1000XPAUSE))
pause = PAUSE_RX;
}
}
if (speed)
*speed = sp;
if (duplex)
*duplex = dplx;
if (fc)
*fc = pause;
return 0;
}
/*
* Enable/disable auto MDI/MDI-X in forced link speed mode.
*/
static int vsc8211_set_automdi(struct cphy *phy, int enable)
{
int err;
if ((err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0x52b5)) != 0 ||
(err = mdio_write(phy, 0, 18, 0x12)) != 0 ||
(err = mdio_write(phy, 0, 17, enable ? 0x2803 : 0x3003)) != 0 ||
(err = mdio_write(phy, 0, 16, 0x87fa)) != 0 ||
(err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0)) != 0)
return err;
return 0;
}
static int vsc8211_set_speed_duplex(struct cphy *phy, int speed, int duplex)
{
int err;
err = t3_set_phy_speed_duplex(phy, speed, duplex);
if (!err)
err = vsc8211_set_automdi(phy, 1);
return err;
}
static int vsc8211_power_down(struct cphy *cphy, int enable)
{
return t3_mdio_change_bits(cphy, 0, MII_BMCR, BMCR_PDOWN,
@ -214,7 +320,6 @@ static int vsc8211_intr_handler(struct cphy *cphy)
#ifdef C99_NOT_SUPPORTED
static struct cphy_ops vsc8211_ops = {
NULL,
vsc8211_reset,
vsc8211_intr_enable,
vsc8211_intr_disable,
@ -224,10 +329,25 @@ static struct cphy_ops vsc8211_ops = {
vsc8211_autoneg_restart,
t3_phy_advertise,
NULL,
t3_set_phy_speed_duplex,
vsc8211_set_speed_duplex,
vsc8211_get_link_status,
vsc8211_power_down,
};
static struct cphy_ops vsc8211_fiber_ops = {
vsc8211_reset,
vsc8211_intr_enable,
vsc8211_intr_disable,
vsc8211_intr_clear,
vsc8211_intr_handler,
vsc8211_autoneg_enable,
vsc8211_autoneg_restart,
t3_phy_advertise_fiber,
NULL,
t3_set_phy_speed_duplex,
vsc8211_get_link_status_fiber,
vsc8211_power_down,
};
#else
static struct cphy_ops vsc8211_ops = {
.reset = vsc8211_reset,
@ -238,15 +358,57 @@ static struct cphy_ops vsc8211_ops = {
.autoneg_enable = vsc8211_autoneg_enable,
.autoneg_restart = vsc8211_autoneg_restart,
.advertise = t3_phy_advertise,
.set_speed_duplex = t3_set_phy_speed_duplex,
.set_speed_duplex = vsc8211_set_speed_duplex,
.get_link_status = vsc8211_get_link_status,
.power_down = vsc8211_power_down,
};
static struct cphy_ops vsc8211_fiber_ops = {
.reset = vsc8211_reset,
.intr_enable = vsc8211_intr_enable,
.intr_disable = vsc8211_intr_disable,
.intr_clear = vsc8211_intr_clear,
.intr_handler = vsc8211_intr_handler,
.autoneg_enable = vsc8211_autoneg_enable,
.autoneg_restart = vsc8211_autoneg_restart,
.advertise = t3_phy_advertise_fiber,
.set_speed_duplex = t3_set_phy_speed_duplex,
.get_link_status = vsc8211_get_link_status_fiber,
.power_down = vsc8211_power_down,
};
#endif
void t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
int t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
const struct mdio_ops *mdio_ops)
{
cphy_init(phy, adapter, phy_addr, &vsc8211_ops, mdio_ops);
t3_os_sleep(20); /* PHY needs ~10ms to start responding to MDIO */
int err;
unsigned int val;
cphy_init(phy, adapter, phy_addr, &vsc8211_ops, mdio_ops,
SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full |
SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII |
SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T");
msleep(20); /* PHY needs ~10ms to start responding to MDIO */
err = mdio_read(phy, 0, VSC8211_EXT_CTRL, &val);
if (err)
return err;
if (val & VSC_CTRL_MEDIA_MODE_HI)
return 0; /* copper interface, done */
phy->caps = SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg |
SUPPORTED_MII | SUPPORTED_FIBRE | SUPPORTED_IRQ;
phy->desc = "1000BASE-X";
phy->ops = &vsc8211_fiber_ops;
if ((err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 1)) != 0 ||
(err = mdio_write(phy, 0, VSC8211_SIGDET_CTRL, 1)) != 0 ||
(err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0)) != 0 ||
(err = mdio_write(phy, 0, VSC8211_EXT_CTRL,
val | VSC_CTRL_CLAUSE37_VIEW)) != 0 ||
(err = vsc8211_reset(phy, 0)) != 0)
return err;
udelay(5); /* delay after reset before next SMI */
return 0;
}

143
sys/dev/cxgb/common/cxgb_xgmac.c
View File

@ -75,6 +75,12 @@ static void xaui_serdes_reset(struct cmac *mac)
}
}
/**
* t3b_pcs_reset - reset the PCS on T3B+ adapters
* @mac: the XGMAC handle
*
* Reset the XGMAC PCS block on T3B+ adapters.
*/
void t3b_pcs_reset(struct cmac *mac)
{
t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset,
@ -84,6 +90,12 @@ void t3b_pcs_reset(struct cmac *mac)
F_PCS_RESET_);
}
/**
* t3_mac_reset - reset a MAC
* @mac: the MAC to reset
*
* Reset the given MAC.
*/
int t3_mac_reset(struct cmac *mac)
{
static struct addr_val_pair mac_reset_avp[] = {
@ -114,6 +126,7 @@ int t3_mac_reset(struct cmac *mac)
t3_set_reg_field(adap, A_XGM_RXFIFO_CFG + oft,
F_RXSTRFRWRD | F_DISERRFRAMES,
uses_xaui(adap) ? 0 : F_RXSTRFRWRD);
t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + oft, 0, F_UNDERUNFIX);
if (uses_xaui(adap)) {
if (adap->params.rev == 0) {
@ -146,8 +159,10 @@ int t3_mac_reset(struct cmac *mac)
t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN);
t3_write_reg(adap, A_XGM_RX_CTRL + oft, F_RXEN);
}
val = F_MAC_RESET_;
t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + oft,
V_RXMAXFRAMERSIZE(M_RXMAXFRAMERSIZE),
V_RXMAXFRAMERSIZE(MAX_FRAME_SIZE) | F_RXENFRAMER);
val = F_MAC_RESET_ | F_XGMAC_STOP_EN;
if (is_10G(adap) || mac->multiport)
val |= F_PCS_RESET_;
else if (uses_xaui(adap))
@ -236,7 +251,14 @@ static void set_addr_filter(struct cmac *mac, int idx, const u8 *addr)
t3_write_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_HIGH_1 + oft, addr_hi);
}
/* Set one of the station's unicast MAC addresses. */
/**
* t3_mac_set_address - set one of the station's unicast MAC addresses
* @mac: the MAC handle
* @idx: index of the exact address match filter to use
* @addr: the Ethernet address
*
* Set one of the station's unicast MAC addresses.
*/
int t3_mac_set_address(struct cmac *mac, unsigned int idx, u8 addr[6])
{
if (mac->multiport)
@ -249,10 +271,14 @@ int t3_mac_set_address(struct cmac *mac, unsigned int idx, u8 addr[6])
return 0;
}
/*
* Specify the number of exact address filters that should be reserved for
* unicast addresses. Caller should reload the unicast and multicast addresses
* after calling this.
/**
* t3_mac_set_num_ucast - set the number of unicast addresses needed
* @mac: the MAC handle
* @n: number of unicast addresses needed
*
* Specify the number of exact address filters that should be reserved for
* unicast addresses. Caller should reload the unicast and multicast
* addresses after calling this.
*/
int t3_mac_set_num_ucast(struct cmac *mac, unsigned char n)
{
@ -298,6 +324,14 @@ static int hash_hw_addr(const u8 *addr)
return hash;
}
/**
* t3_mac_set_rx_mode - set the Rx mode and address filters
* @mac: the MAC to configure
* @rm: structure containing the Rx mode and MAC addresses needed
*
* Configures the MAC Rx mode (promiscuity, etc) and exact and hash
* address filters.
*/
int t3_mac_set_rx_mode(struct cmac *mac, struct t3_rx_mode *rm)
{
u32 hash_lo, hash_hi;
@ -344,10 +378,18 @@ static int rx_fifo_hwm(int mtu)
return min(hwm, MAC_RXFIFO_SIZE - 8192);
}
/**
* t3_mac_set_mtu - set the MAC MTU
* @mac: the MAC to configure
* @mtu: the MTU
*
* Sets the MAC MTU and adjusts the FIFO PAUSE watermarks accordingly.
*/
int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu)
{
int hwm, lwm;
unsigned int thres, v;
int hwm, lwm, divisor;
int ipg;
unsigned int thres, v, reg;
adapter_t *adap = mac->adapter;
/*
@ -362,27 +404,33 @@ int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu)
if (mac->multiport)
return t3_vsc7323_set_mtu(adap, mtu - 4, mac->ext_port);
if (adap->params.rev == T3_REV_B2 &&
if (adap->params.rev >= T3_REV_B2 &&
(t3_read_reg(adap, A_XGM_RX_CTRL + mac->offset) & F_RXEN)) {
disable_exact_filters(mac);
v = t3_read_reg(adap, A_XGM_RX_CFG + mac->offset);
t3_set_reg_field(adap, A_XGM_RX_CFG + mac->offset,
F_ENHASHMCAST | F_COPYALLFRAMES, F_DISBCAST);
/* drain rx FIFO */
if (t3_wait_op_done(adap,
A_XGM_RX_MAX_PKT_SIZE_ERR_CNT + mac->offset,
1 << 31, 1, 20, 5)) {
reg = adap->params.rev == T3_REV_B2 ?
A_XGM_RX_MAX_PKT_SIZE_ERR_CNT : A_XGM_RXFIFO_CFG;
/* drain RX FIFO */
if (t3_wait_op_done(adap, reg + mac->offset,
F_RXFIFO_EMPTY, 1, 20, 5)) {
t3_write_reg(adap, A_XGM_RX_CFG + mac->offset, v);
enable_exact_filters(mac);
return -EIO;
}
t3_write_reg(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, mtu);
t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset,
V_RXMAXPKTSIZE(M_RXMAXPKTSIZE),
V_RXMAXPKTSIZE(mtu));
t3_write_reg(adap, A_XGM_RX_CFG + mac->offset, v);
enable_exact_filters(mac);
} else
t3_write_reg(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, mtu);
t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset,
V_RXMAXPKTSIZE(M_RXMAXPKTSIZE),
V_RXMAXPKTSIZE(mtu));
/*
* Adjust the PAUSE frame watermarks. We always set the LWM, and the
* HWM only if flow-control is enabled.
@ -405,20 +453,34 @@ int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu)
thres /= 10;
thres = mtu > thres ? (mtu - thres + 7) / 8 : 0;
thres = max(thres, 8U); /* need at least 8 */
ipg = (adap->params.rev == T3_REV_C) ? 0 : 1;
t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + mac->offset,
V_TXFIFOTHRESH(M_TXFIFOTHRESH) | V_TXIPG(M_TXIPG),
V_TXFIFOTHRESH(thres) | V_TXIPG(1));
V_TXFIFOTHRESH(thres) | V_TXIPG(ipg));
/* Assuming a minimum drain rate of 2.5Gbps...
*/
if (adap->params.rev > 0)
if (adap->params.rev > 0) {
divisor = (adap->params.rev == T3_REV_C) ? 64 : 8;
t3_write_reg(adap, A_XGM_PAUSE_TIMER + mac->offset,
(hwm - lwm) * 4 / 8);
(hwm - lwm) * 4 / divisor);
}
t3_write_reg(adap, A_XGM_TX_PAUSE_QUANTA + mac->offset,
MAC_RXFIFO_SIZE * 4 * 8 / 512);
return 0;
}
/**
* t3_mac_set_speed_duplex_fc - set MAC speed, duplex and flow control
* @mac: the MAC to configure
* @speed: the desired speed (10/100/1000/10000)
* @duplex: the desired duplex
* @fc: desired Tx/Rx PAUSE configuration
*
* Set the MAC speed, duplex (actually only full-duplex is supported), and
* flow control. If a parameter value is negative the corresponding
* MAC setting is left at its current value.
*/
int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc)
{
u32 val;
@ -466,6 +528,15 @@ int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc)
return 0;
}
/**
* t3_mac_enable - enable the MAC in the given directions
* @mac: the MAC to configure
* @which: bitmap indicating which directions to enable
*
* Enables the MAC for operation in the given directions.
* %MAC_DIRECTION_TX enables the Tx direction, and %MAC_DIRECTION_RX
* enables the Rx one.
*/
int t3_mac_enable(struct cmac *mac, int which)
{
int idx = macidx(mac);
@ -478,9 +549,13 @@ int t3_mac_enable(struct cmac *mac, int which)
if (which & MAC_DIRECTION_TX) {
t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx);
t3_write_reg(adap, A_TP_PIO_DATA, 0xc0ede401);
t3_write_reg(adap, A_TP_PIO_DATA,
adap->params.rev == T3_REV_C ?
0xc4ffff01 : 0xc0ede401);
t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_MODE);
t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx, 1 << idx);
t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx,
adap->params.rev == T3_REV_C ?
0 : 1 << idx);
t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN);
@ -505,6 +580,15 @@ int t3_mac_enable(struct cmac *mac, int which)
return 0;
}
/**
* t3_mac_disable - disable the MAC in the given directions
* @mac: the MAC to configure
* @which: bitmap indicating which directions to disable
*
* Disables the MAC in the given directions.
* %MAC_DIRECTION_TX disables the Tx direction, and %MAC_DIRECTION_RX
* disables the Rx one.
*/
int t3_mac_disable(struct cmac *mac, int which)
{
adapter_t *adap = mac->adapter;
@ -621,12 +705,15 @@ int t3b2_mac_watchdog_task(struct cmac *mac)
return status;
}
/*
* This function is called periodically to accumulate the current values of the
* RMON counters into the port statistics. Since the packet counters are only
* 32 bits they can overflow in ~286 secs at 10G, so the function should be
* called more frequently than that. The byte counters are 45-bit wide, they
* would overflow in ~7.8 hours.
/**
* t3_mac_update_stats - accumulate MAC statistics
* @mac: the MAC handle
*
* This function is called periodically to accumulate the current values
* of the RMON counters into the port statistics. Since the packet
* counters are only 32 bits they can overflow in ~286 secs at 10G, so the
* function should be called more frequently than that. The byte counters
* are 45-bit wide, they would overflow in ~7.8 hours.
*/
const struct mac_stats *t3_mac_update_stats(struct cmac *mac)
{

27
sys/dev/cxgb/cxgb_adapter.h
View File

@ -46,6 +46,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_media.h>
#include <net/if_dl.h>
#include <machine/bus.h>
#include <machine/resource.h>
@ -54,6 +55,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#ifdef CONFIG_DEFINED
#include <cxgb_osdep.h>
#include <t3cdev.h>
@ -144,6 +146,9 @@ enum { /* adapter flags */
QUEUES_BOUND = (1 << 3),
FW_UPTODATE = (1 << 4),
TPS_UPTODATE = (1 << 5),
CXGB_SHUTDOWN = (1 << 6),
CXGB_OFLD_INIT = (1 << 7),
TP_PARITY_INIT = (1 << 8),
};
#define FL_Q_SIZE 4096
@ -203,6 +208,7 @@ struct sge_rspq {
uint32_t holdoff_tmr;
uint32_t next_holdoff;
uint32_t imm_data;
uint32_t async_notif;
uint32_t cntxt_id;
uint32_t offload_pkts;
uint32_t offload_bundles;
@ -348,6 +354,8 @@ struct adapter {
/* PCI register resources */
int regs_rid;
struct resource *regs_res;
int udbs_rid;
struct resource *udbs_res;
bus_space_handle_t bh;
bus_space_tag_t bt;
bus_size_t mmio_len;
@ -508,10 +516,23 @@ static __inline uint8_t *
t3_get_next_mcaddr(struct t3_rx_mode *rm)
{
uint8_t *macaddr = NULL;
if (rm->idx == 0)
macaddr = (uint8_t *)rm->port->hw_addr;
struct ifnet *ifp = rm->port->ifp;
struct ifmultiaddr *ifma;
int i = 0;
IF_ADDR_LOCK(ifp);
TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (ifma->ifma_addr->sa_family != AF_LINK)
continue;
if (i == rm->idx) {
macaddr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
break;
}
i++;
}
IF_ADDR_UNLOCK(ifp);
rm->idx++;
return (macaddr);
}

21
sys/dev/cxgb/cxgb_ioctl.h
View File

@ -101,15 +101,16 @@ struct ch_mem_range {
};
struct ch_qset_params {
uint32_t qset_idx;
int32_t txq_size[3];
int32_t rspq_size;
int32_t fl_size[2];
int32_t intr_lat;
int32_t polling;
int32_t cong_thres;
int32_t vector;
int32_t qnum;
uint32_t qset_idx;
int32_t txq_size[3];
int32_t rspq_size;
int32_t fl_size[2];
int32_t intr_lat;
int32_t polling;
int32_t lro;
int32_t cong_thres;
int32_t vector;
int32_t qnum;
};
struct ch_pktsched_params {
@ -260,4 +261,6 @@ struct mii_data {
#define CHELSIO_SET_FILTER _IOW('f', CH_SET_FILTER, struct ch_filter)
#define CHELSIO_DEL_FILTER _IOW('f', CH_DEL_FILTER, struct ch_filter)
#define CHELSIO_DEVUP _IO('f', CH_DEVUP)
#define CHELSIO_GET_TCB _IOWR('f', CH_GET_TCB, struct ch_tcb)
#endif

27
sys/dev/cxgb/cxgb_l2t.c
View File

@ -175,11 +175,8 @@ t3_l2t_send_slow(struct t3cdev *dev, struct mbuf *m, struct l2t_entry *e)
sin.sin_family = AF_INET;
sin.sin_len = sizeof(struct sockaddr_in);
sin.sin_addr.s_addr = e->addr;
printf("send slow on rt=%p eaddr=0x%08x\n", rt, e->addr);
CTR2(KTR_CXGB, "send slow on rt=%p eaddr=0x%08x\n", rt, e->addr);
again:
switch (e->state) {
case L2T_STATE_STALE: /* entry is stale, kick off revalidation */
@ -199,8 +196,6 @@ t3_l2t_send_slow(struct t3cdev *dev, struct mbuf *m, struct l2t_entry *e)
}
arpq_enqueue(e, m);
mtx_unlock(&e->lock);
printf("enqueueing arp request\n");
/*
* Only the first packet added to the arpq should kick off
* resolution. However, because the m_gethdr below can fail,
@ -209,10 +204,9 @@ t3_l2t_send_slow(struct t3cdev *dev, struct mbuf *m, struct l2t_entry *e)
* A better way would be to use a work request to retry L2T
* entries when there's no memory.
*/
printf("doing arpresolve on 0x%x \n", e->addr);
if (arpresolve(rt->rt_ifp, rt, NULL,
(struct sockaddr *)&sin, e->dmac) == 0) {
printf("mac=%x:%x:%x:%x:%x:%x\n",
CTR6(KTR_CXGB, "mac=%x:%x:%x:%x:%x:%x\n",
e->dmac[0], e->dmac[1], e->dmac[2], e->dmac[3], e->dmac[4], e->dmac[5]);
if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
@ -224,8 +218,7 @@ t3_l2t_send_slow(struct t3cdev *dev, struct mbuf *m, struct l2t_entry *e)
else
m_freem(m);
mtx_unlock(&e->lock);
} else
printf("arpresolve returned non-zero\n");
}
}
return 0;
}
@ -396,8 +389,6 @@ t3_l2t_get(struct t3cdev *dev, struct rtentry *neigh, struct ifnet *ifp,
/* Need to allocate a new entry */
e = alloc_l2e(d);
if (e) {
printf("initializing new entry\n");
mtx_lock(&e->lock); /* avoid race with t3_l2t_free */
e->next = d->l2tab[hash].first;
d->l2tab[hash].first = e;
@ -472,8 +463,6 @@ t3_l2t_update(struct t3cdev *dev, struct rtentry *neigh,
int hash = arp_hash(addr, ifidx, d);
struct llinfo_arp *la;
printf("t3_l2t_update called with arp info\n");
rw_rlock(&d->lock);
for (e = d->l2tab[hash].first; e; e = e->next)
if (e->addr == addr && e->ifindex == ifidx) {
@ -481,7 +470,7 @@ t3_l2t_update(struct t3cdev *dev, struct rtentry *neigh,
goto found;
}
rw_runlock(&d->lock);
printf("addr=0x%08x not found\n", addr);
CTR1(KTR_CXGB, "t3_l2t_update: addr=0x%08x not found", addr);
return;
found:
@ -543,6 +532,12 @@ t3_init_l2t(unsigned int l2t_capacity)
void
t3_free_l2t(struct l2t_data *d)
{
int i;
rw_destroy(&d->lock);
for (i = 0; i < d->nentries; ++i)
mtx_destroy(&d->l2tab[i].lock);
cxgb_free_mem(d);
}

2
sys/dev/cxgb/cxgb_l2t.h
View File

@ -143,8 +143,6 @@ static inline int l2t_send(struct t3cdev *dev, struct mbuf *m,
if (__predict_true(e->state == L2T_STATE_VALID)) {
return cxgb_ofld_send(dev, (struct mbuf *)m);
}
printf("send slow\n");
return t3_l2t_send_slow(dev, (struct mbuf *)m, e);
}

318
sys/dev/cxgb/cxgb_main.c
View File

@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus_dma.h>
#include <sys/ktr.h>
#include <sys/rman.h>
#include <sys/ioccom.h>
#include <sys/mbuf.h>
@ -119,6 +120,7 @@ static int cxgb_get_regs_len(void);
static int offload_open(struct port_info *pi);
static void touch_bars(device_t dev);
static int offload_close(struct t3cdev *tdev);
static void cxgb_link_start(struct port_info *p);
static device_method_t cxgb_controller_methods[] = {
DEVMETHOD(device_probe, cxgb_controller_probe),
@ -281,6 +283,32 @@ struct cxgb_ident {
static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
void
cxgb_log_tcb(struct adapter *sc, unsigned int tid)
{
char buf[TCB_SIZE];
uint64_t *tcb = (uint64_t *)buf;
int i, error;
struct mc7 *mem = &sc->cm;
error = t3_mc7_bd_read(mem, tid*TCB_SIZE/8, TCB_SIZE/8, tcb);
if (error)
printf("cxgb_tcb_log failed\n");
CTR1(KTR_CXGB, "TCB tid=%u", tid);
for (i = 0; i < TCB_SIZE / 32; i++) {
CTR5(KTR_CXGB, "%1d: %08x %08x %08x %08x",
i, (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
(uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
tcb += 2;
CTR4(KTR_CXGB, " %08x %08x %08x %08x",
(uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
(uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
tcb += 2;
}
}
static __inline char
t3rev2char(struct adapter *adapter)
{
@ -397,7 +425,8 @@ cxgb_controller_attach(device_t dev)
int port_qsets = 1;
#ifdef MSI_SUPPORTED
int msi_needed, reg;
#endif
#endif
int must_load = 0;
sc = device_get_softc(dev);
sc->dev = dev;
sc->msi_count = 0;
@ -434,9 +463,16 @@ cxgb_controller_attach(device_t dev)
sc->regs_rid = PCIR_BAR(0);
if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
&sc->regs_rid, RF_ACTIVE)) == NULL) {
device_printf(dev, "Cannot allocate BAR\n");
device_printf(dev, "Cannot allocate BAR region 0\n");
return (ENXIO);
}
sc->udbs_rid = PCIR_BAR(2);
if ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
&sc->udbs_rid, RF_ACTIVE)) == NULL) {
device_printf(dev, "Cannot allocate BAR region 1\n");
error = ENXIO;
goto out;
}
snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
device_get_unit(dev));
@ -449,7 +485,7 @@ cxgb_controller_attach(device_t dev)
snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
device_get_unit(dev));
MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_DEF);
MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
@ -534,7 +570,7 @@ cxgb_controller_attach(device_t dev)
/* Create a periodic callout for checking adapter status */
callout_init(&sc->cxgb_tick_ch, TRUE);
if (t3_check_fw_version(sc) != 0) {
if (t3_check_fw_version(sc, &must_load) != 0 && must_load) {
/*
* Warn user that a firmware update will be attempted in init.
*/
@ -545,7 +581,7 @@ cxgb_controller_attach(device_t dev)
sc->flags |= FW_UPTODATE;
}
if (t3_check_tpsram_version(sc) != 0) {
if (t3_check_tpsram_version(sc, &must_load) != 0 && must_load) {
/*
* Warn user that a firmware update will be attempted in init.
*/
@ -609,6 +645,8 @@ cxgb_controller_attach(device_t dev)
G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
G_FW_VERSION_MICRO(vers));
device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
t3_add_attach_sysctls(sc);
out:
if (error)
@ -634,9 +672,12 @@ cxgb_free(struct adapter *sc)
{
int i;
ADAPTER_LOCK(sc);
sc->flags |= CXGB_SHUTDOWN;
ADAPTER_UNLOCK(sc);
cxgb_pcpu_shutdown_threads(sc);
ADAPTER_LOCK(sc);
/*
* drops the lock
*/
@ -654,11 +695,7 @@ cxgb_free(struct adapter *sc)
bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
sc->msix_regs_res);
}
if (sc->tq != NULL) {
taskqueue_drain(sc->tq, &sc->ext_intr_task);
taskqueue_drain(sc->tq, &sc->tick_task);
}
t3_sge_deinit_sw(sc);
/*
* Wait for last callout
@ -672,8 +709,11 @@ cxgb_free(struct adapter *sc)
}
bus_generic_detach(sc->dev);
if (sc->tq != NULL)
if (sc->tq != NULL) {
taskqueue_free(sc->tq);
sc->tq = NULL;
}
if (is_offload(sc)) {
cxgb_adapter_unofld(sc);
if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT))
@ -682,11 +722,18 @@ cxgb_free(struct adapter *sc)
printf("cxgb_free: DEVMAP_BIT not set\n");
} else
printf("not offloading set\n");
if (sc->flags & CXGB_OFLD_INIT)
cxgb_offload_deactivate(sc);
free(sc->filters, M_DEVBUF);
t3_sge_free(sc);
cxgb_offload_exit();
if (sc->udbs_res != NULL)
bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
sc->udbs_res);
if (sc->regs_res != NULL)
bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
sc->regs_res);
@ -797,8 +844,6 @@ cxgb_setup_msix(adapter_t *sc, int msix_count)
return (EINVAL);
}
sc->msix_irq_rid[k] = rid;
printf("setting up interrupt for port=%d\n",
qs->port->port_id);
if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
INTR_MPSAFE|INTR_TYPE_NET,
#ifdef INTR_FILTERS
@ -828,10 +873,11 @@ cxgb_port_probe(device_t dev)
{
struct port_info *p;
char buf[80];
const char *desc;
p = device_get_softc(dev);
snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, p->port_type->desc);
desc = p->phy.desc;
snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
device_set_desc_copy(dev, buf);
return (0);
}
@ -873,9 +919,11 @@ cxgb_port_attach(device_t dev)
struct port_info *p;
struct ifnet *ifp;
int err, media_flags;
struct adapter *sc;
p = device_get_softc(dev);
sc = p->adapter;
snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
device_get_unit(device_get_parent(dev)), p->port_id);
PORT_LOCK_INIT(p, p->lockbuf);
@ -897,11 +945,12 @@ cxgb_port_attach(device_t dev)
ifp->if_ioctl = cxgb_ioctl;
ifp->if_start = cxgb_start;
#if 0
#ifdef IFNET_MULTIQUEUE
ifp->if_flags |= IFF_MULTIQ;
ifp->if_mq_start = cxgb_pcpu_start;
#endif
#endif
ifp->if_timer = 0; /* Disable ifnet watchdog */
ifp->if_watchdog = NULL;
@ -934,14 +983,14 @@ cxgb_port_attach(device_t dev)
}
ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
cxgb_media_status);
if (!strcmp(p->port_type->desc, "10GBASE-CX4")) {
if (!strcmp(p->phy.desc, "10GBASE-CX4")) {
media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
} else if (!strcmp(p->port_type->desc, "10GBASE-SR")) {
} else if (!strcmp(p->phy.desc, "10GBASE-SR")) {
media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
} else if (!strcmp(p->port_type->desc, "10GBASE-XR")) {
} else if (!strcmp(p->phy.desc, "10GBASE-XR")) {
media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
} else if (!strcmp(p->port_type->desc, "10/100/1000BASE-T")) {
} else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) {
ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
0, NULL);
@ -953,7 +1002,7 @@ cxgb_port_attach(device_t dev)
0, NULL);
media_flags = 0;
} else {
printf("unsupported media type %s\n", p->port_type->desc);
printf("unsupported media type %s\n", p->phy.desc);
return (ENXIO);
}
if (media_flags) {
@ -976,7 +1025,8 @@ cxgb_port_attach(device_t dev)
taskqueue_thread_enqueue, &p->tq);
#endif
t3_sge_init_port(p);
cxgb_link_start(p);
t3_link_changed(sc, p->port_id);
return (0);
}
@ -1119,17 +1169,14 @@ t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
struct port_info *pi = &adapter->port[port_id];
struct cmac *mac = &adapter->port[port_id].mac;
if ((pi->ifp->if_flags & IFF_UP) == 0)
return;
if (link_status) {
t3_mac_enable(mac, MAC_DIRECTION_RX);
if_link_state_change(pi->ifp, LINK_STATE_UP);
} else {
if_link_state_change(pi->ifp, LINK_STATE_DOWN);
pi->phy.ops->power_down(&pi->phy, 1);
t3_mac_disable(mac, MAC_DIRECTION_RX);
t3_link_start(&pi->phy, mac, &pi->link_config);
if_link_state_change(pi->ifp, LINK_STATE_DOWN);
}
}
@ -1195,6 +1242,84 @@ cxgb_link_start(struct port_info *p)
t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
}
static int
await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
unsigned long n)
{
int attempts = 5;
while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
if (!--attempts)
return (ETIMEDOUT);
t3_os_sleep(10);
}
return 0;
}
static int
init_tp_parity(struct adapter *adap)
{
int i;
struct mbuf *m;
struct cpl_set_tcb_field *greq;
unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
t3_tp_set_offload_mode(adap, 1);
for (i = 0; i < 16; i++) {
struct cpl_smt_write_req *req;
m = m_gethdr(M_WAITOK, MT_DATA);
req = mtod(m, struct cpl_smt_write_req *);
m->m_len = m->m_pkthdr.len = sizeof(*req);
memset(req, 0, sizeof(*req));
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
req->iff = i;
t3_mgmt_tx(adap, m);
}
for (i = 0; i < 2048; i++) {
struct cpl_l2t_write_req *req;
m = m_gethdr(M_WAITOK, MT_DATA);
req = mtod(m, struct cpl_l2t_write_req *);
m->m_len = m->m_pkthdr.len = sizeof(*req);
memset(req, 0, sizeof(*req));
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
req->params = htonl(V_L2T_W_IDX(i));
t3_mgmt_tx(adap, m);
}
for (i = 0; i < 2048; i++) {
struct cpl_rte_write_req *req;
m = m_gethdr(M_WAITOK, MT_DATA);
req = mtod(m, struct cpl_rte_write_req *);
m->m_len = m->m_pkthdr.len = sizeof(*req);
memset(req, 0, sizeof(*req));
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
req->l2t_idx = htonl(V_L2T_W_IDX(i));
t3_mgmt_tx(adap, m);
}
m = m_gethdr(M_WAITOK, MT_DATA);
greq = mtod(m, struct cpl_set_tcb_field *);
m->m_len = m->m_pkthdr.len = sizeof(*greq);
memset(greq, 0, sizeof(*greq));
greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
greq->mask = htobe64(1);
t3_mgmt_tx(adap, m);
i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
t3_tp_set_offload_mode(adap, 0);
return (i);
}
/**
* setup_rss - configure Receive Side Steering (per-queue connection demux)
* @adap: the adapter
@ -1224,11 +1349,9 @@ setup_rss(adapter_t *adap)
nq[pi->tx_chan] += pi->nqsets;
}
nq[0] = max(nq[0], 1U);
nq[1] = max(nq[1], 1U);
for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
rspq_map[i] = i % nq[0];
rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq[1]) + nq[0];
rspq_map[i] = nq[0] ? i % nq[0] : 0;
rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
}
/* Calculate the reverse RSS map table */
for (i = 0; i < RSS_TABLE_SIZE; ++i)
@ -1237,7 +1360,8 @@ setup_rss(adapter_t *adap)
t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
V_RRCPLCPUSIZE(6), cpus, rspq_map);
F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
cpus, rspq_map);
}
@ -1470,6 +1594,7 @@ cxgb_up(struct adapter *sc)
if (err)
goto out;
t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
err = setup_sge_qsets(sc);
@ -1510,8 +1635,18 @@ cxgb_up(struct adapter *sc)
t3_sge_start(sc);
t3_intr_enable(sc);
if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
is_offload(sc) && init_tp_parity(sc) == 0)
sc->flags |= TP_PARITY_INIT;
if (sc->flags & TP_PARITY_INIT) {
t3_write_reg(sc, A_TP_INT_CAUSE,
F_CMCACHEPERR | F_ARPLUTPERR);
t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
}
if (!(sc->flags & QUEUES_BOUND)) {
printf("bind qsets\n");
bind_qsets(sc);
sc->flags |= QUEUES_BOUND;
}
@ -1529,7 +1664,6 @@ cxgb_up(struct adapter *sc)
static void
cxgb_down_locked(struct adapter *sc)
{
int i;
t3_sge_stop(sc);
t3_intr_disable(sc);
@ -1546,20 +1680,24 @@ cxgb_down_locked(struct adapter *sc)
sc->irq_res = NULL;
}
if (sc->flags & USING_MSIX)
if (sc->flags & USING_MSIX)
cxgb_teardown_msix(sc);
ADAPTER_UNLOCK(sc);
callout_stop(&sc->cxgb_tick_ch);
callout_stop(&sc->sge_timer_ch);
callout_drain(&sc->cxgb_tick_ch);
callout_drain(&sc->sge_timer_ch);
if (sc->tq != NULL) {
printf("draining slow intr\n");
taskqueue_drain(sc->tq, &sc->slow_intr_task);
for (i = 0; i < sc->params.nports; i++)
taskqueue_drain(sc->tq, &sc->port[i].timer_reclaim_task);
printf("draining ext intr\n");
taskqueue_drain(sc->tq, &sc->ext_intr_task);
printf("draining tick task\n");
taskqueue_drain(sc->tq, &sc->tick_task);
}
ADAPTER_UNLOCK(sc);
}
static int
@ -1573,7 +1711,7 @@ offload_open(struct port_info *pi)
int adap_up = adapter->open_device_map & PORT_MASK;
int err = 0;
printf("device_map=0x%x\n", adapter->open_device_map);
CTR1(KTR_CXGB, "device_map=0x%x", adapter->open_device_map);
if (atomic_cmpset_int(&adapter->open_device_map,
(adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)),
(adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0)
@ -1620,11 +1758,8 @@ offload_close(struct t3cdev *tdev)
{
struct adapter *adapter = tdev2adap(tdev);
if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) {
printf("offload_close: DEVMAP_BIT not set\n");
if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
return (0);
}
/* Call back all registered clients */
cxgb_remove_clients(tdev);
@ -1638,7 +1773,6 @@ offload_close(struct t3cdev *tdev)
cxgb_down_locked(adapter);
else
ADAPTER_UNLOCK(adapter);
cxgb_offload_deactivate(adapter);
return (0);
}
@ -1680,17 +1814,12 @@ cxgb_init_locked(struct port_info *p)
if (err)
log(LOG_WARNING,
"Could not initialize offload capabilities\n");
else
printf("offload opened\n");
}
cxgb_link_start(p);
t3_link_changed(sc, p->port_id);
ifp->if_baudrate = p->link_config.speed * 1000000;
device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id);
t3_port_intr_enable(sc, p->port_id);
callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
t3_sge_reset_adapter(sc);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
@ -1703,10 +1832,10 @@ cxgb_set_rxmode(struct port_info *p)
struct t3_rx_mode rm;
struct cmac *mac = &p->mac;
PORT_LOCK_ASSERT_OWNED(p);
t3_init_rx_mode(&rm, p);
mtx_lock(&p->adapter->mdio_lock);
t3_mac_set_rx_mode(mac, &rm);
mtx_unlock(&p->adapter->mdio_lock);
}
static void
@ -1745,7 +1874,6 @@ cxgb_set_mtu(struct port_info *p, int mtu)
PORT_LOCK(p);
ifp->if_mtu = mtu;
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
callout_stop(&p->adapter->cxgb_tick_ch);
cxgb_stop_locked(p);
cxgb_init_locked(p);
}
@ -1771,19 +1899,18 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
error = cxgb_set_mtu(p, ifr->ifr_mtu);
break;
case SIOCSIFADDR:
case SIOCGIFADDR:
if (ifa->ifa_addr->sa_family == AF_INET) {
PORT_LOCK(p);
ifp->if_flags |= IFF_UP;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
PORT_LOCK(p);
cxgb_init_locked(p);
PORT_UNLOCK(p);
}
arp_ifinit(ifp, ifa);
PORT_UNLOCK(p);
} else
error = ether_ioctl(ifp, command, data);
break;
case SIOCSIFFLAGS:
callout_drain(&p->adapter->cxgb_tick_ch);
PORT_LOCK(p);
if (ifp->if_flags & IFF_UP) {
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
@ -1797,13 +1924,14 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
cxgb_stop_locked(p);
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
adapter_t *sc = p->adapter;
callout_reset(&sc->cxgb_tick_ch, hz,
cxgb_tick, sc);
}
PORT_UNLOCK(p);
break;
case SIOCADDMULTI:
case SIOCDELMULTI:
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
cxgb_set_rxmode(p);
}
break;
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
error = ifmedia_ioctl(ifp, ifr, &p->media, command);
@ -1929,7 +2057,7 @@ check_link_status(adapter_t *sc)
for (i = 0; i < (sc)->params.nports; ++i) {
struct port_info *p = &sc->port[i];
if (!(p->port_type->caps & SUPPORTED_IRQ))
if (!(p->phy.caps & SUPPORTED_IRQ))
t3_link_changed(sc, i);
p->ifp->if_baudrate = p->link_config.speed * 1000000;
}
@ -1940,11 +2068,17 @@ check_t3b2_mac(struct adapter *adapter)
{
int i;
if(adapter->flags & CXGB_SHUTDOWN)
return;
for_each_port(adapter, i) {
struct port_info *p = &adapter->port[i];
struct ifnet *ifp = p->ifp;
int status;
if(adapter->flags & CXGB_SHUTDOWN)
return;
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
continue;
@ -1974,26 +2108,12 @@ static void
cxgb_tick(void *arg)
{
adapter_t *sc = (adapter_t *)arg;
int i, running = 0;
for_each_port(sc, i) {
struct port_info *p = &sc->port[i];
struct ifnet *ifp = p->ifp;
PORT_LOCK(p);
if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
running = 1;
PORT_UNLOCK(p);
}
if (running == 0)
if(sc->flags & CXGB_SHUTDOWN)
return;
taskqueue_enqueue(sc->tq, &sc->tick_task);
if (sc->open_device_map != 0)
callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
}
static void
@ -2002,17 +2122,20 @@ cxgb_tick_handler(void *arg, int count)
adapter_t *sc = (adapter_t *)arg;
const struct adapter_params *p = &sc->params;
if(sc->flags & CXGB_SHUTDOWN)
return;
ADAPTER_LOCK(sc);
if (p->linkpoll_period)
check_link_status(sc);
/*
* adapter lock can currently only be acquire after the
* adapter lock can currently only be acquired after the
* port lock
*/
ADAPTER_UNLOCK(sc);
if (p->rev == T3_REV_B2 && p->nports < 4)
if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
check_t3b2_mac(sc);
}
@ -2180,7 +2303,7 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
}
case CHELSIO_GET_SGE_CONTEXT: {
struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
mtx_lock(&sc->sge.reg_lock);
mtx_lock_spin(&sc->sge.reg_lock);
switch (ecntxt->cntxt_type) {
case CNTXT_TYPE_EGRESS:
error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
@ -2202,7 +2325,7 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
error = EINVAL;
break;
}
mtx_unlock(&sc->sge.reg_lock);
mtx_unlock_spin(&sc->sge.reg_lock);
break;
}
case CHELSIO_GET_SGE_DESC: {
@ -2220,7 +2343,8 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
case CHELSIO_SET_QSET_PARAMS: {
struct qset_params *q;
struct ch_qset_params *t = (struct ch_qset_params *)data;
int i;
if (t->qset_idx >= SGE_QSETS)
return (EINVAL);
if (!in_range(t->intr_lat, 0, M_NEWTIMER) ||
@ -2236,6 +2360,18 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
MAX_RX_JUMBO_BUFFERS) ||
!in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES))
return (EINVAL);
if ((sc->flags & FULL_INIT_DONE) && t->lro > 0)
for_each_port(sc, i) {
pi = adap2pinfo(sc, i);
if (t->qset_idx >= pi->first_qset &&
t->qset_idx < pi->first_qset + pi->nqsets
#if 0
&& !pi->rx_csum_offload
#endif
)
return -EINVAL;
}
if ((sc->flags & FULL_INIT_DONE) &&
(t->rspq_size >= 0 || t->fl_size[0] >= 0 ||
t->fl_size[1] >= 0 || t->txq_size[0] >= 0 ||

13
sys/dev/cxgb/cxgb_multiq.c
View File

@ -422,13 +422,16 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush)
txq = &qs->txq[TXQ_ETH];
mtx_assert(&txq->lock, MA_OWNED);
KASSERT(qs->idx == 0, ("invalid qs %d", qs->idx));
retry:
if (!pi->link_config.link_ok)
initerr = ENXIO;
else if (qs->qs_flags & QS_EXITING)
initerr = ENXIO;
else if ((pi->ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
initerr = ENXIO;
else if ((pi->ifp->if_flags & IFF_UP) == 0)
initerr = ENXIO;
else if (immpkt) {
if (!buf_ring_empty(&txq->txq_mr))
@ -690,15 +693,15 @@ cxgb_pcpu_shutdown_threads(struct adapter *sc)
int i, j;
int nqsets;
for (i = 0; i < sc->params.nports; i++) {
struct port_info *pi = &sc->port[i];
int first = pi->first_qset;
#ifdef IFNET_MULTIQUEUE
nqsets = pi->nqsets;
#else
nqsets = 1;
#endif
for (i = 0; i < sc->params.nports; i++) {
struct port_info *pi = &sc->port[i];
int first = pi->first_qset;
for (j = 0; j < nqsets; j++) {
struct sge_qset *qs = &sc->sge.qs[first + j];

135
sys/dev/cxgb/cxgb_offload.c
View File

@ -105,16 +105,12 @@ cxgb_register_client(struct cxgb_client *client)
TAILQ_INSERT_TAIL(&client_list, client, client_entry);
if (client->add) {
printf("client->add set\n");
TAILQ_FOREACH(tdev, &ofld_dev_list, entry) {
if (offload_activated(tdev)) {
printf("calling add=%p on %p\n",
client->add, tdev);
client->add(tdev);
} else
printf("%p not activated\n", tdev);
CTR1(KTR_CXGB,
"cxgb_register_client: %p not activated", tdev);
}
}
@ -270,11 +266,10 @@ cxgb_ulp_iscsi_ctl(adapter_t *adapter, unsigned int req, void *data)
t3_read_reg(adapter, A_PM1_TX_CFG) >> 17);
/* on rx, the iscsi pdu has to be < rx page size and the
whole pdu + cpl headers has to fit into one sge buffer */
uiip->max_rxsz =
(unsigned int)min(adapter->params.tp.rx_pg_size,
(adapter->sge.qs[0].fl[1].buf_size -
sizeof(struct cpl_rx_data) * 2 -
sizeof(struct cpl_rx_data_ddp)) );
/* also check the max rx data length programmed in TP */
uiip->max_rxsz = min(uiip->max_rxsz,
((t3_read_reg(adapter, A_TP_PARA_REG2))
>> S_MAXRXDATA) & M_MAXRXDATA);
break;
case ULP_ISCSI_SET_PARAMS:
t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask);
@ -297,25 +292,24 @@ cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data)
case RDMA_GET_PARAMS: {
struct rdma_info *req = data;
req->udbell_physbase = rman_get_start(adapter->regs_res);
req->udbell_len = rman_get_size(adapter->regs_res);
req->udbell_physbase = rman_get_start(adapter->udbs_res);
req->udbell_len = rman_get_size(adapter->udbs_res);
req->tpt_base = t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT);
req->tpt_top = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT);
req->pbl_base = t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT);
req->pbl_top = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT);
req->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT);
req->rqt_top = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT);
req->kdb_addr = (void *)(rman_get_start(adapter->regs_res) + A_SG_KDOORBELL);
break;
req->kdb_addr = (void *)((unsigned long)rman_get_virtual(adapter->regs_res) + A_SG_KDOORBELL); break;
}
case RDMA_CQ_OP: {
struct rdma_cq_op *req = data;
/* may be called in any context */
mtx_lock(&adapter->sge.reg_lock);
mtx_lock_spin(&adapter->sge.reg_lock);
ret = t3_sge_cqcntxt_op(adapter, req->id, req->op,
req->credits);
mtx_unlock(&adapter->sge.reg_lock);
mtx_unlock_spin(&adapter->sge.reg_lock);
break;
}
case RDMA_GET_MEM: {
@ -341,28 +335,28 @@ cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data)
case RDMA_CQ_SETUP: {
struct rdma_cq_setup *req = data;
mtx_lock(&adapter->sge.reg_lock);
mtx_lock_spin(&adapter->sge.reg_lock);
ret = t3_sge_init_cqcntxt(adapter, req->id, req->base_addr,
req->size, ASYNC_NOTIF_RSPQ,
req->ovfl_mode, req->credits,
req->credit_thres);
mtx_unlock(&adapter->sge.reg_lock);
mtx_unlock_spin(&adapter->sge.reg_lock);
break;
}
case RDMA_CQ_DISABLE:
mtx_lock(&adapter->sge.reg_lock);
mtx_lock_spin(&adapter->sge.reg_lock);
ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data);
mtx_unlock(&adapter->sge.reg_lock);
mtx_unlock_spin(&adapter->sge.reg_lock);
break;
case RDMA_CTRL_QP_SETUP: {
struct rdma_ctrlqp_setup *req = data;
mtx_lock(&adapter->sge.reg_lock);
mtx_lock_spin(&adapter->sge.reg_lock);
ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0,
SGE_CNTXT_RDMA, ASYNC_NOTIF_RSPQ,
req->base_addr, req->size,
FW_RI_TID_START, 1, 0);
mtx_unlock(&adapter->sge.reg_lock);
mtx_unlock_spin(&adapter->sge.reg_lock);
break;
}
default:
@ -380,6 +374,8 @@ cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data)
struct iff_mac *iffmacp;
struct ddp_params *ddpp;
struct adap_ports *ports;
struct ofld_page_info *rx_page_info;
struct tp_params *tp = &adapter->params.tp;
int port;
switch (req) {
@ -444,6 +440,11 @@ cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data)
case FAILOVER_CLEAR:
t3_failover_clear(adapter);
break;
case GET_RX_PAGE_INFO:
rx_page_info = data;
rx_page_info->page_size = tp->rx_pg_size;
rx_page_info->num = tp->rx_num_pgs;
break;
case ULP_ISCSI_GET_PARAMS:
case ULP_ISCSI_SET_PARAMS:
if (!offload_running(adapter))
@ -472,8 +473,6 @@ cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data)
static int
rx_offload_blackhole(struct t3cdev *dev, struct mbuf **m, int n)
{
CH_ERR(tdev2adap(dev), "%d unexpected offload packets, first data 0x%x\n",
n, *mtod(m[0], uint32_t *));
while (n--)
m_freem(m[n]);
return 0;
@ -629,7 +628,7 @@ cxgb_remove_tid(struct t3cdev *tdev, void *ctx, unsigned int tid)
m = m_get(M_NOWAIT, MT_DATA);
if (__predict_true(m != NULL)) {
mk_tid_release(m, tid);
printf("sending tid release\n");
CTR1(KTR_CXGB, "releasing tid=%u", tid);
cxgb_ofld_send(tdev, m);
t->tid_tab[tid].ctx = NULL;
@ -707,6 +706,19 @@ do_l2t_write_rpl(struct t3cdev *dev, struct mbuf *m)
return CPL_RET_BUF_DONE;
}
static int
do_rte_write_rpl(struct t3cdev *dev, struct mbuf *m)
{
struct cpl_rte_write_rpl *rpl = cplhdr(m);
if (rpl->status != CPL_ERR_NONE)
log(LOG_ERR,
"Unexpected L2T_WRITE_RPL status %u for entry %u\n",
rpl->status, GET_TID(rpl));
return CPL_RET_BUF_DONE;
}
static int
do_act_open_rpl(struct t3cdev *dev, struct mbuf *m)
{
@ -903,7 +915,7 @@ cxgb_arp_update_event(void *unused, struct rtentry *rt0,
uint8_t *enaddr, struct sockaddr *sa)
{
if (TOEDEV(rt0->rt_ifp) == NULL)
if (!is_offloading(rt0->rt_ifp))
return;
RT_ADDREF(rt0);
@ -918,15 +930,21 @@ static void
cxgb_redirect_event(void *unused, int event, struct rtentry *rt0,
struct rtentry *rt1, struct sockaddr *sa)
{
struct toedev *tdev0, *tdev1;
/*
* ignore events on non-offloaded interfaces
*/
tdev0 = TOEDEV(rt0->rt_ifp);
tdev1 = TOEDEV(rt1->rt_ifp);
if (tdev0 == NULL && tdev1 == NULL)
if (!is_offloading(rt0->rt_ifp))
return;
/*
* Cannot redirect to non-offload device.
*/
if (!is_offloading(rt1->rt_ifp)) {
log(LOG_WARNING, "%s: Redirect to non-offload"
"device ignored.\n", __FUNCTION__);
return;
}
/*
* avoid LORs by dropping the route lock but keeping a reference
*
@ -952,14 +970,15 @@ static int
do_bad_cpl(struct t3cdev *dev, struct mbuf *m)
{
log(LOG_ERR, "%s: received bad CPL command 0x%x\n", dev->name,
*mtod(m, uint32_t *));
0xFF & *mtod(m, uint32_t *));
kdb_backtrace();
return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
}
/*
* Handlers for each CPL opcode
*/
static cpl_handler_func cpl_handlers[NUM_CPL_CMDS];
static cpl_handler_func cpl_handlers[256];
/*
* Add a new handler to the CPL dispatch table. A NULL handler may be supplied
@ -1052,7 +1071,7 @@ void
cxgb_neigh_update(struct rtentry *rt, uint8_t *enaddr, struct sockaddr *sa)
{
if (is_offloading(rt->rt_ifp)) {
if (rt->rt_ifp && is_offloading(rt->rt_ifp) && (rt->rt_ifp->if_flags & IFCAP_TOE)) {
struct t3cdev *tdev = T3CDEV(rt->rt_ifp);
PANIC_IF(!tdev);
@ -1159,7 +1178,6 @@ cxgb_free_mem(void *addr)
free(addr, M_CXGB);
}
/*
* Allocate and initialize the TID tables. Returns 0 on success.
*/
@ -1208,6 +1226,8 @@ init_tid_tabs(struct tid_info *t, unsigned int ntids,
static void
free_tid_maps(struct tid_info *t)
{
mtx_destroy(&t->stid_lock);
mtx_destroy(&t->atid_lock);
cxgb_free_mem(t->tid_tab);
}
@ -1227,11 +1247,6 @@ remove_adapter(adapter_t *adap)
rw_wunlock(&adapter_list_lock);
}
/*
* XXX
*/
#define t3_free_l2t(...)
int
cxgb_offload_activate(struct adapter *adapter)
{
@ -1265,8 +1280,6 @@ cxgb_offload_activate(struct adapter *adapter)
device_printf(adapter->dev, "%s: t3_init_l2t failed\n", __FUNCTION__);
goto out_free;
}
natids = min(tid_range.num / 2, MAX_ATIDS);
err = init_tid_tabs(&t->tid_maps, tid_range.num, natids,
stid_range.num, ATID_BASE, stid_range.base);
@ -1295,9 +1308,10 @@ cxgb_offload_activate(struct adapter *adapter)
log(LOG_ERR, "Unable to set offload capabilities\n");
#endif
}
printf("adding adapter %p\n", adapter);
CTR1(KTR_CXGB, "adding adapter %p", adapter);
add_adapter(adapter);
device_printf(adapter->dev, "offload started\n");
adapter->flags |= CXGB_OFLD_INIT;
#if 0
printf("failing as test\n");
return (ENOMEM);
@ -1330,6 +1344,7 @@ cxgb_offload_deactivate(struct adapter *adapter)
T3C_DATA(tdev) = NULL;
t3_free_l2t(L2DATA(tdev));
L2DATA(tdev) = NULL;
mtx_destroy(&t->tid_release_lock);
free(t, M_CXGB);
}
@ -1353,6 +1368,26 @@ unregister_tdev(struct t3cdev *tdev)
mtx_unlock(&cxgb_db_lock);
}
static __inline int
adap2type(struct adapter *adapter)
{
int type = 0;
switch (adapter->params.rev) {
case T3_REV_A:
type = T3A;
break;
case T3_REV_B:
case T3_REV_B2:
type = T3B;
break;
case T3_REV_C:
type = T3C;
break;
}
return type;
}
void
cxgb_adapter_ofld(struct adapter *adapter)
{
@ -1361,9 +1396,8 @@ cxgb_adapter_ofld(struct adapter *adapter)
cxgb_set_dummy_ops(tdev);
tdev->send = t3_offload_tx;
tdev->ctl = cxgb_offload_ctl;
tdev->type = adapter->params.rev == 0 ?
T3A : T3B;
tdev->type = adap2type(adapter);
register_tdev(tdev);
#if 0
offload_proc_dev_init(tdev);
@ -1398,10 +1432,11 @@ cxgb_offload_init(void)
TAILQ_INIT(&ofld_dev_list);
TAILQ_INIT(&adapter_list);
for (i = 0; i < NUM_CPL_CMDS; ++i)
for (i = 0; i < 0x100; ++i)
cpl_handlers[i] = do_bad_cpl;
t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl);
t3_register_cpl_handler(CPL_RTE_WRITE_RPL, do_rte_write_rpl);
t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl);
t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl);
@ -1425,7 +1460,9 @@ cxgb_offload_init(void)
t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl);
t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl);
t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl);
t3_register_cpl_handler(CPL_GET_TCB_RPL, do_hwtid_rpl);
t3_register_cpl_handler(CPL_SET_TCB_RPL, do_hwtid_rpl);
EVENTHANDLER_REGISTER(route_arp_update_event, cxgb_arp_update_event,
NULL, EVENTHANDLER_PRI_ANY);
EVENTHANDLER_REGISTER(route_redirect_event, cxgb_redirect_event,

47
sys/dev/cxgb/cxgb_osdep.h
View File

@ -83,6 +83,9 @@ struct t3_mbuf_hdr {
#define m_set_socket(m, a) ((m)->m_pkthdr.header = (a))
#define m_get_socket(m) ((m)->m_pkthdr.header)
#define KTR_CXGB KTR_SPARE2
void cxgb_log_tcb(struct adapter *sc, unsigned int tid);
#define MT_DONTFREE 128
#if __FreeBSD_version > 700030
@ -338,13 +341,14 @@ static const int debug_flags = DBG_RX;
#define DBG(...)
#endif
#include <sys/syslog.h>
#define promisc_rx_mode(rm) ((rm)->port->ifp->if_flags & IFF_PROMISC)
#define allmulti_rx_mode(rm) ((rm)->port->ifp->if_flags & IFF_ALLMULTI)
#define CH_ERR(adap, fmt, ...)device_printf(adap->dev, fmt, ##__VA_ARGS__);
#define CH_WARN(adap, fmt, ...) device_printf(adap->dev, fmt, ##__VA_ARGS__)
#define CH_ALERT(adap, fmt, ...) device_printf(adap->dev, fmt, ##__VA_ARGS__)
#define CH_ERR(adap, fmt, ...) log(LOG_ERR, fmt, ##__VA_ARGS__)
#define CH_WARN(adap, fmt, ...) log(LOG_WARNING, fmt, ##__VA_ARGS__)
#define CH_ALERT(adap, fmt, ...) log(LOG_ALERT, fmt, ##__VA_ARGS__)
#define t3_os_sleep(x) DELAY((x) * 1000)
@ -370,7 +374,8 @@ static const int debug_flags = DBG_RX;
#define MII_CTRL1000 MII_100T2CR
#define ADVERTISE_PAUSE_CAP ANAR_FC
#define ADVERTISE_PAUSE_ASYM 0x0800
#define ADVERTISE_PAUSE_ASYM ANAR_X_PAUSE_ASYM
#define ADVERTISE_PAUSE ANAR_X_PAUSE_SYM
#define ADVERTISE_1000HALF ANAR_X_HD
#define ADVERTISE_1000FULL ANAR_X_FD
#define ADVERTISE_10FULL ANAR_10_FD
@ -378,6 +383,13 @@ static const int debug_flags = DBG_RX;
#define ADVERTISE_100FULL ANAR_TX_FD
#define ADVERTISE_100HALF ANAR_TX
#define ADVERTISE_1000XHALF ANAR_X_HD
#define ADVERTISE_1000XFULL ANAR_X_FD
#define ADVERTISE_1000XPSE_ASYM ANAR_X_PAUSE_ASYM
#define ADVERTISE_1000XPAUSE ANAR_X_PAUSE_SYM
/* Standard PCI Extended Capaibilities definitions */
#define PCI_CAP_ID_VPD 0x03
#define PCI_VPD_ADDR 2
@ -399,23 +411,26 @@ static const int debug_flags = DBG_RX;
#define udelay(x) DELAY(x)
#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
#define le32_to_cpu(x) le32toh(x)
#define le16_to_cpu(x) le16toh(x)
#define cpu_to_le32(x) htole32(x)
#define swab32(x) bswap32(x)
#define simple_strtoul strtoul
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;
#ifndef LINUX_TYPES_DEFINED
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;
typedef uint8_t __u8;
typedef uint16_t __u16;
typedef uint32_t __u32;
typedef uint8_t __be8;
typedef uint16_t __be16;
typedef uint32_t __be32;
typedef uint64_t __be64;
typedef uint8_t __u8;
typedef uint16_t __u16;
typedef uint32_t __u32;
typedef uint8_t __be8;
typedef uint16_t __be16;
typedef uint32_t __be32;
typedef uint64_t __be64;
#endif
#if BYTE_ORDER == BIG_ENDIAN

279
sys/dev/cxgb/cxgb_sge.c
View File

@ -73,11 +73,16 @@ __FBSDID("$FreeBSD$");
#endif
int txq_fills = 0;
static int recycle_enable = 1;
/*
* XXX don't re-enable this until TOE stops assuming
* we have an m_ext
*/
static int recycle_enable = 0;
extern int cxgb_txq_buf_ring_size;
int cxgb_cached_allocations;
int cxgb_cached;
int cxgb_ext_freed;
int cxgb_ext_freed = 0;
int cxgb_ext_inited = 0;
extern int cxgb_use_16k_clusters;
extern int cxgb_pcpu_cache_enable;
@ -247,7 +252,7 @@ t3_sge_init(adapter_t *adap, struct sge_params *p)
ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */
ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
F_CQCRDTCTRL |
F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
#if SGE_NUM_GENBITS == 1
@ -256,7 +261,6 @@ t3_sge_init(adapter_t *adap, struct sge_params *p)
if (adap->params.rev > 0) {
if (!(adap->flags & (USING_MSIX | USING_MSI)))
ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
ctrl |= F_CQCRDTCTRL | F_AVOIDCQOVFL;
}
t3_write_reg(adap, A_SG_CONTROL, ctrl);
t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
@ -264,7 +268,8 @@ t3_sge_init(adapter_t *adap, struct sge_params *p)
t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
V_TIMEOUT(200 * core_ticks_per_usec(adap)));
t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, 1000);
t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
adap->params.rev < T3_REV_C ? 1000 : 500);
t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
@ -293,13 +298,14 @@ sgl_len(unsigned int n)
* Return a packet containing the immediate data of the given response.
*/
static int
get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl, uint32_t flags)
get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m)
{
m->m_len = m->m_pkthdr.len = IMMED_PKT_SIZE;
m->m_len = m->m_pkthdr.len = IMMED_PKT_SIZE;
m->m_ext.ext_buf = NULL;
m->m_ext.ext_type = 0;
memcpy(mtod(m, uint8_t *), resp->imm_data, IMMED_PKT_SIZE);
return (0);
return (0);
}
static __inline u_int
@ -308,14 +314,33 @@ flits_to_desc(u_int n)
return (flit_desc_map[n]);
}
#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
F_HIRCQPARITYERROR)
#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
F_RSPQDISABLED)
/**
* t3_sge_err_intr_handler - SGE async event interrupt handler
* @adapter: the adapter
*
* Interrupt handler for SGE asynchronous (non-data) events.
*/
void
t3_sge_err_intr_handler(adapter_t *adapter)
{
unsigned int v, status;
status = t3_read_reg(adapter, A_SG_INT_CAUSE);
if (status & SGE_PARERR)
CH_ALERT(adapter, "SGE parity error (0x%x)\n",
status & SGE_PARERR);
if (status & SGE_FRAMINGERR)
CH_ALERT(adapter, "SGE framing error (0x%x)\n",
status & SGE_FRAMINGERR);
if (status & F_RSPQCREDITOVERFOW)
CH_ALERT(adapter, "SGE response queue credit overflow\n");
@ -328,7 +353,7 @@ t3_sge_err_intr_handler(adapter_t *adapter)
}
t3_write_reg(adapter, A_SG_INT_CAUSE, status);
if (status & (F_RSPQCREDITOVERFOW | F_RSPQDISABLED))
if (status & SGE_FATALERR)
t3_fatal_err(adapter);
}
@ -343,8 +368,6 @@ t3_sge_prep(adapter_t *adap, struct sge_params *p)
for (i = 0; i < SGE_QSETS; ++i) {
struct qset_params *q = p->qset + i;
q->polling = adap->params.rev > 0;
if (adap->params.nports > 2) {
q->coalesce_nsecs = 50000;
} else {
@ -354,6 +377,7 @@ t3_sge_prep(adapter_t *adap, struct sge_params *p)
q->coalesce_nsecs = 5000;
#endif
}
q->polling = adap->params.rev > 0;
q->rspq_size = RSPQ_Q_SIZE;
q->fl_size = FL_Q_SIZE;
q->jumbo_size = JUMBO_Q_SIZE;
@ -473,7 +497,7 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
struct rx_desc *d = &q->desc[q->pidx];
struct refill_fl_cb_arg cb_arg;
caddr_t cl;
int err;
int err, count = 0;
int header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t);
cb_arg.error = 0;
@ -527,10 +551,12 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
d = q->desc;
}
q->credits++;
count++;
}
done:
t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
if (count)
t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
}
@ -776,14 +802,6 @@ t3_sge_init_port(struct port_info *pi)
void
t3_sge_deinit_sw(adapter_t *sc)
{
int i;
callout_drain(&sc->sge_timer_ch);
if (sc->tq)
taskqueue_drain(sc->tq, &sc->slow_intr_task);
for (i = 0; i < sc->params.nports; i++)
if (sc->port[i].tq != NULL)
taskqueue_drain(sc->port[i].tq, &sc->port[i].timer_reclaim_task);
mi_deinit();
}
@ -909,8 +927,8 @@ txq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs)
*/
txqs->gen = txq->gen;
txq->unacked += ndesc;
txqs->compl = (txq->unacked & 8) << (S_WR_COMPL - 3);
txq->unacked &= 7;
txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5);
txq->unacked &= 31;
txqs->pidx = txq->pidx;
txq->pidx += ndesc;
#ifdef INVARIANTS
@ -1209,7 +1227,6 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
struct mbuf_iovec *mi;
DPRINTF("t3_encap cpu=%d ", curcpu);
KASSERT(qs->idx == 0, ("invalid qs %d", qs->idx));
mi = NULL;
pi = qs->port;
@ -1310,6 +1327,7 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
undersized = (((tmpmi->mi_len < TCPPKTHDRSIZE) &&
(m0->m_flags & M_VLANTAG)) ||
(tmpmi->mi_len < TCPPKTHDRSIZE - ETHER_VLAN_ENCAP_LEN));
if (__predict_false(undersized)) {
pkthdr = tmp;
dump_mi(mi);
@ -1550,7 +1568,6 @@ again: reclaim_completed_tx_imm(q);
if (ret == 1) {
mtx_unlock(&q->lock);
log(LOG_ERR, "no desc available\n");
return (ENOSPC);
}
goto again;
@ -1610,6 +1627,7 @@ again: reclaim_completed_tx_imm(q);
q->stops++;
}
mtx_unlock(&q->lock);
wmb();
t3_write_reg(adap, A_SG_KDOORBELL,
F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
}
@ -1648,9 +1666,9 @@ t3_free_qset(adapter_t *sc, struct sge_qset *q)
}
for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
if (q->fl[i].desc) {
mtx_lock(&sc->sge.reg_lock);
mtx_lock_spin(&sc->sge.reg_lock);
t3_sge_disable_fl(sc, q->fl[i].cntxt_id);
mtx_unlock(&sc->sge.reg_lock);
mtx_unlock_spin(&sc->sge.reg_lock);
bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map);
bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc,
q->fl[i].desc_map);
@ -1665,9 +1683,9 @@ t3_free_qset(adapter_t *sc, struct sge_qset *q)
for (i = 0; i < SGE_TXQ_PER_SET; i++) {
if (q->txq[i].desc) {
mtx_lock(&sc->sge.reg_lock);
mtx_lock_spin(&sc->sge.reg_lock);
t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0);
mtx_unlock(&sc->sge.reg_lock);
mtx_unlock_spin(&sc->sge.reg_lock);
bus_dmamap_unload(q->txq[i].desc_tag,
q->txq[i].desc_map);
bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc,
@ -1682,9 +1700,9 @@ t3_free_qset(adapter_t *sc, struct sge_qset *q)
}
if (q->rspq.desc) {
mtx_lock(&sc->sge.reg_lock);
mtx_lock_spin(&sc->sge.reg_lock);
t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id);
mtx_unlock(&sc->sge.reg_lock);
mtx_unlock_spin(&sc->sge.reg_lock);
bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map);
bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc,
@ -1893,7 +1911,7 @@ write_ofld_wr(adapter_t *adap, struct mbuf *m,
struct tx_desc *d = &q->desc[pidx];
struct txq_state txqs;
if (immediate(m) && segs == NULL) {
if (immediate(m) && nsegs == 0) {
write_imm(d, m, m->m_len, gen);
return;
}
@ -1927,18 +1945,25 @@ static __inline unsigned int
calc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs)
{
unsigned int flits, cnt = 0;
int ndescs;
if (m->m_len <= WR_LEN)
return 1; /* packet fits as immediate data */
if (m->m_len <= WR_LEN && nsegs == 0)
return (1); /* packet fits as immediate data */
if (m->m_flags & M_IOVEC)
cnt = mtomv(m)->mv_count;
else
cnt = nsegs;
/* headers */
flits = ((uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *)) / 8;
flits = m->m_len / 8;
return flits_to_desc(flits + sgl_len(cnt));
ndescs = flits_to_desc(flits + sgl_len(cnt));
CTR4(KTR_CXGB, "flits=%d sgl_len=%d nsegs=%d ndescs=%d",
flits, sgl_len(cnt), nsegs, ndescs);
return (ndescs);
}
/**
@ -1998,7 +2023,6 @@ again: reclaim_completed_tx_(q, 16);
write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs);
check_ring_tx_db(adap, q);
return (0);
}
@ -2058,6 +2082,7 @@ again: cleaned = reclaim_completed_tx_(q, 16);
set_bit(TXQ_RUNNING, &q->flags);
set_bit(TXQ_LAST_PKT_DB, &q->flags);
#endif
wmb();
t3_write_reg(adap, A_SG_KDOORBELL,
F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
}
@ -2300,7 +2325,7 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
#endif
q->lro.enabled = lro_default;
mtx_lock(&sc->sge.reg_lock);
mtx_lock_spin(&sc->sge.reg_lock);
ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx,
q->rspq.phys_addr, q->rspq.size,
q->fl[0].buf_size, 1, 0);
@ -2356,7 +2381,7 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
device_get_unit(sc->dev), irq_vec_idx);
MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF);
mtx_unlock(&sc->sge.reg_lock);
mtx_unlock_spin(&sc->sge.reg_lock);
t3_update_qset_coalesce(q, p);
q->port = pi;
@ -2370,7 +2395,7 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
return (0);
err_unlock:
mtx_unlock(&sc->sge.reg_lock);
mtx_unlock_spin(&sc->sge.reg_lock);
err:
t3_free_qset(sc, q);
@ -2419,17 +2444,17 @@ t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad)
}
static void
ext_free_handler(void *cl, void * arg)
ext_free_handler(void *arg1, void * arg2)
{
uintptr_t type = (uintptr_t)arg;
uintptr_t type = (uintptr_t)arg2;
uma_zone_t zone;
struct mbuf *m;
m = cl;
m = arg1;
zone = m_getzonefromtype(type);
m->m_ext.ext_type = (int)type;
cxgb_ext_freed++;
cxgb_cache_put(zone, cl);
cxgb_cache_put(zone, m);
}
static void
@ -2443,7 +2468,8 @@ init_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone)
bzero(cl, header_size);
m = (struct mbuf *)cl;
cxgb_ext_inited++;
SLIST_INIT(&m->m_pkthdr.tags);
m->m_type = MT_DATA;
m->m_flags = flags | M_NOFREE | M_EXT;
@ -2721,9 +2747,30 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
eth = (r->rss_hdr.opcode == CPL_RX_PKT);
if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) {
/* XXX */
printf("async notification\n");
struct mbuf *m;
if (cxgb_debug)
printf("async notification\n");
if (rspq->rspq_mh.mh_head == NULL) {
rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA);
m = rspq->rspq_mh.mh_head;
} else {
m = m_gethdr(M_DONTWAIT, MT_DATA);
}
/* XXX m is lost here if rspq->rspq_mbuf is not NULL */
if (m == NULL)
goto no_mem;
memcpy(mtod(m, char *), r, AN_PKT_SIZE);
m->m_len = m->m_pkthdr.len = AN_PKT_SIZE;
*mtod(m, char *) = CPL_ASYNC_NOTIF;
rss_csum = htonl(CPL_ASYNC_NOTIF << 24);
eop = 1;
rspq->async_notif++;
goto skip;
} else if (flags & F_RSPD_IMM_DATA_VALID) {
struct mbuf *m = NULL;
@ -2734,35 +2781,32 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
else
m = m_gethdr(M_DONTWAIT, MT_DATA);
/*
* XXX revisit me
*/
if (rspq->rspq_mh.mh_head == NULL && m == NULL) {
if (rspq->rspq_mh.mh_head == NULL && m == NULL) {
no_mem:
rspq->next_holdoff = NOMEM_INTR_DELAY;
budget_left--;
break;
}
get_imm_packet(adap, r, rspq->rspq_mh.mh_head, m, flags);
get_imm_packet(adap, r, rspq->rspq_mh.mh_head);
eop = 1;
rspq->imm_data++;
} else if (r->len_cq) {
} else if (r->len_cq) {
int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
#ifdef DISABLE_MBUF_IOVEC
eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r);
#else
eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r);
#ifdef IFNET_MULTIQUEUE
rspq->rspq_mbuf->m_pkthdr.rss_hash = rss_hash;
#endif
#endif
#ifdef IFNET_MULTIQUEUE
rspq->rspq_mh.mh_head->m_pkthdr.rss_hash = rss_hash;
#endif
ethpad = 2;
} else {
DPRINTF("pure response\n");
rspq->pure_rsps++;
}
skip:
if (flags & RSPD_CTRL_MASK) {
sleeping |= flags & RSPD_GTS_MASK;
handle_rsp_cntrl_info(qs, flags);
@ -2787,7 +2831,8 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
* XXX size mismatch
*/
m_set_priority(rspq->rspq_mh.mh_head, rss_hash);
ngathered = rx_offload(&adap->tdev, rspq,
rspq->rspq_mh.mh_head, offload_mbufs, ngathered);
rspq->rspq_mh.mh_head = NULL;
@ -2988,12 +3033,8 @@ t3_dump_rspq(SYSCTL_HANDLER_ARGS)
return (err);
}
/*
* broken by recent mbuf changes
*/
static int
t3_dump_txq(SYSCTL_HANDLER_ARGS)
t3_dump_txq_eth(SYSCTL_HANDLER_ARGS)
{
struct sge_txq *txq;
struct sge_qset *qs;
@ -3022,7 +3063,7 @@ t3_dump_txq(SYSCTL_HANDLER_ARGS)
txq->txq_dump_start = 0;
return (EINVAL);
}
err = t3_sge_read_ecntxt(qs->port->adapter, txq->cntxt_id, data);
err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data);
if (err)
return (err);
@ -3066,6 +3107,67 @@ t3_dump_txq(SYSCTL_HANDLER_ARGS)
return (err);
}
static int
t3_dump_txq_ctrl(SYSCTL_HANDLER_ARGS)
{
struct sge_txq *txq;
struct sge_qset *qs;
int i, j, err, dump_end;
static int multiplier = 1;
struct sbuf *sb;
struct tx_desc *txd;
uint32_t *WR, wr_hi, wr_lo, gen;
txq = arg1;
qs = txq_to_qset(txq, TXQ_CTRL);
if (txq->txq_dump_count == 0) {
return (0);
}
if (txq->txq_dump_count > 256) {
log(LOG_WARNING,
"dump count is too large %d\n", txq->txq_dump_count);
txq->txq_dump_count = 1;
return (EINVAL);
}
if (txq->txq_dump_start > 255) {
log(LOG_WARNING,
"dump start of %d is greater than queue size\n",
txq->txq_dump_start);
txq->txq_dump_start = 0;
return (EINVAL);
}
retry_sbufops:
sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN);
sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx,
txq->txq_dump_start,
(txq->txq_dump_start + txq->txq_dump_count) & 255);
dump_end = txq->txq_dump_start + txq->txq_dump_count;
for (i = txq->txq_dump_start; i < dump_end; i++) {
txd = &txq->desc[i & (255)];
WR = (uint32_t *)txd->flit;
wr_hi = ntohl(WR[0]);
wr_lo = ntohl(WR[1]);
gen = G_WR_GEN(wr_lo);
sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n",
wr_hi, wr_lo, gen);
for (j = 2; j < 30; j += 4)
sbuf_printf(sb, "\t%08x %08x %08x %08x \n",
WR[j], WR[j + 1], WR[j + 2], WR[j + 3]);
}
if (sbuf_overflowed(sb)) {
sbuf_delete(sb);
multiplier++;
goto retry_sbufops;
}
sbuf_finish(sb);
err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
sbuf_delete(sb);
return (err);
}
static int
t3_lro_enable(SYSCTL_HANDLER_ARGS)
@ -3162,7 +3264,10 @@ t3_add_attach_sysctls(adapter_t *sc)
CTLTYPE_INT|CTLFLAG_RW, sc,
0, t3_lro_enable,
"I", "enable large receive offload");
SYSCTL_ADD_INT(ctx, children, OID_AUTO,
"hw_revision",
CTLFLAG_RD, &sc->params.rev,
0, "chip model");
SYSCTL_ADD_INT(ctx, children, OID_AUTO,
"enable_debug",
CTLFLAG_RW, &cxgb_debug,
@ -3190,6 +3295,10 @@ t3_add_attach_sysctls(adapter_t *sc)
"ext_freed",
CTLFLAG_RD, &cxgb_ext_freed,
0, "#times a cluster was freed through ext_free");
SYSCTL_ADD_INT(ctx, children, OID_AUTO,
"ext_inited",
CTLFLAG_RD, &cxgb_ext_inited,
0, "#times a cluster was initialized for ext_free");
SYSCTL_ADD_INT(ctx, children, OID_AUTO,
"mbufs_outstanding",
CTLFLAG_RD, &cxgb_mbufs_outstanding,
@ -3240,8 +3349,8 @@ t3_add_configured_sysctls(adapter_t *sc)
for (j = 0; j < pi->nqsets; j++) {
struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j];
struct sysctl_oid *qspoid, *rspqpoid, *txqpoid;
struct sysctl_oid_list *qspoidlist, *rspqpoidlist, *txqpoidlist;
struct sysctl_oid *qspoid, *rspqpoid, *txqpoid, *ctrlqpoid;
struct sysctl_oid_list *qspoidlist, *rspqpoidlist, *txqpoidlist, *ctrlqpoidlist;
struct sge_txq *txq = &qs->txq[TXQ_ETH];
snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j);
@ -3258,8 +3367,10 @@ t3_add_configured_sysctls(adapter_t *sc)
txq_names[0], CTLFLAG_RD, NULL, "txq statistics");
txqpoidlist = SYSCTL_CHILDREN(txqpoid);
ctrlqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
txq_names[2], CTLFLAG_RD, NULL, "ctrlq statistics");
ctrlqpoidlist = SYSCTL_CHILDREN(ctrlqpoid);
SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "size",
CTLFLAG_RD, &qs->rspq.size,
0, "#entries in response queue");
@ -3282,8 +3393,7 @@ t3_add_configured_sysctls(adapter_t *sc)
CTLTYPE_STRING | CTLFLAG_RD, &qs->rspq,
0, t3_dump_rspq, "A", "dump of the response queue");
SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "dropped",
CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_drops,
0, "#tunneled packets dropped");
@ -3340,7 +3450,22 @@ t3_add_configured_sysctls(adapter_t *sc)
0, "txq #entries to dump");
SYSCTL_ADD_PROC(ctx, txqpoidlist, OID_AUTO, "qdump",
CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_ETH],
0, t3_dump_txq, "A", "dump of the transmit queue");
0, t3_dump_txq_eth, "A", "dump of the transmit queue");
SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_start",
CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start,
0, "ctrlq start idx for dump");
SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_count",
CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count,
0, "ctrl #entries to dump");
SYSCTL_ADD_PROC(ctx, ctrlqpoidlist, OID_AUTO, "qdump",
CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_CTRL],
0, t3_dump_txq_ctrl, "A", "dump of the transmit queue");
}
}
}

7
sys/dev/cxgb/sys/cxgb_support.c
View File

@ -331,3 +331,10 @@ buf_ring_alloc(int count, int flags)
return (br);
}
void
buf_ring_free(struct buf_ring *br)
{
free(br->br_ring, M_DEVBUF);
free(br, M_DEVBUF);
}

2
sys/dev/cxgb/sys/mbufq.h
View File

@ -103,7 +103,7 @@ mbufq_dequeue(struct mbuf_head *l)
}
static __inline struct mbuf *
mbufq_peek(struct mbuf_head *l)
mbufq_peek(const struct mbuf_head *l)
{
return (l->head);
}

29
sys/dev/cxgb/sys/mvec.h
View File

@ -31,6 +31,7 @@
#ifndef _MVEC_H_
#define _MVEC_H_
#include <machine/bus.h>
int cxgb_cache_init(void);
@ -48,8 +49,10 @@ extern int cxgb_ext_freed;
extern int cxgb_mbufs_outstanding;
extern int cxgb_pack_outstanding;
#define mtomv(m) ((struct mbuf_vec *)((m)->m_pktdat))
#define M_IOVEC 0x100000 /* mbuf immediate data area is used for cluster ptrs */
#define mtomv(m) ((struct mbuf_vec *)((m)->m_pktdat))
#define M_IOVEC 0x100000 /* mbuf immediate data area is used for cluster ptrs */
#define M_DDP 0x200000 /* direct data placement mbuf */
#define EXT_PHYS 10 /* physical/bus address */
/*
* duplication from mbuf.h - can't use directly because
@ -59,7 +62,8 @@ struct m_ext_ {
caddr_t ext_buf; /* start of buffer */
void (*ext_free) /* free routine if not the usual */
(void *, void *);
void *ext_args; /* optional argument pointer */
void *ext_arg1; /* optional argument pointer */
void *ext_arg2; /* optional argument pointer */
u_int ext_size; /* size of buffer, for ext_free */
volatile u_int *ref_cnt; /* pointer to ref count info */
int ext_type; /* type of external storage */
@ -72,6 +76,11 @@ struct m_ext_ {
#define EXT_CLIOVEC 9
#define EXT_JMPIOVEC 10
#define m_cur_offset m_ext.ext_size /* override to provide ddp offset */
#define m_seq m_pkthdr.csum_data /* stored sequence */
#define m_ddp_gl m_ext.ext_buf /* ddp list */
#define m_ddp_flags m_pkthdr.csum_flags /* ddp flags */
#define m_ulp_mode m_pkthdr.tso_segsz /* upper level protocol */
extern uma_zone_t zone_miovec;
@ -181,11 +190,21 @@ static __inline int busdma_map_sgl(bus_dma_segment_t *vsegs, bus_dma_segment_t *
}
struct mbuf *mi_collapse_mbuf(struct mbuf_iovec *mi, struct mbuf *m);
struct mbuf *mi_collapse_sge(struct mbuf_iovec *mi, bus_dma_segment_t *seg);
void *mcl_alloc(int seg_count, int *type);
void mb_free_ext_fast(struct mbuf_iovec *mi, int type, int idx);
static __inline void
mi_collapse_sge(struct mbuf_iovec *mi, bus_dma_segment_t *seg)
{
mi->mi_flags = 0;
mi->mi_base = (caddr_t)seg->ds_addr;
mi->mi_len = seg->ds_len;
mi->mi_size = 0;
mi->mi_type = EXT_PHYS;
mi->mi_refcnt = NULL;
}
static __inline void
m_free_iovec(struct mbuf *m, int type)
{
@ -279,9 +298,11 @@ m_getzonefromtype(int type)
case EXT_JUMBO16:
zone = zone_jumbo16;
break;
#ifdef PACKET_ZONE
case EXT_PACKET:
zone = zone_pack;
break;
#endif
default:
panic("%s: invalid cluster type %d", __func__, type);
}

4
sys/dev/cxgb/sys/uipc_mvec.c
View File

@ -412,8 +412,8 @@ mb_free_ext_fast(struct mbuf_iovec *mi, int type, int idx)
case EXT_EXTREF:
KASSERT(mi->mi_ext.ext_free != NULL,
("%s: ext_free not set", __func__));
(*(mi->mi_ext.ext_free))(mi->mi_ext.ext_buf,
mi->mi_ext.ext_args);
(*(mi->mi_ext.ext_free))(mi->mi_ext.ext_arg1,
mi->mi_ext.ext_arg2);
break;
default:
dump_mi(mi);

3
sys/dev/cxgb/t3cdev.h
View File

@ -38,7 +38,8 @@ struct cxgb3_client;
enum t3ctype {
T3A = 0,
T3B
T3B,
T3C
};
struct t3cdev {

451
sys/dev/cxgb/t3fw-4.7.0.bin.gz.uu
View File

@ -1,451 +0,0 @@
/**************************************************************************
Copyright (c) 2007, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
$FreeBSD$
***************************************************************************/
begin 755 t3fw-4.7.0.bin.gz
M'XL("-[;X48``W0S9G<M-"XW+C`N8FEN`.R\>5A3U]HWO+,3R+1#$@@0YDT(
M86>.VBH>M8V*5:OG-$XM+1U2DU`Z.-2J.!M&PXR"2D`@H'7`*2AU*+9=H`<G
M6A"L(X&HJ#A1;#U63RO[6SM!VW.^\SS7\[[?/]][72_MREY[[[7N-=Z_^W>O
MM;9&9#&"TZTP+$001``#CEQ#T$R$B5!7&T)#$#?B"0X811(2$O0W$%H&CL*4
M-#<,I""*;$>B2(#<0!AC48^$&9DX,LL`KZWP.AM'9F+PF@BO*?#Z+H[0G0!!
M>H<A=X<A-<,0QS`ZBJ`/=/,5/LB#80B,!R'^Y%6@$Y%7.OG4?_2)4=+I@D53
M![RW_N2/,1Q$RD'^XUL1^0-]0I2(/`M__<E6H%./@O^_I!X5KQXU-F8<;=&(
M![`,?_($T&E'(=I1`NTH7#M*]Z<WWP)=W"@D;I0@;A0>]_P-G7IS-(#\&D.1
M$MP3/3H4/0*&E>#I,'(X@#Q,/3@<2!X,(@\!W47<)YEF1,*0YW^,D3ZP"*J:
M2X:)F)-%/B,P)F/BTF&WX(LEH[W%[P@DMU)9C8@O\B()XQ7X?L!3B1JJ9VJ\
MC2W1-R0LFOKS\Y979.@]/_YD!>PA$5G^'U-MI!)L/(-GX)G169)CDL5C!-F&
MACF;=`TC%H]X9")OP8I:7G5;/G#YDZN`CABE)T8E$*.F$*.FO^@+3TU7@.%T
MC8^G4LL"R+02'`AI4X<A4X;14`3`XM<P.H8%DJM+#2)R-:,31E=1T56,\U14
M^B9,L9+Q(XROA'$!N8*._-^__Y_]F<C?_\MW4;][PPW$Y]1_&8>XX,4&G]:A
MN'@H+AZ*]WKCS=.]\69#%'D<O8&PPN$58LO?%\-X%XY0N$+[Y`;"38!7*Q5N
M()@3A_,0!OI0/!P&',8;J"L,,A@_`J]C88"XA0'J"L-4&#\!K]-A,,!X.W6%
MX6T8/P^O$*-0(XR[J2L,'\-X+[Q^!L-"&!^@KC`LA_%'\)I/A1L(CZI+,0PE
M0_$]5(!Q!KQ29;1[XW0G%?[7XK.F>^.SIGOC[SK^J_&`?;:,"EX,GF6E\!EA
M(/@?][!??Q^$Z9H8")\D]3"O#PQ6!`GI)@>1[KW?_C0>WK],R22;7:A'.@O3
M><8`85L]A3VATD?"]T8QO*4A5MH$;_H"*'<Z]8`%[QEZ3QZ/:<%@/%SOM3(,
M&#"O7"A&AQ@1%B4OG+(W"**GXK3?O742PQ!JA3DH>/`VFH48$(1.DM^1<.YT
M(4@0;"!L"#D`WY&_?NFI![@419Z8`>7!:G!Q7$O"=I^8!&$))B%)!%$AY"#Y
MA*H-E(7"_C)2-27)'OB._(F&"*&,$Q_"4$R2/U'/?C-21?.?P.??"4CR5ZKL
M\_#Y,X3V6SML]VP8!MRP_V`?&ZF:0YE2^"P1]H<QZG?R$JQ+VEI8]C6(ZP@2
M)\!AA_;`]'`L=#`LA/:6RH/#$`_#=(3>7JPTF*Q*PW(8BF'83K7A;5@W/JZ%
MY>@0*VR7IT$(C7RB-!Q`8&#!<@1*0SU,?P"&KZG!$2H-WQ!*PS%X_XW##;L1
MRB>BGB$L>!T+@\$%_^#5#.M6#,/]_Z3OM/\2"?[/^,,%B/Y%$,&^@Q/P17C^
M['D0(P`/IR$O`D[3>YX]#]2S?[^GTOP1C/\2I#0'^:>__U@_Y'48ICSQ7CUQ
MW1_A-?-0@-@T"?*J!(A%XP4X\NJQ/_)/A<^FG/!>J3"9]?\.K\WV7B="S-*?
M@OD_^R,_'6*9"G(T.L2Z,3!.OX\C+\?#*\3?T;#,5Z?@R#!XU4*,5/;AB`)B
MH`S>XPX<T4!\"8<RQ3"O$-93",OP@V7YP2L7XA9KCS<_%1A0)@/BL"\,\3"O
M#M:5_CN]SZ=KN!DJ]M#5_&___7__$_S/GT^,L?Z/Q;Y;2GNO%'V_E/Y!*3+Q
M?Z-:[Q;3WBM&WR^F?U",S/K?R+^E%(DHIE66HE6E].I2!JO4AUWJRREE<O^'
MU=D",Q>C5<7,B&)Z=3%C6;'/\F+?%<7,E?_#ZG1YRG>5HMVE])Y2!J/4QZ?4
MU[>4R2QEL4K9[%(.IY3[W]6E"V8N1KN+N;#\GF)&0['/5\6^AXJ9AXM91XK9
M1XLY7Q=S&[UUH?@(]"N,$(`Y5%;*UT`ZA@G(0<\M==^*=/[Y?B:&G/^7^Q3D
MQS_N/?(XB/"GD("?0NBW*.(:]),X^"=Q)S_U4]]B8T-RJNG)[(`$O@@F?8H@
MGPD8`#_'_^^N2DDD(8D`#E`JE_HJ8M&5.;2KCM-'`WYBJ)Q,PHE<P>DV#I$^
M'9SC6I&K#LO^VYZ<?Y;M>U&OB4;!P)+;_)R7UF[FY+Z\=K^/-A;E#Z)^W&F6
M^C6B_H>A_0\W"]2U5>#,ZN.HD98EGR!+*6;:<+8-LZ2ERFHKNXSF#Q<K<0RT
M8X>%=:G$HFRP95D)3/I&2/\-]E&.?,/6/S*,B5N4)>[O,7\^/&\8-T?$LM,6
MK)?)HQG<QP:[<'Z9&&R\.D#@R;X5\;)\FJ_O9)DT>6DCE*60M8IS7@(GUFY^
M*:R_W?;REL]9/'W%0KY+F(=7ID;TMU8OYC6B%8M`1^62L/XS+J!<2E^[CD6\
MCK#Z4"(67;.9D3=2&2M9O9E1.$H3&[UJ\TC-6!S4`ZOOD7@6:X+9FA[6_TU(
M_S?R0VL0MPTQYO_Y5U-;J4VHU=P7:[97YNA2-[\<UM]@&U$XG'F!*XUF-+@8
M"<(M'U88\Y35EJKDLH\B^_>RR_2^-B$K`?6#M9K;E`]CI>;3%T/Z=Q(XAA4*
M=]GJ^E77L[29;5@75W4KJ^U=:DC`.E#L=S1^N0TUD<M,Y&>M;UW!K^JZ%B9B
M;U*OUPFNX(F\9,IQ`D;38-85'!Q*Y,TUD1<0A6IQGB"1-Z/#<8Z_Z3@H">LO
ML@DK`,LF9-O0LB98>R5L@S19>4RLW%YI&MSOF0>^%_"0_F7A_2N(#U'__E06
M:YK%FI2KXWPRX^.MLY22JOE'QB@VR%(<C`+CBLLZC:3RT)0S?4HC%C=]$/M:
M6%LGK\UV+5R>%X,=XZB:MZ8`ALVX*B=059O5!ED.#[EBI.KWEU/O&&F1\":1
M-\HR(Q"V1U^@NV+DS9N1B)WH=)AKT,P:,(/!F&PB/T>`,W>8]D,&YY^&^4^"
MP<9+3DY9O,*87%_'X4Q6F)-7Y*%&Y,(60:708CBL&"OH!F`R>`BLV!/O()9&
M]4\/Z9\N?[SFD\\Q,%!/(@X;XLY/??H(.')T12/8-KW9.M,V'%O#!0D1_1-+
MKTD_9%2ZX?@5?%IQL_IVV:VH_E>\8\=WH7#0&M8V%51>IT:O-[)_M-J(89>$
M^URUZ5))EO)\&X/D:F595)^']`\GC-3`UO;OLBDSV^#8=CB\0PN'U<IY$K]L
M"AS4%!/Y+J+X;#$XX-(G8MOS!05"X#21)L0S^AFLI\+R&RP[*NZ/LE\K<XO[
MA4ISLEQ2J<RLE6\1RV65)A+UZN_)L'YA:+__1?TE_+(N/3J@7X2^QMV:M736
M"'8)*N700#OZUW@>;\+I"XI#:[IQ^G#4[P8Z;X4/@ML077YZ&BQM@/U&_%IP
MW41V(<"`%F);^_W&^8-AZ,OQ/CX3SKX)\YT<6/U/?B<N[1)T%,][?2COKNO'
M,L!X8*!/HM*=?ANF^U>Y1O;(^*6@V40>`\7HU/CEH,=$7H9S+0[W+16D+/,A
M5CVUI%699ZPEHE$69YJE)D/\X)(RO2KTP84%1RB$4=3"*<?.PSOYG$;,4K]$
MGE[9U&1>/T^*8Z$/VK"_"G>Z*H7JCFS.ZQP/)'V@J-U:)$QQ,/-P3X8WM#]G
MF1].A>4IHADIJ5'9PWP?&^;O#`8_`J<*8LO5^/J6/T'+XI`'+3:U+#HB[,'?
M62GLBH>4Q@QH=`+Q@V:L7^"W#V7UHU4_%[Y\9<#L3NC27=:MO1V@&BLX]D]B
M>%#!Z++'E;_FC6H87?'$3RSJY,_+B28V!.6=+MM>L<-V,N_[LIW$AN"*NK)=
MMK.J5KQLMV*/^(@=O%2?[.O[QJ>]N%\?FV_C^+V"K;WM`XKMMF,VB`"A_O/J
M?.TV>QZ1+PY[4`O:Q0]*(A_8%='A,8\#]OV->.PO?E"<@<MB(U@CT*6[?($1
M%,MC(Q2KGBH?!ZH?!\<\%OE\)@QY4*)Z+,X7RA\'*9NSU+UMRF,IRI:LLWLL
MAB4>_->N>FH>J(*C8!I<BGBA)E^0R!U#O3S']X`+-LU$]D)+<A7.UBEP&"%J
MG'H+8HW2'TG$1GHM4"<?6A[@-#O#H-T)>6#QF!ZVQ_2H.;0AVW.#$NH%*8R@
M%";BP3N;2L#!\`<6UC[AEF)N/PI`U7K5^5IE>J7*BU/9$*?FPCQ7=2I_)(F<
MT.D`,X`5N(!;$XM6"DS/`*4++#GNFU+F0VQY:DD_)H?6K([-GF8I[H06*>C!
M!.R8<'NKICF[J6--)VI$YX4\>/6%22K`V2[,TO&VJCG+W#Q+,[<JY,'+"_I@
MJC>4$/2<S"*<2V*6G\>KYU8V'3-WC-9$XWY.8WD`@8\]22+,5VC<5^J_9=X;
MS6UD\[X5:NZ'^_F-Y_?3Z_OAU:^1JSP6?M&Y9EVX)CIVY;V@N-#?_;[#F`^-
M"/L5&O.5^F7L=:.YJ9P<F2T.&L94B&NT7M"GC([11D?7-W*Y;RAC8U;^%N[\
M1A6-JF-C_%I152PZ9`_TB9P.#V`XBX9IHAE\TC"_SP.6_+)X+9Y<3_+YDR'6
MKZ6$'@7U-IU<)XAX(-P45"K:'+PQ$"P$,T&"#!^K;D?M`E:1"!2S;'J&F>9C
MH?D<CO<=J<\2,%,%IS)`&B\G'E@9T$+:!*Q45/KNH.<N3\FQ";BE`@B+S`24
MB,;CHF,9C2B+$(GN/V7&B6"<D2J$&:`!ZA4R9J-Y.&..@-4H]&E$P3OPIZ*)
M\ZG0IO!)%L"D%2?+3H!ZOEC/A(4DH\QD=%-+U@A@W7B\Z=W,X66G*DZ7-LLD
M(J"O_RA.)BH]N_'OF\[DQ5>TVOY2]OVZ,9M_R!Z]+GA36];+V4&Q+8+,D5F!
MTA9AWJA,D;+%'\[*]BF4P9Y(Z`2%(XJ&0T"(N/]W1IU0W2("Y]4M@05*;4NH
MIB5,V1+"KM+[]@O]ZE".#8V\_XV"(X`1/Q>J;@DB6L0*CE_(_:.4\2Z"QGL[
MJ?DY"\*\WSZNYG'6J;5YLESYCL8*&;<.XW"F*&)CVV93`P3--7>%:7"R1Y>X
MRTR#XQ!$'413!:&H`KVZ,-'GOF6?(C<@W[]`:!I$AE0$ZAKW`TI%JCM#[E>"
M#'L[ZXX0&@7[N9#[!4II,C&W4EE:"Z&!2(8*$O6O7'%&P/W*L/OV\/OVH/MV
MC!"*[MM#[J<QI)SMLETV17J67/4[!D4Y*FKBTK-B7EM_)#4N.RN.8]V1'&NN
M]LC@T$>B&@C58MAEL4%,(O^I.7UT2I^/_/Y3RX8H133*X4[[OB%?)[K_Z>*3
M4G5TS.EN#8<!41(J95&PY>$@G*+&?'$B1PYPR38Z53-B0E7D_0\6V*@YKK@.
MP9VEE"`>J/Y1,:'27--6+@J^_TG4_<3\CYING3Z3FTPLI)W>NB)#J,"QI@/Y
M>*+/S1\=/VPSHKX(U&K(/+91S&/YO7G\;SF:ZUM3W(PB?,W3#V)KL\PSOE1&
M:]<E<U>.Y"[#%A088&46-IU21^.)Z%L%LORX\A!>*[;B^+@<DWV6;6[9S")+
M=:+&*(#8$Y1R>]26MY42L4(24)%4_L[F.07)J[^`=*,L)]A2_)H140"#B:R\
M",WUKBMX7DBBSVK+?IZ);("8N'D.N`8&0'K9V]5)5>^8R!,>+!4GLOM`TP4'
M<"JWT14.P9;@\B"?=ZFA]_T4[5FH-M(+@RH#JP282PC<>^>IN_!$VL7\P+R0
M/7M49GJN+$>^NXYCQ\IEMF0V>XH\-I9IP\J2M1+Q#E(K$YL&SVCFTK31#+]!
MR)5"%!P?@H/(\62_G?'U57Y^DS72Y#446]J9]Y'9.<V\<,I58Y>A9^$5W%,N
M++-*X"DS\KY_(NUH07!1""$)@'9$)JZW$;*`4R0UI_?_>4Y;V@V>>0;;`!N@
M_@QGKD4+@Z[@58+(^[1$6H*)/()`4,*';`?GF.G9`V3!3=3T[)2W/UR&$^Y$
M%D+-;:@9G`;3L^\IDP#GRE8/A[IW`5@C[KG!\3P1ZY_"JH-8'5I>7^DD2FMC
M)E02TN28L>*8UR`'4EW!X3Q(])ED>I;O]6<@[BNB8R#P*_:0',X;BMB8E-(Q
M'Y<-5T9'K]RM@G8*.%22",I).NUUDNXU_O=.DF4_,)$M'OMR40>,X??J(^_M
MU^*^EPU7]"!1=.^0(H@F#T)2GO@0OS^U9'RJC);D"!314DLQ2[&'YJF"=`5D
M%_1(;33*YT^S_,*02P)MPRDG*",Z-QZL-V^5P.E=-`H<XB>-KA`U'"GSYS4*
M*P)./PF[MT/II*T]*,@347-?]J/#O#6\*-6(Q.7[8]N%^Y+5<[,YN]'59R>%
MWLOC?,V!]"?%X:.2("M.CXB=FV6>>5L6K?%]/-+W&;;`A?(?#\TX%4>DCHY*
MI&VD;,YY!ZS)BCR1$6$BU+P^C7BM/..4I?8=T%Z9F+]XR]MY2W*75KQ3GK1I
M3E'J6A/4WO!"P?=N:BQ5_K1$]K"+#O"P6E"T`[3G+MOT7OF[MN4Y*\K>MW]0
MN")W^;[Y2^_1ZNNJ5I0OSUL('-Q"$3=/?\H):H`3;```9)8&E01*\;%@&&L*
MC34VGM4J8,/(F'C&2WKN3@&S4>`7JO>9*,`:44Z",/S>)UP;RJP3<$:@RFB(
M0K%<I8BC$$$DYT+`'H'F*2&TLQI1FPAF8-7!#,8*`R<!92>@N7C(O23X"*(Y
MQ>QFY"E*WBQ]*R,P,P@FK9B=NR1K:=V5+RU+DU`X<OD!H#WRWLOB>Z_XDMC&
MI>5+*#7<5E<N@XKH5<+J5#AL.W=BNX7[&N4'LM29;7OL\D-9IL%)P`UTH`D\
M`9ER?"RLCCHZ%A;+4XL8+B&KC,:OIO';1K..Q[-&"#B_"4&Z#RF`_B^L/ZL.
M9?\0G[=0'AW%[!?0^U&V7,3-$+%&T.C'1XOOJ9@C!'02S<"A2\*SHT6*`OA*
MR!N!0O97H#0;K3;8^V7OVE9D+R_[H&2&_<U-[Y>_!8RIAVD70/B]2.ZG`NX]
M%/L=W3([_%X8]RI*-=L`'.'W?$+N_A,JDB9:BAV!1FU/HZ*A+2=`FY[E[-]%
M:K.SB@1J2:`F5EKO4LL"3[U1*"N0;^NODOFY,!YOBAH"PL!'7FT!QI(9U;/M
M;Y:_E7J8);[[*V%%62WQX?=HWK*?EVDB;?F"UI,4)N<+$UDU5.8FESI6Y8$'
M88%`P[$F^CZAL*"I3Q.K,I$WY=1O#^+Q;X6);#OU+N51>,KCLQU&#XJP:SUX
M3.$!8[R)K()XH,$Q\=T3V$]"%LDB?LY>EO0)=L=+^5@*"<*NPRR.Q!CH(NAF
M;NL/N7L\_&X3'"QEM`J<KQS&;13".;5%!P`<%YMRT\O*U_VWC-HR$M[!R0,.
MEK]4-AS61?USEJRW#:*G^G%6@5`I];W:GL@>[@6VJ^Y$]FJJ:=#G88X3`6N7
M[JH>(N)D$!US3$<G*$\U54^K=YG3WH^\ZPBYZY#?6M-AF'=ER(F"TPB/=8P5
MW?WR4&<1`.W\=2*I'N7GZ$L%_"+]J5]`&B,]'CSD$@*0`"2,V33&LWAT)@T=
MC&>Y!.GA<`;YS:=XBU0U"-R,P7@.?&!#>3:T8'?DW4K6)13>^>XSE#DS'0P"
M<CAJE0+*?`DMK2^JRQ14-YRZ(ZN)`C_[RBATD-7@GB@_E0;Z_-ZG,=>-CJV)
M]2L<[1,K8BX3,.8(?2RHSRV*N?G.03<>BJUA<&X)68E"%H'"9YL.`Z-9GV@N
MGN.[1@@>Y33;CG-?XFZ:5#;9_IKXKKEZ/.-MH9\=C;IK1/TFP[I5Z\LFP/N&
M^"8WO-@3(N_.%-U-TCHP[*EP&[G/KMF0)7O4YF?C$INRJ)D7=??U\+NSE/#]
MM\(]C?O[U=>S9`UMO'%<M=?Q]B$\CO=@E6EP`QP/"#[;`;`L'+#I@(ZU3@1Y
MK\7:"=I!*4@#`['XV,W^W%<$D"`/I\^BT8OBV6_3V.OB@8Y)"K@9\0RHAR:4
M4L5P5JI`/G:071#/<0ERE$P7BAT6P`I##85="Y749XQ>'<VPZQFC!3:<Y1+R
MQJ'>M1B?HP+63K1L/*=1R&N$NDQ$QS))BCJO\Z].8-B$47<#3_7)HJ,\O>^+
MR*)Q&&.EH!63F9'ZL+MB:GCM'B![38YCP-H!L"KA;KNGY41I&V^OM^6P8P*U
M`RC064@<##.182Y]X8PS[@-U]8WJ+H'?LI%E@B(#*Q6KOE1V.6=6T4R[T'ZU
M^LJ:#$Y]:^R[@M/+3&0;8A,4"%FWL<TS]A9<'%AVE5;_:^6LS3/_Z#ESX=\M
MW]68R!*O#89ND5DWV4=GI%PCGU?JWV1:1OM,)4ZNW36"[CN>F4RG?-[QVT>L
M61<IB8Z5A_[^A3(0C<?8:;"=K]#05^JEOA-&TR6<<_P.'1RZ&2;2Z]]!/BH-
M8BIQW]@@WY0C/L2QIY:T:Y2CMY!-K1Z`/!TXL?BP#V2BEHX,T9T6S82JP#O'
M*4>.UB2?*4LI9A?AG7PV=.4>UD.Z"7XTU^PJC(^Z<\Q\;6^YZ/1/!1^?OAIU
MYXQV@+;FEP"*;D)\3Z1/A5:W]N/"%",B@]@2=:?^.>ULREZ>-Y?_'4?[\]:4
M`88-7YLSFR*>,Y&X:`WS]DCF'6Q!)>VBVT,\3T"#FT0>I0QN>2!E<#-T12;@
MKIY5,+<JL7)FO@7:V(\JDG*3-\\I?Z<P90U%-]?8@G[(,R*!E%7>B5!F.2\P
M$;UN<5PTD4V(!RF#$YDS(:TL!O450?)M=(IP]RSL,@!WK)$.R=B=+['WA!2W
M(Y+(B87)7^XL^H@PT_>O3?V-5M^?&UB57/T1K)&\S@YK9)<5IK!84PA(+ONQ
MJA338+]V+BW&ZU/&_T$J^9!45O_)JY2>Z<_[^,S7SREEY)T<R"J!M8SBLD,E
M2ZF*YP<7!"4BB10P=AFN&M6?$=[DJR'1#$*2!@^;R(M(;M#WM1ZG`;8K'.`>
MNDRM0'ALBX=*LN0F\A+B(?_?"K?URZ]G-Z4OSX"=U<E_[LP7X=0(#S1KKV>9
M!XYE#9-%,WS^89B?$GAI(:R_SZ7X^DH>;[):FKR:(L5;*:?PU<`[$PI&9`[W
MKO=M>;AQ@&$1YBHW_Z/T<>6OK#EHQ)VQ359XY=6A&Q]%W!E=_C/O&Y;T>E;<
M^3;I=%QZBUI_=7I;&#7$:9FDB=1B9=2*LXF4A=V)!HZ0.RJ0`2WV6&%U)ZL5
MW7+.WJYHJ)5/J%1(DZE%/4AH!YW4,%,F#"TV#?Y$R7E@&KQ=N55\)P*4E#E8
M*11RA-SQ*ZN)@Y[?A,HXZ`5"_X_BPB][=._4J]2(;K7;97!,O>/9.O,/?Y)Y
MEJH?3!<CB9))(L1W(F,_2]D9%S,V)486M>0+X9+/%:`@5A;1M%EN1@@S$_;]
MWJ;#:DZ?KPK;V8JIL+VM*EF$THPH/.]<3;>U'.M^K586H3$C:NK99P(_\.%E
MO30:M0L9K&F6C$/!?3?DBZH6E$$._$-$7X^\0_:'_=UZ3+NHTOS+(=CDIEZH
M6#OJ"*A8K<N.P\1[0OHN/%^B84%2RV[%+,<WJZ"*'5]?*E`,8]M>+O<O4Q2,
M+`NH5.:,JA39507Q]D#>>Z+*H$IUSFA9+.3Y#'EL.*$CP`9".E:MD[-8D\N"
MV;DTM72<>EA@N9C'&<^Y@BGVW%?$1BVWALH5"+O=B+!>H;%?D6ZAL::.]K,+
M83WA6R(Z*J3O/&N]$;Y"&*_45S(NCY;N^5T:BS<Y./:1[(1PMBT\=US!6)DL
M0"83R62!BEB).C9:CD.R'4[$:J2Q6C9[HEPJ`GIIK%0:&RN%=$<OE8GEPX72
MV!A[Z/P._WDL3GT=4L-`4D+A[#21_3FA)1'`"CHV16F"_.."!,H@X<9(9QV;
M,85=)P0S@#.XKTB#CP-FOP3AIB;>`]IFP$N*]_/3%_AS(7%.C,_!-Y_:=+HP
M@%JG@;;?)0SK^Y):S>@4E?P0VE?#@T;$A5:<+0R$#T]^5W&BXN\5K17?A_:M
M*VN!:<1]RV`*N@LM/Q[05U#9QBI#RYH+@LK.\$)%O#HAO0X-Z-M0?C)71K1(
MB)90HL4_K@6/:Q''M0AD+=&REA!9BS"F1:9L"52VB&):I'EB#GU*4%_VNF"?
M$4)NTFC&"#2P+Y,IU<M:8IGQ^NPHU";@)PB93'UL2PRC$<V*%/6MIC>BM%1!
M0-\J5BJJ;`D6]\W/$TE;PF-:PH`^(P*Z2O!Y7$ND7R,:YR]0M@1%]BT*ZTO4
MM`3D'=7X^^]M!.WJE@AU2Y2Z):Y"3>"^2G\AD?_,;#UB&[-VG1+4E(=\GU@=
M5A4.G:,D<G1A>%%8;@BT*G!*V@4Y>+[`U[/B9D10!#C`1;74EW:"H8IF0*:\
MPR9+SV*U"GUO<67969FO%KQ"#!<P+H@W_8*<8##.BX/ZM+SO9R$)PNV'&-!.
M/^(P_E;Z#W'?=)]2(<.%BOJFE@W`:^7/H),'\>!A1)\8`)6Y3;JH4G6^5CI=
M+%U:"?06P^[<,:"8P]%S2N,[]!37B>@++I<']8T!O;YEJ*9%KFXA-#A'VJ(D
M6E2^_?")0HTS%2V:;$+]]W?`)5F+5BIE:G`S;/R5]I0,)FQ#4Y^)'&G^Z5%X
MGQ^!!^4?][-A2!,#L87N3"F34SL!R/<SO&0\T>>H&O=M[2P<L_8GNM>G3QK\
MR4OW/9CL>\)$GH/11-]F$WG*R\]]ZXO&\/EZ_C9/C2'?@D7GCEB0#BV&!>@#
M;G?##@Z\[?IX`T,:BU[6J_#`<B+H]KTB61S.Y0QBOJELY>>8^=JX0E.!><$Q
MF"L5N<9PGH+U8^UD0798%@=I0,I!S0^;H4L/'7J:D'+HJ;7GX/_.G5?C9G//
MF3PY),V[L>3O`&5'C@E]6EE$<[:&PUIVSQQY^ZC?T%J2#R%!UN3,E#=G1=YN
M-!=/C;K=4U4,,LO6$SBGM(0/!W1#(<[O1ZM*M3C3)M-NY)1M*M^LP7VISK[_
MS.R>&'%[/[4>T9RU]QOI^3;%'ES1DF4B7X=^B>7L`Y!.2"VG'X?=/LOHI<Z@
M*F:FRX>AT+U0O)G.[AT&,YK(*`0Q]QPI,FGP(.0A@]>/(?VA^_95P0':FHLK
MAT?7UY4[Y<.CB>$Q#7;YR.CE.;N4PR7*D=$FTD$UMSI.*@W<<X18@L'>`DV:
M8X-F]W;UL:?F]M6:8\\L_0#V![4RNJ9(SM^/)0^LO0H(_!TB?Y!U-K[':?FV
MQT3.0MR,$@%(X_R"T3-&GRQB7:*S=M)83Z!M"=DY@GX*VTF)IW,G*Z6!&8+3
MWU'#=`QI#=VS!^M"M\19JMX$![B0IK+?D$-+.#@%J8XSD7V48=MIWA\*'IH&
M'T(_31)EZ1]A(M.T,#*H,)'+$$0J]853`T`S&$KM3UH^'Z&<+@8GY6/%H`98
MV3L]VWQI@O#;,T-OSU2TK+GD5+S[;%Z=#V*U(<7Y-CUPL'(F1]VVL+F36=,-
M+-XLADO0T,IH%?#>0RN'45ZHSC:Q$"][F1K!X06R2X;MKDI9H1X4^[UO,+LW
MA]^&<V$</8\+'E775SG5.,=^$(/*>J`(QTBTNJ%0!G6LZE`^H?Z*L^6P7,K4
M0F3!S=K?G[7]>O:KJ-NRB-NC8O"@@D[N"`PYQT!&A#84;#U>(@^XK9#`6:<1
M[N_?*I%+V[34_B#?Y27KX;?%T&>-NAW=D([=@RGV-*H.9,E[VS`)5W+(L[V(
MA-T61-X.44,1'<)]KMT2K21+?KZ-7_1\_Q&T<_?&KSX%^_J^:=!=J(]3+5[9
M[W]E(3BH&BMV&1)]\D"Q36\:O`'ACCX&94Q'789N8R+"*!=<-5Q9F.ASW"8P
M#3ZF^+8&]K5C0U#4[<!\_7XMEC\9Q##.<KG?H&L/HNIOGEQN!\XET%`CZZM%
M\UA_\_N.4U#L=YA[))4W;^3>NLH!7C)VI/K2`#4@"_,W"]<5EPDV!Z;4RP](
M4CM5MO6@_6QA67'<%@&RT+;9'QG(KPZH7']V`3C':Z0O.3O.1-8A]1+-?3%T
M0+(V\%*Q[)+-Q=LOK/Z45C]GXX9-)3D3C$CXP59(G&PB5_&R$E1="6MD6?`M
M]8*>,^%*<590V*U_8)#J)<"QWC`L>^**];*P6]<B;PWD&/R2:9M>AO/2/MPV
M@6/#RB:L>W/'CLUO%B7LWU^=<(X?=NMBCN&_2^#A6@H\4(X'B6Y]P[%CYN()
M&IS+>,!6GL/8QT;NJ<,:,>6/&+59+0T\X\A+*")R)CIW(NL]&$ORX?R+*Y+Q
MCV#+QH1XL#0/ZL4#RIFIY=A1J"L@&CZ$PK"Z(2%[&J&\T_OR$O*)HHG.O1`/
M$%?HWE9J)L==P0_?2_292^6A].LGKQ\6<JLT]%9&^*T,9!@#7)-)(D&[^-8!
M;(QPMXT'(>=&*$0=97.ZH@;ECH&6-9W3.RPN/9T-??6%L>GITO3,ACFQV>D^
M"=BIJ=3:Y:NH-A8]QX?,+C8]@_$2"M])J0>>LL)N?8(IA+LIOZ@A?GF>).S6
M1R&W5G*5'.6!K6PY*\6)+C[*DU?2Y,51\S>P)+597E9_:L.017G:^>),BF_4
MK=SP6[DE@@LZZ&*N9YEF0-;F4Q9?YD]$H_*QSZC#"6D.R_K*7-/J?0&<,0:S
M\QLU_HZE_3#8()>^8T3V0WOU<:>DT,)A3_+C3I!+.0U'""DFOF502KEKK.^!
M$KDT")08D3=3+C+`0:74%Z3EF"-N3;/+['([4=!ICZ,TUA9::$':&<X\KIT&
MVG?9_-COT^S"73:G3<X9A/#'MHT$&V#5V+91:BF3D`;!8DUD+IP2FJYGSC+.
M>S-9KI%JJ=GRTSO0X3#WS!;?&I%'V#)X51B2QJ"0O+$"(ODXZ,YP2,X\4:YQ
M]9&Q-J)0%G(KB-6/M=TF<"[;QB4*:''6`X3U(,NSDR:^-6E/*BN5QOQUU,KM
MH\-N11;,53IIJP]JC`@O^)8.@BN"!-^"&.IN<G*GC6IH]+BQ2>3F<F&;=WDL
MB5QN1(Q-;K^_C8)W$3=_A[Q[+\6[C_0ONS<NXN93WC<<=<?6E'9&(;[ZCDI1
MFU4N/%-7.!?ZM'FRW#CHUW+KO*YMZFX#1!'F2LB;QIH=4.\XP&$BVRDNP/C.
M8IBEQD6\<7I+>S;EU'7I7;I$QNOAMT11M_PO.,YNRZS)<#`8D^D^DS?ZTTTS
MH)\_&%$X=\T7T=!?%'897?KNA3G^!,[L,>RR$5)FTN`:#[Y1T^3L6\_3,->@
M28,_AM\\'W7SC-<K]"R`[#G&V\L%#@BJBA-M5'4J/-NDGKF6)^C2)](S+SBH
M;0IJ[E[11]T$D3>;>-.$%_%+NBQ)/NXSB=L)T&8L=BDJ43TR6WT6:"#0'?>_
M>6SKT=K5RSM'GRX`T<!`C_4<?7A/\WA-AY4R/IYUNYR%[*,L]A7,[%C7`>P+
M8;EO`2-+%K_4<-E$=M*78[5V]C>8V3J).CW!'N,/N]-[AN+LNU#02>/*3_@=
M5NDQ02?_Q5F+G1N.?0%3.^G3XKG<"6?J8;H7Q76`'=W'^JN>E_,:+.=K$]E`
M'2&B3EU8?X0(V@&.K2A?Z%&PF.]9Z#A14W&'E6?'0#M[ZD@PGCYT1`2*_60]
M!O3U"5[1RZ<^REAH(IDO!+M-Y!7O/J,QXN;:X)N?'4^3#Z,I="RH9,MJN,</
MJ/;\8`:+J"59T&0QS)3%HIIV`DYD!><W)2[2XH$R:2"GD<:SL>5IF%(J8I,L
M;19F=C=`'B?W\C@[[:KQ3SR.^6\\SKOQ#2T9<@/")9L[[?2WU+QH?5NE8ZUL
M>@D6=^;N'VFN4.]@R<Y&6!B7/]X\D+WJ\`^@"=M"D?KG$PJ2MH^OG_!NT'S\
ML%$F]845]QYJ4.!<N:?6O#JVJ@;CVK`]=OF7</2F&9'7O5)YX\]>7'68D@>S
M:#SG(.[\*Q?]]S909<)"J.+AQ42V7(7=J3)_-_Z_RA!Q4VHB8SQK:[X+LLQG
MDB#S;DA$'IVI]3;5YSTT$1%3_<&9=JK6,]?KN16H$MH1SYR_J@--B<CET[NI
M*+3YB>@)N4/8Z5BSF:7B/,.VT%3;H#L)]17T4AO=4I&)_/)Y)VHX2"(M<&C=
M(*AW0-P[8)DQ$30%]]Y71J-:"<+E3K,X,]3#$=Z\&>JN9Y;V5,W<JOE]+\L7
M04\?7?Y4I9Y;V?#ZF?.4?AX5;K=K-F2OV:UA/=^OTDB0%7>B%!NR+(YD(Q)$
M1#.B;O*6_:;T.!>T$V>2C,@Y!%%$,T)N,COY*RI84)]IWQH1(84S]"^I@_Q>
M1X1>:S'L&#I5AE.GRNC:3H?942KN/0%F9-;D4VB#89--Y&1Y-`/<+QH&G)><
M?-+`[XN?7R;0X,E[[L/7*FGR*FIE:%-I353OD2.[JAT6PP?@)#&6I:!.4['W
M4JIRMB.JMR&RMP$JC/-UJ*?/>:>X]P#%[A3"[74[)4IIF[PV"^#LJ5SY=FHI
M%K2S(#<#K=2")C4N^3JJ.H-?(0C00;\)U'<5NR`T22)[:YX3O.T4P8N!!.^-
MYP1O2R?(B.JULZJ$U>W4>:1SRL6UU$D`[TI0<J6)G$2M[^"BDY,9(V@Q4I%G
MW(#Q[*RAL8=S)8DLH.8*:]JI]'_1#VS\F>]7'[[W9_6``*\>1H/JH)9RDL@[
M7M8(`&A/I*6:R-W(T#2F[#;A<9H]9P?+@GLA^Q,R;2Q%>K9\NL^*$MBC)2&]
M<U\LZ#S?@%E#I&>%]YK,UB5;K$>RZZ=5I%4(+(:%8,=E(T'UMV^!I[^O1/4F
M1O8FPOZ^I//BHJ>_M3J6UHR$],XA<`S@6+YP1^LNFR2S#?I<`.=\S84.EZ??
MZ07QJQTG*,!O3?0<).K2)PVF#9W,H,/9?CO6G]I:4G(0W\]0.`AHDU<T<&MQ
MS+]W'/94N,->2VH>M1$;X#"P$I_O.WC[&DQAC$"?][4<%T$UK;?+I:*4-"G8
MK^+0H**HI+XK'0S0KHY%7U!OJA=E+ZP:)4N"B^@2FO<,!T16"*MRG9#8XC);
MYRFV=)D='T%/SFR8L."KN1X,.))$_O(G#$@BJZAQQ::=:O9@P#E>):K^$P:`
M)/(2)%MDIXD\H\$#Q;U\+<X%Q78IBV03#S&Y-'!GO^8?&+5[1N7YDXL`W(F(
M:`@#SLX)Z!7$2J)XKPE]YV!;8R"SC)5%^29PI;+(+Z9%G)I)^?P$QPKT4EG4
M3H*011*R".H9)5/ZUR<PF6<5U]-&Z)@NG+]Y/,5\BQ2P(P]!.N8+G6"I:+D5
MC8M%%1[_?D43W^S`GD\^2"202V;#8G4TRN--.^TY`H5HMOQF=J</;;(/K`^]
MT1)^XX2\M@KD+L][3]DL2W$R"O"5EV?*:RO%-_JA/[^Y&/P2<:-+$\VHW,3J
M%U)^X.:J]>!FW(E:*$596ZF1^BJ/Z5@DQ7N5VRN'"*I6AZ0\C1\Z?D5;1#FK
ML%Y*CF\BFB[A(.?X%H.`@&ERI-[J4JD^IO)V>.POG&3>/DUZ=LU$CD6>S\9G
MW2_F`)P\%_#Y65!CK@??*`VY41IQHQ(T`5R1_[NY9J3R0[3A;Q[(W:$T8HJY
MB%?/M.G94+?6WH"Y:CP:YN-1KXU0O<S6HB,Y83>R2M,JK)89[X/#1^*A#^UF
M[XOW\YM@=D\)O9$6=2--NQU"V0L@`PZ-$</ZA#MLVO2LO?T4_&1G*8V^DMZV
ME'J%W!@(#IPN[W"H/T259E_H%ZLMT!,`;M:^^#7.PQ0N2,Q>7*!^+`:.[R?/
M^T+FF8W&1'2DB;P)^Y/P&/YE=I3@($U+FYJ:.L)N+&^2[O[4;)VX;+U>846P
M_9AYX%O>/D@3CJ!4_`"3BN_1[NDS#VS7[+EM=CNT>VZ9!\H55B9K)Y>5BZ[,
M0!4%3RY!T]D(BHU(!N38L#F>CM*D9U-;0D@6>\@`L31S$4X_9G%_H8!]Y9@7
M=>-5<$-C]JVJJ79$W?A48^1LV$I-CMI"(W79IC6*^!_H+0/3]WXKV18,NP?V
MB?9W,>P?T^"'"'6H5\FAL;>@C+^BB;3]$3<TH,DTZ+SLN&+TVM/F@28W>W!T
M$JGW[!48$VF/3(-7>=^R5-N"Y1YIU&Y`=I;9@$%9C&E02O>_8$3(#;^P&WY8
MO)#RZV90?MU")O3GYH0&WQ@/7;J=7R@.I,O7HYQW4,4A:BF)&)XN'9X)9P\+
MNG)_HU:D1J9O<K#?IMQ,"/X0FH9G<9-05>OO*O@"*#U.'<1S+P$J$7A.=8+3
M8$S3SPQ??<0-(;@.%C,GT_S^1MNKKA("O88C,COGSV,EP.0KTRP*)[*F9LS:
MFLDK9^"K:J:F[`I.J1-]42,$U<MKL*:\937,IGUR3J2\`5E5HU0XF=S=W"O.
MU0=1Q3=/KCK.?`FY*^$4Y`@_7M=HKCY"K46:US:8R!\1Q+R@'8QOVJ=H0"!E
M@O7^X5N%)`+6<D6](E^PJCYZQ5%8T=-'`ZZW_%>+@!0*Y0@LU>=-Y"^>/47S
MC(_CAB-,YN2@ZU^KHE$,FV8!Q;DZSA"3R8),)NSZ@?E]LY0'(()09VM6GIZL
MF5MY>,K9GZH$T-PY_^HY62IOSNYI7YXW/O+ZKN?KAPP;OB9'IVK.PGY`+<!$
M$9CK%U=4Q'C6F0N-B!Q113/"KG>L:A13]"6)S#F=:*0M`,!$ME(49[X1\?5R
MH,\LD,MX0`.2+"^U09Y0!^8G4+QF'<5K8`N@*PFVV""I83TQL"KB(8\J\Y]O
M$Q!X\HYU\'6<-#FUD<*4IB.5HN#K:6NW8YN!\@1+TRH`M8H6FL6P\\A?NMS`
M`-8#JU^9]QA]6LCU%>+K*XA-:SZ9X/4+'#8$Y*^>^@@X\G4-ES#,<+8-_.S7
MQ?7(JFZBK.7USZ&UW`ZM9>RC-A5E+;&A77KHSG_.$AO,A>NO0H8&(:T]B4PV
M#6;(<2SLNAFFJQ3NMF]WJ3JR8DO;L-^YJ@O4,D$:*.;DQ(-UGH\A(.?X%&)P
M$%T31//8:UJ7&YIKY,<"4:&P**#I"+5;:L[SMPE"KALV9]BMK$8A*&$WHA5I
MXNMZX)1[F-+0,7KJV.3`'^L.5_3@U\CKKT1<?V7($X0V#D(3Y0L6HQN@+Q@E
M45F?^X*W1=??V'I@VZI%Q\>?29-`IC+D#)YY4WT+.H->3NAQHB2+(M`#+/3O
MF-E0(UD:`=(ZBHE2.FR8@14;OV2@`PXW?16VK=5LG0VB8SF(5X[9\"X4=-*(
MO>R_4;#D$Q[T"-\5_.$/-E.^8\P_K?0OAGS'&(YUZQ=G9\$\F=!Q9&S#7J3M
M*`8S%!PK=@3;N:S.1BR-B/U3Z9`N;D:`$87.W<#?3>1W"$R=!E/7K5`LC?!4
M,Z:2A0X7-0%@0%NQ#BLVG?(7O4?^WX*E?=+T)W]QU=1',8LB3(/7H?1)4#JT
M<%>'S@1$HZ'7KGFXH0L:Y<AK+FI#$+DFOT[M!BHA&6S$+,X.56VEEH.9P6EU
M-(,83@/&JT9J-O\6/_^2X,\'FJF)["1TK-,9I][Q[FPB&__`2$2A$T#:I-(%
M(/;AR-GA9L<DA4X4<>V?2-WP96T^O"ULY8FU8==^37;X\*K9BD?6\&O_B+CV
M>'?%9>?\>A_Y=81]%,.VA#?LWMNJ.)&G/)&?2-MI(DL11*WS#[_V`+DR?%F/
M#U;%5?2N37;Z8-5<Y2.K0B>$TJ%___QH=A+YZ1_[ULA*RMA54]_C7-O-NB2L
M`)ZO<62]U-<XLA=?XY#*/Y_AO8!_/`M[+BR1>IHRTW!DC',Z;'&.5(W[:7#^
M-E=]OT;*=[K44C^BF*DNIF^S.6WU+G4IG2AE)OUVZN,9+P%WNE1AY(NO?:>0
MTN5&OR)<+F5"6E.-5YF]Q)`J0%\N!?52J9]2RG^^5@)UXK*>-U9X"<^5\)*$
MW#HL/9J>A)6G8W/XZ'#NAK2`:\O.XUNS-PG,AG?"KRV,N+8`W*Y]"SP$#O8_
MXSF<":FF-\YL4UU8TUD,]>'KH?F8F98GV#:;T8B9G1M+T[+33\_U<,.%Z*_Q
MJ6X(S#]0AR[O`?Q8CCW-;MV0?HZ_-.:!5/VD-(WJ0S03VYI0*F2\Y`\EG-W]
M7/CSR7XL'8S/$`(C>P2+_EJ\K^^$,V:8Y(^RA8S9V+;,8PF4L`PX[Q_'+W5_
M8R(/([#\R<_+/Y9)O3938Y`5S8L7BJZ]FBW9)F4E8YLD/B.X)=&+QP2<FNLY
MG*([)K='V_$2"747,^Y)2;2)O$'9Z&NQ8==B*0-]CC+00'QM-O8789T-@X:Z
MDUIX';+1?QFRT7'IZ?^RZKID/$<3C?J]0AVF/L>'?%2:GL%_%95FIVN?K[GZ
M7<"/Q`=?PR[K["&6F=-`37F4]D.H9:$4`7XXL4#'^WB&?,LSR_KXHC25`^'G
M&%('5?[77I4&"="?:0L\*_=YUG#W;]RPF5@C%S151"H/+-D2L>@@?+/#B%@0
M_VLO.Z?R*X3*(`$[1#3_>!1XF&^-=/=CX3/]6KF1U]1;(E7-2ZHCUIQ=9D1&
M0KU3&;E1U^2<5K:F"5,U<4"]7RL&W-BW\541><9PMWM_8]0U');%[8<F<K$R
MR&_MO5>N&,'72>1!LZ'42$NG>!H7/.0TLE7UF+*>`S,W8N'71-Q!**+(&.9N
MVT-R22[?D[\Z<M$-H/$70&I7%0ZM,+.5I6W./M2_]MZH4/?)%^=A(;U;<2<6
MVN'3;=I%56'N[^8_B?)8<T:A<>4=D791Y9;0ZC#+@-Z(,*\8D\ABRNX:*8-<
M:#$,%(1TZ:^V%[1[OE-+(D,['>4A9L?E(SLITUM$F5X(2=51ID%8=]6'#+"G
M$%I@OT&#W^[X^:?\"Z,4QN1]S[]9HSXRFFLIOI,3RLN;S.-\P)MN:&AEU`DX
MKZ.,5NJ`:_GZBN(C=LL,M8>A#ZTUF-O?CW*71;KM_[[8`!P%D?D12B.3U\J1
M-R\NB$+MV%[8N9.59J9-)V]9S$HTM.42_@+@]G?G:XW8'TZLDC++W.>'Y\!A
M`%A[XE<YJTUD":7XD`!Q)IN>[07-8O>J2/=7.:%%X6HCA_V4Z^="#Y"1[N5E
MQ5O65Y0`4+E!;<149J9GL6*'A)!DJ<ZWL5Y\#0?P4/<.T%`A+!>8B]\H".7:
MA5WM1ZXJNL2[+A^Q]UB[@4N`M-LJ_1%W/C)@<SFJ`A1[\&YGM4AY#(?/J@*U
M:?0CA;4_5Y\@\@6J5OC,5A6$M.=7!B\_^IW%H,\5FP:?4-_1>=9.D@8[38-W
MD,(P<)#5)N22:/F-,/<;U=<JW)'N\<`)AUII3E;VUE)T?2DT_HL\=/TJ1#\?
MTV#]G[$W>P3X'.B,"*U^!%1B9'$H8RYML?&AQ_^%5,S#X1Y'N%5A[H4GJVU.
M]BXZ.Y=6[E2>0Y=;PR+<GX6[9<#MY]W8;4[G08>V)9W;.^P<OVF`_QUT4=_0
M_HCRJ7LH+\!M"G2_[^]62C\I#G%_$.Y^3^P.#','@7:>5\"!=%D:BM71**A(
MILFR4-_>8:6:+$.)>J-6HM(-\1=6D'NFDF-6A+X4[`X)=_OPU4(>(3QPV<\E
M]+,+]]AVV'>X*H/LPKK<\D"B@*:RULNM!Q56)V$]P*H38G5#YQ)9=>%';$00
MDLC83'QN*/<G@NBY@7G^"H>56T=0GWLQUI]?F!^4%Q#H'L7=3<2\VU0A4K9(
M53KKHM.!^2*%0:_\"X[5$8F,=>`<QS5*/2F^(>'`G`,CEHZ^<B"U2%@89`ML
MZ*^WE056!:V]LW7(I_%L=\%K:,]=A0-)1*$1A=V<1#X-Z^D$)^3UZ*D]V!9(
MS]]0?85B5+]%N061/=\#*\O31>KKZ7SHA-ZBH-33I\$]?P_L^3O`4<]KZ<QT
MR>?H%V_XPEMJA`/<OF(W.\C-9DVA^<IHLB4H\6:Z?\]9V*\QLA")RNB1$=CS
M5<S,JO">`POBH>(JI<VR%#W3AC.HSY<CB)F53<7FM$`9CHEZ]F`?";],52S*
M9I@YU%(7#95.V`H3Y^`,.V8I?B1=E!71L]VLOQ?94Q_0<Q\<I)R!GIVPV^VP
M5U%06IX&C@!'10:4JCCA^:3GS4IQ3YE]?)F^""]YK22A?'+5Q/))D3UEO&=H
MY80LF>8U0?G?RJ>739&_)JQ\W3XUHN>2#WU*2,]!GQ'"DC<J_QK0TV"?1B=1
M`N=H7Q,I7@N@VU#B-7\-SHQ]+<A&:"8IB->"%5*F&O>5XF;HZ[2UG_F;!@\*
M[TF/ZLG9\UG1+XQ^C-IC[`_=5U@E5R^B-E+474;UTJR-<N70FA^Q*"N@9RF1
M;R2&GFZ=JY"V41U\4NL-S]=XB"U/S=8:!;1&CLVY1($,/%EQF:&41)WZ5H6;
MS2"C4-[)]]N');N70Y<JLN<MB.;[6HGF[)">.<M^H98T7GRK5XBS*5^]6-Z<
M)>Z98RY>5U8,'HE[EE6OU^`<,%!=0JV);BC$J4LI@3/+-U=N(C9RM-0!&%\Y
M;M;<?W;F)L!#>J8HFK-V'?4>RM"6MCW?'4HBT\S[?_%,1\1<_:,:#XKLF<1V
M884W0GK&(->H3===1_?NK90#'`J`&;T28/;HY]_4RSA(<(\LK&<Q@5MBOV$0
M^8-F:QGV5^'>;U0=_U!V/%+,0%?53X"_#8V:CE^5%QZMO..C['@,(ZN.*D]6
M(^,9\`ZDP7OEA7\@ME"VC>[KF<;:1>DLDJ9=2NW4&F%9,JDEJB<22BS\&4IV
M]E?]O/)LD+SCL;;C5\CVV23&(C&^5G3JJ1'1<;_'>&K1#Q?!=;]]J)_?&YHY
MJ!^E*YZ#I$GDIK">J>;];\-*>2IJ(K_T%AG5$Z-9E,[O1S6>4KUZT?T3T-&?
M*U90]]W%DZ4A/7\1=__&GDI;ERY_,_W[+T)[1F%*X>XZ26V&HC9S40T]'6\E
M5!^B`/=H;_WJ@!Z=4P/YBB3?P.H7TF&S+!0@:K(#*].]\X83T7T[M+LSIAT)
MZ9$M&+$\O(<#B,">V,B>&&6XX+PSPXT8LC<*$'UNJ7B>X@W^RT+L);94;^6\
MR:D?0<Q=S)O#ULZ(YR<(N0D<UF:,GSJR2`]_)#-&,TFL6K\S53MW25SR8GX9
MAMI&KAO/M(UDI6+[B[3)2S:/I],GU%,><)F;J@LL6=3#DD[XRJQ["WK'[4$]
MS-A6`<,V40D0H"=.(/-&CX6C!KV@S:&I5I&V=DFA87^_Y@'NVS^RRJ#>OL2(
MB%.=[`Q#I2&&ND%C:I=(QN&^DI&2[4L:YARI@X5`:5#04/\.`+WW4)#-$-#=
M?S8MB3P6WGT&4POWME(;]LT9I8:5NT6:YDRH!9QO,#\[QK-C;+G(B/AP=U$1
MNRZP.TLY`U6H?H=6)Q&9JAJ.J%I)U4BD]<2I6DIYO?(E7Y-1W2LU/Z>C_33U
M#+3#H7F<SO-,"UB/?_LW(A@!W:W@'#V&'3,L"S@6O`35,B>L^].3;>+NK9A<
MN,LF2\^`##:D^Z.E,R;%S40TZ5FQ[S(H+GO')S8]$T:6OCU;DYZ)1#.0\Z$7
MW#^ZL02Z)CM=DYW!\50F9D.ZMAZE-BTVI?.I&4Z[Z%CZ=C1,#]]W+(0ID)Y0
MC*1K-J03]2B_GZ;9E,ZB*DNMK'K]%<95738>V/T:P4&(6/3Y*I#/OWV#?"NT
M^U7.ZQQY[=:3WRU=3SO'5VY_88?.G6P+Z1X5UOTR1@AW'EF7E94)',Q9F%FW
M/-M0N*YASL;,)7?H6=D;,U/?_BPKNT$#(6%CYL8LV"+8FO_8#LCWL1%#KR0;
MTND2FL3SG-HD11#XO$%*-6T\PYM0XLG]1RHX5A<<J3,N>),/?=O9+0OKEH1W
M2X*Z)=2WG=V2D&[._\JWG8R0[G<PA7#7KISB$YG+\U:$=4_;O4?>G*%JSF2W
M0O_[8\4&1'E,1VV?O<]5<E0'MH9TLU(`,Z+[KYW\U:=?WR7WGN5(63]\X79Z
MI^-,5<H&P<G]BD-;X8Q+(O=U>$A)0/=$^E9:_K`(UTTL7\1KI:D.;?7\*P-#
M2/O]<T?3VZZK.M`>XKIX!2=TB$3'9$WQ]WX\WH7#<0MS=6"O>PZ89++E7/GV
MS*'O7,>'N,Z(76<QF7!G<NS<[%T$865MG4,=L$@(]4G`S/K7)',S`UU7H+]Z
M6F*+#G8ULS)%S`3:DJD^IU[V?N".L#B3([N#BW#0S.=/W/5,==[!)CF:QU+E
MW$R`<^NXBN3,?Z^GV+7W"A[KK6?BBTKF#&/_TV!VO,,^/3-EEBS"]24V71CE
M^@I9SP!I$,@U#]&]K=`P:?Z!*ILSN4JNLH42_##<58%-$T:Z=A\I8[$FJZ^A
ML/I[&A$RE.(T-U'%@4R.@JLX1#49]D/Q?^X'L2LGR&7#XH3;93(C:^>B6$DV
M\B$#=D5"Z)+Q+(DD$_8`Q?6]?CUPSH^!"EQKQBNS!`&N%4&NRT"__6ME[3YN
MR/B4DO"4#<*4]4R0R\Z*-R*B6([9B/@AL7_IHSX31-9^-SK0]8-R'$U16Z=R
MULN=3@[GKXH]/[/L0CCOEMAC5,X#:N=!5JN0UXJN."TZ,D*I0.H3%DVM-"(:
M*,1$-B/(P@^X/@4BS3A:;&^]NM>Y<#^74R0BH,S,@]K,`U?T2<^FF4@*$;W[
MEU\$NISH5!'*FLBW">5O"JD1'AEZL@A)#97.]&>ETN&8^B:@TC?]O>W3`2<P
MKA.$NQ("71-$KH23@]5"(S(!5KZ:7Y\,W5E+]2-@Y.Q"U^[XZ8K1)P6.H0HX
MX1C"3NMP@/4G]R/V4(Z=+@^B_3#"C+_?B2MQYD5CN$M7((2,?O7.-[UYKAAA
MG@Z'^=4KN<,XN2)&'<V,<_*BN7DB1B-MQ=%Q+L?\&D:!@;<7NZICKD)#74D^
MRSB<*T2$:RV1/I97QX+9+5I')PZ<X:X0$^FH3T:,#`X;1F<C2)=S?CTCWX`=
MPR)<?XUT+>)]3^QS1;H^5]]*,Y%;*;[LXH:XN.&N>6&N^<`*/C=Z_FDIR396
M?9U\BX!;"A7`+-N0O;PY".#SY]ZB#IA+*`!>$KHTZ9*)[(>(E,4S">FQ(O,'
M9R\8Z.J)\R[3R_`&2:?5-T$T;ZJD!"^)+HDID93$EDA+XDID)?(2HD19HBA1
MEZA*M"4:B4IO(M<C2&37W;A;UKA;:=[Y*./0_+MZ0UP315V/`UQ38H:K%7I$
MOF'?_*W4UCJQ9Y#-TZM"FY4<->_L:.4D@]F:DT2^$^AZ)=0U4OF7/LY?:(FT
M5.5X`Z>1Z'0HWT02:=EY\(:EH`XC9(E<+ZNC4=5?^K"QA*^+VN+<HD^DK<[3
MUS6"&A@1N33@>KA+`Q/!%'XO4BRA4B@X:")ML2(:!0W4)Z/4%[KP@176^WBK
M:I+!1"X!-T-<>*`+CW)%1[BBP[OZPURX#9<UK)`UK)$UI&,V0M:0S6+Y[BW8
M7_1EUBZ;)G./+'.W;/8]S>S[LD=*S2.5YOQGLO.?FD@%U1?''>!@$OG&\1IP
MT+^+/*Z7RH+`ZU)9<!+Y.A@K[MIW?+U<%D18B1A9L-E:I.`,5W)H2>3$X[6>
MZ_CCM:!>(0MJWA]#91G7E"[GU)P\"$XU=S23H%\K"XKJVJ>1!:ME0<<OJ&3!
M3<>E,J%"XL^MHWD:J)#Y0W^#4TN5(0M6RH0:B3_%YV3^,(>*$DF(N^Z"S<V[
MF[Y2<6I5LB"E+!B6)Y<%$S+!.3XL'M8^RD3.I]H2WE47UK51W-4=W-43U-43
MV-4#W."\J&M[5-?'$@Y+^S,R3S'B"K@,5B6A_--8>%<UMH78>7Q[LV*D51&;
MN:UY1ZM:FJ^4YFJD>?6-FI'5JI%5JI&5]7O"NZQA7=]#$?^6O^B/_$HJ?WVC
M1P+,KH!"_L@/EH=TK0WK^BZ@ZYO0KF_$7=^*NJQ!7=\R'A'^7=\PDC&&CV_M
MS)V)=>]LG56ADW^)Q&7F0BKWI2(S7YZ9%]QU2-SUU8ZW9",C9",C-2.C@KH.
M!G0=W#Z',0(C9*<T2Q#U[#V:V;LUL^MB9+=B9#<ULMO>/;+@KKR`KF+IHG&2
MX25KM=QY+!\"1HI0$_D+.`&<D5TCJ:]AV!^/EO<2\Q-H%YS`$-)5+.YZIU00
MT14NZT6/$SU.7^UH;2\MJ"L_JFNY8M&D@O;@KNQ*8417ZKQ=D\X;#LY9D23J
M-OAW35_R*:-0Z/<^AEXG]DLD%])^U`=T35[\*2/K_VGC;$.B",(XOCN[9WG>
MY=QYB)QAXYY;ZKWLW%V<==&+YE46E:"]P$%OUZD'ZD*7*'V07(.*(CHU"*07
M*DR+P/LLP0E)"D61%<'.ETBI_!`JH1#F-7MY*1@L[+/[W_DQL\/N\W\^S)AT
M`0-74QP/].\8\#SV"KX[@N]N$7UCOGOVB6N.B>L64C^>751Q,(%YBV7T0-/:
MC+A7B\I#R=D7U7GD%./FK20K$>]^R`Q9!X=Z'Y4H!VW*H6*E9GC?A<L%^F?`
M\9ZY`>6)DV;"ZG8:)-%05&B41'V12+-*'JFZ<C%QIOTRV?B(N0JYC,,)M]R3
M3^YGY%MHJ1#8JALRV$?H8;2/Z.TC6AXB]^EM*J]2"M3O_':#<](@?38Z)_72
M9%8Z[^:I\SE$LA)WHEKLZQ?[!L2^)V+?TQ)/P**.95XQW-X53/::B=,E?%FO
M?EIWPG+WI5.8U-8#C=J%UUE!2^\8_0R#R?SAP0TDU_&+VN%MI>5`\OF7_@,!
M<)QI#RZ^#2X.B!N=P<7FI9QO)AX%`>78:&$HN6`F+NX;YG]@71QWHY["6T)J
MRUI(2F-,%]L->C@3,2J0G<(/3.RT6W/RN=3?F8A-@;;]#">`+FA2YQ.%79!Z
M\$M<)U#8#H9YAUF;MC\<)-9_''4AS9E9P3$I,$=]GN*L!(QCUIMJGQ5C.M@P
MK.H`85C9P;W9D*9]3]-FEVGJ[__0$C#&Q4!,H[['[&[ZU/Q-&$K.0W5NN6\_
M%:@-9+`3+@UIQJP.=\%E\E>-/+"JGQ\P6Y.:3]#LD3XBG<SS._GSP5)JUC0/
M[+4*>J:IRA@-\5R=(2YD"R8NQQ(-`<%!C?JH7YL0JJ6J[;]>!H`IG#J_$KEI
M6C5D,[*@XV9HI)-M.FY6\]U+.IA.^5)9X,$,C60;#V9QNG8`0*LE],SNAG!C
M-"*C/<=09:#BR-[M&!57M$0:SZ/:EC`J;ZE'GBW([?5[R_P>+ZJNK$4>C,N0
MW(SJPHV1-M?I:"1T-AR-U#>'ST5=(;G)+S7(36$II4JM4EWKR<TN7XD#':5Z
FA#:K];:U(8PWNW"9"V/D1&YZ@<LPAOP:IG]3YMP?6%ULKS1=````
`
end

496
sys/dev/cxgb/t3fw-5.0.0.bin.gz.uu Normal file
View File

@ -0,0 +1,496 @@
/**************************************************************************
Copyright (c) 2007, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
$FreeBSD$
***************************************************************************/
begin 644 t3fw-5.0.0.bin.gz
M'XL(")H1MD<``W0S9G<M-2XP+C`N8FEN`.R\>5A3U]HWO+,3R+1#!D((\R:$
MD#EQJ.+1MM'::GL\%6WMH:7MB2:A=%"I57$VC(89%84P!K0.B`K64VFQ[<):
MG&A%M#@1#(J(%2EM/:T^K>QWK01M>[[W/-]SO=\_WW=='^W*7GOO-=QKN'_W
M[UYK;<W8<HRDVV%(Q3!,``.)]6%X%L;$T-6!T3#,@WF#"T:QF3-GFFY@M$P2
MAREI'A@H0135B451`+N!,:;AWA)><Y#8ZTWPV@FOATDL20"O+?":!:]'28S>
M!#"L?QSVW3BL;ASF&D?',?RN<8G:#[L[#H-Q"2:BK@*CF+IRGH_^HS\3)9\C
M6/;\B.]61'T;P\'D'.Q_^U9,?4.?$26FSL!?$=4!C+K)\/^)NLGQNLG38IZD
M+9MP%]8AHHX#HV$R9I@L,$PF#9.-?WCS.3#&3<;B)@OB)I-QC][0T9M/`JE/
M"1S;2GJCGXQ%6\"XK60&C!P)I(Z@!T>"J(\DU,?`>)'T2Z:9L3#LT1]CDA^L
M`HFY8IR8.4OL-X%@,IY9.6X`OE@QQ5?][B!J!\IJQORQQTD83\'W(UXAZE#/
MU/D:N]5T>.:RYW]\U/+*3)/W1T15PAX24Q7_VU3;4()MI\E,,BLZ6W94MGRJ
M("?A\(+MQL,3ED^X9Z$&H*"VISVV?[A%U#I@5$XV*2?/5$Z>K9P\YW%?>"5=
M`\;3]7Y>H58%4NE;22"D/3\.FSV.AF,`5K^!T34NB%I?FB"FUC/.P^@Z%%W'
MN("B\E=@BK6,;V%\+8P+J#5T[/__^W_9GX7Z[3^^B_K-%VY@?B?_8QSB@@\;
M_#K&XM*QN'0LWN^+'TOTQ8^9HZ@O8VY@+"V\&J.HKTIA_#<20[A">^<&QK7"
MJQV%&Q@!\06'SW'Z6#P<!A+&+Z`K#`H8OP2OTV"`N$5XT!6&YV&\'U[GP)``
MXR/H"L.K,'X/7E^'P7P#XZ&RS3"\#>,,>'T/AE08%Z`K#*MA7`RO!2C`.*JS
M!(:M8_%&&)I@7`ZO2+X?85R)\`\%&)_S/X^_WNB+O][HBR?/_D_C`?NL'`4?
M!K_N0?B,,3#RT3W&IRAOW_XV"M.VR4TPKQ\,=@P+Z:5&L=[]GW\_'=X_@<JD
MCCW$O:6S"&@=:/`^E?3>"S"C=TPPMMU;^7V4/Y*&V6DS?/GVP[+?@_;#3E&_
MW&.8O%DP`E[#3=[R[%+TZWL,Q8/OO.4:,3-\"LL*1[8'PTPH3OO-)Y\4AE`[
M3(V@PN63"TO`,#I%?0';!&`R"6PL;"`U`M]1OWSHE07<CZ*.OP_+@]5Q2=)`
MV>&]#4(43`+_1JG[&*;U-@:6A<.^LZ,;BKI&P03?TS`A+.-X.@Q-:'Y2U/?H
M^:]F5#W_/GSWA1&V$=5_#SY_B-%^[83I4F`?"SRP/V&?`R0]+'<F?/X>[)>5
M4;]1EZ`\Z1MA]7T0YS$L3D#"#KT&T\-Q06.3"NTORF."(1&C=Y;`ZWN:!(M=
MD[`:AA(8=J%VK(;R\4D#K,>(V6';O(W":-1]3<(A#`86K$>@26B&Z0_!\"GL
M34RH2?A,J4DX"N\_<WE@5\*R9T<]Q%CP^CH,J3!D0;E*8!CZ3_I/^X_(\/_-
M/U*`F1X',>Q+."$?AT?/'@4I!LAP&O8XD#23]]FC@)[]^SU*\WLP_RG(:2[J
M#W__6_FP^1!#YI7[KB@D_"',9?C"BQ`K7X0X]3>"Q)Z'^CZKY0_Y73`_Q,_Y
MA"_,W?-_#2_>\UWG:&%^F'96RN_Y_2%.3H#YZ!`[GUD%[P=);&H/O(=A.L3/
M63-)+![BPR3X;CSD>N-8)*9+(;$XB+\33Y)8C)3$HB`WY,-R0B$6!\/\P?`J
MAL^X>WSY46!.A`'F8</P-,3KR1!O_>_3!_UZQENALH]=K?_VW__S/\'__/DS
M,?;_<;&OE]+>*,7?+*7_HQ1[YO]`K-=+:&^4X&^6T/]1@KWT?Y"_JA2+**%5
ME^(UI?3:4@:KU(]=ZL\I97+_A^)4P<PE>$T),Z*$7EO"6%7BM[K$?TT)<^W_
M4)P>;_WN4KRWE'ZME,$H]?,K]?<O93)+6:Q2-KN4PRGE_G>R],#,)7AO"1?6
M?ZV$<;C$[Y\E_A^7,(^4L%I*V)^4<#XMX;;Z9$%\!?H=5@%%<5!6Y(M@7>,$
MU*CW%MUW8N?_>)\DP"[\Z3X+^_;W>U2>!^*CB/HE@\0S_GXR&CX143_1!XV,
M8:,?,&XE2Z.WR;PTB>1@PN]#`K\/H0\@`BSY7AK\O?0\/^U=_Q+SX>0TR_V7
M@[+X8ECD`PQ[3\``Y#D^O(I`IUP6J91%A'\?L#Q=HS)AJJK;MLU/QYKH&O*M
M*PFVIE_RQW-'$ZS7&XO51`N-11'6'V+!%C-V6=6)7>CTIA:JR;=B.^FV^LRQ
MM'9OVE4/N@LF@%3B=H)UBX-($A9_4M22<^2*1R$)W/7SQG=IS=_%243$G@1;
M>KEC\D7/P5QGY*I*6O.M[5)0TIVJD@@V1:ZY0:RYS<B3;MS'`.E*B4`M$MCJ
MIP-7[A/`KI)@JVXPK-?]0)VM7J>1A!!S7@;-6HGD:`?6QL!:0YM:-9)@[J^!
M&HGT:NH5<V+0.=`)2I2B$,;K'+4HV""2%DWD+Z7MN\-90].+))IV245NK4,_
M4Z"2OZ4VT1W/-N]5S:0W.5:]:H0-U,7BMKH#&EFD5A:Q]A0--)WZ)'#XF+:)
MJ6S"KI!T!T>9,0><X]JQJR[;P5NH>PLF$/\0HM9OSCLXUR`)*;J_Z9<KGN('
M<1+A;E_[5WPSQ=OR_%6MJ-U!W:D7FT#3FAN<-;?]5"*_C94TT&QKGI?_!'`!
M.VIL'P_460\&F6E/8ECNY/Q)*DD0]Q;!?D`X(U;_2FM*VQY<*^E.!2.;(M;<
MX*VYS=*+_-4BYH9]-/"#[8>`_>\4/Z$7"6O##FP`(VEW:,T--2)'&/`H);2-
M.SAK'OBI1?B:?!KHLWK>UTC$9JP'VS3),3E.(E9*@IBW"-9]HCQJE;>>T-H0
M;SU1OGI4(K92Q%F=C^H9V:47!:I%HN(G&MZI#?^]*G%1>$T@\&S<0:QYP'2(
M#2(Z+`KTV3PKX=AI1/!!$!K!OL<C.!S:-*R7!`?\5Z!:(N4/T[@-M(9W#FSP
MEM;MNNBQC,9M_`BWC-ZUC`YBV,:MN.7AORP/1ZXD6$9G@,V6T<D8')BK)JV)
M"<[I.?9$0H9&!91:1B/`=LNH%/T$_D$7'E\U7HV`?5ZJDONK8_&UN;2K+C3<
MS_UWPXW]4:_\+YKTT3@867&+GSMQ8QDG[XF-!_T,L3A_%`_@SK$U;Q`/3P@=
MGE`FT-77@-/KO\3-M&S5#$5*"=-!LAV$+3U-45_=8[8N7*XA"=!)'!$VI"F7
MY8"J55MATKDAP['L3SBJ+3M^SS`U;EFV=#C"^O[X_''<7#'+25NZ6:&*9G!_
M3G`*EY1+P;:K(THRV;\R7E%`\_>?I9`GKVR%9:D5'=+<B>#XQK*)8<-"QQ-5
M[[-XILI4OEN83U:G10SS:I?S6O'*9:"K>D78,-<--"OI&S>QE"]@K$%<&8MO
M*&/D3]+$RM:7,8HFZV.CUY5-TD\C03.P^[?$LU@SK/:,L+NC(7='51]OP#P.
MS%SPQU]]?;5A9KU^2*K?59UK3"M[(NSNOQP3BL8SN[GR:,9A-V.FL&IAI3E?
M4VNK22Y_*_+N,+O<Y.\0LF;B`5"J16T%,%9J/74QY.YM)4D01<*]CH9A[?5L
M0]99HH>K'<@^^[IWU#>!DH!/XE<[<`NURD*]U_%W.#6,/:F)/!-ZO4EPA4P,
M<*#%!V"VC&9?(<''B0%9%JH;4VN7YPL2`S[H<IWC;_\2;`V[V^D05@*60\AV
MX.5M4'H-;(,\67-4JME5;1D]Z)T'_MUDR-U/P^\>52[$17<_8;'FV.Q)>4;.
M._/>WO&21E:SI&6J>HLBQ<4H-*^Y;-3+JC^>?7I08R;BYHP2GPKK&U3U.>[4
MU?DQQ%&.]MB.%,!PF-?E!FGKL\]^98:.X14SDN_UDZ^9:9'P)C'@5=N\(-@>
M4Z'QBIFW>%XB;^"\RUJ'9]6!>0S&+`OU/@::\L89%C(X_Y6PY'XPV':IB5,>
MKS8G-S=P.+/4UN0U^;@9ZZX25`MM"4?4TP2]`,P"/P`[<=\WB*51=PM#[A:J
M?M[PSOL$&&FF,)<#\Q2D/;@'T=A8/('M,%GM\QWCB0U<,#/B;E9IGWPAH]H#
MQZ_PW<J;M;?*!Z+N;O2-'=^-PT$[O+&ML/HZ&KW^R+MK=&:"N"0\X*[/D,NR
M-1?.,BBN09&-^CSD[@=*,QK8^N&]#DW663BV72[?T,)AM7/NQZ^:#0<UQ4*]
MCJG?6PX.N4V)O),%@D(A:+)0%LP[^IFL!\**&RPG+KUK=O:5>Z1W7])8DU6R
M:DU6O:I*JE)46RC<I[\GPN[.#[W[TD73)?*R,2,Z\.X"_#GNCNR5+XUG;\7E
M'!KHQ/\6S^/-.-6M_GA#+TD?CP?<P!>O\<-(!V8LR$B'M8VPY\9O!-<M5`_L
MP02\B-@Q'/"D"/]7O)_?#'F/@+%\TIEY,/-YLJMD\0MC&7EIQ-[K1S/!=#!"
MGQO/Y\\X]0`F^7.Y">Q_Q:\`)RQ4&S#CS\6O!-<LU.7W!"P]Z5\J3!GT4PX]
ML&6T`I-U_IOJ:#SD;GB%@,.98ZM?+KT;$G$WNN`)C9%>)29:"6L3,)!$D;@V
MD*"$>ZA:D>''G,/#&V_@9CPQ+Y"]%R&+0P2Q)9]DMQ*V0\\H,[)SQ=:,J=82
M<P$9=3<P\J[@BOVR,>*N(%]<>.7;IHU'I"UYJFF"G$FL$\*`-+PLW3]'O*N;
MGXQOM_M)Q>?YBW/C'-M99X0LAWC/);[#J[QUY2Y'1>6.1X_*Z]6-`E45F2>.
M'/JY,#!DZ%>'(%\$E4Q[*/N`6W[A+-'`57^<#2''B[&&C)HKG4OO0P1K5C=.
M4-=#98+B<J"XS2Y#1G5;F_6'<AU)P+ZI#B3<PCWN:I&N*R<OT(NU>;XV%O[>
MQN8T75>V]=S[$#%3;I%YXY">0-"T0?M!)G/>C5=WT)"BR)/7(-#\V"$.'[J0
MKS-$1TB&SG-_96\_1\`6=:J-@K"A3J)!P*G`N0UX19?5/OM*YV7C17N/<>.F
M0.TTP=$KBO&2_"G;NRLO.B8?GE)^B>GKGFCE%DG^J?)=E;L=)_*_+M^CW!)<
MV5"^UW%&VT&6[U,W2EN<DJ&]#C%8_FZ6[(H=E+(<;)Z#PW$2&\OHI8ZC#G:H
MZ+)I<8._T^',5Q9(X5S($P4.U313JNCPN.Y`1;>(SY\+BS#$1FQT^LEC(PP<
M+'AH=_A097Z@WP-A8VL!*>\.5G5+X[J#%-T2>@>N[1:KZK/EA\^JJE)4N[*M
M]M6VA&:?I=,-/;!V7D(06B!,Y'Y_W@7F%3T![O&')]4\(9](MSS<@&$^B$7O
M?>;Q"ID?F,@S6*B;R"Y#_.)IX?3%QIZK+-37&(:K<?I47"NBN1/`$H4\)9$A
M1!B`82?_#G%8(Z(E\H30.L?)(L[SUZZ&),S:%`9M<LB0V6N6V5ZSK./0QNSR
M#50Q`G!A(H^."@H?6E#Q$_@@;.@?K.W"RA$FG)\_R(_7:S*JY3X$SX$(+D.8
MCR1/&OWMO$L.Y]O754(+%>;#AWF!0V^&#;T6/O2:9.@U0BD4#[T6,C2=(>?L
M4NQUJ#.R5=K?""?N=%76Q65DQSRWN24M+B<[CF/?G1QKK?7QBTOD16.&C/>L
MT$]&;,FPDLN9,[B!0U,OD#LR/XC!Y;K[V?;E\^8<S0`EBUD2S.3`4@O2IM`O
MDF!SCGWY%-H%<OE"B'C73Y>#,C`];&@:<''S">X+\1S.C%,-$4,QX4,QVNX-
M;28XKQ[!2YF]*_4<_V@Q\/!?C-]0`K&I&Y!'L[>E;[-OR1CC+IQN(UBFAE@B
M2-GCIVQ\8,MZ11,M4T?+.UY!5F*N.E:.)C_]KC8:5T-;2\RQ?5FAEP7IQN,J
M#BW`3=C.?V\8SXJ\\]#ZT[GBR:`%Y/!;II2+#^<ZA3R'L#SPU*_2(;K23MN8
M&9@O+B`3F?K(.S^Z7=:=`<5OF3%5KI#8#TV`MBN'LP]?5[DH],X0YU..NGY'
MBLM/*\/6G$I0=V7#Y#LNJJ+U%T?8#R:IR6GL^\32PFEG-BJCHUBI9AKW*8SQ
M5/-1QIDI)^9R6SG,5@Z7F,[HH#=W$,3TQM8+3>#8.M<<=73LVGW/F3$M<%FH
MQHLE%NHS#&PY,5>[3DQPIS-:Z<VM7.[T?<OE<\2K79$P]?)]TL96>>AOG*,$
MH\V,L9^BP5IFL[.F<&=R<A6..,C`EF=&`GOYI0)KU>7\Y#Q;Y=6**V7?;OP`
M@L7TLR^9L5"H-!(_.+\9?\43_?<7!D7>.6RK3RI^RT*=J`[R:HH@D1N!GA\`
M\[I=AGHF`'DI9>Z*'L?;N>^4]SJOY;^3]_:^M6FMN+J15OE.Q=L.,W"Q\L4L
MA^EDNAH:IA(E.:TT>)MTJP0D@'%@)B.+QI@2SW(*P`P4?2*>%6_*U'!W"I@+
M!.KH*/8"W&\FS@TQ<=1BIE/@ERU@3,`Y#J'XSA[X"N(@O&4Y$<L!6^"/*CIV
M:R=;)68XA1PG#E/DD9!E-J";T#O%XCM5,.(\EZG>UEU^,5L*\VZ]D!6<*<FS
M%28WG-Z_/NTJ9.UA=^Q,BJBP52<[%`6JG0Y6!U&N((C9VMA8`&K?0I-JKW#?
M/NWALP><^8'*+=G[',KMV9:'7X'-8`0<`L<=YAAR6KF(F$;C_H4&#0?70>/4
MQC,I`:M\"G`1%?%,AX`%T9?"P1[.!!I]ZQ1U-`-*[C=-P.K`.1.$4&#8!GH#
M#IL!`5H3'94IYFK$7"<.E3:?#+_S.JM52&_%(9IGJ@LTUM1L!QR#\IY-;SO>
MJ3SG[*ZX6-9;?LTA`JEIFVC@?>XJ0?"=?[#NXZR3.-.!!]])*N]D3L;++JA>
M'P4CH7=F0DZJB983A7!>-S@@)]5V91<'-K4>[-!V9ZME09I8>7.#6A%D+?GL
ME*E`D:_:V:%5^$,+S>7.UL3&VA).CGD6(-57NT.T]4+:I@#IG7%;0I5V'$2S
MXN(S0J5WGN9^(*#_A-._Q%FPYSLMU/HQ`#;#:<79I(S&4QP^WP3")W"I9!'(
MKW$B!/WB5N`=P7]R;"S4J;9J5:S6C.'PUS=-W0G(FV*=18#:MD<="V'\YLFY
M/NFK%(^%1R!JUHJPI-$O44H?C^;6H7B</`45I8O6%@AT\K?<"8EX&7H.23AW
MMX5Z]@H)O9Y$?ZZ%^@O$)9:*]$\I]U-6/;!E"E31.)L]Q[9E/VB`:23?72<^
M$^YRZ[MRVHYM@)8=_R+DNVN/7:,"DMU!V([M@[UN/?:A;E'-TDLPR8<AWUW0
M0/+=Q"PBN<.$[7JI;E&UM:O($$VV#?$]YA,;#>0TC/44!)/FSUEWIG!;V;S/
MA?JA\("`Z<QA>O,PO`:T<C5'PR\V;?@E7`_!Y(XD+O2W@"\(Y@]><&`^U;R*
MO6D*-\T+#A6!$![2(+_&X\&@)CK&$!V-\&6N)C9F[:_3FHYJ8V.L\RX%?W>V
MV[@[#A(WZ\)OH/OI[R"LZ5_9ZHZK($Z9F:JJ8>OF3O\.[E60]B6N^?I^-V2W
M%T!3\3A]-(-/)2P9]))Z?GF\@4QNIOC\6=`GV0@KI=UU&".^VZ4R"G2=N%/`
M*A:#$@09F2"=EQN/_+1M0=LE9<&E8I`*9BK(:6"^GXW&L-(8+?%^1^+])YFR
M!2R'@)DF8*7A<CBM[?!%OH;C$'!+!9"^,V?BRF@R+CJ6T8JSE&+Q=S7,.#&,
M,]*$,`-TE/J%C)>A:C$6"*!J^;7BX#7X4]G&>5?H4/LE"V#2RA/EQT$S7VI"
MVIN,(ZO<GCT!"O9EV^M9X\M/5IXJ/::0B8&I^:TXA;CTS+:OMI_.CZ_L</RE
M_.M-4\N^R9FR*7C[V>PG<B2Q[8*L2=E!\G9A_N0LL:9=!.=WYVSD6#ZC-`J*
M)A2/AW@0\=WKC`:AKET,+NC:@PHUAO90?7N8ICV$76/R'Q8&-$`4Q"._>UG-
M$<!(@!O7M4N4[5(U)R#DNP3D9!9#)W,7I?\Q&[HC`0>X^I^S3V[,5^2I=K=6
M*K@-!(<S6XUT-\6GNW!$3^X?\S=-B>R=Z&E`(ZZ-A3;\%M)0.+J@*VF4!(.6
MAQ34P43.`")#7H[$N6&ACF'(=M"T$AR:CZNIB7X1MH,'\@(+1(5"R\/&,8IS
MU93(.8UTJ/9\R'=Q(-/9R;HM9"%(#OE.JI$G*Q=5:TKK$3%,KK8\O//'M0IH
M]_,FETI/UEGGZPP<AIY#AWT%=4Y9\,":L3RE7)I2%F20OZ67IZ#ZC052J*^T
M?:@NI(S<.1V?%QB7GXC21<?P:_!3W?I8W#9?ZX4>:2(;=@(8,>RD0T?N]B_*
M+351M_^UU`%50:#[49'2R=7(,)H0YR$"/JK;4FT]=U\G843>'HZZ_4"[B-G6
M?>I,7K)JA'9JYYJM(F@>V@Y!VN!'7G!]L\>,C\,P"`'AMZ]#C[FQ05N?L^[J
MNP&?<71=.U(Z&47D^MMOR.NSK=-/::(-FY*Y:R=Q-Q)+'\Z%@J6VG=1%DXFT
M>86*@KB*8%X'L>;+)W,MSF<<B\IG%-MJ9^O-`@!J)"FW)E<]KY%)U;+`RK]6
MO%#V;&'R>FC0L5);@I<\F"Q4+781ZF$#(K'!B8PCMH-T"W48CE?9LZ`/FJR,
M\N=K_UKS@H4Z[H5.V!]R9-PA`)L%:BO=*>DQH8ZZEJHSTXLDU4$U`NBJ`,_^
MQ;H>,A'K*@C*#VYLU%KI>8I<U;X&2/8K%(YD-GNV*C:6Z2#*DPTRZ6[*H)!:
M1D]CA0LA$O`>0B20@FT`K?7P:N-U!30>;Y9.GKS^5RAX5]Y;5M<*:^JRJ^8>
MT[74*Z2W6EAEC<!;9>3MID3LG\7!2ED@*%$II,T.I2+P)(7F]L$_S^UUWKFC
M(]\^S[>Y3RW],."R<?%>_Z)R597@9)_EX1%?<WV6@CG3\G`7QIB#^[^+Z]XC
MX20JED3>WE8KN)::B#UAH8JP,?\`S?<XRT.`84NWX9:'KD>%'/<D,B>,V0=.
ME.5A+;(ETD1.`'H6=GLA<$EOOP-Q;2MK2,CNP*M_"KG]1M4/Y2/JP_7*+=70
M9T/3?CMD]I3/L#!66!XFP?D/IZ\Z.@::$W4CY>6W,2FE4]\N'Z^)CEZ[3^LU
ME%J?H3SE6P"\_7^S`&@[""Q4NW?]A_FD&$"G[RH<7S`+1,<<-=*5:"4ES41K
M=EO3WXR\/27D]A35P(:NA,57QD@Z\``RUC5-?/LO'Y\O!J"3OTDL-^'\7%.I
M@%]L.OD32&=DQ(,?N$H!F`EDC)=IC(?Q^'P:/AK/<@LRPIG#@H`E"*_DVE'@
M88S&<^`#!\YSX(7[(F__A74)AW?^!Q+*F[)<#"7$;K2*!LN<B)<V%S=D"6H/
MG[RMJ(L"/_HKQ&:,J:@CO5%^&@T,!KQ)8VZ:$EL7&U`TQ2]6S%PE8"P0^MEP
MOP&$V/X+\&T?Q]8Q.`-"5J*0I<3AL^U'@-EJ2K26+/#?(`3W<H\YON1.Y&Y_
MMGR6\SGIX,/:Z8Q7A0%./&KP`1XP"\I6:RJ?`>\/Q[=YX,4Y,W)P2#QXS^`B
MB`?"G=0!IWY+MN+>V0`'%S)"U-=1@_WA@T,:^/YSZ+P>'-9=SU8</LM[DJOS
M+0SY*;T+0Z,UEM$M<#Q`$]@%@"UUQ&$$1M8FQ)IM]O.@$Y2"=#`22TXK$W&?
M$D`3.)[^$HU>',]^E<;>%`^,D%MR,^,9$P1T"TZG\,QP5II`-6V471C/<0MR
M-4PW3AP10('Y;M2U/"?N-]6DBV8X38PI`@?)<@MY3^*^M4*_3P2L/7CY=$ZK
MD->*%ZN5T;%,"IG,3:+:F0R',&KPX,E!1724M_?],44T"6.L%+QR%C/2%#;8
MC(87$F\'7OZ<BB2`O0L0-<)]3F_+E:5G>?M]+8<=TV`8P8'11I%@''1;W::B
M>:<]AQJ:6W4]@H!5D\H%Q0FL-*+V4OGEW)>*YSN%SJNU5S9D<IH[8E\7G%IE
MH<YB#D&AD'6+*)NWO_#BR"K(W'^I?JEL_N\]9RWZRO9%G87:ZEOSAGZ'U3C+
MSP@GS5,TOZ>:7V':IO@]KSRQ<>\$NO]T9C*].=G??_JN"1LV1<JB8U6AOWV@
M"<+C"78Z;.=3-/RI9KG_C"ET&><<O\L(AVZ>A0+><N.B&<"^9)48;/O6KB"3
M&=OBFZT,QBSHJB]_B8[F0.C@HM()X$;XX+NL]<(J(YSZX8-O5H]30E_^6+5R
M9SWRZ-NKS_%A62>3$(LL#4G9[H=T%6$)LL?N!.!)]`]'0*(G">G@7.)[X1Y*
M]6,.<*S.7Q4R.(=]>XQ+YI'L!L+FLL;\F&TUOADV.$?%L6O)%"7Y%AQNKD/(
M<`M9;IS7@8,LW0L">(T8G%$^KMH8,?A,GL8YN6KB_KO0X0``?%3U1,7XJDD0
MP[07SBI_S':;E`6D\N=LC=P?&G&6TP=T5SV)K'XD*>2_W<922=&4;4%6#VZ=
MUP[;`5Z&?%@#^?"A3Y`)AHYXV]5"X_*+?OKH&-OUH\&#$89E-2&#H=[EL:NJ
M+8K'\M=]K5]6'3HHL?8=+XRWGCM5(S[=6?1VU&#DJ=N&$=KZG\3(PD)#FTB?
M]:W+6K^F,,6,J;UF=I#E-;.J^IRVG-7YMLA!OX#/.?KK.U(\C&)RPX.78Z&M
MG1\=%ZUG7IS$O`2=<MK%3J^M/:Z+CDJB5B(F?,$%F?":3)W#`D;*7RI:5)M8
M,[_05OUJP5M52?G)90LJ7\M+68,L[.K<H&_*S9@()%BH/6@3)Q'WV%R_0#N*
M>8V"))$Y'YK1YLH@X()VU&=#>^!8QIKI-8+(08QX0X@LFC*)HA4E?[BG^"VE
ME7YP8]JOM.;AFN3:MZ`PJIU.*(Q349C"8LU60GOJ)JI3+*/?YXL+%EXAB=\2
MDJA_%*84O7VFY\QWT%PF(',9>>L.M)C0S4=FVE?^Z.W\H%/3D7UF-+#4"G]H
MHWWVV=;,\AJ%JV;=>TJ4&8#(6U>K!#T)2:-0R4[Z;&>O<>E-9#MO^Q=5^&SG
M:*2OA85!R';ZK;",\K#BH*]'O(0*MEL,2-E.NG=5FT03!1I-UFL6ZBH<(^(+
MX6XT=]LR5F?"3CS)^>3/<_>(',Y=4_-53\&XL1;&(6H\1W*K(O16>>2M[6TC
M^>,+)L#Y[%NH]TWI;=,AMH&1&I-#4_E<U:SJF;7/0EI</@/JB_\`BQ@6[D8(
M9+">A00"(A`T[QH.IA7Y%P8E8F+?ZAVRUY)$EFS,AK/T%DJ#Q<K?>M122#;D
M*4F4!+WWJ\#5B!IS?+8:S_.:[416M&7T%Z3'3Z.QV^%T*N#H^4:N8_[O')C%
M1F7`=#&R*(4L0GIK=>Q[*7OB8J:EQ"BB5GP@7/&^&A3&*B+:RE163&EEPE[<
MWW9$QQGTUQ)[.@@ML1\ZX!$:*Z;VOG.WW3)P[`<-!D6$WHKIT+/W!`%@X663
M/!IW"AFL.;;,]N!;TU7+:I:6XV9Z;\2M)U5=4-U8:AF&^GQ'AV%9M?6G=MB4
MMGZH0+L;E%"!.E9!5Y5^-.36Q$>N*DL+TT-?]<O=6JA'7]:5"M3CV(XG*@++
MU863RL75FMS)U4%.;6&\4\)[0UP=7*W+G:*(A=R$H8H-5QJ58(M2/DUG5+%8
ML\JE[#R:3OZD;EQ010B/,YUSA5`W#JECHU;;0U5JC-UI1KXM^REY%8WU_)0`
MIQ#*"=]"+17?FLB;;J8QGH(^K'=U2]7XFRJ6;'-Q7ICDF,9QALM),>=*.(/Q
M3.&3RMAH5:Q!H0A4*,0*19`Z5B^70P]-KI#*Y6_)Y2EY4^6Q,?)8N3PV5A<K
MJQ#IHL,YH2:U_.TK)M5XH3Q6ZPQ;TB5:S.(T-V!U#"PE='T^'/3AW+"MD<`.
MNA02@5X2&"<1:B2B;5%-#6S&;':#$,P#3<$#@WKR26`-F"G<WL:[2RL#O*3X
M@`!382"W51"0&)]+EIW<?JI(C!Q42'[<PK!;&'+CSHNW?A,Z\)`'K:@;KSQ3
M)($/3WQ1>;SRJ\J.RJ]#!_K+VV$:Z<`YF(+NQBN^#!RX7'V658Z7'RL,+C_-
M"Q7S&H3T!CQP8+CB1)Y"V2Y3MH<JVT5Q[61<NS2N7:!HCU:TARC:A3'M"DU[
MD*9='-,NSP_AT&=+!JYODOI-$'*3IC`FX$$#UYARDZ(]EAEO4H@$N$/`GRED
M,DVQ[3'0%\Z.$@]<HK?BM#1!X,!%Z$%KVH.E`V?RH4\;'M,>!DR9D;Q6Y%G'
MM4<&M.)Q(J&F71(Y\$W8P'Y]>V#^4;TH<'\KZ-2U1^C:HW3M<94Z)>FO$8F4
M!0^M]H\=HHV;5*"N(O3KO]>&UT1<(9.H)XHBBL/S0J$=@7/4*<@E"P3^'<@[
M-6,TX`(7=7)_VG&&-II!M`AW.Q09V:P.H?\`5Y&3G?5TX5/*\0)&MW3[3]AQ
M!N."5#+@X'W]$C93N.MC!H2(>QS&BZ7_D@[L\2N%6(*+!SXL'X'7ZA_!>6@?
MJWZ(&%@&@-9Z5KZL6GNA7CY'*E]9#4RVA'UY4T$)AV/BE,9WF1!^1`RD5J@D
M`V6@W[\<U[>K=.U*/<F1MVN4[5K_8?A$K2.9ZG9]CE+WU6O@DJ+=()<S]:05
M-OU*9THF$[:A;=!"3;)^?R]\P*(D)07M`0X"G3%PA.Y)*5=9J!X,^WK>V'Z%
M7ZN.].\X7S1UX_=T'\8FC=[Q$7POW/J/6JASR$_W_PWBN&][P?].\50^W\3?
MZ948\DU8==Z$I1G0_MJ`*7`@'O9OT,#DM[<PY+'X99.6#*I02@9,Q8HXDLL9
M)?S3V)KW1=:^)XN2"]]:>A3F2L/Z&$TGH7RL/2S(CLOC-)"^?*3_ILQW`@4Z
MS=Y#*-:FX/_.)=&15NNUT_DJZ#3L(Y*_`,A('!5^V*$\EJ/GL%;=L44.!#XR
MXWY*&;8A]R75L>S(`;&UY(6H@:DU)2"K?+.2Y)1N11MT6XI(_C!>4VH@F0Z%
M81M'N=U?O1WVLS_J[:&'5L^,B`$F<JJ.9>__3'[AK+J15+=G6ZCG]:3MS%V0
MH93;3OT<-A#'Z$??6:CG9ZC&H5TR]2L9[/YQ,)^%BL(PZ[5/BI/UI`3[@<$;
M)K#AT`,':N``[<PC->.CFQLJFE3CHY7C8PX[59.B5^?NU8R7:29%6R@7:FYM
MG%P>U-BB7"&"O07:]$='K9Y=NJ,/K)WK]4<?VH8![`^T(K2A6,4_2"2/;+P*
ME.1KRH)1UIGX:TVVSZ]9J)<P#V.K`*1S?B+HF5-.%+,NT5E[:*S[T-:$[)E`
M/TGL0<73N;,T\J!,P:DOT#`=Q3I"&QN)'KPJSE;S"CC$A32=/5<%K=CH;*PV
MSD(-(@.FLAX,!3]81G_`,+TLRC8\P4*E&V!D5&VA5F&87.Z/MII*+*.AR'^T
MO3]!,T<*3JBF24$=L+/W>+?ATP7A-_>%WMRG;M]PJ4G]^L/%#7Z8W8&5%#A,
MP,7*G15U\QB;.XLU)X'%>XGA%ASN8'0(>&_@U>/0[J31\4P16?X$&L+QA8I+
M";O<U8HB$R@)>#/!ZBD+OPDB;VZGYW/!O=KFFB8=R7%^1$!E/51,$A1>>[A(
M`76LYN,"I>Z?G*HC*CG3`'&%M!I^>WCVES/_C+IIC[A9$$-*"KNY$PCL'`.;
M$'JX<,>76U6!-S-E<-;IA0>'=\A4\K,&M'_/=_N<E?";RR"-B+JYYG`&<0>F
M:&S5'LI6]9\E9%S9Q][M?RSLYCN1-Y?I8!%=P@/N?3*#+%MUX2R_^-'Y`-#)
MW1^__B3LZR'+J*?(%*==OG98="45?*2=AE:J_+X")0Z39?2&;V^1,0=W)_2:
M$S%&A>!JPI741+^'#H%E]&>TSJR'?>W:(HFZ^5Z!Z:"!*)@%8AAGN-S/\(T?
MX;K/[E_N!$TKH.'&-M>*%[->#/B"4U@2<(3;DL9;/&E_0_4(+YEHJ;TT@@8D
MM:!,N*FD7%`6E-*L.B1+.Z]U;`:=9XK*2^*J!%BJHTR$C134!E9O/K,4G..U
MTE><>1*M(C7+]$-2Z(!E;^&E$3E;RTIV=:]_E]:\8-N6[5MS9YBQ\(\Z5%52
MA]A=LFHKKJN&$MF6?HY>T'-G7"G)EH3=G$?P9NEFPK'>,B[GF36;%6$WGXJ\
M.2<W(2"9MOT)."^=XQTS.`ZB?,:F5W;O+GNE>.;!@[4SS_'#;D[,3?CO$GC]
M,349I"(EXILB2'*M)9``<AEWV9IS(O;128T-1"NA^5;DW2,/.NW*GU6LS'VN
M:0^VV8NQ%!_.O[AB!;^%6#4UQ(NE^5`O[D*E!_4<)PYU!42CS8ES(@(M,J%"
M&EMA>:<.Y,\J4!8_U[0?X@'F#MW?@69RW!7RR)U$ORJ4!^G7]SX_-*3_A]!^
M=WB_&QO'`'T*623HE-X,(*8*]SEX$')NA$+4T1S+4-?AW*G0KF9P^L?%962P
MD_'MJ;$9&?*,K,,+8G,R_&82)Y\W1./\IW%#+'Z.#YE>;$8F8R(.W\G1`V]=
M8?W'";5P'_*'#L>OSI>%];>%]'=S-1S-H1UL%2NE"5_^"4]535.51"W9PI+5
M9_OX^LDM8Q;E_N+'9\;\H_H'POL'M@JZC=#%WLRRS(,LSJ\\OERDC,95TQZB
MPT/I+MOFZCS+^@.!G*D)UJ;/=.1KMLXC8(M*_IH9.PCMU=OG944V#OO9`.X,
ME9QSN$4I)Z3]^S5R[@;[&V"K2BX!6\W8*RD7&>`CZ&B"]%QK1+_+J7"JG,K"
M;F<<TEA':)$-ZV0TY7.=--"YUQ'`?I/F%.YU-#E4G%$(?VS')+`%BL9V3-;)
MF4JY!%9KH?+@E-#W/&PJY[PQG^6>I)-;;=^_QN/-LEY[6=KOR%<Z<G@U!):.
MCO,=:*V$2/XDAFDX%&>Q.,^\OF6:0UFD".E/90T39V\I22[;P546TN+LAY3V
MCUC>'01I?UUC&BN-QOQE\MI=4\+Z/RA<I&FBK?](;\9XP?T%$%PQ++@_QT)Y
MVIJX?YU\N-7KOR91917"LWN]/D02M=J,F=L\`2].AG<1_:]`'KX?\?"6X55W
MGHSH?XGWQT5CK;H^NT)XNJ%H$?/6I'Q%7ASS-L%M()96TRYZTO8E0!1AKL63
MJ&E6%]0[#MI`[D1<@/'`EO"2CA3SGC39.G.0L]9C<AL3&5GA_:E1_4NZ76=V
M9M5ENAB,672_6=M$=,L\,,\R&E&T:,,'T5?(?&&/V6WJ3<T5*4GFM82]#J6<
MF32ZP8MO:)J<^?NC-,P->-+HM^']NJA^.;+FGPN]"T"-1WG[N<`%055]_"P2
MY]O?/<1\08\ID9[3#45M\<ZWRR9P,;P_)*P_A#=7>)&\9,R6Y9%^SW+/N_!Z
M8MLRF?;>R:RE>@AT7XK[@W;\N'/#LAOQIPI!-$B@QZ(S3*?>T+1OZ+(OKGAT
M."D5_Y&%WR2L";E;4KTRFUF*^)6=5RW4MQA]`[%S.(`BK/874`D!$T6E`EB,
M?(Z`43H)%G;F=6]A?SBK</0#,%TFPF(Y&.XD6-=9]&=1I:<38+K'57:Y=O_K
MZ()MJ8_K.FJA/L:`'9\=OZKSDH7JPKI<1U<[X7LL9C\+'R]N\X`$?)CHL@>\
M.`E,]Q5YZA58Y#M]!#`US_05N^'Y>YFI%HH)$EC/QJ_H1$>XH*X'WS@)S!$W
MOOTR736.IC:RH(*MJB.^/*1M_,8*/N@RPTI`FRUAGB(6UW=.A+-8S?E50XH-
M9)!"'L1II?$<;%6Z2",7[Z4,V2*KIP5R.)6/PSEI5\U_X'#,?^-PCQ:I$K%;
MOKV6TX>\1K'C-:V1M;9M,JS.ZM'Y+)LOG1N]AY4WM<+ZN/SIUI&<=4=.@S:B
M"G'Z1Q,*<K:WK[<AD.//>?N'%H7<'\KN._:J)KDJK^"\!K:V3L1U$(U.U8<B
MJ_VO9NP%7ZF\Z6<NKCN"RH-9]-ZS.+?_S$7_O1VH3E@)JAY>+%3[5=B?Z=8O
MGOI/&2)NK+90I'=MT7]I=LKI-R#S/IR(_7QZIZ]+_-[`$S'DM+`Y<T[6>^=Z
M,[<2UXP=#AL[2`3:$K$KI_:-;;WA7ZE<0LX.?$,92\MY2%31M#NA1PEU%O2C
M33ZYV$+5_8$D`$\B3>);OX!R2&[,EMZ8;9OW#&@+OC%3$XT;9!B7.\?6E*D;
MC_$6S]/U/+1UIND7U2P9?$*U3)%2@J]^H-4MJC[\PND+2$\_$>YRZK?D;-AG
M8#TZTZ*786MND^HMV3;7VV8L6!G-B+KQUJI?-=X5%/]$VI?6!)49^T8=S0BY
ML?`\?TTE$RHV[7,SA@ZKTMWH0S6?0T*_8DO8[1,;"G.%1`=`Z<GG75;79NF-
M,#`OJZX``0]!S+)0,U71##!4/`XT76KB4PG\P?@EY0(]F=PX!%]KY<GKT#G0
MTM*ZJ!L!+7MK7;:$U\")EGA(03O9^]&QQS/GHFYP(F]P]3]O:'H!*NTC`BJ]
MP48T3RW<U;!'II&?5=5G`Y+]/%>U"UF^3A;D:.!K"]7FW>LI,")91O\9PT%+
M0QH.C5V%NR%&O19Y_9='3&\78GHQD.G-?<3TJLZ#S*CK(ZP:86TG"Y+8<YKE
M]6A;T[>_DUQMH6;Y>$`,*3XQBS&!%B,7>]>"@/G,2V@.P/Z!\R:)*D#SAC7G
M9,:?=(68?OKK]4?N_%%5X$CHQM&@:NCDG"3J]MBD`J`SD;;20NW#QJ8TPE2E
MUW_VGO,M#;[^&726F0Z6.B-'-<=OS5;8I<4AUS]YO-@SMCCD6J/,R`Z__HG5
M_GZ5O26G^:^5Z94"6\)[8/=E,T@'G?Z%WAZ_&'7]0.3U`[#'+QD12'K[VV!D
M&:Q8R/5]2I(`)%$@W-VQUR'+.@M]+T!R/N5"QPOU.[TP?KWKA(5JQ3H2O:?:
M>DQ)HQM]=B&1/L="W8H5^<8`\W\/C0'^P%<R\!A(0G2]E'@@W.VLI_3WSBJW
MP%%@)3[:?_'U,YC-F(`_ZF<5*8;JVNQ4R<4IZ7)P4,NA0471ROW7NAB@4Q>+
M/]8NU(.*Q]8-E24CQ709S7<^#J(LA%B54:BL<EOMB]55/5;76]"CLR;,6/K/
M1:>3X*1O2:)^.EW_&`N2J%K?0OC)8UXL.,>KQG4^+$!I`$BB+J'%CO,6ZK2>
M#))>7V0@N:#$*6=1;.4/(I4\:,^P_E\B"U7CG9]_1@'?*B:4Z\R"P.NV6%D4
M[SFA_P)B1PQDF+&**/^97+DB\H._1IR<CWQ_)<>.EL&B]BB5BDBE(@(]0V7*
M_W8?)K-0EWQS%*WBIRXIFXX8<+$:=N3'D);Y0V=8+EYMQ^-B<;77SU_3QK>Z
MB$<3#Q(*[)(UX0-=-,[CS3G5Y>T]?=6O5D^&#\MM(YM#KT>&7P]7U=>`O-7Y
M;VB.*5*:&(7DVLOS5?75TNNSK27/EY6`GR*N3]9',ZJWLX:%R!\LJ]D,;L8=
MKX>E:.JK]7)_S5$CBT+\5[.KVC=&!B.6\F#*F!HETI8AIW4,J?!/91SL'-^6
M(%#"1+GR1_8HD?8VRHM.'GK7@'U]FO00NMG3L$>S\6'/XSD`)T\WN20;:LOU
MX+ZAB+Y[(7U#H`V0ZH+?K'63-`OQPR]Z(7>WQDRH%V$^'3-DY$"]0F>7L3JO
M=OEY56L;5"VKO;@E-ZROIS2]TFZ;]R8XX@4R#_M`?$#`#*MG=FC?I:B^2X9=
M$,D>XQAPZ<T$,2C<[3!D9.\?1NB3DZTQ^\OZSZ8TJU7F('#H5$672[<0UUC]
MH7^LLT&/`'A8!^(W-!U!F""S^C`!_=@2./[O/.H+A7<VFA/Q)=XCL)C22P)6
M.7$E!VM;V=;6UA76]TV;?-^[5OLSJS:;U':,.$A81S[G'2`@9<!1_!`3Q1L-
MC8/6D5WZQEM6C\O0.&`=J5#;F:P]7%8>OC835Q?>OP1-:"LH,6.9D&O#YG@[
M2I^1LW00]E(V>\P`L?2+,,XP8?-\H(9]Y5H<U;<=W-!;_6OJ:EU1?5_IS9PM
M.]#DJ"\RH\M.@UG,_X?)-C)G_^>RG<&P>V"?&'Z3POZQC"[$T.%['Y(S_@;G
MQZV(OFS09AEMNNRZ8O:1F&,C;1[VZ)0DRN3=!X`]H;6,7N5]SM+N#%9Y2T-'
M\7.RK0D$+`N=D\0E?\*(D+[DL+YD(EZ(_+MYR+]+94*_;D%H<%\E=.WV?*`^
ME*':C'->P]4?HR4EY?@,^?@L.'M8T*5[$2U,3<K8[F*_BMQ-"/P0FL9G<Y-P
M;<=O6O@":+S.'<1R'Q':*EB;"UD-.`6FMOW(\#=%]"6#ZV`Y<Q8MX$7:?EV-
M$)CT'+&U:<EBUDR8?&VZ3=V$;:B;NK%NUMIYY+JZYU/V!J<TB#^H$X+:U75$
M6_ZJ.F;;`14G4G486U>G43<QN?NX5YK6?X2K/[M_U77Z0PMU6-DDR!6^O:G5
M6MN"UB2M&P\C"HU9EW:"Z6T'U(<Q2)V@W-]\KI9%0"G7-*L+!.N:H]=\XOL>
MK2_R/RT&(A3*%=AJ+UBHGU!?LJSSWHX;CS&9LR1](FTTC@[S@I(\(V>,R61#
M)A/6QUDR^)+F$$00=/YV[:E9^D751V:?^;Y&`$U=T]^(H\)=':IC.=<Z5^=/
MC^RC/=X/=)`;<HW:8]G$-[@-6!!_Z1NWIC(&PD(2563&5)@VFA'6IUW7*D7T
M)8G*/95HIBT%P$)U((:SRXSY>WD-;:<-$ID_\QJ:$7W8,@.1FDV(U,`60)<2
M5#D@HV'=3V!5QD,>52Y:XA`HR>3=F^#K.'ER6BO"E+:6:G&PY]+&7409T!QG
MZ3L$H%[=3K,E[&GY2P]T#\!F8`\H]WWNDA[B.2?UG%-NW_#.#)]_X')@H&#]
M\_>`J\!X^!)!))PY"WX,Z.%ZRZIM0];2<Q):RUW06L;>.ZM%UI(8.ZT`W?KW
M6=($:]'FJT;8LC;0F40E6T8S5201YFF%Z:J%^YR[W-JN[-C2L\1O7&TW,M[I
MH(23&P\V>3]:2K-0[T(0EM#U$IJ/,_5XH+FF\0K%1<+BP+86M%5JS1<Y!"&>
MW6693CNK50BVLEOQRG2IIQPTJ;Q$:>QS%\B51D=^7W^X8@*_1'JV17BV\?[J
M]0>AC8/0A#S"$GP+$;LR2J:U6^U^7I_PAMBS<\>AG>N6?3G]=+J,0WOD%)Y^
M13?@\^,:QGPTV;((_!`+_PKZA=6RE1$@O:M$64I'W[&P8N-7C'3!X::O(W9V
M6.T)R#,D_N`9[D2>H37A16^1CQW#/<?`=)`0\U]V^@>$SX.+X<`XZ\P\F"X+
MYB(>)^TJ`?/4'#O10NQ9U>!0KHR(_4/-^RS4AYCO$YH1]#4-!E.GP]0-:]0K
M([PBQE1[?48`?<8.Z#,2<W[W&?\.ZWJG[0\^X[KG[\4LB[",#OB<QI$^"W5U
M[%Q$-![N^8MW^6:SZEA-A&<2VB2D;??N$#[:D'?HCU6#WZQ]=HEG&E05.(]3
MF6MQ-)5_C5^R1V@@DPV_T<:.<Z)/3Q9IC"RPV=I4I#Q*6>WW4PX)K0GKX(Q`
M>(F=YM3C_MYC<>A4S)#N6(Z!Y"R]B#X9C/"$/%KG8!:1/(C[?5-UQ[+;[-9S
M$P(]4CW)1)^`T0Q;.W<-AGC"(SV1:KL@+DO`?H`[NP(:\(KSX'K-N5`/7W4L
MN^'3N)UGV;>YJO9LVS\R'O$#@<8HU!L#L=;QV-!XZ\@:M5$<Y7D":QB_ZGL_
M_N=L;?_&2,^$9)<?OX6MSK(?WBWU&$\\C34S#GS>X\%J0^D4?4F?G[8+(X:(
M"(^:=3(<.Q"ZWZ'.RM>9#\5E%?#=-.2F60\ETBY;*+9Z'$L=R]",8VL0592@
MG5`Y]S#DGH1E=$AG%(5ZPK#+XU==\^-\SE7U;TP&?IPON-I[=K51"$6R4"V/
MCF0F44__87,'VV2A^FMO@*UAU[I9>X25'G3@IB_N0KWF6'6<[QN1=J@]J]&:
M#[8.%B.]]FLN679^:Z?^@D3]+0?TU9QC.7%PPWF!=1\O[XJ3,RVC)__T;2BC
MFWS[)>)1Y21ZFC(_H65JTQPXC+ER'1F@)_D[W<W#>CF_R:V3!RA+F+H2^DY'
MDZ/9K2NE*TN92;\6O3UO(O!DR-5FOO1:GUI.5YD#BDF5G`EI6RU98WUTP"6)
M8E7(0;-<'J"1\Q^M"7G7@'C3A)?(/!DO2<AM(#*BZ4E$10:Q@(^/YVY)#[Q6
M=H'<D;-=8$UX+?Q:<<2U(G"K_N_@!^!B_Q?ZS"3-,O?T3FWWAO,E4-\_'=.Y
MK/1\P<Z7&>CSLFVEZ3D9IQ9YN6\J_DM\F@<:GF_0`;D[@#R:ZTQW>C\[61ES
M5ZZ[7YJ."!&>1>R862ID3!3!$L[L>U3XXT6A##`]4PC,[`DL^G/Q_OXS3EMA
MDM_K%C)>)G9F'9V)"LN$NOUS_$K/9Q;J"`;KG_6H_J-9Z+45C4%V-"]>*+[V
M9HYLIYR53&R7^4W@;HU>/C7PY"+O(23C494SVDENE:&[F"?O;XVV4#<0![GV
M;-BU9Q$!.8<(")!>6TG\1=C@("`1.8\6F,<XR%_&.$A<1L:?5I=73.?HH_&`
MIW`]8AN0;\LS,OE/X_*<#,.CM>6`;K)E2O`U[25C>9AM_A)0IY8(E`MQ+X[\
MS6&,N#:>]<X\=>-#6]W3Q9MUG1B_("%M=*GHVD*Y1(C_2%MR'J)T>^2U-\.N
M*=5F+OB!V\#6U(G4=0+0B4YP/HS/,X=="SS0P&W@<JYR-(>6ZR3\9<A"[M6+
MA)`NUD00GPEWNPU=.1][OPG$/HNZQN%[C7M-9/.PS5-HQC9CHFNSFN;POQ'J
M)4(B7+SD?"@XEU<2UON0$SH?%AQV[6E#_8KJJ+6_-IBQ>1'7IH9=\_LW83Q(
MF%$D3._(_H:(:Q.0/'>0/#51D!19J%0S%@SZ4*&#OD)U]2MJHI:=AZ\67#&#
M3Y.HJC/_,-.>+XXL,F^\+=5U99_JTBRK6=(2'=;;`VE+=>C;31PS1EPQ)U&;
MS9BHT+SVLK]N675%R-D+Z`/:)*K`-H];&-9CNMI9V.G];C:)DIYW58197?=:
M]B**48PH!@)?B<`RN@8#0Z!)O9!1/*XB'*V=W($`':@7"=3FY`.//J-%$'W6
MUB;)#67ESV)Q_L&:DW"X@]$@X+R`,SH$1"M>L;FRI,5IFU?QQU45:^>*J-Y_
M1O;^\]^75=!)8698[UWM0D9AB,-8'!78NY]'<?0_+F_KQ(<)_<_+UYU_.;+W
M@$XD..#4E@CV=Z!EG%+!6CL<MU]4(B$8D?;6RZ`/H!'N;MTC,\C/:@\A9N+D
MJCY&)./$V-J,PT*M\_J^D`-R9ED>YL$2]SM55N9:>POH,Y@YA:&B7I!O=D3@
MK3BOD(M3^$>.K9W:;SFJ"Y*:<Z+>W,H+M5W`4W/>8";T5B;Q*ZS/NVZ@0_7Q
M(!/RUD=&]K:">[7"&H$5S"T,Y78(KW:V7%'V2!M[6CJNE?0VN058IZ-:A'D*
ML!&'VUX3J"P@>UVU8G4C"9_5!!DVTUMJZG^L/:RJ$FB.PF>.&@G665`=O&[O
M%[8$>Y[4,GH&7(_H?5=IYF2$YD>`$4.I`#_$Y3GP0ZU5,5O)FNARF;3W#=0G
M.CC-O7VB[X(RHF\ZNKUK&^@[XN:>$K<I:?2?EM%U&+1]/T7VSL\/K^F$M(K?
MB@-/5.\SE>?T]^HURZKU5J]96`G-@I\W[U68[Y9E=.8?,3]G`G@?&,T8K7D"
M!`]L>2AC$6VY^0?ON@*DN%YN_'-$KSZL=_F)6D<3>R^=G4>K:-*<PU?;PR)Z
M4\-[E<`3X-LX/Y;!:\#5[1G<_G'G^&TC_"^@ZS_7\"W.1_>PO,#>MX)Z%XEZ
ME?)W2D)Z+>&]"Z6]P6&]4M#)\Q5P*$.1CA,--`11R31%-N[?/TZN$&4GQ"B$
ML8I`F=9[AH$EZ7U%P[&J0R<&]TK">YE\G9"G%!ZZ'.`6!CB%C8[=SMWN:HE3
MV)!7$:0LI&GMS2K[1VI[D])^B-4@)!K&SKVR&L);'$H)ELBXIGP_L4*DE-#S
M@O)%:I>=VZ#4B.#SRQ=2"R3Y@4&]4[G[E#&OMU6*->U*K=&^[%10@5B=8-+\
MA20:E(F,L^`<QSU9]ZSI\,Q#"PY-6#GERJ&T8F&1Q!%T>+C941Y4(]EX>\>8
MK^C=3D3G?E/#W)>CW'<BW=<BW`/A[F^!,M!]7=P;K30>DILP*-=B]9(=SRM+
M!,UI5ONKH"Y;@$UG@(58<JC_`IK?AU.(5X3[DF6+MA.O<9;$/:>NEXMZ:2DN
M/WP&:]G>R3&+MO%>P,^,JS'R,_"*\5LF`-F6<8TO`5*]:)NL5"";Z%(W"M3)
MVP+=7TE[_9IG,A@SY%D8.GZ*C<D9ZOY"[<(2<6L2=<&W;I)$=8:Y&\%Q53-^
MLI&H@J[97.T_<0*-;93[XTCW+F!G>8=1=SV##PG1`#(SWG$/=M<$N6L`B7M?
MR^=GR-['/YCK#V_1+`QT]TG=-R7NFZS9-'\%3;$"5[Z2(7+7PK&/443(M*G>
M,H+<13'S:\+=^4OC(9)IY,<4*2:F@V2@?V(B0CF_NJW$FAZD(`FQ.YMX2_AA
MFGI9#L/*04N<-%P^8P=,G$LRG(2MY)Y\67:$>X/5!+N^,-!]$GR$'$&W'4X-
MIQV=A"VM2`<MP%69"4M5'_=^FO)*M=2=ZIQ>;BHFMSZW=6;%K)IG*IZ-=*?R
M'N+5,[(5^N<$%2]6S"F?K7I.6/V"\_D(]V$_^NP0=Y'?!.'6N=5_"W07._]*
MIW`ER3$\)U8_%TAWX,KG1)#:QCXG<2CUSZJ5SP6KY4P=Z2\GK=#//=MY^D4]
M*0EWOQ;E?J/QO>*?&<,$VF<>#CU05*/2+4.;:;H>LVYE]C:59FRM5[DL.]#]
MHK+`K!Q[NF.16GX6=?`)@R\\6M]35CVPVNO450]MKK(\9:$"W%]SF:&119W\
M7$M:K2"S2'6>'W"`2/:LAGP]TFV$_O0!=#(GQ*U?]1-:SGK\'5L1R4;K-"60
M>DO=.FO)IO(2<$_J?JEVLY[D@)':K6@Y?$L1B2ZE2I)945:]7;F-8T"'H/Q5
MI%4_]/#T34"&N$GUL>R]G_C.Y1A*SS[:(4RBTJT'?_*J#&:M_59'2B+=(6PW
M4300X@[$^M#&^]Y/]N^O5@%4`,SH*P%FCW[T[YXH.%AP#Q7F3E"2MMC/&,J"
M4:N]G/B;</]GVJY_:;KNJ>?AZYIGP-_#K?JN7S3=]];>]M-T_0PCZS[1G*B%
M.@?O0#J\UW3_"W.$LAUT?^\T-BS+8%$TPTJT6V^&=2GDMJB>'V&)13_"DIN&
M:WY<>T:BZOK9T/4+]/;8%,&B"+Y!?/*!&3-ROR9X.O$W%\'U@`-X0,!<_0(\
M`.F*]Q!Q$K4]S!UK/?@J%,HK*/0*?55&]8SJEV7PAW&]MU:?7O2<!$;Z(\62
M]'RY?)8\Q"V1]IQE/T_;E*%Z)>/K#T+=@=#,[FN0U6>JZ[.6U=$SR`ZE=B$.
M2*_V-J\/=!-->L-"7%9@9@T+Z;!9-@3:^IR0Z@S?O.%$]+2&]C3&=&(A;GSI
MA-7A/3>!,LA-B^RA-.&""TV9'BPA9YL`,^652A>KY_*?$!(3V7*3G?,*IWF"
M<M%RW@*V89Z)/U/(G<EAE1'\M$G%)O@CFS>=21&UICUIAD4KXI*7\\L)W#%I
MTW2F8Q(KC3A8;$A>43:=3I_1C%8_RCU(EO">`7%/OWS&/ZW&OX/-H%/2<R.V
M0\!P/*,!&#`ICV.+ITR#HP:]X++0-+L8LKVBA(/#^KND__"DF@3=KA5F3)K6
MQ,Y,J$Z(03=X3/T*V9.DOVR2;->*PPM:&F`EL#18T%C_=@"3[V"8(R&PY_29
M]"3J:'C/A]!&[^]`AS:.998FK-TGUA_+@EK`^8P(<!(\)\%6B<V8'W<OBCB-
M03UO:N;A:NUOT#(F8L]KQV/:#DH[">LX?K(>*:^O?-FG5%3/R_H?,_!AFFX>
MWN72_YS!\TX+*,>__3L^C,">/1$]L^DQ[)AQ#M"V="+42\>)+X!+VK.!4`GW
M.A09F9#=A_285LY[-FX^IL_(CGV=@7C^;;_8C"P86?GJR_J,+"R:@1T/[?9\
MZ^'.I.MS,O0YF1RO,#%;,@SG<+19M3V#CV8X[:)KY:O1,#U\WY4*4V!G0[D4
M7;\E0WD.YP_3]-LS6$A8M*KN\^485XTY9%!/E)*#*6/Q1RN`?O_V;R$,A/9(
M.2]P5/4[3GRQ<C/M'%^SZ[&M_%^577M8$U<6GTP2"),`>1F2$&`<DG&2R0NP
ML=+2@D*M]"%;P-*F533`0A7HJM3'M@H8\('6A>C7^K5J!$1`U`39;E?:[1BT
M:$N[K5+7W('M"UVWN]L52G?WVT_-W@FPLMON'YLO7W)GSIS?/>?<.S?GW+GW
MY-S@>YH1:<)(C(22=5[;T=#@9KR1^9(2^^;&O%=W]!4><-?\D=_0>,"]\9FU
M#8U]5C@D''`?:(`:06U^5`]D0"M.FR81+?5\@D>$SX<?EB/P?)^>4RU+,'4A
M$>:^=Q5LJZO>C3^Y.G7Y=-Z%$22>O:UC;ZO8VUS>!?:VAOWZ_\J[H!EQ2$RR
MKJY=OSCOWMRT)7YDWHD>8V"[.>".&I*4,!6F%H1^AYM?]*X0TYBYMTW#?E7.
M1":,P('ZE4NY7<:I]3SES:DO=O"O>#\X7-XB'3QM>JL-]CAGZ!07*2*(8B2)
MW\;;DY+`GI7L448/\<QOM7%SN3,C[4<S0?B47JR=^43#^@!.V1'"'BEZ5#Z5
MQ&($A^T6SYZ0Y(87&;FCC&)CAWLZ3U66AFU3L^T24M999EC=V$51M:*V0FZ1
M3;96F"TIR7R$6.V>P_;!6/X2L7-N'/N:R*V,S.;5+!5>O&]J^PLBPI8DLM_M
MPYE`;.SBKCOF86\4C"O^IJ=7NQD<!ENF,O=_RZEF&P%NF)*SZ-]"[DJ)^F=>
MB??9J$M/E>>3">Q6R>.R)'87TBQ@ZN!`;AU'3P[!'R;K]R@=<(MI,?T^!SRN
M8S=('I,ELMO>?ETD6F+Y$H7B]_0C(2WGTUQ'3;UNS"0VO<6I#.U0\6-VF++=
MJAG;S3*<CGT.@O?TST()VTW-+E>QA9)YL@Z2+!9UKC,0C<@J`31=MK8F2T00
M;F@Q;C7'U!P)XZM*AO=[:PE^J$&J8)>HV"XFL^,LW7I*K,DJ]^C*6V3ES9',
M[JB&^XL1I0$K*49B$$/ZS6(N?_FV]Q;.80_3&3Q3:[?9YS?Z?!CVA*EG0G10
M!OMIS<%DLZ_7XCLC&I)%#Z%;+BG?3J--B#][W=)#Q8@5@H3W_;ZX4BS<J[1F
M\`QC?LN8[\738FR?DH*8[C,V=R_(=-Y1<7MBN`;E]%L_AZU%ERI1T>+8G3+C
M<AG7(QS:P7W(1JW^*;EH(Q_V@8AL5+]</J6?G?$QQ3ND.A;>33%SV)C!;P_+
MBI%%4/@CL?ZR*(VR],@D4XQUH=N._Q44"\NA?<V,#]H7&NVREVD>/(T<U&('
M^485[^.T$GS%%9S&(W]7K`-W]LI@E/)*Y_(I'E`,>2Y[2QX&NU.PW4I!-Z\$
MQYKFBIN4@G[>EE]GC'JKC@KVYD6?E+#VR)=1+6L2;L(P0"6PRZCZ!Z.[19"]
MU.:]@C,^';CA"GG]94BQ`(O2N$(%"#+BJ_(+]N1)WI$DL+I$=E'T1]2IT40V
MRW*CSA5JXWQL<$T#KNG8C'CV(::6^5EQ.*4FT2[R=\,@4KP?WC`E9$OCYH"*
MP:M6W^`V)1#<@%VC?<EYS17Z%HY@#=$N&=^@+%GYX=4\OF5Q99#_.MY'7*F-
MR%96+B4\N&>N)]E#>`P>O6>>A_08/92']I@\%H_98_-884#E"C4C2"+XU;P;
MM?-NU$WU7SGH9=8QJS2L0@G>5;#JY%2+*1,QMIRJ:N-6F5(]=Z.B,\W:0.#=
MZ`\7TCE%);4-SM"SL)VT;`2=?A-+YQ7Q=M-915@_=<5++T>*>-U->5B_R(3!
M8I>2%5KFHN;TFY('J8A1[G'XFYE%O*--F=W]S%%8T(&[D`Z)D:.H`MSFZ/8B
MWL$F.Z2WPH)I+LKT04<]O#,6+>(=^S1V8,B<4^0*;;9B=S1@,@'<B@?C.G!+
M#7Z3!,8;<')R"SFYE9RL)R<;1844TR`41O0\=\)Y\GER^`0UW--5:,GLI<[_
MF3S_)ZK/'#/*(_MH*_9W6#\L4P5KF9LQHRA9L,:*?0Y/!+Z`1U:L-CKZ$6[>
M-[O7%<Y+"^TVX&7..$/9`T>9,W)P?B!33ZJ87#T9Q\WD/J@&:P::C:2*JIV?
M3,:5U.XS8:DTQG.&[A]H#7\[!EH9OXE4!4XG<RRIY^J-V-'!,\S%P.5`B/G6
M1JJ2P#HK&6<A50-7S63<N0$]*3,1<G$W+VP/$RF'L0S6RM5!QM&DS$K(.5^1
ME$,.,P>9H`:GF=<")\[]THRUFDD53<;!^HQD'$5*H1%;.>F5KE`5IXL.5,6#
M9]2@+0ZTJT#['-#.?,$,*\$+28`F,)%M`L:G#L`$F9>=:.PEB0ZX)&]2G0,=
M`9.CUF1PMP>.#UGT>VC];EJ_PZIO\O=;'4?,CL-FQR%_CPXLC@?-#%5I2IN-
MD'\/@>80_/UA#,AMTN^>Q<YLUG``;@6HTX(Z-=BN!-DJ4"^8I.3PLTPB$$:T
M/M59U/UL6_X;=N,Q9)Y[-W04CYG<>XSNICCP<S78?/QITI%`.A*MCB05V*0`
MFSH*!6D2BKQHK4$L!3W6@A/6@NYD\D8R>=U*_F'F>50<6*8`!?IU&42J9YM-
M7"D24K"P#W6%OF/.,[[$X`2W`RNJ8J%Q;'Y5-N^JC\G3@`(U2-@O30B.DF/H
M`/6Y+\*VT#;&4X&\))!A6I>S]Y,XD'M(E@`>J.S*&<X[4[C%J?Q]GAS$UJP1
MO"J+62%!OZ).$\35NL\R%0#;L$;0(!/F2/CYE#^G\Z&NU.-IA.,0X3BLAS9S
M'*''FLQC>Y2`OA*K7_0$8Q<HE1<?JQ1%^-.X4I8K-'$^3PWT2(I`&QQD_)Z.
M<#;2-XX;ZY](KG^2JL\_]^B6'8G82=0\C+PJK1Y;*0^^*WQ88C/(]7,5-H-4
M;Y!]&JL&T3MKF=5;=XR0[<@N*3]B&9-2O3]^Q!L1KX2A2,Y"8;^$OB"G+RCH
M"U+Z`N1`1KSP-"3_@)(8/";(D%BNRVU?*BS7I;;KLIG?=77PK"+XG3SXC\%E
MVN`DDV?HZ#1T=!DZN@T=)_2DT9B:HPKNC-HIH<H09^B@E?A:%_3$K%`>_L!"
M7(]^7FEY7TP3'XF=2OI]B3/TN#.D/N=+"E[3.QXPK4)MCO3I\:,4+4(...\.
M..^^:2`MSKNKIAP+>?#[67EM@^/3>6W]L_/:2H-_:49:>!YT/U\6_*1>ROO&
MWBKCW4KAH@45]"%EP1OUTN1<A$^@+5)9\"PSMT4*_7PW?SM:SZM#D,MV7C(W
M;DB#H_=PWIO!&9^%,UPO502WA7%F`URQ\]+"_!\W(W6\4NG2.K14FEW'_VW2
M#-K)&;2)66@,AY;[GVB,M)G?C#9SJ,-VWF*H_Y;_\0<74_]\(`WVWI/95R_E
M%%R[73JMZDEY<&N+]%Z-[5R-I3^0_S,[+S_<UFA5JNTJ+JP6"!X6;'":H*/(
M^=]I6@)#*I=&KW<)^&42/Q%+R/@*Y7H72IAAD'`QG6LG2`M'^M-^T8>Y+5*/
M+%`,QX`75B4BF0)4I($>P>MRQ*"]7`N/^7R-,`WU*#C>"%$$4BM`^1J/@L%;
MY&'?$\J"?F/GL-`A`_\6C'IBD6I"R!^')6%ULI`_P<4-TW3T5MBOKB8$Z#@L
M52<+T`G[3.R#HEPLQ&5+7EQ>NG9]137^R--X=LZBPB49=IQ:5%.Q=@->4%.*
M/UG]$IZZ`$]9D)ZR,#UE/IZ77X"GVNT+\.HJO*QT;<4FZZKU%:Z2TO45/ZTJ
M7;?>ZJJN3+>55U>6VL)4VT9;V<:5\ZT+K"E&,[X<7E$!&0O2-FW"[?;[K'8[
:?.,6/`4>V.%+*D20C-+)Z_\";.?>SN!F````
`
end

4
sys/dev/cxgb/ulp/toecore/cxgb_toedev.h
View File

@ -41,6 +41,8 @@ enum {
TOE_ID_CHELSIO_T2,
TOE_ID_CHELSIO_T3,
TOE_ID_CHELSIO_T3B,
};
TOE_ID_CHELSIO_T3C,
}
;
#endif

1569
sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
View File

File diff suppressed because it is too large Load Diff

729
sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c
View File

@ -38,14 +38,18 @@ __FBSDID("$FreeBSD$");
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mbuf.h>
#include <sys/condvar.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/smp.h>
#include <sys/socket.h>
#include <sys/syslog.h>
#include <sys/socketvar.h>
#include <sys/uio.h>
#include <sys/file.h>
#include <machine/bus.h>
#include <machine/cpu.h>
#include <net/if.h>
#include <net/route.h>
@ -56,6 +60,7 @@ __FBSDID("$FreeBSD$");
#include <netinet/in_var.h>
#include <dev/cxgb/cxgb_config.h>
#include <dev/cxgb/cxgb_osdep.h>
#include <dev/cxgb/sys/mbufq.h>
@ -72,6 +77,7 @@ __FBSDID("$FreeBSD$");
#include <dev/cxgb/common/cxgb_ctl_defs.h>
#include <dev/cxgb/cxgb_l2t.h>
#include <dev/cxgb/cxgb_offload.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
@ -85,6 +91,7 @@ __FBSDID("$FreeBSD$");
#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h>
#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
#include <dev/cxgb/ulp/tom/cxgb_vm.h>
static int (*pru_sosend)(struct socket *so, struct sockaddr *addr,
struct uio *uio, struct mbuf *top, struct mbuf *control,
@ -94,13 +101,11 @@ static int (*pru_soreceive)(struct socket *so, struct sockaddr **paddr,
struct uio *uio, struct mbuf **mp0, struct mbuf **controlp,
int *flagsp);
#ifdef notyet
#define VM_HOLD_WRITEABLE 0x1
static int vm_fault_hold_user_pages(vm_offset_t addr, int len, vm_page_t *mp,
int *count, int flags);
#endif
static void vm_fault_unhold_pages(vm_page_t *m, int count);
#define TMP_IOV_MAX 16
#ifndef PG_FRAME
#define PG_FRAME ~PAGE_MASK
#endif
#define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK)
void
t3_init_socket_ops(void)
@ -110,20 +115,8 @@ t3_init_socket_ops(void)
prp = pffindtype(AF_INET, SOCK_STREAM);
pru_sosend = prp->pr_usrreqs->pru_sosend;
pru_soreceive = prp->pr_usrreqs->pru_soreceive;
#ifdef TCP_USRREQS_OVERLOAD
tcp_usrreqs.pru_connect = cxgb_tcp_usrreqs.pru_connect;
tcp_usrreqs.pru_abort = cxgb_tcp_usrreqs.pru_abort;
tcp_usrreqs.pru_listen = cxgb_tcp_usrreqs.pru_listen;
tcp_usrreqs.pru_send = cxgb_tcp_usrreqs.pru_send;
tcp_usrreqs.pru_abort = cxgb_tcp_usrreqs.pru_abort;
tcp_usrreqs.pru_disconnect = cxgb_tcp_usrreqs.pru_disconnect;
tcp_usrreqs.pru_close = cxgb_tcp_usrreqs.pru_close;
tcp_usrreqs.pru_shutdown = cxgb_tcp_usrreqs.pru_shutdown;
tcp_usrreqs.pru_rcvd = cxgb_tcp_usrreqs.pru_rcvd;
#endif
}
struct cxgb_dma_info {
size_t cdi_mapped;
int cdi_nsegs;
@ -182,21 +175,172 @@ iov_adj(struct iovec **iov, int *iovcnt, size_t count)
}
}
static void
cxgb_zero_copy_free(void *cl, void *arg) {}
cxgb_zero_copy_free(void *cl, void *arg)
{
struct mbuf_vec *mv;
struct mbuf *m = (struct mbuf *)cl;
mv = mtomv(m);
/*
* Physical addresses, don't try to free should be unheld separately from sbdrop
*
*/
mv->mv_count = 0;
m_free_iovec(m, m->m_type);
}
static int
cxgb_hold_iovec_pages(struct uio *uio, vm_page_t *m, int *held, int flags)
{
struct iovec *iov = uio->uio_iov;
int iovcnt = uio->uio_iovcnt;
int err, i, count, totcount, maxcount, totbytes, npages, curbytes;
uint64_t start, end;
vm_page_t *mp;
totbytes = totcount = 0;
maxcount = *held;
return (EINVAL);
mp = m;
for (totcount = i = 0; (i < iovcnt) && (totcount < maxcount); i++, iov++) {
count = maxcount - totcount;
start = (uintptr_t)iov->iov_base;
end = (uintptr_t)((caddr_t)iov->iov_base + iov->iov_len);
start &= PG_FRAME;
end += PAGE_MASK;
end &= PG_FRAME;
npages = (end - start) >> PAGE_SHIFT;
count = min(count, npages);
err = vm_fault_hold_user_pages((vm_offset_t)iov->iov_base, mp, count, flags);
if (err) {
vm_fault_unhold_pages(m, totcount);
return (err);
}
mp += count;
totcount += count;
curbytes = iov->iov_len;
if (count != npages)
curbytes = count*PAGE_SIZE - (((uintptr_t)iov->iov_base)&PAGE_MASK);
totbytes += curbytes;
}
uio->uio_resid -= totbytes;
return (0);
}
/*
* Returns whether a connection should enable DDP. This happens when all of
* the following conditions are met:
* - the connection's ULP mode is DDP
* - DDP is not already enabled
* - the last receive was above the DDP threshold
* - receive buffers are in user space
* - receive side isn't shutdown (handled by caller)
* - the connection's receive window is big enough so that sizable buffers
* can be posted without closing the window in the middle of DDP (checked
* when the connection is offloaded)
*/
static int
so_should_ddp(const struct toepcb *toep, int last_recv_len)
{
DPRINTF("ulp_mode=%d last_recv_len=%d ddp_thresh=%d rcv_wnd=%ld ddp_copy_limit=%d\n",
toep->tp_ulp_mode, last_recv_len, TOM_TUNABLE(toep->tp_toedev, ddp_thres),
toep->tp_tp->rcv_wnd, (TOM_TUNABLE(toep->tp_toedev, ddp_copy_limit) + DDP_RSVD_WIN));
return toep->tp_ulp_mode == ULP_MODE_TCPDDP && (toep->tp_ddp_state.kbuf[0] == NULL) &&
last_recv_len > TOM_TUNABLE(toep->tp_toedev, ddp_thres) &&
toep->tp_tp->rcv_wnd >
(TOM_TUNABLE(toep->tp_toedev, ddp_copy_limit) + DDP_RSVD_WIN);
}
static inline int
is_ddp(const struct mbuf *m)
{
return (m->m_flags & M_DDP);
}
static inline int
is_ddp_psh(const struct mbuf *m)
{
return is_ddp(m) && (m->m_pkthdr.csum_flags & DDP_BF_PSH);
}
static int
m_uiomove(const struct mbuf *m, int offset, int len, struct uio *uio)
{
int curlen, startlen, resid_init, err = 0;
caddr_t buf;
DPRINTF("m_uiomove(m=%p, offset=%d, len=%d, ...)\n",
m, offset, len);
startlen = len;
resid_init = uio->uio_resid;
while (m && len) {
buf = mtod(m, caddr_t);
curlen = m->m_len;
if (offset && (offset < curlen)) {
curlen -= offset;
buf += offset;
offset = 0;
} else if (offset) {
offset -= curlen;
m = m->m_next;
continue;
}
err = uiomove(buf, min(len, curlen), uio);
if (err) {
printf("uiomove returned %d\n", err);
return (err);
}
len -= min(len, curlen);
m = m->m_next;
}
DPRINTF("copied %d bytes - resid_init=%d uio_resid=%d\n",
startlen - len, resid_init, uio->uio_resid);
return (err);
}
/*
* Copy data from an sk_buff to an iovec. Deals with RX_DATA, which carry the
* data in the sk_buff body, and with RX_DATA_DDP, which place the data in a
* DDP buffer.
*/
static inline int
copy_data(const struct mbuf *m, int offset, int len, struct uio *uio)
{
struct iovec *to = uio->uio_iov;
int err;
if (__predict_true(!is_ddp(m))) { /* RX_DATA */
return m_uiomove(m, offset, len, uio);
} if (__predict_true(m->m_ddp_flags & DDP_BF_NOCOPY)) { /* user DDP */
to->iov_len -= len;
to->iov_base = ((caddr_t)to->iov_base) + len;
uio->uio_iov = to;
uio->uio_resid -= len;
return (0);
}
err = t3_ddp_copy(m, offset, uio, len); /* kernel DDP */
return (err);
}
static void
cxgb_wait_dma_completion(struct toepcb *tp)
cxgb_wait_dma_completion(struct toepcb *toep)
{
struct mtx *lock;
lock = &toep->tp_tp->t_inpcb->inp_mtx;
INP_LOCK(toep->tp_tp->t_inpcb);
cv_wait_unlock(&toep->tp_cv, lock);
}
static int
@ -234,7 +378,13 @@ cxgb_vm_page_to_miov(struct toepcb *toep, struct uio *uio, struct mbuf **m)
mi_collapse_sge(mi, segs);
*m = m0;
/*
* This appears to be a no-op at the moment
* as busdma is all or nothing need to make
* sure the tag values are large enough
*
*/
if (cdi.cdi_mapped < uio->uio_resid) {
uio->uio_resid -= cdi.cdi_mapped;
} else
@ -305,10 +455,11 @@ t3_sosend(struct socket *so, struct uio *uio)
}
uio->uio_resid -= m->m_pkthdr.len;
sent += m->m_pkthdr.len;
sbappend_locked(&so->so_snd, m);
sbappend(&so->so_snd, m);
t3_push_frames(so, TRUE);
iov_adj(&uiotmp.uio_iov, &iovcnt, uiotmp.uio_resid);
}
/*
* Wait for pending I/O to be DMA'd to the card
*
@ -357,7 +508,7 @@ cxgb_sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
zcopy_thres = TOM_TUNABLE(tdev, zcopy_sosend_partial_thres);
zcopy_enabled = TOM_TUNABLE(tdev, zcopy_sosend_enabled);
if ((uio->uio_resid > zcopy_thres) &&
if (uio && (uio->uio_resid > zcopy_thres) &&
(uio->uio_iovcnt < TMP_IOV_MAX) && ((so->so_state & SS_NBIO) == 0)
&& zcopy_enabled) {
rv = t3_sosend(so, uio);
@ -368,36 +519,378 @@ cxgb_sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
return pru_sosend(so, addr, uio, top, control, flags, td);
}
/*
* Following replacement or removal of the first mbuf on the first mbuf chain
* of a socket buffer, push necessary state changes back into the socket
* buffer so that other consumers see the values consistently. 'nextrecord'
* is the callers locally stored value of the original value of
* sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes.
* NOTE: 'nextrecord' may be NULL.
*/
static __inline void
sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord)
{
SOCKBUF_LOCK_ASSERT(sb);
/*
* First, update for the new value of nextrecord. If necessary, make
* it the first record.
*/
if (sb->sb_mb != NULL)
sb->sb_mb->m_nextpkt = nextrecord;
else
sb->sb_mb = nextrecord;
/*
* Now update any dependent socket buffer fields to reflect the new
* state. This is an expanded inline of SB_EMPTY_FIXUP(), with the
* addition of a second clause that takes care of the case where
* sb_mb has been updated, but remains the last record.
*/
if (sb->sb_mb == NULL) {
sb->sb_mbtail = NULL;
sb->sb_lastrecord = NULL;
} else if (sb->sb_mb->m_nextpkt == NULL)
sb->sb_lastrecord = sb->sb_mb;
}
#define IS_NONBLOCKING(so) ((so)->so_state & SS_NBIO)
static int
t3_soreceive(struct socket *so, struct uio *uio)
t3_soreceive(struct socket *so, int *flagsp, struct uio *uio)
{
#ifdef notyet
int i, rv, count, hold_resid, sent, iovcnt;
struct iovec iovtmp[TMP_IOV_MAX], *iovtmpp, *iov;
struct tcpcb *tp = sototcpcb(so);
struct toepcb *toep = tp->t_toe;
struct mbuf *m;
struct uio uiotmp;
uint32_t offset;
int err, flags, avail, len, copied, copied_unacked;
int target; /* Read at least this many bytes */
int user_ddp_ok;
struct ddp_state *p;
struct inpcb *inp = sotoinpcb(so);
avail = offset = copied = copied_unacked = 0;
flags = flagsp ? (*flagsp &~ MSG_EOR) : 0;
err = sblock(&so->so_rcv, SBLOCKWAIT(flags));
p = &toep->tp_ddp_state;
if (err)
return (err);
SOCKBUF_LOCK(&so->so_rcv);
p->user_ddp_pending = 0;
restart:
len = uio->uio_resid;
m = so->so_rcv.sb_mb;
target = (flags & MSG_WAITALL) ? len : so->so_rcv.sb_lowat;
user_ddp_ok = p->ubuf_ddp_ready;
p->cancel_ubuf = 0;
if (len == 0)
goto done;
#if 0
while (m && m->m_len == 0) {
so->so_rcv.sb_mb = m_free(m);
m = so->so_rcv.sb_mb;
}
#endif
if (m)
goto got_mbuf;
/* empty receive queue */
if (copied >= target && (so->so_rcv.sb_mb == NULL) &&
!p->user_ddp_pending)
goto done;
if (copied) {
if (so->so_error || tp->t_state == TCPS_CLOSED ||
(so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)))
goto done;
} else {
if (so->so_state & SS_NOFDREF)
goto done;
if (so->so_error) {
err = so->so_error;
so->so_error = 0;
goto done;
}
if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
goto done;
if (so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED))
goto done;
if (tp->t_state == TCPS_CLOSED) {
err = ENOTCONN;
goto done;
}
}
if (so->so_rcv.sb_mb && !p->user_ddp_pending) {
SOCKBUF_UNLOCK(&so->so_rcv);
INP_LOCK(inp);
t3_cleanup_rbuf(tp, copied_unacked);
INP_UNLOCK(inp);
SOCKBUF_LOCK(&so->so_rcv);
copied_unacked = 0;
goto restart;
}
if (p->kbuf[0] && user_ddp_ok && !p->user_ddp_pending &&
uio->uio_iov->iov_len > p->kbuf[0]->dgl_length &&
p->ubuf_ddp_ready) {
p->user_ddp_pending =
!t3_overlay_ubuf(so, uio, IS_NONBLOCKING(so), flags, 1, 1);
if (p->user_ddp_pending) {
p->kbuf_posted++;
user_ddp_ok = 0;
}
}
if (p->kbuf[0] && (p->kbuf_posted == 0)) {
t3_post_kbuf(so, 1, IS_NONBLOCKING(so));
p->kbuf_posted++;
}
if (p->user_ddp_pending) {
/* One shot at DDP if we already have enough data */
if (copied >= target)
user_ddp_ok = 0;
DPRINTF("sbwaiting 1\n");
if ((err = sbwait(&so->so_rcv)) != 0)
goto done;
//for timers to work await_ddp_completion(sk, flags, &timeo);
} else if (copied >= target)
goto done;
else {
if (copied_unacked) {
int i = 0;
SOCKBUF_UNLOCK(&so->so_rcv);
INP_LOCK(inp);
t3_cleanup_rbuf(tp, copied_unacked);
INP_UNLOCK(inp);
copied_unacked = 0;
if (mp_ncpus > 1)
while (i++ < 200 && so->so_rcv.sb_mb == NULL)
cpu_spinwait();
SOCKBUF_LOCK(&so->so_rcv);
}
if (so->so_rcv.sb_mb)
goto restart;
DPRINTF("sbwaiting 2 copied=%d target=%d avail=%d so=%p mb=%p cc=%d\n", copied, target, avail, so,
so->so_rcv.sb_mb, so->so_rcv.sb_cc);
if ((err = sbwait(&so->so_rcv)) != 0)
goto done;
}
goto restart;
got_mbuf:
KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d m_pktlen=%d\n", !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len, m->m_pkthdr.len));
KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x m->m_len=%d",
m->m_next, m->m_nextpkt, m->m_flags, m->m_len));
if (m->m_pkthdr.len == 0) {
if ((m->m_ddp_flags & DDP_BF_NOCOPY) == 0)
panic("empty mbuf and NOCOPY not set\n");
CTR0(KTR_TOM, "ddp done notification");
p->user_ddp_pending = 0;
sbdroprecord_locked(&so->so_rcv);
goto done;
}
offset = toep->tp_copied_seq + copied_unacked - m->m_seq;
DPRINTF("m=%p copied_seq=0x%x copied_unacked=%d m_seq=0x%x offset=%d pktlen=%d is_ddp(m)=%d\n",
m, toep->tp_copied_seq, copied_unacked, m->m_seq, offset, m->m_pkthdr.len, !!is_ddp(m));
if (offset >= m->m_pkthdr.len)
panic("t3_soreceive: OFFSET >= LEN offset %d copied_seq 0x%x seq 0x%x "
"pktlen %d ddp flags 0x%x", offset, toep->tp_copied_seq + copied_unacked, m->m_seq,
m->m_pkthdr.len, m->m_ddp_flags);
avail = m->m_pkthdr.len - offset;
if (len < avail) {
if (is_ddp(m) && (m->m_ddp_flags & DDP_BF_NOCOPY))
panic("bad state in t3_soreceive len=%d avail=%d offset=%d\n", len, avail, offset);
avail = len;
}
CTR4(KTR_TOM, "t3_soreceive: m_len=%u offset=%u len=%u m_seq=0%08x", m->m_pkthdr.len, offset, len, m->m_seq);
#ifdef URGENT_DATA_SUPPORTED
/*
* Events requiring iteration:
* - number of pages exceeds max hold pages for process or system
* - number of pages exceeds maximum sg entries for a single WR
*
* We're limited to holding 128 pages at once - and we're limited to
* 34 SG entries per work request, but each SG entry can be any number
* of contiguous pages
*
* Check if the data we are preparing to copy contains urgent
* data. Either stop short of urgent data or skip it if it's
* first and we are not delivering urgent data inline.
*/
if (__predict_false(toep->tp_urg_data)) {
uint32_t urg_offset = tp->rcv_up - tp->copied_seq + copied_unacked;
if (urg_offset < avail) {
if (urg_offset) {
/* stop short of the urgent data */
avail = urg_offset;
} else if ((so->so_options & SO_OOBINLINE) == 0) {
/* First byte is urgent, skip */
toep->tp_copied_seq++;
offset++;
avail--;
if (!avail)
goto skip_copy;
}
}
}
#endif
if (is_ddp_psh(m) || offset) {
user_ddp_ok = 0;
#ifdef T3_TRACE
T3_TRACE0(TIDTB(so), "t3_sosend: PSH");
#endif
}
if (user_ddp_ok && !p->user_ddp_pending &&
uio->uio_iov->iov_len > p->kbuf[0]->dgl_length &&
p->ubuf_ddp_ready) {
p->user_ddp_pending =
!t3_overlay_ubuf(so, uio, IS_NONBLOCKING(so), flags, 1, 1);
if (p->user_ddp_pending) {
p->kbuf_posted++;
user_ddp_ok = 0;
}
DPRINTF("user_ddp_pending=%d\n", p->user_ddp_pending);
} else
DPRINTF("user_ddp_ok=%d user_ddp_pending=%d iov_len=%ld dgl_length=%d ubuf_ddp_ready=%d ulp_mode=%d is_ddp(m)=%d flags=0x%x ubuf=%p kbuf_posted=%d\n",
user_ddp_ok, p->user_ddp_pending, uio->uio_iov->iov_len, p->kbuf[0] ? p->kbuf[0]->dgl_length : 0,
p->ubuf_ddp_ready, toep->tp_ulp_mode, !!is_ddp(m), m->m_ddp_flags, p->ubuf, p->kbuf_posted);
/*
* If MSG_TRUNC is specified the data is discarded.
* XXX need to check pr_atomic
*/
KASSERT(avail > 0, ("avail=%d resid=%d offset=%d", avail, uio->uio_resid, offset));
if (__predict_true(!(flags & MSG_TRUNC))) {
int resid = uio->uio_resid;
SOCKBUF_UNLOCK(&so->so_rcv);
if ((err = copy_data(m, offset, avail, uio))) {
if (err)
err = EFAULT;
goto done_unlocked;
}
SOCKBUF_LOCK(&so->so_rcv);
if (avail != (resid - uio->uio_resid))
printf("didn't copy all bytes :-/ avail=%d offset=%d pktlen=%d resid=%d uio_resid=%d copied=%d copied_unacked=%d is_ddp(m)=%d\n",
avail, offset, m->m_pkthdr.len, resid, uio->uio_resid, copied, copied_unacked, is_ddp(m));
}
copied += avail;
copied_unacked += avail;
len -= avail;
#ifdef URGENT_DATA_SUPPORTED
skip_copy:
if (tp->urg_data && after(tp->copied_seq + copied_unacked, tp->urg_seq))
tp->urg_data = 0;
#endif
/*
* If the buffer is fully consumed free it. If it's a DDP
* buffer also handle any events it indicates.
*/
if (avail + offset >= m->m_pkthdr.len) {
unsigned int fl = m->m_ddp_flags;
int exitnow, got_psh = 0, nomoredata = 0;
int count;
struct mbuf *nextrecord;
uiotmp = *uio;
iovcnt = uio->uio_iovcnt;
iov = uio->uio_iov;
sent = 0;
re;
#endif
return (0);
if (p->kbuf[0] != NULL && is_ddp(m) && (fl & 1)) {
if (is_ddp_psh(m) && p->user_ddp_pending)
got_psh = 1;
if (fl & DDP_BF_NOCOPY)
p->user_ddp_pending = 0;
else if ((fl & DDP_BF_NODATA) && IS_NONBLOCKING(so)) {
p->kbuf_posted--;
nomoredata = 1;
} else {
p->kbuf_posted--;
p->ubuf_ddp_ready = 1;
}
}
nextrecord = m->m_nextpkt;
count = m->m_pkthdr.len;
while (count > 0) {
count -= m->m_len;
KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n", !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len));
sbfree(&so->so_rcv, m);
so->so_rcv.sb_mb = m_free(m);
m = so->so_rcv.sb_mb;
}
sockbuf_pushsync(&so->so_rcv, nextrecord);
#if 0
sbdrop_locked(&so->so_rcv, m->m_pkthdr.len);
#endif
exitnow = got_psh || nomoredata;
if ((so->so_rcv.sb_mb == NULL) && exitnow)
goto done;
if (copied_unacked > (so->so_rcv.sb_hiwat >> 2)) {
SOCKBUF_UNLOCK(&so->so_rcv);
INP_LOCK(inp);
t3_cleanup_rbuf(tp, copied_unacked);
INP_UNLOCK(inp);
copied_unacked = 0;
SOCKBUF_LOCK(&so->so_rcv);
}
}
if (len > 0)
goto restart;
done:
/*
* If we can still receive decide what to do in preparation for the
* next receive. Note that RCV_SHUTDOWN is set if the connection
* transitioned to CLOSE but not if it was in that state to begin with.
*/
if (__predict_true((so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)) == 0)) {
if (p->user_ddp_pending) {
SOCKBUF_UNLOCK(&so->so_rcv);
SOCKBUF_LOCK(&so->so_rcv);
user_ddp_ok = 0;
t3_cancel_ubuf(toep);
if (so->so_rcv.sb_mb) {
if (copied < 0)
copied = 0;
if (len > 0)
goto restart;
}
p->user_ddp_pending = 0;
}
if ((p->kbuf[0] != NULL) && (p->kbuf_posted == 0)) {
#ifdef T3_TRACE
T3_TRACE0(TIDTB(so),
"chelsio_recvmsg: about to exit, repost kbuf");
#endif
t3_post_kbuf(so, 1, IS_NONBLOCKING(so));
p->kbuf_posted++;
} else if (so_should_ddp(toep, copied) && uio->uio_iovcnt == 1) {
CTR1(KTR_TOM ,"entering ddp on tid=%u", toep->tp_tid);
if (!t3_enter_ddp(so, TOM_TUNABLE(TOE_DEV(so),
ddp_copy_limit), 0, IS_NONBLOCKING(so)))
p->kbuf_posted = 1;
}
}
#ifdef T3_TRACE
T3_TRACE5(TIDTB(so),
"chelsio_recvmsg <-: copied %d len %d buffers_freed %d "
"kbuf_posted %d user_ddp_pending %u",
copied, len, buffers_freed, p ? p->kbuf_posted : -1,
p->user_ddp_pending);
#endif
SOCKBUF_UNLOCK(&so->so_rcv);
done_unlocked:
if (copied_unacked) {
INP_LOCK(inp);
t3_cleanup_rbuf(tp, copied_unacked);
INP_UNLOCK(inp);
}
sbunlock(&so->so_rcv);
return (err);
}
static int
@ -405,9 +898,11 @@ cxgb_soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
{
struct toedev *tdev;
int rv, zcopy_thres, zcopy_enabled;
int rv, zcopy_thres, zcopy_enabled, flags;
struct tcpcb *tp = sototcpcb(so);
flags = flagsp ? *flagsp &~ MSG_EOR : 0;
/*
* In order to use DMA direct from userspace the following
* conditions must be met:
@ -421,150 +916,30 @@ cxgb_soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
* - iovcnt is 1
*
*/
if (tp->t_flags & TF_TOE) {
if ((tp->t_flags & TF_TOE) && uio && ((flags & (MSG_WAITALL|MSG_OOB|MSG_PEEK|MSG_DONTWAIT)) == 0)
&& (uio->uio_iovcnt == 1) && (mp0 == NULL)) {
tdev = TOE_DEV(so);
zcopy_thres = TOM_TUNABLE(tdev, ddp_thres);
zcopy_enabled = TOM_TUNABLE(tdev, ddp);
if ((uio->uio_resid > zcopy_thres) &&
(uio->uio_iovcnt == 1) && ((so->so_state & SS_NBIO) == 0)
(uio->uio_iovcnt == 1)
&& zcopy_enabled) {
rv = t3_soreceive(so, uio);
rv = t3_soreceive(so, flagsp, uio);
if (rv != EAGAIN)
return (rv);
}
}
else
printf("returned EAGAIN\n");
}
} else if ((tp->t_flags & TF_TOE) && uio && mp0 == NULL)
printf("skipping t3_soreceive flags=0x%x iovcnt=%d sb_state=0x%x\n",
flags, uio->uio_iovcnt, so->so_rcv.sb_state);
return pru_soreceive(so, psa, uio, mp0, controlp, flagsp);
}
void
t3_install_socket_ops(struct socket *so)
{
so->so_proto->pr_usrreqs->pru_sosend = cxgb_sosend;
so->so_proto->pr_usrreqs->pru_soreceive = cxgb_soreceive;
}
/*
* This routine takes a user address range and does the following:
* - validate that the user has access to those pages (flags indicates read or write) - if not fail
* - validate that count is enough to hold range number of pages - if not fail
* - fault in any non-resident pages
* - if the user is doing a read force a write fault for any COWed pages
* - if the user is doing a read mark all pages as dirty
* - hold all pages
* - return number of pages in count
*/
#ifdef notyet
static int
vm_fault_hold_user_pages(vm_offset_t addr, int len, vm_page_t *mp, int *count, int flags)
{
vm_offset_t start, va;
vm_paddr_t pa;
int pageslen, faults, rv;
struct thread *td;
vm_map_t map;
pmap_t pmap;
vm_page_t m, *pages;
vm_prot_t prot;
start = addr & ~PAGE_MASK;
pageslen = roundup2(addr + len, PAGE_SIZE);
if (*count < (pageslen >> PAGE_SHIFT))
return (EFBIG);
*count = pageslen >> PAGE_SHIFT;
/*
* Check that virtual address range is legal
* This check is somewhat bogus as on some architectures kernel
* and user do not share VA - however, it appears that all FreeBSD
* architectures define it
*/
if (addr + len > VM_MAXUSER_ADDRESS)
return (EFAULT);
td = curthread;
map = &td->td_proc->p_vmspace->vm_map;
pmap = &td->td_proc->p_vmspace->vm_pmap;
pages = mp;
prot = (flags & VM_HOLD_WRITEABLE) ? VM_PROT_WRITE : VM_PROT_READ;
bzero(pages, sizeof(vm_page_t *) * (*count));
retry:
/*
* First optimistically assume that all pages are resident (and R/W if for write)
* if so just mark pages as held (and dirty if for write) and return
*/
vm_page_lock_queues();
for (pages = mp, faults = 0, va = start; va < pageslen; va += PAGE_SIZE, pages++) {
/*
* Assure that we only hold the page once
*/
if (*pages == NULL) {
/*
* page queue mutex is recursable so this is OK
* it would be really nice if we had an unlocked version of this so
* we were only acquiring the pmap lock 1 time as opposed to potentially
* many dozens of times
*/
m = pmap_extract_and_hold(pmap, va, prot);
if (m == NULL) {
faults++;
continue;
}
*pages = m;
if (flags & VM_HOLD_WRITEABLE)
vm_page_dirty(m);
}
}
vm_page_unlock_queues();
if (faults == 0)
return (0);
/*
* Pages either have insufficient permissions or are not present
* trigger a fault where neccessary
*
*/
for (va = start; va < pageslen; va += PAGE_SIZE) {
m = NULL;
pa = pmap_extract(pmap, va);
rv = 0;
if (pa)
m = PHYS_TO_VM_PAGE(pa);
if (flags & VM_HOLD_WRITEABLE) {
if (m == NULL || (m->flags & PG_WRITEABLE) == 0)
rv = vm_fault(map, va, VM_PROT_WRITE, VM_FAULT_DIRTY);
} else if (m == NULL)
rv = vm_fault(map, va, VM_PROT_READ, VM_FAULT_NORMAL);
if (rv)
goto error;
}
goto retry;
error:
vm_page_lock_queues();
for (pages = mp, va = start; va < pageslen; va += PAGE_SIZE, pages++)
if (*pages)
vm_page_unhold(*pages);
vm_page_unlock_queues();
return (EFAULT);
}
#endif
static void
vm_fault_unhold_pages(vm_page_t *mp, int count)
{
KASSERT(count >= 0, ("negative count %d", count));
vm_page_lock_queues();
while (count--) {
vm_page_unhold(*mp);
mp++;
}
vm_page_unlock_queues();
}

735
sys/dev/cxgb/ulp/tom/cxgb_ddp.c Normal file
View File

@ -0,0 +1,735 @@
/**************************************************************************
Copyright (c) 2007, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***************************************************************************/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/fcntl.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mbuf.h>
#include <sys/condvar.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/syslog.h>
#include <sys/socketvar.h>
#include <sys/uio.h>
#include <machine/bus.h>
#include <net/if.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/in_pcb.h>
#include <netinet/in_systm.h>
#include <netinet/in_var.h>
#include <dev/cxgb/cxgb_osdep.h>
#include <dev/cxgb/sys/mbufq.h>
#include <netinet/tcp.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_offload.h>
#include <net/route.h>
#include <dev/cxgb/t3cdev.h>
#include <dev/cxgb/common/cxgb_firmware_exports.h>
#include <dev/cxgb/common/cxgb_t3_cpl.h>
#include <dev/cxgb/common/cxgb_tcb.h>
#include <dev/cxgb/common/cxgb_ctl_defs.h>
#include <dev/cxgb/cxgb_l2t.h>
#include <dev/cxgb/cxgb_offload.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <dev/cxgb/sys/mvec.h>
#include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
#include <dev/cxgb/ulp/tom/cxgb_defs.h>
#include <dev/cxgb/ulp/tom/cxgb_tom.h>
#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h>
#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
#include <dev/cxgb/ulp/tom/cxgb_vm.h>
#define MAX_SCHEDULE_TIMEOUT 300
/*
* Return the # of page pods needed to accommodate a # of pages.
*/
static inline unsigned int
pages2ppods(unsigned int pages)
{
return (pages + PPOD_PAGES - 1) / PPOD_PAGES + NUM_SENTINEL_PPODS;
}
/**
* t3_pin_pages - pin a user memory range and prepare it for DDP
* @addr - the starting address
* @len - the length of the range
* @newgl - contains the pages and physical addresses of the pinned range
* @gl - an existing gather list, may be %NULL
*
* Pins the pages in the user-space memory range [addr, addr + len) and
* maps them for DMA. Returns a gather list with the pinned pages and
* their physical addresses. If @gl is non NULL the pages it describes
* are compared against the pages for [addr, addr + len), and if the
* existing gather list already covers the range a new list is not
* allocated. Returns 0 on success, or a negative errno. On success if
* a new gather list was allocated it is returned in @newgl.
*/
static int
t3_pin_pages(bus_dma_tag_t tag, bus_dmamap_t map, vm_offset_t addr,
size_t len, struct ddp_gather_list **newgl,
const struct ddp_gather_list *gl)
{
int i = 0, err;
size_t pg_off;
unsigned int npages;
struct ddp_gather_list *p;
/*
* XXX need x86 agnostic check
*/
if (addr + len > VM_MAXUSER_ADDRESS)
return (EFAULT);
pg_off = addr & PAGE_MASK;
npages = (pg_off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
p = malloc(sizeof(struct ddp_gather_list) + npages * sizeof(vm_page_t *),
M_DEVBUF, M_NOWAIT|M_ZERO);
if (p == NULL)
return (ENOMEM);
err = vm_fault_hold_user_pages(addr, p->dgl_pages, npages, VM_HOLD_WRITEABLE);
if (err)
goto free_gl;
if (gl && gl->dgl_offset == pg_off && gl->dgl_nelem >= npages &&
gl->dgl_length >= len) {
for (i = 0; i < npages; i++)
if (p->dgl_pages[i] != gl->dgl_pages[i])
goto different_gl;
err = 0;
goto unpin;
}
different_gl:
p->dgl_length = len;
p->dgl_offset = pg_off;
p->dgl_nelem = npages;
#ifdef NEED_BUSDMA
p->phys_addr[0] = pci_map_page(pdev, p->pages[0], pg_off,
PAGE_SIZE - pg_off,
PCI_DMA_FROMDEVICE) - pg_off;
for (i = 1; i < npages; ++i)
p->phys_addr[i] = pci_map_page(pdev, p->pages[i], 0, PAGE_SIZE,
PCI_DMA_FROMDEVICE);
#endif
*newgl = p;
return (0);
unpin:
vm_fault_unhold_pages(p->dgl_pages, npages);
free_gl:
free(p, M_DEVBUF);
*newgl = NULL;
return (err);
}
static void
unmap_ddp_gl(const struct ddp_gather_list *gl)
{
#ifdef NEED_BUSDMA
int i;
if (!gl->nelem)
return;
pci_unmap_page(pdev, gl->phys_addr[0] + gl->offset,
PAGE_SIZE - gl->offset, PCI_DMA_FROMDEVICE);
for (i = 1; i < gl->nelem; ++i)
pci_unmap_page(pdev, gl->phys_addr[i], PAGE_SIZE,
PCI_DMA_FROMDEVICE);
#endif
}
static void
ddp_gl_free_pages(struct ddp_gather_list *gl, int dirty)
{
/*
* XXX mark pages as dirty before unholding
*/
vm_fault_unhold_pages(gl->dgl_pages, gl->dgl_nelem);
}
void
t3_free_ddp_gl(struct ddp_gather_list *gl)
{
unmap_ddp_gl(gl);
ddp_gl_free_pages(gl, 0);
free(gl, M_DEVBUF);
}
/* Max # of page pods for a buffer, enough for 1MB buffer at 4KB page size */
#define MAX_PPODS 64U
/*
* Allocate page pods for DDP buffer 1 (the user buffer) and set up the tag in
* the TCB. We allocate page pods in multiples of PPOD_CLUSTER_SIZE. First we
* try to allocate enough page pods to accommodate the whole buffer, subject to
* the MAX_PPODS limit. If that fails we try to allocate PPOD_CLUSTER_SIZE page
* pods before failing entirely.
*/
static int
alloc_buf1_ppods(struct socket *so, struct ddp_state *p,
unsigned long addr, unsigned int len)
{
int err, tag, npages, nppods;
struct tom_data *d = TOM_DATA(TOE_DEV(so));
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
npages = ((addr & PAGE_MASK) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
nppods = min(pages2ppods(npages), MAX_PPODS);
nppods = roundup2(nppods, PPOD_CLUSTER_SIZE);
err = t3_alloc_ppods(d, nppods, &tag);
if (err && nppods > PPOD_CLUSTER_SIZE) {
nppods = PPOD_CLUSTER_SIZE;
err = t3_alloc_ppods(d, nppods, &tag);
}
if (err)
return (ENOMEM);
p->ubuf_nppods = nppods;
p->ubuf_tag = tag;
#if NUM_DDP_KBUF == 1
t3_set_ddp_tag(so, 1, tag << 6);
#endif
return (0);
}
/*
* Starting offset for the user DDP buffer. A non-0 value ensures a DDP flush
* won't block indefinitely if there's nothing to place (which should be rare).
*/
#define UBUF_OFFSET 1
static __inline unsigned long
select_ddp_flags(const struct socket *so, int buf_idx,
int nonblock, int rcv_flags)
{
if (buf_idx == 1) {
if (__predict_false(rcv_flags & MSG_WAITALL))
return V_TF_DDP_PSH_NO_INVALIDATE0(1) |
V_TF_DDP_PSH_NO_INVALIDATE1(1) |
V_TF_DDP_PUSH_DISABLE_1(1);
if (nonblock)
return V_TF_DDP_BUF1_FLUSH(1);
return V_TF_DDP_BUF1_FLUSH(!TOM_TUNABLE(TOE_DEV(so),
ddp_push_wait));
}
if (__predict_false(rcv_flags & MSG_WAITALL))
return V_TF_DDP_PSH_NO_INVALIDATE0(1) |
V_TF_DDP_PSH_NO_INVALIDATE1(1) |
V_TF_DDP_PUSH_DISABLE_0(1);
if (nonblock)
return V_TF_DDP_BUF0_FLUSH(1);
return V_TF_DDP_BUF0_FLUSH(!TOM_TUNABLE(TOE_DEV(so), ddp_push_wait));
}
/*
* Reposts the kernel DDP buffer after it has been previously become full and
* invalidated. We just need to reset the offset and adjust the DDP flags.
* Conveniently, we can set the flags and the offset with a single message.
* Note that this function does not set the buffer length. Again conveniently
* our kernel buffer is of fixed size. If the length needs to be changed it
* needs to be done separately.
*/
static void
t3_repost_kbuf(struct socket *so, unsigned int bufidx, int modulate,
int activate, int nonblock)
{
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
unsigned long flags;
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
p->buf_state[bufidx].cur_offset = p->kbuf[bufidx]->dgl_offset;
p->buf_state[bufidx].flags = p->kbuf_noinval ? DDP_BF_NOINVAL : 0;
p->buf_state[bufidx].gl = p->kbuf[bufidx];
p->cur_buf = bufidx;
p->kbuf_idx = bufidx;
flags = select_ddp_flags(so, bufidx, nonblock, 0);
if (!bufidx)
t3_setup_ddpbufs(toep, 0, 0, 0, 0, flags |
V_TF_DDP_PSH_NO_INVALIDATE0(p->kbuf_noinval) |
V_TF_DDP_PSH_NO_INVALIDATE1(p->kbuf_noinval) |
V_TF_DDP_BUF0_VALID(1),
V_TF_DDP_BUF0_FLUSH(1) |
V_TF_DDP_PSH_NO_INVALIDATE0(1) |
V_TF_DDP_PSH_NO_INVALIDATE1(1) | V_TF_DDP_OFF(1) |
V_TF_DDP_BUF0_VALID(1) |
V_TF_DDP_ACTIVE_BUF(activate), modulate);
else
t3_setup_ddpbufs(toep, 0, 0, 0, 0, flags |
V_TF_DDP_PSH_NO_INVALIDATE0(p->kbuf_noinval) |
V_TF_DDP_PSH_NO_INVALIDATE1(p->kbuf_noinval) |
V_TF_DDP_BUF1_VALID(1) |
V_TF_DDP_ACTIVE_BUF(activate),
V_TF_DDP_BUF1_FLUSH(1) |
V_TF_DDP_PSH_NO_INVALIDATE0(1) |
V_TF_DDP_PSH_NO_INVALIDATE1(1) | V_TF_DDP_OFF(1) |
V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1),
modulate);
}
/**
* setup_uio_ppods - setup HW page pods for a user iovec
* @sk: the associated socket
* @uio: the uio
* @oft: additional bytes to map before the start of the buffer
*
* Pins a user iovec and sets up HW page pods for DDP into it. We allocate
* page pods for user buffers on the first call per socket. Afterwards we
* limit the buffer length to whatever the existing page pods can accommodate.
* Returns a negative error code or the length of the mapped buffer.
*
* The current implementation handles iovecs with only one entry.
*/
static int
setup_uio_ppods(struct socket *so, const struct uio *uio, int oft, int *length)
{
int err;
unsigned int len;
struct ddp_gather_list *gl = NULL;
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
struct iovec *iov = uio->uio_iov;
vm_offset_t addr = (vm_offset_t)iov->iov_base - oft;
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
if (__predict_false(p->ubuf_nppods == 0)) {
err = alloc_buf1_ppods(so, p, addr, iov->iov_len + oft);
if (err)
return (err);
}
len = (p->ubuf_nppods - NUM_SENTINEL_PPODS) * PPOD_PAGES * PAGE_SIZE;
len -= addr & PAGE_MASK;
if (len > M_TCB_RX_DDP_BUF0_LEN)
len = M_TCB_RX_DDP_BUF0_LEN;
len = min(len, sototcpcb(so)->rcv_wnd - 32768);
len = min(len, iov->iov_len + oft);
if (len <= p->kbuf[0]->dgl_length) {
printf("length too short\n");
return (EINVAL);
}
err = t3_pin_pages(toep->tp_rx_dmat, toep->tp_dmamap, addr, len, &gl, p->ubuf);
if (err)
return (err);
if (gl) {
if (p->ubuf)
t3_free_ddp_gl(p->ubuf);
p->ubuf = gl;
t3_setup_ppods(so, gl, pages2ppods(gl->dgl_nelem), p->ubuf_tag, len,
gl->dgl_offset, 0);
}
*length = len;
return (0);
}
/*
*
*/
void
t3_cancel_ubuf(struct toepcb *toep)
{
struct ddp_state *p = &toep->tp_ddp_state;
int ubuf_pending = t3_ddp_ubuf_pending(toep);
struct socket *so = toeptoso(toep);
int err = 0, count=0;
if (p->ubuf == NULL)
return;
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
p->cancel_ubuf = 1;
while (ubuf_pending && !(so->so_rcv.sb_state & SBS_CANTRCVMORE)) {
#ifdef T3_TRACE
T3_TRACE3(TB(p),
"t3_cancel_ubuf: flags0 0x%x flags1 0x%x get_tcb_count %d",
p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
p->get_tcb_count);
#endif
CTR3(KTR_TOM,
"t3_cancel_ubuf: flags0 0x%x flags1 0x%x get_tcb_count %d",
p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
p->get_tcb_count);
if (p->get_tcb_count == 0)
t3_cancel_ddpbuf(toep, p->cur_buf);
else
CTR5(KTR_TOM, "waiting err=%d get_tcb_count=%d timeo=%d so=%p SBS_CANTRCVMORE=%d",
err, p->get_tcb_count, so->so_rcv.sb_timeo, so,
!!(so->so_rcv.sb_state & SBS_CANTRCVMORE));
while (p->get_tcb_count && !(so->so_rcv.sb_state & SBS_CANTRCVMORE)) {
if (count & 0xfffffff)
CTR5(KTR_TOM, "waiting err=%d get_tcb_count=%d timeo=%d so=%p count=%d",
err, p->get_tcb_count, so->so_rcv.sb_timeo, so, count);
count++;
err = sbwait(&so->so_rcv);
}
ubuf_pending = t3_ddp_ubuf_pending(toep);
}
p->cancel_ubuf = 0;
}
#define OVERLAY_MASK (V_TF_DDP_PSH_NO_INVALIDATE0(1) | \
V_TF_DDP_PSH_NO_INVALIDATE1(1) | \
V_TF_DDP_BUF1_FLUSH(1) | \
V_TF_DDP_BUF0_FLUSH(1) | \
V_TF_DDP_PUSH_DISABLE_1(1) | \
V_TF_DDP_PUSH_DISABLE_0(1) | \
V_TF_DDP_INDICATE_OUT(1))
/*
* Post a user buffer as an overlay on top of the current kernel buffer.
*/
int
t3_overlay_ubuf(struct socket *so, const struct uio *uio,
int nonblock, int rcv_flags, int modulate, int post_kbuf)
{
int err, len, ubuf_idx;
unsigned long flags;
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
if (p->kbuf[0] == NULL) {
return (EINVAL);
}
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
err = setup_uio_ppods(so, uio, 0, &len);
if (err) {
return (err);
}
ubuf_idx = p->kbuf_idx;
p->buf_state[ubuf_idx].flags = DDP_BF_NOFLIP;
/* Use existing offset */
/* Don't need to update .gl, user buffer isn't copied. */
p->cur_buf = ubuf_idx;
flags = select_ddp_flags(so, ubuf_idx, nonblock, rcv_flags);
if (post_kbuf) {
struct ddp_buf_state *dbs = &p->buf_state[ubuf_idx ^ 1];
dbs->cur_offset = 0;
dbs->flags = 0;
dbs->gl = p->kbuf[ubuf_idx ^ 1];
p->kbuf_idx ^= 1;
flags |= p->kbuf_idx ?
V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_PUSH_DISABLE_1(0) :
V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_PUSH_DISABLE_0(0);
}
if (ubuf_idx == 0) {
t3_overlay_ddpbuf(toep, 0, p->ubuf_tag << 6, p->kbuf_tag[1] << 6,
len);
t3_setup_ddpbufs(toep, 0, 0, p->kbuf[1]->dgl_length, 0,
flags,
OVERLAY_MASK | flags, 1);
} else {
t3_overlay_ddpbuf(toep, 1, p->kbuf_tag[0] << 6, p->ubuf_tag << 6,
len);
t3_setup_ddpbufs(toep, p->kbuf[0]->dgl_length, 0, 0, 0,
flags,
OVERLAY_MASK | flags, 1);
}
#ifdef T3_TRACE
T3_TRACE5(TIDTB(so),
"t3_overlay_ubuf: tag %u flags 0x%x mask 0x%x ubuf_idx %d "
" kbuf_idx %d",
p->ubuf_tag, flags, OVERLAY_MASK, ubuf_idx, p->kbuf_idx);
#endif
CTR3(KTR_TOM,
"t3_overlay_ubuf: tag %u flags 0x%x mask 0x%x",
p->ubuf_tag, flags, OVERLAY_MASK);
CTR3(KTR_TOM,
"t3_overlay_ubuf: ubuf_idx %d kbuf_idx %d post_kbuf %d",
ubuf_idx, p->kbuf_idx, post_kbuf);
return (0);
}
/*
* Clean up DDP state that needs to survive until socket close time, such as the
* DDP buffers. The buffers are already unmapped at this point as unmapping
* needs the PCI device and a socket may close long after the device is removed.
*/
void
t3_cleanup_ddp(struct toepcb *toep)
{
struct ddp_state *p = &toep->tp_ddp_state;
int idx;
for (idx = 0; idx < NUM_DDP_KBUF; idx++)
if (p->kbuf[idx]) {
ddp_gl_free_pages(p->kbuf[idx], 0);
free(p->kbuf[idx], M_DEVBUF);
}
if (p->ubuf) {
ddp_gl_free_pages(p->ubuf, 0);
free(p->ubuf, M_DEVBUF);
p->ubuf = NULL;
}
toep->tp_ulp_mode = 0;
}
/*
* This is a companion to t3_cleanup_ddp() and releases the HW resources
* associated with a connection's DDP state, such as the page pods.
* It's called when HW is done with a connection. The rest of the state
* remains available until both HW and the app are done with the connection.
*/
void
t3_release_ddp_resources(struct toepcb *toep)
{
struct ddp_state *p = &toep->tp_ddp_state;
struct tom_data *d = TOM_DATA(toep->tp_toedev);
int idx;
for (idx = 0; idx < NUM_DDP_KBUF; idx++) {
t3_free_ppods(d, p->kbuf_tag[idx],
p->kbuf_nppods[idx]);
unmap_ddp_gl(p->kbuf[idx]);
}
if (p->ubuf_nppods) {
t3_free_ppods(d, p->ubuf_tag, p->ubuf_nppods);
p->ubuf_nppods = 0;
}
if (p->ubuf)
unmap_ddp_gl(p->ubuf);
}
void
t3_post_kbuf(struct socket *so, int modulate, int nonblock)
{
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
t3_set_ddp_tag(so, p->cur_buf, p->kbuf_tag[p->cur_buf] << 6);
t3_set_ddp_buf(so, p->cur_buf, 0, p->kbuf[p->cur_buf]->dgl_length);
t3_repost_kbuf(so, p->cur_buf, modulate, 1, nonblock);
#ifdef T3_TRACE
T3_TRACE1(TIDTB(so),
"t3_post_kbuf: cur_buf = kbuf_idx = %u ", p->cur_buf);
#endif
CTR1(KTR_TOM,
"t3_post_kbuf: cur_buf = kbuf_idx = %u ", p->cur_buf);
}
/*
* Prepare a socket for DDP. Must be called when the socket is known to be
* open.
*/
int
t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall, int nonblock)
{
int i, err = ENOMEM;
static vm_pindex_t color;
unsigned int nppods, kbuf_pages, idx = 0;
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
struct tom_data *d = TOM_DATA(toep->tp_toedev);
if (kbuf_size > M_TCB_RX_DDP_BUF0_LEN)
return (EINVAL);
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
kbuf_pages = (kbuf_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
nppods = pages2ppods(kbuf_pages);
p->kbuf_noinval = !!waitall;
p->kbuf_tag[NUM_DDP_KBUF - 1] = -1;
for (idx = 0; idx < NUM_DDP_KBUF; idx++) {
p->kbuf[idx] =
malloc(sizeof (struct ddp_gather_list) + kbuf_pages *
sizeof(vm_page_t *), M_DEVBUF, M_NOWAIT|M_ZERO);
if (p->kbuf[idx] == NULL)
goto err;
err = t3_alloc_ppods(d, nppods, &p->kbuf_tag[idx]);
if (err) {
printf("t3_alloc_ppods failed err=%d\n", err);
goto err;
}
p->kbuf_nppods[idx] = nppods;
p->kbuf[idx]->dgl_length = kbuf_size;
p->kbuf[idx]->dgl_offset = 0;
p->kbuf[idx]->dgl_nelem = kbuf_pages;
for (i = 0; i < kbuf_pages; ++i) {
p->kbuf[idx]->dgl_pages[i] = vm_page_alloc(NULL, color,
VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
VM_ALLOC_ZERO);
if (p->kbuf[idx]->dgl_pages[i] == NULL) {
p->kbuf[idx]->dgl_nelem = i;
printf("failed to allocate kbuf pages\n");
goto err;
}
}
#ifdef NEED_BUSDMA
/*
* XXX we'll need this for VT-d or any platform with an iommu :-/
*
*/
for (i = 0; i < kbuf_pages; ++i)
p->kbuf[idx]->phys_addr[i] =
pci_map_page(p->pdev, p->kbuf[idx]->pages[i],
0, PAGE_SIZE, PCI_DMA_FROMDEVICE);
#endif
t3_setup_ppods(so, p->kbuf[idx], nppods, p->kbuf_tag[idx],
p->kbuf[idx]->dgl_length, 0, 0);
}
cxgb_log_tcb(TOEP_T3C_DEV(toep)->adapter, toep->tp_tid);
t3_set_ddp_tag(so, 0, p->kbuf_tag[0] << 6);
t3_set_ddp_buf(so, 0, 0, p->kbuf[0]->dgl_length);
t3_repost_kbuf(so, 0, 0, 1, nonblock);
t3_set_rcv_coalesce_enable(so,
TOM_TUNABLE(TOE_DEV(so), ddp_rcvcoalesce));
#ifdef T3_TRACE
T3_TRACE4(TIDTB(so),
"t3_enter_ddp: kbuf_size %u waitall %u tag0 %d tag1 %d",
kbuf_size, waitall, p->kbuf_tag[0], p->kbuf_tag[1]);
#endif
CTR4(KTR_TOM,
"t3_enter_ddp: kbuf_size %u waitall %u tag0 %d tag1 %d",
kbuf_size, waitall, p->kbuf_tag[0], p->kbuf_tag[1]);
DELAY(100000);
cxgb_log_tcb(TOEP_T3C_DEV(toep)->adapter, toep->tp_tid);
return (0);
err:
t3_release_ddp_resources(toep);
t3_cleanup_ddp(toep);
return (err);
}
int
t3_ddp_copy(const struct mbuf *m, int offset, struct uio *uio, int len)
{
int page_off, resid_init, err;
struct ddp_gather_list *gl = (struct ddp_gather_list *)m->m_ddp_gl;
resid_init = uio->uio_resid;
if (!gl->dgl_pages)
panic("pages not set\n");
offset += gl->dgl_offset + m->m_cur_offset;
page_off = offset & PAGE_MASK;
KASSERT(len <= gl->dgl_length,
("len=%d > dgl_length=%d in ddp_copy\n", len, gl->dgl_length));
err = uiomove_fromphys(gl->dgl_pages, page_off, len, uio);
return (err);
}
/*
* Allocate n page pods. Returns -1 on failure or the page pod tag.
*/
int
t3_alloc_ppods(struct tom_data *td, unsigned int n, int *ptag)
{
unsigned int i, j;
if (__predict_false(!td->ppod_map)) {
printf("ppod_map not set\n");
return (EINVAL);
}
mtx_lock(&td->ppod_map_lock);
for (i = 0; i < td->nppods; ) {
for (j = 0; j < n; ++j) /* scan ppod_map[i..i+n-1] */
if (td->ppod_map[i + j]) {
i = i + j + 1;
goto next;
}
memset(&td->ppod_map[i], 1, n); /* allocate range */
mtx_unlock(&td->ppod_map_lock);
CTR2(KTR_TOM,
"t3_alloc_ppods: n=%u tag=%u", n, i);
*ptag = i;
return (0);
next: ;
}
mtx_unlock(&td->ppod_map_lock);
return (0);
}
void
t3_free_ppods(struct tom_data *td, unsigned int tag, unsigned int n)
{
/* No need to take ppod_lock here */
memset(&td->ppod_map[tag], 0, n);
}

10
sys/dev/cxgb/ulp/tom/cxgb_defs.h
View File

@ -40,6 +40,13 @@ POSSIBILITY OF SUCH DAMAGE.
#define toeptoso(toep) ((toep)->tp_tp->t_inpcb->inp_socket)
#define sototoep(so) (sototcpcb((so))->t_toe)
#define TRACE_ENTER printf("%s:%s entered\n", __FUNCTION__, __FILE__)
#define TRACE_EXIT printf("%s:%s:%d exited\n", __FUNCTION__, __FILE__, __LINE__)
#define KTR_TOM KTR_SPARE2
#define KTR_TCB KTR_SPARE3
struct toepcb;
struct listen_ctx;
typedef void (*defer_handler_t)(struct toedev *dev, struct mbuf *m);
@ -54,7 +61,8 @@ void t3_init_listen_cpl_handlers(void);
int t3_init_cpl_io(void);
void t3_init_wr_tab(unsigned int wr_len);
uint32_t t3_send_rx_credits(struct tcpcb *tp, uint32_t credits, uint32_t dack, int nofail);
void t3_cleanup_rbuf(struct tcpcb *tp);
void t3_send_rx_modulate(struct toepcb *toep);
void t3_cleanup_rbuf(struct tcpcb *tp, int copied);
void t3_init_socket_ops(void);
void t3_install_socket_ops(struct socket *so);

22
sys/dev/cxgb/ulp/tom/cxgb_listen.c
View File

@ -180,7 +180,6 @@ listen_hash_add(struct tom_data *d, struct socket *so, unsigned int stid)
return p;
}
#if 0
/*
* Given a pointer to a listening socket return its server TID by consulting
* the socket->stid map. Returns -1 if the socket is not in the map.
@ -191,16 +190,15 @@ listen_hash_find(struct tom_data *d, struct socket *so)
int stid = -1, bucket = listen_hashfn(so);
struct listen_info *p;
spin_lock(&d->listen_lock);
mtx_lock(&d->listen_lock);
for (p = d->listen_hash_tab[bucket]; p; p = p->next)
if (p->sk == sk) {
if (p->so == so) {
stid = p->stid;
break;
}
spin_unlock(&d->listen_lock);
mtx_unlock(&d->listen_lock);
return stid;
}
#endif
/*
* Delete the listen_info structure for a listening socket. Returns the server
@ -244,28 +242,24 @@ t3_listen_start(struct toedev *dev, struct socket *so, struct t3cdev *cdev)
if (!TOM_TUNABLE(dev, activated))
return;
printf("start listen\n");
if (listen_hash_find(d, so) != -1)
return;
ctx = malloc(sizeof(*ctx), M_CXGB, M_NOWAIT);
CTR1(KTR_TOM, "start listen on port %u", ntohs(inp->inp_lport));
ctx = malloc(sizeof(*ctx), M_CXGB, M_NOWAIT|M_ZERO);
if (!ctx)
return;
ctx->tom_data = d;
ctx->lso = so;
ctx->ulp_mode = 0; /* DDP if the default */
ctx->ulp_mode = TOM_TUNABLE(dev, ddp) && !(so->so_options & SO_NO_DDP) ? ULP_MODE_TCPDDP : 0;
LIST_INIT(&ctx->synq_head);
stid = cxgb_alloc_stid(d->cdev, d->client, ctx);
if (stid < 0)
goto free_ctx;
#ifdef notyet
/*
* XXX need to mark inpcb as referenced
*/
sock_hold(sk);
#endif
m = m_gethdr(M_NOWAIT, MT_DATA);
if (m == NULL)
goto free_stid;

52
sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h
View File

@ -1,4 +1,3 @@
/**************************************************************************
Copyright (c) 2007, Chelsio Inc.
@ -86,7 +85,6 @@ struct pagepod {
#define M_PPOD_PGSZ 0x3
#define V_PPOD_PGSZ(x) ((x) << S_PPOD_PGSZ)
struct pci_dev;
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <machine/bus.h>
@ -96,8 +94,7 @@ struct ddp_gather_list {
unsigned int dgl_length;
unsigned int dgl_offset;
unsigned int dgl_nelem;
vm_page_t *dgl_pages;
bus_addr_t dgl_phys_addr[0];
vm_page_t dgl_pages[0];
};
struct ddp_buf_state {
@ -107,7 +104,6 @@ struct ddp_buf_state {
};
struct ddp_state {
struct pci_dev *pdev;
struct ddp_buf_state buf_state[2]; /* per buffer state */
int cur_buf;
unsigned short kbuf_noinval;
@ -119,6 +115,7 @@ struct ddp_state {
int get_tcb_count;
unsigned int kbuf_posted;
int cancel_ubuf;
int user_ddp_pending;
unsigned int kbuf_nppods[NUM_DDP_KBUF];
unsigned int kbuf_tag[NUM_DDP_KBUF];
struct ddp_gather_list *kbuf[NUM_DDP_KBUF]; /* kernel buffer for DDP prefetch */
@ -132,54 +129,51 @@ enum {
DDP_BF_PSH = 1 << 3, /* set in skb->flags if the a DDP was
completed with a segment having the
PSH flag set */
DDP_BF_NODATA = 1 << 4, /* buffer completed before filling */
};
#ifdef notyet
#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
/*
* Returns 1 if a UBUF DMA buffer might be active.
*/
static inline int t3_ddp_ubuf_pending(struct sock *so)
static inline int
t3_ddp_ubuf_pending(struct toepcb *toep)
{
struct tcp_sock *tp = tcp_sk(sk);
struct ddp_state *p = DDP_STATE(tp);
struct ddp_state *p = &toep->tp_ddp_state;
/* When the TOM_TUNABLE(ddp) is enabled, we're always in ULP_MODE DDP,
* but DDP_STATE() is only valid if the connection actually enabled
* DDP.
*/
if (!p)
return 0;
if (p->kbuf[0] == NULL)
return (0);
return (p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY)) ||
(p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY));
}
#endif
int t3_setup_ppods(struct socket *so, const struct ddp_gather_list *gl,
unsigned int nppods, unsigned int tag, unsigned int maxoff,
unsigned int pg_off, unsigned int color);
int t3_alloc_ppods(struct tom_data *td, unsigned int n);
int t3_alloc_ppods(struct tom_data *td, unsigned int n, int *tag);
void t3_free_ppods(struct tom_data *td, unsigned int tag, unsigned int n);
void t3_free_ddp_gl(struct pci_dev *pdev, struct ddp_gather_list *gl);
int t3_pin_pages(struct pci_dev *pdev, unsigned long uaddr, size_t len,
struct ddp_gather_list **newgl,
const struct ddp_gather_list *gl);
int t3_ddp_copy(const struct mbuf *skb, int offset, struct iovec *to,
int len);
void t3_free_ddp_gl(struct ddp_gather_list *gl);
int t3_ddp_copy(const struct mbuf *m, int offset, struct uio *uio, int len);
//void t3_repost_kbuf(struct socket *so, int modulate, int activate);
void t3_post_kbuf(struct socket *so, int modulate);
int t3_post_ubuf(struct socket *so, const struct iovec *iov, int nonblock,
void t3_post_kbuf(struct socket *so, int modulate, int nonblock);
int t3_post_ubuf(struct socket *so, const struct uio *uio, int nonblock,
int rcv_flags, int modulate, int post_kbuf);
void t3_cancel_ubuf(struct socket *so);
int t3_overlay_ubuf(struct socket *so, const struct iovec *iov, int nonblock,
int rcv_flags, int modulate, int post_kbuf);
int t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall);
void t3_cleanup_ddp(struct socket *so);
void t3_cancel_ubuf(struct toepcb *toep);
int t3_overlay_ubuf(struct socket *so, const struct uio *uio, int nonblock,
int rcv_flags, int modulate, int post_kbuf);
int t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall, int nonblock);
void t3_cleanup_ddp(struct toepcb *toep);
void t3_release_ddp_resources(struct toepcb *toep);
void t3_cancel_ddpbuf(struct socket *so, unsigned int bufidx);
void t3_overlay_ddpbuf(struct socket *so, unsigned int bufidx, unsigned int tag0,
void t3_cancel_ddpbuf(struct toepcb *, unsigned int bufidx);
void t3_overlay_ddpbuf(struct toepcb *, unsigned int bufidx, unsigned int tag0,
unsigned int tag1, unsigned int len);
void t3_setup_ddpbufs(struct socket *so, unsigned int len0, unsigned int offset0,
void t3_setup_ddpbufs(struct toepcb *, unsigned int len0, unsigned int offset0,
unsigned int len1, unsigned int offset1,
uint64_t ddp_flags, uint64_t flag_mask, int modulate);
#endif /* T3_DDP_H */

694
sys/dev/cxgb/ulp/tom/cxgb_tcp_subr.c
View File

@ -1,694 +0,0 @@
/*-
* Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)tcp_subr.c 8.2 (Berkeley) 5/24/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_compat.h"
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
#include "opt_mac.h"
#include "opt_tcpdebug.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/callout.h>
#include <sys/kernel.h>
#include <sys/sysctl.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#ifdef INET6
#include <sys/domain.h>
#endif
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/protosw.h>
#include <sys/random.h>
#include <vm/uma.h>
#include <net/route.h>
#include <net/if.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#ifdef INET6
#include <netinet/ip6.h>
#endif
#include <netinet/in_pcb.h>
#ifdef INET6
#include <netinet6/in6_pcb.h>
#endif
#include <netinet/in_var.h>
#include <netinet/ip_var.h>
#ifdef INET6
#include <netinet6/ip6_var.h>
#include <netinet6/scope6_var.h>
#include <netinet6/nd6.h>
#endif
#include <netinet/ip_icmp.h>
#include <netinet/tcp.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_syncache.h>
#include <netinet/tcp_offload.h>
#ifdef INET6
#include <netinet6/tcp6_var.h>
#endif
#include <netinet/tcpip.h>
#ifdef TCPDEBUG
#include <netinet/tcp_debug.h>
#endif
#include <netinet6/ip6protosw.h>
#ifdef IPSEC
#include <netipsec/ipsec.h>
#include <netipsec/xform.h>
#ifdef INET6
#include <netipsec/ipsec6.h>
#endif
#include <netipsec/key.h>
#endif /*IPSEC*/
#include <machine/in_cksum.h>
#include <sys/md5.h>
#include <security/mac/mac_framework.h>
#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
SYSCTL_NODE(_net_inet_tcp, 0, cxgb, CTLFLAG_RW, 0, "chelsio TOE");
static int tcp_log_debug = 0;
SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, log_debug, CTLFLAG_RW,
&tcp_log_debug, 0, "Log errors caused by incoming TCP segments");
static int tcp_tcbhashsize = 0;
SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, tcbhashsize, CTLFLAG_RDTUN,
&tcp_tcbhashsize, 0, "Size of TCP control-block hashtable");
static int do_tcpdrain = 1;
SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, do_tcpdrain, CTLFLAG_RW,
&do_tcpdrain, 0,
"Enable tcp_drain routine for extra help when low on mbufs");
SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, pcbcount, CTLFLAG_RD,
&tcbinfo.ipi_count, 0, "Number of active PCBs");
static int icmp_may_rst = 1;
SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, icmp_may_rst, CTLFLAG_RW,
&icmp_may_rst, 0,
"Certain ICMP unreachable messages may abort connections in SYN_SENT");
static int tcp_isn_reseed_interval = 0;
SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, isn_reseed_interval, CTLFLAG_RW,
&tcp_isn_reseed_interval, 0, "Seconds between reseeding of ISN secret");
/*
* TCP bandwidth limiting sysctls. Note that the default lower bound of
* 1024 exists only for debugging. A good production default would be
* something like 6100.
*/
SYSCTL_NODE(_net_inet_tcp, OID_AUTO, inflight, CTLFLAG_RW, 0,
"TCP inflight data limiting");
static int tcp_inflight_enable = 1;
SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, enable, CTLFLAG_RW,
&tcp_inflight_enable, 0, "Enable automatic TCP inflight data limiting");
static int tcp_inflight_debug = 0;
SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, debug, CTLFLAG_RW,
&tcp_inflight_debug, 0, "Debug TCP inflight calculations");
static int tcp_inflight_rttthresh;
SYSCTL_PROC(_net_inet_tcp_inflight, OID_AUTO, rttthresh, CTLTYPE_INT|CTLFLAG_RW,
&tcp_inflight_rttthresh, 0, sysctl_msec_to_ticks, "I",
"RTT threshold below which inflight will deactivate itself");
static int tcp_inflight_min = 6144;
SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, min, CTLFLAG_RW,
&tcp_inflight_min, 0, "Lower-bound for TCP inflight window");
static int tcp_inflight_max = TCP_MAXWIN << TCP_MAX_WINSHIFT;
SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, max, CTLFLAG_RW,
&tcp_inflight_max, 0, "Upper-bound for TCP inflight window");
static int tcp_inflight_stab = 20;
SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, stab, CTLFLAG_RW,
&tcp_inflight_stab, 0, "Inflight Algorithm Stabilization 20 = 2 packets");
uma_zone_t sack_hole_zone;
static struct inpcb *tcp_notify(struct inpcb *, int);
static struct inpcb *cxgb_tcp_drop_syn_sent(struct inpcb *inp, int errno);
/*
* Target size of TCP PCB hash tables. Must be a power of two.
*
* Note that this can be overridden by the kernel environment
* variable net.inet.tcp.tcbhashsize
*/
#ifndef TCBHASHSIZE
#define TCBHASHSIZE 512
#endif
/*
* XXX
* Callouts should be moved into struct tcp directly. They are currently
* separate because the tcpcb structure is exported to userland for sysctl
* parsing purposes, which do not know about callouts.
*/
struct tcpcb_mem {
struct tcpcb tcb;
struct tcp_timer tt;
};
MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
/*
* Drop a TCP connection, reporting
* the specified error. If connection is synchronized,
* then send a RST to peer.
*/
struct tcpcb *
cxgb_tcp_drop(struct tcpcb *tp, int errno)
{
struct socket *so = tp->t_inpcb->inp_socket;
INP_INFO_WLOCK_ASSERT(&tcbinfo);
INP_LOCK_ASSERT(tp->t_inpcb);
if (TCPS_HAVERCVDSYN(tp->t_state)) {
tp->t_state = TCPS_CLOSED;
(void) tcp_gen_reset(tp);
tcpstat.tcps_drops++;
} else
tcpstat.tcps_conndrops++;
if (errno == ETIMEDOUT && tp->t_softerror)
errno = tp->t_softerror;
so->so_error = errno;
return (cxgb_tcp_close(tp));
}
/*
* Attempt to close a TCP control block, marking it as dropped, and freeing
* the socket if we hold the only reference.
*/
struct tcpcb *
cxgb_tcp_close(struct tcpcb *tp)
{
struct inpcb *inp = tp->t_inpcb;
struct socket *so;
INP_INFO_WLOCK_ASSERT(&tcbinfo);
INP_LOCK_ASSERT(inp);
if (tp->t_state == TCPS_LISTEN)
tcp_gen_listen_close(tp);
in_pcbdrop(inp);
tcpstat.tcps_closed++;
KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
so = inp->inp_socket;
soisdisconnected(so);
if (inp->inp_vflag & INP_SOCKREF) {
KASSERT(so->so_state & SS_PROTOREF,
("tcp_close: !SS_PROTOREF"));
inp->inp_vflag &= ~INP_SOCKREF;
INP_UNLOCK(inp);
ACCEPT_LOCK();
SOCK_LOCK(so);
so->so_state &= ~SS_PROTOREF;
sofree(so);
return (NULL);
}
return (tp);
}
/*
* Notify a tcp user of an asynchronous error;
* store error as soft error, but wake up user
* (for now, won't do anything until can select for soft error).
*
* Do not wake up user since there currently is no mechanism for
* reporting soft errors (yet - a kqueue filter may be added).
*/
static struct inpcb *
tcp_notify(struct inpcb *inp, int error)
{
struct tcpcb *tp;
INP_INFO_WLOCK_ASSERT(&tcbinfo);
INP_LOCK_ASSERT(inp);
if ((inp->inp_vflag & INP_TIMEWAIT) ||
(inp->inp_vflag & INP_DROPPED))
return (inp);
tp = intotcpcb(inp);
KASSERT(tp != NULL, ("tcp_notify: tp == NULL"));
/*
* Ignore some errors if we are hooked up.
* If connection hasn't completed, has retransmitted several times,
* and receives a second error, give up now. This is better
* than waiting a long time to establish a connection that
* can never complete.
*/
if (tp->t_state == TCPS_ESTABLISHED &&
(error == EHOSTUNREACH || error == ENETUNREACH ||
error == EHOSTDOWN)) {
return (inp);
} else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 &&
tp->t_softerror) {
tp = cxgb_tcp_drop(tp, error);
if (tp != NULL)
return (inp);
else
return (NULL);
} else {
tp->t_softerror = error;
return (inp);
}
#if 0
wakeup( &so->so_timeo);
sorwakeup(so);
sowwakeup(so);
#endif
}
void
cxgb_tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip)
{
struct ip *ip = vip;
struct tcphdr *th;
struct in_addr faddr;
struct inpcb *inp;
struct tcpcb *tp;
struct inpcb *(*notify)(struct inpcb *, int) = tcp_notify;
struct icmp *icp;
struct in_conninfo inc;
tcp_seq icmp_tcp_seq;
int mtu;
faddr = ((struct sockaddr_in *)sa)->sin_addr;
if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY)
return;
if (cmd == PRC_MSGSIZE)
notify = tcp_mtudisc;
else if (icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB ||
cmd == PRC_UNREACH_PORT || cmd == PRC_TIMXCEED_INTRANS) && ip)
notify = cxgb_tcp_drop_syn_sent;
/*
* Redirects don't need to be handled up here.
*/
else if (PRC_IS_REDIRECT(cmd))
return;
/*
* Source quench is depreciated.
*/
else if (cmd == PRC_QUENCH)
return;
/*
* Hostdead is ugly because it goes linearly through all PCBs.
* XXX: We never get this from ICMP, otherwise it makes an
* excellent DoS attack on machines with many connections.
*/
else if (cmd == PRC_HOSTDEAD)
ip = NULL;
else if ((unsigned)cmd >= PRC_NCMDS || inetctlerrmap[cmd] == 0)
return;
if (ip != NULL) {
icp = (struct icmp *)((caddr_t)ip
- offsetof(struct icmp, icmp_ip));
th = (struct tcphdr *)((caddr_t)ip
+ (ip->ip_hl << 2));
INP_INFO_WLOCK(&tcbinfo);
inp = in_pcblookup_hash(&tcbinfo, faddr, th->th_dport,
ip->ip_src, th->th_sport, 0, NULL);
if (inp != NULL) {
INP_LOCK(inp);
if (!(inp->inp_vflag & INP_TIMEWAIT) &&
!(inp->inp_vflag & INP_DROPPED) &&
!(inp->inp_socket == NULL)) {
icmp_tcp_seq = htonl(th->th_seq);
tp = intotcpcb(inp);
if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) &&
SEQ_LT(icmp_tcp_seq, tp->snd_max)) {
if (cmd == PRC_MSGSIZE) {
/*
* MTU discovery:
* If we got a needfrag set the MTU
* in the route to the suggested new
* value (if given) and then notify.
*/
bzero(&inc, sizeof(inc));
inc.inc_flags = 0; /* IPv4 */
inc.inc_faddr = faddr;
mtu = ntohs(icp->icmp_nextmtu);
/*
* If no alternative MTU was
* proposed, try the next smaller
* one. ip->ip_len has already
* been swapped in icmp_input().
*/
if (!mtu)
mtu = ip_next_mtu(ip->ip_len,
1);
if (mtu < max(296, (tcp_minmss)
+ sizeof(struct tcpiphdr)))
mtu = 0;
if (!mtu)
mtu = tcp_mssdflt
+ sizeof(struct tcpiphdr);
/*
* Only cache the the MTU if it
* is smaller than the interface
* or route MTU. tcp_mtudisc()
* will do right thing by itself.
*/
if (mtu <= tcp_maxmtu(&inc, NULL))
tcp_hc_updatemtu(&inc, mtu);
}
inp = (*notify)(inp, inetctlerrmap[cmd]);
}
}
if (inp != NULL)
INP_UNLOCK(inp);
} else {
inc.inc_fport = th->th_dport;
inc.inc_lport = th->th_sport;
inc.inc_faddr = faddr;
inc.inc_laddr = ip->ip_src;
#ifdef INET6
inc.inc_isipv6 = 0;
#endif
syncache_unreach(&inc, th);
}
INP_INFO_WUNLOCK(&tcbinfo);
} else
in_pcbnotifyall(&tcbinfo, faddr, inetctlerrmap[cmd], notify);
}
#ifdef INET6
void
tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d)
{
struct tcphdr th;
struct inpcb *(*notify)(struct inpcb *, int) = tcp_notify;
struct ip6_hdr *ip6;
struct mbuf *m;
struct ip6ctlparam *ip6cp = NULL;
const struct sockaddr_in6 *sa6_src = NULL;
int off;
struct tcp_portonly {
u_int16_t th_sport;
u_int16_t th_dport;
} *thp;
if (sa->sa_family != AF_INET6 ||
sa->sa_len != sizeof(struct sockaddr_in6))
return;
if (cmd == PRC_MSGSIZE)
notify = tcp_mtudisc;
else if (!PRC_IS_REDIRECT(cmd) &&
((unsigned)cmd >= PRC_NCMDS || inet6ctlerrmap[cmd] == 0))
return;
/* Source quench is depreciated. */
else if (cmd == PRC_QUENCH)
return;
/* if the parameter is from icmp6, decode it. */
if (d != NULL) {
ip6cp = (struct ip6ctlparam *)d;
m = ip6cp->ip6c_m;
ip6 = ip6cp->ip6c_ip6;
off = ip6cp->ip6c_off;
sa6_src = ip6cp->ip6c_src;
} else {
m = NULL;
ip6 = NULL;
off = 0; /* fool gcc */
sa6_src = &sa6_any;
}
if (ip6 != NULL) {
struct in_conninfo inc;
/*
* XXX: We assume that when IPV6 is non NULL,
* M and OFF are valid.
*/
/* check if we can safely examine src and dst ports */
if (m->m_pkthdr.len < off + sizeof(*thp))
return;
bzero(&th, sizeof(th));
m_copydata(m, off, sizeof(*thp), (caddr_t)&th);
in6_pcbnotify(&tcbinfo, sa, th.th_dport,
(struct sockaddr *)ip6cp->ip6c_src,
th.th_sport, cmd, NULL, notify);
inc.inc_fport = th.th_dport;
inc.inc_lport = th.th_sport;
inc.inc6_faddr = ((struct sockaddr_in6 *)sa)->sin6_addr;
inc.inc6_laddr = ip6cp->ip6c_src->sin6_addr;
inc.inc_isipv6 = 1;
INP_INFO_WLOCK(&tcbinfo);
syncache_unreach(&inc, &th);
INP_INFO_WUNLOCK(&tcbinfo);
} else
in6_pcbnotify(&tcbinfo, sa, 0, (const struct sockaddr *)sa6_src,
0, cmd, NULL, notify);
}
#endif /* INET6 */
/*
* Following is where TCP initial sequence number generation occurs.
*
* There are two places where we must use initial sequence numbers:
* 1. In SYN-ACK packets.
* 2. In SYN packets.
*
* All ISNs for SYN-ACK packets are generated by the syncache. See
* tcp_syncache.c for details.
*
* The ISNs in SYN packets must be monotonic; TIME_WAIT recycling
* depends on this property. In addition, these ISNs should be
* unguessable so as to prevent connection hijacking. To satisfy
* the requirements of this situation, the algorithm outlined in
* RFC 1948 is used, with only small modifications.
*
* Implementation details:
*
* Time is based off the system timer, and is corrected so that it
* increases by one megabyte per second. This allows for proper
* recycling on high speed LANs while still leaving over an hour
* before rollover.
*
* As reading the *exact* system time is too expensive to be done
* whenever setting up a TCP connection, we increment the time
* offset in two ways. First, a small random positive increment
* is added to isn_offset for each connection that is set up.
* Second, the function tcp_isn_tick fires once per clock tick
* and increments isn_offset as necessary so that sequence numbers
* are incremented at approximately ISN_BYTES_PER_SECOND. The
* random positive increments serve only to ensure that the same
* exact sequence number is never sent out twice (as could otherwise
* happen when a port is recycled in less than the system tick
* interval.)
*
* net.inet.tcp.isn_reseed_interval controls the number of seconds
* between seeding of isn_secret. This is normally set to zero,
* as reseeding should not be necessary.
*
* Locking of the global variables isn_secret, isn_last_reseed, isn_offset,
* isn_offset_old, and isn_ctx is performed using the TCP pcbinfo lock. In
* general, this means holding an exclusive (write) lock.
*/
#define ISN_BYTES_PER_SECOND 1048576
#define ISN_STATIC_INCREMENT 4096
#define ISN_RANDOM_INCREMENT (4096 - 1)
/*
* When a specific ICMP unreachable message is received and the
* connection state is SYN-SENT, drop the connection. This behavior
* is controlled by the icmp_may_rst sysctl.
*/
static struct inpcb *
cxgb_tcp_drop_syn_sent(struct inpcb *inp, int errno)
{
struct tcpcb *tp;
INP_INFO_WLOCK_ASSERT(&tcbinfo);
INP_LOCK_ASSERT(inp);
if ((inp->inp_vflag & INP_TIMEWAIT) ||
(inp->inp_vflag & INP_DROPPED))
return (inp);
tp = intotcpcb(inp);
if (tp->t_state != TCPS_SYN_SENT)
return (inp);
tp = cxgb_tcp_drop(tp, errno);
if (tp != NULL)
return (inp);
else
return (NULL);
}
static int
cxgb_sysctl_drop(SYSCTL_HANDLER_ARGS)
{
/* addrs[0] is a foreign socket, addrs[1] is a local one. */
struct sockaddr_storage addrs[2];
struct inpcb *inp;
struct tcpcb *tp;
struct tcptw *tw;
struct sockaddr_in *fin, *lin;
#ifdef INET6
struct sockaddr_in6 *fin6, *lin6;
struct in6_addr f6, l6;
#endif
int error;
inp = NULL;
fin = lin = NULL;
#ifdef INET6
fin6 = lin6 = NULL;
#endif
error = 0;
if (req->oldptr != NULL || req->oldlen != 0)
return (EINVAL);
if (req->newptr == NULL)
return (EPERM);
if (req->newlen < sizeof(addrs))
return (ENOMEM);
error = SYSCTL_IN(req, &addrs, sizeof(addrs));
if (error)
return (error);
switch (addrs[0].ss_family) {
#ifdef INET6
case AF_INET6:
fin6 = (struct sockaddr_in6 *)&addrs[0];
lin6 = (struct sockaddr_in6 *)&addrs[1];
if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
lin6->sin6_len != sizeof(struct sockaddr_in6))
return (EINVAL);
if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
return (EINVAL);
in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
fin = (struct sockaddr_in *)&addrs[0];
lin = (struct sockaddr_in *)&addrs[1];
break;
}
error = sa6_embedscope(fin6, ip6_use_defzone);
if (error)
return (error);
error = sa6_embedscope(lin6, ip6_use_defzone);
if (error)
return (error);
break;
#endif
case AF_INET:
fin = (struct sockaddr_in *)&addrs[0];
lin = (struct sockaddr_in *)&addrs[1];
if (fin->sin_len != sizeof(struct sockaddr_in) ||
lin->sin_len != sizeof(struct sockaddr_in))
return (EINVAL);
break;
default:
return (EINVAL);
}
INP_INFO_WLOCK(&tcbinfo);
switch (addrs[0].ss_family) {
#ifdef INET6
case AF_INET6:
inp = in6_pcblookup_hash(&tcbinfo, &f6, fin6->sin6_port,
&l6, lin6->sin6_port, 0, NULL);
break;
#endif
case AF_INET:
inp = in_pcblookup_hash(&tcbinfo, fin->sin_addr, fin->sin_port,
lin->sin_addr, lin->sin_port, 0, NULL);
break;
}
if (inp != NULL) {
INP_LOCK(inp);
if (inp->inp_vflag & INP_TIMEWAIT) {
/*
* XXXRW: There currently exists a state where an
* inpcb is present, but its timewait state has been
* discarded. For now, don't allow dropping of this
* type of inpcb.
*/
tw = intotw(inp);
if (tw != NULL)
tcp_twclose(tw, 0);
else
INP_UNLOCK(inp);
} else if (!(inp->inp_vflag & INP_DROPPED) &&
!(inp->inp_socket->so_options & SO_ACCEPTCONN)) {
tp = intotcpcb(inp);
tp = cxgb_tcp_drop(tp, ECONNABORTED);
if (tp != NULL)
INP_UNLOCK(inp);
} else
INP_UNLOCK(inp);
} else
error = ESRCH;
INP_INFO_WUNLOCK(&tcbinfo);
return (error);
}
SYSCTL_PROC(_net_inet_tcp_cxgb, TCPCTL_DROP, drop,
CTLTYPE_STRUCT|CTLFLAG_WR|CTLFLAG_SKIP, NULL,
0, cxgb_sysctl_drop, "", "Drop TCP connection");

1362
sys/dev/cxgb/ulp/tom/cxgb_tcp_usrreq.c
View File

File diff suppressed because it is too large Load Diff

75
sys/dev/cxgb/ulp/tom/cxgb_toepcb.h
View File

@ -30,45 +30,49 @@
#ifndef CXGB_TOEPCB_H_
#define CXGB_TOEPCB_H_
#include <sys/bus.h>
#include <sys/condvar.h>
#include <dev/cxgb/sys/mbufq.h>
struct toepcb {
struct toedev *tp_toedev;
struct l2t_entry *tp_l2t;
pr_ctloutput_t *tp_ctloutput;
unsigned int tp_tid;
int tp_wr_max;
int tp_wr_avail;
int tp_wr_unacked;
int tp_delack_mode;
int tp_mtu_idx;
int tp_ulp_mode;
int tp_qset_idx;
int tp_mss_clamp;
int tp_qset;
int tp_flags;
int tp_enqueued_bytes;
int tp_page_count;
int tp_state;
struct toedev *tp_toedev;
struct l2t_entry *tp_l2t;
pr_ctloutput_t *tp_ctloutput;
unsigned int tp_tid;
int tp_wr_max;
int tp_wr_avail;
int tp_wr_unacked;
int tp_delack_mode;
int tp_mtu_idx;
int tp_ulp_mode;
int tp_qset_idx;
int tp_mss_clamp;
int tp_qset;
int tp_flags;
int tp_enqueued_bytes;
int tp_page_count;
int tp_state;
tcp_seq tp_iss;
tcp_seq tp_delack_seq;
tcp_seq tp_rcv_wup;
tcp_seq tp_copied_seq;
uint64_t tp_write_seq;
tcp_seq tp_iss;
tcp_seq tp_delack_seq;
tcp_seq tp_rcv_wup;
tcp_seq tp_copied_seq;
uint64_t tp_write_seq;
volatile int tp_refcount;
vm_page_t *tp_pages;
volatile int tp_refcount;
vm_page_t *tp_pages;
struct tcpcb *tp_tp;
struct mbuf *tp_m_last;
bus_dma_tag_t tp_tx_dmat;
bus_dmamap_t tp_dmamap;
struct tcpcb *tp_tp;
struct mbuf *tp_m_last;
bus_dma_tag_t tp_tx_dmat;
bus_dma_tag_t tp_rx_dmat;
bus_dmamap_t tp_dmamap;
LIST_ENTRY(toepcb) synq_entry;
struct mbuf_head wr_list;
struct mbuf_head out_of_order_queue;
struct ddp_state tp_ddp_state;
LIST_ENTRY(toepcb) synq_entry;
struct mbuf_head wr_list;
struct mbuf_head out_of_order_queue;
struct ddp_state tp_ddp_state;
struct cv tp_cv;
};
static inline void
@ -95,7 +99,7 @@ enqueue_wr(struct toepcb *toep, struct mbuf *m)
}
static inline struct mbuf *
peek_wr(struct toepcb *toep)
peek_wr(const struct toepcb *toep)
{
return (mbufq_peek(&toep->wr_list));
@ -108,5 +112,10 @@ dequeue_wr(struct toepcb *toep)
return (mbufq_dequeue(&toep->wr_list));
}
#define wr_queue_walk(toep, m) \
for (m = peek_wr(toep); m; m = m->m_nextpkt)
#endif

102
sys/dev/cxgb/ulp/tom/cxgb_tom.c
View File

@ -34,11 +34,13 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/fcntl.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/eventhandler.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/condvar.h>
#include <sys/mutex.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
@ -90,16 +92,20 @@ static TAILQ_HEAD(, tom_data) cxgb_list;
static struct mtx cxgb_list_lock;
static int t3_toe_attach(struct toedev *dev, const struct offload_id *entry);
static void cxgb_register_listeners(void);
/*
* Handlers for each CPL opcode
*/
static cxgb_cpl_handler_func tom_cpl_handlers[NUM_CPL_CMDS];
static cxgb_cpl_handler_func tom_cpl_handlers[256];
static eventhandler_tag listen_tag;
static struct offload_id t3_toe_id_tab[] = {
{ TOE_ID_CHELSIO_T3, 0 },
{ TOE_ID_CHELSIO_T3B, 0 },
{ TOE_ID_CHELSIO_T3C, 0 },
{ 0 }
};
@ -138,7 +144,7 @@ toepcb_alloc(void)
{
struct toepcb *toep;
toep = malloc(sizeof(struct toepcb), M_DEVBUF, M_NOWAIT);
toep = malloc(sizeof(struct toepcb), M_DEVBUF, M_NOWAIT|M_ZERO);
if (toep == NULL)
return (NULL);
@ -150,8 +156,8 @@ toepcb_alloc(void)
void
toepcb_init(struct toepcb *toep)
{
bzero(toep, sizeof(*toep));
toep->tp_refcount = 1;
cv_init(&toep->tp_cv, "toep cv");
}
void
@ -164,12 +170,9 @@ void
toepcb_release(struct toepcb *toep)
{
if (toep->tp_refcount == 1) {
printf("doing final toepcb free\n");
free(toep, M_DEVBUF);
return;
}
atomic_add_acq_int(&toep->tp_refcount, -1);
}
@ -179,13 +182,30 @@ toepcb_release(struct toepcb *toep)
static void
t3cdev_add(struct tom_data *t)
{
printf("t3cdev_add\n");
mtx_lock(&cxgb_list_lock);
TAILQ_INSERT_TAIL(&cxgb_list, t, entry);
mtx_unlock(&cxgb_list_lock);
}
static inline int
cdev2type(struct t3cdev *cdev)
{
int type = 0;
switch (cdev->type) {
case T3A:
type = TOE_ID_CHELSIO_T3;
break;
case T3B:
type = TOE_ID_CHELSIO_T3B;
break;
case T3C:
type = TOE_ID_CHELSIO_T3C;
break;
}
return (type);
}
/*
* Allocate a TOM data structure,
* initialize its cpl_handlers
@ -200,11 +220,7 @@ t3c_tom_add(struct t3cdev *cdev)
struct toedev *tdev;
struct adap_ports *port_info;
printf("%s called\n", __FUNCTION__);
t = malloc(sizeof(*t), M_CXGB, M_NOWAIT|M_ZERO);
if (t == NULL)
return;
@ -224,8 +240,7 @@ t3c_tom_add(struct t3cdev *cdev)
/* Register TCP offload device */
tdev = &t->tdev;
tdev->tod_ttid = (cdev->type == T3A ?
TOE_ID_CHELSIO_T3 : TOE_ID_CHELSIO_T3B);
tdev->tod_ttid = cdev2type(cdev);
tdev->tod_lldev = cdev->lldev;
if (register_toedev(tdev, "toe%d")) {
@ -234,13 +249,11 @@ t3c_tom_add(struct t3cdev *cdev)
}
TOM_DATA(tdev) = t;
printf("nports=%d\n", port_info->nports);
for (i = 0; i < port_info->nports; i++) {
struct ifnet *ifp = port_info->lldevs[i];
TOEDEV(ifp) = tdev;
printf("enabling toe on %p\n", ifp);
CTR1(KTR_TOM, "enabling toe on %p", ifp);
ifp->if_capabilities |= IFCAP_TOE4;
ifp->if_capenable |= IFCAP_TOE4;
}
@ -251,6 +264,7 @@ t3c_tom_add(struct t3cdev *cdev)
/* Activate TCP offload device */
activate_offload(tdev);
cxgb_register_listeners();
return;
out_free_all:
@ -269,8 +283,8 @@ static int
do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
{
log(LOG_ERR, "%s: received bad CPL command %u\n", cdev->name,
*mtod(m, unsigned int *));
0xFF & *mtod(m, unsigned int *));
kdb_backtrace();
return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
}
@ -282,7 +296,7 @@ do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
void
t3tom_register_cpl_handler(unsigned int opcode, cxgb_cpl_handler_func h)
{
if (opcode < NUM_CPL_CMDS)
if (opcode < 256)
tom_cpl_handlers[opcode] = h ? h : do_bad_cpl;
else
log(LOG_ERR, "Chelsio T3 TOM: handler registration for "
@ -327,7 +341,7 @@ init_cpl_handlers(void)
{
int i;
for (i = 0; i < NUM_CPL_CMDS; ++i)
for (i = 0; i < 256; ++i)
tom_cpl_handlers[i] = do_bad_cpl;
t3_init_listen_cpl_handlers();
@ -349,7 +363,7 @@ t3_toe_attach(struct toedev *dev, const struct offload_id *entry)
#endif
t3_init_tunables(t);
mtx_init(&t->listen_lock, "tom data listeners", NULL, MTX_DEF);
CTR2(KTR_TOM, "t3_toe_attach dev=%p entry=%p", dev, entry);
/* Adjust TOE activation for this module */
t->conf.activated = activated;
@ -374,19 +388,14 @@ t3_toe_attach(struct toedev *dev, const struct offload_id *entry)
t->ddp_ulimit = ddp.ulimit;
t->pdev = ddp.pdev;
t->rx_page_size = rx_page_info.page_size;
#ifdef notyet
/* OK if this fails, we just can't do DDP */
t->nppods = (ddp.ulimit + 1 - ddp.llimit) / PPOD_SIZE;
t->ppod_map = t3_alloc_mem(t->nppods);
#endif
t->ppod_map = malloc(t->nppods, M_DEVBUF, M_WAITOK|M_ZERO);
#if 0
spin_lock_init(&t->ppod_map_lock);
tom_proc_init(dev);
#ifdef CONFIG_SYSCTL
t->sysctl = t3_sysctl_register(dev, &t->conf);
#endif
#endif
mtx_init(&t->ppod_map_lock, "ppod map", NULL, MTX_DEF);
t3_sysctl_register(cdev->adapter, &t->conf);
return (0);
}
@ -411,11 +420,8 @@ cxgb_toe_listen_stop(void *unused, struct tcpcb *tp)
mtx_lock(&cxgb_list_lock);
TAILQ_FOREACH(p, &cxgb_list, entry) {
if (tp->t_state == TCPS_LISTEN) {
printf("stopping listen on port=%d\n",
ntohs(tp->t_inpcb->inp_lport));
if (tp->t_state == TCPS_LISTEN)
t3_listen_stop(&p->tdev, so, p->cdev);
}
}
mtx_unlock(&cxgb_list_lock);
}
@ -439,23 +445,12 @@ cxgb_register_listeners(void)
static int
t3_tom_init(void)
{
#if 0
struct socket *sock;
err = sock_create_kern(PF_INET, SOCK_STREAM, IPPROTO_TCP, &sock);
if (err < 0) {
printk(KERN_ERR "Could not create TCP socket, error %d\n", err);
return err;
}
t3_def_state_change = sock->sk->sk_state_change;
t3_def_data_ready = sock->sk->sk_data_ready;
t3_def_error_report = sock->sk->sk_error_report;
sock_release(sock);
#endif
init_cpl_handlers();
if (t3_init_cpl_io() < 0)
if (t3_init_cpl_io() < 0) {
log(LOG_ERR,
"Unable to initialize cpl io ops\n");
return -1;
}
t3_init_socket_ops();
/* Register with the TOE device layer. */
@ -466,7 +461,6 @@ t3_tom_init(void)
return -1;
}
INP_INFO_WLOCK(&tcbinfo);
INP_INFO_WUNLOCK(&tcbinfo);
mtx_init(&cxgb_list_lock, "cxgb tom list", NULL, MTX_DEF);
@ -477,10 +471,8 @@ t3_tom_init(void)
TAILQ_INIT(&cxgb_list);
/* Register to offloading devices */
printf("setting add to %p\n", t3c_tom_add);
t3c_tom_client.add = t3c_tom_add;
cxgb_register_client(&t3c_tom_client);
cxgb_register_listeners();
return (0);
}
@ -491,8 +483,6 @@ t3_tom_load(module_t mod, int cmd, void *arg)
switch (cmd) {
case MOD_LOAD:
printf("wheeeeee ...\n");
t3_tom_init();
break;
case MOD_QUIESCE:

2
sys/dev/cxgb/ulp/tom/cxgb_tom.h
View File

@ -138,6 +138,8 @@ struct listen_ctx {
void t3_init_tunables(struct tom_data *t);
void t3_sysctl_register(struct adapter *sc, const struct tom_tunables *p);
static __inline struct mbuf *
m_gethdr_nofail(int len)
{

18
sys/dev/cxgb/ulp/tom/cxgb_tom_sysctl.c
View File

@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
#include <dev/cxgb/common/cxgb_ctl_defs.h>
#include <dev/cxgb/common/cxgb_t3_cpl.h>
#include <dev/cxgb/cxgb_offload.h>
#include <dev/cxgb/cxgb_include.h>
#include <dev/cxgb/cxgb_l2t.h>
#include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
#include <dev/cxgb/ulp/tom/cxgb_tom.h>
@ -82,7 +83,7 @@ static struct tom_tunables default_tunable_vals = {
.delack = 1,
.max_conn = -1,
.soft_backlog_limit = 0,
.ddp = 0,
.ddp = 1,
.ddp_thres = 14 * 4096,
.ddp_copy_limit = 13 * 4096,
.ddp_push_wait = 1,
@ -96,7 +97,8 @@ static struct tom_tunables default_tunable_vals = {
.activated = 1,
};
void t3_init_tunables(struct tom_data *t)
void
t3_init_tunables(struct tom_data *t)
{
t->conf = default_tunable_vals;
@ -104,3 +106,15 @@ void t3_init_tunables(struct tom_data *t)
t->conf.mss = T3C_DATA(t->cdev)->tx_max_chunk;
t->conf.max_wrs = T3C_DATA(t->cdev)->max_wrs;
}
void
t3_sysctl_register(struct adapter *sc, const struct tom_tunables *p)
{
struct sysctl_ctx_list *ctx;
struct sysctl_oid_list *children;
ctx = device_get_sysctl_ctx(sc->dev);
children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
}

180
sys/dev/cxgb/ulp/tom/cxgb_vm.c Normal file
View File

@ -0,0 +1,180 @@
/**************************************************************************
Copyright (c) 2007, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***************************************************************************/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/fcntl.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mbuf.h>
#include <sys/condvar.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <dev/cxgb/ulp/tom/cxgb_vm.h>
#define TRACE_ENTER printf("%s:%s entered", __FUNCTION__, __FILE__)
#define TRACE_EXIT printf("%s:%s:%d exited", __FUNCTION__, __FILE__, __LINE__)
/*
* This routine takes a user address range and does the following:
* - validate that the user has access to those pages (flags indicates read or write) - if not fail
* - validate that count is enough to hold range number of pages - if not fail
* - fault in any non-resident pages
* - if the user is doing a read force a write fault for any COWed pages
* - if the user is doing a read mark all pages as dirty
* - hold all pages
* - return number of pages in count
*/
int
vm_fault_hold_user_pages(vm_offset_t addr, vm_page_t *mp, int count, int flags)
{
vm_offset_t end, va;
vm_paddr_t pa;
int faults, rv;
struct thread *td;
vm_map_t map;
pmap_t pmap;
vm_page_t m, *pages;
vm_prot_t prot;
/*
* Check that virtual address range is legal
* This check is somewhat bogus as on some architectures kernel
* and user do not share VA - however, it appears that all FreeBSD
* architectures define it
*/
end = addr + (count * PAGE_SIZE);
if (end > VM_MAXUSER_ADDRESS) {
printf("bad address passed\n");
return (EFAULT);
}
td = curthread;
map = &td->td_proc->p_vmspace->vm_map;
pmap = &td->td_proc->p_vmspace->vm_pmap;
pages = mp;
prot = VM_PROT_READ;
prot |= (flags & VM_HOLD_WRITEABLE) ? VM_PROT_WRITE : 0;
bzero(pages, sizeof(vm_page_t *) * count);
retry:
/*
* First optimistically assume that all pages are resident (and R/W if for write)
* if so just mark pages as held (and dirty if for write) and return
*/
vm_page_lock_queues();
for (pages = mp, faults = 0, va = addr; va < end; va += PAGE_SIZE, pages++) {
/*
* Assure that we only hold the page once
*/
if (*pages == NULL) {
/*
* page queue mutex is recursable so this is OK
* it would be really nice if we had an unlocked version of this so
* we were only acquiring the pmap lock 1 time as opposed to potentially
* many dozens of times
*/
m = pmap_extract_and_hold(pmap, va, prot);
if (m == NULL) {
faults++;
continue;
}
*pages = m;
if (flags & VM_HOLD_WRITEABLE)
vm_page_dirty(m);
}
}
vm_page_unlock_queues();
if (faults == 0) {
return (0);
}
/*
* Pages either have insufficient permissions or are not present
* trigger a fault where neccessary
*
*/
for (va = addr; va < end; va += PAGE_SIZE) {
m = NULL;
pa = pmap_extract(pmap, va);
rv = 0;
if (pa)
m = PHYS_TO_VM_PAGE(pa);
if (flags & VM_HOLD_WRITEABLE) {
if (m == NULL || (m->flags & PG_WRITEABLE) == 0)
rv = vm_fault(map, va, VM_PROT_WRITE, VM_FAULT_DIRTY);
} else if (m == NULL)
rv = vm_fault(map, va, VM_PROT_READ, VM_FAULT_NORMAL);
if (rv) {
printf("vm_fault bad return rv=%d va=0x%zx\n", rv, va);
goto error;
}
}
goto retry;
error:
vm_page_lock_queues();
for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++)
if (*pages)
vm_page_unhold(*pages);
vm_page_unlock_queues();
return (EFAULT);
}
void
vm_fault_unhold_pages(vm_page_t *mp, int count)
{
KASSERT(count >= 0, ("negative count %d", count));
vm_page_lock_queues();
while (count--) {
vm_page_unhold(*mp);
mp++;
}
vm_page_unlock_queues();
}

40
sys/dev/cxgb/ulp/tom/cxgb_vm.h Normal file
View File

@ -0,0 +1,40 @@
/**************************************************************************
Copyright (c) 2007, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
$FreeBSD$
***************************************************************************/
#ifndef CXGB_VM_H_
#define CXGB_VM_H_
#define VM_HOLD_WRITEABLE 0x1
int vm_fault_hold_user_pages(vm_offset_t addr, vm_page_t *mp, int count, int flags);
void vm_fault_unhold_pages(vm_page_t *mp, int count);
#endif

18
sys/modules/cxgb/cxgb/Makefile
View File

@ -7,26 +7,30 @@ KMOD= if_cxgb
SRCS= cxgb_mc5.c cxgb_vsc8211.c cxgb_ael1002.c cxgb_mv88e1xxx.c
SRCS+= cxgb_xgmac.c cxgb_vsc7323.c cxgb_t3_hw.c cxgb_main.c
SRCS+= cxgb_sge.c cxgb_lro.c cxgb_offload.c cxgb_l2t.c
SRCS+= device_if.h bus_if.h pci_if.h opt_zero.h opt_sched.h opt_global.h
SRCS+= device_if.h bus_if.h pci_if.h opt_zero.h opt_sched.h opt_global.h
SRCS+= uipc_mvec.c cxgb_support.c
SRCS+= cxgb_multiq.c
CFLAGS+= -DCONFIG_CHELSIO_T3_CORE -g -DCONFIG_DEFINED -DDEFAULT_JUMBO -I${CXGB} -DSMP
#CFLAGS+= -DDISABLE_MBUF_IOVEC
CFLAGS+= -DDISABLE_MBUF_IOVEC
#CFLAGS+= -DIFNET_MULTIQUEUE
#CFLAGS+= -DDISABLE_MBUF_IOVEC
#CFLAGS+= -DDEBUG -DDEBUG_PRINT
#CFLAGS+= -DINVARIANT_SUPPORT -DINVARIANTS
#CFLAGS+= -DWITNESS
#CFLAGS+= -DDEBUG -DDEBUG_PRINT
#CFLAGS += -DLOCK_PROFILING
#CFLAGS+= -DWITNESS
.if ${MACHINE_ARCH} != "ia64"
# ld is broken on ia64
t3fw-4.7.0.bin: ${CXGB}/t3fw-4.7.0.bin.gz.uu
uudecode -p < ${CXGB}/t3fw-4.7.0.bin.gz.uu \
t3fw-5.0.0.bin: ${CXGB}/t3fw-5.0.0.bin.gz.uu
uudecode -p < ${CXGB}/t3fw-5.0.0.bin.gz.uu \
| gzip -dc > ${.TARGET}
FIRMWS= t3fw-4.7.0.bin:t3fw470
CLEANFILES+= t3fw-4.7.0.bin
FIRMWS= t3fw-5.0.0.bin:t3fw500
CLEANFILES+= t3fw-5.0.0.bin
t3b_protocol_sram-1.1.0.bin: ${CXGB}/t3b_protocol_sram-1.1.0.bin.gz.uu
uudecode -p < ${CXGB}/t3b_protocol_sram-1.1.0.bin.gz.uu \

6
sys/modules/cxgb/tom/Makefile
View File

@ -1,11 +1,13 @@
# $FreeBSD$
TOM = ${.CURDIR}/../../../dev/cxgb/ulp/tom
.PATH: ${TOM}
KMOD= tom
SRCS= cxgb_tom.c cxgb_cpl_io.c cxgb_listen.c cxgb_tom_sysctl.c cxgb_cpl_socket.c
#SRCS+= cxgb_tcp_subr.c cxgb_tcp_usrreq.c
SRCS+= opt_compat.h opt_inet.h opt_inet6.h opt_ipsec.h opt_mac.h opt_tcpdebug.h opt_ddb.h
SRCS+= cxgb_ddp.c cxgb_vm.c
SRCS+= opt_compat.h opt_inet.h opt_inet6.h opt_ipsec.h opt_mac.h
SRCS+= opt_tcpdebug.h opt_ddb.h opt_sched.h opt_global.h opt_ktr.h
SRCS+= device_if.h bus_if.h pci_if.h
#CFLAGS+= -DDEBUG_PRINT -DDEBUG