- update firmware to 5.0

- add support for T3C
- add DDP support (zero-copy receive)
- fix TOE transmit of large requests
- fix shutdown so that sockets don't remain in CLOSING state indefinitely
- register listeners when an interface is brought up after tom is loaded
- fix setting of multicast filter
- enable link at device attach
- exit tick handler if shutdown is in progress
- add helper for logging TCB
- add sysctls for dumping transmit queues

- note that TOE wxill not be MFC'd until after 7.0 has been finalized

MFC after: 3 days

sys/dev/cxgb/common/cxgb_ael1002.c

@@ -36,6 +36,9 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/cxgb_include.h>
 #endif
 
+#undef msleep
+#define msleep t3_os_sleep
+
 enum {
 	AEL100X_TX_DISABLE  = 9,
 	AEL100X_TX_CONFIG1  = 0xc002,
@@ -52,9 +55,9 @@ static void ael100x_txon(struct cphy *phy)
 {
 	int tx_on_gpio = phy->addr == 0 ? F_GPIO7_OUT_VAL : F_GPIO2_OUT_VAL;
 
-	t3_os_sleep(100);
+	msleep(100);
 	t3_set_reg_field(phy->adapter, A_T3DBG_GPIO_EN, 0, tx_on_gpio);
-	t3_os_sleep(30);
+	msleep(30);
 }
 
 static int ael1002_power_down(struct cphy *phy, int enable)
@@ -115,7 +118,6 @@ static int ael100x_get_link_status(struct cphy *phy, int *link_ok,
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops ael1002_ops = {
-	NULL,
 	ael1002_reset,
 	ael1002_intr_noop,
 	ael1002_intr_noop,
@@ -141,11 +143,14 @@ static struct cphy_ops ael1002_ops = {
 };
 #endif
 
-void t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			 const struct mdio_ops *mdio_ops)
+			const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &ael1002_ops, mdio_ops);
+	cphy_init(phy, adapter, phy_addr, &ael1002_ops, mdio_ops,
+		  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE,
+		  "10GBASE-XR");
 	ael100x_txon(phy);
+	return 0;
 }
 
 static int ael1006_reset(struct cphy *phy, int wait)
@@ -188,7 +193,6 @@ static int ael1006_power_down(struct cphy *phy, int enable)
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops ael1006_ops = {
-	NULL,
 	ael1006_reset,
 	ael1006_intr_enable,
 	ael1006_intr_disable,
@@ -214,16 +218,18 @@ static struct cphy_ops ael1006_ops = {
 };
 #endif
 
-void t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			 const struct mdio_ops *mdio_ops)
+			const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &ael1006_ops, mdio_ops);
+	cphy_init(phy, adapter, phy_addr, &ael1006_ops, mdio_ops,
+		  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE,
+		  "10GBASE-SR");
 	ael100x_txon(phy);
+	return 0;
 }
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops qt2045_ops = {
-	NULL,
 	ael1006_reset,
 	ael1006_intr_enable,
 	ael1006_intr_disable,
@@ -249,12 +255,14 @@ static struct cphy_ops qt2045_ops = {
 };
 #endif
 
-void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			const struct mdio_ops *mdio_ops)
+		       const struct mdio_ops *mdio_ops)
 {
 	unsigned int stat;
 
-	cphy_init(phy, adapter, phy_addr, &qt2045_ops, mdio_ops);
+	cphy_init(phy, adapter, phy_addr, &qt2045_ops, mdio_ops,
+		  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_TP,
+		  "10GBASE-CX4");
 
 	/*
 	 * Some cards where the PHY is supposed to be at address 0 actually
@@ -263,6 +271,7 @@ void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 	if (!phy_addr && !mdio_read(phy, MDIO_DEV_PMA_PMD, MII_BMSR, &stat) &&
 	    stat == 0xffff)
 		phy->addr = 1;
+	return 0;
 }
 
 static int xaui_direct_reset(struct cphy *phy, int wait)
@@ -300,7 +309,6 @@ static int xaui_direct_power_down(struct cphy *phy, int enable)
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops xaui_direct_ops = {
-	NULL,
 	xaui_direct_reset,
 	ael1002_intr_noop,
 	ael1002_intr_noop,
@@ -326,8 +334,11 @@ static struct cphy_ops xaui_direct_ops = {
 };
 #endif
 
-void t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			     const struct mdio_ops *mdio_ops)
+			    const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &xaui_direct_ops, mdio_ops);
+	cphy_init(phy, adapter, phy_addr, &xaui_direct_ops, mdio_ops,
+		  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_TP,
+		  "10GBASE-CX4");
+	return 0;
 }

sys/dev/cxgb/common/cxgb_common.h

@@ -98,8 +98,8 @@ enum {
 	    (((x) >> S_TP_VERSION_MICRO) & M_TP_VERSION_MICRO)
 
 enum {
-	FW_VERSION_MAJOR = 4,
+	FW_VERSION_MAJOR = 5,
-	FW_VERSION_MINOR = 7,
+	FW_VERSION_MINOR = 0,
 	FW_VERSION_MICRO = 0
 };
 
@@ -157,10 +157,10 @@ struct adapter_info {
 };
 
 struct port_type_info {
-	void (*phy_prep)(struct cphy *phy, adapter_t *adapter, int phy_addr,
+	int (*phy_prep)(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			 const struct mdio_ops *ops);
+			const struct mdio_ops *ops);
-	unsigned int caps;
+
-	const char *desc;
+
 };
 
 struct mc5_stats {
@@ -508,7 +508,6 @@ enum {
 
 /* PHY operations */
 struct cphy_ops {
-	void (*destroy)(struct cphy *phy);
 	int (*reset)(struct cphy *phy, int wait);
 
 	int (*intr_enable)(struct cphy *phy);
@@ -530,7 +529,9 @@ struct cphy_ops {
 /* A PHY instance */
 struct cphy {
 	int addr;                            /* PHY address */
+	unsigned int caps;                   /* PHY capabilities */
 	adapter_t *adapter;                  /* associated adapter */
+	const char *desc;                    /* PHY description */
 	unsigned long fifo_errors;           /* FIFO over/under-flows */
 	const struct cphy_ops *ops;          /* PHY operations */
 	int (*mdio_read)(adapter_t *adapter, int phy_addr, int mmd_addr,
@@ -555,10 +556,13 @@ static inline int mdio_write(struct cphy *phy, int mmd, int reg,
 /* Convenience initializer */
 static inline void cphy_init(struct cphy *phy, adapter_t *adapter,
 			     int phy_addr, struct cphy_ops *phy_ops,
-			     const struct mdio_ops *mdio_ops)
+                             const struct mdio_ops *mdio_ops, unsigned int caps,
+                             const char *desc)
 {
 	phy->adapter = adapter;
 	phy->addr    = phy_addr;
+	phy->caps    = caps;
+	phy->desc    = desc;
 	phy->ops     = phy_ops;
 	if (mdio_ops) {
 		phy->mdio_read  = mdio_ops->read;
@@ -667,11 +671,12 @@ int t3_seeprom_wp(adapter_t *adapter, int enable);
 int t3_read_flash(adapter_t *adapter, unsigned int addr, unsigned int nwords,
 		  u32 *data, int byte_oriented);
 int t3_get_tp_version(adapter_t *adapter, u32 *vers);
-int t3_check_tpsram_version(adapter_t *adapter);
+int t3_check_tpsram_version(adapter_t *adapter, int *must_load);
 int t3_check_tpsram(adapter_t *adapter, const u8 *tp_ram, unsigned int size);
 int t3_load_fw(adapter_t *adapter, const const u8 *fw_data, unsigned int size);
+int t3_load_boot(adapter_t *adapter, u8 *boot_data, unsigned int size);
 int t3_get_fw_version(adapter_t *adapter, u32 *vers);
-int t3_check_fw_version(adapter_t *adapter);
+int t3_check_fw_version(adapter_t *adapter, int *must_load);
 int t3_init_hw(adapter_t *adapter, u32 fw_params);
 void mac_prep(struct cmac *mac, adapter_t *adapter, int index);
 void early_hw_init(adapter_t *adapter, const struct adapter_info *ai);
@@ -769,18 +774,21 @@ int t3_vsc7323_set_mtu(adapter_t *adap, unsigned int mtu, int port);
 int t3_vsc7323_set_addr(adapter_t *adap, u8 addr[6], int port);
 int t3_vsc7323_enable(adapter_t *adap, int port, int which);
 int t3_vsc7323_disable(adapter_t *adap, int port, int which);
+
+int t3_phy_advertise_fiber(struct cphy *phy, unsigned int advert);
+
 const struct mac_stats *t3_vsc7323_update_stats(struct cmac *mac);
 
-void t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			   const struct mdio_ops *mdio_ops);
-void t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			 const struct mdio_ops *mdio_ops);
-void t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			 const struct mdio_ops *mdio_ops);
-void t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			 const struct mdio_ops *mdio_ops);
-void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			const struct mdio_ops *mdio_ops);
-void t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			     const struct mdio_ops *mdio_ops);
 #endif /* __CHELSIO_COMMON_H */

sys/dev/cxgb/common/cxgb_ctl_defs.h

@@ -125,8 +125,8 @@ struct rdma_info {
 	unsigned int rqt_top;	 /* RQT last entry address */
 	unsigned int udbell_len; /* user doorbell region length */
 	unsigned long udbell_physbase;  /* user doorbell physical start addr */
-	void volatile *kdb_addr;  /* kernel doorbell register address */
+	void *kdb_addr;  /* kernel doorbell register address */
-	struct pci_dev *pdev;    /* associated PCI device */
+	struct device *pdev;    /* associated PCI device */
 };
 
 /*

sys/dev/cxgb/common/cxgb_firmware_exports.h

@@ -74,6 +74,8 @@ POSSIBILITY OF SUCH DAMAGE.
 
 #define FW_WROPCODE_MNGT			0x1D
 #define FW_MNGTOPCODE_PKTSCHED_SET		0x00
+#define FW_MNGTOPCODE_WRC_SET			0x01
+#define FW_MNGTOPCODE_TUNNEL_CR_FLUSH		0x02
 
 /* Maximum size of a WR sent from the host, limited by the SGE. 
  *

sys/dev/cxgb/common/cxgb_mc5.c

@@ -384,7 +384,7 @@ int t3_mc5_init(struct mc5 *mc5, unsigned int nservers, unsigned int nfilters,
 	return err;
 }
 
-/*
+/**
  *	read_mc5_range - dump a part of the memory managed by MC5
  *	@mc5: the MC5 handle
  *	@start: the start address for the dump
@@ -425,8 +425,11 @@ int t3_read_mc5_range(const struct mc5 *mc5, unsigned int start,
 
 #define MC5_INT_FATAL (F_PARITYERR | F_REQQPARERR | F_DISPQPARERR)
 
-/*
+/**
- * MC5 interrupt handler
+ *	t3_mc5_intr_handler - MC5 interrupt handler
+ *	@mc5: the MC5 handle
+ *
+ *	The MC5 interrupt handler.
  */
 void t3_mc5_intr_handler(struct mc5 *mc5)
 {
@@ -462,6 +465,16 @@ void t3_mc5_intr_handler(struct mc5 *mc5)
 	t3_write_reg(adap, A_MC5_DB_INT_CAUSE, cause);
 }
 
+
+/**
+ *	t3_mc5_prep - initialize the SW state for MC5
+ *	@adapter: the adapter
+ *	@mc5: the MC5 handle
+ *	@mode: whether the TCAM will be in 72- or 144-bit mode
+ *
+ *	Initialize the SW state associated with MC5.  Among other things
+ *	this determines the size of the attached TCAM.
+ */
 void __devinit t3_mc5_prep(adapter_t *adapter, struct mc5 *mc5, int mode)
 {
 #define K * 1024

sys/dev/cxgb/common/cxgb_mv88e1xxx.c

@@ -221,6 +221,16 @@ static int mv88e1xxx_get_link_status(struct cphy *cphy, int *link_ok,
 	return 0;
 }
 
+static int mv88e1xxx_set_speed_duplex(struct cphy *phy, int speed, int duplex)
+{
+	int err = t3_set_phy_speed_duplex(phy, speed, duplex);
+
+	/* PHY needs reset for new settings to take effect */
+	if (!err)
+		err = mv88e1xxx_reset(phy, 0);
+	return err;
+}
+
 static int mv88e1xxx_downshift_set(struct cphy *cphy, int downshift_enable)
 {
 	/*
@@ -258,7 +268,6 @@ static int mv88e1xxx_intr_handler(struct cphy *cphy)
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops mv88e1xxx_ops = {
-	NULL,
 	mv88e1xxx_reset,
 	mv88e1xxx_intr_enable,
 	mv88e1xxx_intr_disable,
@@ -268,7 +277,7 @@ static struct cphy_ops mv88e1xxx_ops = {
 	mv88e1xxx_autoneg_restart,
 	t3_phy_advertise,
 	mv88e1xxx_set_loopback,
-	t3_set_phy_speed_duplex,
+	mv88e1xxx_set_speed_duplex,
 	mv88e1xxx_get_link_status,
 	mv88e1xxx_power_down,
 };
@@ -283,20 +292,28 @@ static struct cphy_ops mv88e1xxx_ops = {
 	.autoneg_restart   = mv88e1xxx_autoneg_restart,
 	.advertise         = t3_phy_advertise,
 	.set_loopback      = mv88e1xxx_set_loopback,
-	.set_speed_duplex  = t3_set_phy_speed_duplex,
+	.set_speed_duplex  = mv88e1xxx_set_speed_duplex,
 	.get_link_status   = mv88e1xxx_get_link_status,
 	.power_down        = mv88e1xxx_power_down,
 };
 #endif
 
-void t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			   const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &mv88e1xxx_ops, mdio_ops);
+	int err;
+
+	cphy_init(phy, adapter, phy_addr, &mv88e1xxx_ops, mdio_ops,
+		  SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full |
+		  SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII |
+		  SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T");
 
 	/* Configure copper PHY transmitter as class A to reduce EMI. */
-	mdio_write(phy, 0, MV88E1XXX_EXTENDED_ADDR, 0xb);
+	err = mdio_write(phy, 0, MV88E1XXX_EXTENDED_ADDR, 0xb);
-	mdio_write(phy, 0, MV88E1XXX_EXTENDED_DATA, 0x8004);
 
-	mv88e1xxx_downshift_set(phy, 1);   /* Enable downshift */
+	if (!err)
+		err = mdio_write(phy, 0, MV88E1XXX_EXTENDED_DATA, 0x8004);
+	if (!err)
+		err = mv88e1xxx_downshift_set(phy, 1);   /* Enable downshift */
+	return err;
 }

sys/dev/cxgb/common/cxgb_t3_cpl.h

@@ -173,8 +173,9 @@ enum {                     /* TCP congestion control algorithms */
 
 enum {			   /* RSS hash type */
 	RSS_HASH_NONE = 0,
-	RSS_HASH_2_TUPLE = 1 << 0,
+	RSS_HASH_2_TUPLE = 1,
-	RSS_HASH_4_TUPLE = 1 << 1
+	RSS_HASH_4_TUPLE = 2,
+	RSS_HASH_TCPV6 = 3
 };
 
 union opcode_tid {
@@ -1097,6 +1098,11 @@ struct cpl_rx_data_ddp {
 #define V_DDP_OFFSET(x) ((x) << S_DDP_OFFSET)
 #define G_DDP_OFFSET(x) (((x) >> S_DDP_OFFSET) & M_DDP_OFFSET)
 
+#define S_DDP_DACK_MODE    22
+#define M_DDP_DACK_MODE    0x3
+#define V_DDP_DACK_MODE(x) ((x) << S_DDP_DACK_MODE)
+#define G_DDP_DACK_MODE(x) (((x) >> S_DDP_DACK_MODE) & M_DDP_DACK_MODE)
+
 #define S_DDP_URG    24
 #define V_DDP_URG(x) ((x) << S_DDP_URG)
 #define F_DDP_URG    V_DDP_URG(1U)

sys/dev/cxgb/common/cxgb_t3_hw.c

@@ -403,6 +403,29 @@ int t3_phy_advertise(struct cphy *phy, unsigned int advert)
 	return mdio_write(phy, 0, MII_ADVERTISE, val);
 }
 
+/**
+ *	t3_phy_advertise_fiber - set fiber PHY advertisement register
+ *	@phy: the PHY to operate on
+ *	@advert: bitmap of capabilities the PHY should advertise
+ *
+ *	Sets a fiber PHY's advertisement register to advertise the
+ *	requested capabilities.
+ */
+int t3_phy_advertise_fiber(struct cphy *phy, unsigned int advert)
+{
+	unsigned int val = 0;
+
+	if (advert & ADVERTISED_1000baseT_Half)
+		val |= ADVERTISE_1000XHALF;
+	if (advert & ADVERTISED_1000baseT_Full)
+		val |= ADVERTISE_1000XFULL;
+	if (advert & ADVERTISED_Pause)
+		val |= ADVERTISE_1000XPAUSE;
+	if (advert & ADVERTISED_Asym_Pause)
+		val |= ADVERTISE_1000XPSE_ASYM;
+	return mdio_write(phy, 0, MII_ADVERTISE, val);
+}
+
 /**
  *	t3_set_phy_speed_duplex - force PHY speed and duplex
  *	@phy: the PHY to operate on
@@ -451,8 +474,8 @@ static struct adapter_info t3_adap_info[] = {
 	  &mi1_mdio_ops, "Chelsio T302" },
 	{ 1, 0, 0, 0, 0,
 	  F_GPIO1_OEN | F_GPIO6_OEN | F_GPIO7_OEN | F_GPIO10_OEN |
-	  F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, 0,
+	  F_GPIO11_OEN | F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL,
-	  SUPPORTED_10000baseT_Full | SUPPORTED_AUI,
+	  0, SUPPORTED_10000baseT_Full | SUPPORTED_AUI,
 	  &mi1_mdio_ext_ops, "Chelsio T310" },
 	{ 1, 1, 0, 0, 0,
 	  F_GPIO1_OEN | F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO5_OEN | F_GPIO6_OEN |
@@ -476,31 +499,20 @@ const struct adapter_info *t3_get_adapter_info(unsigned int id)
 	return id < ARRAY_SIZE(t3_adap_info) ? &t3_adap_info[id] : NULL;
 }
 
-#define CAPS_1G (SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full | \
-		 SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII)
-#define CAPS_10G (SUPPORTED_10000baseT_Full | SUPPORTED_AUI)
-
 static struct port_type_info port_types[] = {
 	{ NULL },
-	{ t3_ael1002_phy_prep, CAPS_10G | SUPPORTED_FIBRE,
+	{ t3_ael1002_phy_prep },
-	  "10GBASE-XR" },
+	{ t3_vsc8211_phy_prep },
-	{ t3_vsc8211_phy_prep, CAPS_1G | SUPPORTED_TP | SUPPORTED_IRQ,
+	{ t3_mv88e1xxx_phy_prep },
-	  "10/100/1000BASE-T" },
+	{ t3_xaui_direct_phy_prep },
-	{ t3_mv88e1xxx_phy_prep, CAPS_1G | SUPPORTED_TP | SUPPORTED_IRQ,
+	{ NULL },
-	  "10/100/1000BASE-T" },
+	{ t3_qt2045_phy_prep },
-	{ t3_xaui_direct_phy_prep, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
+	{ t3_ael1006_phy_prep },
-	{ NULL, CAPS_10G, "10GBASE-KX4" },
+	{ NULL },
-	{ t3_qt2045_phy_prep, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
-	{ t3_ael1006_phy_prep, CAPS_10G | SUPPORTED_FIBRE,
-	  "10GBASE-SR" },
-	{ NULL, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
 };
 
-#undef CAPS_1G
-#undef CAPS_10G
-
 #define VPD_ENTRY(name, len) \
-	u8 name##_kword[2]; u8 name##_len; char name##_data[len]
+	u8 name##_kword[2]; u8 name##_len; u8 name##_data[len]
 
 /*
  * Partial EEPROM Vital Product Data structure.  Includes only the ID and
@@ -678,6 +690,15 @@ static int get_vpd_params(adapter_t *adapter, struct vpd_params *p)
 	return 0;
 }
 
+/* BIOS boot header */
+typedef struct boot_header_s {
+	u8	signature[2];	/* signature */
+	u8	length;		/* image length (include header) */
+	u8	offset[4];	/* initialization vector */
+	u8	reserved[19];	/* reserved */
+	u8	exheader[2];	/* offset to expansion header */
+} boot_header_t;
+
 /* serial flash and firmware constants */
 enum {
 	SF_ATTEMPTS = 5,           /* max retries for SF1 operations */
@@ -694,7 +715,14 @@ enum {
 
 	FW_FLASH_BOOT_ADDR = 0x70000, /* start address of FW in flash */
 	FW_VERS_ADDR = 0x77ffc,    /* flash address holding FW version */
-	FW_MIN_SIZE = 8            /* at least version and csum */
+	FW_MIN_SIZE = 8,           /* at least version and csum */
+	FW_MAX_SIZE = FW_VERS_ADDR - FW_FLASH_BOOT_ADDR,
+
+	BOOT_FLASH_BOOT_ADDR = 0x0,/* start address of boot image in flash */
+	BOOT_SIGNATURE = 0xaa55,   /* signature of BIOS boot ROM */
+	BOOT_SIZE_INC = 512,       /* image size measured in 512B chunks */
+	BOOT_MIN_SIZE = sizeof(boot_header_t), /* at least basic header */
+	BOOT_MAX_SIZE = 0xff*BOOT_SIZE_INC /* 1 byte * length increment  */
 };
 
 /**
@@ -817,16 +845,21 @@ int t3_read_flash(adapter_t *adapter, unsigned int addr, unsigned int nwords,
  *	@addr: the start address to write
  *	@n: length of data to write
  *	@data: the data to write
+ *	@byte_oriented: whether to store data as bytes or as words
  *
  *	Writes up to a page of data (256 bytes) to the serial flash starting
  *	at the given address.
+ *	If @byte_oriented is set the write data is stored as a 32-bit
+ *	big-endian array, otherwise in the processor's native endianess.
+ *	
  */
 static int t3_write_flash(adapter_t *adapter, unsigned int addr,
-			  unsigned int n, const u8 *data)
+			  unsigned int n, const u8 *data,
+			  int byte_oriented)
 {
 	int ret;
 	u32 buf[64];
-	unsigned int i, c, left, val, offset = addr & 0xff;
+	unsigned int c, left, val, offset = addr & 0xff;
 
 	if (addr + n > SF_SIZE || offset + n > 256)
 		return -EINVAL;
@@ -839,8 +872,10 @@ static int t3_write_flash(adapter_t *adapter, unsigned int addr,
 
 	for (left = n; left; left -= c) {
 		c = min(left, 4U);
-		for (val = 0, i = 0; i < c; ++i)
+		val = *(const u32*)data;
-			val = (val << 8) + *data++;
+		data += c;
+		if (byte_oriented)
+			val = htonl(val);
 
 		ret = sf1_write(adapter, c, c != left, val);
 		if (ret)
@@ -850,7 +885,8 @@ static int t3_write_flash(adapter_t *adapter, unsigned int addr,
 		return ret;
 
 	/* Read the page to verify the write succeeded */
-	ret = t3_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1);
+	ret = t3_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf,
+			    byte_oriented);
 	if (ret)
 		return ret;
 
@@ -887,16 +923,18 @@ int t3_get_tp_version(adapter_t *adapter, u32 *vers)
  *	@adapter: the adapter
  *
  */
-int t3_check_tpsram_version(adapter_t *adapter)
+int t3_check_tpsram_version(adapter_t *adapter, int *must_load)
 {
 	int ret;
 	u32 vers;
 	unsigned int major, minor;
 
-	/* Get version loaded in SRAM */
+	if (adapter->params.rev == T3_REV_A)
-	t3_write_reg(adapter, A_TP_EMBED_OP_FIELD0, 0);
+		return 0;
-	ret = t3_wait_op_done(adapter, A_TP_EMBED_OP_FIELD0,
+
-			      1, 1, 5, 1);
+	*must_load = 1;
+
+	ret = t3_get_tp_version(adapter, &vers);
 	if (ret)
 		return ret;
 	
@@ -908,9 +946,16 @@ int t3_check_tpsram_version(adapter_t *adapter)
 	if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 
 		return 0;
 
-	CH_WARN(adapter, "found wrong TP version (%u.%u), "
+	if (major != TP_VERSION_MAJOR)
-	       "driver needs version %d.%d\n", major, minor,
+		CH_ERR(adapter, "found wrong TP version (%u.%u), "
-	       TP_VERSION_MAJOR, TP_VERSION_MINOR);
+		       "driver needs version %d.%d\n", major, minor,
+		       TP_VERSION_MAJOR, TP_VERSION_MINOR);
+	else {
+		*must_load = 0;
+		CH_ERR(adapter, "found wrong TP version (%u.%u), "
+		       "driver compiled for version %d.%d\n", major, minor,
+		       TP_VERSION_MAJOR, TP_VERSION_MINOR);
+	}
 	return -EINVAL;
 }
 
@@ -966,12 +1011,13 @@ int t3_get_fw_version(adapter_t *adapter, u32 *vers)
  *	Checks if an adapter's FW is compatible with the driver.  Returns 0
  *	if the versions are compatible, a negative error otherwise.
  */
-int t3_check_fw_version(adapter_t *adapter)
+int t3_check_fw_version(adapter_t *adapter, int *must_load)
 {
 	int ret;
 	u32 vers;
 	unsigned int type, major, minor;
 
+	*must_load = 1;
 	ret = t3_get_fw_version(adapter, &vers);
 	if (ret)
 		return ret;
@@ -984,9 +1030,21 @@ int t3_check_fw_version(adapter_t *adapter)
 	    minor == FW_VERSION_MINOR)
 		return 0;
 
-	CH_WARN(adapter, "found wrong FW version (%u.%u), "
+	if (major != FW_VERSION_MAJOR)
-	       "driver needs version %d.%d\n", major, minor,
+		CH_ERR(adapter, "found wrong FW version(%u.%u), "
-	       FW_VERSION_MAJOR, FW_VERSION_MINOR);
+		       "driver needs version %u.%u\n", major, minor,
+		       FW_VERSION_MAJOR, FW_VERSION_MINOR);
+	else if ((int)minor < FW_VERSION_MINOR) {
+		*must_load = 0;
+		CH_WARN(adapter, "found old FW minor version(%u.%u), "
+		        "driver compiled for version %u.%u\n", major, minor,
+			FW_VERSION_MAJOR, FW_VERSION_MINOR);
+	} else {
+		CH_WARN(adapter, "found newer FW version(%u.%u), "
+		        "driver compiled for version %u.%u\n", major, minor,
+			FW_VERSION_MAJOR, FW_VERSION_MINOR);
+			return 0;
+	}
 	return -EINVAL;
 }
 
@@ -1033,7 +1091,7 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
 
 	if ((size & 3) || size < FW_MIN_SIZE)
 		return -EINVAL;
-	if (size > FW_VERS_ADDR + 8 - FW_FLASH_BOOT_ADDR)
+	if (size - 8 > FW_MAX_SIZE)
 		return -EFBIG;
 
 	for (csum = 0, i = 0; i < size / sizeof(csum); i++)
@@ -1052,7 +1110,7 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
 	for (addr = FW_FLASH_BOOT_ADDR; size; ) {
 		unsigned int chunk_size = min(size, 256U);
 
-		ret = t3_write_flash(adapter, addr, chunk_size, fw_data);
+		ret = t3_write_flash(adapter, addr, chunk_size, fw_data, 1);
 		if (ret)
 			goto out;
 
@@ -1061,13 +1119,71 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
 		size -= chunk_size;
 	}
 
-	ret = t3_write_flash(adapter, FW_VERS_ADDR, 4, fw_data);
+	ret = t3_write_flash(adapter, FW_VERS_ADDR, 4, fw_data, 1);
 out:
 	if (ret)
 		CH_ERR(adapter, "firmware download failed, error %d\n", ret);
 	return ret;
 }
 
+/*
+ *	t3_load_boot - download boot flash
+ *	@adapter: the adapter
+ *	@boot_data: the boot image to write
+ *	@size: image size
+ *
+ *	Write the supplied boot image to the card's serial flash.
+ *	The boot image has the following sections: a 28-byte header and the
+ *	boot image.
+ */
+int t3_load_boot(adapter_t *adapter, u8 *boot_data, unsigned int size)
+{
+	boot_header_t *header = (boot_header_t *)boot_data;
+	int ret;
+	unsigned int addr;
+	unsigned int boot_sector = BOOT_FLASH_BOOT_ADDR >> 16;
+	unsigned int boot_end = (BOOT_FLASH_BOOT_ADDR + size - 1) >> 16;
+
+	/*
+	 * Perform some primitive sanity testing to avoid accidentally
+	 * writing garbage over the boot sectors.  We ought to check for
+	 * more but it's not worth it for now ...
+	 */
+	if (size < BOOT_MIN_SIZE || size > BOOT_MAX_SIZE) {
+		CH_ERR(adapter, "boot image too small/large\n");
+		return -EFBIG;
+	}
+	if (le16_to_cpu(*(u16*)header->signature) != BOOT_SIGNATURE) {
+		CH_ERR(adapter, "boot image missing signature\n");
+		return -EINVAL;
+	}
+	if (header->length * BOOT_SIZE_INC != size) {
+		CH_ERR(adapter, "boot image header length != image length\n");
+		return -EINVAL;
+	}
+
+	ret = t3_flash_erase_sectors(adapter, boot_sector, boot_end);
+	if (ret)
+		goto out;
+
+	for (addr = BOOT_FLASH_BOOT_ADDR; size; ) {
+		unsigned int chunk_size = min(size, 256U);
+
+		ret = t3_write_flash(adapter, addr, chunk_size, boot_data, 0);
+		if (ret)
+			goto out;
+
+		addr += chunk_size;
+		boot_data += chunk_size;
+		size -= chunk_size;
+	}
+
+out:
+	if (ret)
+		CH_ERR(adapter, "boot image download failed, error %d\n", ret);
+	return ret;
+}
+
 #define CIM_CTL_BASE 0x2000
 
 /**
@@ -1175,7 +1291,6 @@ int t3_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc)
 						   fc);
 			/* Also disables autoneg */
 			phy->ops->set_speed_duplex(phy, lc->speed, lc->duplex);
-			phy->ops->reset(phy, 0);
 		} else
 			phy->ops->autoneg_enable(phy);
 	} else {
@@ -1248,7 +1363,13 @@ static int t3_handle_intr_status(adapter_t *adapter, unsigned int reg,
 	return fatal;
 }
 
-#define SGE_INTR_MASK (F_RSPQDISABLED)
+#define SGE_INTR_MASK (F_RSPQDISABLED | \
+		       F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR | \
+		       F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
+		       F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
+		       V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
+		       F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
+		       F_HIRCQPARITYERROR)
 #define MC5_INTR_MASK (F_PARITYERR | F_ACTRGNFULL | F_UNKNOWNCMD | \
 		       F_REQQPARERR | F_DISPQPARERR | F_DELACTEMPTY | \
 		       F_NFASRCHFAIL)
@@ -1265,16 +1386,23 @@ static int t3_handle_intr_status(adapter_t *adapter, unsigned int reg,
 #define PCIE_INTR_MASK (F_UNXSPLCPLERRR | F_UNXSPLCPLERRC | F_PCIE_PIOPARERR |\
 			F_PCIE_WFPARERR | F_PCIE_RFPARERR | F_PCIE_CFPARERR | \
 			/* V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR) | */ \
-			V_BISTERR(M_BISTERR) | F_PEXERR)
+			F_RETRYBUFPARERR | F_RETRYLUTPARERR | F_RXPARERR | \
-#define ULPRX_INTR_MASK F_PARERR
+			F_TXPARERR | V_BISTERR(M_BISTERR))
-#define ULPTX_INTR_MASK 0
+#define ULPRX_INTR_MASK (F_PARERRDATA | F_PARERRPCMD | F_ARBPF1PERR | \
-#define CPLSW_INTR_MASK (F_TP_FRAMING_ERROR | \
+			 F_ARBPF0PERR | F_ARBFPERR | F_PCMDMUXPERR | \
+			 F_DATASELFRAMEERR1 | F_DATASELFRAMEERR0)
+#define ULPTX_INTR_MASK 0xfc
+#define CPLSW_INTR_MASK (F_CIM_OP_MAP_PERR | F_TP_FRAMING_ERROR | \
 			 F_SGE_FRAMING_ERROR | F_CIM_FRAMING_ERROR | \
 			 F_ZERO_SWITCH_ERROR)
 #define CIM_INTR_MASK (F_BLKWRPLINT | F_BLKRDPLINT | F_BLKWRCTLINT | \
 		       F_BLKRDCTLINT | F_BLKWRFLASHINT | F_BLKRDFLASHINT | \
 		       F_SGLWRFLASHINT | F_WRBLKFLASHINT | F_BLKWRBOOTINT | \
-	 	       F_FLASHRANGEINT | F_SDRAMRANGEINT | F_RSVDSPACEINT)
+	 	       F_FLASHRANGEINT | F_SDRAMRANGEINT | F_RSVDSPACEINT | \
+		       F_DRAMPARERR | F_ICACHEPARERR | F_DCACHEPARERR | \
+		       F_OBQSGEPARERR | F_OBQULPHIPARERR | F_OBQULPLOPARERR | \
+		       F_IBQSGELOPARERR | F_IBQSGEHIPARERR | F_IBQULPPARERR | \
+		       F_IBQTPPARERR | F_ITAGPARERR | F_DTAGPARERR)
 #define PMTX_INTR_MASK (F_ZERO_C_CMD_ERROR | ICSPI_FRM_ERR | OESPI_FRM_ERR | \
 			V_ICSPI_PAR_ERROR(M_ICSPI_PAR_ERROR) | \
 			V_OESPI_PAR_ERROR(M_OESPI_PAR_ERROR))
@@ -1343,6 +1471,10 @@ static void pcie_intr_handler(adapter_t *adapter)
 		{ F_PCIE_CFPARERR, "PCI command FIFO parity error", -1, 1 },
 		{ V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR),
 		  "PCI MSI-X table/PBA parity error", -1, 1 },
+		{ F_RETRYBUFPARERR, "PCI retry buffer parity error", -1, 1 },
+		{ F_RETRYLUTPARERR, "PCI retry LUT parity error", -1, 1 },
+		{ F_RXPARERR, "PCI Rx parity error", -1, 1 },
+		{ F_TXPARERR, "PCI Tx parity error", -1, 1 },
 		{ V_BISTERR(M_BISTERR), "PCI BIST error", -1, 1 },
 		{ 0 }
 	};
@@ -1367,9 +1499,16 @@ static void tp_intr_handler(adapter_t *adapter)
 		{ 0x2000000, "TP out of Tx pages", -1, 1 },
 		{ 0 }
 	};
+	static struct intr_info tp_intr_info_t3c[] = {
+		{ 0x1fffffff,  "TP parity error", -1, 1 },
+		{ F_FLMRXFLSTEMPTY, "TP out of Rx pages", -1, 1 },
+		{ F_FLMTXFLSTEMPTY, "TP out of Tx pages", -1, 1 },
+		{ 0 }
+	};
 
 	if (t3_handle_intr_status(adapter, A_TP_INT_CAUSE, 0xffffffff,
-				  tp_intr_info, NULL))
+				  adapter->params.rev < T3_REV_C ?
+					tp_intr_info : tp_intr_info_t3c, NULL))
 		t3_fatal_err(adapter);
 }
 
@@ -1391,10 +1530,22 @@ static void cim_intr_handler(adapter_t *adapter)
 		{ F_BLKWRCTLINT, "CIM block write to CTL space", -1, 1 },
 		{ F_BLKRDPLINT, "CIM block read from PL space", -1, 1 },
 		{ F_BLKWRPLINT, "CIM block write to PL space", -1, 1 },
+		{ F_DRAMPARERR, "CIM DRAM parity error", -1, 1 },
+		{ F_ICACHEPARERR, "CIM icache parity error", -1, 1 },
+		{ F_DCACHEPARERR, "CIM dcache parity error", -1, 1 },
+		{ F_OBQSGEPARERR, "CIM OBQ SGE parity error", -1, 1 },
+		{ F_OBQULPHIPARERR, "CIM OBQ ULPHI parity error", -1, 1 },
+		{ F_OBQULPLOPARERR, "CIM OBQ ULPLO parity error", -1, 1 },
+		{ F_IBQSGELOPARERR, "CIM IBQ SGELO parity error", -1, 1 },
+		{ F_IBQSGEHIPARERR, "CIM IBQ SGEHI parity error", -1, 1 },
+		{ F_IBQULPPARERR, "CIM IBQ ULP parity error", -1, 1 },
+		{ F_IBQTPPARERR, "CIM IBQ TP parity error", -1, 1 },
+		{ F_ITAGPARERR, "CIM itag parity error", -1, 1 },
+		{ F_DTAGPARERR, "CIM dtag parity error", -1, 1 },
 		{ 0 }
         };
 
-	if (t3_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, 0xffffffff,
+	if (t3_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, CIM_INTR_MASK,
 				  cim_intr_info, NULL))
 		t3_fatal_err(adapter);
 }
@@ -1405,7 +1556,14 @@ static void cim_intr_handler(adapter_t *adapter)
 static void ulprx_intr_handler(adapter_t *adapter)
 {
 	static struct intr_info ulprx_intr_info[] = {
-		{ F_PARERR, "ULP RX parity error", -1, 1 },
+		{ F_PARERRDATA, "ULP RX data parity error", -1, 1 },
+		{ F_PARERRPCMD, "ULP RX command parity error", -1, 1 },
+		{ F_ARBPF1PERR, "ULP RX ArbPF1 parity error", -1, 1 },
+		{ F_ARBPF0PERR, "ULP RX ArbPF0 parity error", -1, 1 },
+		{ F_ARBFPERR, "ULP RX ArbF parity error", -1, 1 },
+		{ F_PCMDMUXPERR, "ULP RX PCMDMUX parity error", -1, 1 },
+		{ F_DATASELFRAMEERR1, "ULP RX frame error", -1, 1 },
+		{ F_DATASELFRAMEERR0, "ULP RX frame error", -1, 1 },
 		{ 0 }
         };
 
@@ -1424,6 +1582,7 @@ static void ulptx_intr_handler(adapter_t *adapter)
 		  STAT_ULP_CH0_PBL_OOB, 0 },
 		{ F_PBL_BOUND_ERR_CH1, "ULP TX channel 1 PBL out of bounds",
 		  STAT_ULP_CH1_PBL_OOB, 0 },
+		{ 0xfc, "ULP TX parity error", -1, 1 },
 		{ 0 }
         };
 
@@ -1498,7 +1657,8 @@ static void pmrx_intr_handler(adapter_t *adapter)
 static void cplsw_intr_handler(adapter_t *adapter)
 {
 	static struct intr_info cplsw_intr_info[] = {
-//		{ F_CIM_OVFL_ERROR, "CPL switch CIM overflow", -1, 1 },
+		{ F_CIM_OP_MAP_PERR, "CPL switch CIM parity error", -1, 1 },
+		{ F_CIM_OVFL_ERROR, "CPL switch CIM overflow", -1, 1 },
 		{ F_TP_FRAMING_ERROR, "CPL switch TP framing error", -1, 1 },
 		{ F_SGE_FRAMING_ERROR, "CPL switch SGE framing error", -1, 1 },
 		{ F_CIM_FRAMING_ERROR, "CPL switch CIM framing error", -1, 1 },
@@ -1632,7 +1792,7 @@ int t3_phy_intr_handler(adapter_t *adapter)
 		mask = gpi - (gpi & (gpi - 1));
 		gpi -= mask;
 
-		if (!(p->port_type->caps & SUPPORTED_IRQ))
+		if (!(p->phy.caps & SUPPORTED_IRQ))
 			continue;
 
 		if (cause & mask) {
@@ -1728,7 +1888,6 @@ void t3_intr_enable(adapter_t *adapter)
 			MC7_INTR_MASK },
 		{ A_MC5_DB_INT_ENABLE, MC5_INTR_MASK },
 		{ A_ULPRX_INT_ENABLE, ULPRX_INTR_MASK },
-		{ A_TP_INT_ENABLE, 0x3bfffff },
 		{ A_PM1_TX_INT_ENABLE, PMTX_INTR_MASK },
 		{ A_PM1_RX_INT_ENABLE, PMRX_INTR_MASK },
 		{ A_CIM_HOST_INT_ENABLE, CIM_INTR_MASK },
@@ -1738,6 +1897,8 @@ void t3_intr_enable(adapter_t *adapter)
 	adapter->slow_intr_mask = PL_INTR_MASK;
 
 	t3_write_regs(adapter, intr_en_avp, ARRAY_SIZE(intr_en_avp), 0);
+	t3_write_reg(adapter, A_TP_INT_ENABLE,
+		     adapter->params.rev >= T3_REV_C ? 0x2bfffff : 0x3bfffff);
 
 	if (adapter->params.rev > 0) {
 		t3_write_reg(adapter, A_CPL_INTR_ENABLE,
@@ -1889,6 +2050,15 @@ static int t3_sge_write_context(adapter_t *adapter, unsigned int id,
 			       0, SG_CONTEXT_CMD_ATTEMPTS, 1);
 }
 
+static int clear_sge_ctxt(adapter_t *adap, unsigned int id, unsigned int type)
+{
+	t3_write_reg(adap, A_SG_CONTEXT_DATA0, 0);
+	t3_write_reg(adap, A_SG_CONTEXT_DATA1, 0);
+	t3_write_reg(adap, A_SG_CONTEXT_DATA2, 0);
+	t3_write_reg(adap, A_SG_CONTEXT_DATA3, 0);
+	return t3_sge_write_context(adap, id, type);
+}
+
 /**
  *	t3_sge_init_ecntxt - initialize an SGE egress context
  *	@adapter: the adapter to configure
@@ -2389,20 +2559,6 @@ static void tp_wr_bits_indirect(adapter_t *adap, unsigned int addr,
 	t3_write_reg(adap, A_TP_PIO_DATA, val);
 }
 
-/**
- *	t3_enable_filters - enable the HW filters
- *	@adap: the adapter
- *
- *	Enables the HW filters for NIC traffic.
- */
-void t3_enable_filters(adapter_t *adap)
-{
-	t3_set_reg_field(adap, A_TP_IN_CONFIG, F_NICMODE, 0);
-	t3_set_reg_field(adap, A_MC5_DB_CONFIG, 0, F_FILTEREN);
-	t3_set_reg_field(adap, A_TP_GLOBAL_CONFIG, 0, V_FIVETUPLELOOKUP(3));
-	tp_wr_bits_indirect(adap, A_TP_INGRESS_CONFIG, 0, F_LOOKUPEVERYPKT);
-}
-
 /**
  *	pm_num_pages - calculate the number of pages of the payload memory
  *	@mem_size: the size of the payload memory
@@ -2508,7 +2664,7 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
 		     V_AUTOSTATE2(1) | V_AUTOSTATE1(0) |
 		     V_BYTETHRESHOLD(16384) | V_MSSTHRESHOLD(2) |
 		     F_AUTOCAREFUL | F_AUTOENABLE | V_DACK_MODE(1));
-	t3_set_reg_field(adap, A_TP_IN_CONFIG, F_IPV6ENABLE | F_NICMODE,
+	t3_set_reg_field(adap, A_TP_IN_CONFIG, F_RXFBARBPRIO | F_TXFBARBPRIO,
 			 F_IPV6ENABLE | F_NICMODE);
 	t3_write_reg(adap, A_TP_TX_RESOURCE_LIMIT, 0x18141814);
 	t3_write_reg(adap, A_TP_PARA_REG4, 0x5050105);
@@ -2519,7 +2675,9 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
 			 F_ENABLEEPCMDAFULL,
 			 F_ENABLEOCSPIFULL |F_TXDEFERENABLE | F_HEARBEATDACK |
 			 F_TXCONGESTIONMODE | F_RXCONGESTIONMODE);
-	t3_set_reg_field(adap, A_TP_PC_CONFIG2, F_CHDRAFULL, 0);
+	t3_set_reg_field(adap, A_TP_PC_CONFIG2, F_CHDRAFULL,
+			 F_ENABLEIPV6RSS | F_ENABLENONOFDTNLSYN |
+			 F_ENABLEARPMISS | F_DISBLEDAPARBIT0);
 	t3_write_reg(adap, A_TP_PROXY_FLOW_CNTL, 1080);
 	t3_write_reg(adap, A_TP_PROXY_FLOW_CNTL, 1000);
 
@@ -2534,6 +2692,11 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
 	} else
 		t3_set_reg_field(adap, A_TP_PARA_REG3, 0, F_TXPACEFIXED);
 
+	if (adap->params.rev == T3_REV_C)
+		t3_set_reg_field(adap, A_TP_PC_CONFIG,
+				 V_TABLELATENCYDELTA(M_TABLELATENCYDELTA),
+				 V_TABLELATENCYDELTA(4));
+
 	t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT1, 0);
 	t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0, 0);
 	t3_write_reg(adap, A_TP_MOD_CHANNEL_WEIGHT, 0);
@@ -2972,7 +3135,7 @@ int t3_config_sched(adapter_t *adap, unsigned int kbps, int sched)
 			if (bpt > 0 && bpt <= 255) {
 				v = bpt * tps;
 				delta = v >= kbps ? v - kbps : kbps - v;
-				if (delta <= mindelta) {
+				if (delta < mindelta) {
 					mindelta = delta;
 					selected_cpt = cpt;
 					selected_bpt = bpt;
@@ -3383,7 +3546,8 @@ static void config_pcie(adapter_t *adap)
 			 V_REPLAYLMT(rpllmt));
 
 	t3_write_reg(adap, A_PCIE_PEX_ERR, 0xffffffff);
-	t3_set_reg_field(adap, A_PCIE_CFG, F_PCIE_CLIDECEN, F_PCIE_CLIDECEN);
+	t3_set_reg_field(adap, A_PCIE_CFG, 0,
+			 F_PCIE_DMASTOPEN | F_PCIE_CLIDECEN);
 }
 
 /**
@@ -3401,7 +3565,7 @@ static void config_pcie(adapter_t *adap)
  */
 int t3_init_hw(adapter_t *adapter, u32 fw_params)
 {
-	int err = -EIO, attempts = 100;
+	int err = -EIO, attempts, i;
 	const struct vpd_params *vpd = &adapter->params.vpd;
 
 	if (adapter->params.rev > 0)
@@ -3422,6 +3586,10 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
 			        adapter->params.mc5.nfilters,
 			       	adapter->params.mc5.nroutes))
 			goto out_err;
+
+		for (i = 0; i < 32; i++)
+			if (clear_sge_ctxt(adapter, i, F_CQ))
+				goto out_err;
 	}
 
 	if (tp_init(adapter, &adapter->params.tp))
@@ -3438,7 +3606,12 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
 	if (is_pcie(adapter))
 		config_pcie(adapter);
 	else
-		t3_set_reg_field(adapter, A_PCIX_CFG, 0, F_CLIDECEN);
+		t3_set_reg_field(adapter, A_PCIX_CFG, 0,
+				 F_DMASTOPEN | F_CLIDECEN);
+
+	if (adapter->params.rev == T3_REV_C)
+		t3_set_reg_field(adapter, A_ULPTX_CONFIG, 0,
+				 F_CFG_CQE_SOP_MASK);
 
 	t3_write_reg(adapter, A_PM1_RX_CFG, 0xffffffff);
 	t3_write_reg(adapter, A_PM1_RX_MODE, 0);
@@ -3451,6 +3624,7 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
 		     V_BOOTADDR(FW_FLASH_BOOT_ADDR >> 2));
 	(void) t3_read_reg(adapter, A_CIM_BOOT_CFG);    /* flush */
 
+	attempts = 100;
 	do {                          /* wait for uP to initialize */
 		msleep(20);
 	} while (t3_read_reg(adapter, A_CIM_HOST_ACC_DATA) && --attempts);
@@ -3601,6 +3775,7 @@ void early_hw_init(adapter_t *adapter, const struct adapter_info *ai)
 	t3_write_reg(adapter, A_T3DBG_GPIO_EN,
 		     ai->gpio_out | F_GPIO0_OEN | F_GPIO0_OUT_VAL);
 	t3_write_reg(adapter, A_MC5_DB_SERVER_INDEX, 0);
+	t3_write_reg(adapter, A_SG_OCO_BASE, V_BASE1(0xfff));
 
 	if (adapter->params.rev == 0 || !uses_xaui(adapter))
 		val |= F_ENRGMII;
@@ -3651,6 +3826,36 @@ static int t3_reset_adapter(adapter_t *adapter)
 	return 0;
 }
 
+static int __devinit init_parity(adapter_t *adap)
+{
+	int i, err, addr;
+
+	if (t3_read_reg(adap, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY)
+		return -EBUSY;
+
+	for (err = i = 0; !err && i < 16; i++)
+		err = clear_sge_ctxt(adap, i, F_EGRESS);
+	for (i = 0xfff0; !err && i <= 0xffff; i++)
+		err = clear_sge_ctxt(adap, i, F_EGRESS);
+	for (i = 0; !err && i < SGE_QSETS; i++)
+		err = clear_sge_ctxt(adap, i, F_RESPONSEQ);
+	if (err)
+		return err;
+
+	t3_write_reg(adap, A_CIM_IBQ_DBG_DATA, 0);
+	for (i = 0; i < 4; i++)
+		for (addr = 0; addr <= M_IBQDBGADDR; addr++) {
+			t3_write_reg(adap, A_CIM_IBQ_DBG_CFG, F_IBQDBGEN |
+				     F_IBQDBGWR | V_IBQDBGQID(i) |
+				     V_IBQDBGADDR(addr));
+			err = t3_wait_op_done(adap, A_CIM_IBQ_DBG_CFG,
+					      F_IBQDBGBUSY, 0, 2, 1);
+			if (err)
+				return err;
+		}
+	return 0;
+}
+
 /**
  *	t3_prep_adapter - prepare SW and HW for operation
  *	@adapter: the adapter
@@ -3732,6 +3937,9 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
 	}
 
 	early_hw_init(adapter, ai);
+	ret = init_parity(adapter);
+	if (ret)
+		return ret;
 
 	if (adapter->params.nports > 2 &&
 	    (ret = t3_vsc7323_init(adapter, adapter->params.nports)))
@@ -3739,14 +3947,17 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
 
 	for_each_port(adapter, i) {
 		u8 hw_addr[6];
+		const struct port_type_info *pti;
 		struct port_info *p = adap2pinfo(adapter, i);
 
 		while (!adapter->params.vpd.port_type[j])
 			++j;
 
-		p->port_type = &port_types[adapter->params.vpd.port_type[j]];
+		pti = &port_types[adapter->params.vpd.port_type[j]];
-		p->port_type->phy_prep(&p->phy, adapter, ai->phy_base_addr + j,
+		ret = pti->phy_prep(&p->phy, adapter, ai->phy_base_addr + j,
-				       ai->mdio_ops);
+				    ai->mdio_ops);
+		if (ret)
+			return ret;
 		mac_prep(&p->mac, adapter, j);
 		++j;
 
@@ -3759,9 +3970,9 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
 		hw_addr[5] = adapter->params.vpd.eth_base[5] + i;
 
 		t3_os_set_hw_addr(adapter, i, hw_addr);
-		init_link_config(&p->link_config, p->port_type->caps);
+		init_link_config(&p->link_config, p->phy.caps);
 		p->phy.ops->power_down(&p->phy, 1);
-		if (!(p->port_type->caps & SUPPORTED_IRQ))
+		if (!(p->phy.caps & SUPPORTED_IRQ))
 			adapter->params.linkpoll_period = 10;
 	}
 

sys/dev/cxgb/common/cxgb_tcb.h

@@ -668,7 +668,10 @@ POSSIBILITY OF SUCH DAMAGE.
 #define S_TF_DDP_BUF1_FLUSH    28
 #define V_TF_DDP_BUF1_FLUSH(x) ((x) << S_TF_DDP_BUF1_FLUSH)
 
-#define S_TF_DDP_PSH_NO_INVALIDATE    29
+#define S_TF_DDP_PSH_NO_INVALIDATE0    29
-#define V_TF_DDP_PSH_NO_INVALIDATE(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE)
+#define V_TF_DDP_PSH_NO_INVALIDATE0(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE0)
+
+#define S_TF_DDP_PSH_NO_INVALIDATE1    30
+#define V_TF_DDP_PSH_NO_INVALIDATE1(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE1)
 
 #endif /* _TCB_DEFS_H */

sys/dev/cxgb/common/cxgb_version.h

@@ -37,5 +37,5 @@ POSSIBILITY OF SUCH DAMAGE.
 #define __CHELSIO_VERSION_H
 #define DRV_DESC "Chelsio T3 Network Driver"
 #define DRV_NAME "cxgb"
-#define DRV_VERSION "1.0.086"
+#define DRV_VERSION "1.0.129a"
 #endif

sys/dev/cxgb/common/cxgb_vsc8211.c

@@ -36,11 +36,17 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/cxgb_include.h>
 #endif
 
+#undef msleep
+#define msleep t3_os_sleep
+
 /* VSC8211 PHY specific registers. */
 enum {
+	VSC8211_SIGDET_CTRL   = 19,
+	VSC8211_EXT_CTRL      = 23,
 	VSC8211_INTR_ENABLE   = 25,
 	VSC8211_INTR_STATUS   = 26,
 	VSC8211_AUX_CTRL_STAT = 28,
+	VSC8211_EXT_PAGE_AXS  = 31,
 };
 
 enum {
@@ -55,11 +61,19 @@ enum {
 	VSC_INTR_SYMBOL_ERR = 1 << 8,  /* symbol error */
 	VSC_INTR_NEG_DONE   = 1 << 10, /* autoneg done */
 	VSC_INTR_NEG_ERR    = 1 << 11, /* autoneg error */
+	VSC_INTR_DPLX_CHG   = 1 << 12, /* duplex change */
 	VSC_INTR_LINK_CHG   = 1 << 13, /* link change */
+	VSC_INTR_SPD_CHG    = 1 << 14, /* speed change */
 	VSC_INTR_ENABLE     = 1 << 15, /* interrupt enable */
 };
 
+enum {
+	VSC_CTRL_CLAUSE37_VIEW = 1 << 4,   /* Switch to Clause 37 view */
+	VSC_CTRL_MEDIA_MODE_HI = 0xf000    /* High part of media mode select */
+};
+
 #define CFG_CHG_INTR_MASK (VSC_INTR_LINK_CHG | VSC_INTR_NEG_ERR | \
+			   VSC_INTR_DPLX_CHG | VSC_INTR_SPD_CHG | \
 	 		   VSC_INTR_NEG_DONE)
 #define INTR_MASK (CFG_CHG_INTR_MASK | VSC_INTR_TX_FIFO | VSC_INTR_RX_FIFO | \
 		   VSC_INTR_ENABLE)
@@ -189,6 +203,98 @@ static int vsc8211_get_link_status(struct cphy *cphy, int *link_ok,
 	return 0;
 }
 
+static int vsc8211_get_link_status_fiber(struct cphy *cphy, int *link_ok,
+					 int *speed, int *duplex, int *fc)
+{
+	unsigned int bmcr, status, lpa, adv;
+	int err, sp = -1, dplx = -1, pause = 0;
+
+	err = mdio_read(cphy, 0, MII_BMCR, &bmcr);
+	if (!err)
+		err = mdio_read(cphy, 0, MII_BMSR, &status);
+	if (err)
+		return err;
+
+	if (link_ok) {
+		/*
+		 * BMSR_LSTATUS is latch-low, so if it is 0 we need to read it
+		 * once more to get the current link state.
+		 */
+		if (!(status & BMSR_LSTATUS))
+			err = mdio_read(cphy, 0, MII_BMSR, &status);
+		if (err)
+			return err;
+		*link_ok = (status & BMSR_LSTATUS) != 0;
+	}
+	if (!(bmcr & BMCR_ANENABLE)) {
+		dplx = (bmcr & BMCR_FULLDPLX) ? DUPLEX_FULL : DUPLEX_HALF;
+		if (bmcr & BMCR_SPEED1000)
+			sp = SPEED_1000;
+		else if (bmcr & BMCR_SPEED100)
+			sp = SPEED_100;
+		else
+			sp = SPEED_10;
+	} else if (status & BMSR_ANEGCOMPLETE) {
+		err = mdio_read(cphy, 0, MII_LPA, &lpa);
+		if (!err)
+			err = mdio_read(cphy, 0, MII_ADVERTISE, &adv);
+		if (err)
+			return err;
+
+		if (adv & lpa & ADVERTISE_1000XFULL) {
+			dplx = DUPLEX_FULL;
+			sp = SPEED_1000;
+		} else if (adv & lpa & ADVERTISE_1000XHALF) {
+			dplx = DUPLEX_HALF;
+			sp = SPEED_1000;
+		}
+
+		if (fc && dplx == DUPLEX_FULL) {
+			if (lpa & adv & ADVERTISE_1000XPAUSE)
+				pause = PAUSE_RX | PAUSE_TX;
+			else if ((lpa & ADVERTISE_1000XPAUSE) &&
+				 (adv & lpa & ADVERTISE_1000XPSE_ASYM))
+				pause = PAUSE_TX;
+			else if ((lpa & ADVERTISE_1000XPSE_ASYM) &&
+				 (adv & ADVERTISE_1000XPAUSE))
+				pause = PAUSE_RX;
+		}
+	}
+	if (speed)
+		*speed = sp;
+	if (duplex)
+		*duplex = dplx;
+	if (fc)
+		*fc = pause;
+	return 0;
+}
+
+/*
+ * Enable/disable auto MDI/MDI-X in forced link speed mode.
+ */
+static int vsc8211_set_automdi(struct cphy *phy, int enable)
+{
+	int err;
+
+	if ((err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0x52b5)) != 0 ||
+	    (err = mdio_write(phy, 0, 18, 0x12)) != 0 ||
+	    (err = mdio_write(phy, 0, 17, enable ? 0x2803 : 0x3003)) != 0 ||
+	    (err = mdio_write(phy, 0, 16, 0x87fa)) != 0 ||
+	    (err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0)) != 0)
+		return err;
+	return 0;
+}
+
+static int vsc8211_set_speed_duplex(struct cphy *phy, int speed, int duplex)
+{
+	int err;
+
+	err = t3_set_phy_speed_duplex(phy, speed, duplex);
+	if (!err)
+		err = vsc8211_set_automdi(phy, 1);
+	return err;
+}
+
 static int vsc8211_power_down(struct cphy *cphy, int enable)
 {
 	return t3_mdio_change_bits(cphy, 0, MII_BMCR, BMCR_PDOWN,
@@ -214,7 +320,6 @@ static int vsc8211_intr_handler(struct cphy *cphy)
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops vsc8211_ops = {
-	NULL,
 	vsc8211_reset,
 	vsc8211_intr_enable,
 	vsc8211_intr_disable,
@@ -224,10 +329,25 @@ static struct cphy_ops vsc8211_ops = {
 	vsc8211_autoneg_restart,
 	t3_phy_advertise,
 	NULL,
-	t3_set_phy_speed_duplex,
+	vsc8211_set_speed_duplex,
 	vsc8211_get_link_status,
 	vsc8211_power_down,
 };
+
+static struct cphy_ops vsc8211_fiber_ops = {
+	vsc8211_reset,
+	vsc8211_intr_enable,
+	vsc8211_intr_disable,
+	vsc8211_intr_clear,
+	vsc8211_intr_handler,
+	vsc8211_autoneg_enable,
+	vsc8211_autoneg_restart,
+	t3_phy_advertise_fiber,
+	NULL,
+	t3_set_phy_speed_duplex,
+	vsc8211_get_link_status_fiber,
+	vsc8211_power_down,
+};
 #else
 static struct cphy_ops vsc8211_ops = {
 	.reset             = vsc8211_reset,
@@ -238,15 +358,57 @@ static struct cphy_ops vsc8211_ops = {
 	.autoneg_enable    = vsc8211_autoneg_enable,
 	.autoneg_restart   = vsc8211_autoneg_restart,
 	.advertise         = t3_phy_advertise,
-	.set_speed_duplex  = t3_set_phy_speed_duplex,
+	.set_speed_duplex  = vsc8211_set_speed_duplex,
 	.get_link_status   = vsc8211_get_link_status,
 	.power_down        = vsc8211_power_down,
 };
+
+static struct cphy_ops vsc8211_fiber_ops = {
+	.reset             = vsc8211_reset,
+	.intr_enable       = vsc8211_intr_enable,
+	.intr_disable      = vsc8211_intr_disable,
+	.intr_clear        = vsc8211_intr_clear,
+	.intr_handler      = vsc8211_intr_handler,
+	.autoneg_enable    = vsc8211_autoneg_enable,
+	.autoneg_restart   = vsc8211_autoneg_restart,
+	.advertise         = t3_phy_advertise_fiber,
+	.set_speed_duplex  = t3_set_phy_speed_duplex,
+	.get_link_status   = vsc8211_get_link_status_fiber,
+	.power_down        = vsc8211_power_down,
+};
 #endif
 
-void t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			 const struct mdio_ops *mdio_ops)
+			const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &vsc8211_ops, mdio_ops);
+	int err;
-	t3_os_sleep(20);       /* PHY needs ~10ms to start responding to MDIO */
+	unsigned int val;
+
+	cphy_init(phy, adapter, phy_addr, &vsc8211_ops, mdio_ops,
+		  SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full |
+		  SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII |
+		  SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T");
+	msleep(20);       /* PHY needs ~10ms to start responding to MDIO */
+
+	err = mdio_read(phy, 0, VSC8211_EXT_CTRL, &val);
+	if (err)
+		return err;
+	if (val & VSC_CTRL_MEDIA_MODE_HI)
+		return 0;   /* copper interface, done */
+
+	phy->caps = SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg |
+		    SUPPORTED_MII | SUPPORTED_FIBRE | SUPPORTED_IRQ;
+	phy->desc = "1000BASE-X";
+	phy->ops = &vsc8211_fiber_ops;
+
+	if ((err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 1)) != 0 ||
+	    (err = mdio_write(phy, 0, VSC8211_SIGDET_CTRL, 1)) != 0 ||
+	    (err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0)) != 0 ||
+	    (err = mdio_write(phy, 0, VSC8211_EXT_CTRL,
+			      val | VSC_CTRL_CLAUSE37_VIEW)) != 0 ||
+	    (err = vsc8211_reset(phy, 0)) != 0)
+		return err;
+
+	udelay(5); /* delay after reset before next SMI */
+	return 0;
 }

sys/dev/cxgb/common/cxgb_xgmac.c

@@ -75,6 +75,12 @@ static void xaui_serdes_reset(struct cmac *mac)
 	}
 }
 
+/**
+ *	t3b_pcs_reset - reset the PCS on T3B+ adapters
+ *	@mac: the XGMAC handle
+ *
+ *	Reset the XGMAC PCS block on T3B+ adapters.
+ */
 void t3b_pcs_reset(struct cmac *mac)
 {
 	t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset,
@@ -84,6 +90,12 @@ void t3b_pcs_reset(struct cmac *mac)
 			 F_PCS_RESET_);
 }
 
+/**
+ *	t3_mac_reset - reset a MAC
+ *	@mac: the MAC to reset
+ *
+ *	Reset the given MAC.
+ */
 int t3_mac_reset(struct cmac *mac)
 {
 	static struct addr_val_pair mac_reset_avp[] = {
@@ -114,6 +126,7 @@ int t3_mac_reset(struct cmac *mac)
 	t3_set_reg_field(adap, A_XGM_RXFIFO_CFG + oft,
 			 F_RXSTRFRWRD | F_DISERRFRAMES,
 			 uses_xaui(adap) ? 0 : F_RXSTRFRWRD);
+	t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + oft, 0, F_UNDERUNFIX);
 
 	if (uses_xaui(adap)) {
 		if (adap->params.rev == 0) {
@@ -146,8 +159,10 @@ int t3_mac_reset(struct cmac *mac)
 		t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN);
 		t3_write_reg(adap, A_XGM_RX_CTRL + oft, F_RXEN);
 	}
-
+	t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + oft,
-	val = F_MAC_RESET_;
+			 V_RXMAXFRAMERSIZE(M_RXMAXFRAMERSIZE),
+			 V_RXMAXFRAMERSIZE(MAX_FRAME_SIZE) | F_RXENFRAMER);
+	val = F_MAC_RESET_ | F_XGMAC_STOP_EN;
 	if (is_10G(adap) || mac->multiport)
 		val |= F_PCS_RESET_;
 	else if (uses_xaui(adap))
@@ -236,7 +251,14 @@ static void set_addr_filter(struct cmac *mac, int idx, const u8 *addr)
 	t3_write_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_HIGH_1 + oft, addr_hi);
 }
 
-/* Set one of the station's unicast MAC addresses. */
+/**
+ *	t3_mac_set_address - set one of the station's unicast MAC addresses
+ *	@mac: the MAC handle
+ *	@idx: index of the exact address match filter to use
+ *	@addr: the Ethernet address
+ *
+ *	Set one of the station's unicast MAC addresses.
+ */
 int t3_mac_set_address(struct cmac *mac, unsigned int idx, u8 addr[6])
 {
 	if (mac->multiport)
@@ -249,10 +271,14 @@ int t3_mac_set_address(struct cmac *mac, unsigned int idx, u8 addr[6])
 	return 0;
 }
 
-/*
+/**
- * Specify the number of exact address filters that should be reserved for
+ *	t3_mac_set_num_ucast - set the number of unicast addresses needed
- * unicast addresses.  Caller should reload the unicast and multicast addresses
+ *	@mac: the MAC handle
- * after calling this.
+ *	@n: number of unicast addresses needed
+ *
+ *	Specify the number of exact address filters that should be reserved for
+ *	unicast addresses.  Caller should reload the unicast and multicast
+ *	addresses after calling this.
  */
 int t3_mac_set_num_ucast(struct cmac *mac, unsigned char n)
 {
@@ -298,6 +324,14 @@ static int hash_hw_addr(const u8 *addr)
 	return hash;
 }
 
+/**
+ *	t3_mac_set_rx_mode - set the Rx mode and address filters
+ *	@mac: the MAC to configure
+ *	@rm: structure containing the Rx mode and MAC addresses needed
+ *
+ *	Configures the MAC Rx mode (promiscuity, etc) and exact and hash
+ *	address filters.
+ */
 int t3_mac_set_rx_mode(struct cmac *mac, struct t3_rx_mode *rm)
 {
 	u32 hash_lo, hash_hi;
@@ -344,10 +378,18 @@ static int rx_fifo_hwm(int mtu)
 	return min(hwm, MAC_RXFIFO_SIZE - 8192);
 }
 
+/**
+ *	t3_mac_set_mtu - set the MAC MTU
+ *	@mac: the MAC to configure
+ *	@mtu: the MTU
+ *
+ *	Sets the MAC MTU and adjusts the FIFO PAUSE watermarks accordingly.
+ */
 int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu) 
 {
-	int hwm, lwm;
+	int hwm, lwm, divisor;
-	unsigned int thres, v;
+	int ipg;
+	unsigned int thres, v, reg;
 	adapter_t *adap = mac->adapter;
 
 	/*
@@ -362,26 +404,32 @@ int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu)
 	if (mac->multiport)
 		return t3_vsc7323_set_mtu(adap, mtu - 4, mac->ext_port);
 
-	if (adap->params.rev == T3_REV_B2 &&
+	if (adap->params.rev >= T3_REV_B2 &&
 	    (t3_read_reg(adap, A_XGM_RX_CTRL + mac->offset) & F_RXEN)) {
 		disable_exact_filters(mac);
 		v = t3_read_reg(adap, A_XGM_RX_CFG + mac->offset);
 		t3_set_reg_field(adap, A_XGM_RX_CFG + mac->offset,
 				 F_ENHASHMCAST | F_COPYALLFRAMES, F_DISBCAST);
 
-		/* drain rx FIFO */
+		reg = adap->params.rev == T3_REV_B2 ?
-		if (t3_wait_op_done(adap,
+			A_XGM_RX_MAX_PKT_SIZE_ERR_CNT : A_XGM_RXFIFO_CFG;
-				    A_XGM_RX_MAX_PKT_SIZE_ERR_CNT + mac->offset,
+	
-				    1 << 31, 1, 20, 5)) {
+		/* drain RX FIFO */
+		if (t3_wait_op_done(adap, reg + mac->offset,
+				    F_RXFIFO_EMPTY, 1, 20, 5)) {
 			t3_write_reg(adap, A_XGM_RX_CFG + mac->offset, v);
 			enable_exact_filters(mac);
 			return -EIO;
 		}
-		t3_write_reg(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, mtu);
+		t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset,
+				 V_RXMAXPKTSIZE(M_RXMAXPKTSIZE),
+				 V_RXMAXPKTSIZE(mtu));
 		t3_write_reg(adap, A_XGM_RX_CFG + mac->offset, v);
 		enable_exact_filters(mac);
 	} else
-		t3_write_reg(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, mtu);
+		t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset,
+		                 V_RXMAXPKTSIZE(M_RXMAXPKTSIZE),
+		                 V_RXMAXPKTSIZE(mtu));
 	
 	/*
 	 * Adjust the PAUSE frame watermarks.  We always set the LWM, and the
@@ -405,20 +453,34 @@ int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu)
 		thres /= 10;
 	thres = mtu > thres ? (mtu - thres + 7) / 8 : 0;
 	thres = max(thres, 8U);                          /* need at least 8 */
+	ipg = (adap->params.rev == T3_REV_C) ? 0 : 1;
 	t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + mac->offset,
 			 V_TXFIFOTHRESH(M_TXFIFOTHRESH) | V_TXIPG(M_TXIPG),
-			 V_TXFIFOTHRESH(thres) | V_TXIPG(1));
+			 V_TXFIFOTHRESH(thres) | V_TXIPG(ipg));
 
 	/* Assuming a minimum drain rate of 2.5Gbps...
 	 */
-	if (adap->params.rev > 0)
+	if (adap->params.rev > 0) {
+		divisor = (adap->params.rev == T3_REV_C) ? 64 : 8;
 		t3_write_reg(adap, A_XGM_PAUSE_TIMER + mac->offset, 
-			     (hwm - lwm) * 4 / 8);
+		    (hwm - lwm) * 4 / divisor);
+	}
 	t3_write_reg(adap, A_XGM_TX_PAUSE_QUANTA + mac->offset, 
 		     MAC_RXFIFO_SIZE * 4 * 8 / 512);
 	return 0;
 }
 
+/**
+ *	t3_mac_set_speed_duplex_fc - set MAC speed, duplex and flow control
+ *	@mac: the MAC to configure
+ *	@speed: the desired speed (10/100/1000/10000)
+ *	@duplex: the desired duplex
+ *	@fc: desired Tx/Rx PAUSE configuration
+ *
+ *	Set the MAC speed, duplex (actually only full-duplex is supported), and
+ *	flow control.  If a parameter value is negative the corresponding
+ *	MAC setting is left at its current value.
+ */
 int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc)
 {
 	u32 val;
@@ -466,6 +528,15 @@ int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc)
 	return 0;
 }
 
+/**
+ *	t3_mac_enable - enable the MAC in the given directions
+ *	@mac: the MAC to configure
+ *	@which: bitmap indicating which directions to enable
+ *
+ *	Enables the MAC for operation in the given directions.
+ *	%MAC_DIRECTION_TX enables the Tx direction, and %MAC_DIRECTION_RX
+ *	enables the Rx one.
+ */
 int t3_mac_enable(struct cmac *mac, int which)
 {
 	int idx = macidx(mac);
@@ -478,9 +549,13 @@ int t3_mac_enable(struct cmac *mac, int which)
 
 	if (which & MAC_DIRECTION_TX) {
 		t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx);
-		t3_write_reg(adap, A_TP_PIO_DATA, 0xc0ede401);
+		t3_write_reg(adap, A_TP_PIO_DATA,
+			     adap->params.rev == T3_REV_C ?
+			     0xc4ffff01 : 0xc0ede401);
 		t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_MODE);
-		t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx, 1 << idx);
+		t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx,
+				 adap->params.rev == T3_REV_C ?
+				 0 : 1 << idx);
 
 		t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN);
 
@@ -505,6 +580,15 @@ int t3_mac_enable(struct cmac *mac, int which)
 	return 0;
 }
 
+/**
+ *	t3_mac_disable - disable the MAC in the given directions
+ *	@mac: the MAC to configure
+ *	@which: bitmap indicating which directions to disable
+ *
+ *	Disables the MAC in the given directions.
+ *	%MAC_DIRECTION_TX disables the Tx direction, and %MAC_DIRECTION_RX
+ *	disables the Rx one.
+ */
 int t3_mac_disable(struct cmac *mac, int which)
 {
 	adapter_t *adap = mac->adapter;
@@ -621,12 +705,15 @@ int t3b2_mac_watchdog_task(struct cmac *mac)
 	return status;
 }
 
-/*
+/**
- * This function is called periodically to accumulate the current values of the
+ *	t3_mac_update_stats - accumulate MAC statistics
- * RMON counters into the port statistics.  Since the packet counters are only
+ *	@mac: the MAC handle
- * 32 bits they can overflow in ~286 secs at 10G, so the function should be
+ *
- * called more frequently than that.  The byte counters are 45-bit wide, they
+ *	This function is called periodically to accumulate the current values
- * would overflow in ~7.8 hours.
+ *	of the RMON counters into the port statistics.  Since the packet
+ *	counters are only 32 bits they can overflow in ~286 secs at 10G, so the
+ *	function should be called more frequently than that.  The byte counters
+ *	are 45-bit wide, they would overflow in ~7.8 hours.
  */
 const struct mac_stats *t3_mac_update_stats(struct cmac *mac)
 {

sys/dev/cxgb/cxgb_adapter.h

@@ -46,6 +46,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <net/if_media.h>
+#include <net/if_dl.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
@@ -54,6 +55,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
+
 #ifdef CONFIG_DEFINED
 #include <cxgb_osdep.h>
 #include <t3cdev.h>
@@ -144,6 +146,9 @@ enum {				/* adapter flags */
 	QUEUES_BOUND	= (1 << 3),
 	FW_UPTODATE     = (1 << 4),
 	TPS_UPTODATE    = (1 << 5),
+	CXGB_SHUTDOWN	= (1 << 6),
+	CXGB_OFLD_INIT	= (1 << 7),
+	TP_PARITY_INIT  = (1 << 8),
 };
 
 #define FL_Q_SIZE	4096
@@ -203,6 +208,7 @@ struct sge_rspq {
 	uint32_t	holdoff_tmr;
 	uint32_t	next_holdoff;
 	uint32_t        imm_data;
+	uint32_t        async_notif;
 	uint32_t	cntxt_id;
 	uint32_t        offload_pkts;
 	uint32_t        offload_bundles;
@@ -348,6 +354,8 @@ struct adapter {
 	/* PCI register resources */
 	int			regs_rid;
 	struct resource		*regs_res;
+	int			udbs_rid;
+	struct resource		*udbs_res;
 	bus_space_handle_t	bh;
 	bus_space_tag_t		bt;
 	bus_size_t              mmio_len;
@@ -508,9 +516,22 @@ static __inline uint8_t *
 t3_get_next_mcaddr(struct t3_rx_mode *rm)
 {
 	uint8_t *macaddr = NULL;
+	struct ifnet *ifp = rm->port->ifp;
+	struct ifmultiaddr *ifma;
+	int i = 0;
+
+	IF_ADDR_LOCK(ifp);
+	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+		if (ifma->ifma_addr->sa_family != AF_LINK)
+			continue;
+		if (i == rm->idx) {
+			macaddr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
+			break;
+		}
+		i++;
+	}
+	IF_ADDR_UNLOCK(ifp);
 
-	if (rm->idx == 0)
-		macaddr = (uint8_t *)rm->port->hw_addr;
 	
 	rm->idx++;
 	return (macaddr);

sys/dev/cxgb/cxgb_ioctl.h

@@ -101,15 +101,16 @@ struct ch_mem_range {
 };
 
 struct ch_qset_params {
-	uint32_t qset_idx;
+	uint32_t 	qset_idx;
-	int32_t  txq_size[3];
+	int32_t		txq_size[3];
-	int32_t  rspq_size;
+	int32_t		rspq_size;
-	int32_t  fl_size[2];
+	int32_t		fl_size[2];
-	int32_t  intr_lat;
+	int32_t		intr_lat;
-	int32_t  polling;
+	int32_t		polling;
-	int32_t  cong_thres;
+	int32_t		lro;
-	int32_t  vector;
+	int32_t		cong_thres;
-	int32_t  qnum;
+	int32_t		vector;
+	int32_t		qnum;
 };
 
 struct ch_pktsched_params {
@@ -260,4 +261,6 @@ struct mii_data {
 #define CHELSIO_SET_FILTER          _IOW('f', CH_SET_FILTER, struct ch_filter)
 #define CHELSIO_DEL_FILTER          _IOW('f', CH_DEL_FILTER, struct ch_filter)
 #define CHELSIO_DEVUP               _IO('f', CH_DEVUP)
+
+#define CHELSIO_GET_TCB             _IOWR('f', CH_GET_TCB, struct ch_tcb)
 #endif

sys/dev/cxgb/cxgb_l2t.c

@@ -176,10 +176,7 @@ t3_l2t_send_slow(struct t3cdev *dev, struct mbuf *m, struct l2t_entry *e)
 	sin.sin_len = sizeof(struct sockaddr_in);
 	sin.sin_addr.s_addr = e->addr;
 
-	
+	CTR2(KTR_CXGB, "send slow on rt=%p eaddr=0x%08x\n", rt, e->addr);
-	
-	printf("send slow on rt=%p eaddr=0x%08x\n", rt, e->addr);
-	
 again:
 	switch (e->state) {
 	case L2T_STATE_STALE:     /* entry is stale, kick off revalidation */
@@ -199,8 +196,6 @@ t3_l2t_send_slow(struct t3cdev *dev, struct mbuf *m, struct l2t_entry *e)
 		}
 		arpq_enqueue(e, m);
 		mtx_unlock(&e->lock);
-		printf("enqueueing arp request\n");
-		
 		/*
 		 * Only the first packet added to the arpq should kick off
 		 * resolution.  However, because the m_gethdr below can fail,
@@ -209,10 +204,9 @@ t3_l2t_send_slow(struct t3cdev *dev, struct mbuf *m, struct l2t_entry *e)
 		 * A better way would be to use a work request to retry L2T
 		 * entries when there's no memory.
 		 */
-		printf("doing arpresolve on 0x%x \n", e->addr);
 		if (arpresolve(rt->rt_ifp, rt, NULL,
 		     (struct sockaddr *)&sin, e->dmac) == 0) {
-			printf("mac=%x:%x:%x:%x:%x:%x\n",
+			CTR6(KTR_CXGB, "mac=%x:%x:%x:%x:%x:%x\n",
 			    e->dmac[0], e->dmac[1], e->dmac[2], e->dmac[3], e->dmac[4], e->dmac[5]);
 			
 			if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
@@ -224,8 +218,7 @@ t3_l2t_send_slow(struct t3cdev *dev, struct mbuf *m, struct l2t_entry *e)
 			else
 				m_freem(m);
 			mtx_unlock(&e->lock);
-		} else
+		}
-			printf("arpresolve returned non-zero\n");
 	}
 	return 0;
 }
@@ -396,8 +389,6 @@ t3_l2t_get(struct t3cdev *dev, struct rtentry *neigh, struct ifnet *ifp,
 	/* Need to allocate a new entry */
 	e = alloc_l2e(d);
 	if (e) {
-		printf("initializing new entry\n");
-		
 		mtx_lock(&e->lock);          /* avoid race with t3_l2t_free */
 		e->next = d->l2tab[hash].first;
 		d->l2tab[hash].first = e;
@@ -472,8 +463,6 @@ t3_l2t_update(struct t3cdev *dev, struct rtentry *neigh,
 	int hash = arp_hash(addr, ifidx, d);
 	struct llinfo_arp *la;
 
-	printf("t3_l2t_update called with arp info\n");
-	
 	rw_rlock(&d->lock);
 	for (e = d->l2tab[hash].first; e; e = e->next)
 		if (e->addr == addr && e->ifindex == ifidx) {
@@ -481,7 +470,7 @@ t3_l2t_update(struct t3cdev *dev, struct rtentry *neigh,
 			goto found;
 		}
 	rw_runlock(&d->lock);
-	printf("addr=0x%08x not found\n", addr);
+	CTR1(KTR_CXGB, "t3_l2t_update: addr=0x%08x not found", addr);
 	return;
 
 found:
@@ -543,6 +532,12 @@ t3_init_l2t(unsigned int l2t_capacity)
 void
 t3_free_l2t(struct l2t_data *d)
 {
+	int i;
+
+	rw_destroy(&d->lock);
+	for (i = 0; i < d->nentries; ++i) 
+		mtx_destroy(&d->l2tab[i].lock);
+
 	cxgb_free_mem(d);
 }
 

sys/dev/cxgb/cxgb_l2t.h

@@ -143,8 +143,6 @@ static inline int l2t_send(struct t3cdev *dev, struct mbuf *m,
 	if (__predict_true(e->state == L2T_STATE_VALID)) {
 		return cxgb_ofld_send(dev, (struct mbuf *)m);
 	}
-	printf("send slow\n");
-	
 	return t3_l2t_send_slow(dev, (struct mbuf *)m, e);
 }
 

sys/dev/cxgb/cxgb_main.c

@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <sys/bus_dma.h>
+#include <sys/ktr.h>
 #include <sys/rman.h>
 #include <sys/ioccom.h>
 #include <sys/mbuf.h>
@@ -119,6 +120,7 @@ static int cxgb_get_regs_len(void);
 static int offload_open(struct port_info *pi);
 static void touch_bars(device_t dev);
 static int offload_close(struct t3cdev *tdev);
+static void cxgb_link_start(struct port_info *p);
 
 static device_method_t cxgb_controller_methods[] = {
 	DEVMETHOD(device_probe,		cxgb_controller_probe),
@@ -281,6 +283,32 @@ struct cxgb_ident {
 
 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
 
+
+void
+cxgb_log_tcb(struct adapter *sc, unsigned int tid)
+{
+	char buf[TCB_SIZE];
+	uint64_t *tcb = (uint64_t *)buf;
+	int i, error;
+	struct mc7 *mem = &sc->cm;
+	
+	error = t3_mc7_bd_read(mem, tid*TCB_SIZE/8, TCB_SIZE/8, tcb);
+	if (error)
+		printf("cxgb_tcb_log failed\n");
+	
+	CTR1(KTR_CXGB, "TCB tid=%u", tid);
+	for (i = 0; i < TCB_SIZE / 32; i++) {
+		CTR5(KTR_CXGB, "%1d: %08x %08x %08x %08x",
+		    i, (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
+		    (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
+		tcb += 2;
+		CTR4(KTR_CXGB, "   %08x %08x %08x %08x",
+		    (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
+		    (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
+		tcb += 2;
+	}
+}
+
 static __inline char
 t3rev2char(struct adapter *adapter)
 {
@@ -398,6 +426,7 @@ cxgb_controller_attach(device_t dev)
 #ifdef MSI_SUPPORTED
 	int msi_needed, reg;
 #endif
+	int must_load = 0;
 	sc = device_get_softc(dev);
 	sc->dev = dev;
 	sc->msi_count = 0;
@@ -434,9 +463,16 @@ cxgb_controller_attach(device_t dev)
 	sc->regs_rid = PCIR_BAR(0);
 	if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 	    &sc->regs_rid, RF_ACTIVE)) == NULL) {
-		device_printf(dev, "Cannot allocate BAR\n");
+		device_printf(dev, "Cannot allocate BAR region 0\n");
 		return (ENXIO);
 	}
+	sc->udbs_rid = PCIR_BAR(2);
+	if ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+           &sc->udbs_rid, RF_ACTIVE)) == NULL) {
+		device_printf(dev, "Cannot allocate BAR region 1\n");
+		error = ENXIO;
+		goto out;
+       }
 
 	snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
 	    device_get_unit(dev));
@@ -449,7 +485,7 @@ cxgb_controller_attach(device_t dev)
 	snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
 	    device_get_unit(dev));
 	
-	MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_DEF);
+	MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
 	MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
 	MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
 	
@@ -534,7 +570,7 @@ cxgb_controller_attach(device_t dev)
 	/* Create a periodic callout for checking adapter status */
 	callout_init(&sc->cxgb_tick_ch, TRUE);
 	
-	if (t3_check_fw_version(sc) != 0) {
+	if (t3_check_fw_version(sc, &must_load) != 0 && must_load) {
 		/*
 		 * Warn user that a firmware update will be attempted in init.
 		 */
@@ -545,7 +581,7 @@ cxgb_controller_attach(device_t dev)
 		sc->flags |= FW_UPTODATE;
 	}
 
-	if (t3_check_tpsram_version(sc) != 0) {
+	if (t3_check_tpsram_version(sc, &must_load) != 0 && must_load) {
 		/*
 		 * Warn user that a firmware update will be attempted in init.
 		 */
@@ -609,6 +645,8 @@ cxgb_controller_attach(device_t dev)
 	    G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
 	    G_FW_VERSION_MICRO(vers));
 
+	device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
+	callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
 	t3_add_attach_sysctls(sc);
 out:
 	if (error)
@@ -634,9 +672,12 @@ cxgb_free(struct adapter *sc)
 {
 	int i;
 
-	
+	ADAPTER_LOCK(sc);
+	sc->flags |= CXGB_SHUTDOWN;
+	ADAPTER_UNLOCK(sc);
 	cxgb_pcpu_shutdown_threads(sc);
 	ADAPTER_LOCK(sc);
+
 /*
  * drops the lock
  */
@@ -655,10 +696,6 @@ cxgb_free(struct adapter *sc)
 		    sc->msix_regs_res);
 	}
 
-	if (sc->tq != NULL) {
-		taskqueue_drain(sc->tq, &sc->ext_intr_task);
-		taskqueue_drain(sc->tq, &sc->tick_task);	
-	}	
 	t3_sge_deinit_sw(sc);
 	/*
 	 * Wait for last callout
@@ -672,8 +709,11 @@ cxgb_free(struct adapter *sc)
 	}
 		
 	bus_generic_detach(sc->dev);
-	if (sc->tq != NULL) 
+	if (sc->tq != NULL) {
 		taskqueue_free(sc->tq);
+		sc->tq = NULL;
+	}
+	
 	if (is_offload(sc)) {
 		cxgb_adapter_unofld(sc);
 		if (isset(&sc->open_device_map,	OFFLOAD_DEVMAP_BIT))
@@ -682,11 +722,18 @@ cxgb_free(struct adapter *sc)
 			printf("cxgb_free: DEVMAP_BIT not set\n");
 	} else
 		printf("not offloading set\n");	
+
+	if (sc->flags & CXGB_OFLD_INIT)
+		cxgb_offload_deactivate(sc);
 	free(sc->filters, M_DEVBUF);
 	t3_sge_free(sc);
 	
 	cxgb_offload_exit();
 
+	if (sc->udbs_res != NULL)
+		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
+		    sc->udbs_res);
+
 	if (sc->regs_res != NULL)
 		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
 		    sc->regs_res);
@@ -797,8 +844,6 @@ cxgb_setup_msix(adapter_t *sc, int msix_count)
 				return (EINVAL);
 			}
 			sc->msix_irq_rid[k] = rid;
-			printf("setting up interrupt for port=%d\n",
-			    qs->port->port_id);
 			if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
 				INTR_MPSAFE|INTR_TYPE_NET,
 #ifdef INTR_FILTERS
@@ -828,10 +873,11 @@ cxgb_port_probe(device_t dev)
 {
 	struct port_info *p;
 	char buf[80];
+	const char *desc;
 	
 	p = device_get_softc(dev);
-
+	desc = p->phy.desc;
-	snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, p->port_type->desc);
+	snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
 	device_set_desc_copy(dev, buf);
 	return (0);
 }
@@ -873,9 +919,11 @@ cxgb_port_attach(device_t dev)
 	struct port_info *p;
 	struct ifnet *ifp;
 	int err, media_flags;
+	struct adapter *sc;
+	
 	
 	p = device_get_softc(dev);
-
+	sc = p->adapter;
 	snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
 	    device_get_unit(device_get_parent(dev)), p->port_id);
 	PORT_LOCK_INIT(p, p->lockbuf);
@@ -897,11 +945,12 @@ cxgb_port_attach(device_t dev)
 	ifp->if_ioctl = cxgb_ioctl;
 	ifp->if_start = cxgb_start;
 
+#if 0	
 #ifdef IFNET_MULTIQUEUE
 	ifp->if_flags |= IFF_MULTIQ;
 	ifp->if_mq_start = cxgb_pcpu_start;
 #endif
-	
+#endif	
 	ifp->if_timer = 0;	/* Disable ifnet watchdog */
 	ifp->if_watchdog = NULL;
 
@@ -935,13 +984,13 @@ cxgb_port_attach(device_t dev)
 	ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
 	    cxgb_media_status);
       
-	if (!strcmp(p->port_type->desc, "10GBASE-CX4")) {
+	if (!strcmp(p->phy.desc,	"10GBASE-CX4")) {
 		media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
-	} else if (!strcmp(p->port_type->desc, "10GBASE-SR")) {
+	} else if (!strcmp(p->phy.desc, "10GBASE-SR")) {
 		media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
-	} else if (!strcmp(p->port_type->desc, "10GBASE-XR")) {
+	} else if (!strcmp(p->phy.desc, "10GBASE-XR")) {
 		media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
-	} else if (!strcmp(p->port_type->desc, "10/100/1000BASE-T")) {
+	} else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) {
 		ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
 		ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
 			    0, NULL);
@@ -953,7 +1002,7 @@ cxgb_port_attach(device_t dev)
 			    0, NULL);
 		media_flags = 0;
 	} else {
-	        printf("unsupported media type %s\n", p->port_type->desc);
+	        printf("unsupported media type %s\n", p->phy.desc);
 		return (ENXIO);
 	}
 	if (media_flags) {
@@ -976,7 +1025,8 @@ cxgb_port_attach(device_t dev)
 	    taskqueue_thread_enqueue, &p->tq);
 #endif	
 	t3_sge_init_port(p);
-
+	cxgb_link_start(p);
+	t3_link_changed(sc, p->port_id);
 	return (0);
 }
 
@@ -1119,17 +1169,14 @@ t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
 	struct port_info *pi = &adapter->port[port_id];
 	struct cmac *mac = &adapter->port[port_id].mac;
 
-	if ((pi->ifp->if_flags & IFF_UP) == 0)
-		return;
-
 	if (link_status) {
 		t3_mac_enable(mac, MAC_DIRECTION_RX);
 		if_link_state_change(pi->ifp, LINK_STATE_UP);
 	} else {
-		if_link_state_change(pi->ifp, LINK_STATE_DOWN);
 		pi->phy.ops->power_down(&pi->phy, 1);
 		t3_mac_disable(mac, MAC_DIRECTION_RX);
 		t3_link_start(&pi->phy, mac, &pi->link_config);
+		if_link_state_change(pi->ifp, LINK_STATE_DOWN);
 	}
 }
 
@@ -1195,6 +1242,84 @@ cxgb_link_start(struct port_info *p)
 	t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 }
 
+
+static int
+await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
+			      unsigned long n)
+{
+	int attempts = 5;
+
+	while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
+		if (!--attempts)
+			return (ETIMEDOUT);
+		t3_os_sleep(10);
+	}
+	return 0;
+}
+
+static int
+init_tp_parity(struct adapter *adap)
+{
+	int i;
+	struct mbuf *m;
+	struct cpl_set_tcb_field *greq;
+	unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
+
+	t3_tp_set_offload_mode(adap, 1);
+
+	for (i = 0; i < 16; i++) {
+		struct cpl_smt_write_req *req;
+
+		m = m_gethdr(M_WAITOK, MT_DATA);
+		req = mtod(m, struct cpl_smt_write_req *);
+		m->m_len = m->m_pkthdr.len = sizeof(*req);
+		memset(req, 0, sizeof(*req));
+		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
+		req->iff = i;
+		t3_mgmt_tx(adap, m);
+	}
+
+	for (i = 0; i < 2048; i++) {
+		struct cpl_l2t_write_req *req;
+
+		m = m_gethdr(M_WAITOK, MT_DATA);
+		req = mtod(m, struct cpl_l2t_write_req *);
+		m->m_len = m->m_pkthdr.len = sizeof(*req);
+		memset(req, 0, sizeof(*req));
+		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
+		req->params = htonl(V_L2T_W_IDX(i));
+		t3_mgmt_tx(adap, m);
+	}
+
+	for (i = 0; i < 2048; i++) {
+		struct cpl_rte_write_req *req;
+
+		m = m_gethdr(M_WAITOK, MT_DATA);
+		req = mtod(m, struct cpl_rte_write_req *);
+		m->m_len = m->m_pkthdr.len = sizeof(*req);
+		memset(req, 0, sizeof(*req));
+		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
+		req->l2t_idx = htonl(V_L2T_W_IDX(i));
+		t3_mgmt_tx(adap, m);
+	}
+
+	m = m_gethdr(M_WAITOK, MT_DATA);
+	greq = mtod(m, struct cpl_set_tcb_field *);
+	m->m_len = m->m_pkthdr.len = sizeof(*greq);
+	memset(greq, 0, sizeof(*greq));
+	greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+	OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
+	greq->mask = htobe64(1);
+	t3_mgmt_tx(adap, m);
+
+	i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
+	t3_tp_set_offload_mode(adap, 0);
+	return (i);
+}
+
 /**
  *	setup_rss - configure Receive Side Steering (per-queue connection demux) 
  *	@adap: the adapter
@@ -1224,11 +1349,9 @@ setup_rss(adapter_t *adap)
 
 		nq[pi->tx_chan] += pi->nqsets;
 	}
-	nq[0] = max(nq[0], 1U);
-	nq[1] = max(nq[1], 1U);
 	for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
-		rspq_map[i] = i % nq[0];
+		rspq_map[i] = nq[0] ? i % nq[0] : 0;
-		rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq[1]) + nq[0];
+		rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
 	}
 	/* Calculate the reverse RSS map table */
 	for (i = 0; i < RSS_TABLE_SIZE; ++i)
@@ -1237,7 +1360,8 @@ setup_rss(adapter_t *adap)
 
 	t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 		      F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
-		      V_RRCPLCPUSIZE(6), cpus, rspq_map);
+	              F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
+	              cpus, rspq_map);
 
 }
 
@@ -1470,6 +1594,7 @@ cxgb_up(struct adapter *sc)
 		if (err)
 			goto out;
 
+		t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
 		t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 
 		err = setup_sge_qsets(sc);
@@ -1510,8 +1635,18 @@ cxgb_up(struct adapter *sc)
 	t3_sge_start(sc);
 	t3_intr_enable(sc);
 
+	if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
+	    is_offload(sc) && init_tp_parity(sc) == 0)
+		sc->flags |= TP_PARITY_INIT;
+
+	if (sc->flags & TP_PARITY_INIT) {
+		t3_write_reg(sc, A_TP_INT_CAUSE,
+				F_CMCACHEPERR | F_ARPLUTPERR);
+		t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
+	}
+
+	
 	if (!(sc->flags & QUEUES_BOUND)) {
-		printf("bind qsets\n");
 		bind_qsets(sc);
 		sc->flags |= QUEUES_BOUND;		
 	}
@@ -1529,7 +1664,6 @@ cxgb_up(struct adapter *sc)
 static void
 cxgb_down_locked(struct adapter *sc)
 {
-	int i;
 	
 	t3_sge_stop(sc);
 	t3_intr_disable(sc);
@@ -1548,7 +1682,6 @@ cxgb_down_locked(struct adapter *sc)
 	
 	if (sc->flags & USING_MSIX) 
 		cxgb_teardown_msix(sc);
-	ADAPTER_UNLOCK(sc);
 	
 	callout_stop(&sc->cxgb_tick_ch);
 	callout_stop(&sc->sge_timer_ch);
@@ -1556,10 +1689,15 @@ cxgb_down_locked(struct adapter *sc)
 	callout_drain(&sc->sge_timer_ch);
 	
 	if (sc->tq != NULL) {
+		printf("draining slow intr\n");
+		
 		taskqueue_drain(sc->tq, &sc->slow_intr_task);
-		for (i = 0; i < sc->params.nports; i++) 
+			printf("draining ext intr\n");	
-			taskqueue_drain(sc->tq, &sc->port[i].timer_reclaim_task);
+		taskqueue_drain(sc->tq, &sc->ext_intr_task);
+		printf("draining tick task\n");
+		taskqueue_drain(sc->tq, &sc->tick_task);
 	}
+	ADAPTER_UNLOCK(sc);
 }
 
 static int
@@ -1573,7 +1711,7 @@ offload_open(struct port_info *pi)
 	int adap_up = adapter->open_device_map & PORT_MASK;
 	int err = 0;
 
-	printf("device_map=0x%x\n", adapter->open_device_map); 
+	CTR1(KTR_CXGB, "device_map=0x%x", adapter->open_device_map); 
 	if (atomic_cmpset_int(&adapter->open_device_map,
 		(adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)),
 		(adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0)
@@ -1620,11 +1758,8 @@ offload_close(struct t3cdev *tdev)
 {
 	struct adapter *adapter = tdev2adap(tdev);
 
-	if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) {
+	if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
-		printf("offload_close: DEVMAP_BIT not set\n");
-	
 		return (0);
-	}
 	
 	/* Call back all registered clients */
 	cxgb_remove_clients(tdev);
@@ -1638,7 +1773,6 @@ offload_close(struct t3cdev *tdev)
 		cxgb_down_locked(adapter);
 	else
 		ADAPTER_UNLOCK(adapter);
-	cxgb_offload_deactivate(adapter);
 	return (0);
 }
 
@@ -1680,17 +1814,12 @@ cxgb_init_locked(struct port_info *p)
 		if (err)
 			log(LOG_WARNING,
 			    "Could not initialize offload capabilities\n");
-		else
-			printf("offload opened\n");
 	}
-	cxgb_link_start(p);
-	t3_link_changed(sc, p->port_id);
 	ifp->if_baudrate = p->link_config.speed * 1000000;
 
 	device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id);
 	t3_port_intr_enable(sc, p->port_id);
 
-	callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
 	t3_sge_reset_adapter(sc);
 
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
@@ -1703,10 +1832,10 @@ cxgb_set_rxmode(struct port_info *p)
 	struct t3_rx_mode rm;
 	struct cmac *mac = &p->mac;
 
-	PORT_LOCK_ASSERT_OWNED(p);
-
 	t3_init_rx_mode(&rm, p);
+	mtx_lock(&p->adapter->mdio_lock);
 	t3_mac_set_rx_mode(mac, &rm);
+	mtx_unlock(&p->adapter->mdio_lock);
 }
 
 static void
@@ -1745,7 +1874,6 @@ cxgb_set_mtu(struct port_info *p, int mtu)
 		PORT_LOCK(p);
 		ifp->if_mtu = mtu;
 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-			callout_stop(&p->adapter->cxgb_tick_ch);
 			cxgb_stop_locked(p);
 			cxgb_init_locked(p);
 		}
@@ -1771,19 +1899,18 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
 		error = cxgb_set_mtu(p, ifr->ifr_mtu);
 		break;
 	case SIOCSIFADDR:
-	case SIOCGIFADDR:
 		if (ifa->ifa_addr->sa_family == AF_INET) {
-			PORT_LOCK(p);
 			ifp->if_flags |= IFF_UP;
-			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 
+			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+				PORT_LOCK(p);
 				cxgb_init_locked(p);
+				PORT_UNLOCK(p);
+			}
 			arp_ifinit(ifp, ifa);
-			PORT_UNLOCK(p);
 		} else
 			error = ether_ioctl(ifp, command, data);
 		break;
 	case SIOCSIFFLAGS:
-		callout_drain(&p->adapter->cxgb_tick_ch);
 		PORT_LOCK(p);
 		if (ifp->if_flags & IFF_UP) {
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
@@ -1797,13 +1924,14 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
 		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 			cxgb_stop_locked(p);
 				
-		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-			adapter_t *sc = p->adapter;
-			callout_reset(&sc->cxgb_tick_ch, hz,
-			    cxgb_tick, sc);
-		}
 		PORT_UNLOCK(p);
 		break;
+	case SIOCADDMULTI:
+	case SIOCDELMULTI:
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			cxgb_set_rxmode(p);
+		}
+		break;
 	case SIOCSIFMEDIA:
 	case SIOCGIFMEDIA:
 		error = ifmedia_ioctl(ifp, ifr, &p->media, command);
@@ -1929,7 +2057,7 @@ check_link_status(adapter_t *sc)
 	for (i = 0; i < (sc)->params.nports; ++i) {
 		struct port_info *p = &sc->port[i];
 
-		if (!(p->port_type->caps & SUPPORTED_IRQ)) 
+		if (!(p->phy.caps & SUPPORTED_IRQ)) 
 			t3_link_changed(sc, i);
 		p->ifp->if_baudrate = p->link_config.speed * 1000000;
 	}
@@ -1940,11 +2068,17 @@ check_t3b2_mac(struct adapter *adapter)
 {
 	int i;
 
+	if(adapter->flags & CXGB_SHUTDOWN)
+		return;
+	
 	for_each_port(adapter, i) {
 		struct port_info *p = &adapter->port[i];
 		struct ifnet *ifp = p->ifp;
 		int status;
 		
+		if(adapter->flags & CXGB_SHUTDOWN)
+			return;
+		
 		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 
 			continue;
 		
@@ -1974,26 +2108,12 @@ static void
 cxgb_tick(void *arg)
 {
 	adapter_t *sc = (adapter_t *)arg;
-	int i, running = 0;
 
-	for_each_port(sc, i) {
+	if(sc->flags & CXGB_SHUTDOWN)
-		
-		struct port_info *p = &sc->port[i];
-		struct ifnet *ifp = p->ifp;
-		PORT_LOCK(p);
-
-		if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 
-			running = 1;
-		PORT_UNLOCK(p);
-	}	
-
-	if (running == 0)
 		return;
 
 	taskqueue_enqueue(sc->tq, &sc->tick_task);
-	
+	callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
-	if (sc->open_device_map != 0) 
-		callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
 }
 
 static void
@@ -2002,17 +2122,20 @@ cxgb_tick_handler(void *arg, int count)
 	adapter_t *sc = (adapter_t *)arg;
 	const struct adapter_params *p = &sc->params;
 
+	if(sc->flags & CXGB_SHUTDOWN)
+		return;
+
 	ADAPTER_LOCK(sc);
 	if (p->linkpoll_period)
 		check_link_status(sc);
 
 	/*
-	 * adapter lock can currently only be acquire after the
+	 * adapter lock can currently only be acquired after the
 	 * port lock
 	 */
 	ADAPTER_UNLOCK(sc);
 
-	if (p->rev == T3_REV_B2 && p->nports < 4) 
+	if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 
 		check_t3b2_mac(sc);
 }
 
@@ -2180,7 +2303,7 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 	}
 	case CHELSIO_GET_SGE_CONTEXT: {
 		struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
-		mtx_lock(&sc->sge.reg_lock);
+		mtx_lock_spin(&sc->sge.reg_lock);
 		switch (ecntxt->cntxt_type) {
 		case CNTXT_TYPE_EGRESS:
 			error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
@@ -2202,7 +2325,7 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 			error = EINVAL;
 			break;
 		}
-		mtx_unlock(&sc->sge.reg_lock);
+		mtx_unlock_spin(&sc->sge.reg_lock);
 		break;
 	}
 	case CHELSIO_GET_SGE_DESC: {
@@ -2220,6 +2343,7 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 	case CHELSIO_SET_QSET_PARAMS: {
 		struct qset_params *q;
 		struct ch_qset_params *t = (struct ch_qset_params *)data;
+		int i;
 	
 		if (t->qset_idx >= SGE_QSETS)
 			return (EINVAL);
@@ -2236,6 +2360,18 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 			      MAX_RX_JUMBO_BUFFERS) ||
 		    !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES))
 			return (EINVAL);
+
+		if ((sc->flags & FULL_INIT_DONE) && t->lro > 0)
+			for_each_port(sc, i) {
+				pi = adap2pinfo(sc, i);
+				if (t->qset_idx >= pi->first_qset &&
+				    t->qset_idx < pi->first_qset + pi->nqsets
+#if 0					
+					&& !pi->rx_csum_offload
+#endif					    
+					)
+					return -EINVAL;
+			}
 		if ((sc->flags & FULL_INIT_DONE) &&
 		    (t->rspq_size >= 0 || t->fl_size[0] >= 0 ||
 		     t->fl_size[1] >= 0 || t->txq_size[0] >= 0 ||

sys/dev/cxgb/cxgb_multiq.c

@@ -422,13 +422,16 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush)
 	txq = &qs->txq[TXQ_ETH];
 	
 	mtx_assert(&txq->lock, MA_OWNED);
-	KASSERT(qs->idx == 0, ("invalid qs %d", qs->idx));
 	
  retry:	
 	if (!pi->link_config.link_ok)
 		initerr = ENXIO;
 	else if (qs->qs_flags & QS_EXITING)
 		initerr = ENXIO;
+	else if ((pi->ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
+		initerr = ENXIO;
+	else if ((pi->ifp->if_flags & IFF_UP) == 0)
+		initerr = ENXIO;
 	else if (immpkt) {
 
 		if (!buf_ring_empty(&txq->txq_mr)) 
@@ -690,15 +693,15 @@ cxgb_pcpu_shutdown_threads(struct adapter *sc)
 	int i, j;
 	int nqsets;
 
+	for (i = 0; i < sc->params.nports; i++) {
+		struct port_info *pi = &sc->port[i];
+		int first = pi->first_qset;
+
 #ifdef IFNET_MULTIQUEUE
 	nqsets = pi->nqsets;
 #else
 	nqsets = 1;
 #endif	
-	
-	for (i = 0; i < sc->params.nports; i++) {
-		struct port_info *pi = &sc->port[i];
-		int first = pi->first_qset;
 		for (j = 0; j < nqsets; j++) {
 			struct sge_qset *qs = &sc->sge.qs[first + j];
 

sys/dev/cxgb/cxgb_offload.c

@@ -105,16 +105,12 @@ cxgb_register_client(struct cxgb_client *client)
 	TAILQ_INSERT_TAIL(&client_list, client, client_entry);
 
 	if (client->add) {
-		printf("client->add set\n");
-		
 		TAILQ_FOREACH(tdev, &ofld_dev_list, entry) {
 			if (offload_activated(tdev)) {
-				printf("calling add=%p on %p\n",
-				    client->add, tdev);
-				
 				client->add(tdev);
 			} else
-				printf("%p not activated\n", tdev);
+				CTR1(KTR_CXGB,
+				    "cxgb_register_client: %p not activated", tdev);
 			
 		}
 	}
@@ -270,11 +266,10 @@ cxgb_ulp_iscsi_ctl(adapter_t *adapter, unsigned int req, void *data)
 				     t3_read_reg(adapter, A_PM1_TX_CFG) >> 17);
 		/* on rx, the iscsi pdu has to be < rx page size and the
 		   whole pdu + cpl headers has to fit into one sge buffer */
-		uiip->max_rxsz =
+		/* also check the max rx data length programmed in TP */
-		    (unsigned int)min(adapter->params.tp.rx_pg_size,
+		uiip->max_rxsz = min(uiip->max_rxsz,
-			(adapter->sge.qs[0].fl[1].buf_size -
+				     ((t3_read_reg(adapter, A_TP_PARA_REG2))
-			    sizeof(struct cpl_rx_data) * 2 -
+					>> S_MAXRXDATA) & M_MAXRXDATA);
-			    sizeof(struct cpl_rx_data_ddp)) );
 		break;
 	case ULP_ISCSI_SET_PARAMS:
 		t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask);
@@ -297,25 +292,24 @@ cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data)
 	case RDMA_GET_PARAMS: {
 		struct rdma_info *req = data;
 
-		req->udbell_physbase = rman_get_start(adapter->regs_res);
+		req->udbell_physbase = rman_get_start(adapter->udbs_res);
-		req->udbell_len = rman_get_size(adapter->regs_res);
+		req->udbell_len = rman_get_size(adapter->udbs_res);
 		req->tpt_base = t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT);
 		req->tpt_top  = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT);
 		req->pbl_base = t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT);
 		req->pbl_top  = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT);
 		req->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT);
 		req->rqt_top  = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT);
-		req->kdb_addr = (void *)(rman_get_start(adapter->regs_res) + A_SG_KDOORBELL);
+		req->kdb_addr =  (void *)((unsigned long)rman_get_virtual(adapter->regs_res) + A_SG_KDOORBELL);		break;
-		break;
 	}
 	case RDMA_CQ_OP: {
 		struct rdma_cq_op *req = data;
 
 		/* may be called in any context */
-		mtx_lock(&adapter->sge.reg_lock);
+		mtx_lock_spin(&adapter->sge.reg_lock);
 		ret = t3_sge_cqcntxt_op(adapter, req->id, req->op,
 					req->credits);
-		mtx_unlock(&adapter->sge.reg_lock);
+		mtx_unlock_spin(&adapter->sge.reg_lock);
 		break;
 	}
 	case RDMA_GET_MEM: {
@@ -341,28 +335,28 @@ cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data)
 	case RDMA_CQ_SETUP: {
 		struct rdma_cq_setup *req = data;
 
-		mtx_lock(&adapter->sge.reg_lock);
+		mtx_lock_spin(&adapter->sge.reg_lock);
 		ret = t3_sge_init_cqcntxt(adapter, req->id, req->base_addr,
 					  req->size, ASYNC_NOTIF_RSPQ,
 					  req->ovfl_mode, req->credits,
 					  req->credit_thres);
-		mtx_unlock(&adapter->sge.reg_lock);
+		mtx_unlock_spin(&adapter->sge.reg_lock);
 		break;
 	}
 	case RDMA_CQ_DISABLE:
-		mtx_lock(&adapter->sge.reg_lock);
+		mtx_lock_spin(&adapter->sge.reg_lock);
 		ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data);
-		mtx_unlock(&adapter->sge.reg_lock);
+		mtx_unlock_spin(&adapter->sge.reg_lock);
 		break;
 	case RDMA_CTRL_QP_SETUP: {
 		struct rdma_ctrlqp_setup *req = data;
 
-		mtx_lock(&adapter->sge.reg_lock);
+		mtx_lock_spin(&adapter->sge.reg_lock);
 		ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0,
 					 SGE_CNTXT_RDMA, ASYNC_NOTIF_RSPQ,
 					 req->base_addr, req->size,
 					 FW_RI_TID_START, 1, 0);
-		mtx_unlock(&adapter->sge.reg_lock);
+		mtx_unlock_spin(&adapter->sge.reg_lock);
 		break;
 	}
 	default:
@@ -380,6 +374,8 @@ cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data)
 	struct iff_mac *iffmacp;
 	struct ddp_params *ddpp;
 	struct adap_ports *ports;
+	struct ofld_page_info *rx_page_info;
+	struct tp_params *tp = &adapter->params.tp;
 	int port;
 
 	switch (req) {
@@ -444,6 +440,11 @@ cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data)
 	case FAILOVER_CLEAR:
 		t3_failover_clear(adapter);
 		break;
+	case GET_RX_PAGE_INFO:
+		rx_page_info = data;
+		rx_page_info->page_size = tp->rx_pg_size;
+		rx_page_info->num = tp->rx_num_pgs;
+		break;
 	case ULP_ISCSI_GET_PARAMS:
 	case ULP_ISCSI_SET_PARAMS:
 		if (!offload_running(adapter))
@@ -472,8 +473,6 @@ cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data)
 static int
 rx_offload_blackhole(struct t3cdev *dev, struct mbuf **m, int n)
 {
-	CH_ERR(tdev2adap(dev), "%d unexpected offload packets, first data 0x%x\n",
-	    n, *mtod(m[0], uint32_t *));
 	while (n--)
 		m_freem(m[n]);
 	return 0;
@@ -629,7 +628,7 @@ cxgb_remove_tid(struct t3cdev *tdev, void *ctx, unsigned int tid)
 		m = m_get(M_NOWAIT, MT_DATA);
 		if (__predict_true(m != NULL)) {
 			mk_tid_release(m, tid);
-			printf("sending tid release\n");
+			CTR1(KTR_CXGB, "releasing tid=%u", tid);
 			
 			cxgb_ofld_send(tdev, m);
 			t->tid_tab[tid].ctx = NULL;
@@ -707,6 +706,19 @@ do_l2t_write_rpl(struct t3cdev *dev, struct mbuf *m)
 	return CPL_RET_BUF_DONE;
 }
 
+static int
+do_rte_write_rpl(struct t3cdev *dev, struct mbuf *m)
+{
+	struct cpl_rte_write_rpl *rpl = cplhdr(m);
+
+	if (rpl->status != CPL_ERR_NONE)
+		log(LOG_ERR,
+		       "Unexpected L2T_WRITE_RPL status %u for entry %u\n",
+		       rpl->status, GET_TID(rpl));
+
+	return CPL_RET_BUF_DONE;
+}
+
 static int
 do_act_open_rpl(struct t3cdev *dev, struct mbuf *m)
 {
@@ -903,7 +915,7 @@ cxgb_arp_update_event(void *unused, struct rtentry *rt0,
     uint8_t *enaddr, struct sockaddr *sa)
 {
 
-	if (TOEDEV(rt0->rt_ifp) == NULL)
+	if (!is_offloading(rt0->rt_ifp))
 		return;
 
 	RT_ADDREF(rt0);
@@ -918,15 +930,21 @@ static void
 cxgb_redirect_event(void *unused, int event, struct rtentry *rt0,
     struct rtentry *rt1, struct sockaddr *sa)
 {
-	struct toedev *tdev0, *tdev1;
-
 	/* 
 	 * ignore events on non-offloaded interfaces
 	 */
-	tdev0 = TOEDEV(rt0->rt_ifp);
+	if (!is_offloading(rt0->rt_ifp))
-	tdev1 = TOEDEV(rt1->rt_ifp);
-	if (tdev0 == NULL && tdev1 == NULL)
 		return;
+
+	/*
+	 * Cannot redirect to non-offload device.
+	 */
+	if (!is_offloading(rt1->rt_ifp)) {
+		log(LOG_WARNING, "%s: Redirect to non-offload"
+		    "device ignored.\n", __FUNCTION__);
+		return;
+	}
+
         /*
 	 * avoid LORs by dropping the route lock but keeping a reference
 	 * 
@@ -952,14 +970,15 @@ static int
 do_bad_cpl(struct t3cdev *dev, struct mbuf *m)
 {
 	log(LOG_ERR, "%s: received bad CPL command 0x%x\n", dev->name,
-	    *mtod(m, uint32_t *));
+	    0xFF & *mtod(m, uint32_t *));
+	kdb_backtrace();
 	return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
 }
 
 /*
  * Handlers for each CPL opcode
  */
-static cpl_handler_func cpl_handlers[NUM_CPL_CMDS];
+static cpl_handler_func cpl_handlers[256];
 
 /*
  * Add a new handler to the CPL dispatch table.  A NULL handler may be supplied
@@ -1052,7 +1071,7 @@ void
 cxgb_neigh_update(struct rtentry *rt, uint8_t *enaddr, struct sockaddr *sa)
 {
 
-	if (is_offloading(rt->rt_ifp)) {
+	if (rt->rt_ifp && is_offloading(rt->rt_ifp) && (rt->rt_ifp->if_flags & IFCAP_TOE)) {
 		struct t3cdev *tdev = T3CDEV(rt->rt_ifp);
 
 		PANIC_IF(!tdev);
@@ -1159,7 +1178,6 @@ cxgb_free_mem(void *addr)
 	free(addr, M_CXGB);
 }
 
-
 /*
  * Allocate and initialize the TID tables.  Returns 0 on success.
  */
@@ -1208,6 +1226,8 @@ init_tid_tabs(struct tid_info *t, unsigned int ntids,
 static void
 free_tid_maps(struct tid_info *t)
 {
+	mtx_destroy(&t->stid_lock);
+	mtx_destroy(&t->atid_lock);
 	cxgb_free_mem(t->tid_tab);
 }
 
@@ -1227,11 +1247,6 @@ remove_adapter(adapter_t *adap)
 	rw_wunlock(&adapter_list_lock);
 }
 
-/*
- * XXX
- */
-#define t3_free_l2t(...)
-
 int
 cxgb_offload_activate(struct adapter *adapter)
 {
@@ -1265,8 +1280,6 @@ cxgb_offload_activate(struct adapter *adapter)
 		device_printf(adapter->dev, "%s: t3_init_l2t failed\n", __FUNCTION__);
 		goto out_free;
 	}
-	
-
 	natids = min(tid_range.num / 2, MAX_ATIDS);
 	err = init_tid_tabs(&t->tid_maps, tid_range.num, natids,
 			    stid_range.num, ATID_BASE, stid_range.base);
@@ -1295,9 +1308,10 @@ cxgb_offload_activate(struct adapter *adapter)
 			log(LOG_ERR, "Unable to set offload capabilities\n");
 #endif
 	}
-	printf("adding adapter %p\n", adapter); 
+	CTR1(KTR_CXGB, "adding adapter %p", adapter); 
 	add_adapter(adapter);
 	device_printf(adapter->dev, "offload started\n");
+	adapter->flags |= CXGB_OFLD_INIT;
 #if 0
 	printf("failing as test\n");
 	return (ENOMEM);
@@ -1330,6 +1344,7 @@ cxgb_offload_deactivate(struct adapter *adapter)
 	T3C_DATA(tdev) = NULL;
 	t3_free_l2t(L2DATA(tdev));
 	L2DATA(tdev) = NULL;
+	mtx_destroy(&t->tid_release_lock);
 	free(t, M_CXGB);
 }
 
@@ -1353,6 +1368,26 @@ unregister_tdev(struct t3cdev *tdev)
 	mtx_unlock(&cxgb_db_lock);	
 }
 
+static __inline int
+adap2type(struct adapter *adapter) 
+{ 
+        int type = 0; 
+ 
+        switch (adapter->params.rev) { 
+        case T3_REV_A: 
+                type = T3A; 
+                break; 
+        case T3_REV_B: 
+        case T3_REV_B2: 
+                type = T3B; 
+                break; 
+        case T3_REV_C: 
+                type = T3C; 
+                break; 
+        } 
+        return type; 
+} 
+                 
 void
 cxgb_adapter_ofld(struct adapter *adapter)
 {
@@ -1361,8 +1396,7 @@ cxgb_adapter_ofld(struct adapter *adapter)
 	cxgb_set_dummy_ops(tdev);
 	tdev->send = t3_offload_tx;
 	tdev->ctl = cxgb_offload_ctl;
-	tdev->type = adapter->params.rev == 0 ?
+	tdev->type = adap2type(adapter);
-		     T3A : T3B;
 	
 	register_tdev(tdev);
 #if 0	
@@ -1398,10 +1432,11 @@ cxgb_offload_init(void)
 	TAILQ_INIT(&ofld_dev_list);
 	TAILQ_INIT(&adapter_list);
 	
-	for (i = 0; i < NUM_CPL_CMDS; ++i)
+	for (i = 0; i < 0x100; ++i)
 		cpl_handlers[i] = do_bad_cpl;
 	
 	t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl);
+	t3_register_cpl_handler(CPL_RTE_WRITE_RPL, do_rte_write_rpl);
 	t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
 	t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl);
 	t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl);
@@ -1425,6 +1460,8 @@ cxgb_offload_init(void)
 	t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl);
 	t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl);
 	t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_GET_TCB_RPL, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_SET_TCB_RPL, do_hwtid_rpl);
 	
 	EVENTHANDLER_REGISTER(route_arp_update_event, cxgb_arp_update_event,
 	    NULL, EVENTHANDLER_PRI_ANY);

sys/dev/cxgb/cxgb_osdep.h

@@ -83,6 +83,9 @@ struct t3_mbuf_hdr {
 #define m_set_socket(m, a) ((m)->m_pkthdr.header = (a))
 #define m_get_socket(m) ((m)->m_pkthdr.header)
 
+#define	KTR_CXGB	KTR_SPARE2
+void cxgb_log_tcb(struct adapter *sc, unsigned int tid);
+
 #define MT_DONTFREE  128
 
 #if __FreeBSD_version > 700030
@@ -338,13 +341,14 @@ static const int debug_flags = DBG_RX;
 #define DBG(...)
 #endif
 
+#include <sys/syslog.h>
+
 #define promisc_rx_mode(rm)  ((rm)->port->ifp->if_flags & IFF_PROMISC) 
 #define allmulti_rx_mode(rm) ((rm)->port->ifp->if_flags & IFF_ALLMULTI) 
 
-#define CH_ERR(adap, fmt, ...)device_printf(adap->dev, fmt, ##__VA_ARGS__);
+#define CH_ERR(adap, fmt, ...) log(LOG_ERR, fmt, ##__VA_ARGS__)
-
+#define CH_WARN(adap, fmt, ...)	log(LOG_WARNING, fmt, ##__VA_ARGS__)
-#define CH_WARN(adap, fmt, ...)	device_printf(adap->dev, fmt, ##__VA_ARGS__)
+#define CH_ALERT(adap, fmt, ...) log(LOG_ALERT, fmt, ##__VA_ARGS__)
-#define CH_ALERT(adap, fmt, ...) device_printf(adap->dev, fmt, ##__VA_ARGS__)
 
 #define t3_os_sleep(x) DELAY((x) * 1000)
 
@@ -370,7 +374,8 @@ static const int debug_flags = DBG_RX;
 #define MII_CTRL1000		MII_100T2CR
 
 #define ADVERTISE_PAUSE_CAP	ANAR_FC
-#define ADVERTISE_PAUSE_ASYM	0x0800
+#define ADVERTISE_PAUSE_ASYM	ANAR_X_PAUSE_ASYM
+#define ADVERTISE_PAUSE		ANAR_X_PAUSE_SYM
 #define ADVERTISE_1000HALF	ANAR_X_HD
 #define ADVERTISE_1000FULL	ANAR_X_FD
 #define ADVERTISE_10FULL	ANAR_10_FD
@@ -378,6 +383,13 @@ static const int debug_flags = DBG_RX;
 #define ADVERTISE_100FULL	ANAR_TX_FD
 #define ADVERTISE_100HALF	ANAR_TX
 
+
+#define ADVERTISE_1000XHALF	ANAR_X_HD
+#define ADVERTISE_1000XFULL	ANAR_X_FD
+#define ADVERTISE_1000XPSE_ASYM	ANAR_X_PAUSE_ASYM
+#define ADVERTISE_1000XPAUSE	ANAR_X_PAUSE_SYM
+
+
 /* Standard PCI Extended Capaibilities definitions */
 #define PCI_CAP_ID_VPD	0x03
 #define PCI_VPD_ADDR	2
@@ -399,23 +411,26 @@ static const int debug_flags = DBG_RX;
 #define udelay(x) DELAY(x)
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 #define le32_to_cpu(x) le32toh(x)
+#define le16_to_cpu(x) le16toh(x)
 #define cpu_to_le32(x) htole32(x)
 #define swab32(x) bswap32(x)
 #define simple_strtoul strtoul
 
 
-typedef uint8_t u8;
+#ifndef LINUX_TYPES_DEFINED
-typedef uint16_t u16;
+typedef uint8_t 	u8;
-typedef uint32_t u32;
+typedef uint16_t 	u16;
-typedef uint64_t u64;
+typedef uint32_t 	u32;
+typedef uint64_t 	u64;
  
-typedef uint8_t       __u8;
+typedef uint8_t		__u8;
-typedef uint16_t __u16;
+typedef uint16_t	__u16;
-typedef uint32_t __u32;
+typedef uint32_t	__u32;
-typedef uint8_t __be8;
+typedef uint8_t		__be8;
-typedef uint16_t __be16;
+typedef uint16_t	__be16;
-typedef uint32_t __be32;
+typedef uint32_t	__be32;
-typedef uint64_t __be64;
+typedef uint64_t	__be64;
+#endif
 
 
 #if BYTE_ORDER == BIG_ENDIAN

sys/dev/cxgb/cxgb_sge.c

@@ -73,11 +73,16 @@ __FBSDID("$FreeBSD$");
 #endif
 
 int      txq_fills = 0;
-static int recycle_enable = 1;
+/*
+ * XXX don't re-enable this until TOE stops assuming
+ * we have an m_ext
+ */
+static int recycle_enable = 0;
 extern int cxgb_txq_buf_ring_size;
 int cxgb_cached_allocations;
 int cxgb_cached;
-int cxgb_ext_freed;
+int cxgb_ext_freed = 0;
+int cxgb_ext_inited = 0;
 extern int cxgb_use_16k_clusters;
 extern int cxgb_pcpu_cache_enable;
 
@@ -247,7 +252,7 @@ t3_sge_init(adapter_t *adap, struct sge_params *p)
 	ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */
 
 	ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
-	       F_CQCRDTCTRL |
+	       F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
 	       V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
 	       V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
 #if SGE_NUM_GENBITS == 1
@@ -256,7 +261,6 @@ t3_sge_init(adapter_t *adap, struct sge_params *p)
 	if (adap->params.rev > 0) {
 		if (!(adap->flags & (USING_MSIX | USING_MSI)))
 			ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
-		ctrl |= F_CQCRDTCTRL | F_AVOIDCQOVFL;
 	}
 	t3_write_reg(adap, A_SG_CONTROL, ctrl);
 	t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
@@ -264,7 +268,8 @@ t3_sge_init(adapter_t *adap, struct sge_params *p)
 	t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
 	t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
 		     V_TIMEOUT(200 * core_ticks_per_usec(adap)));
-	t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, 1000);
+	t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
+		     adap->params.rev < T3_REV_C ? 1000 : 500);
 	t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
 	t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
 	t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
@@ -293,13 +298,14 @@ sgl_len(unsigned int n)
  *	Return a packet containing the immediate data of the given response.
  */
 static int
-get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl, uint32_t flags)
+get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m)
 {
 
 	m->m_len = m->m_pkthdr.len = IMMED_PKT_SIZE;
+	m->m_ext.ext_buf = NULL;
+	m->m_ext.ext_type = 0;
 	memcpy(mtod(m, uint8_t *), resp->imm_data, IMMED_PKT_SIZE); 
 	return (0);	
-	
 }
 
 static __inline u_int
@@ -308,14 +314,33 @@ flits_to_desc(u_int n)
 	return (flit_desc_map[n]);
 }
 
+#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
+		    F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
+		    V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
+		    F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
+		    F_HIRCQPARITYERROR)
+#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
+#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
+		      F_RSPQDISABLED)
+
+/**
+ *	t3_sge_err_intr_handler - SGE async event interrupt handler
+ *	@adapter: the adapter
+ *
+ *	Interrupt handler for SGE asynchronous (non-data) events.
+ */
 void
 t3_sge_err_intr_handler(adapter_t *adapter)
 {
 	unsigned int v, status;
 
-	
 	status = t3_read_reg(adapter, A_SG_INT_CAUSE);
-	
+	if (status & SGE_PARERR)
+		CH_ALERT(adapter, "SGE parity error (0x%x)\n",
+			 status & SGE_PARERR);
+	if (status & SGE_FRAMINGERR)
+		CH_ALERT(adapter, "SGE framing error (0x%x)\n",
+			 status & SGE_FRAMINGERR);
 	if (status & F_RSPQCREDITOVERFOW)
 		CH_ALERT(adapter, "SGE response queue credit overflow\n");
 
@@ -328,7 +353,7 @@ t3_sge_err_intr_handler(adapter_t *adapter)
 	}
 
 	t3_write_reg(adapter, A_SG_INT_CAUSE, status);
-	if (status & (F_RSPQCREDITOVERFOW | F_RSPQDISABLED))
+	if (status & SGE_FATALERR)
 		t3_fatal_err(adapter);
 }
 
@@ -343,8 +368,6 @@ t3_sge_prep(adapter_t *adap, struct sge_params *p)
 	for (i = 0; i < SGE_QSETS; ++i) {
 		struct qset_params *q = p->qset + i;
 
-		q->polling = adap->params.rev > 0;
-
 		if (adap->params.nports > 2) {
 			q->coalesce_nsecs = 50000;
 		} else {
@@ -354,6 +377,7 @@ t3_sge_prep(adapter_t *adap, struct sge_params *p)
 			q->coalesce_nsecs = 5000;
 #endif			
 		}
+		q->polling = adap->params.rev > 0;
 		q->rspq_size = RSPQ_Q_SIZE;
 		q->fl_size = FL_Q_SIZE;
 		q->jumbo_size = JUMBO_Q_SIZE;
@@ -473,7 +497,7 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
 	struct rx_desc *d = &q->desc[q->pidx];
 	struct refill_fl_cb_arg cb_arg;
 	caddr_t cl;
-	int err;
+	int err, count = 0;
 	int header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t);
 	
 	cb_arg.error = 0;
@@ -527,10 +551,12 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
 			d = q->desc;
 		}
 		q->credits++;
+		count++;
 	}
 
 done:
-	t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
+	if (count)
+		t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
 }
 
 
@@ -776,14 +802,6 @@ t3_sge_init_port(struct port_info *pi)
 void
 t3_sge_deinit_sw(adapter_t *sc)
 {
-	int i;
-	
-	callout_drain(&sc->sge_timer_ch);
-	if (sc->tq) 
-		taskqueue_drain(sc->tq, &sc->slow_intr_task);
-	for (i = 0; i < sc->params.nports; i++) 
-		if (sc->port[i].tq != NULL)
-			taskqueue_drain(sc->port[i].tq, &sc->port[i].timer_reclaim_task);
 
 	mi_deinit();
 }
@@ -909,8 +927,8 @@ txq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs)
 	 */
 	txqs->gen = txq->gen;
 	txq->unacked += ndesc;
-	txqs->compl = (txq->unacked & 8) << (S_WR_COMPL - 3);
+	txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5);
-	txq->unacked &= 7;
+	txq->unacked &= 31;
 	txqs->pidx = txq->pidx;
 	txq->pidx += ndesc;
 #ifdef INVARIANTS
@@ -1209,7 +1227,6 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
 	struct mbuf_iovec *mi;
 		
 	DPRINTF("t3_encap cpu=%d ", curcpu);
-	KASSERT(qs->idx == 0, ("invalid qs %d", qs->idx));
 
 	mi = NULL;
 	pi = qs->port;
@@ -1310,6 +1327,7 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
 		undersized = (((tmpmi->mi_len < TCPPKTHDRSIZE) &&
 			(m0->m_flags & M_VLANTAG)) ||
 		    (tmpmi->mi_len < TCPPKTHDRSIZE - ETHER_VLAN_ENCAP_LEN));
+
 		if (__predict_false(undersized)) {
 			pkthdr = tmp;
 			dump_mi(mi);
@@ -1550,7 +1568,6 @@ again:	reclaim_completed_tx_imm(q);
 		if (ret == 1) {
 			mtx_unlock(&q->lock);
 			log(LOG_ERR, "no desc available\n");
-			
 			return (ENOSPC);
 		}
 		goto again;
@@ -1610,6 +1627,7 @@ again:	reclaim_completed_tx_imm(q);
 		q->stops++;
 	}
 	mtx_unlock(&q->lock);
+	wmb();
 	t3_write_reg(adap, A_SG_KDOORBELL,
 		     F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
 }
@@ -1648,9 +1666,9 @@ t3_free_qset(adapter_t *sc, struct sge_qset *q)
 		}
 	for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
 		if (q->fl[i].desc) {
-			mtx_lock(&sc->sge.reg_lock);
+			mtx_lock_spin(&sc->sge.reg_lock);
 			t3_sge_disable_fl(sc, q->fl[i].cntxt_id);
-			mtx_unlock(&sc->sge.reg_lock);
+			mtx_unlock_spin(&sc->sge.reg_lock);
 			bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map);
 			bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc,
 					q->fl[i].desc_map);
@@ -1665,9 +1683,9 @@ t3_free_qset(adapter_t *sc, struct sge_qset *q)
 
 	for (i = 0; i < SGE_TXQ_PER_SET; i++) {
 		if (q->txq[i].desc) {
-			mtx_lock(&sc->sge.reg_lock);
+			mtx_lock_spin(&sc->sge.reg_lock);
 			t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0);
-			mtx_unlock(&sc->sge.reg_lock);
+			mtx_unlock_spin(&sc->sge.reg_lock);
 			bus_dmamap_unload(q->txq[i].desc_tag,
 					q->txq[i].desc_map);
 			bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc,
@@ -1682,9 +1700,9 @@ t3_free_qset(adapter_t *sc, struct sge_qset *q)
 	}
 
 	if (q->rspq.desc) {
-		mtx_lock(&sc->sge.reg_lock);
+		mtx_lock_spin(&sc->sge.reg_lock);
 		t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id);
-		mtx_unlock(&sc->sge.reg_lock);
+		mtx_unlock_spin(&sc->sge.reg_lock);
 		
 		bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map);
 		bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc,
@@ -1893,7 +1911,7 @@ write_ofld_wr(adapter_t *adap, struct mbuf *m,
 	struct tx_desc *d = &q->desc[pidx];
 	struct txq_state txqs;
 	
-	if (immediate(m) && segs == NULL) {
+	if (immediate(m) && nsegs == 0) {
 		write_imm(d, m, m->m_len, gen);
 		return;
 	}
@@ -1927,18 +1945,25 @@ static __inline unsigned int
 calc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs)
 {
 	unsigned int flits, cnt = 0;
+	int ndescs;
 
-
+	if (m->m_len <= WR_LEN && nsegs == 0)
-	if (m->m_len <= WR_LEN)
+		return (1);                 /* packet fits as immediate data */
-		return 1;                 /* packet fits as immediate data */
 
 	if (m->m_flags & M_IOVEC)
 		cnt = mtomv(m)->mv_count;
+	else
+		cnt = nsegs;
 
 	/* headers */
-	flits = ((uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *)) / 8;
+	flits = m->m_len / 8;
 
-	return flits_to_desc(flits + sgl_len(cnt));
+	ndescs = flits_to_desc(flits + sgl_len(cnt));
+
+	CTR4(KTR_CXGB, "flits=%d sgl_len=%d nsegs=%d ndescs=%d",
+	    flits, sgl_len(cnt), nsegs, ndescs);
+
+	return (ndescs);
 }
 
 /**
@@ -1998,7 +2023,6 @@ again:	reclaim_completed_tx_(q, 16);
 
 	write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs);
 	check_ring_tx_db(adap, q);
-
 	return (0);
 }
 
@@ -2058,6 +2082,7 @@ again:	cleaned = reclaim_completed_tx_(q, 16);
 	set_bit(TXQ_RUNNING, &q->flags);
 	set_bit(TXQ_LAST_PKT_DB, &q->flags);
 #endif
+	wmb();
 	t3_write_reg(adap, A_SG_KDOORBELL,
 		     F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
 }
@@ -2300,7 +2325,7 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
 #endif
 	q->lro.enabled = lro_default;
 
-	mtx_lock(&sc->sge.reg_lock);
+	mtx_lock_spin(&sc->sge.reg_lock);
 	ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx,
 				   q->rspq.phys_addr, q->rspq.size,
 				   q->fl[0].buf_size, 1, 0);
@@ -2356,7 +2381,7 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
 	    device_get_unit(sc->dev), irq_vec_idx);
 	MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF);
 	
-	mtx_unlock(&sc->sge.reg_lock);
+	mtx_unlock_spin(&sc->sge.reg_lock);
 	t3_update_qset_coalesce(q, p);
 	q->port = pi;
 	
@@ -2370,7 +2395,7 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
 	return (0);
 
 err_unlock:
-	mtx_unlock(&sc->sge.reg_lock);
+	mtx_unlock_spin(&sc->sge.reg_lock);
 err:	
 	t3_free_qset(sc, q);
 
@@ -2419,17 +2444,17 @@ t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad)
 }
 
 static void
-ext_free_handler(void *cl, void * arg)
+ext_free_handler(void *arg1, void * arg2)
 {
-	uintptr_t type = (uintptr_t)arg;
+	uintptr_t type = (uintptr_t)arg2;
 	uma_zone_t zone;
 	struct mbuf *m;
 
-	m = cl;
+	m = arg1;
 	zone = m_getzonefromtype(type);
 	m->m_ext.ext_type = (int)type;
 	cxgb_ext_freed++;
-	cxgb_cache_put(zone, cl);
+	cxgb_cache_put(zone, m);
 }
 
 static void
@@ -2444,6 +2469,7 @@ init_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone)
 	bzero(cl, header_size);
 	m = (struct mbuf *)cl;
 	
+	cxgb_ext_inited++;
 	SLIST_INIT(&m->m_pkthdr.tags);
 	m->m_type = MT_DATA;
 	m->m_flags = flags | M_NOFREE | M_EXT;
@@ -2721,9 +2747,30 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 		eth = (r->rss_hdr.opcode == CPL_RX_PKT);
 		
 		if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) {
-			/* XXX */
+			struct mbuf *m;
-			printf("async notification\n");
 
+			if (cxgb_debug)
+				printf("async notification\n");
+
+			if (rspq->rspq_mh.mh_head == NULL) {
+				rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA);
+				m = rspq->rspq_mh.mh_head;
+			} else {
+				m = m_gethdr(M_DONTWAIT, MT_DATA);
+			}
+
+			/* XXX m is lost here if rspq->rspq_mbuf is not NULL */
+
+			if (m == NULL)
+				goto no_mem;
+
+                        memcpy(mtod(m, char *), r, AN_PKT_SIZE);
+			m->m_len = m->m_pkthdr.len = AN_PKT_SIZE;
+                        *mtod(m, char *) = CPL_ASYNC_NOTIF;
+			rss_csum = htonl(CPL_ASYNC_NOTIF << 24);
+			eop = 1;
+                        rspq->async_notif++;
+			goto skip;
 		} else if  (flags & F_RSPD_IMM_DATA_VALID) {
 			struct mbuf *m = NULL;
 
@@ -2734,16 +2781,13 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
                         else 
 				m = m_gethdr(M_DONTWAIT, MT_DATA);
 
-			/*
-			 * XXX revisit me
-			 */
 			if (rspq->rspq_mh.mh_head == NULL &&  m == NULL) {	
+		no_mem:
 				rspq->next_holdoff = NOMEM_INTR_DELAY;
 				budget_left--;
 				break;
 			}
-			get_imm_packet(adap, r, rspq->rspq_mh.mh_head, m, flags);
+			get_imm_packet(adap, r, rspq->rspq_mh.mh_head);
-			
 			eop = 1;
 			rspq->imm_data++;
 		} else if (r->len_cq) {
@@ -2753,16 +2797,16 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 			eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r);
 #else
 			eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r);
-#ifdef IFNET_MULTIQUEUE
-			rspq->rspq_mbuf->m_pkthdr.rss_hash = rss_hash;
 #endif
+#ifdef IFNET_MULTIQUEUE
+			rspq->rspq_mh.mh_head->m_pkthdr.rss_hash = rss_hash;
 #endif			
 			ethpad = 2;
 		} else {
 			DPRINTF("pure response\n");
 			rspq->pure_rsps++;
 		}
-		
+	skip:
 		if (flags & RSPD_CTRL_MASK) {
 			sleeping |= flags & RSPD_GTS_MASK;
 			handle_rsp_cntrl_info(qs, flags);
@@ -2788,6 +2832,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 			 */
 			m_set_priority(rspq->rspq_mh.mh_head, rss_hash);
 
+			
 			ngathered = rx_offload(&adap->tdev, rspq,
 			    rspq->rspq_mh.mh_head, offload_mbufs, ngathered);
 			rspq->rspq_mh.mh_head = NULL;
@@ -2988,12 +3033,8 @@ t3_dump_rspq(SYSCTL_HANDLER_ARGS)
 	return (err);
 }	
 
-
-/* 
- * broken by recent mbuf changes 
- */ 
 static int
-t3_dump_txq(SYSCTL_HANDLER_ARGS)
+t3_dump_txq_eth(SYSCTL_HANDLER_ARGS)
 {
 	struct sge_txq *txq;
 	struct sge_qset *qs;
@@ -3022,7 +3063,7 @@ t3_dump_txq(SYSCTL_HANDLER_ARGS)
 		txq->txq_dump_start = 0;
 		return (EINVAL);
 	}
-	err = t3_sge_read_ecntxt(qs->port->adapter, txq->cntxt_id, data);
+	err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data);
 	if (err)
 		return (err);
 	
@@ -3066,6 +3107,67 @@ t3_dump_txq(SYSCTL_HANDLER_ARGS)
 	return (err);
 }
 
+static int
+t3_dump_txq_ctrl(SYSCTL_HANDLER_ARGS)
+{
+	struct sge_txq *txq;
+	struct sge_qset *qs;
+	int i, j, err, dump_end;
+	static int multiplier = 1;
+	struct sbuf *sb;
+	struct tx_desc *txd;
+	uint32_t *WR, wr_hi, wr_lo, gen;
+	
+	txq = arg1;
+	qs = txq_to_qset(txq, TXQ_CTRL);
+	if (txq->txq_dump_count == 0) {
+		return (0);
+	}
+	if (txq->txq_dump_count > 256) {
+		log(LOG_WARNING,
+		    "dump count is too large %d\n", txq->txq_dump_count);
+		txq->txq_dump_count = 1;
+		return (EINVAL);
+	}
+	if (txq->txq_dump_start > 255) {
+		log(LOG_WARNING,
+		    "dump start of %d is greater than queue size\n",
+		    txq->txq_dump_start);
+		txq->txq_dump_start = 0;
+		return (EINVAL);
+	}
+
+retry_sbufops:
+	sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN);
+	sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx,
+	    txq->txq_dump_start,
+	    (txq->txq_dump_start + txq->txq_dump_count) & 255);
+
+	dump_end = txq->txq_dump_start + txq->txq_dump_count;
+	for (i = txq->txq_dump_start; i < dump_end; i++) {
+		txd = &txq->desc[i & (255)];
+		WR = (uint32_t *)txd->flit;
+		wr_hi = ntohl(WR[0]);
+		wr_lo = ntohl(WR[1]);		
+		gen = G_WR_GEN(wr_lo);
+		
+		sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n",
+		    wr_hi, wr_lo, gen);
+		for (j = 2; j < 30; j += 4) 
+			sbuf_printf(sb, "\t%08x %08x %08x %08x \n",
+			    WR[j], WR[j + 1], WR[j + 2], WR[j + 3]);
+
+	}
+	if (sbuf_overflowed(sb)) {
+		sbuf_delete(sb);
+		multiplier++;
+		goto retry_sbufops;
+	}
+	sbuf_finish(sb);
+	err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
+	sbuf_delete(sb);
+	return (err);
+}
 
 static int
 t3_lro_enable(SYSCTL_HANDLER_ARGS)
@@ -3162,7 +3264,10 @@ t3_add_attach_sysctls(adapter_t *sc)
 	    CTLTYPE_INT|CTLFLAG_RW, sc,
 	    0, t3_lro_enable,
 	    "I", "enable large receive offload");
-
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, 
+	    "hw_revision",
+	    CTLFLAG_RD, &sc->params.rev,
+	    0, "chip model");
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, 
 	    "enable_debug",
 	    CTLFLAG_RW, &cxgb_debug,
@@ -3190,6 +3295,10 @@ t3_add_attach_sysctls(adapter_t *sc)
 	    "ext_freed",
 	    CTLFLAG_RD, &cxgb_ext_freed,
 	    0, "#times a cluster was freed through ext_free");
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, 
+	    "ext_inited",
+	    CTLFLAG_RD, &cxgb_ext_inited,
+	    0, "#times a cluster was initialized for ext_free");
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, 
 	    "mbufs_outstanding",
 	    CTLFLAG_RD, &cxgb_mbufs_outstanding,
@@ -3240,8 +3349,8 @@ t3_add_configured_sysctls(adapter_t *sc)
 		
 		for (j = 0; j < pi->nqsets; j++) {
 			struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j];
-			struct sysctl_oid *qspoid, *rspqpoid, *txqpoid;
+			struct sysctl_oid *qspoid, *rspqpoid, *txqpoid, *ctrlqpoid;
-			struct sysctl_oid_list *qspoidlist, *rspqpoidlist, *txqpoidlist;
+			struct sysctl_oid_list *qspoidlist, *rspqpoidlist, *txqpoidlist, *ctrlqpoidlist;
 			struct sge_txq *txq = &qs->txq[TXQ_ETH];
 			
 			snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j);
@@ -3258,7 +3367,9 @@ t3_add_configured_sysctls(adapter_t *sc)
 			    txq_names[0], CTLFLAG_RD, NULL, "txq statistics");
 			txqpoidlist = SYSCTL_CHILDREN(txqpoid);
 
-			
+			ctrlqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 
+			    txq_names[2], CTLFLAG_RD, NULL, "ctrlq statistics");
+			ctrlqpoidlist = SYSCTL_CHILDREN(ctrlqpoid);
 
 			SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "size",
 			    CTLFLAG_RD, &qs->rspq.size,
@@ -3283,7 +3394,6 @@ t3_add_configured_sysctls(adapter_t *sc)
 			    0, t3_dump_rspq, "A", "dump of the response queue");
 
 
-			
 			SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "dropped",
 			    CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_drops,
 			    0, "#tunneled packets dropped");
@@ -3340,7 +3450,22 @@ t3_add_configured_sysctls(adapter_t *sc)
 			    0, "txq #entries to dump");			
 			SYSCTL_ADD_PROC(ctx, txqpoidlist, OID_AUTO, "qdump",
 			    CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_ETH],
-			    0, t3_dump_txq, "A", "dump of the transmit queue");
+			    0, t3_dump_txq_eth, "A", "dump of the transmit queue");
+
+			SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_start",
+			    CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start,
+			    0, "ctrlq start idx for dump");
+			SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_count",
+			    CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count,
+			    0, "ctrl #entries to dump");			
+			SYSCTL_ADD_PROC(ctx, ctrlqpoidlist, OID_AUTO, "qdump",
+			    CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_CTRL],
+			    0, t3_dump_txq_ctrl, "A", "dump of the transmit queue");
+
+
+			
+
+			
 		}
 	}
 }

sys/dev/cxgb/sys/cxgb_support.c

@@ -331,3 +331,10 @@ buf_ring_alloc(int count, int flags)
 
 	return (br);
 }
+
+void
+buf_ring_free(struct buf_ring *br)
+{
+	free(br->br_ring, M_DEVBUF);
+	free(br, M_DEVBUF);
+}

sys/dev/cxgb/sys/mbufq.h

@@ -103,7 +103,7 @@ mbufq_dequeue(struct mbuf_head *l)
 }
 
 static __inline struct mbuf *
-mbufq_peek(struct mbuf_head *l)
+mbufq_peek(const struct mbuf_head *l)
 {
 	return (l->head);
 }

sys/dev/cxgb/sys/mvec.h

@@ -31,6 +31,7 @@
 
 #ifndef _MVEC_H_
 #define _MVEC_H_
+#include <machine/bus.h>
 
 int cxgb_cache_init(void);
 
@@ -48,8 +49,10 @@ extern int cxgb_ext_freed;
 extern int cxgb_mbufs_outstanding;
 extern int cxgb_pack_outstanding;
 
-#define mtomv(m)          ((struct mbuf_vec *)((m)->m_pktdat))
+#define	mtomv(m)          ((struct mbuf_vec *)((m)->m_pktdat))
-#define M_IOVEC               0x100000 /* mbuf immediate data area is used for cluster ptrs */
+#define	M_IOVEC		0x100000	/* mbuf immediate data area is used for cluster ptrs */
+#define	M_DDP		0x200000	/* direct data placement mbuf */
+#define	EXT_PHYS	10		/* physical/bus address  */
 
 /*
  * duplication from mbuf.h - can't use directly because
@@ -59,7 +62,8 @@ struct m_ext_ {
 	caddr_t		 ext_buf;	/* start of buffer */
 	void		(*ext_free)	/* free routine if not the usual */
 			    (void *, void *);
-	void		*ext_args;	/* optional argument pointer */
+	void		*ext_arg1;	/* optional argument pointer */
+	void		*ext_arg2;	/* optional argument pointer */
 	u_int		 ext_size;	/* size of buffer, for ext_free */
 	volatile u_int	*ref_cnt;	/* pointer to ref count info */
 	int		 ext_type;	/* type of external storage */
@@ -72,6 +76,11 @@ struct m_ext_ {
 #define EXT_CLIOVEC     9
 #define EXT_JMPIOVEC    10
 
+#define m_cur_offset	m_ext.ext_size		/* override to provide ddp offset */
+#define m_seq		m_pkthdr.csum_data	/* stored sequence */
+#define m_ddp_gl	m_ext.ext_buf		/* ddp list	*/
+#define m_ddp_flags	m_pkthdr.csum_flags	/* ddp flags	*/
+#define m_ulp_mode	m_pkthdr.tso_segsz	/* upper level protocol	*/
 
 extern uma_zone_t zone_miovec;
 
@@ -181,11 +190,21 @@ static __inline int busdma_map_sgl(bus_dma_segment_t *vsegs, bus_dma_segment_t *
 }
 
 struct mbuf *mi_collapse_mbuf(struct mbuf_iovec *mi, struct mbuf *m);
-struct mbuf *mi_collapse_sge(struct mbuf_iovec *mi, bus_dma_segment_t *seg);
 void *mcl_alloc(int seg_count, int *type);
 
 void mb_free_ext_fast(struct mbuf_iovec *mi, int type, int idx);
 
+static __inline void
+mi_collapse_sge(struct mbuf_iovec *mi, bus_dma_segment_t *seg)
+{
+	mi->mi_flags = 0;
+	mi->mi_base = (caddr_t)seg->ds_addr;
+	mi->mi_len = seg->ds_len;
+	mi->mi_size = 0;
+	mi->mi_type = EXT_PHYS;
+	mi->mi_refcnt = NULL;
+}
+
 static __inline void
 m_free_iovec(struct mbuf *m, int type)
 {
@@ -279,9 +298,11 @@ m_getzonefromtype(int type)
 	case EXT_JUMBO16:
 		zone = zone_jumbo16;
 		break;
+#ifdef PACKET_ZONE		
 	case EXT_PACKET:
 		zone = zone_pack;
 		break;
+#endif		
 	default:
 		panic("%s: invalid cluster type %d", __func__, type);
 	}

sys/dev/cxgb/sys/uipc_mvec.c

@@ -412,8 +412,8 @@ mb_free_ext_fast(struct mbuf_iovec *mi, int type, int idx)
 	case EXT_EXTREF:
 		KASSERT(mi->mi_ext.ext_free != NULL,
 		    ("%s: ext_free not set", __func__));
-		(*(mi->mi_ext.ext_free))(mi->mi_ext.ext_buf,
+		(*(mi->mi_ext.ext_free))(mi->mi_ext.ext_arg1,
-		    mi->mi_ext.ext_args);
+		    mi->mi_ext.ext_arg2);
 		break;
 	default:
 		dump_mi(mi);

sys/dev/cxgb/t3cdev.h

@@ -38,7 +38,8 @@ struct cxgb3_client;
 
 enum t3ctype {
         T3A = 0,
-        T3B
+        T3B,
+	T3C
 };
 
 struct t3cdev {

sys/dev/cxgb/t3fw-4.7.0.bin.gz.uu

@@ -1,451 +0,0 @@
-/**************************************************************************
-
-Copyright (c) 2007, Chelsio Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Neither the name of the Chelsio Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
- 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-$FreeBSD$
-
-***************************************************************************/
-
-begin 755 t3fw-4.7.0.bin.gz
-M'XL("-[;X48``W0S9G<M-"XW+C`N8FEN`.R\>5A3U]HWO+,3R+1#$@@0YDT(
-M86>.VBH>M8V*5:OG-$XM+1U2DU`Z.-2J.!M&PXR"2D`@H'7`*2AU*+9=H`<G
-M6A"L(X&HJ#A1;#U63RO[6SM!VW.^\SS7\[[?/]][72_MREY[[[7N-=Z_^W>O
-MM;9&9#&"TZTP+$001``#CEQ#T$R$B5!7&T)#$#?B"0X811(2$O0W$%H&CL*4
-M-#<,I""*;$>B2(#<0!AC48^$&9DX,LL`KZWP.AM'9F+PF@BO*?#Z+H[0G0!!
-M>H<A=X<A-<,0QS`ZBJ`/=/,5/LB#80B,!R'^Y%6@$Y%7.OG4?_2)4=+I@D53
-M![RW_N2/,1Q$RD'^XUL1^0-]0I2(/`M__<E6H%./@O^_I!X5KQXU-F8<;=&(
-M![`,?_($T&E'(=I1`NTH7#M*]Z<WWP)=W"@D;I0@;A0>]_P-G7IS-(#\&D.1
-M$MP3/3H4/0*&E>#I,'(X@#Q,/3@<2!X,(@\!W47<)YEF1,*0YW^,D3ZP"*J:
-M2X:)F)-%/B,P)F/BTF&WX(LEH[W%[P@DMU)9C8@O\B()XQ7X?L!3B1JJ9VJ\
-MC2W1-R0LFOKS\Y979.@]/_YD!>PA$5G^'U-MI!)L/(-GX)G169)CDL5C!-F&
-MACF;=`TC%H]X9")OP8I:7G5;/G#YDZN`CABE)T8E$*.F$*.FO^@+3TU7@.%T
-MC8^G4LL"R+02'`AI4X<A4X;14`3`XM<P.H8%DJM+#2)R-:,31E=1T56,\U14
-M^B9,L9+Q(XROA'$!N8*._-^__Y_]F<C?_\MW4;][PPW$Y]1_&8>XX,4&G]:A
-MN'@H+AZ*]WKCS=.]\69#%'D<O8&PPN$58LO?%\-X%XY0N$+[Y`;"38!7*Q5N
-M()@3A_,0!OI0/!P&',8;J"L,,A@_`J]C88"XA0'J"L-4&#\!K]-A,,!X.W6%
-MX6T8/P^O$*-0(XR[J2L,'\-X+[Q^!L-"&!^@KC`LA_%'\)I/A1L(CZI+,0PE
-M0_$]5(!Q!KQ29;1[XW0G%?[7XK.F>^.SIGOC[SK^J_&`?;:,"EX,GF6E\!EA
-M(/@?][!??Q^$Z9H8")\D]3"O#PQ6!`GI)@>1[KW?_C0>WK],R22;7:A'.@O3
-M><8`85L]A3VATD?"]T8QO*4A5MH$;_H"*'<Z]8`%[QEZ3QZ/:<%@/%SOM3(,
-M&#"O7"A&AQ@1%B4OG+(W"**GXK3?O742PQ!JA3DH>/`VFH48$(1.DM^1<.YT
-M(4@0;"!L"#D`WY&_?NFI![@419Z8`>7!:G!Q7$O"=I^8!&$))B%)!%$AY"#Y
-MA*H-E(7"_C)2-27)'OB._(F&"*&,$Q_"4$R2/U'/?C-21?.?P.??"4CR5ZKL
-M\_#Y,X3V6SML]VP8!MRP_V`?&ZF:0YE2^"P1]H<QZG?R$JQ+VEI8]C6(ZP@2
-M)\!AA_;`]'`L=#`LA/:6RH/#$`_#=(3>7JPTF*Q*PW(8BF'83K7A;5@W/JZ%
-MY>@0*VR7IT$(C7RB-!Q`8&#!<@1*0SU,?P"&KZG!$2H-WQ!*PS%X_XW##;L1
-MRB>BGB$L>!T+@\$%_^#5#.M6#,/]_Z3OM/\2"?[/^,,%B/Y%$,&^@Q/P17C^
-M['D0(P`/IR$O`D[3>YX]#]2S?[^GTOP1C/\2I#0'^:>__U@_Y'48ICSQ7CUQ
-MW1_A-?-0@-@T"?*J!(A%XP4X\NJQ/_)/A<^FG/!>J3"9]?\.K\WV7B="S-*?
-M@OD_^R,_'6*9"G(T.L2Z,3!.OX\C+\?#*\3?T;#,5Z?@R#!XU4*,5/;AB`)B
-MH`S>XPX<T4!\"8<RQ3"O$-93",OP@V7YP2L7XA9KCS<_%1A0)@/BL"\,\3"O
-M#M:5_CN]SZ=KN!DJ]M#5_&___7__$_S/GT^,L?Z/Q;Y;2GNO%'V_E/Y!*3+Q
-M?Z-:[Q;3WBM&WR^F?U",S/K?R+^E%(DHIE66HE6E].I2!JO4AUWJRREE<O^'
-MU=D",Q>C5<7,B&)Z=3%C6;'/\F+?%<7,E?_#ZG1YRG>5HMVE])Y2!J/4QZ?4
-MU[>4R2QEL4K9[%(.IY3[W]6E"V8N1KN+N;#\GF)&0['/5\6^AXJ9AXM91XK9
-M1XLY7Q=S&[UUH?@(]"N,$(`Y5%;*UT`ZA@G(0<\M==^*=/[Y?B:&G/^7^Q3D
-MQS_N/?(XB/"GD("?0NBW*.(:]),X^"=Q)S_U4]]B8T-RJNG)[(`$O@@F?8H@
-MGPD8`#_'_^^N2DDD(8D`#E`JE_HJ8M&5.;2KCM-'`WYBJ)Q,PHE<P>DV#I$^
-M'9SC6I&K#LO^VYZ<?Y;M>U&OB4;!P)+;_)R7UF[FY+Z\=K^/-A;E#Z)^W&F6
-M^C6B_H>A_0\W"]2U5>#,ZN.HD98EGR!+*6;:<+8-LZ2ERFHKNXSF#Q<K<0RT
-M8X>%=:G$HFRP95D)3/I&2/\-]E&.?,/6/S*,B5N4)>[O,7\^/&\8-T?$LM,6
-MK)?)HQG<QP:[<'Z9&&R\.D#@R;X5\;)\FJ_O9)DT>6DCE*60M8IS7@(GUFY^
-M*:R_W?;REL]9/'W%0KY+F(=7ID;TMU8OYC6B%8M`1^62L/XS+J!<2E^[CD6\
-MCK#Z4"(67;.9D3=2&2M9O9E1.$H3&[UJ\TC-6!S4`ZOOD7@6:X+9FA[6_TU(
-M_S?R0VL0MPTQYO_Y5U-;J4VHU=P7:[97YNA2-[\<UM]@&U$XG'F!*XUF-+@8
-M"<(M'U88\Y35EJKDLH\B^_>RR_2^-B$K`?6#M9K;E`]CI>;3%T/Z=Q(XAA4*
-M=]GJ^E77L[29;5@75W4KJ^U=:DC`.E#L=S1^N0TUD<M,Y&>M;UW!K^JZ%B9B
-M;U*OUPFNX(F\9,IQ`D;38-85'!Q*Y,TUD1<0A6IQGB"1-Z/#<8Z_Z3@H">LO
-ML@DK`,LF9-O0LB98>R5L@S19>4RLW%YI&MSOF0>^%_"0_F7A_2N(#U'__E06
-M:YK%FI2KXWPRX^.MLY22JOE'QB@VR%(<C`+CBLLZC:3RT)0S?4HC%C=]$/M:
-M6%LGK\UV+5R>%X,=XZB:MZ8`ALVX*B=059O5!ED.#[EBI.KWEU/O&&F1\":1
-M-\HR(Q"V1U^@NV+DS9N1B)WH=)AKT,P:,(/!F&PB/T>`,W>8]D,&YY^&^4^"
-MP<9+3DY9O,*87%_'X4Q6F)-7Y*%&Y,(60:708CBL&"OH!F`R>`BLV!/O()9&
-M]4\/Z9\N?[SFD\\Q,%!/(@X;XLY/??H(.')T12/8-KW9.M,V'%O#!0D1_1-+
-MKTD_9%2ZX?@5?%IQL_IVV:VH_E>\8\=WH7#0&M8V%51>IT:O-[)_M-J(89>$
-M^URUZ5))EO)\&X/D:F595)^']`\GC-3`UO;OLBDSV^#8=CB\0PN'U<IY$K]L
-M"AS4%!/Y+J+X;#$XX-(G8MOS!05"X#21)L0S^AFLI\+R&RP[*NZ/LE\K<XO[
-MA4ISLEQ2J<RLE6\1RV65)A+UZN_)L'YA:+__1?TE_+(N/3J@7X2^QMV:M736
-M"'8)*N700#OZUW@>;\+I"XI#:[IQ^G#4[P8Z;X4/@ML077YZ&BQM@/U&_%IP
-MW41V(<"`%F);^_W&^8-AZ,OQ/CX3SKX)\YT<6/U/?B<N[1)T%,][?2COKNO'
-M,L!X8*!/HM*=?ANF^U>Y1O;(^*6@V40>`\7HU/CEH,=$7H9S+0[W+16D+/,A
-M5CVUI%699ZPEHE$69YJE)D/\X)(RO2KTP84%1RB$4=3"*<?.PSOYG$;,4K]$
-MGE[9U&1>/T^*8Z$/VK"_"G>Z*H7JCFS.ZQP/)'V@J-U:)$QQ,/-P3X8WM#]G
-MF1].A>4IHADIJ5'9PWP?&^;O#`8_`J<*8LO5^/J6/T'+XI`'+3:U+#HB[,'?
-M62GLBH>4Q@QH=`+Q@V:L7^"W#V7UHU4_%[Y\9<#L3NC27=:MO1V@&BLX]D]B
-M>%#!Z++'E;_FC6H87?'$3RSJY,_+B28V!.6=+MM>L<-V,N_[LIW$AN"*NK)=
-MMK.J5KQLMV*/^(@=O%2?[.O[QJ>]N%\?FV_C^+V"K;WM`XKMMF,VB`"A_O/J
-M?.TV>QZ1+PY[4`O:Q0]*(A_8%='A,8\#]OV->.PO?E"<@<MB(U@CT*6[?($1
-M%,MC(Q2KGBH?!ZH?!\<\%OE\)@QY4*)Z+,X7RA\'*9NSU+UMRF,IRI:LLWLL
-MAB4>_->N>FH>J(*C8!I<BGBA)E^0R!U#O3S']X`+-LU$]D)+<A7.UBEP&"%J
-MG'H+8HW2'TG$1GHM4"<?6A[@-#O#H-T)>6#QF!ZVQ_2H.;0AVW.#$NH%*8R@
-M%";BP3N;2L#!\`<6UC[AEF)N/PI`U7K5^5IE>J7*BU/9$*?FPCQ7=2I_)(F<
-MT.D`,X`5N(!;$XM6"DS/`*4++#GNFU+F0VQY:DD_)H?6K([-GF8I[H06*>C!
-M!.R8<'NKICF[J6--)VI$YX4\>/6%22K`V2[,TO&VJCG+W#Q+,[<JY,'+"_I@
-MJC>4$/2<S"*<2V*6G\>KYU8V'3-WC-9$XWY.8WD`@8\]22+,5VC<5^J_9=X;
-MS6UD\[X5:NZ'^_F-Y_?3Z_OAU:^1JSP6?M&Y9EVX)CIVY;V@N-#?_;[#F`^-
-M"/L5&O.5^F7L=:.YJ9P<F2T.&L94B&NT7M"GC([11D?7-W*Y;RAC8U;^%N[\
-M1A6-JF-C_%I152PZ9`_TB9P.#V`XBX9IHAE\TC"_SP.6_+)X+9Y<3_+YDR'6
-MKZ6$'@7U-IU<)XAX(-P45"K:'+PQ$"P$,T&"#!^K;D?M`E:1"!2S;'J&F>9C
-MH?D<CO<=J<\2,%,%IS)`&B\G'E@9T$+:!*Q45/KNH.<N3\FQ";BE`@B+S`24
-MB,;CHF,9C2B+$(GN/V7&B6"<D2J$&:`!ZA4R9J-Y.&..@-4H]&E$P3OPIZ*)
-M\ZG0IO!)%L"D%2?+3H!ZOEC/A(4DH\QD=%-+U@A@W7B\Z=W,X66G*DZ7-LLD
-M(J"O_RA.)BH]N_'OF\[DQ5>TVOY2]OVZ,9M_R!Z]+GA36];+V4&Q+8+,D5F!
-MTA9AWJA,D;+%'\[*]BF4P9Y(Z`2%(XJ&0T"(N/]W1IU0W2("Y]4M@05*;4NH
-MIB5,V1+"KM+[]@O]ZE".#8V\_XV"(X`1/Q>J;@DB6L0*CE_(_:.4\2Z"QGL[
-MJ?DY"\*\WSZNYG'6J;5YLESYCL8*&;<.XW"F*&)CVV93`P3--7>%:7"R1Y>X
-MRTR#XQ!$'413!:&H`KVZ,-'GOF6?(C<@W[]`:!I$AE0$ZAKW`TI%JCM#[E>"
-M#'L[ZXX0&@7[N9#[!4II,C&W4EE:"Z&!2(8*$O6O7'%&P/W*L/OV\/OVH/MV
-MC!"*[MM#[J<QI)SMLETV17J67/4[!D4Y*FKBTK-B7EM_)#4N.RN.8]V1'&NN
-M]LC@T$>B&@C58MAEL4%,(O^I.7UT2I^/_/Y3RX8H133*X4[[OB%?)[K_Z>*3
-M4G5TS.EN#8<!41(J95&PY>$@G*+&?'$B1PYPR38Z53-B0E7D_0\6V*@YKK@.
-MP9VEE"`>J/Y1,:'27--6+@J^_TG4_<3\CYING3Z3FTPLI)W>NB)#J,"QI@/Y
-M>*+/S1\=/VPSHKX(U&K(/+91S&/YO7G\;SF:ZUM3W(PB?,W3#V)KL\PSOE1&
-M:]<E<U>.Y"[#%A088&46-IU21^.)Z%L%LORX\A!>*[;B^+@<DWV6;6[9S")+
-M=:+&*(#8$Y1R>]26MY42L4(24)%4_L[F.07)J[^`=*,L)]A2_)H140"#B:R\
-M",WUKBMX7DBBSVK+?IZ);("8N'D.N`8&0'K9V]5)5>^8R!,>+!4GLOM`TP4'
-M<"JWT14.P9;@\B"?=ZFA]_T4[5FH-M(+@RH#JP282PC<>^>IN_!$VL7\P+R0
-M/7M49GJN+$>^NXYCQ\IEMF0V>XH\-I9IP\J2M1+Q#E(K$YL&SVCFTK31#+]!
-MR)5"%!P?@H/(\62_G?'U57Y^DS72Y#446]J9]Y'9.<V\<,I58Y>A9^$5W%,N
-M++-*X"DS\KY_(NUH07!1""$)@'9$)JZW$;*`4R0UI_?_>4Y;V@V>>0;;`!N@
-M_@QGKD4+@Z[@58+(^[1$6H*)/()`4,*';`?GF.G9`V3!3=3T[)2W/UR&$^Y$
-M%D+-;:@9G`;3L^\IDP#GRE8/A[IW`5@C[KG!\3P1ZY_"JH-8'5I>7^DD2FMC
-M)E02TN28L>*8UR`'4EW!X3Q(])ED>I;O]6<@[BNB8R#P*_:0',X;BMB8E-(Q
-M'Y<-5T9'K]RM@G8*.%22",I).NUUDNXU_O=.DF4_,)$M'OMR40>,X??J(^_M
-MU^*^EPU7]"!1=.^0(H@F#T)2GO@0OS^U9'RJC);D"!314DLQ2[&'YJF"=`5D
-M%_1(;33*YT^S_,*02P)MPRDG*",Z-QZL-V^5P.E=-`H<XB>-KA`U'"GSYS4*
-M*P)./PF[MT/II*T]*,@347-?]J/#O#6\*-6(Q.7[8]N%^Y+5<[,YN]'59R>%
-MWLOC?,V!]"?%X:.2("M.CXB=FV6>>5L6K?%]/-+W&;;`A?(?#\TX%4>DCHY*
-MI&VD;,YY!ZS)BCR1$6$BU+P^C7BM/..4I?8=T%Z9F+]XR]MY2W*75KQ3GK1I
-M3E'J6A/4WO!"P?=N:BQ5_K1$]K"+#O"P6E"T`[3G+MOT7OF[MN4Y*\K>MW]0
-MN")W^;[Y2^_1ZNNJ5I0OSUL('-Q"$3=/?\H):H`3;```9)8&E01*\;%@&&L*
-MC34VGM4J8,/(F'C&2WKN3@&S4>`7JO>9*,`:44Z",/S>)UP;RJP3<$:@RFB(
-M0K%<I8BC$$$DYT+`'H'F*2&TLQI1FPAF8-7!#,8*`R<!92>@N7C(O23X"*(Y
-MQ>QFY"E*WBQ]*R,P,P@FK9B=NR1K:=V5+RU+DU`X<OD!H#WRWLOB>Z_XDMC&
-MI>5+*#7<5E<N@XKH5<+J5#AL.W=BNX7[&N4'LM29;7OL\D-9IL%)P`UTH`D\
-M`9ER?"RLCCHZ%A;+4XL8+B&KC,:OIO';1K..Q[-&"#B_"4&Z#RF`_B^L/ZL.
-M9?\0G[=0'AW%[!?0^U&V7,3-$+%&T.C'1XOOJ9@C!'02S<"A2\*SHT6*`OA*
-MR!N!0O97H#0;K3;8^V7OVE9D+R_[H&2&_<U-[Y>_!8RIAVD70/B]2.ZG`NX]
-M%/L=W3([_%X8]RI*-=L`'.'W?$+N_A,JDB9:BAV!1FU/HZ*A+2=`FY[E[-]%
-M:K.SB@1J2:`F5EKO4LL"3[U1*"N0;^NODOFY,!YOBAH"PL!'7FT!QI(9U;/M
-M;Y:_E7J8);[[*V%%62WQX?=HWK*?EVDB;?F"UI,4)N<+$UDU5.8FESI6Y8$'
-M88%`P[$F^CZAL*"I3Q.K,I$WY=1O#^+Q;X6);#OU+N51>,KCLQU&#XJP:SUX
-M3.$!8[R)K()XH,$Q\=T3V$]"%LDB?LY>EO0)=L=+^5@*"<*NPRR.Q!CH(NAF
-M;NL/N7L\_&X3'"QEM`J<KQS&;13".;5%!P`<%YMRT\O*U_VWC-HR$M[!R0,.
-MEK]4-AS61?USEJRW#:*G^G%6@5`I];W:GL@>[@6VJ^Y$]FJJ:=#G88X3`6N7
-M[JH>(N)D$!US3$<G*$\U54^K=YG3WH^\ZPBYZY#?6M-AF'=ER(F"TPB/=8P5
-MW?WR4&<1`.W\=2*I'N7GZ$L%_"+]J5]`&B,]'CSD$@*0`"2,V33&LWAT)@T=
-MC&>Y!.GA<`;YS:=XBU0U"-R,P7@.?&!#>3:T8'?DW4K6)13>^>XSE#DS'0P"
-M<CAJE0+*?`DMK2^JRQ14-YRZ(ZN)`C_[RBATD-7@GB@_E0;Z_-ZG,=>-CJV)
-M]2L<[1,K8BX3,.8(?2RHSRV*N?G.03<>BJUA<&X)68E"%H'"9YL.`Z-9GV@N
-MGN.[1@@>Y33;CG-?XFZ:5#;9_IKXKKEZ/.-MH9\=C;IK1/TFP[I5Z\LFP/N&
-M^"8WO-@3(N_.%-U-TCHP[*EP&[G/KMF0)7O4YF?C$INRJ)D7=??U\+NSE/#]
-MM\(]C?O[U=>S9`UMO'%<M=?Q]B$\CO=@E6EP`QP/"#[;`;`L'+#I@(ZU3@1Y
-MK\7:"=I!*4@#`['XV,W^W%<$D"`/I\^BT8OBV6_3V.OB@8Y)"K@9\0RHAR:4
-M4L5P5JI`/G:071#/<0ERE$P7BAT6P`I##85="Y749XQ>'<VPZQFC!3:<Y1+R
-MQJ'>M1B?HP+63K1L/*=1R&N$NDQ$QS))BCJO\Z].8-B$47<#3_7)HJ,\O>^+
-MR*)Q&&.EH!63F9'ZL+MB:GCM'B![38YCP-H!L"KA;KNGY41I&V^OM^6P8P*U
-M`RC064@<##.182Y]X8PS[@-U]8WJ+H'?LI%E@B(#*Q6KOE1V.6=6T4R[T'ZU
-M^LJ:#$Y]:^R[@M/+3&0;8A,4"%FWL<TS]A9<'%AVE5;_:^6LS3/_Z#ESX=\M
-MW]68R!*O#89ND5DWV4=GI%PCGU?JWV1:1OM,)4ZNW36"[CN>F4RG?-[QVT>L
-M61<IB8Z5A_[^A3(0C<?8:;"=K]#05^JEOA-&TR6<<_P.'1RZ&2;2Z]]!/BH-
-M8BIQW]@@WY0C/L2QIY:T:Y2CMY!-K1Z`/!TXL?BP#V2BEHX,T9T6S82JP#O'
-M*4>.UB2?*4LI9A?AG7PV=.4>UD.Z"7XTU^PJC(^Z<\Q\;6^YZ/1/!1^?OAIU
-MYXQV@+;FEP"*;D)\3Z1/A5:W]N/"%",B@]@2=:?^.>ULREZ>-Y?_'4?[\]:4
-M`88-7YLSFR*>,Y&X:`WS]DCF'6Q!)>VBVT,\3T"#FT0>I0QN>2!E<#-T12;@
-MKIY5,+<JL7)FO@7:V(\JDG*3-\\I?Z<P90U%-]?8@G[(,R*!E%7>B5!F.2\P
-M$;UN<5PTD4V(!RF#$YDS(:TL!O450?)M=(IP]RSL,@!WK)$.R=B=+['WA!2W
-M(Y+(B87)7^XL^H@PT_>O3?V-5M^?&UB57/T1K)&\S@YK9)<5IK!84PA(+ONQ
-MJA338+]V+BW&ZU/&_T$J^9!45O_)JY2>Z<_[^,S7SREEY)T<R"J!M8SBLD,E
-M2ZF*YP<7!"4BB10P=AFN&M6?$=[DJR'1#$*2!@^;R(M(;M#WM1ZG`;8K'.`>
-MNDRM0'ALBX=*LN0F\A+B(?_?"K?URZ]G-Z4OSX"=U<E_[LP7X=0(#S1KKV>9
-M!XYE#9-%,WS^89B?$GAI(:R_SZ7X^DH>;[):FKR:(L5;*:?PU<`[$PI&9`[W
-MKO=M>;AQ@&$1YBHW_Z/T<>6OK#EHQ)VQ359XY=6A&Q]%W!E=_C/O&Y;T>E;<
-M^3;I=%QZBUI_=7I;&#7$:9FDB=1B9=2*LXF4A=V)!HZ0.RJ0`2WV6&%U)ZL5
-MW7+.WJYHJ)5/J%1(DZE%/4AH!YW4,%,F#"TV#?Y$R7E@&KQ=N55\)P*4E#E8
-M*11RA-SQ*ZN)@Y[?A,HXZ`5"_X_BPB][=._4J]2(;K7;97!,O>/9.O,/?Y)Y
-MEJH?3!<CB9))(L1W(F,_2]D9%S,V)486M>0+X9+/%:`@5A;1M%EN1@@S$_;]
-MWJ;#:DZ?KPK;V8JIL+VM*EF$THPH/.]<3;>U'.M^K586H3$C:NK99P(_\.%E
-MO30:M0L9K&F6C$/!?3?DBZH6E$$._$-$7X^\0_:'_=UZ3+NHTOS+(=CDIEZH
-M6#OJ"*A8K<N.P\1[0OHN/%^B84%2RV[%+,<WJZ"*'5]?*E`,8]M>+O<O4Q2,
-M+`NH5.:,JA39507Q]D#>>Z+*H$IUSFA9+.3Y#'EL.*$CP`9".E:MD[-8D\N"
-MV;DTM72<>EA@N9C'&<^Y@BGVW%?$1BVWALH5"+O=B+!>H;%?D6ZAL::.]K,+
-M83WA6R(Z*J3O/&N]$;Y"&*_45S(NCY;N^5T:BS<Y./:1[(1PMBT\=US!6)DL
-M0"83R62!BEB).C9:CD.R'4[$:J2Q6C9[HEPJ`GIIK%0:&RN%=$<OE8GEPX72
-MV!A[Z/P._WDL3GT=4L-`4D+A[#21_3FA)1'`"CHV16F"_.."!,H@X<9(9QV;
-M,85=)P0S@#.XKTB#CP-FOP3AIB;>`]IFP$N*]_/3%_AS(7%.C,_!-Y_:=+HP
-M@%JG@;;?)0SK^Y):S>@4E?P0VE?#@T;$A5:<+0R$#T]^5W&BXN\5K17?A_:M
-M*VN!:<1]RV`*N@LM/Q[05U#9QBI#RYH+@LK.\$)%O#HAO0X-Z-M0?C)71K1(
-MB)90HL4_K@6/:Q''M0AD+=&REA!9BS"F1:9L"52VB&):I'EB#GU*4%_VNF"?
-M$4)NTFC&"#2P+Y,IU<M:8IGQ^NPHU";@)PB93'UL2PRC$<V*%/6MIC>BM%1!
-M0-\J5BJJ;`D6]\W/$TE;PF-:PH`^(P*Z2O!Y7$ND7R,:YR]0M@1%]BT*ZTO4
-MM`3D'=7X^^]M!.WJE@AU2Y2Z):Y"3>"^2G\AD?_,;#UB&[-VG1+4E(=\GU@=
-M5A4.G:,D<G1A>%%8;@BT*G!*V@4Y>+[`U[/B9D10!#C`1;74EW:"H8IF0*:\
-MPR9+SV*U"GUO<67969FO%KQ"#!<P+H@W_8*<8##.BX/ZM+SO9R$)PNV'&-!.
-M/^(P_E;Z#W'?=)]2(<.%BOJFE@W`:^7/H),'\>!A1)\8`)6Y3;JH4G6^5CI=
-M+%U:"?06P^[<,:"8P]%S2N,[]!37B>@++I<']8T!O;YEJ*9%KFXA-#A'VJ(D
-M6E2^_?")0HTS%2V:;$+]]W?`)5F+5BIE:G`S;/R5]I0,)FQ#4Y^)'&G^Z5%X
-MGQ^!!^4?][-A2!,#L87N3"F34SL!R/<SO&0\T>>H&O=M[2P<L_8GNM>G3QK\
-MR4OW/9CL>\)$GH/11-]F$WG*R\]]ZXO&\/EZ_C9/C2'?@D7GCEB0#BV&!>@#
-M;G?##@Z\[?IX`T,:BU[6J_#`<B+H]KTB61S.Y0QBOJELY>>8^=JX0E.!><$Q
-MF"L5N<9PGH+U8^UD0798%@=I0,I!S0^;H4L/'7J:D'+HJ;7GX/_.G5?C9G//
-MF3PY),V[L>3O`&5'C@E]6EE$<[:&PUIVSQQY^ZC?T%J2#R%!UN3,E#=G1=YN
-M-!=/C;K=4U4,,LO6$SBGM(0/!W1#(<[O1ZM*M3C3)M-NY)1M*M^LP7VISK[_
-MS.R>&'%[/[4>T9RU]QOI^3;%'ES1DF4B7X=^B>7L`Y!.2"VG'X?=/LOHI<Z@
-M*F:FRX>AT+U0O)G.[AT&,YK(*`0Q]QPI,FGP(.0A@]>/(?VA^_95P0':FHLK
-MAT?7UY4[Y<.CB>$Q#7;YR.CE.;N4PR7*D=$FTD$UMSI.*@W<<X18@L'>`DV:
-M8X-F]W;UL:?F]M6:8\\L_0#V![4RNJ9(SM^/)0^LO0H(_!TB?Y!U-K[':?FV
-MQT3.0MR,$@%(X_R"T3-&GRQB7:*S=M)83Z!M"=DY@GX*VTF)IW,G*Z6!&8+3
-MWU'#=`QI#=VS!^M"M\19JMX$![B0IK+?D$-+.#@%J8XSD7V48=MIWA\*'IH&
-M'T(_31)EZ1]A(M.T,#*H,)'+$$0J]853`T`S&$KM3UH^'Z&<+@8GY6/%H`98
-MV3L]VWQI@O#;,T-OSU2TK+GD5+S[;%Z=#V*U(<7Y-CUPL'(F1]VVL+F36=,-
-M+-XLADO0T,IH%?#>0RN'45ZHSC:Q$"][F1K!X06R2X;MKDI9H1X4^[UO,+LW
-MA]^&<V$</8\+'E775SG5.,=^$(/*>J`(QTBTNJ%0!G6LZE`^H?Z*L^6P7,K4
-M0F3!S=K?G[7]>O:KJ-NRB-NC8O"@@D[N"`PYQT!&A#84;#U>(@^XK9#`6:<1
-M[N_?*I%+V[34_B#?Y27KX;?%T&>-NAW=D([=@RGV-*H.9,E[VS`)5W+(L[V(
-MA-T61-X.44,1'<)]KMT2K21+?KZ-7_1\_Q&T<_?&KSX%^_J^:=!=J(]3+5[9
-M[W]E(3BH&BMV&1)]\D"Q36\:O`'ACCX&94Q'789N8R+"*!=<-5Q9F.ASW"8P
-M#3ZF^+8&]K5C0U#4[<!\_7XMEC\9Q##.<KG?H&L/HNIOGEQN!\XET%`CZZM%
-M\UA_\_N.4U#L=YA[))4W;^3>NLH!7C)VI/K2`#4@"_,W"]<5EPDV!Z;4RP](
-M4CM5MO6@_6QA67'<%@&RT+;9'QG(KPZH7']V`3C':Z0O.3O.1-8A]1+-?3%T
-M0+(V\%*Q[)+-Q=LOK/Z45C]GXX9-)3D3C$CXP59(G&PB5_&R$E1="6MD6?`M
-M]8*>,^%*<590V*U_8)#J)<"QWC`L>^**];*P6]<B;PWD&/R2:9M>AO/2/MPV
-M@6/#RB:L>W/'CLUO%B7LWU^=<(X?=NMBCN&_2^#A6@H\4(X'B6Y]P[%CYN()
-M&IS+>,!6GL/8QT;NJ<,:,>6/&+59+0T\X\A+*")R)CIW(NL]&$ORX?R+*Y+Q
-MCV#+QH1XL#0/ZL4#RIFIY=A1J"L@&CZ$PK"Z(2%[&J&\T_OR$O*)HHG.O1`/
-M$%?HWE9J)L==P0_?2_292^6A].LGKQ\6<JLT]%9&^*T,9!@#7)-)(D&[^-8!
-M;(QPMXT'(>=&*$0=97.ZH@;ECH&6-9W3.RPN/9T-??6%L>GITO3,ACFQV>D^
-M"=BIJ=3:Y:NH-A8]QX?,+C8]@_$2"M])J0>>LL)N?8(IA+LIOZ@A?GF>).S6
-M1R&W5G*5'.6!K6PY*\6)+C[*DU?2Y,51\S>P)+597E9_:L.017G:^>),BF_4
-MK=SP6[DE@@LZZ&*N9YEF0-;F4Q9?YD]$H_*QSZC#"6D.R_K*7-/J?0&<,0:S
-M\QLU_HZE_3#8()>^8T3V0WOU<:>DT,)A3_+C3I!+.0U'""DFOF502KEKK.^!
-M$KDT")08D3=3+C+`0:74%Z3EF"-N3;/+['([4=!ICZ,TUA9::$':&<X\KIT&
-MVG?9_-COT^S"73:G3<X9A/#'MHT$&V#5V+91:BF3D`;!8DUD+IP2FJYGSC+.
-M>S-9KI%JJ=GRTSO0X3#WS!;?&I%'V#)X51B2QJ"0O+$"(ODXZ,YP2,X\4:YQ
-M]9&Q-J)0%G(KB-6/M=TF<"[;QB4*:''6`X3U(,NSDR:^-6E/*BN5QOQUU,KM
-MH\-N11;,53IIJP]JC`@O^)8.@BN"!-^"&.IN<G*GC6IH]+BQ2>3F<F&;=WDL
-MB5QN1(Q-;K^_C8)W$3=_A[Q[+\6[C_0ONS<NXN93WC<<=<?6E'9&(;[ZCDI1
-MFU4N/%-7.!?ZM'FRW#CHUW+KO*YMZFX#1!'F2LB;QIH=4.\XP&$BVRDNP/C.
-M8IBEQD6\<7I+>S;EU'7I7;I$QNOAMT11M_PO.,YNRZS)<#`8D^D^DS?ZTTTS
-MH)\_&%$X=\T7T=!?%'897?KNA3G^!,[L,>RR$5)FTN`:#[Y1T^3L6\_3,->@
-M28,_AM\\'W7SC-<K]"R`[#G&V\L%#@BJBA-M5'4J/-NDGKF6)^C2)](S+SBH
-M;0IJ[E[11]T$D3>;>-.$%_%+NBQ)/NXSB=L)T&8L=BDJ43TR6WT6:"#0'?>_
-M>6SKT=K5RSM'GRX`T<!`C_4<?7A/\WA-AY4R/IYUNYR%[*,L]A7,[%C7`>P+
-M8;EO`2-+%K_4<-E$=M*78[5V]C>8V3J).CW!'N,/N]-[AN+LNU#02>/*3_@=
-M5NDQ02?_Q5F+G1N.?0%3.^G3XKG<"6?J8;H7Q76`'=W'^JN>E_,:+.=K$]E`
-M'2&B3EU8?X0(V@&.K2A?Z%&PF.]9Z#A14W&'E6?'0#M[ZD@PGCYT1`2*_60]
-M!O3U"5[1RZ<^REAH(IDO!+M-Y!7O/J,QXN;:X)N?'4^3#Z,I="RH9,MJN,</
-MJ/;\8`:+J"59T&0QS)3%HIIV`DYD!><W)2[2XH$R:2"GD<:SL>5IF%(J8I,L
-M;19F=C=`'B?W\C@[[:KQ3SR.^6\\SKOQ#2T9<@/")9L[[?2WU+QH?5NE8ZUL
-M>@D6=^;N'VFN4.]@R<Y&6!B7/]X\D+WJ\`^@"=M"D?KG$PJ2MH^OG_!NT'S\
-ML%$F]845]QYJ4.!<N:?6O#JVJ@;CVK`]=OF7</2F&9'7O5)YX\]>7'68D@>S
-M:#SG(.[\*Q?]]S909<)"J.+AQ42V7(7=J3)_-_Z_RA!Q4VHB8SQK:[X+LLQG
-MDB#S;DA$'IVI]3;5YSTT$1%3_<&9=JK6,]?KN16H$MH1SYR_J@--B<CET[NI
-M*+3YB>@)N4/8Z5BSF:7B/,.VT%3;H#L)]17T4AO=4I&)_/)Y)VHX2"(M<&C=
-M(*AW0-P[8)DQ$30%]]Y71J-:"<+E3K,X,]3#$=Z\&>JN9Y;V5,W<JOE]+\L7
-M04\?7?Y4I9Y;V?#ZF?.4?AX5;K=K-F2OV:UA/=^OTDB0%7>B%!NR+(YD(Q)$
-M1#.B;O*6_:;T.!>T$V>2C,@Y!%%$,T)N,COY*RI84)]IWQH1(84S]"^I@_Q>
-M1X1>:S'L&#I5AE.GRNC:3H?942KN/0%F9-;D4VB#89--Y&1Y-`/<+QH&G)><
-M?-+`[XN?7R;0X,E[[L/7*FGR*FIE:%-I353OD2.[JAT6PP?@)#&6I:!.4['W
-M4JIRMB.JMR&RMP$JC/-UJ*?/>:>X]P#%[A3"[74[)4IIF[PV"^#LJ5SY=FHI
-M%K2S(#<#K=2")C4N^3JJ.H-?(0C00;\)U'<5NR`T22)[:YX3O.T4P8N!!.^-
-MYP1O2R?(B.JULZJ$U>W4>:1SRL6UU$D`[TI0<J6)G$2M[^"BDY,9(V@Q4I%G
-MW(#Q[*RAL8=S)8DLH.8*:]JI]'_1#VS\F>]7'[[W9_6``*\>1H/JH)9RDL@[
-M7M8(`&A/I*6:R-W(T#2F[#;A<9H]9P?+@GLA^Q,R;2Q%>K9\NL^*$MBC)2&]
-M<U\LZ#S?@%E#I&>%]YK,UB5;K$>RZZ=5I%4(+(:%8,=E(T'UMV^!I[^O1/4F
-M1O8FPOZ^I//BHJ>_M3J6UHR$],XA<`S@6+YP1^LNFR2S#?I<`.=\S84.EZ??
-MZ07QJQTG*,!O3?0<).K2)PVF#9W,H,/9?CO6G]I:4G(0W\]0.`AHDU<T<&MQ
-MS+]W'/94N,->2VH>M1$;X#"P$I_O.WC[&DQAC$"?][4<%T$UK;?+I:*4-"G8
-MK^+0H**HI+XK'0S0KHY%7U!OJA=E+ZP:)4N"B^@2FO<,!T16"*MRG9#8XC);
-MYRFV=)D='T%/SFR8L."KN1X,.))$_O(G#$@BJZAQQ::=:O9@P#E>):K^$P:`
-M)/(2)%MDIXD\H\$#Q;U\+<X%Q78IBV03#S&Y-'!GO^8?&+5[1N7YDXL`W(F(
-M:`@#SLX)Z!7$2J)XKPE]YV!;8R"SC)5%^29PI;+(+Z9%G)I)^?P$QPKT4EG4
-M3H*011*R".H9)5/ZUR<PF6<5U]-&Z)@NG+]Y/,5\BQ2P(P]!.N8+G6"I:+D5
-MC8M%%1[_?D43W^S`GD\^2"202V;#8G4TRN--.^TY`H5HMOQF=J</;;(/K`^]
-MT1)^XX2\M@KD+L][3]DL2W$R"O"5EV?*:RO%-_JA/[^Y&/P2<:-+$\VHW,3J
-M%U)^X.:J]>!FW(E:*$596ZF1^BJ/Z5@DQ7N5VRN'"*I6AZ0\C1\Z?D5;1#FK
-ML%Y*CF\BFB[A(.?X%H.`@&ERI-[J4JD^IO)V>.POG&3>/DUZ=LU$CD6>S\9G
-MW2_F`)P\%_#Y65!CK@??*`VY41IQHQ(T`5R1_[NY9J3R0[3A;Q[(W:$T8HJY
-MB%?/M.G94+?6WH"Y:CP:YN-1KXU0O<S6HB,Y83>R2M,JK)89[X/#1^*A#^UF
-M[XOW\YM@=D\)O9$6=2--NQU"V0L@`PZ-$</ZA#MLVO2LO?T4_&1G*8V^DMZV
-ME'J%W!@(#IPN[W"H/T259E_H%ZLMT!,`;M:^^#7.PQ0N2,Q>7*!^+`:.[R?/
-M^T+FF8W&1'2DB;P)^Y/P&/YE=I3@($U+FYJ:.L)N+&^2[O[4;)VX;+U>846P
-M_9AYX%O>/D@3CJ!4_`"3BN_1[NDS#VS7[+EM=CNT>VZ9!\H55B9K)Y>5BZ[,
-M0!4%3RY!T]D(BHU(!N38L#F>CM*D9U-;0D@6>\@`L31S$4X_9G%_H8!]Y9@7
-M=>-5<$-C]JVJJ79$W?A48^1LV$I-CMI"(W79IC6*^!_H+0/3]WXKV18,NP?V
-MB?9W,>P?T^"'"'6H5\FAL;>@C+^BB;3]$3<TH,DTZ+SLN&+TVM/F@28W>W!T
-M$JGW[!48$VF/3(-7>=^R5-N"Y1YIU&Y`=I;9@$%9C&E02O>_8$3(#;^P&WY8
-MO)#RZV90?MU")O3GYH0&WQ@/7;J=7R@.I,O7HYQW4,4A:BF)&)XN'9X)9P\+
-MNG)_HU:D1J9O<K#?IMQ,"/X0FH9G<9-05>OO*O@"*#U.'<1S+P$J$7A.=8+3
-M8$S3SPQ??<0-(;@.%C,GT_S^1MNKKA("O88C,COGSV,EP.0KTRP*)[*F9LS:
-MFLDK9^"K:J:F[`I.J1-]42,$U<MKL*:\937,IGUR3J2\`5E5HU0XF=S=W"O.
-MU0=1Q3=/KCK.?`FY*^$4Y`@_7M=HKCY"K46:US:8R!\1Q+R@'8QOVJ=H0"!E
-M@O7^X5N%)`+6<D6](E^PJCYZQ5%8T=-'`ZZW_%>+@!0*Y0@LU>=-Y"^>/47S
-MC(_CAB-,YN2@ZU^KHE$,FV8!Q;DZSA"3R8),)NSZ@?E]LY0'(()09VM6GIZL
-MF5MY>,K9GZH$T-PY_^HY62IOSNYI7YXW/O+ZKN?KAPP;OB9'IVK.PGY`+<!$
-M$9CK%U=4Q'C6F0N-B!Q113/"KG>L:A13]"6)S#F=:*0M`,!$ME(49[X1\?5R
-MH,\LD,MX0`.2+"^U09Y0!^8G4+QF'<5K8`N@*PFVV""I83TQL"KB(8\J\Y]O
-M$Q!X\HYU\'6<-#FUD<*4IB.5HN#K:6NW8YN!\@1+TRH`M8H6FL6P\\A?NMS`
-M`-8#JU^9]QA]6LCU%>+K*XA-:SZ9X/4+'#8$Y*^>^@@X\G4-ES#,<+8-_.S7
-MQ?7(JFZBK.7USZ&UW`ZM9>RC-A5E+;&A77KHSG_.$AO,A>NO0H8&(:T]B4PV
-M#6;(<2SLNAFFJQ3NMF]WJ3JR8DO;L-^YJ@O4,D$:*.;DQ(-UGH\A(.?X%&)P
-M$%T31//8:UJ7&YIKY,<"4:&P**#I"+5;:L[SMPE"KALV9]BMK$8A*&$WHA5I
-MXNMZX)1[F-+0,7KJV.3`'^L.5_3@U\CKKT1<?V7($X0V#D(3Y0L6HQN@+Q@E
-M45F?^X*W1=??V'I@VZI%Q\>?29-`IC+D#)YY4WT+.H->3NAQHB2+(M`#+/3O
-MF-E0(UD:`=(ZBHE2.FR8@14;OV2@`PXW?16VK=5LG0VB8SF(5X[9\"X4=-*(
-MO>R_4;#D$Q[T"-\5_.$/-E.^8\P_K?0OAGS'&(YUZQ=G9\$\F=!Q9&S#7J3M
-M*`8S%!PK=@3;N:S.1BR-B/U3Z9`N;D:`$87.W<#?3>1W"$R=!E/7K5`LC?!4
-M,Z:2A0X7-0%@0%NQ#BLVG?(7O4?^WX*E?=+T)W]QU=1',8LB3(/7H?1)4#JT
-M<%>'S@1$HZ'7KGFXH0L:Y<AK+FI#$+DFOT[M!BHA&6S$+,X.56VEEH.9P6EU
-M-(,83@/&JT9J-O\6/_^2X,\'FJF)["1TK-,9I][Q[FPB&__`2$2A$T#:I-(%
-M(/;AR-GA9L<DA4X4<>V?2-WP96T^O"ULY8FU8==^37;X\*K9BD?6\&O_B+CV
-M>'?%9>?\>A_Y=81]%,.VA#?LWMNJ.)&G/)&?2-MI(DL11*WS#[_V`+DR?%F/
-M#U;%5?2N37;Z8-5<Y2.K0B>$TJ%___QH=A+YZ1_[ULA*RMA54]_C7-O-NB2L
-M`)ZO<62]U-<XLA=?XY#*/Y_AO8!_/`M[+BR1>IHRTW!DC',Z;'&.5(W[:7#^
-M-E=]OT;*=[K44C^BF*DNIF^S.6WU+G4IG2AE)OUVZN,9+P%WNE1AY(NO?:>0
-MTN5&OR)<+F5"6E.-5YF]Q)`J0%\N!?52J9]2RG^^5@)UXK*>-U9X"<^5\)*$
-MW#HL/9J>A)6G8W/XZ'#NAK2`:\O.XUNS-PG,AG?"KRV,N+8`W*Y]"SP$#O8_
-MXSF<":FF-\YL4UU8TUD,]>'KH?F8F98GV#:;T8B9G1M+T[+33\_U<,.%Z*_Q
-MJ6X(S#]0AR[O`?Q8CCW-;MV0?HZ_-.:!5/VD-(WJ0S03VYI0*F2\Y`\EG-W]
-M7/CSR7XL'8S/$`(C>P2+_EJ\K^^$,V:8Y(^RA8S9V+;,8PF4L`PX[Q_'+W5_
-M8R(/([#\R<_+/Y9)O3938Y`5S8L7BJZ]FBW9)F4E8YLD/B.X)=&+QP2<FNLY
-MG*([)K='V_$2"747,^Y)2;2)O$'9Z&NQ8==B*0-]CC+00'QM-O8789T-@X:Z
-MDUIX';+1?QFRT7'IZ?^RZKID/$<3C?J]0AVF/L>'?%2:GL%_%95FIVN?K[GZ
-M7<"/Q`=?PR[K["&6F=-`37F4]D.H9:$4`7XXL4#'^WB&?,LSR_KXHC25`^'G
-M&%('5?[77I4&"="?:0L\*_=YUG#W;]RPF5@C%S151"H/+-D2L>@@?+/#B%@0
-M_VLO.Z?R*X3*(`$[1#3_>!1XF&^-=/=CX3/]6KF1U]1;(E7-2ZHCUIQ=9D1&
-M0KU3&;E1U^2<5K:F"5,U<4"]7RL&W-BW\541><9PMWM_8]0U');%[8<F<K$R
-MR&_MO5>N&,'72>1!LZ'42$NG>!H7/.0TLE7UF+*>`S,W8N'71-Q!**+(&.9N
-MVT-R22[?D[\Z<M$-H/$70&I7%0ZM,+.5I6W./M2_]MZH4/?)%^=A(;U;<2<6
-MVN'3;=I%56'N[^8_B?)8<T:A<>4=D791Y9;0ZC#+@-Z(,*\8D\ABRNX:*8-<
-M:#$,%(1TZ:^V%[1[OE-+(D,['>4A9L?E(SLITUM$F5X(2=51ID%8=]6'#+"G
-M$%I@OT&#W^[X^:?\"Z,4QN1]S[]9HSXRFFLIOI,3RLN;S.-\P)MN:&AEU`DX
-MKZ.,5NJ`:_GZBN(C=LL,M8>A#ZTUF-O?CW*71;KM_[[8`!P%D?D12B.3U\J1
-M-R\NB$+MV%[8N9.59J9-)V]9S$HTM.42_@+@]G?G:XW8'TZLDC++W.>'Y\!A
-M`%A[XE<YJTUD":7XD`!Q)IN>[07-8O>J2/=7.:%%X6HCA_V4Z^="#Y"1[N5E
-MQ5O65Y0`4+E!;<149J9GL6*'A)!DJ<ZWL5Y\#0?P4/<.T%`A+!>8B]\H".7:
-MA5WM1ZXJNL2[+A^Q]UB[@4N`M-LJ_1%W/C)@<SFJ`A1[\&YGM4AY#(?/J@*U
-M:?0CA;4_5Y\@\@6J5OC,5A6$M.=7!B\_^IW%H,\5FP:?4-_1>=9.D@8[38-W
-MD,(P<)#5)N22:/F-,/<;U=<JW)'N\<`)AUII3E;VUE)T?2DT_HL\=/TJ1#\?
-MTV#]G[$W>P3X'.B,"*U^!%1B9'$H8RYML?&AQ_^%5,S#X1Y'N%5A[H4GJVU.
-M]BXZ.Y=6[E2>0Y=;PR+<GX6[9<#MY]W8;4[G08>V)9W;.^P<OVF`_QUT4=_0
-M_HCRJ7LH+\!M"G2_[^]62C\I#G%_$.Y^3^P.#','@7:>5\"!=%D:BM71**A(
-MILFR4-_>8:6:+$.)>J-6HM(-\1=6D'NFDF-6A+X4[`X)=_OPU4(>(3QPV<\E
-M]+,+]]AVV'>X*H/LPKK<\D"B@*:RULNM!Q56)V$]P*H38G5#YQ)9=>%';$00
-MDLC83'QN*/<G@NBY@7G^"H>56T=0GWLQUI]?F!^4%Q#H'L7=3<2\VU0A4K9(
-M53KKHM.!^2*%0:_\"X[5$8F,=>`<QS5*/2F^(>'`G`,CEHZ^<B"U2%@89`ML
-MZ*^WE056!:V]LW7(I_%L=\%K:,]=A0-)1*$1A=V<1#X-Z^D$)^3UZ*D]V!9(
-MS]]0?85B5+]%N061/=\#*\O31>KKZ7SHA-ZBH-33I\$]?P_L^3O`4<]KZ<QT
-MR>?H%V_XPEMJA`/<OF(W.\C-9DVA^<IHLB4H\6:Z?\]9V*\QLA")RNB1$=CS
-M5<S,JO">`POBH>(JI<VR%#W3AC.HSY<CB)F53<7FM$`9CHEZ]F`?";],52S*
-M9I@YU%(7#95.V`H3Y^`,.V8I?B1=E!71L]VLOQ?94Q_0<Q\<I)R!GIVPV^VP
-M5U%06IX&C@!'10:4JCCA^:3GS4IQ3YE]?)F^""]YK22A?'+5Q/))D3UEO&=H
-MY80LF>8U0?G?RJ>739&_)JQ\W3XUHN>2#WU*2,]!GQ'"DC<J_QK0TV"?1B=1
-M`N=H7Q,I7@N@VU#B-7\-SHQ]+<A&:"8IB->"%5*F&O>5XF;HZ[2UG_F;!@\*
-M[TF/ZLG9\UG1+XQ^C-IC[`_=5U@E5R^B-E+474;UTJR-<N70FA^Q*"N@9RF1
-M;R2&GFZ=JY"V41U\4NL-S]=XB"U/S=8:!;1&CLVY1($,/%EQF:&41)WZ5H6;
-MS2"C4-[)]]N');N70Y<JLN<MB.;[6HGF[)">.<M^H98T7GRK5XBS*5^]6-Z<
-M)>Z98RY>5U8,'HE[EE6OU^`<,%!=0JV);BC$J4LI@3/+-U=N(C9RM-0!&%\Y
-M;M;<?W;F)L!#>J8HFK-V'?4>RM"6MCW?'4HBT\S[?_%,1\1<_:,:#XKLF<1V
-M884W0GK&(->H3===1_?NK90#'`J`&;T28/;HY]_4RSA(<(\LK&<Q@5MBOV$0
-M^8-F:QGV5^'>;U0=_U!V/%+,0%?53X"_#8V:CE^5%QZMO..C['@,(ZN.*D]6
-M(^,9\`ZDP7OEA7\@ME"VC>[KF<;:1>DLDJ9=2NW4&F%9,JDEJB<22BS\&4IV
-M]E?]O/)LD+SCL;;C5\CVV23&(C&^5G3JJ1'1<;_'>&K1#Q?!=;]]J)_?&YHY
-MJ!^E*YZ#I$GDIK">J>;];\-*>2IJ(K_T%AG5$Z-9E,[O1S6>4KUZT?T3T-&?
-M*U90]]W%DZ4A/7\1=__&GDI;ERY_,_W[+T)[1F%*X>XZ26V&HC9S40T]'6\E
-M5!^B`/=H;_WJ@!Z=4P/YBB3?P.H7TF&S+!0@:K(#*].]\X83T7T[M+LSIAT)
-MZ9$M&+$\O(<#B,">V,B>&&6XX+PSPXT8LC<*$'UNJ7B>X@W^RT+L);94;^6\
-MR:D?0<Q=S)O#ULZ(YR<(N0D<UF:,GSJR2`]_)#-&,TFL6K\S53MW25SR8GX9
-MAMI&KAO/M(UDI6+[B[3)2S:/I],GU%,><)F;J@LL6=3#DD[XRJQ["WK'[4$]
-MS-A6`<,V40D0H"=.(/-&CX6C!KV@S:&I5I&V=DFA87^_Y@'NVS^RRJ#>OL2(
-MB%.=[`Q#I2&&ND%C:I=(QN&^DI&2[4L:YARI@X5`:5#04/\.`+WW4)#-$-#=
-M?S8MB3P6WGT&4POWME(;]LT9I8:5NT6:YDRH!9QO,#\[QK-C;+G(B/AP=U$1
-MNRZP.TLY`U6H?H=6)Q&9JAJ.J%I)U4BD]<2I6DIYO?(E7Y-1W2LU/Z>C_33U
-M#+3#H7F<SO-,"UB/?_LW(A@!W:W@'#V&'3,L"S@6O`35,B>L^].3;>+NK9A<
-MN,LF2\^`##:D^Z.E,R;%S40TZ5FQ[S(H+GO')S8]$T:6OCU;DYZ)1#.0\Z$7
-MW#^ZL02Z)CM=DYW!\50F9D.ZMAZE-BTVI?.I&4Z[Z%CZ=C1,#]]W+(0ID)Y0
-MC*1K-J03]2B_GZ;9E,ZB*DNMK'K]%<95738>V/T:P4&(6/3Y*I#/OWV#?"NT
-M^U7.ZQQY[=:3WRU=3SO'5VY_88?.G6P+Z1X5UOTR1@AW'EF7E94)',Q9F%FW
-M/-M0N*YASL;,)7?H6=D;,U/?_BPKNT$#(6%CYL8LV"+8FO_8#LCWL1%#KR0;
-MTND2FL3SG-HD11#XO$%*-6T\PYM0XLG]1RHX5A<<J3,N>),/?=O9+0OKEH1W
-M2X*Z)=2WG=V2D&[._\JWG8R0[G<PA7#7KISB$YG+\U:$=4_;O4?>G*%JSF2W
-M0O_[8\4&1'E,1VV?O<]5<E0'MH9TLU(`,Z+[KYW\U:=?WR7WGN5(63]\X79Z
-MI^-,5<H&P<G]BD-;X8Q+(O=U>$A)0/=$^E9:_K`(UTTL7\1KI:D.;?7\*P-#
-M2/O]<T?3VZZK.M`>XKIX!2=TB$3'9$WQ]WX\WH7#<0MS=6"O>PZ89++E7/GV
-MS*'O7,>'N,Z(76<QF7!G<NS<[%T$865MG4,=L$@(]4G`S/K7)',S`UU7H+]Z
-M6F*+#G8ULS)%S`3:DJD^IU[V?N".L#B3([N#BW#0S.=/W/5,==[!)CF:QU+E
-MW$R`<^NXBN3,?Z^GV+7W"A[KK6?BBTKF#&/_TV!VO,,^/3-EEBS"]24V71CE
-M^@I9SP!I$,@U#]&]K=`P:?Z!*ILSN4JNLH42_##<58%-$T:Z=A\I8[$FJZ^A
-ML/I[&A$RE.(T-U'%@4R.@JLX1#49]D/Q?^X'L2LGR&7#XH3;93(C:^>B6$DV
-M\B$#=D5"Z)+Q+(DD$_8`Q?6]?CUPSH^!"EQKQBNS!`&N%4&NRT"__6ME[3YN
-MR/B4DO"4#<*4]4R0R\Z*-R*B6([9B/@AL7_IHSX31-9^-SK0]8-R'$U16Z=R
-MULN=3@[GKXH]/[/L0CCOEMAC5,X#:N=!5JN0UXJN."TZ,D*I0.H3%DVM-"(:
-M*,1$-B/(P@^X/@4BS3A:;&^]NM>Y<#^74R0BH,S,@]K,`U?T2<^FF4@*$;W[
-MEU\$NISH5!'*FLBW">5O"JD1'AEZL@A)#97.]&>ETN&8^B:@TC?]O>W3`2<P
-MKA.$NQ("71-$KH23@]5"(S(!5KZ:7Y\,W5E+]2-@Y.Q"U^[XZ8K1)P6.H0HX
-MX1C"3NMP@/4G]R/V4(Z=+@^B_3#"C+_?B2MQYD5CN$M7((2,?O7.-[UYKAAA
-MG@Z'^=4KN<,XN2)&'<V,<_*BN7DB1B-MQ=%Q+L?\&D:!@;<7NZICKD)#74D^
-MRSB<*T2$:RV1/I97QX+9+5I')PZ<X:X0$^FH3T:,#`X;1F<C2)=S?CTCWX`=
-MPR)<?XUT+>)]3^QS1;H^5]]*,Y%;*;[LXH:XN.&N>6&N^<`*/C=Z_FDIR396
-M?9U\BX!;"A7`+-N0O;PY".#SY]ZB#IA+*`!>$KHTZ9*)[(>(E,4S">FQ(O,'
-M9R\8Z.J)\R[3R_`&2:?5-T$T;ZJD!"^)+HDID93$EDA+XDID)?(2HD19HBA1
-MEZA*M"4:B4IO(M<C2&37W;A;UKA;:=[Y*./0_+MZ0UP315V/`UQ38H:K%7I$
-MOF'?_*W4UCJQ9Y#-TZM"FY4<->_L:.4D@]F:DT2^$^AZ)=0U4OF7/LY?:(FT
-M5.5X`Z>1Z'0HWT02:=EY\(:EH`XC9(E<+ZNC4=5?^K"QA*^+VN+<HD^DK<[3
-MUS6"&A@1N33@>KA+`Q/!%'XO4BRA4B@X:")ML2(:!0W4)Z/4%[KP@176^WBK
-M:I+!1"X!-T-<>*`+CW)%1[BBP[OZPURX#9<UK)`UK)$UI&,V0M:0S6+Y[BW8
-M7_1EUBZ;)G./+'.W;/8]S>S[LD=*S2.5YOQGLO.?FD@%U1?''>!@$OG&\1IP
-MT+^+/*Z7RH+`ZU)9<!+Y.A@K[MIW?+U<%D18B1A9L-E:I.`,5W)H2>3$X[6>
-MZ_CCM:!>(0MJWA]#91G7E"[GU)P\"$XU=S23H%\K"XKJVJ>1!:ME0<<OJ&3!
-M3<>E,J%"XL^MHWD:J)#Y0W^#4TN5(0M6RH0:B3_%YV3^,(>*$DF(N^Z"S<V[
-MF[Y2<6I5LB"E+!B6)Y<%$S+!.3XL'M8^RD3.I]H2WE47UK51W-4=W-43U-43
-MV-4#W."\J&M[5-?'$@Y+^S,R3S'B"K@,5B6A_--8>%<UMH78>7Q[LV*D51&;
-MN:UY1ZM:FJ^4YFJD>?6-FI'5JI%5JI&5]7O"NZQA7=]#$?^6O^B/_$HJ?WVC
-M1P+,KH!"_L@/EH=TK0WK^BZ@ZYO0KF_$7=^*NJQ!7=\R'A'^7=\PDC&&CV_M
-MS)V)=>]LG56ADW^)Q&7F0BKWI2(S7YZ9%]QU2-SUU8ZW9",C9",C-2.C@KH.
-M!G0=W#Z',0(C9*<T2Q#U[#V:V;LUL^MB9+=B9#<ULMO>/;+@KKR`KF+IHG&2
-MX25KM=QY+!\"1HI0$_D+.`&<D5TCJ:]AV!^/EO<2\Q-H%YS`$-)5+.YZIU00
-MT14NZT6/$SU.7^UH;2\MJ"L_JFNY8M&D@O;@KNQ*8417ZKQ=D\X;#LY9D23J
-M-OAW35_R*:-0Z/<^AEXG]DLD%])^U`=T35[\*2/K_VGC;$.B",(XOCN[9WG>
-MY=QYB)QAXYY;ZKWLW%V<==&+YE46E:"]P$%OUZD'ZD*7*'V07(.*(CHU"*07
-M*DR+P/LLP0E)"D61%<'.ETBI_!`JH1#F-7MY*1@L[+/[W_DQL\/N\W\^S)AT
-M`0-74QP/].\8\#SV"KX[@N]N$7UCOGOVB6N.B>L64C^>751Q,(%YBV7T0-/:
-MC+A7B\I#R=D7U7GD%./FK20K$>]^R`Q9!X=Z'Y4H!VW*H6*E9GC?A<L%^F?`
-M\9ZY`>6)DV;"ZG8:)-%05&B41'V12+-*'JFZ<C%QIOTRV?B(N0JYC,,)M]R3
-M3^YGY%MHJ1#8JALRV$?H8;2/Z.TC6AXB]^EM*J]2"M3O_':#<](@?38Z)_72
-M9%8Z[^:I\SE$LA)WHEKLZQ?[!L2^)V+?TQ)/P**.95XQW-X53/::B=,E?%FO
-M?EIWPG+WI5.8U-8#C=J%UUE!2^\8_0R#R?SAP0TDU_&+VN%MI>5`\OF7_@,!
-M<)QI#RZ^#2X.B!N=P<7FI9QO)AX%`>78:&$HN6`F+NX;YG]@71QWHY["6T)J
-MRUI(2F-,%]L->C@3,2J0G<(/3.RT6W/RN=3?F8A-@;;]#">`+FA2YQ.%79!Z
-M\$M<)U#8#H9YAUF;MC\<)-9_''4AS9E9P3$I,$=]GN*L!(QCUIMJGQ5C.M@P
-MK.H`85C9P;W9D*9]3]-FEVGJ[__0$C#&Q4!,H[['[&[ZU/Q-&$K.0W5NN6\_
-M%:@-9+`3+@UIQJP.=\%E\E>-/+"JGQ\P6Y.:3]#LD3XBG<SS._GSP5)JUC0/
-M[+4*>J:IRA@-\5R=(2YD"R8NQQ(-`<%!C?JH7YL0JJ6J[;]>!H`IG#J_$KEI
-M6C5D,[*@XV9HI)-M.FY6\]U+.IA.^5)9X,$,C60;#V9QNG8`0*LE],SNAG!C
-M-"*C/<=09:#BR-[M&!57M$0:SZ/:EC`J;ZE'GBW([?5[R_P>+ZJNK$4>C,N0
-MW(SJPHV1-M?I:"1T-AR-U#>'ST5=(;G)+S7(36$II4JM4EWKR<TN7XD#':5Z
-FA#:K];:U(8PWNW"9"V/D1&YZ@<LPAOP:IG]3YMP?6%ULKS1=````
-`
-end

sys/dev/cxgb/t3fw-5.0.0.bin.gz.uu

@@ -0,0 +1,496 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+ 
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+$FreeBSD$
+
+***************************************************************************/
+
+begin 644 t3fw-5.0.0.bin.gz
+M'XL(")H1MD<``W0S9G<M-2XP+C`N8FEN`.R\>5A3U]HWO+,3R+1#!D((\R:$
+MD#EQJ.+1MM'::GL\%6WMH:7MB2:A=%"I57$VC(89%84P!K0.B`K64VFQ[<):
+MG&A%M#@1#(J(%2EM/:T^K>QWK01M>[[W/-]SO=\_WW=='^W*7GOO-=QKN'_W
+M[UYK;<W8<HRDVV%(Q3!,``.)]6%X%L;$T-6!T3#,@WF#"T:QF3-GFFY@M$P2
+MAREI'A@H0135B451`+N!,:;AWA)><Y#8ZTWPV@FOATDL20"O+?":!:]'28S>
+M!#"L?QSVW3BL;ASF&D?',?RN<8G:#[L[#H-Q"2:BK@*CF+IRGH_^HS\3)9\C
+M6/;\B.]61'T;P\'D'.Q_^U9,?4.?$26FSL!?$=4!C+K)\/^)NLGQNLG38IZD
+M+9MP%]8AHHX#HV$R9I@L,$PF#9.-?WCS.3#&3<;B)@OB)I-QC][0T9M/`JE/
+M"1S;2GJCGXQ%6\"XK60&C!P)I(Z@!T>"J(\DU,?`>)'T2Z:9L3#LT1]CDA^L
+M`HFY8IR8.4OL-X%@,IY9.6X`OE@QQ5?][B!J!\IJQORQQTD83\'W(UXAZE#/
+MU/D:N]5T>.:RYW]\U/+*3)/W1T15PAX24Q7_VU3;4()MI\E,,BLZ6W94MGRJ
+M("?A\(+MQL,3ED^X9Z$&H*"VISVV?[A%U#I@5$XV*2?/5$Z>K9P\YW%?>"5=
+M`\;3]7Y>H58%4NE;22"D/3\.FSV.AF,`5K^!T34NB%I?FB"FUC/.P^@Z%%W'
+MN("B\E=@BK6,;V%\+8P+J#5T[/__^W_9GX7Z[3^^B_K-%VY@?B?_8QSB@@\;
+M_#K&XM*QN'0LWN^+'TOTQ8^9HZ@O8VY@+"V\&J.HKTIA_#<20[A">^<&QK7"
+MJQV%&Q@!\06'SW'Z6#P<!A+&+Z`K#`H8OP2OTV"`N$5XT!6&YV&\'U[GP)``
+MXR/H"L.K,'X/7E^'P7P#XZ&RS3"\#>,,>'T/AE08%Z`K#*MA7`RO!2C`.*JS
+M!(:M8_%&&)I@7`ZO2+X?85R)\`\%&)_S/X^_WNB+O][HBR?/_D_C`?NL'`4?
+M!K_N0?B,,3#RT3W&IRAOW_XV"M.VR4TPKQ\,=@P+Z:5&L=[]GW\_'=X_@<JD
+MCCW$O:6S"&@=:/`^E?3>"S"C=TPPMMU;^7V4/Y*&V6DS?/GVP[+?@_;#3E&_
+MW&.8O%DP`E[#3=[R[%+TZWL,Q8/OO.4:,3-\"LL*1[8'PTPH3OO-)Y\4AE`[
+M3(V@PN63"TO`,#I%?0';!&`R"6PL;"`U`M]1OWSHE07<CZ*.OP_+@]5Q2=)`
+MV>&]#4(43`+_1JG[&*;U-@:6A<.^LZ,;BKI&P03?TS`A+.-X.@Q-:'Y2U/?H
+M^:]F5#W_/GSWA1&V$=5_#SY_B-%^[83I4F`?"SRP/V&?`R0]+'<F?/X>[)>5
+M4;]1EZ`\Z1MA]7T0YS$L3D#"#KT&T\-Q06.3"NTORF."(1&C=Y;`ZWN:!(M=
+MD[`:AA(8=J%VK(;R\4D#K,>(V6';O(W":-1]3<(A#`86K$>@26B&Z0_!\"GL
+M34RH2?A,J4DX"N\_<WE@5\*R9T<]Q%CP^CH,J3!D0;E*8!CZ3_I/^X_(\/_-
+M/U*`F1X',>Q+."$?AT?/'@4I!LAP&O8XD#23]]FC@)[]^SU*\WLP_RG(:2[J
+M#W__6_FP^1!#YI7[KB@D_"',9?C"BQ`K7X0X]3>"Q)Z'^CZKY0_Y73`_Q,_Y
+MA"_,W?-_#2_>\UWG:&%^F'96RN_Y_2%.3H#YZ!`[GUD%[P=);&H/O(=A.L3/
+M63-)+![BPR3X;CSD>N-8)*9+(;$XB+\33Y)8C)3$HB`WY,-R0B$6!\/\P?`J
+MAL^X>WSY46!.A`'F8</P-,3KR1!O_>_3!_UZQENALH]=K?_VW__S/\'__/DS
+M,?;_<;&OE]+>*,7?+*7_HQ1[YO]`K-=+:&^4X&^6T/]1@KWT?Y"_JA2+**%5
+ME^(UI?3:4@:KU(]=ZL\I97+_A^)4P<PE>$T),Z*$7EO"6%7BM[K$?TT)<^W_
+M4)P>;_WN4KRWE'ZME,$H]?,K]?<O93)+6:Q2-KN4PRGE_G>R],#,)7AO"1?6
+M?ZV$<;C$[Y\E_A^7,(^4L%I*V)^4<#XMX;;Z9$%\!?H=5@%%<5!6Y(M@7>,$
+MU*CW%MUW8N?_>)\DP"[\Z3X+^_;W>U2>!^*CB/HE@\0S_GXR&CX143_1!XV,
+M8:,?,&XE2Z.WR;PTB>1@PN]#`K\/H0\@`BSY7AK\O?0\/^U=_Q+SX>0TR_V7
+M@[+X8ECD`PQ[3\``Y#D^O(I`IUP6J91%A'\?L#Q=HS)AJJK;MLU/QYKH&O*M
+M*PFVIE_RQW-'$ZS7&XO51`N-11'6'V+!%C-V6=6)7>CTIA:JR;=B.^FV^LRQ
+MM'9OVE4/N@LF@%3B=H)UBX-($A9_4M22<^2*1R$)W/7SQG=IS=_%243$G@1;
+M>KEC\D7/P5QGY*I*6O.M[5)0TIVJD@@V1:ZY0:RYS<B3;MS'`.E*B4`M$MCJ
+MIP-7[A/`KI)@JVXPK-?]0)VM7J>1A!!S7@;-6HGD:`?6QL!:0YM:-9)@[J^!
+M&HGT:NH5<V+0.=`)2I2B$,;K'+4HV""2%DWD+Z7MN\-90].+))IV245NK4,_
+M4Z"2OZ4VT1W/-N]5S:0W.5:]:H0-U,7BMKH#&EFD5A:Q]A0--)WZ)'#XF+:)
+MJ6S"KI!T!T>9,0><X]JQJR[;P5NH>PLF$/\0HM9OSCLXUR`)*;J_Z9<KGN('
+M<1+A;E_[5WPSQ=OR_%6MJ-U!W:D7FT#3FAN<-;?]5"*_C94TT&QKGI?_!'`!
+M.VIL'P_460\&F6E/8ECNY/Q)*DD0]Q;!?D`X(U;_2FM*VQY<*^E.!2.;(M;<
+MX*VYS=*+_-4BYH9]-/"#[8>`_>\4/Z$7"6O##FP`(VEW:,T--2)'&/`H);2-
+M.SAK'OBI1?B:?!KHLWK>UTC$9JP'VS3),3E.(E9*@IBW"-9]HCQJE;>>T-H0
+M;SU1OGI4(K92Q%F=C^H9V:47!:I%HN(G&MZI#?^]*G%1>$T@\&S<0:QYP'2(
+M#2(Z+`KTV3PKX=AI1/!!$!K!OL<C.!S:-*R7!`?\5Z!:(N4/T[@-M(9W#FSP
+MEM;MNNBQC,9M_`BWC-ZUC`YBV,:MN.7AORP/1ZXD6$9G@,V6T<D8')BK)JV)
+M"<[I.?9$0H9&!91:1B/`=LNH%/T$_D$7'E\U7HV`?5ZJDONK8_&UN;2K+C3<
+MS_UWPXW]4:_\+YKTT3@867&+GSMQ8QDG[XF-!_T,L3A_%`_@SK$U;Q`/3P@=
+MGE`FT-77@-/KO\3-M&S5#$5*"=-!LAV$+3U-45_=8[8N7*XA"=!)'!$VI"F7
+MY8"J55MATKDAP['L3SBJ+3M^SS`U;EFV=#C"^O[X_''<7#'+25NZ6:&*9G!_
+M3G`*EY1+P;:K(THRV;\R7E%`\_>?I9`GKVR%9:D5'=+<B>#XQK*)8<-"QQ-5
+M[[-XILI4OEN83U:G10SS:I?S6O'*9:"K>D78,-<--"OI&S>QE"]@K$%<&8MO
+M*&/D3]+$RM:7,8HFZV.CUY5-TD\C03.P^[?$LU@SK/:,L+NC(7='51]OP#P.
+MS%SPQU]]?;5A9KU^2*K?59UK3"M[(NSNOQP3BL8SN[GR:,9A-V.FL&IAI3E?
+M4VNK22Y_*_+N,+O<Y.\0LF;B`5"J16T%,%9J/74QY.YM)4D01<*]CH9A[?5L
+M0]99HH>K'<@^^[IWU#>!DH!/XE<[<`NURD*]U_%W.#6,/:F)/!-ZO4EPA4P,
+M<*#%!V"VC&9?(<''B0%9%JH;4VN7YPL2`S[H<IWC;_\2;`V[V^D05@*60\AV
+MX.5M4'H-;(,\67-4JME5;1D]Z)T'_MUDR-U/P^\>52[$17<_8;'FV.Q)>4;.
+M._/>WO&21E:SI&6J>HLBQ<4H-*^Y;-3+JC^>?7I08R;BYHP2GPKK&U3U.>[4
+MU?DQQ%&.]MB.%,!PF-?E!FGKL\]^98:.X14SDN_UDZ^9:9'P)C'@5=N\(-@>
+M4Z'QBIFW>%XB;^"\RUJ'9]6!>0S&+`OU/@::\L89%C(X_Y6PY'XPV':IB5,>
+MKS8G-S=P.+/4UN0U^;@9ZZX25`MM"4?4TP2]`,P"/P`[<=\WB*51=PM#[A:J
+M?M[PSOL$&&FF,)<#\Q2D/;@'T=A8/('M,%GM\QWCB0U<,#/B;E9IGWPAH]H#
+MQZ_PW<J;M;?*!Z+N;O2-'=^-PT$[O+&ML/HZ&KW^R+MK=&:"N"0\X*[/D,NR
+M-1?.,BBN09&-^CSD[@=*,QK8^N&]#DW663BV72[?T,)AM7/NQZ^:#0<UQ4*]
+MCJG?6PX.N4V)O),%@D(A:+)0%LP[^IFL!\**&RPG+KUK=O:5>Z1W7])8DU6R
+M:DU6O:I*JE)46RC<I[\GPN[.#[W[TD73)?*R,2,Z\.X"_#GNCNR5+XUG;\7E
+M'!KHQ/\6S^/-.-6M_GA#+TD?CP?<P!>O\<-(!V8LR$B'M8VPY\9O!-<M5`_L
+MP02\B-@Q'/"D"/]7O)_?#'F/@+%\TIEY,/-YLJMD\0MC&7EIQ-[K1S/!=#!"
+MGQO/Y\\X]0`F^7.Y">Q_Q:\`)RQ4&S#CS\6O!-<LU.7W!"P]Z5\J3!GT4PX]
+ML&6T`I-U_IOJ:#SD;GB%@,.98ZM?+KT;$G$WNN`)C9%>)29:"6L3,)!$D;@V
+MD*"$>ZA:D>''G,/#&V_@9CPQ+Y"]%R&+0P2Q)9]DMQ*V0\\H,[)SQ=:,J=82
+M<P$9=3<P\J[@BOVR,>*N(%]<>.7;IHU'I"UYJFF"G$FL$\*`-+PLW3]'O*N;
+MGXQOM_M)Q>?YBW/C'-M99X0LAWC/);[#J[QUY2Y'1>6.1X_*Z]6-`E45F2>.
+M'/JY,#!DZ%>'(%\$E4Q[*/N`6W[A+-'`57^<#2''B[&&C)HKG4OO0P1K5C=.
+M4-=#98+B<J"XS2Y#1G5;F_6'<AU)P+ZI#B3<PCWN:I&N*R<OT(NU>;XV%O[>
+MQN8T75>V]=S[$#%3;I%YXY">0-"T0?M!)G/>C5=WT)"BR)/7(-#\V"$.'[J0
+MKS-$1TB&SG-_96\_1\`6=:J-@K"A3J)!P*G`N0UX19?5/OM*YV7C17N/<>.F
+M0.TTP=$KBO&2_"G;NRLO.B8?GE)^B>GKGFCE%DG^J?)=E;L=)_*_+M^CW!)<
+MV5"^UW%&VT&6[U,W2EN<DJ&]#C%8_FZ6[(H=E+(<;)Z#PW$2&\OHI8ZC#G:H
+MZ+)I<8._T^',5Q9(X5S($P4.U313JNCPN.Y`1;>(SY\+BS#$1FQT^LEC(PP<
+M+'AH=_A097Z@WP-A8VL!*>\.5G5+X[J#%-T2>@>N[1:KZK/EA\^JJE)4N[*M
+M]M6VA&:?I=,-/;!V7D(06B!,Y'Y_W@7F%3T![O&')]4\(9](MSS<@&$^B$7O
+M?>;Q"ID?F,@S6*B;R"Y#_.)IX?3%QIZK+-37&(:K<?I47"NBN1/`$H4\)9$A
+M1!B`82?_#G%8(Z(E\H30.L?)(L[SUZZ&),S:%`9M<LB0V6N6V5ZSK./0QNSR
+M#50Q`G!A(H^."@H?6E#Q$_@@;.@?K.W"RA$FG)\_R(_7:S*JY3X$SX$(+D.8
+MCR1/&OWMO$L.Y]O754(+%>;#AWF!0V^&#;T6/O2:9.@U0BD4#[T6,C2=(>?L
+M4NQUJ#.R5=K?""?N=%76Q65DQSRWN24M+B<[CF/?G1QKK?7QBTOD16.&C/>L
+MT$]&;,FPDLN9,[B!0U,OD#LR/XC!Y;K[V?;E\^8<S0`EBUD2S.3`4@O2IM`O
+MDF!SCGWY%-H%<OE"B'C73Y>#,C`];&@:<''S">X+\1S.C%,-$4,QX4,QVNX-
+M;28XKQ[!2YF]*_4<_V@Q\/!?C-]0`K&I&Y!'L[>E;[-OR1CC+IQN(UBFAE@B
+M2-GCIVQ\8,MZ11,M4T?+.UY!5F*N.E:.)C_]KC8:5T-;2\RQ?5FAEP7IQN,J
+M#BW`3=C.?V\8SXJ\\]#ZT[GBR:`%Y/!;II2+#^<ZA3R'L#SPU*_2(;K23MN8
+M&9@O+B`3F?K(.S^Z7=:=`<5OF3%5KI#8#TV`MBN'LP]?5[DH],X0YU..NGY'
+MBLM/*\/6G$I0=V7#Y#LNJJ+U%T?8#R:IR6GL^\32PFEG-BJCHUBI9AKW*8SQ
+M5/-1QIDI)^9R6SG,5@Z7F,[HH#=W$,3TQM8+3>#8.M<<=73LVGW/F3$M<%FH
+MQHLE%NHS#&PY,5>[3DQPIS-:Z<VM7.[T?<OE<\2K79$P]?)]TL96>>AOG*,$
+MH\V,L9^BP5IFL[.F<&=R<A6..,C`EF=&`GOYI0)KU>7\Y#Q;Y=6**V7?;OP`
+M@L7TLR^9L5"H-!(_.+\9?\43_?<7!D7>.6RK3RI^RT*=J`[R:HH@D1N!GA\`
+M\[I=AGHF`'DI9>Z*'L?;N>^4]SJOY;^3]_:^M6FMN+J15OE.Q=L.,W"Q\L4L
+MA^EDNAH:IA(E.:TT>)MTJP0D@'%@)B.+QI@2SW(*P`P4?2*>%6_*U'!W"I@+
+M!.KH*/8"W&\FS@TQ<=1BIE/@ERU@3,`Y#J'XSA[X"N(@O&4Y$<L!6^"/*CIV
+M:R=;)68XA1PG#E/DD9!E-J";T#O%XCM5,.(\EZG>UEU^,5L*\VZ]D!6<*<FS
+M%28WG-Z_/NTJ9.UA=^Q,BJBP52<[%`6JG0Y6!U&N((C9VMA8`&K?0I-JKW#?
+M/NWALP><^8'*+=G[',KMV9:'7X'-8`0<`L<=YAAR6KF(F$;C_H4&#0?70>/4
+MQC,I`:M\"G`1%?%,AX`%T9?"P1[.!!I]ZQ1U-`-*[C=-P.K`.1.$4&#8!GH#
+M#IL!`5H3'94IYFK$7"<.E3:?#+_S.JM52&_%(9IGJ@LTUM1L!QR#\IY-;SO>
+MJ3SG[*ZX6-9;?LTA`JEIFVC@?>XJ0?"=?[#NXZR3.-.!!]])*N]D3L;++JA>
+M'P4CH7=F0DZJB983A7!>-S@@)]5V91<'-K4>[-!V9ZME09I8>7.#6A%D+?GL
+ME*E`D:_:V:%5^$,+S>7.UL3&VA).CGD6(-57NT.T]4+:I@#IG7%;0I5V'$2S
+MXN(S0J5WGN9^(*#_A-._Q%FPYSLMU/HQ`#;#:<79I(S&4QP^WP3")W"I9!'(
+MKW$B!/WB5N`=P7]R;"S4J;9J5:S6C.'PUS=-W0G(FV*=18#:MD<="V'\YLFY
+M/NFK%(^%1R!JUHJPI-$O44H?C^;6H7B</`45I8O6%@AT\K?<"8EX&7H.23AW
+MMX5Z]@H)O9Y$?ZZ%^@O$)9:*]$\I]U-6/;!E"E31.)L]Q[9E/VB`:23?72<^
+M$^YRZ[MRVHYM@)8=_R+DNVN/7:,"DMU!V([M@[UN/?:A;E'-TDLPR8<AWUW0
+M0/+=Q"PBN<.$[7JI;E&UM:O($$VV#?$]YA,;#>0TC/44!)/FSUEWIG!;V;S/
+MA?JA\("`Z<QA>O,PO`:T<C5'PR\V;?@E7`_!Y(XD+O2W@"\(Y@]><&`^U;R*
+MO6D*-\T+#A6!$![2(+_&X\&@)CK&$!V-\&6N)C9F[:_3FHYJ8V.L\RX%?W>V
+MV[@[#A(WZ\)OH/OI[R"LZ5_9ZHZK($Z9F:JJ8>OF3O\.[E60]B6N^?I^-V2W
+M%T!3\3A]-(-/)2P9]))Z?GF\@4QNIOC\6=`GV0@KI=UU&".^VZ4R"G2=N%/`
+M*A:#$@09F2"=EQN/_+1M0=LE9<&E8I`*9BK(:6"^GXW&L-(8+?%^1^+])YFR
+M!2R'@)DF8*7A<CBM[?!%OH;C$'!+!9"^,V?BRF@R+CJ6T8JSE&+Q=S7,.#&,
+M,]*$,`-TE/J%C)>A:C$6"*!J^;7BX#7X4]G&>5?H4/LE"V#2RA/EQT$S7VI"
+MVIN,(ZO<GCT!"O9EV^M9X\M/5IXJ/::0B8&I^:TXA;CTS+:OMI_.CZ_L</RE
+M_.M-4\N^R9FR*7C[V>PG<B2Q[8*L2=E!\G9A_N0LL:9=!.=WYVSD6#ZC-`J*
+M)A2/AW@0\=WKC`:AKET,+NC:@PHUAO90?7N8ICV$76/R'Q8&-$`4Q"._>UG-
+M$<!(@!O7M4N4[5(U)R#DNP3D9!9#)W,7I?\Q&[HC`0>X^I^S3V[,5^2I=K=6
+M*K@-!(<S6XUT-\6GNW!$3^X?\S=-B>R=Z&E`(ZZ-A3;\%M)0.+J@*VF4!(.6
+MAQ34P43.`")#7H[$N6&ACF'(=M"T$AR:CZNIB7X1MH,'\@(+1(5"R\/&,8IS
+MU93(.8UTJ/9\R'=Q(-/9R;HM9"%(#OE.JI$G*Q=5:TKK$3%,KK8\O//'M0IH
+M]_,FETI/UEGGZPP<AIY#AWT%=4Y9\,":L3RE7)I2%F20OZ67IZ#ZC052J*^T
+M?:@NI(S<.1V?%QB7GXC21<?P:_!3W?I8W#9?ZX4>:2(;=@(8,>RD0T?N]B_*
+M+351M_^UU`%50:#[49'2R=7(,)H0YR$"/JK;4FT]=U\G843>'HZZ_4"[B-G6
+M?>I,7K)JA'9JYYJM(F@>V@Y!VN!'7G!]L\>,C\,P"`'AMZ]#C[FQ05N?L^[J
+MNP&?<71=.U(Z&47D^MMOR.NSK=-/::(-FY*Y:R=Q-Q)+'\Z%@J6VG=1%DXFT
+M>86*@KB*8%X'L>;+)W,MSF<<B\IG%-MJ9^O-`@!J)"FW)E<]KY%)U;+`RK]6
+MO%#V;&'R>FC0L5);@I<\F"Q4+781ZF$#(K'!B8PCMH-T"W48CE?9LZ`/FJR,
+M\N=K_UKS@H4Z[H5.V!]R9-PA`)L%:BO=*>DQH8ZZEJHSTXLDU4$U`NBJ`,_^
+MQ;H>,A'K*@C*#VYLU%KI>8I<U;X&2/8K%(YD-GNV*C:6Z2#*DPTRZ6[*H)!:
+M1D]CA0LA$O`>0B20@FT`K?7P:N-U!30>;Y9.GKS^5RAX5]Y;5M<*:^JRJ^8>
+MT[74*Z2W6EAEC<!;9>3MID3LG\7!2ED@*%$II,T.I2+P)(7F]L$_S^UUWKFC
+M(]\^S[>Y3RW],."R<?%>_Z)R597@9)_EX1%?<WV6@CG3\G`7QIB#^[^+Z]XC
+MX20JED3>WE8KN)::B#UAH8JP,?\`S?<XRT.`84NWX9:'KD>%'/<D,B>,V0=.
+ME.5A+;(ETD1.`'H6=GLA<$EOOP-Q;2MK2,CNP*M_"KG]1M4/Y2/JP_7*+=70
+M9T/3?CMD]I3/L#!66!XFP?D/IZ\Z.@::$W4CY>6W,2FE4]\N'Z^)CEZ[3^LU
+ME%J?H3SE6P"\_7^S`&@[""Q4NW?]A_FD&$"G[RH<7S`+1,<<-=*5:"4ES41K
+M=EO3WXR\/27D]A35P(:NA,57QD@Z\``RUC5-?/LO'Y\O!J"3OTDL-^'\7%.I
+M@%]L.OD32&=DQ(,?N$H!F`EDC)=IC(?Q^'P:/AK/<@LRPIG#@H`E"*_DVE'@
+M88S&<^`#!\YSX(7[(F__A74)AW?^!Q+*F[)<#"7$;K2*!LN<B)<V%S=D"6H/
+MG[RMJ(L"/_HKQ&:,J:@CO5%^&@T,!KQ)8VZ:$EL7&U`TQ2]6S%PE8"P0^MEP
+MOP&$V/X+\&T?Q]8Q.`-"5J*0I<3AL^U'@-EJ2K26+/#?(`3W<H\YON1.Y&Y_
+MMGR6\SGIX,/:Z8Q7A0%./&KP`1XP"\I6:RJ?`>\/Q[=YX,4Y,W)P2#QXS^`B
+MB`?"G=0!IWY+MN+>V0`'%S)"U-=1@_WA@T,:^/YSZ+P>'-9=SU8</LM[DJOS
+M+0SY*;T+0Z,UEM$M<#Q`$]@%@"UUQ&$$1M8FQ)IM]O.@$Y2"=#`22TXK$W&?
+M$D`3.)[^$HU>',]^E<;>%`^,D%MR,^,9$P1T"TZG\,QP5II`-6V471C/<0MR
+M-4PW3AP10('Y;M2U/"?N-]6DBV8X38PI`@?)<@MY3^*^M4*_3P2L/7CY=$ZK
+MD->*%ZN5T;%,"IG,3:+:F0R',&KPX,E!1724M_?],44T"6.L%+QR%C/2%#;8
+MC(87$F\'7OZ<BB2`O0L0-<)]3F_+E:5G>?M]+8<=TV`8P8'11I%@''1;W::B
+M>:<]AQJ:6W4]@H!5D\H%Q0FL-*+V4OGEW)>*YSN%SJNU5S9D<IH[8E\7G%IE
+MH<YB#D&AD'6+*)NWO_#BR"K(W'^I?JEL_N\]9RWZRO9%G87:ZEOSAGZ'U3C+
+MSP@GS5,TOZ>:7V':IO@]KSRQ<>\$NO]T9C*].=G??_JN"1LV1<JB8U6AOWV@
+M"<+C"78Z;.=3-/RI9KG_C"ET&><<O\L(AVZ>A0+><N.B&<"^9)48;/O6KB"3
+M&=OBFZT,QBSHJB]_B8[F0.C@HM()X$;XX+NL]<(J(YSZX8-O5H]30E_^6+5R
+M9SWRZ-NKS_%A62>3$(LL#4G9[H=T%6$)LL?N!.!)]`]'0*(G">G@7.)[X1Y*
+M]6,.<*S.7Q4R.(=]>XQ+YI'L!L+FLL;\F&TUOADV.$?%L6O)%"7Y%AQNKD/(
+M<`M9;IS7@8,LW0L">(T8G%$^KMH8,?A,GL8YN6KB_KO0X0``?%3U1,7XJDD0
+MP[07SBI_S':;E`6D\N=LC=P?&G&6TP=T5SV)K'XD*>2_W<922=&4;4%6#VZ=
+MUP[;`5Z&?%@#^?"A3Y`)AHYXV]5"X_*+?OKH&-OUH\&#$89E-2&#H=[EL:NJ
+M+8K'\M=]K5]6'3HHL?8=+XRWGCM5(S[=6?1VU&#DJ=N&$=KZG\3(PD)#FTB?
+M]:W+6K^F,,6,J;UF=I#E-;.J^IRVG-7YMLA!OX#/.?KK.U(\C&)RPX.78Z&M
+MG1\=%ZUG7IS$O`2=<MK%3J^M/:Z+CDJB5B(F?,$%F?":3)W#`D;*7RI:5)M8
+M,[_05OUJP5M52?G)90LJ7\M+68,L[.K<H&_*S9@()%BH/6@3)Q'WV%R_0#N*
+M>8V"))$Y'YK1YLH@X()VU&=#>^!8QIKI-8+(08QX0X@LFC*)HA4E?[BG^"VE
+ME7YP8]JOM.;AFN3:MZ`PJIU.*(Q349C"8LU60GOJ)JI3+*/?YXL+%EXAB=\2
+MDJA_%*84O7VFY\QWT%PF(',9>>L.M)C0S4=FVE?^Z.W\H%/3D7UF-+#4"G]H
+MHWWVV=;,\AJ%JV;=>TJ4&8#(6U>K!#T)2:-0R4[Z;&>O<>E-9#MO^Q=5^&SG
+M:*2OA85!R';ZK;",\K#BH*]'O(0*MEL,2-E.NG=5FT03!1I-UFL6ZBH<(^(+
+MX6XT=]LR5F?"3CS)^>3/<_>(',Y=4_-53\&XL1;&(6H\1W*K(O16>>2M[6TC
+M^>,+)L#Y[%NH]TWI;=,AMH&1&I-#4_E<U:SJF;7/0EI</@/JB_\`BQ@6[D8(
+M9+">A00"(A`T[QH.IA7Y%P8E8F+?ZAVRUY)$EFS,AK/T%DJ#Q<K?>M122#;D
+M*4F4!+WWJ\#5B!IS?+8:S_.:[416M&7T%Z3'3Z.QV^%T*N#H^4:N8_[O')C%
+M1F7`=#&R*(4L0GIK=>Q[*7OB8J:EQ"BB5GP@7/&^&A3&*B+:RE163&EEPE[<
+MWW9$QQGTUQ)[.@@ML1\ZX!$:*Z;VOG.WW3)P[`<-!D6$WHKIT+/W!`%@X663
+M/!IW"AFL.;;,]N!;TU7+:I:6XV9Z;\2M)U5=4-U8:AF&^GQ'AV%9M?6G=MB4
+MMGZH0+L;E%"!.E9!5Y5^-.36Q$>N*DL+TT-?]<O=6JA'7]:5"M3CV(XG*@++
+MU863RL75FMS)U4%.;6&\4\)[0UP=7*W+G:*(A=R$H8H-5QJ58(M2/DUG5+%8
+ML\JE[#R:3OZD;EQ010B/,YUSA5`W#JECHU;;0U5JC-UI1KXM^REY%8WU_)0`
+MIQ#*"=]"+17?FLB;;J8QGH(^K'=U2]7XFRJ6;'-Q7ICDF,9QALM),>=*.(/Q
+M3.&3RMAH5:Q!H0A4*,0*19`Z5B^70P]-KI#*Y6_)Y2EY4^6Q,?)8N3PV5A<K
+MJQ#IHL,YH2:U_.TK)M5XH3Q6ZPQ;TB5:S.(T-V!U#"PE='T^'/3AW+"MD<`.
+MNA02@5X2&"<1:B2B;5%-#6S&;':#$,P#3<$#@WKR26`-F"G<WL:[2RL#O*3X
+M@`!382"W51"0&)]+EIW<?JI(C!Q42'[<PK!;&'+CSHNW?A,Z\)`'K:@;KSQ3
+M)($/3WQ1>;SRJ\J.RJ]#!_K+VV$:Z<`YF(+NQBN^#!RX7'V658Z7'RL,+C_-
+M"Q7S&H3T!CQP8+CB1)Y"V2Y3MH<JVT5Q[61<NS2N7:!HCU:TARC:A3'M"DU[
+MD*9='-,NSP_AT&=+!JYODOI-$'*3IC`FX$$#UYARDZ(]EAEO4H@$N$/`GRED
+M,DVQ[3'0%\Z.$@]<HK?BM#1!X,!%Z$%KVH.E`V?RH4\;'M,>!DR9D;Q6Y%G'
+MM4<&M.)Q(J&F71(Y\$W8P'Y]>V#^4;TH<'\KZ-2U1^C:HW3M<94Z)>FO$8F4
+M!0^M]H\=HHV;5*"N(O3KO]>&UT1<(9.H)XHBBL/S0J$=@7/4*<@E"P3^'<@[
+M-6,TX`(7=7)_VG&&-II!M`AW.Q09V:P.H?\`5Y&3G?5TX5/*\0)&MW3[3]AQ
+M!N."5#+@X'W]$C93N.MC!H2(>QS&BZ7_D@[L\2N%6(*+!SXL'X'7ZA_!>6@?
+MJWZ(&%@&@-9Z5KZL6GNA7CY'*E]9#4RVA'UY4T$)AV/BE,9WF1!^1`RD5J@D
+M`V6@W[\<U[>K=.U*/<F1MVN4[5K_8?A$K2.9ZG9]CE+WU6O@DJ+=()<S]:05
+M-OU*9THF$[:A;=!"3;)^?R]\P*(D)07M`0X"G3%PA.Y)*5=9J!X,^WK>V'Z%
+M7ZN.].\X7S1UX_=T'\8FC=[Q$7POW/J/6JASR$_W_PWBN&][P?].\50^W\3?
+MZ948\DU8==Z$I1G0_MJ`*7`@'O9OT,#DM[<PY+'X99.6#*I02@9,Q8HXDLL9
+M)?S3V)KW1=:^)XN2"]]:>A3F2L/Z&$TGH7RL/2S(CLOC-)"^?*3_ILQW`@4Z
+MS=Y#*-:FX/_.)=&15NNUT_DJZ#3L(Y*_`,A('!5^V*$\EJ/GL%;=L44.!#XR
+MXWY*&;8A]R75L>S(`;&UY(6H@:DU)2"K?+.2Y)1N11MT6XI(_C!>4VH@F0Z%
+M81M'N=U?O1WVLS_J[:&'5L^,B`$F<JJ.9>__3'[AK+J15+=G6ZCG]:3MS%V0
+MH93;3OT<-A#'Z$??6:CG9ZC&H5TR]2L9[/YQ,)^%BL(PZ[5/BI/UI`3[@<$;
+M)K#AT`,':N``[<PC->.CFQLJFE3CHY7C8PX[59.B5^?NU8R7:29%6R@7:FYM
+MG%P>U-BB7"&"O07:]$='K9Y=NJ,/K)WK]4<?VH8![`^T(K2A6,4_2"2/;+P*
+ME.1KRH)1UIGX:TVVSZ]9J)<P#V.K`*1S?B+HF5-.%+,NT5E[:*S[T-:$[)E`
+M/TGL0<73N;,T\J!,P:DOT#`=Q3I"&QN)'KPJSE;S"CC$A32=/5<%K=CH;*PV
+MSD(-(@.FLAX,!3]81G_`,+TLRC8\P4*E&V!D5&VA5F&87.Z/MII*+*.AR'^T
+MO3]!,T<*3JBF24$=L+/W>+?ATP7A-_>%WMRG;M]PJ4G]^L/%#7Z8W8&5%#A,
+MP,7*G15U\QB;.XLU)X'%>XGA%ASN8'0(>&_@U>/0[J31\4P16?X$&L+QA8I+
+M";O<U8HB$R@)>#/!ZBD+OPDB;VZGYW/!O=KFFB8=R7%^1$!E/51,$A1>>[A(
+M`76LYN,"I>Z?G*HC*CG3`'&%M!I^>WCVES/_C+IIC[A9$$-*"KNY$PCL'`.;
+M$'JX<,>76U6!-S-E<-;IA0>'=\A4\K,&M'_/=_N<E?";RR"-B+JYYG`&<0>F
+M:&S5'LI6]9\E9%S9Q][M?RSLYCN1-Y?I8!%=P@/N?3*#+%MUX2R_^-'Y`-#)
+MW1^__B3LZR'+J*?(%*==OG98="45?*2=AE:J_+X")0Z39?2&;V^1,0=W)_2:
+M$S%&A>!JPI741+^'#H%E]&>TSJR'?>W:(HFZ^5Z!Z:"!*)@%8AAGN-S/\(T?
+MX;K/[E_N!$TKH.'&-M>*%[->#/B"4U@2<(3;DL9;/&E_0_4(+YEHJ;TT@@8D
+MM:!,N*FD7%`6E-*L.B1+.Z]U;`:=9XK*2^*J!%BJHTR$C134!E9O/K,4G..U
+MTE><>1*M(C7+]$-2Z(!E;^&E$3E;RTIV=:]_E]:\8-N6[5MS9YBQ\(\Z5%52
+MA]A=LFHKKJN&$MF6?HY>T'-G7"G)EH3=G$?P9NEFPK'>,B[GF36;%6$WGXJ\
+M.2<W(2"9MOT)."^=XQTS.`ZB?,:F5W;O+GNE>.;!@[4SS_'#;D[,3?CO$GC]
+M,349I"(EXILB2'*M)9``<AEWV9IS(O;128T-1"NA^5;DW2,/.NW*GU6LS'VN
+M:0^VV8NQ%!_.O[AB!;^%6#4UQ(NE^5`O[D*E!_4<)PYU!42CS8ES(@(M,J%"
+M&EMA>:<.Y,\J4!8_U[0?X@'F#MW?@69RW!7RR)U$ORJ4!^G7]SX_-*3_A]!^
+M=WB_&QO'`'T*623HE-X,(*8*]SEX$')NA$+4T1S+4-?AW*G0KF9P^L?%962P
+MD_'MJ;$9&?*,K,,+8G,R_&82)Y\W1./\IW%#+'Z.#YE>;$8F8R(.W\G1`V]=
+M8?W'";5P'_*'#L>OSI>%];>%]'=S-1S-H1UL%2NE"5_^"4]535.51"W9PI+5
+M9_OX^LDM8Q;E_N+'9\;\H_H'POL'M@JZC=#%WLRRS(,LSJ\\OERDC,95TQZB
+MPT/I+MOFZCS+^@.!G*D)UJ;/=.1KMLXC8(M*_IH9.PCMU=OG944V#OO9`.X,
+ME9QSN$4I)Z3]^S5R[@;[&V"K2BX!6\W8*RD7&>`CZ&B"]%QK1+_+J7"JG,K"
+M;F<<TEA':)$-ZV0TY7.=--"YUQ'`?I/F%.YU-#E4G%$(?VS')+`%BL9V3-;)
+MF4JY!%9KH?+@E-#W/&PJY[PQG^6>I)-;;=^_QN/-LEY[6=KOR%<Z<G@U!):.
+MCO,=:*V$2/XDAFDX%&>Q.,^\OF6:0UFD".E/90T39V\I22[;P546TN+LAY3V
+MCUC>'01I?UUC&BN-QOQE\MI=4\+Z/RA<I&FBK?](;\9XP?T%$%PQ++@_QT)Y
+MVIJX?YU\N-7KOR91917"LWN]/D02M=J,F=L\`2].AG<1_:]`'KX?\?"6X55W
+MGHSH?XGWQT5CK;H^NT)XNJ%H$?/6I'Q%7ASS-L%M()96TRYZTO8E0!1AKL63
+MJ&E6%]0[#MI`[D1<@/'`EO"2CA3SGC39.G.0L]9C<AL3&5GA_:E1_4NZ76=V
+M9M5ENAB,672_6=M$=,L\,,\R&E&T:,,'T5?(?&&/V6WJ3<T5*4GFM82]#J6<
+MF32ZP8MO:)J<^?NC-,P->-+HM^']NJA^.;+FGPN]"T"-1WG[N<`%055]_"P2
+MY]O?/<1\08\ID9[3#45M\<ZWRR9P,;P_)*P_A#=7>)&\9,R6Y9%^SW+/N_!Z
+M8MLRF?;>R:RE>@AT7XK[@W;\N'/#LAOQIPI!-$B@QZ(S3*?>T+1OZ+(OKGAT
+M."D5_Y&%WR2L";E;4KTRFUF*^)6=5RW4MQA]`[%S.(`BK/874`D!$T6E`EB,
+M?(Z`43H)%G;F=6]A?SBK</0#,%TFPF(Y&.XD6-=9]&=1I:<38+K'57:Y=O_K
+MZ()MJ8_K.FJA/L:`'9\=OZKSDH7JPKI<1U<[X7LL9C\+'R]N\X`$?)CHL@>\
+M.`E,]Q5YZA58Y#M]!#`US_05N^'Y>YFI%HH)$EC/QJ_H1$>XH*X'WS@)S!$W
+MOOTR736.IC:RH(*MJB.^/*1M_,8*/N@RPTI`FRUAGB(6UW=.A+-8S?E50XH-
+M9)!"'L1II?$<;%6Z2",7[Z4,V2*KIP5R.)6/PSEI5\U_X'#,?^-PCQ:I$K%;
+MOKV6TX>\1K'C-:V1M;9M,JS.ZM'Y+)LOG1N]AY4WM<+ZN/SIUI&<=4=.@S:B
+M"G'Z1Q,*<K:WK[<AD.//>?N'%H7<'\KN._:J)KDJK^"\!K:V3L1U$(U.U8<B
+MJ_VO9NP%7ZF\Z6<NKCN"RH-9]-ZS.+?_S$7_O1VH3E@)JAY>+%3[5=B?Z=8O
+MGOI/&2)NK+90I'=MT7]I=LKI-R#S/IR(_7QZIZ]+_-[`$S'DM+`Y<T[6>^=Z
+M,[<2UXP=#AL[2`3:$K$KI_:-;;WA7ZE<0LX.?$,92\MY2%31M#NA1PEU%O2C
+M33ZYV$+5_8$D`$\B3>);OX!R2&[,EMZ8;9OW#&@+OC%3$XT;9!B7.\?6E*D;
+MC_$6S]/U/+1UIND7U2P9?$*U3)%2@J]^H-4MJC[\PND+2$\_$>YRZK?D;-AG
+M8#TZTZ*786MND^HMV3;7VV8L6!G-B+KQUJI?-=X5%/]$VI?6!)49^T8=S0BY
+ML?`\?TTE$RHV[7,SA@ZKTMWH0S6?0T*_8DO8[1,;"G.%1`=`Z<GG75;79NF-
+M,#`OJZX``0]!S+)0,U71##!4/`XT76KB4PG\P?@EY0(]F=PX!%]KY<GKT#G0
+MTM*ZJ!L!+7MK7;:$U\")EGA(03O9^]&QQS/GHFYP(F]P]3]O:'H!*NTC`BJ]
+MP48T3RW<U;!'II&?5=5G`Y+]/%>U"UF^3A;D:.!K"]7FW>LI,")91O\9PT%+
+M0QH.C5V%NR%&O19Y_9='3&\78GHQD.G-?<3TJLZ#S*CK(ZP:86TG"Y+8<YKE
+M]6A;T[>_DUQMH6;Y>$`,*3XQBS&!%B,7>]>"@/G,2V@.P/Z!\R:)*D#SAC7G
+M9,:?=(68?OKK]4?N_%%5X$CHQM&@:NCDG"3J]MBD`J`SD;;20NW#QJ8TPE2E
+MUW_VGO,M#;[^&726F0Z6.B-'-<=OS5;8I<4AUS]YO-@SMCCD6J/,R`Z__HG5
+M_GZ5O26G^:^5Z94"6\)[8/=E,T@'G?Z%WAZ_&'7]0.3U`[#'+QD12'K[VV!D
+M&:Q8R/5]2I(`)%$@W-VQUR'+.@M]+T!R/N5"QPOU.[TP?KWKA(5JQ3H2O:?:
+M>DQ)HQM]=B&1/L="W8H5^<8`\W\/C0'^P%<R\!A(0G2]E'@@W.VLI_3WSBJW
+MP%%@)3[:?_'U,YC-F(`_ZF<5*8;JVNQ4R<4IZ7)P4,NA0471ROW7NAB@4Q>+
+M/]8NU(.*Q]8-E24CQ709S7<^#J(LA%B54:BL<EOMB]55/5;76]"CLR;,6/K/
+M1:>3X*1O2:)^.EW_&`N2J%K?0OC)8UXL.,>KQG4^+$!I`$BB+J'%CO,6ZK2>
+M#))>7V0@N:#$*6=1;.4/(I4\:,^P_E\B"U7CG9]_1@'?*B:4Z\R"P.NV6%D4
+M[SFA_P)B1PQDF+&**/^97+DB\H._1IR<CWQ_)<>.EL&B]BB5BDBE(@(]0V7*
+M_W8?)K-0EWQS%*WBIRXIFXX8<+$:=N3'D);Y0V=8+EYMQ^-B<;77SU_3QK>Z
+MB$<3#Q(*[)(UX0-=-,[CS3G5Y>T]?=6O5D^&#\MM(YM#KT>&7P]7U=>`O-7Y
+M;VB.*5*:&(7DVLOS5?75TNNSK27/EY6`GR*N3]9',ZJWLX:%R!\LJ]D,;L8=
+MKX>E:.JK]7)_S5$CBT+\5[.KVC=&!B.6\F#*F!HETI8AIW4,J?!/91SL'-^6
+M(%#"1+GR1_8HD?8VRHM.'GK7@'U]FO00NMG3L$>S\6'/XSD`)T\WN20;:LOU
+MX+ZAB+Y[(7U#H`V0ZH+?K'63-`OQPR]Z(7>WQDRH%V$^'3-DY$"]0F>7L3JO
+M=OEY56L;5"VKO;@E-ZROIS2]TFZ;]R8XX@4R#_M`?$#`#*MG=FC?I:B^2X9=
+M$,D>XQAPZ<T$,2C<[3!D9.\?1NB3DZTQ^\OZSZ8TJU7F('#H5$672[<0UUC]
+MH7^LLT&/`'A8!^(W-!U!F""S^C`!_=@2./[O/.H+A7<VFA/Q)=XCL)C22P)6
+M.7$E!VM;V=;6UA76]TV;?-^[5OLSJS:;U':,.$A81S[G'2`@9<!1_!`3Q1L-
+MC8/6D5WZQEM6C\O0.&`=J5#;F:P]7%8>OC835Q?>OP1-:"LH,6.9D&O#YG@[
+M2I^1LW00]E(V>\P`L?2+,,XP8?-\H(9]Y5H<U;<=W-!;_6OJ:EU1?5_IS9PM
+M.]#DJ"\RH\M.@UG,_X?)-C)G_^>RG<&P>V"?&'Z3POZQC"[$T.%['Y(S_@;G
+MQZV(OFS09AEMNNRZ8O:1F&,C;1[VZ)0DRN3=!X`]H;6,7N5]SM+N#%9Y2T-'
+M\7.RK0D$+`N=D\0E?\*(D+[DL+YD(EZ(_+MYR+]+94*_;D%H<%\E=.WV?*`^
+ME*':C'->P]4?HR4EY?@,^?@L.'M8T*5[$2U,3<K8[F*_BMQ-"/P0FL9G<Y-P
+M;<=O6O@":+S.'<1R'Q':*EB;"UD-.`6FMOW(\#=%]"6#ZV`Y<Q8MX$7:?EV-
+M$)CT'+&U:<EBUDR8?&VZ3=V$;:B;NK%NUMIYY+JZYU/V!J<TB#^H$X+:U75$
+M6_ZJ.F;;`14G4G486U>G43<QN?NX5YK6?X2K/[M_U77Z0PMU6-DDR!6^O:G5
+M6MN"UB2M&P\C"HU9EW:"Z6T'U(<Q2)V@W-]\KI9%0"G7-*L+!.N:H]=\XOL>
+MK2_R/RT&(A3*%=AJ+UBHGU!?LJSSWHX;CS&9LR1](FTTC@[S@I(\(V>,R61#
+M)A/6QUDR^)+F$$00=/YV[:E9^D751V:?^;Y&`$U=T]^(H\)=':IC.=<Z5^=/
+MC^RC/=X/=)`;<HW:8]G$-[@-6!!_Z1NWIC(&PD(2563&5)@VFA'6IUW7*D7T
+M)8G*/95HIBT%P$)U((:SRXSY>WD-;:<-$ID_\QJ:$7W8,@.1FDV(U,`60)<2
+M5#D@HV'=3V!5QD,>52Y:XA`HR>3=F^#K.'ER6BO"E+:6:G&PY]+&7409T!QG
+MZ3L$H%[=3K,E[&GY2P]T#\!F8`\H]WWNDA[B.2?UG%-NW_#.#)]_X')@H&#]
+M\_>`J\!X^!)!))PY"WX,Z.%ZRZIM0];2<Q):RUW06L;>.ZM%UI(8.ZT`W?KW
+M6=($:]'FJT;8LC;0F40E6T8S5201YFF%Z:J%^YR[W-JN[-C2L\1O7&TW,M[I
+MH(23&P\V>3]:2K-0[T(0EM#U$IJ/,_5XH+FF\0K%1<+BP+86M%5JS1<Y!"&>
+MW6693CNK50BVLEOQRG2IIQPTJ;Q$:>QS%\B51D=^7W^X8@*_1'JV17BV\?[J
+M]0>AC8/0A#S"$GP+$;LR2J:U6^U^7I_PAMBS<\>AG>N6?3G]=+J,0WOD%)Y^
+M13?@\^,:QGPTV;((_!`+_PKZA=6RE1$@O:M$64I'W[&P8N-7C'3!X::O(W9V
+M6.T)R#,D_N`9[D2>H37A16^1CQW#/<?`=)`0\U]V^@>$SX.+X<`XZ\P\F"X+
+MYB(>)^TJ`?/4'#O10NQ9U>!0KHR(_4/-^RS4AYCO$YH1]#4-!E.GP]0-:]0K
+M([PBQE1[?48`?<8.Z#,2<W[W&?\.ZWJG[0\^X[KG[\4LB[",#OB<QI$^"W5U
+M[%Q$-![N^8MW^6:SZEA-A&<2VB2D;??N$#[:D'?HCU6#WZQ]=HEG&E05.(]3
+MF6MQ-)5_C5^R1V@@DPV_T<:.<Z)/3Q9IC"RPV=I4I#Q*6>WW4PX)K0GKX(Q`
+M>(F=YM3C_MYC<>A4S)#N6(Z!Y"R]B#X9C/"$/%KG8!:1/(C[?5-UQ[+;[-9S
+M$P(]4CW)1)^`T0Q;.W<-AGC"(SV1:KL@+DO`?H`[NP(:\(KSX'K-N5`/7W4L
+MN^'3N)UGV;>YJO9LVS\R'O$#@<8HU!L#L=;QV-!XZ\@:M5$<Y7D":QB_ZGL_
+M_N=L;?_&2,^$9)<?OX6MSK(?WBWU&$\\C34S#GS>X\%J0^D4?4F?G[8+(X:(
+M"(^:=3(<.Q"ZWZ'.RM>9#\5E%?#=-.2F60\ETBY;*+9Z'$L=R]",8VL0592@
+MG5`Y]S#DGH1E=$AG%(5ZPK#+XU==\^-\SE7U;TP&?IPON-I[=K51"$6R4"V/
+MCF0F44__87,'VV2A^FMO@*UAU[I9>X25'G3@IB_N0KWF6'6<[QN1=J@]J]&:
+M#[8.%B.]]FLN679^:Z?^@D3]+0?TU9QC.7%PPWF!=1\O[XJ3,RVC)__T;2BC
+MFWS[)>)1Y21ZFC(_H65JTQPXC+ER'1F@)_D[W<W#>CF_R:V3!RA+F+H2^DY'
+MDZ/9K2NE*TN92;\6O3UO(O!DR-5FOO1:GUI.5YD#BDF5G`EI6RU98WUTP"6)
+M8E7(0;-<'J"1\Q^M"7G7@'C3A)?(/!DO2<AM(#*BZ4E$10:Q@(^/YVY)#[Q6
+M=H'<D;-=8$UX+?Q:<<2U(G"K_N_@!^!B_Q?ZS"3-,O?T3FWWAO,E4-\_'=.Y
+MK/1\P<Z7&>CSLFVEZ3D9IQ9YN6\J_DM\F@<:GF_0`;D[@#R:ZTQW>C\[61ES
+M5ZZ[7YJ."!&>1>R862ID3!3!$L[L>U3XXT6A##`]4PC,[`DL^G/Q_OXS3EMA
+MDM_K%C)>)G9F'9V)"LN$NOUS_$K/9Q;J"`;KG_6H_J-9Z+45C4%V-"]>*+[V
+M9HYLIYR53&R7^4W@;HU>/C7PY"+O(23C494SVDENE:&[F"?O;XVV4#<0![GV
+M;-BU9Q$!.8<(")!>6TG\1=C@("`1.8\6F,<XR%_&.$A<1L:?5I=73.?HH_&`
+MIW`]8AN0;\LS,OE/X_*<#,.CM>6`;K)E2O`U[25C>9AM_A)0IY8(E`MQ+X[\
+MS6&,N#:>]<X\=>-#6]W3Q9MUG1B_("%M=*GHVD*Y1(C_2%MR'J)T>^2U-\.N
+M*=5F+OB!V\#6U(G4=0+0B4YP/HS/,X=="SS0P&W@<JYR-(>6ZR3\9<A"[M6+
+MA)`NUD00GPEWNPU=.1][OPG$/HNZQN%[C7M-9/.PS5-HQC9CHFNSFN;POQ'J
+M)4(B7+SD?"@XEU<2UON0$SH?%AQV[6E#_8KJJ+6_-IBQ>1'7IH9=\_LW83Q(
+MF%$D3._(_H:(:Q.0/'>0/#51D!19J%0S%@SZ4*&#OD)U]2MJHI:=AZ\67#&#
+M3Y.HJC/_,-.>+XXL,F^\+=5U99_JTBRK6=(2'=;;`VE+=>C;31PS1EPQ)U&;
+MS9BHT+SVLK]N675%R-D+Z`/:)*K`-H];&-9CNMI9V.G];C:)DIYW58197?=:
+M]B**48PH!@)?B<`RN@8#0Z!)O9!1/*XB'*V=W($`':@7"=3FY`.//J-%$'W6
+MUB;)#67ESV)Q_L&:DW"X@]$@X+R`,SH$1"M>L;FRI,5IFU?QQU45:^>*J-Y_
+M1O;^\]^75=!)8698[UWM0D9AB,-8'!78NY]'<?0_+F_KQ(<)_<_+UYU_.;+W
+M@$XD..#4E@CV=Z!EG%+!6CL<MU]4(B$8D?;6RZ`/H!'N;MTC,\C/:@\A9N+D
+MJCY&)./$V-J,PT*M\_J^D`-R9ED>YL$2]SM55N9:>POH,Y@YA:&B7I!O=D3@
+MK3BOD(M3^$>.K9W:;SFJ"Y*:<Z+>W,H+M5W`4W/>8";T5B;Q*ZS/NVZ@0_7Q
+M(!/RUD=&]K:">[7"&H$5S"T,Y78(KW:V7%'V2!M[6CJNE?0VN058IZ-:A'D*
+ML!&'VUX3J"P@>UVU8G4C"9_5!!DVTUMJZG^L/:RJ$FB.PF>.&@G665`=O&[O
+M%[8$>Y[4,GH&7(_H?5=IYF2$YD>`$4.I`#_$Y3GP0ZU5,5O)FNARF;3W#=0G
+M.CC-O7VB[X(RHF\ZNKUK&^@[XN:>$K<I:?2?EM%U&+1]/T7VSL\/K^F$M(K?
+MB@-/5.\SE>?T]^HURZKU5J]96`G-@I\W[U68[Y9E=.8?,3]G`G@?&,T8K7D"
+M!`]L>2AC$6VY^0?ON@*DN%YN_'-$KSZL=_F)6D<3>R^=G4>K:-*<PU?;PR)Z
+M4\-[E<`3X-LX/Y;!:\#5[1G<_G'G^&TC_"^@ZS_7\"W.1_>PO,#>MX)Z%XEZ
+ME?)W2D)Z+>&]"Z6]P6&]4M#)\Q5P*$.1CA,--`11R31%-N[?/TZN$&4GQ"B$
+ML8I`F=9[AH$EZ7U%P[&J0R<&]TK">YE\G9"G%!ZZ'.`6!CB%C8[=SMWN:HE3
+MV)!7$:0LI&GMS2K[1VI[D])^B-4@)!K&SKVR&L);'$H)ELBXIGP_L4*DE-#S
+M@O)%:I>=VZ#4B.#SRQ=2"R3Y@4&]4[G[E#&OMU6*->U*K=&^[%10@5B=8-+\
+MA20:E(F,L^`<QSU9]ZSI\,Q#"PY-6#GERJ&T8F&1Q!%T>+C941Y4(]EX>\>8
+MK^C=3D3G?E/#W)>CW'<BW=<BW`/A[F^!,M!]7=P;K30>DILP*-=B]9(=SRM+
+M!,UI5ONKH"Y;@$UG@(58<JC_`IK?AU.(5X3[DF6+MA.O<9;$/:>NEXMZ:2DN
+M/WP&:]G>R3&+MO%>P,^,JS'R,_"*\5LF`-F6<8TO`5*]:)NL5"";Z%(W"M3)
+MVP+=7TE[_9IG,A@SY%D8.GZ*C<D9ZOY"[<(2<6L2=<&W;I)$=8:Y&\%Q53-^
+MLI&H@J[97.T_<0*-;93[XTCW+F!G>8=1=SV##PG1`#(SWG$/=M<$N6L`B7M?
+MR^=GR-['/YCK#V_1+`QT]TG=-R7NFZS9-'\%3;$"5[Z2(7+7PK&/443(M*G>
+M,H+<13'S:\+=^4OC(9)IY,<4*2:F@V2@?V(B0CF_NJW$FAZD(`FQ.YMX2_AA
+MFGI9#L/*04N<-%P^8P=,G$LRG(2MY)Y\67:$>X/5!+N^,-!]$GR$'$&W'4X-
+MIQV=A"VM2`<MP%69"4M5'_=^FO)*M=2=ZIQ>;BHFMSZW=6;%K)IG*IZ-=*?R
+M'N+5,[(5^N<$%2]6S"F?K7I.6/V"\_D(]V$_^NP0=Y'?!.'6N=5_"W07._]*
+MIW`ER3$\)U8_%TAWX,KG1)#:QCXG<2CUSZJ5SP6KY4P=Z2\GK=#//=MY^D4]
+M*0EWOQ;E?J/QO>*?&<,$VF<>#CU05*/2+4.;:;H>LVYE]C:59FRM5[DL.]#]
+MHK+`K!Q[NF.16GX6=?`)@R\\6M]35CVPVNO450]MKK(\9:$"W%]SF:&119W\
+M7$M:K2"S2'6>'W"`2/:LAGP]TFV$_O0!=#(GQ*U?]1-:SGK\'5L1R4;K-"60
+M>DO=.FO)IO(2<$_J?JEVLY[D@)':K6@Y?$L1B2ZE2I)945:]7;F-8T"'H/Q5
+MI%4_]/#T34"&N$GUL>R]G_C.Y1A*SS[:(4RBTJT'?_*J#&:M_59'2B+=(6PW
+M4300X@[$^M#&^]Y/]N^O5@%4`,SH*P%FCW[T[YXH.%AP#Q7F3E"2MMC/&,J"
+M4:N]G/B;</]GVJY_:;KNJ>?AZYIGP-_#K?JN7S3=]];>]M-T_0PCZS[1G*B%
+M.@?O0#J\UW3_"W.$LAUT?^\T-BS+8%$TPTJT6V^&=2GDMJB>'V&)13_"DIN&
+M:WY<>T:BZOK9T/4+]/;8%,&B"+Y!?/*!&3-ROR9X.O$W%\'U@`-X0,!<_0(\
+M`.F*]Q!Q$K4]S!UK/?@J%,HK*/0*?55&]8SJEV7PAW&]MU:?7O2<!$;Z(\62
+M]'RY?)8\Q"V1]IQE/T_;E*%Z)>/K#T+=@=#,[FN0U6>JZ[.6U=$SR`ZE=B$.
+M2*_V-J\/=!-->L-"7%9@9@T+Z;!9-@3:^IR0Z@S?O.%$]+2&]C3&=&(A;GSI
+MA-7A/3>!,LA-B^RA-.&""TV9'BPA9YL`,^652A>KY_*?$!(3V7*3G?,*IWF"
+M<M%RW@*V89Z)/U/(G<EAE1'\M$G%)O@CFS>=21&UICUIAD4KXI*7\\L)W#%I
+MTW2F8Q(KC3A8;$A>43:=3I_1C%8_RCU(EO">`7%/OWS&/ZW&OX/-H%/2<R.V
+M0\!P/*,!&#`ICV.+ITR#HP:]X++0-+L8LKVBA(/#^KND__"DF@3=KA5F3)K6
+MQ,Y,J$Z(03=X3/T*V9.DOVR2;->*PPM:&F`EL#18T%C_=@"3[V"8(R&PY_29
+M]"3J:'C/A]!&[^]`AS:.998FK-TGUA_+@EK`^8P(<!(\)\%6B<V8'W<OBCB-
+M03UO:N;A:NUOT#(F8L]KQV/:#DH[">LX?K(>*:^O?-FG5%3/R_H?,_!AFFX>
+MWN72_YS!\TX+*,>__3L^C,">/1$]L^DQ[)AQ#M"V="+42\>)+X!+VK.!4`GW
+M.A09F9#=A_285LY[-FX^IL_(CGV=@7C^;;_8C"P86?GJR_J,+"R:@1T/[?9\
+MZ^'.I.MS,O0YF1RO,#%;,@SG<+19M3V#CV8X[:)KY:O1,#U\WY4*4V!G0[D4
+M7;\E0WD.YP_3]-LS6$A8M*KN\^485XTY9%!/E)*#*6/Q1RN`?O_V;R$,A/9(
+M.2]P5/4[3GRQ<C/M'%^SZ[&M_%^577M8$U<6GTP2"),`>1F2$&`<DG&2R0NP
+ML=+2@D*M]"%;P-*F533`0A7HJM3'M@H8\('6A>C7^K5J!$1`U`39;E?:[1BT
+M:$N[K5+7W('M"UVWN]L52G?WVT_-W@FPLMON'YLO7W)GSIS?/>?<.S?GW+GW
+MY-S@>YH1:<)(C(22=5[;T=#@9KR1^9(2^^;&O%=W]!4><-?\D=_0>,"]\9FU
+M#8U]5C@D''`?:(`:06U^5`]D0"M.FR81+?5\@D>$SX<?EB/P?)^>4RU+,'4A
+M$>:^=Q5LJZO>C3^Y.G7Y=-Z%$22>O:UC;ZO8VUS>!?:VAOWZ_\J[H!EQ2$RR
+MKJY=OSCOWMRT)7YDWHD>8V"[.>".&I*4,!6F%H1^AYM?]*X0TYBYMTW#?E7.
+M1":,P('ZE4NY7<:I]3SES:DO=O"O>#\X7-XB'3QM>JL-]CAGZ!07*2*(8B2)
+MW\;;DY+`GI7L448/\<QOM7%SN3,C[4<S0?B47JR=^43#^@!.V1'"'BEZ5#Z5
+MQ&($A^T6SYZ0Y(87&;FCC&)CAWLZ3U66AFU3L^T24M999EC=V$51M:*V0FZ1
+M3;96F"TIR7R$6.V>P_;!6/X2L7-N'/N:R*V,S.;5+!5>O&]J^PLBPI8DLM_M
+MPYE`;.SBKCOF86\4C"O^IJ=7NQD<!ENF,O=_RZEF&P%NF)*SZ-]"[DJ)^F=>
+MB??9J$M/E>>3">Q6R>.R)'87TBQ@ZN!`;AU'3P[!'R;K]R@=<(MI,?T^!SRN
+M8S=('I,ELMO>?ETD6F+Y$H7B]_0C(2WGTUQ'3;UNS"0VO<6I#.U0\6-VF++=
+MJAG;S3*<CGT.@O?TST()VTW-+E>QA9)YL@Z2+!9UKC,0C<@J`31=MK8F2T00
+M;F@Q;C7'U!P)XZM*AO=[:PE^J$&J8)>HV"XFL^,LW7I*K,DJ]^C*6V3ES9',
+M[JB&^XL1I0$K*49B$$/ZS6(N?_FV]Q;.80_3&3Q3:[?9YS?Z?!CVA*EG0G10
+M!OMIS<%DLZ_7XCLC&I)%#Z%;+BG?3J--B#][W=)#Q8@5@H3W_;ZX4BS<J[1F
+M\`QC?LN8[\738FR?DH*8[C,V=R_(=-Y1<7MBN`;E]%L_AZU%ERI1T>+8G3+C
+M<AG7(QS:P7W(1JW^*;EH(Q_V@8AL5+]</J6?G?$QQ3ND.A;>33%SV)C!;P_+
+MBI%%4/@CL?ZR*(VR],@D4XQUH=N._Q44"\NA?<V,#]H7&NVREVD>/(T<U&('
+M^485[^.T$GS%%9S&(W]7K`-W]LI@E/)*Y_(I'E`,>2Y[2QX&NU.PW4I!-Z\$
+MQYKFBIN4@G[>EE]GC'JKC@KVYD6?E+#VR)=1+6L2;L(P0"6PRZCZ!Z.[19"]
+MU.:]@C,^';CA"GG]94BQ`(O2N$(%"#+BJ_(+]N1)WI$DL+I$=E'T1]2IT40V
+MRW*CSA5JXWQL<$T#KNG8C'CV(::6^5EQ.*4FT2[R=\,@4KP?WC`E9$OCYH"*
+MP:M6W^`V)1#<@%VC?<EYS17Z%HY@#=$N&=^@+%GYX=4\OF5Q99#_.MY'7*F-
+MR%96+B4\N&>N)]E#>`P>O6>>A_08/92']I@\%H_98_-884#E"C4C2"+XU;P;
+MM?-NU$WU7SGH9=8QJS2L0@G>5;#JY%2+*1,QMIRJ:N-6F5(]=Z.B,\W:0.#=
+MZ`\7TCE%);4-SM"SL)VT;`2=?A-+YQ7Q=M-915@_=<5++T>*>-U->5B_R(3!
+M8I>2%5KFHN;TFY('J8A1[G'XFYE%O*--F=W]S%%8T(&[D`Z)D:.H`MSFZ/8B
+MWL$F.Z2WPH)I+LKT04<]O#,6+>(=^S1V8,B<4^0*;;9B=S1@,@'<B@?C.G!+
+M#7Z3!,8;<')R"SFYE9RL)R<;1844TR`41O0\=\)Y\GER^`0UW--5:,GLI<[_
+MF3S_)ZK/'#/*(_MH*_9W6#\L4P5KF9LQHRA9L,:*?0Y/!+Z`1U:L-CKZ$6[>
+M-[O7%<Y+"^TVX&7..$/9`T>9,W)P?B!33ZJ87#T9Q\WD/J@&:P::C:2*JIV?
+M3,:5U.XS8:DTQG.&[A]H#7\[!EH9OXE4!4XG<RRIY^J-V-'!,\S%P.5`B/G6
+M1JJ2P#HK&6<A50-7S63<N0$]*3,1<G$W+VP/$RF'L0S6RM5!QM&DS$K(.5^1
+ME$,.,P>9H`:GF=<")\[]THRUFDD53<;!^HQD'$5*H1%;.>F5KE`5IXL.5,6#
+M9]2@+0ZTJT#['-#.?,$,*\$+28`F,)%M`L:G#L`$F9>=:.PEB0ZX)&]2G0,=
+M`9.CUF1PMP>.#UGT>VC];EJ_PZIO\O=;'4?,CL-FQR%_CPXLC@?-#%5I2IN-
+MD'\/@>80_/UA#,AMTN^>Q<YLUG``;@6HTX(Z-=BN!-DJ4"^8I.3PLTPB$$:T
+M/M59U/UL6_X;=N,Q9)Y[-W04CYG<>XSNICCP<S78?/QITI%`.A*MCB05V*0`
+MFSH*!6D2BKQHK4$L!3W6@A/6@NYD\D8R>=U*_F'F>50<6*8`!?IU&42J9YM-
+M7"D24K"P#W6%OF/.,[[$X`2W`RNJ8J%Q;'Y5-N^JC\G3@`(U2-@O30B.DF/H
+M`/6Y+\*VT#;&4X&\))!A6I>S]Y,XD'M(E@`>J.S*&<X[4[C%J?Q]GAS$UJP1
+MO"J+62%!OZ).$\35NL\R%0#;L$;0(!/F2/CYE#^G\Z&NU.-IA.,0X3BLAS9S
+M'*''FLQC>Y2`OA*K7_0$8Q<HE1<?JQ1%^-.X4I8K-'$^3PWT2(I`&QQD_)Z.
+M<#;2-XX;ZY](KG^2JL\_]^B6'8G82=0\C+PJK1Y;*0^^*WQ88C/(]7,5-H-4
+M;Y!]&JL&T3MKF=5;=XR0[<@N*3]B&9-2O3]^Q!L1KX2A2,Y"8;^$OB"G+RCH
+M"U+Z`N1`1KSP-"3_@)(8/";(D%BNRVU?*BS7I;;KLIG?=77PK"+XG3SXC\%E
+MVN`DDV?HZ#1T=!DZN@T=)_2DT9B:HPKNC-HIH<H09^B@E?A:%_3$K%`>_L!"
+M7(]^7FEY7TP3'XF=2OI]B3/TN#.D/N=+"E[3.QXPK4)MCO3I\:,4+4(...\.
+M..^^:2`MSKNKIAP+>?#[67EM@^/3>6W]L_/:2H-_:49:>!YT/U\6_*1>ROO&
+MWBKCW4KAH@45]"%EP1OUTN1<A$^@+5)9\"PSMT4*_7PW?SM:SZM#D,MV7C(W
+M;DB#H_=PWIO!&9^%,UPO502WA7%F`URQ\]+"_!\W(W6\4NG2.K14FEW'_VW2
+M#-K)&;2)66@,AY;[GVB,M)G?C#9SJ,-VWF*H_Y;_\0<74_]\(`WVWI/95R_E
+M%%R[73JMZDEY<&N+]%Z-[5R-I3^0_S,[+S_<UFA5JNTJ+JP6"!X6;'":H*/(
+M^=]I6@)#*I=&KW<)^&42/Q%+R/@*Y7H72IAAD'`QG6LG2`M'^M-^T8>Y+5*/
+M+%`,QX`75B4BF0)4I($>P>MRQ*"]7`N/^7R-,`WU*#C>"%$$4BM`^1J/@L%;
+MY&'?$\J"?F/GL-`A`_\6C'IBD6I"R!^')6%ULI`_P<4-TW3T5MBOKB8$Z#@L
+M52<+T`G[3.R#HEPLQ&5+7EQ>NG9]137^R--X=LZBPB49=IQ:5%.Q=@->4%.*
+M/UG]$IZZ`$]9D)ZR,#UE/IZ77X"GVNT+\.HJO*QT;<4FZZKU%:Z2TO45/ZTJ
+M7;?>ZJJN3+>55U>6VL)4VT9;V<:5\ZT+K"E&,[X<7E$!&0O2-FW"[?;[K'8[
+:?.,6/`4>V.%+*D20C-+)Z_\";.?>SN!F````
+`
+end

sys/dev/cxgb/ulp/toecore/cxgb_toedev.h

@@ -41,6 +41,8 @@ enum {
 	TOE_ID_CHELSIO_T2,
 	TOE_ID_CHELSIO_T3,
 	TOE_ID_CHELSIO_T3B,
-};
+	TOE_ID_CHELSIO_T3C,
+}
+	;
 
 #endif

sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c

@@ -38,14 +38,18 @@ __FBSDID("$FreeBSD$");
 #include <sys/limits.h>
 #include <sys/lock.h>
 #include <sys/mbuf.h>
+#include <sys/condvar.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/smp.h>
 #include <sys/socket.h>
 #include <sys/syslog.h>
 #include <sys/socketvar.h>
 #include <sys/uio.h>
+#include <sys/file.h>
 
 #include <machine/bus.h>
+#include <machine/cpu.h>
 
 #include <net/if.h>
 #include <net/route.h>
@@ -56,6 +60,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in_var.h>
 
 
+#include <dev/cxgb/cxgb_config.h>
 #include <dev/cxgb/cxgb_osdep.h>
 #include <dev/cxgb/sys/mbufq.h>
 
@@ -72,6 +77,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/common/cxgb_ctl_defs.h>
 #include <dev/cxgb/cxgb_l2t.h>
 #include <dev/cxgb/cxgb_offload.h>
+
 #include <vm/vm.h>
 #include <vm/vm_page.h>
 #include <vm/vm_map.h>
@@ -85,6 +91,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h>
 #include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
 #include <dev/cxgb/ulp/tom/cxgb_tcp.h>
+#include <dev/cxgb/ulp/tom/cxgb_vm.h>
 
 static int	(*pru_sosend)(struct socket *so, struct sockaddr *addr,
     struct uio *uio, struct mbuf *top, struct mbuf *control,
@@ -94,13 +101,11 @@ static int	(*pru_soreceive)(struct socket *so, struct sockaddr **paddr,
     struct uio *uio, struct mbuf **mp0, struct mbuf **controlp,
     int *flagsp);
 
-#ifdef notyet
-#define VM_HOLD_WRITEABLE	0x1
-static int  vm_fault_hold_user_pages(vm_offset_t addr, int len, vm_page_t *mp,
-    int *count, int flags);
-#endif
-static void vm_fault_unhold_pages(vm_page_t *m, int count);
 #define TMP_IOV_MAX 16
+#ifndef PG_FRAME
+#define PG_FRAME	~PAGE_MASK
+#endif
+#define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK)
 
 void
 t3_init_socket_ops(void)
@@ -110,20 +115,8 @@ t3_init_socket_ops(void)
 	prp = pffindtype(AF_INET, SOCK_STREAM);
 	pru_sosend = prp->pr_usrreqs->pru_sosend;
 	pru_soreceive = prp->pr_usrreqs->pru_soreceive;
-#ifdef TCP_USRREQS_OVERLOAD	
-	tcp_usrreqs.pru_connect = cxgb_tcp_usrreqs.pru_connect;
-	tcp_usrreqs.pru_abort = cxgb_tcp_usrreqs.pru_abort;
-	tcp_usrreqs.pru_listen = cxgb_tcp_usrreqs.pru_listen;
-	tcp_usrreqs.pru_send = cxgb_tcp_usrreqs.pru_send;
-	tcp_usrreqs.pru_abort = cxgb_tcp_usrreqs.pru_abort;
-	tcp_usrreqs.pru_disconnect = cxgb_tcp_usrreqs.pru_disconnect;
-	tcp_usrreqs.pru_close = cxgb_tcp_usrreqs.pru_close;
-	tcp_usrreqs.pru_shutdown = cxgb_tcp_usrreqs.pru_shutdown;
-	tcp_usrreqs.pru_rcvd = cxgb_tcp_usrreqs.pru_rcvd;
-#endif
 }
 
-
 struct cxgb_dma_info {
 	size_t			cdi_mapped;
 	int			cdi_nsegs;
@@ -182,21 +175,172 @@ iov_adj(struct iovec **iov, int *iovcnt, size_t count)
 	}
 }
 
-
 static void
-cxgb_zero_copy_free(void *cl, void *arg) {}
+cxgb_zero_copy_free(void *cl, void *arg)
+{
+	struct mbuf_vec *mv;
+	struct mbuf *m = (struct mbuf *)cl;
+
+	mv = mtomv(m);
+	/*
+	 * Physical addresses, don't try to free should be unheld separately from sbdrop
+	 *
+	 */
+	mv->mv_count = 0;
+	m_free_iovec(m, m->m_type);
+}
+
 
 static int
 cxgb_hold_iovec_pages(struct uio *uio, vm_page_t *m, int *held, int flags)
 {
+	struct iovec *iov = uio->uio_iov;
+	int iovcnt = uio->uio_iovcnt;
+	int err, i, count, totcount, maxcount, totbytes, npages, curbytes;
+	uint64_t start, end;
+	vm_page_t *mp;
 	
-	return (EINVAL);
+	totbytes = totcount = 0;
+	maxcount = *held;
+
+	mp = m;
+	for (totcount = i = 0; (i < iovcnt) && (totcount < maxcount);  i++, iov++) {
+		count = maxcount - totcount;
+		    
+		start = (uintptr_t)iov->iov_base;
+		end = (uintptr_t)((caddr_t)iov->iov_base + iov->iov_len);
+		start &= PG_FRAME;
+		end += PAGE_MASK;
+		end &= PG_FRAME;
+		npages = (end - start) >> PAGE_SHIFT;
+		
+		count = min(count, npages);
+
+		err = vm_fault_hold_user_pages((vm_offset_t)iov->iov_base, mp, count, flags);
+		if (err) {
+			vm_fault_unhold_pages(m, totcount);
+			return (err);
+		}
+		mp += count;
+		totcount += count;
+		curbytes = iov->iov_len;
+		if (count != npages)
+			curbytes = count*PAGE_SIZE - (((uintptr_t)iov->iov_base)&PAGE_MASK);
+		totbytes += curbytes;
+	}
+	uio->uio_resid -= totbytes;
+
+	return (0);
+}
+
+/*
+ * Returns whether a connection should enable DDP.  This happens when all of
+ * the following conditions are met:
+ * - the connection's ULP mode is DDP
+ * - DDP is not already enabled
+ * - the last receive was above the DDP threshold
+ * - receive buffers are in user space
+ * - receive side isn't shutdown (handled by caller)
+ * - the connection's receive window is big enough so that sizable buffers
+ *   can be posted without closing the window in the middle of DDP (checked
+ *   when the connection is offloaded)
+ */
+static int
+so_should_ddp(const struct toepcb *toep, int last_recv_len)
+{
+
+	DPRINTF("ulp_mode=%d last_recv_len=%d ddp_thresh=%d rcv_wnd=%ld ddp_copy_limit=%d\n",
+	    toep->tp_ulp_mode, last_recv_len,  TOM_TUNABLE(toep->tp_toedev, ddp_thres),
+	    toep->tp_tp->rcv_wnd, (TOM_TUNABLE(toep->tp_toedev, ddp_copy_limit) + DDP_RSVD_WIN));
+
+	return toep->tp_ulp_mode == ULP_MODE_TCPDDP && (toep->tp_ddp_state.kbuf[0] == NULL) &&
+	       last_recv_len > TOM_TUNABLE(toep->tp_toedev, ddp_thres) &&
+	       toep->tp_tp->rcv_wnd > 
+	           (TOM_TUNABLE(toep->tp_toedev, ddp_copy_limit) + DDP_RSVD_WIN);
+}
+
+static inline int
+is_ddp(const struct mbuf *m)
+{
+	return (m->m_flags & M_DDP);
+}
+
+static inline int
+is_ddp_psh(const struct mbuf *m)
+{
+        return is_ddp(m) && (m->m_pkthdr.csum_flags & DDP_BF_PSH);
+}
+
+static int
+m_uiomove(const struct mbuf *m, int offset, int len, struct uio *uio)
+{
+	int curlen, startlen, resid_init, err = 0;
+	caddr_t buf;
+
+	DPRINTF("m_uiomove(m=%p, offset=%d, len=%d, ...)\n",
+	    m, offset, len);
+
+	startlen = len;
+	resid_init = uio->uio_resid;
+	while (m && len) {
+		buf = mtod(m, caddr_t);
+		curlen = m->m_len;
+		if (offset && (offset < curlen)) {
+			curlen -= offset;
+			buf += offset;
+			offset = 0;
+		} else if (offset) {
+			offset -= curlen;
+			m = m->m_next;
+			continue;
+		}
+		err = uiomove(buf, min(len, curlen), uio);
+		if (err) {
+			printf("uiomove returned %d\n", err);
+			return (err);
+		}
+		
+		len -= min(len, curlen);
+		m = m->m_next;
+	}
+	DPRINTF("copied %d bytes - resid_init=%d uio_resid=%d\n",
+	    startlen - len, resid_init, uio->uio_resid);
+	return (err);
+}
+
+/*
+ * Copy data from an sk_buff to an iovec.  Deals with RX_DATA, which carry the
+ * data in the sk_buff body, and with RX_DATA_DDP, which place the data in a
+ * DDP buffer.
+ */
+static inline int
+copy_data(const struct mbuf *m, int offset, int len, struct uio *uio)
+{
+	struct iovec *to = uio->uio_iov;
+	int err;
+
+	
+	if (__predict_true(!is_ddp(m))) {                             /* RX_DATA */
+		return m_uiomove(m, offset, len, uio);
+	} if (__predict_true(m->m_ddp_flags & DDP_BF_NOCOPY)) { /* user DDP */
+		to->iov_len -= len;
+		to->iov_base = ((caddr_t)to->iov_base) + len;
+		uio->uio_iov = to;
+		uio->uio_resid -= len;
+		return (0);
+	}
+	err = t3_ddp_copy(m, offset, uio, len);             /* kernel DDP */
+	return (err);
 }
 
 static void
-cxgb_wait_dma_completion(struct toepcb *tp)
+cxgb_wait_dma_completion(struct toepcb *toep)
 {
+	struct mtx *lock;
 	
+	lock = &toep->tp_tp->t_inpcb->inp_mtx;
+	INP_LOCK(toep->tp_tp->t_inpcb);
+	cv_wait_unlock(&toep->tp_cv, lock);
 }
 
 static int
@@ -235,6 +379,12 @@ cxgb_vm_page_to_miov(struct toepcb *toep, struct uio *uio, struct mbuf **m)
 
 	*m = m0;
 
+	/*
+	 * This appears to be a no-op at the moment
+	 * as busdma is all or nothing need to make
+	 * sure the tag values are large enough
+	 *
+	 */
 	if (cdi.cdi_mapped < uio->uio_resid) {
 		uio->uio_resid -= cdi.cdi_mapped;
 	} else
@@ -305,10 +455,11 @@ t3_sosend(struct socket *so, struct uio *uio)
 		}
 		uio->uio_resid -= m->m_pkthdr.len;
 		sent += m->m_pkthdr.len;
-		sbappend_locked(&so->so_snd, m);
+		sbappend(&so->so_snd, m);
 		t3_push_frames(so, TRUE);
 		iov_adj(&uiotmp.uio_iov, &iovcnt, uiotmp.uio_resid);
 	}
+
 	/*
 	 * Wait for pending I/O to be DMA'd to the card 
 	 * 
@@ -357,7 +508,7 @@ cxgb_sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
 		zcopy_thres = TOM_TUNABLE(tdev, zcopy_sosend_partial_thres);
 		zcopy_enabled = TOM_TUNABLE(tdev, zcopy_sosend_enabled);
 
-		if ((uio->uio_resid > zcopy_thres) &&
+		if (uio && (uio->uio_resid > zcopy_thres) &&
 		    (uio->uio_iovcnt < TMP_IOV_MAX) &&  ((so->so_state & SS_NBIO) == 0)
 		    && zcopy_enabled) {
 			rv = t3_sosend(so, uio);
@@ -368,36 +519,378 @@ cxgb_sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
 	return pru_sosend(so, addr, uio, top, control, flags, td);
 }
 
+/*
+ * Following replacement or removal of the first mbuf on the first mbuf chain
+ * of a socket buffer, push necessary state changes back into the socket
+ * buffer so that other consumers see the values consistently.  'nextrecord'
+ * is the callers locally stored value of the original value of
+ * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes.
+ * NOTE: 'nextrecord' may be NULL.
+ */
+static __inline void
+sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord)
+{
+
+	SOCKBUF_LOCK_ASSERT(sb);
+	/*
+	 * First, update for the new value of nextrecord.  If necessary, make
+	 * it the first record.
+	 */
+	if (sb->sb_mb != NULL)
+		sb->sb_mb->m_nextpkt = nextrecord;
+	else
+		sb->sb_mb = nextrecord;
+
+        /*
+         * Now update any dependent socket buffer fields to reflect the new
+         * state.  This is an expanded inline of SB_EMPTY_FIXUP(), with the
+	 * addition of a second clause that takes care of the case where
+	 * sb_mb has been updated, but remains the last record.
+         */
+        if (sb->sb_mb == NULL) {
+                sb->sb_mbtail = NULL;
+                sb->sb_lastrecord = NULL;
+        } else if (sb->sb_mb->m_nextpkt == NULL)
+                sb->sb_lastrecord = sb->sb_mb;
+}
+
+#define IS_NONBLOCKING(so)	((so)->so_state & SS_NBIO)
+
 
 static int
-t3_soreceive(struct socket *so, struct uio *uio)
+t3_soreceive(struct socket *so, int *flagsp, struct uio *uio)
 {
-#ifdef notyet
-	int i, rv, count, hold_resid, sent, iovcnt;
-	struct iovec iovtmp[TMP_IOV_MAX], *iovtmpp, *iov;
 	struct tcpcb *tp = sototcpcb(so);
 	struct toepcb *toep = tp->t_toe;
 	struct mbuf *m;
-	struct uio uiotmp;
+	uint32_t offset;
+	int err, flags, avail, len, copied, copied_unacked;
+	int target;		/* Read at least this many bytes */
+	int user_ddp_ok;
+	struct ddp_state *p;
+	struct inpcb *inp = sotoinpcb(so);
+
+	avail = offset = copied = copied_unacked = 0;
+	flags = flagsp ? (*flagsp &~ MSG_EOR) : 0;
+	err = sblock(&so->so_rcv, SBLOCKWAIT(flags));
+	p = &toep->tp_ddp_state;
+
+	if (err)
+		return (err);
+
+	SOCKBUF_LOCK(&so->so_rcv);
+	p->user_ddp_pending = 0;
+restart:
+	len = uio->uio_resid;
+	m = so->so_rcv.sb_mb;
+	target = (flags & MSG_WAITALL) ? len : so->so_rcv.sb_lowat;
+	user_ddp_ok = p->ubuf_ddp_ready;
+	p->cancel_ubuf = 0;
+
+	if (len == 0)
+		goto done;
+#if 0	
+	while (m && m->m_len == 0) {
+		so->so_rcv.sb_mb = m_free(m);
+		m = so->so_rcv.sb_mb;
+	}
+#endif	
+	if (m) 
+		goto got_mbuf;
+
+	/* empty receive queue */
+	if (copied >= target && (so->so_rcv.sb_mb == NULL) &&
+	    !p->user_ddp_pending)
+		goto done;
+
+	if (copied) {
+		if (so->so_error || tp->t_state == TCPS_CLOSED || 
+		    (so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)))
+			goto done;
+	} else {
+		if (so->so_state & SS_NOFDREF)
+			goto done;
+		if (so->so_error) {
+			err = so->so_error;
+			so->so_error = 0;
+			goto done;
+		}
+		if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 
+			goto done;
+		if (so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED))
+			goto done;
+		if (tp->t_state == TCPS_CLOSED) {
+			err = ENOTCONN; 
+			goto done;
+		}
+	}
+	if (so->so_rcv.sb_mb && !p->user_ddp_pending) {
+		SOCKBUF_UNLOCK(&so->so_rcv);
+		INP_LOCK(inp);
+		t3_cleanup_rbuf(tp, copied_unacked);
+		INP_UNLOCK(inp);
+		SOCKBUF_LOCK(&so->so_rcv);
+		copied_unacked = 0;
+		goto restart;
+	}
+	if (p->kbuf[0] && user_ddp_ok && !p->user_ddp_pending && 
+	    uio->uio_iov->iov_len > p->kbuf[0]->dgl_length &&
+	    p->ubuf_ddp_ready) {
+		p->user_ddp_pending =
+		    !t3_overlay_ubuf(so, uio, IS_NONBLOCKING(so), flags, 1, 1);
+		if (p->user_ddp_pending) {
+			p->kbuf_posted++;
+			user_ddp_ok = 0;
+		}
+	}
+	if (p->kbuf[0] && (p->kbuf_posted == 0)) {
+		t3_post_kbuf(so, 1, IS_NONBLOCKING(so));
+		p->kbuf_posted++;
+	}
+	if (p->user_ddp_pending) {
+		/* One shot at DDP if we already have enough data */
+		if (copied >= target)
+			user_ddp_ok = 0;
+
+		DPRINTF("sbwaiting 1\n");
+		if ((err = sbwait(&so->so_rcv)) != 0)
+			goto done;
+//for timers to work			await_ddp_completion(sk, flags, &timeo);
+	} else if (copied >= target)
+		goto done;
+	else {
+		if (copied_unacked) {
+			int i = 0;
+
+			SOCKBUF_UNLOCK(&so->so_rcv);
+			INP_LOCK(inp);
+			t3_cleanup_rbuf(tp, copied_unacked);
+			INP_UNLOCK(inp);
+			copied_unacked = 0;
+			if (mp_ncpus > 1)
+				while (i++ < 200 && so->so_rcv.sb_mb == NULL)
+					cpu_spinwait();
+			SOCKBUF_LOCK(&so->so_rcv);
+		}
+		
+		if (so->so_rcv.sb_mb)
+			goto restart;
+		DPRINTF("sbwaiting 2 copied=%d target=%d avail=%d so=%p mb=%p cc=%d\n", copied, target, avail, so,
+		    so->so_rcv.sb_mb, so->so_rcv.sb_cc);
+		if ((err = sbwait(&so->so_rcv)) != 0)
+			goto done;
+	}
+     	goto restart;
+got_mbuf:
+	KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d m_pktlen=%d\n", !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len, m->m_pkthdr.len));
+	KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x m->m_len=%d",
+		m->m_next, m->m_nextpkt, m->m_flags, m->m_len));
+	if (m->m_pkthdr.len == 0) {
+		if ((m->m_ddp_flags & DDP_BF_NOCOPY) == 0)
+			panic("empty mbuf and NOCOPY not set\n");
+		CTR0(KTR_TOM, "ddp done notification");
+		p->user_ddp_pending = 0;
+		sbdroprecord_locked(&so->so_rcv);
+		goto done;
+	}
+	
+	offset = toep->tp_copied_seq + copied_unacked - m->m_seq;
+	DPRINTF("m=%p copied_seq=0x%x copied_unacked=%d m_seq=0x%x offset=%d pktlen=%d is_ddp(m)=%d\n",
+	    m, toep->tp_copied_seq, copied_unacked, m->m_seq, offset, m->m_pkthdr.len, !!is_ddp(m));
+
+	if (offset >= m->m_pkthdr.len)
+		panic("t3_soreceive: OFFSET >= LEN offset %d copied_seq 0x%x seq 0x%x "
+		    "pktlen %d ddp flags 0x%x", offset, toep->tp_copied_seq + copied_unacked, m->m_seq,
+		    m->m_pkthdr.len, m->m_ddp_flags);
+
+	avail = m->m_pkthdr.len - offset;
+	if (len < avail) {
+		if (is_ddp(m) && (m->m_ddp_flags & DDP_BF_NOCOPY)) 
+			panic("bad state in t3_soreceive len=%d avail=%d offset=%d\n", len, avail, offset);
+		avail = len;
+	}
+	CTR4(KTR_TOM, "t3_soreceive: m_len=%u offset=%u len=%u m_seq=0%08x", m->m_pkthdr.len, offset, len, m->m_seq);
+	
+#ifdef URGENT_DATA_SUPPORTED
+	/*
+	 * Check if the data we are preparing to copy contains urgent
+	 * data.  Either stop short of urgent data or skip it if it's
+	 * first and we are not delivering urgent data inline.
+	 */
+	if (__predict_false(toep->tp_urg_data)) {
+		uint32_t urg_offset = tp->rcv_up - tp->copied_seq + copied_unacked;
+		
+		if (urg_offset < avail) {
+			if (urg_offset) {
+				/* stop short of the urgent data */
+				avail = urg_offset;
+			} else if ((so->so_options & SO_OOBINLINE) == 0) {
+				/* First byte is urgent, skip */
+				toep->tp_copied_seq++;
+				offset++;
+				avail--;
+				if (!avail)
+					goto skip_copy;
+			}	
+		}	
+	}	
+#endif
+	if (is_ddp_psh(m) || offset) {
+		user_ddp_ok = 0;
+#ifdef T3_TRACE	
+		T3_TRACE0(TIDTB(so), "t3_sosend: PSH");
+#endif	
+	}
+	
+	if (user_ddp_ok && !p->user_ddp_pending &&
+	    uio->uio_iov->iov_len > p->kbuf[0]->dgl_length &&
+	    p->ubuf_ddp_ready) {
+		p->user_ddp_pending = 
+		    !t3_overlay_ubuf(so, uio, IS_NONBLOCKING(so), flags, 1, 1);
+		if (p->user_ddp_pending) {
+			p->kbuf_posted++;
+			user_ddp_ok = 0;
+		}
+		DPRINTF("user_ddp_pending=%d\n", p->user_ddp_pending);
+	} else
+		DPRINTF("user_ddp_ok=%d user_ddp_pending=%d iov_len=%ld dgl_length=%d ubuf_ddp_ready=%d ulp_mode=%d is_ddp(m)=%d flags=0x%x ubuf=%p kbuf_posted=%d\n",
+		    user_ddp_ok, p->user_ddp_pending, uio->uio_iov->iov_len, p->kbuf[0] ? p->kbuf[0]->dgl_length : 0,
+		    p->ubuf_ddp_ready, toep->tp_ulp_mode, !!is_ddp(m), m->m_ddp_flags, p->ubuf, p->kbuf_posted);
 	
 	/*
-	 * Events requiring iteration:
+	 * If MSG_TRUNC is specified the data is discarded.
-	 *  - number of pages exceeds max hold pages for process or system
+	 * XXX need to check pr_atomic
-	 *  - number of pages exceeds maximum sg entries for a single WR
-	 *
-	 * We're limited to holding 128 pages at once - and we're limited to
-	 * 34 SG entries per work request, but each SG entry can be any number 
-	 * of contiguous pages
-	 *
 	 */
+	KASSERT(avail > 0, ("avail=%d resid=%d offset=%d", avail,  uio->uio_resid, offset));
+	if (__predict_true(!(flags & MSG_TRUNC))) {
+		int resid = uio->uio_resid;
 		
-	uiotmp = *uio;
+		SOCKBUF_UNLOCK(&so->so_rcv);
-	iovcnt = uio->uio_iovcnt;
+		if ((err = copy_data(m, offset, avail, uio))) {
-	iov = uio->uio_iov;
+			if (err)
-	sent = 0;
+				err = EFAULT;
-	re;
+			goto done_unlocked;
+		}
+		SOCKBUF_LOCK(&so->so_rcv);
+		if (avail != (resid - uio->uio_resid))
+			printf("didn't copy all bytes :-/ avail=%d offset=%d pktlen=%d resid=%d uio_resid=%d copied=%d copied_unacked=%d is_ddp(m)=%d\n",
+			    avail, offset, m->m_pkthdr.len, resid, uio->uio_resid, copied, copied_unacked, is_ddp(m));
+	}
+	
+	copied += avail;
+	copied_unacked += avail;
+	len -= avail;
+	
+#ifdef URGENT_DATA_SUPPORTED
+skip_copy:
+	if (tp->urg_data && after(tp->copied_seq + copied_unacked, tp->urg_seq))
+		tp->urg_data = 0;
 #endif
-	return (0);
+	/*
+	 * If the buffer is fully consumed free it.  If it's a DDP
+	 * buffer also handle any events it indicates.
+	 */
+	if (avail + offset >= m->m_pkthdr.len) {
+		unsigned int fl = m->m_ddp_flags;
+		int exitnow, got_psh = 0, nomoredata = 0;
+		int count;
+		struct mbuf *nextrecord;
+
+		if (p->kbuf[0] != NULL && is_ddp(m) && (fl & 1)) {
+			if (is_ddp_psh(m) && p->user_ddp_pending)
+				got_psh = 1;
+			
+			if (fl & DDP_BF_NOCOPY)
+				p->user_ddp_pending = 0;
+			else if ((fl & DDP_BF_NODATA) && IS_NONBLOCKING(so)) {
+				p->kbuf_posted--;
+				nomoredata = 1;
+			} else {
+				p->kbuf_posted--;
+				p->ubuf_ddp_ready = 1;
+			}
+		}
+
+		nextrecord = m->m_nextpkt;
+		count = m->m_pkthdr.len;
+		while (count > 0) {
+			count -= m->m_len;
+			KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n", !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len));
+			sbfree(&so->so_rcv, m);
+			so->so_rcv.sb_mb = m_free(m);
+			m = so->so_rcv.sb_mb;
+		}
+		sockbuf_pushsync(&so->so_rcv, nextrecord);
+#if 0
+		sbdrop_locked(&so->so_rcv, m->m_pkthdr.len);
+#endif		
+		exitnow = got_psh || nomoredata;
+		if  ((so->so_rcv.sb_mb == NULL) && exitnow)
+			goto done;
+		if (copied_unacked > (so->so_rcv.sb_hiwat >> 2)) {
+			SOCKBUF_UNLOCK(&so->so_rcv);
+			INP_LOCK(inp);
+			t3_cleanup_rbuf(tp, copied_unacked);
+			INP_UNLOCK(inp);
+			copied_unacked = 0;
+			SOCKBUF_LOCK(&so->so_rcv);
+		}
+	} 
+	if (len > 0)
+		goto restart;
+
+	done:
+	/*
+	 * If we can still receive decide what to do in preparation for the
+	 * next receive.  Note that RCV_SHUTDOWN is set if the connection
+	 * transitioned to CLOSE but not if it was in that state to begin with.
+	 */
+	if (__predict_true((so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)) == 0)) {
+		if (p->user_ddp_pending) {
+			SOCKBUF_UNLOCK(&so->so_rcv);
+			SOCKBUF_LOCK(&so->so_rcv);
+			user_ddp_ok = 0;
+			t3_cancel_ubuf(toep);
+			if (so->so_rcv.sb_mb) {
+				if (copied < 0)
+					copied = 0;
+				if (len > 0)
+					goto restart;
+			}
+			p->user_ddp_pending = 0;
+		}
+		if ((p->kbuf[0] != NULL) && (p->kbuf_posted == 0)) {
+#ifdef T3_TRACE
+			T3_TRACE0(TIDTB(so),
+			  "chelsio_recvmsg: about to exit, repost kbuf");
+#endif
+
+			t3_post_kbuf(so, 1, IS_NONBLOCKING(so));
+			p->kbuf_posted++;
+		} else if (so_should_ddp(toep, copied) && uio->uio_iovcnt == 1) {
+			CTR1(KTR_TOM ,"entering ddp on tid=%u", toep->tp_tid);
+			if (!t3_enter_ddp(so, TOM_TUNABLE(TOE_DEV(so),
+				    ddp_copy_limit), 0, IS_NONBLOCKING(so)))
+				p->kbuf_posted = 1;
+		}
+	}
+#ifdef T3_TRACE
+	T3_TRACE5(TIDTB(so),
+		  "chelsio_recvmsg <-: copied %d len %d buffers_freed %d "
+		  "kbuf_posted %d user_ddp_pending %u",
+		  copied, len, buffers_freed, p ? p->kbuf_posted : -1, 
+	    p->user_ddp_pending);
+#endif
+	SOCKBUF_UNLOCK(&so->so_rcv);
+done_unlocked:	
+	if (copied_unacked) {
+		INP_LOCK(inp);
+		t3_cleanup_rbuf(tp, copied_unacked);
+		INP_UNLOCK(inp);
+	}
+	sbunlock(&so->so_rcv);
+
+	return (err);
 }
 
 static int
@@ -405,9 +898,11 @@ cxgb_soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
     struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
 {
 	struct toedev *tdev;
-	int rv, zcopy_thres, zcopy_enabled;
+	int rv, zcopy_thres, zcopy_enabled, flags;
 	struct tcpcb *tp = sototcpcb(so);
 
+	flags = flagsp ? *flagsp &~ MSG_EOR : 0;
+	
 	/*
 	 * In order to use DMA direct from userspace the following
 	 * conditions must be met:
@@ -421,150 +916,30 @@ cxgb_soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
 	 *  - iovcnt is 1
 	 *
 	 */
-	if (tp->t_flags & TF_TOE) {
+	
+	if ((tp->t_flags & TF_TOE) && uio && ((flags & (MSG_WAITALL|MSG_OOB|MSG_PEEK|MSG_DONTWAIT)) == 0)
+	    && (uio->uio_iovcnt == 1) && (mp0 == NULL)) {
 		tdev =  TOE_DEV(so);
 		zcopy_thres = TOM_TUNABLE(tdev, ddp_thres);
 		zcopy_enabled = TOM_TUNABLE(tdev, ddp);
 		if ((uio->uio_resid > zcopy_thres) &&
-		    (uio->uio_iovcnt == 1) &&  ((so->so_state & SS_NBIO) == 0)
+		    (uio->uio_iovcnt == 1)
 		    && zcopy_enabled) {
-			rv = t3_soreceive(so, uio);
+			rv = t3_soreceive(so, flagsp, uio);
 			if (rv != EAGAIN)
 				return (rv);
+			else
+				printf("returned EAGAIN\n");
 		} 
-	}
+	} else if ((tp->t_flags & TF_TOE) && uio && mp0 == NULL)
-	
+		printf("skipping t3_soreceive flags=0x%x iovcnt=%d sb_state=0x%x\n",
+		    flags, uio->uio_iovcnt, so->so_rcv.sb_state);
 	return pru_soreceive(so, psa, uio, mp0, controlp, flagsp);
 }
 
-
 void
 t3_install_socket_ops(struct socket *so)
 {
 	so->so_proto->pr_usrreqs->pru_sosend = cxgb_sosend;
 	so->so_proto->pr_usrreqs->pru_soreceive = cxgb_soreceive;
 }
-
-/*
- * This routine takes a user address range and does the following:
- *  - validate that the user has access to those pages (flags indicates read or write) - if not fail
- *  - validate that count is enough to hold range number of pages - if not fail
- *  - fault in any non-resident pages
- *  - if the user is doing a read force a write fault for any COWed pages
- *  - if the user is doing a read mark all pages as dirty
- *  - hold all pages
- *  - return number of pages in count
- */
-#ifdef notyet
-static int
-vm_fault_hold_user_pages(vm_offset_t addr, int len, vm_page_t *mp, int *count, int flags)
-{
-
-	vm_offset_t start, va;
-	vm_paddr_t pa;
-	int pageslen, faults, rv;
-	
-	struct thread *td;
-	vm_map_t map;
-	pmap_t pmap;
-	vm_page_t m, *pages;
-	vm_prot_t prot;
-	
-	start = addr & ~PAGE_MASK;
-	pageslen = roundup2(addr + len, PAGE_SIZE);
-	if (*count < (pageslen >> PAGE_SHIFT))
-		return (EFBIG);
-
-	*count = pageslen >> PAGE_SHIFT;
-	/*
-	 * Check that virtual address range is legal
-	 * This check is somewhat bogus as on some architectures kernel
-	 * and user do not share VA - however, it appears that all FreeBSD
-	 * architectures define it
-	 */
-	if (addr + len > VM_MAXUSER_ADDRESS)
-		return (EFAULT);
-	
-	td = curthread;
-	map = &td->td_proc->p_vmspace->vm_map;
-	pmap = &td->td_proc->p_vmspace->vm_pmap;
-	pages = mp;
-
-	prot = (flags & VM_HOLD_WRITEABLE) ? VM_PROT_WRITE : VM_PROT_READ;
-	bzero(pages, sizeof(vm_page_t *) * (*count));
-retry:
-	
-	/*
-	 * First optimistically assume that all pages are resident (and R/W if for write)
-	 * if so just mark pages as held (and dirty if for write) and return
-	 */
-	vm_page_lock_queues();
-	for (pages = mp, faults = 0, va = start; va < pageslen; va += PAGE_SIZE, pages++) {
-		/*
-		 * Assure that we only hold the page once
-		 */
-		if (*pages == NULL) {
-			/*
-			 * page queue mutex is recursable so this is OK
-			 * it would be really nice if we had an unlocked version of this so
-			 * we were only acquiring the pmap lock 1 time as opposed to potentially
-			 * many dozens of times
-			 */
-			m = pmap_extract_and_hold(pmap, va, prot);
-			if (m == NULL) {
-				faults++;
-				continue;
-			}
-			*pages = m;
-		if (flags & VM_HOLD_WRITEABLE)
-			vm_page_dirty(m);
-		}
-	}
-	vm_page_unlock_queues();
-	
-	if (faults == 0) 
-		return (0);
-	/*
-	 * Pages either have insufficient permissions or are not present
-	 * trigger a fault where neccessary
-	 * 
-	 */
-	for (va = start; va < pageslen; va += PAGE_SIZE) {
-		m = NULL;
-		pa = pmap_extract(pmap, va);
-		rv = 0;
-		if (pa)
-			m = PHYS_TO_VM_PAGE(pa);
-		if (flags & VM_HOLD_WRITEABLE) {
-			if (m == NULL  || (m->flags & PG_WRITEABLE) == 0)
-				rv = vm_fault(map, va, VM_PROT_WRITE, VM_FAULT_DIRTY);
-		} else if (m == NULL)
-			rv = vm_fault(map, va, VM_PROT_READ, VM_FAULT_NORMAL);
-		if (rv)
-			goto error;
-	} 
-	goto retry;
-
-error:	
-	vm_page_lock_queues();
-	for (pages = mp, va = start; va < pageslen; va += PAGE_SIZE, pages++) 
-		if (*pages)
-			vm_page_unhold(*pages);
-	vm_page_unlock_queues();
-	return (EFAULT);
-}
-#endif
-
-static void
-vm_fault_unhold_pages(vm_page_t *mp, int count)
-{
-
-	KASSERT(count >= 0, ("negative count %d", count));
-	vm_page_lock_queues();
-	while (count--) {
-		vm_page_unhold(*mp);
-		mp++;
-	}
-	vm_page_unlock_queues();
-}
-

sys/dev/cxgb/ulp/tom/cxgb_ddp.c

@@ -0,0 +1,735 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+***************************************************************************/
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+#include <sys/fcntl.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/limits.h>
+#include <sys/lock.h>
+#include <sys/mbuf.h>
+#include <sys/condvar.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/syslog.h>
+#include <sys/socketvar.h>
+#include <sys/uio.h>
+
+#include <machine/bus.h>
+
+#include <net/if.h>
+#include <net/route.h>
+
+#include <netinet/in.h>
+#include <netinet/in_pcb.h>
+#include <netinet/in_systm.h>
+#include <netinet/in_var.h>
+
+
+#include <dev/cxgb/cxgb_osdep.h>
+#include <dev/cxgb/sys/mbufq.h>
+
+#include <netinet/tcp.h>
+#include <netinet/tcp_var.h>
+#include <netinet/tcp_fsm.h>
+#include <netinet/tcp_offload.h>
+#include <net/route.h>
+
+#include <dev/cxgb/t3cdev.h>
+#include <dev/cxgb/common/cxgb_firmware_exports.h>
+#include <dev/cxgb/common/cxgb_t3_cpl.h>
+#include <dev/cxgb/common/cxgb_tcb.h>
+#include <dev/cxgb/common/cxgb_ctl_defs.h>
+#include <dev/cxgb/cxgb_l2t.h>
+#include <dev/cxgb/cxgb_offload.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
+
+#include <dev/cxgb/sys/mvec.h>
+#include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
+#include <dev/cxgb/ulp/tom/cxgb_defs.h>
+#include <dev/cxgb/ulp/tom/cxgb_tom.h>
+#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h>
+#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
+#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
+#include <dev/cxgb/ulp/tom/cxgb_vm.h>
+
+#define MAX_SCHEDULE_TIMEOUT	300
+
+/*
+ * Return the # of page pods needed to accommodate a # of pages.
+ */
+static inline unsigned int
+pages2ppods(unsigned int pages)
+{
+	return (pages + PPOD_PAGES - 1) / PPOD_PAGES + NUM_SENTINEL_PPODS;
+}
+
+/**
+ *	t3_pin_pages - pin a user memory range and prepare it for DDP
+ *	@addr - the starting address
+ *	@len - the length of the range
+ *	@newgl - contains the pages and physical addresses of the pinned range
+ *	@gl - an existing gather list, may be %NULL
+ *
+ *	Pins the pages in the user-space memory range [addr, addr + len) and
+ *	maps them for DMA.  Returns a gather list with the pinned pages and
+ *	their physical addresses.  If @gl is non NULL the pages it describes
+ *	are compared against the pages for [addr, addr + len), and if the
+ *	existing gather list already covers the range a new list is not
+ *	allocated.  Returns 0 on success, or a negative errno.  On success if
+ *	a new gather list was allocated it is returned in @newgl.
+ */ 
+static int
+t3_pin_pages(bus_dma_tag_t tag, bus_dmamap_t map, vm_offset_t addr,
+    size_t len, struct ddp_gather_list **newgl,
+    const struct ddp_gather_list *gl)
+{
+	int i = 0, err;
+	size_t pg_off;
+	unsigned int npages;
+	struct ddp_gather_list *p;
+
+	/*
+	 * XXX need x86 agnostic check
+	 */
+	if (addr + len > VM_MAXUSER_ADDRESS)
+		return (EFAULT);
+
+	pg_off = addr & PAGE_MASK;
+	npages = (pg_off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	p = malloc(sizeof(struct ddp_gather_list) + npages * sizeof(vm_page_t *),
+	    M_DEVBUF, M_NOWAIT|M_ZERO);
+	if (p == NULL)
+		return (ENOMEM);
+
+	err = vm_fault_hold_user_pages(addr, p->dgl_pages, npages, VM_HOLD_WRITEABLE);
+	if (err)
+		goto free_gl;
+
+	if (gl && gl->dgl_offset == pg_off && gl->dgl_nelem >= npages &&
+	    gl->dgl_length >= len) {
+		for (i = 0; i < npages; i++)
+			if (p->dgl_pages[i] != gl->dgl_pages[i])
+				goto different_gl;
+		err = 0;
+		goto unpin;
+	}
+
+different_gl:
+	p->dgl_length = len;
+	p->dgl_offset = pg_off;
+	p->dgl_nelem = npages;
+#ifdef NEED_BUSDMA
+	p->phys_addr[0] = pci_map_page(pdev, p->pages[0], pg_off,
+				       PAGE_SIZE - pg_off,
+				       PCI_DMA_FROMDEVICE) - pg_off;
+	for (i = 1; i < npages; ++i)
+		p->phys_addr[i] = pci_map_page(pdev, p->pages[i], 0, PAGE_SIZE,
+					       PCI_DMA_FROMDEVICE);
+#endif	
+	*newgl = p;
+	return (0);
+unpin:
+	vm_fault_unhold_pages(p->dgl_pages, npages);
+
+free_gl:
+	
+	free(p, M_DEVBUF);
+	*newgl = NULL;
+	return (err);
+}
+
+static void
+unmap_ddp_gl(const struct ddp_gather_list *gl)
+{
+#ifdef NEED_BUSDMA	
+	int i;
+
+	if (!gl->nelem)
+		return;
+
+	pci_unmap_page(pdev, gl->phys_addr[0] + gl->offset,
+		       PAGE_SIZE - gl->offset, PCI_DMA_FROMDEVICE);
+	for (i = 1; i < gl->nelem; ++i)
+		pci_unmap_page(pdev, gl->phys_addr[i], PAGE_SIZE,
+			       PCI_DMA_FROMDEVICE);
+
+#endif
+}
+
+static void
+ddp_gl_free_pages(struct ddp_gather_list *gl, int dirty)
+{
+	/*
+	 * XXX mark pages as dirty before unholding 
+	 */
+	vm_fault_unhold_pages(gl->dgl_pages, gl->dgl_nelem);
+}
+
+void
+t3_free_ddp_gl(struct ddp_gather_list *gl)
+{
+	unmap_ddp_gl(gl);
+	ddp_gl_free_pages(gl, 0);
+	free(gl, M_DEVBUF);
+}
+
+/* Max # of page pods for a buffer, enough for 1MB buffer at 4KB page size */
+#define MAX_PPODS 64U
+
+/*
+ * Allocate page pods for DDP buffer 1 (the user buffer) and set up the tag in
+ * the TCB.  We allocate page pods in multiples of PPOD_CLUSTER_SIZE.  First we
+ * try to allocate enough page pods to accommodate the whole buffer, subject to
+ * the MAX_PPODS limit.  If that fails we try to allocate PPOD_CLUSTER_SIZE page
+ * pods before failing entirely.
+ */
+static int
+alloc_buf1_ppods(struct socket *so, struct ddp_state *p,
+			    unsigned long addr, unsigned int len)
+{
+	int err, tag, npages, nppods;
+	struct tom_data *d = TOM_DATA(TOE_DEV(so));
+
+	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	npages = ((addr & PAGE_MASK) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	nppods = min(pages2ppods(npages), MAX_PPODS);
+	nppods = roundup2(nppods, PPOD_CLUSTER_SIZE);
+	err = t3_alloc_ppods(d, nppods, &tag);
+	if (err && nppods > PPOD_CLUSTER_SIZE) {
+		nppods = PPOD_CLUSTER_SIZE;
+		err = t3_alloc_ppods(d, nppods, &tag);
+	}
+	if (err)
+		return (ENOMEM);
+
+	p->ubuf_nppods = nppods;
+	p->ubuf_tag = tag;
+#if NUM_DDP_KBUF == 1
+	t3_set_ddp_tag(so, 1, tag << 6);
+#endif
+	return (0);
+}
+
+/*
+ * Starting offset for the user DDP buffer.  A non-0 value ensures a DDP flush
+ * won't block indefinitely if there's nothing to place (which should be rare).
+ */
+#define UBUF_OFFSET 1
+
+static __inline unsigned long
+select_ddp_flags(const struct socket *so, int buf_idx,
+                 int nonblock, int rcv_flags)
+{
+	if (buf_idx == 1) {
+		if (__predict_false(rcv_flags & MSG_WAITALL))
+			return V_TF_DDP_PSH_NO_INVALIDATE0(1) |
+			       V_TF_DDP_PSH_NO_INVALIDATE1(1) |
+			       V_TF_DDP_PUSH_DISABLE_1(1);
+		if (nonblock)
+			return V_TF_DDP_BUF1_FLUSH(1);
+
+		return V_TF_DDP_BUF1_FLUSH(!TOM_TUNABLE(TOE_DEV(so),
+							ddp_push_wait));
+	}
+
+	if (__predict_false(rcv_flags & MSG_WAITALL))
+		return V_TF_DDP_PSH_NO_INVALIDATE0(1) |
+		       V_TF_DDP_PSH_NO_INVALIDATE1(1) |
+		       V_TF_DDP_PUSH_DISABLE_0(1);
+	if (nonblock)
+		return V_TF_DDP_BUF0_FLUSH(1);
+
+	return V_TF_DDP_BUF0_FLUSH(!TOM_TUNABLE(TOE_DEV(so), ddp_push_wait));
+}
+
+/*
+ * Reposts the kernel DDP buffer after it has been previously become full and
+ * invalidated.  We just need to reset the offset and adjust the DDP flags.
+ * Conveniently, we can set the flags and the offset with a single message.
+ * Note that this function does not set the buffer length.  Again conveniently
+ * our kernel buffer is of fixed size.  If the length needs to be changed it
+ * needs to be done separately.
+ */
+static void
+t3_repost_kbuf(struct socket *so, unsigned int bufidx, int modulate, 
+    int activate, int nonblock)
+{
+	struct toepcb *toep = sototcpcb(so)->t_toe;
+	struct ddp_state *p = &toep->tp_ddp_state;
+	unsigned long flags;
+	
+	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	p->buf_state[bufidx].cur_offset = p->kbuf[bufidx]->dgl_offset;
+	p->buf_state[bufidx].flags = p->kbuf_noinval ? DDP_BF_NOINVAL : 0;
+	p->buf_state[bufidx].gl = p->kbuf[bufidx];
+	p->cur_buf = bufidx;
+	p->kbuf_idx = bufidx;
+
+	flags = select_ddp_flags(so, bufidx, nonblock, 0);
+	if (!bufidx)
+		t3_setup_ddpbufs(toep, 0, 0, 0, 0, flags |
+			 V_TF_DDP_PSH_NO_INVALIDATE0(p->kbuf_noinval) |
+			 V_TF_DDP_PSH_NO_INVALIDATE1(p->kbuf_noinval) |
+		         V_TF_DDP_BUF0_VALID(1),
+		         V_TF_DDP_BUF0_FLUSH(1) |
+			 V_TF_DDP_PSH_NO_INVALIDATE0(1) |
+		         V_TF_DDP_PSH_NO_INVALIDATE1(1) | V_TF_DDP_OFF(1) |
+			 V_TF_DDP_BUF0_VALID(1) |
+			 V_TF_DDP_ACTIVE_BUF(activate), modulate);
+	else
+		t3_setup_ddpbufs(toep, 0, 0, 0, 0, flags |
+			 V_TF_DDP_PSH_NO_INVALIDATE0(p->kbuf_noinval) |	
+		         V_TF_DDP_PSH_NO_INVALIDATE1(p->kbuf_noinval) | 
+			 V_TF_DDP_BUF1_VALID(1) | 
+			 V_TF_DDP_ACTIVE_BUF(activate),
+		         V_TF_DDP_BUF1_FLUSH(1) | 
+			 V_TF_DDP_PSH_NO_INVALIDATE0(1) |
+		         V_TF_DDP_PSH_NO_INVALIDATE1(1) | V_TF_DDP_OFF(1) |
+			 V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1), 
+			 modulate);
+	
+}
+
+/**
+ * setup_uio_ppods - setup HW page pods for a user iovec
+ * @sk: the associated socket
+ * @uio: the uio
+ * @oft: additional bytes to map before the start of the buffer
+ *
+ * Pins a user iovec and sets up HW page pods for DDP into it.  We allocate
+ * page pods for user buffers on the first call per socket.  Afterwards we
+ * limit the buffer length to whatever the existing page pods can accommodate.
+ * Returns a negative error code or the length of the mapped buffer.
+ *
+ * The current implementation handles iovecs with only one entry.
+ */
+static int
+setup_uio_ppods(struct socket *so, const struct uio *uio, int oft, int *length)
+{
+	int err;
+	unsigned int len;
+	struct ddp_gather_list *gl = NULL;
+	struct toepcb *toep = sototcpcb(so)->t_toe;
+	struct ddp_state *p = &toep->tp_ddp_state;
+	struct iovec *iov = uio->uio_iov;
+	vm_offset_t addr = (vm_offset_t)iov->iov_base - oft;
+
+	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	if (__predict_false(p->ubuf_nppods == 0)) {
+		err = alloc_buf1_ppods(so, p, addr, iov->iov_len + oft);
+		if (err)
+			return (err);
+	}
+
+	len = (p->ubuf_nppods - NUM_SENTINEL_PPODS) * PPOD_PAGES * PAGE_SIZE;
+	len -= addr & PAGE_MASK;
+	if (len > M_TCB_RX_DDP_BUF0_LEN)
+		len = M_TCB_RX_DDP_BUF0_LEN;
+	len = min(len, sototcpcb(so)->rcv_wnd - 32768);
+	len = min(len, iov->iov_len + oft);
+
+	if (len <= p->kbuf[0]->dgl_length) {
+		printf("length too short\n");
+		return (EINVAL);
+	}
+	
+	err = t3_pin_pages(toep->tp_rx_dmat, toep->tp_dmamap, addr, len, &gl, p->ubuf);
+	if (err)
+		return (err);
+	if (gl) {
+		if (p->ubuf)
+			t3_free_ddp_gl(p->ubuf);
+		p->ubuf = gl;
+		t3_setup_ppods(so, gl, pages2ppods(gl->dgl_nelem), p->ubuf_tag, len,
+			       gl->dgl_offset, 0);
+	}
+	*length = len;
+	return (0);
+}
+
+/*
+ * 
+ */
+void
+t3_cancel_ubuf(struct toepcb *toep)
+{
+	struct ddp_state *p = &toep->tp_ddp_state;
+	int ubuf_pending = t3_ddp_ubuf_pending(toep);
+	struct socket *so = toeptoso(toep);
+	int err = 0, count=0;
+	
+	if (p->ubuf == NULL)
+		return;
+
+	SOCKBUF_LOCK_ASSERT(&so->so_rcv);	
+	p->cancel_ubuf = 1;
+	while (ubuf_pending && !(so->so_rcv.sb_state & SBS_CANTRCVMORE)) {
+#ifdef T3_TRACE
+		T3_TRACE3(TB(p), 
+		  "t3_cancel_ubuf: flags0 0x%x flags1 0x%x get_tcb_count %d",
+		  p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY), 
+		  p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
+		  p->get_tcb_count);
+#endif
+		CTR3(KTR_TOM,
+		  "t3_cancel_ubuf: flags0 0x%x flags1 0x%x get_tcb_count %d",
+		  p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY), 
+		  p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
+		  p->get_tcb_count);	
+		if (p->get_tcb_count == 0)
+			t3_cancel_ddpbuf(toep, p->cur_buf);
+		else
+			CTR5(KTR_TOM, "waiting err=%d get_tcb_count=%d timeo=%d so=%p SBS_CANTRCVMORE=%d",
+			    err, p->get_tcb_count, so->so_rcv.sb_timeo, so,
+			    !!(so->so_rcv.sb_state & SBS_CANTRCVMORE));
+		
+		while (p->get_tcb_count && !(so->so_rcv.sb_state & SBS_CANTRCVMORE)) {
+			if (count & 0xfffffff)
+				CTR5(KTR_TOM, "waiting err=%d get_tcb_count=%d timeo=%d so=%p count=%d",
+				    err, p->get_tcb_count, so->so_rcv.sb_timeo, so, count);
+			count++;
+			err = sbwait(&so->so_rcv);
+		}
+		ubuf_pending = t3_ddp_ubuf_pending(toep);
+	}
+	p->cancel_ubuf = 0;
+}
+
+#define OVERLAY_MASK (V_TF_DDP_PSH_NO_INVALIDATE0(1) | \
+	              V_TF_DDP_PSH_NO_INVALIDATE1(1) | \
+		      V_TF_DDP_BUF1_FLUSH(1) | \
+		      V_TF_DDP_BUF0_FLUSH(1) | \
+		      V_TF_DDP_PUSH_DISABLE_1(1) | \
+		      V_TF_DDP_PUSH_DISABLE_0(1) | \
+		      V_TF_DDP_INDICATE_OUT(1))
+
+/*
+ * Post a user buffer as an overlay on top of the current kernel buffer.
+ */
+int
+t3_overlay_ubuf(struct socket *so, const struct uio *uio,
+    int nonblock, int rcv_flags, int modulate, int post_kbuf)
+{
+	int err, len, ubuf_idx;
+	unsigned long flags;
+	struct toepcb *toep = sototcpcb(so)->t_toe;
+	struct ddp_state *p = &toep->tp_ddp_state;
+
+	if (p->kbuf[0] == NULL) {
+		return (EINVAL);
+	}
+	
+	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	err = setup_uio_ppods(so, uio, 0, &len);
+	if (err) {
+		return (err);
+	}
+	
+	ubuf_idx = p->kbuf_idx;
+	p->buf_state[ubuf_idx].flags = DDP_BF_NOFLIP;
+	/* Use existing offset */
+	/* Don't need to update .gl, user buffer isn't copied. */
+	p->cur_buf = ubuf_idx;
+
+	flags = select_ddp_flags(so, ubuf_idx, nonblock, rcv_flags);
+
+	if (post_kbuf) {
+		struct ddp_buf_state *dbs = &p->buf_state[ubuf_idx ^ 1];
+		
+		dbs->cur_offset = 0;
+		dbs->flags = 0;
+		dbs->gl = p->kbuf[ubuf_idx ^ 1];
+		p->kbuf_idx ^= 1;
+		flags |= p->kbuf_idx ?
+		    V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_PUSH_DISABLE_1(0) :
+		    V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_PUSH_DISABLE_0(0);
+	}
+	
+	if (ubuf_idx == 0) {
+		t3_overlay_ddpbuf(toep, 0, p->ubuf_tag << 6, p->kbuf_tag[1] << 6,
+				  len);
+		t3_setup_ddpbufs(toep, 0, 0, p->kbuf[1]->dgl_length, 0,
+				 flags,
+				 OVERLAY_MASK | flags, 1);
+	} else {
+		t3_overlay_ddpbuf(toep, 1, p->kbuf_tag[0] << 6, p->ubuf_tag << 6,
+				  len);
+		t3_setup_ddpbufs(toep, p->kbuf[0]->dgl_length, 0, 0, 0,
+				 flags,
+				 OVERLAY_MASK | flags, 1);
+	}
+#ifdef T3_TRACE
+	T3_TRACE5(TIDTB(so),
+		  "t3_overlay_ubuf: tag %u flags 0x%x mask 0x%x ubuf_idx %d "
+		  " kbuf_idx %d",
+		   p->ubuf_tag, flags, OVERLAY_MASK, ubuf_idx, p->kbuf_idx);
+#endif
+	CTR3(KTR_TOM,
+	    "t3_overlay_ubuf: tag %u flags 0x%x mask 0x%x",
+	    p->ubuf_tag, flags, OVERLAY_MASK);
+	CTR3(KTR_TOM,
+	    "t3_overlay_ubuf:  ubuf_idx %d kbuf_idx %d post_kbuf %d",
+	    ubuf_idx, p->kbuf_idx, post_kbuf);
+	    
+	return (0);
+}
+
+/*
+ * Clean up DDP state that needs to survive until socket close time, such as the
+ * DDP buffers.  The buffers are already unmapped at this point as unmapping
+ * needs the PCI device and a socket may close long after the device is removed.
+ */
+void
+t3_cleanup_ddp(struct toepcb *toep)
+{
+	struct ddp_state *p = &toep->tp_ddp_state;
+	int idx;
+
+	for (idx = 0; idx < NUM_DDP_KBUF; idx++)
+		if (p->kbuf[idx]) {
+			ddp_gl_free_pages(p->kbuf[idx], 0);
+			free(p->kbuf[idx], M_DEVBUF);
+		}
+	if (p->ubuf) {
+		ddp_gl_free_pages(p->ubuf, 0);
+		free(p->ubuf, M_DEVBUF);
+		p->ubuf = NULL;
+	}
+	toep->tp_ulp_mode = 0;
+}
+
+/*
+ * This is a companion to t3_cleanup_ddp() and releases the HW resources
+ * associated with a connection's DDP state, such as the page pods.
+ * It's called when HW is done with a connection.   The rest of the state
+ * remains available until both HW and the app are done with the connection.
+ */
+void
+t3_release_ddp_resources(struct toepcb *toep)
+{
+	struct ddp_state *p = &toep->tp_ddp_state;
+	struct tom_data *d = TOM_DATA(toep->tp_toedev);
+	int idx;
+	
+	for (idx = 0; idx < NUM_DDP_KBUF; idx++) {
+		t3_free_ppods(d, p->kbuf_tag[idx], 
+		    p->kbuf_nppods[idx]);
+		unmap_ddp_gl(p->kbuf[idx]);
+	}
+
+	if (p->ubuf_nppods) {
+		t3_free_ppods(d, p->ubuf_tag, p->ubuf_nppods);
+		p->ubuf_nppods = 0;
+	}
+	if (p->ubuf)
+		unmap_ddp_gl(p->ubuf);
+	
+}
+
+void
+t3_post_kbuf(struct socket *so, int modulate, int nonblock)
+{
+	struct toepcb *toep = sototcpcb(so)->t_toe;
+	struct ddp_state *p = &toep->tp_ddp_state;
+
+	t3_set_ddp_tag(so, p->cur_buf, p->kbuf_tag[p->cur_buf] << 6);
+	t3_set_ddp_buf(so, p->cur_buf, 0, p->kbuf[p->cur_buf]->dgl_length);
+	t3_repost_kbuf(so, p->cur_buf, modulate, 1, nonblock);
+#ifdef T3_TRACE
+	T3_TRACE1(TIDTB(so),
+		  "t3_post_kbuf: cur_buf = kbuf_idx = %u ", p->cur_buf);
+#endif
+	CTR1(KTR_TOM,
+		  "t3_post_kbuf: cur_buf = kbuf_idx = %u ", p->cur_buf);
+}
+
+/*
+ * Prepare a socket for DDP.  Must be called when the socket is known to be
+ * open.
+ */
+int
+t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall, int nonblock)
+{
+	int i, err = ENOMEM;
+	static vm_pindex_t color;
+	unsigned int nppods, kbuf_pages, idx = 0;
+	struct toepcb *toep = sototcpcb(so)->t_toe;
+	struct ddp_state *p = &toep->tp_ddp_state;
+	struct tom_data *d = TOM_DATA(toep->tp_toedev);
+
+	
+	if (kbuf_size > M_TCB_RX_DDP_BUF0_LEN)
+		return (EINVAL);
+
+	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	
+	kbuf_pages = (kbuf_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	nppods = pages2ppods(kbuf_pages);
+
+	p->kbuf_noinval = !!waitall;
+	p->kbuf_tag[NUM_DDP_KBUF - 1] = -1;
+	for (idx = 0; idx < NUM_DDP_KBUF; idx++) {
+		p->kbuf[idx] = 
+		    malloc(sizeof (struct ddp_gather_list) + kbuf_pages *
+			sizeof(vm_page_t *), M_DEVBUF, M_NOWAIT|M_ZERO);
+		if (p->kbuf[idx] == NULL)
+			goto err;
+		err = t3_alloc_ppods(d, nppods, &p->kbuf_tag[idx]);
+		if (err) {
+			printf("t3_alloc_ppods failed err=%d\n", err);
+			goto err;
+		}
+		
+		p->kbuf_nppods[idx] = nppods;
+		p->kbuf[idx]->dgl_length = kbuf_size;
+		p->kbuf[idx]->dgl_offset = 0;
+		p->kbuf[idx]->dgl_nelem = kbuf_pages;
+
+		for (i = 0; i < kbuf_pages; ++i) {
+			p->kbuf[idx]->dgl_pages[i] = vm_page_alloc(NULL, color,
+			    VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
+			    VM_ALLOC_ZERO);
+			if (p->kbuf[idx]->dgl_pages[i] == NULL) {
+				p->kbuf[idx]->dgl_nelem = i;
+				printf("failed to allocate kbuf pages\n");
+				goto err;
+			}
+		}
+#ifdef NEED_BUSDMA
+		/*
+		 * XXX we'll need this for VT-d or any platform with an iommu :-/
+		 *
+		 */
+		for (i = 0; i < kbuf_pages; ++i)
+			p->kbuf[idx]->phys_addr[i] = 
+			    pci_map_page(p->pdev, p->kbuf[idx]->pages[i],
+					 0, PAGE_SIZE, PCI_DMA_FROMDEVICE);
+#endif
+		t3_setup_ppods(so, p->kbuf[idx], nppods, p->kbuf_tag[idx], 
+			       p->kbuf[idx]->dgl_length, 0, 0);
+	}
+	cxgb_log_tcb(TOEP_T3C_DEV(toep)->adapter, toep->tp_tid);
+
+	t3_set_ddp_tag(so, 0, p->kbuf_tag[0] << 6);
+	t3_set_ddp_buf(so, 0, 0, p->kbuf[0]->dgl_length);
+	t3_repost_kbuf(so, 0, 0, 1, nonblock);
+
+	t3_set_rcv_coalesce_enable(so, 
+	    TOM_TUNABLE(TOE_DEV(so), ddp_rcvcoalesce));
+
+#ifdef T3_TRACE
+	T3_TRACE4(TIDTB(so),
+		  "t3_enter_ddp: kbuf_size %u waitall %u tag0 %d tag1 %d",
+		   kbuf_size, waitall, p->kbuf_tag[0], p->kbuf_tag[1]);
+#endif
+	CTR4(KTR_TOM,
+		  "t3_enter_ddp: kbuf_size %u waitall %u tag0 %d tag1 %d",
+		   kbuf_size, waitall, p->kbuf_tag[0], p->kbuf_tag[1]);
+	DELAY(100000);
+	cxgb_log_tcb(TOEP_T3C_DEV(toep)->adapter, toep->tp_tid);
+	return (0);
+
+err:
+	t3_release_ddp_resources(toep);
+	t3_cleanup_ddp(toep);
+	return (err);
+}
+
+int
+t3_ddp_copy(const struct mbuf *m, int offset, struct uio *uio, int len)
+{
+	int page_off, resid_init, err;
+	struct ddp_gather_list *gl = (struct ddp_gather_list *)m->m_ddp_gl;
+	
+	resid_init = uio->uio_resid;
+	
+	if (!gl->dgl_pages)
+		panic("pages not set\n");
+
+	offset += gl->dgl_offset + m->m_cur_offset;
+	page_off = offset & PAGE_MASK;
+	KASSERT(len <= gl->dgl_length,
+	    ("len=%d > dgl_length=%d in ddp_copy\n", len, gl->dgl_length));
+
+	err = uiomove_fromphys(gl->dgl_pages, page_off, len, uio);
+	return (err);
+}
+
+
+/*
+ * Allocate n page pods.  Returns -1 on failure or the page pod tag.
+ */
+int
+t3_alloc_ppods(struct tom_data *td, unsigned int n, int *ptag)
+{
+	unsigned int i, j;
+
+	if (__predict_false(!td->ppod_map)) {
+		printf("ppod_map not set\n");
+		return (EINVAL);
+	}
+
+	mtx_lock(&td->ppod_map_lock);
+	for (i = 0; i < td->nppods; ) {
+		
+		for (j = 0; j < n; ++j)           /* scan ppod_map[i..i+n-1] */
+			if (td->ppod_map[i + j]) {
+				i = i + j + 1;
+				goto next;
+			}
+		memset(&td->ppod_map[i], 1, n);   /* allocate range */
+		mtx_unlock(&td->ppod_map_lock);
+		CTR2(KTR_TOM,
+		    "t3_alloc_ppods: n=%u tag=%u", n, i);
+		*ptag = i;
+		return (0);
+	next: ;
+	}
+	mtx_unlock(&td->ppod_map_lock);
+	return (0);
+}
+
+void
+t3_free_ppods(struct tom_data *td, unsigned int tag, unsigned int n)
+{
+	/* No need to take ppod_lock here */
+	memset(&td->ppod_map[tag], 0, n);
+}

sys/dev/cxgb/ulp/tom/cxgb_defs.h

@@ -40,6 +40,13 @@ POSSIBILITY OF SUCH DAMAGE.
 #define toeptoso(toep) ((toep)->tp_tp->t_inpcb->inp_socket)
 #define sototoep(so) (sototcpcb((so))->t_toe)
 
+#define TRACE_ENTER printf("%s:%s entered\n", __FUNCTION__, __FILE__)
+#define TRACE_EXIT printf("%s:%s:%d exited\n", __FUNCTION__, __FILE__, __LINE__)
+	
+#define	KTR_TOM	KTR_SPARE2
+#define	KTR_TCB	KTR_SPARE3
+
+struct toepcb;
 struct listen_ctx;
 
 typedef void (*defer_handler_t)(struct toedev *dev, struct mbuf *m);
@@ -54,7 +61,8 @@ void t3_init_listen_cpl_handlers(void);
 int t3_init_cpl_io(void);
 void t3_init_wr_tab(unsigned int wr_len);
 uint32_t t3_send_rx_credits(struct tcpcb *tp, uint32_t credits, uint32_t dack, int nofail);
-void t3_cleanup_rbuf(struct tcpcb *tp);
+void t3_send_rx_modulate(struct toepcb *toep);
+void t3_cleanup_rbuf(struct tcpcb *tp, int copied);
 
 void t3_init_socket_ops(void);
 void t3_install_socket_ops(struct socket *so);

sys/dev/cxgb/ulp/tom/cxgb_listen.c

@@ -180,7 +180,6 @@ listen_hash_add(struct tom_data *d, struct socket *so, unsigned int stid)
 	return p;
 }
 
-#if 0
 /*
  * Given a pointer to a listening socket return its server TID by consulting
  * the socket->stid map.  Returns -1 if the socket is not in the map.
@@ -191,16 +190,15 @@ listen_hash_find(struct tom_data *d, struct socket *so)
 	int stid = -1, bucket = listen_hashfn(so);
 	struct listen_info *p;
 
-	spin_lock(&d->listen_lock);
+	mtx_lock(&d->listen_lock);
 	for (p = d->listen_hash_tab[bucket]; p; p = p->next)
-		if (p->sk == sk) {
+		if (p->so == so) {
 			stid = p->stid;
 			break;
 		}
-	spin_unlock(&d->listen_lock);
+	mtx_unlock(&d->listen_lock);
 	return stid;
 }
-#endif
 
 /*
  * Delete the listen_info structure for a listening socket.  Returns the server
@@ -244,28 +242,24 @@ t3_listen_start(struct toedev *dev, struct socket *so, struct t3cdev *cdev)
 	if (!TOM_TUNABLE(dev, activated))
 		return;
 
-	printf("start listen\n");
+	if (listen_hash_find(d, so) != -1)
+		return;
 	
-	ctx = malloc(sizeof(*ctx), M_CXGB, M_NOWAIT);
+	CTR1(KTR_TOM, "start listen on port %u", ntohs(inp->inp_lport));
+	ctx = malloc(sizeof(*ctx), M_CXGB, M_NOWAIT|M_ZERO);
 
 	if (!ctx)
 		return;
 
 	ctx->tom_data = d;
 	ctx->lso = so;
-	ctx->ulp_mode = 0; /* DDP if the default */
+	ctx->ulp_mode = TOM_TUNABLE(dev, ddp) && !(so->so_options & SO_NO_DDP) ? ULP_MODE_TCPDDP : 0;
 	LIST_INIT(&ctx->synq_head);
 	
 	stid = cxgb_alloc_stid(d->cdev, d->client, ctx);
 	if (stid < 0)
 		goto free_ctx;
 
-#ifdef notyet
-	/*
-	 * XXX need to mark inpcb as referenced
-	 */
-	sock_hold(sk);
-#endif
 	m = m_gethdr(M_NOWAIT, MT_DATA);
 	if (m == NULL)
 		goto free_stid;

sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h

@@ -1,4 +1,3 @@
-
 /**************************************************************************
 
 Copyright (c) 2007, Chelsio Inc.
@@ -86,7 +85,6 @@ struct pagepod {
 #define M_PPOD_PGSZ    0x3
 #define V_PPOD_PGSZ(x) ((x) << S_PPOD_PGSZ)
 
-struct pci_dev;
 #include <vm/vm.h>
 #include <vm/vm_page.h>
 #include <machine/bus.h>
@@ -96,8 +94,7 @@ struct ddp_gather_list {
 	unsigned int	dgl_length;
 	unsigned int	dgl_offset;
 	unsigned int	dgl_nelem;
-	vm_page_t   	*dgl_pages;
+	vm_page_t   	dgl_pages[0];
-	bus_addr_t 	dgl_phys_addr[0];
 };
 
 struct ddp_buf_state {
@@ -107,7 +104,6 @@ struct ddp_buf_state {
 };
 
 struct ddp_state {
-	struct pci_dev *pdev;
 	struct ddp_buf_state buf_state[2];   /* per buffer state */
 	int cur_buf;
 	unsigned short kbuf_noinval;
@@ -119,6 +115,7 @@ struct ddp_state {
 	int get_tcb_count;
 	unsigned int kbuf_posted;
 	int cancel_ubuf;
+	int user_ddp_pending;
 	unsigned int kbuf_nppods[NUM_DDP_KBUF];
 	unsigned int kbuf_tag[NUM_DDP_KBUF];
 	struct ddp_gather_list *kbuf[NUM_DDP_KBUF]; /* kernel buffer for DDP prefetch */
@@ -132,54 +129,51 @@ enum {
 	DDP_BF_PSH     = 1 << 3,   /* set in skb->flags if the a DDP was 
 	                              completed with a segment having the
 				      PSH flag set */
+	DDP_BF_NODATA  = 1 << 4,   /* buffer completed before filling */ 
 };
 
-#ifdef notyet
+#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
+
 /*
  * Returns 1 if a UBUF DMA buffer might be active.
  */
-static inline int t3_ddp_ubuf_pending(struct sock *so)
+static inline int
+t3_ddp_ubuf_pending(struct toepcb *toep)
 {
-	struct tcp_sock *tp = tcp_sk(sk);
+	struct ddp_state *p = &toep->tp_ddp_state;
-	struct ddp_state *p = DDP_STATE(tp);
 
 	/* When the TOM_TUNABLE(ddp) is enabled, we're always in ULP_MODE DDP,
 	 * but DDP_STATE() is only valid if the connection actually enabled
 	 * DDP.
 	 */
-	if (!p)
+	if (p->kbuf[0] == NULL)
-		return 0;
+		return (0);
 
 	return (p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY)) || 
 	       (p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY));
 }
-#endif
 
 int t3_setup_ppods(struct socket *so, const struct ddp_gather_list *gl,
 		   unsigned int nppods, unsigned int tag, unsigned int maxoff,
 		   unsigned int pg_off, unsigned int color);
-int t3_alloc_ppods(struct tom_data *td, unsigned int n);
+int t3_alloc_ppods(struct tom_data *td, unsigned int n, int *tag);
 void t3_free_ppods(struct tom_data *td, unsigned int tag, unsigned int n);
-void t3_free_ddp_gl(struct pci_dev *pdev, struct ddp_gather_list *gl);
+void t3_free_ddp_gl(struct ddp_gather_list *gl);
-int t3_pin_pages(struct pci_dev *pdev, unsigned long uaddr, size_t len,
+int t3_ddp_copy(const struct mbuf *m, int offset, struct uio *uio, int len);
-		 struct ddp_gather_list **newgl,
-		 const struct ddp_gather_list *gl);
-int t3_ddp_copy(const struct mbuf *skb, int offset, struct iovec *to,
-		int len);
 //void t3_repost_kbuf(struct socket *so, int modulate, int activate);
-void t3_post_kbuf(struct socket *so, int modulate);
+void t3_post_kbuf(struct socket *so, int modulate, int nonblock);
-int t3_post_ubuf(struct socket *so, const struct iovec *iov, int nonblock,
+int t3_post_ubuf(struct socket *so, const struct uio *uio, int nonblock,
 		 int rcv_flags, int modulate, int post_kbuf);
-void t3_cancel_ubuf(struct socket *so);
+void t3_cancel_ubuf(struct toepcb *toep);
-int t3_overlay_ubuf(struct socket *so, const struct iovec *iov, int nonblock,
+int t3_overlay_ubuf(struct socket *so, const struct uio *uio, int nonblock,
-		    int rcv_flags, int modulate, int post_kbuf);
+    int rcv_flags, int modulate, int post_kbuf);
-int t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall);
+int t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall, int nonblock);
-void t3_cleanup_ddp(struct socket *so);
+void t3_cleanup_ddp(struct toepcb *toep);
 void t3_release_ddp_resources(struct toepcb *toep);
-void t3_cancel_ddpbuf(struct socket *so, unsigned int bufidx);
+void t3_cancel_ddpbuf(struct toepcb *, unsigned int bufidx);
-void t3_overlay_ddpbuf(struct socket *so, unsigned int bufidx, unsigned int tag0,
+void t3_overlay_ddpbuf(struct toepcb *, unsigned int bufidx, unsigned int tag0,
 		       unsigned int tag1, unsigned int len);
-void t3_setup_ddpbufs(struct socket *so, unsigned int len0, unsigned int offset0,
+void t3_setup_ddpbufs(struct toepcb *, unsigned int len0, unsigned int offset0,
 		      unsigned int len1, unsigned int offset1,
 		      uint64_t ddp_flags, uint64_t flag_mask, int modulate);
 #endif  /* T3_DDP_H */

sys/dev/cxgb/ulp/tom/cxgb_tcp_subr.c

@@ -1,694 +0,0 @@
-/*-
- * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)tcp_subr.c	8.2 (Berkeley) 5/24/95
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include "opt_compat.h"
-#include "opt_inet.h"
-#include "opt_inet6.h"
-#include "opt_ipsec.h"
-#include "opt_mac.h"
-#include "opt_tcpdebug.h"
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/callout.h>
-#include <sys/kernel.h>
-#include <sys/sysctl.h>
-#include <sys/malloc.h>
-#include <sys/mbuf.h>
-#ifdef INET6
-#include <sys/domain.h>
-#endif
-#include <sys/priv.h>
-#include <sys/proc.h>
-#include <sys/socket.h>
-#include <sys/socketvar.h>
-#include <sys/protosw.h>
-#include <sys/random.h>
-
-#include <vm/uma.h>
-
-#include <net/route.h>
-#include <net/if.h>
-
-#include <netinet/in.h>
-#include <netinet/in_systm.h>
-#include <netinet/ip.h>
-#ifdef INET6
-#include <netinet/ip6.h>
-#endif
-#include <netinet/in_pcb.h>
-#ifdef INET6
-#include <netinet6/in6_pcb.h>
-#endif
-#include <netinet/in_var.h>
-#include <netinet/ip_var.h>
-#ifdef INET6
-#include <netinet6/ip6_var.h>
-#include <netinet6/scope6_var.h>
-#include <netinet6/nd6.h>
-#endif
-#include <netinet/ip_icmp.h>
-#include <netinet/tcp.h>
-#include <netinet/tcp_fsm.h>
-#include <netinet/tcp_seq.h>
-#include <netinet/tcp_timer.h>
-#include <netinet/tcp_var.h>
-#include <netinet/tcp_syncache.h>
-#include <netinet/tcp_offload.h>
-#ifdef INET6
-#include <netinet6/tcp6_var.h>
-#endif
-#include <netinet/tcpip.h>
-#ifdef TCPDEBUG
-#include <netinet/tcp_debug.h>
-#endif
-#include <netinet6/ip6protosw.h>
-
-#ifdef IPSEC
-#include <netipsec/ipsec.h>
-#include <netipsec/xform.h>
-#ifdef INET6
-#include <netipsec/ipsec6.h>
-#endif
-#include <netipsec/key.h>
-#endif /*IPSEC*/
-
-#include <machine/in_cksum.h>
-#include <sys/md5.h>
-
-#include <security/mac/mac_framework.h>
-
-#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
-
-
-SYSCTL_NODE(_net_inet_tcp, 0,	cxgb,	CTLFLAG_RW, 0,	"chelsio TOE");
-
-static int	tcp_log_debug = 0;
-SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, log_debug, CTLFLAG_RW,
-    &tcp_log_debug, 0, "Log errors caused by incoming TCP segments");
-
-static int	tcp_tcbhashsize = 0;
-SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, tcbhashsize, CTLFLAG_RDTUN,
-    &tcp_tcbhashsize, 0, "Size of TCP control-block hashtable");
-
-static int	do_tcpdrain = 1;
-SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, do_tcpdrain, CTLFLAG_RW,
-    &do_tcpdrain, 0,
-    "Enable tcp_drain routine for extra help when low on mbufs");
-
-SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, pcbcount, CTLFLAG_RD,
-    &tcbinfo.ipi_count, 0, "Number of active PCBs");
-
-static int	icmp_may_rst = 1;
-SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, icmp_may_rst, CTLFLAG_RW,
-    &icmp_may_rst, 0,
-    "Certain ICMP unreachable messages may abort connections in SYN_SENT");
-
-static int	tcp_isn_reseed_interval = 0;
-SYSCTL_INT(_net_inet_tcp_cxgb, OID_AUTO, isn_reseed_interval, CTLFLAG_RW,
-    &tcp_isn_reseed_interval, 0, "Seconds between reseeding of ISN secret");
-
-/*
- * TCP bandwidth limiting sysctls.  Note that the default lower bound of
- * 1024 exists only for debugging.  A good production default would be
- * something like 6100.
- */
-SYSCTL_NODE(_net_inet_tcp, OID_AUTO, inflight, CTLFLAG_RW, 0,
-    "TCP inflight data limiting");
-
-static int	tcp_inflight_enable = 1;
-SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, enable, CTLFLAG_RW,
-    &tcp_inflight_enable, 0, "Enable automatic TCP inflight data limiting");
-
-static int	tcp_inflight_debug = 0;
-SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, debug, CTLFLAG_RW,
-    &tcp_inflight_debug, 0, "Debug TCP inflight calculations");
-
-static int	tcp_inflight_rttthresh;
-SYSCTL_PROC(_net_inet_tcp_inflight, OID_AUTO, rttthresh, CTLTYPE_INT|CTLFLAG_RW,
-    &tcp_inflight_rttthresh, 0, sysctl_msec_to_ticks, "I",
-    "RTT threshold below which inflight will deactivate itself");
-
-static int	tcp_inflight_min = 6144;
-SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, min, CTLFLAG_RW,
-    &tcp_inflight_min, 0, "Lower-bound for TCP inflight window");
-
-static int	tcp_inflight_max = TCP_MAXWIN << TCP_MAX_WINSHIFT;
-SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, max, CTLFLAG_RW,
-    &tcp_inflight_max, 0, "Upper-bound for TCP inflight window");
-
-static int	tcp_inflight_stab = 20;
-SYSCTL_INT(_net_inet_tcp_inflight, OID_AUTO, stab, CTLFLAG_RW,
-    &tcp_inflight_stab, 0, "Inflight Algorithm Stabilization 20 = 2 packets");
-
-uma_zone_t sack_hole_zone;
-
-static struct inpcb *tcp_notify(struct inpcb *, int);
-static struct inpcb *cxgb_tcp_drop_syn_sent(struct inpcb *inp, int errno);
-
-/*
- * Target size of TCP PCB hash tables. Must be a power of two.
- *
- * Note that this can be overridden by the kernel environment
- * variable net.inet.tcp.tcbhashsize
- */
-#ifndef TCBHASHSIZE
-#define TCBHASHSIZE	512
-#endif
-
-/*
- * XXX
- * Callouts should be moved into struct tcp directly.  They are currently
- * separate because the tcpcb structure is exported to userland for sysctl
- * parsing purposes, which do not know about callouts.
- */
-struct tcpcb_mem {
-	struct	tcpcb		tcb;
-	struct	tcp_timer	tt;
-};
-
-MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
-
-/*
- * Drop a TCP connection, reporting
- * the specified error.  If connection is synchronized,
- * then send a RST to peer.
- */
-struct tcpcb *
-cxgb_tcp_drop(struct tcpcb *tp, int errno)
-{
-	struct socket *so = tp->t_inpcb->inp_socket;
-
-	INP_INFO_WLOCK_ASSERT(&tcbinfo);
-	INP_LOCK_ASSERT(tp->t_inpcb);
-
-	if (TCPS_HAVERCVDSYN(tp->t_state)) {
-		tp->t_state = TCPS_CLOSED;
-		(void) tcp_gen_reset(tp);
-		tcpstat.tcps_drops++;
-	} else
-		tcpstat.tcps_conndrops++;
-	if (errno == ETIMEDOUT && tp->t_softerror)
-		errno = tp->t_softerror;
-	so->so_error = errno;
-	return (cxgb_tcp_close(tp));
-}
-
-/*
- * Attempt to close a TCP control block, marking it as dropped, and freeing
- * the socket if we hold the only reference.
- */
-struct tcpcb *
-cxgb_tcp_close(struct tcpcb *tp)
-{
-	struct inpcb *inp = tp->t_inpcb;
-	struct socket *so;
-
-	INP_INFO_WLOCK_ASSERT(&tcbinfo);
-	INP_LOCK_ASSERT(inp);
-
-	if (tp->t_state == TCPS_LISTEN)
-		tcp_gen_listen_close(tp);
-	in_pcbdrop(inp);
-	tcpstat.tcps_closed++;
-	KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
-	so = inp->inp_socket;
-	soisdisconnected(so);
-	if (inp->inp_vflag & INP_SOCKREF) {
-		KASSERT(so->so_state & SS_PROTOREF,
-		    ("tcp_close: !SS_PROTOREF"));
-		inp->inp_vflag &= ~INP_SOCKREF;
-		INP_UNLOCK(inp);
-		ACCEPT_LOCK();
-		SOCK_LOCK(so);
-		so->so_state &= ~SS_PROTOREF;
-		sofree(so);
-		return (NULL);
-	}
-	return (tp);
-}
-
-/*
- * Notify a tcp user of an asynchronous error;
- * store error as soft error, but wake up user
- * (for now, won't do anything until can select for soft error).
- *
- * Do not wake up user since there currently is no mechanism for
- * reporting soft errors (yet - a kqueue filter may be added).
- */
-static struct inpcb *
-tcp_notify(struct inpcb *inp, int error)
-{
-	struct tcpcb *tp;
-
-	INP_INFO_WLOCK_ASSERT(&tcbinfo);
-	INP_LOCK_ASSERT(inp);
-
-	if ((inp->inp_vflag & INP_TIMEWAIT) ||
-	    (inp->inp_vflag & INP_DROPPED))
-		return (inp);
-
-	tp = intotcpcb(inp);
-	KASSERT(tp != NULL, ("tcp_notify: tp == NULL"));
-
-	/*
-	 * Ignore some errors if we are hooked up.
-	 * If connection hasn't completed, has retransmitted several times,
-	 * and receives a second error, give up now.  This is better
-	 * than waiting a long time to establish a connection that
-	 * can never complete.
-	 */
-	if (tp->t_state == TCPS_ESTABLISHED &&
-	    (error == EHOSTUNREACH || error == ENETUNREACH ||
-	     error == EHOSTDOWN)) {
-		return (inp);
-	} else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 &&
-	    tp->t_softerror) {
-		tp = cxgb_tcp_drop(tp, error);
-		if (tp != NULL)
-			return (inp);
-		else
-			return (NULL);
-	} else {
-		tp->t_softerror = error;
-		return (inp);
-	}
-#if 0
-	wakeup( &so->so_timeo);
-	sorwakeup(so);
-	sowwakeup(so);
-#endif
-}
-
-void
-cxgb_tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip)
-{
-	struct ip *ip = vip;
-	struct tcphdr *th;
-	struct in_addr faddr;
-	struct inpcb *inp;
-	struct tcpcb *tp;
-	struct inpcb *(*notify)(struct inpcb *, int) = tcp_notify;
-	struct icmp *icp;
-	struct in_conninfo inc;
-	tcp_seq icmp_tcp_seq;
-	int mtu;
-
-	faddr = ((struct sockaddr_in *)sa)->sin_addr;
-	if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY)
-		return;
-
-	if (cmd == PRC_MSGSIZE)
-		notify = tcp_mtudisc;
-	else if (icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB ||
-		cmd == PRC_UNREACH_PORT || cmd == PRC_TIMXCEED_INTRANS) && ip)
-		notify = cxgb_tcp_drop_syn_sent;
-	/*
-	 * Redirects don't need to be handled up here.
-	 */
-	else if (PRC_IS_REDIRECT(cmd))
-		return;
-	/*
-	 * Source quench is depreciated.
-	 */
-	else if (cmd == PRC_QUENCH)
-		return;
-	/*
-	 * Hostdead is ugly because it goes linearly through all PCBs.
-	 * XXX: We never get this from ICMP, otherwise it makes an
-	 * excellent DoS attack on machines with many connections.
-	 */
-	else if (cmd == PRC_HOSTDEAD)
-		ip = NULL;
-	else if ((unsigned)cmd >= PRC_NCMDS || inetctlerrmap[cmd] == 0)
-		return;
-	if (ip != NULL) {
-		icp = (struct icmp *)((caddr_t)ip
-				      - offsetof(struct icmp, icmp_ip));
-		th = (struct tcphdr *)((caddr_t)ip
-				       + (ip->ip_hl << 2));
-		INP_INFO_WLOCK(&tcbinfo);
-		inp = in_pcblookup_hash(&tcbinfo, faddr, th->th_dport,
-		    ip->ip_src, th->th_sport, 0, NULL);
-		if (inp != NULL)  {
-			INP_LOCK(inp);
-			if (!(inp->inp_vflag & INP_TIMEWAIT) &&
-			    !(inp->inp_vflag & INP_DROPPED) &&
-			    !(inp->inp_socket == NULL)) {
-				icmp_tcp_seq = htonl(th->th_seq);
-				tp = intotcpcb(inp);
-				if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) &&
-				    SEQ_LT(icmp_tcp_seq, tp->snd_max)) {
-					if (cmd == PRC_MSGSIZE) {
-					    /*
-					     * MTU discovery:
-					     * If we got a needfrag set the MTU
-					     * in the route to the suggested new
-					     * value (if given) and then notify.
-					     */
-					    bzero(&inc, sizeof(inc));
-					    inc.inc_flags = 0;	/* IPv4 */
-					    inc.inc_faddr = faddr;
-
-					    mtu = ntohs(icp->icmp_nextmtu);
-					    /*
-					     * If no alternative MTU was
-					     * proposed, try the next smaller
-					     * one.  ip->ip_len has already
-					     * been swapped in icmp_input().
-					     */
-					    if (!mtu)
-						mtu = ip_next_mtu(ip->ip_len,
-						 1);
-					    if (mtu < max(296, (tcp_minmss)
-						 + sizeof(struct tcpiphdr)))
-						mtu = 0;
-					    if (!mtu)
-						mtu = tcp_mssdflt
-						 + sizeof(struct tcpiphdr);
-					    /*
-					     * Only cache the the MTU if it
-					     * is smaller than the interface
-					     * or route MTU.  tcp_mtudisc()
-					     * will do right thing by itself.
-					     */
-					    if (mtu <= tcp_maxmtu(&inc, NULL))
-						tcp_hc_updatemtu(&inc, mtu);
-					}
-
-					inp = (*notify)(inp, inetctlerrmap[cmd]);
-				}
-			}
-			if (inp != NULL)
-				INP_UNLOCK(inp);
-		} else {
-			inc.inc_fport = th->th_dport;
-			inc.inc_lport = th->th_sport;
-			inc.inc_faddr = faddr;
-			inc.inc_laddr = ip->ip_src;
-#ifdef INET6
-			inc.inc_isipv6 = 0;
-#endif
-			syncache_unreach(&inc, th);
-		}
-		INP_INFO_WUNLOCK(&tcbinfo);
-	} else
-		in_pcbnotifyall(&tcbinfo, faddr, inetctlerrmap[cmd], notify);
-}
-
-#ifdef INET6
-void
-tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d)
-{
-	struct tcphdr th;
-	struct inpcb *(*notify)(struct inpcb *, int) = tcp_notify;
-	struct ip6_hdr *ip6;
-	struct mbuf *m;
-	struct ip6ctlparam *ip6cp = NULL;
-	const struct sockaddr_in6 *sa6_src = NULL;
-	int off;
-	struct tcp_portonly {
-		u_int16_t th_sport;
-		u_int16_t th_dport;
-	} *thp;
-
-	if (sa->sa_family != AF_INET6 ||
-	    sa->sa_len != sizeof(struct sockaddr_in6))
-		return;
-
-	if (cmd == PRC_MSGSIZE)
-		notify = tcp_mtudisc;
-	else if (!PRC_IS_REDIRECT(cmd) &&
-		 ((unsigned)cmd >= PRC_NCMDS || inet6ctlerrmap[cmd] == 0))
-		return;
-	/* Source quench is depreciated. */
-	else if (cmd == PRC_QUENCH)
-		return;
-
-	/* if the parameter is from icmp6, decode it. */
-	if (d != NULL) {
-		ip6cp = (struct ip6ctlparam *)d;
-		m = ip6cp->ip6c_m;
-		ip6 = ip6cp->ip6c_ip6;
-		off = ip6cp->ip6c_off;
-		sa6_src = ip6cp->ip6c_src;
-	} else {
-		m = NULL;
-		ip6 = NULL;
-		off = 0;	/* fool gcc */
-		sa6_src = &sa6_any;
-	}
-
-	if (ip6 != NULL) {
-		struct in_conninfo inc;
-		/*
-		 * XXX: We assume that when IPV6 is non NULL,
-		 * M and OFF are valid.
-		 */
-
-		/* check if we can safely examine src and dst ports */
-		if (m->m_pkthdr.len < off + sizeof(*thp))
-			return;
-
-		bzero(&th, sizeof(th));
-		m_copydata(m, off, sizeof(*thp), (caddr_t)&th);
-
-		in6_pcbnotify(&tcbinfo, sa, th.th_dport,
-		    (struct sockaddr *)ip6cp->ip6c_src,
-		    th.th_sport, cmd, NULL, notify);
-
-		inc.inc_fport = th.th_dport;
-		inc.inc_lport = th.th_sport;
-		inc.inc6_faddr = ((struct sockaddr_in6 *)sa)->sin6_addr;
-		inc.inc6_laddr = ip6cp->ip6c_src->sin6_addr;
-		inc.inc_isipv6 = 1;
-		INP_INFO_WLOCK(&tcbinfo);
-		syncache_unreach(&inc, &th);
-		INP_INFO_WUNLOCK(&tcbinfo);
-	} else
-		in6_pcbnotify(&tcbinfo, sa, 0, (const struct sockaddr *)sa6_src,
-			      0, cmd, NULL, notify);
-}
-#endif /* INET6 */
-
-
-/*
- * Following is where TCP initial sequence number generation occurs.
- *
- * There are two places where we must use initial sequence numbers:
- * 1.  In SYN-ACK packets.
- * 2.  In SYN packets.
- *
- * All ISNs for SYN-ACK packets are generated by the syncache.  See
- * tcp_syncache.c for details.
- *
- * The ISNs in SYN packets must be monotonic; TIME_WAIT recycling
- * depends on this property.  In addition, these ISNs should be
- * unguessable so as to prevent connection hijacking.  To satisfy
- * the requirements of this situation, the algorithm outlined in
- * RFC 1948 is used, with only small modifications.
- *
- * Implementation details:
- *
- * Time is based off the system timer, and is corrected so that it
- * increases by one megabyte per second.  This allows for proper
- * recycling on high speed LANs while still leaving over an hour
- * before rollover.
- *
- * As reading the *exact* system time is too expensive to be done
- * whenever setting up a TCP connection, we increment the time
- * offset in two ways.  First, a small random positive increment
- * is added to isn_offset for each connection that is set up.
- * Second, the function tcp_isn_tick fires once per clock tick
- * and increments isn_offset as necessary so that sequence numbers
- * are incremented at approximately ISN_BYTES_PER_SECOND.  The
- * random positive increments serve only to ensure that the same
- * exact sequence number is never sent out twice (as could otherwise
- * happen when a port is recycled in less than the system tick
- * interval.)
- *
- * net.inet.tcp.isn_reseed_interval controls the number of seconds
- * between seeding of isn_secret.  This is normally set to zero,
- * as reseeding should not be necessary.
- *
- * Locking of the global variables isn_secret, isn_last_reseed, isn_offset,
- * isn_offset_old, and isn_ctx is performed using the TCP pcbinfo lock.  In
- * general, this means holding an exclusive (write) lock.
- */
-
-#define ISN_BYTES_PER_SECOND 1048576
-#define ISN_STATIC_INCREMENT 4096
-#define ISN_RANDOM_INCREMENT (4096 - 1)
-
-
-/*
- * When a specific ICMP unreachable message is received and the
- * connection state is SYN-SENT, drop the connection.  This behavior
- * is controlled by the icmp_may_rst sysctl.
- */
-static struct inpcb *
-cxgb_tcp_drop_syn_sent(struct inpcb *inp, int errno)
-{
-	struct tcpcb *tp;
-
-	INP_INFO_WLOCK_ASSERT(&tcbinfo);
-	INP_LOCK_ASSERT(inp);
-
-	if ((inp->inp_vflag & INP_TIMEWAIT) ||
-	    (inp->inp_vflag & INP_DROPPED))
-		return (inp);
-
-	tp = intotcpcb(inp);
-	if (tp->t_state != TCPS_SYN_SENT)
-		return (inp);
-
-	tp = cxgb_tcp_drop(tp, errno);
-	if (tp != NULL)
-		return (inp);
-	else
-		return (NULL);
-}
-
-static int
-cxgb_sysctl_drop(SYSCTL_HANDLER_ARGS)
-{
-	/* addrs[0] is a foreign socket, addrs[1] is a local one. */
-	struct sockaddr_storage addrs[2];
-	struct inpcb *inp;
-	struct tcpcb *tp;
-	struct tcptw *tw;
-	struct sockaddr_in *fin, *lin;
-#ifdef INET6
-	struct sockaddr_in6 *fin6, *lin6;
-	struct in6_addr f6, l6;
-#endif
-	int error;
-
-	inp = NULL;
-	fin = lin = NULL;
-#ifdef INET6
-	fin6 = lin6 = NULL;
-#endif
-	error = 0;
-
-	if (req->oldptr != NULL || req->oldlen != 0)
-		return (EINVAL);
-	if (req->newptr == NULL)
-		return (EPERM);
-	if (req->newlen < sizeof(addrs))
-		return (ENOMEM);
-	error = SYSCTL_IN(req, &addrs, sizeof(addrs));
-	if (error)
-		return (error);
-
-	switch (addrs[0].ss_family) {
-#ifdef INET6
-	case AF_INET6:
-		fin6 = (struct sockaddr_in6 *)&addrs[0];
-		lin6 = (struct sockaddr_in6 *)&addrs[1];
-		if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
-		    lin6->sin6_len != sizeof(struct sockaddr_in6))
-			return (EINVAL);
-		if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
-			if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
-				return (EINVAL);
-			in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
-			in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
-			fin = (struct sockaddr_in *)&addrs[0];
-			lin = (struct sockaddr_in *)&addrs[1];
-			break;
-		}
-		error = sa6_embedscope(fin6, ip6_use_defzone);
-		if (error)
-			return (error);
-		error = sa6_embedscope(lin6, ip6_use_defzone);
-		if (error)
-			return (error);
-		break;
-#endif
-	case AF_INET:
-		fin = (struct sockaddr_in *)&addrs[0];
-		lin = (struct sockaddr_in *)&addrs[1];
-		if (fin->sin_len != sizeof(struct sockaddr_in) ||
-		    lin->sin_len != sizeof(struct sockaddr_in))
-			return (EINVAL);
-		break;
-	default:
-		return (EINVAL);
-	}
-	INP_INFO_WLOCK(&tcbinfo);
-	switch (addrs[0].ss_family) {
-#ifdef INET6
-	case AF_INET6:
-		inp = in6_pcblookup_hash(&tcbinfo, &f6, fin6->sin6_port,
-		    &l6, lin6->sin6_port, 0, NULL);
-		break;
-#endif
-	case AF_INET:
-		inp = in_pcblookup_hash(&tcbinfo, fin->sin_addr, fin->sin_port,
-		    lin->sin_addr, lin->sin_port, 0, NULL);
-		break;
-	}
-	if (inp != NULL) {
-		INP_LOCK(inp);
-		if (inp->inp_vflag & INP_TIMEWAIT) {
-			/*
-			 * XXXRW: There currently exists a state where an
-			 * inpcb is present, but its timewait state has been
-			 * discarded.  For now, don't allow dropping of this
-			 * type of inpcb.
-			 */
-			tw = intotw(inp);
-			if (tw != NULL)
-				tcp_twclose(tw, 0);
-			else
-				INP_UNLOCK(inp);
-		} else if (!(inp->inp_vflag & INP_DROPPED) &&
-			   !(inp->inp_socket->so_options & SO_ACCEPTCONN)) {
-			tp = intotcpcb(inp);
-			tp = cxgb_tcp_drop(tp, ECONNABORTED);
-			if (tp != NULL)
-				INP_UNLOCK(inp);
-		} else
-			INP_UNLOCK(inp);
-	} else
-		error = ESRCH;
-	INP_INFO_WUNLOCK(&tcbinfo);
-	return (error);
-}
-
-SYSCTL_PROC(_net_inet_tcp_cxgb, TCPCTL_DROP, drop,
-    CTLTYPE_STRUCT|CTLFLAG_WR|CTLFLAG_SKIP, NULL,
-    0, cxgb_sysctl_drop, "", "Drop TCP connection");
-

sys/dev/cxgb/ulp/tom/cxgb_toepcb.h

@@ -30,45 +30,49 @@
 #ifndef CXGB_TOEPCB_H_
 #define CXGB_TOEPCB_H_
 #include <sys/bus.h>
+#include <sys/condvar.h>
 #include <dev/cxgb/sys/mbufq.h>
 
 struct toepcb {
-	struct toedev *tp_toedev;
+	struct toedev 		*tp_toedev;
-	struct l2t_entry *tp_l2t;
+	struct l2t_entry 	*tp_l2t;
-	pr_ctloutput_t *tp_ctloutput;
+	pr_ctloutput_t 		*tp_ctloutput;
-	unsigned int tp_tid;
+	unsigned int 		tp_tid;
-	int tp_wr_max;
+	int 			tp_wr_max;
-	int tp_wr_avail;
+	int 			tp_wr_avail;
-	int tp_wr_unacked;
+	int 			tp_wr_unacked;
-	int tp_delack_mode;
+	int 			tp_delack_mode;
-	int tp_mtu_idx;
+	int 			tp_mtu_idx;
-	int tp_ulp_mode;
+	int 			tp_ulp_mode;
-	int tp_qset_idx;
+	int 			tp_qset_idx;
-	int tp_mss_clamp;
+	int 			tp_mss_clamp;
-	int tp_qset;
+	int 			tp_qset;
-	int tp_flags;
+	int 			tp_flags;
-	int tp_enqueued_bytes;
+	int 			tp_enqueued_bytes;
-	int tp_page_count;
+	int 			tp_page_count;
-	int tp_state;
+	int 			tp_state;
 
-	tcp_seq tp_iss;
+	tcp_seq 		tp_iss;
-	tcp_seq tp_delack_seq;
+	tcp_seq 		tp_delack_seq;
-	tcp_seq tp_rcv_wup;
+	tcp_seq 		tp_rcv_wup;
-	tcp_seq tp_copied_seq;
+	tcp_seq 		tp_copied_seq;
-	uint64_t tp_write_seq;
+	uint64_t 		tp_write_seq;
 
-	volatile int tp_refcount;
+	volatile int 		tp_refcount;
-	vm_page_t *tp_pages;
+	vm_page_t 		*tp_pages;
 	
-	struct tcpcb *tp_tp;
+	struct tcpcb 		*tp_tp;
-	struct mbuf  *tp_m_last;
+	struct mbuf  		*tp_m_last;
-	bus_dma_tag_t	tp_tx_dmat;
+	bus_dma_tag_t		tp_tx_dmat;
-	bus_dmamap_t	tp_dmamap;
+	bus_dma_tag_t		tp_rx_dmat;
+	bus_dmamap_t		tp_dmamap;
+
+	LIST_ENTRY(toepcb) 	synq_entry;
+	struct mbuf_head 	wr_list;
+	struct mbuf_head 	out_of_order_queue;
+	struct ddp_state 	tp_ddp_state;
+	struct cv		tp_cv;
 			   
-	LIST_ENTRY(toepcb) synq_entry;
-	struct mbuf_head wr_list;
-	struct mbuf_head out_of_order_queue;
-	struct ddp_state tp_ddp_state;
 };
 
 static inline void
@@ -95,7 +99,7 @@ enqueue_wr(struct toepcb *toep, struct mbuf *m)
 }
 
 static inline struct mbuf *
-peek_wr(struct toepcb *toep)
+peek_wr(const struct toepcb *toep)
 {
 
 	return (mbufq_peek(&toep->wr_list));
@@ -108,5 +112,10 @@ dequeue_wr(struct toepcb *toep)
 	return (mbufq_dequeue(&toep->wr_list));
 }
 
+#define wr_queue_walk(toep, m) \
+	for (m = peek_wr(toep); m; m = m->m_nextpkt)
+
+
+
 #endif
 

sys/dev/cxgb/ulp/tom/cxgb_tom.c

@@ -34,11 +34,13 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/fcntl.h>
+#include <sys/ktr.h>
 #include <sys/limits.h>
 #include <sys/lock.h>
 #include <sys/eventhandler.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
+#include <sys/condvar.h>
 #include <sys/mutex.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
@@ -90,16 +92,20 @@ static TAILQ_HEAD(, tom_data) cxgb_list;
 static struct mtx cxgb_list_lock;
 
 static int t3_toe_attach(struct toedev *dev, const struct offload_id *entry);
+static void cxgb_register_listeners(void);
+
 /*
  * Handlers for each CPL opcode
  */
-static cxgb_cpl_handler_func tom_cpl_handlers[NUM_CPL_CMDS];
+static cxgb_cpl_handler_func tom_cpl_handlers[256];
+
 
 static eventhandler_tag listen_tag;
 
 static struct offload_id t3_toe_id_tab[] = {
 	{ TOE_ID_CHELSIO_T3, 0 },
 	{ TOE_ID_CHELSIO_T3B, 0 },
+	{ TOE_ID_CHELSIO_T3C, 0 },
 	{ 0 }
 };
 
@@ -138,7 +144,7 @@ toepcb_alloc(void)
 {
 	struct toepcb *toep;
 	
-	toep = malloc(sizeof(struct toepcb), M_DEVBUF, M_NOWAIT);
+	toep = malloc(sizeof(struct toepcb), M_DEVBUF, M_NOWAIT|M_ZERO);
 	
 	if (toep == NULL)
 		return (NULL);
@@ -150,8 +156,8 @@ toepcb_alloc(void)
 void
 toepcb_init(struct toepcb *toep)
 {
-	bzero(toep, sizeof(*toep));
 	toep->tp_refcount = 1;
+	cv_init(&toep->tp_cv, "toep cv");
 }
 
 void
@@ -164,12 +170,9 @@ void
 toepcb_release(struct toepcb *toep)
 {
 	if (toep->tp_refcount == 1) {
-		printf("doing final toepcb free\n");
-		
 		free(toep, M_DEVBUF);
 		return;
 	}
-	
 	atomic_add_acq_int(&toep->tp_refcount, -1);
 }
 
@@ -179,13 +182,30 @@ toepcb_release(struct toepcb *toep)
 static void
 t3cdev_add(struct tom_data *t)
 {
-	printf("t3cdev_add\n");
-	
 	mtx_lock(&cxgb_list_lock);
 	TAILQ_INSERT_TAIL(&cxgb_list, t, entry);
 	mtx_unlock(&cxgb_list_lock);
 }
 
+static inline int
+cdev2type(struct t3cdev *cdev)
+{
+	int type = 0;
+
+	switch (cdev->type) {
+	case T3A:
+		type = TOE_ID_CHELSIO_T3;
+		break;
+	case T3B:
+		type = TOE_ID_CHELSIO_T3B;
+		break;
+	case T3C:
+		type = TOE_ID_CHELSIO_T3C;
+		break;
+	}
+	return (type);
+}
+
 /*
  * Allocate a TOM data structure,
  * initialize its cpl_handlers
@@ -200,11 +220,7 @@ t3c_tom_add(struct t3cdev *cdev)
 	struct toedev *tdev;
 	struct adap_ports *port_info;
 
-	printf("%s called\n", __FUNCTION__);
-	
-	
 	t = malloc(sizeof(*t), M_CXGB, M_NOWAIT|M_ZERO);
-	
 	if (t == NULL)
 		return;
 
@@ -224,8 +240,7 @@ t3c_tom_add(struct t3cdev *cdev)
 
 	/* Register TCP offload device */
 	tdev = &t->tdev;
-	tdev->tod_ttid = (cdev->type == T3A ?
+	tdev->tod_ttid = cdev2type(cdev);
-		      TOE_ID_CHELSIO_T3 : TOE_ID_CHELSIO_T3B);
 	tdev->tod_lldev = cdev->lldev;
 
 	if (register_toedev(tdev, "toe%d")) {
@@ -234,13 +249,11 @@ t3c_tom_add(struct t3cdev *cdev)
 	}
 	TOM_DATA(tdev) = t;
 
-	printf("nports=%d\n", port_info->nports);
 	for (i = 0; i < port_info->nports; i++) {
 		struct ifnet *ifp = port_info->lldevs[i];
 		TOEDEV(ifp) = tdev;
 
-		printf("enabling toe on %p\n", ifp);
+		CTR1(KTR_TOM, "enabling toe on %p", ifp);
-		
 		ifp->if_capabilities |= IFCAP_TOE4;
 		ifp->if_capenable |= IFCAP_TOE4;
 	}
@@ -251,6 +264,7 @@ t3c_tom_add(struct t3cdev *cdev)
 
 	/* Activate TCP offload device */
 	activate_offload(tdev);
+	cxgb_register_listeners();
 	return;
 
 out_free_all:
@@ -269,8 +283,8 @@ static int
 do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
 {
 	log(LOG_ERR, "%s: received bad CPL command %u\n", cdev->name,
-	    *mtod(m, unsigned int *));
+	    0xFF & *mtod(m, unsigned int *));
-
+	kdb_backtrace();
 	return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
 }
 
@@ -282,7 +296,7 @@ do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
 void
 t3tom_register_cpl_handler(unsigned int opcode, cxgb_cpl_handler_func h)
 {
-	if (opcode < NUM_CPL_CMDS)
+	if (opcode < 256)
 		tom_cpl_handlers[opcode] = h ? h : do_bad_cpl;
 	else
 		log(LOG_ERR, "Chelsio T3 TOM: handler registration for "
@@ -327,7 +341,7 @@ init_cpl_handlers(void)
 {
 	int i;
 
-	for (i = 0; i < NUM_CPL_CMDS; ++i)
+	for (i = 0; i < 256; ++i)
 		tom_cpl_handlers[i] = do_bad_cpl;
 
 	t3_init_listen_cpl_handlers();
@@ -349,7 +363,7 @@ t3_toe_attach(struct toedev *dev, const struct offload_id *entry)
 #endif
 	t3_init_tunables(t);
 	mtx_init(&t->listen_lock, "tom data listeners", NULL, MTX_DEF);
-
+	CTR2(KTR_TOM, "t3_toe_attach dev=%p entry=%p", dev, entry);
 	/* Adjust TOE activation for this module */
 	t->conf.activated = activated;
 
@@ -374,19 +388,14 @@ t3_toe_attach(struct toedev *dev, const struct offload_id *entry)
 	t->ddp_ulimit = ddp.ulimit;
 	t->pdev = ddp.pdev;
 	t->rx_page_size = rx_page_info.page_size;
-#ifdef notyet
 	/* OK if this fails, we just can't do DDP */
 	t->nppods = (ddp.ulimit + 1 - ddp.llimit) / PPOD_SIZE;
-	t->ppod_map = t3_alloc_mem(t->nppods);
+	t->ppod_map = malloc(t->nppods, M_DEVBUF, M_WAITOK|M_ZERO);
-#endif
 
-#if 0
+	mtx_init(&t->ppod_map_lock, "ppod map", NULL, MTX_DEF);
-	spin_lock_init(&t->ppod_map_lock);
+
-	tom_proc_init(dev);
+
-#ifdef CONFIG_SYSCTL
+	t3_sysctl_register(cdev->adapter, &t->conf);
-	t->sysctl = t3_sysctl_register(dev, &t->conf);
-#endif
-#endif
 	return (0);
 }
 
@@ -411,11 +420,8 @@ cxgb_toe_listen_stop(void *unused, struct tcpcb *tp)
 	
 	mtx_lock(&cxgb_list_lock);
 	TAILQ_FOREACH(p, &cxgb_list, entry) {
-		if (tp->t_state == TCPS_LISTEN) {
+		if (tp->t_state == TCPS_LISTEN)
-			printf("stopping listen on port=%d\n",
-			    ntohs(tp->t_inpcb->inp_lport));
 			t3_listen_stop(&p->tdev, so, p->cdev);
-		}
 	}
 	mtx_unlock(&cxgb_list_lock);
 }
@@ -439,23 +445,12 @@ cxgb_register_listeners(void)
 static int
 t3_tom_init(void)
 {
-
-#if 0
-	struct socket *sock;
-	err = sock_create_kern(PF_INET, SOCK_STREAM, IPPROTO_TCP, &sock);
-	if (err < 0) {
-		printk(KERN_ERR "Could not create TCP socket, error %d\n", err);
-		return err;
-	}
-
-	t3_def_state_change = sock->sk->sk_state_change;
-	t3_def_data_ready = sock->sk->sk_data_ready;
-	t3_def_error_report = sock->sk->sk_error_report;
-	sock_release(sock);
-#endif
 	init_cpl_handlers();
-	if (t3_init_cpl_io() < 0)
+	if (t3_init_cpl_io() < 0) {
+		log(LOG_ERR,
+		    "Unable to initialize cpl io ops\n");
 		return -1;
+	}
 	t3_init_socket_ops();
 
 	 /* Register with the TOE device layer. */
@@ -466,7 +461,6 @@ t3_tom_init(void)
 		return -1;
 	}
 	INP_INFO_WLOCK(&tcbinfo);
-
 	INP_INFO_WUNLOCK(&tcbinfo);	    
 
 	mtx_init(&cxgb_list_lock, "cxgb tom list", NULL, MTX_DEF);
@@ -477,10 +471,8 @@ t3_tom_init(void)
 	TAILQ_INIT(&cxgb_list);
 	
 	/* Register to offloading devices */
-	printf("setting add to %p\n", t3c_tom_add);
 	t3c_tom_client.add = t3c_tom_add;
 	cxgb_register_client(&t3c_tom_client);
-	cxgb_register_listeners();
 	return (0);
 }
 
@@ -491,8 +483,6 @@ t3_tom_load(module_t mod, int cmd, void *arg)
 
 	switch (cmd) {
 	case MOD_LOAD:
-		printf("wheeeeee ...\n");
-		
 		t3_tom_init();
 		break;
 	case MOD_QUIESCE:

sys/dev/cxgb/ulp/tom/cxgb_tom.h

@@ -138,6 +138,8 @@ struct listen_ctx {
 
 void t3_init_tunables(struct tom_data *t);
 
+void t3_sysctl_register(struct adapter *sc, const struct tom_tunables *p);
+
 static __inline struct mbuf *
 m_gethdr_nofail(int len)
 {

sys/dev/cxgb/ulp/tom/cxgb_tom_sysctl.c

@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/common/cxgb_ctl_defs.h>
 #include <dev/cxgb/common/cxgb_t3_cpl.h>
 #include <dev/cxgb/cxgb_offload.h>
+#include <dev/cxgb/cxgb_include.h>
 #include <dev/cxgb/cxgb_l2t.h>
 #include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
 #include <dev/cxgb/ulp/tom/cxgb_tom.h>
@@ -82,7 +83,7 @@ static struct tom_tunables default_tunable_vals = {
 	.delack = 1,
 	.max_conn = -1,
 	.soft_backlog_limit = 0,
-	.ddp = 0,
+	.ddp = 1,
 	.ddp_thres = 14 * 4096,
 	.ddp_copy_limit = 13 * 4096,
 	.ddp_push_wait = 1,
@@ -96,7 +97,8 @@ static struct tom_tunables default_tunable_vals = {
 	.activated = 1,
 };
 
-void t3_init_tunables(struct tom_data *t)
+void
+t3_init_tunables(struct tom_data *t)
 {
 	t->conf = default_tunable_vals;
 
@@ -104,3 +106,15 @@ void t3_init_tunables(struct tom_data *t)
 	t->conf.mss = T3C_DATA(t->cdev)->tx_max_chunk;
 	t->conf.max_wrs = T3C_DATA(t->cdev)->max_wrs;
 }
+
+void
+t3_sysctl_register(struct adapter *sc, const struct tom_tunables *p)
+{
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid_list *children;
+
+	ctx = device_get_sysctl_ctx(sc->dev);
+	children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
+	
+}
+

sys/dev/cxgb/ulp/tom/cxgb_vm.c

@@ -0,0 +1,180 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+***************************************************************************/
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+#include <sys/fcntl.h>
+#include <sys/kernel.h>
+#include <sys/limits.h>
+#include <sys/lock.h>
+#include <sys/mbuf.h>
+#include <sys/condvar.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
+#include <dev/cxgb/ulp/tom/cxgb_vm.h>
+
+#define TRACE_ENTER printf("%s:%s entered", __FUNCTION__, __FILE__)
+#define TRACE_EXIT printf("%s:%s:%d exited", __FUNCTION__, __FILE__, __LINE__)
+
+/*
+ * This routine takes a user address range and does the following:
+ *  - validate that the user has access to those pages (flags indicates read or write) - if not fail
+ *  - validate that count is enough to hold range number of pages - if not fail
+ *  - fault in any non-resident pages
+ *  - if the user is doing a read force a write fault for any COWed pages
+ *  - if the user is doing a read mark all pages as dirty
+ *  - hold all pages
+ *  - return number of pages in count
+ */
+int
+vm_fault_hold_user_pages(vm_offset_t addr, vm_page_t *mp, int count, int flags)
+{
+
+	vm_offset_t end, va;
+	vm_paddr_t pa;
+	int faults, rv;
+
+	struct thread *td;
+	vm_map_t map;
+	pmap_t pmap;
+	vm_page_t m, *pages;
+	vm_prot_t prot;
+	
+
+	/*
+	 * Check that virtual address range is legal
+	 * This check is somewhat bogus as on some architectures kernel
+	 * and user do not share VA - however, it appears that all FreeBSD
+	 * architectures define it
+	 */
+	end = addr + (count * PAGE_SIZE);
+	if (end > VM_MAXUSER_ADDRESS) {
+		printf("bad address passed\n");
+		return (EFAULT);
+	}
+
+	td = curthread;
+	map = &td->td_proc->p_vmspace->vm_map;
+	pmap = &td->td_proc->p_vmspace->vm_pmap;
+	pages = mp;
+
+	prot = VM_PROT_READ;
+	prot |= (flags & VM_HOLD_WRITEABLE) ? VM_PROT_WRITE : 0;
+	bzero(pages, sizeof(vm_page_t *) * count);
+retry:
+
+	/*
+	 * First optimistically assume that all pages are resident (and R/W if for write)
+	 * if so just mark pages as held (and dirty if for write) and return
+	 */
+	vm_page_lock_queues();
+	for (pages = mp, faults = 0, va = addr; va < end; va += PAGE_SIZE, pages++) {
+		/*
+		 * Assure that we only hold the page once
+		 */
+		if (*pages == NULL) {
+			/*
+			 * page queue mutex is recursable so this is OK
+			 * it would be really nice if we had an unlocked version of this so
+			 * we were only acquiring the pmap lock 1 time as opposed to potentially
+			 * many dozens of times
+			 */
+			m = pmap_extract_and_hold(pmap, va, prot);
+			if (m == NULL) {
+				faults++;
+				continue;
+			}
+			
+			*pages = m;
+			if (flags & VM_HOLD_WRITEABLE)
+				vm_page_dirty(m);
+		}
+	}
+	vm_page_unlock_queues();
+	
+	if (faults == 0) {
+		return (0);
+	}
+	
+	/*
+	 * Pages either have insufficient permissions or are not present
+	 * trigger a fault where neccessary
+	 * 
+	 */
+	for (va = addr; va < end; va += PAGE_SIZE) {
+		m = NULL;
+		pa = pmap_extract(pmap, va);
+		rv = 0;
+		if (pa)
+			m = PHYS_TO_VM_PAGE(pa);
+		if (flags & VM_HOLD_WRITEABLE) {
+			if (m == NULL  || (m->flags & PG_WRITEABLE) == 0)
+				rv = vm_fault(map, va, VM_PROT_WRITE, VM_FAULT_DIRTY);
+		} else if (m == NULL)
+			rv = vm_fault(map, va, VM_PROT_READ, VM_FAULT_NORMAL);
+		if (rv) {
+			printf("vm_fault bad return rv=%d va=0x%zx\n", rv, va);
+			
+			goto error;
+		} 
+	}
+	
+	goto retry;
+
+error:	
+	vm_page_lock_queues();
+	for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++)
+		if (*pages)
+			vm_page_unhold(*pages);
+	vm_page_unlock_queues();
+	return (EFAULT);
+}
+
+void
+vm_fault_unhold_pages(vm_page_t *mp, int count)
+{
+
+	KASSERT(count >= 0, ("negative count %d", count));
+	vm_page_lock_queues();
+	while (count--) {
+		vm_page_unhold(*mp);
+		mp++;
+	}
+	vm_page_unlock_queues();
+}

sys/dev/cxgb/ulp/tom/cxgb_vm.h

@@ -0,0 +1,40 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+$FreeBSD$
+
+***************************************************************************/
+#ifndef CXGB_VM_H_
+#define CXGB_VM_H_
+
+#define VM_HOLD_WRITEABLE	0x1
+
+int vm_fault_hold_user_pages(vm_offset_t addr, vm_page_t *mp, int count, int flags);
+void vm_fault_unhold_pages(vm_page_t *mp, int count);
+
+#endif

sys/modules/cxgb/cxgb/Makefile

@@ -12,21 +12,25 @@ SRCS+=	uipc_mvec.c cxgb_support.c
 SRCS+=	cxgb_multiq.c 
 
 CFLAGS+= -DCONFIG_CHELSIO_T3_CORE -g -DCONFIG_DEFINED -DDEFAULT_JUMBO -I${CXGB} -DSMP
-#CFLAGS+= -DDISABLE_MBUF_IOVEC
+CFLAGS+= -DDISABLE_MBUF_IOVEC
 #CFLAGS+= -DIFNET_MULTIQUEUE
+#CFLAGS+= -DDISABLE_MBUF_IOVEC
+#CFLAGS+= -DDEBUG -DDEBUG_PRINT
 #CFLAGS+= -DINVARIANT_SUPPORT -DINVARIANTS 
 #CFLAGS+= -DWITNESS
-#CFLAGS+= -DDEBUG -DDEBUG_PRINT
+#CFLAGS += -DLOCK_PROFILING
+
+#CFLAGS+= -DWITNESS
 
 
 .if ${MACHINE_ARCH} != "ia64"
 # ld is broken on ia64
-t3fw-4.7.0.bin: ${CXGB}/t3fw-4.7.0.bin.gz.uu
+t3fw-5.0.0.bin: ${CXGB}/t3fw-5.0.0.bin.gz.uu
-	uudecode -p < ${CXGB}/t3fw-4.7.0.bin.gz.uu \
+	uudecode -p < ${CXGB}/t3fw-5.0.0.bin.gz.uu \
 	| gzip -dc > ${.TARGET}
 
-FIRMWS= t3fw-4.7.0.bin:t3fw470
+FIRMWS= t3fw-5.0.0.bin:t3fw500
-CLEANFILES+= t3fw-4.7.0.bin
+CLEANFILES+= t3fw-5.0.0.bin
 
 t3b_protocol_sram-1.1.0.bin: ${CXGB}/t3b_protocol_sram-1.1.0.bin.gz.uu
 	uudecode -p < ${CXGB}/t3b_protocol_sram-1.1.0.bin.gz.uu \

sys/modules/cxgb/tom/Makefile

@@ -1,11 +1,13 @@
 # $FreeBSD$
+
 TOM = ${.CURDIR}/../../../dev/cxgb/ulp/tom
 .PATH: ${TOM} 
 
 KMOD=	tom
 SRCS=   cxgb_tom.c cxgb_cpl_io.c cxgb_listen.c cxgb_tom_sysctl.c cxgb_cpl_socket.c
-#SRCS+=  cxgb_tcp_subr.c cxgb_tcp_usrreq.c
+SRCS+=  cxgb_ddp.c cxgb_vm.c
-SRCS+=	opt_compat.h opt_inet.h opt_inet6.h opt_ipsec.h opt_mac.h opt_tcpdebug.h opt_ddb.h
+SRCS+=	opt_compat.h opt_inet.h opt_inet6.h opt_ipsec.h opt_mac.h 
+SRCS+=	opt_tcpdebug.h opt_ddb.h opt_sched.h opt_global.h opt_ktr.h
 SRCS+=	device_if.h bus_if.h pci_if.h
 
 #CFLAGS+= -DDEBUG_PRINT -DDEBUG