Loading Documentation/infiniband/ipoib.txt +45 −0 Original line number Diff line number Diff line Loading @@ -24,6 +24,49 @@ Partitions and P_Keys The P_Key for any interface is given by the "pkey" file, and the main interface for a subinterface is in "parent." Datagram vs Connected modes The IPoIB driver supports two modes of operation: datagram and connected. The mode is set and read through an interface's /sys/class/net/<intf name>/mode file. In datagram mode, the IB UD (Unreliable Datagram) transport is used and so the interface MTU has is equal to the IB L2 MTU minus the IPoIB encapsulation header (4 bytes). For example, in a typical IB fabric with a 2K MTU, the IPoIB MTU will be 2048 - 4 = 2044 bytes. In connected mode, the IB RC (Reliable Connected) transport is used. Connected mode is to takes advantage of the connected nature of the IB transport and allows an MTU up to the maximal IP packet size of 64K, which reduces the number of IP packets needed for handling large UDP datagrams, TCP segments, etc and increases the performance for large messages. In connected mode, the interface's UD QP is still used for multicast and communication with peers that don't support connected mode. In this case, RX emulation of ICMP PMTU packets is used to cause the networking stack to use the smaller UD MTU for these neighbours. Stateless offloads If the IB HW supports IPoIB stateless offloads, IPoIB advertises TCP/IP checksum and/or Large Send (LSO) offloading capability to the network stack. Large Receive (LRO) offloading is also implemented and may be turned on/off using ethtool calls. Currently LRO is supported only for checksum offload capable devices. Stateless offloads are supported only in datagram mode. Interrupt moderation If the underlying IB device supports CQ event moderation, one can use ethtool to set interrupt mitigation parameters and thus reduce the overhead incurred by handling interrupts. The main code path of IPoIB doesn't use events for TX completion signaling so only RX moderation is supported. Debugging Information By compiling the IPoIB driver with CONFIG_INFINIBAND_IPOIB_DEBUG set Loading Loading @@ -55,3 +98,5 @@ References http://ietf.org/rfc/rfc4391.txt IP over InfiniBand (IPoIB) Architecture (RFC 4392) http://ietf.org/rfc/rfc4392.txt IP over InfiniBand: Connected Mode (RFC 4755) http://ietf.org/rfc/rfc4755.txt drivers/infiniband/hw/cxgb3/cxio_hal.c +5 −5 Original line number Diff line number Diff line Loading @@ -152,7 +152,7 @@ static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid) sge_cmd = qpid << 8 | 3; wqe->sge_cmd = cpu_to_be64(sge_cmd); skb->priority = CPL_PRIORITY_CONTROL; return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb)); return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb); } int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) Loading Loading @@ -571,7 +571,7 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p) (unsigned long long) rdev_p->ctrl_qp.dma_addr, rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2); skb->priority = CPL_PRIORITY_CONTROL; return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb)); return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb); err: kfree_skb(skb); return err; Loading Loading @@ -701,7 +701,7 @@ static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry, u32 stag_idx; u32 wptr; if (rdev_p->flags) if (cxio_fatal_error(rdev_p)) return -EIO; stag_state = stag_state > 0; Loading Loading @@ -858,7 +858,7 @@ int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr) wqe->qp_dma_size = cpu_to_be32(attr->qp_dma_size); wqe->irs = cpu_to_be32(attr->irs); skb->priority = 0; /* 0=>ToeQ; 1=>CtrlQ */ return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb)); return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb); } void cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb) Loading Loading @@ -1041,9 +1041,9 @@ void cxio_rdev_close(struct cxio_rdev *rdev_p) cxio_hal_pblpool_destroy(rdev_p); cxio_hal_rqtpool_destroy(rdev_p); list_del(&rdev_p->entry); rdev_p->t3cdev_p->ulp = NULL; cxio_hal_destroy_ctrl_qp(rdev_p); cxio_hal_destroy_resource(rdev_p->rscp); rdev_p->t3cdev_p->ulp = NULL; } } Loading drivers/infiniband/hw/cxgb3/cxio_hal.h +6 −0 Original line number Diff line number Diff line Loading @@ -115,6 +115,11 @@ struct cxio_rdev { #define CXIO_ERROR_FATAL 1 }; static inline int cxio_fatal_error(struct cxio_rdev *rdev_p) { return rdev_p->flags & CXIO_ERROR_FATAL; } static inline int cxio_num_stags(struct cxio_rdev *rdev_p) { return min((int)T3_MAX_NUM_STAG, (int)((rdev_p->rnic_info.tpt_top - rdev_p->rnic_info.tpt_base) >> 5)); Loading Loading @@ -188,6 +193,7 @@ void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count); void cxio_flush_hw_cq(struct t3_cq *cq); int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe, u8 *cqe_flushed, u64 *cookie, u32 *credit); int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb); #define MOD "iw_cxgb3: " #define PDBG(fmt, args...) pr_debug(MOD fmt, ## args) Loading drivers/infiniband/hw/cxgb3/iwch.c +9 −2 Original line number Diff line number Diff line Loading @@ -165,12 +165,19 @@ static void close_rnic_dev(struct t3cdev *tdev) static void iwch_err_handler(struct t3cdev *tdev, u32 status, u32 error) { struct cxio_rdev *rdev = tdev->ulp; struct iwch_dev *rnicp = rdev_to_iwch_dev(rdev); struct ib_event event; if (status == OFFLOAD_STATUS_DOWN) if (status == OFFLOAD_STATUS_DOWN) { rdev->flags = CXIO_ERROR_FATAL; return; event.device = &rnicp->ibdev; event.event = IB_EVENT_DEVICE_FATAL; event.element.port_num = 0; ib_dispatch_event(&event); } return; } static int __init iwch_init_module(void) Loading drivers/infiniband/hw/cxgb3/iwch.h +5 −0 Original line number Diff line number Diff line Loading @@ -117,6 +117,11 @@ static inline struct iwch_dev *to_iwch_dev(struct ib_device *ibdev) return container_of(ibdev, struct iwch_dev, ibdev); } static inline struct iwch_dev *rdev_to_iwch_dev(struct cxio_rdev *rdev) { return container_of(rdev, struct iwch_dev, rdev); } static inline int t3b_device(const struct iwch_dev *rhp) { return rhp->rdev.t3cdev_p->type == T3B; Loading Loading
Documentation/infiniband/ipoib.txt +45 −0 Original line number Diff line number Diff line Loading @@ -24,6 +24,49 @@ Partitions and P_Keys The P_Key for any interface is given by the "pkey" file, and the main interface for a subinterface is in "parent." Datagram vs Connected modes The IPoIB driver supports two modes of operation: datagram and connected. The mode is set and read through an interface's /sys/class/net/<intf name>/mode file. In datagram mode, the IB UD (Unreliable Datagram) transport is used and so the interface MTU has is equal to the IB L2 MTU minus the IPoIB encapsulation header (4 bytes). For example, in a typical IB fabric with a 2K MTU, the IPoIB MTU will be 2048 - 4 = 2044 bytes. In connected mode, the IB RC (Reliable Connected) transport is used. Connected mode is to takes advantage of the connected nature of the IB transport and allows an MTU up to the maximal IP packet size of 64K, which reduces the number of IP packets needed for handling large UDP datagrams, TCP segments, etc and increases the performance for large messages. In connected mode, the interface's UD QP is still used for multicast and communication with peers that don't support connected mode. In this case, RX emulation of ICMP PMTU packets is used to cause the networking stack to use the smaller UD MTU for these neighbours. Stateless offloads If the IB HW supports IPoIB stateless offloads, IPoIB advertises TCP/IP checksum and/or Large Send (LSO) offloading capability to the network stack. Large Receive (LRO) offloading is also implemented and may be turned on/off using ethtool calls. Currently LRO is supported only for checksum offload capable devices. Stateless offloads are supported only in datagram mode. Interrupt moderation If the underlying IB device supports CQ event moderation, one can use ethtool to set interrupt mitigation parameters and thus reduce the overhead incurred by handling interrupts. The main code path of IPoIB doesn't use events for TX completion signaling so only RX moderation is supported. Debugging Information By compiling the IPoIB driver with CONFIG_INFINIBAND_IPOIB_DEBUG set Loading Loading @@ -55,3 +98,5 @@ References http://ietf.org/rfc/rfc4391.txt IP over InfiniBand (IPoIB) Architecture (RFC 4392) http://ietf.org/rfc/rfc4392.txt IP over InfiniBand: Connected Mode (RFC 4755) http://ietf.org/rfc/rfc4755.txt
drivers/infiniband/hw/cxgb3/cxio_hal.c +5 −5 Original line number Diff line number Diff line Loading @@ -152,7 +152,7 @@ static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid) sge_cmd = qpid << 8 | 3; wqe->sge_cmd = cpu_to_be64(sge_cmd); skb->priority = CPL_PRIORITY_CONTROL; return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb)); return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb); } int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) Loading Loading @@ -571,7 +571,7 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p) (unsigned long long) rdev_p->ctrl_qp.dma_addr, rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2); skb->priority = CPL_PRIORITY_CONTROL; return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb)); return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb); err: kfree_skb(skb); return err; Loading Loading @@ -701,7 +701,7 @@ static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry, u32 stag_idx; u32 wptr; if (rdev_p->flags) if (cxio_fatal_error(rdev_p)) return -EIO; stag_state = stag_state > 0; Loading Loading @@ -858,7 +858,7 @@ int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr) wqe->qp_dma_size = cpu_to_be32(attr->qp_dma_size); wqe->irs = cpu_to_be32(attr->irs); skb->priority = 0; /* 0=>ToeQ; 1=>CtrlQ */ return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb)); return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb); } void cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb) Loading Loading @@ -1041,9 +1041,9 @@ void cxio_rdev_close(struct cxio_rdev *rdev_p) cxio_hal_pblpool_destroy(rdev_p); cxio_hal_rqtpool_destroy(rdev_p); list_del(&rdev_p->entry); rdev_p->t3cdev_p->ulp = NULL; cxio_hal_destroy_ctrl_qp(rdev_p); cxio_hal_destroy_resource(rdev_p->rscp); rdev_p->t3cdev_p->ulp = NULL; } } Loading
drivers/infiniband/hw/cxgb3/cxio_hal.h +6 −0 Original line number Diff line number Diff line Loading @@ -115,6 +115,11 @@ struct cxio_rdev { #define CXIO_ERROR_FATAL 1 }; static inline int cxio_fatal_error(struct cxio_rdev *rdev_p) { return rdev_p->flags & CXIO_ERROR_FATAL; } static inline int cxio_num_stags(struct cxio_rdev *rdev_p) { return min((int)T3_MAX_NUM_STAG, (int)((rdev_p->rnic_info.tpt_top - rdev_p->rnic_info.tpt_base) >> 5)); Loading Loading @@ -188,6 +193,7 @@ void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count); void cxio_flush_hw_cq(struct t3_cq *cq); int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe, u8 *cqe_flushed, u64 *cookie, u32 *credit); int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb); #define MOD "iw_cxgb3: " #define PDBG(fmt, args...) pr_debug(MOD fmt, ## args) Loading
drivers/infiniband/hw/cxgb3/iwch.c +9 −2 Original line number Diff line number Diff line Loading @@ -165,12 +165,19 @@ static void close_rnic_dev(struct t3cdev *tdev) static void iwch_err_handler(struct t3cdev *tdev, u32 status, u32 error) { struct cxio_rdev *rdev = tdev->ulp; struct iwch_dev *rnicp = rdev_to_iwch_dev(rdev); struct ib_event event; if (status == OFFLOAD_STATUS_DOWN) if (status == OFFLOAD_STATUS_DOWN) { rdev->flags = CXIO_ERROR_FATAL; return; event.device = &rnicp->ibdev; event.event = IB_EVENT_DEVICE_FATAL; event.element.port_num = 0; ib_dispatch_event(&event); } return; } static int __init iwch_init_module(void) Loading
drivers/infiniband/hw/cxgb3/iwch.h +5 −0 Original line number Diff line number Diff line Loading @@ -117,6 +117,11 @@ static inline struct iwch_dev *to_iwch_dev(struct ib_device *ibdev) return container_of(ibdev, struct iwch_dev, ibdev); } static inline struct iwch_dev *rdev_to_iwch_dev(struct cxio_rdev *rdev) { return container_of(rdev, struct iwch_dev, rdev); } static inline int t3b_device(const struct iwch_dev *rhp) { return rhp->rdev.t3cdev_p->type == T3B; Loading