Loading drivers/usb/host/xhci-mem.c +298 −66 Original line number Diff line number Diff line Loading @@ -1831,19 +1831,155 @@ void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst) erst->entries = NULL; } void xhci_mem_cleanup(struct xhci_hcd *xhci) static void xhci_handle_sec_intr_events(struct xhci_hcd *xhci, int intr_num) { union xhci_trb *erdp_trb, *current_trb; struct xhci_segment *seg; u64 erdp_reg; u32 iman_reg; dma_addr_t deq; unsigned long segment_offset; /* disable irq, ack pending interrupt and ack all pending events */ iman_reg = readl_relaxed(&xhci->sec_ir_set[intr_num]->irq_pending); iman_reg &= ~IMAN_IE; writel_relaxed(iman_reg, &xhci->sec_ir_set[intr_num]->irq_pending); iman_reg = readl_relaxed(&xhci->sec_ir_set[intr_num]->irq_pending); if (iman_reg & IMAN_IP) writel_relaxed(iman_reg, &xhci->sec_ir_set[intr_num]->irq_pending); /* last acked event trb is in erdp reg */ erdp_reg = xhci_read_64(xhci, &xhci->sec_ir_set[intr_num]->erst_dequeue); deq = (dma_addr_t)(erdp_reg & ~ERST_PTR_MASK); if (!deq) { pr_debug("%s: event ring handling not required\n", __func__); return; } seg = xhci->sec_event_ring[intr_num]->first_seg; segment_offset = deq - seg->dma; /* find out virtual address of the last acked event trb */ erdp_trb = current_trb = &seg->trbs[0] + (segment_offset/sizeof(*current_trb)); /* read cycle state of the last acked trb to find out CCS */ xhci->sec_event_ring[intr_num]->cycle_state = le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE; while (1) { /* last trb of the event ring: toggle cycle state */ if (current_trb == &seg->trbs[TRBS_PER_SEGMENT - 1]) { xhci->sec_event_ring[intr_num]->cycle_state ^= 1; current_trb = &seg->trbs[0]; } else { current_trb++; } /* cycle state transition */ if ((le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE) != xhci->sec_event_ring[intr_num]->cycle_state) break; } if (erdp_trb != current_trb) { deq = xhci_trb_virt_to_dma(xhci->sec_event_ring[intr_num]->deq_seg, current_trb); if (deq == 0) xhci_warn(xhci, "WARN invalid SW event ring dequeue ptr.\n"); /* Update HC event ring dequeue pointer */ erdp_reg &= ERST_PTR_MASK; erdp_reg |= ((u64) deq & (u64) ~ERST_PTR_MASK); } /* Clear the event handler busy flag (RW1C); event ring is empty. */ erdp_reg |= ERST_EHB; xhci_write_64(xhci, erdp_reg, &xhci->sec_ir_set[intr_num]->erst_dequeue); } static int sec_event_ring_cleanup(struct xhci_hcd *xhci, unsigned int intr_num) { int size; struct device *dev = xhci_to_hcd(xhci)->self.sysdev; int i, j, num_ports; cancel_delayed_work_sync(&xhci->cmd_timer); if (intr_num >= xhci->max_interrupters) { xhci_err(xhci, "invalid secondary interrupter num %d\n", intr_num); return -EINVAL; } xhci_free_erst(xhci, &xhci->erst); size = sizeof(struct xhci_erst_entry)*(xhci->sec_erst[intr_num].num_entries); if (xhci->sec_erst[intr_num].entries) { xhci_handle_sec_intr_events(xhci, intr_num); dma_free_coherent(dev, size, xhci->sec_erst[intr_num].entries, xhci->sec_erst[intr_num].erst_dma_addr); xhci->sec_erst[intr_num].entries = NULL; } xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed SEC ERST#%d", intr_num); if (xhci->sec_event_ring[intr_num]) xhci_ring_free(xhci, xhci->sec_event_ring[intr_num]); xhci->sec_event_ring[intr_num] = NULL; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed sec event ring"); return 0; } int xhci_sec_event_ring_cleanup(struct usb_device *udev, unsigned int intr_num) { struct usb_hcd *hcd = bus_to_hcd(udev->bus); if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; return sec_event_ring_cleanup(hcd_to_xhci(hcd), intr_num); } EXPORT_SYMBOL(xhci_sec_event_ring_cleanup); static void xhci_event_ring_cleanup(struct xhci_hcd *xhci) { unsigned int i; /* sec event ring clean up */ for (i = 1; i < xhci->max_interrupters; i++) sec_event_ring_cleanup(xhci, i); kfree(xhci->sec_ir_set); xhci->sec_ir_set = NULL; kfree(xhci->sec_erst); xhci->sec_erst = NULL; kfree(xhci->sec_event_ring); xhci->sec_event_ring = NULL; /* primary event ring clean up */ xhci_free_erst(xhci, &xhci->erst); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed primary ERST"); if (xhci->event_ring) xhci_ring_free(xhci, xhci->event_ring); xhci->event_ring = NULL; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring"); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed priamry event ring"); } void xhci_mem_cleanup(struct xhci_hcd *xhci) { struct device *dev = xhci_to_hcd(xhci)->self.sysdev; int i, j, num_ports; cancel_delayed_work_sync(&xhci->cmd_timer); xhci_event_ring_cleanup(xhci); if (xhci->lpm_command) xhci_free_command(xhci, xhci->lpm_command); Loading Loading @@ -2084,30 +2220,6 @@ static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci) return 0; } static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) { u64 temp; dma_addr_t deq; deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, xhci->event_ring->dequeue); if (deq == 0 && !in_interrupt()) xhci_warn(xhci, "WARN something wrong with SW event ring " "dequeue ptr.\n"); /* Update HC event ring dequeue pointer */ temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); temp &= ERST_PTR_MASK; /* Don't clear the EHB bit (which is RW1C) because * there might be more events to service. */ temp &= ~ERST_EHB; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Write event ring dequeue pointer, " "preserving EHB bit"); xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, &xhci->ir_set->erst_dequeue); } static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, __le32 __iomem *addr, int max_caps) { Loading Loading @@ -2344,6 +2456,159 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) return 0; } static int xhci_event_ring_setup(struct xhci_hcd *xhci, struct xhci_ring **er, struct xhci_intr_reg __iomem *ir_set, struct xhci_erst *erst, unsigned int intr_num, gfp_t flags) { dma_addr_t deq; u64 val_64; unsigned int val; int ret; *er = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, 0, flags); if (!*er) return -ENOMEM; ret = xhci_alloc_erst(xhci, *er, erst, flags); if (ret) return ret; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "intr# %d: num segs = %i, virt addr = %pK, dma addr = 0x%llx", intr_num, erst->num_entries, erst->entries, (unsigned long long)erst->erst_dma_addr); /* set ERST count with the number of entries in the segment table */ val = readl_relaxed(&ir_set->erst_size); val &= ERST_SIZE_MASK; val |= ERST_NUM_SEGS; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Write ERST size = %i to ir_set %d (some bits preserved)", val, intr_num); writel_relaxed(val, &ir_set->erst_size); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "intr# %d: Set ERST entries to point to event ring.", intr_num); /* set the segment table base address */ xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Set ERST base address for ir_set %d = 0x%llx", intr_num, (unsigned long long)erst->erst_dma_addr); val_64 = xhci_read_64(xhci, &ir_set->erst_base); val_64 &= ERST_PTR_MASK; val_64 |= (erst->erst_dma_addr & (u64) ~ERST_PTR_MASK); xhci_write_64(xhci, val_64, &ir_set->erst_base); /* Set the event ring dequeue address */ deq = xhci_trb_virt_to_dma((*er)->deq_seg, (*er)->dequeue); if (deq == 0 && !in_interrupt()) xhci_warn(xhci, "intr# %d:WARN something wrong with SW event ring deq ptr.\n", intr_num); /* Update HC event ring dequeue pointer */ val_64 = xhci_read_64(xhci, &ir_set->erst_dequeue); val_64 &= ERST_PTR_MASK; /* Don't clear the EHB bit (which is RW1C) because * there might be more events to service. */ val_64 &= ~ERST_EHB; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "intr# %d:Write event ring dequeue pointer, preserving EHB bit", intr_num); xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | val_64, &ir_set->erst_dequeue); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Wrote ERST address to ir_set %d.", intr_num); return 0; } int xhci_sec_event_ring_setup(struct usb_device *udev, unsigned int intr_num) { int ret; struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct xhci_hcd *xhci = hcd_to_xhci(hcd); if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; if ((xhci->xhc_state & XHCI_STATE_HALTED) || !xhci->sec_ir_set || !xhci->sec_event_ring || !xhci->sec_erst || intr_num >= xhci->max_interrupters) { xhci_err(xhci, "%s:state %x ir_set %pK evt_ring %pK erst %pK intr# %d\n", __func__, xhci->xhc_state, xhci->sec_ir_set, xhci->sec_event_ring, xhci->sec_erst, intr_num); return -EINVAL; } if (xhci->sec_event_ring && xhci->sec_event_ring[intr_num] && xhci->sec_event_ring[intr_num]->first_seg) goto done; xhci->sec_ir_set[intr_num] = &xhci->run_regs->ir_set[intr_num]; ret = xhci_event_ring_setup(xhci, &xhci->sec_event_ring[intr_num], xhci->sec_ir_set[intr_num], &xhci->sec_erst[intr_num], intr_num, GFP_KERNEL); if (ret) { xhci_err(xhci, "sec event ring setup failed inter#%d\n", intr_num); return ret; } done: return 0; } EXPORT_SYMBOL(xhci_sec_event_ring_setup); static int xhci_event_ring_init(struct xhci_hcd *xhci, gfp_t flags) { int ret = 0; /* primary + secondary */ xhci->max_interrupters = HCS_MAX_INTRS(xhci->hcs_params1); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating primary event ring"); /* Set ir_set to interrupt register set 0 */ xhci->ir_set = &xhci->run_regs->ir_set[0]; ret = xhci_event_ring_setup(xhci, &xhci->event_ring, xhci->ir_set, &xhci->erst, 0, flags); if (ret) { xhci_err(xhci, "failed to setup primary event ring\n"); goto fail; } xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating sec event ring related pointers"); xhci->sec_ir_set = kcalloc(xhci->max_interrupters, sizeof(*xhci->sec_ir_set), flags); if (!xhci->sec_ir_set) { ret = -ENOMEM; goto fail; } xhci->sec_event_ring = kcalloc(xhci->max_interrupters, sizeof(*xhci->sec_event_ring), flags); if (!xhci->sec_event_ring) { ret = -ENOMEM; goto fail; } xhci->sec_erst = kcalloc(xhci->max_interrupters, sizeof(*xhci->sec_erst), flags); if (!xhci->sec_erst) ret = -ENOMEM; fail: return ret; } int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) { dma_addr_t dma; Loading @@ -2351,7 +2616,7 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) unsigned int val, val2; u64 val_64; u32 page_size, temp; int i, ret; int i; INIT_LIST_HEAD(&xhci->cmd_list); Loading Loading @@ -2471,50 +2736,17 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) "// Doorbell array is located at offset 0x%x" " from cap regs base addr", val); xhci->dba = (void __iomem *) xhci->cap_regs + val; /* Set ir_set to interrupt register set 0 */ xhci->ir_set = &xhci->run_regs->ir_set[0]; /* * Event ring setup: Allocate a normal ring, but also setup * the event ring segment table (ERST). Section 4.9.3. */ xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring"); xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, 0, flags); if (!xhci->event_ring) goto fail; if (xhci_check_trb_in_td_math(xhci) < 0) if (xhci_event_ring_init(xhci, GFP_KERNEL)) goto fail; ret = xhci_alloc_erst(xhci, xhci->event_ring, &xhci->erst, flags); if (ret) if (xhci_check_trb_in_td_math(xhci) < 0) goto fail; /* set ERST count with the number of entries in the segment table */ val = readl(&xhci->ir_set->erst_size); val &= ERST_SIZE_MASK; val |= ERST_NUM_SEGS; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Write ERST size = %i to ir_set 0 (some bits preserved)", val); writel(val, &xhci->ir_set->erst_size); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Set ERST entries to point to event ring."); /* set the segment table base address */ xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Set ERST base address for ir_set 0 = 0x%llx", (unsigned long long)xhci->erst.erst_dma_addr); val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base); val_64 &= ERST_PTR_MASK; val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK); xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base); /* Set the event ring dequeue address */ xhci_set_hc_event_deq(xhci); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Wrote ERST address to ir_set 0."); /* * XXX: Might need to set the Interrupter Moderation Register to * something other than the default (~1ms minimum between interrupts). Loading drivers/usb/host/xhci.c +140 −0 Original line number Diff line number Diff line Loading @@ -5300,6 +5300,146 @@ static void xhci_clear_tt_buffer_complete(struct usb_hcd *hcd, spin_unlock_irqrestore(&xhci->lock, flags); } phys_addr_t xhci_get_sec_event_ring_phys_addr(struct usb_device *udev, unsigned int intr_num, dma_addr_t *dma) { struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct xhci_hcd *xhci = hcd_to_xhci(hcd); struct device *dev = hcd->self.sysdev; struct sg_table sgt; phys_addr_t pa; if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; if (intr_num >= xhci->max_interrupters) { xhci_err(xhci, "intr num %d >= max intrs %d\n", intr_num, xhci->max_interrupters); return 0; } if (!(xhci->xhc_state & XHCI_STATE_HALTED) && xhci->sec_event_ring && xhci->sec_event_ring[intr_num] && xhci->sec_event_ring[intr_num]->first_seg) { dma_get_sgtable(dev, &sgt, xhci->sec_event_ring[intr_num]->first_seg->trbs, xhci->sec_event_ring[intr_num]->first_seg->dma, TRB_SEGMENT_SIZE); *dma = xhci->sec_event_ring[intr_num]->first_seg->dma; pa = page_to_phys(sg_page(sgt.sgl)); sg_free_table(&sgt); return pa; } return 0; } EXPORT_SYMBOL(xhci_get_sec_event_ring_phys_addr); phys_addr_t xhci_get_xfer_ring_phys_addr(struct usb_device *udev, struct usb_host_endpoint *ep, dma_addr_t *dma) { int ret; unsigned int ep_index; struct xhci_virt_device *virt_dev; struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct device *dev = hcd->self.sysdev; struct xhci_hcd *xhci = hcd_to_xhci(hcd); struct sg_table sgt; phys_addr_t pa; if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; ret = xhci_check_args(hcd, udev, ep, 1, true, __func__); if (ret <= 0) { xhci_err(xhci, "%s: invalid args\n", __func__); return 0; } virt_dev = xhci->devs[udev->slot_id]; ep_index = xhci_get_endpoint_index(&ep->desc); if (virt_dev->eps[ep_index].ring && virt_dev->eps[ep_index].ring->first_seg) { dma_get_sgtable(dev, &sgt, virt_dev->eps[ep_index].ring->first_seg->trbs, virt_dev->eps[ep_index].ring->first_seg->dma, TRB_SEGMENT_SIZE); *dma = virt_dev->eps[ep_index].ring->first_seg->dma; pa = page_to_phys(sg_page(sgt.sgl)); sg_free_table(&sgt); return pa; } return 0; } EXPORT_SYMBOL(xhci_get_xfer_ring_phys_addr); int xhci_stop_endpoint(struct usb_device *udev, struct usb_host_endpoint *ep) { struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct xhci_hcd *xhci = hcd_to_xhci(hcd); unsigned int ep_index; struct xhci_virt_device *virt_dev; struct xhci_command *cmd; unsigned long flags; int ret = 0; if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; cmd = xhci_alloc_command(xhci, true, GFP_NOIO); if (!cmd) return -ENOMEM; spin_lock_irqsave(&xhci->lock, flags); virt_dev = xhci->devs[udev->slot_id]; if (!virt_dev) { ret = -ENODEV; goto err; } ep_index = xhci_get_endpoint_index(&ep->desc); if (virt_dev->eps[ep_index].ring && virt_dev->eps[ep_index].ring->dequeue) { ret = xhci_queue_stop_endpoint(xhci, cmd, udev->slot_id, ep_index, 0); if (ret) goto err; xhci_ring_cmd_db(xhci); spin_unlock_irqrestore(&xhci->lock, flags); /* Wait for stop endpoint command to finish */ wait_for_completion(cmd->completion); if (cmd->status == COMP_COMMAND_ABORTED || cmd->status == COMP_STOPPED) { xhci_warn(xhci, "stop endpoint command timeout for ep%d%s\n", usb_endpoint_num(&ep->desc), usb_endpoint_dir_in(&ep->desc) ? "in" : "out"); ret = -ETIME; } goto free_cmd; } err: spin_unlock_irqrestore(&xhci->lock, flags); free_cmd: xhci_free_command(xhci, cmd); return ret; } EXPORT_SYMBOL(xhci_stop_endpoint); static const struct hc_driver xhci_hc_driver = { .description = "xhci-hcd", .product_desc = "xHCI Host Controller", Loading drivers/usb/host/xhci.h +8 −0 Original line number Diff line number Diff line Loading @@ -16,6 +16,7 @@ #include <linux/timer.h> #include <linux/kernel.h> #include <linux/usb/hcd.h> #include <linux/usb/xhci-sec.h> #include <linux/io-64-nonatomic-lo-hi.h> /* Code sharing between pci-quirks and xhci hcd */ Loading Loading @@ -1735,6 +1736,8 @@ struct xhci_hcd { struct xhci_doorbell_array __iomem *dba; /* Our HCD's current interrupter register set */ struct xhci_intr_reg __iomem *ir_set; /* secondary interrupter */ struct xhci_intr_reg __iomem **sec_ir_set; /* Cached register copies of read-only HC data */ __u32 hcs_params1; Loading Loading @@ -1778,6 +1781,11 @@ struct xhci_hcd { struct xhci_command *current_cmd; struct xhci_ring *event_ring; struct xhci_erst erst; /* secondary event ring and erst */ struct xhci_ring **sec_event_ring; struct xhci_erst *sec_erst; /* Scratchpad */ struct xhci_scratchpad *scratchpad; /* Store LPM test failed devices' information */ Loading include/linux/usb/xhci-sec.h 0 → 100644 +54 −0 Original line number Diff line number Diff line /* SPDX-License-Identifier: GPL-2.0-only */ /* * xHCI secondary ring APIs * * Copyright (c) 2019, The Linux Foundation. All rights reserved. */ #ifndef __LINUX_XHCI_SEC_H #define __LINUX_XHCI_SEC_H #include <linux/usb.h> #if IS_ENABLED(CONFIG_USB_XHCI_HCD) int xhci_sec_event_ring_setup(struct usb_device *udev, unsigned int intr_num); int xhci_sec_event_ring_cleanup(struct usb_device *udev, unsigned int intr_num); phys_addr_t xhci_get_sec_event_ring_phys_addr(struct usb_device *udev, unsigned int intr_num, dma_addr_t *dma); phys_addr_t xhci_get_xfer_ring_phys_addr(struct usb_device *udev, struct usb_host_endpoint *ep, dma_addr_t *dma); int xhci_stop_endpoint(struct usb_device *udev, struct usb_host_endpoint *ep); #else static inline int xhci_sec_event_ring_setup(struct usb_device *udev, unsigned int intr_num); { return -ENODEV; } static inline int xhci_sec_event_ring_cleanup(struct usb_device *udev, unsigned int intr_num) { return -ENODEV; } static inline phys_addr_t xhci_get_sec_event_ring_phys_addr( struct usb_device *udev, unsigned int intr_num, dma_addr_t *dma) { return NULL; } static inline phys_addr_t xhci_get_xfer_ring_phys_addr(struct usb_device *udev, struct usb_host_endpoint *ep, dma_addr_t *dma) { return NULL; } static inline int xhci_stop_endpoint(struct usb_device *udev, struct usb_host_endpoint *ep) { return -ENODEV; } #endif #endif /* __LINUX_XHCI_SEC_H */ Loading
drivers/usb/host/xhci-mem.c +298 −66 Original line number Diff line number Diff line Loading @@ -1831,19 +1831,155 @@ void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst) erst->entries = NULL; } void xhci_mem_cleanup(struct xhci_hcd *xhci) static void xhci_handle_sec_intr_events(struct xhci_hcd *xhci, int intr_num) { union xhci_trb *erdp_trb, *current_trb; struct xhci_segment *seg; u64 erdp_reg; u32 iman_reg; dma_addr_t deq; unsigned long segment_offset; /* disable irq, ack pending interrupt and ack all pending events */ iman_reg = readl_relaxed(&xhci->sec_ir_set[intr_num]->irq_pending); iman_reg &= ~IMAN_IE; writel_relaxed(iman_reg, &xhci->sec_ir_set[intr_num]->irq_pending); iman_reg = readl_relaxed(&xhci->sec_ir_set[intr_num]->irq_pending); if (iman_reg & IMAN_IP) writel_relaxed(iman_reg, &xhci->sec_ir_set[intr_num]->irq_pending); /* last acked event trb is in erdp reg */ erdp_reg = xhci_read_64(xhci, &xhci->sec_ir_set[intr_num]->erst_dequeue); deq = (dma_addr_t)(erdp_reg & ~ERST_PTR_MASK); if (!deq) { pr_debug("%s: event ring handling not required\n", __func__); return; } seg = xhci->sec_event_ring[intr_num]->first_seg; segment_offset = deq - seg->dma; /* find out virtual address of the last acked event trb */ erdp_trb = current_trb = &seg->trbs[0] + (segment_offset/sizeof(*current_trb)); /* read cycle state of the last acked trb to find out CCS */ xhci->sec_event_ring[intr_num]->cycle_state = le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE; while (1) { /* last trb of the event ring: toggle cycle state */ if (current_trb == &seg->trbs[TRBS_PER_SEGMENT - 1]) { xhci->sec_event_ring[intr_num]->cycle_state ^= 1; current_trb = &seg->trbs[0]; } else { current_trb++; } /* cycle state transition */ if ((le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE) != xhci->sec_event_ring[intr_num]->cycle_state) break; } if (erdp_trb != current_trb) { deq = xhci_trb_virt_to_dma(xhci->sec_event_ring[intr_num]->deq_seg, current_trb); if (deq == 0) xhci_warn(xhci, "WARN invalid SW event ring dequeue ptr.\n"); /* Update HC event ring dequeue pointer */ erdp_reg &= ERST_PTR_MASK; erdp_reg |= ((u64) deq & (u64) ~ERST_PTR_MASK); } /* Clear the event handler busy flag (RW1C); event ring is empty. */ erdp_reg |= ERST_EHB; xhci_write_64(xhci, erdp_reg, &xhci->sec_ir_set[intr_num]->erst_dequeue); } static int sec_event_ring_cleanup(struct xhci_hcd *xhci, unsigned int intr_num) { int size; struct device *dev = xhci_to_hcd(xhci)->self.sysdev; int i, j, num_ports; cancel_delayed_work_sync(&xhci->cmd_timer); if (intr_num >= xhci->max_interrupters) { xhci_err(xhci, "invalid secondary interrupter num %d\n", intr_num); return -EINVAL; } xhci_free_erst(xhci, &xhci->erst); size = sizeof(struct xhci_erst_entry)*(xhci->sec_erst[intr_num].num_entries); if (xhci->sec_erst[intr_num].entries) { xhci_handle_sec_intr_events(xhci, intr_num); dma_free_coherent(dev, size, xhci->sec_erst[intr_num].entries, xhci->sec_erst[intr_num].erst_dma_addr); xhci->sec_erst[intr_num].entries = NULL; } xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed SEC ERST#%d", intr_num); if (xhci->sec_event_ring[intr_num]) xhci_ring_free(xhci, xhci->sec_event_ring[intr_num]); xhci->sec_event_ring[intr_num] = NULL; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed sec event ring"); return 0; } int xhci_sec_event_ring_cleanup(struct usb_device *udev, unsigned int intr_num) { struct usb_hcd *hcd = bus_to_hcd(udev->bus); if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; return sec_event_ring_cleanup(hcd_to_xhci(hcd), intr_num); } EXPORT_SYMBOL(xhci_sec_event_ring_cleanup); static void xhci_event_ring_cleanup(struct xhci_hcd *xhci) { unsigned int i; /* sec event ring clean up */ for (i = 1; i < xhci->max_interrupters; i++) sec_event_ring_cleanup(xhci, i); kfree(xhci->sec_ir_set); xhci->sec_ir_set = NULL; kfree(xhci->sec_erst); xhci->sec_erst = NULL; kfree(xhci->sec_event_ring); xhci->sec_event_ring = NULL; /* primary event ring clean up */ xhci_free_erst(xhci, &xhci->erst); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed primary ERST"); if (xhci->event_ring) xhci_ring_free(xhci, xhci->event_ring); xhci->event_ring = NULL; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring"); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed priamry event ring"); } void xhci_mem_cleanup(struct xhci_hcd *xhci) { struct device *dev = xhci_to_hcd(xhci)->self.sysdev; int i, j, num_ports; cancel_delayed_work_sync(&xhci->cmd_timer); xhci_event_ring_cleanup(xhci); if (xhci->lpm_command) xhci_free_command(xhci, xhci->lpm_command); Loading Loading @@ -2084,30 +2220,6 @@ static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci) return 0; } static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) { u64 temp; dma_addr_t deq; deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, xhci->event_ring->dequeue); if (deq == 0 && !in_interrupt()) xhci_warn(xhci, "WARN something wrong with SW event ring " "dequeue ptr.\n"); /* Update HC event ring dequeue pointer */ temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); temp &= ERST_PTR_MASK; /* Don't clear the EHB bit (which is RW1C) because * there might be more events to service. */ temp &= ~ERST_EHB; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Write event ring dequeue pointer, " "preserving EHB bit"); xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, &xhci->ir_set->erst_dequeue); } static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, __le32 __iomem *addr, int max_caps) { Loading Loading @@ -2344,6 +2456,159 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) return 0; } static int xhci_event_ring_setup(struct xhci_hcd *xhci, struct xhci_ring **er, struct xhci_intr_reg __iomem *ir_set, struct xhci_erst *erst, unsigned int intr_num, gfp_t flags) { dma_addr_t deq; u64 val_64; unsigned int val; int ret; *er = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, 0, flags); if (!*er) return -ENOMEM; ret = xhci_alloc_erst(xhci, *er, erst, flags); if (ret) return ret; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "intr# %d: num segs = %i, virt addr = %pK, dma addr = 0x%llx", intr_num, erst->num_entries, erst->entries, (unsigned long long)erst->erst_dma_addr); /* set ERST count with the number of entries in the segment table */ val = readl_relaxed(&ir_set->erst_size); val &= ERST_SIZE_MASK; val |= ERST_NUM_SEGS; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Write ERST size = %i to ir_set %d (some bits preserved)", val, intr_num); writel_relaxed(val, &ir_set->erst_size); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "intr# %d: Set ERST entries to point to event ring.", intr_num); /* set the segment table base address */ xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Set ERST base address for ir_set %d = 0x%llx", intr_num, (unsigned long long)erst->erst_dma_addr); val_64 = xhci_read_64(xhci, &ir_set->erst_base); val_64 &= ERST_PTR_MASK; val_64 |= (erst->erst_dma_addr & (u64) ~ERST_PTR_MASK); xhci_write_64(xhci, val_64, &ir_set->erst_base); /* Set the event ring dequeue address */ deq = xhci_trb_virt_to_dma((*er)->deq_seg, (*er)->dequeue); if (deq == 0 && !in_interrupt()) xhci_warn(xhci, "intr# %d:WARN something wrong with SW event ring deq ptr.\n", intr_num); /* Update HC event ring dequeue pointer */ val_64 = xhci_read_64(xhci, &ir_set->erst_dequeue); val_64 &= ERST_PTR_MASK; /* Don't clear the EHB bit (which is RW1C) because * there might be more events to service. */ val_64 &= ~ERST_EHB; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "intr# %d:Write event ring dequeue pointer, preserving EHB bit", intr_num); xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | val_64, &ir_set->erst_dequeue); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Wrote ERST address to ir_set %d.", intr_num); return 0; } int xhci_sec_event_ring_setup(struct usb_device *udev, unsigned int intr_num) { int ret; struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct xhci_hcd *xhci = hcd_to_xhci(hcd); if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; if ((xhci->xhc_state & XHCI_STATE_HALTED) || !xhci->sec_ir_set || !xhci->sec_event_ring || !xhci->sec_erst || intr_num >= xhci->max_interrupters) { xhci_err(xhci, "%s:state %x ir_set %pK evt_ring %pK erst %pK intr# %d\n", __func__, xhci->xhc_state, xhci->sec_ir_set, xhci->sec_event_ring, xhci->sec_erst, intr_num); return -EINVAL; } if (xhci->sec_event_ring && xhci->sec_event_ring[intr_num] && xhci->sec_event_ring[intr_num]->first_seg) goto done; xhci->sec_ir_set[intr_num] = &xhci->run_regs->ir_set[intr_num]; ret = xhci_event_ring_setup(xhci, &xhci->sec_event_ring[intr_num], xhci->sec_ir_set[intr_num], &xhci->sec_erst[intr_num], intr_num, GFP_KERNEL); if (ret) { xhci_err(xhci, "sec event ring setup failed inter#%d\n", intr_num); return ret; } done: return 0; } EXPORT_SYMBOL(xhci_sec_event_ring_setup); static int xhci_event_ring_init(struct xhci_hcd *xhci, gfp_t flags) { int ret = 0; /* primary + secondary */ xhci->max_interrupters = HCS_MAX_INTRS(xhci->hcs_params1); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating primary event ring"); /* Set ir_set to interrupt register set 0 */ xhci->ir_set = &xhci->run_regs->ir_set[0]; ret = xhci_event_ring_setup(xhci, &xhci->event_ring, xhci->ir_set, &xhci->erst, 0, flags); if (ret) { xhci_err(xhci, "failed to setup primary event ring\n"); goto fail; } xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating sec event ring related pointers"); xhci->sec_ir_set = kcalloc(xhci->max_interrupters, sizeof(*xhci->sec_ir_set), flags); if (!xhci->sec_ir_set) { ret = -ENOMEM; goto fail; } xhci->sec_event_ring = kcalloc(xhci->max_interrupters, sizeof(*xhci->sec_event_ring), flags); if (!xhci->sec_event_ring) { ret = -ENOMEM; goto fail; } xhci->sec_erst = kcalloc(xhci->max_interrupters, sizeof(*xhci->sec_erst), flags); if (!xhci->sec_erst) ret = -ENOMEM; fail: return ret; } int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) { dma_addr_t dma; Loading @@ -2351,7 +2616,7 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) unsigned int val, val2; u64 val_64; u32 page_size, temp; int i, ret; int i; INIT_LIST_HEAD(&xhci->cmd_list); Loading Loading @@ -2471,50 +2736,17 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) "// Doorbell array is located at offset 0x%x" " from cap regs base addr", val); xhci->dba = (void __iomem *) xhci->cap_regs + val; /* Set ir_set to interrupt register set 0 */ xhci->ir_set = &xhci->run_regs->ir_set[0]; /* * Event ring setup: Allocate a normal ring, but also setup * the event ring segment table (ERST). Section 4.9.3. */ xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring"); xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, 0, flags); if (!xhci->event_ring) goto fail; if (xhci_check_trb_in_td_math(xhci) < 0) if (xhci_event_ring_init(xhci, GFP_KERNEL)) goto fail; ret = xhci_alloc_erst(xhci, xhci->event_ring, &xhci->erst, flags); if (ret) if (xhci_check_trb_in_td_math(xhci) < 0) goto fail; /* set ERST count with the number of entries in the segment table */ val = readl(&xhci->ir_set->erst_size); val &= ERST_SIZE_MASK; val |= ERST_NUM_SEGS; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Write ERST size = %i to ir_set 0 (some bits preserved)", val); writel(val, &xhci->ir_set->erst_size); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Set ERST entries to point to event ring."); /* set the segment table base address */ xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Set ERST base address for ir_set 0 = 0x%llx", (unsigned long long)xhci->erst.erst_dma_addr); val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base); val_64 &= ERST_PTR_MASK; val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK); xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base); /* Set the event ring dequeue address */ xhci_set_hc_event_deq(xhci); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Wrote ERST address to ir_set 0."); /* * XXX: Might need to set the Interrupter Moderation Register to * something other than the default (~1ms minimum between interrupts). Loading
drivers/usb/host/xhci.c +140 −0 Original line number Diff line number Diff line Loading @@ -5300,6 +5300,146 @@ static void xhci_clear_tt_buffer_complete(struct usb_hcd *hcd, spin_unlock_irqrestore(&xhci->lock, flags); } phys_addr_t xhci_get_sec_event_ring_phys_addr(struct usb_device *udev, unsigned int intr_num, dma_addr_t *dma) { struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct xhci_hcd *xhci = hcd_to_xhci(hcd); struct device *dev = hcd->self.sysdev; struct sg_table sgt; phys_addr_t pa; if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; if (intr_num >= xhci->max_interrupters) { xhci_err(xhci, "intr num %d >= max intrs %d\n", intr_num, xhci->max_interrupters); return 0; } if (!(xhci->xhc_state & XHCI_STATE_HALTED) && xhci->sec_event_ring && xhci->sec_event_ring[intr_num] && xhci->sec_event_ring[intr_num]->first_seg) { dma_get_sgtable(dev, &sgt, xhci->sec_event_ring[intr_num]->first_seg->trbs, xhci->sec_event_ring[intr_num]->first_seg->dma, TRB_SEGMENT_SIZE); *dma = xhci->sec_event_ring[intr_num]->first_seg->dma; pa = page_to_phys(sg_page(sgt.sgl)); sg_free_table(&sgt); return pa; } return 0; } EXPORT_SYMBOL(xhci_get_sec_event_ring_phys_addr); phys_addr_t xhci_get_xfer_ring_phys_addr(struct usb_device *udev, struct usb_host_endpoint *ep, dma_addr_t *dma) { int ret; unsigned int ep_index; struct xhci_virt_device *virt_dev; struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct device *dev = hcd->self.sysdev; struct xhci_hcd *xhci = hcd_to_xhci(hcd); struct sg_table sgt; phys_addr_t pa; if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; ret = xhci_check_args(hcd, udev, ep, 1, true, __func__); if (ret <= 0) { xhci_err(xhci, "%s: invalid args\n", __func__); return 0; } virt_dev = xhci->devs[udev->slot_id]; ep_index = xhci_get_endpoint_index(&ep->desc); if (virt_dev->eps[ep_index].ring && virt_dev->eps[ep_index].ring->first_seg) { dma_get_sgtable(dev, &sgt, virt_dev->eps[ep_index].ring->first_seg->trbs, virt_dev->eps[ep_index].ring->first_seg->dma, TRB_SEGMENT_SIZE); *dma = virt_dev->eps[ep_index].ring->first_seg->dma; pa = page_to_phys(sg_page(sgt.sgl)); sg_free_table(&sgt); return pa; } return 0; } EXPORT_SYMBOL(xhci_get_xfer_ring_phys_addr); int xhci_stop_endpoint(struct usb_device *udev, struct usb_host_endpoint *ep) { struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct xhci_hcd *xhci = hcd_to_xhci(hcd); unsigned int ep_index; struct xhci_virt_device *virt_dev; struct xhci_command *cmd; unsigned long flags; int ret = 0; if (udev->state == USB_STATE_NOTATTACHED || !HCD_RH_RUNNING(hcd)) return 0; cmd = xhci_alloc_command(xhci, true, GFP_NOIO); if (!cmd) return -ENOMEM; spin_lock_irqsave(&xhci->lock, flags); virt_dev = xhci->devs[udev->slot_id]; if (!virt_dev) { ret = -ENODEV; goto err; } ep_index = xhci_get_endpoint_index(&ep->desc); if (virt_dev->eps[ep_index].ring && virt_dev->eps[ep_index].ring->dequeue) { ret = xhci_queue_stop_endpoint(xhci, cmd, udev->slot_id, ep_index, 0); if (ret) goto err; xhci_ring_cmd_db(xhci); spin_unlock_irqrestore(&xhci->lock, flags); /* Wait for stop endpoint command to finish */ wait_for_completion(cmd->completion); if (cmd->status == COMP_COMMAND_ABORTED || cmd->status == COMP_STOPPED) { xhci_warn(xhci, "stop endpoint command timeout for ep%d%s\n", usb_endpoint_num(&ep->desc), usb_endpoint_dir_in(&ep->desc) ? "in" : "out"); ret = -ETIME; } goto free_cmd; } err: spin_unlock_irqrestore(&xhci->lock, flags); free_cmd: xhci_free_command(xhci, cmd); return ret; } EXPORT_SYMBOL(xhci_stop_endpoint); static const struct hc_driver xhci_hc_driver = { .description = "xhci-hcd", .product_desc = "xHCI Host Controller", Loading
drivers/usb/host/xhci.h +8 −0 Original line number Diff line number Diff line Loading @@ -16,6 +16,7 @@ #include <linux/timer.h> #include <linux/kernel.h> #include <linux/usb/hcd.h> #include <linux/usb/xhci-sec.h> #include <linux/io-64-nonatomic-lo-hi.h> /* Code sharing between pci-quirks and xhci hcd */ Loading Loading @@ -1735,6 +1736,8 @@ struct xhci_hcd { struct xhci_doorbell_array __iomem *dba; /* Our HCD's current interrupter register set */ struct xhci_intr_reg __iomem *ir_set; /* secondary interrupter */ struct xhci_intr_reg __iomem **sec_ir_set; /* Cached register copies of read-only HC data */ __u32 hcs_params1; Loading Loading @@ -1778,6 +1781,11 @@ struct xhci_hcd { struct xhci_command *current_cmd; struct xhci_ring *event_ring; struct xhci_erst erst; /* secondary event ring and erst */ struct xhci_ring **sec_event_ring; struct xhci_erst *sec_erst; /* Scratchpad */ struct xhci_scratchpad *scratchpad; /* Store LPM test failed devices' information */ Loading
include/linux/usb/xhci-sec.h 0 → 100644 +54 −0 Original line number Diff line number Diff line /* SPDX-License-Identifier: GPL-2.0-only */ /* * xHCI secondary ring APIs * * Copyright (c) 2019, The Linux Foundation. All rights reserved. */ #ifndef __LINUX_XHCI_SEC_H #define __LINUX_XHCI_SEC_H #include <linux/usb.h> #if IS_ENABLED(CONFIG_USB_XHCI_HCD) int xhci_sec_event_ring_setup(struct usb_device *udev, unsigned int intr_num); int xhci_sec_event_ring_cleanup(struct usb_device *udev, unsigned int intr_num); phys_addr_t xhci_get_sec_event_ring_phys_addr(struct usb_device *udev, unsigned int intr_num, dma_addr_t *dma); phys_addr_t xhci_get_xfer_ring_phys_addr(struct usb_device *udev, struct usb_host_endpoint *ep, dma_addr_t *dma); int xhci_stop_endpoint(struct usb_device *udev, struct usb_host_endpoint *ep); #else static inline int xhci_sec_event_ring_setup(struct usb_device *udev, unsigned int intr_num); { return -ENODEV; } static inline int xhci_sec_event_ring_cleanup(struct usb_device *udev, unsigned int intr_num) { return -ENODEV; } static inline phys_addr_t xhci_get_sec_event_ring_phys_addr( struct usb_device *udev, unsigned int intr_num, dma_addr_t *dma) { return NULL; } static inline phys_addr_t xhci_get_xfer_ring_phys_addr(struct usb_device *udev, struct usb_host_endpoint *ep, dma_addr_t *dma) { return NULL; } static inline int xhci_stop_endpoint(struct usb_device *udev, struct usb_host_endpoint *ep) { return -ENODEV; } #endif #endif /* __LINUX_XHCI_SEC_H */