{net,IB}/mlx5: Refactor page fault handling (d9aaed83) · Commits · e / devices / android_kernel_fairphone_FP5

drivers/infiniband/hw/mlx5/main.c

+3 −11

Original line number	Diff line number	Diff line
		@@ -3319,6 +3319,9 @@ static struct mlx5_interface mlx5_ib_interface = {
		.add = mlx5_ib_add,
		.remove = mlx5_ib_remove,
		.event = mlx5_ib_event,
		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		.pfault = mlx5_ib_pfault,
		#endif
		.protocol = MLX5_INTERFACE_PROTOCOL_IB,
		};

		@@ -3329,25 +3332,14 @@ static int __init mlx5_ib_init(void)
		if (deprecated_prof_sel != 2)
		pr_warn("prof_sel is deprecated for mlx5_ib, set it for mlx5_core\n");

		err = mlx5_ib_odp_init();
		if (err)
		return err;

		err = mlx5_register_interface(&mlx5_ib_interface);
		if (err)
		goto clean_odp;

		return err;

		clean_odp:
		mlx5_ib_odp_cleanup();
		return err;
		}

		static void __exit mlx5_ib_cleanup(void)
		{
		mlx5_unregister_interface(&mlx5_ib_interface);
		mlx5_ib_odp_cleanup();
		}

		module_init(mlx5_ib_init);

drivers/infiniband/hw/mlx5/mlx5_ib.h

+2 −47

Original line number	Diff line number	Diff line
		@@ -277,29 +277,6 @@ struct mlx5_ib_rwq_ind_table {
		u32 rqtn;
		};

		/*
		* Connect-IB can trigger up to four concurrent pagefaults
		* per-QP.
		*/
		enum mlx5_ib_pagefault_context {
		MLX5_IB_PAGEFAULT_RESPONDER_READ,
		MLX5_IB_PAGEFAULT_REQUESTOR_READ,
		MLX5_IB_PAGEFAULT_RESPONDER_WRITE,
		MLX5_IB_PAGEFAULT_REQUESTOR_WRITE,
		MLX5_IB_PAGEFAULT_CONTEXTS
		};

		static inline enum mlx5_ib_pagefault_context
		mlx5_ib_get_pagefault_context(struct mlx5_pagefault *pagefault)
		{
		return pagefault->flags & (MLX5_PFAULT_REQUESTOR \| MLX5_PFAULT_WRITE);
		}

		struct mlx5_ib_pfault {
		struct work_struct work;
		struct mlx5_pagefault mpfault;
		};

		struct mlx5_ib_ubuffer {
		struct ib_umem *umem;
		int buf_size;
		@@ -385,20 +362,6 @@ struct mlx5_ib_qp {
		/* Store signature errors */
		bool signature_en;

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		/*
		* A flag that is true for QP's that are in a state that doesn't
		* allow page faults, and shouldn't schedule any more faults.
		*/
		int disable_page_faults;
		/*
		* The disable_page_faults_lock protects a QP's disable_page_faults
		* field, allowing for a thread to atomically check whether the QP
		* allows page faults, and if so schedule a page fault.
		*/
		spinlock_t disable_page_faults_lock;
		struct mlx5_ib_pfault pagefaults[MLX5_IB_PAGEFAULT_CONTEXTS];
		#endif
		struct list_head qps_list;
		struct list_head cq_recv_list;
		struct list_head cq_send_list;
		@@ -869,18 +832,13 @@ struct ib_rwq_ind_table mlx5_ib_create_rwq_ind_table(struct ib_device device,
		int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		extern struct workqueue_struct *mlx5_ib_page_fault_wq;

		void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);
		void mlx5_ib_mr_pfault_handler(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault);
		void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp);
		void mlx5_ib_pfault(struct mlx5_core_dev mdev, void context,
		struct mlx5_pagefault *pfault);
		int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
		void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev);
		int __init mlx5_ib_odp_init(void);
		void mlx5_ib_odp_cleanup(void);
		void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp);
		void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp);
		void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
		unsigned long end);
		#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
		@@ -889,13 +847,10 @@ static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
		return;
		}

		static inline void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp) {}
		static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
		static inline void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev) {}
		static inline int mlx5_ib_odp_init(void) { return 0; }
		static inline void mlx5_ib_odp_cleanup(void) {}
		static inline void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp) {}
		static inline void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp) {}

		#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */

drivers/infiniband/hw/mlx5/odp.c

+110 −190

Original line number	Diff line number	Diff line
		@@ -41,8 +41,6 @@
		* a pagefault. */
		#define MMU_NOTIFIER_TIMEOUT 1000

		struct workqueue_struct *mlx5_ib_page_fault_wq;

		void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
		unsigned long end)
		{
		@@ -162,38 +160,38 @@ static struct mlx5_ib_mr mlx5_ib_odp_find_mr_lkey(struct mlx5_ib_dev dev,
		return container_of(mmkey, struct mlx5_ib_mr, mmkey);
		}

		static void mlx5_ib_page_fault_resume(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault,
		static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,
		struct mlx5_pagefault *pfault,
		int error)
		{
		struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);
		u32 qpn = qp->trans_qp.base.mqp.qpn;
		int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?
		pfault->wqe.wq_num : pfault->token;
		int ret = mlx5_core_page_fault_resume(dev->mdev,
		qpn,
		pfault->mpfault.flags,
		pfault->token,
		wq_num,
		pfault->type,
		error);
		if (ret)
		pr_err("Failed to resolve the page fault on QP 0x%x\n", qpn);
		mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x\n",
		wq_num);
		}

		/*
		* Handle a single data segment in a page-fault WQE.
		* Handle a single data segment in a page-fault WQE or RDMA region.
		*
		* Returns number of pages retrieved on success. The caller will continue to
		* Returns number of pages retrieved on success. The caller may continue to
		* the next data segment.
		* Can return the following error codes:
		* -EAGAIN to designate a temporary error. The caller will abort handling the
		* page fault and resolve it.
		* -EFAULT when there's an error mapping the requested pages. The caller will
		* abort the page fault handling and possibly move the QP to an error state.
		* On other errors the QP should also be closed with an error.
		* abort the page fault handling.
		*/
		static int pagefault_single_data_segment(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault,
		static int pagefault_single_data_segment(struct mlx5_ib_dev *mib_dev,
		u32 key, u64 io_virt, size_t bcnt,
		u32 *bytes_committed,
		u32 *bytes_mapped)
		{
		struct mlx5_ib_dev *mib_dev = to_mdev(qp->ibqp.pd->device);
		int srcu_key;
		unsigned int current_seq;
		u64 start_idx;
		@@ -219,12 +217,7 @@ static int pagefault_single_data_segment(struct mlx5_ib_qp *qp,
		key);
		if (bytes_mapped)
		*bytes_mapped +=
		(bcnt - pfault->mpfault.bytes_committed);
		goto srcu_unlock;
		}
		if (mr->ibmr.pd != qp->ibqp.pd) {
		pr_err("Page-fault with different PDs for QP and MR.\n");
		ret = -EFAULT;
		(bcnt - *bytes_committed);
		goto srcu_unlock;
		}

		@@ -240,8 +233,8 @@ static int pagefault_single_data_segment(struct mlx5_ib_qp *qp,
		* in all iterations (in iteration 2 and above,
		* bytes_committed == 0).
		*/
		io_virt += pfault->mpfault.bytes_committed;
		bcnt -= pfault->mpfault.bytes_committed;
		io_virt += *bytes_committed;
		bcnt -= *bytes_committed;

		start_idx = (io_virt - (mr->mmkey.iova & PAGE_MASK)) >> PAGE_SHIFT;

		@@ -300,7 +293,7 @@ static int pagefault_single_data_segment(struct mlx5_ib_qp *qp,
		}
		}
		srcu_read_unlock(&mib_dev->mr_srcu, srcu_key);
		pfault->mpfault.bytes_committed = 0;
		*bytes_committed = 0;
		return ret ? ret : npages;
		}

		@@ -322,8 +315,9 @@ static int pagefault_single_data_segment(struct mlx5_ib_qp *qp,
		* Returns the number of pages loaded if positive, zero for an empty WQE, or a
		* negative error code.
		*/
		static int pagefault_data_segments(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault pfault, void wqe,
		static int pagefault_data_segments(struct mlx5_ib_dev *dev,
		struct mlx5_pagefault *pfault,
		struct mlx5_ib_qp qp, void wqe,
		void wqe_end, u32 bytes_mapped,
		u32 *total_wqe_bytes, int receive_queue)
		{
		@@ -367,22 +361,23 @@ static int pagefault_data_segments(struct mlx5_ib_qp *qp,

		if (!inline_segment && total_wqe_bytes) {
		*total_wqe_bytes += bcnt - min_t(size_t, bcnt,
		pfault->mpfault.bytes_committed);
		pfault->bytes_committed);
		}

		/* A zero length data segment designates a length of 2GB. */
		if (bcnt == 0)
		bcnt = 1U << 31;

		if (inline_segment \|\| bcnt <= pfault->mpfault.bytes_committed) {
		pfault->mpfault.bytes_committed -=
		if (inline_segment \|\| bcnt <= pfault->bytes_committed) {
		pfault->bytes_committed -=
		min_t(size_t, bcnt,
		pfault->mpfault.bytes_committed);
		pfault->bytes_committed);
		continue;
		}

		ret = pagefault_single_data_segment(qp, pfault, key, io_virt,
		bcnt, bytes_mapped);
		ret = pagefault_single_data_segment(dev, key, io_virt, bcnt,
		&pfault->bytes_committed,
		bytes_mapped);
		if (ret < 0)
		break;
		npages += ret;
		@@ -396,12 +391,11 @@ static int pagefault_data_segments(struct mlx5_ib_qp *qp,
		* scatter-gather list, and set wqe_end to the end of the WQE.
		*/
		static int mlx5_ib_mr_initiator_pfault_handler(
		struct mlx5_ib_qp qp, struct mlx5_ib_pfault pfault,
		void wqe, void wqe_end, int wqe_length)
		struct mlx5_ib_dev dev, struct mlx5_pagefault pfault,
		struct mlx5_ib_qp qp, void wqe, void *wqe_end, int wqe_length)
		{
		struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);
		struct mlx5_wqe_ctrl_seg ctrl = wqe;
		u16 wqe_index = pfault->mpfault.wqe.wqe_index;
		u16 wqe_index = pfault->wqe.wqe_index;
		unsigned ds, opcode;
		#if defined(DEBUG)
		u32 ctrl_wqe_index, ctrl_qpn;
		@@ -502,10 +496,9 @@ static int mlx5_ib_mr_initiator_pfault_handler(
		* scatter-gather list, and set wqe_end to the end of the WQE.
		*/
		static int mlx5_ib_mr_responder_pfault_handler(
		struct mlx5_ib_qp qp, struct mlx5_ib_pfault pfault,
		void wqe, void wqe_end, int wqe_length)
		struct mlx5_ib_dev dev, struct mlx5_pagefault pfault,
		struct mlx5_ib_qp qp, void wqe, void *wqe_end, int wqe_length)
		{
		struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);
		struct mlx5_ib_wq *wq = &qp->rq;
		int wqe_size = 1 << wq->wqe_shift;

		@@ -542,70 +535,83 @@ static int mlx5_ib_mr_responder_pfault_handler(
		return 0;
		}

		static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault)
		static struct mlx5_ib_qp mlx5_ib_odp_find_qp(struct mlx5_ib_dev dev,
		u32 wq_num)
		{
		struct mlx5_core_qp *mqp = __mlx5_qp_lookup(dev->mdev, wq_num);

		if (!mqp) {
		mlx5_ib_err(dev, "QPN 0x%6x not found\n", wq_num);
		return NULL;
		}

		return to_mibqp(mqp);
		}

		static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,
		struct mlx5_pagefault *pfault)
		{
		struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);
		int ret;
		void wqe, wqe_end;
		u32 bytes_mapped, total_wqe_bytes;
		char *buffer = NULL;
		int resume_with_error = 0;
		u16 wqe_index = pfault->mpfault.wqe.wqe_index;
		int requestor = pfault->mpfault.flags & MLX5_PFAULT_REQUESTOR;
		u32 qpn = qp->trans_qp.base.mqp.qpn;
		int resume_with_error = 1;
		u16 wqe_index = pfault->wqe.wqe_index;
		int requestor = pfault->type & MLX5_PFAULT_REQUESTOR;
		struct mlx5_ib_qp *qp;

		buffer = (char *)__get_free_page(GFP_KERNEL);
		if (!buffer) {
		mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n");
		resume_with_error = 1;
		goto resolve_page_fault;
		}

		qp = mlx5_ib_odp_find_qp(dev, pfault->wqe.wq_num);
		if (!qp)
		goto resolve_page_fault;

		ret = mlx5_ib_read_user_wqe(qp, requestor, wqe_index, buffer,
		PAGE_SIZE, &qp->trans_qp.base);
		if (ret < 0) {
		mlx5_ib_err(dev, "Failed reading a WQE following page fault, error=%x, wqe_index=%x, qpn=%x\n",
		-ret, wqe_index, qpn);
		resume_with_error = 1;
		mlx5_ib_err(dev, "Failed reading a WQE following page fault, error=%d, wqe_index=%x, qpn=%x\n",
		ret, wqe_index, pfault->token);
		goto resolve_page_fault;
		}

		wqe = buffer;
		if (requestor)
		ret = mlx5_ib_mr_initiator_pfault_handler(qp, pfault, &wqe,
		ret = mlx5_ib_mr_initiator_pfault_handler(dev, pfault, qp, &wqe,
		&wqe_end, ret);
		else
		ret = mlx5_ib_mr_responder_pfault_handler(qp, pfault, &wqe,
		ret = mlx5_ib_mr_responder_pfault_handler(dev, pfault, qp, &wqe,
		&wqe_end, ret);
		if (ret < 0) {
		resume_with_error = 1;
		if (ret < 0)
		goto resolve_page_fault;
		}

		if (wqe >= wqe_end) {
		mlx5_ib_err(dev, "ODP fault on invalid WQE.\n");
		resume_with_error = 1;
		goto resolve_page_fault;
		}

		ret = pagefault_data_segments(qp, pfault, wqe, wqe_end, &bytes_mapped,
		&total_wqe_bytes, !requestor);
		ret = pagefault_data_segments(dev, pfault, qp, wqe, wqe_end,
		&bytes_mapped, &total_wqe_bytes,
		!requestor);
		if (ret == -EAGAIN) {
		resume_with_error = 0;
		goto resolve_page_fault;
		} else if (ret < 0 \|\| total_wqe_bytes > bytes_mapped) {
		mlx5_ib_err(dev, "Error getting user pages for page fault. Error: 0x%x\n",
		-ret);
		resume_with_error = 1;
		if (ret != -ENOENT)
		mlx5_ib_err(dev, "Error getting user pages for page fault. Error: %d\n",
		ret);
		goto resolve_page_fault;
		}

		resume_with_error = 0;
		resolve_page_fault:
		mlx5_ib_page_fault_resume(qp, pfault, resume_with_error);
		mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, flags: 0x%x\n",
		qpn, resume_with_error,
		pfault->mpfault.flags);

		mlx5_ib_page_fault_resume(dev, pfault, resume_with_error);
		mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n",
		pfault->token, resume_with_error,
		pfault->type);
		free_page((unsigned long)buffer);
		}

		@@ -615,15 +621,14 @@ static int pages_in_range(u64 address, u32 length)
		(address & PAGE_MASK)) >> PAGE_SHIFT;
		}

		static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault)
		static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev,
		struct mlx5_pagefault *pfault)
		{
		struct mlx5_pagefault *mpfault = &pfault->mpfault;
		u64 address;
		u32 length;
		u32 prefetch_len = mpfault->bytes_committed;
		u32 prefetch_len = pfault->bytes_committed;
		int prefetch_activated = 0;
		u32 rkey = mpfault->rdma.r_key;
		u32 rkey = pfault->rdma.r_key;
		int ret;

		/* The RDMA responder handler handles the page fault in two parts.
		@@ -632,38 +637,40 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_qp *qp,
		* prefetches more pages. The second operation cannot use the pfault
		* context and therefore uses the dummy_pfault context allocated on
		* the stack */
		struct mlx5_ib_pfault dummy_pfault = {};

		dummy_pfault.mpfault.bytes_committed = 0;
		pfault->rdma.rdma_va += pfault->bytes_committed;
		pfault->rdma.rdma_op_len -= min(pfault->bytes_committed,
		pfault->rdma.rdma_op_len);
		pfault->bytes_committed = 0;

		mpfault->rdma.rdma_va += mpfault->bytes_committed;
		mpfault->rdma.rdma_op_len -= min(mpfault->bytes_committed,
		mpfault->rdma.rdma_op_len);
		mpfault->bytes_committed = 0;

		address = mpfault->rdma.rdma_va;
		length = mpfault->rdma.rdma_op_len;
		address = pfault->rdma.rdma_va;
		length = pfault->rdma.rdma_op_len;

		/* For some operations, the hardware cannot tell the exact message
		* length, and in those cases it reports zero. Use prefetch
		* logic. */
		if (length == 0) {
		prefetch_activated = 1;
		length = mpfault->rdma.packet_size;
		length = pfault->rdma.packet_size;
		prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len);
		}

		ret = pagefault_single_data_segment(qp, pfault, rkey, address, length,
		NULL);
		ret = pagefault_single_data_segment(dev, rkey, address, length,
		&pfault->bytes_committed, NULL);
		if (ret == -EAGAIN) {
		/* We're racing with an invalidation, don't prefetch */
		prefetch_activated = 0;
		} else if (ret < 0 \|\| pages_in_range(address, length) > ret) {
		mlx5_ib_page_fault_resume(qp, pfault, 1);
		mlx5_ib_page_fault_resume(dev, pfault, 1);
		if (ret != -ENOENT)
		mlx5_ib_warn(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n",
		ret, pfault->token, pfault->type);
		return;
		}

		mlx5_ib_page_fault_resume(qp, pfault, 0);
		mlx5_ib_page_fault_resume(dev, pfault, 0);
		mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n",
		pfault->token, pfault->type,
		prefetch_activated);

		/* At this point, there might be a new pagefault already arriving in
		* the eq, switch to the dummy pagefault for the rest of the
		@@ -671,112 +678,39 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_qp *qp,
		* work-queue is being fenced. */

		if (prefetch_activated) {
		ret = pagefault_single_data_segment(qp, &dummy_pfault, rkey,
		address,
		u32 bytes_committed = 0;

		ret = pagefault_single_data_segment(dev, rkey, address,
		prefetch_len,
		NULL);
		&bytes_committed, NULL);
		if (ret < 0) {
		pr_warn("Prefetch failed (ret = %d, prefetch_activated = %d) for QPN %d, address: 0x%.16llx, length = 0x%.16x\n",
		ret, prefetch_activated,
		qp->ibqp.qp_num, address, prefetch_len);
		mlx5_ib_warn(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",
		ret, pfault->token, address,
		prefetch_len);
		}
		}
		}

		void mlx5_ib_mr_pfault_handler(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault)
		void mlx5_ib_pfault(struct mlx5_core_dev mdev, void context,
		struct mlx5_pagefault *pfault)
		{
		u8 event_subtype = pfault->mpfault.event_subtype;
		struct mlx5_ib_dev *dev = context;
		u8 event_subtype = pfault->event_subtype;

		switch (event_subtype) {
		case MLX5_PFAULT_SUBTYPE_WQE:
		mlx5_ib_mr_wqe_pfault_handler(qp, pfault);
		mlx5_ib_mr_wqe_pfault_handler(dev, pfault);
		break;
		case MLX5_PFAULT_SUBTYPE_RDMA:
		mlx5_ib_mr_rdma_pfault_handler(qp, pfault);
		mlx5_ib_mr_rdma_pfault_handler(dev, pfault);
		break;
		default:
		pr_warn("Invalid page fault event subtype: 0x%x\n",
		mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n",
		event_subtype);
		mlx5_ib_page_fault_resume(qp, pfault, 1);
		break;
		mlx5_ib_page_fault_resume(dev, pfault, 1);
		}
		}

		static void mlx5_ib_qp_pfault_action(struct work_struct *work)
		{
		struct mlx5_ib_pfault *pfault = container_of(work,
		struct mlx5_ib_pfault,
		work);
		enum mlx5_ib_pagefault_context context =
		mlx5_ib_get_pagefault_context(&pfault->mpfault);
		struct mlx5_ib_qp *qp = container_of(pfault, struct mlx5_ib_qp,
		pagefaults[context]);
		mlx5_ib_mr_pfault_handler(qp, pfault);
		}

		void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp)
		{
		unsigned long flags;

		spin_lock_irqsave(&qp->disable_page_faults_lock, flags);
		qp->disable_page_faults = 1;
		spin_unlock_irqrestore(&qp->disable_page_faults_lock, flags);

		/*
		* Note that at this point, we are guarenteed that no more
		* work queue elements will be posted to the work queue with
		* the QP we are closing.
		*/
		flush_workqueue(mlx5_ib_page_fault_wq);
		}

		void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp)
		{
		unsigned long flags;

		spin_lock_irqsave(&qp->disable_page_faults_lock, flags);
		qp->disable_page_faults = 0;
		spin_unlock_irqrestore(&qp->disable_page_faults_lock, flags);
		}

		static void mlx5_ib_pfault_handler(struct mlx5_core_qp *qp,
		struct mlx5_pagefault *pfault)
		{
		/*
		* Note that we will only get one fault event per QP per context
		* (responder/initiator, read/write), until we resolve the page fault
		* with the mlx5_ib_page_fault_resume command. Since this function is
		* called from within the work element, there is no risk of missing
		* events.
		*/
		struct mlx5_ib_qp *mibqp = to_mibqp(qp);
		enum mlx5_ib_pagefault_context context =
		mlx5_ib_get_pagefault_context(pfault);
		struct mlx5_ib_pfault *qp_pfault = &mibqp->pagefaults[context];

		qp_pfault->mpfault = *pfault;

		/* No need to stop interrupts here since we are in an interrupt */
		spin_lock(&mibqp->disable_page_faults_lock);
		if (!mibqp->disable_page_faults)
		queue_work(mlx5_ib_page_fault_wq, &qp_pfault->work);
		spin_unlock(&mibqp->disable_page_faults_lock);
		}

		void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp)
		{
		int i;

		qp->disable_page_faults = 1;
		spin_lock_init(&qp->disable_page_faults_lock);

		qp->trans_qp.base.mqp.pfault_handler = mlx5_ib_pfault_handler;

		for (i = 0; i < MLX5_IB_PAGEFAULT_CONTEXTS; ++i)
		INIT_WORK(&qp->pagefaults[i].work, mlx5_ib_qp_pfault_action);
		}

		int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev)
		{
		int ret;
		@@ -793,17 +727,3 @@ void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev)
		cleanup_srcu_struct(&ibdev->mr_srcu);
		}

		int __init mlx5_ib_odp_init(void)
		{
		mlx5_ib_page_fault_wq = alloc_ordered_workqueue("mlx5_ib_page_faults",
		WQ_MEM_RECLAIM);
		if (!mlx5_ib_page_fault_wq)
		return -ENOMEM;

		return 0;
		}

		void mlx5_ib_odp_cleanup(void)
		{
		destroy_workqueue(mlx5_ib_page_fault_wq);
		}

drivers/infiniband/hw/mlx5/qp.c

+0 −26

Original line number	Diff line number	Diff line
		@@ -1526,9 +1526,6 @@ static int create_qp_common(struct mlx5_ib_dev dev, struct ib_pd pd,
		&qp->raw_packet_qp.rq.base :
		&qp->trans_qp.base;

		if (init_attr->qp_type != IB_QPT_RAW_PACKET)
		mlx5_ib_odp_create_qp(qp);

		mutex_init(&qp->mutex);
		spin_lock_init(&qp->sq.lock);
		spin_lock_init(&qp->rq.lock);
		@@ -1923,7 +1920,6 @@ static void destroy_qp_common(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp)

		if (qp->state != IB_QPS_RESET) {
		if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET) {
		mlx5_ib_qp_disable_pagefaults(qp);
		err = mlx5_core_qp_modify(dev->mdev,
		MLX5_CMD_OP_2RST_QP, 0,
		NULL, &base->mqp);
		@@ -2823,16 +2819,6 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
		if (mlx5_st < 0)
		goto out;

		/* If moving to a reset or error state, we must disable page faults on
		* this QP and flush all current page faults. Otherwise a stale page
		* fault may attempt to work on this QP after it is reset and moved
		* again to RTS, and may cause the driver and the device to get out of
		* sync. */
		if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
		(new_state == IB_QPS_RESET \|\| new_state == IB_QPS_ERR) &&
		(qp->ibqp.qp_type != IB_QPT_RAW_PACKET))
		mlx5_ib_qp_disable_pagefaults(qp);

		if (mlx5_cur >= MLX5_QP_NUM_STATE \|\| mlx5_new >= MLX5_QP_NUM_STATE \|\|
		!optab[mlx5_cur][mlx5_new])
		goto out;
		@@ -2864,10 +2850,6 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
		if (err)
		goto out;

		if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT &&
		(qp->ibqp.qp_type != IB_QPT_RAW_PACKET))
		mlx5_ib_qp_enable_pagefaults(qp);

		qp->state = new_state;

		if (attr_mask & IB_QP_ACCESS_FLAGS)
		@@ -4533,14 +4515,6 @@ int mlx5_ib_query_qp(struct ib_qp ibqp, struct ib_qp_attr qp_attr,
		return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
		qp_init_attr);

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		/*
		* Wait for any outstanding page faults, in case the user frees memory
		* based upon this query's result.
		*/
		flush_workqueue(mlx5_ib_page_fault_wq);
		#endif

		mutex_lock(&qp->mutex);

		if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {

drivers/net/ethernet/mellanox/mlx5/core/dev.c

+33 −0

Original line number	Diff line number	Diff line
		@@ -71,6 +71,16 @@ void mlx5_add_device(struct mlx5_interface intf, struct mlx5_priv priv)
		if (dev_ctx->context) {
		spin_lock_irq(&priv->ctx_lock);
		list_add_tail(&dev_ctx->list, &priv->ctx_list);
		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		if (dev_ctx->intf->pfault) {
		if (priv->pfault) {
		mlx5_core_err(dev, "multiple page fault handlers not supported");
		} else {
		priv->pfault_ctx = dev_ctx->context;
		priv->pfault = dev_ctx->intf->pfault;
		}
		}
		#endif
		spin_unlock_irq(&priv->ctx_lock);
		} else {
		kfree(dev_ctx);
		@@ -97,6 +107,15 @@ void mlx5_remove_device(struct mlx5_interface intf, struct mlx5_priv priv)
		if (!dev_ctx)
		return;

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		spin_lock_irq(&priv->ctx_lock);
		if (priv->pfault == dev_ctx->intf->pfault)
		priv->pfault = NULL;
		spin_unlock_irq(&priv->ctx_lock);

		synchronize_srcu(&priv->pfault_srcu);
		#endif

		spin_lock_irq(&priv->ctx_lock);
		list_del(&dev_ctx->list);
		spin_unlock_irq(&priv->ctx_lock);
		@@ -329,6 +348,20 @@ void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
		spin_unlock_irqrestore(&priv->ctx_lock, flags);
		}

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		void mlx5_core_page_fault(struct mlx5_core_dev *dev,
		struct mlx5_pagefault *pfault)
		{
		struct mlx5_priv *priv = &dev->priv;
		int srcu_idx;

		srcu_idx = srcu_read_lock(&priv->pfault_srcu);
		if (priv->pfault)
		priv->pfault(dev, priv->pfault_ctx, pfault);
		srcu_read_unlock(&priv->pfault_srcu, srcu_idx);
		}
		#endif

		void mlx5_dev_list_lock(void)
		{
		mutex_lock(&mlx5_intf_mutex);