Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 45bb912b authored by Lars Ellenberg's avatar Lars Ellenberg Committed by Philipp Reisner
Browse files

drbd: Allow drbd_epoch_entries to use multiple bios. 


This should allow for better performance if the lower level IO stack
of the peers differs in limits exposed either via the queue,
or via some merge_bvec_fn.

Signed-off-by: default avatarPhilipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: default avatarLars Ellenberg <lars.ellenberg@linbit.com>
parent 708d740e

drivers/block/drbd/drbd_int.h

+74 −16
Original line number Original line Diff line number Diff line
@@ -740,18 +740,6 @@ enum epoch_event { 
	EV_CLEANUP = 32, /* used as flag */

	EV_CLEANUP = 32, /* used as flag */

};

};





struct drbd_epoch_entry {

	struct drbd_work    w;

	struct drbd_conf *mdev;

	struct bio *private_bio;

	struct hlist_node colision;

	sector_t sector;

	unsigned int size;

	unsigned int flags;

	struct drbd_epoch *epoch;

	u64    block_id;

};



struct drbd_wq_barrier {

struct drbd_wq_barrier {

	struct drbd_work w;

	struct drbd_work w;

	struct completion done;

	struct completion done;
@@ -762,17 +750,49 @@ struct digest_info { 
	void *digest;

	void *digest;

};

};





/* ee flag bits */

struct drbd_epoch_entry {

	struct drbd_work w;

	struct hlist_node colision;

	struct drbd_epoch *epoch;

	struct drbd_conf *mdev;

	struct page *pages;

	atomic_t pending_bios;

	unsigned int size;

	/* see comments on ee flag bits below */

	unsigned long flags;

	sector_t sector;

	u64 block_id;

};



/* ee flag bits.

 * While corresponding bios are in flight, the only modification will be

 * set_bit WAS_ERROR, which has to be atomic.

 * If no bios are in flight yet, or all have been completed,

 * non-atomic modification to ee->flags is ok.

 */

enum {

enum {

	__EE_CALL_AL_COMPLETE_IO,

	__EE_CALL_AL_COMPLETE_IO,

	__EE_CONFLICT_PENDING,

	__EE_MAY_SET_IN_SYNC,

	__EE_MAY_SET_IN_SYNC,



	/* This epoch entry closes an epoch using a barrier.

	 * On sucessful completion, the epoch is released,

	 * and the P_BARRIER_ACK send. */

	__EE_IS_BARRIER,

	__EE_IS_BARRIER,



	/* In case a barrier failed,

	 * we need to resubmit without the barrier flag. */

	__EE_RESUBMITTED,



	/* we may have several bios per epoch entry.

	 * if any of those fail, we set this flag atomically

	 * from the endio callback */

	__EE_WAS_ERROR,

};

};

#define EE_CALL_AL_COMPLETE_IO (1<<__EE_CALL_AL_COMPLETE_IO)

#define EE_CALL_AL_COMPLETE_IO (1<<__EE_CALL_AL_COMPLETE_IO)

#define EE_CONFLICT_PENDING    (1<<__EE_CONFLICT_PENDING)

#define EE_MAY_SET_IN_SYNC     (1<<__EE_MAY_SET_IN_SYNC)

#define EE_MAY_SET_IN_SYNC     (1<<__EE_MAY_SET_IN_SYNC)

#define EE_IS_BARRIER          (1<<__EE_IS_BARRIER)

#define EE_IS_BARRIER          (1<<__EE_IS_BARRIER)

#define	EE_RESUBMITTED         (1<<__EE_RESUBMITTED)

#define EE_WAS_ERROR           (1<<__EE_WAS_ERROR)





/* global flag bits */

/* global flag bits */

enum {

enum {
@@ -1441,7 +1461,8 @@ static inline void ov_oos_print(struct drbd_conf *mdev) 
}

}









extern void drbd_csum(struct drbd_conf *, struct crypto_hash *, struct bio *, void *);

extern void drbd_csum_bio(struct drbd_conf *, struct crypto_hash *, struct bio *, void *);

extern void drbd_csum_ee(struct drbd_conf *, struct crypto_hash *, struct drbd_epoch_entry *, void *);

/* worker callbacks */

/* worker callbacks */

extern int w_req_cancel_conflict(struct drbd_conf *, struct drbd_work *, int);

extern int w_req_cancel_conflict(struct drbd_conf *, struct drbd_work *, int);

extern int w_read_retry_remote(struct drbd_conf *, struct drbd_work *, int);

extern int w_read_retry_remote(struct drbd_conf *, struct drbd_work *, int);
@@ -1465,6 +1486,8 @@ extern int w_e_reissue(struct drbd_conf *, struct drbd_work *, int); 
extern void resync_timer_fn(unsigned long data);

extern void resync_timer_fn(unsigned long data);





/* drbd_receiver.c */

/* drbd_receiver.c */

extern int drbd_submit_ee(struct drbd_conf *mdev, struct drbd_epoch_entry *e,

		const unsigned rw, const int fault_type);

extern int drbd_release_ee(struct drbd_conf *mdev, struct list_head *list);

extern int drbd_release_ee(struct drbd_conf *mdev, struct list_head *list);

extern struct drbd_epoch_entry *drbd_alloc_ee(struct drbd_conf *mdev,

extern struct drbd_epoch_entry *drbd_alloc_ee(struct drbd_conf *mdev,

					    u64 id,

					    u64 id,
@@ -1620,6 +1643,41 @@ void drbd_bcast_ee(struct drbd_conf *mdev, 
 * inline helper functions

 * inline helper functions

 *************************/

 *************************/





/* see also page_chain_add and friends in drbd_receiver.c */

static inline struct page *page_chain_next(struct page *page)

{

	return (struct page *)page_private(page);

}

#define page_chain_for_each(page) \

	for (; page && ({ prefetch(page_chain_next(page)); 1; }); \

			page = page_chain_next(page))

#define page_chain_for_each_safe(page, n) \

	for (; page && ({ n = page_chain_next(page); 1; }); page = n)



static inline int drbd_bio_has_active_page(struct bio *bio)

{

	struct bio_vec *bvec;

	int i;



	__bio_for_each_segment(bvec, bio, i, 0) {

		if (page_count(bvec->bv_page) > 1)

			return 1;

	}



	return 0;

}



static inline int drbd_ee_has_active_page(struct drbd_epoch_entry *e)

{

	struct page *page = e->pages;

	page_chain_for_each(page) {

		if (page_count(page) > 1)

			return 1;

	}

	return 0;

}





static inline void drbd_state_lock(struct drbd_conf *mdev)

static inline void drbd_state_lock(struct drbd_conf *mdev)

{

{

	wait_event(mdev->misc_wait,

	wait_event(mdev->misc_wait,

drivers/block/drbd/drbd_main.c

+16 −3
Original line number Original line Diff line number Diff line
@@ -2354,6 +2354,19 @@ static int _drbd_send_zc_bio(struct drbd_conf *mdev, struct bio *bio) 
	return 1;

	return 1;

}

}





static int _drbd_send_zc_ee(struct drbd_conf *mdev, struct drbd_epoch_entry *e)

{

	struct page *page = e->pages;

	unsigned len = e->size;

	page_chain_for_each(page) {

		unsigned l = min_t(unsigned, len, PAGE_SIZE);

		if (!_drbd_send_page(mdev, page, 0, l))

			return 0;

		len -= l;

	}

	return 1;

}



static void consider_delay_probes(struct drbd_conf *mdev)

static void consider_delay_probes(struct drbd_conf *mdev)

{

{

	if (mdev->state.conn != C_SYNC_SOURCE || mdev->agreed_pro_version < 93)

	if (mdev->state.conn != C_SYNC_SOURCE || mdev->agreed_pro_version < 93)
@@ -2430,7 +2443,7 @@ int drbd_send_dblock(struct drbd_conf *mdev, struct drbd_request *req) 
		drbd_send(mdev, mdev->data.socket, &p, sizeof(p), MSG_MORE));

		drbd_send(mdev, mdev->data.socket, &p, sizeof(p), MSG_MORE));

	if (ok && dgs) {

	if (ok && dgs) {

		dgb = mdev->int_dig_out;

		dgb = mdev->int_dig_out;

		drbd_csum(mdev, mdev->integrity_w_tfm, req->master_bio, dgb);

		drbd_csum_bio(mdev, mdev->integrity_w_tfm, req->master_bio, dgb);

		ok = drbd_send(mdev, mdev->data.socket, dgb, dgs, MSG_MORE);

		ok = drbd_send(mdev, mdev->data.socket, dgb, dgs, MSG_MORE);

	}

	}

	if (ok) {

	if (ok) {
@@ -2483,11 +2496,11 @@ int drbd_send_block(struct drbd_conf *mdev, enum drbd_packets cmd, 
					sizeof(p), MSG_MORE);

					sizeof(p), MSG_MORE);

	if (ok && dgs) {

	if (ok && dgs) {

		dgb = mdev->int_dig_out;

		dgb = mdev->int_dig_out;

		drbd_csum(mdev, mdev->integrity_w_tfm, e->private_bio, dgb);

		drbd_csum_ee(mdev, mdev->integrity_w_tfm, e, dgb);

		ok = drbd_send(mdev, mdev->data.socket, dgb, dgs, MSG_MORE);

		ok = drbd_send(mdev, mdev->data.socket, dgb, dgs, MSG_MORE);

	}

	}

	if (ok)

	if (ok)

		ok = _drbd_send_zc_bio(mdev, e->private_bio);

		ok = _drbd_send_zc_ee(mdev, e);





	drbd_put_data_sock(mdev);

	drbd_put_data_sock(mdev);

drivers/block/drbd/drbd_nl.c

+11 −7
Original line number Original line Diff line number Diff line
@@ -2215,9 +2215,9 @@ void drbd_bcast_ee(struct drbd_conf *mdev, 
{

{

	struct cn_msg *cn_reply;

	struct cn_msg *cn_reply;

	struct drbd_nl_cfg_reply *reply;

	struct drbd_nl_cfg_reply *reply;

	struct bio_vec *bvec;

	unsigned short *tl;

	unsigned short *tl;

	int i;

	struct page *page;

	unsigned len;





	if (!e)

	if (!e)

		return;

		return;
@@ -2255,11 +2255,15 @@ void drbd_bcast_ee(struct drbd_conf *mdev, 
	put_unaligned(T_ee_data, tl++);

	put_unaligned(T_ee_data, tl++);

	put_unaligned(e->size, tl++);

	put_unaligned(e->size, tl++);





	__bio_for_each_segment(bvec, e->private_bio, i, 0) {

	len = e->size;

		void *d = kmap(bvec->bv_page);

	page = e->pages;

		memcpy(tl, d + bvec->bv_offset, bvec->bv_len);

	page_chain_for_each(page) {

		kunmap(bvec->bv_page);

		void *d = kmap_atomic(page, KM_USER0);

		tl=(unsigned short*)((char*)tl + bvec->bv_len);

		unsigned l = min_t(unsigned, len, PAGE_SIZE);

		memcpy(tl, d, l);

		kunmap_atomic(d, KM_USER0);

		tl = (unsigned short*)((char*)tl + l);

		len -= l;

	}

	}

	put_unaligned(TT_END, tl++); /* Close the tag list */

	put_unaligned(TT_END, tl++); /* Close the tag list */

drivers/block/drbd/drbd_receiver.c

+274 −209

File changed.

Preview size limit exceeded, changes collapsed.

drivers/block/drbd/drbd_worker.c

+104 −74

File changed.

Preview size limit exceeded, changes collapsed.