Loading fs/xfs/xfs_aops.c +170 −164 Original line number Diff line number Diff line Loading @@ -84,23 +84,71 @@ xfs_find_bdev_for_inode( } /* * We're now finished for good with this ioend structure. * Update the page state via the associated buffer_heads, * release holds on the inode and bio, and finally free * up memory. Do not use the ioend after this. * We're now finished for good with this page. Update the page state via the * associated buffer_heads, paying attention to the start and end offsets that * we need to process on the page. */ static void xfs_finish_page_writeback( struct inode *inode, struct bio_vec *bvec, int error) { unsigned int end = bvec->bv_offset + bvec->bv_len - 1; struct buffer_head *head, *bh; unsigned int off = 0; ASSERT(bvec->bv_offset < PAGE_SIZE); ASSERT((bvec->bv_offset & ((1 << inode->i_blkbits) - 1)) == 0); ASSERT(end < PAGE_SIZE); ASSERT((bvec->bv_len & ((1 << inode->i_blkbits) - 1)) == 0); bh = head = page_buffers(bvec->bv_page); do { if (off < bvec->bv_offset) goto next_bh; if (off > end) break; bh->b_end_io(bh, !error); next_bh: off += bh->b_size; } while ((bh = bh->b_this_page) != head); } /* * We're now finished for good with this ioend structure. Update the page * state, release holds on bios, and finally free up memory. Do not use the * ioend after this. */ STATIC void xfs_destroy_ioend( xfs_ioend_t *ioend) struct xfs_ioend *ioend, int error) { struct buffer_head *bh, *next; struct inode *inode = ioend->io_inode; struct bio *last = ioend->io_bio; struct bio *bio, *next; for (bh = ioend->io_buffer_head; bh; bh = next) { next = bh->b_private; bh->b_end_io(bh, !ioend->io_error); } for (bio = &ioend->io_inline_bio; bio; bio = next) { struct bio_vec *bvec; int i; /* * For the last bio, bi_private points to the ioend, so we * need to explicitly end the iteration here. */ if (bio == last) next = NULL; else next = bio->bi_private; mempool_free(ioend, xfs_ioend_pool); /* walk each page on bio, ending page IO on them */ bio_for_each_segment_all(bvec, bio, i) xfs_finish_page_writeback(inode, bvec, error); bio_put(bio); } } /* Loading Loading @@ -174,7 +222,8 @@ xfs_setfilesize( STATIC int xfs_setfilesize_ioend( struct xfs_ioend *ioend) struct xfs_ioend *ioend, int error) { struct xfs_inode *ip = XFS_I(ioend->io_inode); struct xfs_trans *tp = ioend->io_append_trans; Loading @@ -188,36 +237,14 @@ xfs_setfilesize_ioend( __sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS); /* we abort the update if there was an IO error */ if (ioend->io_error) { if (error) { xfs_trans_cancel(tp); return ioend->io_error; return error; } return xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size); } /* * Schedule IO completion handling on the final put of an ioend. * * If there is no work to do we might as well call it a day and free the * ioend right now. */ STATIC void xfs_finish_ioend( struct xfs_ioend *ioend) { if (atomic_dec_and_test(&ioend->io_remaining)) { struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; if (ioend->io_type == XFS_IO_UNWRITTEN) queue_work(mp->m_unwritten_workqueue, &ioend->io_work); else if (ioend->io_append_trans) queue_work(mp->m_data_workqueue, &ioend->io_work); else xfs_destroy_ioend(ioend); } } /* * IO write completion. */ Loading @@ -225,16 +252,17 @@ STATIC void xfs_end_io( struct work_struct *work) { xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work); struct xfs_ioend *ioend = container_of(work, struct xfs_ioend, io_work); struct xfs_inode *ip = XFS_I(ioend->io_inode); int error = 0; int error = ioend->io_bio->bi_error; /* * Set an error if the mount has shut down and proceed with end I/O * processing so it can perform whatever cleanups are necessary. */ if (XFS_FORCED_SHUTDOWN(ip->i_mount)) ioend->io_error = -EIO; error = -EIO; /* * For unwritten extents we need to issue transactions to convert a Loading @@ -244,55 +272,33 @@ xfs_end_io( * on error. */ if (ioend->io_type == XFS_IO_UNWRITTEN) { if (ioend->io_error) if (error) goto done; error = xfs_iomap_write_unwritten(ip, ioend->io_offset, ioend->io_size); } else if (ioend->io_append_trans) { error = xfs_setfilesize_ioend(ioend); error = xfs_setfilesize_ioend(ioend, error); } else { ASSERT(!xfs_ioend_is_append(ioend)); } done: if (error) ioend->io_error = error; xfs_destroy_ioend(ioend); xfs_destroy_ioend(ioend, error); } /* * Allocate and initialise an IO completion structure. * We need to track unwritten extent write completion here initially. * We'll need to extend this for updating the ondisk inode size later * (vs. incore size). */ STATIC xfs_ioend_t * xfs_alloc_ioend( struct inode *inode, unsigned int type) STATIC void xfs_end_bio( struct bio *bio) { xfs_ioend_t *ioend; ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS); /* * Set the count to 1 initially, which will prevent an I/O * completion callback from happening before we have started * all the I/O from calling the completion routine too early. */ atomic_set(&ioend->io_remaining, 1); ioend->io_error = 0; INIT_LIST_HEAD(&ioend->io_list); ioend->io_type = type; ioend->io_inode = inode; ioend->io_buffer_head = NULL; ioend->io_buffer_tail = NULL; ioend->io_offset = 0; ioend->io_size = 0; ioend->io_append_trans = NULL; struct xfs_ioend *ioend = bio->bi_private; struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; INIT_WORK(&ioend->io_work, xfs_end_io); return ioend; if (ioend->io_type == XFS_IO_UNWRITTEN) queue_work(mp->m_unwritten_workqueue, &ioend->io_work); else if (ioend->io_append_trans) queue_work(mp->m_data_workqueue, &ioend->io_work); else xfs_destroy_ioend(ioend, bio->bi_error); } STATIC int Loading Loading @@ -364,50 +370,6 @@ xfs_imap_valid( offset < imap->br_startoff + imap->br_blockcount; } /* * BIO completion handler for buffered IO. */ STATIC void xfs_end_bio( struct bio *bio) { xfs_ioend_t *ioend = bio->bi_private; if (!ioend->io_error) ioend->io_error = bio->bi_error; /* Toss bio and pass work off to an xfsdatad thread */ bio->bi_private = NULL; bio->bi_end_io = NULL; bio_put(bio); xfs_finish_ioend(ioend); } STATIC void xfs_submit_ioend_bio( struct writeback_control *wbc, xfs_ioend_t *ioend, struct bio *bio) { atomic_inc(&ioend->io_remaining); bio->bi_private = ioend; bio->bi_end_io = xfs_end_bio; submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, bio); } STATIC struct bio * xfs_alloc_ioend_bio( struct buffer_head *bh) { struct bio *bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); ASSERT(bio->bi_private == NULL); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio->bi_bdev = bh->b_bdev; return bio; } STATIC void xfs_start_buffer_writeback( struct buffer_head *bh) Loading Loading @@ -452,28 +414,35 @@ static inline int xfs_bio_add_buffer(struct bio *bio, struct buffer_head *bh) } /* * Submit all of the bios for an ioend. We are only passed a single ioend at a * time; the caller is responsible for chaining prior to submission. * Submit the bio for an ioend. We are passed an ioend with a bio attached to * it, and we submit that bio. The ioend may be used for multiple bio * submissions, so we only want to allocate an append transaction for the ioend * once. In the case of multiple bio submission, each bio will take an IO * reference to the ioend to ensure that the ioend completion is only done once * all bios have been submitted and the ioend is really done. * * If @fail is non-zero, it means that we have a situation where some part of * the submission process has failed after we have marked paged for writeback * and unlocked them. In this situation, we need to fail the ioend chain rather * than submit it to IO. This typically only happens on a filesystem shutdown. * and unlocked them. In this situation, we need to fail the bio and ioend * rather than submit it to IO. This typically only happens on a filesystem * shutdown. */ STATIC int xfs_submit_ioend( struct writeback_control *wbc, xfs_ioend_t *ioend, struct xfs_ioend *ioend, int status) { struct buffer_head *bh; struct bio *bio; sector_t lastblock = 0; /* Reserve log space if we might write beyond the on-disk inode size. */ if (!status && ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend)) ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend) && !ioend->io_append_trans) status = xfs_setfilesize_trans_alloc(ioend); ioend->io_bio->bi_private = ioend; ioend->io_bio->bi_end_io = xfs_end_bio; /* * If we are failing the IO now, just mark the ioend with an * error and finish it. This will run IO completion immediately Loading @@ -481,33 +450,73 @@ xfs_submit_ioend( * time. */ if (status) { ioend->io_error = status; xfs_finish_ioend(ioend); ioend->io_bio->bi_error = status; bio_endio(ioend->io_bio); return status; } bio = NULL; for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) { if (!bio) { retry: bio = xfs_alloc_ioend_bio(bh); } else if (bh->b_blocknr != lastblock + 1) { xfs_submit_ioend_bio(wbc, ioend, bio); goto retry; submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, ioend->io_bio); return 0; } if (xfs_bio_add_buffer(bio, bh) != bh->b_size) { xfs_submit_ioend_bio(wbc, ioend, bio); goto retry; static void xfs_init_bio_from_bh( struct bio *bio, struct buffer_head *bh) { bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio->bi_bdev = bh->b_bdev; } lastblock = bh->b_blocknr; static struct xfs_ioend * xfs_alloc_ioend( struct inode *inode, unsigned int type, xfs_off_t offset, struct buffer_head *bh) { struct xfs_ioend *ioend; struct bio *bio; bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, xfs_ioend_bioset); xfs_init_bio_from_bh(bio, bh); ioend = container_of(bio, struct xfs_ioend, io_inline_bio); INIT_LIST_HEAD(&ioend->io_list); ioend->io_type = type; ioend->io_inode = inode; ioend->io_size = 0; ioend->io_offset = offset; INIT_WORK(&ioend->io_work, xfs_end_io); ioend->io_append_trans = NULL; ioend->io_bio = bio; return ioend; } if (bio) xfs_submit_ioend_bio(wbc, ioend, bio); xfs_finish_ioend(ioend); return 0; /* * Allocate a new bio, and chain the old bio to the new one. * * Note that we have to do perform the chaining in this unintuitive order * so that the bi_private linkage is set up in the right direction for the * traversal in xfs_destroy_ioend(). */ static void xfs_chain_bio( struct xfs_ioend *ioend, struct writeback_control *wbc, struct buffer_head *bh) { struct bio *new; new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES); xfs_init_bio_from_bh(new, bh); bio_chain(ioend->io_bio, new); bio_get(ioend->io_bio); /* for xfs_destroy_ioend */ submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, ioend->io_bio); ioend->io_bio = new; } /* Loading @@ -523,27 +532,24 @@ xfs_add_to_ioend( struct buffer_head *bh, xfs_off_t offset, struct xfs_writepage_ctx *wpc, struct writeback_control *wbc, struct list_head *iolist) { if (!wpc->ioend || wpc->io_type != wpc->ioend->io_type || bh->b_blocknr != wpc->last_block + 1 || offset != wpc->ioend->io_offset + wpc->ioend->io_size) { struct xfs_ioend *new; if (wpc->ioend) list_add(&wpc->ioend->io_list, iolist); new = xfs_alloc_ioend(inode, wpc->io_type); new->io_offset = offset; new->io_buffer_head = bh; new->io_buffer_tail = bh; wpc->ioend = new; } else { wpc->ioend->io_buffer_tail->b_private = bh; wpc->ioend->io_buffer_tail = bh; wpc->ioend = xfs_alloc_ioend(inode, wpc->io_type, offset, bh); } bh->b_private = NULL; /* * If the buffer doesn't fit into the bio we need to allocate a new * one. This shouldn't happen more than once for a given buffer. */ while (xfs_bio_add_buffer(wpc->ioend->io_bio, bh) != bh->b_size) xfs_chain_bio(wpc->ioend, wbc, bh); wpc->ioend->io_size += bh->b_size; wpc->last_block = bh->b_blocknr; xfs_start_buffer_writeback(bh); Loading Loading @@ -803,7 +809,7 @@ xfs_writepage_map( lock_buffer(bh); if (wpc->io_type != XFS_IO_OVERWRITE) xfs_map_at_offset(inode, bh, &wpc->imap, offset); xfs_add_to_ioend(inode, bh, offset, wpc, &submit_list); xfs_add_to_ioend(inode, bh, offset, wpc, wbc, &submit_list); count++; } Loading fs/xfs/xfs_aops.h +6 −9 Original line number Diff line number Diff line Loading @@ -18,7 +18,7 @@ #ifndef __XFS_AOPS_H__ #define __XFS_AOPS_H__ extern mempool_t *xfs_ioend_pool; extern struct bio_set *xfs_ioend_bioset; /* * Types of I/O for bmap clustering and I/O completion tracking. Loading @@ -37,22 +37,19 @@ enum { { XFS_IO_OVERWRITE, "overwrite" } /* * xfs_ioend struct manages large extent writes for XFS. * It can manage several multi-page bio's at once. * Structure for buffered I/O completions. */ typedef struct xfs_ioend { struct xfs_ioend { struct list_head io_list; /* next ioend in chain */ unsigned int io_type; /* delalloc / unwritten */ int io_error; /* I/O error code */ atomic_t io_remaining; /* hold count */ struct inode *io_inode; /* file being written to */ struct buffer_head *io_buffer_head;/* buffer linked list head */ struct buffer_head *io_buffer_tail;/* buffer linked list tail */ size_t io_size; /* size of the extent */ xfs_off_t io_offset; /* offset in the file */ struct work_struct io_work; /* xfsdatad work queue */ struct xfs_trans *io_append_trans;/* xact. for size update */ } xfs_ioend_t; struct bio *io_bio; /* bio being built */ struct bio io_inline_bio; /* MUST BE LAST! */ }; extern const struct address_space_operations xfs_address_space_operations; Loading fs/xfs/xfs_super.c +8 −18 Original line number Diff line number Diff line Loading @@ -58,8 +58,7 @@ #include <linux/parser.h> static const struct super_operations xfs_super_operations; static kmem_zone_t *xfs_ioend_zone; mempool_t *xfs_ioend_pool; struct bio_set *xfs_ioend_bioset; static struct kset *xfs_kset; /* top-level xfs sysfs dir */ #ifdef DEBUG Loading Loading @@ -1699,20 +1698,15 @@ MODULE_ALIAS_FS("xfs"); STATIC int __init xfs_init_zones(void) { xfs_ioend_zone = kmem_zone_init(sizeof(xfs_ioend_t), "xfs_ioend"); if (!xfs_ioend_zone) xfs_ioend_bioset = bioset_create(4 * MAX_BUF_PER_PAGE, offsetof(struct xfs_ioend, io_inline_bio)); if (!xfs_ioend_bioset) goto out; xfs_ioend_pool = mempool_create_slab_pool(4 * MAX_BUF_PER_PAGE, xfs_ioend_zone); if (!xfs_ioend_pool) goto out_destroy_ioend_zone; xfs_log_ticket_zone = kmem_zone_init(sizeof(xlog_ticket_t), "xfs_log_ticket"); if (!xfs_log_ticket_zone) goto out_destroy_ioend_pool; goto out_free_ioend_bioset; xfs_bmap_free_item_zone = kmem_zone_init(sizeof(xfs_bmap_free_item_t), "xfs_bmap_free_item"); Loading Loading @@ -1808,10 +1802,8 @@ xfs_init_zones(void) kmem_zone_destroy(xfs_bmap_free_item_zone); out_destroy_log_ticket_zone: kmem_zone_destroy(xfs_log_ticket_zone); out_destroy_ioend_pool: mempool_destroy(xfs_ioend_pool); out_destroy_ioend_zone: kmem_zone_destroy(xfs_ioend_zone); out_free_ioend_bioset: bioset_free(xfs_ioend_bioset); out: return -ENOMEM; } Loading @@ -1837,9 +1829,7 @@ xfs_destroy_zones(void) kmem_zone_destroy(xfs_btree_cur_zone); kmem_zone_destroy(xfs_bmap_free_item_zone); kmem_zone_destroy(xfs_log_ticket_zone); mempool_destroy(xfs_ioend_pool); kmem_zone_destroy(xfs_ioend_zone); bioset_free(xfs_ioend_bioset); } STATIC int __init Loading Loading
fs/xfs/xfs_aops.c +170 −164 Original line number Diff line number Diff line Loading @@ -84,23 +84,71 @@ xfs_find_bdev_for_inode( } /* * We're now finished for good with this ioend structure. * Update the page state via the associated buffer_heads, * release holds on the inode and bio, and finally free * up memory. Do not use the ioend after this. * We're now finished for good with this page. Update the page state via the * associated buffer_heads, paying attention to the start and end offsets that * we need to process on the page. */ static void xfs_finish_page_writeback( struct inode *inode, struct bio_vec *bvec, int error) { unsigned int end = bvec->bv_offset + bvec->bv_len - 1; struct buffer_head *head, *bh; unsigned int off = 0; ASSERT(bvec->bv_offset < PAGE_SIZE); ASSERT((bvec->bv_offset & ((1 << inode->i_blkbits) - 1)) == 0); ASSERT(end < PAGE_SIZE); ASSERT((bvec->bv_len & ((1 << inode->i_blkbits) - 1)) == 0); bh = head = page_buffers(bvec->bv_page); do { if (off < bvec->bv_offset) goto next_bh; if (off > end) break; bh->b_end_io(bh, !error); next_bh: off += bh->b_size; } while ((bh = bh->b_this_page) != head); } /* * We're now finished for good with this ioend structure. Update the page * state, release holds on bios, and finally free up memory. Do not use the * ioend after this. */ STATIC void xfs_destroy_ioend( xfs_ioend_t *ioend) struct xfs_ioend *ioend, int error) { struct buffer_head *bh, *next; struct inode *inode = ioend->io_inode; struct bio *last = ioend->io_bio; struct bio *bio, *next; for (bh = ioend->io_buffer_head; bh; bh = next) { next = bh->b_private; bh->b_end_io(bh, !ioend->io_error); } for (bio = &ioend->io_inline_bio; bio; bio = next) { struct bio_vec *bvec; int i; /* * For the last bio, bi_private points to the ioend, so we * need to explicitly end the iteration here. */ if (bio == last) next = NULL; else next = bio->bi_private; mempool_free(ioend, xfs_ioend_pool); /* walk each page on bio, ending page IO on them */ bio_for_each_segment_all(bvec, bio, i) xfs_finish_page_writeback(inode, bvec, error); bio_put(bio); } } /* Loading Loading @@ -174,7 +222,8 @@ xfs_setfilesize( STATIC int xfs_setfilesize_ioend( struct xfs_ioend *ioend) struct xfs_ioend *ioend, int error) { struct xfs_inode *ip = XFS_I(ioend->io_inode); struct xfs_trans *tp = ioend->io_append_trans; Loading @@ -188,36 +237,14 @@ xfs_setfilesize_ioend( __sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS); /* we abort the update if there was an IO error */ if (ioend->io_error) { if (error) { xfs_trans_cancel(tp); return ioend->io_error; return error; } return xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size); } /* * Schedule IO completion handling on the final put of an ioend. * * If there is no work to do we might as well call it a day and free the * ioend right now. */ STATIC void xfs_finish_ioend( struct xfs_ioend *ioend) { if (atomic_dec_and_test(&ioend->io_remaining)) { struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; if (ioend->io_type == XFS_IO_UNWRITTEN) queue_work(mp->m_unwritten_workqueue, &ioend->io_work); else if (ioend->io_append_trans) queue_work(mp->m_data_workqueue, &ioend->io_work); else xfs_destroy_ioend(ioend); } } /* * IO write completion. */ Loading @@ -225,16 +252,17 @@ STATIC void xfs_end_io( struct work_struct *work) { xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work); struct xfs_ioend *ioend = container_of(work, struct xfs_ioend, io_work); struct xfs_inode *ip = XFS_I(ioend->io_inode); int error = 0; int error = ioend->io_bio->bi_error; /* * Set an error if the mount has shut down and proceed with end I/O * processing so it can perform whatever cleanups are necessary. */ if (XFS_FORCED_SHUTDOWN(ip->i_mount)) ioend->io_error = -EIO; error = -EIO; /* * For unwritten extents we need to issue transactions to convert a Loading @@ -244,55 +272,33 @@ xfs_end_io( * on error. */ if (ioend->io_type == XFS_IO_UNWRITTEN) { if (ioend->io_error) if (error) goto done; error = xfs_iomap_write_unwritten(ip, ioend->io_offset, ioend->io_size); } else if (ioend->io_append_trans) { error = xfs_setfilesize_ioend(ioend); error = xfs_setfilesize_ioend(ioend, error); } else { ASSERT(!xfs_ioend_is_append(ioend)); } done: if (error) ioend->io_error = error; xfs_destroy_ioend(ioend); xfs_destroy_ioend(ioend, error); } /* * Allocate and initialise an IO completion structure. * We need to track unwritten extent write completion here initially. * We'll need to extend this for updating the ondisk inode size later * (vs. incore size). */ STATIC xfs_ioend_t * xfs_alloc_ioend( struct inode *inode, unsigned int type) STATIC void xfs_end_bio( struct bio *bio) { xfs_ioend_t *ioend; ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS); /* * Set the count to 1 initially, which will prevent an I/O * completion callback from happening before we have started * all the I/O from calling the completion routine too early. */ atomic_set(&ioend->io_remaining, 1); ioend->io_error = 0; INIT_LIST_HEAD(&ioend->io_list); ioend->io_type = type; ioend->io_inode = inode; ioend->io_buffer_head = NULL; ioend->io_buffer_tail = NULL; ioend->io_offset = 0; ioend->io_size = 0; ioend->io_append_trans = NULL; struct xfs_ioend *ioend = bio->bi_private; struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; INIT_WORK(&ioend->io_work, xfs_end_io); return ioend; if (ioend->io_type == XFS_IO_UNWRITTEN) queue_work(mp->m_unwritten_workqueue, &ioend->io_work); else if (ioend->io_append_trans) queue_work(mp->m_data_workqueue, &ioend->io_work); else xfs_destroy_ioend(ioend, bio->bi_error); } STATIC int Loading Loading @@ -364,50 +370,6 @@ xfs_imap_valid( offset < imap->br_startoff + imap->br_blockcount; } /* * BIO completion handler for buffered IO. */ STATIC void xfs_end_bio( struct bio *bio) { xfs_ioend_t *ioend = bio->bi_private; if (!ioend->io_error) ioend->io_error = bio->bi_error; /* Toss bio and pass work off to an xfsdatad thread */ bio->bi_private = NULL; bio->bi_end_io = NULL; bio_put(bio); xfs_finish_ioend(ioend); } STATIC void xfs_submit_ioend_bio( struct writeback_control *wbc, xfs_ioend_t *ioend, struct bio *bio) { atomic_inc(&ioend->io_remaining); bio->bi_private = ioend; bio->bi_end_io = xfs_end_bio; submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, bio); } STATIC struct bio * xfs_alloc_ioend_bio( struct buffer_head *bh) { struct bio *bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); ASSERT(bio->bi_private == NULL); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio->bi_bdev = bh->b_bdev; return bio; } STATIC void xfs_start_buffer_writeback( struct buffer_head *bh) Loading Loading @@ -452,28 +414,35 @@ static inline int xfs_bio_add_buffer(struct bio *bio, struct buffer_head *bh) } /* * Submit all of the bios for an ioend. We are only passed a single ioend at a * time; the caller is responsible for chaining prior to submission. * Submit the bio for an ioend. We are passed an ioend with a bio attached to * it, and we submit that bio. The ioend may be used for multiple bio * submissions, so we only want to allocate an append transaction for the ioend * once. In the case of multiple bio submission, each bio will take an IO * reference to the ioend to ensure that the ioend completion is only done once * all bios have been submitted and the ioend is really done. * * If @fail is non-zero, it means that we have a situation where some part of * the submission process has failed after we have marked paged for writeback * and unlocked them. In this situation, we need to fail the ioend chain rather * than submit it to IO. This typically only happens on a filesystem shutdown. * and unlocked them. In this situation, we need to fail the bio and ioend * rather than submit it to IO. This typically only happens on a filesystem * shutdown. */ STATIC int xfs_submit_ioend( struct writeback_control *wbc, xfs_ioend_t *ioend, struct xfs_ioend *ioend, int status) { struct buffer_head *bh; struct bio *bio; sector_t lastblock = 0; /* Reserve log space if we might write beyond the on-disk inode size. */ if (!status && ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend)) ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend) && !ioend->io_append_trans) status = xfs_setfilesize_trans_alloc(ioend); ioend->io_bio->bi_private = ioend; ioend->io_bio->bi_end_io = xfs_end_bio; /* * If we are failing the IO now, just mark the ioend with an * error and finish it. This will run IO completion immediately Loading @@ -481,33 +450,73 @@ xfs_submit_ioend( * time. */ if (status) { ioend->io_error = status; xfs_finish_ioend(ioend); ioend->io_bio->bi_error = status; bio_endio(ioend->io_bio); return status; } bio = NULL; for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) { if (!bio) { retry: bio = xfs_alloc_ioend_bio(bh); } else if (bh->b_blocknr != lastblock + 1) { xfs_submit_ioend_bio(wbc, ioend, bio); goto retry; submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, ioend->io_bio); return 0; } if (xfs_bio_add_buffer(bio, bh) != bh->b_size) { xfs_submit_ioend_bio(wbc, ioend, bio); goto retry; static void xfs_init_bio_from_bh( struct bio *bio, struct buffer_head *bh) { bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio->bi_bdev = bh->b_bdev; } lastblock = bh->b_blocknr; static struct xfs_ioend * xfs_alloc_ioend( struct inode *inode, unsigned int type, xfs_off_t offset, struct buffer_head *bh) { struct xfs_ioend *ioend; struct bio *bio; bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, xfs_ioend_bioset); xfs_init_bio_from_bh(bio, bh); ioend = container_of(bio, struct xfs_ioend, io_inline_bio); INIT_LIST_HEAD(&ioend->io_list); ioend->io_type = type; ioend->io_inode = inode; ioend->io_size = 0; ioend->io_offset = offset; INIT_WORK(&ioend->io_work, xfs_end_io); ioend->io_append_trans = NULL; ioend->io_bio = bio; return ioend; } if (bio) xfs_submit_ioend_bio(wbc, ioend, bio); xfs_finish_ioend(ioend); return 0; /* * Allocate a new bio, and chain the old bio to the new one. * * Note that we have to do perform the chaining in this unintuitive order * so that the bi_private linkage is set up in the right direction for the * traversal in xfs_destroy_ioend(). */ static void xfs_chain_bio( struct xfs_ioend *ioend, struct writeback_control *wbc, struct buffer_head *bh) { struct bio *new; new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES); xfs_init_bio_from_bh(new, bh); bio_chain(ioend->io_bio, new); bio_get(ioend->io_bio); /* for xfs_destroy_ioend */ submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, ioend->io_bio); ioend->io_bio = new; } /* Loading @@ -523,27 +532,24 @@ xfs_add_to_ioend( struct buffer_head *bh, xfs_off_t offset, struct xfs_writepage_ctx *wpc, struct writeback_control *wbc, struct list_head *iolist) { if (!wpc->ioend || wpc->io_type != wpc->ioend->io_type || bh->b_blocknr != wpc->last_block + 1 || offset != wpc->ioend->io_offset + wpc->ioend->io_size) { struct xfs_ioend *new; if (wpc->ioend) list_add(&wpc->ioend->io_list, iolist); new = xfs_alloc_ioend(inode, wpc->io_type); new->io_offset = offset; new->io_buffer_head = bh; new->io_buffer_tail = bh; wpc->ioend = new; } else { wpc->ioend->io_buffer_tail->b_private = bh; wpc->ioend->io_buffer_tail = bh; wpc->ioend = xfs_alloc_ioend(inode, wpc->io_type, offset, bh); } bh->b_private = NULL; /* * If the buffer doesn't fit into the bio we need to allocate a new * one. This shouldn't happen more than once for a given buffer. */ while (xfs_bio_add_buffer(wpc->ioend->io_bio, bh) != bh->b_size) xfs_chain_bio(wpc->ioend, wbc, bh); wpc->ioend->io_size += bh->b_size; wpc->last_block = bh->b_blocknr; xfs_start_buffer_writeback(bh); Loading Loading @@ -803,7 +809,7 @@ xfs_writepage_map( lock_buffer(bh); if (wpc->io_type != XFS_IO_OVERWRITE) xfs_map_at_offset(inode, bh, &wpc->imap, offset); xfs_add_to_ioend(inode, bh, offset, wpc, &submit_list); xfs_add_to_ioend(inode, bh, offset, wpc, wbc, &submit_list); count++; } Loading
fs/xfs/xfs_aops.h +6 −9 Original line number Diff line number Diff line Loading @@ -18,7 +18,7 @@ #ifndef __XFS_AOPS_H__ #define __XFS_AOPS_H__ extern mempool_t *xfs_ioend_pool; extern struct bio_set *xfs_ioend_bioset; /* * Types of I/O for bmap clustering and I/O completion tracking. Loading @@ -37,22 +37,19 @@ enum { { XFS_IO_OVERWRITE, "overwrite" } /* * xfs_ioend struct manages large extent writes for XFS. * It can manage several multi-page bio's at once. * Structure for buffered I/O completions. */ typedef struct xfs_ioend { struct xfs_ioend { struct list_head io_list; /* next ioend in chain */ unsigned int io_type; /* delalloc / unwritten */ int io_error; /* I/O error code */ atomic_t io_remaining; /* hold count */ struct inode *io_inode; /* file being written to */ struct buffer_head *io_buffer_head;/* buffer linked list head */ struct buffer_head *io_buffer_tail;/* buffer linked list tail */ size_t io_size; /* size of the extent */ xfs_off_t io_offset; /* offset in the file */ struct work_struct io_work; /* xfsdatad work queue */ struct xfs_trans *io_append_trans;/* xact. for size update */ } xfs_ioend_t; struct bio *io_bio; /* bio being built */ struct bio io_inline_bio; /* MUST BE LAST! */ }; extern const struct address_space_operations xfs_address_space_operations; Loading
fs/xfs/xfs_super.c +8 −18 Original line number Diff line number Diff line Loading @@ -58,8 +58,7 @@ #include <linux/parser.h> static const struct super_operations xfs_super_operations; static kmem_zone_t *xfs_ioend_zone; mempool_t *xfs_ioend_pool; struct bio_set *xfs_ioend_bioset; static struct kset *xfs_kset; /* top-level xfs sysfs dir */ #ifdef DEBUG Loading Loading @@ -1699,20 +1698,15 @@ MODULE_ALIAS_FS("xfs"); STATIC int __init xfs_init_zones(void) { xfs_ioend_zone = kmem_zone_init(sizeof(xfs_ioend_t), "xfs_ioend"); if (!xfs_ioend_zone) xfs_ioend_bioset = bioset_create(4 * MAX_BUF_PER_PAGE, offsetof(struct xfs_ioend, io_inline_bio)); if (!xfs_ioend_bioset) goto out; xfs_ioend_pool = mempool_create_slab_pool(4 * MAX_BUF_PER_PAGE, xfs_ioend_zone); if (!xfs_ioend_pool) goto out_destroy_ioend_zone; xfs_log_ticket_zone = kmem_zone_init(sizeof(xlog_ticket_t), "xfs_log_ticket"); if (!xfs_log_ticket_zone) goto out_destroy_ioend_pool; goto out_free_ioend_bioset; xfs_bmap_free_item_zone = kmem_zone_init(sizeof(xfs_bmap_free_item_t), "xfs_bmap_free_item"); Loading Loading @@ -1808,10 +1802,8 @@ xfs_init_zones(void) kmem_zone_destroy(xfs_bmap_free_item_zone); out_destroy_log_ticket_zone: kmem_zone_destroy(xfs_log_ticket_zone); out_destroy_ioend_pool: mempool_destroy(xfs_ioend_pool); out_destroy_ioend_zone: kmem_zone_destroy(xfs_ioend_zone); out_free_ioend_bioset: bioset_free(xfs_ioend_bioset); out: return -ENOMEM; } Loading @@ -1837,9 +1829,7 @@ xfs_destroy_zones(void) kmem_zone_destroy(xfs_btree_cur_zone); kmem_zone_destroy(xfs_bmap_free_item_zone); kmem_zone_destroy(xfs_log_ticket_zone); mempool_destroy(xfs_ioend_pool); kmem_zone_destroy(xfs_ioend_zone); bioset_free(xfs_ioend_bioset); } STATIC int __init Loading
