2 * Copyright (c) 2014 Red Hat, Inc.
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include "xfs_shared.h"
21 #include "xfs_format.h"
22 #include "xfs_log_format.h"
23 #include "xfs_trans_resv.h"
26 #include "xfs_mount.h"
27 #include "xfs_defer.h"
28 #include "xfs_da_format.h"
29 #include "xfs_da_btree.h"
30 #include "xfs_btree.h"
31 #include "xfs_trans.h"
32 #include "xfs_alloc.h"
34 #include "xfs_rmap_btree.h"
35 #include "xfs_trans_space.h"
36 #include "xfs_trace.h"
37 #include "xfs_error.h"
38 #include "xfs_extent_busy.h"
40 #include "xfs_inode.h"
43 * Lookup the first record less than or equal to [bno, len, owner, offset]
44 * in the btree given by cur.
48 struct xfs_btree_cur *cur,
56 cur->bc_rec.r.rm_startblock = bno;
57 cur->bc_rec.r.rm_blockcount = len;
58 cur->bc_rec.r.rm_owner = owner;
59 cur->bc_rec.r.rm_offset = offset;
60 cur->bc_rec.r.rm_flags = flags;
61 return xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat);
65 * Lookup the record exactly matching [bno, len, owner, offset]
66 * in the btree given by cur.
70 struct xfs_btree_cur *cur,
78 cur->bc_rec.r.rm_startblock = bno;
79 cur->bc_rec.r.rm_blockcount = len;
80 cur->bc_rec.r.rm_owner = owner;
81 cur->bc_rec.r.rm_offset = offset;
82 cur->bc_rec.r.rm_flags = flags;
83 return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat);
87 * Update the record referred to by cur to the value given
88 * by [bno, len, owner, offset].
89 * This either works (return 0) or gets an EFSCORRUPTED error.
93 struct xfs_btree_cur *cur,
94 struct xfs_rmap_irec *irec)
96 union xfs_btree_rec rec;
99 trace_xfs_rmap_update(cur->bc_mp, cur->bc_private.a.agno,
100 irec->rm_startblock, irec->rm_blockcount,
101 irec->rm_owner, irec->rm_offset, irec->rm_flags);
103 rec.rmap.rm_startblock = cpu_to_be32(irec->rm_startblock);
104 rec.rmap.rm_blockcount = cpu_to_be32(irec->rm_blockcount);
105 rec.rmap.rm_owner = cpu_to_be64(irec->rm_owner);
106 rec.rmap.rm_offset = cpu_to_be64(
107 xfs_rmap_irec_offset_pack(irec));
108 error = xfs_btree_update(cur, &rec);
110 trace_xfs_rmap_update_error(cur->bc_mp,
111 cur->bc_private.a.agno, error, _RET_IP_);
117 struct xfs_btree_cur *rcur,
127 trace_xfs_rmap_insert(rcur->bc_mp, rcur->bc_private.a.agno, agbno,
128 len, owner, offset, flags);
130 error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
133 XFS_WANT_CORRUPTED_GOTO(rcur->bc_mp, i == 0, done);
135 rcur->bc_rec.r.rm_startblock = agbno;
136 rcur->bc_rec.r.rm_blockcount = len;
137 rcur->bc_rec.r.rm_owner = owner;
138 rcur->bc_rec.r.rm_offset = offset;
139 rcur->bc_rec.r.rm_flags = flags;
140 error = xfs_btree_insert(rcur, &i);
143 XFS_WANT_CORRUPTED_GOTO(rcur->bc_mp, i == 1, done);
146 trace_xfs_rmap_insert_error(rcur->bc_mp,
147 rcur->bc_private.a.agno, error, _RET_IP_);
153 struct xfs_btree_cur *rcur,
163 trace_xfs_rmap_delete(rcur->bc_mp, rcur->bc_private.a.agno, agbno,
164 len, owner, offset, flags);
166 error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
169 XFS_WANT_CORRUPTED_GOTO(rcur->bc_mp, i == 1, done);
171 error = xfs_btree_delete(rcur, &i);
174 XFS_WANT_CORRUPTED_GOTO(rcur->bc_mp, i == 1, done);
177 trace_xfs_rmap_delete_error(rcur->bc_mp,
178 rcur->bc_private.a.agno, error, _RET_IP_);
183 xfs_rmap_btrec_to_irec(
184 union xfs_btree_rec *rec,
185 struct xfs_rmap_irec *irec)
188 irec->rm_startblock = be32_to_cpu(rec->rmap.rm_startblock);
189 irec->rm_blockcount = be32_to_cpu(rec->rmap.rm_blockcount);
190 irec->rm_owner = be64_to_cpu(rec->rmap.rm_owner);
191 return xfs_rmap_irec_offset_unpack(be64_to_cpu(rec->rmap.rm_offset),
196 * Get the data from the pointed-to record.
200 struct xfs_btree_cur *cur,
201 struct xfs_rmap_irec *irec,
204 union xfs_btree_rec *rec;
207 error = xfs_btree_get_rec(cur, &rec, stat);
211 return xfs_rmap_btrec_to_irec(rec, irec);
214 struct xfs_find_left_neighbor_info {
215 struct xfs_rmap_irec high;
216 struct xfs_rmap_irec *irec;
220 /* For each rmap given, figure out if it matches the key we want. */
222 xfs_rmap_find_left_neighbor_helper(
223 struct xfs_btree_cur *cur,
224 struct xfs_rmap_irec *rec,
227 struct xfs_find_left_neighbor_info *info = priv;
229 trace_xfs_rmap_find_left_neighbor_candidate(cur->bc_mp,
230 cur->bc_private.a.agno, rec->rm_startblock,
231 rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
234 if (rec->rm_owner != info->high.rm_owner)
235 return XFS_BTREE_QUERY_RANGE_CONTINUE;
236 if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
237 !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
238 rec->rm_offset + rec->rm_blockcount - 1 != info->high.rm_offset)
239 return XFS_BTREE_QUERY_RANGE_CONTINUE;
243 return XFS_BTREE_QUERY_RANGE_ABORT;
247 * Find the record to the left of the given extent, being careful only to
248 * return a match with the same owner and adjacent physical and logical
252 xfs_rmap_find_left_neighbor(
253 struct xfs_btree_cur *cur,
258 struct xfs_rmap_irec *irec,
261 struct xfs_find_left_neighbor_info info;
267 info.high.rm_startblock = bno - 1;
268 info.high.rm_owner = owner;
269 if (!XFS_RMAP_NON_INODE_OWNER(owner) &&
270 !(flags & XFS_RMAP_BMBT_BLOCK)) {
273 info.high.rm_offset = offset - 1;
275 info.high.rm_offset = 0;
276 info.high.rm_flags = flags;
277 info.high.rm_blockcount = 0;
281 trace_xfs_rmap_find_left_neighbor_query(cur->bc_mp,
282 cur->bc_private.a.agno, bno, 0, owner, offset, flags);
284 error = xfs_rmap_query_range(cur, &info.high, &info.high,
285 xfs_rmap_find_left_neighbor_helper, &info);
286 if (error == XFS_BTREE_QUERY_RANGE_ABORT)
289 trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
290 cur->bc_private.a.agno, irec->rm_startblock,
291 irec->rm_blockcount, irec->rm_owner,
292 irec->rm_offset, irec->rm_flags);
296 /* For each rmap given, figure out if it matches the key we want. */
298 xfs_rmap_lookup_le_range_helper(
299 struct xfs_btree_cur *cur,
300 struct xfs_rmap_irec *rec,
303 struct xfs_find_left_neighbor_info *info = priv;
305 trace_xfs_rmap_lookup_le_range_candidate(cur->bc_mp,
306 cur->bc_private.a.agno, rec->rm_startblock,
307 rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
310 if (rec->rm_owner != info->high.rm_owner)
311 return XFS_BTREE_QUERY_RANGE_CONTINUE;
312 if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
313 !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
314 (rec->rm_offset > info->high.rm_offset ||
315 rec->rm_offset + rec->rm_blockcount <= info->high.rm_offset))
316 return XFS_BTREE_QUERY_RANGE_CONTINUE;
320 return XFS_BTREE_QUERY_RANGE_ABORT;
324 * Find the record to the left of the given extent, being careful only to
325 * return a match with the same owner and overlapping physical and logical
326 * block ranges. This is the overlapping-interval version of
327 * xfs_rmap_lookup_le.
330 xfs_rmap_lookup_le_range(
331 struct xfs_btree_cur *cur,
336 struct xfs_rmap_irec *irec,
339 struct xfs_find_left_neighbor_info info;
342 info.high.rm_startblock = bno;
343 info.high.rm_owner = owner;
344 if (!XFS_RMAP_NON_INODE_OWNER(owner) && !(flags & XFS_RMAP_BMBT_BLOCK))
345 info.high.rm_offset = offset;
347 info.high.rm_offset = 0;
348 info.high.rm_flags = flags;
349 info.high.rm_blockcount = 0;
354 trace_xfs_rmap_lookup_le_range(cur->bc_mp,
355 cur->bc_private.a.agno, bno, 0, owner, offset, flags);
356 error = xfs_rmap_query_range(cur, &info.high, &info.high,
357 xfs_rmap_lookup_le_range_helper, &info);
358 if (error == XFS_BTREE_QUERY_RANGE_ABORT)
361 trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
362 cur->bc_private.a.agno, irec->rm_startblock,
363 irec->rm_blockcount, irec->rm_owner,
364 irec->rm_offset, irec->rm_flags);
369 * Find the extent in the rmap btree and remove it.
371 * The record we find should always be an exact match for the extent that we're
372 * looking for, since we insert them into the btree without modification.
374 * Special Case #1: when growing the filesystem, we "free" an extent when
375 * growing the last AG. This extent is new space and so it is not tracked as
376 * used space in the btree. The growfs code will pass in an owner of
377 * XFS_RMAP_OWN_NULL to indicate that it expected that there is no owner of this
378 * extent. We verify that - the extent lookup result in a record that does not
381 * Special Case #2: EFIs do not record the owner of the extent, so when
382 * recovering EFIs from the log we pass in XFS_RMAP_OWN_UNKNOWN to tell the rmap
383 * btree to ignore the owner (i.e. wildcard match) so we don't trigger
384 * corruption checks during log recovery.
388 struct xfs_btree_cur *cur,
392 struct xfs_owner_info *oinfo)
394 struct xfs_mount *mp = cur->bc_mp;
395 struct xfs_rmap_irec ltrec;
404 xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
405 ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
406 (flags & XFS_RMAP_BMBT_BLOCK);
408 flags |= XFS_RMAP_UNWRITTEN;
409 trace_xfs_rmap_unmap(mp, cur->bc_private.a.agno, bno, len,
413 * We should always have a left record because there's a static record
414 * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
415 * will not ever be removed from the tree.
417 error = xfs_rmap_lookup_le(cur, bno, len, owner, offset, flags, &i);
420 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
422 error = xfs_rmap_get_rec(cur, <rec, &i);
425 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
426 trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
427 cur->bc_private.a.agno, ltrec.rm_startblock,
428 ltrec.rm_blockcount, ltrec.rm_owner,
429 ltrec.rm_offset, ltrec.rm_flags);
430 ltoff = ltrec.rm_offset;
433 * For growfs, the incoming extent must be beyond the left record we
434 * just found as it is new space and won't be used by anyone. This is
435 * just a corruption check as we don't actually do anything with this
436 * extent. Note that we need to use >= instead of > because it might
437 * be the case that the "left" extent goes all the way to EOFS.
439 if (owner == XFS_RMAP_OWN_NULL) {
440 XFS_WANT_CORRUPTED_GOTO(mp, bno >= ltrec.rm_startblock +
441 ltrec.rm_blockcount, out_error);
445 /* Make sure the unwritten flag matches. */
446 XFS_WANT_CORRUPTED_GOTO(mp, (flags & XFS_RMAP_UNWRITTEN) ==
447 (ltrec.rm_flags & XFS_RMAP_UNWRITTEN), out_error);
449 /* Make sure the extent we found covers the entire freeing range. */
450 XFS_WANT_CORRUPTED_GOTO(mp, ltrec.rm_startblock <= bno &&
451 ltrec.rm_startblock + ltrec.rm_blockcount >=
452 bno + len, out_error);
454 /* Make sure the owner matches what we expect to find in the tree. */
455 XFS_WANT_CORRUPTED_GOTO(mp, owner == ltrec.rm_owner ||
456 XFS_RMAP_NON_INODE_OWNER(owner), out_error);
458 /* Check the offset, if necessary. */
459 if (!XFS_RMAP_NON_INODE_OWNER(owner)) {
460 if (flags & XFS_RMAP_BMBT_BLOCK) {
461 XFS_WANT_CORRUPTED_GOTO(mp,
462 ltrec.rm_flags & XFS_RMAP_BMBT_BLOCK,
465 XFS_WANT_CORRUPTED_GOTO(mp,
466 ltrec.rm_offset <= offset, out_error);
467 XFS_WANT_CORRUPTED_GOTO(mp,
468 ltoff + ltrec.rm_blockcount >= offset + len,
473 if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
474 /* exact match, simply remove the record from rmap tree */
475 trace_xfs_rmap_delete(mp, cur->bc_private.a.agno,
476 ltrec.rm_startblock, ltrec.rm_blockcount,
477 ltrec.rm_owner, ltrec.rm_offset,
479 error = xfs_btree_delete(cur, &i);
482 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
483 } else if (ltrec.rm_startblock == bno) {
485 * overlap left hand side of extent: move the start, trim the
486 * length and update the current record.
489 * Orig: |oooooooooooooooooooo|
490 * Freeing: |fffffffff|
491 * Result: |rrrrrrrrrr|
494 ltrec.rm_startblock += len;
495 ltrec.rm_blockcount -= len;
497 ltrec.rm_offset += len;
498 error = xfs_rmap_update(cur, <rec);
501 } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
503 * overlap right hand side of extent: trim the length and update
504 * the current record.
507 * Orig: |oooooooooooooooooooo|
508 * Freeing: |fffffffff|
509 * Result: |rrrrrrrrrr|
512 ltrec.rm_blockcount -= len;
513 error = xfs_rmap_update(cur, <rec);
519 * overlap middle of extent: trim the length of the existing
520 * record to the length of the new left-extent size, increment
521 * the insertion position so we can insert a new record
522 * containing the remaining right-extent space.
525 * Orig: |oooooooooooooooooooo|
526 * Freeing: |fffffffff|
527 * Result: |rrrrr| |rrrr|
530 xfs_extlen_t orig_len = ltrec.rm_blockcount;
532 ltrec.rm_blockcount = bno - ltrec.rm_startblock;
533 error = xfs_rmap_update(cur, <rec);
537 error = xfs_btree_increment(cur, 0, &i);
541 cur->bc_rec.r.rm_startblock = bno + len;
542 cur->bc_rec.r.rm_blockcount = orig_len - len -
544 cur->bc_rec.r.rm_owner = ltrec.rm_owner;
546 cur->bc_rec.r.rm_offset = 0;
548 cur->bc_rec.r.rm_offset = offset + len;
549 cur->bc_rec.r.rm_flags = flags;
550 trace_xfs_rmap_insert(mp, cur->bc_private.a.agno,
551 cur->bc_rec.r.rm_startblock,
552 cur->bc_rec.r.rm_blockcount,
553 cur->bc_rec.r.rm_owner,
554 cur->bc_rec.r.rm_offset,
555 cur->bc_rec.r.rm_flags);
556 error = xfs_btree_insert(cur, &i);
562 trace_xfs_rmap_unmap_done(mp, cur->bc_private.a.agno, bno, len,
566 trace_xfs_rmap_unmap_error(mp, cur->bc_private.a.agno,
572 * Remove a reference to an extent in the rmap btree.
576 struct xfs_trans *tp,
577 struct xfs_buf *agbp,
581 struct xfs_owner_info *oinfo)
583 struct xfs_mount *mp = tp->t_mountp;
584 struct xfs_btree_cur *cur;
587 if (!xfs_sb_version_hasrmapbt(&mp->m_sb))
590 cur = xfs_rmapbt_init_cursor(mp, tp, agbp, agno);
592 error = xfs_rmap_unmap(cur, bno, len, false, oinfo);
596 xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
600 xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
605 * A mergeable rmap must have the same owner and the same values for
606 * the unwritten, attr_fork, and bmbt flags. The startblock and
607 * offset are checked separately.
610 xfs_rmap_is_mergeable(
611 struct xfs_rmap_irec *irec,
615 if (irec->rm_owner == XFS_RMAP_OWN_NULL)
617 if (irec->rm_owner != owner)
619 if ((flags & XFS_RMAP_UNWRITTEN) ^
620 (irec->rm_flags & XFS_RMAP_UNWRITTEN))
622 if ((flags & XFS_RMAP_ATTR_FORK) ^
623 (irec->rm_flags & XFS_RMAP_ATTR_FORK))
625 if ((flags & XFS_RMAP_BMBT_BLOCK) ^
626 (irec->rm_flags & XFS_RMAP_BMBT_BLOCK))
632 * When we allocate a new block, the first thing we do is add a reference to
633 * the extent in the rmap btree. This takes the form of a [agbno, length,
634 * owner, offset] record. Flags are encoded in the high bits of the offset
639 struct xfs_btree_cur *cur,
643 struct xfs_owner_info *oinfo)
645 struct xfs_mount *mp = cur->bc_mp;
646 struct xfs_rmap_irec ltrec;
647 struct xfs_rmap_irec gtrec;
654 unsigned int flags = 0;
657 xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
659 ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
660 (flags & XFS_RMAP_BMBT_BLOCK);
662 flags |= XFS_RMAP_UNWRITTEN;
663 trace_xfs_rmap_map(mp, cur->bc_private.a.agno, bno, len,
667 * For the initial lookup, look for an exact match or the left-adjacent
668 * record for our insertion point. This will also give us the record for
669 * start block contiguity tests.
671 error = xfs_rmap_lookup_le(cur, bno, len, owner, offset, flags,
675 XFS_WANT_CORRUPTED_GOTO(mp, have_lt == 1, out_error);
677 error = xfs_rmap_get_rec(cur, <rec, &have_lt);
680 XFS_WANT_CORRUPTED_GOTO(mp, have_lt == 1, out_error);
681 trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
682 cur->bc_private.a.agno, ltrec.rm_startblock,
683 ltrec.rm_blockcount, ltrec.rm_owner,
684 ltrec.rm_offset, ltrec.rm_flags);
686 if (!xfs_rmap_is_mergeable(<rec, owner, flags))
689 XFS_WANT_CORRUPTED_GOTO(mp,
691 ltrec.rm_startblock + ltrec.rm_blockcount <= bno, out_error);
694 * Increment the cursor to see if we have a right-adjacent record to our
695 * insertion point. This will give us the record for end block
698 error = xfs_btree_increment(cur, 0, &have_gt);
702 error = xfs_rmap_get_rec(cur, >rec, &have_gt);
705 XFS_WANT_CORRUPTED_GOTO(mp, have_gt == 1, out_error);
706 XFS_WANT_CORRUPTED_GOTO(mp, bno + len <= gtrec.rm_startblock,
708 trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
709 cur->bc_private.a.agno, gtrec.rm_startblock,
710 gtrec.rm_blockcount, gtrec.rm_owner,
711 gtrec.rm_offset, gtrec.rm_flags);
712 if (!xfs_rmap_is_mergeable(>rec, owner, flags))
717 * Note: cursor currently points one record to the right of ltrec, even
718 * if there is no record in the tree to the right.
721 ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
722 (ignore_off || ltrec.rm_offset + ltrec.rm_blockcount == offset)) {
724 * left edge contiguous, merge into left record.
728 * adding: |aaaaaaaaa|
729 * result: |rrrrrrrrrrrrrrrrrrr|
732 ltrec.rm_blockcount += len;
734 bno + len == gtrec.rm_startblock &&
735 (ignore_off || offset + len == gtrec.rm_offset) &&
736 (unsigned long)ltrec.rm_blockcount + len +
737 gtrec.rm_blockcount <= XFS_RMAP_LEN_MAX) {
739 * right edge also contiguous, delete right record
740 * and merge into left record.
742 * ltbno ltlen gtbno gtlen
743 * orig: |ooooooooo| |ooooooooo|
744 * adding: |aaaaaaaaa|
745 * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
747 ltrec.rm_blockcount += gtrec.rm_blockcount;
748 trace_xfs_rmap_delete(mp, cur->bc_private.a.agno,
754 error = xfs_btree_delete(cur, &i);
757 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
760 /* point the cursor back to the left record and update */
761 error = xfs_btree_decrement(cur, 0, &have_gt);
764 error = xfs_rmap_update(cur, <rec);
767 } else if (have_gt &&
768 bno + len == gtrec.rm_startblock &&
769 (ignore_off || offset + len == gtrec.rm_offset)) {
771 * right edge contiguous, merge into right record.
775 * adding: |aaaaaaaaa|
776 * Result: |rrrrrrrrrrrrrrrrrrr|
779 gtrec.rm_startblock = bno;
780 gtrec.rm_blockcount += len;
782 gtrec.rm_offset = offset;
783 error = xfs_rmap_update(cur, >rec);
788 * no contiguous edge with identical owner, insert
789 * new record at current cursor position.
791 cur->bc_rec.r.rm_startblock = bno;
792 cur->bc_rec.r.rm_blockcount = len;
793 cur->bc_rec.r.rm_owner = owner;
794 cur->bc_rec.r.rm_offset = offset;
795 cur->bc_rec.r.rm_flags = flags;
796 trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, bno, len,
797 owner, offset, flags);
798 error = xfs_btree_insert(cur, &i);
801 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
804 trace_xfs_rmap_map_done(mp, cur->bc_private.a.agno, bno, len,
808 trace_xfs_rmap_map_error(mp, cur->bc_private.a.agno,
814 * Add a reference to an extent in the rmap btree.
818 struct xfs_trans *tp,
819 struct xfs_buf *agbp,
823 struct xfs_owner_info *oinfo)
825 struct xfs_mount *mp = tp->t_mountp;
826 struct xfs_btree_cur *cur;
829 if (!xfs_sb_version_hasrmapbt(&mp->m_sb))
832 cur = xfs_rmapbt_init_cursor(mp, tp, agbp, agno);
833 error = xfs_rmap_map(cur, bno, len, false, oinfo);
837 xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
841 xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
845 #define RMAP_LEFT_CONTIG (1 << 0)
846 #define RMAP_RIGHT_CONTIG (1 << 1)
847 #define RMAP_LEFT_FILLING (1 << 2)
848 #define RMAP_RIGHT_FILLING (1 << 3)
849 #define RMAP_LEFT_VALID (1 << 6)
850 #define RMAP_RIGHT_VALID (1 << 7)
858 * Convert an unwritten extent to a real extent or vice versa.
859 * Does not handle overlapping extents.
863 struct xfs_btree_cur *cur,
867 struct xfs_owner_info *oinfo)
869 struct xfs_mount *mp = cur->bc_mp;
870 struct xfs_rmap_irec r[4]; /* neighbor extent entries */
871 /* left is 0, right is 1, prev is 2 */
878 unsigned int flags = 0;
883 xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
884 ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
885 (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
886 oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
887 new_endoff = offset + len;
888 trace_xfs_rmap_convert(mp, cur->bc_private.a.agno, bno, len,
892 * For the initial lookup, look for an exact match or the left-adjacent
893 * record for our insertion point. This will also give us the record for
894 * start block contiguity tests.
896 error = xfs_rmap_lookup_le(cur, bno, len, owner, offset, oldext, &i);
899 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
901 error = xfs_rmap_get_rec(cur, &PREV, &i);
904 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
905 trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
906 cur->bc_private.a.agno, PREV.rm_startblock,
907 PREV.rm_blockcount, PREV.rm_owner,
908 PREV.rm_offset, PREV.rm_flags);
910 ASSERT(PREV.rm_offset <= offset);
911 ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
912 ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
913 newext = ~oldext & XFS_RMAP_UNWRITTEN;
916 * Set flags determining what part of the previous oldext allocation
917 * extent is being replaced by a newext allocation.
919 if (PREV.rm_offset == offset)
920 state |= RMAP_LEFT_FILLING;
921 if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
922 state |= RMAP_RIGHT_FILLING;
925 * Decrement the cursor to see if we have a left-adjacent record to our
926 * insertion point. This will give us the record for end block
929 error = xfs_btree_decrement(cur, 0, &i);
933 state |= RMAP_LEFT_VALID;
934 error = xfs_rmap_get_rec(cur, &LEFT, &i);
937 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
938 XFS_WANT_CORRUPTED_GOTO(mp,
939 LEFT.rm_startblock + LEFT.rm_blockcount <= bno,
941 trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
942 cur->bc_private.a.agno, LEFT.rm_startblock,
943 LEFT.rm_blockcount, LEFT.rm_owner,
944 LEFT.rm_offset, LEFT.rm_flags);
945 if (LEFT.rm_startblock + LEFT.rm_blockcount == bno &&
946 LEFT.rm_offset + LEFT.rm_blockcount == offset &&
947 xfs_rmap_is_mergeable(&LEFT, owner, newext))
948 state |= RMAP_LEFT_CONTIG;
952 * Increment the cursor to see if we have a right-adjacent record to our
953 * insertion point. This will give us the record for end block
956 error = xfs_btree_increment(cur, 0, &i);
959 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
960 error = xfs_btree_increment(cur, 0, &i);
964 state |= RMAP_RIGHT_VALID;
965 error = xfs_rmap_get_rec(cur, &RIGHT, &i);
968 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
969 XFS_WANT_CORRUPTED_GOTO(mp, bno + len <= RIGHT.rm_startblock,
971 trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
972 cur->bc_private.a.agno, RIGHT.rm_startblock,
973 RIGHT.rm_blockcount, RIGHT.rm_owner,
974 RIGHT.rm_offset, RIGHT.rm_flags);
975 if (bno + len == RIGHT.rm_startblock &&
976 offset + len == RIGHT.rm_offset &&
977 xfs_rmap_is_mergeable(&RIGHT, owner, newext))
978 state |= RMAP_RIGHT_CONTIG;
981 /* check that left + prev + right is not too long */
982 if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
983 RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
984 (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
985 RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
986 (unsigned long)LEFT.rm_blockcount + len +
987 RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
988 state &= ~RMAP_RIGHT_CONTIG;
990 trace_xfs_rmap_convert_state(mp, cur->bc_private.a.agno, state,
993 /* reset the cursor back to PREV */
994 error = xfs_rmap_lookup_le(cur, bno, len, owner, offset, oldext, &i);
997 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1000 * Switch out based on the FILLING and CONTIG state bits.
1002 switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1003 RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1004 case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1005 RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1007 * Setting all of a previous oldext extent to newext.
1008 * The left and right neighbors are both contiguous with new.
1010 error = xfs_btree_increment(cur, 0, &i);
1013 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1014 trace_xfs_rmap_delete(mp, cur->bc_private.a.agno,
1015 RIGHT.rm_startblock, RIGHT.rm_blockcount,
1016 RIGHT.rm_owner, RIGHT.rm_offset,
1018 error = xfs_btree_delete(cur, &i);
1021 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1022 error = xfs_btree_decrement(cur, 0, &i);
1025 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1026 trace_xfs_rmap_delete(mp, cur->bc_private.a.agno,
1027 PREV.rm_startblock, PREV.rm_blockcount,
1028 PREV.rm_owner, PREV.rm_offset,
1030 error = xfs_btree_delete(cur, &i);
1033 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1034 error = xfs_btree_decrement(cur, 0, &i);
1037 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1039 NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1040 error = xfs_rmap_update(cur, &NEW);
1045 case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1047 * Setting all of a previous oldext extent to newext.
1048 * The left neighbor is contiguous, the right is not.
1050 trace_xfs_rmap_delete(mp, cur->bc_private.a.agno,
1051 PREV.rm_startblock, PREV.rm_blockcount,
1052 PREV.rm_owner, PREV.rm_offset,
1054 error = xfs_btree_delete(cur, &i);
1057 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1058 error = xfs_btree_decrement(cur, 0, &i);
1061 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1063 NEW.rm_blockcount += PREV.rm_blockcount;
1064 error = xfs_rmap_update(cur, &NEW);
1069 case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1071 * Setting all of a previous oldext extent to newext.
1072 * The right neighbor is contiguous, the left is not.
1074 error = xfs_btree_increment(cur, 0, &i);
1077 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1078 trace_xfs_rmap_delete(mp, cur->bc_private.a.agno,
1079 RIGHT.rm_startblock, RIGHT.rm_blockcount,
1080 RIGHT.rm_owner, RIGHT.rm_offset,
1082 error = xfs_btree_delete(cur, &i);
1085 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1086 error = xfs_btree_decrement(cur, 0, &i);
1089 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1091 NEW.rm_blockcount = len + RIGHT.rm_blockcount;
1092 NEW.rm_flags = newext;
1093 error = xfs_rmap_update(cur, &NEW);
1098 case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1100 * Setting all of a previous oldext extent to newext.
1101 * Neither the left nor right neighbors are contiguous with
1105 NEW.rm_flags = newext;
1106 error = xfs_rmap_update(cur, &NEW);
1111 case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1113 * Setting the first part of a previous oldext extent to newext.
1114 * The left neighbor is contiguous.
1117 NEW.rm_offset += len;
1118 NEW.rm_startblock += len;
1119 NEW.rm_blockcount -= len;
1120 error = xfs_rmap_update(cur, &NEW);
1123 error = xfs_btree_decrement(cur, 0, &i);
1127 NEW.rm_blockcount += len;
1128 error = xfs_rmap_update(cur, &NEW);
1133 case RMAP_LEFT_FILLING:
1135 * Setting the first part of a previous oldext extent to newext.
1136 * The left neighbor is not contiguous.
1139 NEW.rm_startblock += len;
1140 NEW.rm_offset += len;
1141 NEW.rm_blockcount -= len;
1142 error = xfs_rmap_update(cur, &NEW);
1145 NEW.rm_startblock = bno;
1146 NEW.rm_owner = owner;
1147 NEW.rm_offset = offset;
1148 NEW.rm_blockcount = len;
1149 NEW.rm_flags = newext;
1150 cur->bc_rec.r = NEW;
1151 trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, bno,
1152 len, owner, offset, newext);
1153 error = xfs_btree_insert(cur, &i);
1156 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1159 case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1161 * Setting the last part of a previous oldext extent to newext.
1162 * The right neighbor is contiguous with the new allocation.
1165 NEW.rm_blockcount -= len;
1166 error = xfs_rmap_update(cur, &NEW);
1169 error = xfs_btree_increment(cur, 0, &i);
1173 NEW.rm_offset = offset;
1174 NEW.rm_startblock = bno;
1175 NEW.rm_blockcount += len;
1176 error = xfs_rmap_update(cur, &NEW);
1181 case RMAP_RIGHT_FILLING:
1183 * Setting the last part of a previous oldext extent to newext.
1184 * The right neighbor is not contiguous.
1187 NEW.rm_blockcount -= len;
1188 error = xfs_rmap_update(cur, &NEW);
1191 error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1195 XFS_WANT_CORRUPTED_GOTO(mp, i == 0, done);
1196 NEW.rm_startblock = bno;
1197 NEW.rm_owner = owner;
1198 NEW.rm_offset = offset;
1199 NEW.rm_blockcount = len;
1200 NEW.rm_flags = newext;
1201 cur->bc_rec.r = NEW;
1202 trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, bno,
1203 len, owner, offset, newext);
1204 error = xfs_btree_insert(cur, &i);
1207 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1212 * Setting the middle part of a previous oldext extent to
1213 * newext. Contiguity is impossible here.
1214 * One extent becomes three extents.
1216 /* new right extent - oldext */
1217 NEW.rm_startblock = bno + len;
1218 NEW.rm_owner = owner;
1219 NEW.rm_offset = new_endoff;
1220 NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1222 NEW.rm_flags = PREV.rm_flags;
1223 error = xfs_rmap_update(cur, &NEW);
1226 /* new left extent - oldext */
1228 NEW.rm_blockcount = offset - PREV.rm_offset;
1229 cur->bc_rec.r = NEW;
1230 trace_xfs_rmap_insert(mp, cur->bc_private.a.agno,
1231 NEW.rm_startblock, NEW.rm_blockcount,
1232 NEW.rm_owner, NEW.rm_offset,
1234 error = xfs_btree_insert(cur, &i);
1237 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1239 * Reset the cursor to the position of the new extent
1240 * we are about to insert as we can't trust it after
1241 * the previous insert.
1243 error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1247 XFS_WANT_CORRUPTED_GOTO(mp, i == 0, done);
1248 /* new middle extent - newext */
1249 cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN;
1250 cur->bc_rec.r.rm_flags |= newext;
1251 trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, bno, len,
1252 owner, offset, newext);
1253 error = xfs_btree_insert(cur, &i);
1256 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1259 case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1260 case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1261 case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1262 case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1263 case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1264 case RMAP_LEFT_CONTIG:
1265 case RMAP_RIGHT_CONTIG:
1267 * These cases are all impossible.
1272 trace_xfs_rmap_convert_done(mp, cur->bc_private.a.agno, bno, len,
1276 trace_xfs_rmap_convert_error(cur->bc_mp,
1277 cur->bc_private.a.agno, error, _RET_IP_);
1282 * Convert an unwritten extent to a real extent or vice versa. If there is no
1283 * possibility of overlapping extents, delegate to the simpler convert
1287 xfs_rmap_convert_shared(
1288 struct xfs_btree_cur *cur,
1292 struct xfs_owner_info *oinfo)
1294 struct xfs_mount *mp = cur->bc_mp;
1295 struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1296 /* left is 0, right is 1, prev is 2 */
1300 uint64_t new_endoff;
1301 unsigned int oldext;
1302 unsigned int newext;
1303 unsigned int flags = 0;
1308 xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1309 ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1310 (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1311 oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1312 new_endoff = offset + len;
1313 trace_xfs_rmap_convert(mp, cur->bc_private.a.agno, bno, len,
1317 * For the initial lookup, look for and exact match or the left-adjacent
1318 * record for our insertion point. This will also give us the record for
1319 * start block contiguity tests.
1321 error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags,
1323 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1325 ASSERT(PREV.rm_offset <= offset);
1326 ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1327 ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1328 newext = ~oldext & XFS_RMAP_UNWRITTEN;
1331 * Set flags determining what part of the previous oldext allocation
1332 * extent is being replaced by a newext allocation.
1334 if (PREV.rm_offset == offset)
1335 state |= RMAP_LEFT_FILLING;
1336 if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1337 state |= RMAP_RIGHT_FILLING;
1339 /* Is there a left record that abuts our range? */
1340 error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, newext,
1345 state |= RMAP_LEFT_VALID;
1346 XFS_WANT_CORRUPTED_GOTO(mp,
1347 LEFT.rm_startblock + LEFT.rm_blockcount <= bno,
1349 if (xfs_rmap_is_mergeable(&LEFT, owner, newext))
1350 state |= RMAP_LEFT_CONTIG;
1353 /* Is there a right record that abuts our range? */
1354 error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
1359 state |= RMAP_RIGHT_VALID;
1360 error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1363 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1364 XFS_WANT_CORRUPTED_GOTO(mp, bno + len <= RIGHT.rm_startblock,
1366 trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1367 cur->bc_private.a.agno, RIGHT.rm_startblock,
1368 RIGHT.rm_blockcount, RIGHT.rm_owner,
1369 RIGHT.rm_offset, RIGHT.rm_flags);
1370 if (xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1371 state |= RMAP_RIGHT_CONTIG;
1374 /* check that left + prev + right is not too long */
1375 if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1376 RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1377 (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1378 RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1379 (unsigned long)LEFT.rm_blockcount + len +
1380 RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1381 state &= ~RMAP_RIGHT_CONTIG;
1383 trace_xfs_rmap_convert_state(mp, cur->bc_private.a.agno, state,
1386 * Switch out based on the FILLING and CONTIG state bits.
1388 switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1389 RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1390 case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1391 RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1393 * Setting all of a previous oldext extent to newext.
1394 * The left and right neighbors are both contiguous with new.
1396 error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1397 RIGHT.rm_blockcount, RIGHT.rm_owner,
1398 RIGHT.rm_offset, RIGHT.rm_flags);
1401 error = xfs_rmap_delete(cur, PREV.rm_startblock,
1402 PREV.rm_blockcount, PREV.rm_owner,
1403 PREV.rm_offset, PREV.rm_flags);
1407 error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1408 NEW.rm_blockcount, NEW.rm_owner,
1409 NEW.rm_offset, NEW.rm_flags, &i);
1412 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1413 NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1414 error = xfs_rmap_update(cur, &NEW);
1419 case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1421 * Setting all of a previous oldext extent to newext.
1422 * The left neighbor is contiguous, the right is not.
1424 error = xfs_rmap_delete(cur, PREV.rm_startblock,
1425 PREV.rm_blockcount, PREV.rm_owner,
1426 PREV.rm_offset, PREV.rm_flags);
1430 error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1431 NEW.rm_blockcount, NEW.rm_owner,
1432 NEW.rm_offset, NEW.rm_flags, &i);
1435 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1436 NEW.rm_blockcount += PREV.rm_blockcount;
1437 error = xfs_rmap_update(cur, &NEW);
1442 case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1444 * Setting all of a previous oldext extent to newext.
1445 * The right neighbor is contiguous, the left is not.
1447 error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1448 RIGHT.rm_blockcount, RIGHT.rm_owner,
1449 RIGHT.rm_offset, RIGHT.rm_flags);
1453 error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1454 NEW.rm_blockcount, NEW.rm_owner,
1455 NEW.rm_offset, NEW.rm_flags, &i);
1458 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1459 NEW.rm_blockcount += RIGHT.rm_blockcount;
1460 NEW.rm_flags = RIGHT.rm_flags;
1461 error = xfs_rmap_update(cur, &NEW);
1466 case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1468 * Setting all of a previous oldext extent to newext.
1469 * Neither the left nor right neighbors are contiguous with
1473 error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1474 NEW.rm_blockcount, NEW.rm_owner,
1475 NEW.rm_offset, NEW.rm_flags, &i);
1478 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1479 NEW.rm_flags = newext;
1480 error = xfs_rmap_update(cur, &NEW);
1485 case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1487 * Setting the first part of a previous oldext extent to newext.
1488 * The left neighbor is contiguous.
1491 error = xfs_rmap_delete(cur, NEW.rm_startblock,
1492 NEW.rm_blockcount, NEW.rm_owner,
1493 NEW.rm_offset, NEW.rm_flags);
1496 NEW.rm_offset += len;
1497 NEW.rm_startblock += len;
1498 NEW.rm_blockcount -= len;
1499 error = xfs_rmap_insert(cur, NEW.rm_startblock,
1500 NEW.rm_blockcount, NEW.rm_owner,
1501 NEW.rm_offset, NEW.rm_flags);
1505 error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1506 NEW.rm_blockcount, NEW.rm_owner,
1507 NEW.rm_offset, NEW.rm_flags, &i);
1510 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1511 NEW.rm_blockcount += len;
1512 error = xfs_rmap_update(cur, &NEW);
1517 case RMAP_LEFT_FILLING:
1519 * Setting the first part of a previous oldext extent to newext.
1520 * The left neighbor is not contiguous.
1523 error = xfs_rmap_delete(cur, NEW.rm_startblock,
1524 NEW.rm_blockcount, NEW.rm_owner,
1525 NEW.rm_offset, NEW.rm_flags);
1528 NEW.rm_offset += len;
1529 NEW.rm_startblock += len;
1530 NEW.rm_blockcount -= len;
1531 error = xfs_rmap_insert(cur, NEW.rm_startblock,
1532 NEW.rm_blockcount, NEW.rm_owner,
1533 NEW.rm_offset, NEW.rm_flags);
1536 error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1541 case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1543 * Setting the last part of a previous oldext extent to newext.
1544 * The right neighbor is contiguous with the new allocation.
1547 error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1548 NEW.rm_blockcount, NEW.rm_owner,
1549 NEW.rm_offset, NEW.rm_flags, &i);
1552 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1553 NEW.rm_blockcount = offset - NEW.rm_offset;
1554 error = xfs_rmap_update(cur, &NEW);
1558 error = xfs_rmap_delete(cur, NEW.rm_startblock,
1559 NEW.rm_blockcount, NEW.rm_owner,
1560 NEW.rm_offset, NEW.rm_flags);
1563 NEW.rm_offset = offset;
1564 NEW.rm_startblock = bno;
1565 NEW.rm_blockcount += len;
1566 error = xfs_rmap_insert(cur, NEW.rm_startblock,
1567 NEW.rm_blockcount, NEW.rm_owner,
1568 NEW.rm_offset, NEW.rm_flags);
1573 case RMAP_RIGHT_FILLING:
1575 * Setting the last part of a previous oldext extent to newext.
1576 * The right neighbor is not contiguous.
1579 error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1580 NEW.rm_blockcount, NEW.rm_owner,
1581 NEW.rm_offset, NEW.rm_flags, &i);
1584 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1585 NEW.rm_blockcount -= len;
1586 error = xfs_rmap_update(cur, &NEW);
1589 error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1596 * Setting the middle part of a previous oldext extent to
1597 * newext. Contiguity is impossible here.
1598 * One extent becomes three extents.
1600 /* new right extent - oldext */
1601 NEW.rm_startblock = bno + len;
1602 NEW.rm_owner = owner;
1603 NEW.rm_offset = new_endoff;
1604 NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1606 NEW.rm_flags = PREV.rm_flags;
1607 error = xfs_rmap_insert(cur, NEW.rm_startblock,
1608 NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
1612 /* new left extent - oldext */
1614 error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1615 NEW.rm_blockcount, NEW.rm_owner,
1616 NEW.rm_offset, NEW.rm_flags, &i);
1619 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, done);
1620 NEW.rm_blockcount = offset - NEW.rm_offset;
1621 error = xfs_rmap_update(cur, &NEW);
1624 /* new middle extent - newext */
1625 NEW.rm_startblock = bno;
1626 NEW.rm_blockcount = len;
1627 NEW.rm_owner = owner;
1628 NEW.rm_offset = offset;
1629 NEW.rm_flags = newext;
1630 error = xfs_rmap_insert(cur, NEW.rm_startblock,
1631 NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
1637 case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1638 case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1639 case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1640 case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1641 case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1642 case RMAP_LEFT_CONTIG:
1643 case RMAP_RIGHT_CONTIG:
1645 * These cases are all impossible.
1650 trace_xfs_rmap_convert_done(mp, cur->bc_private.a.agno, bno, len,
1654 trace_xfs_rmap_convert_error(cur->bc_mp,
1655 cur->bc_private.a.agno, error, _RET_IP_);
1665 * Find an extent in the rmap btree and unmap it. For rmap extent types that
1666 * can overlap (data fork rmaps on reflink filesystems) we must be careful
1667 * that the prev/next records in the btree might belong to another owner.
1668 * Therefore we must use delete+insert to alter any of the key fields.
1670 * For every other situation there can only be one owner for a given extent,
1671 * so we can call the regular _free function.
1674 xfs_rmap_unmap_shared(
1675 struct xfs_btree_cur *cur,
1679 struct xfs_owner_info *oinfo)
1681 struct xfs_mount *mp = cur->bc_mp;
1682 struct xfs_rmap_irec ltrec;
1690 xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1692 flags |= XFS_RMAP_UNWRITTEN;
1693 trace_xfs_rmap_unmap(mp, cur->bc_private.a.agno, bno, len,
1697 * We should always have a left record because there's a static record
1698 * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
1699 * will not ever be removed from the tree.
1701 error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags,
1705 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
1706 ltoff = ltrec.rm_offset;
1708 /* Make sure the extent we found covers the entire freeing range. */
1709 XFS_WANT_CORRUPTED_GOTO(mp, ltrec.rm_startblock <= bno &&
1710 ltrec.rm_startblock + ltrec.rm_blockcount >=
1711 bno + len, out_error);
1713 /* Make sure the owner matches what we expect to find in the tree. */
1714 XFS_WANT_CORRUPTED_GOTO(mp, owner == ltrec.rm_owner, out_error);
1716 /* Make sure the unwritten flag matches. */
1717 XFS_WANT_CORRUPTED_GOTO(mp, (flags & XFS_RMAP_UNWRITTEN) ==
1718 (ltrec.rm_flags & XFS_RMAP_UNWRITTEN), out_error);
1720 /* Check the offset. */
1721 XFS_WANT_CORRUPTED_GOTO(mp, ltrec.rm_offset <= offset, out_error);
1722 XFS_WANT_CORRUPTED_GOTO(mp, offset <= ltoff + ltrec.rm_blockcount,
1725 if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
1726 /* Exact match, simply remove the record from rmap tree. */
1727 error = xfs_rmap_delete(cur, ltrec.rm_startblock,
1728 ltrec.rm_blockcount, ltrec.rm_owner,
1729 ltrec.rm_offset, ltrec.rm_flags);
1732 } else if (ltrec.rm_startblock == bno) {
1734 * Overlap left hand side of extent: move the start, trim the
1735 * length and update the current record.
1738 * Orig: |oooooooooooooooooooo|
1739 * Freeing: |fffffffff|
1740 * Result: |rrrrrrrrrr|
1744 /* Delete prev rmap. */
1745 error = xfs_rmap_delete(cur, ltrec.rm_startblock,
1746 ltrec.rm_blockcount, ltrec.rm_owner,
1747 ltrec.rm_offset, ltrec.rm_flags);
1751 /* Add an rmap at the new offset. */
1752 ltrec.rm_startblock += len;
1753 ltrec.rm_blockcount -= len;
1754 ltrec.rm_offset += len;
1755 error = xfs_rmap_insert(cur, ltrec.rm_startblock,
1756 ltrec.rm_blockcount, ltrec.rm_owner,
1757 ltrec.rm_offset, ltrec.rm_flags);
1760 } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
1762 * Overlap right hand side of extent: trim the length and
1763 * update the current record.
1766 * Orig: |oooooooooooooooooooo|
1767 * Freeing: |fffffffff|
1768 * Result: |rrrrrrrrrr|
1771 error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
1772 ltrec.rm_blockcount, ltrec.rm_owner,
1773 ltrec.rm_offset, ltrec.rm_flags, &i);
1776 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
1777 ltrec.rm_blockcount -= len;
1778 error = xfs_rmap_update(cur, <rec);
1783 * Overlap middle of extent: trim the length of the existing
1784 * record to the length of the new left-extent size, increment
1785 * the insertion position so we can insert a new record
1786 * containing the remaining right-extent space.
1789 * Orig: |oooooooooooooooooooo|
1790 * Freeing: |fffffffff|
1791 * Result: |rrrrr| |rrrr|
1794 xfs_extlen_t orig_len = ltrec.rm_blockcount;
1796 /* Shrink the left side of the rmap */
1797 error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
1798 ltrec.rm_blockcount, ltrec.rm_owner,
1799 ltrec.rm_offset, ltrec.rm_flags, &i);
1802 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
1803 ltrec.rm_blockcount = bno - ltrec.rm_startblock;
1804 error = xfs_rmap_update(cur, <rec);
1808 /* Add an rmap at the new offset */
1809 error = xfs_rmap_insert(cur, bno + len,
1810 orig_len - len - ltrec.rm_blockcount,
1811 ltrec.rm_owner, offset + len,
1817 trace_xfs_rmap_unmap_done(mp, cur->bc_private.a.agno, bno, len,
1821 trace_xfs_rmap_unmap_error(cur->bc_mp,
1822 cur->bc_private.a.agno, error, _RET_IP_);
1827 * Find an extent in the rmap btree and map it. For rmap extent types that
1828 * can overlap (data fork rmaps on reflink filesystems) we must be careful
1829 * that the prev/next records in the btree might belong to another owner.
1830 * Therefore we must use delete+insert to alter any of the key fields.
1832 * For every other situation there can only be one owner for a given extent,
1833 * so we can call the regular _alloc function.
1836 xfs_rmap_map_shared(
1837 struct xfs_btree_cur *cur,
1841 struct xfs_owner_info *oinfo)
1843 struct xfs_mount *mp = cur->bc_mp;
1844 struct xfs_rmap_irec ltrec;
1845 struct xfs_rmap_irec gtrec;
1852 unsigned int flags = 0;
1854 xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1856 flags |= XFS_RMAP_UNWRITTEN;
1857 trace_xfs_rmap_map(mp, cur->bc_private.a.agno, bno, len,
1860 /* Is there a left record that abuts our range? */
1861 error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, flags,
1866 !xfs_rmap_is_mergeable(<rec, owner, flags))
1869 /* Is there a right record that abuts our range? */
1870 error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
1875 error = xfs_rmap_get_rec(cur, >rec, &have_gt);
1878 XFS_WANT_CORRUPTED_GOTO(mp, have_gt == 1, out_error);
1879 trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1880 cur->bc_private.a.agno, gtrec.rm_startblock,
1881 gtrec.rm_blockcount, gtrec.rm_owner,
1882 gtrec.rm_offset, gtrec.rm_flags);
1884 if (!xfs_rmap_is_mergeable(>rec, owner, flags))
1889 ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
1890 ltrec.rm_offset + ltrec.rm_blockcount == offset) {
1892 * Left edge contiguous, merge into left record.
1896 * adding: |aaaaaaaaa|
1897 * result: |rrrrrrrrrrrrrrrrrrr|
1900 ltrec.rm_blockcount += len;
1902 bno + len == gtrec.rm_startblock &&
1903 offset + len == gtrec.rm_offset) {
1905 * Right edge also contiguous, delete right record
1906 * and merge into left record.
1908 * ltbno ltlen gtbno gtlen
1909 * orig: |ooooooooo| |ooooooooo|
1910 * adding: |aaaaaaaaa|
1911 * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
1913 ltrec.rm_blockcount += gtrec.rm_blockcount;
1914 error = xfs_rmap_delete(cur, gtrec.rm_startblock,
1915 gtrec.rm_blockcount, gtrec.rm_owner,
1916 gtrec.rm_offset, gtrec.rm_flags);
1921 /* Point the cursor back to the left record and update. */
1922 error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
1923 ltrec.rm_blockcount, ltrec.rm_owner,
1924 ltrec.rm_offset, ltrec.rm_flags, &i);
1927 XFS_WANT_CORRUPTED_GOTO(mp, i == 1, out_error);
1929 error = xfs_rmap_update(cur, <rec);
1932 } else if (have_gt &&
1933 bno + len == gtrec.rm_startblock &&
1934 offset + len == gtrec.rm_offset) {
1936 * Right edge contiguous, merge into right record.
1940 * adding: |aaaaaaaaa|
1941 * Result: |rrrrrrrrrrrrrrrrrrr|
1944 /* Delete the old record. */
1945 error = xfs_rmap_delete(cur, gtrec.rm_startblock,
1946 gtrec.rm_blockcount, gtrec.rm_owner,
1947 gtrec.rm_offset, gtrec.rm_flags);
1951 /* Move the start and re-add it. */
1952 gtrec.rm_startblock = bno;
1953 gtrec.rm_blockcount += len;
1954 gtrec.rm_offset = offset;
1955 error = xfs_rmap_insert(cur, gtrec.rm_startblock,
1956 gtrec.rm_blockcount, gtrec.rm_owner,
1957 gtrec.rm_offset, gtrec.rm_flags);
1962 * No contiguous edge with identical owner, insert
1963 * new record at current cursor position.
1965 error = xfs_rmap_insert(cur, bno, len, owner, offset, flags);
1970 trace_xfs_rmap_map_done(mp, cur->bc_private.a.agno, bno, len,
1974 trace_xfs_rmap_map_error(cur->bc_mp,
1975 cur->bc_private.a.agno, error, _RET_IP_);
1979 struct xfs_rmap_query_range_info {
1980 xfs_rmap_query_range_fn fn;
1984 /* Format btree record and pass to our callback. */
1986 xfs_rmap_query_range_helper(
1987 struct xfs_btree_cur *cur,
1988 union xfs_btree_rec *rec,
1991 struct xfs_rmap_query_range_info *query = priv;
1992 struct xfs_rmap_irec irec;
1995 error = xfs_rmap_btrec_to_irec(rec, &irec);
1998 return query->fn(cur, &irec, query->priv);
2001 /* Find all rmaps between two keys. */
2003 xfs_rmap_query_range(
2004 struct xfs_btree_cur *cur,
2005 struct xfs_rmap_irec *low_rec,
2006 struct xfs_rmap_irec *high_rec,
2007 xfs_rmap_query_range_fn fn,
2010 union xfs_btree_irec low_brec;
2011 union xfs_btree_irec high_brec;
2012 struct xfs_rmap_query_range_info query;
2014 low_brec.r = *low_rec;
2015 high_brec.r = *high_rec;
2018 return xfs_btree_query_range(cur, &low_brec, &high_brec,
2019 xfs_rmap_query_range_helper, &query);
2022 /* Clean up after calling xfs_rmap_finish_one. */
2024 xfs_rmap_finish_one_cleanup(
2025 struct xfs_trans *tp,
2026 struct xfs_btree_cur *rcur,
2029 struct xfs_buf *agbp;
2033 agbp = rcur->bc_private.a.agbp;
2034 xfs_btree_del_cursor(rcur, error ? XFS_BTREE_ERROR : XFS_BTREE_NOERROR);
2036 xfs_trans_brelse(tp, agbp);
2040 * Process one of the deferred rmap operations. We pass back the
2041 * btree cursor to maintain our lock on the rmapbt between calls.
2042 * This saves time and eliminates a buffer deadlock between the
2043 * superblock and the AGF because we'll always grab them in the same
2047 xfs_rmap_finish_one(
2048 struct xfs_trans *tp,
2049 enum xfs_rmap_intent_type type,
2052 xfs_fileoff_t startoff,
2053 xfs_fsblock_t startblock,
2054 xfs_filblks_t blockcount,
2056 struct xfs_btree_cur **pcur)
2058 struct xfs_mount *mp = tp->t_mountp;
2059 struct xfs_btree_cur *rcur;
2060 struct xfs_buf *agbp = NULL;
2062 xfs_agnumber_t agno;
2063 struct xfs_owner_info oinfo;
2067 agno = XFS_FSB_TO_AGNO(mp, startblock);
2068 ASSERT(agno != NULLAGNUMBER);
2069 bno = XFS_FSB_TO_AGBNO(mp, startblock);
2071 trace_xfs_rmap_deferred(mp, agno, type, bno, owner, whichfork,
2072 startoff, blockcount, state);
2074 if (XFS_TEST_ERROR(false, mp,
2075 XFS_ERRTAG_RMAP_FINISH_ONE,
2076 XFS_RANDOM_RMAP_FINISH_ONE))
2080 * If we haven't gotten a cursor or the cursor AG doesn't match
2081 * the startblock, get one now.
2084 if (rcur != NULL && rcur->bc_private.a.agno != agno) {
2085 xfs_rmap_finish_one_cleanup(tp, rcur, 0);
2091 * Refresh the freelist before we start changing the
2092 * rmapbt, because a shape change could cause us to
2095 error = xfs_free_extent_fix_freelist(tp, agno, &agbp);
2099 return -EFSCORRUPTED;
2101 rcur = xfs_rmapbt_init_cursor(mp, tp, agbp, agno);
2109 xfs_rmap_ino_owner(&oinfo, owner, whichfork, startoff);
2110 unwritten = state == XFS_EXT_UNWRITTEN;
2111 bno = XFS_FSB_TO_AGBNO(rcur->bc_mp, startblock);
2114 case XFS_RMAP_ALLOC:
2116 error = xfs_rmap_map(rcur, bno, blockcount, unwritten, &oinfo);
2118 case XFS_RMAP_MAP_SHARED:
2119 error = xfs_rmap_map_shared(rcur, bno, blockcount, unwritten,
2123 case XFS_RMAP_UNMAP:
2124 error = xfs_rmap_unmap(rcur, bno, blockcount, unwritten,
2127 case XFS_RMAP_UNMAP_SHARED:
2128 error = xfs_rmap_unmap_shared(rcur, bno, blockcount, unwritten,
2131 case XFS_RMAP_CONVERT:
2132 error = xfs_rmap_convert(rcur, bno, blockcount, !unwritten,
2135 case XFS_RMAP_CONVERT_SHARED:
2136 error = xfs_rmap_convert_shared(rcur, bno, blockcount,
2137 !unwritten, &oinfo);
2141 error = -EFSCORRUPTED;
2146 xfs_trans_brelse(tp, agbp);
2152 * Don't defer an rmap if we aren't an rmap filesystem.
2155 xfs_rmap_update_is_needed(
2156 struct xfs_mount *mp,
2159 return xfs_sb_version_hasrmapbt(&mp->m_sb) && whichfork != XFS_COW_FORK;
2163 * Record a rmap intent; the list is kept sorted first by AG and then by
2168 struct xfs_mount *mp,
2169 struct xfs_defer_ops *dfops,
2170 enum xfs_rmap_intent_type type,
2173 struct xfs_bmbt_irec *bmap)
2175 struct xfs_rmap_intent *ri;
2177 trace_xfs_rmap_defer(mp, XFS_FSB_TO_AGNO(mp, bmap->br_startblock),
2179 XFS_FSB_TO_AGBNO(mp, bmap->br_startblock),
2182 bmap->br_blockcount,
2185 ri = kmem_alloc(sizeof(struct xfs_rmap_intent), KM_SLEEP | KM_NOFS);
2186 INIT_LIST_HEAD(&ri->ri_list);
2188 ri->ri_owner = owner;
2189 ri->ri_whichfork = whichfork;
2190 ri->ri_bmap = *bmap;
2192 xfs_defer_add(dfops, XFS_DEFER_OPS_TYPE_RMAP, &ri->ri_list);
2196 /* Map an extent into a file. */
2198 xfs_rmap_map_extent(
2199 struct xfs_mount *mp,
2200 struct xfs_defer_ops *dfops,
2201 struct xfs_inode *ip,
2203 struct xfs_bmbt_irec *PREV)
2205 if (!xfs_rmap_update_is_needed(mp, whichfork))
2208 return __xfs_rmap_add(mp, dfops, xfs_is_reflink_inode(ip) ?
2209 XFS_RMAP_MAP_SHARED : XFS_RMAP_MAP, ip->i_ino,
2213 /* Unmap an extent out of a file. */
2215 xfs_rmap_unmap_extent(
2216 struct xfs_mount *mp,
2217 struct xfs_defer_ops *dfops,
2218 struct xfs_inode *ip,
2220 struct xfs_bmbt_irec *PREV)
2222 if (!xfs_rmap_update_is_needed(mp, whichfork))
2225 return __xfs_rmap_add(mp, dfops, xfs_is_reflink_inode(ip) ?
2226 XFS_RMAP_UNMAP_SHARED : XFS_RMAP_UNMAP, ip->i_ino,
2230 /* Convert a data fork extent from unwritten to real or vice versa. */
2232 xfs_rmap_convert_extent(
2233 struct xfs_mount *mp,
2234 struct xfs_defer_ops *dfops,
2235 struct xfs_inode *ip,
2237 struct xfs_bmbt_irec *PREV)
2239 if (!xfs_rmap_update_is_needed(mp, whichfork))
2242 return __xfs_rmap_add(mp, dfops, xfs_is_reflink_inode(ip) ?
2243 XFS_RMAP_CONVERT_SHARED : XFS_RMAP_CONVERT, ip->i_ino,
2247 /* Schedule the creation of an rmap for non-file data. */
2249 xfs_rmap_alloc_extent(
2250 struct xfs_mount *mp,
2251 struct xfs_defer_ops *dfops,
2252 xfs_agnumber_t agno,
2257 struct xfs_bmbt_irec bmap;
2259 if (!xfs_rmap_update_is_needed(mp, XFS_DATA_FORK))
2262 bmap.br_startblock = XFS_AGB_TO_FSB(mp, agno, bno);
2263 bmap.br_blockcount = len;
2264 bmap.br_startoff = 0;
2265 bmap.br_state = XFS_EXT_NORM;
2267 return __xfs_rmap_add(mp, dfops, XFS_RMAP_ALLOC, owner,
2268 XFS_DATA_FORK, &bmap);
2271 /* Schedule the deletion of an rmap for non-file data. */
2273 xfs_rmap_free_extent(
2274 struct xfs_mount *mp,
2275 struct xfs_defer_ops *dfops,
2276 xfs_agnumber_t agno,
2281 struct xfs_bmbt_irec bmap;
2283 if (!xfs_rmap_update_is_needed(mp, XFS_DATA_FORK))
2286 bmap.br_startblock = XFS_AGB_TO_FSB(mp, agno, bno);
2287 bmap.br_blockcount = len;
2288 bmap.br_startoff = 0;
2289 bmap.br_state = XFS_EXT_NORM;
2291 return __xfs_rmap_add(mp, dfops, XFS_RMAP_FREE, owner,
2292 XFS_DATA_FORK, &bmap);