fs/btrfs/scrub.c

   1 /*
   2  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/blkdev.h>
  20 #include <linux/ratelimit.h>
  21 #include "ctree.h"
  22 #include "volumes.h"
  23 #include "disk-io.h"
  24 #include "ordered-data.h"
  25 #include "transaction.h"
  26 #include "backref.h"
  27 #include "extent_io.h"
  28 #include "dev-replace.h"
  29 #include "check-integrity.h"
  30 #include "rcu-string.h"
  31 #include "raid56.h"
  32
  33 /*
  34  * This is only the first step towards a full-features scrub. It reads all
  35  * extent and super block and verifies the checksums. In case a bad checksum
  36  * is found or the extent cannot be read, good data will be written back if
  37  * any can be found.
  38  *
  39  * Future enhancements:
  40  *  - In case an unrepairable extent is encountered, track which files are
  41  *    affected and report them
  42  *  - track and record media errors, throw out bad devices
  43  *  - add a mode to also read unallocated space
  44  */
  45
  46 struct scrub_block;
  47 struct scrub_ctx;
  48
  49 /*
  50  * the following three values only influence the performance.
  51  * The last one configures the number of parallel and outstanding I/O
  52  * operations. The first two values configure an upper limit for the number
  53  * of (dynamically allocated) pages that are added to a bio.
  54  */
  55 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
  56 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
  57 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
  58
  59 /*
  60  * the following value times PAGE_SIZE needs to be large enough to match the
  61  * largest node/leaf/sector size that shall be supported.
  62  * Values larger than BTRFS_STRIPE_LEN are not supported.
  63  */
  64 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
  65
  66 struct scrub_page {
  67         struct scrub_block      *sblock;
  68         struct page             *page;
  69         struct btrfs_device     *dev;
  70         u64                     flags;  /* extent flags */
  71         u64                     generation;
  72         u64                     logical;
  73         u64                     physical;
  74         u64                     physical_for_dev_replace;
  75         atomic_t                ref_count;
  76         struct {
  77                 unsigned int    mirror_num:8;
  78                 unsigned int    have_csum:1;
  79                 unsigned int    io_error:1;
  80         };
  81         u8                      csum[BTRFS_CSUM_SIZE];
  82 };
  83
  84 struct scrub_bio {
  85         int                     index;
  86         struct scrub_ctx        *sctx;
  87         struct btrfs_device     *dev;
  88         struct bio              *bio;
  89         int                     err;
  90         u64                     logical;
  91         u64                     physical;
  92 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
  93         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
  94 #else
  95         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
  96 #endif
  97         int                     page_count;
  98         int                     next_free;
  99         struct btrfs_work       work;
 100 };
 101
 102 struct scrub_block {
 103         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
 104         int                     page_count;
 105         atomic_t                outstanding_pages;
 106         atomic_t                ref_count; /* free mem on transition to zero */
 107         struct scrub_ctx        *sctx;
 108         struct {
 109                 unsigned int    header_error:1;
 110                 unsigned int    checksum_error:1;
 111                 unsigned int    no_io_error_seen:1;
 112                 unsigned int    generation_error:1; /* also sets header_error */
 113         };
 114 };
 115
 116 struct scrub_wr_ctx {
 117         struct scrub_bio *wr_curr_bio;
 118         struct btrfs_device *tgtdev;
 119         int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
 120         atomic_t flush_all_writes;
 121         struct mutex wr_lock;
 122 };
 123
 124 struct scrub_ctx {
 125         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
 126         struct btrfs_root       *dev_root;
 127         int                     first_free;
 128         int                     curr;
 129         atomic_t                bios_in_flight;
 130         atomic_t                workers_pending;
 131         spinlock_t              list_lock;
 132         wait_queue_head_t       list_wait;
 133         u16                     csum_size;
 134         struct list_head        csum_list;
 135         atomic_t                cancel_req;
 136         int                     readonly;
 137         int                     pages_per_rd_bio;
 138         u32                     sectorsize;
 139         u32                     nodesize;
 140         u32                     leafsize;
 141
 142         int                     is_dev_replace;
 143         struct scrub_wr_ctx     wr_ctx;
 144
 145         /*
 146          * statistics
 147          */
 148         struct btrfs_scrub_progress stat;
 149         spinlock_t              stat_lock;
 150 };
 151
 152 struct scrub_fixup_nodatasum {
 153         struct scrub_ctx        *sctx;
 154         struct btrfs_device     *dev;
 155         u64                     logical;
 156         struct btrfs_root       *root;
 157         struct btrfs_work       work;
 158         int                     mirror_num;
 159 };
 160
 161 struct scrub_nocow_inode {
 162         u64                     inum;
 163         u64                     offset;
 164         u64                     root;
 165         struct list_head        list;
 166 };
 167
 168 struct scrub_copy_nocow_ctx {
 169         struct scrub_ctx        *sctx;
 170         u64                     logical;
 171         u64                     len;
 172         int                     mirror_num;
 173         u64                     physical_for_dev_replace;
 174         struct list_head        inodes;
 175         struct btrfs_work       work;
 176 };
 177
 178 struct scrub_warning {
 179         struct btrfs_path       *path;
 180         u64                     extent_item_size;
 181         char                    *scratch_buf;
 182         char                    *msg_buf;
 183         const char              *errstr;
 184         sector_t                sector;
 185         u64                     logical;
 186         struct btrfs_device     *dev;
 187         int                     msg_bufsize;
 188         int                     scratch_bufsize;
 189 };
 190
 191
 192 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
 193 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
 194 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
 195 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
 196 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
 197 static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
 198                                      struct btrfs_fs_info *fs_info,
 199                                      struct scrub_block *original_sblock,
 200                                      u64 length, u64 logical,
 201                                      struct scrub_block *sblocks_for_recheck);
 202 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
 203                                 struct scrub_block *sblock, int is_metadata,
 204                                 int have_csum, u8 *csum, u64 generation,
 205                                 u16 csum_size);
 206 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
 207                                          struct scrub_block *sblock,
 208                                          int is_metadata, int have_csum,
 209                                          const u8 *csum, u64 generation,
 210                                          u16 csum_size);
 211 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
 212                                              struct scrub_block *sblock_good,
 213                                              int force_write);
 214 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
 215                                             struct scrub_block *sblock_good,
 216                                             int page_num, int force_write);
 217 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
 218 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
 219                                            int page_num);
 220 static int scrub_checksum_data(struct scrub_block *sblock);
 221 static int scrub_checksum_tree_block(struct scrub_block *sblock);
 222 static int scrub_checksum_super(struct scrub_block *sblock);
 223 static void scrub_block_get(struct scrub_block *sblock);
 224 static void scrub_block_put(struct scrub_block *sblock);
 225 static void scrub_page_get(struct scrub_page *spage);
 226 static void scrub_page_put(struct scrub_page *spage);
 227 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
 228                                     struct scrub_page *spage);
 229 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 230                        u64 physical, struct btrfs_device *dev, u64 flags,
 231                        u64 gen, int mirror_num, u8 *csum, int force,
 232                        u64 physical_for_dev_replace);
 233 static void scrub_bio_end_io(struct bio *bio, int err);
 234 static void scrub_bio_end_io_worker(struct btrfs_work *work);
 235 static void scrub_block_complete(struct scrub_block *sblock);
 236 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
 237                                u64 extent_logical, u64 extent_len,
 238                                u64 *extent_physical,
 239                                struct btrfs_device **extent_dev,
 240                                int *extent_mirror_num);
 241 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
 242                               struct scrub_wr_ctx *wr_ctx,
 243                               struct btrfs_fs_info *fs_info,
 244                               struct btrfs_device *dev,
 245                               int is_dev_replace);
 246 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
 247 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
 248                                     struct scrub_page *spage);
 249 static void scrub_wr_submit(struct scrub_ctx *sctx);
 250 static void scrub_wr_bio_end_io(struct bio *bio, int err);
 251 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
 252 static int write_page_nocow(struct scrub_ctx *sctx,
 253                             u64 physical_for_dev_replace, struct page *page);
 254 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
 255                                       struct scrub_copy_nocow_ctx *ctx);
 256 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 257                             int mirror_num, u64 physical_for_dev_replace);
 258 static void copy_nocow_pages_worker(struct btrfs_work *work);
 259 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 260 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 261
 262
 263 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
 264 {
 265         atomic_inc(&sctx->bios_in_flight);
 266 }
 267
 268 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
 269 {
 270         atomic_dec(&sctx->bios_in_flight);
 271         wake_up(&sctx->list_wait);
 272 }
 273
 274 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 275 {
 276         while (atomic_read(&fs_info->scrub_pause_req)) {
 277                 mutex_unlock(&fs_info->scrub_lock);
 278                 wait_event(fs_info->scrub_pause_wait,
 279                    atomic_read(&fs_info->scrub_pause_req) == 0);
 280                 mutex_lock(&fs_info->scrub_lock);
 281         }
 282 }
 283
 284 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 285 {
 286         atomic_inc(&fs_info->scrubs_paused);
 287         wake_up(&fs_info->scrub_pause_wait);
 288
 289         mutex_lock(&fs_info->scrub_lock);
 290         __scrub_blocked_if_needed(fs_info);
 291         atomic_dec(&fs_info->scrubs_paused);
 292         mutex_unlock(&fs_info->scrub_lock);
 293
 294         wake_up(&fs_info->scrub_pause_wait);
 295 }
 296
 297 /*
 298  * used for workers that require transaction commits (i.e., for the
 299  * NOCOW case)
 300  */
 301 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
 302 {
 303         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 304
 305         /*
 306          * increment scrubs_running to prevent cancel requests from
 307          * completing as long as a worker is running. we must also
 308          * increment scrubs_paused to prevent deadlocking on pause
 309          * requests used for transactions commits (as the worker uses a
 310          * transaction context). it is safe to regard the worker
 311          * as paused for all matters practical. effectively, we only
 312          * avoid cancellation requests from completing.
 313          */
 314         mutex_lock(&fs_info->scrub_lock);
 315         atomic_inc(&fs_info->scrubs_running);
 316         atomic_inc(&fs_info->scrubs_paused);
 317         mutex_unlock(&fs_info->scrub_lock);
 318
 319         /*
 320          * check if @scrubs_running=@scrubs_paused condition
 321          * inside wait_event() is not an atomic operation.
 322          * which means we may inc/dec @scrub_running/paused
 323          * at any time. Let's wake up @scrub_pause_wait as
 324          * much as we can to let commit transaction blocked less.
 325          */
 326         wake_up(&fs_info->scrub_pause_wait);
 327
 328         atomic_inc(&sctx->workers_pending);
 329 }
 330
 331 /* used for workers that require transaction commits */
 332 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
 333 {
 334         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 335
 336         /*
 337          * see scrub_pending_trans_workers_inc() why we're pretending
 338          * to be paused in the scrub counters
 339          */
 340         mutex_lock(&fs_info->scrub_lock);
 341         atomic_dec(&fs_info->scrubs_running);
 342         atomic_dec(&fs_info->scrubs_paused);
 343         mutex_unlock(&fs_info->scrub_lock);
 344         atomic_dec(&sctx->workers_pending);
 345         wake_up(&fs_info->scrub_pause_wait);
 346         wake_up(&sctx->list_wait);
 347 }
 348
 349 static void scrub_free_csums(struct scrub_ctx *sctx)
 350 {
 351         while (!list_empty(&sctx->csum_list)) {
 352                 struct btrfs_ordered_sum *sum;
 353                 sum = list_first_entry(&sctx->csum_list,
 354                                        struct btrfs_ordered_sum, list);
 355                 list_del(&sum->list);
 356                 kfree(sum);
 357         }
 358 }
 359
 360 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
 361 {
 362         int i;
 363
 364         if (!sctx)
 365                 return;
 366
 367         scrub_free_wr_ctx(&sctx->wr_ctx);
 368
 369         /* this can happen when scrub is cancelled */
 370         if (sctx->curr != -1) {
 371                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
 372
 373                 for (i = 0; i < sbio->page_count; i++) {
 374                         WARN_ON(!sbio->pagev[i]->page);
 375                         scrub_block_put(sbio->pagev[i]->sblock);
 376                 }
 377                 bio_put(sbio->bio);
 378         }
 379
 380         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 381                 struct scrub_bio *sbio = sctx->bios[i];
 382
 383                 if (!sbio)
 384                         break;
 385                 kfree(sbio);
 386         }
 387
 388         scrub_free_csums(sctx);
 389         kfree(sctx);
 390 }
 391
 392 static noinline_for_stack
 393 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
 394 {
 395         struct scrub_ctx *sctx;
 396         int             i;
 397         struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
 398         int pages_per_rd_bio;
 399         int ret;
 400
 401         /*
 402          * the setting of pages_per_rd_bio is correct for scrub but might
 403          * be wrong for the dev_replace code where we might read from
 404          * different devices in the initial huge bios. However, that
 405          * code is able to correctly handle the case when adding a page
 406          * to a bio fails.
 407          */
 408         if (dev->bdev)
 409                 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
 410                                          bio_get_nr_vecs(dev->bdev));
 411         else
 412                 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
 413         sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
 414         if (!sctx)
 415                 goto nomem;
 416         sctx->is_dev_replace = is_dev_replace;
 417         sctx->pages_per_rd_bio = pages_per_rd_bio;
 418         sctx->curr = -1;
 419         sctx->dev_root = dev->dev_root;
 420         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 421                 struct scrub_bio *sbio;
 422
 423                 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
 424                 if (!sbio)
 425                         goto nomem;
 426                 sctx->bios[i] = sbio;
 427
 428                 sbio->index = i;
 429                 sbio->sctx = sctx;
 430                 sbio->page_count = 0;
 431                 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
 432                                 scrub_bio_end_io_worker, NULL, NULL);
 433
 434                 if (i != SCRUB_BIOS_PER_SCTX - 1)
 435                         sctx->bios[i]->next_free = i + 1;
 436                 else
 437                         sctx->bios[i]->next_free = -1;
 438         }
 439         sctx->first_free = 0;
 440         sctx->nodesize = dev->dev_root->nodesize;
 441         sctx->leafsize = dev->dev_root->leafsize;
 442         sctx->sectorsize = dev->dev_root->sectorsize;
 443         atomic_set(&sctx->bios_in_flight, 0);
 444         atomic_set(&sctx->workers_pending, 0);
 445         atomic_set(&sctx->cancel_req, 0);
 446         sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
 447         INIT_LIST_HEAD(&sctx->csum_list);
 448
 449         spin_lock_init(&sctx->list_lock);
 450         spin_lock_init(&sctx->stat_lock);
 451         init_waitqueue_head(&sctx->list_wait);
 452
 453         ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
 454                                  fs_info->dev_replace.tgtdev, is_dev_replace);
 455         if (ret) {
 456                 scrub_free_ctx(sctx);
 457                 return ERR_PTR(ret);
 458         }
 459         return sctx;
 460
 461 nomem:
 462         scrub_free_ctx(sctx);
 463         return ERR_PTR(-ENOMEM);
 464 }
 465
 466 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
 467                                      void *warn_ctx)
 468 {
 469         u64 isize;
 470         u32 nlink;
 471         int ret;
 472         int i;
 473         struct extent_buffer *eb;
 474         struct btrfs_inode_item *inode_item;
 475         struct scrub_warning *swarn = warn_ctx;
 476         struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
 477         struct inode_fs_paths *ipath = NULL;
 478         struct btrfs_root *local_root;
 479         struct btrfs_key root_key;
 480
 481         root_key.objectid = root;
 482         root_key.type = BTRFS_ROOT_ITEM_KEY;
 483         root_key.offset = (u64)-1;
 484         local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
 485         if (IS_ERR(local_root)) {
 486                 ret = PTR_ERR(local_root);
 487                 goto err;
 488         }
 489
 490         ret = inode_item_info(inum, 0, local_root, swarn->path);
 491         if (ret) {
 492                 btrfs_release_path(swarn->path);
 493                 goto err;
 494         }
 495
 496         eb = swarn->path->nodes[0];
 497         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
 498                                         struct btrfs_inode_item);
 499         isize = btrfs_inode_size(eb, inode_item);
 500         nlink = btrfs_inode_nlink(eb, inode_item);
 501         btrfs_release_path(swarn->path);
 502
 503         ipath = init_ipath(4096, local_root, swarn->path);
 504         if (IS_ERR(ipath)) {
 505                 ret = PTR_ERR(ipath);
 506                 ipath = NULL;
 507                 goto err;
 508         }
 509         ret = paths_from_inode(inum, ipath);
 510
 511         if (ret < 0)
 512                 goto err;
 513
 514         /*
 515          * we deliberately ignore the bit ipath might have been too small to
 516          * hold all of the paths here
 517          */
 518         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
 519                 printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
 520                         "%s, sector %llu, root %llu, inode %llu, offset %llu, "
 521                         "length %llu, links %u (path: %s)\n", swarn->errstr,
 522                         swarn->logical, rcu_str_deref(swarn->dev->name),
 523                         (unsigned long long)swarn->sector, root, inum, offset,
 524                         min(isize - offset, (u64)PAGE_SIZE), nlink,
 525                         (char *)(unsigned long)ipath->fspath->val[i]);
 526
 527         free_ipath(ipath);
 528         return 0;
 529
 530 err:
 531         printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
 532                 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
 533                 "resolving failed with ret=%d\n", swarn->errstr,
 534                 swarn->logical, rcu_str_deref(swarn->dev->name),
 535                 (unsigned long long)swarn->sector, root, inum, offset, ret);
 536
 537         free_ipath(ipath);
 538         return 0;
 539 }
 540
 541 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
 542 {
 543         struct btrfs_device *dev;
 544         struct btrfs_fs_info *fs_info;
 545         struct btrfs_path *path;
 546         struct btrfs_key found_key;
 547         struct extent_buffer *eb;
 548         struct btrfs_extent_item *ei;
 549         struct scrub_warning swarn;
 550         unsigned long ptr = 0;
 551         u64 extent_item_pos;
 552         u64 flags = 0;
 553         u64 ref_root;
 554         u32 item_size;
 555         u8 ref_level;
 556         const int bufsize = 4096;
 557         int ret;
 558
 559         WARN_ON(sblock->page_count < 1);
 560         dev = sblock->pagev[0]->dev;
 561         fs_info = sblock->sctx->dev_root->fs_info;
 562
 563         path = btrfs_alloc_path();
 564
 565         swarn.scratch_buf = kmalloc(bufsize, GFP_NOFS);
 566         swarn.msg_buf = kmalloc(bufsize, GFP_NOFS);
 567         swarn.sector = (sblock->pagev[0]->physical) >> 9;
 568         swarn.logical = sblock->pagev[0]->logical;
 569         swarn.errstr = errstr;
 570         swarn.dev = NULL;
 571         swarn.msg_bufsize = bufsize;
 572         swarn.scratch_bufsize = bufsize;
 573
 574         if (!path || !swarn.scratch_buf || !swarn.msg_buf)
 575                 goto out;
 576
 577         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
 578                                   &flags);
 579         if (ret < 0)
 580                 goto out;
 581
 582         extent_item_pos = swarn.logical - found_key.objectid;
 583         swarn.extent_item_size = found_key.offset;
 584
 585         eb = path->nodes[0];
 586         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
 587         item_size = btrfs_item_size_nr(eb, path->slots[0]);
 588
 589         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 590                 do {
 591                         ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
 592                                                       item_size, &ref_root,
 593                                                       &ref_level);
 594                         printk_in_rcu(KERN_WARNING
 595                                 "BTRFS: %s at logical %llu on dev %s, "
 596                                 "sector %llu: metadata %s (level %d) in tree "
 597                                 "%llu\n", errstr, swarn.logical,
 598                                 rcu_str_deref(dev->name),
 599                                 (unsigned long long)swarn.sector,
 600                                 ref_level ? "node" : "leaf",
 601                                 ret < 0 ? -1 : ref_level,
 602                                 ret < 0 ? -1 : ref_root);
 603                 } while (ret != 1);
 604                 btrfs_release_path(path);
 605         } else {
 606                 btrfs_release_path(path);
 607                 swarn.path = path;
 608                 swarn.dev = dev;
 609                 iterate_extent_inodes(fs_info, found_key.objectid,
 610                                         extent_item_pos, 1,
 611                                         scrub_print_warning_inode, &swarn);
 612         }
 613
 614 out:
 615         btrfs_free_path(path);
 616         kfree(swarn.scratch_buf);
 617         kfree(swarn.msg_buf);
 618 }
 619
 620 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
 621 {
 622         struct page *page = NULL;
 623         unsigned long index;
 624         struct scrub_fixup_nodatasum *fixup = fixup_ctx;
 625         int ret;
 626         int corrected = 0;
 627         struct btrfs_key key;
 628         struct inode *inode = NULL;
 629         struct btrfs_fs_info *fs_info;
 630         u64 end = offset + PAGE_SIZE - 1;
 631         struct btrfs_root *local_root;
 632         int srcu_index;
 633
 634         key.objectid = root;
 635         key.type = BTRFS_ROOT_ITEM_KEY;
 636         key.offset = (u64)-1;
 637
 638         fs_info = fixup->root->fs_info;
 639         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
 640
 641         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
 642         if (IS_ERR(local_root)) {
 643                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 644                 return PTR_ERR(local_root);
 645         }
 646
 647         key.type = BTRFS_INODE_ITEM_KEY;
 648         key.objectid = inum;
 649         key.offset = 0;
 650         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
 651         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 652         if (IS_ERR(inode))
 653                 return PTR_ERR(inode);
 654
 655         index = offset >> PAGE_CACHE_SHIFT;
 656
 657         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
 658         if (!page) {
 659                 ret = -ENOMEM;
 660                 goto out;
 661         }
 662
 663         if (PageUptodate(page)) {
 664                 if (PageDirty(page)) {
 665                         /*
 666                          * we need to write the data to the defect sector. the
 667                          * data that was in that sector is not in memory,
 668                          * because the page was modified. we must not write the
 669                          * modified page to that sector.
 670                          *
 671                          * TODO: what could be done here: wait for the delalloc
 672                          *       runner to write out that page (might involve
 673                          *       COW) and see whether the sector is still
 674                          *       referenced afterwards.
 675                          *
 676                          * For the meantime, we'll treat this error
 677                          * incorrectable, although there is a chance that a
 678                          * later scrub will find the bad sector again and that
 679                          * there's no dirty page in memory, then.
 680                          */
 681                         ret = -EIO;
 682                         goto out;
 683                 }
 684                 fs_info = BTRFS_I(inode)->root->fs_info;
 685                 ret = repair_io_failure(fs_info, offset, PAGE_SIZE,
 686                                         fixup->logical, page,
 687                                         fixup->mirror_num);
 688                 unlock_page(page);
 689                 corrected = !ret;
 690         } else {
 691                 /*
 692                  * we need to get good data first. the general readpage path
 693                  * will call repair_io_failure for us, we just have to make
 694                  * sure we read the bad mirror.
 695                  */
 696                 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 697                                         EXTENT_DAMAGED, GFP_NOFS);
 698                 if (ret) {
 699                         /* set_extent_bits should give proper error */
 700                         WARN_ON(ret > 0);
 701                         if (ret > 0)
 702                                 ret = -EFAULT;
 703                         goto out;
 704                 }
 705
 706                 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
 707                                                 btrfs_get_extent,
 708                                                 fixup->mirror_num);
 709                 wait_on_page_locked(page);
 710
 711                 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
 712                                                 end, EXTENT_DAMAGED, 0, NULL);
 713                 if (!corrected)
 714                         clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 715                                                 EXTENT_DAMAGED, GFP_NOFS);
 716         }
 717
 718 out:
 719         if (page)
 720                 put_page(page);
 721
 722         iput(inode);
 723
 724         if (ret < 0)
 725                 return ret;
 726
 727         if (ret == 0 && corrected) {
 728                 /*
 729                  * we only need to call readpage for one of the inodes belonging
 730                  * to this extent. so make iterate_extent_inodes stop
 731                  */
 732                 return 1;
 733         }
 734
 735         return -EIO;
 736 }
 737
 738 static void scrub_fixup_nodatasum(struct btrfs_work *work)
 739 {
 740         int ret;
 741         struct scrub_fixup_nodatasum *fixup;
 742         struct scrub_ctx *sctx;
 743         struct btrfs_trans_handle *trans = NULL;
 744         struct btrfs_path *path;
 745         int uncorrectable = 0;
 746
 747         fixup = container_of(work, struct scrub_fixup_nodatasum, work);
 748         sctx = fixup->sctx;
 749
 750         path = btrfs_alloc_path();
 751         if (!path) {
 752                 spin_lock(&sctx->stat_lock);
 753                 ++sctx->stat.malloc_errors;
 754                 spin_unlock(&sctx->stat_lock);
 755                 uncorrectable = 1;
 756                 goto out;
 757         }
 758
 759         trans = btrfs_join_transaction(fixup->root);
 760         if (IS_ERR(trans)) {
 761                 uncorrectable = 1;
 762                 goto out;
 763         }
 764
 765         /*
 766          * the idea is to trigger a regular read through the standard path. we
 767          * read a page from the (failed) logical address by specifying the
 768          * corresponding copynum of the failed sector. thus, that readpage is
 769          * expected to fail.
 770          * that is the point where on-the-fly error correction will kick in
 771          * (once it's finished) and rewrite the failed sector if a good copy
 772          * can be found.
 773          */
 774         ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
 775                                                 path, scrub_fixup_readpage,
 776                                                 fixup);
 777         if (ret < 0) {
 778                 uncorrectable = 1;
 779                 goto out;
 780         }
 781         WARN_ON(ret != 1);
 782
 783         spin_lock(&sctx->stat_lock);
 784         ++sctx->stat.corrected_errors;
 785         spin_unlock(&sctx->stat_lock);
 786
 787 out:
 788         if (trans && !IS_ERR(trans))
 789                 btrfs_end_transaction(trans, fixup->root);
 790         if (uncorrectable) {
 791                 spin_lock(&sctx->stat_lock);
 792                 ++sctx->stat.uncorrectable_errors;
 793                 spin_unlock(&sctx->stat_lock);
 794                 btrfs_dev_replace_stats_inc(
 795                         &sctx->dev_root->fs_info->dev_replace.
 796                         num_uncorrectable_read_errors);
 797                 printk_ratelimited_in_rcu(KERN_ERR "BTRFS: "
 798                     "unable to fixup (nodatasum) error at logical %llu on dev %s\n",
 799                         fixup->logical, rcu_str_deref(fixup->dev->name));
 800         }
 801
 802         btrfs_free_path(path);
 803         kfree(fixup);
 804
 805         scrub_pending_trans_workers_dec(sctx);
 806 }
 807
 808 /*
 809  * scrub_handle_errored_block gets called when either verification of the
 810  * pages failed or the bio failed to read, e.g. with EIO. In the latter
 811  * case, this function handles all pages in the bio, even though only one
 812  * may be bad.
 813  * The goal of this function is to repair the errored block by using the
 814  * contents of one of the mirrors.
 815  */
 816 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
 817 {
 818         struct scrub_ctx *sctx = sblock_to_check->sctx;
 819         struct btrfs_device *dev;
 820         struct btrfs_fs_info *fs_info;
 821         u64 length;
 822         u64 logical;
 823         u64 generation;
 824         unsigned int failed_mirror_index;
 825         unsigned int is_metadata;
 826         unsigned int have_csum;
 827         u8 *csum;
 828         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
 829         struct scrub_block *sblock_bad;
 830         int ret;
 831         int mirror_index;
 832         int page_num;
 833         int success;
 834         static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
 835                                       DEFAULT_RATELIMIT_BURST);
 836
 837         BUG_ON(sblock_to_check->page_count < 1);
 838         fs_info = sctx->dev_root->fs_info;
 839         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
 840                 /*
 841                  * if we find an error in a super block, we just report it.
 842                  * They will get written with the next transaction commit
 843                  * anyway
 844                  */
 845                 spin_lock(&sctx->stat_lock);
 846                 ++sctx->stat.super_errors;
 847                 spin_unlock(&sctx->stat_lock);
 848                 return 0;
 849         }
 850         length = sblock_to_check->page_count * PAGE_SIZE;
 851         logical = sblock_to_check->pagev[0]->logical;
 852         generation = sblock_to_check->pagev[0]->generation;
 853         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
 854         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
 855         is_metadata = !(sblock_to_check->pagev[0]->flags &
 856                         BTRFS_EXTENT_FLAG_DATA);
 857         have_csum = sblock_to_check->pagev[0]->have_csum;
 858         csum = sblock_to_check->pagev[0]->csum;
 859         dev = sblock_to_check->pagev[0]->dev;
 860
 861         if (sctx->is_dev_replace && !is_metadata && !have_csum) {
 862                 sblocks_for_recheck = NULL;
 863                 goto nodatasum_case;
 864         }
 865
 866         /*
 867          * read all mirrors one after the other. This includes to
 868          * re-read the extent or metadata block that failed (that was
 869          * the cause that this fixup code is called) another time,
 870          * page by page this time in order to know which pages
 871          * caused I/O errors and which ones are good (for all mirrors).
 872          * It is the goal to handle the situation when more than one
 873          * mirror contains I/O errors, but the errors do not
 874          * overlap, i.e. the data can be repaired by selecting the
 875          * pages from those mirrors without I/O error on the
 876          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
 877          * would be that mirror #1 has an I/O error on the first page,
 878          * the second page is good, and mirror #2 has an I/O error on
 879          * the second page, but the first page is good.
 880          * Then the first page of the first mirror can be repaired by
 881          * taking the first page of the second mirror, and the
 882          * second page of the second mirror can be repaired by
 883          * copying the contents of the 2nd page of the 1st mirror.
 884          * One more note: if the pages of one mirror contain I/O
 885          * errors, the checksum cannot be verified. In order to get
 886          * the best data for repairing, the first attempt is to find
 887          * a mirror without I/O errors and with a validated checksum.
 888          * Only if this is not possible, the pages are picked from
 889          * mirrors with I/O errors without considering the checksum.
 890          * If the latter is the case, at the end, the checksum of the
 891          * repaired area is verified in order to correctly maintain
 892          * the statistics.
 893          */
 894
 895         sblocks_for_recheck = kzalloc(BTRFS_MAX_MIRRORS *
 896                                      sizeof(*sblocks_for_recheck),
 897                                      GFP_NOFS);
 898         if (!sblocks_for_recheck) {
 899                 spin_lock(&sctx->stat_lock);
 900                 sctx->stat.malloc_errors++;
 901                 sctx->stat.read_errors++;
 902                 sctx->stat.uncorrectable_errors++;
 903                 spin_unlock(&sctx->stat_lock);
 904                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 905                 goto out;
 906         }
 907
 908         /* setup the context, map the logical blocks and alloc the pages */
 909         ret = scrub_setup_recheck_block(sctx, fs_info, sblock_to_check, length,
 910                                         logical, sblocks_for_recheck);
 911         if (ret) {
 912                 spin_lock(&sctx->stat_lock);
 913                 sctx->stat.read_errors++;
 914                 sctx->stat.uncorrectable_errors++;
 915                 spin_unlock(&sctx->stat_lock);
 916                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 917                 goto out;
 918         }
 919         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
 920         sblock_bad = sblocks_for_recheck + failed_mirror_index;
 921
 922         /* build and submit the bios for the failed mirror, check checksums */
 923         scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
 924                             csum, generation, sctx->csum_size);
 925
 926         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
 927             sblock_bad->no_io_error_seen) {
 928                 /*
 929                  * the error disappeared after reading page by page, or
 930                  * the area was part of a huge bio and other parts of the
 931                  * bio caused I/O errors, or the block layer merged several
 932                  * read requests into one and the error is caused by a
 933                  * different bio (usually one of the two latter cases is
 934                  * the cause)
 935                  */
 936                 spin_lock(&sctx->stat_lock);
 937                 sctx->stat.unverified_errors++;
 938                 spin_unlock(&sctx->stat_lock);
 939
 940                 if (sctx->is_dev_replace)
 941                         scrub_write_block_to_dev_replace(sblock_bad);
 942                 goto out;
 943         }
 944
 945         if (!sblock_bad->no_io_error_seen) {
 946                 spin_lock(&sctx->stat_lock);
 947                 sctx->stat.read_errors++;
 948                 spin_unlock(&sctx->stat_lock);
 949                 if (__ratelimit(&_rs))
 950                         scrub_print_warning("i/o error", sblock_to_check);
 951                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 952         } else if (sblock_bad->checksum_error) {
 953                 spin_lock(&sctx->stat_lock);
 954                 sctx->stat.csum_errors++;
 955                 spin_unlock(&sctx->stat_lock);
 956                 if (__ratelimit(&_rs))
 957                         scrub_print_warning("checksum error", sblock_to_check);
 958                 btrfs_dev_stat_inc_and_print(dev,
 959                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
 960         } else if (sblock_bad->header_error) {
 961                 spin_lock(&sctx->stat_lock);
 962                 sctx->stat.verify_errors++;
 963                 spin_unlock(&sctx->stat_lock);
 964                 if (__ratelimit(&_rs))
 965                         scrub_print_warning("checksum/header error",
 966                                             sblock_to_check);
 967                 if (sblock_bad->generation_error)
 968                         btrfs_dev_stat_inc_and_print(dev,
 969                                 BTRFS_DEV_STAT_GENERATION_ERRS);
 970                 else
 971                         btrfs_dev_stat_inc_and_print(dev,
 972                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
 973         }
 974
 975         if (sctx->readonly) {
 976                 ASSERT(!sctx->is_dev_replace);
 977                 goto out;
 978         }
 979
 980         if (!is_metadata && !have_csum) {
 981                 struct scrub_fixup_nodatasum *fixup_nodatasum;
 982
 983 nodatasum_case:
 984                 WARN_ON(sctx->is_dev_replace);
 985
 986                 /*
 987                  * !is_metadata and !have_csum, this means that the data
 988                  * might not be COW'ed, that it might be modified
 989                  * concurrently. The general strategy to work on the
 990                  * commit root does not help in the case when COW is not
 991                  * used.
 992                  */
 993                 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
 994                 if (!fixup_nodatasum)
 995                         goto did_not_correct_error;
 996                 fixup_nodatasum->sctx = sctx;
 997                 fixup_nodatasum->dev = dev;
 998                 fixup_nodatasum->logical = logical;
 999                 fixup_nodatasum->root = fs_info->extent_root;
1000                 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
1001                 scrub_pending_trans_workers_inc(sctx);
1002                 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1003                                 scrub_fixup_nodatasum, NULL, NULL);
1004                 btrfs_queue_work(fs_info->scrub_workers,
1005                                  &fixup_nodatasum->work);
1006                 goto out;
1007         }
1008
1009         /*
1010          * now build and submit the bios for the other mirrors, check
1011          * checksums.
1012          * First try to pick the mirror which is completely without I/O
1013          * errors and also does not have a checksum error.
1014          * If one is found, and if a checksum is present, the full block
1015          * that is known to contain an error is rewritten. Afterwards
1016          * the block is known to be corrected.
1017          * If a mirror is found which is completely correct, and no
1018          * checksum is present, only those pages are rewritten that had
1019          * an I/O error in the block to be repaired, since it cannot be
1020          * determined, which copy of the other pages is better (and it
1021          * could happen otherwise that a correct page would be
1022          * overwritten by a bad one).
1023          */
1024         for (mirror_index = 0;
1025              mirror_index < BTRFS_MAX_MIRRORS &&
1026              sblocks_for_recheck[mirror_index].page_count > 0;
1027              mirror_index++) {
1028                 struct scrub_block *sblock_other;
1029
1030                 if (mirror_index == failed_mirror_index)
1031                         continue;
1032                 sblock_other = sblocks_for_recheck + mirror_index;
1033
1034                 /* build and submit the bios, check checksums */
1035                 scrub_recheck_block(fs_info, sblock_other, is_metadata,
1036                                     have_csum, csum, generation,
1037                                     sctx->csum_size);
1038
1039                 if (!sblock_other->header_error &&
1040                     !sblock_other->checksum_error &&
1041                     sblock_other->no_io_error_seen) {
1042                         if (sctx->is_dev_replace) {
1043                                 scrub_write_block_to_dev_replace(sblock_other);
1044                         } else {
1045                                 int force_write = is_metadata || have_csum;
1046
1047                                 ret = scrub_repair_block_from_good_copy(
1048                                                 sblock_bad, sblock_other,
1049                                                 force_write);
1050                         }
1051                         if (0 == ret)
1052                                 goto corrected_error;
1053                 }
1054         }
1055
1056         /*
1057          * for dev_replace, pick good pages and write to the target device.
1058          */
1059         if (sctx->is_dev_replace) {
1060                 success = 1;
1061                 for (page_num = 0; page_num < sblock_bad->page_count;
1062                      page_num++) {
1063                         int sub_success;
1064
1065                         sub_success = 0;
1066                         for (mirror_index = 0;
1067                              mirror_index < BTRFS_MAX_MIRRORS &&
1068                              sblocks_for_recheck[mirror_index].page_count > 0;
1069                              mirror_index++) {
1070                                 struct scrub_block *sblock_other =
1071                                         sblocks_for_recheck + mirror_index;
1072                                 struct scrub_page *page_other =
1073                                         sblock_other->pagev[page_num];
1074
1075                                 if (!page_other->io_error) {
1076                                         ret = scrub_write_page_to_dev_replace(
1077                                                         sblock_other, page_num);
1078                                         if (ret == 0) {
1079                                                 /* succeeded for this page */
1080                                                 sub_success = 1;
1081                                                 break;
1082                                         } else {
1083                                                 btrfs_dev_replace_stats_inc(
1084                                                         &sctx->dev_root->
1085                                                         fs_info->dev_replace.
1086                                                         num_write_errors);
1087                                         }
1088                                 }
1089                         }
1090
1091                         if (!sub_success) {
1092                                 /*
1093                                  * did not find a mirror to fetch the page
1094                                  * from. scrub_write_page_to_dev_replace()
1095                                  * handles this case (page->io_error), by
1096                                  * filling the block with zeros before
1097                                  * submitting the write request
1098                                  */
1099                                 success = 0;
1100                                 ret = scrub_write_page_to_dev_replace(
1101                                                 sblock_bad, page_num);
1102                                 if (ret)
1103                                         btrfs_dev_replace_stats_inc(
1104                                                 &sctx->dev_root->fs_info->
1105                                                 dev_replace.num_write_errors);
1106                         }
1107                 }
1108
1109                 goto out;
1110         }
1111
1112         /*
1113          * for regular scrub, repair those pages that are errored.
1114          * In case of I/O errors in the area that is supposed to be
1115          * repaired, continue by picking good copies of those pages.
1116          * Select the good pages from mirrors to rewrite bad pages from
1117          * the area to fix. Afterwards verify the checksum of the block
1118          * that is supposed to be repaired. This verification step is
1119          * only done for the purpose of statistic counting and for the
1120          * final scrub report, whether errors remain.
1121          * A perfect algorithm could make use of the checksum and try
1122          * all possible combinations of pages from the different mirrors
1123          * until the checksum verification succeeds. For example, when
1124          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1125          * of mirror #2 is readable but the final checksum test fails,
1126          * then the 2nd page of mirror #3 could be tried, whether now
1127          * the final checksum succeedes. But this would be a rare
1128          * exception and is therefore not implemented. At least it is
1129          * avoided that the good copy is overwritten.
1130          * A more useful improvement would be to pick the sectors
1131          * without I/O error based on sector sizes (512 bytes on legacy
1132          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1133          * mirror could be repaired by taking 512 byte of a different
1134          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1135          * area are unreadable.
1136          */
1137
1138         /* can only fix I/O errors from here on */
1139         if (sblock_bad->no_io_error_seen)
1140                 goto did_not_correct_error;
1141
1142         success = 1;
1143         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1144                 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1145
1146                 if (!page_bad->io_error)
1147                         continue;
1148
1149                 for (mirror_index = 0;
1150                      mirror_index < BTRFS_MAX_MIRRORS &&
1151                      sblocks_for_recheck[mirror_index].page_count > 0;
1152                      mirror_index++) {
1153                         struct scrub_block *sblock_other = sblocks_for_recheck +
1154                                                            mirror_index;
1155                         struct scrub_page *page_other = sblock_other->pagev[
1156                                                         page_num];
1157
1158                         if (!page_other->io_error) {
1159                                 ret = scrub_repair_page_from_good_copy(
1160                                         sblock_bad, sblock_other, page_num, 0);
1161                                 if (0 == ret) {
1162                                         page_bad->io_error = 0;
1163                                         break; /* succeeded for this page */
1164                                 }
1165                         }
1166                 }
1167
1168                 if (page_bad->io_error) {
1169                         /* did not find a mirror to copy the page from */
1170                         success = 0;
1171                 }
1172         }
1173
1174         if (success) {
1175                 if (is_metadata || have_csum) {
1176                         /*
1177                          * need to verify the checksum now that all
1178                          * sectors on disk are repaired (the write
1179                          * request for data to be repaired is on its way).
1180                          * Just be lazy and use scrub_recheck_block()
1181                          * which re-reads the data before the checksum
1182                          * is verified, but most likely the data comes out
1183                          * of the page cache.
1184                          */
1185                         scrub_recheck_block(fs_info, sblock_bad,
1186                                             is_metadata, have_csum, csum,
1187                                             generation, sctx->csum_size);
1188                         if (!sblock_bad->header_error &&
1189                             !sblock_bad->checksum_error &&
1190                             sblock_bad->no_io_error_seen)
1191                                 goto corrected_error;
1192                         else
1193                                 goto did_not_correct_error;
1194                 } else {
1195 corrected_error:
1196                         spin_lock(&sctx->stat_lock);
1197                         sctx->stat.corrected_errors++;
1198                         spin_unlock(&sctx->stat_lock);
1199                         printk_ratelimited_in_rcu(KERN_ERR
1200                                 "BTRFS: fixed up error at logical %llu on dev %s\n",
1201                                 logical, rcu_str_deref(dev->name));
1202                 }
1203         } else {
1204 did_not_correct_error:
1205                 spin_lock(&sctx->stat_lock);
1206                 sctx->stat.uncorrectable_errors++;
1207                 spin_unlock(&sctx->stat_lock);
1208                 printk_ratelimited_in_rcu(KERN_ERR
1209                         "BTRFS: unable to fixup (regular) error at logical %llu on dev %s\n",
1210                         logical, rcu_str_deref(dev->name));
1211         }
1212
1213 out:
1214         if (sblocks_for_recheck) {
1215                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1216                      mirror_index++) {
1217                         struct scrub_block *sblock = sblocks_for_recheck +
1218                                                      mirror_index;
1219                         int page_index;
1220
1221                         for (page_index = 0; page_index < sblock->page_count;
1222                              page_index++) {
1223                                 sblock->pagev[page_index]->sblock = NULL;
1224                                 scrub_page_put(sblock->pagev[page_index]);
1225                         }
1226                 }
1227                 kfree(sblocks_for_recheck);
1228         }
1229
1230         return 0;
1231 }
1232
1233 static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
1234                                      struct btrfs_fs_info *fs_info,
1235                                      struct scrub_block *original_sblock,
1236                                      u64 length, u64 logical,
1237                                      struct scrub_block *sblocks_for_recheck)
1238 {
1239         int page_index;
1240         int mirror_index;
1241         int ret;
1242
1243         /*
1244          * note: the two members ref_count and outstanding_pages
1245          * are not used (and not set) in the blocks that are used for
1246          * the recheck procedure
1247          */
1248
1249         page_index = 0;
1250         while (length > 0) {
1251                 u64 sublen = min_t(u64, length, PAGE_SIZE);
1252                 u64 mapped_length = sublen;
1253                 struct btrfs_bio *bbio = NULL;
1254
1255                 /*
1256                  * with a length of PAGE_SIZE, each returned stripe
1257                  * represents one mirror
1258                  */
1259                 ret = btrfs_map_block(fs_info, REQ_GET_READ_MIRRORS, logical,
1260                                       &mapped_length, &bbio, 0);
1261                 if (ret || !bbio || mapped_length < sublen) {
1262                         kfree(bbio);
1263                         return -EIO;
1264                 }
1265
1266                 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
1267                 for (mirror_index = 0; mirror_index < (int)bbio->num_stripes;
1268                      mirror_index++) {
1269                         struct scrub_block *sblock;
1270                         struct scrub_page *page;
1271
1272                         if (mirror_index >= BTRFS_MAX_MIRRORS)
1273                                 continue;
1274
1275                         sblock = sblocks_for_recheck + mirror_index;
1276                         sblock->sctx = sctx;
1277                         page = kzalloc(sizeof(*page), GFP_NOFS);
1278                         if (!page) {
1279 leave_nomem:
1280                                 spin_lock(&sctx->stat_lock);
1281                                 sctx->stat.malloc_errors++;
1282                                 spin_unlock(&sctx->stat_lock);
1283                                 kfree(bbio);
1284                                 return -ENOMEM;
1285                         }
1286                         scrub_page_get(page);
1287                         sblock->pagev[page_index] = page;
1288                         page->logical = logical;
1289                         page->physical = bbio->stripes[mirror_index].physical;
1290                         BUG_ON(page_index >= original_sblock->page_count);
1291                         page->physical_for_dev_replace =
1292                                 original_sblock->pagev[page_index]->
1293                                 physical_for_dev_replace;
1294                         /* for missing devices, dev->bdev is NULL */
1295                         page->dev = bbio->stripes[mirror_index].dev;
1296                         page->mirror_num = mirror_index + 1;
1297                         sblock->page_count++;
1298                         page->page = alloc_page(GFP_NOFS);
1299                         if (!page->page)
1300                                 goto leave_nomem;
1301                 }
1302                 kfree(bbio);
1303                 length -= sublen;
1304                 logical += sublen;
1305                 page_index++;
1306         }
1307
1308         return 0;
1309 }
1310
1311 /*
1312  * this function will check the on disk data for checksum errors, header
1313  * errors and read I/O errors. If any I/O errors happen, the exact pages
1314  * which are errored are marked as being bad. The goal is to enable scrub
1315  * to take those pages that are not errored from all the mirrors so that
1316  * the pages that are errored in the just handled mirror can be repaired.
1317  */
1318 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1319                                 struct scrub_block *sblock, int is_metadata,
1320                                 int have_csum, u8 *csum, u64 generation,
1321                                 u16 csum_size)
1322 {
1323         int page_num;
1324
1325         sblock->no_io_error_seen = 1;
1326         sblock->header_error = 0;
1327         sblock->checksum_error = 0;
1328
1329         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1330                 struct bio *bio;
1331                 struct scrub_page *page = sblock->pagev[page_num];
1332
1333                 if (page->dev->bdev == NULL) {
1334                         page->io_error = 1;
1335                         sblock->no_io_error_seen = 0;
1336                         continue;
1337                 }
1338
1339                 WARN_ON(!page->page);
1340                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1341                 if (!bio) {
1342                         page->io_error = 1;
1343                         sblock->no_io_error_seen = 0;
1344                         continue;
1345                 }
1346                 bio->bi_bdev = page->dev->bdev;
1347                 bio->bi_iter.bi_sector = page->physical >> 9;
1348
1349                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1350                 if (btrfsic_submit_bio_wait(READ, bio))
1351                         sblock->no_io_error_seen = 0;
1352
1353                 bio_put(bio);
1354         }
1355
1356         if (sblock->no_io_error_seen)
1357                 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1358                                              have_csum, csum, generation,
1359                                              csum_size);
1360
1361         return;
1362 }
1363
1364 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1365                                          struct scrub_block *sblock,
1366                                          int is_metadata, int have_csum,
1367                                          const u8 *csum, u64 generation,
1368                                          u16 csum_size)
1369 {
1370         int page_num;
1371         u8 calculated_csum[BTRFS_CSUM_SIZE];
1372         u32 crc = ~(u32)0;
1373         void *mapped_buffer;
1374
1375         WARN_ON(!sblock->pagev[0]->page);
1376         if (is_metadata) {
1377                 struct btrfs_header *h;
1378
1379                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1380                 h = (struct btrfs_header *)mapped_buffer;
1381
1382                 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||
1383                     memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE) ||
1384                     memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1385                            BTRFS_UUID_SIZE)) {
1386                         sblock->header_error = 1;
1387                 } else if (generation != btrfs_stack_header_generation(h)) {
1388                         sblock->header_error = 1;
1389                         sblock->generation_error = 1;
1390                 }
1391                 csum = h->csum;
1392         } else {
1393                 if (!have_csum)
1394                         return;
1395
1396                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1397         }
1398
1399         for (page_num = 0;;) {
1400                 if (page_num == 0 && is_metadata)
1401                         crc = btrfs_csum_data(
1402                                 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1403                                 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1404                 else
1405                         crc = btrfs_csum_data(mapped_buffer, crc, PAGE_SIZE);
1406
1407                 kunmap_atomic(mapped_buffer);
1408                 page_num++;
1409                 if (page_num >= sblock->page_count)
1410                         break;
1411                 WARN_ON(!sblock->pagev[page_num]->page);
1412
1413                 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
1414         }
1415
1416         btrfs_csum_final(crc, calculated_csum);
1417         if (memcmp(calculated_csum, csum, csum_size))
1418                 sblock->checksum_error = 1;
1419 }
1420
1421 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1422                                              struct scrub_block *sblock_good,
1423                                              int force_write)
1424 {
1425         int page_num;
1426         int ret = 0;
1427
1428         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1429                 int ret_sub;
1430
1431                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1432                                                            sblock_good,
1433                                                            page_num,
1434                                                            force_write);
1435                 if (ret_sub)
1436                         ret = ret_sub;
1437         }
1438
1439         return ret;
1440 }
1441
1442 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1443                                             struct scrub_block *sblock_good,
1444                                             int page_num, int force_write)
1445 {
1446         struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1447         struct scrub_page *page_good = sblock_good->pagev[page_num];
1448
1449         BUG_ON(page_bad->page == NULL);
1450         BUG_ON(page_good->page == NULL);
1451         if (force_write || sblock_bad->header_error ||
1452             sblock_bad->checksum_error || page_bad->io_error) {
1453                 struct bio *bio;
1454                 int ret;
1455
1456                 if (!page_bad->dev->bdev) {
1457                         printk_ratelimited(KERN_WARNING "BTRFS: "
1458                                 "scrub_repair_page_from_good_copy(bdev == NULL) "
1459                                 "is unexpected!\n");
1460                         return -EIO;
1461                 }
1462
1463                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1464                 if (!bio)
1465                         return -EIO;
1466                 bio->bi_bdev = page_bad->dev->bdev;
1467                 bio->bi_iter.bi_sector = page_bad->physical >> 9;
1468
1469                 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1470                 if (PAGE_SIZE != ret) {
1471                         bio_put(bio);
1472                         return -EIO;
1473                 }
1474
1475                 if (btrfsic_submit_bio_wait(WRITE, bio)) {
1476                         btrfs_dev_stat_inc_and_print(page_bad->dev,
1477                                 BTRFS_DEV_STAT_WRITE_ERRS);
1478                         btrfs_dev_replace_stats_inc(
1479                                 &sblock_bad->sctx->dev_root->fs_info->
1480                                 dev_replace.num_write_errors);
1481                         bio_put(bio);
1482                         return -EIO;
1483                 }
1484                 bio_put(bio);
1485         }
1486
1487         return 0;
1488 }
1489
1490 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1491 {
1492         int page_num;
1493
1494         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1495                 int ret;
1496
1497                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1498                 if (ret)
1499                         btrfs_dev_replace_stats_inc(
1500                                 &sblock->sctx->dev_root->fs_info->dev_replace.
1501                                 num_write_errors);
1502         }
1503 }
1504
1505 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1506                                            int page_num)
1507 {
1508         struct scrub_page *spage = sblock->pagev[page_num];
1509
1510         BUG_ON(spage->page == NULL);
1511         if (spage->io_error) {
1512                 void *mapped_buffer = kmap_atomic(spage->page);
1513
1514                 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1515                 flush_dcache_page(spage->page);
1516                 kunmap_atomic(mapped_buffer);
1517         }
1518         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1519 }
1520
1521 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1522                                     struct scrub_page *spage)
1523 {
1524         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1525         struct scrub_bio *sbio;
1526         int ret;
1527
1528         mutex_lock(&wr_ctx->wr_lock);
1529 again:
1530         if (!wr_ctx->wr_curr_bio) {
1531                 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1532                                               GFP_NOFS);
1533                 if (!wr_ctx->wr_curr_bio) {
1534                         mutex_unlock(&wr_ctx->wr_lock);
1535                         return -ENOMEM;
1536                 }
1537                 wr_ctx->wr_curr_bio->sctx = sctx;
1538                 wr_ctx->wr_curr_bio->page_count = 0;
1539         }
1540         sbio = wr_ctx->wr_curr_bio;
1541         if (sbio->page_count == 0) {
1542                 struct bio *bio;
1543
1544                 sbio->physical = spage->physical_for_dev_replace;
1545                 sbio->logical = spage->logical;
1546                 sbio->dev = wr_ctx->tgtdev;
1547                 bio = sbio->bio;
1548                 if (!bio) {
1549                         bio = btrfs_io_bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
1550                         if (!bio) {
1551                                 mutex_unlock(&wr_ctx->wr_lock);
1552                                 return -ENOMEM;
1553                         }
1554                         sbio->bio = bio;
1555                 }
1556
1557                 bio->bi_private = sbio;
1558                 bio->bi_end_io = scrub_wr_bio_end_io;
1559                 bio->bi_bdev = sbio->dev->bdev;
1560                 bio->bi_iter.bi_sector = sbio->physical >> 9;
1561                 sbio->err = 0;
1562         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1563                    spage->physical_for_dev_replace ||
1564                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1565                    spage->logical) {
1566                 scrub_wr_submit(sctx);
1567                 goto again;
1568         }
1569
1570         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1571         if (ret != PAGE_SIZE) {
1572                 if (sbio->page_count < 1) {
1573                         bio_put(sbio->bio);
1574                         sbio->bio = NULL;
1575                         mutex_unlock(&wr_ctx->wr_lock);
1576                         return -EIO;
1577                 }
1578                 scrub_wr_submit(sctx);
1579                 goto again;
1580         }
1581
1582         sbio->pagev[sbio->page_count] = spage;
1583         scrub_page_get(spage);
1584         sbio->page_count++;
1585         if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1586                 scrub_wr_submit(sctx);
1587         mutex_unlock(&wr_ctx->wr_lock);
1588
1589         return 0;
1590 }
1591
1592 static void scrub_wr_submit(struct scrub_ctx *sctx)
1593 {
1594         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1595         struct scrub_bio *sbio;
1596
1597         if (!wr_ctx->wr_curr_bio)
1598                 return;
1599
1600         sbio = wr_ctx->wr_curr_bio;
1601         wr_ctx->wr_curr_bio = NULL;
1602         WARN_ON(!sbio->bio->bi_bdev);
1603         scrub_pending_bio_inc(sctx);
1604         /* process all writes in a single worker thread. Then the block layer
1605          * orders the requests before sending them to the driver which
1606          * doubled the write performance on spinning disks when measured
1607          * with Linux 3.5 */
1608         btrfsic_submit_bio(WRITE, sbio->bio);
1609 }
1610
1611 static void scrub_wr_bio_end_io(struct bio *bio, int err)
1612 {
1613         struct scrub_bio *sbio = bio->bi_private;
1614         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1615
1616         sbio->err = err;
1617         sbio->bio = bio;
1618
1619         btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
1620                          scrub_wr_bio_end_io_worker, NULL, NULL);
1621         btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
1622 }
1623
1624 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1625 {
1626         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1627         struct scrub_ctx *sctx = sbio->sctx;
1628         int i;
1629
1630         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1631         if (sbio->err) {
1632                 struct btrfs_dev_replace *dev_replace =
1633                         &sbio->sctx->dev_root->fs_info->dev_replace;
1634
1635                 for (i = 0; i < sbio->page_count; i++) {
1636                         struct scrub_page *spage = sbio->pagev[i];
1637
1638                         spage->io_error = 1;
1639                         btrfs_dev_replace_stats_inc(&dev_replace->
1640                                                     num_write_errors);
1641                 }
1642         }
1643
1644         for (i = 0; i < sbio->page_count; i++)
1645                 scrub_page_put(sbio->pagev[i]);
1646
1647         bio_put(sbio->bio);
1648         kfree(sbio);
1649         scrub_pending_bio_dec(sctx);
1650 }
1651
1652 static int scrub_checksum(struct scrub_block *sblock)
1653 {
1654         u64 flags;
1655         int ret;
1656
1657         WARN_ON(sblock->page_count < 1);
1658         flags = sblock->pagev[0]->flags;
1659         ret = 0;
1660         if (flags & BTRFS_EXTENT_FLAG_DATA)
1661                 ret = scrub_checksum_data(sblock);
1662         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1663                 ret = scrub_checksum_tree_block(sblock);
1664         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1665                 (void)scrub_checksum_super(sblock);
1666         else
1667                 WARN_ON(1);
1668         if (ret)
1669                 scrub_handle_errored_block(sblock);
1670
1671         return ret;
1672 }
1673
1674 static int scrub_checksum_data(struct scrub_block *sblock)
1675 {
1676         struct scrub_ctx *sctx = sblock->sctx;
1677         u8 csum[BTRFS_CSUM_SIZE];
1678         u8 *on_disk_csum;
1679         struct page *page;
1680         void *buffer;
1681         u32 crc = ~(u32)0;
1682         int fail = 0;
1683         u64 len;
1684         int index;
1685
1686         BUG_ON(sblock->page_count < 1);
1687         if (!sblock->pagev[0]->have_csum)
1688                 return 0;
1689
1690         on_disk_csum = sblock->pagev[0]->csum;
1691         page = sblock->pagev[0]->page;
1692         buffer = kmap_atomic(page);
1693
1694         len = sctx->sectorsize;
1695         index = 0;
1696         for (;;) {
1697                 u64 l = min_t(u64, len, PAGE_SIZE);
1698
1699                 crc = btrfs_csum_data(buffer, crc, l);
1700                 kunmap_atomic(buffer);
1701                 len -= l;
1702                 if (len == 0)
1703                         break;
1704                 index++;
1705                 BUG_ON(index >= sblock->page_count);
1706                 BUG_ON(!sblock->pagev[index]->page);
1707                 page = sblock->pagev[index]->page;
1708                 buffer = kmap_atomic(page);
1709         }
1710
1711         btrfs_csum_final(crc, csum);
1712         if (memcmp(csum, on_disk_csum, sctx->csum_size))
1713                 fail = 1;
1714
1715         return fail;
1716 }
1717
1718 static int scrub_checksum_tree_block(struct scrub_block *sblock)
1719 {
1720         struct scrub_ctx *sctx = sblock->sctx;
1721         struct btrfs_header *h;
1722         struct btrfs_root *root = sctx->dev_root;
1723         struct btrfs_fs_info *fs_info = root->fs_info;
1724         u8 calculated_csum[BTRFS_CSUM_SIZE];
1725         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1726         struct page *page;
1727         void *mapped_buffer;
1728         u64 mapped_size;
1729         void *p;
1730         u32 crc = ~(u32)0;
1731         int fail = 0;
1732         int crc_fail = 0;
1733         u64 len;
1734         int index;
1735
1736         BUG_ON(sblock->page_count < 1);
1737         page = sblock->pagev[0]->page;
1738         mapped_buffer = kmap_atomic(page);
1739         h = (struct btrfs_header *)mapped_buffer;
1740         memcpy(on_disk_csum, h->csum, sctx->csum_size);
1741
1742         /*
1743          * we don't use the getter functions here, as we
1744          * a) don't have an extent buffer and
1745          * b) the page is already kmapped
1746          */
1747
1748         if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
1749                 ++fail;
1750
1751         if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))
1752                 ++fail;
1753
1754         if (memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
1755                 ++fail;
1756
1757         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1758                    BTRFS_UUID_SIZE))
1759                 ++fail;
1760
1761         WARN_ON(sctx->nodesize != sctx->leafsize);
1762         len = sctx->nodesize - BTRFS_CSUM_SIZE;
1763         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1764         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1765         index = 0;
1766         for (;;) {
1767                 u64 l = min_t(u64, len, mapped_size);
1768
1769                 crc = btrfs_csum_data(p, crc, l);
1770                 kunmap_atomic(mapped_buffer);
1771                 len -= l;
1772                 if (len == 0)
1773                         break;
1774                 index++;
1775                 BUG_ON(index >= sblock->page_count);
1776                 BUG_ON(!sblock->pagev[index]->page);
1777                 page = sblock->pagev[index]->page;
1778                 mapped_buffer = kmap_atomic(page);
1779                 mapped_size = PAGE_SIZE;
1780                 p = mapped_buffer;
1781         }
1782
1783         btrfs_csum_final(crc, calculated_csum);
1784         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1785                 ++crc_fail;
1786
1787         return fail || crc_fail;
1788 }
1789
1790 static int scrub_checksum_super(struct scrub_block *sblock)
1791 {
1792         struct btrfs_super_block *s;
1793         struct scrub_ctx *sctx = sblock->sctx;
1794         struct btrfs_root *root = sctx->dev_root;
1795         struct btrfs_fs_info *fs_info = root->fs_info;
1796         u8 calculated_csum[BTRFS_CSUM_SIZE];
1797         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1798         struct page *page;
1799         void *mapped_buffer;
1800         u64 mapped_size;
1801         void *p;
1802         u32 crc = ~(u32)0;
1803         int fail_gen = 0;
1804         int fail_cor = 0;
1805         u64 len;
1806         int index;
1807
1808         BUG_ON(sblock->page_count < 1);
1809         page = sblock->pagev[0]->page;
1810         mapped_buffer = kmap_atomic(page);
1811         s = (struct btrfs_super_block *)mapped_buffer;
1812         memcpy(on_disk_csum, s->csum, sctx->csum_size);
1813
1814         if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
1815                 ++fail_cor;
1816
1817         if (sblock->pagev[0]->generation != btrfs_super_generation(s))
1818                 ++fail_gen;
1819
1820         if (memcmp(s->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
1821                 ++fail_cor;
1822
1823         len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
1824         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1825         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1826         index = 0;
1827         for (;;) {
1828                 u64 l = min_t(u64, len, mapped_size);
1829
1830                 crc = btrfs_csum_data(p, crc, l);
1831                 kunmap_atomic(mapped_buffer);
1832                 len -= l;
1833                 if (len == 0)
1834                         break;
1835                 index++;
1836                 BUG_ON(index >= sblock->page_count);
1837                 BUG_ON(!sblock->pagev[index]->page);
1838                 page = sblock->pagev[index]->page;
1839                 mapped_buffer = kmap_atomic(page);
1840                 mapped_size = PAGE_SIZE;
1841                 p = mapped_buffer;
1842         }
1843
1844         btrfs_csum_final(crc, calculated_csum);
1845         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1846                 ++fail_cor;
1847
1848         if (fail_cor + fail_gen) {
1849                 /*
1850                  * if we find an error in a super block, we just report it.
1851                  * They will get written with the next transaction commit
1852                  * anyway
1853                  */
1854                 spin_lock(&sctx->stat_lock);
1855                 ++sctx->stat.super_errors;
1856                 spin_unlock(&sctx->stat_lock);
1857                 if (fail_cor)
1858                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1859                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1860                 else
1861                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1862                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1863         }
1864
1865         return fail_cor + fail_gen;
1866 }
1867
1868 static void scrub_block_get(struct scrub_block *sblock)
1869 {
1870         atomic_inc(&sblock->ref_count);
1871 }
1872
1873 static void scrub_block_put(struct scrub_block *sblock)
1874 {
1875         if (atomic_dec_and_test(&sblock->ref_count)) {
1876                 int i;
1877
1878                 for (i = 0; i < sblock->page_count; i++)
1879                         scrub_page_put(sblock->pagev[i]);
1880                 kfree(sblock);
1881         }
1882 }
1883
1884 static void scrub_page_get(struct scrub_page *spage)
1885 {
1886         atomic_inc(&spage->ref_count);
1887 }
1888
1889 static void scrub_page_put(struct scrub_page *spage)
1890 {
1891         if (atomic_dec_and_test(&spage->ref_count)) {
1892                 if (spage->page)
1893                         __free_page(spage->page);
1894                 kfree(spage);
1895         }
1896 }
1897
1898 static void scrub_submit(struct scrub_ctx *sctx)
1899 {
1900         struct scrub_bio *sbio;
1901
1902         if (sctx->curr == -1)
1903                 return;
1904
1905         sbio = sctx->bios[sctx->curr];
1906         sctx->curr = -1;
1907         scrub_pending_bio_inc(sctx);
1908
1909         if (!sbio->bio->bi_bdev) {
1910                 /*
1911                  * this case should not happen. If btrfs_map_block() is
1912                  * wrong, it could happen for dev-replace operations on
1913                  * missing devices when no mirrors are available, but in
1914                  * this case it should already fail the mount.
1915                  * This case is handled correctly (but _very_ slowly).
1916                  */
1917                 printk_ratelimited(KERN_WARNING
1918                         "BTRFS: scrub_submit(bio bdev == NULL) is unexpected!\n");
1919                 bio_endio(sbio->bio, -EIO);
1920         } else {
1921                 btrfsic_submit_bio(READ, sbio->bio);
1922         }
1923 }
1924
1925 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
1926                                     struct scrub_page *spage)
1927 {
1928         struct scrub_block *sblock = spage->sblock;
1929         struct scrub_bio *sbio;
1930         int ret;
1931
1932 again:
1933         /*
1934          * grab a fresh bio or wait for one to become available
1935          */
1936         while (sctx->curr == -1) {
1937                 spin_lock(&sctx->list_lock);
1938                 sctx->curr = sctx->first_free;
1939                 if (sctx->curr != -1) {
1940                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
1941                         sctx->bios[sctx->curr]->next_free = -1;
1942                         sctx->bios[sctx->curr]->page_count = 0;
1943                         spin_unlock(&sctx->list_lock);
1944                 } else {
1945                         spin_unlock(&sctx->list_lock);
1946                         wait_event(sctx->list_wait, sctx->first_free != -1);
1947                 }
1948         }
1949         sbio = sctx->bios[sctx->curr];
1950         if (sbio->page_count == 0) {
1951                 struct bio *bio;
1952
1953                 sbio->physical = spage->physical;
1954                 sbio->logical = spage->logical;
1955                 sbio->dev = spage->dev;
1956                 bio = sbio->bio;
1957                 if (!bio) {
1958                         bio = btrfs_io_bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
1959                         if (!bio)
1960                                 return -ENOMEM;
1961                         sbio->bio = bio;
1962                 }
1963
1964                 bio->bi_private = sbio;
1965                 bio->bi_end_io = scrub_bio_end_io;
1966                 bio->bi_bdev = sbio->dev->bdev;
1967                 bio->bi_iter.bi_sector = sbio->physical >> 9;
1968                 sbio->err = 0;
1969         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1970                    spage->physical ||
1971                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1972                    spage->logical ||
1973                    sbio->dev != spage->dev) {
1974                 scrub_submit(sctx);
1975                 goto again;
1976         }
1977
1978         sbio->pagev[sbio->page_count] = spage;
1979         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1980         if (ret != PAGE_SIZE) {
1981                 if (sbio->page_count < 1) {
1982                         bio_put(sbio->bio);
1983                         sbio->bio = NULL;
1984                         return -EIO;
1985                 }
1986                 scrub_submit(sctx);
1987                 goto again;
1988         }
1989
1990         scrub_block_get(sblock); /* one for the page added to the bio */
1991         atomic_inc(&sblock->outstanding_pages);
1992         sbio->page_count++;
1993         if (sbio->page_count == sctx->pages_per_rd_bio)
1994                 scrub_submit(sctx);
1995
1996         return 0;
1997 }
1998
1999 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2000                        u64 physical, struct btrfs_device *dev, u64 flags,
2001                        u64 gen, int mirror_num, u8 *csum, int force,
2002                        u64 physical_for_dev_replace)
2003 {
2004         struct scrub_block *sblock;
2005         int index;
2006
2007         sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2008         if (!sblock) {
2009                 spin_lock(&sctx->stat_lock);
2010                 sctx->stat.malloc_errors++;
2011                 spin_unlock(&sctx->stat_lock);
2012                 return -ENOMEM;
2013         }
2014
2015         /* one ref inside this function, plus one for each page added to
2016          * a bio later on */
2017         atomic_set(&sblock->ref_count, 1);
2018         sblock->sctx = sctx;
2019         sblock->no_io_error_seen = 1;
2020
2021         for (index = 0; len > 0; index++) {
2022                 struct scrub_page *spage;
2023                 u64 l = min_t(u64, len, PAGE_SIZE);
2024
2025                 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2026                 if (!spage) {
2027 leave_nomem:
2028                         spin_lock(&sctx->stat_lock);
2029                         sctx->stat.malloc_errors++;
2030                         spin_unlock(&sctx->stat_lock);
2031                         scrub_block_put(sblock);
2032                         return -ENOMEM;
2033                 }
2034                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2035                 scrub_page_get(spage);
2036                 sblock->pagev[index] = spage;
2037                 spage->sblock = sblock;
2038                 spage->dev = dev;
2039                 spage->flags = flags;
2040                 spage->generation = gen;
2041                 spage->logical = logical;
2042                 spage->physical = physical;
2043                 spage->physical_for_dev_replace = physical_for_dev_replace;
2044                 spage->mirror_num = mirror_num;
2045                 if (csum) {
2046                         spage->have_csum = 1;
2047                         memcpy(spage->csum, csum, sctx->csum_size);
2048                 } else {
2049                         spage->have_csum = 0;
2050                 }
2051                 sblock->page_count++;
2052                 spage->page = alloc_page(GFP_NOFS);
2053                 if (!spage->page)
2054                         goto leave_nomem;
2055                 len -= l;
2056                 logical += l;
2057                 physical += l;
2058                 physical_for_dev_replace += l;
2059         }
2060
2061         WARN_ON(sblock->page_count == 0);
2062         for (index = 0; index < sblock->page_count; index++) {
2063                 struct scrub_page *spage = sblock->pagev[index];
2064                 int ret;
2065
2066                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2067                 if (ret) {
2068                         scrub_block_put(sblock);
2069                         return ret;
2070                 }
2071         }
2072
2073         if (force)
2074                 scrub_submit(sctx);
2075
2076         /* last one frees, either here or in bio completion for last page */
2077         scrub_block_put(sblock);
2078         return 0;
2079 }
2080
2081 static void scrub_bio_end_io(struct bio *bio, int err)
2082 {
2083         struct scrub_bio *sbio = bio->bi_private;
2084         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
2085
2086         sbio->err = err;
2087         sbio->bio = bio;
2088
2089         btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2090 }
2091
2092 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2093 {
2094         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2095         struct scrub_ctx *sctx = sbio->sctx;
2096         int i;
2097
2098         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2099         if (sbio->err) {
2100                 for (i = 0; i < sbio->page_count; i++) {
2101                         struct scrub_page *spage = sbio->pagev[i];
2102
2103                         spage->io_error = 1;
2104                         spage->sblock->no_io_error_seen = 0;
2105                 }
2106         }
2107
2108         /* now complete the scrub_block items that have all pages completed */
2109         for (i = 0; i < sbio->page_count; i++) {
2110                 struct scrub_page *spage = sbio->pagev[i];
2111                 struct scrub_block *sblock = spage->sblock;
2112
2113                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2114                         scrub_block_complete(sblock);
2115                 scrub_block_put(sblock);
2116         }
2117
2118         bio_put(sbio->bio);
2119         sbio->bio = NULL;
2120         spin_lock(&sctx->list_lock);
2121         sbio->next_free = sctx->first_free;
2122         sctx->first_free = sbio->index;
2123         spin_unlock(&sctx->list_lock);
2124
2125         if (sctx->is_dev_replace &&
2126             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2127                 mutex_lock(&sctx->wr_ctx.wr_lock);
2128                 scrub_wr_submit(sctx);
2129                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2130         }
2131
2132         scrub_pending_bio_dec(sctx);
2133 }
2134
2135 static void scrub_block_complete(struct scrub_block *sblock)
2136 {
2137         if (!sblock->no_io_error_seen) {
2138                 scrub_handle_errored_block(sblock);
2139         } else {
2140                 /*
2141                  * if has checksum error, write via repair mechanism in
2142                  * dev replace case, otherwise write here in dev replace
2143                  * case.
2144                  */
2145                 if (!scrub_checksum(sblock) && sblock->sctx->is_dev_replace)
2146                         scrub_write_block_to_dev_replace(sblock);
2147         }
2148 }
2149
2150 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
2151                            u8 *csum)
2152 {
2153         struct btrfs_ordered_sum *sum = NULL;
2154         unsigned long index;
2155         unsigned long num_sectors;
2156
2157         while (!list_empty(&sctx->csum_list)) {
2158                 sum = list_first_entry(&sctx->csum_list,
2159                                        struct btrfs_ordered_sum, list);
2160                 if (sum->bytenr > logical)
2161                         return 0;
2162                 if (sum->bytenr + sum->len > logical)
2163                         break;
2164
2165                 ++sctx->stat.csum_discards;
2166                 list_del(&sum->list);
2167                 kfree(sum);
2168                 sum = NULL;
2169         }
2170         if (!sum)
2171                 return 0;
2172
2173         index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
2174         num_sectors = sum->len / sctx->sectorsize;
2175         memcpy(csum, sum->sums + index, sctx->csum_size);
2176         if (index == num_sectors - 1) {
2177                 list_del(&sum->list);
2178                 kfree(sum);
2179         }
2180         return 1;
2181 }
2182
2183 /* scrub extent tries to collect up to 64 kB for each bio */
2184 static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
2185                         u64 physical, struct btrfs_device *dev, u64 flags,
2186                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2187 {
2188         int ret;
2189         u8 csum[BTRFS_CSUM_SIZE];
2190         u32 blocksize;
2191
2192         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2193                 blocksize = sctx->sectorsize;
2194                 spin_lock(&sctx->stat_lock);
2195                 sctx->stat.data_extents_scrubbed++;
2196                 sctx->stat.data_bytes_scrubbed += len;
2197                 spin_unlock(&sctx->stat_lock);
2198         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2199                 WARN_ON(sctx->nodesize != sctx->leafsize);
2200                 blocksize = sctx->nodesize;
2201                 spin_lock(&sctx->stat_lock);
2202                 sctx->stat.tree_extents_scrubbed++;
2203                 sctx->stat.tree_bytes_scrubbed += len;
2204                 spin_unlock(&sctx->stat_lock);
2205         } else {
2206                 blocksize = sctx->sectorsize;
2207                 WARN_ON(1);
2208         }
2209
2210         while (len) {
2211                 u64 l = min_t(u64, len, blocksize);
2212                 int have_csum = 0;
2213
2214                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2215                         /* push csums to sbio */
2216                         have_csum = scrub_find_csum(sctx, logical, l, csum);
2217                         if (have_csum == 0)
2218                                 ++sctx->stat.no_csum;
2219                         if (sctx->is_dev_replace && !have_csum) {
2220                                 ret = copy_nocow_pages(sctx, logical, l,
2221                                                        mirror_num,
2222                                                       physical_for_dev_replace);
2223                                 goto behind_scrub_pages;
2224                         }
2225                 }
2226                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2227                                   mirror_num, have_csum ? csum : NULL, 0,
2228                                   physical_for_dev_replace);
2229 behind_scrub_pages:
2230                 if (ret)
2231                         return ret;
2232                 len -= l;
2233                 logical += l;
2234                 physical += l;
2235                 physical_for_dev_replace += l;
2236         }
2237         return 0;
2238 }
2239
2240 /*
2241  * Given a physical address, this will calculate it's
2242  * logical offset. if this is a parity stripe, it will return
2243  * the most left data stripe's logical offset.
2244  *
2245  * return 0 if it is a data stripe, 1 means parity stripe.
2246  */
2247 static int get_raid56_logic_offset(u64 physical, int num,
2248                                    struct map_lookup *map, u64 *offset)
2249 {
2250         int i;
2251         int j = 0;
2252         u64 stripe_nr;
2253         u64 last_offset;
2254         int stripe_index;
2255         int rot;
2256
2257         last_offset = (physical - map->stripes[num].physical) *
2258                       nr_data_stripes(map);
2259         *offset = last_offset;
2260         for (i = 0; i < nr_data_stripes(map); i++) {
2261                 *offset = last_offset + i * map->stripe_len;
2262
2263                 stripe_nr = *offset;
2264                 do_div(stripe_nr, map->stripe_len);
2265                 do_div(stripe_nr, nr_data_stripes(map));
2266
2267                 /* Work out the disk rotation on this stripe-set */
2268                 rot = do_div(stripe_nr, map->num_stripes);
2269                 /* calculate which stripe this data locates */
2270                 rot += i;
2271                 stripe_index = rot % map->num_stripes;
2272                 if (stripe_index == num)
2273                         return 0;
2274                 if (stripe_index < num)
2275                         j++;
2276         }
2277         *offset = last_offset + j * map->stripe_len;
2278         return 1;
2279 }
2280
2281 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
2282                                            struct map_lookup *map,
2283                                            struct btrfs_device *scrub_dev,
2284                                            int num, u64 base, u64 length,
2285                                            int is_dev_replace)
2286 {
2287         struct btrfs_path *path;
2288         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2289         struct btrfs_root *root = fs_info->extent_root;
2290         struct btrfs_root *csum_root = fs_info->csum_root;
2291         struct btrfs_extent_item *extent;
2292         struct blk_plug plug;
2293         u64 flags;
2294         int ret;
2295         int slot;
2296         u64 nstripes;
2297         struct extent_buffer *l;
2298         struct btrfs_key key;
2299         u64 physical;
2300         u64 logical;
2301         u64 logic_end;
2302         u64 physical_end;
2303         u64 generation;
2304         int mirror_num;
2305         struct reada_control *reada1;
2306         struct reada_control *reada2;
2307         struct btrfs_key key_start;
2308         struct btrfs_key key_end;
2309         u64 increment = map->stripe_len;
2310         u64 offset;
2311         u64 extent_logical;
2312         u64 extent_physical;
2313         u64 extent_len;
2314         struct btrfs_device *extent_dev;
2315         int extent_mirror_num;
2316         int stop_loop = 0;
2317
2318         nstripes = length;
2319         physical = map->stripes[num].physical;
2320         offset = 0;
2321         do_div(nstripes, map->stripe_len);
2322         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
2323                 offset = map->stripe_len * num;
2324                 increment = map->stripe_len * map->num_stripes;
2325                 mirror_num = 1;
2326         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
2327                 int factor = map->num_stripes / map->sub_stripes;
2328                 offset = map->stripe_len * (num / map->sub_stripes);
2329                 increment = map->stripe_len * factor;
2330                 mirror_num = num % map->sub_stripes + 1;
2331         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
2332                 increment = map->stripe_len;
2333                 mirror_num = num % map->num_stripes + 1;
2334         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
2335                 increment = map->stripe_len;
2336                 mirror_num = num % map->num_stripes + 1;
2337         } else if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
2338                                 BTRFS_BLOCK_GROUP_RAID6)) {
2339                 get_raid56_logic_offset(physical, num, map, &offset);
2340                 increment = map->stripe_len * nr_data_stripes(map);
2341                 mirror_num = 1;
2342         } else {
2343                 increment = map->stripe_len;
2344                 mirror_num = 1;
2345         }
2346
2347         path = btrfs_alloc_path();
2348         if (!path)
2349                 return -ENOMEM;
2350
2351         /*
2352          * work on commit root. The related disk blocks are static as
2353          * long as COW is applied. This means, it is save to rewrite
2354          * them to repair disk errors without any race conditions
2355          */
2356         path->search_commit_root = 1;
2357         path->skip_locking = 1;
2358
2359         /*
2360          * trigger the readahead for extent tree csum tree and wait for
2361          * completion. During readahead, the scrub is officially paused
2362          * to not hold off transaction commits
2363          */
2364         logical = base + offset;
2365         physical_end = physical + nstripes * map->stripe_len;
2366         if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
2367                          BTRFS_BLOCK_GROUP_RAID6)) {
2368                 get_raid56_logic_offset(physical_end, num,
2369                                         map, &logic_end);
2370                 logic_end += base;
2371         } else {
2372                 logic_end = logical + increment * nstripes;
2373         }
2374         wait_event(sctx->list_wait,
2375                    atomic_read(&sctx->bios_in_flight) == 0);
2376         scrub_blocked_if_needed(fs_info);
2377
2378         /* FIXME it might be better to start readahead at commit root */
2379         key_start.objectid = logical;
2380         key_start.type = BTRFS_EXTENT_ITEM_KEY;
2381         key_start.offset = (u64)0;
2382         key_end.objectid = logic_end;
2383         key_end.type = BTRFS_METADATA_ITEM_KEY;
2384         key_end.offset = (u64)-1;
2385         reada1 = btrfs_reada_add(root, &key_start, &key_end);
2386
2387         key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
2388         key_start.type = BTRFS_EXTENT_CSUM_KEY;
2389         key_start.offset = logical;
2390         key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
2391         key_end.type = BTRFS_EXTENT_CSUM_KEY;
2392         key_end.offset = logic_end;
2393         reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
2394
2395         if (!IS_ERR(reada1))
2396                 btrfs_reada_wait(reada1);
2397         if (!IS_ERR(reada2))
2398                 btrfs_reada_wait(reada2);
2399
2400
2401         /*
2402          * collect all data csums for the stripe to avoid seeking during
2403          * the scrub. This might currently (crc32) end up to be about 1MB
2404          */
2405         blk_start_plug(&plug);
2406
2407         /*
2408          * now find all extents for each stripe and scrub them
2409          */
2410         ret = 0;
2411         while (physical < physical_end) {
2412                 /* for raid56, we skip parity stripe */
2413                 if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
2414                                 BTRFS_BLOCK_GROUP_RAID6)) {
2415                         ret = get_raid56_logic_offset(physical, num,
2416                                         map, &logical);
2417                         logical += base;
2418                         if (ret)
2419                                 goto skip;
2420                 }
2421                 /*
2422                  * canceled?
2423                  */
2424                 if (atomic_read(&fs_info->scrub_cancel_req) ||
2425                     atomic_read(&sctx->cancel_req)) {
2426                         ret = -ECANCELED;
2427                         goto out;
2428                 }
2429                 /*
2430                  * check to see if we have to pause
2431                  */
2432                 if (atomic_read(&fs_info->scrub_pause_req)) {
2433                         /* push queued extents */
2434                         atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
2435                         scrub_submit(sctx);
2436                         mutex_lock(&sctx->wr_ctx.wr_lock);
2437                         scrub_wr_submit(sctx);
2438                         mutex_unlock(&sctx->wr_ctx.wr_lock);
2439                         wait_event(sctx->list_wait,
2440                                    atomic_read(&sctx->bios_in_flight) == 0);
2441                         atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
2442                         scrub_blocked_if_needed(fs_info);
2443                 }
2444
2445                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2446                         key.type = BTRFS_METADATA_ITEM_KEY;
2447                 else
2448                         key.type = BTRFS_EXTENT_ITEM_KEY;
2449                 key.objectid = logical;
2450                 key.offset = (u64)-1;
2451
2452                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2453                 if (ret < 0)
2454                         goto out;
2455
2456                 if (ret > 0) {
2457                         ret = btrfs_previous_extent_item(root, path, 0);
2458                         if (ret < 0)
2459                                 goto out;
2460                         if (ret > 0) {
2461                                 /* there's no smaller item, so stick with the
2462                                  * larger one */
2463                                 btrfs_release_path(path);
2464                                 ret = btrfs_search_slot(NULL, root, &key,
2465                                                         path, 0, 0);
2466                                 if (ret < 0)
2467                                         goto out;
2468                         }
2469                 }
2470
2471                 stop_loop = 0;
2472                 while (1) {
2473                         u64 bytes;
2474
2475                         l = path->nodes[0];
2476                         slot = path->slots[0];
2477                         if (slot >= btrfs_header_nritems(l)) {
2478                                 ret = btrfs_next_leaf(root, path);
2479                                 if (ret == 0)
2480                                         continue;
2481                                 if (ret < 0)
2482                                         goto out;
2483
2484                                 stop_loop = 1;
2485                                 break;
2486                         }
2487                         btrfs_item_key_to_cpu(l, &key, slot);
2488
2489                         if (key.type == BTRFS_METADATA_ITEM_KEY)
2490                                 bytes = root->leafsize;
2491                         else
2492                                 bytes = key.offset;
2493
2494                         if (key.objectid + bytes <= logical)
2495                                 goto next;
2496
2497                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2498                             key.type != BTRFS_METADATA_ITEM_KEY)
2499                                 goto next;
2500
2501                         if (key.objectid >= logical + map->stripe_len) {
2502                                 /* out of this device extent */
2503                                 if (key.objectid >= logic_end)
2504                                         stop_loop = 1;
2505                                 break;
2506                         }
2507
2508                         extent = btrfs_item_ptr(l, slot,
2509                                                 struct btrfs_extent_item);
2510                         flags = btrfs_extent_flags(l, extent);
2511                         generation = btrfs_extent_generation(l, extent);
2512
2513                         if (key.objectid < logical &&
2514                             (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
2515                                 btrfs_err(fs_info,
2516                                            "scrub: tree block %llu spanning "
2517                                            "stripes, ignored. logical=%llu",
2518                                        key.objectid, logical);
2519                                 goto next;
2520                         }
2521
2522 again:
2523                         extent_logical = key.objectid;
2524                         extent_len = bytes;
2525
2526                         /*
2527                          * trim extent to this stripe
2528                          */
2529                         if (extent_logical < logical) {
2530                                 extent_len -= logical - extent_logical;
2531                                 extent_logical = logical;
2532                         }
2533                         if (extent_logical + extent_len >
2534                             logical + map->stripe_len) {
2535                                 extent_len = logical + map->stripe_len -
2536                                              extent_logical;
2537                         }
2538
2539                         extent_physical = extent_logical - logical + physical;
2540                         extent_dev = scrub_dev;
2541                         extent_mirror_num = mirror_num;
2542                         if (is_dev_replace)
2543                                 scrub_remap_extent(fs_info, extent_logical,
2544                                                    extent_len, &extent_physical,
2545                                                    &extent_dev,
2546                                                    &extent_mirror_num);
2547
2548                         ret = btrfs_lookup_csums_range(csum_root, logical,
2549                                                 logical + map->stripe_len - 1,
2550                                                 &sctx->csum_list, 1);
2551                         if (ret)
2552                                 goto out;
2553
2554                         ret = scrub_extent(sctx, extent_logical, extent_len,
2555                                            extent_physical, extent_dev, flags,
2556                                            generation, extent_mirror_num,
2557                                            extent_logical - logical + physical);
2558                         if (ret)
2559                                 goto out;
2560
2561                         scrub_free_csums(sctx);
2562                         if (extent_logical + extent_len <
2563                             key.objectid + bytes) {
2564                                 if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
2565                                         BTRFS_BLOCK_GROUP_RAID6)) {
2566                                         /*
2567                                          * loop until we find next data stripe
2568                                          * or we have finished all stripes.
2569                                          */
2570                                         do {
2571                                                 physical += map->stripe_len;
2572                                                 ret = get_raid56_logic_offset(
2573                                                                 physical, num,
2574                                                                 map, &logical);
2575                                                 logical += base;
2576                                         } while (physical < physical_end && ret);
2577                                 } else {
2578                                         physical += map->stripe_len;
2579                                         logical += increment;
2580                                 }
2581                                 if (logical < key.objectid + bytes) {
2582                                         cond_resched();
2583                                         goto again;
2584                                 }
2585
2586                                 if (physical >= physical_end) {
2587                                         stop_loop = 1;
2588                                         break;
2589                                 }
2590                         }
2591 next:
2592                         path->slots[0]++;
2593                 }
2594                 btrfs_release_path(path);
2595 skip:
2596                 logical += increment;
2597                 physical += map->stripe_len;
2598                 spin_lock(&sctx->stat_lock);
2599                 if (stop_loop)
2600                         sctx->stat.last_physical = map->stripes[num].physical +
2601                                                    length;
2602                 else
2603                         sctx->stat.last_physical = physical;
2604                 spin_unlock(&sctx->stat_lock);
2605                 if (stop_loop)
2606                         break;
2607         }
2608 out:
2609         /* push queued extents */
2610         scrub_submit(sctx);
2611         mutex_lock(&sctx->wr_ctx.wr_lock);
2612         scrub_wr_submit(sctx);
2613         mutex_unlock(&sctx->wr_ctx.wr_lock);
2614
2615         blk_finish_plug(&plug);
2616         btrfs_free_path(path);
2617         return ret < 0 ? ret : 0;
2618 }
2619
2620 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
2621                                           struct btrfs_device *scrub_dev,
2622                                           u64 chunk_tree, u64 chunk_objectid,
2623                                           u64 chunk_offset, u64 length,
2624                                           u64 dev_offset, int is_dev_replace)
2625 {
2626         struct btrfs_mapping_tree *map_tree =
2627                 &sctx->dev_root->fs_info->mapping_tree;
2628         struct map_lookup *map;
2629         struct extent_map *em;
2630         int i;
2631         int ret = 0;
2632
2633         read_lock(&map_tree->map_tree.lock);
2634         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
2635         read_unlock(&map_tree->map_tree.lock);
2636
2637         if (!em)
2638                 return -EINVAL;
2639
2640         map = (struct map_lookup *)em->bdev;
2641         if (em->start != chunk_offset)
2642                 goto out;
2643
2644         if (em->len < length)
2645                 goto out;
2646
2647         for (i = 0; i < map->num_stripes; ++i) {
2648                 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
2649                     map->stripes[i].physical == dev_offset) {
2650                         ret = scrub_stripe(sctx, map, scrub_dev, i,
2651                                            chunk_offset, length,
2652                                            is_dev_replace);
2653                         if (ret)
2654                                 goto out;
2655                 }
2656         }
2657 out:
2658         free_extent_map(em);
2659
2660         return ret;
2661 }
2662
2663 static noinline_for_stack
2664 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
2665                            struct btrfs_device *scrub_dev, u64 start, u64 end,
2666                            int is_dev_replace)
2667 {
2668         struct btrfs_dev_extent *dev_extent = NULL;
2669         struct btrfs_path *path;
2670         struct btrfs_root *root = sctx->dev_root;
2671         struct btrfs_fs_info *fs_info = root->fs_info;
2672         u64 length;
2673         u64 chunk_tree;
2674         u64 chunk_objectid;
2675         u64 chunk_offset;
2676         int ret;
2677         int slot;
2678         struct extent_buffer *l;
2679         struct btrfs_key key;
2680         struct btrfs_key found_key;
2681         struct btrfs_block_group_cache *cache;
2682         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
2683
2684         path = btrfs_alloc_path();
2685         if (!path)
2686                 return -ENOMEM;
2687
2688         path->reada = 2;
2689         path->search_commit_root = 1;
2690         path->skip_locking = 1;
2691
2692         key.objectid = scrub_dev->devid;
2693         key.offset = 0ull;
2694         key.type = BTRFS_DEV_EXTENT_KEY;
2695
2696         while (1) {
2697                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2698                 if (ret < 0)
2699                         break;
2700                 if (ret > 0) {
2701                         if (path->slots[0] >=
2702                             btrfs_header_nritems(path->nodes[0])) {
2703                                 ret = btrfs_next_leaf(root, path);
2704                                 if (ret)
2705                                         break;
2706                         }
2707                 }
2708
2709                 l = path->nodes[0];
2710                 slot = path->slots[0];
2711
2712                 btrfs_item_key_to_cpu(l, &found_key, slot);
2713
2714                 if (found_key.objectid != scrub_dev->devid)
2715                         break;
2716
2717                 if (btrfs_key_type(&found_key) != BTRFS_DEV_EXTENT_KEY)
2718                         break;
2719
2720                 if (found_key.offset >= end)
2721                         break;
2722
2723                 if (found_key.offset < key.offset)
2724                         break;
2725
2726                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
2727                 length = btrfs_dev_extent_length(l, dev_extent);
2728
2729                 if (found_key.offset + length <= start)
2730                         goto skip;
2731
2732                 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
2733                 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
2734                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
2735
2736                 /*
2737                  * get a reference on the corresponding block group to prevent
2738                  * the chunk from going away while we scrub it
2739                  */
2740                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
2741
2742                 /* some chunks are removed but not committed to disk yet,
2743                  * continue scrubbing */
2744                 if (!cache)
2745                         goto skip;
2746
2747                 dev_replace->cursor_right = found_key.offset + length;
2748                 dev_replace->cursor_left = found_key.offset;
2749                 dev_replace->item_needs_writeback = 1;
2750                 ret = scrub_chunk(sctx, scrub_dev, chunk_tree, chunk_objectid,
2751                                   chunk_offset, length, found_key.offset,
2752                                   is_dev_replace);
2753
2754                 /*
2755                  * flush, submit all pending read and write bios, afterwards
2756                  * wait for them.
2757                  * Note that in the dev replace case, a read request causes
2758                  * write requests that are submitted in the read completion
2759                  * worker. Therefore in the current situation, it is required
2760                  * that all write requests are flushed, so that all read and
2761                  * write requests are really completed when bios_in_flight
2762                  * changes to 0.
2763                  */
2764                 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
2765                 scrub_submit(sctx);
2766                 mutex_lock(&sctx->wr_ctx.wr_lock);
2767                 scrub_wr_submit(sctx);
2768                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2769
2770                 wait_event(sctx->list_wait,
2771                            atomic_read(&sctx->bios_in_flight) == 0);
2772                 atomic_inc(&fs_info->scrubs_paused);
2773                 wake_up(&fs_info->scrub_pause_wait);
2774
2775                 /*
2776                  * must be called before we decrease @scrub_paused.
2777                  * make sure we don't block transaction commit while
2778                  * we are waiting pending workers finished.
2779                  */
2780                 wait_event(sctx->list_wait,
2781                            atomic_read(&sctx->workers_pending) == 0);
2782                 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
2783
2784                 mutex_lock(&fs_info->scrub_lock);
2785                 __scrub_blocked_if_needed(fs_info);
2786                 atomic_dec(&fs_info->scrubs_paused);
2787                 mutex_unlock(&fs_info->scrub_lock);
2788                 wake_up(&fs_info->scrub_pause_wait);
2789
2790                 btrfs_put_block_group(cache);
2791                 if (ret)
2792                         break;
2793                 if (is_dev_replace &&
2794                     atomic64_read(&dev_replace->num_write_errors) > 0) {
2795                         ret = -EIO;
2796                         break;
2797                 }
2798                 if (sctx->stat.malloc_errors > 0) {
2799                         ret = -ENOMEM;
2800                         break;
2801                 }
2802
2803                 dev_replace->cursor_left = dev_replace->cursor_right;
2804                 dev_replace->item_needs_writeback = 1;
2805 skip:
2806                 key.offset = found_key.offset + length;
2807                 btrfs_release_path(path);
2808         }
2809
2810         btrfs_free_path(path);
2811
2812         /*
2813          * ret can still be 1 from search_slot or next_leaf,
2814          * that's not an error
2815          */
2816         return ret < 0 ? ret : 0;
2817 }
2818
2819 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
2820                                            struct btrfs_device *scrub_dev)
2821 {
2822         int     i;
2823         u64     bytenr;
2824         u64     gen;
2825         int     ret;
2826         struct btrfs_root *root = sctx->dev_root;
2827
2828         if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
2829                 return -EIO;
2830
2831         gen = root->fs_info->last_trans_committed;
2832
2833         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
2834                 bytenr = btrfs_sb_offset(i);
2835                 if (bytenr + BTRFS_SUPER_INFO_SIZE > scrub_dev->total_bytes)
2836                         break;
2837
2838                 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
2839                                   scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
2840                                   NULL, 1, bytenr);
2841                 if (ret)
2842                         return ret;
2843         }
2844         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
2845
2846         return 0;
2847 }
2848
2849 /*
2850  * get a reference count on fs_info->scrub_workers. start worker if necessary
2851  */
2852 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
2853                                                 int is_dev_replace)
2854 {
2855         int ret = 0;
2856         int flags = WQ_FREEZABLE | WQ_UNBOUND;
2857         int max_active = fs_info->thread_pool_size;
2858
2859         if (fs_info->scrub_workers_refcnt == 0) {
2860                 if (is_dev_replace)
2861                         fs_info->scrub_workers =
2862                                 btrfs_alloc_workqueue("btrfs-scrub", flags,
2863                                                       1, 4);
2864                 else
2865                         fs_info->scrub_workers =
2866                                 btrfs_alloc_workqueue("btrfs-scrub", flags,
2867                                                       max_active, 4);
2868                 if (!fs_info->scrub_workers) {
2869                         ret = -ENOMEM;
2870                         goto out;
2871                 }
2872                 fs_info->scrub_wr_completion_workers =
2873                         btrfs_alloc_workqueue("btrfs-scrubwrc", flags,
2874                                               max_active, 2);
2875                 if (!fs_info->scrub_wr_completion_workers) {
2876                         ret = -ENOMEM;
2877                         goto out;
2878                 }
2879                 fs_info->scrub_nocow_workers =
2880                         btrfs_alloc_workqueue("btrfs-scrubnc", flags, 1, 0);
2881                 if (!fs_info->scrub_nocow_workers) {
2882                         ret = -ENOMEM;
2883                         goto out;
2884                 }
2885         }
2886         ++fs_info->scrub_workers_refcnt;
2887 out:
2888         return ret;
2889 }
2890
2891 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
2892 {
2893         if (--fs_info->scrub_workers_refcnt == 0) {
2894                 btrfs_destroy_workqueue(fs_info->scrub_workers);
2895                 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
2896                 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
2897         }
2898         WARN_ON(fs_info->scrub_workers_refcnt < 0);
2899 }
2900
2901 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
2902                     u64 end, struct btrfs_scrub_progress *progress,
2903                     int readonly, int is_dev_replace)
2904 {
2905         struct scrub_ctx *sctx;
2906         int ret;
2907         struct btrfs_device *dev;
2908
2909         if (btrfs_fs_closing(fs_info))
2910                 return -EINVAL;
2911
2912         /*
2913          * check some assumptions
2914          */
2915         if (fs_info->chunk_root->nodesize != fs_info->chunk_root->leafsize) {
2916                 btrfs_err(fs_info,
2917                            "scrub: size assumption nodesize == leafsize (%d == %d) fails",
2918                        fs_info->chunk_root->nodesize,
2919                        fs_info->chunk_root->leafsize);
2920                 return -EINVAL;
2921         }
2922
2923         if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
2924                 /*
2925                  * in this case scrub is unable to calculate the checksum
2926                  * the way scrub is implemented. Do not handle this
2927                  * situation at all because it won't ever happen.
2928                  */
2929                 btrfs_err(fs_info,
2930                            "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
2931                        fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
2932                 return -EINVAL;
2933         }
2934
2935         if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
2936                 /* not supported for data w/o checksums */
2937                 btrfs_err(fs_info,
2938                            "scrub: size assumption sectorsize != PAGE_SIZE "
2939                            "(%d != %lu) fails",
2940                        fs_info->chunk_root->sectorsize, PAGE_SIZE);
2941                 return -EINVAL;
2942         }
2943
2944         if (fs_info->chunk_root->nodesize >
2945             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
2946             fs_info->chunk_root->sectorsize >
2947             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
2948                 /*
2949                  * would exhaust the array bounds of pagev member in
2950                  * struct scrub_block
2951                  */
2952                 btrfs_err(fs_info, "scrub: size assumption nodesize and sectorsize "
2953                            "<= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
2954                        fs_info->chunk_root->nodesize,
2955                        SCRUB_MAX_PAGES_PER_BLOCK,
2956                        fs_info->chunk_root->sectorsize,
2957                        SCRUB_MAX_PAGES_PER_BLOCK);
2958                 return -EINVAL;
2959         }
2960
2961
2962         mutex_lock(&fs_info->fs_devices->device_list_mutex);
2963         dev = btrfs_find_device(fs_info, devid, NULL, NULL);
2964         if (!dev || (dev->missing && !is_dev_replace)) {
2965                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2966                 return -ENODEV;
2967         }
2968
2969         mutex_lock(&fs_info->scrub_lock);
2970         if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
2971                 mutex_unlock(&fs_info->scrub_lock);
2972                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2973                 return -EIO;
2974         }
2975
2976         btrfs_dev_replace_lock(&fs_info->dev_replace);
2977         if (dev->scrub_device ||
2978             (!is_dev_replace &&
2979              btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
2980                 btrfs_dev_replace_unlock(&fs_info->dev_replace);
2981                 mutex_unlock(&fs_info->scrub_lock);
2982                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2983                 return -EINPROGRESS;
2984         }
2985         btrfs_dev_replace_unlock(&fs_info->dev_replace);
2986
2987         ret = scrub_workers_get(fs_info, is_dev_replace);
2988         if (ret) {
2989                 mutex_unlock(&fs_info->scrub_lock);
2990                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2991                 return ret;
2992         }
2993
2994         sctx = scrub_setup_ctx(dev, is_dev_replace);
2995         if (IS_ERR(sctx)) {
2996                 mutex_unlock(&fs_info->scrub_lock);
2997                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2998                 scrub_workers_put(fs_info);
2999                 return PTR_ERR(sctx);
3000         }
3001         sctx->readonly = readonly;
3002         dev->scrub_device = sctx;
3003         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3004
3005         /*
3006          * checking @scrub_pause_req here, we can avoid
3007          * race between committing transaction and scrubbing.
3008          */
3009         __scrub_blocked_if_needed(fs_info);
3010         atomic_inc(&fs_info->scrubs_running);
3011         mutex_unlock(&fs_info->scrub_lock);
3012
3013         if (!is_dev_replace) {
3014                 /*
3015                  * by holding device list mutex, we can
3016                  * kick off writing super in log tree sync.
3017                  */
3018                 mutex_lock(&fs_info->fs_devices->device_list_mutex);
3019                 ret = scrub_supers(sctx, dev);
3020                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3021         }
3022
3023         if (!ret)
3024                 ret = scrub_enumerate_chunks(sctx, dev, start, end,
3025                                              is_dev_replace);
3026
3027         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3028         atomic_dec(&fs_info->scrubs_running);
3029         wake_up(&fs_info->scrub_pause_wait);
3030
3031         wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
3032
3033         if (progress)
3034                 memcpy(progress, &sctx->stat, sizeof(*progress));
3035
3036         mutex_lock(&fs_info->scrub_lock);
3037         dev->scrub_device = NULL;
3038         scrub_workers_put(fs_info);
3039         mutex_unlock(&fs_info->scrub_lock);
3040
3041         scrub_free_ctx(sctx);
3042
3043         return ret;
3044 }
3045
3046 void btrfs_scrub_pause(struct btrfs_root *root)
3047 {
3048         struct btrfs_fs_info *fs_info = root->fs_info;
3049
3050         mutex_lock(&fs_info->scrub_lock);
3051         atomic_inc(&fs_info->scrub_pause_req);
3052         while (atomic_read(&fs_info->scrubs_paused) !=
3053                atomic_read(&fs_info->scrubs_running)) {
3054                 mutex_unlock(&fs_info->scrub_lock);
3055                 wait_event(fs_info->scrub_pause_wait,
3056                            atomic_read(&fs_info->scrubs_paused) ==
3057                            atomic_read(&fs_info->scrubs_running));
3058                 mutex_lock(&fs_info->scrub_lock);
3059         }
3060         mutex_unlock(&fs_info->scrub_lock);
3061 }
3062
3063 void btrfs_scrub_continue(struct btrfs_root *root)
3064 {
3065         struct btrfs_fs_info *fs_info = root->fs_info;
3066
3067         atomic_dec(&fs_info->scrub_pause_req);
3068         wake_up(&fs_info->scrub_pause_wait);
3069 }
3070
3071 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
3072 {
3073         mutex_lock(&fs_info->scrub_lock);
3074         if (!atomic_read(&fs_info->scrubs_running)) {
3075                 mutex_unlock(&fs_info->scrub_lock);
3076                 return -ENOTCONN;
3077         }
3078
3079         atomic_inc(&fs_info->scrub_cancel_req);
3080         while (atomic_read(&fs_info->scrubs_running)) {
3081                 mutex_unlock(&fs_info->scrub_lock);
3082                 wait_event(fs_info->scrub_pause_wait,
3083                            atomic_read(&fs_info->scrubs_running) == 0);
3084                 mutex_lock(&fs_info->scrub_lock);
3085         }
3086         atomic_dec(&fs_info->scrub_cancel_req);
3087         mutex_unlock(&fs_info->scrub_lock);
3088
3089         return 0;
3090 }
3091
3092 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
3093                            struct btrfs_device *dev)
3094 {
3095         struct scrub_ctx *sctx;
3096
3097         mutex_lock(&fs_info->scrub_lock);
3098         sctx = dev->scrub_device;
3099         if (!sctx) {
3100                 mutex_unlock(&fs_info->scrub_lock);
3101                 return -ENOTCONN;
3102         }
3103         atomic_inc(&sctx->cancel_req);
3104         while (dev->scrub_device) {
3105                 mutex_unlock(&fs_info->scrub_lock);
3106                 wait_event(fs_info->scrub_pause_wait,
3107                            dev->scrub_device == NULL);
3108                 mutex_lock(&fs_info->scrub_lock);
3109         }
3110         mutex_unlock(&fs_info->scrub_lock);
3111
3112         return 0;
3113 }
3114
3115 int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
3116                          struct btrfs_scrub_progress *progress)
3117 {
3118         struct btrfs_device *dev;
3119         struct scrub_ctx *sctx = NULL;
3120
3121         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
3122         dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
3123         if (dev)
3124                 sctx = dev->scrub_device;
3125         if (sctx)
3126                 memcpy(progress, &sctx->stat, sizeof(*progress));
3127         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
3128
3129         return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
3130 }
3131
3132 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
3133                                u64 extent_logical, u64 extent_len,
3134                                u64 *extent_physical,
3135                                struct btrfs_device **extent_dev,
3136                                int *extent_mirror_num)
3137 {
3138         u64 mapped_length;
3139         struct btrfs_bio *bbio = NULL;
3140         int ret;
3141
3142         mapped_length = extent_len;
3143         ret = btrfs_map_block(fs_info, READ, extent_logical,
3144                               &mapped_length, &bbio, 0);
3145         if (ret || !bbio || mapped_length < extent_len ||
3146             !bbio->stripes[0].dev->bdev) {
3147                 kfree(bbio);
3148                 return;
3149         }
3150
3151         *extent_physical = bbio->stripes[0].physical;
3152         *extent_mirror_num = bbio->mirror_num;
3153         *extent_dev = bbio->stripes[0].dev;
3154         kfree(bbio);
3155 }
3156
3157 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
3158                               struct scrub_wr_ctx *wr_ctx,
3159                               struct btrfs_fs_info *fs_info,
3160                               struct btrfs_device *dev,
3161                               int is_dev_replace)
3162 {
3163         WARN_ON(wr_ctx->wr_curr_bio != NULL);
3164
3165         mutex_init(&wr_ctx->wr_lock);
3166         wr_ctx->wr_curr_bio = NULL;
3167         if (!is_dev_replace)
3168                 return 0;
3169
3170         WARN_ON(!dev->bdev);
3171         wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO,
3172                                          bio_get_nr_vecs(dev->bdev));
3173         wr_ctx->tgtdev = dev;
3174         atomic_set(&wr_ctx->flush_all_writes, 0);
3175         return 0;
3176 }
3177
3178 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
3179 {
3180         mutex_lock(&wr_ctx->wr_lock);
3181         kfree(wr_ctx->wr_curr_bio);
3182         wr_ctx->wr_curr_bio = NULL;
3183         mutex_unlock(&wr_ctx->wr_lock);
3184 }
3185
3186 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
3187                             int mirror_num, u64 physical_for_dev_replace)
3188 {
3189         struct scrub_copy_nocow_ctx *nocow_ctx;
3190         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3191
3192         nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
3193         if (!nocow_ctx) {
3194                 spin_lock(&sctx->stat_lock);
3195                 sctx->stat.malloc_errors++;
3196                 spin_unlock(&sctx->stat_lock);
3197                 return -ENOMEM;
3198         }
3199
3200         scrub_pending_trans_workers_inc(sctx);
3201
3202         nocow_ctx->sctx = sctx;
3203         nocow_ctx->logical = logical;
3204         nocow_ctx->len = len;
3205         nocow_ctx->mirror_num = mirror_num;
3206         nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
3207         btrfs_init_work(&nocow_ctx->work, btrfs_scrubnc_helper,
3208                         copy_nocow_pages_worker, NULL, NULL);
3209         INIT_LIST_HEAD(&nocow_ctx->inodes);
3210         btrfs_queue_work(fs_info->scrub_nocow_workers,
3211                          &nocow_ctx->work);
3212
3213         return 0;
3214 }
3215
3216 static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
3217 {
3218         struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
3219         struct scrub_nocow_inode *nocow_inode;
3220
3221         nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
3222         if (!nocow_inode)
3223                 return -ENOMEM;
3224         nocow_inode->inum = inum;
3225         nocow_inode->offset = offset;
3226         nocow_inode->root = root;
3227         list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
3228         return 0;
3229 }
3230
3231 #define COPY_COMPLETE 1
3232
3233 static void copy_nocow_pages_worker(struct btrfs_work *work)
3234 {
3235         struct scrub_copy_nocow_ctx *nocow_ctx =
3236                 container_of(work, struct scrub_copy_nocow_ctx, work);
3237         struct scrub_ctx *sctx = nocow_ctx->sctx;
3238         u64 logical = nocow_ctx->logical;
3239         u64 len = nocow_ctx->len;
3240         int mirror_num = nocow_ctx->mirror_num;
3241         u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3242         int ret;
3243         struct btrfs_trans_handle *trans = NULL;
3244         struct btrfs_fs_info *fs_info;
3245         struct btrfs_path *path;
3246         struct btrfs_root *root;
3247         int not_written = 0;
3248
3249         fs_info = sctx->dev_root->fs_info;
3250         root = fs_info->extent_root;
3251
3252         path = btrfs_alloc_path();
3253         if (!path) {
3254                 spin_lock(&sctx->stat_lock);
3255                 sctx->stat.malloc_errors++;
3256                 spin_unlock(&sctx->stat_lock);
3257                 not_written = 1;
3258                 goto out;
3259         }
3260
3261         trans = btrfs_join_transaction(root);
3262         if (IS_ERR(trans)) {
3263                 not_written = 1;
3264                 goto out;
3265         }
3266
3267         ret = iterate_inodes_from_logical(logical, fs_info, path,
3268                                           record_inode_for_nocow, nocow_ctx);
3269         if (ret != 0 && ret != -ENOENT) {
3270                 btrfs_warn(fs_info, "iterate_inodes_from_logical() failed: log %llu, "
3271                         "phys %llu, len %llu, mir %u, ret %d",
3272                         logical, physical_for_dev_replace, len, mirror_num,
3273                         ret);
3274                 not_written = 1;
3275                 goto out;
3276         }
3277
3278         btrfs_end_transaction(trans, root);
3279         trans = NULL;
3280         while (!list_empty(&nocow_ctx->inodes)) {
3281                 struct scrub_nocow_inode *entry;
3282                 entry = list_first_entry(&nocow_ctx->inodes,
3283                                          struct scrub_nocow_inode,
3284                                          list);
3285                 list_del_init(&entry->list);
3286                 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
3287                                                  entry->root, nocow_ctx);
3288                 kfree(entry);
3289                 if (ret == COPY_COMPLETE) {
3290                         ret = 0;
3291                         break;
3292                 } else if (ret) {
3293                         break;
3294                 }
3295         }
3296 out:
3297         while (!list_empty(&nocow_ctx->inodes)) {
3298                 struct scrub_nocow_inode *entry;
3299                 entry = list_first_entry(&nocow_ctx->inodes,
3300                                          struct scrub_nocow_inode,
3301                                          list);
3302                 list_del_init(&entry->list);
3303                 kfree(entry);
3304         }
3305         if (trans && !IS_ERR(trans))
3306                 btrfs_end_transaction(trans, root);
3307         if (not_written)
3308                 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
3309                                             num_uncorrectable_read_errors);
3310
3311         btrfs_free_path(path);
3312         kfree(nocow_ctx);
3313
3314         scrub_pending_trans_workers_dec(sctx);
3315 }
3316
3317 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
3318                                       struct scrub_copy_nocow_ctx *nocow_ctx)
3319 {
3320         struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
3321         struct btrfs_key key;
3322         struct inode *inode;
3323         struct page *page;
3324         struct btrfs_root *local_root;
3325         struct btrfs_ordered_extent *ordered;
3326         struct extent_map *em;
3327         struct extent_state *cached_state = NULL;
3328         struct extent_io_tree *io_tree;
3329         u64 physical_for_dev_replace;
3330         u64 len = nocow_ctx->len;
3331         u64 lockstart = offset, lockend = offset + len - 1;
3332         unsigned long index;
3333         int srcu_index;
3334         int ret = 0;
3335         int err = 0;
3336
3337         key.objectid = root;
3338         key.type = BTRFS_ROOT_ITEM_KEY;
3339         key.offset = (u64)-1;
3340
3341         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
3342
3343         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
3344         if (IS_ERR(local_root)) {
3345                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
3346                 return PTR_ERR(local_root);
3347         }
3348
3349         key.type = BTRFS_INODE_ITEM_KEY;
3350         key.objectid = inum;
3351         key.offset = 0;
3352         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
3353         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
3354         if (IS_ERR(inode))
3355                 return PTR_ERR(inode);
3356
3357         /* Avoid truncate/dio/punch hole.. */
3358         mutex_lock(&inode->i_mutex);
3359         inode_dio_wait(inode);
3360
3361         physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3362         io_tree = &BTRFS_I(inode)->io_tree;
3363
3364         lock_extent_bits(io_tree, lockstart, lockend, 0, &cached_state);
3365         ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
3366         if (ordered) {
3367                 btrfs_put_ordered_extent(ordered);
3368                 goto out_unlock;
3369         }
3370
3371         em = btrfs_get_extent(inode, NULL, 0, lockstart, len, 0);
3372         if (IS_ERR(em)) {
3373                 ret = PTR_ERR(em);
3374                 goto out_unlock;
3375         }
3376
3377         /*
3378          * This extent does not actually cover the logical extent anymore,
3379          * move on to the next inode.
3380          */
3381         if (em->block_start > nocow_ctx->logical ||
3382             em->block_start + em->block_len < nocow_ctx->logical + len) {
3383                 free_extent_map(em);
3384                 goto out_unlock;
3385         }
3386         free_extent_map(em);
3387
3388         while (len >= PAGE_CACHE_SIZE) {
3389                 index = offset >> PAGE_CACHE_SHIFT;
3390 again:
3391                 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
3392                 if (!page) {
3393                         btrfs_err(fs_info, "find_or_create_page() failed");
3394                         ret = -ENOMEM;
3395                         goto out;
3396                 }
3397
3398                 if (PageUptodate(page)) {
3399                         if (PageDirty(page))
3400                                 goto next_page;
3401                 } else {
3402                         ClearPageError(page);
3403                         err = extent_read_full_page_nolock(io_tree, page,
3404                                                            btrfs_get_extent,
3405                                                            nocow_ctx->mirror_num);
3406                         if (err) {
3407                                 ret = err;
3408                                 goto next_page;
3409                         }
3410
3411                         lock_page(page);
3412                         /*
3413                          * If the page has been remove from the page cache,
3414                          * the data on it is meaningless, because it may be
3415                          * old one, the new data may be written into the new
3416                          * page in the page cache.
3417                          */
3418                         if (page->mapping != inode->i_mapping) {
3419                                 unlock_page(page);
3420                                 page_cache_release(page);
3421                                 goto again;
3422                         }
3423                         if (!PageUptodate(page)) {
3424                                 ret = -EIO;
3425                                 goto next_page;
3426                         }
3427                 }
3428                 err = write_page_nocow(nocow_ctx->sctx,
3429                                        physical_for_dev_replace, page);
3430                 if (err)
3431                         ret = err;
3432 next_page:
3433                 unlock_page(page);
3434                 page_cache_release(page);
3435
3436                 if (ret)
3437                         break;
3438
3439                 offset += PAGE_CACHE_SIZE;
3440                 physical_for_dev_replace += PAGE_CACHE_SIZE;
3441                 len -= PAGE_CACHE_SIZE;
3442         }
3443         ret = COPY_COMPLETE;
3444 out_unlock:
3445         unlock_extent_cached(io_tree, lockstart, lockend, &cached_state,
3446                              GFP_NOFS);
3447 out:
3448         mutex_unlock(&inode->i_mutex);
3449         iput(inode);
3450         return ret;
3451 }
3452
3453 static int write_page_nocow(struct scrub_ctx *sctx,
3454                             u64 physical_for_dev_replace, struct page *page)
3455 {
3456         struct bio *bio;
3457         struct btrfs_device *dev;
3458         int ret;
3459
3460         dev = sctx->wr_ctx.tgtdev;
3461         if (!dev)
3462                 return -EIO;
3463         if (!dev->bdev) {
3464                 printk_ratelimited(KERN_WARNING
3465                         "BTRFS: scrub write_page_nocow(bdev == NULL) is unexpected!\n");
3466                 return -EIO;
3467         }
3468         bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
3469         if (!bio) {
3470                 spin_lock(&sctx->stat_lock);
3471                 sctx->stat.malloc_errors++;
3472                 spin_unlock(&sctx->stat_lock);
3473                 return -ENOMEM;
3474         }
3475         bio->bi_iter.bi_size = 0;
3476         bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
3477         bio->bi_bdev = dev->bdev;
3478         ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
3479         if (ret != PAGE_CACHE_SIZE) {
3480 leave_with_eio:
3481                 bio_put(bio);
3482                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
3483                 return -EIO;
3484         }
3485
3486         if (btrfsic_submit_bio_wait(WRITE_SYNC, bio))
3487                 goto leave_with_eio;
3488
3489         bio_put(bio);
3490         return 0;
3491 }