drivers/xen/grant-table.c

   1 /******************************************************************************
   2  * grant_table.c
   3  *
   4  * Granting foreign access to our memory reservation.
   5  *
   6  * Copyright (c) 2005-2006, Christopher Clark
   7  * Copyright (c) 2004-2005, K A Fraser
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License version 2
  11  * as published by the Free Software Foundation; or, when distributed
  12  * separately from the Linux kernel or incorporated into other
  13  * software packages, subject to the following license:
  14  *
  15  * Permission is hereby granted, free of charge, to any person obtaining a copy
  16  * of this source file (the "Software"), to deal in the Software without
  17  * restriction, including without limitation the rights to use, copy, modify,
  18  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19  * and to permit persons to whom the Software is furnished to do so, subject to
  20  * the following conditions:
  21  *
  22  * The above copyright notice and this permission notice shall be included in
  23  * all copies or substantial portions of the Software.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31  * IN THE SOFTWARE.
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/sched.h>
  36 #include <linux/mm.h>
  37 #include <linux/slab.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/io.h>
  41 #include <linux/delay.h>
  42 #include <linux/hardirq.h>
  43
  44 #include <xen/xen.h>
  45 #include <xen/interface/xen.h>
  46 #include <xen/page.h>
  47 #include <xen/grant_table.h>
  48 #include <xen/interface/memory.h>
  49 #include <xen/hvc-console.h>
  50 #include <asm/xen/hypercall.h>
  51 #include <asm/xen/interface.h>
  52
  53 #include <asm/pgtable.h>
  54 #include <asm/sync_bitops.h>
  55
  56 /* External tools reserve first few grant table entries. */
  57 #define NR_RESERVED_ENTRIES 8
  58 #define GNTTAB_LIST_END 0xffffffff
  59
  60 static grant_ref_t **gnttab_list;
  61 static unsigned int nr_grant_frames;
  62 static unsigned int boot_max_nr_grant_frames;
  63 static int gnttab_free_count;
  64 static grant_ref_t gnttab_free_head;
  65 static DEFINE_SPINLOCK(gnttab_list_lock);
  66 unsigned long xen_hvm_resume_frames;
  67 EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
  68
  69 static union {
  70         struct grant_entry_v1 *v1;
  71         union grant_entry_v2 *v2;
  72         void *addr;
  73 } gnttab_shared;
  74
  75 /*This is a structure of function pointers for grant table*/
  76 struct gnttab_ops {
  77         /*
  78          * Mapping a list of frames for storing grant entries. Frames parameter
  79          * is used to store grant table address when grant table being setup,
  80          * nr_gframes is the number of frames to map grant table. Returning
  81          * GNTST_okay means success and negative value means failure.
  82          */
  83         int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
  84         /*
  85          * Release a list of frames which are mapped in map_frames for grant
  86          * entry status.
  87          */
  88         void (*unmap_frames)(void);
  89         /*
  90          * Introducing a valid entry into the grant table, granting the frame of
  91          * this grant entry to domain for accessing or transfering. Ref
  92          * parameter is reference of this introduced grant entry, domid is id of
  93          * granted domain, frame is the page frame to be granted, and flags is
  94          * status of the grant entry to be updated.
  95          */
  96         void (*update_entry)(grant_ref_t ref, domid_t domid,
  97                              unsigned long frame, unsigned flags);
  98         /*
  99          * Stop granting a grant entry to domain for accessing. Ref parameter is
 100          * reference of a grant entry whose grant access will be stopped,
 101          * readonly is not in use in this function. If the grant entry is
 102          * currently mapped for reading or writing, just return failure(==0)
 103          * directly and don't tear down the grant access. Otherwise, stop grant
 104          * access for this entry and return success(==1).
 105          */
 106         int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
 107         /*
 108          * Stop granting a grant entry to domain for transfer. Ref parameter is
 109          * reference of a grant entry whose grant transfer will be stopped. If
 110          * tranfer has not started, just reclaim the grant entry and return
 111          * failure(==0). Otherwise, wait for the transfer to complete and then
 112          * return the frame.
 113          */
 114         unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
 115         /*
 116          * Query the status of a grant entry. Ref parameter is reference of
 117          * queried grant entry, return value is the status of queried entry.
 118          * Detailed status(writing/reading) can be gotten from the return value
 119          * by bit operations.
 120          */
 121         int (*query_foreign_access)(grant_ref_t ref);
 122         /*
 123          * Grant a domain to access a range of bytes within the page referred by
 124          * an available grant entry. Ref parameter is reference of a grant entry
 125          * which will be sub-page accessed, domid is id of grantee domain, frame
 126          * is frame address of subpage grant, flags is grant type and flag
 127          * information, page_off is offset of the range of bytes, and length is
 128          * length of bytes to be accessed.
 129          */
 130         void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
 131                                      unsigned long frame, int flags,
 132                                      unsigned page_off, unsigned length);
 133         /*
 134          * Redirect an available grant entry on domain A to another grant
 135          * reference of domain B, then allow domain C to use grant reference
 136          * of domain B transitively. Ref parameter is an available grant entry
 137          * reference on domain A, domid is id of domain C which accesses grant
 138          * entry transitively, flags is grant type and flag information,
 139          * trans_domid is id of domain B whose grant entry is finally accessed
 140          * transitively, trans_gref is grant entry transitive reference of
 141          * domain B.
 142          */
 143         void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
 144                                    domid_t trans_domid, grant_ref_t trans_gref);
 145 };
 146
 147 static struct gnttab_ops *gnttab_interface;
 148
 149 /*This reflects status of grant entries, so act as a global value*/
 150 static grant_status_t *grstatus;
 151
 152 static int grant_table_version;
 153 static int grefs_per_grant_frame;
 154
 155 static struct gnttab_free_callback *gnttab_free_callback_list;
 156
 157 static int gnttab_expand(unsigned int req_entries);
 158
 159 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
 160 #define SPP (PAGE_SIZE / sizeof(grant_status_t))
 161
 162 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
 163 {
 164         return &gnttab_list[(entry) / RPP][(entry) % RPP];
 165 }
 166 /* This can be used as an l-value */
 167 #define gnttab_entry(entry) (*__gnttab_entry(entry))
 168
 169 static int get_free_entries(unsigned count)
 170 {
 171         unsigned long flags;
 172         int ref, rc = 0;
 173         grant_ref_t head;
 174
 175         spin_lock_irqsave(&gnttab_list_lock, flags);
 176
 177         if ((gnttab_free_count < count) &&
 178             ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 179                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 180                 return rc;
 181         }
 182
 183         ref = head = gnttab_free_head;
 184         gnttab_free_count -= count;
 185         while (count-- > 1)
 186                 head = gnttab_entry(head);
 187         gnttab_free_head = gnttab_entry(head);
 188         gnttab_entry(head) = GNTTAB_LIST_END;
 189
 190         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 191
 192         return ref;
 193 }
 194
 195 static void do_free_callbacks(void)
 196 {
 197         struct gnttab_free_callback *callback, *next;
 198
 199         callback = gnttab_free_callback_list;
 200         gnttab_free_callback_list = NULL;
 201
 202         while (callback != NULL) {
 203                 next = callback->next;
 204                 if (gnttab_free_count >= callback->count) {
 205                         callback->next = NULL;
 206                         callback->fn(callback->arg);
 207                 } else {
 208                         callback->next = gnttab_free_callback_list;
 209                         gnttab_free_callback_list = callback;
 210                 }
 211                 callback = next;
 212         }
 213 }
 214
 215 static inline void check_free_callbacks(void)
 216 {
 217         if (unlikely(gnttab_free_callback_list))
 218                 do_free_callbacks();
 219 }
 220
 221 static void put_free_entry(grant_ref_t ref)
 222 {
 223         unsigned long flags;
 224         spin_lock_irqsave(&gnttab_list_lock, flags);
 225         gnttab_entry(ref) = gnttab_free_head;
 226         gnttab_free_head = ref;
 227         gnttab_free_count++;
 228         check_free_callbacks();
 229         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 230 }
 231
 232 /*
 233  * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
 234  * Introducing a valid entry into the grant table:
 235  *  1. Write ent->domid.
 236  *  2. Write ent->frame:
 237  *      GTF_permit_access:   Frame to which access is permitted.
 238  *      GTF_accept_transfer: Pseudo-phys frame slot being filled by new
 239  *                           frame, or zero if none.
 240  *  3. Write memory barrier (WMB).
 241  *  4. Write ent->flags, inc. valid type.
 242  */
 243 static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
 244                                    unsigned long frame, unsigned flags)
 245 {
 246         gnttab_shared.v1[ref].domid = domid;
 247         gnttab_shared.v1[ref].frame = frame;
 248         wmb();
 249         gnttab_shared.v1[ref].flags = flags;
 250 }
 251
 252 static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
 253                                    unsigned long frame, unsigned flags)
 254 {
 255         gnttab_shared.v2[ref].hdr.domid = domid;
 256         gnttab_shared.v2[ref].full_page.frame = frame;
 257         wmb();
 258         gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
 259 }
 260
 261 /*
 262  * Public grant-issuing interface functions
 263  */
 264 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 265                                      unsigned long frame, int readonly)
 266 {
 267         gnttab_interface->update_entry(ref, domid, frame,
 268                            GTF_permit_access | (readonly ? GTF_readonly : 0));
 269 }
 270 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 271
 272 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 273                                 int readonly)
 274 {
 275         int ref;
 276
 277         ref = get_free_entries(1);
 278         if (unlikely(ref < 0))
 279                 return -ENOSPC;
 280
 281         gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
 282
 283         return ref;
 284 }
 285 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 286
 287 static void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
 288                                            unsigned long frame, int flags,
 289                                            unsigned page_off, unsigned length)
 290 {
 291         gnttab_shared.v2[ref].sub_page.frame = frame;
 292         gnttab_shared.v2[ref].sub_page.page_off = page_off;
 293         gnttab_shared.v2[ref].sub_page.length = length;
 294         gnttab_shared.v2[ref].hdr.domid = domid;
 295         wmb();
 296         gnttab_shared.v2[ref].hdr.flags =
 297                                 GTF_permit_access | GTF_sub_page | flags;
 298 }
 299
 300 int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
 301                                             unsigned long frame, int flags,
 302                                             unsigned page_off,
 303                                             unsigned length)
 304 {
 305         if (flags & (GTF_accept_transfer | GTF_reading |
 306                      GTF_writing | GTF_transitive))
 307                 return -EPERM;
 308
 309         if (gnttab_interface->update_subpage_entry == NULL)
 310                 return -ENOSYS;
 311
 312         gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
 313                                                page_off, length);
 314
 315         return 0;
 316 }
 317 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
 318
 319 int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
 320                                         int flags, unsigned page_off,
 321                                         unsigned length)
 322 {
 323         int ref, rc;
 324
 325         ref = get_free_entries(1);
 326         if (unlikely(ref < 0))
 327                 return -ENOSPC;
 328
 329         rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
 330                                                      page_off, length);
 331         if (rc < 0) {
 332                 put_free_entry(ref);
 333                 return rc;
 334         }
 335
 336         return ref;
 337 }
 338 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
 339
 340 bool gnttab_subpage_grants_available(void)
 341 {
 342         return gnttab_interface->update_subpage_entry != NULL;
 343 }
 344 EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
 345
 346 static void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
 347                                          int flags, domid_t trans_domid,
 348                                          grant_ref_t trans_gref)
 349 {
 350         gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
 351         gnttab_shared.v2[ref].transitive.gref = trans_gref;
 352         gnttab_shared.v2[ref].hdr.domid = domid;
 353         wmb();
 354         gnttab_shared.v2[ref].hdr.flags =
 355                                 GTF_permit_access | GTF_transitive | flags;
 356 }
 357
 358 int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
 359                                           int flags, domid_t trans_domid,
 360                                           grant_ref_t trans_gref)
 361 {
 362         if (flags & (GTF_accept_transfer | GTF_reading |
 363                      GTF_writing | GTF_sub_page))
 364                 return -EPERM;
 365
 366         if (gnttab_interface->update_trans_entry == NULL)
 367                 return -ENOSYS;
 368
 369         gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
 370                                              trans_gref);
 371
 372         return 0;
 373 }
 374 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
 375
 376 int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
 377                                       domid_t trans_domid,
 378                                       grant_ref_t trans_gref)
 379 {
 380         int ref, rc;
 381
 382         ref = get_free_entries(1);
 383         if (unlikely(ref < 0))
 384                 return -ENOSPC;
 385
 386         rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
 387                                                    trans_domid, trans_gref);
 388         if (rc < 0) {
 389                 put_free_entry(ref);
 390                 return rc;
 391         }
 392
 393         return ref;
 394 }
 395 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
 396
 397 bool gnttab_trans_grants_available(void)
 398 {
 399         return gnttab_interface->update_trans_entry != NULL;
 400 }
 401 EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
 402
 403 static int gnttab_query_foreign_access_v1(grant_ref_t ref)
 404 {
 405         return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
 406 }
 407
 408 static int gnttab_query_foreign_access_v2(grant_ref_t ref)
 409 {
 410         return grstatus[ref] & (GTF_reading|GTF_writing);
 411 }
 412
 413 int gnttab_query_foreign_access(grant_ref_t ref)
 414 {
 415         return gnttab_interface->query_foreign_access(ref);
 416 }
 417 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
 418
 419 static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
 420 {
 421         u16 flags, nflags;
 422         u16 *pflags;
 423
 424         pflags = &gnttab_shared.v1[ref].flags;
 425         nflags = *pflags;
 426         do {
 427                 flags = nflags;
 428                 if (flags & (GTF_reading|GTF_writing))
 429                         return 0;
 430         } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
 431
 432         return 1;
 433 }
 434
 435 static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
 436 {
 437         gnttab_shared.v2[ref].hdr.flags = 0;
 438         mb();
 439         if (grstatus[ref] & (GTF_reading|GTF_writing)) {
 440                 return 0;
 441         } else {
 442                 /* The read of grstatus needs to have acquire
 443                 semantics.  On x86, reads already have
 444                 that, and we just need to protect against
 445                 compiler reorderings.  On other
 446                 architectures we may need a full
 447                 barrier. */
 448 #ifdef CONFIG_X86
 449                 barrier();
 450 #else
 451                 mb();
 452 #endif
 453         }
 454
 455         return 1;
 456 }
 457
 458 static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 459 {
 460         return gnttab_interface->end_foreign_access_ref(ref, readonly);
 461 }
 462
 463 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 464 {
 465         if (_gnttab_end_foreign_access_ref(ref, readonly))
 466                 return 1;
 467         pr_warn("WARNING: g.e. %#x still in use!\n", ref);
 468         return 0;
 469 }
 470 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 471
 472 struct deferred_entry {
 473         struct list_head list;
 474         grant_ref_t ref;
 475         bool ro;
 476         uint16_t warn_delay;
 477         struct page *page;
 478 };
 479 static LIST_HEAD(deferred_list);
 480 static void gnttab_handle_deferred(unsigned long);
 481 static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
 482
 483 static void gnttab_handle_deferred(unsigned long unused)
 484 {
 485         unsigned int nr = 10;
 486         struct deferred_entry *first = NULL;
 487         unsigned long flags;
 488
 489         spin_lock_irqsave(&gnttab_list_lock, flags);
 490         while (nr--) {
 491                 struct deferred_entry *entry
 492                         = list_first_entry(&deferred_list,
 493                                            struct deferred_entry, list);
 494
 495                 if (entry == first)
 496                         break;
 497                 list_del(&entry->list);
 498                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 499                 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
 500                         put_free_entry(entry->ref);
 501                         if (entry->page) {
 502                                 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
 503                                          entry->ref, page_to_pfn(entry->page));
 504                                 __free_page(entry->page);
 505                         } else
 506                                 pr_info("freeing g.e. %#x\n", entry->ref);
 507                         kfree(entry);
 508                         entry = NULL;
 509                 } else {
 510                         if (!--entry->warn_delay)
 511                                 pr_info("g.e. %#x still pending\n",
 512                                         entry->ref);
 513                         if (!first)
 514                                 first = entry;
 515                 }
 516                 spin_lock_irqsave(&gnttab_list_lock, flags);
 517                 if (entry)
 518                         list_add_tail(&entry->list, &deferred_list);
 519                 else if (list_empty(&deferred_list))
 520                         break;
 521         }
 522         if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
 523                 deferred_timer.expires = jiffies + HZ;
 524                 add_timer(&deferred_timer);
 525         }
 526         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 527 }
 528
 529 static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
 530                                 struct page *page)
 531 {
 532         struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 533         const char *what = KERN_WARNING "leaking";
 534
 535         if (entry) {
 536                 unsigned long flags;
 537
 538                 entry->ref = ref;
 539                 entry->ro = readonly;
 540                 entry->page = page;
 541                 entry->warn_delay = 60;
 542                 spin_lock_irqsave(&gnttab_list_lock, flags);
 543                 list_add_tail(&entry->list, &deferred_list);
 544                 if (!timer_pending(&deferred_timer)) {
 545                         deferred_timer.expires = jiffies + HZ;
 546                         add_timer(&deferred_timer);
 547                 }
 548                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 549                 what = KERN_DEBUG "deferring";
 550         }
 551         printk("%s g.e. %#x (pfn %#lx)\n",
 552                what, ref, page ? page_to_pfn(page) : -1);
 553 }
 554
 555 void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
 556                                unsigned long page)
 557 {
 558         if (gnttab_end_foreign_access_ref(ref, readonly)) {
 559                 put_free_entry(ref);
 560                 if (page != 0)
 561                         free_page(page);
 562         } else
 563                 gnttab_add_deferred(ref, readonly,
 564                                     page ? virt_to_page(page) : NULL);
 565 }
 566 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 567
 568 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
 569 {
 570         int ref;
 571
 572         ref = get_free_entries(1);
 573         if (unlikely(ref < 0))
 574                 return -ENOSPC;
 575         gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 576
 577         return ref;
 578 }
 579 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
 580
 581 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
 582                                        unsigned long pfn)
 583 {
 584         gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
 585 }
 586 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
 587
 588 static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
 589 {
 590         unsigned long frame;
 591         u16           flags;
 592         u16          *pflags;
 593
 594         pflags = &gnttab_shared.v1[ref].flags;
 595
 596         /*
 597          * If a transfer is not even yet started, try to reclaim the grant
 598          * reference and return failure (== 0).
 599          */
 600         while (!((flags = *pflags) & GTF_transfer_committed)) {
 601                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 602                         return 0;
 603                 cpu_relax();
 604         }
 605
 606         /* If a transfer is in progress then wait until it is completed. */
 607         while (!(flags & GTF_transfer_completed)) {
 608                 flags = *pflags;
 609                 cpu_relax();
 610         }
 611
 612         rmb();  /* Read the frame number /after/ reading completion status. */
 613         frame = gnttab_shared.v1[ref].frame;
 614         BUG_ON(frame == 0);
 615
 616         return frame;
 617 }
 618
 619 static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
 620 {
 621         unsigned long frame;
 622         u16           flags;
 623         u16          *pflags;
 624
 625         pflags = &gnttab_shared.v2[ref].hdr.flags;
 626
 627         /*
 628          * If a transfer is not even yet started, try to reclaim the grant
 629          * reference and return failure (== 0).
 630          */
 631         while (!((flags = *pflags) & GTF_transfer_committed)) {
 632                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 633                         return 0;
 634                 cpu_relax();
 635         }
 636
 637         /* If a transfer is in progress then wait until it is completed. */
 638         while (!(flags & GTF_transfer_completed)) {
 639                 flags = *pflags;
 640                 cpu_relax();
 641         }
 642
 643         rmb();  /* Read the frame number /after/ reading completion status. */
 644         frame = gnttab_shared.v2[ref].full_page.frame;
 645         BUG_ON(frame == 0);
 646
 647         return frame;
 648 }
 649
 650 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
 651 {
 652         return gnttab_interface->end_foreign_transfer_ref(ref);
 653 }
 654 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
 655
 656 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
 657 {
 658         unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
 659         put_free_entry(ref);
 660         return frame;
 661 }
 662 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
 663
 664 void gnttab_free_grant_reference(grant_ref_t ref)
 665 {
 666         put_free_entry(ref);
 667 }
 668 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 669
 670 void gnttab_free_grant_references(grant_ref_t head)
 671 {
 672         grant_ref_t ref;
 673         unsigned long flags;
 674         int count = 1;
 675         if (head == GNTTAB_LIST_END)
 676                 return;
 677         spin_lock_irqsave(&gnttab_list_lock, flags);
 678         ref = head;
 679         while (gnttab_entry(ref) != GNTTAB_LIST_END) {
 680                 ref = gnttab_entry(ref);
 681                 count++;
 682         }
 683         gnttab_entry(ref) = gnttab_free_head;
 684         gnttab_free_head = head;
 685         gnttab_free_count += count;
 686         check_free_callbacks();
 687         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 688 }
 689 EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 690
 691 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 692 {
 693         int h = get_free_entries(count);
 694
 695         if (h < 0)
 696                 return -ENOSPC;
 697
 698         *head = h;
 699
 700         return 0;
 701 }
 702 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 703
 704 int gnttab_empty_grant_references(const grant_ref_t *private_head)
 705 {
 706         return (*private_head == GNTTAB_LIST_END);
 707 }
 708 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 709
 710 int gnttab_claim_grant_reference(grant_ref_t *private_head)
 711 {
 712         grant_ref_t g = *private_head;
 713         if (unlikely(g == GNTTAB_LIST_END))
 714                 return -ENOSPC;
 715         *private_head = gnttab_entry(g);
 716         return g;
 717 }
 718 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 719
 720 void gnttab_release_grant_reference(grant_ref_t *private_head,
 721                                     grant_ref_t release)
 722 {
 723         gnttab_entry(release) = *private_head;
 724         *private_head = release;
 725 }
 726 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 727
 728 void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 729                                   void (*fn)(void *), void *arg, u16 count)
 730 {
 731         unsigned long flags;
 732         struct gnttab_free_callback *cb;
 733
 734         spin_lock_irqsave(&gnttab_list_lock, flags);
 735
 736         /* Check if the callback is already on the list */
 737         cb = gnttab_free_callback_list;
 738         while (cb) {
 739                 if (cb == callback)
 740                         goto out;
 741                 cb = cb->next;
 742         }
 743
 744         callback->fn = fn;
 745         callback->arg = arg;
 746         callback->count = count;
 747         callback->next = gnttab_free_callback_list;
 748         gnttab_free_callback_list = callback;
 749         check_free_callbacks();
 750 out:
 751         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 752 }
 753 EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 754
 755 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 756 {
 757         struct gnttab_free_callback **pcb;
 758         unsigned long flags;
 759
 760         spin_lock_irqsave(&gnttab_list_lock, flags);
 761         for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 762                 if (*pcb == callback) {
 763                         *pcb = callback->next;
 764                         break;
 765                 }
 766         }
 767         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 768 }
 769 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 770
 771 static int grow_gnttab_list(unsigned int more_frames)
 772 {
 773         unsigned int new_nr_grant_frames, extra_entries, i;
 774         unsigned int nr_glist_frames, new_nr_glist_frames;
 775
 776         BUG_ON(grefs_per_grant_frame == 0);
 777
 778         new_nr_grant_frames = nr_grant_frames + more_frames;
 779         extra_entries       = more_frames * grefs_per_grant_frame;
 780
 781         nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
 782         new_nr_glist_frames =
 783                 (new_nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
 784         for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
 785                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 786                 if (!gnttab_list[i])
 787                         goto grow_nomem;
 788         }
 789
 790
 791         for (i = grefs_per_grant_frame * nr_grant_frames;
 792              i < grefs_per_grant_frame * new_nr_grant_frames - 1; i++)
 793                 gnttab_entry(i) = i + 1;
 794
 795         gnttab_entry(i) = gnttab_free_head;
 796         gnttab_free_head = grefs_per_grant_frame * nr_grant_frames;
 797         gnttab_free_count += extra_entries;
 798
 799         nr_grant_frames = new_nr_grant_frames;
 800
 801         check_free_callbacks();
 802
 803         return 0;
 804
 805 grow_nomem:
 806         for ( ; i >= nr_glist_frames; i--)
 807                 free_page((unsigned long) gnttab_list[i]);
 808         return -ENOMEM;
 809 }
 810
 811 static unsigned int __max_nr_grant_frames(void)
 812 {
 813         struct gnttab_query_size query;
 814         int rc;
 815
 816         query.dom = DOMID_SELF;
 817
 818         rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 819         if ((rc < 0) || (query.status != GNTST_okay))
 820                 return 4; /* Legacy max supported number of frames */
 821
 822         return query.max_nr_frames;
 823 }
 824
 825 unsigned int gnttab_max_grant_frames(void)
 826 {
 827         unsigned int xen_max = __max_nr_grant_frames();
 828
 829         if (xen_max > boot_max_nr_grant_frames)
 830                 return boot_max_nr_grant_frames;
 831         return xen_max;
 832 }
 833 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 834
 835 /* Handling of paged out grant targets (GNTST_eagain) */
 836 #define MAX_DELAY 256
 837 static inline void
 838 gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
 839                                                 const char *func)
 840 {
 841         unsigned delay = 1;
 842
 843         do {
 844                 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
 845                 if (*status == GNTST_eagain)
 846                         msleep(delay++);
 847         } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
 848
 849         if (delay >= MAX_DELAY) {
 850                 printk(KERN_ERR "%s: %s eagain grant\n", func, current->comm);
 851                 *status = GNTST_bad_page;
 852         }
 853 }
 854
 855 void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
 856 {
 857         struct gnttab_map_grant_ref *op;
 858
 859         if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
 860                 BUG();
 861         for (op = batch; op < batch + count; op++)
 862                 if (op->status == GNTST_eagain)
 863                         gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
 864                                                 &op->status, __func__);
 865 }
 866 EXPORT_SYMBOL_GPL(gnttab_batch_map);
 867
 868 void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
 869 {
 870         struct gnttab_copy *op;
 871
 872         if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
 873                 BUG();
 874         for (op = batch; op < batch + count; op++)
 875                 if (op->status == GNTST_eagain)
 876                         gnttab_retry_eagain_gop(GNTTABOP_copy, op,
 877                                                 &op->status, __func__);
 878 }
 879 EXPORT_SYMBOL_GPL(gnttab_batch_copy);
 880
 881 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
 882                     struct gnttab_map_grant_ref *kmap_ops,
 883                     struct page **pages, unsigned int count)
 884 {
 885         int i, ret;
 886         bool lazy = false;
 887         pte_t *pte;
 888         unsigned long mfn;
 889
 890         ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
 891         if (ret)
 892                 return ret;
 893
 894         /* Retry eagain maps */
 895         for (i = 0; i < count; i++)
 896                 if (map_ops[i].status == GNTST_eagain)
 897                         gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
 898                                                 &map_ops[i].status, __func__);
 899
 900         if (xen_feature(XENFEAT_auto_translated_physmap))
 901                 return ret;
 902
 903         if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
 904                 arch_enter_lazy_mmu_mode();
 905                 lazy = true;
 906         }
 907
 908         for (i = 0; i < count; i++) {
 909                 /* Do not add to override if the map failed. */
 910                 if (map_ops[i].status)
 911                         continue;
 912
 913                 if (map_ops[i].flags & GNTMAP_contains_pte) {
 914                         pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
 915                                 (map_ops[i].host_addr & ~PAGE_MASK));
 916                         mfn = pte_mfn(*pte);
 917                 } else {
 918                         mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
 919                 }
 920                 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
 921                                        &kmap_ops[i] : NULL);
 922                 if (ret)
 923                         goto out;
 924         }
 925
 926  out:
 927         if (lazy)
 928                 arch_leave_lazy_mmu_mode();
 929
 930         return ret;
 931 }
 932 EXPORT_SYMBOL_GPL(gnttab_map_refs);
 933
 934 int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
 935                       struct gnttab_map_grant_ref *kmap_ops,
 936                       struct page **pages, unsigned int count)
 937 {
 938         int i, ret;
 939         bool lazy = false;
 940
 941         ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
 942         if (ret)
 943                 return ret;
 944
 945         if (xen_feature(XENFEAT_auto_translated_physmap))
 946                 return ret;
 947
 948         if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
 949                 arch_enter_lazy_mmu_mode();
 950                 lazy = true;
 951         }
 952
 953         for (i = 0; i < count; i++) {
 954                 ret = m2p_remove_override(pages[i], kmap_ops ?
 955                                        &kmap_ops[i] : NULL);
 956                 if (ret)
 957                         goto out;
 958         }
 959
 960  out:
 961         if (lazy)
 962                 arch_leave_lazy_mmu_mode();
 963
 964         return ret;
 965 }
 966 EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
 967
 968 static unsigned nr_status_frames(unsigned nr_grant_frames)
 969 {
 970         BUG_ON(grefs_per_grant_frame == 0);
 971         return (nr_grant_frames * grefs_per_grant_frame + SPP - 1) / SPP;
 972 }
 973
 974 static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
 975 {
 976         int rc;
 977
 978         rc = arch_gnttab_map_shared(frames, nr_gframes,
 979                                     gnttab_max_grant_frames(),
 980                                     &gnttab_shared.addr);
 981         BUG_ON(rc);
 982
 983         return 0;
 984 }
 985
 986 static void gnttab_unmap_frames_v1(void)
 987 {
 988         arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
 989 }
 990
 991 static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
 992 {
 993         uint64_t *sframes;
 994         unsigned int nr_sframes;
 995         struct gnttab_get_status_frames getframes;
 996         int rc;
 997
 998         nr_sframes = nr_status_frames(nr_gframes);
 999
1000         /* No need for kzalloc as it is initialized in following hypercall
1001          * GNTTABOP_get_status_frames.
1002          */
1003         sframes = kmalloc(nr_sframes  * sizeof(uint64_t), GFP_ATOMIC);
1004         if (!sframes)
1005                 return -ENOMEM;
1006
1007         getframes.dom        = DOMID_SELF;
1008         getframes.nr_frames  = nr_sframes;
1009         set_xen_guest_handle(getframes.frame_list, sframes);
1010
1011         rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1012                                        &getframes, 1);
1013         if (rc == -ENOSYS) {
1014                 kfree(sframes);
1015                 return -ENOSYS;
1016         }
1017
1018         BUG_ON(rc || getframes.status);
1019
1020         rc = arch_gnttab_map_status(sframes, nr_sframes,
1021                                     nr_status_frames(gnttab_max_grant_frames()),
1022                                     &grstatus);
1023         BUG_ON(rc);
1024         kfree(sframes);
1025
1026         rc = arch_gnttab_map_shared(frames, nr_gframes,
1027                                     gnttab_max_grant_frames(),
1028                                     &gnttab_shared.addr);
1029         BUG_ON(rc);
1030
1031         return 0;
1032 }
1033
1034 static void gnttab_unmap_frames_v2(void)
1035 {
1036         arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1037         arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
1038 }
1039
1040 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1041 {
1042         struct gnttab_setup_table setup;
1043         xen_pfn_t *frames;
1044         unsigned int nr_gframes = end_idx + 1;
1045         int rc;
1046
1047         if (xen_hvm_domain()) {
1048                 struct xen_add_to_physmap xatp;
1049                 unsigned int i = end_idx;
1050                 rc = 0;
1051                 /*
1052                  * Loop backwards, so that the first hypercall has the largest
1053                  * index, ensuring that the table will grow only once.
1054                  */
1055                 do {
1056                         xatp.domid = DOMID_SELF;
1057                         xatp.idx = i;
1058                         xatp.space = XENMAPSPACE_grant_table;
1059                         xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
1060                         rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1061                         if (rc != 0) {
1062                                 printk(KERN_WARNING
1063                                                 "grant table add_to_physmap failed, err=%d\n", rc);
1064                                 break;
1065                         }
1066                 } while (i-- > start_idx);
1067
1068                 return rc;
1069         }
1070
1071         /* No need for kzalloc as it is initialized in following hypercall
1072          * GNTTABOP_setup_table.
1073          */
1074         frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1075         if (!frames)
1076                 return -ENOMEM;
1077
1078         setup.dom        = DOMID_SELF;
1079         setup.nr_frames  = nr_gframes;
1080         set_xen_guest_handle(setup.frame_list, frames);
1081
1082         rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1083         if (rc == -ENOSYS) {
1084                 kfree(frames);
1085                 return -ENOSYS;
1086         }
1087
1088         BUG_ON(rc || setup.status);
1089
1090         rc = gnttab_interface->map_frames(frames, nr_gframes);
1091
1092         kfree(frames);
1093
1094         return rc;
1095 }
1096
1097 static struct gnttab_ops gnttab_v1_ops = {
1098         .map_frames                     = gnttab_map_frames_v1,
1099         .unmap_frames                   = gnttab_unmap_frames_v1,
1100         .update_entry                   = gnttab_update_entry_v1,
1101         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v1,
1102         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v1,
1103         .query_foreign_access           = gnttab_query_foreign_access_v1,
1104 };
1105
1106 static struct gnttab_ops gnttab_v2_ops = {
1107         .map_frames                     = gnttab_map_frames_v2,
1108         .unmap_frames                   = gnttab_unmap_frames_v2,
1109         .update_entry                   = gnttab_update_entry_v2,
1110         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v2,
1111         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v2,
1112         .query_foreign_access           = gnttab_query_foreign_access_v2,
1113         .update_subpage_entry           = gnttab_update_subpage_entry_v2,
1114         .update_trans_entry             = gnttab_update_trans_entry_v2,
1115 };
1116
1117 static void gnttab_request_version(void)
1118 {
1119         int rc;
1120         struct gnttab_set_version gsv;
1121
1122         if (xen_hvm_domain())
1123                 gsv.version = 1;
1124         else
1125                 gsv.version = 2;
1126         rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1127         if (rc == 0 && gsv.version == 2) {
1128                 grant_table_version = 2;
1129                 grefs_per_grant_frame = PAGE_SIZE / sizeof(union grant_entry_v2);
1130                 gnttab_interface = &gnttab_v2_ops;
1131         } else if (grant_table_version == 2) {
1132                 /*
1133                  * If we've already used version 2 features,
1134                  * but then suddenly discover that they're not
1135                  * available (e.g. migrating to an older
1136                  * version of Xen), almost unbounded badness
1137                  * can happen.
1138                  */
1139                 panic("we need grant tables version 2, but only version 1 is available");
1140         } else {
1141                 grant_table_version = 1;
1142                 grefs_per_grant_frame = PAGE_SIZE / sizeof(struct grant_entry_v1);
1143                 gnttab_interface = &gnttab_v1_ops;
1144         }
1145         printk(KERN_INFO "Grant tables using version %d layout.\n",
1146                 grant_table_version);
1147 }
1148
1149 static int gnttab_setup(void)
1150 {
1151         unsigned int max_nr_gframes;
1152
1153         max_nr_gframes = gnttab_max_grant_frames();
1154         if (max_nr_gframes < nr_grant_frames)
1155                 return -ENOSYS;
1156
1157         if (xen_pv_domain())
1158                 return gnttab_map(0, nr_grant_frames - 1);
1159
1160         if (gnttab_shared.addr == NULL) {
1161                 gnttab_shared.addr = xen_remap(xen_hvm_resume_frames,
1162                                                 PAGE_SIZE * max_nr_gframes);
1163                 if (gnttab_shared.addr == NULL) {
1164                         printk(KERN_WARNING
1165                                         "Failed to ioremap gnttab share frames!");
1166                         return -ENOMEM;
1167                 }
1168         }
1169
1170         gnttab_map(0, nr_grant_frames - 1);
1171
1172         return 0;
1173 }
1174
1175 int gnttab_resume(void)
1176 {
1177         gnttab_request_version();
1178         return gnttab_setup();
1179 }
1180
1181 int gnttab_suspend(void)
1182 {
1183         gnttab_interface->unmap_frames();
1184         return 0;
1185 }
1186
1187 static int gnttab_expand(unsigned int req_entries)
1188 {
1189         int rc;
1190         unsigned int cur, extra;
1191
1192         BUG_ON(grefs_per_grant_frame == 0);
1193         cur = nr_grant_frames;
1194         extra = ((req_entries + (grefs_per_grant_frame-1)) /
1195                  grefs_per_grant_frame);
1196         if (cur + extra > gnttab_max_grant_frames())
1197                 return -ENOSPC;
1198
1199         rc = gnttab_map(cur, cur + extra - 1);
1200         if (rc == 0)
1201                 rc = grow_gnttab_list(extra);
1202
1203         return rc;
1204 }
1205
1206 int gnttab_init(void)
1207 {
1208         int i;
1209         unsigned int max_nr_glist_frames, nr_glist_frames;
1210         unsigned int nr_init_grefs;
1211         int ret;
1212
1213         gnttab_request_version();
1214         nr_grant_frames = 1;
1215         boot_max_nr_grant_frames = __max_nr_grant_frames();
1216
1217         /* Determine the maximum number of frames required for the
1218          * grant reference free list on the current hypervisor.
1219          */
1220         BUG_ON(grefs_per_grant_frame == 0);
1221         max_nr_glist_frames = (boot_max_nr_grant_frames *
1222                                grefs_per_grant_frame / RPP);
1223
1224         gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1225                               GFP_KERNEL);
1226         if (gnttab_list == NULL)
1227                 return -ENOMEM;
1228
1229         nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
1230         for (i = 0; i < nr_glist_frames; i++) {
1231                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1232                 if (gnttab_list[i] == NULL) {
1233                         ret = -ENOMEM;
1234                         goto ini_nomem;
1235                 }
1236         }
1237
1238         if (gnttab_setup() < 0) {
1239                 ret = -ENODEV;
1240                 goto ini_nomem;
1241         }
1242
1243         nr_init_grefs = nr_grant_frames * grefs_per_grant_frame;
1244
1245         for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1246                 gnttab_entry(i) = i + 1;
1247
1248         gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1249         gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1250         gnttab_free_head  = NR_RESERVED_ENTRIES;
1251
1252         printk("Grant table initialized\n");
1253         return 0;
1254
1255  ini_nomem:
1256         for (i--; i >= 0; i--)
1257                 free_page((unsigned long)gnttab_list[i]);
1258         kfree(gnttab_list);
1259         return ret;
1260 }
1261 EXPORT_SYMBOL_GPL(gnttab_init);
1262
1263 static int __gnttab_init(void)
1264 {
1265         /* Delay grant-table initialization in the PV on HVM case */
1266         if (xen_hvm_domain())
1267                 return 0;
1268
1269         if (!xen_pv_domain())
1270                 return -ENODEV;
1271
1272         return gnttab_init();
1273 }
1274
1275 core_initcall(__gnttab_init);