2 * Copyright 2012 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #define nv50_instmem(p) container_of((p), struct nv50_instmem, base)
27 #include <core/memory.h>
28 #include <subdev/bar.h>
29 #include <subdev/fb.h>
30 #include <subdev/mmu.h>
33 struct nvkm_instmem base;
36 /* Mappings that can be evicted when BAR2 space has been exhausted. */
40 /******************************************************************************
41 * instmem object implementation
42 *****************************************************************************/
43 #define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)
46 struct nvkm_instobj base;
47 struct nv50_instmem *imem;
48 struct nvkm_memory *ram;
56 nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
58 struct nv50_instobj *iobj = nv50_instobj(memory);
59 struct nv50_instmem *imem = iobj->imem;
60 struct nvkm_device *device = imem->base.subdev.device;
61 u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
62 u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
65 spin_lock_irqsave(&imem->base.lock, flags);
66 if (unlikely(imem->addr != base)) {
67 nvkm_wr32(device, 0x001700, base >> 16);
70 nvkm_wr32(device, 0x700000 + addr, data);
71 spin_unlock_irqrestore(&imem->base.lock, flags);
75 nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)
77 struct nv50_instobj *iobj = nv50_instobj(memory);
78 struct nv50_instmem *imem = iobj->imem;
79 struct nvkm_device *device = imem->base.subdev.device;
80 u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
81 u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
85 spin_lock_irqsave(&imem->base.lock, flags);
86 if (unlikely(imem->addr != base)) {
87 nvkm_wr32(device, 0x001700, base >> 16);
90 data = nvkm_rd32(device, 0x700000 + addr);
91 spin_unlock_irqrestore(&imem->base.lock, flags);
95 static const struct nvkm_memory_ptrs
97 .rd32 = nv50_instobj_rd32_slow,
98 .wr32 = nv50_instobj_wr32_slow,
102 nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
104 iowrite32_native(data, nv50_instobj(memory)->map + offset);
108 nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
110 return ioread32_native(nv50_instobj(memory)->map + offset);
113 static const struct nvkm_memory_ptrs
114 nv50_instobj_fast = {
115 .rd32 = nv50_instobj_rd32,
116 .wr32 = nv50_instobj_wr32,
120 nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
122 struct nv50_instmem *imem = iobj->imem;
123 struct nv50_instobj *eobj;
124 struct nvkm_memory *memory = &iobj->base.memory;
125 struct nvkm_subdev *subdev = &imem->base.subdev;
126 struct nvkm_device *device = subdev->device;
127 struct nvkm_vma *bar = NULL, *ebar;
128 u64 size = nvkm_memory_size(memory);
132 /* Attempt to allocate BAR2 address-space and map the object
133 * into it. The lock has to be dropped while doing this due
134 * to the possibility of recursion for page table allocation.
136 mutex_unlock(&subdev->mutex);
137 while ((ret = nvkm_vmm_get(vmm, 12, size, &bar))) {
138 /* Evict unused mappings, and keep retrying until we either
139 * succeed,or there's no more objects left on the LRU.
141 mutex_lock(&subdev->mutex);
142 eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru);
144 nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n",
145 nvkm_memory_addr(&eobj->base.memory),
146 nvkm_memory_size(&eobj->base.memory),
148 list_del_init(&eobj->lru);
154 mutex_unlock(&subdev->mutex);
158 nvkm_vmm_put(vmm, &ebar);
162 ret = nvkm_memory_map(memory, 0, vmm, bar, NULL, 0);
163 mutex_lock(&subdev->mutex);
164 if (ret || iobj->bar) {
165 /* We either failed, or another thread beat us. */
166 mutex_unlock(&subdev->mutex);
167 nvkm_vmm_put(vmm, &bar);
168 mutex_lock(&subdev->mutex);
172 /* Make the mapping visible to the host. */
174 iobj->map = ioremap_wc(device->func->resource_addr(device, 3) +
175 (u32)iobj->bar->addr, size);
177 nvkm_warn(subdev, "PRAMIN ioremap failed\n");
178 nvkm_vmm_put(vmm, &iobj->bar);
183 nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
184 struct nvkm_vma *vma, void *argv, u32 argc)
186 memory = nv50_instobj(memory)->ram;
187 return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
191 nv50_instobj_release(struct nvkm_memory *memory)
193 struct nv50_instobj *iobj = nv50_instobj(memory);
194 struct nv50_instmem *imem = iobj->imem;
195 struct nvkm_subdev *subdev = &imem->base.subdev;
198 nvkm_bar_flush(subdev->device->bar);
200 if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) {
201 /* Add the now-unused mapping to the LRU instead of directly
202 * unmapping it here, in case we need to map it again later.
204 if (likely(iobj->lru.next) && iobj->map) {
205 BUG_ON(!list_empty(&iobj->lru));
206 list_add_tail(&iobj->lru, &imem->lru);
209 /* Switch back to NULL accessors when last map is gone. */
210 iobj->base.memory.ptrs = NULL;
211 mutex_unlock(&subdev->mutex);
215 static void __iomem *
216 nv50_instobj_acquire(struct nvkm_memory *memory)
218 struct nv50_instobj *iobj = nv50_instobj(memory);
219 struct nvkm_instmem *imem = &iobj->imem->base;
220 struct nvkm_vmm *vmm;
221 void __iomem *map = NULL;
223 /* Already mapped? */
224 if (refcount_inc_not_zero(&iobj->maps))
227 /* Take the lock, and re-check that another thread hasn't
228 * already mapped the object in the meantime.
230 mutex_lock(&imem->subdev.mutex);
231 if (refcount_inc_not_zero(&iobj->maps)) {
232 mutex_unlock(&imem->subdev.mutex);
236 /* Attempt to get a direct CPU mapping of the object. */
237 if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {
239 nv50_instobj_kmap(iobj, vmm);
243 if (!refcount_inc_not_zero(&iobj->maps)) {
244 /* Exclude object from eviction while it's being accessed. */
245 if (likely(iobj->lru.next))
246 list_del_init(&iobj->lru);
249 iobj->base.memory.ptrs = &nv50_instobj_fast;
251 iobj->base.memory.ptrs = &nv50_instobj_slow;
252 refcount_set(&iobj->maps, 1);
255 mutex_unlock(&imem->subdev.mutex);
260 nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
262 struct nv50_instobj *iobj = nv50_instobj(memory);
263 struct nvkm_instmem *imem = &iobj->imem->base;
265 /* Exclude bootstrapped objects (ie. the page tables for the
266 * instmem BAR itself) from eviction.
268 mutex_lock(&imem->subdev.mutex);
269 if (likely(iobj->lru.next)) {
270 list_del_init(&iobj->lru);
271 iobj->lru.next = NULL;
274 nv50_instobj_kmap(iobj, vmm);
275 nvkm_instmem_boot(imem);
276 mutex_unlock(&imem->subdev.mutex);
280 nv50_instobj_size(struct nvkm_memory *memory)
282 return nvkm_memory_size(nv50_instobj(memory)->ram);
286 nv50_instobj_addr(struct nvkm_memory *memory)
288 return nvkm_memory_addr(nv50_instobj(memory)->ram);
292 nv50_instobj_bar2(struct nvkm_memory *memory)
294 struct nv50_instobj *iobj = nv50_instobj(memory);
296 if (nv50_instobj_acquire(&iobj->base.memory)) {
297 iobj->lru.next = NULL; /* Exclude from eviction. */
298 addr = iobj->bar->addr;
300 nv50_instobj_release(&iobj->base.memory);
304 static enum nvkm_memory_target
305 nv50_instobj_target(struct nvkm_memory *memory)
307 return nvkm_memory_target(nv50_instobj(memory)->ram);
311 nv50_instobj_dtor(struct nvkm_memory *memory)
313 struct nv50_instobj *iobj = nv50_instobj(memory);
314 struct nvkm_instmem *imem = &iobj->imem->base;
315 struct nvkm_vma *bar;
318 mutex_lock(&imem->subdev.mutex);
319 if (likely(iobj->lru.next))
320 list_del(&iobj->lru);
323 mutex_unlock(&imem->subdev.mutex);
326 struct nvkm_vmm *vmm = nvkm_bar_bar2_vmm(imem->subdev.device);
328 if (likely(vmm)) /* Can be NULL during BAR destructor. */
329 nvkm_vmm_put(vmm, &bar);
332 nvkm_memory_unref(&iobj->ram);
333 nvkm_instobj_dtor(imem, &iobj->base);
337 static const struct nvkm_memory_func
338 nv50_instobj_func = {
339 .dtor = nv50_instobj_dtor,
340 .target = nv50_instobj_target,
341 .bar2 = nv50_instobj_bar2,
342 .addr = nv50_instobj_addr,
343 .size = nv50_instobj_size,
344 .boot = nv50_instobj_boot,
345 .acquire = nv50_instobj_acquire,
346 .release = nv50_instobj_release,
347 .map = nv50_instobj_map,
351 nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
352 struct nvkm_memory **pmemory)
354 struct nv50_instmem *imem = nv50_instmem(base);
355 struct nv50_instobj *iobj;
356 struct nvkm_device *device = imem->base.subdev.device;
357 u8 page = max(order_base_2(align), 12);
359 if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
361 *pmemory = &iobj->base.memory;
363 nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
365 refcount_set(&iobj->maps, 0);
366 INIT_LIST_HEAD(&iobj->lru);
368 return nvkm_ram_get(device, 0, 1, page, size, true, true, &iobj->ram);
371 /******************************************************************************
372 * instmem subdev implementation
373 *****************************************************************************/
376 nv50_instmem_fini(struct nvkm_instmem *base)
378 nv50_instmem(base)->addr = ~0ULL;
381 static const struct nvkm_instmem_func
383 .fini = nv50_instmem_fini,
384 .memory_new = nv50_instobj_new,
389 nv50_instmem_new(struct nvkm_device *device, int index,
390 struct nvkm_instmem **pimem)
392 struct nv50_instmem *imem;
394 if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))
396 nvkm_instmem_ctor(&nv50_instmem, device, index, &imem->base);
397 INIT_LIST_HEAD(&imem->lru);
398 *pimem = &imem->base;