From f07069da6b4c5f937d6df2de6504394845513964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 17 Nov 2020 13:52:28 +0100 Subject: drm/ttm: move memory accounting into vmwgfx v4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is just another feature which is only used by VMWGFX, so move it into the driver instead. I've tried to add the accounting sysfs file to the kobject of the drm minor, but I'm not 100% sure if this works as expected. v2: fix typo in KFD and avoid 64bit divide v3: fix init order in VMWGFX v4: use pdev sysfs reference instead of drm Signed-off-by: Christian König Reviewed-by: Zack Rusin (v3) Tested-by: Nirmoy Das Link: https://patchwork.freedesktop.org/patch/msgid/20210208133226.36955-2-christian.koenig@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 16 +- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 8 +- drivers/gpu/drm/drm_gem_vram_helper.c | 6 +- drivers/gpu/drm/nouveau/nouveau_bo.c | 7 +- drivers/gpu/drm/nouveau/nouveau_drv.h | 1 - drivers/gpu/drm/qxl/qxl_object.c | 4 +- drivers/gpu/drm/radeon/radeon_object.c | 8 +- drivers/gpu/drm/ttm/Makefile | 7 +- drivers/gpu/drm/ttm/ttm_bo.c | 33 +- drivers/gpu/drm/ttm/ttm_bo_util.c | 1 - drivers/gpu/drm/ttm/ttm_device.c | 22 +- drivers/gpu/drm/ttm/ttm_memory.c | 686 ----------------------- drivers/gpu/drm/ttm/ttm_pool.c | 13 +- drivers/gpu/drm/vmwgfx/Makefile | 2 +- drivers/gpu/drm/vmwgfx/ttm_memory.c | 681 ++++++++++++++++++++++ drivers/gpu/drm/vmwgfx/ttm_memory.h | 96 ++++ drivers/gpu/drm/vmwgfx/ttm_object.h | 3 +- drivers/gpu/drm/vmwgfx/vmwgfx_bo.c | 22 +- drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 5 + drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c | 28 +- 20 files changed, 874 insertions(+), 775 deletions(-) delete mode 100644 drivers/gpu/drm/ttm/ttm_memory.c create mode 100644 drivers/gpu/drm/vmwgfx/ttm_memory.c create mode 100644 drivers/gpu/drm/vmwgfx/ttm_memory.h (limited to 'drivers/gpu/drm') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 0849b68e784f..e440af37dde8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -118,6 +118,16 @@ void amdgpu_amdkfd_gpuvm_init_mem_limits(void) */ #define ESTIMATE_PT_SIZE(mem_size) ((mem_size) >> 14) +static size_t amdgpu_amdkfd_acc_size(uint64_t size) +{ + size >>= PAGE_SHIFT; + size *= sizeof(dma_addr_t) + sizeof(void *); + + return __roundup_pow_of_two(sizeof(struct amdgpu_bo)) + + __roundup_pow_of_two(sizeof(struct ttm_tt)) + + PAGE_ALIGN(size); +} + static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg) { @@ -126,8 +136,7 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed; int ret = 0; - acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, - sizeof(struct amdgpu_bo)); + acc_size = amdgpu_amdkfd_acc_size(size); vram_needed = 0; if (domain == AMDGPU_GEM_DOMAIN_GTT) { @@ -174,8 +183,7 @@ static void unreserve_mem_limit(struct amdgpu_device *adev, { size_t acc_size; - acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, - sizeof(struct amdgpu_bo)); + acc_size = amdgpu_amdkfd_acc_size(size); spin_lock(&kfd_mem_limit.mem_limit_lock); if (domain == AMDGPU_GEM_DOMAIN_GTT) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 6cc9919b12cc..599c9a132eb6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -523,7 +523,6 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, }; struct amdgpu_bo *bo; unsigned long page_align, size = bp->size; - size_t acc_size; int r; /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ @@ -546,9 +545,6 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, *bo_ptr = NULL; - acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, - sizeof(struct amdgpu_bo)); - bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); if (bo == NULL) return -ENOMEM; @@ -577,8 +573,8 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, bo->tbo.priority = 1; r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type, - &bo->placement, page_align, &ctx, acc_size, - NULL, bp->resv, &amdgpu_bo_destroy); + &bo->placement, page_align, &ctx, NULL, + bp->resv, &amdgpu_bo_destroy); if (unlikely(r != 0)) return r; diff --git a/drivers/gpu/drm/drm_gem_vram_helper.c b/drivers/gpu/drm/drm_gem_vram_helper.c index 0b13c8507688..a0992f0b8afd 100644 --- a/drivers/gpu/drm/drm_gem_vram_helper.c +++ b/drivers/gpu/drm/drm_gem_vram_helper.c @@ -189,7 +189,6 @@ struct drm_gem_vram_object *drm_gem_vram_create(struct drm_device *dev, struct drm_vram_mm *vmm = dev->vram_mm; struct ttm_device *bdev; int ret; - size_t acc_size; if (WARN_ONCE(!vmm, "VRAM MM not initialized")) return ERR_PTR(-EINVAL); @@ -216,7 +215,6 @@ struct drm_gem_vram_object *drm_gem_vram_create(struct drm_device *dev, } bdev = &vmm->bdev; - acc_size = ttm_bo_dma_acc_size(bdev, size, sizeof(*gbo)); gbo->bo.bdev = bdev; drm_gem_vram_placement(gbo, DRM_GEM_VRAM_PL_FLAG_SYSTEM); @@ -226,8 +224,8 @@ struct drm_gem_vram_object *drm_gem_vram_create(struct drm_device *dev, * to release gbo->bo.base and kfree gbo. */ ret = ttm_bo_init(bdev, &gbo->bo, size, ttm_bo_type_device, - &gbo->placement, pg_align, false, acc_size, - NULL, NULL, ttm_buffer_object_destroy); + &gbo->placement, pg_align, false, NULL, NULL, + ttm_buffer_object_destroy); if (ret) return ERR_PTR(ret); diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index c177940d6e2c..ca2a8ae1938e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -300,18 +300,15 @@ nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain, struct sg_table *sg, struct dma_resv *robj) { int type = sg ? ttm_bo_type_sg : ttm_bo_type_device; - size_t acc_size; int ret; - acc_size = ttm_bo_dma_acc_size(nvbo->bo.bdev, size, sizeof(*nvbo)); - nvbo->bo.mem.num_pages = size >> PAGE_SHIFT; nouveau_bo_placement_set(nvbo, domain, 0); INIT_LIST_HEAD(&nvbo->io_reserve_lru); ret = ttm_bo_init(nvbo->bo.bdev, &nvbo->bo, size, type, - &nvbo->placement, align >> PAGE_SHIFT, false, - acc_size, sg, robj, nouveau_bo_del_ttm); + &nvbo->placement, align >> PAGE_SHIFT, false, sg, + robj, nouveau_bo_del_ttm); if (ret) { /* ttm will call nouveau_bo_del_ttm if it fails.. */ return ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index edf9d1ee9d58..a491c2c1c56e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -54,7 +54,6 @@ #include #include #include -#include #include diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c index ceebc5881f68..705b51535492 100644 --- a/drivers/gpu/drm/qxl/qxl_object.c +++ b/drivers/gpu/drm/qxl/qxl_object.c @@ -138,8 +138,8 @@ int qxl_bo_create(struct qxl_device *qdev, qxl_ttm_placement_from_domain(bo, domain); r = ttm_bo_init_reserved(&qdev->mman.bdev, &bo->tbo, size, type, - &bo->placement, 0, &ctx, size, - NULL, NULL, &qxl_ttm_bo_destroy); + &bo->placement, 0, &ctx, NULL, NULL, + &qxl_ttm_bo_destroy); if (unlikely(r != 0)) { if (r != -ERESTARTSYS) dev_err(qdev->ddev.dev, diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c index 6a336284466f..804f7a427be7 100644 --- a/drivers/gpu/drm/radeon/radeon_object.c +++ b/drivers/gpu/drm/radeon/radeon_object.c @@ -159,7 +159,6 @@ int radeon_bo_create(struct radeon_device *rdev, struct radeon_bo *bo; enum ttm_bo_type type; unsigned long page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; - size_t acc_size; int r; size = ALIGN(size, PAGE_SIZE); @@ -173,9 +172,6 @@ int radeon_bo_create(struct radeon_device *rdev, } *bo_ptr = NULL; - acc_size = ttm_bo_dma_acc_size(&rdev->mman.bdev, size, - sizeof(struct radeon_bo)); - bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL); if (bo == NULL) return -ENOMEM; @@ -230,8 +226,8 @@ int radeon_bo_create(struct radeon_device *rdev, /* Kernel allocation are uninterruptible */ down_read(&rdev->pm.mclk_lock); r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type, - &bo->placement, page_align, !kernel, acc_size, - sg, resv, &radeon_ttm_bo_destroy); + &bo->placement, page_align, !kernel, sg, resv, + &radeon_ttm_bo_destroy); up_read(&rdev->pm.mclk_lock); if (unlikely(r != 0)) { return r; diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index 8e6437eadabe..40e5e9da7953 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -2,10 +2,9 @@ # # Makefile for the drm device driver. This driver provides support for the -ttm-y := ttm_memory.o ttm_tt.o ttm_bo.o \ - ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ - ttm_execbuf_util.o ttm_range_manager.o \ - ttm_resource.o ttm_pool.o ttm_device.o +ttm-y := ttm_tt.o ttm_bo.o ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ + ttm_execbuf_util.o ttm_range_manager.o ttm_resource.o ttm_pool.o \ + ttm_device.o ttm-$(CONFIG_AGP) += ttm_agp_backend.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 643befc1a6f2..e38102282fd5 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -425,7 +425,6 @@ static void ttm_bo_release(struct kref *kref) struct ttm_buffer_object *bo = container_of(kref, struct ttm_buffer_object, kref); struct ttm_device *bdev = bo->bdev; - size_t acc_size = bo->acc_size; int ret; if (!bo->deleted) { @@ -485,7 +484,6 @@ static void ttm_bo_release(struct kref *kref) if (!ttm_bo_uses_embedded_gem_object(bo)) dma_resv_fini(&bo->base._resv); bo->destroy(bo); - ttm_mem_global_free(&ttm_mem_glob, acc_size); } void ttm_bo_put(struct ttm_buffer_object *bo) @@ -1046,25 +1044,13 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_placement *placement, uint32_t page_alignment, struct ttm_operation_ctx *ctx, - size_t acc_size, struct sg_table *sg, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { - struct ttm_mem_global *mem_glob = &ttm_mem_glob; bool locked; int ret = 0; - ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); - if (ret) { - pr_err("Out of kernel memory\n"); - if (destroy) - (*destroy)(bo); - else - kfree(bo); - return -ENOMEM; - } - bo->destroy = destroy ? destroy : ttm_bo_default_destroy; kref_init(&bo->kref); @@ -1081,7 +1067,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, bo->mem.bus.addr = NULL; bo->moving = NULL; bo->mem.placement = 0; - bo->acc_size = acc_size; bo->pin_count = 0; bo->sg = sg; if (resv) { @@ -1142,7 +1127,6 @@ int ttm_bo_init(struct ttm_device *bdev, struct ttm_placement *placement, uint32_t page_alignment, bool interruptible, - size_t acc_size, struct sg_table *sg, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) @@ -1151,8 +1135,7 @@ int ttm_bo_init(struct ttm_device *bdev, int ret; ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, - page_alignment, &ctx, acc_size, - sg, resv, destroy); + page_alignment, &ctx, sg, resv, destroy); if (ret) return ret; @@ -1163,20 +1146,6 @@ int ttm_bo_init(struct ttm_device *bdev, } EXPORT_SYMBOL(ttm_bo_init); -size_t ttm_bo_dma_acc_size(struct ttm_device *bdev, - unsigned long bo_size, - unsigned struct_size) -{ - unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; - size_t size = 0; - - size += ttm_round_pot(struct_size); - size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); - size += ttm_round_pot(sizeof(struct ttm_tt)); - return size; -} -EXPORT_SYMBOL(ttm_bo_dma_acc_size); - /* * buffer object vm functions. */ diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index db0f2661d504..031e5819fec4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -309,7 +309,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, kref_init(&fbo->base.kref); fbo->base.destroy = &ttm_transfered_destroy; - fbo->base.acc_size = 0; fbo->base.pin_count = 0; if (bo->type != ttm_bo_type_sg) fbo->base.base.resv = &fbo->base.base._resv; diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c index 8bb61dd26437..95e1b7b1f2e6 100644 --- a/drivers/gpu/drm/ttm/ttm_device.c +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -27,9 +27,12 @@ #define pr_fmt(fmt) "[TTM DEVICE] " fmt +#include + #include -#include +#include #include +#include #include "ttm_module.h" @@ -49,7 +52,9 @@ static void ttm_global_release(void) if (--ttm_glob_use_count > 0) goto out; - ttm_mem_global_release(&ttm_mem_glob); + ttm_pool_mgr_fini(); + ttm_tt_mgr_fini(); + __free_page(glob->dummy_read_page); memset(glob, 0, sizeof(*glob)); out: @@ -59,6 +64,8 @@ out: static int ttm_global_init(void) { struct ttm_global *glob = &ttm_glob; + unsigned long num_pages; + struct sysinfo si; int ret = 0; unsigned i; @@ -66,9 +73,14 @@ static int ttm_global_init(void) if (++ttm_glob_use_count > 1) goto out; - ret = ttm_mem_global_init(&ttm_mem_glob); - if (ret) - goto out; + si_meminfo(&si); + + /* Limit the number of pages in the pool to about 50% of the total + * system memory. + */ + num_pages = ((u64)si.totalram * si.mem_unit) >> PAGE_SHIFT; + ttm_pool_mgr_init(num_pages * 50 / 100); + ttm_tt_mgr_init(); spin_lock_init(&glob->lru_lock); glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c deleted file mode 100644 index 634a85c2dc4c..000000000000 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ /dev/null @@ -1,686 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR MIT */ -/************************************************************************** - * - * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ - -#define pr_fmt(fmt) "[TTM] " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ttm_module.h" - -#define TTM_MEMORY_ALLOC_RETRIES 4 - -struct ttm_mem_global ttm_mem_glob; -EXPORT_SYMBOL(ttm_mem_glob); - -struct ttm_mem_zone { - struct kobject kobj; - struct ttm_mem_global *glob; - const char *name; - uint64_t zone_mem; - uint64_t emer_mem; - uint64_t max_mem; - uint64_t swap_limit; - uint64_t used_mem; -}; - -static struct attribute ttm_mem_sys = { - .name = "zone_memory", - .mode = S_IRUGO -}; -static struct attribute ttm_mem_emer = { - .name = "emergency_memory", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_mem_max = { - .name = "available_memory", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_mem_swap = { - .name = "swap_limit", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_mem_used = { - .name = "used_memory", - .mode = S_IRUGO -}; - -static void ttm_mem_zone_kobj_release(struct kobject *kobj) -{ - struct ttm_mem_zone *zone = - container_of(kobj, struct ttm_mem_zone, kobj); - - pr_info("Zone %7s: Used memory at exit: %llu KiB\n", - zone->name, (unsigned long long)zone->used_mem >> 10); - kfree(zone); -} - -static ssize_t ttm_mem_zone_show(struct kobject *kobj, - struct attribute *attr, - char *buffer) -{ - struct ttm_mem_zone *zone = - container_of(kobj, struct ttm_mem_zone, kobj); - uint64_t val = 0; - - spin_lock(&zone->glob->lock); - if (attr == &ttm_mem_sys) - val = zone->zone_mem; - else if (attr == &ttm_mem_emer) - val = zone->emer_mem; - else if (attr == &ttm_mem_max) - val = zone->max_mem; - else if (attr == &ttm_mem_swap) - val = zone->swap_limit; - else if (attr == &ttm_mem_used) - val = zone->used_mem; - spin_unlock(&zone->glob->lock); - - return snprintf(buffer, PAGE_SIZE, "%llu\n", - (unsigned long long) val >> 10); -} - -static void ttm_check_swapping(struct ttm_mem_global *glob); - -static ssize_t ttm_mem_zone_store(struct kobject *kobj, - struct attribute *attr, - const char *buffer, - size_t size) -{ - struct ttm_mem_zone *zone = - container_of(kobj, struct ttm_mem_zone, kobj); - int chars; - unsigned long val; - uint64_t val64; - - chars = sscanf(buffer, "%lu", &val); - if (chars == 0) - return size; - - val64 = val; - val64 <<= 10; - - spin_lock(&zone->glob->lock); - if (val64 > zone->zone_mem) - val64 = zone->zone_mem; - if (attr == &ttm_mem_emer) { - zone->emer_mem = val64; - if (zone->max_mem > val64) - zone->max_mem = val64; - } else if (attr == &ttm_mem_max) { - zone->max_mem = val64; - if (zone->emer_mem < val64) - zone->emer_mem = val64; - } else if (attr == &ttm_mem_swap) - zone->swap_limit = val64; - spin_unlock(&zone->glob->lock); - - ttm_check_swapping(zone->glob); - - return size; -} - -static struct attribute *ttm_mem_zone_attrs[] = { - &ttm_mem_sys, - &ttm_mem_emer, - &ttm_mem_max, - &ttm_mem_swap, - &ttm_mem_used, - NULL -}; - -static const struct sysfs_ops ttm_mem_zone_ops = { - .show = &ttm_mem_zone_show, - .store = &ttm_mem_zone_store -}; - -static struct kobj_type ttm_mem_zone_kobj_type = { - .release = &ttm_mem_zone_kobj_release, - .sysfs_ops = &ttm_mem_zone_ops, - .default_attrs = ttm_mem_zone_attrs, -}; - -static struct attribute ttm_mem_global_lower_mem_limit = { - .name = "lower_mem_limit", - .mode = S_IRUGO | S_IWUSR -}; - -static ssize_t ttm_mem_global_show(struct kobject *kobj, - struct attribute *attr, - char *buffer) -{ - struct ttm_mem_global *glob = - container_of(kobj, struct ttm_mem_global, kobj); - uint64_t val = 0; - - spin_lock(&glob->lock); - val = glob->lower_mem_limit; - spin_unlock(&glob->lock); - /* convert from number of pages to KB */ - val <<= (PAGE_SHIFT - 10); - return snprintf(buffer, PAGE_SIZE, "%llu\n", - (unsigned long long) val); -} - -static ssize_t ttm_mem_global_store(struct kobject *kobj, - struct attribute *attr, - const char *buffer, - size_t size) -{ - int chars; - uint64_t val64; - unsigned long val; - struct ttm_mem_global *glob = - container_of(kobj, struct ttm_mem_global, kobj); - - chars = sscanf(buffer, "%lu", &val); - if (chars == 0) - return size; - - val64 = val; - /* convert from KB to number of pages */ - val64 >>= (PAGE_SHIFT - 10); - - spin_lock(&glob->lock); - glob->lower_mem_limit = val64; - spin_unlock(&glob->lock); - - return size; -} - -static struct attribute *ttm_mem_global_attrs[] = { - &ttm_mem_global_lower_mem_limit, - NULL -}; - -static const struct sysfs_ops ttm_mem_global_ops = { - .show = &ttm_mem_global_show, - .store = &ttm_mem_global_store, -}; - -static struct kobj_type ttm_mem_glob_kobj_type = { - .sysfs_ops = &ttm_mem_global_ops, - .default_attrs = ttm_mem_global_attrs, -}; - -static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, - bool from_wq, uint64_t extra) -{ - unsigned int i; - struct ttm_mem_zone *zone; - uint64_t target; - - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - - if (from_wq) - target = zone->swap_limit; - else if (capable(CAP_SYS_ADMIN)) - target = zone->emer_mem; - else - target = zone->max_mem; - - target = (extra > target) ? 0ULL : target; - - if (zone->used_mem > target) - return true; - } - return false; -} - -/* - * At this point we only support a single shrink callback. - * Extend this if needed, perhaps using a linked list of callbacks. - * Note that this function is reentrant: - * many threads may try to swap out at any given time. - */ - -static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, - uint64_t extra, struct ttm_operation_ctx *ctx) -{ - int ret; - - spin_lock(&glob->lock); - - while (ttm_zones_above_swap_target(glob, from_wq, extra)) { - spin_unlock(&glob->lock); - ret = ttm_bo_swapout(ctx, GFP_KERNEL); - spin_lock(&glob->lock); - if (unlikely(ret < 0)) - break; - } - - spin_unlock(&glob->lock); -} - -static void ttm_shrink_work(struct work_struct *work) -{ - struct ttm_operation_ctx ctx = { - .interruptible = false, - .no_wait_gpu = false - }; - struct ttm_mem_global *glob = - container_of(work, struct ttm_mem_global, work); - - ttm_shrink(glob, true, 0ULL, &ctx); -} - -static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, - const struct sysinfo *si) -{ - struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); - uint64_t mem; - int ret; - - if (unlikely(!zone)) - return -ENOMEM; - - mem = si->totalram - si->totalhigh; - mem *= si->mem_unit; - - zone->name = "kernel"; - zone->zone_mem = mem; - zone->max_mem = mem >> 1; - zone->emer_mem = (mem >> 1) + (mem >> 2); - zone->swap_limit = zone->max_mem - (mem >> 3); - zone->used_mem = 0; - zone->glob = glob; - glob->zone_kernel = zone; - ret = kobject_init_and_add( - &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); - if (unlikely(ret != 0)) { - kobject_put(&zone->kobj); - return ret; - } - glob->zones[glob->num_zones++] = zone; - return 0; -} - -#ifdef CONFIG_HIGHMEM -static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob, - const struct sysinfo *si) -{ - struct ttm_mem_zone *zone; - uint64_t mem; - int ret; - - if (si->totalhigh == 0) - return 0; - - zone = kzalloc(sizeof(*zone), GFP_KERNEL); - if (unlikely(!zone)) - return -ENOMEM; - - mem = si->totalram; - mem *= si->mem_unit; - - zone->name = "highmem"; - zone->zone_mem = mem; - zone->max_mem = mem >> 1; - zone->emer_mem = (mem >> 1) + (mem >> 2); - zone->swap_limit = zone->max_mem - (mem >> 3); - zone->used_mem = 0; - zone->glob = glob; - glob->zone_highmem = zone; - ret = kobject_init_and_add( - &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s", - zone->name); - if (unlikely(ret != 0)) { - kobject_put(&zone->kobj); - return ret; - } - glob->zones[glob->num_zones++] = zone; - return 0; -} -#else -static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, - const struct sysinfo *si) -{ - struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); - uint64_t mem; - int ret; - - if (unlikely(!zone)) - return -ENOMEM; - - mem = si->totalram; - mem *= si->mem_unit; - - /** - * No special dma32 zone needed. - */ - - if (mem <= ((uint64_t) 1ULL << 32)) { - kfree(zone); - return 0; - } - - /* - * Limit max dma32 memory to 4GB for now - * until we can figure out how big this - * zone really is. - */ - - mem = ((uint64_t) 1ULL << 32); - zone->name = "dma32"; - zone->zone_mem = mem; - zone->max_mem = mem >> 1; - zone->emer_mem = (mem >> 1) + (mem >> 2); - zone->swap_limit = zone->max_mem - (mem >> 3); - zone->used_mem = 0; - zone->glob = glob; - glob->zone_dma32 = zone; - ret = kobject_init_and_add( - &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); - if (unlikely(ret != 0)) { - kobject_put(&zone->kobj); - return ret; - } - glob->zones[glob->num_zones++] = zone; - return 0; -} -#endif - -int ttm_mem_global_init(struct ttm_mem_global *glob) -{ - struct sysinfo si; - int ret; - int i; - struct ttm_mem_zone *zone; - - spin_lock_init(&glob->lock); - glob->swap_queue = create_singlethread_workqueue("ttm_swap"); - INIT_WORK(&glob->work, ttm_shrink_work); - ret = kobject_init_and_add( - &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting"); - if (unlikely(ret != 0)) { - kobject_put(&glob->kobj); - return ret; - } - - si_meminfo(&si); - - /* set it as 0 by default to keep original behavior of OOM */ - glob->lower_mem_limit = 0; - - ret = ttm_mem_init_kernel_zone(glob, &si); - if (unlikely(ret != 0)) - goto out_no_zone; -#ifdef CONFIG_HIGHMEM - ret = ttm_mem_init_highmem_zone(glob, &si); - if (unlikely(ret != 0)) - goto out_no_zone; -#else - ret = ttm_mem_init_dma32_zone(glob, &si); - if (unlikely(ret != 0)) - goto out_no_zone; -#endif - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - pr_info("Zone %7s: Available graphics memory: %llu KiB\n", - zone->name, (unsigned long long)zone->max_mem >> 10); - } - ttm_pool_mgr_init(glob->zone_kernel->max_mem/(2*PAGE_SIZE)); - ttm_tt_mgr_init(); - return 0; -out_no_zone: - ttm_mem_global_release(glob); - return ret; -} - -void ttm_mem_global_release(struct ttm_mem_global *glob) -{ - struct ttm_mem_zone *zone; - unsigned int i; - - /* let the page allocator first stop the shrink work. */ - ttm_pool_mgr_fini(); - ttm_tt_mgr_fini(); - - flush_workqueue(glob->swap_queue); - destroy_workqueue(glob->swap_queue); - glob->swap_queue = NULL; - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - kobject_del(&zone->kobj); - kobject_put(&zone->kobj); - } - kobject_del(&glob->kobj); - kobject_put(&glob->kobj); - memset(glob, 0, sizeof(*glob)); -} - -static void ttm_check_swapping(struct ttm_mem_global *glob) -{ - bool needs_swapping = false; - unsigned int i; - struct ttm_mem_zone *zone; - - spin_lock(&glob->lock); - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - if (zone->used_mem > zone->swap_limit) { - needs_swapping = true; - break; - } - } - - spin_unlock(&glob->lock); - - if (unlikely(needs_swapping)) - (void)queue_work(glob->swap_queue, &glob->work); - -} - -static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, - struct ttm_mem_zone *single_zone, - uint64_t amount) -{ - unsigned int i; - struct ttm_mem_zone *zone; - - spin_lock(&glob->lock); - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - if (single_zone && zone != single_zone) - continue; - zone->used_mem -= amount; - } - spin_unlock(&glob->lock); -} - -void ttm_mem_global_free(struct ttm_mem_global *glob, - uint64_t amount) -{ - return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount); -} -EXPORT_SYMBOL(ttm_mem_global_free); - -/* - * check if the available mem is under lower memory limit - * - * a. if no swap disk at all or free swap space is under swap_mem_limit - * but available system mem is bigger than sys_mem_limit, allow TTM - * allocation; - * - * b. if the available system mem is less than sys_mem_limit but free - * swap disk is bigger than swap_mem_limit, allow TTM allocation. - */ -bool -ttm_check_under_lowerlimit(struct ttm_mem_global *glob, - uint64_t num_pages, - struct ttm_operation_ctx *ctx) -{ - int64_t available; - - /* We allow over commit during suspend */ - if (ctx->force_alloc) - return false; - - available = get_nr_swap_pages() + si_mem_available(); - available -= num_pages; - if (available < glob->lower_mem_limit) - return true; - - return false; -} - -static int ttm_mem_global_reserve(struct ttm_mem_global *glob, - struct ttm_mem_zone *single_zone, - uint64_t amount, bool reserve) -{ - uint64_t limit; - int ret = -ENOMEM; - unsigned int i; - struct ttm_mem_zone *zone; - - spin_lock(&glob->lock); - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - if (single_zone && zone != single_zone) - continue; - - limit = (capable(CAP_SYS_ADMIN)) ? - zone->emer_mem : zone->max_mem; - - if (zone->used_mem > limit) - goto out_unlock; - } - - if (reserve) { - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - if (single_zone && zone != single_zone) - continue; - zone->used_mem += amount; - } - } - - ret = 0; -out_unlock: - spin_unlock(&glob->lock); - ttm_check_swapping(glob); - - return ret; -} - - -static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, - struct ttm_mem_zone *single_zone, - uint64_t memory, - struct ttm_operation_ctx *ctx) -{ - int count = TTM_MEMORY_ALLOC_RETRIES; - - while (unlikely(ttm_mem_global_reserve(glob, - single_zone, - memory, true) - != 0)) { - if (ctx->no_wait_gpu) - return -ENOMEM; - if (unlikely(count-- == 0)) - return -ENOMEM; - ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx); - } - - return 0; -} - -int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, - struct ttm_operation_ctx *ctx) -{ - /** - * Normal allocations of kernel memory are registered in - * the kernel zone. - */ - - return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx); -} -EXPORT_SYMBOL(ttm_mem_global_alloc); - -int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, - struct page *page, uint64_t size, - struct ttm_operation_ctx *ctx) -{ - struct ttm_mem_zone *zone = NULL; - - /** - * Page allocations may be registed in a single zone - * only if highmem or !dma32. - */ - -#ifdef CONFIG_HIGHMEM - if (PageHighMem(page) && glob->zone_highmem != NULL) - zone = glob->zone_highmem; -#else - if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) - zone = glob->zone_kernel; -#endif - return ttm_mem_global_alloc_zone(glob, zone, size, ctx); -} - -void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, - uint64_t size) -{ - struct ttm_mem_zone *zone = NULL; - -#ifdef CONFIG_HIGHMEM - if (PageHighMem(page) && glob->zone_highmem != NULL) - zone = glob->zone_highmem; -#else - if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) - zone = glob->zone_kernel; -#endif - ttm_mem_global_free_zone(glob, zone, size); -} - -size_t ttm_round_pot(size_t size) -{ - if ((size & (size - 1)) == 0) - return size; - else if (size > PAGE_SIZE) - return PAGE_ALIGN(size); - else { - size_t tmp_size = 4; - - while (tmp_size < size) - tmp_size <<= 1; - - return tmp_size; - } - return 0; -} -EXPORT_SYMBOL(ttm_round_pot); diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index e0617717113f..6b0f957d63d5 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -404,16 +404,10 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, caching = pages + (1 << order); } - r = ttm_mem_global_alloc_page(&ttm_mem_glob, p, - (1 << order) * PAGE_SIZE, - ctx); - if (r) - goto error_free_page; - if (dma_addr) { r = ttm_pool_map(pool, order, p, &dma_addr); if (r) - goto error_global_free; + goto error_free_page; } num_pages -= 1 << order; @@ -427,9 +421,6 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, return 0; -error_global_free: - ttm_mem_global_free_page(&ttm_mem_glob, p, (1 << order) * PAGE_SIZE); - error_free_page: ttm_pool_free_page(pool, tt->caching, order, p); @@ -464,8 +455,6 @@ void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) order = ttm_pool_page_order(pool, p); num_pages = 1ULL << order; - ttm_mem_global_free_page(&ttm_mem_glob, p, - num_pages * PAGE_SIZE); if (tt->dma_address) ttm_pool_unmap(pool, tt->dma_address[i], num_pages); diff --git a/drivers/gpu/drm/vmwgfx/Makefile b/drivers/gpu/drm/vmwgfx/Makefile index cc4cdca7176e..8c02fa5852e7 100644 --- a/drivers/gpu/drm/vmwgfx/Makefile +++ b/drivers/gpu/drm/vmwgfx/Makefile @@ -9,7 +9,7 @@ vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \ vmwgfx_cotable.o vmwgfx_so.o vmwgfx_binding.o vmwgfx_msg.o \ vmwgfx_simple_resource.o vmwgfx_va.o vmwgfx_blit.o \ vmwgfx_validation.o vmwgfx_page_dirty.o vmwgfx_streamoutput.o \ - ttm_object.o ttm_lock.o + ttm_object.o ttm_lock.o ttm_memory.o vmwgfx-$(CONFIG_TRANSPARENT_HUGEPAGE) += vmwgfx_thp.o obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o diff --git a/drivers/gpu/drm/vmwgfx/ttm_memory.c b/drivers/gpu/drm/vmwgfx/ttm_memory.c new file mode 100644 index 000000000000..e972af07d029 --- /dev/null +++ b/drivers/gpu/drm/vmwgfx/ttm_memory.c @@ -0,0 +1,681 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ttm_memory.h" + +#define TTM_MEMORY_ALLOC_RETRIES 4 + +struct ttm_mem_global ttm_mem_glob; +EXPORT_SYMBOL(ttm_mem_glob); + +struct ttm_mem_zone { + struct kobject kobj; + struct ttm_mem_global *glob; + const char *name; + uint64_t zone_mem; + uint64_t emer_mem; + uint64_t max_mem; + uint64_t swap_limit; + uint64_t used_mem; +}; + +static struct attribute ttm_mem_sys = { + .name = "zone_memory", + .mode = S_IRUGO +}; +static struct attribute ttm_mem_emer = { + .name = "emergency_memory", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_mem_max = { + .name = "available_memory", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_mem_swap = { + .name = "swap_limit", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_mem_used = { + .name = "used_memory", + .mode = S_IRUGO +}; + +static void ttm_mem_zone_kobj_release(struct kobject *kobj) +{ + struct ttm_mem_zone *zone = + container_of(kobj, struct ttm_mem_zone, kobj); + + pr_info("Zone %7s: Used memory at exit: %llu KiB\n", + zone->name, (unsigned long long)zone->used_mem >> 10); + kfree(zone); +} + +static ssize_t ttm_mem_zone_show(struct kobject *kobj, + struct attribute *attr, + char *buffer) +{ + struct ttm_mem_zone *zone = + container_of(kobj, struct ttm_mem_zone, kobj); + uint64_t val = 0; + + spin_lock(&zone->glob->lock); + if (attr == &ttm_mem_sys) + val = zone->zone_mem; + else if (attr == &ttm_mem_emer) + val = zone->emer_mem; + else if (attr == &ttm_mem_max) + val = zone->max_mem; + else if (attr == &ttm_mem_swap) + val = zone->swap_limit; + else if (attr == &ttm_mem_used) + val = zone->used_mem; + spin_unlock(&zone->glob->lock); + + return snprintf(buffer, PAGE_SIZE, "%llu\n", + (unsigned long long) val >> 10); +} + +static void ttm_check_swapping(struct ttm_mem_global *glob); + +static ssize_t ttm_mem_zone_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, + size_t size) +{ + struct ttm_mem_zone *zone = + container_of(kobj, struct ttm_mem_zone, kobj); + int chars; + unsigned long val; + uint64_t val64; + + chars = sscanf(buffer, "%lu", &val); + if (chars == 0) + return size; + + val64 = val; + val64 <<= 10; + + spin_lock(&zone->glob->lock); + if (val64 > zone->zone_mem) + val64 = zone->zone_mem; + if (attr == &ttm_mem_emer) { + zone->emer_mem = val64; + if (zone->max_mem > val64) + zone->max_mem = val64; + } else if (attr == &ttm_mem_max) { + zone->max_mem = val64; + if (zone->emer_mem < val64) + zone->emer_mem = val64; + } else if (attr == &ttm_mem_swap) + zone->swap_limit = val64; + spin_unlock(&zone->glob->lock); + + ttm_check_swapping(zone->glob); + + return size; +} + +static struct attribute *ttm_mem_zone_attrs[] = { + &ttm_mem_sys, + &ttm_mem_emer, + &ttm_mem_max, + &ttm_mem_swap, + &ttm_mem_used, + NULL +}; + +static const struct sysfs_ops ttm_mem_zone_ops = { + .show = &ttm_mem_zone_show, + .store = &ttm_mem_zone_store +}; + +static struct kobj_type ttm_mem_zone_kobj_type = { + .release = &ttm_mem_zone_kobj_release, + .sysfs_ops = &ttm_mem_zone_ops, + .default_attrs = ttm_mem_zone_attrs, +}; + +static struct attribute ttm_mem_global_lower_mem_limit = { + .name = "lower_mem_limit", + .mode = S_IRUGO | S_IWUSR +}; + +static ssize_t ttm_mem_global_show(struct kobject *kobj, + struct attribute *attr, + char *buffer) +{ + struct ttm_mem_global *glob = + container_of(kobj, struct ttm_mem_global, kobj); + uint64_t val = 0; + + spin_lock(&glob->lock); + val = glob->lower_mem_limit; + spin_unlock(&glob->lock); + /* convert from number of pages to KB */ + val <<= (PAGE_SHIFT - 10); + return snprintf(buffer, PAGE_SIZE, "%llu\n", + (unsigned long long) val); +} + +static ssize_t ttm_mem_global_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, + size_t size) +{ + int chars; + uint64_t val64; + unsigned long val; + struct ttm_mem_global *glob = + container_of(kobj, struct ttm_mem_global, kobj); + + chars = sscanf(buffer, "%lu", &val); + if (chars == 0) + return size; + + val64 = val; + /* convert from KB to number of pages */ + val64 >>= (PAGE_SHIFT - 10); + + spin_lock(&glob->lock); + glob->lower_mem_limit = val64; + spin_unlock(&glob->lock); + + return size; +} + +static struct attribute *ttm_mem_global_attrs[] = { + &ttm_mem_global_lower_mem_limit, + NULL +}; + +static const struct sysfs_ops ttm_mem_global_ops = { + .show = &ttm_mem_global_show, + .store = &ttm_mem_global_store, +}; + +static struct kobj_type ttm_mem_glob_kobj_type = { + .sysfs_ops = &ttm_mem_global_ops, + .default_attrs = ttm_mem_global_attrs, +}; + +static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, + bool from_wq, uint64_t extra) +{ + unsigned int i; + struct ttm_mem_zone *zone; + uint64_t target; + + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + + if (from_wq) + target = zone->swap_limit; + else if (capable(CAP_SYS_ADMIN)) + target = zone->emer_mem; + else + target = zone->max_mem; + + target = (extra > target) ? 0ULL : target; + + if (zone->used_mem > target) + return true; + } + return false; +} + +/* + * At this point we only support a single shrink callback. + * Extend this if needed, perhaps using a linked list of callbacks. + * Note that this function is reentrant: + * many threads may try to swap out at any given time. + */ + +static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, + uint64_t extra, struct ttm_operation_ctx *ctx) +{ + int ret; + + spin_lock(&glob->lock); + + while (ttm_zones_above_swap_target(glob, from_wq, extra)) { + spin_unlock(&glob->lock); + ret = ttm_bo_swapout(ctx, GFP_KERNEL); + spin_lock(&glob->lock); + if (unlikely(ret < 0)) + break; + } + + spin_unlock(&glob->lock); +} + +static void ttm_shrink_work(struct work_struct *work) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + struct ttm_mem_global *glob = + container_of(work, struct ttm_mem_global, work); + + ttm_shrink(glob, true, 0ULL, &ctx); +} + +static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, + const struct sysinfo *si) +{ + struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); + uint64_t mem; + int ret; + + if (unlikely(!zone)) + return -ENOMEM; + + mem = si->totalram - si->totalhigh; + mem *= si->mem_unit; + + zone->name = "kernel"; + zone->zone_mem = mem; + zone->max_mem = mem >> 1; + zone->emer_mem = (mem >> 1) + (mem >> 2); + zone->swap_limit = zone->max_mem - (mem >> 3); + zone->used_mem = 0; + zone->glob = glob; + glob->zone_kernel = zone; + ret = kobject_init_and_add( + &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); + if (unlikely(ret != 0)) { + kobject_put(&zone->kobj); + return ret; + } + glob->zones[glob->num_zones++] = zone; + return 0; +} + +#ifdef CONFIG_HIGHMEM +static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob, + const struct sysinfo *si) +{ + struct ttm_mem_zone *zone; + uint64_t mem; + int ret; + + if (si->totalhigh == 0) + return 0; + + zone = kzalloc(sizeof(*zone), GFP_KERNEL); + if (unlikely(!zone)) + return -ENOMEM; + + mem = si->totalram; + mem *= si->mem_unit; + + zone->name = "highmem"; + zone->zone_mem = mem; + zone->max_mem = mem >> 1; + zone->emer_mem = (mem >> 1) + (mem >> 2); + zone->swap_limit = zone->max_mem - (mem >> 3); + zone->used_mem = 0; + zone->glob = glob; + glob->zone_highmem = zone; + ret = kobject_init_and_add( + &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s", + zone->name); + if (unlikely(ret != 0)) { + kobject_put(&zone->kobj); + return ret; + } + glob->zones[glob->num_zones++] = zone; + return 0; +} +#else +static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, + const struct sysinfo *si) +{ + struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); + uint64_t mem; + int ret; + + if (unlikely(!zone)) + return -ENOMEM; + + mem = si->totalram; + mem *= si->mem_unit; + + /** + * No special dma32 zone needed. + */ + + if (mem <= ((uint64_t) 1ULL << 32)) { + kfree(zone); + return 0; + } + + /* + * Limit max dma32 memory to 4GB for now + * until we can figure out how big this + * zone really is. + */ + + mem = ((uint64_t) 1ULL << 32); + zone->name = "dma32"; + zone->zone_mem = mem; + zone->max_mem = mem >> 1; + zone->emer_mem = (mem >> 1) + (mem >> 2); + zone->swap_limit = zone->max_mem - (mem >> 3); + zone->used_mem = 0; + zone->glob = glob; + glob->zone_dma32 = zone; + ret = kobject_init_and_add( + &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); + if (unlikely(ret != 0)) { + kobject_put(&zone->kobj); + return ret; + } + glob->zones[glob->num_zones++] = zone; + return 0; +} +#endif + +int ttm_mem_global_init(struct ttm_mem_global *glob, struct device *dev) +{ + struct sysinfo si; + int ret; + int i; + struct ttm_mem_zone *zone; + + spin_lock_init(&glob->lock); + glob->swap_queue = create_singlethread_workqueue("ttm_swap"); + INIT_WORK(&glob->work, ttm_shrink_work); + + ret = kobject_init_and_add(&glob->kobj, &ttm_mem_glob_kobj_type, + &dev->kobj, "memory_accounting"); + if (unlikely(ret != 0)) { + kobject_put(&glob->kobj); + return ret; + } + + si_meminfo(&si); + + /* set it as 0 by default to keep original behavior of OOM */ + glob->lower_mem_limit = 0; + + ret = ttm_mem_init_kernel_zone(glob, &si); + if (unlikely(ret != 0)) + goto out_no_zone; +#ifdef CONFIG_HIGHMEM + ret = ttm_mem_init_highmem_zone(glob, &si); + if (unlikely(ret != 0)) + goto out_no_zone; +#else + ret = ttm_mem_init_dma32_zone(glob, &si); + if (unlikely(ret != 0)) + goto out_no_zone; +#endif + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + pr_info("Zone %7s: Available graphics memory: %llu KiB\n", + zone->name, (unsigned long long)zone->max_mem >> 10); + } + return 0; +out_no_zone: + ttm_mem_global_release(glob); + return ret; +} + +void ttm_mem_global_release(struct ttm_mem_global *glob) +{ + struct ttm_mem_zone *zone; + unsigned int i; + + flush_workqueue(glob->swap_queue); + destroy_workqueue(glob->swap_queue); + glob->swap_queue = NULL; + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + kobject_del(&zone->kobj); + kobject_put(&zone->kobj); + } + kobject_del(&glob->kobj); + kobject_put(&glob->kobj); + memset(glob, 0, sizeof(*glob)); +} + +static void ttm_check_swapping(struct ttm_mem_global *glob) +{ + bool needs_swapping = false; + unsigned int i; + struct ttm_mem_zone *zone; + + spin_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (zone->used_mem > zone->swap_limit) { + needs_swapping = true; + break; + } + } + + spin_unlock(&glob->lock); + + if (unlikely(needs_swapping)) + (void)queue_work(glob->swap_queue, &glob->work); + +} + +static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t amount) +{ + unsigned int i; + struct ttm_mem_zone *zone; + + spin_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + zone->used_mem -= amount; + } + spin_unlock(&glob->lock); +} + +void ttm_mem_global_free(struct ttm_mem_global *glob, + uint64_t amount) +{ + return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount); +} +EXPORT_SYMBOL(ttm_mem_global_free); + +/* + * check if the available mem is under lower memory limit + * + * a. if no swap disk at all or free swap space is under swap_mem_limit + * but available system mem is bigger than sys_mem_limit, allow TTM + * allocation; + * + * b. if the available system mem is less than sys_mem_limit but free + * swap disk is bigger than swap_mem_limit, allow TTM allocation. + */ +bool +ttm_check_under_lowerlimit(struct ttm_mem_global *glob, + uint64_t num_pages, + struct ttm_operation_ctx *ctx) +{ + int64_t available; + + /* We allow over commit during suspend */ + if (ctx->force_alloc) + return false; + + available = get_nr_swap_pages() + si_mem_available(); + available -= num_pages; + if (available < glob->lower_mem_limit) + return true; + + return false; +} + +static int ttm_mem_global_reserve(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t amount, bool reserve) +{ + uint64_t limit; + int ret = -ENOMEM; + unsigned int i; + struct ttm_mem_zone *zone; + + spin_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + + limit = (capable(CAP_SYS_ADMIN)) ? + zone->emer_mem : zone->max_mem; + + if (zone->used_mem > limit) + goto out_unlock; + } + + if (reserve) { + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + zone->used_mem += amount; + } + } + + ret = 0; +out_unlock: + spin_unlock(&glob->lock); + ttm_check_swapping(glob); + + return ret; +} + + +static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t memory, + struct ttm_operation_ctx *ctx) +{ + int count = TTM_MEMORY_ALLOC_RETRIES; + + while (unlikely(ttm_mem_global_reserve(glob, + single_zone, + memory, true) + != 0)) { + if (ctx->no_wait_gpu) + return -ENOMEM; + if (unlikely(count-- == 0)) + return -ENOMEM; + ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx); + } + + return 0; +} + +int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, + struct ttm_operation_ctx *ctx) +{ + /** + * Normal allocations of kernel memory are registered in + * the kernel zone. + */ + + return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx); +} +EXPORT_SYMBOL(ttm_mem_global_alloc); + +int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, + struct page *page, uint64_t size, + struct ttm_operation_ctx *ctx) +{ + struct ttm_mem_zone *zone = NULL; + + /** + * Page allocations may be registed in a single zone + * only if highmem or !dma32. + */ + +#ifdef CONFIG_HIGHMEM + if (PageHighMem(page) && glob->zone_highmem != NULL) + zone = glob->zone_highmem; +#else + if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) + zone = glob->zone_kernel; +#endif + return ttm_mem_global_alloc_zone(glob, zone, size, ctx); +} + +void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, + uint64_t size) +{ + struct ttm_mem_zone *zone = NULL; + +#ifdef CONFIG_HIGHMEM + if (PageHighMem(page) && glob->zone_highmem != NULL) + zone = glob->zone_highmem; +#else + if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) + zone = glob->zone_kernel; +#endif + ttm_mem_global_free_zone(glob, zone, size); +} + +size_t ttm_round_pot(size_t size) +{ + if ((size & (size - 1)) == 0) + return size; + else if (size > PAGE_SIZE) + return PAGE_ALIGN(size); + else { + size_t tmp_size = 4; + + while (tmp_size < size) + tmp_size <<= 1; + + return tmp_size; + } + return 0; +} +EXPORT_SYMBOL(ttm_round_pot); diff --git a/drivers/gpu/drm/vmwgfx/ttm_memory.h b/drivers/gpu/drm/vmwgfx/ttm_memory.h new file mode 100644 index 000000000000..c50dba774485 --- /dev/null +++ b/drivers/gpu/drm/vmwgfx/ttm_memory.h @@ -0,0 +1,96 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#ifndef TTM_MEMORY_H +#define TTM_MEMORY_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +/** + * struct ttm_mem_global - Global memory accounting structure. + * + * @shrink: A single callback to shrink TTM memory usage. Extend this + * to a linked list to be able to handle multiple callbacks when needed. + * @swap_queue: A workqueue to handle shrinking in low memory situations. We + * need a separate workqueue since it will spend a lot of time waiting + * for the GPU, and this will otherwise block other workqueue tasks(?) + * At this point we use only a single-threaded workqueue. + * @work: The workqueue callback for the shrink queue. + * @lock: Lock to protect the @shrink - and the memory accounting members, + * that is, essentially the whole structure with some exceptions. + * @lower_mem_limit: include lower limit of swap space and lower limit of + * system memory. + * @zones: Array of pointers to accounting zones. + * @num_zones: Number of populated entries in the @zones array. + * @zone_kernel: Pointer to the kernel zone. + * @zone_highmem: Pointer to the highmem zone if there is one. + * @zone_dma32: Pointer to the dma32 zone if there is one. + * + * Note that this structure is not per device. It should be global for all + * graphics devices. + */ + +#define TTM_MEM_MAX_ZONES 2 +struct ttm_mem_zone; +extern struct ttm_mem_global { + struct kobject kobj; + struct workqueue_struct *swap_queue; + struct work_struct work; + spinlock_t lock; + uint64_t lower_mem_limit; + struct ttm_mem_zone *zones[TTM_MEM_MAX_ZONES]; + unsigned int num_zones; + struct ttm_mem_zone *zone_kernel; +#ifdef CONFIG_HIGHMEM + struct ttm_mem_zone *zone_highmem; +#else + struct ttm_mem_zone *zone_dma32; +#endif +} ttm_mem_glob; + +int ttm_mem_global_init(struct ttm_mem_global *glob, struct device *dev); +void ttm_mem_global_release(struct ttm_mem_global *glob); +int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, + struct ttm_operation_ctx *ctx); +void ttm_mem_global_free(struct ttm_mem_global *glob, uint64_t amount); +int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, + struct page *page, uint64_t size, + struct ttm_operation_ctx *ctx); +void ttm_mem_global_free_page(struct ttm_mem_global *glob, + struct page *page, uint64_t size); +size_t ttm_round_pot(size_t size); +bool ttm_check_under_lowerlimit(struct ttm_mem_global *glob, uint64_t num_pages, + struct ttm_operation_ctx *ctx); +#endif diff --git a/drivers/gpu/drm/vmwgfx/ttm_object.h b/drivers/gpu/drm/vmwgfx/ttm_object.h index ede26df87c93..49b064f0cb19 100644 --- a/drivers/gpu/drm/vmwgfx/ttm_object.h +++ b/drivers/gpu/drm/vmwgfx/ttm_object.h @@ -43,7 +43,8 @@ #include #include -#include + +#include "ttm_memory.h" /** * enum ttm_ref_type diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c index 6b3bfd8c678a..50e529a01677 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c @@ -507,11 +507,16 @@ int vmw_bo_create_kernel(struct vmw_private *dev_priv, unsigned long size, acc_size = ttm_round_pot(sizeof(*bo)); acc_size += ttm_round_pot(npages * sizeof(void *)); acc_size += ttm_round_pot(sizeof(struct ttm_tt)); + + ret = ttm_mem_global_alloc(&ttm_mem_glob, acc_size, &ctx); + if (unlikely(ret)) + goto error_free; + ret = ttm_bo_init_reserved(&dev_priv->bdev, bo, size, ttm_bo_type_device, placement, 0, - &ctx, acc_size, NULL, NULL, NULL); + &ctx, NULL, NULL, NULL); if (unlikely(ret)) - goto error_free; + goto error_account; ttm_bo_pin(bo); ttm_bo_unreserve(bo); @@ -519,6 +524,9 @@ int vmw_bo_create_kernel(struct vmw_private *dev_priv, unsigned long size, return 0; +error_account: + ttm_mem_global_free(&ttm_mem_glob, acc_size); + error_free: kfree(bo); return ret; @@ -558,11 +566,17 @@ int vmw_bo_init(struct vmw_private *dev_priv, vmw_bo->base.priority = 3; vmw_bo->res_tree = RB_ROOT; + ret = ttm_mem_global_alloc(&ttm_mem_glob, acc_size, &ctx); + if (unlikely(ret)) + return ret; + ret = ttm_bo_init_reserved(bdev, &vmw_bo->base, size, ttm_bo_type_device, placement, - 0, &ctx, acc_size, NULL, NULL, bo_free); - if (unlikely(ret)) + 0, &ctx, NULL, NULL, bo_free); + if (unlikely(ret)) { + ttm_mem_global_free(&ttm_mem_glob, acc_size); return ret; + } if (pin) ttm_bo_pin(&vmw_bo->base); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index 9a9afa087802..4efed3bf0ef9 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -1268,6 +1268,7 @@ static void vmw_remove(struct pci_dev *pdev) { struct drm_device *dev = pci_get_drvdata(pdev); + ttm_mem_global_release(&ttm_mem_glob); drm_dev_unregister(dev); vmw_driver_unload(dev); } @@ -1518,6 +1519,10 @@ static int vmw_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_drvdata(pdev, &vmw->drm); + ret = ttm_mem_global_init(&ttm_mem_glob, &pdev->dev); + if (ret) + return ret; + ret = vmw_driver_load(vmw, ent->device); if (ret) return ret; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c index d1bfa59579f1..63f10c865061 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c @@ -576,11 +576,31 @@ static void vmw_ttm_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) static int vmw_ttm_populate(struct ttm_device *bdev, struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { + unsigned int i; + int ret; + /* TODO: maybe completely drop this ? */ if (ttm_tt_is_populated(ttm)) return 0; - return ttm_pool_alloc(&bdev->pool, ttm, ctx); + ret = ttm_pool_alloc(&bdev->pool, ttm, ctx); + if (ret) + return ret; + + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_mem_global_alloc_page(&ttm_mem_glob, ttm->pages[i], + PAGE_SIZE, ctx); + if (ret) + goto error; + } + return 0; + +error: + while (i--) + ttm_mem_global_free_page(&ttm_mem_glob, ttm->pages[i], + PAGE_SIZE); + ttm_pool_free(&bdev->pool, ttm); + return ret; } static void vmw_ttm_unpopulate(struct ttm_device *bdev, @@ -588,6 +608,7 @@ static void vmw_ttm_unpopulate(struct ttm_device *bdev, { struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt, dma_ttm); + unsigned int i; if (vmw_tt->mob) { vmw_mob_destroy(vmw_tt->mob); @@ -595,6 +616,11 @@ static void vmw_ttm_unpopulate(struct ttm_device *bdev, } vmw_ttm_unmap_dma(vmw_tt); + + for (i = 0; i < ttm->num_pages; ++i) + ttm_mem_global_free_page(&ttm_mem_glob, ttm->pages[i], + PAGE_SIZE); + ttm_pool_free(&bdev->pool, ttm); } -- cgit v1.2.3