drm/i915: Make the inactive object shrinker per-device
Eliminate the racy device unload by embedding a shrinker into each device. Smaller, simpler code. Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
This commit is contained in:
parent
176f28ebf4
commit
17250b7155
4 changed files with 54 additions and 139 deletions
|
@ -2096,6 +2096,9 @@ int i915_driver_unload(struct drm_device *dev)
|
||||||
i915_mch_dev = NULL;
|
i915_mch_dev = NULL;
|
||||||
spin_unlock(&mchdev_lock);
|
spin_unlock(&mchdev_lock);
|
||||||
|
|
||||||
|
if (dev_priv->mm.inactive_shrinker.shrink)
|
||||||
|
unregister_shrinker(&dev_priv->mm.inactive_shrinker);
|
||||||
|
|
||||||
mutex_lock(&dev->struct_mutex);
|
mutex_lock(&dev->struct_mutex);
|
||||||
ret = i915_gpu_idle(dev);
|
ret = i915_gpu_idle(dev);
|
||||||
if (ret)
|
if (ret)
|
||||||
|
|
|
@ -660,8 +660,6 @@ static int __init i915_init(void)
|
||||||
|
|
||||||
driver.num_ioctls = i915_max_ioctl;
|
driver.num_ioctls = i915_max_ioctl;
|
||||||
|
|
||||||
i915_gem_shrinker_init();
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If CONFIG_DRM_I915_KMS is set, default to KMS unless
|
* If CONFIG_DRM_I915_KMS is set, default to KMS unless
|
||||||
* explicitly disabled with the module pararmeter.
|
* explicitly disabled with the module pararmeter.
|
||||||
|
@ -693,7 +691,6 @@ static int __init i915_init(void)
|
||||||
|
|
||||||
static void __exit i915_exit(void)
|
static void __exit i915_exit(void)
|
||||||
{
|
{
|
||||||
i915_gem_shrinker_exit();
|
|
||||||
drm_exit(&driver);
|
drm_exit(&driver);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -542,14 +542,7 @@ typedef struct drm_i915_private {
|
||||||
struct io_mapping *gtt_mapping;
|
struct io_mapping *gtt_mapping;
|
||||||
int gtt_mtrr;
|
int gtt_mtrr;
|
||||||
|
|
||||||
/**
|
struct shrinker inactive_shrinker;
|
||||||
* Membership on list of all loaded devices, used to evict
|
|
||||||
* inactive buffers under memory pressure.
|
|
||||||
*
|
|
||||||
* Modifications should only be done whilst holding the
|
|
||||||
* shrink_list_lock spinlock.
|
|
||||||
*/
|
|
||||||
struct list_head shrink_list;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* List of objects currently involved in rendering.
|
* List of objects currently involved in rendering.
|
||||||
|
@ -1079,9 +1072,6 @@ void i915_gem_detach_phys_object(struct drm_device *dev,
|
||||||
void i915_gem_free_all_phys_object(struct drm_device *dev);
|
void i915_gem_free_all_phys_object(struct drm_device *dev);
|
||||||
void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv);
|
void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv);
|
||||||
|
|
||||||
void i915_gem_shrinker_init(void);
|
|
||||||
void i915_gem_shrinker_exit(void);
|
|
||||||
|
|
||||||
/* i915_gem_evict.c */
|
/* i915_gem_evict.c */
|
||||||
int i915_gem_evict_something(struct drm_device *dev, int min_size,
|
int i915_gem_evict_something(struct drm_device *dev, int min_size,
|
||||||
unsigned alignment, bool mappable);
|
unsigned alignment, bool mappable);
|
||||||
|
|
|
@ -65,8 +65,10 @@ i915_gem_object_get_pages(struct drm_gem_object *obj,
|
||||||
static void
|
static void
|
||||||
i915_gem_object_put_pages(struct drm_gem_object *obj);
|
i915_gem_object_put_pages(struct drm_gem_object *obj);
|
||||||
|
|
||||||
static LIST_HEAD(shrink_list);
|
static int i915_gem_inactive_shrink(struct shrinker *shrinker,
|
||||||
static DEFINE_SPINLOCK(shrink_list_lock);
|
int nr_to_scan,
|
||||||
|
gfp_t gfp_mask);
|
||||||
|
|
||||||
|
|
||||||
/* some bookkeeping */
|
/* some bookkeeping */
|
||||||
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
|
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
|
||||||
|
@ -4765,9 +4767,6 @@ i915_gem_load(struct drm_device *dev)
|
||||||
INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
|
INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
|
||||||
i915_gem_retire_work_handler);
|
i915_gem_retire_work_handler);
|
||||||
init_completion(&dev_priv->error_completion);
|
init_completion(&dev_priv->error_completion);
|
||||||
spin_lock(&shrink_list_lock);
|
|
||||||
list_add(&dev_priv->mm.shrink_list, &shrink_list);
|
|
||||||
spin_unlock(&shrink_list_lock);
|
|
||||||
|
|
||||||
/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
|
/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
|
||||||
if (IS_GEN3(dev)) {
|
if (IS_GEN3(dev)) {
|
||||||
|
@ -4810,6 +4809,10 @@ i915_gem_load(struct drm_device *dev)
|
||||||
}
|
}
|
||||||
i915_gem_detect_bit_6_swizzle(dev);
|
i915_gem_detect_bit_6_swizzle(dev);
|
||||||
init_waitqueue_head(&dev_priv->pending_flip_queue);
|
init_waitqueue_head(&dev_priv->pending_flip_queue);
|
||||||
|
|
||||||
|
dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
|
||||||
|
dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
|
||||||
|
register_shrinker(&dev_priv->mm.inactive_shrinker);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -5022,152 +5025,74 @@ i915_gpu_is_active(struct drm_device *dev)
|
||||||
int lists_empty;
|
int lists_empty;
|
||||||
|
|
||||||
lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
|
lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
|
||||||
list_empty(&dev_priv->render_ring.active_list) &&
|
list_empty(&dev_priv->mm.active_list);
|
||||||
list_empty(&dev_priv->bsd_ring.active_list) &&
|
|
||||||
list_empty(&dev_priv->blt_ring.active_list);
|
|
||||||
|
|
||||||
return !lists_empty;
|
return !lists_empty;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
|
i915_gem_inactive_shrink(struct shrinker *shrinker,
|
||||||
|
int nr_to_scan,
|
||||||
|
gfp_t gfp_mask)
|
||||||
{
|
{
|
||||||
drm_i915_private_t *dev_priv, *next_dev;
|
struct drm_i915_private *dev_priv =
|
||||||
struct drm_i915_gem_object *obj_priv, *next_obj;
|
container_of(shrinker,
|
||||||
int cnt = 0;
|
struct drm_i915_private,
|
||||||
int would_deadlock = 1;
|
mm.inactive_shrinker);
|
||||||
|
struct drm_device *dev = dev_priv->dev;
|
||||||
|
struct drm_i915_gem_object *obj, *next;
|
||||||
|
int cnt;
|
||||||
|
|
||||||
|
if (!mutex_trylock(&dev->struct_mutex))
|
||||||
|
return nr_to_scan ? 0 : -1;
|
||||||
|
|
||||||
/* "fast-path" to count number of available objects */
|
/* "fast-path" to count number of available objects */
|
||||||
if (nr_to_scan == 0) {
|
if (nr_to_scan == 0) {
|
||||||
spin_lock(&shrink_list_lock);
|
cnt = 0;
|
||||||
list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
|
list_for_each_entry(obj,
|
||||||
struct drm_device *dev = dev_priv->dev;
|
&dev_priv->mm.inactive_list,
|
||||||
|
mm_list)
|
||||||
if (mutex_trylock(&dev->struct_mutex)) {
|
cnt++;
|
||||||
list_for_each_entry(obj_priv,
|
mutex_unlock(&dev->struct_mutex);
|
||||||
&dev_priv->mm.inactive_list,
|
return cnt / 100 * sysctl_vfs_cache_pressure;
|
||||||
mm_list)
|
|
||||||
cnt++;
|
|
||||||
mutex_unlock(&dev->struct_mutex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
spin_unlock(&shrink_list_lock);
|
|
||||||
|
|
||||||
return (cnt / 100) * sysctl_vfs_cache_pressure;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
spin_lock(&shrink_list_lock);
|
|
||||||
|
|
||||||
rescan:
|
rescan:
|
||||||
/* first scan for clean buffers */
|
/* first scan for clean buffers */
|
||||||
list_for_each_entry_safe(dev_priv, next_dev,
|
i915_gem_retire_requests(dev);
|
||||||
&shrink_list, mm.shrink_list) {
|
|
||||||
struct drm_device *dev = dev_priv->dev;
|
|
||||||
|
|
||||||
if (! mutex_trylock(&dev->struct_mutex))
|
list_for_each_entry_safe(obj, next,
|
||||||
continue;
|
&dev_priv->mm.inactive_list,
|
||||||
|
mm_list) {
|
||||||
spin_unlock(&shrink_list_lock);
|
if (i915_gem_object_is_purgeable(obj)) {
|
||||||
i915_gem_retire_requests(dev);
|
i915_gem_object_unbind(&obj->base);
|
||||||
|
if (--nr_to_scan == 0)
|
||||||
list_for_each_entry_safe(obj_priv, next_obj,
|
break;
|
||||||
&dev_priv->mm.inactive_list,
|
|
||||||
mm_list) {
|
|
||||||
if (i915_gem_object_is_purgeable(obj_priv)) {
|
|
||||||
i915_gem_object_unbind(&obj_priv->base);
|
|
||||||
if (--nr_to_scan <= 0)
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
spin_lock(&shrink_list_lock);
|
|
||||||
mutex_unlock(&dev->struct_mutex);
|
|
||||||
|
|
||||||
would_deadlock = 0;
|
|
||||||
|
|
||||||
if (nr_to_scan <= 0)
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* second pass, evict/count anything still on the inactive list */
|
/* second pass, evict/count anything still on the inactive list */
|
||||||
list_for_each_entry_safe(dev_priv, next_dev,
|
cnt = 0;
|
||||||
&shrink_list, mm.shrink_list) {
|
list_for_each_entry_safe(obj, next,
|
||||||
struct drm_device *dev = dev_priv->dev;
|
&dev_priv->mm.inactive_list,
|
||||||
|
mm_list) {
|
||||||
if (! mutex_trylock(&dev->struct_mutex))
|
if (nr_to_scan) {
|
||||||
continue;
|
i915_gem_object_unbind(&obj->base);
|
||||||
|
nr_to_scan--;
|
||||||
spin_unlock(&shrink_list_lock);
|
} else
|
||||||
|
cnt++;
|
||||||
list_for_each_entry_safe(obj_priv, next_obj,
|
|
||||||
&dev_priv->mm.inactive_list,
|
|
||||||
mm_list) {
|
|
||||||
if (nr_to_scan > 0) {
|
|
||||||
i915_gem_object_unbind(&obj_priv->base);
|
|
||||||
nr_to_scan--;
|
|
||||||
} else
|
|
||||||
cnt++;
|
|
||||||
}
|
|
||||||
|
|
||||||
spin_lock(&shrink_list_lock);
|
|
||||||
mutex_unlock(&dev->struct_mutex);
|
|
||||||
|
|
||||||
would_deadlock = 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (nr_to_scan) {
|
if (nr_to_scan && i915_gpu_is_active(dev)) {
|
||||||
int active = 0;
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* We are desperate for pages, so as a last resort, wait
|
* We are desperate for pages, so as a last resort, wait
|
||||||
* for the GPU to finish and discard whatever we can.
|
* for the GPU to finish and discard whatever we can.
|
||||||
* This has a dramatic impact to reduce the number of
|
* This has a dramatic impact to reduce the number of
|
||||||
* OOM-killer events whilst running the GPU aggressively.
|
* OOM-killer events whilst running the GPU aggressively.
|
||||||
*/
|
*/
|
||||||
list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
|
if (i915_gpu_idle(dev) == 0)
|
||||||
struct drm_device *dev = dev_priv->dev;
|
|
||||||
|
|
||||||
if (!mutex_trylock(&dev->struct_mutex))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
spin_unlock(&shrink_list_lock);
|
|
||||||
|
|
||||||
if (i915_gpu_is_active(dev)) {
|
|
||||||
i915_gpu_idle(dev);
|
|
||||||
active++;
|
|
||||||
}
|
|
||||||
|
|
||||||
spin_lock(&shrink_list_lock);
|
|
||||||
mutex_unlock(&dev->struct_mutex);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (active)
|
|
||||||
goto rescan;
|
goto rescan;
|
||||||
}
|
}
|
||||||
|
mutex_unlock(&dev->struct_mutex);
|
||||||
spin_unlock(&shrink_list_lock);
|
return cnt / 100 * sysctl_vfs_cache_pressure;
|
||||||
|
|
||||||
if (would_deadlock)
|
|
||||||
return -1;
|
|
||||||
else if (cnt > 0)
|
|
||||||
return (cnt / 100) * sysctl_vfs_cache_pressure;
|
|
||||||
else
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static struct shrinker shrinker = {
|
|
||||||
.shrink = i915_gem_shrink,
|
|
||||||
.seeks = DEFAULT_SEEKS,
|
|
||||||
};
|
|
||||||
|
|
||||||
__init void
|
|
||||||
i915_gem_shrinker_init(void)
|
|
||||||
{
|
|
||||||
register_shrinker(&shrinker);
|
|
||||||
}
|
|
||||||
|
|
||||||
__exit void
|
|
||||||
i915_gem_shrinker_exit(void)
|
|
||||||
{
|
|
||||||
unregister_shrinker(&shrinker);
|
|
||||||
}
|
}
|
||||||
|
|
Reference in a new issue