diff options
Diffstat (limited to 'drivers/gpu')
| -rw-r--r-- | drivers/gpu/ion/ion.c | 115 | ||||
| -rw-r--r-- | drivers/gpu/ion/ion_heap.c | 107 | ||||
| -rw-r--r-- | drivers/gpu/ion/ion_page_pool.c | 122 | ||||
| -rw-r--r-- | drivers/gpu/ion/ion_priv.h | 57 | ||||
| -rw-r--r-- | drivers/gpu/ion/ion_system_heap.c | 49 | 
5 files changed, 276 insertions, 174 deletions
diff --git a/drivers/gpu/ion/ion.c b/drivers/gpu/ion/ion.c index d2d05139a2b..6c93365c3db 100644 --- a/drivers/gpu/ion/ion.c +++ b/drivers/gpu/ion/ion.c @@ -29,8 +29,6 @@  #include <linux/mm.h>  #include <linux/mm_types.h>  #include <linux/rbtree.h> -#include <linux/rtmutex.h> -#include <linux/sched.h>  #include <linux/slab.h>  #include <linux/seq_file.h>  #include <linux/uaccess.h> @@ -143,7 +141,6 @@ static void ion_buffer_add(struct ion_device *dev,  static int ion_buffer_alloc_dirty(struct ion_buffer *buffer); -static bool ion_heap_drain_freelist(struct ion_heap *heap);  /* this function should only be called while dev->lock is held */  static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,  				     struct ion_device *dev, @@ -170,7 +167,7 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,  		if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE))  			goto err2; -		ion_heap_drain_freelist(heap); +		ion_heap_freelist_drain(heap, 0);  		ret = heap->ops->allocate(heap, buffer, len, align,  					  flags);  		if (ret) @@ -230,7 +227,7 @@ err2:  	return ERR_PTR(ret);  } -static void _ion_buffer_destroy(struct ion_buffer *buffer) +void ion_buffer_destroy(struct ion_buffer *buffer)  {  	if (WARN_ON(buffer->kmap_cnt > 0))  		buffer->heap->ops->unmap_kernel(buffer->heap, buffer); @@ -241,7 +238,7 @@ static void _ion_buffer_destroy(struct ion_buffer *buffer)  	kfree(buffer);  } -static void ion_buffer_destroy(struct kref *kref) +static void _ion_buffer_destroy(struct kref *kref)  {  	struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref);  	struct ion_heap *heap = buffer->heap; @@ -251,14 +248,10 @@ static void ion_buffer_destroy(struct kref *kref)  	rb_erase(&buffer->node, &dev->buffers);  	mutex_unlock(&dev->buffer_lock); -	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) { -		rt_mutex_lock(&heap->lock); -		list_add(&buffer->list, &heap->free_list); -		rt_mutex_unlock(&heap->lock); -		wake_up(&heap->waitqueue); -		return; -	} -	_ion_buffer_destroy(buffer); +	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) +		ion_heap_freelist_add(heap, buffer); +	else +		ion_buffer_destroy(buffer);  }  static void ion_buffer_get(struct ion_buffer *buffer) @@ -268,7 +261,7 @@ static void ion_buffer_get(struct ion_buffer *buffer)  static int ion_buffer_put(struct ion_buffer *buffer)  { -	return kref_put(&buffer->ref, ion_buffer_destroy); +	return kref_put(&buffer->ref, _ion_buffer_destroy);  }  static void ion_buffer_add_to_handle(struct ion_buffer *buffer) @@ -1298,80 +1291,53 @@ static const struct file_operations debug_heap_fops = {  	.release = single_release,  }; -static size_t ion_heap_free_list_is_empty(struct ion_heap *heap) +#ifdef DEBUG_HEAP_SHRINKER +static int debug_shrink_set(void *data, u64 val)  { -	bool is_empty; +        struct ion_heap *heap = data; +        struct shrink_control sc; +        int objs; -	rt_mutex_lock(&heap->lock); -	is_empty = list_empty(&heap->free_list); -	rt_mutex_unlock(&heap->lock); +        sc.gfp_mask = -1; +        sc.nr_to_scan = 0; -	return is_empty; -} - -static int ion_heap_deferred_free(void *data) -{ -	struct ion_heap *heap = data; +        if (!val) +                return 0; -	while (true) { -		struct ion_buffer *buffer; - -		wait_event_freezable(heap->waitqueue, -				     !ion_heap_free_list_is_empty(heap)); - -		rt_mutex_lock(&heap->lock); -		if (list_empty(&heap->free_list)) { -			rt_mutex_unlock(&heap->lock); -			continue; -		} -		buffer = list_first_entry(&heap->free_list, struct ion_buffer, -					  list); -		list_del(&buffer->list); -		rt_mutex_unlock(&heap->lock); -		_ion_buffer_destroy(buffer); -	} +        objs = heap->shrinker.shrink(&heap->shrinker, &sc); +        sc.nr_to_scan = objs; -	return 0; +        heap->shrinker.shrink(&heap->shrinker, &sc); +        return 0;  } -static bool ion_heap_drain_freelist(struct ion_heap *heap) +static int debug_shrink_get(void *data, u64 *val)  { -	struct ion_buffer *buffer, *tmp; +        struct ion_heap *heap = data; +        struct shrink_control sc; +        int objs; -	if (ion_heap_free_list_is_empty(heap)) -		return false; -	rt_mutex_lock(&heap->lock); -	list_for_each_entry_safe(buffer, tmp, &heap->free_list, list) { -		list_del(&buffer->list); -		_ion_buffer_destroy(buffer); -	} -	BUG_ON(!list_empty(&heap->free_list)); -	rt_mutex_unlock(&heap->lock); +        sc.gfp_mask = -1; +        sc.nr_to_scan = 0; - -	return true; +        objs = heap->shrinker.shrink(&heap->shrinker, &sc); +        *val = objs; +        return 0;  } +DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get, +                        debug_shrink_set, "%llu\n"); +#endif +  void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)  { -	struct sched_param param = { .sched_priority = 0 }; -  	if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma ||  	    !heap->ops->unmap_dma)  		pr_err("%s: can not add heap with invalid ops struct.\n",  		       __func__); -	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) { -		INIT_LIST_HEAD(&heap->free_list); -		rt_mutex_init(&heap->lock); -		init_waitqueue_head(&heap->waitqueue); -		heap->task = kthread_run(ion_heap_deferred_free, heap, -					 "%s", heap->name); -		sched_setscheduler(heap->task, SCHED_IDLE, ¶m); -		if (IS_ERR(heap->task)) -			pr_err("%s: creating thread for deferred free failed\n", -			       __func__); -	} +	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) +		ion_heap_init_deferred_free(heap);  	heap->dev = dev;  	down_write(&dev->lock); @@ -1381,6 +1347,15 @@ void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)  	plist_add(&heap->node, &dev->heaps);  	debugfs_create_file(heap->name, 0664, dev->debug_root, heap,  			    &debug_heap_fops); +#ifdef DEBUG_HEAP_SHRINKER +	if (heap->shrinker.shrink) { +		char debug_name[64]; + +		snprintf(debug_name, 64, "%s_shrink", heap->name); +		debugfs_create_file(debug_name, 0644, dev->debug_root, heap, +				    &debug_shrink_fops); +	} +#endif  	up_write(&dev->lock);  } diff --git a/drivers/gpu/ion/ion_heap.c b/drivers/gpu/ion/ion_heap.c index 225ef94655d..4a16aa27384 100644 --- a/drivers/gpu/ion/ion_heap.c +++ b/drivers/gpu/ion/ion_heap.c @@ -15,8 +15,12 @@   */  #include <linux/err.h> +#include <linux/freezer.h>  #include <linux/ion.h> +#include <linux/kthread.h>  #include <linux/mm.h> +#include <linux/rtmutex.h> +#include <linux/sched.h>  #include <linux/scatterlist.h>  #include <linux/vmalloc.h>  #include "ion_priv.h" @@ -130,6 +134,109 @@ end:  	return ret;  } +void ion_heap_free_page(struct ion_buffer *buffer, struct page *page, +		       unsigned int order) +{ +	int i; + +	if (!ion_buffer_fault_user_mappings(buffer)) { +		__free_pages(page, order); +		return; +	} +	for (i = 0; i < (1 << order); i++) +		__free_page(page + i); +} + +void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer * buffer) +{ +	rt_mutex_lock(&heap->lock); +	list_add(&buffer->list, &heap->free_list); +	heap->free_list_size += buffer->size; +	rt_mutex_unlock(&heap->lock); +	wake_up(&heap->waitqueue); +} + +size_t ion_heap_freelist_size(struct ion_heap *heap) +{ +	size_t size; + +	rt_mutex_lock(&heap->lock); +	size = heap->free_list_size; +	rt_mutex_unlock(&heap->lock); + +	return size; +} + +size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size) +{ +	struct ion_buffer *buffer, *tmp; +	size_t total_drained = 0; + +	if (ion_heap_freelist_size(heap) == 0) +		return 0; + +	rt_mutex_lock(&heap->lock); +	if (size == 0) +		size = heap->free_list_size; + +	list_for_each_entry_safe(buffer, tmp, &heap->free_list, list) { +		if (total_drained >= size) +			break; +		list_del(&buffer->list); +		ion_buffer_destroy(buffer); +		heap->free_list_size -= buffer->size; +		total_drained += buffer->size; +	} +	rt_mutex_unlock(&heap->lock); + +	return total_drained; +} + +int ion_heap_deferred_free(void *data) +{ +	struct ion_heap *heap = data; + +	while (true) { +		struct ion_buffer *buffer; + +		wait_event_freezable(heap->waitqueue, +				     ion_heap_freelist_size(heap) > 0); + +		rt_mutex_lock(&heap->lock); +		if (list_empty(&heap->free_list)) { +			rt_mutex_unlock(&heap->lock); +			continue; +		} +		buffer = list_first_entry(&heap->free_list, struct ion_buffer, +					  list); +		list_del(&buffer->list); +		heap->free_list_size -= buffer->size; +		rt_mutex_unlock(&heap->lock); +		ion_buffer_destroy(buffer); +	} + +	return 0; +} + +int ion_heap_init_deferred_free(struct ion_heap *heap) +{ +	struct sched_param param = { .sched_priority = 0 }; + +	INIT_LIST_HEAD(&heap->free_list); +	heap->free_list_size = 0; +	rt_mutex_init(&heap->lock); +	init_waitqueue_head(&heap->waitqueue); +	heap->task = kthread_run(ion_heap_deferred_free, heap, +				 "%s", heap->name); +	sched_setscheduler(heap->task, SCHED_IDLE, ¶m); +	if (IS_ERR(heap->task)) { +		pr_err("%s: creating thread for deferred free failed\n", +		       __func__); +		return PTR_RET(heap->task); +	} +	return 0; +} +  struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data)  {  	struct ion_heap *heap = NULL; diff --git a/drivers/gpu/ion/ion_page_pool.c b/drivers/gpu/ion/ion_page_pool.c index 1658c9444b3..8708bbfd61b 100644 --- a/drivers/gpu/ion/ion_page_pool.c +++ b/drivers/gpu/ion/ion_page_pool.c @@ -21,14 +21,8 @@  #include <linux/list.h>  #include <linux/module.h>  #include <linux/slab.h> -#include <linux/shrinker.h>  #include "ion_priv.h" -/* #define DEBUG_PAGE_POOL_SHRINKER */ - -static struct plist_head pools = PLIST_HEAD_INIT(pools); -static struct shrinker shrinker; -  struct ion_page_pool_item {  	struct page *page;  	struct list_head list; @@ -126,109 +120,46 @@ void ion_page_pool_free(struct ion_page_pool *pool, struct page* page)  		ion_page_pool_free_pages(pool, page);  } -#ifdef DEBUG_PAGE_POOL_SHRINKER -static int debug_drop_pools_set(void *data, u64 val) +static int ion_page_pool_total(struct ion_page_pool *pool, bool high)  { -	struct shrink_control sc; -	int objs; - -	sc.gfp_mask = -1; -	sc.nr_to_scan = 0; - -	if (!val) -		return 0; - -	objs = shrinker.shrink(&shrinker, &sc); -	sc.nr_to_scan = objs; - -	shrinker.shrink(&shrinker, &sc); -	return 0; -} - -static int debug_drop_pools_get(void *data, u64 *val) -{ -	struct shrink_control sc; -	int objs; - -	sc.gfp_mask = -1; -	sc.nr_to_scan = 0; - -	objs = shrinker.shrink(&shrinker, &sc); -	*val = objs; -	return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(debug_drop_pools_fops, debug_drop_pools_get, -                        debug_drop_pools_set, "%llu\n"); - -static int debug_grow_pools_set(void *data, u64 val) -{ -	struct ion_page_pool *pool; -	struct page *page; - -	plist_for_each_entry(pool, &pools, list) { -		if (val != pool->list.prio) -			continue; -		page = ion_page_pool_alloc_pages(pool); -		if (page) -			ion_page_pool_add(pool, page); -	} - -	return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(debug_grow_pools_fops, debug_drop_pools_get, -			debug_grow_pools_set, "%llu\n"); -#endif - -static int ion_page_pool_total(bool high) -{ -	struct ion_page_pool *pool;  	int total = 0; -	plist_for_each_entry(pool, &pools, list) { -		total += high ? (pool->high_count + pool->low_count) * -			(1 << pool->order) : +	total += high ? (pool->high_count + pool->low_count) * +		(1 << pool->order) :  			pool->low_count * (1 << pool->order); -	}  	return total;  } -static int ion_page_pool_shrink(struct shrinker *shrinker, -				 struct shrink_control *sc) +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, +				int nr_to_scan)  { -	struct ion_page_pool *pool;  	int nr_freed = 0;  	int i;  	bool high; -	int nr_to_scan = sc->nr_to_scan; -	high = sc->gfp_mask & __GFP_HIGHMEM; +	high = gfp_mask & __GFP_HIGHMEM;  	if (nr_to_scan == 0) -		return ion_page_pool_total(high); +		return ion_page_pool_total(pool, high); -	plist_for_each_entry(pool, &pools, list) { -		for (i = 0; i < nr_to_scan; i++) { -			struct page *page; +	for (i = 0; i < nr_to_scan; i++) { +		struct page *page; -			mutex_lock(&pool->mutex); -			if (high && pool->high_count) { -				page = ion_page_pool_remove(pool, true); -			} else if (pool->low_count) { -				page = ion_page_pool_remove(pool, false); -			} else { -				mutex_unlock(&pool->mutex); -				break; -			} +		mutex_lock(&pool->mutex); +		if (high && pool->high_count) { +			page = ion_page_pool_remove(pool, true); +		} else if (pool->low_count) { +			page = ion_page_pool_remove(pool, false); +		} else {  			mutex_unlock(&pool->mutex); -			ion_page_pool_free_pages(pool, page); -			nr_freed += (1 << pool->order); +			break;  		} -		nr_to_scan -= i; +		mutex_unlock(&pool->mutex); +		ion_page_pool_free_pages(pool, page); +		nr_freed += (1 << pool->order);  	} -	return ion_page_pool_total(high); +	return nr_freed;  }  struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) @@ -245,35 +176,22 @@ struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order)  	pool->order = order;  	mutex_init(&pool->mutex);  	plist_node_init(&pool->list, order); -	plist_add(&pool->list, &pools);  	return pool;  }  void ion_page_pool_destroy(struct ion_page_pool *pool)  { -	plist_del(&pool->list, &pools);  	kfree(pool);  }  static int __init ion_page_pool_init(void)  { -	shrinker.shrink = ion_page_pool_shrink; -	shrinker.seeks = DEFAULT_SEEKS; -	shrinker.batch = 0; -	register_shrinker(&shrinker); -#ifdef DEBUG_PAGE_POOL_SHRINKER -	debugfs_create_file("ion_pools_shrink", 0644, NULL, NULL, -			    &debug_drop_pools_fops); -	debugfs_create_file("ion_pools_grow", 0644, NULL, NULL, -			    &debug_grow_pools_fops); -#endif  	return 0;  }  static void __exit ion_page_pool_exit(void)  { -	unregister_shrinker(&shrinker);  }  module_init(ion_page_pool_init); diff --git a/drivers/gpu/ion/ion_priv.h b/drivers/gpu/ion/ion_priv.h index 505681479c0..f9bccbc131e 100644 --- a/drivers/gpu/ion/ion_priv.h +++ b/drivers/gpu/ion/ion_priv.h @@ -81,6 +81,7 @@ struct ion_buffer {  	char task_comm[TASK_COMM_LEN];  	pid_t pid;  }; +void ion_buffer_destroy(struct ion_buffer *buffer);  /**   * struct ion_heap_ops - ops to operate on a given heap @@ -126,7 +127,12 @@ struct ion_heap_ops {   *			allocating.  These are specified by platform data and   *			MUST be unique   * @name:		used for debugging + * @shrinker:		a shrinker for the heap, if the heap caches system + *			memory, it must define a shrinker to return it on low + *			memory conditions, this includes system memory cached + *			in the deferred free lists for heaps that support it   * @free_list:		free list head if deferred free is used + * @free_list_size	size of the deferred free list in bytes   * @lock:		protects the free list   * @waitqueue:		queue to wait on from deferred free thread   * @task:		task struct of deferred free thread @@ -146,7 +152,9 @@ struct ion_heap {  	unsigned long flags;  	unsigned int id;  	const char *name; +	struct shrinker shrinker;  	struct list_head free_list; +	size_t free_list_size;  	struct rt_mutex lock;  	wait_queue_head_t waitqueue;  	struct task_struct *task; @@ -204,6 +212,43 @@ int ion_heap_map_user(struct ion_heap *, struct ion_buffer *,  			struct vm_area_struct *);  int ion_heap_buffer_zero(struct ion_buffer *buffer); +/** + * ion_heap_init_deferred_free -- initialize deferred free functionality + * @heap:		the heap + * + * If a heap sets the ION_HEAP_FLAG_DEFER_FREE flag this function will + * be called to setup deferred frees. Calls to free the buffer will + * return immediately and the actual free will occur some time later + */ +int ion_heap_init_deferred_free(struct ion_heap *heap); + +/** + * ion_heap_freelist_add - add a buffer to the deferred free list + * @heap:		the heap + * @buffer: 		the buffer + * + * Adds an item to the deferred freelist. + */ +void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer); + +/** + * ion_heap_freelist_drain - drain the deferred free list + * @heap:		the heap + * @size:		ammount of memory to drain in bytes + * + * Drains the indicated amount of memory from the deferred freelist immediately. + * Returns the total amount freed.  The total freed may be higher depending + * on the size of the items in the list, or lower if there is insufficient + * total memory on the freelist. + */ +size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size); + +/** + * ion_heap_freelist_size - returns the size of the freelist in bytes + * @heap:		the heap + */ +size_t ion_heap_freelist_size(struct ion_heap *heap); +  /**   * functions for creating and destroying the built in ion heaps. @@ -273,8 +318,6 @@ struct ion_page_pool {  	struct list_head high_items;  	struct list_head low_items;  	struct mutex mutex; -	void *(*alloc)(struct ion_page_pool *pool); -	void (*free)(struct ion_page_pool *pool, struct page *page);  	gfp_t gfp_mask;  	unsigned int order;  	struct plist_node list; @@ -285,4 +328,14 @@ void ion_page_pool_destroy(struct ion_page_pool *);  void *ion_page_pool_alloc(struct ion_page_pool *);  void ion_page_pool_free(struct ion_page_pool *, struct page *); +/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool + * @pool:		the pool + * @gfp_mask:		the memory type to reclaim + * @nr_to_scan:		number of items to shrink in pages + * + * returns the number of items freed in pages + */ +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, +			  int nr_to_scan); +  #endif /* _ION_PRIV_H */ diff --git a/drivers/gpu/ion/ion_system_heap.c b/drivers/gpu/ion/ion_system_heap.c index 6369fe8f305..37e1871ee02 100644 --- a/drivers/gpu/ion/ion_system_heap.c +++ b/drivers/gpu/ion/ion_system_heap.c @@ -253,6 +253,50 @@ static struct ion_heap_ops system_heap_ops = {  	.map_user = ion_heap_map_user,  }; +static int ion_system_heap_shrink(struct shrinker *shrinker, +				  struct shrink_control *sc) { + +	struct ion_heap *heap = container_of(shrinker, struct ion_heap, +					     shrinker); +	struct ion_system_heap *sys_heap = container_of(heap, +							struct ion_system_heap, +							heap); +	int nr_total = 0; +	int nr_freed = 0; +	int i; + +	if (sc->nr_to_scan == 0) +		goto end; + +	/* shrink the free list first, no point in zeroing the memory if +	   we're just going to reclaim it */ +	nr_freed += ion_heap_freelist_drain(heap, sc->nr_to_scan * PAGE_SIZE) / +		PAGE_SIZE; + +	if (nr_freed >= sc->nr_to_scan) +		goto end; + +	for (i = 0; i < num_orders; i++) { +		struct ion_page_pool *pool = sys_heap->pools[i]; + +		nr_freed += ion_page_pool_shrink(pool, sc->gfp_mask, +						 sc->nr_to_scan); +		if (nr_freed >= sc->nr_to_scan) +			break; +	} + +end: +	/* total number of items is whatever the page pools are holding +	   plus whatever's in the freelist */ +	for (i = 0; i < num_orders; i++) { +		struct ion_page_pool *pool = sys_heap->pools[i]; +		nr_total += ion_page_pool_shrink(pool, sc->gfp_mask, 0); +	} +	nr_total += ion_heap_freelist_size(heap) / PAGE_SIZE; +	return nr_total; + +} +  static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s,  				      void *unused)  { @@ -299,6 +343,11 @@ struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused)  			goto err_create_pool;  		heap->pools[i] = pool;  	} + +	heap->heap.shrinker.shrink = ion_system_heap_shrink; +	heap->heap.shrinker.seeks = DEFAULT_SEEKS; +	heap->heap.shrinker.batch = 0; +	register_shrinker(&heap->heap.shrinker);  	heap->heap.debug_show = ion_system_heap_debug_show;  	return &heap->heap;  err_create_pool:  |