From aabfb57296e3dd9761e47736ec69305c95461d7d Mon Sep 17 00:00:00 2001 From: Michal Hocko Date: Thu, 9 Oct 2014 15:28:52 -0700 Subject: mm: memcontrol: do not kill uncharge batching in free_pages_and_swap_cache free_pages_and_swap_cache limits release_pages to PAGEVEC_SIZE chunks. This is not a big deal for the normal release path but it completely kills memcg uncharge batching which reduces res_counter spin_lock contention. Dave has noticed this with his page fault scalability test case on a large machine when the lock was basically dominating on all CPUs: 80.18% 80.18% [kernel] [k] _raw_spin_lock | --- _raw_spin_lock | |--66.59%-- res_counter_uncharge_until | res_counter_uncharge | uncharge_batch | uncharge_list | mem_cgroup_uncharge_list | release_pages | free_pages_and_swap_cache | tlb_flush_mmu_free | | | |--90.12%-- unmap_single_vma | | unmap_vmas | | unmap_region | | do_munmap | | vm_munmap | | sys_munmap | | system_call_fastpath | | __GI___munmap | | | --9.88%-- tlb_flush_mmu | tlb_finish_mmu | unmap_region | do_munmap | vm_munmap | sys_munmap | system_call_fastpath | __GI___munmap In his case the load was running in the root memcg and that part has been handled by reverting 05b843012335 ("mm: memcontrol: use root_mem_cgroup res_counter") because this is a clear regression, but the problem remains inside dedicated memcgs. There is no reason to limit release_pages to PAGEVEC_SIZE batches other than lru_lock held times. This logic, however, can be moved inside the function. mem_cgroup_uncharge_list and free_hot_cold_page_list do not hold any lock for the whole pages_to_free list so it is safe to call them in a single run. The release_pages() code was previously breaking the lru_lock each PAGEVEC_SIZE pages (ie, 14 pages). However this code has no usage of pagevecs so switch to breaking the lock at least every SWAP_CLUSTER_MAX (32) pages. This means that the lock acquisition frequency is approximately halved and the max hold times are approximately doubled. The now unneeded batching is removed from free_pages_and_swap_cache(). Also update the grossly out-of-date release_pages documentation. Signed-off-by: Michal Hocko Signed-off-by: Johannes Weiner Reported-by: Dave Hansen Cc: Vladimir Davydov Cc: Greg Thelen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/swap.c | 30 +++++++++++++++++++----------- mm/swap_state.c | 14 ++++---------- 2 files changed, 23 insertions(+), 21 deletions(-) (limited to 'mm') diff --git a/mm/swap.c b/mm/swap.c index 6b2dc3897cd5..8a12b33936b4 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -887,18 +887,14 @@ void lru_add_drain_all(void) mutex_unlock(&lock); } -/* - * Batched page_cache_release(). Decrement the reference count on all the - * passed pages. If it fell to zero then remove the page from the LRU and - * free it. - * - * Avoid taking zone->lru_lock if possible, but if it is taken, retain it - * for the remainder of the operation. +/** + * release_pages - batched page_cache_release() + * @pages: array of pages to release + * @nr: number of pages + * @cold: whether the pages are cache cold * - * The locking in this function is against shrink_inactive_list(): we recheck - * the page count inside the lock to see whether shrink_inactive_list() - * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() - * will free it. + * Decrement the reference count on all the pages in @pages. If it + * fell to zero, remove the page from the LRU and free it. */ void release_pages(struct page **pages, int nr, bool cold) { @@ -907,6 +903,7 @@ void release_pages(struct page **pages, int nr, bool cold) struct zone *zone = NULL; struct lruvec *lruvec; unsigned long uninitialized_var(flags); + unsigned int uninitialized_var(lock_batch); for (i = 0; i < nr; i++) { struct page *page = pages[i]; @@ -920,6 +917,16 @@ void release_pages(struct page **pages, int nr, bool cold) continue; } + /* + * Make sure the IRQ-safe lock-holding time does not get + * excessive with a continuous string of pages from the + * same zone. The lock is held only if zone != NULL. + */ + if (zone && ++lock_batch == SWAP_CLUSTER_MAX) { + spin_unlock_irqrestore(&zone->lru_lock, flags); + zone = NULL; + } + if (!put_page_testzero(page)) continue; @@ -930,6 +937,7 @@ void release_pages(struct page **pages, int nr, bool cold) if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); + lock_batch = 0; zone = pagezone; spin_lock_irqsave(&zone->lru_lock, flags); } diff --git a/mm/swap_state.c b/mm/swap_state.c index ef1f39139b71..154444918685 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -265,18 +265,12 @@ void free_page_and_swap_cache(struct page *page) void free_pages_and_swap_cache(struct page **pages, int nr) { struct page **pagep = pages; + int i; lru_add_drain(); - while (nr) { - int todo = min(nr, PAGEVEC_SIZE); - int i; - - for (i = 0; i < todo; i++) - free_swap_cache(pagep[i]); - release_pages(pagep, todo, false); - pagep += todo; - nr -= todo; - } + for (i = 0; i < nr; i++) + free_swap_cache(pagep[i]); + release_pages(pagep, nr, false); } /* -- cgit v1.2.3