diff --git a/include/linux/kidled.h b/include/linux/kidled.h index a212b9b6adf458..d6d3dc1c8f69b7 100644 --- a/include/linux/kidled.h +++ b/include/linux/kidled.h @@ -221,6 +221,10 @@ void kidled_mem_cgroup_move_stats(struct mem_cgroup *from, unsigned int nr_pages); #endif /* CONFIG_MEMCG */ +#ifdef KIDLED_AGE_NOT_IN_PAGE_FLAGS +void kidled_free_page_age(pg_data_t *pgdat); +#endif + #else /* !CONFIG_KIDLED */ #ifdef CONFIG_MEMCG diff --git a/include/linux/mm.h b/include/linux/mm.h index 66399f1df6ff66..9ad745f94af372 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1155,38 +1155,54 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) #ifdef KIDLED_AGE_NOT_IN_PAGE_FLAGS static inline int kidled_get_page_age(pg_data_t *pgdat, unsigned long pfn) { - u8 *age = pgdat->node_page_age; + u8 *age, age_val; - if (unlikely(!age)) + rcu_read_lock(); + age = rcu_dereference(pgdat->node_page_age); + if (unlikely(!age)) { + rcu_read_unlock(); return -EINVAL; + } age += (pfn - pgdat->node_start_pfn); - return *age; + age_val = *age; + rcu_read_unlock(); + return age_val; } static inline int kidled_inc_page_age(pg_data_t *pgdat, unsigned long pfn) { - u8 *age = pgdat->node_page_age; + u8 *age, age_val; - if (unlikely(!age)) + rcu_read_lock(); + age = rcu_dereference(pgdat->node_page_age); + if (unlikely(!age)) { + rcu_read_unlock(); return -EINVAL; + } age += (pfn - pgdat->node_start_pfn); - *age += 1; + age_val = ++*age; + rcu_read_unlock(); - return *age; + return age_val; } static inline void kidled_set_page_age(pg_data_t *pgdat, unsigned long pfn, int val) { - u8 *age = pgdat->node_page_age; + u8 *age; - if (unlikely(!age)) + rcu_read_lock(); + age = rcu_dereference(pgdat->node_page_age); + if (unlikely(!age)) { + rcu_read_unlock(); return; + } age += (pfn - pgdat->node_start_pfn); *age = val; + rcu_read_unlock(); } #else static inline int kidled_get_page_age(pg_data_t *pgdat, unsigned long pfn) diff --git a/mm/kidled.c b/mm/kidled.c index 00f282b9a46231..163f793ad5d025 100644 --- a/mm/kidled.c +++ b/mm/kidled.c @@ -400,13 +400,16 @@ static bool kidled_scan_node(pg_data_t *pgdat, #ifdef KIDLED_AGE_NOT_IN_PAGE_FLAGS if (unlikely(!pgdat->node_page_age)) { + u8 *age; + /* This node has none memory, skip it. */ if (!pgdat->node_spanned_pages) return true; - pgdat->node_page_age = vzalloc(pgdat->node_spanned_pages); - if (unlikely(!pgdat->node_page_age)) + age = vzalloc(pgdat->node_spanned_pages); + if (unlikely(!age)) return false; + rcu_assign_pointer(pgdat->node_page_age, age); } #endif /* KIDLED_AGE_NOT_IN_PAGE_FLAGS */ @@ -429,6 +432,20 @@ static bool kidled_scan_node(pg_data_t *pgdat, return pfn >= node_end; } +#ifdef KIDLED_AGE_NOT_IN_PAGE_FLAGS +void kidled_free_page_age(pg_data_t *pgdat) +{ + u8 *age; + + age = rcu_access_pointer(pgdat->node_page_age); + if (age) { + rcu_assign_pointer(pgdat->node_page_age, NULL); + synchronize_rcu(); + vfree(age); + } +} +#endif + static inline void kidled_scan_done(struct kidled_scan_period scan_period) { kidled_mem_cgroup_scan_done(scan_period); @@ -448,10 +465,9 @@ static inline void kidled_reset(bool free) if (!pgdat->node_page_age) continue; - if (free) { - vfree(pgdat->node_page_age); - pgdat->node_page_age = NULL; - } else { + if (free) + kidled_free_page_age(pgdat); + else { memset(pgdat->node_page_age, 0, pgdat->node_spanned_pages); } diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index d5beb2b0b9a015..4c93c61052261e 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -36,6 +36,7 @@ #include #include #include +#include #include @@ -750,12 +751,6 @@ static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned lon pgdat->node_start_pfn = start_pfn; pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn; -#ifdef KIDLED_AGE_NOT_IN_PAGE_FLAGS - if (pgdat->node_page_age) { - vfree(pgdat->node_page_age); - pgdat->node_page_age = NULL; - } -#endif } void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, @@ -765,6 +760,10 @@ void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, int nid = pgdat->node_id; unsigned long flags; +#ifdef KIDLED_AGE_NOT_IN_PAGE_FLAGS + kidled_free_page_age(pgdat); +#endif + if (zone_is_empty(zone)) init_currently_empty_zone(zone, start_pfn, nr_pages); @@ -1927,10 +1926,7 @@ void try_offline_node(int nid) return; #ifdef KIDLED_AGE_NOT_IN_PAGE_FLAGS - if (pgdat->node_page_age) { - vfree(pgdat->node_page_age); - pgdat->node_page_age = NULL; - } + kidled_free_page_age(pgdat); #endif /*