Skip to content

Commit

Permalink
Merge tag 'x86-mm-2020-06-05' of git://git.kernel.org/pub/scm/linux/k…
Browse files Browse the repository at this point in the history 
…ernel/git/tip/tip

Pull x86 mm updates from Ingo Molnar:
 "Misc changes:

   - Unexport various PAT primitives

   - Unexport per-CPU tlbstate and uninline TLB helpers"

* tag 'x86-mm-2020-06-05' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (23 commits)
  x86/tlb/uv: Add a forward declaration for struct flush_tlb_info
  x86/cpu: Export native_write_cr4() only when CONFIG_LKTDM=m
  x86/tlb: Restrict access to tlbstate
  xen/privcmd: Remove unneeded asm/tlb.h include
  x86/tlb: Move PCID helpers where they are used
  x86/tlb: Uninline nmi_uaccess_okay()
  x86/tlb: Move cr4_set_bits_and_update_boot() to the usage site
  x86/tlb: Move paravirt_tlb_remove_table() to the usage site
  x86/tlb: Move __flush_tlb_all() out of line
  x86/tlb: Move flush_tlb_others() out of line
  x86/tlb: Move __flush_tlb_one_kernel() out of line
  x86/tlb: Move __flush_tlb_one_user() out of line
  x86/tlb: Move __flush_tlb_global() out of line
  x86/tlb: Move __flush_tlb() out of line
  x86/alternatives: Move temporary_mm helpers into C
  x86/cr4: Sanitize CR4.PCE update
  x86/cpu: Uninline CR4 accessors
  x86/tlb: Uninline __get_current_cr3_fast()
  x86/mm: Use pgprotval_t in protval_4k_2_large() and protval_large_2_4k()
  x86/mm: Unexport __cachemode2pte_tbl
  ...
  • Loading branch information
@torvalds
torvalds committed Jun 5, 2020
2 parents 435faf5 + bd1de2a commit f4dd60a
Show file tree
Hide file tree
Showing 24 changed files with 608 additions and 586 deletions.
11 changes: 3 additions & 8 deletions arch/x86/events/core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2166,11 +2166,6 @@ static int x86_pmu_event_init(struct perf_event *event)
return err;
}

static void refresh_pce(void *ignored)
{
load_mm_cr4_irqsoff(this_cpu_read(cpu_tlbstate.loaded_mm));
}

static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
{
if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
Expand All @@ -2189,7 +2184,7 @@ static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
lockdep_assert_held_write(&mm->mmap_sem);

if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
}

static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
Expand All @@ -2199,7 +2194,7 @@ static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *m
return;

if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
}

static int x86_pmu_event_idx(struct perf_event *event)
Expand Down Expand Up @@ -2257,7 +2252,7 @@ static ssize_t set_attr_rdpmc(struct device *cdev,
else if (x86_pmu.attr_rdpmc == 2)
static_branch_dec(&rdpmc_always_available_key);

on_each_cpu(refresh_pce, NULL, 1);
on_each_cpu(cr4_update_pce, NULL, 1);
x86_pmu.attr_rdpmc = val;
}

Expand Down
3 changes: 3 additions & 0 deletions arch/x86/include/asm/memtype.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,7 @@ extern void memtype_free_io(resource_size_t start, resource_size_t end);

extern bool pat_pfn_immune_to_uc_mtrr(unsigned long pfn);

bool x86_has_pat_wp(void);
enum page_cache_mode pgprot2cachemode(pgprot_t pgprot);

#endif /* _ASM_X86_MEMTYPE_H */
88 changes: 2 additions & 86 deletions arch/x86/include/asm/mmu_context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,9 @@ static inline void paravirt_activate_mm(struct mm_struct *prev,
#endif /* !CONFIG_PARAVIRT_XXL */

#ifdef CONFIG_PERF_EVENTS

DECLARE_STATIC_KEY_FALSE(rdpmc_never_available_key);
DECLARE_STATIC_KEY_FALSE(rdpmc_always_available_key);

static inline void load_mm_cr4_irqsoff(struct mm_struct *mm)
{
if (static_branch_unlikely(&rdpmc_always_available_key) ||
(!static_branch_unlikely(&rdpmc_never_available_key) &&
atomic_read(&mm->context.perf_rdpmc_allowed)))
cr4_set_bits_irqsoff(X86_CR4_PCE);
else
cr4_clear_bits_irqsoff(X86_CR4_PCE);
}
#else
static inline void load_mm_cr4_irqsoff(struct mm_struct *mm) {}
void cr4_update_pce(void *ignored);
#endif

#ifdef CONFIG_MODIFY_LDT_SYSCALL
Expand Down Expand Up @@ -225,78 +213,6 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
return __pkru_allows_pkey(vma_pkey(vma), write);
}

/*
* This can be used from process context to figure out what the value of
* CR3 is without needing to do a (slow) __read_cr3().
*
* It's intended to be used for code like KVM that sneakily changes CR3
* and needs to restore it. It needs to be used very carefully.
*/
static inline unsigned long __get_current_cr3_fast(void)
{
unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
this_cpu_read(cpu_tlbstate.loaded_mm_asid));

/* For now, be very restrictive about when this can be called. */
VM_WARN_ON(in_nmi() || preemptible());

VM_BUG_ON(cr3 != __read_cr3());
return cr3;
}

typedef struct {
struct mm_struct *mm;
} temp_mm_state_t;

/*
* Using a temporary mm allows to set temporary mappings that are not accessible
* by other CPUs. Such mappings are needed to perform sensitive memory writes
* that override the kernel memory protections (e.g., W^X), without exposing the
* temporary page-table mappings that are required for these write operations to
* other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the
* mapping is torn down.
*
* Context: The temporary mm needs to be used exclusively by a single core. To
* harden security IRQs must be disabled while the temporary mm is
* loaded, thereby preventing interrupt handler bugs from overriding
* the kernel memory protection.
*/
static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
{
temp_mm_state_t temp_state;

lockdep_assert_irqs_disabled();
temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm);
switch_mm_irqs_off(NULL, mm, current);

/*
* If breakpoints are enabled, disable them while the temporary mm is
* used. Userspace might set up watchpoints on addresses that are used
* in the temporary mm, which would lead to wrong signals being sent or
* crashes.
*
* Note that breakpoints are not disabled selectively, which also causes
* kernel breakpoints (e.g., perf's) to be disabled. This might be
* undesirable, but still seems reasonable as the code that runs in the
* temporary mm should be short.
*/
if (hw_breakpoint_active())
hw_breakpoint_disable();

return temp_state;
}

static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
{
lockdep_assert_irqs_disabled();
switch_mm_irqs_off(NULL, prev_state.mm, current);

/*
* Restore the breakpoints if they were disabled before the temporary mm
* was loaded.
*/
if (hw_breakpoint_active())
hw_breakpoint_restore();
}
unsigned long __get_current_cr3_fast(void);

#endif /* _ASM_X86_MMU_CONTEXT_H */
12 changes: 9 additions & 3 deletions arch/x86/include/asm/paravirt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,13 @@ static inline void slow_down_io(void)
#endif
}

static inline void __flush_tlb(void)
void native_flush_tlb_local(void);
void native_flush_tlb_global(void);
void native_flush_tlb_one_user(unsigned long addr);
void native_flush_tlb_others(const struct cpumask *cpumask,
const struct flush_tlb_info *info);

static inline void __flush_tlb_local(void)
{
PVOP_VCALL0(mmu.flush_tlb_user);
}
Expand All @@ -62,8 +68,8 @@ static inline void __flush_tlb_one_user(unsigned long addr)
PVOP_VCALL1(mmu.flush_tlb_one_user, addr);
}

static inline void flush_tlb_others(const struct cpumask *cpumask,
const struct flush_tlb_info *info)
static inline void __flush_tlb_others(const struct cpumask *cpumask,
const struct flush_tlb_info *info)
{
PVOP_VCALL2(mmu.flush_tlb_others, cpumask, info);
}
Expand Down
2 changes: 1 addition & 1 deletion arch/x86/include/asm/pgtable_32.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ void sync_initial_page_table(void);
#define kpte_clear_flush(ptep, vaddr) \
do { \
pte_clear(&init_mm, (vaddr), (ptep)); \
__flush_tlb_one_kernel((vaddr)); \
flush_tlb_one_kernel((vaddr)); \
} while (0)

#endif /* !__ASSEMBLY__ */
Expand Down
44 changes: 12 additions & 32 deletions arch/x86/include/asm/pgtable_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -471,9 +471,6 @@ static inline pteval_t pte_flags(pte_t pte)
return native_pte_val(pte) & PTE_FLAGS_MASK;
}

extern uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM];
extern uint8_t __pte2cachemode_tbl[8];

#define __pte2cm_idx(cb) \
((((cb) >> (_PAGE_BIT_PAT - 2)) & 4) | \
(((cb) >> (_PAGE_BIT_PCD - 1)) & 2) | \
Expand All @@ -483,43 +480,26 @@ extern uint8_t __pte2cachemode_tbl[8];
(((i) & 2) << (_PAGE_BIT_PCD - 1)) | \
(((i) & 1) << _PAGE_BIT_PWT))

static inline unsigned long cachemode2protval(enum page_cache_mode pcm)
unsigned long cachemode2protval(enum page_cache_mode pcm);

static inline pgprotval_t protval_4k_2_large(pgprotval_t val)
{
if (likely(pcm == 0))
return 0;
return __cachemode2pte_tbl[pcm];
return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) |
((val & _PAGE_PAT) << (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT));
}
static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm)
static inline pgprot_t pgprot_4k_2_large(pgprot_t pgprot)
{
return __pgprot(cachemode2protval(pcm));
return __pgprot(protval_4k_2_large(pgprot_val(pgprot)));
}
static inline enum page_cache_mode pgprot2cachemode(pgprot_t pgprot)
static inline pgprotval_t protval_large_2_4k(pgprotval_t val)
{
unsigned long masked;

masked = pgprot_val(pgprot) & _PAGE_CACHE_MASK;
if (likely(masked == 0))
return 0;
return __pte2cachemode_tbl[__pte2cm_idx(masked)];
}
static inline pgprot_t pgprot_4k_2_large(pgprot_t pgprot)
{
pgprotval_t val = pgprot_val(pgprot);
pgprot_t new;

pgprot_val(new) = (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) |
((val & _PAGE_PAT) << (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT));
return new;
return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) |
((val & _PAGE_PAT_LARGE) >>
(_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT));
}
static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot)
{
pgprotval_t val = pgprot_val(pgprot);
pgprot_t new;

pgprot_val(new) = (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) |
((val & _PAGE_PAT_LARGE) >>
(_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT));
return new;
return __pgprot(protval_large_2_4k(pgprot_val(pgprot)));
}


Expand Down
Loading

0 comments on commit f4dd60a

Please sign in to comment.