From 4db7327194dba0cb91f274b0f606785a9ee5108d Mon Sep 17 00:00:00 2001 From: Kevin Hao Date: Sat, 23 Jul 2016 14:42:41 +0530 Subject: powerpc: Add option to use jump label for cpu_has_feature() We do binary patching of asm code using CPU features, which is a one-time operation, done during early boot. However checks of CPU features in C code are currently done at run time, even though the set of CPU features can never change after boot. We can optimise this by using jump labels to implement cpu_has_feature(), meaning checks in C code are binary patched into a single nop or branch. For a C sequence along the lines of: if (cpu_has_feature(FOO)) return 2; The generated code before is roughly: ld r9,-27640(r2) ld r9,0(r9) lwz r9,32(r9) cmpwi cr7,r9,0 bge cr7, 1f li r3,2 blr 1: ... After (true): nop li r3,2 blr After (false): b 1f li r3,2 blr 1: ... mpe: Rename MAX_CPU_FEATURES as we already have a #define with that name, and define it simply as a constant, rather than doing tricks with sizeof and NULL pointers. Rename the array to cpu_feature_keys. Use the kconfig we added to guard it. Add BUILD_BUG_ON() if the feature is not a compile time constant. Rewrite the change log. Signed-off-by: Kevin Hao Signed-off-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/cpu_has_feature.h | 24 ++++++++++++++++++++++++ arch/powerpc/include/asm/cputable.h | 6 ++++++ arch/powerpc/include/asm/mmu.h | 1 + arch/powerpc/kernel/cputable.c | 20 ++++++++++++++++++++ arch/powerpc/lib/feature-fixups.c | 1 + 5 files changed, 52 insertions(+) (limited to 'arch/powerpc') diff --git a/arch/powerpc/include/asm/cpu_has_feature.h b/arch/powerpc/include/asm/cpu_has_feature.h index c935c5854fed..b9ce6425fdea 100644 --- a/arch/powerpc/include/asm/cpu_has_feature.h +++ b/arch/powerpc/include/asm/cpu_has_feature.h @@ -11,10 +11,34 @@ static inline bool early_cpu_has_feature(unsigned long feature) (CPU_FTRS_POSSIBLE & cur_cpu_spec->cpu_features & feature)); } +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS +#include + +#define NUM_CPU_FTR_KEYS 64 + +extern struct static_key_true cpu_feature_keys[NUM_CPU_FTR_KEYS]; + +static __always_inline bool cpu_has_feature(unsigned long feature) +{ + int i; + + BUILD_BUG_ON(!__builtin_constant_p(feature)); + + if (CPU_FTRS_ALWAYS & feature) + return true; + + if (!(CPU_FTRS_POSSIBLE & feature)) + return false; + + i = __builtin_ctzl(feature); + return static_branch_likely(&cpu_feature_keys[i]); +} +#else static inline bool cpu_has_feature(unsigned long feature) { return early_cpu_has_feature(feature); } +#endif #endif /* __ASSEMBLY__ */ #endif /* __ASM_POWERPC_CPUFEATURE_H */ diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index 92961bcfbe3f..82026b419341 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -123,6 +123,12 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start, extern const char *powerpc_base_platform; +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS +extern void cpu_feature_keys_init(void); +#else +static inline void cpu_feature_keys_init(void) { } +#endif + /* TLB flush actions. Used as argument to cpu_spec.flush_tlb() hook */ enum { TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */ diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index 08b4c06604d6..37f55169df3e 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -112,6 +112,7 @@ #define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \ MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B #ifndef __ASSEMBLY__ +#include #include #ifdef CONFIG_PPC_FSL_BOOK3E diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c index d81f826d1029..e3da41f8ef93 100644 --- a/arch/powerpc/kernel/cputable.c +++ b/arch/powerpc/kernel/cputable.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -2224,3 +2225,22 @@ struct cpu_spec * __init identify_cpu(unsigned long offset, unsigned int pvr) return NULL; } + +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS +struct static_key_true cpu_feature_keys[NUM_CPU_FTR_KEYS] = { + [0 ... NUM_CPU_FTR_KEYS - 1] = STATIC_KEY_TRUE_INIT +}; +EXPORT_SYMBOL_GPL(cpu_feature_keys); + +void __init cpu_feature_keys_init(void) +{ + int i; + + for (i = 0; i < NUM_CPU_FTR_KEYS; i++) { + unsigned long f = 1ul << i; + + if (!(cur_cpu_spec->cpu_features & f)) + static_branch_disable(&cpu_feature_keys[i]); + } +} +#endif diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c index ae57de78e2aa..7a6ccbb74d91 100644 --- a/arch/powerpc/lib/feature-fixups.c +++ b/arch/powerpc/lib/feature-fixups.c @@ -195,6 +195,7 @@ void __init apply_feature_fixups(void) * CPU/MMU features. */ jump_label_init(); + cpu_feature_keys_init(); } static int __init check_features(void) -- cgit v1.2.3