diff options
Diffstat (limited to 'arch/s390/include')
| -rw-r--r-- | arch/s390/include/asm/cpacf.h | 178 | ||||
| -rw-r--r-- | arch/s390/include/asm/facilities_src.h | 24 | ||||
| -rw-r--r-- | arch/s390/include/asm/fpu/api.h | 32 | ||||
| -rw-r--r-- | arch/s390/include/asm/lowcore.h | 3 | ||||
| -rw-r--r-- | arch/s390/include/asm/mmu.h | 1 | ||||
| -rw-r--r-- | arch/s390/include/asm/mmu_context.h | 1 | ||||
| -rw-r--r-- | arch/s390/include/asm/pci.h | 5 | ||||
| -rw-r--r-- | arch/s390/include/asm/pgtable.h | 74 | ||||
| -rw-r--r-- | arch/s390/include/asm/tlbflush.h | 72 | ||||
| -rw-r--r-- | arch/s390/include/asm/vx-insn.h | 148 | ||||
| -rw-r--r-- | arch/s390/include/uapi/asm/Kbuild | 1 |
11 files changed, 293 insertions, 246 deletions
diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index d28621de8e0b..2c680db7e5c1 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -28,67 +28,51 @@ #define CPACF_PPNO 0xb93c /* MSA5 */ /* - * Function codes for the KM (CIPHER MESSAGE) - * instruction (0x80 is the decipher modifier bit) + * Decryption modifier bit + */ +#define CPACF_DECRYPT 0x80 + +/* + * Function codes for the KM (CIPHER MESSAGE) instruction */ #define CPACF_KM_QUERY 0x00 -#define CPACF_KM_DEA_ENC 0x01 -#define CPACF_KM_DEA_DEC 0x81 -#define CPACF_KM_TDEA_128_ENC 0x02 -#define CPACF_KM_TDEA_128_DEC 0x82 -#define CPACF_KM_TDEA_192_ENC 0x03 -#define CPACF_KM_TDEA_192_DEC 0x83 -#define CPACF_KM_AES_128_ENC 0x12 -#define CPACF_KM_AES_128_DEC 0x92 -#define CPACF_KM_AES_192_ENC 0x13 -#define CPACF_KM_AES_192_DEC 0x93 -#define CPACF_KM_AES_256_ENC 0x14 -#define CPACF_KM_AES_256_DEC 0x94 -#define CPACF_KM_XTS_128_ENC 0x32 -#define CPACF_KM_XTS_128_DEC 0xb2 -#define CPACF_KM_XTS_256_ENC 0x34 -#define CPACF_KM_XTS_256_DEC 0xb4 +#define CPACF_KM_DEA 0x01 +#define CPACF_KM_TDEA_128 0x02 +#define CPACF_KM_TDEA_192 0x03 +#define CPACF_KM_AES_128 0x12 +#define CPACF_KM_AES_192 0x13 +#define CPACF_KM_AES_256 0x14 +#define CPACF_KM_XTS_128 0x32 +#define CPACF_KM_XTS_256 0x34 /* * Function codes for the KMC (CIPHER MESSAGE WITH CHAINING) - * instruction (0x80 is the decipher modifier bit) + * instruction */ #define CPACF_KMC_QUERY 0x00 -#define CPACF_KMC_DEA_ENC 0x01 -#define CPACF_KMC_DEA_DEC 0x81 -#define CPACF_KMC_TDEA_128_ENC 0x02 -#define CPACF_KMC_TDEA_128_DEC 0x82 -#define CPACF_KMC_TDEA_192_ENC 0x03 -#define CPACF_KMC_TDEA_192_DEC 0x83 -#define CPACF_KMC_AES_128_ENC 0x12 -#define CPACF_KMC_AES_128_DEC 0x92 -#define CPACF_KMC_AES_192_ENC 0x13 -#define CPACF_KMC_AES_192_DEC 0x93 -#define CPACF_KMC_AES_256_ENC 0x14 -#define CPACF_KMC_AES_256_DEC 0x94 +#define CPACF_KMC_DEA 0x01 +#define CPACF_KMC_TDEA_128 0x02 +#define CPACF_KMC_TDEA_192 0x03 +#define CPACF_KMC_AES_128 0x12 +#define CPACF_KMC_AES_192 0x13 +#define CPACF_KMC_AES_256 0x14 #define CPACF_KMC_PRNG 0x43 /* * Function codes for the KMCTR (CIPHER MESSAGE WITH COUNTER) - * instruction (0x80 is the decipher modifier bit) + * instruction */ -#define CPACF_KMCTR_QUERY 0x00 -#define CPACF_KMCTR_DEA_ENC 0x01 -#define CPACF_KMCTR_DEA_DEC 0x81 -#define CPACF_KMCTR_TDEA_128_ENC 0x02 -#define CPACF_KMCTR_TDEA_128_DEC 0x82 -#define CPACF_KMCTR_TDEA_192_ENC 0x03 -#define CPACF_KMCTR_TDEA_192_DEC 0x83 -#define CPACF_KMCTR_AES_128_ENC 0x12 -#define CPACF_KMCTR_AES_128_DEC 0x92 -#define CPACF_KMCTR_AES_192_ENC 0x13 -#define CPACF_KMCTR_AES_192_DEC 0x93 -#define CPACF_KMCTR_AES_256_ENC 0x14 -#define CPACF_KMCTR_AES_256_DEC 0x94 +#define CPACF_KMCTR_QUERY 0x00 +#define CPACF_KMCTR_DEA 0x01 +#define CPACF_KMCTR_TDEA_128 0x02 +#define CPACF_KMCTR_TDEA_192 0x03 +#define CPACF_KMCTR_AES_128 0x12 +#define CPACF_KMCTR_AES_192 0x13 +#define CPACF_KMCTR_AES_256 0x14 /* * Function codes for the KIMD (COMPUTE INTERMEDIATE MESSAGE DIGEST) - * instruction (0x80 is the decipher modifier bit) + * instruction */ #define CPACF_KIMD_QUERY 0x00 #define CPACF_KIMD_SHA_1 0x01 @@ -98,7 +82,7 @@ /* * Function codes for the KLMD (COMPUTE LAST MESSAGE DIGEST) - * instruction (0x80 is the decipher modifier bit) + * instruction */ #define CPACF_KLMD_QUERY 0x00 #define CPACF_KLMD_SHA_1 0x01 @@ -107,7 +91,7 @@ /* * function codes for the KMAC (COMPUTE MESSAGE AUTHENTICATION CODE) - * instruction (0x80 is the decipher modifier bit) + * instruction */ #define CPACF_KMAC_QUERY 0x00 #define CPACF_KMAC_DEA 0x01 @@ -116,12 +100,14 @@ /* * Function codes for the PPNO (PERFORM PSEUDORANDOM NUMBER OPERATION) - * instruction (0x80 is the decipher modifier bit) + * instruction */ #define CPACF_PPNO_QUERY 0x00 #define CPACF_PPNO_SHA512_DRNG_GEN 0x03 #define CPACF_PPNO_SHA512_DRNG_SEED 0x83 +typedef struct { unsigned char bytes[16]; } cpacf_mask_t; + /** * cpacf_query() - check if a specific CPACF function is available * @opcode: the opcode of the crypto instruction @@ -132,55 +118,66 @@ * * Returns 1 if @func is available for @opcode, 0 otherwise */ -static inline void __cpacf_query(unsigned int opcode, unsigned char *status) +static inline void __cpacf_query(unsigned int opcode, cpacf_mask_t *mask) { - typedef struct { unsigned char _[16]; } status_type; register unsigned long r0 asm("0") = 0; /* query function */ - register unsigned long r1 asm("1") = (unsigned long) status; + register unsigned long r1 asm("1") = (unsigned long) mask; asm volatile( " spm 0\n" /* pckmo doesn't change the cc */ /* Parameter registers are ignored, but may not be 0 */ "0: .insn rrf,%[opc] << 16,2,2,2,0\n" " brc 1,0b\n" /* handle partial completion */ - : "=m" (*(status_type *) status) + : "=m" (*mask) : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (opcode) : "cc"); } -static inline int cpacf_query(unsigned int opcode, unsigned int func) +static inline int __cpacf_check_opcode(unsigned int opcode) { - unsigned char status[16]; - switch (opcode) { case CPACF_KMAC: case CPACF_KM: case CPACF_KMC: case CPACF_KIMD: case CPACF_KLMD: - if (!test_facility(17)) /* check for MSA */ - return 0; - break; + return test_facility(17); /* check for MSA */ case CPACF_PCKMO: - if (!test_facility(76)) /* check for MSA3 */ - return 0; - break; + return test_facility(76); /* check for MSA3 */ case CPACF_KMF: case CPACF_KMO: case CPACF_PCC: case CPACF_KMCTR: - if (!test_facility(77)) /* check for MSA4 */ - return 0; - break; + return test_facility(77); /* check for MSA4 */ case CPACF_PPNO: - if (!test_facility(57)) /* check for MSA5 */ - return 0; - break; + return test_facility(57); /* check for MSA5 */ default: BUG(); } - __cpacf_query(opcode, status); - return (status[func >> 3] & (0x80 >> (func & 7))) != 0; +} + +static inline int cpacf_query(unsigned int opcode, cpacf_mask_t *mask) +{ + if (__cpacf_check_opcode(opcode)) { + __cpacf_query(opcode, mask); + return 1; + } + memset(mask, 0, sizeof(*mask)); + return 0; +} + +static inline int cpacf_test_func(cpacf_mask_t *mask, unsigned int func) +{ + return (mask->bytes[func >> 3] & (0x80 >> (func & 7))) != 0; +} + +static inline int cpacf_query_func(unsigned int opcode, unsigned int func) +{ + cpacf_mask_t mask; + + if (cpacf_query(opcode, &mask)) + return cpacf_test_func(&mask, func); + return 0; } /** @@ -194,7 +191,7 @@ static inline int cpacf_query(unsigned int opcode, unsigned int func) * Returns 0 for the query func, number of processed bytes for * encryption/decryption funcs */ -static inline int cpacf_km(long func, void *param, +static inline int cpacf_km(unsigned long func, void *param, u8 *dest, const u8 *src, long src_len) { register unsigned long r0 asm("0") = (unsigned long) func; @@ -224,7 +221,7 @@ static inline int cpacf_km(long func, void *param, * Returns 0 for the query func, number of processed bytes for * encryption/decryption funcs */ -static inline int cpacf_kmc(long func, void *param, +static inline int cpacf_kmc(unsigned long func, void *param, u8 *dest, const u8 *src, long src_len) { register unsigned long r0 asm("0") = (unsigned long) func; @@ -250,11 +247,9 @@ static inline int cpacf_kmc(long func, void *param, * @param: address of parameter block; see POP for details on each func * @src: address of source memory area * @src_len: length of src operand in bytes - * - * Returns 0 for the query func, number of processed bytes for digest funcs */ -static inline int cpacf_kimd(long func, void *param, - const u8 *src, long src_len) +static inline void cpacf_kimd(unsigned long func, void *param, + const u8 *src, long src_len) { register unsigned long r0 asm("0") = (unsigned long) func; register unsigned long r1 asm("1") = (unsigned long) param; @@ -267,8 +262,6 @@ static inline int cpacf_kimd(long func, void *param, : [src] "+a" (r2), [len] "+d" (r3) : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KIMD) : "cc", "memory"); - - return src_len - r3; } /** @@ -277,11 +270,9 @@ static inline int cpacf_kimd(long func, void *param, * @param: address of parameter block; see POP for details on each func * @src: address of source memory area * @src_len: length of src operand in bytes - * - * Returns 0 for the query func, number of processed bytes for digest funcs */ -static inline int cpacf_klmd(long func, void *param, - const u8 *src, long src_len) +static inline void cpacf_klmd(unsigned long func, void *param, + const u8 *src, long src_len) { register unsigned long r0 asm("0") = (unsigned long) func; register unsigned long r1 asm("1") = (unsigned long) param; @@ -294,8 +285,6 @@ static inline int cpacf_klmd(long func, void *param, : [src] "+a" (r2), [len] "+d" (r3) : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KLMD) : "cc", "memory"); - - return src_len - r3; } /** @@ -308,7 +297,7 @@ static inline int cpacf_klmd(long func, void *param, * * Returns 0 for the query func, number of processed bytes for digest funcs */ -static inline int cpacf_kmac(long func, void *param, +static inline int cpacf_kmac(unsigned long func, void *param, const u8 *src, long src_len) { register unsigned long r0 asm("0") = (unsigned long) func; @@ -338,7 +327,7 @@ static inline int cpacf_kmac(long func, void *param, * Returns 0 for the query func, number of processed bytes for * encryption/decryption funcs */ -static inline int cpacf_kmctr(long func, void *param, u8 *dest, +static inline int cpacf_kmctr(unsigned long func, void *param, u8 *dest, const u8 *src, long src_len, u8 *counter) { register unsigned long r0 asm("0") = (unsigned long) func; @@ -368,13 +357,10 @@ static inline int cpacf_kmctr(long func, void *param, u8 *dest, * @dest_len: size of destination memory area in bytes * @seed: address of seed data * @seed_len: size of seed data in bytes - * - * Returns 0 for the query func, number of random bytes stored in - * dest buffer for generate function */ -static inline int cpacf_ppno(long func, void *param, - u8 *dest, long dest_len, - const u8 *seed, long seed_len) +static inline void cpacf_ppno(unsigned long func, void *param, + u8 *dest, long dest_len, + const u8 *seed, long seed_len) { register unsigned long r0 asm("0") = (unsigned long) func; register unsigned long r1 asm("1") = (unsigned long) param; @@ -390,8 +376,6 @@ static inline int cpacf_ppno(long func, void *param, : [fc] "d" (r0), [pba] "a" (r1), [seed] "a" (r4), [slen] "d" (r5), [opc] "i" (CPACF_PPNO) : "cc", "memory"); - - return dest_len - r3; } /** @@ -399,10 +383,8 @@ static inline int cpacf_ppno(long func, void *param, * instruction * @func: the function code passed to PCC; see CPACF_KM_xxx defines * @param: address of parameter block; see POP for details on each func - * - * Returns 0. */ -static inline int cpacf_pcc(long func, void *param) +static inline void cpacf_pcc(unsigned long func, void *param) { register unsigned long r0 asm("0") = (unsigned long) func; register unsigned long r1 asm("1") = (unsigned long) param; @@ -413,8 +395,6 @@ static inline int cpacf_pcc(long func, void *param) : : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_PCC) : "cc", "memory"); - - return 0; } #endif /* _ASM_S390_CPACF_H */ diff --git a/arch/s390/include/asm/facilities_src.h b/arch/s390/include/asm/facilities_src.h index 4917728e5828..3b758f66e48b 100644 --- a/arch/s390/include/asm/facilities_src.h +++ b/arch/s390/include/asm/facilities_src.h @@ -55,4 +55,28 @@ static struct facility_def facility_defs[] = { -1 /* END */ } }, + { + .name = "FACILITIES_KVM", + .bits = (int[]){ + 0, /* N3 instructions */ + 1, /* z/Arch mode installed */ + 2, /* z/Arch mode active */ + 3, /* DAT-enhancement */ + 4, /* idte segment table */ + 5, /* idte region table */ + 6, /* ASN-and-LX reuse */ + 7, /* stfle */ + 8, /* enhanced-DAT 1 */ + 9, /* sense-running-status */ + 10, /* conditional sske */ + 13, /* ipte-range */ + 14, /* nonquiescing key-setting */ + 73, /* transactional execution */ + 75, /* access-exception-fetch/store indication */ + 76, /* msa extension 3 */ + 77, /* msa extension 4 */ + 78, /* enhanced-DAT 2 */ + -1 /* END */ + } + }, }; diff --git a/arch/s390/include/asm/fpu/api.h b/arch/s390/include/asm/fpu/api.h index 6aba6fc406ad..02124d66bfb5 100644 --- a/arch/s390/include/asm/fpu/api.h +++ b/arch/s390/include/asm/fpu/api.h @@ -64,18 +64,18 @@ static inline int test_fp_ctl(u32 fpc) return rc; } -#define KERNEL_VXR_V0V7 1 -#define KERNEL_VXR_V8V15 2 -#define KERNEL_VXR_V16V23 4 -#define KERNEL_VXR_V24V31 8 -#define KERNEL_FPR 16 -#define KERNEL_FPC 256 +#define KERNEL_FPC 1 +#define KERNEL_VXR_V0V7 2 +#define KERNEL_VXR_V8V15 4 +#define KERNEL_VXR_V16V23 8 +#define KERNEL_VXR_V24V31 16 #define KERNEL_VXR_LOW (KERNEL_VXR_V0V7|KERNEL_VXR_V8V15) #define KERNEL_VXR_MID (KERNEL_VXR_V8V15|KERNEL_VXR_V16V23) #define KERNEL_VXR_HIGH (KERNEL_VXR_V16V23|KERNEL_VXR_V24V31) -#define KERNEL_FPU_MASK (KERNEL_VXR_LOW|KERNEL_VXR_HIGH|KERNEL_FPR) +#define KERNEL_VXR (KERNEL_VXR_LOW|KERNEL_VXR_HIGH) +#define KERNEL_FPR (KERNEL_FPC|KERNEL_VXR_V0V7) struct kernel_fpu; @@ -87,18 +87,28 @@ struct kernel_fpu; * Prefer using the kernel_fpu_begin()/kernel_fpu_end() pair of functions. */ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags); -void __kernel_fpu_end(struct kernel_fpu *state); +void __kernel_fpu_end(struct kernel_fpu *state, u32 flags); static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) { preempt_disable(); - __kernel_fpu_begin(state, flags); + state->mask = S390_lowcore.fpu_flags; + if (!test_cpu_flag(CIF_FPU)) + /* Save user space FPU state and register contents */ + save_fpu_regs(); + else if (state->mask & flags) + /* Save FPU/vector register in-use by the kernel */ + __kernel_fpu_begin(state, flags); + S390_lowcore.fpu_flags |= flags; } -static inline void kernel_fpu_end(struct kernel_fpu *state) +static inline void kernel_fpu_end(struct kernel_fpu *state, u32 flags) { - __kernel_fpu_end(state); + S390_lowcore.fpu_flags = state->mask; + if (state->mask & flags) + /* Restore FPU/vector register in-use by the kernel */ + __kernel_fpu_end(state, flags); preempt_enable(); } diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index d79ba7cf75b0..7b93b78f423c 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -129,7 +129,8 @@ struct lowcore { __u8 pad_0x0390[0x0398-0x0390]; /* 0x0390 */ __u64 gmap; /* 0x0398 */ __u32 spinlock_lockval; /* 0x03a0 */ - __u8 pad_0x03a0[0x0400-0x03a4]; /* 0x03a4 */ + __u32 fpu_flags; /* 0x03a4 */ + __u8 pad_0x03a8[0x0400-0x03a8]; /* 0x03a8 */ /* Per cpu primary space access list */ __u32 paste[16]; /* 0x0400 */ diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index 6d39329c894b..bea785d7f853 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -12,6 +12,7 @@ typedef struct { struct list_head pgtable_list; spinlock_t gmap_lock; struct list_head gmap_list; + unsigned long gmap_asce; unsigned long asce; unsigned long asce_limit; unsigned long vdso_base; diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index c6a088c91aee..515fea5a3fc4 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -21,6 +21,7 @@ static inline int init_new_context(struct task_struct *tsk, INIT_LIST_HEAD(&mm->context.gmap_list); cpumask_clear(&mm->context.cpu_attach_mask); atomic_set(&mm->context.flush_count, 0); + mm->context.gmap_asce = 0; mm->context.flush_mm = 0; #ifdef CONFIG_PGSTE mm->context.alloc_pgste = page_table_allocate_pgste; diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index 0da91c4d30fd..6611f798d2be 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -11,6 +11,7 @@ #include <asm-generic/pci.h> #include <asm/pci_clp.h> #include <asm/pci_debug.h> +#include <asm/sclp.h> #define PCIBIOS_MIN_IO 0x1000 #define PCIBIOS_MIN_MEM 0x10000000 @@ -117,6 +118,7 @@ struct zpci_dev { spinlock_t iommu_bitmap_lock; unsigned long *iommu_bitmap; + unsigned long *lazy_bitmap; unsigned long iommu_size; unsigned long iommu_pages; unsigned int next_bit; @@ -216,6 +218,9 @@ void zpci_debug_init_device(struct zpci_dev *, const char *); void zpci_debug_exit_device(struct zpci_dev *); void zpci_debug_info(struct zpci_dev *, struct seq_file *); +/* Error reporting */ +int zpci_report_error(struct pci_dev *, struct zpci_report_error_header *); + #ifdef CONFIG_NUMA /* Returns the node based on PCI bus */ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 72c7f60bfe83..0362cd5fa187 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -874,35 +874,31 @@ static inline pte_t pte_mkhuge(pte_t pte) } #endif -static inline void __ptep_ipte(unsigned long address, pte_t *ptep) -{ - unsigned long pto = (unsigned long) ptep; - - /* Invalidation + global TLB flush for the pte */ - asm volatile( - " ipte %2,%3" - : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); -} +#define IPTE_GLOBAL 0 +#define IPTE_LOCAL 1 -static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep) +static inline void __ptep_ipte(unsigned long address, pte_t *ptep, int local) { unsigned long pto = (unsigned long) ptep; - /* Invalidation + local TLB flush for the pte */ + /* Invalidation + TLB flush for the pte */ asm volatile( - " .insn rrf,0xb2210000,%2,%3,0,1" - : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); + " .insn rrf,0xb2210000,%[r1],%[r2],0,%[m4]" + : "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address), + [m4] "i" (local)); } -static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep) +static inline void __ptep_ipte_range(unsigned long address, int nr, + pte_t *ptep, int local) { unsigned long pto = (unsigned long) ptep; - /* Invalidate a range of ptes + global TLB flush of the ptes */ + /* Invalidate a range of ptes + TLB flush of the ptes */ do { asm volatile( - " .insn rrf,0xb2210000,%2,%0,%1,0" - : "+a" (address), "+a" (nr) : "a" (pto) : "memory"); + " .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]" + : [r2] "+a" (address), [r3] "+a" (nr) + : [r1] "a" (pto), [m4] "i" (local) : "memory"); } while (nr != 255); } @@ -1239,53 +1235,33 @@ static inline void __pmdp_csp(pmd_t *pmdp) pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); } -static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp) -{ - unsigned long sto; - - sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t); - asm volatile( - " .insn rrf,0xb98e0000,%2,%3,0,0" - : "=m" (*pmdp) - : "m" (*pmdp), "a" (sto), "a" ((address & HPAGE_MASK)) - : "cc" ); -} - -static inline void __pudp_idte(unsigned long address, pud_t *pudp) -{ - unsigned long r3o; - - r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t); - r3o |= _ASCE_TYPE_REGION3; - asm volatile( - " .insn rrf,0xb98e0000,%2,%3,0,0" - : "=m" (*pudp) - : "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK)) - : "cc"); -} +#define IDTE_GLOBAL 0 +#define IDTE_LOCAL 1 -static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp) +static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp, int local) { unsigned long sto; sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t); asm volatile( - " .insn rrf,0xb98e0000,%2,%3,0,1" - : "=m" (*pmdp) - : "m" (*pmdp), "a" (sto), "a" ((address & HPAGE_MASK)) + " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]" + : "+m" (*pmdp) + : [r1] "a" (sto), [r2] "a" ((address & HPAGE_MASK)), + [m4] "i" (local) : "cc" ); } -static inline void __pudp_idte_local(unsigned long address, pud_t *pudp) +static inline void __pudp_idte(unsigned long address, pud_t *pudp, int local) { unsigned long r3o; r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t); r3o |= _ASCE_TYPE_REGION3; asm volatile( - " .insn rrf,0xb98e0000,%2,%3,0,1" - : "=m" (*pudp) - : "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK)) + " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]" + : "+m" (*pudp) + : [r1] "a" (r3o), [r2] "a" ((address & PUD_MASK)), + [m4] "i" (local) : "cc"); } diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index 1a691ef740cf..39846100682a 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -26,17 +26,6 @@ static inline void __tlb_flush_idte(unsigned long asce) : : "a" (2048), "a" (asce) : "cc"); } -/* - * Flush TLB entries for a specific ASCE on the local CPU - */ -static inline void __tlb_flush_idte_local(unsigned long asce) -{ - /* Local TLB flush for the mm */ - asm volatile( - " .insn rrf,0xb98e0000,0,%0,%1,1" - : : "a" (2048), "a" (asce) : "cc"); -} - #ifdef CONFIG_SMP void smp_ptlb_all(void); @@ -65,35 +54,33 @@ static inline void __tlb_flush_full(struct mm_struct *mm) /* Global TLB flush */ __tlb_flush_global(); /* Reset TLB flush mask */ - if (MACHINE_HAS_TLB_LC) - cpumask_copy(mm_cpumask(mm), - &mm->context.cpu_attach_mask); + cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask); } atomic_dec(&mm->context.flush_count); preempt_enable(); } -/* - * Flush TLB entries for a specific ASCE on all CPUs. Should never be used - * when more than one asce (e.g. gmap) ran on this mm. - */ -static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) +static inline void __tlb_flush_mm(struct mm_struct *mm) { + unsigned long gmap_asce; + + /* + * If the machine has IDTE we prefer to do a per mm flush + * on all cpus instead of doing a local flush if the mm + * only ran on the local cpu. + */ preempt_disable(); atomic_inc(&mm->context.flush_count); - if (MACHINE_HAS_TLB_LC && - cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { - __tlb_flush_idte_local(asce); + gmap_asce = READ_ONCE(mm->context.gmap_asce); + if (MACHINE_HAS_IDTE && gmap_asce != -1UL) { + if (gmap_asce) + __tlb_flush_idte(gmap_asce); + __tlb_flush_idte(mm->context.asce); } else { - if (MACHINE_HAS_IDTE) - __tlb_flush_idte(asce); - else - __tlb_flush_global(); - /* Reset TLB flush mask */ - if (MACHINE_HAS_TLB_LC) - cpumask_copy(mm_cpumask(mm), - &mm->context.cpu_attach_mask); + __tlb_flush_full(mm); } + /* Reset TLB flush mask */ + cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask); atomic_dec(&mm->context.flush_count); preempt_enable(); } @@ -112,36 +99,17 @@ static inline void __tlb_flush_kernel(void) /* * Flush TLB entries for a specific ASCE on all CPUs. */ -static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) +static inline void __tlb_flush_mm(struct mm_struct *mm) { - if (MACHINE_HAS_TLB_LC) - __tlb_flush_idte_local(asce); - else - __tlb_flush_local(); + __tlb_flush_local(); } static inline void __tlb_flush_kernel(void) { - if (MACHINE_HAS_TLB_LC) - __tlb_flush_idte_local(init_mm.context.asce); - else - __tlb_flush_local(); + __tlb_flush_local(); } #endif -static inline void __tlb_flush_mm(struct mm_struct * mm) -{ - /* - * If the machine has IDTE we prefer to do a per mm flush - * on all cpus instead of doing a local flush if the mm - * only ran on the local cpu. - */ - if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list)) - __tlb_flush_asce(mm, mm->context.asce); - else - __tlb_flush_full(mm); -} - static inline void __tlb_flush_mm_lazy(struct mm_struct * mm) { if (mm->context.flush_mm) { diff --git a/arch/s390/include/asm/vx-insn.h b/arch/s390/include/asm/vx-insn.h index 4a3135620f5e..49c24a2afce0 100644 --- a/arch/s390/include/asm/vx-insn.h +++ b/arch/s390/include/asm/vx-insn.h @@ -16,15 +16,13 @@ /* Macros to generate vector instruction byte code */ -#define REG_NUM_INVALID 255 - /* GR_NUM - Retrieve general-purpose register number * * @opd: Operand to store register number * @r64: String designation register in the format "%rN" */ .macro GR_NUM opd gr - \opd = REG_NUM_INVALID + \opd = 255 .ifc \gr,%r0 \opd = 0 .endif @@ -73,14 +71,11 @@ .ifc \gr,%r15 \opd = 15 .endif - .if \opd == REG_NUM_INVALID - .error "Invalid general-purpose register designation: \gr" + .if \opd == 255 + \opd = \gr .endif .endm -/* VX_R() - Macro to encode the VX_NUM into the instruction */ -#define VX_R(v) (v & 0x0F) - /* VX_NUM - Retrieve vector register number * * @opd: Operand to store register number @@ -88,11 +83,10 @@ * * The vector register number is used for as input number to the * instruction and, as well as, to compute the RXB field of the - * instruction. To encode the particular vector register number, - * use the VX_R(v) macro to extract the instruction opcode. + * instruction. */ .macro VX_NUM opd vxr - \opd = REG_NUM_INVALID + \opd = 255 .ifc \vxr,%v0 \opd = 0 .endif @@ -189,8 +183,8 @@ .ifc \vxr,%v31 \opd = 31 .endif - .if \opd == REG_NUM_INVALID - .error "Invalid vector register designation: \vxr" + .if \opd == 255 + \opd = \vxr .endif .endm @@ -251,7 +245,7 @@ /* VECTOR GENERATE BYTE MASK */ .macro VGBM vr imm2 VX_NUM v1, \vr - .word (0xE700 | (VX_R(v1) << 4)) + .word (0xE700 | ((v1&15) << 4)) .word \imm2 MRXBOPC 0, 0x44, v1 .endm @@ -267,7 +261,7 @@ VX_NUM v1, \v GR_NUM b2, "%r0" GR_NUM r3, \gr - .word 0xE700 | (VX_R(v1) << 4) | r3 + .word 0xE700 | ((v1&15) << 4) | r3 .word (b2 << 12) | (\disp) MRXBOPC \m, 0x22, v1 .endm @@ -284,12 +278,21 @@ VLVG \v, \gr, \index, 3 .endm +/* VECTOR LOAD REGISTER */ +.macro VLR v1, v2 + VX_NUM v1, \v1 + VX_NUM v2, \v2 + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word 0 + MRXBOPC 0, 0x56, v1, v2 +.endm + /* VECTOR LOAD */ .macro VL v, disp, index="%r0", base VX_NUM v1, \v GR_NUM x2, \index GR_NUM b2, \base - .word 0xE700 | (VX_R(v1) << 4) | x2 + .word 0xE700 | ((v1&15) << 4) | x2 .word (b2 << 12) | (\disp) MRXBOPC 0, 0x06, v1 .endm @@ -299,7 +302,7 @@ VX_NUM v1, \vr1 GR_NUM x2, \index GR_NUM b2, \base - .word 0xE700 | (VX_R(v1) << 4) | x2 + .word 0xE700 | ((v1&15) << 4) | x2 .word (b2 << 12) | (\disp) MRXBOPC \m3, \opc, v1 .endm @@ -319,7 +322,7 @@ /* VECTOR LOAD ELEMENT IMMEDIATE */ .macro VLEIx vr1, imm2, m3, opc VX_NUM v1, \vr1 - .word 0xE700 | (VX_R(v1) << 4) + .word 0xE700 | ((v1&15) << 4) .word \imm2 MRXBOPC \m3, \opc, v1 .endm @@ -341,7 +344,7 @@ GR_NUM r1, \gr GR_NUM b2, \base VX_NUM v3, \vr - .word 0xE700 | (r1 << 4) | VX_R(v3) + .word 0xE700 | (r1 << 4) | (v3&15) .word (b2 << 12) | (\disp) MRXBOPC \m, 0x21, v3 .endm @@ -363,7 +366,7 @@ VX_NUM v1, \vfrom VX_NUM v3, \vto GR_NUM b2, \base /* Base register */ - .word 0xE700 | (VX_R(v1) << 4) | VX_R(v3) + .word 0xE700 | ((v1&15) << 4) | (v3&15) .word (b2 << 12) | (\disp) MRXBOPC 0, 0x36, v1, v3 .endm @@ -373,7 +376,7 @@ VX_NUM v1, \vfrom VX_NUM v3, \vto GR_NUM b2, \base /* Base register */ - .word 0xE700 | (VX_R(v1) << 4) | VX_R(v3) + .word 0xE700 | ((v1&15) << 4) | (v3&15) .word (b2 << 12) | (\disp) MRXBOPC 0, 0x3E, v1, v3 .endm @@ -384,16 +387,16 @@ VX_NUM v2, \vr2 VX_NUM v3, \vr3 VX_NUM v4, \vr4 - .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2) - .word (VX_R(v3) << 12) - MRXBOPC VX_R(v4), 0x8C, v1, v2, v3, v4 + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) + MRXBOPC (v4&15), 0x8C, v1, v2, v3, v4 .endm /* VECTOR UNPACK LOGICAL LOW */ .macro VUPLL vr1, vr2, m3 VX_NUM v1, \vr1 VX_NUM v2, \vr2 - .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2) + .word 0xE700 | ((v1&15) << 4) | (v2&15) .word 0x0000 MRXBOPC \m3, 0xD4, v1, v2 .endm @@ -410,13 +413,23 @@ /* Vector integer instructions */ +/* VECTOR AND */ +.macro VN vr1, vr2, vr3 + VX_NUM v1, \vr1 + VX_NUM v2, \vr2 + VX_NUM v3, \vr3 + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) + MRXBOPC 0, 0x68, v1, v2, v3 +.endm + /* VECTOR EXCLUSIVE OR */ .macro VX vr1, vr2, vr3 VX_NUM v1, \vr1 VX_NUM v2, \vr2 VX_NUM v3, \vr3 - .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2) - .word (VX_R(v3) << 12) + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) MRXBOPC 0, 0x6D, v1, v2, v3 .endm @@ -425,8 +438,8 @@ VX_NUM v1, \vr1 VX_NUM v2, \vr2 VX_NUM v3, \vr3 - .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2) - .word (VX_R(v3) << 12) + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) MRXBOPC \m4, 0xB4, v1, v2, v3 .endm .macro VGFMB vr1, vr2, vr3 @@ -448,9 +461,9 @@ VX_NUM v2, \vr2 VX_NUM v3, \vr3 VX_NUM v4, \vr4 - .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2) - .word (VX_R(v3) << 12) | (\m5 << 8) - MRXBOPC VX_R(v4), 0xBC, v1, v2, v3, v4 + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) | (\m5 << 8) + MRXBOPC (v4&15), 0xBC, v1, v2, v3, v4 .endm .macro VGFMAB vr1, vr2, vr3, vr4 VGFMA \vr1, \vr2, \vr3, \vr4, 0 @@ -470,11 +483,78 @@ VX_NUM v1, \vr1 VX_NUM v2, \vr2 VX_NUM v3, \vr3 - .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2) - .word (VX_R(v3) << 12) + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) MRXBOPC 0, 0x7D, v1, v2, v3 .endm +/* VECTOR REPLICATE IMMEDIATE */ +.macro VREPI vr1, imm2, m3 + VX_NUM v1, \vr1 + .word 0xE700 | ((v1&15) << 4) + .word \imm2 + MRXBOPC \m3, 0x45, v1 +.endm +.macro VREPIB vr1, imm2 + VREPI \vr1, \imm2, 0 +.endm +.macro VREPIH vr1, imm2 + VREPI \vr1, \imm2, 1 +.endm +.macro VREPIF vr1, imm2 + VREPI \vr1, \imm2, 2 +.endm +.macro VREPIG vr1, imm2 + VREP \vr1, \imm2, 3 +.endm + +/* VECTOR ADD */ +.macro VA vr1, vr2, vr3, m4 + VX_NUM v1, \vr1 + VX_NUM v2, \vr2 + VX_NUM v3, \vr3 + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) + MRXBOPC \m4, 0xF3, v1, v2, v3 +.endm +.macro VAB vr1, vr2, vr3 + VA \vr1, \vr2, \vr3, 0 +.endm +.macro VAH vr1, vr2, vr3 + VA \vr1, \vr2, \vr3, 1 +.endm +.macro VAF vr1, vr2, vr3 + VA \vr1, \vr2, \vr3, 2 +.endm +.macro VAG vr1, vr2, vr3 + VA \vr1, \vr2, \vr3, 3 +.endm +.macro VAQ vr1, vr2, vr3 + VA \vr1, \vr2, \vr3, 4 +.endm + +/* VECTOR ELEMENT SHIFT RIGHT ARITHMETIC */ +.macro VESRAV vr1, vr2, vr3, m4 + VX_NUM v1, \vr1 + VX_NUM v2, \vr2 + VX_NUM v3, \vr3 + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) + MRXBOPC \m4, 0x7A, v1, v2, v3 +.endm + +.macro VESRAVB vr1, vr2, vr3 + VESRAV \vr1, \vr2, \vr3, 0 +.endm +.macro VESRAVH vr1, vr2, vr3 + VESRAV \vr1, \vr2, \vr3, 1 +.endm +.macro VESRAVF vr1, vr2, vr3 + VESRAV \vr1, \vr2, \vr3, 2 +.endm +.macro VESRAVG vr1, vr2, vr3 + VESRAV \vr1, \vr2, \vr3, 3 +.endm #endif /* __ASSEMBLY__ */ #endif /* __ASM_S390_VX_INSN_H */ diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild index 08fe6dad9026..cc44b09c25fc 100644 --- a/arch/s390/include/uapi/asm/Kbuild +++ b/arch/s390/include/uapi/asm/Kbuild @@ -6,6 +6,7 @@ header-y += bitsperlong.h header-y += byteorder.h header-y += chpid.h header-y += chsc.h +header-y += clp.h header-y += cmb.h header-y += dasd.h header-y += debug.h |