1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
#include <linux/bitops.h>
#undef find_first_zero_bit
#undef find_first_bit
static inline long
__find_first_zero_bit(const unsigned long * addr, unsigned long size)
{
long d0, d1, d2;
long res;
/*
* We must test the size in words, not in bits, because
* otherwise incoming sizes in the range -63..-1 will not run
* any scasq instructions, and then the flags used by the je
* instruction will have whatever random value was in place
* before. Nobody should call us like that, but
* find_next_zero_bit() does when offset and size are at the
* same word and it fails to find a zero itself.
*/
size += 63;
size >>= 6;
if (!size)
return 0;
asm volatile(
" repe; scasq\n"
" je 1f\n"
" xorq -8(%%rdi),%%rax\n"
" subq $8,%%rdi\n"
" bsfq %%rax,%%rdx\n"
"1: subq %[addr],%%rdi\n"
" shlq $3,%%rdi\n"
" addq %%rdi,%%rdx"
:"=d" (res), "=&c" (d0), "=&D" (d1), "=&a" (d2)
:"0" (0ULL), "1" (size), "2" (addr), "3" (-1ULL),
[addr] "S" (addr) : "memory");
/*
* Any register would do for [addr] above, but GCC tends to
* prefer rbx over rsi, even though rsi is readily available
* and doesn't have to be saved.
*/
return res;
}
/**
* find_first_zero_bit - find the first zero bit in a memory region
* @addr: The address to start the search at
* @size: The maximum size to search
*
* Returns the bit-number of the first zero bit, not the number of the byte
* containing a bit.
*/
long find_first_zero_bit(const unsigned long * addr, unsigned long size)
{
return __find_first_zero_bit (addr, size);
}
static inline long
__find_first_bit(const unsigned long * addr, unsigned long size)
{
long d0, d1;
long res;
/*
* We must test the size in words, not in bits, because
* otherwise incoming sizes in the range -63..-1 will not run
* any scasq instructions, and then the flags used by the jz
* instruction will have whatever random value was in place
* before. Nobody should call us like that, but
* find_next_bit() does when offset and size are at the same
* word and it fails to find a one itself.
*/
size += 63;
size >>= 6;
if (!size)
return 0;
asm volatile(
" repe; scasq\n"
" jz 1f\n"
" subq $8,%%rdi\n"
" bsfq (%%rdi),%%rax\n"
"1: subq %[addr],%%rdi\n"
" shlq $3,%%rdi\n"
" addq %%rdi,%%rax"
:"=a" (res), "=&c" (d0), "=&D" (d1)
:"0" (0ULL), "1" (size), "2" (addr),
[addr] "r" (addr) : "memory");
return res;
}
/**
* find_first_bit - find the first set bit in a memory region
* @addr: The address to start the search at
* @size: The maximum size to search
*
* Returns the bit-number of the first set bit, not the number of the byte
* containing a bit.
*/
long find_first_bit(const unsigned long * addr, unsigned long size)
{
return __find_first_bit(addr,size);
}
#include <linux/module.h>
EXPORT_SYMBOL(find_first_bit);
EXPORT_SYMBOL(find_first_zero_bit);
|
