1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
// SPDX-License-Identifier: GPL-2.0
#include <sys/mman.h>
#include <inttypes.h>
#include <asm/bug.h>
#include <errno.h>
#include <linux/ring_buffer.h>
#include <linux/perf_event.h>
#include <perf/mmap.h>
#include <internal/mmap.h>
#include <internal/lib.h>
#include <linux/kernel.h>
#include "internal.h"
void perf_mmap__init(struct perf_mmap *map, bool overwrite,
libperf_unmap_cb_t unmap_cb)
{
map->fd = -1;
map->overwrite = overwrite;
map->unmap_cb = unmap_cb;
refcount_set(&map->refcnt, 0);
}
size_t perf_mmap__mmap_len(struct perf_mmap *map)
{
return map->mask + 1 + page_size;
}
int perf_mmap__mmap(struct perf_mmap *map, struct perf_mmap_param *mp,
int fd, int cpu)
{
map->prev = 0;
map->mask = mp->mask;
map->base = mmap(NULL, perf_mmap__mmap_len(map), mp->prot,
MAP_SHARED, fd, 0);
if (map->base == MAP_FAILED) {
map->base = NULL;
return -1;
}
map->fd = fd;
map->cpu = cpu;
return 0;
}
void perf_mmap__munmap(struct perf_mmap *map)
{
if (map && map->base != NULL) {
munmap(map->base, perf_mmap__mmap_len(map));
map->base = NULL;
map->fd = -1;
refcount_set(&map->refcnt, 0);
}
if (map && map->unmap_cb)
map->unmap_cb(map);
}
void perf_mmap__get(struct perf_mmap *map)
{
refcount_inc(&map->refcnt);
}
void perf_mmap__put(struct perf_mmap *map)
{
BUG_ON(map->base && refcount_read(&map->refcnt) == 0);
if (refcount_dec_and_test(&map->refcnt))
perf_mmap__munmap(map);
}
static inline void perf_mmap__write_tail(struct perf_mmap *md, u64 tail)
{
ring_buffer_write_tail(md->base, tail);
}
u64 perf_mmap__read_head(struct perf_mmap *map)
{
return ring_buffer_read_head(map->base);
}
static bool perf_mmap__empty(struct perf_mmap *map)
{
struct perf_event_mmap_page *pc = map->base;
return perf_mmap__read_head(map) == map->prev && !pc->aux_size;
}
void perf_mmap__consume(struct perf_mmap *map)
{
if (!map->overwrite) {
u64 old = map->prev;
perf_mmap__write_tail(map, old);
}
if (refcount_read(&map->refcnt) == 1 && perf_mmap__empty(map))
perf_mmap__put(map);
}
static int overwrite_rb_find_range(void *buf, int mask, u64 *start, u64 *end)
{
struct perf_event_header *pheader;
u64 evt_head = *start;
int size = mask + 1;
pr_debug2("%s: buf=%p, start=%"PRIx64"\n", __func__, buf, *start);
pheader = (struct perf_event_header *)(buf + (*start & mask));
while (true) {
if (evt_head - *start >= (unsigned int)size) {
pr_debug("Finished reading overwrite ring buffer: rewind\n");
if (evt_head - *start > (unsigned int)size)
evt_head -= pheader->size;
*end = evt_head;
return 0;
}
pheader = (struct perf_event_header *)(buf + (evt_head & mask));
if (pheader->size == 0) {
pr_debug("Finished reading overwrite ring buffer: get start\n");
*end = evt_head;
return 0;
}
evt_head += pheader->size;
pr_debug3("move evt_head: %"PRIx64"\n", evt_head);
}
WARN_ONCE(1, "Shouldn't get here\n");
return -1;
}
/*
* Report the start and end of the available data in ringbuffer
*/
static int __perf_mmap__read_init(struct perf_mmap *md)
{
u64 head = perf_mmap__read_head(md);
u64 old = md->prev;
unsigned char *data = md->base + page_size;
unsigned long size;
md->start = md->overwrite ? head : old;
md->end = md->overwrite ? old : head;
if ((md->end - md->start) < md->flush)
return -EAGAIN;
size = md->end - md->start;
if (size > (unsigned long)(md->mask) + 1) {
if (!md->overwrite) {
WARN_ONCE(1, "failed to keep up with mmap data. (warn only once)\n");
md->prev = head;
perf_mmap__consume(md);
return -EAGAIN;
}
/*
* Backward ring buffer is full. We still have a chance to read
* most of data from it.
*/
if (overwrite_rb_find_range(data, md->mask, &md->start, &md->end))
return -EINVAL;
}
return 0;
}
int perf_mmap__read_init(struct perf_mmap *map)
{
/*
* Check if event was unmapped due to a POLLHUP/POLLERR.
*/
if (!refcount_read(&map->refcnt))
return -ENOENT;
return __perf_mmap__read_init(map);
}
|
