1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
|
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
/*
* Copyright(c) 2018 Intel Corporation.
*
*/
#include "hfi.h"
#include "verbs.h"
#include "tid_rdma.h"
/*
* J_KEY for kernel contexts when TID RDMA is used.
* See generate_jkey() in hfi.h for more information.
*/
#define TID_RDMA_JKEY 32
#define HFI1_KERNEL_MIN_JKEY HFI1_ADMIN_JKEY_RANGE
#define HFI1_KERNEL_MAX_JKEY (2 * HFI1_ADMIN_JKEY_RANGE - 1)
#define TID_RDMA_MAX_READ_SEGS_PER_REQ 6
#define TID_RDMA_MAX_WRITE_SEGS_PER_REQ 4
#define TID_OPFN_QP_CTXT_MASK 0xff
#define TID_OPFN_QP_CTXT_SHIFT 56
#define TID_OPFN_QP_KDETH_MASK 0xff
#define TID_OPFN_QP_KDETH_SHIFT 48
#define TID_OPFN_MAX_LEN_MASK 0x7ff
#define TID_OPFN_MAX_LEN_SHIFT 37
#define TID_OPFN_TIMEOUT_MASK 0x1f
#define TID_OPFN_TIMEOUT_SHIFT 32
#define TID_OPFN_RESERVED_MASK 0x3f
#define TID_OPFN_RESERVED_SHIFT 26
#define TID_OPFN_URG_MASK 0x1
#define TID_OPFN_URG_SHIFT 25
#define TID_OPFN_VER_MASK 0x7
#define TID_OPFN_VER_SHIFT 22
#define TID_OPFN_JKEY_MASK 0x3f
#define TID_OPFN_JKEY_SHIFT 16
#define TID_OPFN_MAX_READ_MASK 0x3f
#define TID_OPFN_MAX_READ_SHIFT 10
#define TID_OPFN_MAX_WRITE_MASK 0x3f
#define TID_OPFN_MAX_WRITE_SHIFT 4
/*
* OPFN TID layout
*
* 63 47 31 15
* NNNNNNNNKKKKKKKK MMMMMMMMMMMTTTTT DDDDDDUVVVJJJJJJ RRRRRRWWWWWWCCCC
* 3210987654321098 7654321098765432 1098765432109876 5432109876543210
* N - the context Number
* K - the Kdeth_qp
* M - Max_len
* T - Timeout
* D - reserveD
* V - version
* U - Urg capable
* J - Jkey
* R - max_Read
* W - max_Write
* C - Capcode
*/
static u64 tid_rdma_opfn_encode(struct tid_rdma_params *p)
{
return
(((u64)p->qp & TID_OPFN_QP_CTXT_MASK) <<
TID_OPFN_QP_CTXT_SHIFT) |
((((u64)p->qp >> 16) & TID_OPFN_QP_KDETH_MASK) <<
TID_OPFN_QP_KDETH_SHIFT) |
(((u64)((p->max_len >> PAGE_SHIFT) - 1) &
TID_OPFN_MAX_LEN_MASK) << TID_OPFN_MAX_LEN_SHIFT) |
(((u64)p->timeout & TID_OPFN_TIMEOUT_MASK) <<
TID_OPFN_TIMEOUT_SHIFT) |
(((u64)p->urg & TID_OPFN_URG_MASK) << TID_OPFN_URG_SHIFT) |
(((u64)p->jkey & TID_OPFN_JKEY_MASK) << TID_OPFN_JKEY_SHIFT) |
(((u64)p->max_read & TID_OPFN_MAX_READ_MASK) <<
TID_OPFN_MAX_READ_SHIFT) |
(((u64)p->max_write & TID_OPFN_MAX_WRITE_MASK) <<
TID_OPFN_MAX_WRITE_SHIFT);
}
static void tid_rdma_opfn_decode(struct tid_rdma_params *p, u64 data)
{
p->max_len = (((data >> TID_OPFN_MAX_LEN_SHIFT) &
TID_OPFN_MAX_LEN_MASK) + 1) << PAGE_SHIFT;
p->jkey = (data >> TID_OPFN_JKEY_SHIFT) & TID_OPFN_JKEY_MASK;
p->max_write = (data >> TID_OPFN_MAX_WRITE_SHIFT) &
TID_OPFN_MAX_WRITE_MASK;
p->max_read = (data >> TID_OPFN_MAX_READ_SHIFT) &
TID_OPFN_MAX_READ_MASK;
p->qp =
((((data >> TID_OPFN_QP_KDETH_SHIFT) & TID_OPFN_QP_KDETH_MASK)
<< 16) |
((data >> TID_OPFN_QP_CTXT_SHIFT) & TID_OPFN_QP_CTXT_MASK));
p->urg = (data >> TID_OPFN_URG_SHIFT) & TID_OPFN_URG_MASK;
p->timeout = (data >> TID_OPFN_TIMEOUT_SHIFT) & TID_OPFN_TIMEOUT_MASK;
}
void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p)
{
struct hfi1_qp_priv *priv = qp->priv;
p->qp = (kdeth_qp << 16) | priv->rcd->ctxt;
p->max_len = TID_RDMA_MAX_SEGMENT_SIZE;
p->jkey = priv->rcd->jkey;
p->max_read = TID_RDMA_MAX_READ_SEGS_PER_REQ;
p->max_write = TID_RDMA_MAX_WRITE_SEGS_PER_REQ;
p->timeout = qp->timeout;
p->urg = is_urg_masked(priv->rcd);
}
bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data)
{
struct hfi1_qp_priv *priv = qp->priv;
*data = tid_rdma_opfn_encode(&priv->tid_rdma.local);
return true;
}
bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data)
{
struct hfi1_qp_priv *priv = qp->priv;
struct tid_rdma_params *remote, *old;
bool ret = true;
old = rcu_dereference_protected(priv->tid_rdma.remote,
lockdep_is_held(&priv->opfn.lock));
data &= ~0xfULL;
/*
* If data passed in is zero, return true so as not to continue the
* negotiation process
*/
if (!data || !HFI1_CAP_IS_KSET(TID_RDMA))
goto null;
/*
* If kzalloc fails, return false. This will result in:
* * at the requester a new OPFN request being generated to retry
* the negotiation
* * at the responder, 0 being returned to the requester so as to
* disable TID RDMA at both the requester and the responder
*/
remote = kzalloc(sizeof(*remote), GFP_ATOMIC);
if (!remote) {
ret = false;
goto null;
}
tid_rdma_opfn_decode(remote, data);
priv->tid_timer_timeout_jiffies =
usecs_to_jiffies((((4096UL * (1UL << remote->timeout)) /
1000UL) << 3) * 7);
rcu_assign_pointer(priv->tid_rdma.remote, remote);
/*
* A TID RDMA READ request's segment size is not equal to
* remote->max_len only when the request's data length is smaller
* than remote->max_len. In that case, there will be only one segment.
* Therefore, when priv->pkts_ps is used to calculate req->cur_seg
* during retry, it will lead to req->cur_seg = 0, which is exactly
* what is expected.
*/
priv->pkts_ps = (u16)rvt_div_mtu(qp, remote->max_len);
priv->timeout_shift = ilog2(priv->pkts_ps - 1) + 1;
goto free;
null:
RCU_INIT_POINTER(priv->tid_rdma.remote, NULL);
priv->timeout_shift = 0;
free:
if (old)
kfree_rcu(old, rcu_head);
return ret;
}
bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data)
{
bool ret;
ret = tid_rdma_conn_reply(qp, *data);
*data = 0;
/*
* If tid_rdma_conn_reply() returns error, set *data as 0 to indicate
* TID RDMA could not be enabled. This will result in TID RDMA being
* disabled at the requester too.
*/
if (ret)
(void)tid_rdma_conn_req(qp, data);
return ret;
}
void tid_rdma_conn_error(struct rvt_qp *qp)
{
struct hfi1_qp_priv *priv = qp->priv;
struct tid_rdma_params *old;
old = rcu_dereference_protected(priv->tid_rdma.remote,
lockdep_is_held(&priv->opfn.lock));
RCU_INIT_POINTER(priv->tid_rdma.remote, NULL);
if (old)
kfree_rcu(old, rcu_head);
}
/* This is called at context initialization time */
int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit)
{
if (reinit)
return 0;
BUILD_BUG_ON(TID_RDMA_JKEY < HFI1_KERNEL_MIN_JKEY);
BUILD_BUG_ON(TID_RDMA_JKEY > HFI1_KERNEL_MAX_JKEY);
rcd->jkey = TID_RDMA_JKEY;
hfi1_set_ctxt_jkey(rcd->dd, rcd, rcd->jkey);
return 0;
}
/**
* qp_to_rcd - determine the receive context used by a qp
* @qp - the qp
*
* This routine returns the receive context associated
* with a a qp's qpn.
*
* Returns the context.
*/
static struct hfi1_ctxtdata *qp_to_rcd(struct rvt_dev_info *rdi,
struct rvt_qp *qp)
{
struct hfi1_ibdev *verbs_dev = container_of(rdi,
struct hfi1_ibdev,
rdi);
struct hfi1_devdata *dd = container_of(verbs_dev,
struct hfi1_devdata,
verbs_dev);
unsigned int ctxt;
if (qp->ibqp.qp_num == 0)
ctxt = 0;
else
ctxt = ((qp->ibqp.qp_num >> dd->qos_shift) %
(dd->n_krcv_queues - 1)) + 1;
return dd->rcd[ctxt];
}
int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp,
struct ib_qp_init_attr *init_attr)
{
struct hfi1_qp_priv *qpriv = qp->priv;
qpriv->rcd = qp_to_rcd(rdi, qp);
return 0;
}
|
