#include "../util/tevent_ntstatus.h"
#include "../lib/tsocket/tsocket.h"
+#include "lib/util/util_net.h" //TODO
+
#ifdef SMB_TRANSPORT_ENABLE_RDMA
#include <rdma/rdma_cma_abi.h>
#include <rdma/rdma_cma.h>
enum rdma_cm_event_type expected_event;
struct rdma_cm_event *cm_event;
} rdma;
+ struct {
+ struct ibv_pd *pd;
+ struct ibv_comp_channel *comp_channel;
+ struct tevent_fd *fde_channel;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ } ibv;
};
static int smb_direct_transport_destructor(struct smb_direct_transport *t);
-static struct smb_direct_transport *smb_direct_transport_create(TALLOC_CTX *mem_ctx)
+struct smb_direct_transport *smb_direct_transport_create(TALLOC_CTX *mem_ctx);
+
+struct smb_direct_transport *smb_direct_transport_create(TALLOC_CTX *mem_ctx)
{
struct smb_direct_transport *t;
int ret;
static int smb_direct_transport_destructor(struct smb_direct_transport *t)
{
+ TALLOC_FREE(t->ibv.fde_channel);
TALLOC_FREE(t->rdma.fde_channel);
+ if (t->ibv.qp != NULL) {
+ ibv_destroy_qp(t->ibv.qp);
+ t->ibv.qp = NULL;
+ }
+
+ if (t->ibv.cq != NULL) {
+ ibv_destroy_cq(t->ibv.cq);
+ t->ibv.cq = NULL;
+ }
+
+ if (t->ibv.comp_channel != NULL) {
+ ibv_destroy_comp_channel(t->ibv.comp_channel);
+ t->ibv.comp_channel = NULL;
+ }
+
+ if (t->ibv.pd != NULL) {
+ ibv_dealloc_pd(t->ibv.pd);
+ t->ibv.pd = NULL;
+ }
+
if (t->rdma.cm_event != NULL) {
rdma_ack_cm_event(t->rdma.cm_event);
t->rdma.cm_event = NULL;
struct smb_direct_transport *t;
};
-static struct tevent_req *smb_direct_rdma_connect_send(TALLOC_CTX *mem_ctx,
+struct tevent_req *smb_direct_rdma_connect_send(TALLOC_CTX *mem_ctx,
struct tevent_context *ev,
struct smb_direct_transport *transport,
+ const struct sockaddr_storage *addr,
+ struct tsocket_address *local_addr,
+ struct tsocket_address *remote_addr);
+NTSTATUS smb_direct_rdma_connect_recv(struct tevent_req *req);
+
+static void smb_direct_rdma_connect_handler(struct tevent_context *ev,
+ struct tevent_fd *fde,
+ uint16_t flags,
+ void *private_data);
+
+struct tevent_req *smb_direct_rdma_connect_send(TALLOC_CTX *mem_ctx,
+ struct tevent_context *ev,
+ struct smb_direct_transport *transport,
+ const struct sockaddr_storage *_addr,
struct tsocket_address *local_addr,
struct tsocket_address *remote_addr)
{
struct tevent_req *req;
struct smb_direct_rdma_connect_state *state;
int ret;
- struct sockaddr *src_addr, *dst_addr;
+ struct sockaddr_storage addr = *_addr;
+ struct sockaddr *src_addr = NULL, *dst_addr = &addr;
+
+ set_sockaddr_port(dst_addr, 5445);
req = tevent_req_create(mem_ctx, &state,
struct smb_direct_rdma_connect_state);
- if (!req) {
+ if (req == NULL) {
return NULL;
}
state->t = transport;
-/*
- * ret = smb_direct_transport_setup_ev(transport, ev);
- if (ret != 0) {
-
+ transport->rdma.fde_channel = tevent_add_fd(ev, transport,
+ transport->rdma.cm_channel->fd,
+ TEVENT_FD_READ,
+ smb_direct_rdma_connect_handler,
+ req);
+ if (tevent_req_nomem(transport->rdma.fde_channel, req)) {
+ return tevent_req_post(req, ev);
}
-*/
+
+ errno = 0;
ret = rdma_resolve_addr(state->t->rdma.cm_id,
src_addr, dst_addr,
5000);
if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
}
state->t->rdma.expected_event = RDMA_CM_EVENT_ADDR_RESOLVED;
struct smb_direct_rdma_connect_state *state =
tevent_req_data(req,
struct smb_direct_rdma_connect_state);
+ struct ibv_qp_init_attr init_attr;
+ struct rdma_conn_param conn_param;
int ret;
+ errno = 0;
+
ret = rdma_get_cm_event(state->t->rdma.cm_channel,
&state->t->rdma.cm_event);
if (ret != 0) {
-
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
}
if (state->t->rdma.cm_event->status != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
}
if (state->t->rdma.cm_event->event != state->t->rdma.expected_event) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
}
switch (state->t->rdma.cm_event->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
+ errno = 0;
ret = rdma_resolve_route(state->t->rdma.cm_id, 5000);
if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
}
+ state->t->rdma.expected_event = RDMA_CM_EVENT_ROUTE_RESOLVED;
break;
case RDMA_CM_EVENT_ADDR_ERROR:
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ errno = 0;
+ ret = 0;
+ state->t->ibv.pd = ibv_alloc_pd(state->t->rdma.cm_id->verbs);
+ if (state->t->ibv.pd == NULL) {
+
+ }
+
+ state->t->ibv.comp_channel = ibv_create_comp_channel(state->t->rdma.cm_id->verbs);
+ if (state->t->ibv.comp_channel == NULL) {
+ }
+
+ ZERO_STRUCT(init_attr);
+ init_attr.cap.max_send_wr = 16;
+ init_attr.cap.max_recv_wr = 2;
+ init_attr.cap.max_recv_sge = 1;
+ init_attr.cap.max_send_sge = 1;
+ init_attr.qp_type = IBV_QPT_RC;
+
+ state->t->ibv.cq = ibv_create_cq(state->t->rdma.cm_id->verbs,
+ init_attr.cap.max_send_wr * 2,
+ state->t,
+ state->t->ibv.comp_channel,
+ 0);
+ if (state->t->ibv.cq == NULL) {
+
+ }
+ init_attr.send_cq = state->t->ibv.cq;
+ init_attr.recv_cq = state->t->ibv.cq;
+
+ errno = 0;
+ ret = ibv_req_notify_cq(state->t->ibv.cq, 0);
+ if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+
+ }
+
+
+ errno = 0;
+ ret = rdma_create_qp(state->t->rdma.cm_id, state->t->ibv.pd,
+ &init_attr);
+ if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+
+ }
+ state->t->ibv.qp = state->t->rdma.cm_id->qp;
+
+ ZERO_STRUCT(conn_param);
+ conn_param.responder_resources = 1;
+ conn_param.initiator_depth = 1;
+ conn_param.retry_count = 10;
+
+ errno = 0;
+ ret = rdma_connect(state->t->rdma.cm_id, &conn_param);
+ if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+
+ }
+ state->t->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED;
+ break;
+
+ case RDMA_CM_EVENT_ESTABLISHED:
+ errno = 0;
+ ret = 0;
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+
+ state->t->rdma.expected_event = RDMA_CM_EVENT_DISCONNECTED;
+ TALLOC_FREE(state->t->rdma.fde_channel);
+ tevent_req_done(req);
+ break;
case RDMA_CM_EVENT_ROUTE_ERROR:
case RDMA_CM_EVENT_CONNECT_REQUEST:
case RDMA_CM_EVENT_CONNECT_ERROR:
case RDMA_CM_EVENT_UNREACHABLE:
case RDMA_CM_EVENT_REJECTED:
- case RDMA_CM_EVENT_ESTABLISHED:
case RDMA_CM_EVENT_DISCONNECTED:
case RDMA_CM_EVENT_DEVICE_REMOVAL:
case RDMA_CM_EVENT_MULTICAST_JOIN:
}
}
-static NTSTATUS smb_transport_direct_rdma_connect_recv(struct tevent_req *req)
+NTSTATUS smb_direct_rdma_connect_recv(struct tevent_req *req)
{
struct smb_direct_rdma_connect_state *state =
tevent_req_data(req,
struct smb_direct_rdma_connect_state);
NTSTATUS status;
+ TALLOC_FREE(state->t->rdma.fde_channel);
+
if (tevent_req_is_nterror(req, &status)) {
tevent_req_received(req);
return status;
}
tevent_req_received(req);
- return NT_STATUS_NOT_IMPLEMENTED;
+ return NT_STATUS_OK;
+}
+
+struct smb_direct_negotiate_state {
+ struct smb_direct_transport *t;
+ struct {
+ uint8_t buffer[0x14];
+ struct ibv_mr *mr;
+/* uint8_t buffer2[0x14];
+ struct ibv_mr *mr2;
+ struct ibv_sge sge[2];
+*/ struct ibv_sge sge[1];
+ struct ibv_send_wr wr;
+ } req;
+ struct {
+ uint8_t buffer[0x1C];
+ struct ibv_mr *mr;
+ struct ibv_sge sge[1];
+ struct ibv_recv_wr wr;
+ } rep;
+};
+
+struct tevent_req *smb_direct_negotiate_send(TALLOC_CTX *mem_ctx,
+ struct tevent_context *ev,
+ struct smb_direct_transport *transport);
+NTSTATUS smb_direct_negotiate_recv(struct tevent_req *req);
+
+static void smb_direct_negotiate_rdma_handler(struct tevent_context *ev,
+ struct tevent_fd *fde,
+ uint16_t flags,
+ void *private_data);
+static void smb_direct_negotiate_ibv_handler(struct tevent_context *ev,
+ struct tevent_fd *fde,
+ uint16_t flags,
+ void *private_data);
+
+struct tevent_req *smb_direct_negotiate_send(TALLOC_CTX *mem_ctx,
+ struct tevent_context *ev,
+ struct smb_direct_transport *transport)
+{
+ struct tevent_req *req;
+ struct smb_direct_negotiate_state *state;
+ struct ibv_recv_wr *bad_recv_wr = NULL;
+ struct ibv_send_wr *bad_send_wr = NULL;
+ int ret;
+
+ req = tevent_req_create(mem_ctx, &state,
+ struct smb_direct_negotiate_state);
+ if (req == NULL) {
+ return NULL;
+ }
+ state->t = transport;
+
+ transport->ibv.fde_channel = tevent_add_fd(ev, transport,
+ transport->ibv.comp_channel->fd,
+ TEVENT_FD_READ,
+ smb_direct_negotiate_ibv_handler,
+ req);
+ if (tevent_req_nomem(transport->ibv.fde_channel, req)) {
+ return tevent_req_post(req, ev);
+ }
+ transport->rdma.fde_channel = tevent_add_fd(ev, transport,
+ transport->rdma.cm_channel->fd,
+ TEVENT_FD_READ,
+ smb_direct_negotiate_rdma_handler,
+ req);
+ if (tevent_req_nomem(transport->rdma.fde_channel, req)) {
+ return tevent_req_post(req, ev);
+ }
+
+ SSVAL(state->req.buffer, 0x00, 0x0100);
+ SSVAL(state->req.buffer, 0x02, 0x0100);
+ SSVAL(state->req.buffer, 0x04, 0x0000);
+ SSVAL(state->req.buffer, 0x06, 0x000A);
+ SIVAL(state->req.buffer, 0x08, 0x00000400);
+ SIVAL(state->req.buffer, 0x0C, 0x00000400);
+ SIVAL(state->req.buffer, 0x10, 0x00020000);
+
+ state->req.mr = ibv_reg_mr(transport->ibv.pd,
+ state->req.buffer,
+ sizeof(state->req.buffer),
+ IBV_ACCESS_LOCAL_WRITE);
+ if (tevent_req_nomem(state->req.mr, req)) {
+ return tevent_req_post(req, ev);
+ }
+
+/* state->req.mr2 = ibv_reg_mr(transport->ibv.pd,
+ state->req.buffer2,
+ sizeof(state->req.buffer2),
+ IBV_ACCESS_LOCAL_WRITE);
+ if (tevent_req_nomem(state->req.mr2, req)) {
+ return tevent_req_post(req, ev);
+ }
+*/ state->req.sge[0].addr = (uint64_t) (uintptr_t) state->req.buffer;
+ state->req.sge[0].length = sizeof(state->req.buffer);
+ state->req.sge[0].lkey = state->req.mr->lkey;
+/* state->req.sge[1].addr = (uint64_t) (uintptr_t) state->req.buffer2;
+ state->req.sge[1].length = sizeof(state->req.buffer2);
+ state->req.sge[1].lkey = state->req.mr2->lkey;
+*/ state->req.wr.opcode = IBV_WR_SEND;
+ state->req.wr.send_flags = IBV_SEND_SIGNALED;
+ state->req.wr.sg_list = state->req.sge;
+ state->req.wr.num_sge = ARRAY_SIZE(state->req.sge);
+
+ state->rep.mr = ibv_reg_mr(transport->ibv.pd,
+ state->rep.buffer,
+ sizeof(state->rep.buffer),
+ 0);
+ if (tevent_req_nomem(state->rep.mr, req)) {
+ return tevent_req_post(req, ev);
+ }
+
+ state->rep.sge[0].addr = (uint64_t) (uintptr_t) state->rep.buffer;
+ state->rep.sge[0].length = sizeof(state->rep.buffer);;
+ state->rep.sge[0].lkey = state->rep.mr->lkey;
+ state->rep.wr.sg_list = state->rep.sge;
+ state->rep.wr.num_sge = ARRAY_SIZE(state->rep.sge);
+
+ errno = 0;
+ ret = ibv_post_recv(transport->ibv.qp, &state->rep.wr, &bad_recv_wr);
+ if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+
+ }
+
+ errno = 0;
+ ret = ibv_post_send(transport->ibv.qp, &state->req.wr, &bad_send_wr);
+ if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+
+ }
+
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+ return req;
}
+static void smb_direct_negotiate_ibv_handler(struct tevent_context *ev,
+ struct tevent_fd *fde,
+ uint16_t flags,
+ void *private_data)
+{
+ struct tevent_req *req =
+ talloc_get_type_abort(private_data,
+ struct tevent_req);
+ struct smb_direct_negotiate_state *state =
+ tevent_req_data(req,
+ struct smb_direct_negotiate_state);
+ struct ibv_cq *cq = NULL;
+ void *cq_context = NULL;
+ struct ibv_wc wc;
+ struct ibv_recv_wr *bad_wr;
+ int ret;
+
+ errno = 0;
+
+ ret = ibv_get_cq_event(state->t->ibv.comp_channel,
+ &cq, &cq_context);
+ if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+ }
+
+ if (cq != state->t->ibv.cq) {
+ }
+ if (cq_context != state->t) {
+ }
+
+ ret = ibv_req_notify_cq(state->t->ibv.cq, 0);
+ if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+ }
+
+ ret = ibv_poll_cq(state->t->ibv.cq, 1, &wc);
+ if (ret != 1) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+ }
+
+ ret = 0;
+
+ if (wc.status != IBV_WC_SUCCESS) {
+ }
+
+ switch (wc.opcode) {
+ case IBV_WC_SEND:
+ break;
+ case IBV_WC_RDMA_WRITE:
+ break;
+ case IBV_WC_RDMA_READ:
+ break;
+ case IBV_WC_RECV:
+ //ret = ibv_post_recv(cb->qp, &cb->rq_wr, &bad_wr);
+ break;
+ default:
+ break;
+ }
+
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+ ibv_ack_cq_events(state->t->ibv.cq, 1);
+}
+
+static void smb_direct_negotiate_rdma_handler(struct tevent_context *ev,
+ struct tevent_fd *fde,
+ uint16_t flags,
+ void *private_data)
+{
+ struct tevent_req *req =
+ talloc_get_type_abort(private_data,
+ struct tevent_req);
+ struct smb_direct_negotiate_state *state =
+ tevent_req_data(req,
+ struct smb_direct_negotiate_state);
+ int ret;
+
+ errno = 0;
+
+ ret = rdma_get_cm_event(state->t->rdma.cm_channel,
+ &state->t->rdma.cm_event);
+ if (ret != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+ }
+
+ if (state->t->rdma.cm_event->status != 0) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+
+ }
+
+ if (state->t->rdma.cm_event->event != state->t->rdma.expected_event) {
+ DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+ __location__, __FUNCTION__, ret, errno));
+
+ }
+
+ switch (state->t->rdma.cm_event->event) {
+ case RDMA_CM_EVENT_ADDR_RESOLVED:
+ case RDMA_CM_EVENT_ADDR_ERROR:
+ case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ case RDMA_CM_EVENT_ESTABLISHED:
+ case RDMA_CM_EVENT_ROUTE_ERROR:
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ case RDMA_CM_EVENT_CONNECT_RESPONSE:
+ case RDMA_CM_EVENT_CONNECT_ERROR:
+ case RDMA_CM_EVENT_UNREACHABLE:
+ case RDMA_CM_EVENT_REJECTED:
+ case RDMA_CM_EVENT_DISCONNECTED:
+ case RDMA_CM_EVENT_DEVICE_REMOVAL:
+ case RDMA_CM_EVENT_MULTICAST_JOIN:
+ case RDMA_CM_EVENT_MULTICAST_ERROR:
+ case RDMA_CM_EVENT_ADDR_CHANGE:
+ case RDMA_CM_EVENT_TIMEWAIT_EXIT:
+ //error
+ break;
+ }
+
+ if (state->t->rdma.cm_event != NULL) {
+ rdma_ack_cm_event(state->t->rdma.cm_event);
+ state->t->rdma.cm_event = NULL;
+ }
+}
+
+NTSTATUS smb_direct_negotiate_recv(struct tevent_req *req)
+{
+ struct smb_direct_negotiate_state *state =
+ tevent_req_data(req,
+ struct smb_direct_negotiate_state);
+ NTSTATUS status;
+
+ TALLOC_FREE(state->t->ibv.fde_channel);
+ TALLOC_FREE(state->t->rdma.fde_channel);
+
+ if (tevent_req_is_nterror(req, &status)) {
+ tevent_req_received(req);
+ return status;
+ }
+
+ tevent_req_received(req);
+ return NT_STATUS_OK;
+}
#endif /* SMB_TRANSPORT_ENABLE_RDMA */
git.samba.org / metze / samba / wip.git / blobdiff