#include "libcli/util/tstream.h"
#include "lib/util/byteorder.h"
#include "lib/util/samba_util.h"
+#include "lib/util/blocking.h"
+
+#ifdef SMB_TRANSPORT_ENABLE_RDMA
+#include <rdma/rdma_cma_abi.h>
#include <rdma/rdma_cma.h>
#include "smb_direct.h"
#include "smb_direct_util.h"
return NT_STATUS_OK;
}
+static void rdma_cm_handler(struct tevent_context *ev,
+ struct tevent_fd *fde,
+ uint16_t flags,
+ void *private_data);
+
static NTSTATUS smbd_direct_daemon_listen(
struct smb_direct_daemon_conn *conn,
struct sdd_packet_listen_request *listen_request,
struct sdd_packet_listen_response *listen_response)
{
struct smb_direct_daemon_state *daemon_state = conn->daemon_state;
+ struct sockaddr_in inaddr;
+ int result;
if (daemon_state->listening_conn != NULL) {
/* There can be only one... */
return NT_STATUS_INTERNAL_ERROR;
}
+ set_close_on_exec(daemon_state->rdma.cm_channel->fd);
+ set_blocking(daemon_state->rdma.cm_channel->fd, false);
+
#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
result = rdma_create_id(daemon_state->rdma.cm_channel,
&daemon_state->rdma.cm_id,
if (result != 0) {
DBG_ERR("rdma_create_id failed [%s]\n",
strerror(errno));
- goto cleanup;
+ goto fail;
}
daemon_state->rdma.cm_channel_fde = tevent_add_fd(
daemon_state->rdma.cm_channel->fd,
TEVENT_FD_READ,
rdma_cm_handler,
- conn);
+ daemon_state);
if (daemon_state->rdma.cm_channel_fde == NULL) {
DBG_ERR("tevent_add_fd failed\n");
- goto cleanup;
+ goto fail;
+ }
+
+ ZERO_STRUCT(inaddr);
+ inaddr.sin_family = AF_INET;
+ inaddr.sin_port = 5445;
+ inaddr.sin_addr.s_addr = INADDR_ANY;
+
+ result = rdma_bind_addr(daemon_state->rdma.cm_id,
+ (struct sockaddr *)&inaddr);
+ if (result != 0) {
+ DBG_ERR("rdma_bind_addr failed\n");
+ goto fail;
+ }
+
+ result = rdma_listen(daemon_state->rdma.cm_id,
+ SMB_DIRECT_LISTEN_BACKLOG);
+ if (result != 0) {
+ DBG_ERR("rdma_bind_addr failed\n");
+ goto fail;
}
conn->listening = true;
return NT_STATUS_OK;
-cleanup:
- rdma_destroy_id(daemon_state->cm_id);
- daemon_state->cm_id = NULL;
- rdma_destroy_event_channel(daemon_state->cm_channel);
- daemon_state->cm_channel = NULL;
+fail:
+ rdma_destroy_id(daemon_state->rdma.cm_id);
+ daemon_state->rdma.cm_id = NULL;
+ rdma_destroy_event_channel(daemon_state->rdma.cm_channel);
+ daemon_state->rdma.cm_channel = NULL;
return NT_STATUS_INTERNAL_ERROR;
}
daemon_state = conn->daemon_state;
daemon_state->listening_conn = NULL;
- result = rdma_destroy_event_channel(daemon_state->rdma_ec);
- if (result != 0) {
- DBG_ERR("rdma_destroy_event_channel failed [%s]\n",
- strerror(errno));
- }
- daemon_state->rdma_ec = 0;
+ rdma_destroy_event_channel(daemon_state->rdma.cm_channel);
+ daemon_state->rdma.cm_channel = NULL;
- result = rdma_destroy_id(daemon_state->rdma_cm_id);
+ result = rdma_destroy_id(daemon_state->rdma.cm_id);
if (result != 0) {
DBG_ERR("rdma_destroy_id failed [%s]\n",
strerror(errno));
}
- daemon_state->rdma_cm_id = 0;
+ daemon_state->rdma.cm_id = NULL;
}
/**
.privileged = privileged,
};
+ tevent_req_set_cleanup_fn(req, smb_direct_daemon_conn_cleanup);
+
subreq = smbd_direct_daemon_read_send(conn);
if (tevent_req_nomem(subreq, req)) {
tevent_req_post(req, ev);
}
tevent_req_set_callback(subreq, smbd_direct_daemon_conn_handler, req);
- tevent_req_set_cleanup_fn(subreq, smb_direct_daemon_conn_cleanup);
return req;
}
uint16_t flags,
void *private_data)
{
- struct smb_direct_daemon_conn *conn = talloc_get_type_abort(
- private_data, struct smb_direct_daemon_conn);
- struct smb_direct_daemon_state *daemon_state = conn->daemon_state;
+ struct smb_direct_daemon_state *daemon_state = talloc_get_type_abort(
+ private_data, struct smb_direct_daemon_state);
+ struct smb_direct_connection *sconn = NULL;
struct rdma_cm_event *cm_ev = NULL;
- struct rdma_cm_id *cm_id = NULL;
+ struct rdma_cm_event cm_ev_copy;
int result;
- result = rdma_get_cm_event(daemon_state->cm_channel, &cm_ev);
+ result = rdma_get_cm_event(daemon_state->rdma.cm_channel, &cm_ev);
if (result != 0) {
DBG_ERR("rdma_get_cm_event failed [%s]\n", strerror(errno));
- goto fail;
+ return;
}
- cm_id = cm_ev->id;
-
- DBG_DEBUG("cm_event type [%d]\n", cm_ev->event);
+ cm_ev_copy = *cm_ev;
+ result = rdma_ack_cm_event(cm_ev);
+ if (result != 0) {
+ DBG_ERR("rdma_ack_cm_event failed [%s]\n", strerror(errno));
+ goto fail;
+ }
+ cm_ev = NULL;
- switch (cm_ev->event) {
+ DBG_DEBUG("cm_event type [%d] cm_id [%p]\n",
+ cm_ev_copy.event, cm_ev_copy.id);
- case RDMA_CM_EVENT_ADDR_RESOLVED:
- DBG_DEBUG("RDMA_CM_EVENT_ADDR_RESOLVED\n");
+ switch (cm_ev_copy.event) {
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ DBG_DEBUG("RDMA_CM_EVENT_CONNECT_REQUEST\n");
- result = rdma_resolve_route(cm_id, 2000);
+ result = rdma_accept(cm_ev_copy.id, NULL);
if (result != 0) {
- DBG_ERR("rdma_resolve_route failed [%s]\n", strerror(errno));
+ DBG_ERR("rdma_accept failed [%s]\n", strerror(errno));
goto fail;
}
- break;
-
- case RDMA_CM_EVENT_ROUTE_RESOLVED: {
- DBG_DEBUG("RDMA_CM_EVENT_ROUTE_RESOLVED\n");
-
- conn = talloc_get_type(cma_id->context, struct ibw_conn);
- result = ibw_manage_connect(conn);
- if (rc)
- goto error;
-
- break;
- }
-
- case RDMA_CM_EVENT_CONNECT_REQUEST:
- DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_CONNECT_REQUEST\n"));
- ctx->state = IBWS_CONNECT_REQUEST;
- conn = ibw_conn_new(ctx, ctx);
- pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
- pconn->cm_id = cma_id; /* !!! event will be freed but id not */
- cma_id->context = (void *)conn;
- DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p\n", pconn->cm_id));
-
- if (ibw_setup_cq_qp(conn))
- goto error;
-
- conn->state = IBWC_INIT;
- pctx->connstate_func(ctx, conn);
-
- /* continued at ibw_accept when invoked by the func above */
- if (!pconn->is_accepted) {
- rc = rdma_reject(cma_id, NULL, 0);
- if (rc)
- DEBUG(DEBUG_ERR, ("rdma_reject failed with rc=%d\n", rc));
- talloc_free(conn);
- DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p wasn't accepted\n", pconn->cm_id));
+ sconn = smb_direct_connection_listener(daemon_state,
+ cm_ev_copy.id);
+ if (sconn == NULL) {
+ DBG_ERR("smb_direct_connection_create failed\n");
+ return;
}
- /* TODO: clarify whether if it's needed by upper layer: */
- ctx->state = IBWS_READY;
- pctx->connstate_func(ctx, NULL);
-
- /* NOTE: more requests can arrive until RDMA_CM_EVENT_ESTABLISHED ! */
+ cm_ev_copy.id->context = sconn;
break;
case RDMA_CM_EVENT_ESTABLISHED:
- /* expected after ibw_accept and ibw_connect[not directly] */
- DEBUG(DEBUG_INFO, ("ESTABLISHED (conn: %p)\n", cma_id->context));
- conn = talloc_get_type(cma_id->context, struct ibw_conn);
- assert(conn!=NULL); /* important assumption */
-
- DEBUG(DEBUG_DEBUG, ("ibw_setup_cq_qp succeeded (cmid=%p)\n", cma_id));
-
- /* client conn is up */
- conn->state = IBWC_CONNECTED;
-
- /* both ctx and conn have changed */
- pctx->connstate_func(ctx, conn);
+ DBG_DEBUG("RDMA_CM_EVENT_ESTABLISHED\n");
+
+ sconn = talloc_get_type_abort(cm_ev_copy.id->context,
+ struct smb_direct_connection);
+ sconn = smb_direct_connection_complete_alloc(sconn);
+ /*
+ * TODO: get things going...
+ */
break;
- case RDMA_CM_EVENT_ADDR_ERROR:
- sprintf(ibw_lasterr, "RDMA_CM_EVENT_ADDR_ERROR, error %d\n", event->status);
- case RDMA_CM_EVENT_ROUTE_ERROR:
- sprintf(ibw_lasterr, "RDMA_CM_EVENT_ROUTE_ERROR, error %d\n", event->status);
- case RDMA_CM_EVENT_CONNECT_ERROR:
- sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status);
- case RDMA_CM_EVENT_UNREACHABLE:
- sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status);
- goto error;
- case RDMA_CM_EVENT_REJECTED:
- sprintf(ibw_lasterr, "RDMA_CM_EVENT_REJECTED, error %d\n", event->status);
- DEBUG(DEBUG_INFO, ("cm event handler: %s", ibw_lasterr));
- conn = talloc_get_type(cma_id->context, struct ibw_conn);
- if (conn) {
- /* must be done BEFORE connstate */
- if ((rc=rdma_ack_cm_event(event)))
- DEBUG(DEBUG_ERR, ("reject/rdma_ack_cm_event failed with %d\n", rc));
- event = NULL; /* not to touch cma_id or conn */
- conn->state = IBWC_ERROR;
- /* it should free the conn */
- pctx->connstate_func(NULL, conn);
- }
- break; /* this is not strictly an error */
-
case RDMA_CM_EVENT_DISCONNECTED:
- DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_DISCONNECTED\n"));
- if ((rc=rdma_ack_cm_event(event)))
- DEBUG(DEBUG_ERR, ("disc/rdma_ack_cm_event failed with %d\n", rc));
- event = NULL; /* don't ack more */
-
- if (cma_id!=pctx->cm_id) {
- DEBUG(DEBUG_ERR, ("client DISCONNECT event cm_id=%p\n", cma_id));
- conn = talloc_get_type(cma_id->context, struct ibw_conn);
- conn->state = IBWC_DISCONNECTED;
- pctx->connstate_func(NULL, conn);
- }
+ DBG_DEBUG("RDMA_CM_EVENT_DISCONNECTED\n");
+
+ sconn = talloc_get_type(cm_ev_copy.id->context,
+ struct smb_direct_connection);
+ TALLOC_FREE(sconn);
break;
case RDMA_CM_EVENT_DEVICE_REMOVAL:
- sprintf(ibw_lasterr, "cma detected device removal!\n");
- goto error;
+ DBG_ERR("RDMA device removal\n");
+ goto fail;
default:
- sprintf(ibw_lasterr, "unknown event %d\n", event->event);
- goto error;
+ DBG_ERR("event %d\n", cm_ev_copy.event);
+ goto fail;
}
- if (event!=NULL && (rc=rdma_ack_cm_event(event))) {
- sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n", rc);
- goto error;
- }
return;
-error:
- DEBUG(DEBUG_ERR, ("cm event handler: %s", ibw_lasterr));
-
- if (event!=NULL) {
- if (cma_id!=NULL && cma_id!=pctx->cm_id) {
- conn = talloc_get_type(cma_id->context, struct ibw_conn);
- if (conn) {
- conn->state = IBWC_ERROR;
- pctx->connstate_func(NULL, conn);
- }
- } else {
- ctx->state = IBWS_ERROR;
- pctx->connstate_func(ctx, NULL);
- }
- if ((rc=rdma_ack_cm_event(event))!=0) {
- DEBUG(DEBUG_ERR, ("rdma_ack_cm_event failed with %d\n", rc));
- }
+fail:
+ if (cm_ev_copy.id == daemon_state->rdma.cm_id) {
+ /* Listening conn is erroring out, shut it down and die */
+ TALLOC_FREE(daemon_state->listening_conn);
}
+ TALLOC_FREE(sconn);
return;
}
+
+#endif /* SMB_TRANSPORT_ENABLE_RDMA */
git.samba.org / metze / samba / wip.git / commitdiff