WIP

diff --git a/libcli/smb/smb_direct_daemon.c b/libcli/smb/smb_direct_daemon.c
index 50cd9cac09a3132dd7cc703821867b3c7cdabc16..1e7c47c26e099a13cc4b9364afc1fe6e9b170e53 100644 (file)
--- a/libcli/smb/smb_direct_daemon.c
+++ b/libcli/smb/smb_direct_daemon.c
@@ -34,6 +34,7 @@
 #include "libcli/util/tstream.h"
 #include "lib/util/byteorder.h"
 #include "lib/util/samba_util.h"
+#include <rdma/rdma_cma.h>
 #include "smb_direct.h"
 #include "smb_direct_util.h"
 
@@ -62,6 +63,12 @@ struct smb_direct_daemon_state {
         * only one and it will receive all new SMB-D connections.
         */
        struct smb_direct_daemon_conn *listening_conn;
+
+       struct {
+               struct rdma_cm_id *cm_id;
+               struct rdma_event_channel *cm_channel;
+               struct tevent_fd *cm_channel_fde;
+       } rdma;
 };
 
 struct smb_direct_daemon_conn {
@@ -513,22 +520,62 @@ static NTSTATUS smbd_direct_daemon_listen(
        struct sdd_packet_listen_request *listen_request,
        struct sdd_packet_listen_response *listen_response)
 {
-       if (conn->daemon_state->listening_conn != NULL) {
+       struct smb_direct_daemon_state *daemon_state = conn->daemon_state;
+
+       if (daemon_state->listening_conn != NULL) {
                /* There can be only one... */
                return NT_STATUS_INVALID_PARAMETER;
        }
 
-       conn->daemon_state->params = listen_request->params;
-       conn->listening = true;
-       conn->daemon_state->listening_conn = conn;
-
-       ok = smb_direct_daemon_rdma_listen(conn->daemon_state);
-       if (!ok) {
+       daemon_state->rdma.cm_channel = rdma_create_event_channel();
+       if (daemon_state->rdma.cm_channel == NULL) {
+               DBG_ERR("rdma_create_event_channel failed [%s]\n",
+                       strerror(errno));
                return NT_STATUS_INTERNAL_ERROR;
        }
+
+#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
+       result = rdma_create_id(daemon_state->rdma.cm_channel,
+                               &daemon_state->rdma.cm_id,
+                               conn,
+                               RDMA_PS_TCP);
+#else
+       result = rdma_create_id(daemon_state->rdma.cm_channel,
+                               &daemon_state->rdma.cm_id,
+                               conn);
+#endif
+       if (result != 0) {
+               DBG_ERR("rdma_create_id failed [%s]\n",
+                       strerror(errno));
+               goto cleanup;
+       }
+
+       daemon_state->rdma.cm_channel_fde = tevent_add_fd(
+               conn->ev,
+               conn,
+               daemon_state->rdma.cm_channel->fd,
+               TEVENT_FD_READ,
+               rdma_cm_handler,
+               conn);
+       if (daemon_state->rdma.cm_channel_fde == NULL) {
+               DBG_ERR("tevent_add_fd failed\n");
+               goto cleanup;
+       }
+
+       conn->listening = true;
+
+       daemon_state->params = listen_request->params;
+       daemon_state->listening_conn = conn;
        listen_response->status = 0;
 
        return NT_STATUS_OK;
+
+cleanup:
+       rdma_destroy_id(daemon_state->cm_id);
+       daemon_state->cm_id = NULL;
+       rdma_destroy_event_channel(daemon_state->cm_channel);
+       daemon_state->cm_channel = NULL;
+       return NT_STATUS_INTERNAL_ERROR;
 }
 
 static NTSTATUS smbd_direct_daemon_dispatch(struct smb_direct_daemon_conn *conn,
@@ -627,15 +674,34 @@ static void smb_direct_daemon_conn_cleanup(struct tevent_req *req,
 {
        struct smb_direct_daemon_conn *conn = tevent_req_data(
                req, struct smb_direct_daemon_conn);
+       struct smb_direct_daemon_state *daemon_state = NULL;
+       int result;
+
+       if (conn->conn_fd != -1) {
+               close(conn->conn_fd);
+               conn->conn_fd = -1;
+       }
 
-       if (conn->fd != -1) {
-               close(conn->fd);
-               conn->fd = -1;
+       if (conn->daemon_state->listening_conn != conn) {
+               return;
        }
 
-       if (conn->daemon_state->conn == conn) {
-               conn->daemon_state->conn = NULL;
+       daemon_state = conn->daemon_state;
+       daemon_state->listening_conn = NULL;
+
+       result = rdma_destroy_event_channel(daemon_state->rdma_ec);
+       if (result != 0) {
+               DBG_ERR("rdma_destroy_event_channel failed [%s]\n",
+                       strerror(errno));
        }
+       daemon_state->rdma_ec = 0;
+
+       result = rdma_destroy_id(daemon_state->rdma_cm_id);
+       if (result != 0) {
+               DBG_ERR("rdma_destroy_id failed [%s]\n",
+                       strerror(errno));
+       }
+       daemon_state->rdma_cm_id = 0;
 }
 
 /**
@@ -742,3 +808,171 @@ static int smb_direct_daemon_conn_recv(struct tevent_req *req)
 {
        return tevent_req_simple_recv_unix(req);
 }
+
+static void rdma_cm_handler(struct tevent_context *ev,
+                           struct tevent_fd *fde,
+                           uint16_t flags,
+                           void *private_data)
+{
+       struct smb_direct_daemon_conn *conn = talloc_get_type_abort(
+               private_data, struct smb_direct_daemon_conn);
+       struct smb_direct_daemon_state *daemon_state = conn->daemon_state;
+       struct rdma_cm_event *cm_ev = NULL;
+       struct rdma_cm_id *cm_id = NULL;
+       int result;
+
+       result = rdma_get_cm_event(daemon_state->cm_channel, &cm_ev);
+       if (result != 0) {
+               DBG_ERR("rdma_get_cm_event failed [%s]\n", strerror(errno));
+               goto fail;
+       }
+
+       cm_id = cm_ev->id;
+
+       DBG_DEBUG("cm_event type [%d]\n", cm_ev->event);
+
+       switch (cm_ev->event) {
+
+       case RDMA_CM_EVENT_ADDR_RESOLVED:
+               DBG_DEBUG("RDMA_CM_EVENT_ADDR_RESOLVED\n");
+
+               result = rdma_resolve_route(cm_id, 2000);
+               if (result != 0) {
+                       DBG_ERR("rdma_resolve_route failed [%s]\n", strerror(errno));
+                       goto fail;
+               }
+               break;
+
+       case RDMA_CM_EVENT_ROUTE_RESOLVED: {
+               DBG_DEBUG("RDMA_CM_EVENT_ROUTE_RESOLVED\n");
+
+               conn = talloc_get_type(cma_id->context, struct ibw_conn);
+
+               result = ibw_manage_connect(conn);
+               if (rc)
+                       goto error;
+
+               break;
+       }
+
+       case RDMA_CM_EVENT_CONNECT_REQUEST:
+               DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_CONNECT_REQUEST\n"));
+               ctx->state = IBWS_CONNECT_REQUEST;
+               conn = ibw_conn_new(ctx, ctx);
+               pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+               pconn->cm_id = cma_id; /* !!! event will be freed but id not */
+               cma_id->context = (void *)conn;
+               DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p\n", pconn->cm_id));
+
+               if (ibw_setup_cq_qp(conn))
+                       goto error;
+
+               conn->state = IBWC_INIT;
+               pctx->connstate_func(ctx, conn);
+
+               /* continued at ibw_accept when invoked by the func above */
+               if (!pconn->is_accepted) {
+                       rc = rdma_reject(cma_id, NULL, 0);
+                       if (rc)
+                               DEBUG(DEBUG_ERR, ("rdma_reject failed with rc=%d\n", rc));
+                       talloc_free(conn);
+                       DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p wasn't accepted\n", pconn->cm_id));
+               }
+
+               /* TODO: clarify whether if it's needed by upper layer: */
+               ctx->state = IBWS_READY;
+               pctx->connstate_func(ctx, NULL);
+
+               /* NOTE: more requests can arrive until RDMA_CM_EVENT_ESTABLISHED ! */
+               break;
+
+       case RDMA_CM_EVENT_ESTABLISHED:
+               /* expected after ibw_accept and ibw_connect[not directly] */
+               DEBUG(DEBUG_INFO, ("ESTABLISHED (conn: %p)\n", cma_id->context));
+               conn = talloc_get_type(cma_id->context, struct ibw_conn);
+               assert(conn!=NULL); /* important assumption */
+
+               DEBUG(DEBUG_DEBUG, ("ibw_setup_cq_qp succeeded (cmid=%p)\n", cma_id));
+
+               /* client conn is up */
+               conn->state = IBWC_CONNECTED;
+
+               /* both ctx and conn have changed */
+               pctx->connstate_func(ctx, conn);
+               break;
+
+       case RDMA_CM_EVENT_ADDR_ERROR:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_ADDR_ERROR, error %d\n", event->status);
+       case RDMA_CM_EVENT_ROUTE_ERROR:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_ROUTE_ERROR, error %d\n", event->status);
+       case RDMA_CM_EVENT_CONNECT_ERROR:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status);
+       case RDMA_CM_EVENT_UNREACHABLE:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status);
+               goto error;
+       case RDMA_CM_EVENT_REJECTED:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_REJECTED, error %d\n", event->status);
+               DEBUG(DEBUG_INFO, ("cm event handler: %s", ibw_lasterr));
+               conn = talloc_get_type(cma_id->context, struct ibw_conn);
+               if (conn) {
+                       /* must be done BEFORE connstate */
+                       if ((rc=rdma_ack_cm_event(event)))
+                               DEBUG(DEBUG_ERR, ("reject/rdma_ack_cm_event failed with %d\n", rc));
+                       event = NULL; /* not to touch cma_id or conn */
+                       conn->state = IBWC_ERROR;
+                       /* it should free the conn */
+                       pctx->connstate_func(NULL, conn);
+               }
+               break; /* this is not strictly an error */
+
+       case RDMA_CM_EVENT_DISCONNECTED:
+               DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_DISCONNECTED\n"));
+               if ((rc=rdma_ack_cm_event(event)))
+                       DEBUG(DEBUG_ERR, ("disc/rdma_ack_cm_event failed with %d\n", rc));
+               event = NULL; /* don't ack more */
+
+               if (cma_id!=pctx->cm_id) {
+                       DEBUG(DEBUG_ERR, ("client DISCONNECT event cm_id=%p\n", cma_id));
+                       conn = talloc_get_type(cma_id->context, struct ibw_conn);
+                       conn->state = IBWC_DISCONNECTED;
+                       pctx->connstate_func(NULL, conn);
+               }
+               break;
+
+       case RDMA_CM_EVENT_DEVICE_REMOVAL:
+               sprintf(ibw_lasterr, "cma detected device removal!\n");
+               goto error;
+
+       default:
+               sprintf(ibw_lasterr, "unknown event %d\n", event->event);
+               goto error;
+       }
+
+       if (event!=NULL && (rc=rdma_ack_cm_event(event))) {
+               sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n", rc);
+               goto error;
+       }
+
+       return;
+error:
+       DEBUG(DEBUG_ERR, ("cm event handler: %s", ibw_lasterr));
+
+       if (event!=NULL) {
+               if (cma_id!=NULL && cma_id!=pctx->cm_id) {
+                       conn = talloc_get_type(cma_id->context, struct ibw_conn);
+                       if (conn) {
+                               conn->state = IBWC_ERROR;
+                               pctx->connstate_func(NULL, conn);
+                       }
+               } else {
+                       ctx->state = IBWS_ERROR;
+                       pctx->connstate_func(ctx, NULL);
+               }
+
+               if ((rc=rdma_ack_cm_event(event))!=0) {
+                       DEBUG(DEBUG_ERR, ("rdma_ack_cm_event failed with %d\n", rc));
+               }
+       }
+
+       return;
+}