HACK start with smb_transport_direct...

diff --git a/libcli/smb/smb_transport_direct.c b/libcli/smb/smb_transport_direct.c
new file mode 100644 (file)
index 0000000..d77bc3f
--- /dev/null
+++ b/libcli/smb/smb_transport_direct.c
@@ -0,0 +1,353 @@
+/*
+   Unix SMB/CIFS implementation.
+   Infrastructure for SMB-Direct RDMA as transport
+   Copyright (C) Stefan Metzmacher 2012
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "system/network.h"
+#include <tevent.h>
+#include "../util/tevent_ntstatus.h"
+#include "../lib/tsocket/tsocket.h"
+
+#include "lib/util/util_net.h" //TODO
+
+#ifdef SMB_TRANSPORT_ENABLE_RDMA
+#include <rdma/rdma_cma_abi.h>
+#include <rdma/rdma_cma.h>
+#include <infiniband/verbs.h>
+
+struct smb_direct_transport {
+       struct {
+               struct rdma_cm_id *cm_id;
+               struct rdma_event_channel *cm_channel;
+               struct tevent_fd *fde_channel;
+               enum rdma_cm_event_type expected_event;
+               struct rdma_cm_event *cm_event;
+       } rdma;
+       struct {
+               struct ibv_pd *pd;
+               struct ibv_comp_channel *comp_channel;
+               struct tevent_fd *fde_channel;
+               struct ibv_cq *cq;
+               struct ibv_qp *qp;
+       } ibv;
+};
+
+static int smb_direct_transport_destructor(struct smb_direct_transport *t);
+
+struct smb_direct_transport *smb_direct_transport_create(TALLOC_CTX *mem_ctx);
+
+struct smb_direct_transport *smb_direct_transport_create(TALLOC_CTX *mem_ctx)
+{
+       struct smb_direct_transport *t;
+       int ret;
+
+       t = talloc_zero(mem_ctx, struct smb_direct_transport);
+       if (t == NULL) {
+               return NULL;
+       }
+       talloc_set_destructor(t, smb_direct_transport_destructor);
+
+       t->rdma.cm_channel = rdma_create_event_channel();
+       if (t->rdma.cm_channel == NULL) {
+               talloc_free(t);
+               return NULL;
+       }
+
+#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
+       ret = rdma_create_id(t->rdma.cm_channel,
+                            &t->rdma.cm_id,
+                            t, RDMA_PS_TCP);
+#else
+       ret = rdma_create_id(t->rdma.cm_channel,
+                            &t->rdma.cm_id,
+                            t);
+#endif
+       if (ret != 0) {
+               talloc_free(t);
+               return NULL;
+       }
+
+       return t;
+}
+
+static int smb_direct_transport_destructor(struct smb_direct_transport *t)
+{
+       TALLOC_FREE(t->ibv.fde_channel);
+       TALLOC_FREE(t->rdma.fde_channel);
+
+       if (t->ibv.qp != NULL) {
+               ibv_destroy_qp(t->ibv.qp);
+               t->ibv.qp = NULL;
+       }
+
+       if (t->ibv.cq != NULL) {
+               ibv_destroy_cq(t->ibv.cq);
+               t->ibv.cq = NULL;
+       }
+
+       if (t->ibv.comp_channel != NULL) {
+               ibv_destroy_comp_channel(t->ibv.comp_channel);
+               t->ibv.comp_channel = NULL;
+       }
+
+       if (t->ibv.pd != NULL) {
+               ibv_dealloc_pd(t->ibv.pd);
+               t->ibv.pd = NULL;
+       }
+
+       if (t->rdma.cm_event != NULL) {
+               rdma_ack_cm_event(t->rdma.cm_event);
+               t->rdma.cm_event = NULL;
+       }
+
+       if (t->rdma.cm_id != NULL) {
+               rdma_destroy_id(t->rdma.cm_id);
+               t->rdma.cm_id = NULL;
+       }
+
+       if (t->rdma.cm_channel != NULL) {
+               rdma_destroy_event_channel(t->rdma.cm_channel);
+               t->rdma.cm_channel = NULL;
+       }
+
+       return 0;
+}
+
+struct smb_direct_rdma_connect_state {
+       struct smb_direct_transport *t;
+};
+
+struct tevent_req *smb_direct_rdma_connect_send(TALLOC_CTX *mem_ctx,
+                                       struct tevent_context *ev,
+                                       struct smb_direct_transport *transport,
+                                       const struct sockaddr_storage *addr,
+                                       struct tsocket_address *local_addr,
+                                       struct tsocket_address *remote_addr);
+NTSTATUS smb_transport_direct_rdma_connect_recv(struct tevent_req *req);
+
+static void smb_direct_rdma_connect_handler(struct tevent_context *ev,
+                                           struct tevent_fd *fde,
+                                           uint16_t flags,
+                                           void *private_data);
+
+struct tevent_req *smb_direct_rdma_connect_send(TALLOC_CTX *mem_ctx,
+                                       struct tevent_context *ev,
+                                       struct smb_direct_transport *transport,
+                                       const struct sockaddr_storage *_addr,
+                                       struct tsocket_address *local_addr,
+                                       struct tsocket_address *remote_addr)
+{
+       struct tevent_req *req;
+       struct smb_direct_rdma_connect_state *state;
+       int ret;
+       struct sockaddr_storage addr = *_addr;
+       struct sockaddr *src_addr = NULL, *dst_addr = &addr;
+
+       set_sockaddr_port(dst_addr, 5445);
+
+       req = tevent_req_create(mem_ctx, &state,
+                               struct smb_direct_rdma_connect_state);
+       if (req == NULL) {
+               return NULL;
+       }
+       state->t = transport;
+
+       transport->rdma.fde_channel = tevent_add_fd(ev, transport,
+                                               transport->rdma.cm_channel->fd,
+                                               TEVENT_FD_READ,
+                                               smb_direct_rdma_connect_handler,
+                                               req);
+       if (tevent_req_nomem(transport->rdma.fde_channel, req)) {
+               return tevent_req_post(req, ev);
+       }
+
+       errno = 0;
+       ret = rdma_resolve_addr(state->t->rdma.cm_id,
+                               src_addr, dst_addr,
+                               5000);
+       if (ret != 0) {
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+       }
+       state->t->rdma.expected_event = RDMA_CM_EVENT_ADDR_RESOLVED;
+
+       return req;
+}
+
+static void smb_direct_rdma_connect_handler(struct tevent_context *ev,
+                                           struct tevent_fd *fde,
+                                           uint16_t flags,
+                                           void *private_data)
+{
+       struct tevent_req *req =
+               talloc_get_type_abort(private_data,
+               struct tevent_req);
+       struct smb_direct_rdma_connect_state *state =
+               tevent_req_data(req,
+               struct smb_direct_rdma_connect_state);
+       struct ibv_qp_init_attr init_attr;
+       struct rdma_conn_param conn_param;
+       int ret;
+
+       errno = 0;
+
+       ret = rdma_get_cm_event(state->t->rdma.cm_channel,
+                               &state->t->rdma.cm_event);
+       if (ret != 0) {
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+       }
+
+       if (state->t->rdma.cm_event->status != 0) {
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+
+       }
+
+       if (state->t->rdma.cm_event->event != state->t->rdma.expected_event) {
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+
+       }
+
+       switch (state->t->rdma.cm_event->event) {
+       case RDMA_CM_EVENT_ADDR_RESOLVED:
+       errno = 0;
+               ret = rdma_resolve_route(state->t->rdma.cm_id, 5000);
+               if (ret != 0) {
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+
+               }
+               state->t->rdma.expected_event = RDMA_CM_EVENT_ROUTE_RESOLVED;
+               break;
+       case RDMA_CM_EVENT_ADDR_ERROR:
+               break;
+       case RDMA_CM_EVENT_ROUTE_RESOLVED:
+       errno = 0;
+       ret = 0;
+               state->t->ibv.pd = ibv_alloc_pd(state->t->rdma.cm_id->verbs);
+               if (state->t->ibv.pd == NULL) {
+
+               }
+
+               state->t->ibv.comp_channel = ibv_create_comp_channel(state->t->rdma.cm_id->verbs);
+               if (state->t->ibv.comp_channel == NULL) {
+               }
+
+               ZERO_STRUCT(init_attr);
+               init_attr.cap.max_send_wr = 16;
+               init_attr.cap.max_recv_wr = 2;
+               init_attr.cap.max_recv_sge = 1;
+               init_attr.cap.max_send_sge = 1;
+               init_attr.qp_type = IBV_QPT_RC;
+
+               state->t->ibv.cq = ibv_create_cq(state->t->rdma.cm_id->verbs,
+                                                init_attr.cap.max_send_wr * 2,
+                                                state->t,
+                                                state->t->ibv.comp_channel,
+                                                0);
+               if (state->t->ibv.cq == NULL) {
+
+               }
+               init_attr.send_cq = state->t->ibv.cq;
+               init_attr.recv_cq = state->t->ibv.cq;
+
+       errno = 0;
+               ret = ibv_req_notify_cq(state->t->ibv.cq, 0);
+               if (ret != 0) {
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+
+               }
+
+
+       errno = 0;
+               ret = rdma_create_qp(state->t->rdma.cm_id, state->t->ibv.pd,
+                                    &init_attr);
+               if (ret != 0) {
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+
+               }
+               state->t->ibv.qp = state->t->rdma.cm_id->qp;
+
+               ZERO_STRUCT(conn_param);
+               conn_param.responder_resources = 1;
+               conn_param.initiator_depth = 1;
+               conn_param.retry_count = 10;
+
+               errno = 0;
+               ret = rdma_connect(state->t->rdma.cm_id, &conn_param);
+               if (ret != 0) {
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+
+               }
+               state->t->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED;
+               break;
+
+       case RDMA_CM_EVENT_ESTABLISHED:
+               errno = 0;
+               ret = 0;
+               DEBUG(0,("%s:%s: ret[%d] errno[%d]\n",
+                       __location__, __FUNCTION__, ret, errno));
+
+               break;
+
+       case RDMA_CM_EVENT_ROUTE_ERROR:
+       case RDMA_CM_EVENT_CONNECT_REQUEST:
+       case RDMA_CM_EVENT_CONNECT_RESPONSE:
+       case RDMA_CM_EVENT_CONNECT_ERROR:
+       case RDMA_CM_EVENT_UNREACHABLE:
+       case RDMA_CM_EVENT_REJECTED:
+       case RDMA_CM_EVENT_DISCONNECTED:
+       case RDMA_CM_EVENT_DEVICE_REMOVAL:
+       case RDMA_CM_EVENT_MULTICAST_JOIN:
+       case RDMA_CM_EVENT_MULTICAST_ERROR:
+       case RDMA_CM_EVENT_ADDR_CHANGE:
+       case RDMA_CM_EVENT_TIMEWAIT_EXIT:
+               //error
+               break;
+       }
+
+       if (state->t->rdma.cm_event != NULL) {
+               rdma_ack_cm_event(state->t->rdma.cm_event);
+               state->t->rdma.cm_event = NULL;
+       }
+}
+
+NTSTATUS smb_transport_direct_rdma_connect_recv(struct tevent_req *req)
+{
+       struct smb_direct_rdma_connect_state *state =
+               tevent_req_data(req,
+               struct smb_direct_rdma_connect_state);
+       NTSTATUS status;
+
+       TALLOC_FREE(state->t->rdma.fde_channel);
+
+       if (tevent_req_is_nterror(req, &status)) {
+               tevent_req_received(req);
+               return status;
+       }
+
+       tevent_req_received(req);
+       return NT_STATUS_OK;
+}
+
+#endif /* SMB_TRANSPORT_ENABLE_RDMA */

diff --git a/libcli/smb/wscript b/libcli/smb/wscript
index ae65d684df10701847eaff3faedaac32c1c1272b..b3dc353da241727ef23b4cd6ae5399525fbc390f 100755 (executable)
--- a/libcli/smb/wscript
+++ b/libcli/smb/wscript
@@ -1,12 +1,39 @@
 #!/usr/bin/env python
 
 
+def set_options(opt):
+    return
+
+def configure(conf):
+    rdma = conf.CHECK_FUNCS_IN('rdma_connect', 'rdmacm',
+                               headers='rdma/rdma_cma.h')
+
+    ibverbs = conf.CHECK_FUNCS_IN('ibv_create_qp', 'ibverbs',
+                                  headers='infiniband/verbs.h')
+
+    hdrs = conf.CHECK_HEADERS('''
+                              sys/socket.h
+                              netinet/in.h
+                              rdma/rdma_cma_abi.h
+                              rdma/rdma_cma.h
+                              infiniband/verbs.h
+                              ''',
+                              together=True)
+
+    conf.COMPOUND_START('Checking for complete RDMA support')
+    if rdma and ibverbs and hdrs:
+        conf.DEFINE('SMB_TRANSPORT_ENABLE_RDMA', 1)
+        conf.COMPOUND_END(True)
+    else:
+        conf.COMPOUND_END(False)
+
 def build(bld):
     bld.SAMBA_LIBRARY('smb_transport',
         source='''
             read_smb.c
+            smb_transport_direct.c
         ''',
-        deps='errors LIBASYNC_REQ',
+        deps='errors LIBASYNC_REQ rdmacm ibverbs',
         public_deps='talloc tevent samba-util',
         private_library=True,
         public_headers='''

diff --git a/wscript b/wscript
index 4f823102dd5cf31131d6331671ae614e9fdd3ceb..707da37b2a56d485257b94b39c6bea6b9c2c008b 100755 (executable)
--- a/wscript
+++ b/wscript
@@ -43,6 +43,7 @@ def set_options(opt):
     opt.RECURSE('lib/nss_wrapper')
     opt.RECURSE('lib/socket_wrapper')
     opt.RECURSE('lib/uid_wrapper')
+    opt.RECURSE('libcli/smb')
     opt.RECURSE('pidl')
     opt.RECURSE('source3')
     opt.RECURSE('lib/util')
@@ -140,6 +141,7 @@ def configure(conf):
     conf.RECURSE('lib/subunit/c')
     conf.RECURSE('libcli/smbreadline')
     conf.RECURSE('lib/crypto')
+    conf.RECURSE('libcli/smb')
     conf.RECURSE('pidl')
     conf.RECURSE('selftest')
     conf.RECURSE('source3')