ib: a trivial approach of integration

TODO: bugfix (not yet complete, something happens differently than expected
at tcpip when connection is refused in the beginning)

diff --git a/common/ctdb.c b/common/ctdb.c
index 59a14d4dbdc19ee25befe0377591d9cf342c419d..6dc78fa22a57d8b920dfb2b1707e80104427a556 100644 (file)
--- a/common/ctdb.c
+++ b/common/ctdb.c
@@ -32,10 +32,19 @@
 int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport)
 {
        int ctdb_tcp_init(struct ctdb_context *ctdb);
+#ifdef USE_INFINIBAND
+       int ctdb_ibw_init(struct ctdb_context *ctdb);
+#endif /* USE_INFINIBAND */
 
        if (strcmp(transport, "tcp") == 0) {
                return ctdb_tcp_init(ctdb);
        }
+#ifdef USE_INFINIBAND
+       if (strcmp(transport, "ib") == 0) {
+               return ctdb_ibw_init(ctdb);
+       }
+#endif /* USE_INFINIBAND */
+
        ctdb_set_error(ctdb, "Unknown transport '%s'\n", transport);
        return -1;
 }

diff --git a/ib/ibw_ctdb.c b/ib/ibw_ctdb.c
index 4379cb7508608f27b626ab84d47067a266719c9c..5152062d246aa4f33edb5229fc6a51f13d0a9ac2 100644 (file)
--- a/ib/ibw_ctdb.c
+++ b/ib/ibw_ctdb.c
@@ -63,19 +63,24 @@ int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn)
                        break;
                case IBWC_CONNECTED: { /* after ibw_accept or ibw_connect */
                        struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
-                       if (node!=NULL) /* after ibw_connect */
+                       if (node!=NULL) { /* after ibw_connect */
+                               node->private = (void *)conn;
                                node->ctdb->upcalls->node_connected(node);
-                       else { /* after ibw_accept */
+                       } else { /* after ibw_accept */
                                /* NOP in CTDB case */
                        }
                } break;
-               case IBWC_DISCONNECTED: /* after ibw_disconnect */
+               case IBWC_DISCONNECTED: { /* after ibw_disconnect */
                        /* TODO: have a CTDB upcall */
-                       break;
-               case IBWC_ERROR: {
                        struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
                        if (node!=NULL)
-                               node->ctdb->upcalls->node_connected(node);
+                               node->ctdb->upcalls->node_dead(node);
+                       talloc_free(conn);
+               } break;
+               case IBWC_ERROR: {
+/*                     struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
+                       if (node!=NULL)
+                               node->ctdb->upcalls->node_dead(node);*/
                } break;
                default:
                        assert(0);

diff --git a/ib/ibw_ctdb_init.c b/ib/ibw_ctdb_init.c
index 0d4f01d763592244acad2d1289e22ff6d4eb4187..a1eaca9202bdae71eb8c8c9529e31c03785be36d 100644 (file)
--- a/ib/ibw_ctdb_init.c
+++ b/ib/ibw_ctdb_init.c
@@ -29,9 +29,6 @@
 #include "ibwrapper.h"
 #include "ibw_ctdb.h"
 
-/* not nice; temporary workaround for the current implementation... */
-static void *last_key = NULL;
-
 static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
 {
        struct ibw_ctx *ictx = talloc_get_type(ctdb->private, struct ibw_ctx);
@@ -60,6 +57,7 @@ int ctdb_ibw_node_connect(struct ibw_ctx *ictx, struct ctdb_node *node)
 {
         struct sockaddr_in sock_out;
 
+       memset(&sock_out, 0, sizeof(struct sockaddr_in));
        inet_pton(AF_INET, node->address.address, &sock_out.sin_addr);
        sock_out.sin_port = htons(node->address.port);
        sock_out.sin_family = PF_INET;
@@ -112,37 +110,36 @@ static int ctdb_ibw_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t le
 {
        struct ibw_conn *conn = talloc_get_type(node->private, struct ibw_conn);
        int     rc;
+       void    *buf, *key;
+
+       assert(length>=sizeof(uint32_t));
+
+       if (conn==NULL) {
+               DEBUG(0, ("ctdb_ibw_queue_pkt: conn is NULL\n"));
+               return -1;
+       }
+
+       if (ibw_alloc_send_buf(conn, &buf, &key, length)) {
+               DEBUG(0, ("queue_pkt/ibw_alloc_send_buf failed\n"));
+               return -1;
+       }
 
-       rc = ibw_send(conn, data, last_key, length);
-       last_key = NULL;
+       memcpy(buf, data, length);
+       rc = ibw_send(conn, buf, key, length);
 
        return rc;
 }
 
-#ifdef __NOTDEF__
 /*
  * transport packet allocator - allows transport to control memory for packets
  */
-static void *ctdb_ibw_allocate_pkt(struct ctdb_node *node, size_t size)
+static void *ctdb_ibw_allocate_pkt(struct ctdb_context *ctdb, size_t size)
 {
-       struct ibw_conn *conn = NULL;
-       void *buf = NULL;
-
-       if (ibw_alloc_send_buf(conn, &buf, &last_key, size))
-               return NULL;
-
-       return buf;
+       /* TODO: use ibw_alloc_send_buf instead... */
+       return talloc_size(ctdb, size);
 }
 
-static void ctdb_ibw_dealloc_pkt(struct ctdb_node *node, void *data)
-{
-       if (last_key) {
-               struct ibw_conn *conn = talloc_get_type(node->private, struct ibw_conn);
-       
-               assert(conn!=NULL);
-               ibw_cancel_send_buf(conn, data, last_key);
-       } /* else ibw_send is already using it and will free it after completion */
-}
+#ifdef __NOTDEF__
 
 static int ctdb_ibw_stop(struct ctdb_context *cctx)
 {
@@ -158,9 +155,8 @@ static const struct ctdb_methods ctdb_ibw_methods = {
        .start     = ctdb_ibw_start,
        .add_node  = ctdb_ibw_add_node,
        .queue_pkt = ctdb_ibw_queue_pkt,
-//     .allocate_pkt = ctdb_ibw_allocate_pkt,
+       .allocate_pkt = ctdb_ibw_allocate_pkt,
 
-//     .dealloc_pkt = ctdb_ibw_dealloc_pkt,
 //     .stop = ctdb_ibw_stop
 };
 
@@ -171,6 +167,7 @@ int ctdb_ibw_init(struct ctdb_context *ctdb)
 {
        struct ibw_ctx *ictx;
 
+       DEBUG(10, ("ctdb_ibw_init invoked...\n"));
        ictx = ibw_init(
                NULL, //struct ibw_initattr *attr, /* TODO */
                0, //int nattr, /* TODO */
@@ -186,5 +183,7 @@ int ctdb_ibw_init(struct ctdb_context *ctdb)
 
        ctdb->methods = &ctdb_ibw_methods;
        ctdb->private = ictx;
+       
+       DEBUG(10, ("ctdb_ibw_init succeeded.\n"));
        return 0;
 }

diff --git a/ib/ibwrapper.c b/ib/ibwrapper.c
index 2f8dd256f91ac25bfac12dfe4d2f5f47d72463d8..fef312678a5498e8e96b447bfd03f5c4531d5d3e 100644 (file)
--- a/ib/ibwrapper.c
+++ b/ib/ibwrapper.c
@@ -420,6 +420,7 @@ static void ibw_event_handler_cm(struct event_context *ev,
 
        switch (event->event) {
        case RDMA_CM_EVENT_ADDR_RESOLVED:
+               DEBUG(11, ("RDMA_CM_EVENT_ADDR_RESOLVED\n"));
                /* continuing from ibw_connect ... */
                rc = rdma_resolve_route(cma_id, 2000);
                if (rc) {
@@ -430,6 +431,7 @@ static void ibw_event_handler_cm(struct event_context *ev,
                break;
 
        case RDMA_CM_EVENT_ROUTE_RESOLVED:
+               DEBUG(11, ("RDMA_CM_EVENT_ROUTE_RESOLVED\n"));
                /* after RDMA_CM_EVENT_ADDR_RESOLVED: */
                assert(cma_id->context!=NULL);
                conn = talloc_get_type(cma_id->context, struct ibw_conn);
@@ -441,6 +443,7 @@ static void ibw_event_handler_cm(struct event_context *ev,
                break;
 
        case RDMA_CM_EVENT_CONNECT_REQUEST:
+               DEBUG(11, ("RDMA_CM_EVENT_CONNECT_REQUEST\n"));
                ctx->state = IBWS_CONNECT_REQUEST;
                conn = ibw_conn_new(ctx);
                pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
@@ -481,14 +484,23 @@ static void ibw_event_handler_cm(struct event_context *ev,
                break;
 
        case RDMA_CM_EVENT_ADDR_ERROR:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_ADDR_ERROR, error %d\n", event->status);
+               goto error;
        case RDMA_CM_EVENT_ROUTE_ERROR:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_ROUTE_ERROR, error %d\n", event->status);
+               goto error;
        case RDMA_CM_EVENT_CONNECT_ERROR:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status);
+               goto error;
        case RDMA_CM_EVENT_UNREACHABLE:
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status);
+               goto error;
        case RDMA_CM_EVENT_REJECTED:
-               sprintf(ibw_lasterr, "cma event %d, error %d\n", event->event, event->status);
+               sprintf(ibw_lasterr, "RDMA_CM_EVENT_REJECTED, error %d\n", event->status);
                goto error;
 
        case RDMA_CM_EVENT_DISCONNECTED:
+               DEBUG(11, ("RDMA_CM_EVENT_DISCONNECTED\n"));
                if (cma_id!=pctx->cm_id) {
                        DEBUG(0, ("client DISCONNECT event cm_id=%p\n", cma_id));
                        conn = talloc_get_type(cma_id->context, struct ibw_conn);
@@ -944,7 +956,7 @@ int ibw_bind(struct ibw_ctx *ctx, struct sockaddr_in *my_addr)
        int     rc;
 
        DEBUG(10, ("ibw_bind: addr=%s, port=%u\n",
-               inet_ntoa(my_addr->sin_addr), my_addr->sin_port));
+               inet_ntoa(my_addr->sin_addr), ntohs(my_addr->sin_port)));
        rc = rdma_bind_addr(pctx->cm_id, (struct sockaddr *) my_addr);
        if (rc) {
                sprintf(ibw_lasterr, "rdma_bind_addr error %d\n", rc);
@@ -1008,7 +1020,8 @@ int ibw_connect(struct ibw_ctx *ctx, struct sockaddr_in *serv_addr, void *conn_u
        conn = ibw_conn_new(ctx);
        conn->conn_userdata = conn_userdata;
        pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
-       DEBUG(10, ("ibw_connect: addr=%s, port=%u\n", inet_ntoa(serv_addr->sin_addr), serv_addr->sin_port));
+       DEBUG(10, ("ibw_connect: addr=%s, port=%u\n", inet_ntoa(serv_addr->sin_addr),
+               ntohs(serv_addr->sin_port)));
 
        /* init cm */
        rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP);
@@ -1023,6 +1036,7 @@ int ibw_connect(struct ibw_ctx *ctx, struct sockaddr_in *serv_addr, void *conn_u
        if (rc) {
                sprintf(ibw_lasterr, "rdma_resolve_addr error %d\n", rc);
                DEBUG(0, (ibw_lasterr));
+               talloc_free(conn);
                return -1;
        }
 

diff --git a/ib/ibwrapper_test.c b/ib/ibwrapper_test.c
index 65877872fd9b1335ecb33c5c8d81664fe0e1315f..dc07a1c313bdaacc3b5038264443d2ca27bd678b 100644 (file)
--- a/ib/ibwrapper_test.c
+++ b/ib/ibwrapper_test.c
@@ -477,7 +477,7 @@ int ibwtest_getdests(struct ibwtest_ctx *tcx, char op)
                p = tcx->addrs + i;
                p->sin_family = AF_INET;
                p->sin_addr.s_addr = inet_addr(attrs[i].name);
-               p->sin_port = atoi(attrs[i].value);
+               p->sin_port = htons(atoi(attrs[i].value));
        }
 
        return 0;