* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <malloc.h>
+#include "replace.h"
+#include "system/network.h"
+
#include <assert.h>
-#include <unistd.h>
+#include <talloc.h>
+#include <tevent.h>
-#include "includes.h"
-#include "lib/events/events.h"
-#include "ibwrapper.h"
+#include "lib/util/dlinklist.h"
+#include "lib/util/debug.h"
+
+#include "common/logging.h"
#include <infiniband/kern-abi.h>
#include <rdma/rdma_cma_abi.h>
#include <rdma/rdma_cma.h>
+#include "ibwrapper.h"
#include "ibwrapper_internal.h"
-#include "lib/util/dlinklist.h"
#define IBW_LASTERR_BUFSIZE 512
static char ibw_lasterr[IBW_LASTERR_BUFSIZE];
-#define IBW_MAX_SEND_WR 256
-#define IBW_MAX_RECV_WR 1024
-#define IBW_RECV_BUFSIZE 256
+#define IBW_MAX_SEND_WR 16
+#define IBW_MAX_RECV_WR 16
+#define IBW_RECV_BUFSIZE 64
#define IBW_RECV_THRESHOLD (1 * 1024 * 1024)
-static void ibw_event_handler_verbs(struct event_context *ev,
- struct fd_event *fde, uint16_t flags, void *private_data);
+static void ibw_event_handler_verbs(struct tevent_context *ev,
+ struct tevent_fd *fde, uint16_t flags, void *private_data);
static int ibw_fill_cq(struct ibw_conn *conn);
static int ibw_wc_recv(struct ibw_conn *conn, struct ibv_wc *wc);
static int ibw_wc_send(struct ibw_conn *conn, struct ibv_wc *wc);
return NULL;
}
+ DEBUG(DEBUG_DEBUG, ("2 ibw_alloc_mr(cmid=%p, n=%u)\n", pconn->cm_id, n));
*ppmr = ibv_reg_mr(pconn->pd, buf, n, IBV_ACCESS_LOCAL_WRITE);
if (!*ppmr) {
sprintf(ibw_lasterr, "couldn't allocate mr\n");
return NULL;
}
+ DEBUG(DEBUG_DEBUG, ("3 ibw_alloc_mr(cmid=%p, n=%u)\n", pconn->cm_id, n));
return buf;
}
return -1;
}
+ pconn->buf_recv = ibw_alloc_mr(pctx, pconn,
+ opts->max_send_wr * opts->recv_bufsize, &pconn->mr_send);
+ if (!pconn->buf_send) {
+ sprintf(ibw_lasterr, "METZEcouldn't allocate work send buf\n");
+ return -1;
+ }
+
pconn->buf_recv = ibw_alloc_mr(pctx, pconn,
opts->max_recv_wr * opts->recv_bufsize, &pconn->mr_recv);
if (!pconn->buf_recv) {
{
DEBUG(DEBUG_DEBUG, ("ibw_ctx_priv_destruct(%p)\n", pctx));
+ /*
+ * tevent_fd must be removed before the fd is closed
+ */
+ TALLOC_FREE(pctx->cm_channel_event);
+
/* destroy cm */
if (pctx->cm_channel) {
rdma_destroy_event_channel(pctx->cm_channel);
pctx->cm_channel = NULL;
}
- if (pctx->cm_channel_event) {
- /* TODO: do we have to do this here? */
- talloc_free(pctx->cm_channel_event);
- pctx->cm_channel_event = NULL;
- }
if (pctx->cm_id) {
rdma_destroy_id(pctx->cm_id);
pctx->cm_id = NULL;
/* pconn->wr_index is freed by talloc */
/* pconn->wr_index[i] are freed by talloc */
+ /*
+ * tevent_fd must be removed before the fd is closed
+ */
+ TALLOC_FREE(pconn->verbs_channel_event);
+
/* destroy verbs */
if (pconn->cm_id!=NULL && pconn->cm_id->qp!=NULL) {
rdma_destroy_qp(pconn->cm_id);
pconn->verbs_channel = NULL;
}
- /* must be freed here because its order is important */
- if (pconn->verbs_channel_event) {
- talloc_free(pconn->verbs_channel_event);
- pconn->verbs_channel_event = NULL;
- }
-
/* free memory regions */
ibw_free_mr(&pconn->buf_send, &pconn->mr_send);
ibw_free_mr(&pconn->buf_recv, &pconn->mr_recv);
}
DEBUG(DEBUG_DEBUG, ("created channel %p\n", pconn->verbs_channel));
- pconn->verbs_channel_event = event_add_fd(pctx->ectx, NULL, /* not pconn or conn */
- pconn->verbs_channel->fd, EVENT_FD_READ, ibw_event_handler_verbs, conn);
+ pconn->verbs_channel_event = tevent_add_fd(pctx->ectx, NULL, /* not pconn or conn */
+ pconn->verbs_channel->fd, TEVENT_FD_READ, ibw_event_handler_verbs, conn);
pconn->pd = ibv_alloc_pd(pconn->cm_id->verbs);
if (!pconn->pd) {
rc = ibv_post_recv(pconn->cm_id->qp, &wr, &bad_wr);
if (rc) {
sprintf(ibw_lasterr, "refill/ibv_post_recv failed with %d\n", rc);
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
return -2;
}
rc = ibv_post_recv(pconn->cm_id->qp, &wr, &bad_wr);
if (rc) {
sprintf(ibw_lasterr, "fill/ibv_post_recv failed with %d\n", rc);
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
return -2;
}
}
return rc;
}
-static void ibw_event_handler_cm(struct event_context *ev,
- struct fd_event *fde, uint16_t flags, void *private_data)
+static void ibw_event_handler_cm(struct tevent_context *ev,
+ struct tevent_fd *fde, uint16_t flags, void *private_data)
{
int rc;
struct ibw_ctx *ctx = talloc_get_type(private_data, struct ibw_ctx);
case RDMA_CM_EVENT_ADDR_ERROR:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_ADDR_ERROR, error %d\n", event->status);
+ goto error;
case RDMA_CM_EVENT_ROUTE_ERROR:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_ROUTE_ERROR, error %d\n", event->status);
+ goto error;
case RDMA_CM_EVENT_CONNECT_ERROR:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status);
+ goto error;
case RDMA_CM_EVENT_UNREACHABLE:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status);
goto error;
return;
}
-static void ibw_event_handler_verbs(struct event_context *ev,
- struct fd_event *fde, uint16_t flags, void *private_data)
+static void ibw_event_handler_verbs(struct tevent_context *ev,
+ struct tevent_fd *fde, uint16_t flags, void *private_data)
{
struct ibw_conn *conn = talloc_get_type(private_data, struct ibw_conn);
struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
error:
ibv_ack_cq_events(pconn->cq, 1);
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
if (conn->state!=IBWC_ERROR) {
conn->state = IBWC_ERROR;
void *ctx_userdata,
ibw_connstate_fn_t ibw_connstate,
ibw_receive_fn_t ibw_receive,
- struct event_context *ectx)
+ struct tevent_context *ectx)
{
struct ibw_ctx *ctx = talloc_zero(NULL, struct ibw_ctx);
struct ibw_ctx_priv *pctx;
goto cleanup;
}
- pctx->cm_channel_event = event_add_fd(pctx->ectx, pctx,
- pctx->cm_channel->fd, EVENT_FD_READ, ibw_event_handler_cm, ctx);
+ pctx->cm_channel_event = tevent_add_fd(pctx->ectx, pctx,
+ pctx->cm_channel->fd, TEVENT_FD_READ, ibw_event_handler_cm, ctx);
#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx, RDMA_PS_TCP);
return ctx;
/* don't put code here */
cleanup:
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
if (ctx)
talloc_free(ctx);
DEBUG(DEBUG_DEBUG, ("ibw_stop\n"));
for(p=ctx->conn_list; p!=NULL; p=p->next) {
- if (ctx->state==IBWC_ERROR || ctx->state==IBWC_CONNECTED) {
+ if (p->state==IBWC_ERROR || p->state==IBWC_CONNECTED) {
if (ibw_disconnect(p))
return -1;
}
rc = rdma_bind_addr(pctx->cm_id, (struct sockaddr *) my_addr);
if (rc) {
sprintf(ibw_lasterr, "rdma_bind_addr error %d\n", rc);
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
return rc;
}
DEBUG(DEBUG_DEBUG, ("rdma_bind_addr successful\n"));
rc = rdma_listen(pctx->cm_id, backlog);
if (rc) {
sprintf(ibw_lasterr, "rdma_listen failed: %d\n", rc);
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
return rc;
}
rc = rdma_accept(pconn->cm_id, &conn_param);
if (rc) {
sprintf(ibw_lasterr, "rdma_accept failed %d\n", rc);
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
return -1;;
}
rc = rdma_resolve_addr(pconn->cm_id, NULL, (struct sockaddr *) serv_addr, 2000);
if (rc) {
sprintf(ibw_lasterr, "rdma_resolve_addr error %d\n", rc);
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
talloc_free(conn);
return -1;
}
rc = rdma_disconnect(pconn->cm_id);
if (rc) {
sprintf(ibw_lasterr, "ibw_disconnect failed with %d\n", rc);
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
return rc;
}
break;
return 0;
error:
- DEBUG(DEBUG_ERR, (ibw_lasterr));
+ DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
return -1;
}
git.samba.org / metze / samba / wip.git / blobdiff