ctdb: use mlockall, cautiously

We don't want ctdb stalling due to paging; this can be far worse than
scheduling delays.  But if we simply do mlockall(MCL_FUTURE), it
increases the risk that mmap (ie. tdb open) or malloc will fail,
causing us to abort.

This patch is a compromise: we mlock all current pages (including
10k of future stack for expansion) and then relock when a client
asks us to open a TDB.  We warn, but don't exit, if it fails.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

diff --git a/common/ctdb_util.c b/common/ctdb_util.c
index fe2ada45ff54d6a7efae08e49c19c16655146654..0b34f2adda427180ec1b640ba92d25c1763147bb 100644 (file)
--- a/common/ctdb_util.c
+++ b/common/ctdb_util.c
@@ -23,6 +23,7 @@
 #include "system/network.h"
 #include "system/filesys.h"
 #include "system/wait.h"
+#include "system/shmem.h"
 #include "../include/ctdb_private.h"
 
 int LogLevel = DEBUG_NOTICE;
@@ -629,6 +630,28 @@ int32_t get_debug_by_desc(const char *desc)
        return DEBUG_ERR;
 }
 
+/* we don't lock future pages here; it would increase the chance that
+ * we'd fail to mmap later on. */
+void ctdb_lockdown_memory(struct ctdb_context *ctdb)
+{
+#ifdef HAVE_MLOCKALL
+       /* Extra stack, please! */
+       char dummy[10000];
+       memset(dummy, 0, sizeof(dummy));
+
+       if (ctdb->valgrinding) {
+               return;
+       }
+
+       /* Avoid compiler optimizing out dummy. */
+       mlock(dummy, sizeof(dummy));
+       if (mlockall(MCL_CURRENT) != 0) {
+               DEBUG(DEBUG_WARNING,("Failed to lock memory: %s'\n",
+                                    strerror(errno)));
+       }
+#endif
+}
+
 const char *ctdb_eventscript_call_names[] = {
        "startup",
        "startrecovery",

diff --git a/configure.ac b/configure.ac
index aaec420c086ec0e6263a7ed3ee461b42102f0af8..d4098a6a5ac0d5a16115b0377ffde38498eec3f1 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -71,6 +71,7 @@ m4_include(ib/config.m4)
 
 AC_CHECK_HEADERS(sched.h)
 AC_CHECK_FUNCS(sched_setscheduler)
+AC_CHECK_FUNCS(mlockall)
 
 AC_CACHE_CHECK([for sin_len in sock],ctdb_cv_HAVE_SOCK_SIN_LEN,[
 AC_TRY_COMPILE([#include <sys/types.h>

diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 9fc2be74d147e38b45a95bffbcebd88bd3ba08d4..de74bb88fdf29b7ffae0de4ad35cdc2aca73ab77 100644 (file)
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -1442,7 +1442,7 @@ void ctdb_block_signal(int signum);
 void ctdb_unblock_signal(int signum);
 int32_t ctdb_monitoring_mode(struct ctdb_context *ctdb);
 int ctdb_set_child_logging(struct ctdb_context *ctdb);
-
+void ctdb_lockdown_memory(struct ctdb_context *ctdb);
 
 typedef void (*client_async_callback)(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data);
 

diff --git a/server/ctdb_daemon.c b/server/ctdb_daemon.c
index 97051625353a85ab4ddbb820d96f7593e38c4c49..cc076fbc8390e795c8bd8649af9de51f90a44929 100644 (file)
--- a/server/ctdb_daemon.c
+++ b/server/ctdb_daemon.c
@@ -805,6 +805,7 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
                }
        }
 
+       ctdb_lockdown_memory(ctdb);
          
        /* go into a wait loop to allow other nodes to complete */
        event_loop_wait(ctdb->ev);

diff --git a/server/ctdb_ltdb_server.c b/server/ctdb_ltdb_server.c
index 6c204821dcb2485c177215d13a1c87e79a987ad1..c24d4214d54f90ae3005b79dbc77eddb905be9a7 100644 (file)
--- a/server/ctdb_ltdb_server.c
+++ b/server/ctdb_ltdb_server.c
@@ -361,6 +361,9 @@ int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata,
        outdata->dptr  = (uint8_t *)&db->db_id;
        outdata->dsize = sizeof(db->db_id);
 
+       /* Try to ensure it's locked in mem */
+       ctdb_lockdown_memory(ctdb);
+
        /* tell all the other nodes about this database */
        ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL, 0,
                                 persistent?CTDB_CONTROL_DB_ATTACH_PERSISTENT: