eventscript: don't do debugging system() from inside signal handler

In the case of a timeout, we dump a log of what's happening to a file
in /tmp.  We do it from the signal handler, which is an unreliable hack
(BZ58365).

Instead, create another (lower-priority) child to do the dump, then
kill the timedout script.

Note that this doesn't quite work as intended (the dump is often run
after the script has been killed), so the next patch resolves this.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

diff --git a/common/ctdb_util.c b/common/ctdb_util.c
index 63abc02437820b78ca313a3284c3eaecfdc7eb76..433a2ad8578b29421449984c6af44814f6666af7 100644 (file)
--- a/common/ctdb_util.c
+++ b/common/ctdb_util.c
@@ -339,6 +339,18 @@ void ctdb_high_priority(struct ctdb_context *ctdb)
        }
 }
 
+/*
+  make ourselves slightly nicer: eg. a ctdb child.
+ */
+void ctdb_reduce_priority(struct ctdb_context *ctdb)
+{
+       errno = 0;
+       if (nice(10) == -1 && errno != 0) {
+               DEBUG(DEBUG_WARNING,("Unable to lower priority: %s\n",
+                                    strerror(errno)));
+       }
+}
+
 void set_nonblocking(int fd)
 {
        unsigned v;

diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 888b626e1e6e8cd05ab8994ec4bd590cff626337..b65d1af7b3a5e9308066c559267d51fc5f696d42 100644 (file)
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -1263,6 +1263,7 @@ void ctdb_node_dead(struct ctdb_node *node);
 void ctdb_node_connected(struct ctdb_node *node);
 bool ctdb_blocking_freeze(struct ctdb_context *ctdb);
 void ctdb_high_priority(struct ctdb_context *ctdb);
+void ctdb_reduce_priority(struct ctdb_context *ctdb);
 int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, 
                                 struct ctdb_req_control *c,
                                 TDB_DATA indata, 

diff --git a/server/eventscript.c b/server/eventscript.c
index 5e2772bed6e713577c871c1b8e3f88bec9f98070..41da296031b81e6cce0195c0162ae3d8772dc84e 100644 (file)
--- a/server/eventscript.c
+++ b/server/eventscript.c
@@ -27,11 +27,6 @@
 #include "lib/events/events.h"
 #include "../common/rb_tree.h"
 
-static struct {
-       struct timeval start;
-       const char *script_running;
-} child_state;
-
 static void ctdb_event_script_timeout(struct event_context *ev, struct timed_event *te, struct timeval t, void *p);
 
 /*
@@ -39,21 +34,6 @@ static void ctdb_event_script_timeout(struct event_context *ev, struct timed_eve
  */
 static void sigterm(int sig)
 {
-       char tbuf[100], buf[200];
-       time_t t;
-
-       DEBUG(DEBUG_ERR,("Timed out running script '%s' after %.1f seconds pid :%d\n", 
-                child_state.script_running, timeval_elapsed(&child_state.start), getpid()));
-
-       t = time(NULL);
-
-       strftime(tbuf, sizeof(tbuf)-1, "%Y%m%d%H%M%S",  localtime(&t));
-       sprintf(buf, "{ pstree -p; cat /proc/locks; ls -li /var/ctdb/ /var/ctdb/persistent; }"
-               " >/tmp/ctdb.event.%s.%d", tbuf, getpid());
-       system(buf);
-
-       DEBUG(DEBUG_ERR,("Logged timedout eventscript : %s\n", buf));
-
        /* all the child processes will be running in the same process group */
        kill(-getpgrp(), SIGKILL);
        _exit(1);
@@ -368,7 +348,6 @@ static int child_run_script(struct ctdb_context *ctdb,
        int ret;
        TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 
-       child_state.start = timeval_current();
        ret = child_setup(ctdb);
        if (ret != 0)
                goto out;
@@ -376,7 +355,6 @@ static int child_run_script(struct ctdb_context *ctdb,
        cmdstr = child_command_string(ctdb, tmp_ctx, from_user,
                                      current->name, call, options);
        CTDB_NO_MEMORY(ctdb, cmdstr);
-       child_state.script_running = cmdstr;
 
        DEBUG(DEBUG_DEBUG,("Executing event script %s\n",cmdstr));
 
@@ -518,6 +496,42 @@ static void ctdb_event_script_handler(struct event_context *ev, struct fd_event
        }
 }
 
+static void debug_timeout(struct ctdb_event_script_state *state)
+{
+       struct ctdb_script_wire *current = get_current_script(state);
+       char *cmd;
+       pid_t pid;
+       time_t t;
+       char tbuf[100], buf[200];
+
+       cmd = child_command_string(state->ctdb, state,
+                                  state->from_user, current->name,
+                                  state->call, state->options);
+       CTDB_NO_MEMORY_VOID(state->ctdb, cmd);
+
+       DEBUG(DEBUG_ERR,("Timed out running script '%s' after %.1f seconds pid :%d\n",
+                        cmd, timeval_elapsed(&current->start), state->child));
+       talloc_free(cmd);
+
+       t = time(NULL);
+       strftime(tbuf, sizeof(tbuf)-1, "%Y%m%d%H%M%S",  localtime(&t));
+       sprintf(buf, "{ pstree -p; cat /proc/locks; ls -li /var/ctdb/ /var/ctdb/persistent; }"
+                       " >/tmp/ctdb.event.%s.%d", tbuf, getpid());
+
+       pid = fork();
+       if (pid == 0) {
+               ctdb_reduce_priority(state->ctdb);
+               system(buf);
+               exit(0);
+       }
+       if (pid == -1) {
+               DEBUG(DEBUG_ERR,("Fork for debug script failed : %s\n",
+                                strerror(errno)));
+       } else {
+               DEBUG(DEBUG_ERR,("Logged timedout eventscript : %s\n", buf));
+       }
+}
+
 /* called when child times out */
 static void ctdb_event_script_timeout(struct event_context *ev, struct timed_event *te, 
                                      struct timeval t, void *p)
@@ -543,6 +557,7 @@ static void ctdb_event_script_timeout(struct event_context *ev, struct timed_eve
                break;
         default:
                state->scripts->scripts[state->current].status = -ETIME;
+               debug_timeout(state);
        }
 
        if (kill(state->child, 0) != 0) {