3 provide API to do non-blocking locks for single or all databases
5 Copyright (C) Amitay Isaacs 2012
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #include "include/ctdb_private.h"
22 #include "include/ctdb_protocol.h"
26 #include "system/filesys.h"
27 #include "lib/util/dlinklist.h"
30 * Non-blocking Locking API
32 * 1. Create a child process to do blocking locks.
33 * 2. Once the locks are obtained, signal parent process via fd.
34 * 3. Invoke registered callback routine with locking status.
35 * 4. If the child process cannot get locks within certain time,
36 * diagnose using /proc/locks and log warning message
38 * ctdb_lock_record() - get a lock on a record
39 * ctdb_lock_db() - get a lock on a DB
40 * ctdb_lock_alldb_prio() - get a lock on all DBs with given priority
41 * ctdb_lock_alldb() - get a lock on all DBs
43 * auto_mark - whether to mark/unmark DBs in before/after callback
46 /* FIXME: Add a tunable max_lock_processes_per_db */
47 #define MAX_LOCK_PROCESSES_PER_DB (100)
56 static const char * const lock_type_str[] = {
65 /* lock_context is the common part for a lock request */
67 struct lock_context *next, *prev;
69 struct ctdb_context *ctdb;
70 struct ctdb_db_context *ctdb_db;
74 struct lock_request *request;
77 struct tevent_fd *tfd;
78 struct tevent_timer *ttimer;
79 struct timeval start_time;
84 /* lock_request is the client specific part for a lock request */
86 struct lock_context *lctx;
87 void (*callback)(void *, bool);
93 * Support samba 3.6.x (and older) versions which do not set db priority.
95 * By default, all databases are set to priority 1. So only when priority
96 * is set to 1, check for databases that need higher priority.
98 static bool later_db(struct ctdb_context *ctdb, const char *name)
100 if (ctdb->tunable.samba3_hack == 0) {
104 if (strstr(name, "brlock") ||
105 strstr(name, "g_lock") ||
106 strstr(name, "notify_onelevel") ||
107 strstr(name, "serverid") ||
108 strstr(name, "xattr_tdb")) {
115 typedef int (*db_handler_t)(struct ctdb_db_context *ctdb_db,
119 static int ctdb_db_iterator(struct ctdb_context *ctdb, uint32_t priority,
120 db_handler_t handler, void *private_data)
122 struct ctdb_db_context *ctdb_db;
125 for (ctdb_db = ctdb->db_list; ctdb_db; ctdb_db = ctdb_db->next) {
126 if (ctdb_db->priority != priority) {
129 if (later_db(ctdb, ctdb_db->db_name)) {
132 ret = handler(ctdb_db, priority, private_data);
138 /* If priority != 1, later_db check is not required and can return */
143 for (ctdb_db = ctdb->db_list; ctdb_db; ctdb_db = ctdb_db->next) {
144 if (!later_db(ctdb, ctdb_db->db_name)) {
147 ret = handler(ctdb_db, priority, private_data);
158 * lock all databases - mark only
160 static int db_lock_mark_handler(struct ctdb_db_context *ctdb_db, uint32_t priority,
163 int tdb_transaction_write_lock_mark(struct tdb_context *);
165 DEBUG(DEBUG_INFO, ("marking locked database %s, priority:%u\n",
166 ctdb_db->db_name, priority));
168 if (tdb_transaction_write_lock_mark(ctdb_db->ltdb->tdb) != 0) {
169 DEBUG(DEBUG_ERR, ("Failed to mark (transaction lock) database %s\n",
174 if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
175 DEBUG(DEBUG_ERR, ("Failed to mark (all lock) database %s\n",
183 int ctdb_lockall_mark_prio(struct ctdb_context *ctdb, uint32_t priority)
186 * This function is only used by the main dameon during recovery.
187 * At this stage, the databases have already been locked, by a
188 * dedicated child process. The freeze_mode variable is used to track
189 * whether the actual locks are held by the child process or not.
192 if (ctdb->freeze_mode[priority] != CTDB_FREEZE_FROZEN) {
193 DEBUG(DEBUG_ERR, ("Attempt to mark all databases locked when not frozen\n"));
197 return ctdb_db_iterator(ctdb, priority, db_lock_mark_handler, NULL);
200 static int ctdb_lockall_mark(struct ctdb_context *ctdb)
204 for (priority=1; priority<=NUM_DB_PRIORITIES; priority++) {
205 if (ctdb_db_iterator(ctdb, priority, db_lock_mark_handler, NULL) != 0) {
215 * lock all databases - unmark only
217 static int db_lock_unmark_handler(struct ctdb_db_context *ctdb_db, uint32_t priority,
220 int tdb_transaction_write_lock_unmark(struct tdb_context *);
222 DEBUG(DEBUG_INFO, ("unmarking locked database %s, priority:%u\n",
223 ctdb_db->db_name, priority));
225 if (tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb) != 0) {
226 DEBUG(DEBUG_ERR, ("Failed to unmark (transaction lock) database %s\n",
231 if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
232 DEBUG(DEBUG_ERR, ("Failed to unmark (all lock) database %s\n",
240 int ctdb_lockall_unmark_prio(struct ctdb_context *ctdb, uint32_t priority)
243 * This function is only used by the main daemon during recovery.
244 * At this stage, the databases have already been locked, by a
245 * dedicated child process. The freeze_mode variable is used to track
246 * whether the actual locks are held by the child process or not.
249 if (ctdb->freeze_mode[priority] != CTDB_FREEZE_FROZEN) {
250 DEBUG(DEBUG_ERR, ("Attempt to unmark all databases locked when not frozen\n"));
254 return ctdb_db_iterator(ctdb, priority, db_lock_unmark_handler, NULL);
257 static int ctdb_lockall_unmark(struct ctdb_context *ctdb)
261 for (priority=NUM_DB_PRIORITIES; priority>0; priority--) {
262 if (ctdb_db_iterator(ctdb, priority, db_lock_unmark_handler, NULL) != 0) {
271 static void ctdb_lock_schedule(struct ctdb_context *ctdb);
274 * Destructor to kill the child locking process
276 static int ctdb_lock_context_destructor(struct lock_context *lock_ctx)
278 if (lock_ctx->child > 0) {
279 ctdb_kill(lock_ctx->ctdb, lock_ctx->child, SIGKILL);
280 DLIST_REMOVE(lock_ctx->ctdb->lock_current, lock_ctx);
281 if (lock_ctx->ctdb_db) {
282 lock_ctx->ctdb_db->lock_num_current--;
284 CTDB_DECREMENT_STAT(lock_ctx->ctdb, locks.num_current);
285 if (lock_ctx->type == LOCK_RECORD || lock_ctx->type == LOCK_DB) {
286 CTDB_DECREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_current);
289 DLIST_REMOVE(lock_ctx->ctdb->lock_pending, lock_ctx);
290 lock_ctx->ctdb->lock_num_pending--;
291 CTDB_DECREMENT_STAT(lock_ctx->ctdb, locks.num_pending);
292 if (lock_ctx->type == LOCK_RECORD || lock_ctx->type == LOCK_DB) {
293 CTDB_DECREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_pending);
297 ctdb_lock_schedule(lock_ctx->ctdb);
304 * Destructor to remove lock request
306 static int ctdb_lock_request_destructor(struct lock_request *lock_request)
308 lock_request->lctx->request = NULL;
312 void ctdb_lock_free_request_context(struct lock_request *lock_req)
314 struct lock_context *lock_ctx;
316 lock_ctx = lock_req->lctx;
317 talloc_free(lock_req);
318 talloc_free(lock_ctx);
323 * Process all the callbacks waiting for lock
325 * If lock has failed, callback is executed with locked=false
327 static void process_callbacks(struct lock_context *lock_ctx, bool locked)
329 struct lock_request *request;
331 if (lock_ctx->auto_mark && locked) {
332 switch (lock_ctx->type) {
334 tdb_chainlock_mark(lock_ctx->ctdb_db->ltdb->tdb, lock_ctx->key);
338 tdb_lockall_mark(lock_ctx->ctdb_db->ltdb->tdb);
341 case LOCK_ALLDB_PRIO:
342 ctdb_lockall_mark_prio(lock_ctx->ctdb, lock_ctx->priority);
346 ctdb_lockall_mark(lock_ctx->ctdb);
351 request = lock_ctx->request;
352 if (lock_ctx->auto_mark) {
353 /* Reset the destructor, so request is not removed from the list */
354 talloc_set_destructor(request, NULL);
356 request->callback(request->private_data, locked);
358 if (lock_ctx->auto_mark && locked) {
359 switch (lock_ctx->type) {
361 tdb_chainlock_unmark(lock_ctx->ctdb_db->ltdb->tdb, lock_ctx->key);
365 tdb_lockall_unmark(lock_ctx->ctdb_db->ltdb->tdb);
368 case LOCK_ALLDB_PRIO:
369 ctdb_lockall_unmark_prio(lock_ctx->ctdb, lock_ctx->priority);
373 ctdb_lockall_unmark(lock_ctx->ctdb);
380 static int lock_bucket_id(double t)
382 double ms = 1.e-3, s = 1;
387 } else if (t < 10*ms) {
389 } else if (t < 100*ms) {
391 } else if (t < 1*s) {
393 } else if (t < 2*s) {
395 } else if (t < 4*s) {
397 } else if (t < 8*s) {
399 } else if (t < 16*s) {
401 } else if (t < 32*s) {
403 } else if (t < 64*s) {
413 * Callback routine when the required locks are obtained.
414 * Called from parent context
416 static void ctdb_lock_handler(struct tevent_context *ev,
417 struct tevent_fd *tfd,
421 struct lock_context *lock_ctx;
422 TALLOC_CTX *tmp_ctx = NULL;
428 lock_ctx = talloc_get_type_abort(private_data, struct lock_context);
430 /* cancel the timeout event */
431 if (lock_ctx->ttimer) {
432 TALLOC_FREE(lock_ctx->ttimer);
435 t = timeval_elapsed(&lock_ctx->start_time);
436 id = lock_bucket_id(t);
438 if (lock_ctx->auto_mark) {
439 tmp_ctx = talloc_new(ev);
440 talloc_steal(tmp_ctx, lock_ctx);
443 /* Read the status from the child process */
444 if (read(lock_ctx->fd[0], &c, 1) != 1) {
447 locked = (c == 0 ? true : false);
450 /* Update statistics */
451 CTDB_DECREMENT_STAT(lock_ctx->ctdb, locks.num_pending);
452 CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.num_calls);
453 if (lock_ctx->ctdb_db) {
454 CTDB_DECREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_pending);
455 CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_calls);
459 if (lock_ctx->ctdb_db) {
460 CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.buckets[id]);
461 CTDB_UPDATE_LATENCY(lock_ctx->ctdb, lock_ctx->ctdb_db,
462 lock_type_str[lock_ctx->type], locks.latency,
463 lock_ctx->start_time);
465 CTDB_UPDATE_DB_LATENCY(lock_ctx->ctdb_db, lock_type_str[lock_ctx->type], locks.latency, t);
466 CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db, locks.buckets[id]);
469 CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.num_failed);
470 if (lock_ctx->ctdb_db) {
471 CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_failed);
475 process_callbacks(lock_ctx, locked);
477 if (lock_ctx->auto_mark) {
478 talloc_free(tmp_ctx);
484 * Callback routine when required locks are not obtained within timeout
485 * Called from parent context
487 static void ctdb_lock_timeout_handler(struct tevent_context *ev,
488 struct tevent_timer *ttimer,
489 struct timeval current_time,
492 static const char * debug_locks = NULL;
493 struct lock_context *lock_ctx;
494 struct ctdb_context *ctdb;
497 lock_ctx = talloc_get_type_abort(private_data, struct lock_context);
498 ctdb = lock_ctx->ctdb;
500 if (lock_ctx->type == LOCK_RECORD || lock_ctx->type == LOCK_DB) {
502 ("Unable to get %s lock on database %s for %.0lf seconds\n",
503 (lock_ctx->type == LOCK_RECORD ? "RECORD" : "DB"),
504 lock_ctx->ctdb_db->db_name,
505 timeval_elapsed(&lock_ctx->start_time)));
508 ("Unable to get ALLDB locks for %.0lf seconds\n",
509 timeval_elapsed(&lock_ctx->start_time)));
512 /* Fire a child process to find the blocking process. */
513 if (debug_locks == NULL) {
514 debug_locks = getenv("CTDB_DEBUG_LOCKS");
515 if (debug_locks == NULL) {
516 debug_locks = talloc_asprintf(ctdb,
518 getenv("CTDB_BASE"));
521 if (debug_locks != NULL) {
524 execl(debug_locks, debug_locks, NULL);
527 ctdb_track_child(ctdb, pid);
531 " Unable to setup lock debugging - no memory?\n"));
534 /* reset the timeout timer */
535 // talloc_free(lock_ctx->ttimer);
536 lock_ctx->ttimer = tevent_add_timer(ctdb->ev,
538 timeval_current_ofs(10, 0),
539 ctdb_lock_timeout_handler,
544 static int db_count_handler(struct ctdb_db_context *ctdb_db, uint32_t priority,
547 int *count = (int *)private_data;
559 static int db_name_handler(struct ctdb_db_context *ctdb_db, uint32_t priority,
562 struct db_namelist *list = (struct db_namelist *)private_data;
564 list->names[list->n] = talloc_strdup(list->names, ctdb_db->db_path);
570 static char **lock_helper_args(TALLOC_CTX *mem_ctx, struct lock_context *lock_ctx, int fd)
572 struct ctdb_context *ctdb = lock_ctx->ctdb;
576 struct db_namelist list;
578 switch (lock_ctx->type) {
587 case LOCK_ALLDB_PRIO:
589 ctdb_db_iterator(ctdb, lock_ctx->priority, db_count_handler, &nargs);
594 for (priority=1; priority<NUM_DB_PRIORITIES; priority++) {
595 ctdb_db_iterator(ctdb, priority, db_count_handler, &nargs);
600 /* Add extra argument for null termination */
603 args = talloc_array(mem_ctx, char *, nargs);
608 args[0] = talloc_strdup(args, "ctdb_lock_helper");
609 args[1] = talloc_asprintf(args, "%d", getpid());
610 args[2] = talloc_asprintf(args, "%d", fd);
612 switch (lock_ctx->type) {
614 args[3] = talloc_strdup(args, "RECORD");
615 args[4] = talloc_strdup(args, lock_ctx->ctdb_db->db_path);
616 if (lock_ctx->key.dsize == 0) {
617 args[5] = talloc_strdup(args, "NULL");
619 args[5] = hex_encode_talloc(args, lock_ctx->key.dptr, lock_ctx->key.dsize);
624 args[3] = talloc_strdup(args, "DB");
625 args[4] = talloc_strdup(args, lock_ctx->ctdb_db->db_path);
628 case LOCK_ALLDB_PRIO:
629 args[3] = talloc_strdup(args, "DB");
632 ctdb_db_iterator(ctdb, lock_ctx->priority, db_name_handler, &list);
636 args[3] = talloc_strdup(args, "DB");
639 for (priority=1; priority<NUM_DB_PRIORITIES; priority++) {
640 ctdb_db_iterator(ctdb, priority, db_name_handler, &list);
645 /* Make sure last argument is NULL */
646 args[nargs-1] = NULL;
648 for (i=0; i<nargs-1; i++) {
649 if (args[i] == NULL) {
660 * Schedule a new lock child process
661 * Set up callback handler and timeout handler
663 static void ctdb_lock_schedule(struct ctdb_context *ctdb)
665 struct lock_context *lock_ctx, *next_ctx;
668 const char *helper = BINDIR "/ctdb_lock_helper";
669 static const char *prog = NULL;
675 t = getenv("CTDB_LOCK_HELPER");
677 prog = talloc_strdup(ctdb, t);
679 prog = talloc_strdup(ctdb, helper);
681 CTDB_NO_MEMORY_VOID(ctdb, prog);
684 if (ctdb->lock_pending == NULL) {
688 /* Find a lock context with requests */
689 lock_ctx = ctdb->lock_pending;
690 while (lock_ctx != NULL) {
691 next_ctx = lock_ctx->next;
692 if (! lock_ctx->request) {
693 DEBUG(DEBUG_INFO, ("Removing lock context without lock request\n"));
694 DLIST_REMOVE(ctdb->lock_pending, lock_ctx);
695 ctdb->lock_num_pending--;
696 CTDB_DECREMENT_STAT(ctdb, locks.num_pending);
697 if (lock_ctx->ctdb_db) {
698 CTDB_DECREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_pending);
700 talloc_free(lock_ctx);
702 if (lock_ctx->ctdb_db == NULL ||
703 lock_ctx->ctdb_db->lock_num_current < MAX_LOCK_PROCESSES_PER_DB) {
704 /* Found a lock context with lock requests */
711 if (lock_ctx == NULL) {
715 lock_ctx->child = -1;
716 ret = pipe(lock_ctx->fd);
718 DEBUG(DEBUG_ERR, ("Failed to create pipe in ctdb_lock_schedule\n"));
722 set_close_on_exec(lock_ctx->fd[0]);
724 /* Create data for child process */
725 tmp_ctx = talloc_new(lock_ctx);
726 if (tmp_ctx == NULL) {
727 DEBUG(DEBUG_ERR, ("Failed to allocate memory for helper args\n"));
728 close(lock_ctx->fd[0]);
729 close(lock_ctx->fd[1]);
733 /* Create arguments for lock helper */
734 args = lock_helper_args(tmp_ctx, lock_ctx, lock_ctx->fd[1]);
736 DEBUG(DEBUG_ERR, ("Failed to create lock helper args\n"));
737 close(lock_ctx->fd[0]);
738 close(lock_ctx->fd[1]);
739 talloc_free(tmp_ctx);
743 lock_ctx->child = vfork();
745 if (lock_ctx->child == (pid_t)-1) {
746 DEBUG(DEBUG_ERR, ("Failed to create a child in ctdb_lock_schedule\n"));
747 close(lock_ctx->fd[0]);
748 close(lock_ctx->fd[1]);
749 talloc_free(tmp_ctx);
755 if (lock_ctx->child == 0) {
756 ret = execv(prog, args);
758 DEBUG(DEBUG_ERR, ("Failed to execute helper %s (%d, %s)\n",
759 prog, errno, strerror(errno)));
765 ctdb_track_child(ctdb, lock_ctx->child);
766 close(lock_ctx->fd[1]);
768 talloc_set_destructor(lock_ctx, ctdb_lock_context_destructor);
770 talloc_free(tmp_ctx);
772 /* Set up timeout handler */
773 lock_ctx->ttimer = tevent_add_timer(ctdb->ev,
775 timeval_current_ofs(10, 0),
776 ctdb_lock_timeout_handler,
778 if (lock_ctx->ttimer == NULL) {
779 ctdb_kill(ctdb, lock_ctx->child, SIGKILL);
780 lock_ctx->child = -1;
781 talloc_set_destructor(lock_ctx, NULL);
782 close(lock_ctx->fd[0]);
786 /* Set up callback */
787 lock_ctx->tfd = tevent_add_fd(ctdb->ev,
793 if (lock_ctx->tfd == NULL) {
794 TALLOC_FREE(lock_ctx->ttimer);
795 ctdb_kill(ctdb, lock_ctx->child, SIGKILL);
796 lock_ctx->child = -1;
797 talloc_set_destructor(lock_ctx, NULL);
798 close(lock_ctx->fd[0]);
801 tevent_fd_set_auto_close(lock_ctx->tfd);
803 /* Move the context from pending to current */
804 DLIST_REMOVE(ctdb->lock_pending, lock_ctx);
805 ctdb->lock_num_pending--;
806 DLIST_ADD_END(ctdb->lock_current, lock_ctx, NULL);
807 if (lock_ctx->ctdb_db) {
808 lock_ctx->ctdb_db->lock_num_current++;
809 CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.num_current);
810 CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_current);
816 * Lock record / db depending on type
818 static struct lock_request *ctdb_lock_internal(struct ctdb_context *ctdb,
819 struct ctdb_db_context *ctdb_db,
822 void (*callback)(void *, bool),
827 struct lock_context *lock_ctx = NULL;
828 struct lock_request *request;
830 if (callback == NULL) {
831 DEBUG(DEBUG_WARNING, ("No callback function specified, not locking\n"));
835 lock_ctx = talloc_zero(ctdb, struct lock_context);
836 if (lock_ctx == NULL) {
837 DEBUG(DEBUG_ERR, ("Failed to create a new lock context\n"));
841 if ((request = talloc_zero(lock_ctx, struct lock_request)) == NULL) {
842 talloc_free(lock_ctx);
846 lock_ctx->type = type;
847 lock_ctx->ctdb = ctdb;
848 lock_ctx->ctdb_db = ctdb_db;
849 lock_ctx->key.dsize = key.dsize;
851 lock_ctx->key.dptr = talloc_memdup(lock_ctx, key.dptr, key.dsize);
852 if (lock_ctx->key.dptr == NULL) {
853 DEBUG(DEBUG_ERR, (__location__ "Memory allocation error\n"));
854 talloc_free(lock_ctx);
857 lock_ctx->key_hash = ctdb_hash(&key);
859 lock_ctx->key.dptr = NULL;
861 lock_ctx->priority = priority;
862 lock_ctx->auto_mark = auto_mark;
864 lock_ctx->request = request;
865 lock_ctx->child = -1;
867 DLIST_ADD_END(ctdb->lock_pending, lock_ctx, NULL);
868 ctdb->lock_num_pending++;
869 CTDB_INCREMENT_STAT(ctdb, locks.num_pending);
871 CTDB_INCREMENT_DB_STAT(ctdb_db, locks.num_pending);
874 /* Start the timer when we activate the context */
875 lock_ctx->start_time = timeval_current();
877 request->lctx = lock_ctx;
878 request->callback = callback;
879 request->private_data = private_data;
881 talloc_set_destructor(request, ctdb_lock_request_destructor);
883 ctdb_lock_schedule(ctdb);
890 * obtain a lock on a record in a database
892 struct lock_request *ctdb_lock_record(struct ctdb_db_context *ctdb_db,
895 void (*callback)(void *, bool),
898 return ctdb_lock_internal(ctdb_db->ctdb,
910 * obtain a lock on a database
912 struct lock_request *ctdb_lock_db(struct ctdb_db_context *ctdb_db,
914 void (*callback)(void *, bool),
917 return ctdb_lock_internal(ctdb_db->ctdb,
929 * obtain locks on all databases of specified priority
931 struct lock_request *ctdb_lock_alldb_prio(struct ctdb_context *ctdb,
934 void (*callback)(void *, bool),
937 if (priority < 1 || priority > NUM_DB_PRIORITIES) {
938 DEBUG(DEBUG_ERR, ("Invalid db priority: %u\n", priority));
942 return ctdb_lock_internal(ctdb,
954 * obtain locks on all databases
956 struct lock_request *ctdb_lock_alldb(struct ctdb_context *ctdb,
958 void (*callback)(void *, bool),
961 return ctdb_lock_internal(ctdb,