1 Added some DB-access routines to help rsync keep extra filesystem info
2 about the files it is dealing with. This adds both the --db=CONFIG_FILE
3 option and the "db config" daemon parameter.
5 Future improvements may include:
7 - Updating of MD4 checksums when transferring any file, even w/o -c.
8 To make that work we'd need to make the sender force checksum_seed to
9 0 when using a DB and having the receiving side check to see if it
10 got a 0 checksum_seed.
12 - Caching of path info that allows for the finding of files to use for
13 moving/linking/copying/alternate-basis-use.
15 - Extend DB support beyond MySQL and SQLite (PostgreSQL?).
17 To use this patch, run these commands for a successful build:
19 patch -p1 <patches/db.diff
24 based-on: c4a3f55be35726d0a033996dc37b0fb248b45cb5
25 diff --git a/.gitignore b/.gitignore
28 @@ -16,6 +16,7 @@ aclocal.m4
36 @@ -24,6 +25,7 @@ aclocal.m4
44 diff --git a/Makefile.in b/Makefile.in
47 @@ -6,6 +6,7 @@ datarootdir=@datarootdir@
48 exec_prefix=@exec_prefix@
55 @@ -29,7 +30,7 @@ VERSION=@RSYNC_VERSION@
59 -GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsyncd.conf.5
60 +GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsyncdb.1 rsyncd.conf.5
61 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
63 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
64 @@ -39,7 +40,7 @@ zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
65 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
66 util.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
67 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
68 - fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
69 + fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
70 OBJS3=progress.o pipe.o
71 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
72 popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
73 @@ -63,14 +64,17 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash.
74 $(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -c $< @CC_SHOBJ_FLAG@
77 -all: Makefile rsync$(EXEEXT) rsync-ssl stunnel-rsync stunnel-rsyncd.conf @MAKE_MAN@
78 +all: Makefile rsync$(EXEEXT) rsyncdb$(EXEEXT) rsync-ssl stunnel-rsync stunnel-rsyncd.conf @MAKE_MAN@
81 - -${MKDIR_P} ${DESTDIR}${bindir}
82 + -${MKDIR_P} ${DESTDIR}${bindir} ${DESTDIR}${sbindir}
83 ${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir}
84 + rsync -ilt rsyncdb$(EXEEXT) ${DESTDIR}${bindir}/
85 + ${INSTALLCMD} -m 755 rsyncdb-mountinfo ${DESTDIR}${sbindir}
86 -${MKDIR_P} ${DESTDIR}${mandir}/man1
87 -${MKDIR_P} ${DESTDIR}${mandir}/man5
88 if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi
89 + if test -f rsyncdb.1; then ${INSTALLMAN} -m 644 rsyncdb.1 ${DESTDIR}${mandir}/man1; fi
90 if test -f rsyncd.conf.5; then ${INSTALLMAN} -m 644 rsyncd.conf.5 ${DESTDIR}${mandir}/man5; fi
92 install-ssl-client: rsync-ssl stunnel-rsync
93 @@ -93,6 +97,9 @@ install-strip:
94 rsync$(EXEEXT): $(OBJS)
95 $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
97 +rsyncdb$(EXEEXT): rsync$(EXEEXT)
98 + ln -s rsync$(EXEEXT) rsyncdb$(EXEEXT)
101 $(CHECK_OBJS): $(HEADERS)
103 @@ -212,22 +219,27 @@ proto.h: proto.h-tstamp
104 proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h
105 perl $(srcdir)/mkproto.pl $(srcdir)/*.c $(srcdir)/lib/compat.c
107 -man: rsync.1 rsyncd.conf.5 man-copy
108 +man: rsync.1 rsyncdb.1 rsyncd.conf.5 man-copy
111 @if test -f rsync.1; then :; elif test -f $(srcdir)/rsync.1; then echo 'Copying srcdir rsync.1'; cp -p $(srcdir)/rsync.1 .; else echo "NOTE: rsync.1 cannot be created."; fi
112 + @if test -f rsyncdb.1; then :; elif test -f $(srcdir)/rsyncdb.1; then echo 'Copying srcdir rsyncdb.1'; cp -p $(srcdir)/rsyncdb.1 .; else echo "NOTE: rsyncdb.1 cannot be created."; fi
113 @if test -f rsyncd.conf.5; then :; elif test -f $(srcdir)/rsyncd.conf.5; then echo 'Copying srcdir rsyncd.conf.5'; cp -p $(srcdir)/rsyncd.conf.5 .; else echo "NOTE: rsyncd.conf.5 cannot be created."; fi
116 yodl2man -o rsync.1 $(srcdir)/rsync.yo
117 -$(srcdir)/tweak_manpage rsync.1
119 +rsyncdb.1: rsyncdb.yo
120 + yodl2man -o rsyncdb.1 $(srcdir)/rsyncdb.yo
121 + -$(srcdir)/tweak_manpage rsyncdb.1
123 rsyncd.conf.5: rsyncd.conf.yo
124 yodl2man -o rsyncd.conf.5 $(srcdir)/rsyncd.conf.yo
125 -$(srcdir)/tweak_manpage rsyncd.conf.5
128 - rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \
129 + rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) rsyncdb$(EXEEXT) \
130 rounding rounding.h *.old
133 diff --git a/checksum.c b/checksum.c
137 extern int checksum_seed;
138 extern int protocol_version;
139 extern int proper_seed_order;
141 extern char *checksum_choice;
144 @@ -220,6 +221,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
145 md5_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
147 md5_result(&m, (uchar *)sum);
149 + db_set_checksum(5, st_p, sum);
153 @@ -241,6 +244,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
154 mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
156 mdfour_result(&m, (uchar *)sum);
158 + db_set_checksum(4, st_p, sum);
161 rprintf(FERROR, "invalid checksum-choice for the --checksum option (%d)\n", checksum_type);
162 diff --git a/cleanup.c b/cleanup.c
165 @@ -27,6 +27,7 @@ extern int am_server;
166 extern int am_daemon;
167 extern int am_receiver;
170 extern int keep_partial;
171 extern int got_xfer_error;
172 extern int protocol_version;
173 @@ -142,6 +143,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
178 + db_disconnect(False);
183 if (cleanup_child_pid != -1) {
185 int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
186 diff --git a/clientserver.c b/clientserver.c
189 @@ -42,12 +42,15 @@ extern int numeric_ids;
190 extern int filesfrom_fd;
191 extern int remote_protocol;
192 extern int protocol_version;
193 +extern int always_checksum;
195 extern int io_timeout;
196 extern int no_detach;
197 extern int write_batch;
198 extern int default_af_hint;
199 extern int logfile_format_has_i;
200 extern int logfile_format_has_o_or_i;
201 +extern char *db_config;
202 extern char *bind_address;
203 extern char *config_file;
204 extern char *logfile_format;
205 @@ -687,6 +690,11 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
209 + if (*lp_db_config(i)) {
210 + db_read_config(FLOG, lp_db_config(i));
211 + db_lax = lp_db_lax(i);
215 if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) {
217 @@ -894,6 +902,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
219 am_server = 1; /* Don't let someone try to be tricky. */
223 if (lp_ignore_errors(module_id))
226 diff --git a/configure.ac b/configure.ac
229 @@ -344,6 +344,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
230 unistd.h utime.h grp.h compat.h sys/param.h ctype.h sys/wait.h \
231 sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h \
232 sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
233 + mysql/mysql.h sqlite3.h \
234 netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
235 sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \
236 popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \
237 @@ -1099,6 +1100,48 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
241 +AC_MSG_CHECKING([whether to include mysql DB support])
242 +AC_ARG_ENABLE(mysql,
243 + AC_HELP_STRING([--disable-mysql],
244 + [disable mysql DB support]))
246 +if test x"$enable_mysql" = x"no"; then
249 + AC_MSG_RESULT([yes])
250 + AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
251 + if test x$MYSQL_CONFIG = x1; then
252 + AC_MSG_CHECKING(for mysql version >= 4)
253 + mysql_version=`mysql_config --version`
254 + mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
255 + if test $mysql_major_version -lt 4; then
256 + AC_MSG_RESULT(no.. skipping MySQL)
260 + MYSQL_CFLAGS=`mysql_config --cflags`
261 + MYSQL_LIBS=`mysql_config --libs`
263 + CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
264 + LIBS="$MYSQL_LIBS $LIBS"
266 + AC_CHECK_LIB(mysqlclient, mysql_init)
271 +AC_MSG_CHECKING([whether to include sqlite DB support])
272 +AC_ARG_ENABLE(sqlite,
273 + AC_HELP_STRING([--disable-sqlite],
274 + [disable sqlite DB support]))
276 +if test x"$enable_sqlite" = x"no"; then
279 + AC_CHECK_LIB(sqlite3, sqlite3_open)
280 + AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
284 ' checker'*|checker*)
285 AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
286 diff --git a/db.c b/db.c
292 + * Routines to access extended file info via DB.
294 + * Copyright (C) 2008-2013 Wayne Davison
296 + * This program is free software; you can redistribute it and/or modify
297 + * it under the terms of the GNU General Public License as published by
298 + * the Free Software Foundation; either version 3 of the License, or
299 + * (at your option) any later version.
301 + * This program is distributed in the hope that it will be useful,
302 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
303 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
304 + * GNU General Public License for more details.
306 + * You should have received a copy of the GNU General Public License along
307 + * with this program; if not, visit the http://fsf.org website.
317 +extern int am_receiver;
318 +extern int am_generator;
319 +extern int checksum_type;
320 +extern int db_clean, db_check, db_do_md4, db_do_md5, db_update, db_lax, db_init, db_mounts;
321 +extern int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
322 +extern int saw_db_output_opt, saw_db_sum_opt;
323 +extern char *db_config;
325 +/* TODO: make this configurable */
326 +#define RSYNCDB_MOUNTS "/usr/sbin/rsyncdb-mountinfo"
328 +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
330 +#include <mysql/mysql.h>
331 +#include <mysql/errmsg.h>
334 +#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
336 +#include <sqlite3.h>
337 +#ifndef HAVE_SQLITE3_OPEN_V2
338 +#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
339 + sqlite3_open(dbname, dbhptr)
341 +#ifndef HAVE_SQLITE3_PREPARE_V2
342 +#define sqlite3_prepare_v2 sqlite3_prepare
344 +#define MAX_LOCK_FAILURES 10
345 +#define LOCK_FAIL_MSLEEP 100
348 +#define DB_TYPE_NONE 0
349 +#define DB_TYPE_MYSQL 1
350 +#define DB_TYPE_SQLITE 2
352 +int use_db = DB_TYPE_NONE;
353 +int select_many_sums = 0;
356 +#define PREP_MOUNT 1
358 +static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
359 +static unsigned int dbport = 0;
360 +static int transaction_state = -1;
377 +#define UPD_MOUNT 5 /* SQLite only */
381 +#define INS_PRESENT 9
382 +#define MAX_PREP_CNT 10
384 +#define MAX_BIND_CNT 7
385 +#define MAX_RESULT_BINDS 32
392 + sqlite3_stmt *sqlite;
395 +} statements[MAX_PREP_CNT];
398 +static enum logcode log_code;
401 +static unsigned int bind_disk_id, bind_mdnum;
402 +static int64 bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
403 +static char bind_sum[MAX_DIGEST_LEN];
404 +static unsigned long result_length[MAX_RESULT_BINDS];
405 +static my_bool result_is_null[MAX_RESULT_BINDS], result_error[MAX_RESULT_BINDS];
407 +static int64 bind_mtime;
409 +static char bind_thishost[128+1], bind_mount_uniq[128+1];
410 +static unsigned long bind_thishost_len, bind_mount_uniq_len;
412 +static char *error_log;
413 +#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
414 +static FILE *error_log_fp;
417 +#define PTR_SIZE (sizeof (struct file_struct *))
419 +static void update_mounts(void);
422 + struct name_list *next;
426 +int db_read_config(enum logcode code, const char *config_file)
428 + char buf[2048], *cp;
434 + bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
436 + if (!(fp = fopen(config_file, "r"))) {
437 + rsyserr(log_code, errno, "unable to open %s", config_file);
440 + if (DEBUG_GTE(DB, 1))
441 + rprintf(FCLIENT, "[%s] Reading DB config from %s\n", who_am_i(), config_file);
442 + while (fgets(buf, sizeof buf, fp)) {
444 + if ((cp = strchr(buf, '#')) == NULL
445 + && (cp = strchr(buf, '\r')) == NULL
446 + && (cp = strchr(buf, '\n')) == NULL)
447 + cp = buf + strlen(buf);
448 + while (cp != buf && isSpace(cp-1)) cp--;
454 + if (!(cp = strchr(buf, ':')))
458 + while (isSpace(cp)) cp++;
459 + if (strcasecmp(buf, "dbhost") == 0)
460 + dbhost = strdup(cp);
461 + else if (strcasecmp(buf, "dbuser") == 0)
462 + dbuser = strdup(cp);
463 + else if (strcasecmp(buf, "dbpass") == 0)
464 + dbpass = strdup(cp);
465 + else if (strcasecmp(buf, "dbname") == 0)
466 + dbname = strdup(cp);
467 + else if (strcasecmp(buf, "dbport") == 0)
469 + else if (strcasecmp(buf, "transaction") == 0)
470 + transaction_state = atoi(cp) ? 0 : -1;
471 + else if (strcasecmp(buf, "errlog") == 0)
472 + error_log = strdup(cp);
473 + else if (strcasecmp(buf, "thishost") == 0)
474 + bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
475 + else if (strcasecmp(buf, "dbtype") == 0) {
477 + if (strcasecmp(cp, "mysql") == 0) {
478 + use_db = DB_TYPE_MYSQL;
483 + if (strcasecmp(cp, "sqlite") == 0) {
484 + use_db = DB_TYPE_SQLITE;
489 + "Unsupported dbtype on line #%d in %s.\n",
490 + lineno, config_file);
491 + use_db = DB_TYPE_NONE;
495 + rprintf(log_code, "Invalid line #%d in %s\n",
496 + lineno, config_file);
497 + use_db = DB_TYPE_NONE;
503 + if (bind_thishost_len >= (int)sizeof bind_thishost)
504 + bind_thishost_len = sizeof bind_thishost - 1;
506 + if (!use_db || !dbname) {
507 + rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
508 + use_db = DB_TYPE_NONE;
512 + md_num = checksum_type == 5 ? 5 : 4;
515 + if (use_db != DB_TYPE_SQLITE)
516 + rprintf(log_code, "Ignoring errlog setting for non-SQLite DB.\n");
517 +#ifndef SQLITE_CONFIG_LOG
519 + rprintf(log_code, "Your sqlite doesn't support SQLITE_CONFIG_LOG.\n");
526 +#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
527 +static void errorLogCallback(UNUSED(void *pArg), int iErrCode, const char *zMsg)
529 + fprintf(error_log_fp, "[%d] %s (%d)\n", (int)getpid(), zMsg, iErrCode);
533 +static int run_sql(const char *fmt, ...)
540 + qlen = vasprintf(&query, fmt, ap);
543 + out_of_memory("run_sql");
544 + if (DEBUG_GTE(DB, 3))
545 + rprintf(FCLIENT, "[%s] SQL being run: %s\n", who_am_i(), query);
549 + case DB_TYPE_MYSQL:
550 + if (mysql_query(dbh.mysql, query) < 0) {
551 + rprintf(FERROR, "Failed to run sql: %s\n", mysql_error(dbh.mysql));
552 + rprintf(FERROR, "%s\n", query);
558 + case DB_TYPE_SQLITE: {
559 + int rc, lock_failures = 0;
561 + if ((rc = sqlite3_exec(dbh.sqlite, query, NULL, NULL, NULL)) == 0)
563 + if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
565 + if (++lock_failures > MAX_LOCK_FAILURES)
567 + msleep(LOCK_FAIL_MSLEEP);
570 + rprintf(FERROR, "[%s] Failed to run sql: %s\n", who_am_i(), sqlite3_errmsg(dbh.sqlite));
571 + rprintf(FERROR, "%s\n", query);
585 +static int prepare_mysql(int ndx, MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
589 + int qlen, param_cnt;
590 + MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
593 + out_of_memory("prepare_mysql");
596 + qlen = vasprintf(&query, fmt, ap);
599 + out_of_memory("prepare_mysql");
600 + if (DEBUG_GTE(DB, 3))
601 + rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
603 + if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
604 + rprintf(log_code, "[%s] Prepare failed: %s\n", who_am_i(), mysql_stmt_error(stmt));
605 + rprintf(log_code, "%s\n", query);
610 + if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
611 + rprintf(log_code, "[%s] Parameters in statement = %d, bind vars = %d\n",
612 + who_am_i(), param_cnt, bind_cnt);
613 + rprintf(log_code, "%s\n", query);
618 + mysql_stmt_bind_param(stmt, binds);
620 + statements[ndx].mysql = stmt;
628 +static int prepare_mysql_queries(int type)
630 + MYSQL_BIND binds[MAX_BIND_CNT];
635 + sql="SELECT disk_id"
637 + " WHERE host = ? AND devno = ?";
638 + memset(binds, 0, sizeof binds);
639 + binds[0].buffer_type = MYSQL_TYPE_STRING;
640 + binds[0].buffer = &bind_thishost;
641 + binds[0].buffer_length = bind_thishost_len;
642 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
643 + binds[1].buffer = &bind_devno;
644 + if (!prepare_mysql(SEL_DEV, binds, 2, sql))
647 + memset(binds, 0, sizeof binds);
648 + binds[0].buffer_type = MYSQL_TYPE_LONG;
649 + binds[0].buffer = &bind_disk_id;
650 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
651 + binds[1].buffer = &bind_ino;
652 + if (select_many_sums) {
653 + sql="SELECT checksum, sum_type, size, mtime, ctime"
655 + " WHERE disk_id = ? AND ino = ?";
656 + if (!prepare_mysql(SEL_SUM, binds, 2, sql))
659 + sql="SELECT checksum"
661 + " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
662 + " AND size = ? AND mtime = ? %s";
663 + binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
664 + binds[2].buffer = &bind_size;
665 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
666 + binds[3].buffer = &bind_mtime;
668 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
669 + binds[4].buffer = &bind_ctime;
671 + if (!prepare_mysql(SEL_SUM, binds, 4 + !db_lax, sql, md_num, db_lax ? "" : "AND ctime = ?"))
675 + sql="INSERT INTO inode_map"
676 + " SET disk_id = ?, ino = ?, sum_type = ?,"
677 + " size = ?, mtime = ?, ctime = ?, checksum = ?"
678 + " ON DUPLICATE KEY"
679 + " UPDATE size = VALUES(size), mtime = VALUES(mtime),"
680 + " ctime = VALUES(ctime), checksum = VALUES(checksum)";
681 + memset(binds, 0, sizeof binds);
682 + binds[0].buffer_type = MYSQL_TYPE_LONG;
683 + binds[0].buffer = &bind_disk_id;
684 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
685 + binds[1].buffer = &bind_ino;
686 + binds[2].buffer_type = MYSQL_TYPE_LONG;
687 + binds[2].buffer = &bind_mdnum;
688 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
689 + binds[3].buffer = &bind_size;
690 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
691 + binds[4].buffer = &bind_mtime;
692 + binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
693 + binds[5].buffer = &bind_ctime;
694 + binds[6].buffer_type = MYSQL_TYPE_BLOB;
695 + binds[6].buffer = &bind_sum;
696 + binds[6].buffer_length = MD5_DIGEST_LEN; /* Same as MD4_DIGEST_LEN */
697 + if (!prepare_mysql(REP_SUM, binds, 7, sql))
700 + sql="UPDATE inode_map"
702 + " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
703 + memset(binds, 0, sizeof binds);
704 + binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
705 + binds[0].buffer = &bind_ctime;
706 + binds[1].buffer_type = MYSQL_TYPE_LONG;
707 + binds[1].buffer = &bind_disk_id;
708 + binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
709 + binds[2].buffer = &bind_ino;
710 + binds[3].buffer_type = MYSQL_TYPE_LONG;
711 + binds[3].buffer = &bind_mdnum;
712 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
713 + binds[4].buffer = &bind_size;
714 + binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
715 + binds[5].buffer = &bind_mtime;
716 + if (!prepare_mysql(UPD_CTIME, binds, 6, sql))
721 + sql="INSERT INTO disk"
722 + " SET host = ?, last_seen = ?, mount_uniq = ?, devno = ?"
723 + " ON DUPLICATE KEY"
724 + " UPDATE last_seen = VALUES(last_seen), devno = VALUES(devno)";
725 + memset(binds, 0, sizeof binds);
726 + binds[0].buffer_type = MYSQL_TYPE_STRING;
727 + binds[0].buffer = &bind_thishost;
728 + binds[0].buffer_length = bind_thishost_len;
729 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
730 + binds[1].buffer = &bind_mtime; /* we abuse mtime to hold the last_seen value */
731 + binds[2].buffer_type = MYSQL_TYPE_STRING;
732 + binds[2].buffer = &bind_mount_uniq;
733 + binds[2].buffer_length = sizeof bind_mount_uniq;
734 + binds[2].length = &bind_mount_uniq_len;
735 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
736 + binds[3].buffer = &bind_devno;
737 + if (!prepare_mysql(INS_MOUNT, binds, 4, sql))
740 + sql="SELECT mount_uniq"
742 + " WHERE host = ? AND last_seen < ? AND devno != 0";
743 + /* Reusing first 2 binds from INS_MOUNT */
744 + if (!prepare_mysql(SEL_MOUNT, binds, 2, sql))
749 + " WHERE host = ? AND last_seen < ? AND devno != 0";
750 + /* Reusing binds from SEL_MOUNT */
751 + if (!prepare_mysql(UN_MOUNT, binds, 2, sql))
761 +static int db_connect_mysql(void)
763 + const char *open_dbname = db_init ? "mysql" : dbname;
765 + if (!(dbh.mysql = mysql_init(NULL)))
766 + out_of_memory("db_read_config");
768 + if (DEBUG_GTE(DB, 1)) {
769 + rprintf(FCLIENT, "[%s] connecting: host=%s user=%s db=%s port=%d\n",
770 + who_am_i(), dbhost, dbuser, open_dbname, dbport);
772 + if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, open_dbname, dbport, NULL, 0)) {
773 + rprintf(log_code, "[%s] Unable to connect to DB: %s\n", who_am_i(), mysql_error(dbh.mysql));
778 + if (db_output_msgs)
779 + rprintf(FCLIENT, "Creating DB %s (if it does not exist)\n", dbname);
780 + if (!run_sql("CREATE DATABASE IF NOT EXISTS `%s`", dbname)
781 + || !run_sql("USE `%s`", dbname))
782 + exit_cleanup(RERR_IPC);
784 + if (db_output_msgs)
785 + rprintf(FCLIENT, "Dropping old tables (if they exist))\n");
786 + if (!run_sql("DROP TABLE IF EXISTS disk")
787 + || !run_sql("DROP TABLE IF EXISTS inode_map"))
788 + exit_cleanup(RERR_IPC);
790 + if (db_output_msgs)
791 + rprintf(FCLIENT, "Creating empty tables ...\n");
793 + "CREATE TABLE disk (\n"
794 + " disk_id integer unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n"
795 + " host varchar(128) NOT NULL default 'localhost',\n"
796 + " mount_uniq varchar(128) default NULL,\n"
797 + " devno bigint unsigned NOT NULL,\n" /* This is 0 when not mounted */
798 + " last_seen bigint NOT NULL,\n"
799 + " UNIQUE KEY mount_lookup (host, mount_uniq),\n"
800 + " KEY dev_lookup (devno, host)\n"
802 + exit_cleanup(RERR_IPC);
805 + "CREATE TABLE inode_map (\n"
806 + " disk_id integer unsigned NOT NULL,\n"
807 + " ino bigint unsigned NOT NULL,\n"
808 + " sum_type tinyint NOT NULL default '0',\n"
809 + " size bigint unsigned NOT NULL,\n"
810 + " mtime bigint NOT NULL,\n"
811 + " ctime bigint NOT NULL,\n"
812 + " checksum binary(16) NOT NULL,\n"
813 + " PRIMARY KEY (disk_id,ino,sum_type)\n"
815 + exit_cleanup(RERR_IPC);
822 + if (!prepare_mysql_queries(PREP_MOUNT))
823 + exit_cleanup(RERR_IPC);
828 + if (!prepare_mysql_queries(PREP_NORM))
836 +static int prepare_sqlite(int ndx, const char *fmt, ...)
840 + int rc, qlen, lock_failures = 0;
843 + qlen = vasprintf(&query, fmt, ap);
846 + out_of_memory("prepare_sqlite");
847 + if (DEBUG_GTE(DB, 3))
848 + rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
850 + while ((rc = sqlite3_prepare_v2(dbh.sqlite, query, -1, &statements[ndx].sqlite, NULL)) != 0) {
851 + if (DEBUG_GTE(DB, 4)) {
852 + rprintf(FCLIENT, "[%s] sqlite3_prepare_v2(,%s,,) returned %d\n",
853 + who_am_i(), query, rc);
855 + if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
857 + if (++lock_failures > MAX_LOCK_FAILURES)
859 + msleep(LOCK_FAIL_MSLEEP);
862 + rprintf(log_code, "[%s] Failed to prepare SQL: %s (%d)\n", who_am_i(), sqlite3_errmsg(dbh.sqlite), rc);
863 + rprintf(log_code, "%s\n", query);
874 +static int prepare_sqlite_queries(int type)
880 + sql="SELECT disk_id"
882 + " WHERE host = ? AND devno = ?";
883 + if (!prepare_sqlite(SEL_DEV, sql))
886 + if (select_many_sums) {
887 + sql="SELECT checksum, sum_type, size, mtime, ctime"
889 + " WHERE disk_id = ? AND ino = ?";
890 + if (!prepare_sqlite(SEL_SUM, sql))
893 + sql="SELECT checksum"
895 + " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
896 + " AND size = ? AND mtime = ? %s";
897 + if (!prepare_sqlite(SEL_SUM, sql, md_num, db_lax ? "" : "AND ctime = ?"))
901 + sql="INSERT OR REPLACE INTO inode_map"
902 + " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
903 + " VALUES (?, ?, ?, ?, ?, ?, ?)";
904 + if (!prepare_sqlite(REP_SUM, sql))
907 + sql="UPDATE inode_map"
909 + " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
910 + if (!prepare_sqlite(UPD_CTIME, sql))
915 + sql="INSERT OR IGNORE INTO disk"
916 + " (host, last_seen, mount_uniq, devno)"
917 + " VALUES (?, ?, ?, ?)";
918 + if (!prepare_sqlite(INS_MOUNT, sql))
922 + " SET last_seen = ?, devno = ?"
923 + " WHERE host = ? AND mount_uniq = ?";
924 + if (!prepare_sqlite(UPD_MOUNT, sql))
927 + sql="SELECT mount_uniq"
929 + " WHERE host = ? AND last_seen < ? AND devno != 0";
930 + if (!prepare_sqlite(SEL_MOUNT, sql))
935 + " WHERE host = ? AND last_seen < ? AND devno != 0";
936 + if (!prepare_sqlite(UN_MOUNT, sql))
946 +static int db_connect_sqlite(void)
948 + int lock_failures = 0;
951 +#ifdef SQLITE_CONFIG_LOG
953 + if (DEBUG_GTE(DB, 1))
954 + rprintf(FCLIENT, "[%s] Setting sqlite errlog to %s\n", who_am_i(), error_log);
955 + if (!(error_log_fp = fopen(error_log, "a"))) {
956 + rsyserr(log_code, errno, "unable to append to logfile %s", error_log);
958 + } else if (sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, NULL) != 0)
959 + rprintf(log_code, "Failed to set errorLogCallback: %s\n", sqlite3_errmsg(dbh.sqlite));
964 + int open_flags = SQLITE_OPEN_READWRITE;
966 + open_flags |= SQLITE_OPEN_CREATE;
967 + if (DEBUG_GTE(DB, 1))
968 + rprintf(FCLIENT, "[%s] opening %s (%d)\n", who_am_i(), dbname, open_flags);
969 + if ((rc = sqlite3_open_v2(dbname, &dbh.sqlite, open_flags, NULL)) == 0) {
972 + if (DEBUG_GTE(DB, 4)) {
973 + rprintf(FCLIENT, "[%s] sqlite3_open_v2(%s,,%d,NULL) returned %d\n",
974 + who_am_i(), dbname, open_flags, rc);
976 + if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
978 + if (++lock_failures > MAX_LOCK_FAILURES)
980 + msleep(LOCK_FAIL_MSLEEP);
984 + rprintf(log_code, "Unable to connect to DB: %s (%d)\n", sqlite3_errmsg(dbh.sqlite), rc);
990 + if (db_output_msgs)
991 + rprintf(FCLIENT, "Dropping old tables (if they exist) ...\n");
992 + if (!run_sql("DROP TABLE IF EXISTS disk")
993 + || !run_sql("DROP TABLE IF EXISTS inode_map"))
994 + exit_cleanup(RERR_IPC);
996 + if (db_output_msgs)
997 + rprintf(FCLIENT, "Creating empty tables ...\n");
998 + sql="CREATE TABLE disk (\n"
999 + " disk_id integer NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
1000 + " host varchar(128) NOT NULL default 'localhost',\n"
1001 + " mount_uniq varchar(128) default NULL,\n"
1002 + " devno bigint NOT NULL,\n" /* This is 0 when not mounted */
1003 + " last_seen bigint NOT NULL,\n"
1004 + " UNIQUE (host, mount_uniq)\n"
1006 + if (!run_sql(sql))
1007 + exit_cleanup(RERR_IPC);
1009 + sql="CREATE TABLE inode_map (\n"
1010 + " disk_id integer NOT NULL,\n"
1011 + " ino bigint NOT NULL,\n"
1012 + " size bigint NOT NULL,\n"
1013 + " mtime bigint NOT NULL,\n"
1014 + " ctime bigint NOT NULL,\n"
1015 + " sum_type tinyint NOT NULL default '0',\n"
1016 + " checksum binary(16) NOT NULL,\n"
1017 + " PRIMARY KEY (disk_id,ino,sum_type)\n"
1019 + if (!run_sql(sql))
1020 + exit_cleanup(RERR_IPC);
1022 +#if SQLITE_VERSION_NUMBER >= 3007000
1023 + /* Using WAL locking makes concurrency much better (requires sqlite 3.7.0). */
1024 + sql="PRAGMA journal_mode = wal";
1025 + run_sql(sql); /* We don't check this for success. */
1033 + if (!prepare_sqlite_queries(PREP_MOUNT))
1034 + exit_cleanup(RERR_IPC);
1039 + if (!prepare_sqlite_queries(PREP_NORM)) {
1040 + db_disconnect(False);
1048 +int db_connect(int select_many)
1050 + select_many_sums = select_many;
1054 + case DB_TYPE_MYSQL:
1055 + if (db_connect_mysql())
1060 + case DB_TYPE_SQLITE:
1061 + if (db_connect_sqlite())
1067 + db_disconnect(False);
1072 +void db_disconnect(BOOL commit)
1079 + if (transaction_state > 0) {
1080 + if (DEBUG_GTE(DB, 1)) {
1081 + rprintf(FCLIENT, "[%s] %s our DB transaction\n",
1082 + who_am_i(), commit ? "Committing" : "Rolling back");
1084 + transaction_state = 0;
1086 + run_sql("COMMIT");
1088 + run_sql("ROLLBACK");
1091 + if (DEBUG_GTE(DB, 1))
1092 + rprintf(FCLIENT, "[%s] Disconnecting from the DB\n", who_am_i());
1094 + for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
1095 + if (statements[ndx].all) {
1098 + case DB_TYPE_MYSQL:
1099 + mysql_stmt_close(statements[ndx].mysql);
1103 + case DB_TYPE_SQLITE:
1104 + sqlite3_finalize(statements[ndx].sqlite);
1108 + statements[ndx].all = NULL;
1114 + case DB_TYPE_MYSQL:
1115 + mysql_close(dbh.mysql);
1119 + case DB_TYPE_SQLITE:
1120 + sqlite3_close(dbh.sqlite);
1126 + use_db = DB_TYPE_NONE;
1130 +static MYSQL_STMT *exec_mysql(int ndx)
1132 + MYSQL_STMT *stmt = statements[ndx].mysql;
1135 + if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
1136 + db_disconnect(False);
1137 + use_db = DB_TYPE_MYSQL;
1138 + if (db_connect(select_many_sums)) {
1139 + stmt = statements[ndx].mysql;
1140 + rc = mysql_stmt_execute(stmt);
1144 + rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
1153 +/* This stores up to max_rows into the values pointed to by the bind data arrays.
1154 + * If max_rows is > 1, then all the buffer pointers MUST be set to an array long
1155 + * enough to hold the max count of rows. The buffer pointer will be incremented
1156 + * to read additional rows (but never past the end). If stmt_ptr is non-NULL, it
1157 + * will be set to the "stmt" pointer IFF we didn't run out of rows before hitting
1158 + * the max. In this case, the caller should call mysql_stmt_fetch() to read any
1159 + * remaining rows (the buffer pointers will point at the final array element) and
1160 + * then call mysql_stmt_free_result(). If *stmt_ptr is a NULL value, there were
1161 + * not enough rows to fill the max_rows arrays, and the stmt was already freed. */
1162 +static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx, int max_rows, MYSQL_STMT **stmt_ptr)
1165 + int i, rc, rows = 0;
1167 + if (bind_cnt > MAX_RESULT_BINDS) {
1168 + fprintf(stderr, "Internal error: MAX_RESULT_BINDS overflow\n");
1169 + exit_cleanup(RERR_UNSUPPORTED);
1172 + if ((stmt = exec_mysql(ndx)) == NULL)
1175 + for (i = 0; i < bind_cnt; i++) {
1176 + binds[i].is_null = &result_is_null[i];
1177 + binds[i].length = &result_length[i];
1178 + binds[i].error = &result_error[i];
1180 + mysql_stmt_bind_result(stmt, binds);
1182 + while (rows < max_rows) {
1183 + if ((rc = mysql_stmt_fetch(stmt)) != 0) {
1184 + if (rc != MYSQL_NO_DATA)
1185 + rprintf(log_code, "SELECT fetch failed: %s\n", mysql_stmt_error(stmt));
1188 + if (++rows >= max_rows)
1190 + for (i = 0; i < bind_cnt; i++) {
1191 + switch (binds[i].buffer_type) {
1192 + case MYSQL_TYPE_BLOB:
1193 + case MYSQL_TYPE_STRING:
1194 + binds[i].buffer += binds[i].buffer_length;
1196 + case MYSQL_TYPE_LONG:
1197 + binds[i].buffer += sizeof (int);
1199 + case MYSQL_TYPE_LONGLONG:
1200 + binds[i].buffer += sizeof (int64);
1203 + fprintf(stderr, "Unknown MYSQL_TYPE_* in multi-row read: %d.\n", binds[i].buffer_type);
1204 + exit_cleanup(RERR_UNSUPPORTED);
1209 + if (!stmt_ptr || rows < max_rows) {
1210 + mysql_stmt_free_result(stmt);
1220 +static void update_mounts(void)
1222 + char buf[2048], *argv[2];
1223 + int f_from, f_to, len;
1227 + if (DEBUG_GTE(DB, 2))
1228 + printf("Running %s to grab mount info\n", RSYNCDB_MOUNTS);
1229 + argv[0] = RSYNCDB_MOUNTS;
1231 + pid = piped_child(argv, &f_from, &f_to);
1234 + bind_mtime = time(NULL); /* abuse mtime slightly to hold our last_seen value */
1236 + /* Strict format has 2 items with one tab as separator: MOUNT_UNIQ\tPATH */
1237 + while ((len = read_line(f_from, buf, sizeof buf, 0)) > 0) {
1238 + char *mount_uniq, *path;
1240 + if (DEBUG_GTE(DB, 3))
1241 + printf("Parsing mount info: %s\n", buf);
1242 + mount_uniq = strtok(buf, "\t");
1243 + path = mount_uniq ? strtok(NULL, "\r\n") : NULL;
1245 + fprintf(stderr, "Failed to parse line from %s output\n", RSYNCDB_MOUNTS);
1246 + exit_cleanup(RERR_SYNTAX);
1249 + if (lstat(path, &st) < 0) {
1250 + fprintf(stderr, "Failed to lstat(%s): %s\n", path, strerror(errno));
1251 + exit_cleanup(RERR_IPC);
1254 + bind_mount_uniq_len = strlcpy(bind_mount_uniq, mount_uniq, sizeof bind_mount_uniq);
1255 + if (bind_mount_uniq_len >= (int)sizeof bind_mount_uniq)
1256 + bind_mount_uniq_len = sizeof bind_mount_uniq - 1;
1258 + if (db_output_msgs) {
1259 + printf("Marking mount \"%s\" (%s) as a recent mount\n",
1260 + bind_mount_uniq, big_num(st.st_dev));
1264 + case DB_TYPE_MYSQL:
1265 + bind_devno = st.st_dev;
1266 + if (exec_mysql(INS_MOUNT) == NULL) {
1267 + fprintf(stderr, "Failed to update mount info for \"%s\" - %s\n",
1268 + bind_mount_uniq, mysql_error(dbh.mysql));
1269 + exit_cleanup(RERR_IPC);
1274 + case DB_TYPE_SQLITE: {
1275 + int rc, change_cnt;
1276 + sqlite3_stmt *stmt = statements[INS_MOUNT].sqlite;
1277 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1278 + sqlite3_bind_int64(stmt, 2, bind_mtime);
1279 + sqlite3_bind_text(stmt, 3, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
1280 + sqlite3_bind_int64(stmt, 4, st.st_dev);
1281 + rc = sqlite3_step(stmt);
1282 + if (rc != SQLITE_DONE) {
1283 + fprintf(stderr, "Failed to insert mount info for \"%s\" - %s (%d)\n",
1284 + bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
1285 + exit_cleanup(RERR_IPC);
1287 + change_cnt = sqlite3_changes(dbh.sqlite);
1288 + sqlite3_reset(stmt);
1289 + if (change_cnt == 0) {
1290 + stmt = statements[UPD_MOUNT].sqlite;
1291 + sqlite3_bind_int64(stmt, 1, bind_mtime);
1292 + sqlite3_bind_int64(stmt, 2, st.st_dev);
1293 + sqlite3_bind_text(stmt, 3, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1294 + sqlite3_bind_text(stmt, 4, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
1295 + rc = sqlite3_step(stmt);
1296 + if (rc != SQLITE_DONE) {
1297 + fprintf(stderr, "Failed to update mount info for \"%s\" - %s (%d)\n",
1298 + bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
1299 + exit_cleanup(RERR_IPC);
1301 + sqlite3_reset(stmt);
1310 + waitpid(pid, &status, 0);
1314 + case DB_TYPE_MYSQL: {
1315 + if (db_output_msgs) {
1316 + MYSQL_BIND binds[1];
1319 + binds[0].buffer_type = MYSQL_TYPE_BLOB;
1320 + binds[0].buffer = bind_mount_uniq;
1321 + binds[0].buffer_length = sizeof bind_mount_uniq;
1322 + if (fetch_mysql(binds, 1, SEL_MOUNT, 1, &stmt)) {
1324 + printf("Marking mount \"%s\" as unmounted.\n", bind_mount_uniq);
1325 + if (mysql_stmt_fetch(stmt) != 0)
1328 + mysql_stmt_free_result(stmt);
1332 + if (exec_mysql(UN_MOUNT) == NULL) {
1333 + fprintf(stderr, "Failed to update old mount info - %s\n",
1334 + mysql_error(dbh.mysql));
1335 + exit_cleanup(RERR_IPC);
1341 + case DB_TYPE_SQLITE: {
1342 + sqlite3_stmt *stmt;
1345 + if (db_output_msgs) {
1346 + stmt = statements[SEL_MOUNT].sqlite;
1347 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1348 + sqlite3_bind_int64(stmt, 2, bind_mtime);
1350 + if (sqlite3_step(stmt) != SQLITE_ROW)
1352 + printf("Marking mount \"%s\" as unmounted.\n", sqlite3_column_text(stmt, 0));
1354 + sqlite3_reset(stmt);
1357 + stmt = statements[UN_MOUNT].sqlite;
1358 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1359 + sqlite3_bind_int64(stmt, 2, bind_mtime);
1360 + rc = sqlite3_step(stmt);
1361 + sqlite3_reset(stmt);
1362 + if (rc != SQLITE_DONE) {
1363 + fprintf(stderr, "Failed to update old mount info - %s (%d)\n",
1364 + sqlite3_errmsg(dbh.sqlite), rc);
1365 + exit_cleanup(RERR_IPC);
1373 +unsigned int get_disk_id(int64 devno)
1375 + static unsigned int prior_disk_id = 0;
1376 + static int64 prior_devno = 0;
1378 + if (prior_devno == devno && prior_disk_id) {
1379 + if (DEBUG_GTE(DB, 5))
1380 + rprintf(FCLIENT, "get_disk_id(%s,%s) = %d (cached)\n", bind_thishost, big_num(devno), prior_disk_id);
1381 + return prior_disk_id;
1383 + prior_devno = devno;
1387 + case DB_TYPE_MYSQL: {
1388 + MYSQL_BIND binds[1];
1390 + bind_devno = devno; /* The one changing SEL_DEV input value. */
1392 + /* Bind where to put the output. */
1393 + binds[0].buffer_type = MYSQL_TYPE_LONG;
1394 + binds[0].buffer = &prior_disk_id;
1395 + if (!fetch_mysql(binds, 1, SEL_DEV, 1, NULL))
1396 + prior_disk_id = 0;
1401 + case DB_TYPE_SQLITE: {
1402 + sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
1403 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1404 + sqlite3_bind_int64(stmt, 2, devno);
1405 + if (sqlite3_step(stmt) == SQLITE_ROW)
1406 + prior_disk_id = sqlite3_column_int(stmt, 0);
1408 + prior_disk_id = 0;
1409 + sqlite3_reset(stmt);
1415 + if (DEBUG_GTE(DB, 2))
1416 + rprintf(FCLIENT, "get_disk_id(%s,%s) = %d\n", bind_thishost, big_num(devno), prior_disk_id);
1417 + return prior_disk_id;
1420 +int db_get_checksum(const STRUCT_STAT *st_p, char *sum)
1422 + unsigned int disk_id = get_disk_id(st_p->st_dev);
1430 + case DB_TYPE_MYSQL: {
1431 + MYSQL_BIND binds[1];
1433 + bind_disk_id = disk_id;
1434 + bind_ino = st_p->st_ino;
1435 + bind_size = st_p->st_size;
1436 + bind_mtime = st_p->st_mtime;
1438 + bind_ctime = st_p->st_ctime;
1440 + binds[0].buffer_type = MYSQL_TYPE_BLOB;
1441 + binds[0].buffer = sum;
1442 + binds[0].buffer_length = MD5_DIGEST_LEN;
1443 + ok = fetch_mysql(binds, 1, SEL_SUM, 1, NULL);
1448 + case DB_TYPE_SQLITE: {
1449 + sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
1450 + sqlite3_bind_int(stmt, 1, disk_id);
1451 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1452 + sqlite3_bind_int64(stmt, 3, st_p->st_size);
1453 + sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
1455 + sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
1456 + if (sqlite3_step(stmt) == SQLITE_ROW) {
1457 + int len = sqlite3_column_bytes(stmt, 0);
1458 + if (len > MAX_DIGEST_LEN)
1459 + len = MAX_DIGEST_LEN;
1460 + memcpy(sum, sqlite3_column_blob(stmt, 0), len);
1463 + sqlite3_reset(stmt);
1469 + if (DEBUG_GTE(DB, 2)) {
1471 + rprintf(FCLIENT, "[%s] Found DB checksum for %s,%s,%d: %s\n",
1472 + who_am_i(), big_num(st_p->st_dev),
1473 + big_num(st_p->st_ino), md_num, sum_as_hex(md_num, sum, 0));
1475 + rprintf(FCLIENT, "[%s] No DB checksum for %s,%s,%d\n",
1476 + who_am_i(), big_num(st_p->st_dev),
1477 + big_num(st_p->st_ino), md_num);
1484 +int db_get_both_checksums(const STRUCT_STAT *st_p, int *right_sum_cnt, int *wrong_sum_cnt, char **sum4, char **sum5)
1486 + static char dbsum[MD5_DIGEST_LEN*2];
1487 + int rows, j, sum_type[2];
1488 + int64 dbsize[2], dbmtime[2], dbctime[2];
1489 + unsigned int disk_id = get_disk_id(st_p->st_dev);
1496 + case DB_TYPE_MYSQL: {
1497 + MYSQL_BIND binds[5];
1499 + bind_disk_id = disk_id;
1500 + bind_ino = st_p->st_ino;
1502 + binds[0].buffer_type = MYSQL_TYPE_BLOB;
1503 + binds[0].buffer = dbsum;
1504 + binds[0].buffer_length = MD5_DIGEST_LEN;
1505 + binds[1].buffer_type = MYSQL_TYPE_LONG;
1506 + binds[1].buffer = (char*)sum_type;
1507 + binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
1508 + binds[2].buffer = (char*)dbsize;
1509 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
1510 + binds[3].buffer = (char*)dbmtime;
1511 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
1512 + binds[4].buffer = (char*)dbctime;
1513 + rows = fetch_mysql(binds, 5, SEL_SUM, 2, NULL);
1518 + case DB_TYPE_SQLITE: {
1519 + sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
1520 + sqlite3_bind_int(stmt, 1, disk_id);
1521 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1522 + for (j = 0; j < 2; j++) {
1524 + if (sqlite3_step(stmt) != SQLITE_ROW)
1526 + len = sqlite3_column_bytes(stmt, 0);
1527 + if (len > MD5_DIGEST_LEN)
1528 + len = MD5_DIGEST_LEN;
1529 + memcpy(dbsum + MD5_DIGEST_LEN*j, sqlite3_column_blob(stmt, 0), len);
1530 + sum_type[j] = sqlite3_column_int(stmt, 1);
1531 + dbsize[j] = sqlite3_column_int(stmt, 2);
1532 + dbmtime[j] = sqlite3_column_int64(stmt, 3);
1533 + dbctime[j] = sqlite3_column_int64(stmt, 4);
1535 + sqlite3_reset(stmt);
1548 + *right_sum_cnt = *wrong_sum_cnt = 0;
1549 + for (j = 0; j < rows; j++) {
1550 + if (DEBUG_GTE(DB, 3)) {
1551 + rprintf(FCLIENT, "DB checksum for %s,%s,%d: %s\n",
1552 + big_num(st_p->st_dev), big_num(st_p->st_ino), sum_type[j],
1553 + sum_as_hex(sum_type[j], dbsum + MD5_DIGEST_LEN*j, 0));
1556 + if (sum_type[j] == 4) {
1559 + *sum4 = dbsum + MD5_DIGEST_LEN*j;
1563 + *sum5 = dbsum + MD5_DIGEST_LEN*j;
1565 + if (st_p->st_size == dbsize[j] && st_p->st_mtime == dbmtime[j] && (db_lax || st_p->st_ctime == dbctime[j]))
1574 +int db_set_checksum(int mdnum, const STRUCT_STAT *st_p, const char *sum)
1576 + unsigned int disk_id;
1577 + const char *errmsg = NULL;
1580 + if (am_receiver || (am_generator && same_db)) {
1581 + /* Forward the setting to a single process. The receiver always
1582 + * forward to the generator, and the generator will forward to
1583 + * the receiver ONLY if this is a local transfer. */
1584 + char data[MSG_CHECKSUM_LEN];
1585 + SIVAL64(data, 0, st_p->st_dev);
1586 + SIVAL64(data, 8, st_p->st_ino);
1587 + SIVAL64(data, 16, st_p->st_size);
1588 + SIVAL64(data, 24, st_p->st_mtime);
1589 + SIVAL64(data, 32, st_p->st_ctime);
1590 +#if MSG_CHECKSUM_LONGS != 5
1591 +#error Fix the setting of checksum long values
1593 + SIVAL(data, MSG_CHECKSUM_LONGS*8, mdnum);
1594 + memcpy(data + MSG_CHECKSUM_LONGS*8 + 4, sum, MAX_DIGEST_LEN);
1595 + return send_msg(MSG_CHECKSUM, data, sizeof data, 0);
1598 + if ((disk_id = get_disk_id(st_p->st_dev)) == 0)
1603 + case DB_TYPE_MYSQL:
1604 + if (transaction_state == 0) {
1605 + if (!run_sql("BEGIN"))
1607 + transaction_state = 1;
1610 + bind_disk_id = disk_id;
1611 + bind_ino = st_p->st_ino;
1612 + bind_mdnum = mdnum;
1613 + bind_size = st_p->st_size;
1614 + bind_mtime = st_p->st_mtime;
1615 + bind_ctime = st_p->st_ctime;
1616 + memcpy(bind_sum, sum, MD5_DIGEST_LEN);
1617 + if (exec_mysql(REP_SUM) == NULL)
1618 + errmsg = mysql_error(dbh.mysql);
1622 + case DB_TYPE_SQLITE: {
1623 + sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
1624 + int lock_failures = 0;
1626 + if (transaction_state == 0) {
1627 + if (!run_sql("BEGIN"))
1629 + transaction_state = 1;
1632 + sqlite3_bind_int(stmt, 1, disk_id);
1633 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1634 + sqlite3_bind_int(stmt, 3, mdnum);
1635 + sqlite3_bind_int64(stmt, 4, st_p->st_size);
1636 + sqlite3_bind_int64(stmt, 5, st_p->st_mtime);
1637 + sqlite3_bind_int64(stmt, 6, st_p->st_ctime);
1638 + sqlite3_bind_blob(stmt, 7, sum, MD5_DIGEST_LEN, SQLITE_TRANSIENT);
1640 + rc = sqlite3_step(stmt);
1641 + if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
1643 + if (++lock_failures > MAX_LOCK_FAILURES)
1645 + sqlite3_reset(stmt);
1646 + msleep(LOCK_FAIL_MSLEEP);
1648 + if (rc != SQLITE_DONE)
1649 + errmsg = sqlite3_errmsg(dbh.sqlite);
1650 + sqlite3_reset(stmt);
1657 + if (DEBUG_GTE(DB, 2)) {
1658 + rprintf(FCLIENT, "[%s] Set DB checksum for %s,%s,%d: %s\n",
1659 + who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
1660 + md_num, sum_as_hex(md_num, sum, 0));
1663 + rprintf(log_code, "[%s] Failed to set checksum for %s,%s,%d: %s (%d) -- closing DB\n",
1664 + who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
1665 + md_num, errmsg, rc);
1666 + db_disconnect(False);
1669 + return errmsg ? 0 : 1;
1672 +/* For a delayed-update copy, we set the checksum on the file when it was
1673 + * inside the partial-dir. Since renaming the file changes its ctime, we need
1674 + * to update the ctime to its new value (we can skip this in db_lax mode). */
1675 +int db_update_ctime(int mdnum, const STRUCT_STAT *st_p)
1677 + unsigned int disk_id = get_disk_id(st_p->st_dev);
1684 + case DB_TYPE_MYSQL:
1685 + bind_ctime = st_p->st_ctime;
1686 + bind_disk_id = disk_id;
1687 + bind_ino = st_p->st_ino;
1688 + bind_mdnum = mdnum;
1689 + bind_size = st_p->st_size;
1690 + bind_mtime = st_p->st_mtime;
1691 + return exec_mysql(UPD_CTIME) != NULL;
1694 + case DB_TYPE_SQLITE: {
1697 + sqlite3_stmt *stmt = statements[UPD_CTIME].sqlite;
1700 + sqlite3_bind_int64(stmt, 1, st_p->st_ctime);
1701 + sqlite3_bind_int(stmt, 2, disk_id);
1702 + sqlite3_bind_int64(stmt, 3, st_p->st_ino);
1703 + sqlite3_bind_int(stmt, 4, mdnum);
1704 + sqlite3_bind_int64(stmt, 5, st_p->st_size);
1705 + sqlite3_bind_int64(stmt, 6, st_p->st_mtime);
1706 + rc = sqlite3_step(stmt);
1707 + sqlite3_reset(stmt);
1708 + return rc == SQLITE_DONE;
1716 +int db_clean_init(void)
1720 + case DB_TYPE_MYSQL: {
1721 + MYSQL_BIND binds[MAX_BIND_CNT];
1724 + mysql_query(dbh.mysql,
1725 + "CREATE TEMPORARY TABLE inode_present ("
1726 + " disk_id integer unsigned NOT NULL,"
1727 + " ino bigint unsigned NOT NULL,"
1728 + " present tinyint NOT NULL default '1',"
1729 + " PRIMARY KEY (disk_id,ino)"
1733 + sql="INSERT IGNORE INTO inode_present"
1734 + " SET disk_id = ?, ino = ?, present = 1";
1735 + memset(binds, 0, sizeof binds);
1736 + binds[0].buffer_type = MYSQL_TYPE_LONG;
1737 + binds[0].buffer = &bind_disk_id;
1738 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
1739 + binds[1].buffer = &bind_ino;
1740 + if (!prepare_mysql(INS_PRESENT, binds, 2, sql))
1741 + exit_cleanup(RERR_SYNTAX);
1744 + " FROM inode_map AS m"
1745 + " LEFT JOIN inode_present USING(disk_id, ino)"
1746 + " JOIN disk AS d ON(m.disk_id = d.disk_id)"
1747 + " WHERE host = ? AND devno != 0 AND present IS NULL";
1748 + memset(binds, 0, sizeof binds);
1749 + binds[0].buffer_type = MYSQL_TYPE_STRING;
1750 + binds[0].buffer = &bind_thishost;
1751 + binds[0].buffer_length = bind_thishost_len;
1752 + if (!prepare_mysql(DEL_SUMS, binds, 1, sql))
1753 + exit_cleanup(RERR_SYNTAX);
1759 + case DB_TYPE_SQLITE: {
1761 + sql="ATTACH DATABASE '' AS aux1;"; /* Private temp DB, probably in-memory */
1762 + if (!run_sql(sql))
1763 + exit_cleanup(RERR_IPC);
1765 + sql="CREATE TABLE aux1.inode_present ("
1766 + " disk_id integer NOT NULL,"
1767 + " ino bigint NOT NULL,"
1768 + " present tinyint NOT NULL default '1',"
1769 + " PRIMARY KEY (disk_id,ino)"
1771 + if (!run_sql(sql))
1772 + exit_cleanup(RERR_IPC);
1774 + sql="INSERT OR IGNORE INTO aux1.inode_present"
1775 + " (disk_id, ino, present)"
1776 + " VALUES (?, ?, 1)";
1777 + if (!prepare_sqlite(INS_PRESENT, sql))
1778 + exit_cleanup(RERR_IPC);
1780 + sql="DELETE FROM inode_map"
1781 + " WHERE ROWID IN ("
1783 + " FROM inode_map AS m"
1784 + " LEFT JOIN aux1.inode_present USING(disk_id, ino)"
1785 + " JOIN disk AS d ON(m.disk_id = d.disk_id)"
1786 + " WHERE host = ? AND devno != 0 AND present IS NULL"
1788 + if (!prepare_sqlite(DEL_SUMS, sql))
1789 + exit_cleanup(RERR_IPC);
1791 + transaction_state = -1; /* bug work-around -- force transaction off when cleaning XXX */
1801 +int db_note_present(int disk_id, int64 ino)
1805 + case DB_TYPE_MYSQL:
1806 + bind_disk_id = disk_id;
1808 + return exec_mysql(INS_PRESENT) != NULL;
1811 + case DB_TYPE_SQLITE: {
1813 + sqlite3_stmt *stmt = statements[INS_PRESENT].sqlite;
1814 + sqlite3_bind_int(stmt, 1, disk_id);
1815 + sqlite3_bind_int64(stmt, 2, ino);
1816 + rc = sqlite3_step(stmt);
1817 + sqlite3_reset(stmt);
1818 + return rc == SQLITE_DONE;
1826 +/* This function requires the user to have populated all disk_id+inode pairs
1827 + * into the inode_present table. */
1828 +int db_clean_inodes(void)
1834 + case DB_TYPE_MYSQL: {
1835 + MYSQL_STMT *stmt = exec_mysql(DEL_SUMS);
1837 + del_cnt = mysql_affected_rows(dbh.mysql);
1842 + case DB_TYPE_SQLITE: {
1844 + sqlite3_stmt *stmt = statements[DEL_SUMS].sqlite;
1845 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1846 + rc = sqlite3_step(stmt);
1847 + if (rc == SQLITE_DONE)
1848 + del_cnt = sqlite3_changes(dbh.sqlite);
1849 + sqlite3_reset(stmt);
1858 +static int abs_path(char *buf, int bufsiz, const char *curdir, const char *dir)
1861 + strlcpy(buf, dir, bufsiz);
1863 + snprintf(buf, bufsiz, "%s/%s", curdir, dir);
1865 + return clean_fname(buf, CFN_DROP_TRAILING_DOT_DIR | CFN_COLLAPSE_DOT_DOT_DIRS);
1868 +static struct name_list *new_name(const char *basename, const char *filename)
1870 + struct name_list *n;
1871 + int blen = strlen(basename);
1872 + int slen = filename ? (int)strlen(filename) : -1;
1873 + int len = blen + 1 + slen;
1875 + if (len >= MAXPATHLEN) {
1877 + rprintf(FERROR, "Filename too long: %s/%s\n", basename, filename);
1879 + rprintf(FERROR, "Filename too long: %s\n", basename);
1883 + if (!(n = (struct name_list *)malloc(sizeof (struct name_list) + len)))
1884 + out_of_memory("new_name");
1886 + memcpy(n->name, basename, blen);
1888 + n->name[blen] = '/';
1889 + memcpy(n->name + 1 + blen, filename, slen);
1891 + n->name[len] = '\0';
1897 +static int name_compare(const void *n1, const void *n2)
1899 + struct name_list *p1 = *(struct name_list **)n1;
1900 + struct name_list *p2 = *(struct name_list **)n2;
1901 + return strcmp(p1->name, p2->name);
1904 +static struct name_list *get_sorted_names(const char *dir)
1906 + struct name_list *add, **sortbuf, *names = NULL, *prior_name = NULL;
1907 + struct dirent *di;
1911 + if (!(d = opendir("."))) {
1912 + rprintf(FERROR, "Unable to opendir %s: %s\n", dir, strerror(errno));
1915 + while ((di = readdir(d)) != NULL) {
1916 + char *dname = d_name(di);
1917 + if (dname[0] == '.' && (dname[1] == '\0' || (dname[1] == '.' && dname[2] == '\0')))
1919 + if (!(add = new_name(dname, NULL)))
1922 + prior_name->next = add;
1933 + if (!(sortbuf = new_array(struct name_list *, cnt)))
1934 + out_of_memory("get_sorted_names");
1935 + for (j = 0; j < cnt; j++) {
1936 + sortbuf[j] = names;
1937 + names = names->next;
1940 + qsort(sortbuf, cnt, PTR_SIZE, name_compare);
1942 + names = prior_name = NULL;
1943 + for (j = 0; j < cnt; j++) {
1946 + prior_name->next = add;
1953 + prior_name->next = NULL;
1960 +static inline int sums_ne(const char *sum1, const char *sum2)
1962 + return memcmp(sum1, sum2, MD5_DIGEST_LEN) != 0;
1965 +/* Returns 1 if there is a checksum change, else 0. */
1966 +static int mention_file(const char *dir, const char *name, int right_cnt, int wrong_cnt,
1967 + const char *dbsum4, const char *dbsum5, const char *sum4, const char *sum5)
1969 + char *info_str = wrong_cnt && !right_cnt ? "!i " : " ";
1970 + char *md4_str = !db_do_md4 ? NULL : !dbsum4 ? "+4 " : !sum4 ? "?4 " : sums_ne(sum4, dbsum4) ? "!4 " : " ";
1971 + char *md5_str = !db_do_md5 ? NULL : !dbsum5 ? "+5 " : !sum5 ? "?5 " : sums_ne(sum5, dbsum5) ? "!5 " : " ";
1972 + int chg = *info_str != ' ' || (md4_str && *md4_str != ' ') || (md5_str && *md5_str != ' ');
1973 + if (chg || db_output_unchanged) {
1974 + if (db_output_info) {
1975 + fputs(info_str, stdout);
1977 + fputs(md4_str, stdout);
1979 + fputs(md5_str, stdout);
1981 + if (db_output_sum) {
1983 + printf("%s ", sum_as_hex(4, sum4, 0));
1985 + printf("%s ", sum_as_hex(5, sum5, 0));
1987 + if (db_output_name) {
1988 + if (db_output_sum)
1989 + putchar(' '); /* We want 2 spaces, like md5sum. */
1990 + if (*dir != '.' || dir[1]) {
1991 + fputs(dir, stdout);
2001 +NORETURN void run_dbonly(const char **args)
2003 + char start_dir[MAXPATHLEN], dirbuf[MAXPATHLEN];
2004 + int need_sum_cnt, start_dir_len;
2005 + struct name_list *prior_dir;
2006 + struct name_list *names;
2007 + int exit_code = 0;
2009 + checksum_type = 5;
2011 + need_sum_cnt = db_do_md4 + db_do_md5;
2013 + if (!db_read_config(FERROR, db_config) || !db_connect(1))
2014 + exit_cleanup(RERR_FILEIO);
2019 + if (getcwd(start_dir, sizeof start_dir - 1) == NULL) {
2020 + rsyserr(FERROR, errno, "getcwd()");
2021 + exit_cleanup(RERR_FILESELECT);
2023 + start_dir_len = strlen(start_dir);
2028 + struct name_list *add;
2029 + if (abs_path(dirbuf, sizeof dirbuf, start_dir, *args++) <= 0)
2031 + if (!(add = new_name(dirbuf, NULL)))
2034 + prior_dir->next = add;
2040 + dirs_list = new_name(start_dir, NULL);
2043 + while (dirs_list) {
2044 + struct name_list *subdirs, *prior_subdir, *prior_name;
2045 + const char *dir = dirs_list->name;
2046 + const char *reldir = dir;
2049 + free((void*)prior_dir);
2050 + prior_dir = dirs_list;
2051 + dirs_list = dirs_list->next;
2053 + if (strncmp(reldir, start_dir, start_dir_len) == 0) {
2054 + if (reldir[start_dir_len] == '\0')
2056 + else if (reldir[start_dir_len] == '/')
2057 + reldir += start_dir_len + 1;
2059 + if (db_output_dirs)
2060 + printf("... %s/ ...\n", reldir);
2062 + if (chdir(dir) < 0) {
2063 + rprintf(FERROR, "Unable to chdir to %s: %s\n", dir, strerror(errno));
2066 + if (!(names = get_sorted_names(dir)))
2069 + subdirs = prior_subdir = prior_name = NULL;
2072 + char *dbsum4, *sum4, sumbuf4[MD5_DIGEST_LEN];
2073 + char *dbsum5, *sum5, sumbuf5[MD5_DIGEST_LEN];
2074 + int right_sum_cnt, wrong_sum_cnt;
2075 + const char *name = names->name;
2076 + unsigned int disk_id;
2079 + free((void*)prior_name);
2080 + prior_name = names;
2081 + names = names->next;
2083 + dbsum4 = dbsum5 = sum4 = sum5 = NULL;
2085 + if (lstat(name, &st) < 0) {
2086 + rprintf(FERROR, "Failed to lstat(%s): %s\n", name, strerror(errno));
2089 + if (S_ISLNK(st.st_mode))
2091 + if (S_ISDIR(st.st_mode)) {
2092 + /* add optional excluding of things like /^(CVS|\.svn|\.git|\.bzr)$/; */
2094 + struct name_list *add = new_name(dir, name);
2097 + prior_subdir->next = add;
2100 + prior_subdir = add;
2105 + if (!S_ISREG(st.st_mode))
2108 + if (!(disk_id = get_disk_id(st.st_dev)))
2111 + db_note_present(disk_id, st.st_ino);
2112 + if (!db_update && !db_check)
2115 + db_get_both_checksums(&st, &right_sum_cnt, &wrong_sum_cnt,
2116 + db_do_md4 ? &dbsum4 : NULL, db_do_md5 ? &dbsum5 : NULL);
2118 + if (!db_check && right_sum_cnt == need_sum_cnt) {
2119 + mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, dbsum4, dbsum5);
2123 + if (db_update || (db_check && right_sum_cnt) || db_output_sum) {
2126 + md_context m4, m5;
2127 + struct map_struct *buf;
2128 + OFF_T off, len = st.st_size;
2129 + int fd = do_open(name, O_RDONLY, 0);
2132 + rprintf(FERROR, "ERROR: unable to read %s: %s\n", name, strerror(errno));
2137 + mdfour_begin(&m4);
2141 + buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
2143 + for (off = 0; off + CSUM_CHUNK <= len; off += CSUM_CHUNK) {
2144 + data = (uchar*)map_ptr(buf, off, CSUM_CHUNK);
2146 + mdfour_update(&m4, data, CSUM_CHUNK);
2148 + md5_update(&m5, data, CSUM_CHUNK);
2151 + remainder = (int32)(len - off);
2152 + data = (uchar*)map_ptr(buf, off, remainder);
2154 + mdfour_update(&m4, data, remainder);
2155 + mdfour_result(&m4, (uchar*)(sum4 = sumbuf4));
2158 + md5_update(&m5, data, remainder);
2159 + md5_result(&m5, (uchar*)(sum5 = sumbuf5));
2166 + int chg = mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, sum4, sum5);
2168 + /* Only db_check should get here... */
2169 + } else if (!db_update) {
2173 + if (db_do_md4 && !db_set_checksum(4, &st, sum4))
2175 + if (db_do_md5 && !db_set_checksum(5, &st, sum5))
2178 + fprintf(stderr, "Failed to set checksum on %s/%s\n", reldir, name);
2179 + exit_cleanup(RERR_FILEIO);
2184 + free((void*)prior_name);
2186 + if (recurse && subdirs) {
2187 + prior_subdir->next = dirs_list;
2188 + dirs_list = subdirs;
2192 + free((void*)prior_dir);
2195 + int rows = db_clean_inodes();
2196 + if (db_output_msgs)
2197 + printf("Cleaned out %d old inode%s.\n", rows, rows == 1 ? "" : "s");
2200 + db_disconnect(True);
2203 diff --git a/flist.c b/flist.c
2206 @@ -54,6 +54,7 @@ extern int preserve_devices;
2207 extern int preserve_specials;
2208 extern int delete_during;
2209 extern int missing_args;
2211 extern int eol_nulls;
2212 extern int relative_paths;
2213 extern int implied_dirs;
2214 @@ -1294,11 +1295,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
2215 extra_len += EXTRA_LEN;
2218 - if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
2219 - file_checksum(thisname, &st, tmp_sum);
2220 - if (sender_keeps_checksum)
2221 - extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
2223 + if (sender_keeps_checksum && S_ISREG(st.st_mode))
2224 + extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
2226 #if EXTRA_ROUNDING > 0
2227 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
2228 @@ -1383,8 +1381,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
2232 - if (sender_keeps_checksum && S_ISREG(st.st_mode))
2233 - memcpy(F_SUM(file), tmp_sum, flist_csum_len);
2234 + if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
2235 + if (!use_db || !db_get_checksum(&st, tmp_sum))
2236 + file_checksum(thisname, &st, tmp_sum);
2237 + if (sender_keeps_checksum)
2238 + memcpy(F_SUM(file), tmp_sum, flist_csum_len);
2242 F_NDX(file) = stats.num_dirs;
2243 @@ -2045,6 +2047,9 @@ void send_extra_file_list(int f, int at_least)
2245 if (io_error != save_io_error && protocol_version == 30 && !ignore_errors)
2246 send_msg_int(MSG_IO_ERROR, io_error);
2248 + if (use_db && flist_eof)
2249 + db_disconnect(True);
2252 struct file_list *send_file_list(int f, int argc, char *argv[])
2253 @@ -2068,6 +2073,13 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
2254 | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
2255 int implied_dot_dir = 0;
2258 + if (always_checksum)
2259 + db_connect(0); /* Will reset use_db on error. */
2264 rprintf(FLOG, "building file list\n");
2265 if (show_filelist_progress)
2266 start_filelist_progress("building file list");
2267 @@ -2414,6 +2426,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
2268 rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i());
2271 + if (use_db && (!inc_recurse || flist_eof))
2272 + db_disconnect(True);
2277 diff --git a/generator.c b/generator.c
2280 @@ -59,6 +59,7 @@ extern int ignore_existing;
2281 extern int ignore_non_existing;
2282 extern int want_xattr_optim;
2285 extern int append_mode;
2286 extern int make_backups;
2287 extern int csum_length;
2288 @@ -582,7 +583,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
2289 of the file time to determine whether to sync */
2290 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
2291 char sum[MAX_DIGEST_LEN];
2292 - file_checksum(fn, st, sum);
2293 + if (!use_db || !db_get_checksum(st, sum))
2294 + file_checksum(fn, st, sum);
2295 return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
2298 @@ -2225,6 +2227,13 @@ void generate_files(int f_out, const char *local_name)
2303 + if (always_checksum || (append_mode != 1 && protocol_version >= 30))
2304 + db_connect(0); /* Will reset use_db on error. */
2309 dflt_perms = (ACCESSPERMS & ~orig_umask);
2312 @@ -2350,6 +2359,9 @@ void generate_files(int f_out, const char *local_name)
2313 wait_for_receiver();
2317 + db_disconnect(True);
2319 info_levels[INFO_FLIST] = save_info_flist;
2320 info_levels[INFO_PROGRESS] = save_info_progress;
2322 diff --git a/io.c b/io.c
2325 @@ -41,8 +41,10 @@ extern int am_server;
2326 extern int am_sender;
2327 extern int am_receiver;
2328 extern int am_generator;
2329 +extern int local_server;
2330 extern int msgs2stderr;
2331 extern int inc_recurse;
2332 +extern int same_db;
2333 extern int io_error;
2334 extern int eol_nulls;
2335 extern int flist_eof;
2336 @@ -1481,6 +1483,32 @@ static void read_a_msg(void)
2338 maybe_send_keepalive(time(NULL), MSK_ALLOW_FLUSH);
2340 + case MSG_CHECKSUM:
2341 + /* This receives some checksum info that we want to make a note of
2342 + * (which allows a single process to do all the writing to the db). */
2343 + if (msg_bytes != MSG_CHECKSUM_LEN)
2345 + raw_read_buf(data, MSG_CHECKSUM_LEN);
2346 + if (am_generator && same_db) {
2347 + iobuf.in_multiplexed = 1;
2348 + send_msg(MSG_CHECKSUM, data, MSG_CHECKSUM_LEN, 0);
2349 + } if (am_receiver || (am_sender && !local_server))
2352 + /* The received data is a set of numbers followed by the checksum. */
2354 + st.st_dev = IVAL64(data, 0);
2355 + st.st_ino = IVAL64(data, 8);
2356 + st.st_size = IVAL64(data, 16);
2357 + st.st_mtime = IVAL64(data, 24);
2358 + st.st_ctime = IVAL64(data, 32);
2359 +#if MSG_CHECKSUM_LONGS != 5
2360 +#error Fix the parsing of checksum long values
2362 + iobuf.in_multiplexed = 1;
2363 + db_set_checksum(IVAL(data, MSG_CHECKSUM_LONGS*8), &st, data + MSG_CHECKSUM_LONGS*8 + 4);
2367 if (msg_bytes >= sizeof data)
2369 @@ -1632,6 +1660,7 @@ static void read_a_msg(void)
2370 * with a duplicate exit message. */
2371 _exit_cleanup(val, __FILE__, 0 - __LINE__);
2374 rprintf(FERROR, "unexpected tag %d [%s%s]\n",
2375 tag, who_am_i(), inc_recurse ? "/inc" : "");
2376 exit_cleanup(RERR_STREAMIO);
2377 diff --git a/loadparm.c b/loadparm.c
2380 @@ -112,6 +112,7 @@ typedef struct {
2385 char *dont_compress;
2388 @@ -143,6 +144,7 @@ typedef struct {
2389 int syslog_facility;
2394 BOOL forward_lookup;
2396 @@ -192,6 +194,7 @@ static const all_vars Defaults = {
2397 /* auth_users; */ NULL,
2398 /* charset; */ NULL,
2399 /* comment; */ NULL,
2400 + /* db_config; */ NULL,
2401 /* dont_compress; */ DEFAULT_DONT_COMPRESS,
2402 /* exclude; */ NULL,
2403 /* exclude_from; */ NULL,
2404 @@ -221,6 +224,7 @@ static const all_vars Defaults = {
2405 /* syslog_facility; */ LOG_DAEMON,
2408 + /* db_lax; */ False,
2409 /* fake_super; */ False,
2410 /* forward_lookup; */ True,
2411 /* ignore_errors; */ False,
2412 @@ -333,6 +337,8 @@ static struct parm_struct parm_table[] =
2413 {"auth users", P_STRING, P_LOCAL, &Vars.l.auth_users, NULL,0},
2414 {"charset", P_STRING, P_LOCAL, &Vars.l.charset, NULL,0},
2415 {"comment", P_STRING, P_LOCAL, &Vars.l.comment, NULL,0},
2416 + {"db config", P_STRING, P_LOCAL, &Vars.l.db_config, NULL,0},
2417 + {"db lax", P_BOOL, P_LOCAL, &Vars.l.db_lax, NULL,0},
2418 {"dont compress", P_STRING, P_LOCAL, &Vars.l.dont_compress, NULL,0},
2419 {"exclude from", P_STRING, P_LOCAL, &Vars.l.exclude_from, NULL,0},
2420 {"exclude", P_STRING, P_LOCAL, &Vars.l.exclude, NULL,0},
2421 @@ -469,6 +475,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Vars.g.rsync_port)
2422 FN_LOCAL_STRING(lp_auth_users, auth_users)
2423 FN_LOCAL_STRING(lp_charset, charset)
2424 FN_LOCAL_STRING(lp_comment, comment)
2425 +FN_LOCAL_STRING(lp_db_config, db_config)
2426 FN_LOCAL_STRING(lp_dont_compress, dont_compress)
2427 FN_LOCAL_STRING(lp_exclude, exclude)
2428 FN_LOCAL_STRING(lp_exclude_from, exclude_from)
2429 @@ -498,6 +505,7 @@ FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity)
2430 FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility)
2431 FN_LOCAL_INTEGER(lp_timeout, timeout)
2433 +FN_LOCAL_BOOL(lp_db_lax, db_lax)
2434 FN_LOCAL_BOOL(lp_fake_super, fake_super)
2435 FN_LOCAL_BOOL(lp_forward_lookup, forward_lookup)
2436 FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
2437 diff --git a/main.c b/main.c
2440 @@ -34,6 +34,7 @@ extern int am_root;
2441 extern int am_server;
2442 extern int am_sender;
2443 extern int am_daemon;
2444 +extern int am_dbadmin;
2445 extern int inc_recurse;
2446 extern int blocking_io;
2447 extern int always_checksum;
2448 @@ -51,6 +52,7 @@ extern int copy_unsafe_links;
2449 extern int keep_dirlinks;
2450 extern int preserve_hard_links;
2451 extern int protocol_version;
2452 +extern int always_checksum;
2453 extern int file_total;
2455 extern int xfer_dirs;
2456 @@ -85,6 +87,7 @@ extern char *filesfrom_host;
2457 extern char *partial_dir;
2458 extern char *dest_option;
2459 extern char *rsync_path;
2460 +extern char *db_config;
2461 extern char *shell_cmd;
2462 extern char *batch_name;
2463 extern char *password_file;
2464 @@ -1103,6 +1106,9 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
2465 if (am_daemon && io_timeout && protocol_version >= 31)
2466 send_msg_int(MSG_IO_TIMEOUT, io_timeout);
2469 + db_read_config(FERROR, db_config);
2472 keep_dirlinks = 0; /* Must be disabled on the sender. */
2473 if (need_messages_from_generator)
2474 @@ -1384,6 +1390,9 @@ static int start_client(int argc, char *argv[])
2479 + db_read_config(FERROR, db_config);
2481 if (daemon_over_rsh < 0)
2482 return start_socket_client(shell_machine, remote_argc, remote_argv, argc, argv);
2484 diff --git a/mkproto.pl b/mkproto.pl
2487 @@ -13,6 +13,8 @@ if (open(IN, 'proto.h')) {
2491 +@ARGV = grep !m{/rsyncdb\.c$}, @ARGV;
2494 $protos = qq|/* This file is automatically generated with "make proto". DO NOT EDIT */\n\n|;
2496 diff --git a/options.c b/options.c
2499 @@ -82,6 +82,7 @@ int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
2502 int am_starting_up = 1;
2503 +int am_dbadmin = 0;
2504 int relative_paths = -1;
2505 int implied_dirs = 1;
2506 int missing_args = 0; /* 0 = FERROR_XFER, 1 = ignore, 2 = delete */
2507 @@ -95,6 +96,7 @@ int use_qsort = 0;
2508 char *files_from = NULL;
2509 int filesfrom_fd = -1;
2510 char *filesfrom_host = NULL;
2511 +char *db_config = NULL;
2513 int protect_args = -1;
2514 int human_readable = 1;
2515 @@ -102,6 +104,9 @@ int recurse = 0;
2516 int allow_inc_recurse = 1;
2519 +int db_clean, db_check, db_do_md4, db_do_md5, db_update = 1, db_lax, db_init, db_mounts;
2520 +int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
2521 +int saw_db_output_opt, saw_db_sum_opt;
2522 int connect_timeout = 0;
2523 int keep_partial = 0;
2524 int safe_symlinks = 0;
2525 @@ -271,6 +276,7 @@ static struct output_struct debug_words[COUNT_DEBUG+1] = {
2526 DEBUG_WORD(CHDIR, W_CLI|W_SRV, "Debug when the current directory changes"),
2527 DEBUG_WORD(CONNECT, W_CLI, "Debug connection events (levels 1-2)"),
2528 DEBUG_WORD(CMD, W_CLI, "Debug commands+options that are issued (levels 1-2)"),
2529 + DEBUG_WORD(DB, W_SND|W_REC, "Debug DB operations (levels 1-5)"),
2530 DEBUG_WORD(DEL, W_REC, "Debug delete actions (levels 1-3)"),
2531 DEBUG_WORD(DELTASUM, W_SND|W_REC, "Debug delta-transfer checksumming (levels 1-4)"),
2532 DEBUG_WORD(DUP, W_REC, "Debug weeding of duplicate names"),
2533 @@ -573,6 +579,7 @@ static void print_rsync_version(enum logcode f)
2534 char const *links = "no ";
2535 char const *iconv = "no ";
2536 char const *ipv6 = "no ";
2537 + char const *db = "no ";
2538 STRUCT_STAT *dumstat;
2540 #if SUBPROTOCOL_VERSION != 0
2541 @@ -609,6 +616,11 @@ static void print_rsync_version(enum logcode f)
2542 #ifdef CAN_SET_SYMLINK_TIMES
2545 +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
2547 +#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
2551 rprintf(f, "%s version %s protocol version %d%s\n",
2552 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
2553 @@ -622,8 +634,8 @@ static void print_rsync_version(enum logcode f)
2554 (int)(sizeof (int64) * 8));
2555 rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
2556 got_socketpair, hardlinks, links, ipv6, have_inplace);
2557 - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc\n",
2558 - have_inplace, acls, xattrs, iconv, symtimes, prealloc);
2559 + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc, %sdb\n",
2560 + have_inplace, acls, xattrs, iconv, symtimes, prealloc, db);
2562 #ifdef MAINTAINER_MODE
2563 rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
2564 @@ -672,6 +684,9 @@ void usage(enum logcode F)
2565 rprintf(F," -q, --quiet suppress non-error messages\n");
2566 rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
2567 rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
2568 + rprintf(F," --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums\n");
2569 + rprintf(F," --db-only=CONFIG_FILE behave like rsyncdb\n");
2570 + rprintf(F," --db-lax ignore ctime changes (use with CAUTION)\n");
2571 rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
2572 rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
2573 rprintf(F," -r, --recursive recurse into directories\n");
2574 @@ -820,6 +835,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
2575 OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
2576 OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
2577 OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG,
2578 + OPT_NO_DB, OPT_DBONLY,
2579 OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT,
2580 OPT_SERVER, OPT_REFUSED_BASE = 9000};
2582 @@ -960,6 +976,10 @@ static struct poptOption long_options[] = {
2583 {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
2584 {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
2585 {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
2586 + {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
2587 + {"no-db", 0, POPT_ARG_NONE, 0, OPT_NO_DB, 0, 0 },
2588 + {"db-lax", 0, POPT_ARG_VAL, &db_lax, 1, 0, 0 },
2589 + {"no-db-lax", 0, POPT_ARG_VAL, &db_lax, 0, 0, 0 },
2590 {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
2591 {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
2592 {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
2593 @@ -1050,6 +1070,9 @@ static struct poptOption long_options[] = {
2594 {"dparam", 0, POPT_ARG_STRING, 0, OPT_DAEMON, 0, 0 },
2595 {"detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 },
2596 {"no-detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 },
2597 + /* All the following options switch us into DB-admin option-parsing. */
2598 + {"db-help", 0, POPT_ARG_NONE, 0, OPT_DBONLY, 0, 0 },
2599 + {"db-only", 0, POPT_ARG_STRING, 0, OPT_DBONLY, 0, 0 },
2603 @@ -1103,6 +1126,50 @@ static struct poptOption long_daemon_options[] = {
2607 +static void dbonly_usage(enum logcode F)
2609 + rprintf(F,"Usage: rsyncdb --db=CONFIG_FILE [OPTIONS] [DIRS]\n");
2611 + rprintf(F,"Options:\n");
2612 + rprintf(F," --db=CONFIG Specify the CONFIG file to read for the DB info.\n");
2613 + rprintf(F," --db-lax Ignore ctime changes (use with CAUTION).\n");
2614 + rprintf(F,"-r, --recursive Scan files in subdirs (the default w/o --no-recursive).\n");
2615 + rprintf(F,"-s, --sums=SUMS List which checksums to update (default: 4,5).\n");
2616 + rprintf(F,"-o, --output=STR One or more letters of what to output (default is nothing).\n");
2617 + rprintf(F,"-c, --check Check the checksums (by reading the files) and fix issues.\n");
2618 + rprintf(F," --clean Note all inodes in the DIRS and remove DB extras.\n");
2619 + rprintf(F,"-N, --no-update Avoids updating/adding info with --check and/or --clean.\n");
2620 + rprintf(F," --init Initialize a DB by (re-)creating its tables.\n");
2621 + rprintf(F," --mounts Scan for mounted filesystems and update the DB.\n");
2622 + rprintf(F,"-q, --quiet Disable the default non-error output.\n");
2623 + rprintf(F,"-h, --help Display this help message.\n");
2626 +static struct poptOption long_dbonly_options[] = {
2627 + /* longName, shortName, argInfo, argPtr, value, descrip, argDesc */
2628 + {"check", 'c', POPT_ARG_NONE, &db_check, 0, 0, 0},
2629 + {"clean", 0, POPT_ARG_NONE, &db_clean, 0, 0, 0},
2630 + {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
2631 + {"db-only", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
2632 + {"db-lax", 0, POPT_ARG_VAL, &db_lax, 1, 0, 0 },
2633 + {"no-db-lax", 0, POPT_ARG_VAL, &db_lax, 0, 0, 0 },
2634 + {"info", 0, POPT_ARG_STRING, 0, OPT_INFO, 0, 0 },
2635 + {"debug", 0, POPT_ARG_STRING, 0, OPT_DEBUG, 0, 0 },
2636 + {"update", 'u', POPT_ARG_VAL, &db_update, 1, 0, 0 },
2637 + {"no-update", 'N', POPT_ARG_VAL, &db_update, 0, 0, 0 },
2638 + {"no-u", 0, POPT_ARG_VAL, &db_update, 0, 0, 0 },
2639 + {"output", 'o', POPT_ARG_STRING, 0, 'o', 0, 0 },
2640 + {"recursive", 'r', POPT_ARG_VAL, &recurse, 1, 0, 0 },
2641 + {"no-recursive", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 },
2642 + {"no-r", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 },
2643 + {"sums", 's', POPT_ARG_STRING, 0, 's', 0, 0 },
2644 + {"init", 0, POPT_ARG_NONE, &db_init, 0, 0, 0 },
2645 + {"mounts", 0, POPT_ARG_NONE, &db_mounts, 0, 0, 0 },
2646 + {"quiet", 'q', POPT_ARG_NONE, &quiet, 0, 0, 0 },
2647 + {"help", 'h', POPT_ARG_NONE, 0, 'h', 0, 0 },
2648 + {"db-help", 0, POPT_ARG_NONE, 0, 'h', 0, 0 },
2649 + {0,0,0,0, 0, 0, 0}
2652 static char err_buf[200];
2654 @@ -1281,6 +1348,101 @@ static void create_refuse_error(int which)
2658 +static NORETURN void parse_dbonly_args(int argc, const char **argv)
2660 + poptContext pc = poptGetContext(RSYNC_NAME, argc, argv, long_dbonly_options, 0);
2667 + while ((opt = poptGetNextOpt(pc)) != -1) {
2671 + for (cp = poptGetOptArg(pc); *cp; cp++) {
2672 + switch (toLower(cp)) {
2674 + db_output_name = 1;
2678 + db_output_sum = db_output_name = 1;
2681 + db_output_info = db_output_name = 1;
2684 + db_output_unchanged = db_output_name = 1;
2687 + db_output_dirs = 1;
2691 + saw_db_output_opt = 1;
2695 + for (cp = poptGetOptArg(pc); *cp; cp++) {
2705 + saw_db_sum_opt = 1;
2709 + dbonly_usage(FINFO);
2713 + arg = poptGetOptArg(pc);
2714 + parse_output_words(info_words, info_levels, arg, USER_PRIORITY);
2718 + arg = poptGetOptArg(pc);
2719 + parse_output_words(debug_words, debug_levels, arg, USER_PRIORITY);
2724 + "rsyncdb: %s: %s\n",
2725 + poptBadOption(pc, POPT_BADOPTION_NOALIAS),
2726 + poptStrerror(opt));
2727 + goto dbonly_usage;
2732 + rprintf(FERROR, "You must specify the --db=FILE option.\n");
2735 + "(Type \"rsyncdb --help\" for assistance.)\n");
2736 + exit_cleanup(RERR_SYNTAX);
2740 + db_output_info = 1;
2741 + if (!saw_db_output_opt && !quiet)
2742 + db_output_dirs = db_output_name = 1;
2744 + db_output_msgs = 1;
2745 + if (!saw_db_sum_opt)
2748 + am_starting_up = 0;
2749 + run_dbonly(poptGetArgs(pc));
2750 + exit(42); /* NOT REACHED */
2753 /* This is used to make sure that --daemon & --server cannot be aliased to
2754 * something else. These options have always disabled popt aliases for the
2755 * parsing of a daemon or server command-line, but we have to make sure that
2756 @@ -1315,10 +1477,18 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2758 int orig_protect_args = protect_args;
2760 + arg = *argv + strlen(*argv);
2761 + if (arg - *argv > 2 && strcmp(arg-2, "db") == 0) {
2762 + parse_dbonly_args(argc, argv);
2767 set_refuse_options(ref);
2769 set_refuse_options("log-file*");
2770 + set_refuse_options("db");
2771 + set_refuse_options("db-lax");
2773 if (!*lp_charset(module_id))
2774 set_refuse_options("iconv");
2775 @@ -1444,6 +1614,12 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2781 + poptFreeContext(pc);
2782 + parse_dbonly_args(argc, argv);
2783 + break; /* NOT REACHED */
2785 case OPT_MODIFY_WINDOW:
2786 /* The value has already been set by popt, but
2787 * we need to remember that we're using a
2788 @@ -1518,6 +1694,10 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2789 preserve_devices = preserve_specials = 0;
2799 diff --git a/pipe.c b/pipe.c
2802 @@ -27,11 +27,16 @@ extern int am_server;
2803 extern int blocking_io;
2804 extern int filesfrom_fd;
2805 extern int munge_symlinks;
2806 +extern int always_checksum;
2808 +extern char *db_config;
2809 extern char *logfile_name;
2810 extern int remote_option_cnt;
2811 extern const char **remote_options;
2812 extern struct chmod_mode_struct *chmod_modes;
2817 * Create a child connected to us via its stdin/stdout.
2819 @@ -142,13 +147,22 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
2822 if (remote_option_cnt) {
2823 + const char *db_config_save = db_config;
2824 int rc = remote_option_cnt + 1;
2825 const char **rv = remote_options;
2826 if (!parse_arguments(&rc, &rv)) {
2828 exit_cleanup(RERR_SYNTAX);
2831 + if (db_config == db_config_save)
2832 + same_db = db_config != NULL;
2833 + else if (!db_config || !db_config_save || strcmp(db_config, db_config_save) != 0) {
2836 + db_read_config(FERROR, db_config);
2838 + } else if (use_db)
2841 if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
2842 close(to_child_pipe[1]) < 0 ||
2843 diff --git a/receiver.c b/receiver.c
2849 extern int do_xfers;
2853 extern int am_server;
2854 extern int inc_recurse;
2855 @@ -429,6 +431,11 @@ static void handle_delayed_updates(char *local_name)
2856 "rename failed for %s (from %s)",
2857 full_fname(fname), partialptr);
2859 + if (use_db && !db_lax) {
2861 + if (do_lstat(fname, &st) == 0)
2862 + db_update_ctime(5, &st);
2864 if (remove_source_files
2865 || (preserve_hard_links && F_IS_HLINKED(file)))
2866 send_msg_int(MSG_SUCCESS, ndx);
2867 @@ -535,6 +542,9 @@ int recv_files(int f_in, int f_out, char *local_name)
2869 delayed_bits = bitbag_create(cur_flist->used + 1);
2871 + if (use_db && (append_mode == 1 || protocol_version < 30))
2872 + use_db = 0; /* We can't note finished md5 values */
2877 @@ -863,6 +873,8 @@ int recv_files(int f_in, int f_out, char *local_name)
2878 do_unlink(partialptr);
2879 handle_partial_dir(partialptr, PDIR_DELETE);
2881 + if (use_db && do_lstat(fname, &st) == 0)
2882 + db_set_checksum(5, &st, sender_file_sum);
2883 } else if (keep_partial && partialptr) {
2884 if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
2886 @@ -876,6 +888,8 @@ int recv_files(int f_in, int f_out, char *local_name)
2888 else if (delay_updates && recv_ok) {
2889 bitbag_set_bit(delayed_bits, ndx);
2890 + if (use_db && do_lstat(partialptr, &st) == 0)
2891 + db_set_checksum(5, &st, sender_file_sum);
2895 diff --git a/rsync.c b/rsync.c
2898 @@ -39,6 +39,7 @@ extern int am_daemon;
2899 extern int am_sender;
2900 extern int am_receiver;
2901 extern int am_generator;
2902 +extern int am_dbadmin;
2903 extern int am_starting_up;
2904 extern int allow_8bit_chars;
2905 extern int protocol_version;
2906 @@ -748,6 +749,8 @@ struct file_list *flist_for_ndx(int ndx, const char *fatal_error_loc)
2908 const char *who_am_i(void)
2913 return am_server ? "server" : "client";
2914 return am_sender ? "sender"
2915 diff --git a/rsync.h b/rsync.h
2918 @@ -241,12 +241,16 @@ enum msgcode {
2919 MSG_IO_ERROR=22,/* the sending side had an I/O error */
2920 MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */
2921 MSG_NOOP=42, /* a do-nothing message (legacy protocol-30 only) */
2922 + MSG_CHECKSUM=55,/* sent via rcvr -> gen pipe and local-host-only gen -> sender */
2923 MSG_ERROR_EXIT=86, /* synchronize an error exit (siblings and protocol >= 31) */
2924 MSG_SUCCESS=100,/* successfully updated indicated flist index */
2925 MSG_DELETED=101,/* successfully deleted a file on receiving side */
2926 MSG_NO_SEND=102,/* sender failed to open a file we wanted */
2929 +#define MSG_CHECKSUM_LONGS 5
2930 +#define MSG_CHECKSUM_LEN (MSG_CHECKSUM_LONGS*8 + 4 + MAX_DIGEST_LEN)
2933 #define NDX_FLIST_EOF -2
2934 #define NDX_DEL_STATS -3
2935 @@ -1267,7 +1271,8 @@ extern short info_levels[], debug_levels[];
2936 #define DEBUG_CHDIR (DEBUG_BIND+1)
2937 #define DEBUG_CONNECT (DEBUG_CHDIR+1)
2938 #define DEBUG_CMD (DEBUG_CONNECT+1)
2939 -#define DEBUG_DEL (DEBUG_CMD+1)
2940 +#define DEBUG_DB (DEBUG_CMD+1)
2941 +#define DEBUG_DEL (DEBUG_DB+1)
2942 #define DEBUG_DELTASUM (DEBUG_DEL+1)
2943 #define DEBUG_DUP (DEBUG_DELTASUM+1)
2944 #define DEBUG_EXIT (DEBUG_DUP+1)
2945 diff --git a/rsync.yo b/rsync.yo
2948 @@ -340,6 +340,9 @@ to the detailed description below for a complete description. verb(
2949 -q, --quiet suppress non-error messages
2950 --no-motd suppress daemon-mode MOTD (see caveat)
2951 -c, --checksum skip based on checksum, not mod-time & size
2952 + --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
2953 + --db-only=CONFIG_FILE Behave like rsyncdb (see that manpage).
2954 + --db-lax Ignore ctime changes (use with CAUTION).
2955 -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
2956 --no-OPTION turn off an implied OPTION (e.g. --no-D)
2957 -r, --recursive recurse into directories
2958 @@ -650,6 +653,67 @@ option's before-the-transfer "Does this file need to be updated?" check.
2959 For protocol 30 and beyond (first supported in 3.0.0), the checksum used is
2960 MD5. For older protocols, the checksum used is MD4.
2962 +dit(bf(--db=CONFIG_FILE)) This option specifies a CONFIG_FILE to read
2963 +that holds connection details for a database of checksum information.
2964 +When combined with the bf(--checksum) (bf(-c)) option, rsync will try to
2965 +use cached checksum information from the DB, and will update it if it is
2968 +The currently supported DB choices are MySQL and SQLite. For example, a
2969 +MySQL configuration might look like this:
2971 +verb( dbtype: mysql
2977 + thishost: hostname )
2979 +And a SQLite configuration might look like this:
2981 +verb( dbtype: SQLite
2982 + dbname: /var/cache/rsync/sum.db
2985 +Both the bf(--db) and bf(--db-lax) options only affect the side where the
2986 +option is used. To affect the remote side of a remote-shell connection,
2987 +use the bf(--remote-option) (bf(-M)) option. For example, to specify the
2988 +same options on both sides, you could specify something like this:
2990 +verb( rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ )
2992 +For a local copy, this option affects both the source and the destination.
2993 +If you wish a local copy to enable this option just for the destination
2994 +files, specify bf(-M--db=CONFIG) (the same for bf(-M--db-lax)). If you wish
2995 +a local copy to enable this option just for the source files, combine
2996 +bf(--db=CONFIG) with bf(-M--no-db) (similarly use bf(-M--no-db-lax)).
2998 +See the perl script "rsyncdb" in the support directory of the source code
2999 +(which may also be installed in /usr/bin) for a way to create the tables,
3000 +populate the mounted-disk information, check files against their checksums,
3001 +and update both the MD4 and MD5 checksums for files at the same time (since
3002 +an rsync copy will only update one or the other).
3004 +You can use a single MySQL DB for all your hosts if you give each one
3005 +their own "thishost" name and setup their device-mapping data. Or feel
3006 +free to use separate databases, separate servers, etc. See the rsync
3007 +daemon's "db config" parameter for how to configure a daemon to use a DB
3008 +(since a client cannot control this parameter on a daemon).
3010 +dit(bf(--db-lax)) This option can be used to modify the inode-matching
3011 +algorithm used by bf(--db) to one that ignores the ctime. This can be very
3012 +DANGEROUS unless your files are known to ALWAYS be updated in a safe manner.
3013 +If unsure, don't use it.
3015 +The reason you might want to use it is that the ctime (inode change time) is
3016 +changed by an added hard-link, or the file being moving around. To use this
3017 +option safely you must be CERTAIN that either rsync w/--db is the only program
3018 +adding files into the cached hierarchies, OR that all new files will have new
3019 +modify times (never a historical mtime that might match an orphaned inode).
3020 +So, for certain applications, such as mirrors of new tar releases, this option
3021 +can save a lot of unneeded checksum re-computation due to ctime changes.
3023 dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
3024 way of saying you want recursion and want to preserve almost
3025 everything (with -H being a notable omission).
3026 diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
3027 --- a/rsyncd.conf.yo
3028 +++ b/rsyncd.conf.yo
3029 @@ -326,6 +326,22 @@ is daemon. This setting has no effect if the "log file" setting is a
3030 non-empty string (either set in the per-modules settings, or inherited
3031 from the global settings).
3033 +dit(bf(db config)) This parameter specifies a config file to read that
3034 +holds connection details for a database of checksum information.
3036 +The config file will be read-in prior to any chroot restrictions, but
3037 +the connection occurs from inside the chroot. This means that you
3038 +should use a socket connection (e.g. 127.0.0.1 rather than localhost)
3039 +for a MySQL config from inside a chroot. For SQLite, the DB file must
3040 +be placed inside the chroot (though it can be placed outside the
3041 +transfer dir if you configured an inside-chroot path).
3043 +See the bf(--db=CONFIG_FILE) option for full details.
3045 +dit(bf(db lax)) This parameter specifies that a "db config" setup should use
3046 +lax (no ctime) lookups. See the rsync manpage's section -n bf(--db-lax) for
3047 +some warnings about using this setting.
3049 dit(bf(syslog tag)) This parameter allows you to specify the syslog
3050 tag to use when logging messages from the rsync daemon. The default is
3051 "rsyncd". This setting has no effect if the "log file" setting is a
3052 diff --git a/rsyncdb-mountinfo b/rsyncdb-mountinfo
3053 new file mode 100755
3055 +++ b/rsyncdb-mountinfo
3059 +# This script outputs data for rsyncdb --mounts. It must output a complete
3060 +# list of the mounts for the current host in a strict format -- 2 fields
3061 +# with a Tab between: $MOUNT_UNIQ\t$PATH
3063 +# The list of mounts MUST NOT contain any entry that has the same devnum
3064 +# (st_dev) as any other entry in the list (as checked via its PATH).
3066 +# MOUNT_UNIQ is a unique string that identifies the mount on this host.
3067 +# This cannot be the devnum (st_dev) because that can vary depending on the
3068 +# mount order or be reused for different mounts if they are not mounted at
3069 +# the same time. By default the value is "Mount of $devname", which should
3070 +# be adequate for situations that don't want removable media in the DB
3071 +# (though you may need to take steps to weed-out removable media from the
3072 +# list to ensure that such inodes stay out of the DB).
3074 +# You can override the MOUNT_UNIQ value by putting a .rsyndb_mount_uniq
3075 +# file in the root directory of any mount, at which point it is up to you
3076 +# to make sure that the value stays unique (note that all sequences of
3077 +# whitespace are transformed into a single space, and leading/trailing
3078 +# whitespace is removed).
3080 +# MOUNT_UNIQ may never contain a Tab but it would be legal for PATH to have
3081 +# a Tab (just really weird). Neither may have a CR or LF in it.
3083 +# The maximum size for MOUNT_UNIQ is 256 characters.
3085 +# If this script doesn't meet your needs, feel free to edit it and choose
3086 +# some other method of finding a unique value for each mount. If you come
3087 +# up with a good idiom that might be useful to others, please share it back
3093 +my $MOUNT_FILE = '/etc/mtab';
3094 +my $VALID_DEVICE_REGEX = qr{^/dev};
3098 +open MOUNTS, $MOUNT_FILE or die "Unable to open $MOUNT_FILE: $!\n";
3100 + my ($devname, $path) = (split)[0,1];
3101 + next unless $devname =~ /$VALID_DEVICE_REGEX/;
3103 + my ($devno) = (stat($path))[0];
3104 + next unless defined $devno; # Skip if mount is invalid.
3105 + next if $hash{$devno}++; # SKip if we've seen this devno earlier.
3107 + my $mount_uniq = "Mount of $devname";
3108 + if (open UNIQ, '<', "$path/.rsyndb_mount_uniq") {
3109 + $mount_uniq = <UNIQ>;
3111 + $mount_uniq =~ s/\s+/ /g; # This ensures no tab, CR, nor LF.
3112 + $mount_uniq =~ s/^ | $//g; # .. and no leading or trailing whitespace.
3114 + print $mount_uniq, "\t", $path, "\n";
3117 diff --git a/rsyncdb.yo b/rsyncdb.yo
3118 new file mode 100644
3122 +mailto(rsync-bugs@samba.org)
3123 +manpage(rsync)(1)(23 Jun 2013)()()
3124 +manpagename(rsyncdb)(Maintain an rsync checksum DB)
3127 +verb(rsyncdb --db=CONFIG [OPTION...] [DIR...])
3129 +manpagedescription()
3131 +Rsyncdb can maintain a checksum-caching DB that rsync can use to make its
3132 +bf(--checksum) option more optimal. You must specify a config file via
3133 +the bf(--db=CONFIG_FILE) option in order for rsyncdb to know what DB to
3134 +manipulate. See the rsync manpage's bf(--db) option for full details on
3137 +You can specify one or more directory args for rsyncdb to scan. If no
3138 +DIR args are specified, the current directory is assumed to be the spot
3141 +Note that the rsyncdb program is usually just a symlink to the rsync program.
3142 +You can force rsync to behave as rsyncdb either by having a symlink (or
3143 +hardlink) name that ends with "db" or by bf(starting) the rsync args with
3144 +bf(--db-only=CONFIG) (and that option works just like bf(--db=CONFIG) to
3145 +a program named rsyncdb).
3147 +manpagesection(EXAMPLES)
3149 +The following command will update checksum information in the database
3150 +described in the /etc/db.conf file:
3152 +verb( rsyncdb --db=/etc/db.conf -o n --clean /dir1 /dir2)
3154 +It scans 2 directory hierarchies (/dir1 & /dir2) and cleans out any
3155 +checksums whose inodes are no longer found in those directories (so that
3156 +directory args are presumed to be complete for this host's DB contents).
3158 +The following command will scan all the files in the /dir2 directory (without
3159 +recursive scanning, due to the bf(--no-r) option) and check them against
3162 +verb( rsyncdb --db=/etc/db.conf --check --no-r /dir2)
3164 +Any errors found are output as well as being fixed in the DB. (See
3165 +bf(--no-update) for how to check without updating.)
3167 +The following command will output MD5 sums for all the files found in the
3168 +directories mentioned, even if they are unchanged (due to the
3169 +bf(--output=u) option):
3171 +verb( rsyncdb --db=/etc/db.conf -rous /dir* >/tmp/md5sums.txt)
3173 +This is just like running md5sum, only faster. Unlike md5sum, you can't
3174 +specify a single file, so use bf(--no-r) and grep the output if you just
3175 +want to see a single file's value.
3177 +The following command initializes a new DB, and is required for any new DB:
3179 +verb( rsyncdb --db=/etc/db.conf --init --mounts)
3181 +The bf(--init) option should only be used once (unless you want to
3182 +destroy existing data). The bf(--mounts) option may need to be used
3185 +manpagesection(OPTIONS SUMMARY)
3187 +Rsyncdb accepts the following options: verb(
3188 + --db=CONFIG Specify the CONFIG file to read for the DB info.
3189 + --db-lax Ignore ctime changes (use with CAUTION).
3190 + --no-recursive Avoid the default --recursive (-r) scanning behavior.
3191 + -s, --sums=SUMS List which checksums to update (default: md5).
3192 + -o, --output=STR One or more letters of what to output (default is nothing).
3193 + -c, --check Check the checksums (by reading the files) and fix any
3194 + issues. Enables --output=i.
3195 + --clean Note all inodes in the DIRS and remove DB extras.
3196 + -N, --no-update Avoids updating/adding info with --check and/or --clean.
3197 + --init Initialize a DB by (re-)creating its tables.
3198 + --mounts Scan for mounted filesystems and update the DB.
3199 + -q, --quiet Disables the default non-error output.
3200 + -h, --help Display this help message.)
3204 +Rsyncdb accepts both long (double-dash + word) and short (single-dash + letter)
3205 +options. The full list of the available options are described below. If an
3206 +option can be specified in more than one way, the choices are comma-separated.
3207 +Some options only have a long variant, not a short. If the option takes a
3208 +parameter, the parameter is only listed after the long variant, even though it
3209 +must also be specified for the short. When specifying a parameter, you can
3210 +either use the form --option=param or replace the '=' with whitespace. The
3211 +parameter may need to be quoted in some manner for it to survive the shell's
3212 +command-line parsing.
3215 +dit(bf(--db=CONFIG_FILE)) This tells rsyncdb what DB-config file to read
3216 +for the DB setup. This is the same as the option in rsync, so refer to
3217 +that manpage for full details.
3219 +dit(bf(--db-lax)) This option works just like it does in rsync, so refer to
3220 +that manpage for full details.
3222 +dit(bf(--no-recursive, --no-r)) This disables the default recursive
3223 +directory scan that is performed on the listed directory args. The
3224 +options bf(--recursive) and bf(-r) are also accepted, if someone wants
3225 +to override an earlier bf(--no-r) override.
3227 +dit(bf(--sums=SUMS, -s)) Only output/update the listed checksum types. By
3228 +default we deal with just the newer md5 checksums (i.e. bf(--sums=5)).
3230 +Note that this option does NOT affect the order that checksums are output
3231 +if "-o s" is enabled, so bf(-s5,4) is the same as bf(-s4,5).
3233 +dit(bf(--output=STR, -o)) The output option lets you specify one or more
3234 +letters indicating what information should be output. The default is to
3235 +output "d" and "n" if bf(--output) is not specified.
3237 +The following letters are accepted:
3240 + it() bf(d) outputs "... dir_name ..." lines for each directory in our scan.
3241 + it() bf(n) outputs the names of files with changes (implied by all but "d").
3242 + it() bf(s) outputs checksum info for changes (implies bf(n)).
3243 + it() bf(u) outputs unchanged files too (implies bf(n)).
3244 + it() bf(i) outputs prefixed change info. The output strings are:
3245 + quote(itemization(
3246 + it() bf(!i) indicates that the time and/or size is wrong.
3247 + it() bf(+4) indicates the MD4 sum is missing.
3248 + it() bf(+5) indicates the MD5 sum is missing.
3249 + it() bf(!4) indicates the MD4 sum is wrong.
3250 + it() bf(!5) indicates the MD5 sum is wrong.
3251 + it() bf(?4) indicates an unknown MD4 difference. This can happen if we
3252 + didn't need to read the file; i.e. if the time/size is wrong and no sum
3253 + info was requested.
3254 + it() bf(?5) indicates an unknown MD5 difference.
3258 +dit(bf(--check, -c)) Check the checksums (forcing the reading of all the
3259 +files) and fix any issues that are found. Forces bf(--output=ni) on.
3261 +dit(bf(--clean)) Makes a temp-DB of all the inodes that we find in all the
3262 +listed directories and removes any extraneous checksums from the DB. You
3263 +will need to specify all the mounted directories that are present (and
3264 +listed as mounted) in the DB on this host or else the checksums from the
3265 +unvisited directories will be discarded from the DB. If you want to just
3266 +--clean without adding or updating the info of new or changed files,
3267 +specify bf(--no-update) as well.
3269 +See the bf(--mount)
3271 +dit(bf(--no-update, -N)) Avoids updating/adding info with bf(--check)
3272 +and/or bf(--clean).
3274 +dit(bf(--quiet, -q)) Disable the default (non-error) output settings. This
3275 +turns off the messages that bf(--init), bf(--mount), and bf(--clean) output,
3276 +and makes the default for bf(--output) be nothing (though an explicit
3277 +bf(--output) option is not affected).
3279 +dit(bf(--init)) Create the tables in the DB. If it is used on an existing
3280 +DB, all the existing tables are dropped and re-created.
3282 +This option disables scanning for checksum information, but may be combined
3285 +dit(bf(--mounts)) Populate the "disk" DB with the available device numbers
3286 +and change any mounted/unmount information for devices. This should be run
3287 +every time a mount-change happens that may affect a directory hierarchy in
3288 +the DB. Rsyncdb will not save any checksums for a device that is not
3289 +listed in the "disk" table.
3291 +Some advanced users may want to maintain the disk table themselves in order
3292 +to support mounting a drive in different (or multiple) locations, etc.
3294 +This option disables scanning for checksum information, but may be combined
3297 +dit(bf(--help, -h)) Display a summary of the options.
3307 +Rsyncdb was written by Wayne Davison.