1 This builds on the checksum-reading patch and adds the ability to
2 create and/or update the .rsyncsums files using extended mode args to
3 the --sumfiles=MODE option and the "checksum files = MODE" daemon
6 CAUTION: This patch is only lightly tested. If you're interested
7 in using it, please help out.
9 To use this patch, run these commands for a successful build:
11 patch -p1 <patches/checksum-reading.diff
12 patch -p1 <patches/checksum-updating.diff
13 ./configure (optional if already run)
18 - Fix the code that removes .rsyncsums files when a dir becomes empty.
20 based-on: patch/master/checksum-reading
21 diff --git a/flist.c b/flist.c
32 @@ -109,6 +110,9 @@ extern iconv_t ic_send, ic_recv;
34 #define PTR_SIZE (sizeof (struct file_struct *))
36 +#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */
37 +#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */
41 dev_t filesystem_dev; /* used to implement -x */
42 @@ -151,8 +155,13 @@ static char empty_sum[MAX_DIGEST_LEN];
43 static int flist_count_offset; /* for --delete --progress */
44 static int show_filelist_progress;
46 +#define REGULAR_SKIPPED(flist) ((flist)->to_redo)
48 static struct csum_cache {
49 struct file_list *flist;
50 + const char *dirname;
51 + int checksum_matches;
52 + int checksum_updates;
55 static struct file_list *flist_new(int flags, const char *msg);
56 @@ -345,7 +354,79 @@ static void flist_done_allocating(struct file_list *flist)
57 flist->pool_boundary = ptr;
60 -void reset_checksum_cache()
61 +static void checksum_filename(int slot, const char *dirname, char *fbuf)
63 + if (dirname && *dirname) {
66 + len = strlcpy(fbuf, basis_dir[slot-1], MAXPATHLEN);
67 + if (len >= MAXPATHLEN)
71 + if (pathjoin(fbuf+len, MAXPATHLEN-len, dirname, RSYNCSUMS_FILE) >= MAXPATHLEN-len)
74 + strlcpy(fbuf, RSYNCSUMS_FILE, MAXPATHLEN);
77 +static void write_checksums(int slot, struct file_list *flist, int whole_dir)
81 + char fbuf[MAXPATHLEN];
82 + int new_entries = csum_cache[slot].checksum_updates != 0;
83 + int counts_match = flist->used == csum_cache[slot].checksum_matches;
84 + int no_skipped = whole_dir && REGULAR_SKIPPED(flist) == 0;
85 + const char *dirname = csum_cache[slot].dirname;
87 + flist_sort_and_clean(flist, 0);
89 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
92 + checksum_filename(slot, dirname, fbuf);
94 + if (flist->high - flist->low < 0 && no_skipped) {
99 + if (!new_entries && (counts_match || !whole_dir))
102 + if (!(out_fp = fopen(fbuf, "w")))
105 + for (i = flist->low; i <= flist->high; i++) {
106 + struct file_struct *file = flist->sorted[i];
107 + const char *cp = F_SUM(file);
108 + const char *end = cp + flist_csum_len;
109 + const char *alt_sum = file->basename + strlen(file->basename) + 1;
110 + if (whole_dir && !(file->flags & FLAG_SUM_KEEP))
112 + if (checksum_type == 5)
113 + fprintf(out_fp, "%s ", alt_sum);
114 + if (file->flags & FLAG_SUM_MISSING) {
116 + fputs("==", out_fp);
117 + } while (++cp != end);
120 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
121 + } while (++cp != end);
123 + if (checksum_type != 5)
124 + fprintf(out_fp, " %s", alt_sum);
125 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
126 + (double)F_LENGTH(file), (double)file->modtime,
127 + (long)F_CTIME(file), (long)F_INODE(file), file->basename);
133 +void reset_checksum_cache(int whole_dir)
135 int slot, slots = am_sender ? 1 : basis_dir_cnt + 1;
137 @@ -356,6 +437,9 @@ void reset_checksum_cache()
138 struct file_list *flist = csum_cache[slot].flist;
141 + if (checksum_files & CSF_UPDATE && flist->next)
142 + write_checksums(slot, flist, whole_dir);
144 /* Reset the pool memory and empty the file-list array. */
145 pool_free_old(flist->file_pool,
146 pool_boundary(flist->file_pool, 0));
147 @@ -366,6 +450,10 @@ void reset_checksum_cache()
152 + csum_cache[slot].checksum_matches = 0;
153 + csum_cache[slot].checksum_updates = 0;
154 + REGULAR_SKIPPED(flist) = 0;
158 @@ -373,7 +461,7 @@ void reset_checksum_cache()
159 static int add_checksum(struct file_list *flist, const char *dirname,
160 const char *basename, int basename_len, OFF_T file_length,
161 time_t mtime, uint32 ctime, uint32 inode,
163 + const char *sum, const char *alt_sum, int flags)
165 struct file_struct *file;
166 int alloc_len, extra_len;
167 @@ -390,7 +478,7 @@ static int add_checksum(struct file_list *flist, const char *dirname,
168 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
169 extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
171 - alloc_len = FILE_STRUCT_LEN + extra_len + basename_len;
172 + alloc_len = FILE_STRUCT_LEN + extra_len + basename_len + flist_csum_len*2 + 1;
173 bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum");
175 memset(bp, 0, extra_len + FILE_STRUCT_LEN);
176 @@ -399,7 +487,14 @@ static int add_checksum(struct file_list *flist, const char *dirname,
177 bp += FILE_STRUCT_LEN;
179 memcpy(bp, basename, basename_len);
181 + strlcpy(bp+basename_len, alt_sum, flist_csum_len*2 + 1);
183 + memset(bp+basename_len, '=', flist_csum_len*2);
184 + bp[basename_len+flist_csum_len*2] = '\0';
187 + file->flags = flags;
188 file->mode = S_IFREG;
189 file->modtime = mtime;
190 file->len32 = (uint32)file_length;
191 @@ -428,10 +523,11 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
192 char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
200 + const char *alt_sum = NULL;
201 int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0;
203 if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN))
204 @@ -452,7 +548,7 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
205 while (fgets(line, sizeof line, fp)) {
207 if (checksum_type == 5) {
208 - char *alt_sum = cp;
211 while (*++cp == '=') {}
213 @@ -463,7 +559,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
218 + for (i = 0; i < flist_csum_len*2; i++, cp++) {
224 + memset(sum, 0, flist_csum_len);
225 + flags = FLAG_SUM_MISSING;
227 for (i = 0; i < flist_csum_len*2; i++, cp++) {
229 @@ -481,13 +584,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
237 while (*++cp == ' ') {}
239 if (checksum_type != 5) {
240 - char *alt_sum = cp;
243 while (*++cp == '=') {}
245 @@ -537,24 +641,112 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
248 strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
249 + if (is_excluded(fbuf, 0, ALL_FILTERS)) {
250 + flags |= FLAG_SUM_KEEP;
251 + csum_cache[slot].checksum_matches++;
254 add_checksum(flist, dirname, cp, len, file_length,
257 + sum, alt_sum, flags);
261 flist_sort_and_clean(flist, CLEAN_KEEP_LAST);
264 +void set_cached_checksum(struct file_list *file_flist, struct file_struct *file)
269 + char fbuf[MAXPATHLEN];
270 + const char *fn = f_name(file, NULL);
271 + struct file_list *flist = csum_cache[0].flist;
273 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
276 + if (stat(fn, &st) < 0)
279 + checksum_filename(0, file->dirname, fbuf);
281 + if (file_flist != flist->next) {
282 + const char *cp = F_SUM(file);
283 + const char *end = cp + flist_csum_len;
285 + if (!(out_fp = fopen(fbuf, "a")))
288 + if (checksum_type == 5) {
289 + for (j = 0; j < flist_csum_len; j++)
290 + fputs("==", out_fp);
291 + fputc(' ', out_fp);
294 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
295 + } while (++cp != end);
296 + if (checksum_type != 5) {
297 + fputc(' ', out_fp);
298 + for (j = 0; j < flist_csum_len; j++)
299 + fputs("==", out_fp);
301 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
302 + (double)st.st_size, (double)st.st_mtime,
303 + (long)(uint32)st.st_ctime, (long)(uint32)st.st_ino,
310 + if ((j = flist_find(flist, file)) >= 0) {
311 + struct file_struct *fp = flist->sorted[j];
313 + if (F_LENGTH(fp) != st.st_size) {
314 + fp->len32 = (uint32)st.st_size;
315 + if (st.st_size > 0xFFFFFFFFu) {
316 + OPT_EXTRA(fp, 0)->unum = (uint32)(st.st_size >> 32);
317 + fp->flags |= FLAG_LENGTH64;
319 + fp->flags &= FLAG_LENGTH64;
322 + if (fp->modtime != st.st_mtime) {
323 + fp->modtime = st.st_mtime;
326 + if (F_CTIME(fp) != (uint32)st.st_ctime) {
327 + F_CTIME(fp) = (uint32)st.st_ctime;
330 + if (F_INODE(fp) != (uint32)st.st_ino) {
331 + F_INODE(fp) = (uint32)st.st_ino;
334 + memcpy(F_SUM(fp), F_SUM(file), MAX_DIGEST_LEN);
335 + csum_cache[0].checksum_updates += inc;
336 + fp->flags &= ~FLAG_SUM_MISSING;
337 + fp->flags |= FLAG_SUM_KEEP;
341 + csum_cache[0].checksum_updates +=
342 + add_checksum(flist, file->dirname, file->basename, strlen(file->basename) + 1,
343 + st.st_size, (uint32)st.st_mtime, (uint32)st.st_ctime,
344 + st.st_ino, F_SUM(file), NULL, FLAG_SUM_KEEP);
347 void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
348 - STRUCT_STAT *stp, char *sum_buf)
349 + int basename_len, STRUCT_STAT *stp, char *sum_buf)
351 struct file_list *flist = csum_cache[slot].flist;
355 flist->next = cur_flist; /* next points from checksum flist to file flist */
356 + csum_cache[slot].dirname = file->dirname;
357 read_checksums(slot, flist, file->dirname);
360 @@ -566,12 +758,31 @@ void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
361 && (checksum_files & CSF_LAX
362 || (F_CTIME(fp) == (uint32)stp->st_ctime
363 && F_INODE(fp) == (uint32)stp->st_ino))) {
364 - memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
365 + if (fp->flags & FLAG_SUM_MISSING) {
366 + fp->flags &= ~FLAG_SUM_MISSING;
367 + csum_cache[slot].checksum_updates++;
368 + file_checksum(fname, stp, sum_buf);
369 + memcpy(F_SUM(fp), sum_buf, MAX_DIGEST_LEN);
371 + csum_cache[slot].checksum_matches++;
372 + memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
374 + fp->flags |= FLAG_SUM_KEEP;
380 file_checksum(fname, stp, sum_buf);
382 + if (checksum_files & CSF_UPDATE) {
383 + if (basename_len < 0)
384 + basename_len = strlen(file->basename) + 1;
385 + csum_cache[slot].checksum_updates +=
386 + add_checksum(flist, file->dirname, file->basename, basename_len,
387 + stp->st_size, stp->st_mtime, (uint32)stp->st_ctime,
388 + (uint32)stp->st_ino, sum_buf, NULL, FLAG_SUM_KEEP);
392 /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's
393 @@ -1511,6 +1722,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
394 if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) {
395 if (ignore_perishable)
396 non_perishable_cnt++;
397 + if (S_ISREG(st.st_mode))
398 + REGULAR_SKIPPED(flist)++;
402 @@ -1557,13 +1770,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
405 if (checksum_files && am_sender && flist)
406 - reset_checksum_cache();
407 + reset_checksum_cache(0);
411 if (checksum_files && am_sender && flist && lastdir_len == -2) {
413 - reset_checksum_cache();
414 + reset_checksum_cache(0);
417 basename_len = strlen(basename) + 1; /* count the '\0' */
418 @@ -1671,7 +1884,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
420 if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
421 if (flist && checksum_files)
422 - get_cached_checksum(0, thisname, file, &st, tmp_sum);
423 + get_cached_checksum(0, thisname, file, basename_len, &st, tmp_sum);
425 file_checksum(thisname, &st, tmp_sum);
426 if (sender_keeps_checksum)
427 @@ -2063,6 +2276,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len,
431 + if (checksum_files & CSF_UPDATE && am_sender && f >= 0)
432 + reset_checksum_cache(1);
434 if (f >= 0 && recurse && !divert_dirs) {
435 int i, end = flist->used - 1;
436 /* send_if_directory() bumps flist->used, so use "end". */
437 @@ -2726,6 +2942,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
438 rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i());
441 + if (checksum_files & CSF_UPDATE && flist_eof)
442 + reset_checksum_cache(0); /* writes any last updates */
447 diff --git a/generator.c b/generator.c
450 @@ -111,6 +111,7 @@ static int dir_tweaking;
451 static int symlink_timeset_failed_flags;
452 static int need_retouch_dir_times;
453 static int need_retouch_dir_perms;
454 +static int started_whole_dir, upcoming_whole_dir;
455 static const char *solo_file = NULL;
458 @@ -591,7 +592,7 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot
459 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
460 char sum[MAX_DIGEST_LEN];
461 if (checksum_files && slot >= 0)
462 - get_cached_checksum(slot, fn, file, st, sum);
463 + get_cached_checksum(slot, fn, file, -1, st, sum);
465 file_checksum(fn, st, sum);
466 return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
467 @@ -1335,7 +1336,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
470 if (checksum_files) {
471 - reset_checksum_cache();
472 + reset_checksum_cache(started_whole_dir);
473 + started_whole_dir = upcoming_whole_dir;
475 need_new_dirscan = 0;
477 @@ -1506,6 +1508,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
479 change_local_filter_dir(fname, strlen(fname), F_DEPTH(file));
481 + upcoming_whole_dir = file->flags & FLAG_CONTENT_DIR && f_out != -1 ? 1 : 0;
482 prior_dir_file = file;
485 @@ -1781,6 +1784,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
486 handle_partial_dir(partialptr, PDIR_DELETE);
488 set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT | maybe_ATTRS_ACCURATE_TIME);
489 + if (checksum_files & CSF_UPDATE)
490 + set_cached_checksum(cur_flist, file);
492 itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL);
493 #ifdef SUPPORT_HARD_LINKS
494 @@ -2277,6 +2282,7 @@ void generate_files(int f_out, const char *local_name)
496 change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
498 + upcoming_whole_dir = fp->flags & FLAG_CONTENT_DIR ? 1 : 0;
500 for (i = cur_flist->low; i <= cur_flist->high; i++) {
501 struct file_struct *file = cur_flist->sorted[i];
502 @@ -2371,6 +2377,9 @@ void generate_files(int f_out, const char *local_name)
506 + if (checksum_files)
507 + reset_checksum_cache(started_whole_dir);
509 info_levels[INFO_FLIST] = save_info_flist;
510 info_levels[INFO_PROGRESS] = save_info_progress;
512 diff --git a/io.c b/io.c
515 @@ -54,6 +54,7 @@ extern int read_batch;
516 extern int compat_flags;
517 extern int protect_args;
518 extern int checksum_seed;
519 +extern int checksum_files;
520 extern int protocol_version;
521 extern int remove_source_files;
522 extern int preserve_hard_links;
523 @@ -1061,6 +1062,9 @@ static void got_flist_entry_status(enum festatus status, int ndx)
525 flist->in_progress++;
527 + } else if (checksum_files & CSF_UPDATE) {
528 + struct file_struct *file = flist->files[ndx - flist->ndx_start];
529 + set_cached_checksum(flist, file);
533 diff --git a/loadparm.c b/loadparm.c
536 @@ -166,6 +166,10 @@ static struct enum_list enum_checksum_files[] = {
537 { CSF_IGNORE_FILES, "none" },
538 { CSF_LAX_MODE, "lax" },
539 { CSF_STRICT_MODE, "strict" },
540 + { CSF_LAX_MODE|CSF_UPDATE, "+lax" },
541 + { CSF_STRICT_MODE|CSF_UPDATE, "+strict" },
542 + { CSF_LAX_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++lax" },
543 + { CSF_STRICT_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++strict" },
547 diff --git a/options.c b/options.c
550 @@ -1939,7 +1939,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
553 arg = poptGetOptArg(pc);
554 - checksum_files = 0;
557 + checksum_files = CSF_UPDATE;
560 + checksum_files |= CSF_AFFECT_DRYRUN;
563 + checksum_files = 0;
564 if (strcmp(arg, "lax") == 0)
565 checksum_files |= CSF_LAX_MODE;
566 else if (strcmp(arg, "strict") == 0)
567 diff --git a/receiver.c b/receiver.c
570 @@ -50,6 +50,7 @@ extern int sparse_files;
571 extern int preallocate_files;
572 extern int keep_partial;
573 extern int checksum_seed;
574 +extern int checksum_files;
575 extern int whole_file;
577 extern int inplace_partial;
578 @@ -433,7 +434,7 @@ static void handle_delayed_updates(char *local_name)
579 "rename failed for %s (from %s)",
580 full_fname(fname), partialptr);
582 - if (remove_source_files
583 + if (remove_source_files || checksum_files & CSF_UPDATE
584 || (preserve_hard_links && F_IS_HLINKED(file)))
585 send_msg_int(MSG_SUCCESS, ndx);
586 handle_partial_dir(partialptr, PDIR_DELETE);
587 @@ -906,7 +907,7 @@ int recv_files(int f_in, int f_out, char *local_name)
591 - if (remove_source_files || inc_recurse
592 + if (remove_source_files || inc_recurse || checksum_files & CSF_UPDATE
593 || (preserve_hard_links && F_IS_HLINKED(file)))
594 send_msg_int(MSG_SUCCESS, ndx);
596 diff --git a/rsync.1.md b/rsync.1.md
599 @@ -707,9 +707,13 @@ your home directory (remove the '=' for that).
601 The MODE value is either "lax", for relaxed checking (which compares size
602 and mtime), "strict" (which also compares ctime and inode), or "none" to
603 - ignore any .rsyncsums files ("none" is the default). Rsync does not create
604 - or update these files, but there is a perl script in the support directory
605 - named "rsyncsums" that can be used for that.
606 + ignore any .rsyncsums files ("none" is the default).
607 + If you want rsync to create and/or update these files, specify a prefixed
608 + plus ("+lax" or "+strict"). Adding a second prefixed '+' causes the
609 + checksum-file updates to happen even when the transfer is in `--dry-run`
610 + mode ("++lax" or "++strict"). There is also a perl script in the support
611 + directory named "rsyncsums" that can be used to update the .rsyncsums
614 This option has no effect unless `--checksum`, `-c` was also specified. It
615 also only affects the current side of the transfer, so if you want the
616 diff --git a/rsync.h b/rsync.h
619 @@ -1088,6 +1088,8 @@ typedef struct {
621 #define CSF_ENABLE (1<<1)
622 #define CSF_LAX (1<<2)
623 +#define CSF_UPDATE (1<<3)
624 +#define CSF_AFFECT_DRYRUN (1<<4)
626 #define CSF_IGNORE_FILES 0
627 #define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX)
628 diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md
629 --- a/rsyncd.conf.5.md
630 +++ b/rsyncd.conf.5.md
631 @@ -424,13 +424,15 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details.
632 This parameter tells rsync to make use of any cached checksum information
633 it finds in per-directory .rsyncsums files when the current transfer is
634 using the `--checksum` option. The value can be set to either "lax",
635 - "strict", or "none". See the client's `--sumfiles` option for what these
637 + "strict", "+lax", "+strict", "++lax", "++strict", or +"none". See the
638 + client's `--sumfiles` option for what these choices do.
640 Note also that the client's command-line option, `--sumfiles`, has no
641 effect on a daemon. A daemon will only access checksum files if this
642 - config option tells it to. See also the `exclude` directive for a way to
643 - hide the .rsyncsums files from the user.
644 + config option tells it to. You can configure updating of the .rsyncsums
645 + files even if the module itself is configured to be read-only. See also
646 + the `exclude` directive for a way to hide the .rsyncsums files from the