1 This builds on the checksum-reading patch and adds the ability to
2 create and/or update the .rsyncsums files using extended mode args to
3 the --sumfiles=MODE option and the "checksum files = MODE" daemon
6 CAUTION: This patch is only lightly tested. If you're interested
7 in using it, please help out.
9 To use this patch, run these commands for a successful build:
11 patch -p1 <patches/checksum-reading.diff
12 patch -p1 <patches/checksum-updating.diff
13 ./configure (optional if already run)
18 - Fix the code that removes .rsyncsums files when a dir becomes empty.
20 based-on: patch/master/checksum-reading
21 diff --git a/flist.c b/flist.c
32 @@ -111,6 +112,9 @@ extern iconv_t ic_send, ic_recv;
34 #define PTR_SIZE (sizeof (struct file_struct *))
36 +#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */
37 +#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */
41 dev_t filesystem_dev; /* used to implement -x */
42 @@ -153,8 +157,13 @@ static char empty_sum[MAX_DIGEST_LEN];
43 static int flist_count_offset; /* for --delete --progress */
44 static int show_filelist_progress;
46 +#define REGULAR_SKIPPED(flist) ((flist)->to_redo)
48 static struct csum_cache {
49 struct file_list *flist;
50 + const char *dirname;
51 + int checksum_matches;
52 + int checksum_updates;
55 static struct file_list *flist_new(int flags, const char *msg);
56 @@ -349,7 +358,79 @@ static void flist_done_allocating(struct file_list *flist)
57 flist->pool_boundary = ptr;
60 -void reset_checksum_cache()
61 +static void checksum_filename(int slot, const char *dirname, char *fbuf)
63 + if (dirname && *dirname) {
66 + len = strlcpy(fbuf, basis_dir[slot-1], MAXPATHLEN);
67 + if (len >= MAXPATHLEN)
71 + if (pathjoin(fbuf+len, MAXPATHLEN-len, dirname, RSYNCSUMS_FILE) >= MAXPATHLEN-len)
74 + strlcpy(fbuf, RSYNCSUMS_FILE, MAXPATHLEN);
77 +static void write_checksums(int slot, struct file_list *flist, int whole_dir)
81 + char fbuf[MAXPATHLEN];
82 + int new_entries = csum_cache[slot].checksum_updates != 0;
83 + int counts_match = flist->used == csum_cache[slot].checksum_matches;
84 + int no_skipped = whole_dir && REGULAR_SKIPPED(flist) == 0;
85 + const char *dirname = csum_cache[slot].dirname;
87 + flist_sort_and_clean(flist, 0);
89 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
92 + checksum_filename(slot, dirname, fbuf);
94 + if (flist->high - flist->low < 0 && no_skipped) {
99 + if (!new_entries && (counts_match || !whole_dir))
102 + if (!(out_fp = fopen(fbuf, "w")))
105 + for (i = flist->low; i <= flist->high; i++) {
106 + struct file_struct *file = flist->sorted[i];
107 + const char *cp = F_SUM(file);
108 + const char *end = cp + flist_csum_len;
109 + const char *alt_sum = file->basename + strlen(file->basename) + 1;
110 + if (whole_dir && !(file->flags & FLAG_SUM_KEEP))
112 + if (checksum_type == 5)
113 + fprintf(out_fp, "%s ", alt_sum);
114 + if (file->flags & FLAG_SUM_MISSING) {
116 + fputs("==", out_fp);
117 + } while (++cp != end);
120 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
121 + } while (++cp != end);
123 + if (checksum_type != 5)
124 + fprintf(out_fp, " %s", alt_sum);
125 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
126 + (double)F_LENGTH(file), (double)file->modtime,
127 + (long)F_CTIME(file), (long)F_INODE(file), file->basename);
133 +void reset_checksum_cache(int whole_dir)
135 int slot, slots = am_sender ? 1 : basis_dir_cnt + 1;
137 @@ -360,6 +441,9 @@ void reset_checksum_cache()
138 struct file_list *flist = csum_cache[slot].flist;
141 + if (checksum_files & CSF_UPDATE && flist->next)
142 + write_checksums(slot, flist, whole_dir);
144 /* Reset the pool memory and empty the file-list array. */
145 pool_free_old(flist->file_pool,
146 pool_boundary(flist->file_pool, 0));
147 @@ -370,6 +454,10 @@ void reset_checksum_cache()
152 + csum_cache[slot].checksum_matches = 0;
153 + csum_cache[slot].checksum_updates = 0;
154 + REGULAR_SKIPPED(flist) = 0;
158 @@ -377,7 +465,7 @@ void reset_checksum_cache()
159 static int add_checksum(struct file_list *flist, const char *dirname,
160 const char *basename, int basename_len, OFF_T file_length,
161 time_t mtime, uint32 ctime, uint32 inode,
163 + const char *sum, const char *alt_sum, int flags)
165 struct file_struct *file;
166 int alloc_len, extra_len;
167 @@ -394,7 +482,7 @@ static int add_checksum(struct file_list *flist, const char *dirname,
168 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
169 extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
171 - alloc_len = FILE_STRUCT_LEN + extra_len + basename_len;
172 + alloc_len = FILE_STRUCT_LEN + extra_len + basename_len + flist_csum_len*2 + 1;
173 bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum");
175 memset(bp, 0, extra_len + FILE_STRUCT_LEN);
176 @@ -403,7 +491,14 @@ static int add_checksum(struct file_list *flist, const char *dirname,
177 bp += FILE_STRUCT_LEN;
179 memcpy(bp, basename, basename_len);
181 + strlcpy(bp+basename_len, alt_sum, flist_csum_len*2 + 1);
183 + memset(bp+basename_len, '=', flist_csum_len*2);
184 + bp[basename_len+flist_csum_len*2] = '\0';
187 + file->flags = flags;
188 file->mode = S_IFREG;
189 file->modtime = mtime;
190 file->len32 = (uint32)file_length;
191 @@ -432,10 +527,11 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
192 char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
200 + const char *alt_sum = NULL;
201 int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0;
203 if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN))
204 @@ -456,7 +552,7 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
205 while (fgets(line, sizeof line, fp)) {
207 if (checksum_type == 5) {
208 - char *alt_sum = cp;
211 while (*++cp == '=') {}
213 @@ -467,7 +563,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
218 + for (i = 0; i < flist_csum_len*2; i++, cp++) {
224 + memset(sum, 0, flist_csum_len);
225 + flags = FLAG_SUM_MISSING;
227 for (i = 0; i < flist_csum_len*2; i++, cp++) {
229 @@ -485,13 +588,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
237 while (*++cp == ' ') {}
239 if (checksum_type != 5) {
240 - char *alt_sum = cp;
243 while (*++cp == '=') {}
245 @@ -541,24 +645,112 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
248 strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
249 + if (is_excluded(fbuf, 0, ALL_FILTERS)) {
250 + flags |= FLAG_SUM_KEEP;
251 + csum_cache[slot].checksum_matches++;
254 add_checksum(flist, dirname, cp, len, file_length,
257 + sum, alt_sum, flags);
261 flist_sort_and_clean(flist, CLEAN_KEEP_LAST);
264 +void set_cached_checksum(struct file_list *file_flist, struct file_struct *file)
269 + char fbuf[MAXPATHLEN];
270 + const char *fn = f_name(file, NULL);
271 + struct file_list *flist = csum_cache[0].flist;
273 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
276 + if (stat(fn, &st) < 0)
279 + checksum_filename(0, file->dirname, fbuf);
281 + if (file_flist != flist->next) {
282 + const char *cp = F_SUM(file);
283 + const char *end = cp + flist_csum_len;
285 + if (!(out_fp = fopen(fbuf, "a")))
288 + if (checksum_type == 5) {
289 + for (j = 0; j < flist_csum_len; j++)
290 + fputs("==", out_fp);
291 + fputc(' ', out_fp);
294 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
295 + } while (++cp != end);
296 + if (checksum_type != 5) {
297 + fputc(' ', out_fp);
298 + for (j = 0; j < flist_csum_len; j++)
299 + fputs("==", out_fp);
301 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
302 + (double)st.st_size, (double)st.st_mtime,
303 + (long)(uint32)st.st_ctime, (long)(uint32)st.st_ino,
310 + if ((j = flist_find(flist, file)) >= 0) {
311 + struct file_struct *fp = flist->sorted[j];
313 + if (F_LENGTH(fp) != st.st_size) {
314 + fp->len32 = (uint32)st.st_size;
315 + if (st.st_size > 0xFFFFFFFFu) {
316 + OPT_EXTRA(fp, 0)->unum = (uint32)(st.st_size >> 32);
317 + fp->flags |= FLAG_LENGTH64;
319 + fp->flags &= FLAG_LENGTH64;
322 + if (fp->modtime != st.st_mtime) {
323 + fp->modtime = st.st_mtime;
326 + if (F_CTIME(fp) != (uint32)st.st_ctime) {
327 + F_CTIME(fp) = (uint32)st.st_ctime;
330 + if (F_INODE(fp) != (uint32)st.st_ino) {
331 + F_INODE(fp) = (uint32)st.st_ino;
334 + memcpy(F_SUM(fp), F_SUM(file), MAX_DIGEST_LEN);
335 + csum_cache[0].checksum_updates += inc;
336 + fp->flags &= ~FLAG_SUM_MISSING;
337 + fp->flags |= FLAG_SUM_KEEP;
341 + csum_cache[0].checksum_updates +=
342 + add_checksum(flist, file->dirname, file->basename, strlen(file->basename) + 1,
343 + st.st_size, (uint32)st.st_mtime, (uint32)st.st_ctime,
344 + st.st_ino, F_SUM(file), NULL, FLAG_SUM_KEEP);
347 void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
348 - STRUCT_STAT *stp, char *sum_buf)
349 + int basename_len, STRUCT_STAT *stp, char *sum_buf)
351 struct file_list *flist = csum_cache[slot].flist;
355 flist->next = cur_flist; /* next points from checksum flist to file flist */
356 + csum_cache[slot].dirname = file->dirname;
357 read_checksums(slot, flist, file->dirname);
360 @@ -570,12 +762,31 @@ void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
361 && (checksum_files & CSF_LAX
362 || (F_CTIME(fp) == (uint32)stp->st_ctime
363 && F_INODE(fp) == (uint32)stp->st_ino))) {
364 - memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
365 + if (fp->flags & FLAG_SUM_MISSING) {
366 + fp->flags &= ~FLAG_SUM_MISSING;
367 + csum_cache[slot].checksum_updates++;
368 + file_checksum(fname, stp, sum_buf);
369 + memcpy(F_SUM(fp), sum_buf, MAX_DIGEST_LEN);
371 + csum_cache[slot].checksum_matches++;
372 + memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
374 + fp->flags |= FLAG_SUM_KEEP;
380 file_checksum(fname, stp, sum_buf);
382 + if (checksum_files & CSF_UPDATE) {
383 + if (basename_len < 0)
384 + basename_len = strlen(file->basename) + 1;
385 + csum_cache[slot].checksum_updates +=
386 + add_checksum(flist, file->dirname, file->basename, basename_len,
387 + stp->st_size, stp->st_mtime, (uint32)stp->st_ctime,
388 + (uint32)stp->st_ino, sum_buf, NULL, FLAG_SUM_KEEP);
392 /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's
393 @@ -1581,6 +1792,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
394 if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) {
395 if (ignore_perishable)
396 non_perishable_cnt++;
397 + if (S_ISREG(st.st_mode))
398 + REGULAR_SKIPPED(flist)++;
402 @@ -1627,13 +1840,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
405 if (checksum_files && am_sender && flist)
406 - reset_checksum_cache();
407 + reset_checksum_cache(0);
411 if (checksum_files && am_sender && flist && lastdir_len == -2) {
413 - reset_checksum_cache();
414 + reset_checksum_cache(0);
417 basename_len = strlen(basename) + 1; /* count the '\0' */
418 @@ -1757,7 +1970,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
420 if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
421 if (flist && checksum_files)
422 - get_cached_checksum(0, thisname, file, &st, tmp_sum);
423 + get_cached_checksum(0, thisname, file, basename_len, &st, tmp_sum);
425 file_checksum(thisname, &st, tmp_sum);
426 if (sender_keeps_checksum)
427 @@ -2149,6 +2362,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len,
431 + if (checksum_files & CSF_UPDATE && am_sender && f >= 0)
432 + reset_checksum_cache(1);
434 if (f >= 0 && recurse && !divert_dirs) {
435 int i, end = flist->used - 1;
436 /* send_if_directory() bumps flist->used, so use "end". */
437 @@ -2814,6 +3030,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
438 rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i());
441 + if (checksum_files & CSF_UPDATE && flist_eof)
442 + reset_checksum_cache(0); /* writes any last updates */
447 diff --git a/generator.c b/generator.c
450 @@ -113,6 +113,7 @@ static int dir_tweaking;
451 static int symlink_timeset_failed_flags;
452 static int need_retouch_dir_times;
453 static int need_retouch_dir_perms;
454 +static int started_whole_dir, upcoming_whole_dir;
455 static const char *solo_file = NULL;
457 /* Forward declarations. */
458 @@ -627,7 +628,7 @@ int quick_check_ok(enum filetype ftype, const char *fn, struct file_struct *file
459 if (always_checksum > 0) {
460 char sum[MAX_DIGEST_LEN];
461 if (checksum_files && slot >= 0)
462 - get_cached_checksum(slot, fn, file, st, sum);
463 + get_cached_checksum(slot, fn, file, -1, st, sum);
465 file_checksum(fn, st, sum);
466 return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
467 @@ -1356,7 +1357,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
470 if (checksum_files) {
471 - reset_checksum_cache();
472 + reset_checksum_cache(started_whole_dir);
473 + started_whole_dir = upcoming_whole_dir;
475 need_new_dirscan = 0;
477 @@ -1544,6 +1546,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
479 change_local_filter_dir(fname, strlen(fname), F_DEPTH(file));
481 + upcoming_whole_dir = file->flags & FLAG_CONTENT_DIR && f_out != -1 ? 1 : 0;
482 prior_dir_file = file;
485 @@ -1816,6 +1819,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
486 handle_partial_dir(partialptr, PDIR_DELETE);
488 set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT | maybe_ATTRS_ACCURATE_TIME);
489 + if (checksum_files & CSF_UPDATE)
490 + set_cached_checksum(cur_flist, file);
492 itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL);
493 #ifdef SUPPORT_HARD_LINKS
494 @@ -2308,6 +2313,7 @@ void generate_files(int f_out, const char *local_name)
496 change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
498 + upcoming_whole_dir = fp->flags & FLAG_CONTENT_DIR ? 1 : 0;
500 for (i = cur_flist->low; i <= cur_flist->high; i++) {
501 struct file_struct *file = cur_flist->sorted[i];
502 @@ -2402,6 +2408,9 @@ void generate_files(int f_out, const char *local_name)
506 + if (checksum_files)
507 + reset_checksum_cache(started_whole_dir);
509 info_levels[INFO_FLIST] = save_info_flist;
510 info_levels[INFO_PROGRESS] = save_info_progress;
512 diff --git a/io.c b/io.c
515 @@ -54,6 +54,7 @@ extern int read_batch;
516 extern int compat_flags;
517 extern int protect_args;
518 extern int checksum_seed;
519 +extern int checksum_files;
520 extern int daemon_connection;
521 extern int protocol_version;
522 extern int remove_source_files;
523 @@ -1089,6 +1090,9 @@ static void got_flist_entry_status(enum festatus status, int ndx)
525 flist->in_progress++;
527 + } else if (checksum_files & CSF_UPDATE) {
528 + struct file_struct *file = flist->files[ndx - flist->ndx_start];
529 + set_cached_checksum(flist, file);
533 diff --git a/loadparm.c b/loadparm.c
536 @@ -166,6 +166,10 @@ static struct enum_list enum_checksum_files[] = {
537 { CSF_IGNORE_FILES, "none" },
538 { CSF_LAX_MODE, "lax" },
539 { CSF_STRICT_MODE, "strict" },
540 + { CSF_LAX_MODE|CSF_UPDATE, "+lax" },
541 + { CSF_STRICT_MODE|CSF_UPDATE, "+strict" },
542 + { CSF_LAX_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++lax" },
543 + { CSF_STRICT_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++strict" },
547 diff --git a/options.c b/options.c
550 @@ -1744,7 +1744,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
553 arg = poptGetOptArg(pc);
554 - checksum_files = 0;
557 + checksum_files = CSF_UPDATE;
560 + checksum_files |= CSF_AFFECT_DRYRUN;
563 + checksum_files = 0;
564 if (strcmp(arg, "lax") == 0)
565 checksum_files |= CSF_LAX_MODE;
566 else if (strcmp(arg, "strict") == 0)
567 diff --git a/receiver.c b/receiver.c
570 @@ -51,6 +51,7 @@ extern int sparse_files;
571 extern int preallocate_files;
572 extern int keep_partial;
573 extern int checksum_seed;
574 +extern int checksum_files;
575 extern int whole_file;
577 extern int inplace_partial;
578 @@ -439,7 +440,7 @@ static void handle_delayed_updates(char *local_name)
579 "rename failed for %s (from %s)",
580 full_fname(fname), partialptr);
582 - if (remove_source_files
583 + if (remove_source_files || checksum_files & CSF_UPDATE
584 || (preserve_hard_links && F_IS_HLINKED(file)))
585 send_msg_int(MSG_SUCCESS, ndx);
586 handle_partial_dir(partialptr, PDIR_DELETE);
587 @@ -926,7 +927,7 @@ int recv_files(int f_in, int f_out, char *local_name)
591 - if (remove_source_files || inc_recurse
592 + if (remove_source_files || inc_recurse || checksum_files & CSF_UPDATE
593 || (preserve_hard_links && F_IS_HLINKED(file)))
594 send_msg_int(MSG_SUCCESS, ndx);
596 diff --git a/rsync.1.md b/rsync.1.md
599 @@ -772,9 +772,13 @@ option name from the pathname using a space if you want the shell to expand it.
601 The MODE value is either "lax", for relaxed checking (which compares size
602 and mtime), "strict" (which also compares ctime and inode), or "none" to
603 - ignore any .rsyncsums files ("none" is the default). Rsync does not create
604 - or update these files, but there is a perl script in the support directory
605 - named "rsyncsums" that can be used for that.
606 + ignore any .rsyncsums files ("none" is the default).
607 + If you want rsync to create and/or update these files, specify a prefixed
608 + plus ("+lax" or "+strict"). Adding a second prefixed '+' causes the
609 + checksum-file updates to happen even when the transfer is in [`--dry-run`](#opt)
610 + mode ("++lax" or "++strict"). There is also a perl script in the support
611 + directory named "rsyncsums" that can be used to update the .rsyncsums
614 This option has no effect unless [`--checksum`](#opt) (`-c`) was also
615 specified. It also only affects the current side of the transfer, so if
616 diff --git a/rsync.h b/rsync.h
619 @@ -1121,6 +1121,8 @@ typedef struct {
621 #define CSF_ENABLE (1<<1)
622 #define CSF_LAX (1<<2)
623 +#define CSF_UPDATE (1<<3)
624 +#define CSF_AFFECT_DRYRUN (1<<4)
626 #define CSF_IGNORE_FILES 0
627 #define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX)
628 diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md
629 --- a/rsyncd.conf.5.md
630 +++ b/rsyncd.conf.5.md
631 @@ -426,13 +426,15 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details.
632 This parameter tells rsync to make use of any cached checksum information
633 it finds in per-directory .rsyncsums files when the current transfer is
634 using the `--checksum` option. The value can be set to either "lax",
635 - "strict", or "none". See the client's `--sumfiles` option for what these
637 + "strict", "+lax", "+strict", "++lax", "++strict", or +"none". See the
638 + client's `--sumfiles` option for what these choices do.
640 Note also that the client's command-line option, `--sumfiles`, has no
641 effect on a daemon. A daemon will only access checksum files if this
642 - config option tells it to. See also the `exclude` directive for a way to
643 - hide the .rsyncsums files from the user.
644 + config option tells it to. You can configure updating of the .rsyncsums
645 + files even if the module itself is configured to be read-only. See also
646 + the `exclude` directive for a way to hide the .rsyncsums files from the