1 This patch to rsync adds a %C log escape that expands to the sender's
2 post-transfer checksum of a file for protocol 30 or above. This way, if
3 you need the MD5 checksums of transferred files, you can have rsync log
4 them instead of spending extra processor time on a separate command to
7 -- Matt McCutchen <hashproduct@gmail.com>
9 To use this patch, run these commands for a successful build:
11 patch -p1 <patches/log-checksum.diff
12 ./configure (optional if already run)
15 based-on: 8750f64ec7893c263b51e538895af3092bce9f4c
16 diff --git a/flist.c b/flist.c
19 @@ -66,6 +66,7 @@ extern int munge_symlinks;
20 extern int use_safe_inc_flist;
21 extern int need_unsorted_flist;
22 extern int sender_symlink_iconv;
23 +extern int sender_keeps_checksum;
24 extern int unsort_ndx;
26 extern struct stats stats;
27 @@ -1232,6 +1233,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
28 extra_len += EXTRA_LEN;
31 + if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
32 + file_checksum(thisname, tmp_sum, st.st_size);
33 + if (sender_keeps_checksum)
34 + extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
37 #if EXTRA_ROUNDING > 0
38 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
39 extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
40 @@ -1298,9 +1305,6 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
41 memcpy(bp + basename_len, linkname, linkname_len);
44 - if (always_checksum && am_sender && S_ISREG(st.st_mode))
45 - file_checksum(thisname, tmp_sum, st.st_size);
48 F_PATHNAME(file) = pathname;
50 @@ -1312,6 +1316,9 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
54 + if (sender_keeps_checksum && S_ISREG(st.st_mode))
55 + memcpy(F_SUM(file), tmp_sum, checksum_len);
58 F_NDX(file) = dir_count;
60 diff --git a/log.c b/log.c
63 @@ -32,8 +32,10 @@ extern int local_server;
66 extern int msg_fd_out;
67 +extern int checksum_len;
68 extern int allow_8bit_chars;
69 extern int protocol_version;
70 +extern int always_checksum;
71 extern int preserve_times;
72 extern int progress_is_active;
73 extern int stdout_format_has_i;
74 @@ -55,6 +57,7 @@ extern iconv_t ic_recv;
75 extern char curr_dir[MAXPATHLEN];
76 extern char *full_module_path;
77 extern unsigned int module_dirlen;
78 +extern char sender_file_sum[MAX_DIGEST_LEN];
80 static int log_initialised;
81 static int logfile_was_closed;
82 @@ -630,6 +633,28 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
83 snprintf(buf2, sizeof buf2, fmt, (double)b);
87 + if (protocol_version >= 30
88 + && (iflags & ITEM_TRANSFER
89 + || (always_checksum && S_ISREG(file->mode)))) {
91 + const char *sum = iflags & ITEM_TRANSFER
92 + ? sender_file_sum : F_SUM(file);
93 + c = buf2 + checksum_len*2;
95 + for (i = checksum_len; --i >= 0; ) {
99 + *--c = x1 <= 9 ? x1 + '0' : x1 + 'a' - 10;
100 + *--c = x2 <= 9 ? x2 + '0' : x2 + 'a' - 10;
103 + memset(buf2, ' ', checksum_len*2);
104 + buf2[checksum_len*2] = '\0';
109 if (iflags & ITEM_DELETED) {
111 diff --git a/main.c b/main.c
114 @@ -36,6 +36,7 @@ extern int am_sender;
115 extern int am_daemon;
116 extern int inc_recurse;
117 extern int blocking_io;
118 +extern int always_checksum;
119 extern int remove_source_files;
120 extern int need_messages_from_generator;
121 extern int kluge_around_eof;
122 @@ -68,6 +69,8 @@ extern dev_t filesystem_dev;
123 extern pid_t cleanup_child_pid;
124 extern unsigned int module_dirlen;
125 extern struct stats stats;
126 +extern char *stdout_format;
127 +extern char *logfile_format;
128 extern char *filesfrom_host;
129 extern char *partial_dir;
130 extern char *dest_option;
131 @@ -87,6 +90,7 @@ int local_server = 0;
132 int daemon_over_rsh = 0;
133 mode_t orig_umask = 0;
134 int batch_gen_fd = -1;
135 +int sender_keeps_checksum = 0;
137 /* There's probably never more than at most 2 outstanding child processes,
138 * but set it higher, just in case. */
139 @@ -1016,6 +1020,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
142 keep_dirlinks = 0; /* Must be disabled on the sender. */
144 + if (always_checksum
145 + && (log_format_has(stdout_format, 'C')
146 + || log_format_has(logfile_format, 'C')))
147 + sender_keeps_checksum = 1;
149 if (protocol_version >= 30)
150 io_start_multiplex_out();
152 diff --git a/match.c b/match.c
155 @@ -25,8 +25,10 @@ extern int verbose;
156 extern int do_progress;
157 extern int checksum_seed;
158 extern int append_mode;
159 +extern int checksum_len;
161 int updating_basis_file;
162 +char sender_file_sum[MAX_DIGEST_LEN];
164 static int false_alarms;
165 static int hash_hits;
166 @@ -328,9 +330,6 @@ static void hash_search(int f,struct sum_struct *s,
168 void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
170 - char file_sum[MAX_DIGEST_LEN];
176 @@ -378,18 +377,28 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
177 matched(f, s, buf, len, -1);
180 - sum_len = sum_end(file_sum);
181 - /* If we had a read error, send a bad checksum. */
182 - if (buf && buf->status != 0)
184 + if (sum_end(sender_file_sum) != checksum_len)
185 + overflow_exit("checksum_len"); /* Impossible... */
187 + /* If we had a read error, send a bad checksum. We use all bits
188 + * off as long as the checksum doesn't happen to be that, in
189 + * which case we turn the last 0 bit into a 1. */
190 + if (buf && buf->status != 0) {
192 + for (i = 0; i < checksum_len && sender_file_sum[i] == 0; i++) {}
193 + memset(sender_file_sum, 0, checksum_len);
194 + if (i == checksum_len)
195 + sender_file_sum[i-1]++;
199 rprintf(FINFO,"sending file_sum\n");
200 - write_buf(f, file_sum, sum_len);
201 + write_buf(f, sender_file_sum, checksum_len);
205 rprintf(FINFO, "false_alarms=%d hash_hits=%d matches=%d\n",
206 false_alarms, hash_hits, matches);
209 total_hash_hits += hash_hits;
210 total_false_alarms += false_alarms;
211 diff --git a/options.c b/options.c
214 @@ -1535,7 +1535,8 @@ int parse_arguments(int *argc_p, const char ***argv_p)
215 else if (log_format_has(stdout_format, 'i'))
216 stdout_format_has_i = itemize_changes | 1;
217 if (!log_format_has(stdout_format, 'b')
218 - && !log_format_has(stdout_format, 'c'))
219 + && !log_format_has(stdout_format, 'c')
220 + && !log_format_has(stdout_format, 'C'))
221 log_before_transfer = !am_server;
222 } else if (itemize_changes) {
223 stdout_format = "%i %n%L";
224 diff --git a/receiver.c b/receiver.c
227 @@ -47,6 +47,7 @@ extern int remove_source_files;
228 extern int append_mode;
229 extern int sparse_files;
230 extern int keep_partial;
231 +extern int checksum_len;
232 extern int checksum_seed;
234 extern int delay_updates;
235 @@ -55,6 +56,7 @@ extern struct stats stats;
237 extern char *partial_dir;
238 extern char *basis_dir[MAX_BASIS_DIRS+1];
239 +extern char sender_file_sum[MAX_DIGEST_LEN];
240 extern struct file_list *cur_flist, *first_flist, *dir_flist;
241 extern struct filter_list_struct daemon_filter_list;
243 @@ -186,10 +188,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
244 const char *fname, int fd, OFF_T total_size)
246 static char file_sum1[MAX_DIGEST_LEN];
247 - static char file_sum2[MAX_DIGEST_LEN];
248 struct map_struct *mapbuf;
249 struct sum_struct sum;
250 - int32 len, sum_len;
255 @@ -322,15 +323,16 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
256 exit_cleanup(RERR_FILEIO);
259 - sum_len = sum_end(file_sum1);
260 + if (sum_end(file_sum1) != checksum_len)
261 + overflow_exit("checksum_len"); /* Impossible... */
266 - read_buf(f_in, file_sum2, sum_len);
267 + read_buf(f_in, sender_file_sum, checksum_len);
269 rprintf(FINFO,"got file_sum\n");
270 - if (fd != -1 && memcmp(file_sum1, file_sum2, sum_len) != 0)
271 + if (fd != -1 && memcmp(file_sum1, sender_file_sum, checksum_len) != 0)
275 diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
278 @@ -510,6 +510,7 @@ quote(itemization(
279 it() %b the number of bytes actually transferred
280 it() %B the permission bits of the file (e.g. rwxrwxrwt)
281 it() %c the total size of the block checksums received for the basis file (only when sending)
282 + it() %C the full-file MD5 checksum if bf(--checksum) is enabled or a file was transferred (only for protocol 30 or above).
283 it() %f the filename (long form on sender; no trailing "/")
284 it() %G the gid of the file (decimal) or "DEFAULT"
285 it() %h the remote host name