Jason M. Felice wrote:
-This patch adds the --link-by-hash=DIR option, which hard links received
-files in a link farm arranged by MD4 file hash. The result is that the system
-will only store one copy of the unique contents of each file, regardless of
-the file's name.
+This patch adds the --link-by-hash=DIR option, which hard links received files
+in a link farm arranged by MD4 or MD5 file hash. The result is that the system
+will only store one copy of the unique contents of each file, regardless of the
+file's name.
To use this patch, run these commands for a successful build:
./configure
make
+based-on: 6c8ca91c731b7bf2b081694bda85b7dadc2b7aff
diff --git a/Makefile.in b/Makefile.in
-index feacb90..b27b1e7 100644
--- a/Makefile.in
+++ b/Makefile.in
-@@ -37,7 +37,7 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
- util.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
+@@ -47,7 +47,7 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
+ util1.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
- fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
--OBJS3=progress.o pipe.o
-+OBJS3=progress.o pipe.o hashlink.o
+ usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
+-OBJS3=progress.o pipe.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@
++OBJS3=progress.o pipe.o hashlink.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@
DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
popt/popthelp.o popt/poptparse.o
-diff --git a/flist.c b/flist.c
-index 09b4fc5..570bcee 100644
---- a/flist.c
-+++ b/flist.c
-@@ -73,6 +73,7 @@ extern int sender_keeps_checksum;
- extern int unsort_ndx;
- extern struct stats stats;
- extern char *filesfrom_host;
+diff --git a/checksum.c b/checksum.c
+--- a/checksum.c
++++ b/checksum.c
+@@ -40,6 +40,8 @@ extern int whole_file;
+ extern int checksum_seed;
+ extern int protocol_version;
+ extern int proper_seed_order;
+extern char *link_by_hash_dir;
- extern char *usermap, *groupmap;
++extern char link_by_hash_extra_sum[MAX_DIGEST_LEN];
+ extern const char *checksum_choice;
- extern char curr_dir[MAXPATHLEN];
-@@ -881,7 +882,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
- extra_len += EXTRA_LEN;
- #endif
+ #define NNI_BUILTIN (1<<0)
+@@ -539,7 +541,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
+ }
-- if (always_checksum && S_ISREG(mode))
-+ if ((always_checksum || link_by_hash_dir) && S_ISREG(mode))
- extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
+ static int32 sumresidue;
+-static md_context ctx_md;
++static md_context ctx_md, ctx2_md;
+ #ifdef SUPPORT_XXHASH
+ static XXH64_state_t* xxh64_state;
+ #endif
+@@ -597,6 +599,8 @@ int sum_init(struct name_num_item *nni, int seed)
+ #endif
+ case CSUM_MD5:
+ md5_begin(&ctx_md);
++ if (link_by_hash_dir)
++ md5_begin(&ctx2_md);
+ break;
+ case CSUM_MD4:
+ mdfour_begin(&ctx_md);
+@@ -643,6 +647,8 @@ void sum_update(const char *p, int32 len)
+ #endif
+ case CSUM_MD5:
+ md5_update(&ctx_md, (uchar *)p, len);
++ if (link_by_hash_dir)
++ md5_update(&ctx2_md, (uchar *)p, len);
+ break;
+ case CSUM_MD4:
+ case CSUM_MD4_OLD:
+@@ -709,6 +715,8 @@ void sum_end(char *sum)
+ #endif
+ case CSUM_MD5:
+ md5_result(&ctx_md, (uchar *)sum);
++ if (link_by_hash_dir)
++ md5_result(&ctx2_md, (uchar *)link_by_hash_extra_sum);
+ break;
+ case CSUM_MD4:
+ case CSUM_MD4_OLD:
+diff --git a/clientserver.c b/clientserver.c
+--- a/clientserver.c
++++ b/clientserver.c
+@@ -53,6 +53,7 @@ extern int logfile_format_has_i;
+ extern int logfile_format_has_o_or_i;
+ extern char *bind_address;
+ extern char *config_file;
++extern char *link_by_hash_dir;
+ extern char *logfile_format;
+ extern char *files_from;
+ extern char *tmpdir;
+@@ -736,6 +737,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
+ return -1;
+ }
- #if SIZEOF_INT64 >= 8
++ if (*lp_link_by_hash_dir(i))
++ link_by_hash_dir = lp_link_by_hash_dir(i);
++
+ if (am_daemon > 0) {
+ rprintf(FLOG, "rsync allowed access on module %s from %s (%s)\n",
+ name, host, addr);
+diff --git a/daemon-parm.txt b/daemon-parm.txt
+--- a/daemon-parm.txt
++++ b/daemon-parm.txt
+@@ -29,6 +29,7 @@ STRING hosts_deny NULL
+ STRING include NULL
+ STRING include_from NULL
+ STRING incoming_chmod NULL
++STRING link_by_hash_dir NULL
+ STRING lock_file DEFAULT_LOCK_FILE
+ STRING log_file NULL
+ STRING log_format "%o %h [%a] %m (%u) %f %l"
diff --git a/hashlink.c b/hashlink.c
new file mode 100644
-index 0000000..15e2a73
--- /dev/null
+++ b/hashlink.c
-@@ -0,0 +1,339 @@
+@@ -0,0 +1,92 @@
+/*
+ Copyright (C) Cronosys, LLC 2004
+
+/* This file contains code used by the --link-by-hash option. */
+
+#include "rsync.h"
++#include "inums.h"
+
++extern int protocol_version;
+extern char *link_by_hash_dir;
++extern char sender_file_sum[MAX_DIGEST_LEN];
+
-+#ifdef HAVE_LINK
-+
-+char *make_hash_name(struct file_struct *file)
-+{
-+ char hash[33], *dst;
-+ uchar c, *src = (uchar*)F_SUM(file);
-+ int i;
-+
-+ for (dst = hash, i = 0; i < 4; i++, src++) {
-+ c = *src >> 4;
-+ *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0');
-+ c = *src & 0x0f;
-+ *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0');
-+ }
-+ *dst++ = '/';
-+ for (i = 0; i < 12; i++, src++) {
-+ c = *src >> 4;
-+ *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0');
-+ c = *src & 0x0f;
-+ *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0');
-+ }
-+ *dst = 0;
-+
-+ if (asprintf(&dst,"%s/%s",link_by_hash_dir,hash) < 0)
-+ out_of_memory("make_hash_name");
-+ return dst;
-+}
-+
-+
-+void kill_hashfile(struct hashfile_struct *hashfile)
-+{
-+ if (!hashfile)
-+ return;
-+ free(hashfile->name);
-+ close(hashfile->fd);
-+ free(hashfile);
-+}
-+
-+
-+void kill_hashfiles(struct hashfile_struct *hashfiles)
-+{
-+ struct hashfile_struct *iter, *next;
-+ if ((iter = hashfiles) != NULL) {
-+ do {
-+ next = iter->next;
-+ kill_hashfile(iter);
-+ iter = next;
-+ } while (iter != hashfiles);
-+ }
-+}
++char link_by_hash_extra_sum[MAX_DIGEST_LEN]; /* Only used when md4 sums are in the transfer */
+
++#ifdef HAVE_LINK
+
-+struct hashfile_struct *find_hashfiles(char *hashname, int64 size, long *fnbr)
++/* This function is always called after a file is received, so the
++ * sender_file_sum buffer has whatever the last checksum was for the
++ * transferred file. */
++void link_by_hash(const char *fname, const char *fnametmp, struct file_struct *file)
+{
-+ DIR *d;
-+ struct dirent *di;
-+ struct hashfile_struct *hashfiles = NULL, *hashfile;
+ STRUCT_STAT st;
-+ long this_fnbr;
++ char *hashname, *last_slash, *num_str;
++ const char *hex;
++ int num = 0;
+
-+ *fnbr = 0;
-+
-+ /* Build a list of potential candidates and open
-+ * them. */
-+ if ((d = opendir(hashname)) == NULL) {
-+ rsyserr(FERROR, errno, "opendir failed: \"%s\"", hashname);
-+ free(hashname);
-+ return NULL;
-+ }
-+ while ((di = readdir(d)) != NULL) {
-+ if (!strcmp(di->d_name,".") || !strcmp(di->d_name,"..")) {
-+ continue;
-+ }
-+
-+ /* We need to have the largest fnbr in case we need to store
-+ * a new file. */
-+ this_fnbr = atol(di->d_name);
-+ if (this_fnbr > *fnbr)
-+ *fnbr = this_fnbr;
-+
-+ hashfile = new_array(struct hashfile_struct, 1);
-+ if (asprintf(&hashfile->name,"%s/%s",hashname, di->d_name) < 0)
-+ out_of_memory("find_hashfiles");
-+ if (do_stat(hashfile->name,&st) == -1) {
-+ rsyserr(FERROR, errno, "stat failed: %s", hashfile->name);
-+ kill_hashfile(hashfile);
-+ continue;
-+ }
-+ if (st.st_size != size) {
-+ kill_hashfile(hashfile);
-+ continue;
-+ }
-+ hashfile->nlink = st.st_nlink;
-+ hashfile->fd = open(hashfile->name,O_RDONLY|O_BINARY);
-+ if (hashfile->fd == -1) {
-+ rsyserr(FERROR, errno, "open failed: %s", hashfile->name);
-+ kill_hashfile(hashfile);
-+ continue;
-+ }
-+ if (hashfiles == NULL)
-+ hashfiles = hashfile->next = hashfile->prev = hashfile;
-+ else {
-+ hashfile->next = hashfiles;
-+ hashfile->prev = hashfiles->prev;
-+ hashfile->next->prev = hashfile;
-+ hashfile->prev->next = hashfile;
-+ }
-+ }
-+ closedir(d);
-+
-+ return hashfiles;
-+}
-+
-+
-+struct hashfile_struct *compare_hashfiles(int fd,struct hashfile_struct *files)
-+{
-+ int amt, hamt;
-+ char buffer[BUFSIZ], cmpbuffer[BUFSIZ];
-+ struct hashfile_struct *iter, *next, *best;
-+ uint32 nlink;
-+
-+ if (!files)
-+ return NULL;
-+
-+ iter = files; /* in case files are 0 bytes */
-+ while ((amt = read(fd, buffer, BUFSIZ)) > 0) {
-+ iter = files;
-+ do {
-+ /* Icky bit to resync when we steal the first node. */
-+ if (!files)
-+ files = iter;
-+
-+ next = iter->next;
-+
-+ hamt = read(iter->fd, cmpbuffer, BUFSIZ);
-+ if (amt != hamt || memcmp(buffer, cmpbuffer, amt)) {
-+ if (iter == files) {
-+ files = files->prev;
-+ }
-+ if (iter->next == iter) {
-+ files = next = NULL;
-+ } else {
-+ next = iter->next;
-+ if (iter == files) {
-+ /* So we know to resync */
-+ files = NULL;
-+ }
-+ }
-+ iter->next->prev = iter->prev;
-+ iter->prev->next = iter->next;
-+ kill_hashfile(iter);
-+ }
-+
-+ iter = next;
-+ } while (iter != files);
-+
-+ if (iter == NULL && files == NULL) {
-+ /* There are no matches. */
-+ return NULL;
-+ }
-+ }
-+
-+ if (amt == -1) {
-+ rsyserr(FERROR, errno, "read failed in compare_hashfiles()");
-+ kill_hashfiles(files);
-+ return NULL;
-+ }
++ /* We don't bother to hard-link 0-length files. */
++ if (F_LENGTH(file) == 0)
++ return;
+
-+ /* If we only have one file left, use it. */
-+ if (files == files->next) {
-+ return files;
++ hex = sum_as_hex(5, protocol_version >= 30 ? sender_file_sum : link_by_hash_extra_sum, 0);
++ if (asprintf(&hashname, "%s/%.3s/%.3s/%.3s/%s.%s.000000",
++ link_by_hash_dir, hex, hex+3, hex+6, hex+9, big_num(F_LENGTH(file))) < 0)
++ {
++ out_of_memory("make_hash_name");
+ }
+
-+ /* All files which remain in the list are identical and should have
-+ * the same size. We pick the one with the lowest link count (we
-+ * may have rolled over because we hit the maximum link count for
-+ * the filesystem). */
-+ best = iter = files;
-+ nlink = iter->nlink;
-+ do {
-+ if (iter->nlink < nlink) {
-+ nlink = iter->nlink;
-+ best = iter;
-+ }
-+ iter = iter->next;
-+ } while (iter != files);
-+
-+ best->next->prev = best->prev;
-+ best->prev->next = best->next;
-+ if (files == best)
-+ files = files->next;
-+ kill_hashfiles(files);
-+ return best;
-+}
-+
-+
-+int link_by_hash(const char *fnametmp, const char *fname, struct file_struct *file)
-+{
-+ STRUCT_STAT st;
-+ char *hashname = make_hash_name(file);
-+ int first = 0, rc;
-+ char *linkname;
-+ long last_fnbr;
++ last_slash = strrchr(hashname, '/');
++ num_str = strrchr(last_slash, '.') + 1;
+
-+ if (F_LENGTH(file) == 0)
-+ return robust_rename(fnametmp, fname, NULL, 0644);
-+
-+ if (do_stat(hashname, &st) == -1) {
-+ char *dirname;
-+
-+ /* Directory does not exist. */
-+ dirname = strdup(hashname);
-+ *strrchr(dirname,'/') = 0;
-+ if (do_mkdir(dirname, 0755) == -1 && errno != EEXIST) {
-+ rsyserr(FERROR, errno, "mkdir failed: %s", dirname);
-+ free(hashname);
-+ free(dirname);
-+ return robust_rename(fnametmp, fname, NULL, 0644);
++ while (1) {
++ if (num >= 999999) { /* Surely we'll never reach this... */
++ if (DEBUG_GTE(HASHLINK, 1))
++ rprintf(FINFO, "link-by-hash: giving up after \"%s\".\n", hashname);
++ goto cleanup;
+ }
-+ free(dirname);
++ if (num > 0 && DEBUG_GTE(HASHLINK, 1))
++ rprintf(FINFO, "link-by-hash: max link count exceeded, starting new file \"%s\".\n", hashname);
+
-+ if (do_mkdir(hashname, 0755) == -1 && errno != EEXIST) {
-+ rsyserr(FERROR, errno, "mkdir failed: %s", hashname);
-+ free(hashname);
-+ return robust_rename(fnametmp, fname, NULL, 0644);
-+ }
-+
-+ first = 1;
-+ if (asprintf(&linkname,"%s/0",hashname) < 0)
-+ out_of_memory("link_by_hash");
-+ rprintf(FINFO, "(1) linkname = %s\n", linkname);
-+ } else {
-+ struct hashfile_struct *hashfiles, *hashfile;
-+
-+ if (do_stat(fnametmp,&st) == -1) {
-+ rsyserr(FERROR, errno, "stat failed: %s", fname);
-+ return -1;
-+ }
-+ hashfiles = find_hashfiles(hashname, st.st_size, &last_fnbr);
++ snprintf(num_str, 7, "%d", num++);
++ if (do_stat(hashname, &st) < 0)
++ break;
+
-+ if (hashfiles == NULL) {
-+ first = 1;
-+ if (asprintf(&linkname,"%s/0",hashname) < 0)
-+ out_of_memory("link_by_hash");
-+ rprintf(FINFO, "(2) linkname = %s\n", linkname);
++ if (do_link(hashname, fnametmp) < 0) {
++ if (errno == EMLINK)
++ continue;
++ rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"", hashname, full_fname(fname));
+ } else {
-+ int fd;
-+ /* Search for one identical to us. */
-+ if ((fd = open(fnametmp,O_RDONLY|O_BINARY)) == -1) {
-+ rsyserr(FERROR, errno, "open failed: %s", fnametmp);
-+ kill_hashfiles(hashfiles);
-+ return -1;
-+ }
-+ hashfile = compare_hashfiles(fd, hashfiles);
-+ hashfiles = NULL;
-+ close(fd);
-+
-+ if (hashfile) {
-+ first = 0;
-+ linkname = strdup(hashfile->name);
-+ rprintf(FINFO, "(3) linkname = %s\n", linkname);
-+ kill_hashfile(hashfile);
-+ } else {
-+ first = 1;
-+ if (asprintf(&linkname, "%s/%ld", hashname, last_fnbr + 1) < 0)
-+ out_of_memory("link_by_hash");
-+ rprintf(FINFO, "(4) linkname = %s\n", linkname);
-+ }
++ if (DEBUG_GTE(HASHLINK, 2))
++ rprintf(FINFO, "link-by-hash (existing): \"%s\" -> %s\n", hashname, full_fname(fname));
++ robust_rename(fnametmp, fname, NULL, 0644);
+ }
-+ }
+
-+ if (!first) {
-+ rprintf(FINFO, "link-by-hash (existing): \"%s\" -> %s\n",
-+ linkname, full_fname(fname));
-+ robust_unlink(fname);
-+ rc = do_link(linkname, fname);
-+ if (rc == -1) {
-+ if (errno == EMLINK) {
-+ first = 1;
-+ free(linkname);
-+ if (asprintf(&linkname,"%s/%ld",hashname, last_fnbr + 1) < 0)
-+ out_of_memory("link_by_hash");
-+ rprintf(FINFO, "(5) linkname = %s\n", linkname);
-+ rprintf(FINFO,"link-by-hash: max link count exceeded, starting new file \"%s\".\n", linkname);
-+ } else {
-+ rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"",
-+ linkname, full_fname(fname));
-+ rc = robust_rename(fnametmp, fname, NULL, 0644);
-+ }
-+ } else {
-+ do_unlink(fnametmp);
-+ }
++ goto cleanup;
+ }
+
-+ if (first) {
-+ rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n",
-+ full_fname(fname),linkname);
++ if (DEBUG_GTE(HASHLINK, 2))
++ rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n", full_fname(fname), hashname);
+
-+ rc = robust_rename(fnametmp, fname, NULL, 0644);
-+ if (rc != 0) {
-+ rsyserr(FERROR, errno, "rename \"%s\" -> \"%s\"",
-+ full_fname(fnametmp), full_fname(fname));
-+ }
-+ rc = do_link(fname,linkname);
-+ if (rc != 0) {
-+ rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"",
-+ full_fname(fname), linkname);
-+ }
-+ }
++ if (do_link(fname, hashname) < 0
++ && (errno != ENOENT || make_path(hashname, MKP_DROP_NAME) < 0 || do_link(fname, hashname) < 0))
++ rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"", full_fname(fname), hashname);
+
-+ free(linkname);
++ cleanup:
+ free(hashname);
-+ return rc;
+}
+#endif
diff --git a/options.c b/options.c
-index e7c6c61..73b1aa4 100644
--- a/options.c
+++ b/options.c
-@@ -158,6 +158,7 @@ char *backup_suffix = NULL;
+@@ -173,6 +173,7 @@ char *backup_suffix = NULL;
char *tmpdir = NULL;
char *partial_dir = NULL;
char *basis_dir[MAX_BASIS_DIRS+1];
char *config_file = NULL;
char *shell_cmd = NULL;
char *logfile_name = NULL;
-@@ -745,6 +746,7 @@ void usage(enum logcode F)
- rprintf(F," --compare-dest=DIR also compare destination files relative to DIR\n");
- rprintf(F," --copy-dest=DIR ... and include copies of unchanged files\n");
- rprintf(F," --link-dest=DIR hardlink to files in DIR when unchanged\n");
-+ rprintf(F," --link-by-hash=DIR create hardlinks by hash into DIR\n");
- rprintf(F," -z, --compress compress file data during the transfer\n");
- rprintf(F," --compress-level=NUM explicitly set compression level\n");
- rprintf(F," --skip-compress=LIST skip compressing files with a suffix in LIST\n");
-@@ -798,7 +800,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
+@@ -231,7 +232,7 @@ static const char *debug_verbosity[] = {
+ /*2*/ "BIND,CMD,CONNECT,DEL,DELTASUM,DUP,FILTER,FLIST,ICONV",
+ /*3*/ "ACL,BACKUP,CONNECT2,DELTASUM2,DEL2,EXIT,FILTER2,FLIST2,FUZZY,GENR,OWN,RECV,SEND,TIME",
+ /*4*/ "CMD2,DELTASUM3,DEL3,EXIT2,FLIST3,ICONV2,OWN2,PROTO,TIME2",
+- /*5*/ "CHDIR,DELTASUM4,FLIST4,FUZZY2,HASH,HLINK",
++ /*5*/ "CHDIR,DELTASUM4,FLIST4,FUZZY2,HASH,HASHLINK,HLINK",
+ };
+
+ #define MAX_VERBOSITY ((int)(sizeof debug_verbosity / sizeof debug_verbosity[0]) - 1)
+@@ -302,6 +303,7 @@ static struct output_struct debug_words[COUNT_DEBUG+1] = {
+ DEBUG_WORD(FUZZY, W_REC, "Debug fuzzy scoring (levels 1-2)"),
+ DEBUG_WORD(GENR, W_REC, "Debug generator functions"),
+ DEBUG_WORD(HASH, W_SND|W_REC, "Debug hashtable code"),
++ DEBUG_WORD(HASHLINK, W_REC, "Debug hashlink code (levels 1-2)"),
+ DEBUG_WORD(HLINK, W_SND|W_REC, "Debug hard-link actions (levels 1-3)"),
+ DEBUG_WORD(ICONV, W_CLI|W_SRV, "Debug iconv character conversions (levels 1-2)"),
+ DEBUG_WORD(IO, W_CLI|W_SRV, "Debug I/O routines (levels 1-4)"),
+@@ -582,7 +584,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
- OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG,
-- OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN,
-+ OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_LINK_BY_HASH,
- OPT_SERVER, OPT_REFUSED_BASE = 9000};
-
- static struct poptOption long_options[] = {
-@@ -937,6 +939,7 @@ static struct poptOption long_options[] = {
+ OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE,
+- OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR,
++ OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, OPT_LINK_BY_HASH,
+ OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS,
+ OPT_STOP_AFTER, OPT_STOP_AT,
+ OPT_REFUSED_BASE = 9000};
+@@ -743,6 +745,7 @@ static struct poptOption long_options[] = {
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
{"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 },
+ {"link-by-hash", 0, POPT_ARG_STRING, 0, OPT_LINK_BY_HASH, 0, 0},
- {"fuzzy", 'y', POPT_ARG_VAL, &fuzzy_basis, 1, 0, 0 },
+ {"fuzzy", 'y', POPT_ARG_NONE, 0, 'y', 0, 0 },
{"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 },
{"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 },
-@@ -1742,6 +1745,21 @@ int parse_arguments(int *argc_p, const char ***argv_p)
- return 0;
+@@ -990,6 +993,9 @@ static void set_refuse_options(void)
+ ref = cp + 1;
+ }
+
++ if (*lp_link_by_hash_dir(module_id))
++ parse_one_refuse_match(0, "link-by-hash", list_end);
++
+ if (am_daemon) {
+ #ifdef ICONV_OPTION
+ if (!*lp_charset(module_id))
+@@ -1867,6 +1873,20 @@ int parse_arguments(int *argc_p, const char ***argv_p)
+ goto cleanup;
#endif
+ case OPT_LINK_BY_HASH:
+ snprintf(err_buf, sizeof err_buf,
+ "hard links are not supported on this %s\n",
+ am_server ? "server" : "client");
-+ rprintf(FERROR, "ERROR: %s", err_buf);
+ return 0;
+#endif
+
- default:
- /* A large opt value means that set_refuse_options()
- * turned this option off. */
-@@ -2584,6 +2602,11 @@ void server_options(char **args, int *argc_p)
- } else if (inplace)
- args[ac++] = "--inplace";
+ case OPT_STOP_AFTER: {
+ long val;
+ arg = poptGetOptArg(pc);
+@@ -2252,6 +2272,8 @@ int parse_arguments(int *argc_p, const char ***argv_p)
+ tmpdir = sanitize_path(NULL, tmpdir, NULL, 0, SP_DEFAULT);
+ if (backup_dir)
+ backup_dir = sanitize_path(NULL, backup_dir, NULL, 0, SP_DEFAULT);
++ if (link_by_hash_dir)
++ link_by_hash_dir = sanitize_path(NULL, link_by_hash_dir, NULL, 0, SP_DEFAULT);
+ }
+ if (daemon_filter_list.head && !am_sender) {
+ filter_rule_list *elp = &daemon_filter_list;
+@@ -2941,6 +2963,12 @@ void server_options(char **args, int *argc_p)
+ args[ac++] = "--no-W";
+ }
+ if (link_by_hash_dir && am_sender) {
+ args[ac++] = "--link-by-hash";
+ args[ac++] = link_by_hash_dir;
++ link_by_hash_dir = NULL; /* optimize sending-side checksums */
+ }
+
if (files_from && (!am_sender || filesfrom_host)) {
if (filesfrom_host) {
args[ac++] = "--files-from";
-diff --git a/receiver.c b/receiver.c
-index 4325e30..2709d5e 100644
---- a/receiver.c
-+++ b/receiver.c
-@@ -164,11 +164,13 @@ int open_tmpfile(char *fnametmp, const char *fname, struct file_struct *file)
- }
-
- static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
-- const char *fname, int fd, OFF_T total_size)
-+ const char *fname, int fd, OFF_T total_size,
-+ const char *md4)
- {
- static char file_sum1[MAX_DIGEST_LEN];
- struct map_struct *mapbuf;
- struct sum_struct sum;
-+ md_context mdfour_data;
- int32 len;
- OFF_T offset = 0;
- OFF_T offset2;
-@@ -188,6 +190,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
- } else
- mapbuf = NULL;
+diff --git a/rsync.1.md b/rsync.1.md
+--- a/rsync.1.md
++++ b/rsync.1.md
+@@ -510,6 +510,7 @@ has its own detailed description later in this manpage.
+ --compare-dest=DIR also compare destination files relative to DIR
+ --copy-dest=DIR ... and include copies of unchanged files
+ --link-dest=DIR hardlink to files in DIR when unchanged
++--link-by-hash=DIR create hardlinks by hash into DIR
+ --compress, -z compress file data during the transfer
+ --compress-choice=STR choose the compression algorithm (aka --zc)
+ --compress-level=NUM explicitly set compression level (aka --zl)
+@@ -2720,6 +2721,50 @@ expand it.
+ this bug by avoiding the `-o` option (or using `--no-o`) when sending to an
+ old rsync.
-+ if (md4)
-+ mdfour_begin(&mdfour_data);
++0. `--link-by-hash=DIR`
+
- sum_init(checksum_seed);
-
- if (append_mode > 0) {
-@@ -232,6 +237,8 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
- cleanup_got_literal = 1;
-
- sum_update(data, i);
-+ if (md4)
-+ mdfour_update(&mdfour_data, (uchar*)data, i);
-
- if (fd != -1 && write_file(fd,data,i) != i)
- goto report_write_error;
-@@ -258,6 +265,8 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
-
- see_token(map, len);
- sum_update(map, len);
-+ if (md4)
-+ mdfour_update(&mdfour_data, (uchar*)map, len);
- }
-
- if (updating_basis_or_equiv) {
-@@ -305,6 +314,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
- if (sum_end(file_sum1) != checksum_len)
- overflow_exit("checksum_len"); /* Impossible... */
-
-+ if (md4)
-+ mdfour_result(&mdfour_data, (uchar*)md4);
++ This option hard links the destination files into _DIR_, a link farm
++ arranged by MD5 file hash. The result is that the system will only store
++ (usually) one copy of the unique contents of each file, regardless of the
++ file's name (it will use extra files if the links overflow the available
++ maximum).
+
- if (mapbuf)
- unmap_file(mapbuf);
-
-@@ -319,7 +331,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
-
- static void discard_receive_data(int f_in, OFF_T length)
- {
-- receive_data(f_in, NULL, -1, 0, NULL, -1, length);
-+ receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL);
- }
-
- static void handle_delayed_updates(char *local_name)
-@@ -744,7 +756,7 @@ int recv_files(int f_in, char *local_name)
-
- /* recv file data */
- recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size,
-- fname, fd2, F_LENGTH(file));
-+ fname, fd2, F_LENGTH(file), F_SUM(file));
-
- log_item(log_code, file, &initial_stats, iflags, NULL);
++ This patch does not take into account file permissions, extended
++ attributes, or ACLs when linking things together, so you should only use
++ this if you don't care about preserving those extra file attributes (or if
++ they are always the same for identical files).
++
++ The _DIR_ is relative to the destination directory, so either specify a full
++ path to the hash hierarchy, or specify a relative path that puts the links
++ outside the destination (e.g. "../links").
++
++ Keep in mind that the hierarchy is never pruned, so if you need to reclaim
++ space, you should remove any files that have just one link (since they are
++ not linked into any destination dirs anymore):
++
++ > find $DIR -links 1 -delete
++
++ The link farm's directory hierarchy is determined by the file's (32-char)
++ MD5 hash and the file-length. The hash is split up into directory shards.
++ For example, if a file is 54321 bytes long, it could be stored like this:
++
++ > $DIR/123/456/789/01234567890123456789012.54321.0
++
++ Note that the directory layout in this patch was modified for version
++ 3.1.0, so anyone using an older version of this patch should move their
++ existing link hierarchy out of the way and then use the newer rsync to copy
++ the saved hierarchy into its new layout. Assuming that no files have
++ overflowed their link limits, this would work:
++
++ > mv $DIR $DIR.old
++ > rsync -aiv --link-by-hash=$DIR $DIR.old/ $DIR.tmp/
++ > rm -rf $DIR.tmp
++ > rm -rf $DIR.old
++
++ If some of your files are at their link limit, you'd be better of using a
++ script to calculate the md5 sum of each file in the hierarchy and move it
++ to its new location.
++
+ 0. `--compress`, `-z`
+ With this option, rsync compresses the file data as it is sent to the
diff --git a/rsync.c b/rsync.c
-index 2c026a2..87f6d54 100644
--- a/rsync.c
+++ b/rsync.c
-@@ -48,6 +48,7 @@ extern int flist_eof;
- extern int msgs2stderr;
+@@ -52,6 +52,7 @@ extern int flist_eof;
+ extern int file_old_total;
extern int keep_dirlinks;
extern int make_backups;
+extern char *link_by_hash_dir;
+ extern int sanitize_paths;
extern struct file_list *cur_flist, *first_flist, *dir_flist;
extern struct chmod_mode_struct *daemon_chmod_modes;
- #ifdef ICONV_OPTION
-@@ -575,8 +576,15 @@ int finish_transfer(const char *fname, const char *fnametmp,
- /* move tmp file over real file */
- if (DEBUG_GTE(RECV, 1))
- rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname);
-- ret = robust_rename(fnametmp, fname, temp_copy_name,
-- file->mode & INITACCESSPERMS);
+@@ -760,6 +761,10 @@ int finish_transfer(const char *fname, const char *fnametmp,
+ }
+ if (ret == 0) {
+ /* The file was moved into place (not copied), so it's done. */
+#ifdef HAVE_LINK
-+ if (link_by_hash_dir)
-+ ret = link_by_hash(fnametmp, fname, file);
-+ else
++ if (link_by_hash_dir)
++ link_by_hash(fname, fnametmp, file);
+#endif
-+ {
-+ ret = robust_rename(fnametmp, fname, temp_copy_name,
-+ file->mode & INITACCESSPERMS);
-+ }
- if (ret < 0) {
- rsyserr(FERROR_XFER, errno, "%s %s -> \"%s\"",
- ret == -2 ? "copy" : "rename",
+ return 1;
+ }
+ /* The file was copied, so tweak the perms of the copied file. If it
diff --git a/rsync.h b/rsync.h
-index be7cf8a..d4e2aca 100644
--- a/rsync.h
+++ b/rsync.h
-@@ -853,6 +853,14 @@ struct stats {
- int xferred_files;
- };
+@@ -1446,7 +1446,8 @@ extern short info_levels[], debug_levels[];
+ #define DEBUG_FUZZY (DEBUG_FLIST+1)
+ #define DEBUG_GENR (DEBUG_FUZZY+1)
+ #define DEBUG_HASH (DEBUG_GENR+1)
+-#define DEBUG_HLINK (DEBUG_HASH+1)
++#define DEBUG_HASHLINK (DEBUG_HASH+1)
++#define DEBUG_HLINK (DEBUG_HASHLINK+1)
+ #define DEBUG_ICONV (DEBUG_HLINK+1)
+ #define DEBUG_IO (DEBUG_ICONV+1)
+ #define DEBUG_NSTR (DEBUG_IO+1)
+diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md
+--- a/rsyncd.conf.5.md
++++ b/rsyncd.conf.5.md
+@@ -388,6 +388,23 @@ in the values of parameters. See that section for details.
+ is 0, which means no limit. A negative value disables the module. See
+ also the "[lock file](#)" parameter.
-+struct hashfile_struct {
-+ struct hashfile_struct *next;
-+ struct hashfile_struct *prev;
-+ char *name;
-+ int fd;
-+ uint32 nlink;
-+};
-+
- struct chmod_mode_struct;
++0. `link by hash dir`
++
++ When the "link by hash dir" parameter is set to a non-empty string,
++ received files will be hard linked into **DIR**, a link farm arranged by
++ MD5 file hash. See the `--link-by-hash` option for a full explanation.
++
++ The **DIR** must be accessible inside any chroot restrictions for the
++ module, but can exist outside the transfer location if there is an
++ inside-the-chroot path to the module (see "use chroot"). Note that a
++ user-specified option does not allow this outside-the-transfer-area
++ placement.
++
++ If this parameter is set, it will disable the `--link-by-hash` command-line
++ option for copies into the module.
++
++The default is for this parameter to be unset.
++
+ 0. `log file`
- struct flist_ndx_item {
-diff --git a/rsync.yo b/rsync.yo
-index 941f7a5..568b481 100644
---- a/rsync.yo
-+++ b/rsync.yo
-@@ -400,6 +400,7 @@ to the detailed description below for a complete description. verb(
- --compare-dest=DIR also compare received files relative to DIR
- --copy-dest=DIR ... and include copies of unchanged files
- --link-dest=DIR hardlink to files in DIR when unchanged
-+ --link-by-hash=DIR create hardlinks by hash into DIR
- -z, --compress compress file data during the transfer
- --compress-level=NUM explicitly set compression level
- --skip-compress=LIST skip compressing files with suffix in LIST
+ When the "log file" parameter is set to a non-empty string, the rsync
git.samba.org / rsync-patches.git / blobdiff