Reduce memory usage (#235)

In 2004, an allocation optimization has been added to the file
list handling code, that preallocates 32k of file_struct pointers
in a file_list. This optimization predates the incremental
recursion feature, for which it is not appropriate anymore. When
copying a tree containing a large number of small directories,
using the incremental recursion, rsync allocates many short
file_lists. Suddenly, the unused file_struct pointers can easily
take 90-95% of the memory allocated by rsync.

diff --git a/flist.c b/flist.c
index 3442d868f63ff239d7a5bb6921c9a160142f1df2..803490bda1a3e17ae071e355d1d31b0c38185848 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -295,6 +295,8 @@ static void flist_expand(struct file_list *flist, int extra)
                flist->malloced = FLIST_START;
        else if (flist->malloced >= FLIST_LINEAR)
                flist->malloced += FLIST_LINEAR;
+       else if (flist->malloced < FLIST_START_LARGE/16)
+               flist->malloced *= 4;
        else
                flist->malloced *= 2;
 
@@ -305,7 +307,7 @@ static void flist_expand(struct file_list *flist, int extra)
 
        new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced);
 
-       if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) {
+       if (DEBUG_GTE(FLIST, 1) && flist->files) {
                rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n",
                    who_am_i(),
                    big_num(sizeof flist->files[0] * flist->malloced),
@@ -2186,8 +2188,10 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 #endif
 
        flist = cur_flist = flist_new(0, "send_file_list");
+       flist_expand(flist, FLIST_START_LARGE);
        if (inc_recurse) {
                dir_flist = flist_new(FLIST_TEMP, "send_file_list");
+               flist_expand(dir_flist, FLIST_START_LARGE);
                flags |= FLAG_DIVERT_DIRS;
        } else
                dir_flist = cur_flist;
@@ -2541,10 +2545,13 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 #endif
 
        flist = flist_new(0, "recv_file_list");
+       flist_expand(flist, FLIST_START_LARGE);
 
        if (inc_recurse) {
-               if (flist->ndx_start == 1)
+               if (flist->ndx_start == 1) {
                        dir_flist = flist_new(FLIST_TEMP, "recv_file_list");
+                       flist_expand(dir_flist, FLIST_START_LARGE);
+               }
                dstart = dir_flist->used;
        } else {
                dir_flist = flist;

diff --git a/rsync.h b/rsync.h
index 883197324388614cf528264629eae59ff5350db7..f8fcbffbda53f29f29b8216b7cf3df0d1ed3334b 100644 (file)
--- a/rsync.h
+++ b/rsync.h
@@ -918,8 +918,9 @@ extern int xattrs_ndx;
  * Start the flist array at FLIST_START entries and grow it
  * by doubling until FLIST_LINEAR then grow by FLIST_LINEAR
  */
-#define FLIST_START    (32 * 1024)
-#define FLIST_LINEAR   (FLIST_START * 512)
+#define FLIST_START            (32)
+#define FLIST_START_LARGE      (32 * 1024)
+#define FLIST_LINEAR           (FLIST_START_LARGE * 512)
 
 /*
  * Extent size for allocation pools: A minimum size of 128KB