checksum-reading.diff

Optimize the --checksum option using externally created .rsyncsums files.

This adds a new option, --sumfiles=MODE, that allows you to use a cache of
checksums when performing a --checksum transfer.  These checksum files
(.rsyncsums) must be created by some other process -- see the perl script,
rsyncsums, in the support dir for one way.

This option can be particularly helpful to a public mirror that wants to
pre-compute their .rsyncsums files, set the "checksum files = strict" option
in their daemon config file, and thus make it quite efficient for a client
rsync to make use of the --checksum option on their server.

To use this patch, run these commands for a successful build:

    patch -p1 <patches/checksum-reading.diff
    ./configure                               (optional if already run)
    make

based-on: 6c8ca91c731b7bf2b081694bda85b7dadc2b7aff
diff --git a/clientserver.c b/clientserver.c
--- a/clientserver.c
+++ b/clientserver.c
@@ -44,6 +44,8 @@ extern int numeric_ids;
 extern int filesfrom_fd;
 extern int remote_protocol;
 extern int protocol_version;
+extern int always_checksum;
+extern int checksum_files;
 extern int io_timeout;
 extern int no_detach;
 extern int write_batch;
@@ -1106,6 +1108,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	} else if (am_root < 0) /* Treat --fake-super from client as --super. */
 		am_root = 2;
 
+	checksum_files = always_checksum ? lp_checksum_files(i)
+					 : CSF_IGNORE_FILES;
+
 	if (filesfrom_fd == 0)
 		filesfrom_fd = f_in;
 
diff --git a/daemon-parm.txt b/daemon-parm.txt
--- a/daemon-parm.txt
+++ b/daemon-parm.txt
@@ -49,6 +49,7 @@ INTEGER	max_connections		0
 INTEGER	max_verbosity		1
 INTEGER	timeout			0
 
+ENUM	checksum_files		CSF_IGNORE_FILES
 ENUM	syslog_facility		LOG_DAEMON
 
 BOOL	fake_super		False
diff --git a/flist.c b/flist.c
--- a/flist.c
+++ b/flist.c
@@ -22,6 +22,7 @@
 
 #include "rsync.h"
 #include "ifuncs.h"
+#include "itypes.h"
 #include "rounding.h"
 #include "inums.h"
 #include "io.h"
@@ -33,6 +34,7 @@ extern int am_sender;
 extern int am_generator;
 extern int inc_recurse;
 extern int always_checksum;
+extern int basis_dir_cnt;
 extern int module_id;
 extern int ignore_errors;
 extern int numeric_ids;
@@ -62,6 +64,7 @@ extern int implied_dirs;
 extern int ignore_perishable;
 extern int non_perishable_cnt;
 extern int prune_empty_dirs;
+extern int checksum_files;
 extern int copy_links;
 extern int copy_unsafe_links;
 extern int protocol_version;
@@ -74,6 +77,7 @@ extern int output_needs_newline;
 extern int sender_keeps_checksum;
 extern int trust_sender_filter;
 extern int unsort_ndx;
+extern char *basis_dir[];
 extern uid_t our_uid;
 extern struct stats stats;
 extern char *filesfrom_host;
@@ -92,6 +96,20 @@ extern int filesfrom_convert;
 extern iconv_t ic_send, ic_recv;
 #endif
 
+#ifdef HAVE_UTIMENSAT
+#ifdef HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC
+#define ST_MTIME_NSEC st_mtim.tv_nsec
+#elif defined(HAVE_STRUCT_STAT_ST_MTIMENSEC)
+#define ST_MTIME_NSEC st_mtimensec
+#endif
+#endif
+
+#define RSYNCSUMS_FILE ".rsyncsums"
+#define RSYNCSUMS_LEN (sizeof RSYNCSUMS_FILE-1)
+
+#define CLEAN_STRIP_ROOT (1<<0)
+#define CLEAN_KEEP_LAST (1<<1)
+
 #define PTR_SIZE (sizeof (struct file_struct *))
 
 int io_error;
@@ -136,8 +154,12 @@ static char empty_sum[MAX_DIGEST_LEN];
 static int flist_count_offset; /* for --delete --progress */
 static int show_filelist_progress;
 
+static struct csum_cache {
+	struct file_list *flist;
+} *csum_cache = NULL;
+
 static struct file_list *flist_new(int flags, const char *msg);
-static void flist_sort_and_clean(struct file_list *flist, int strip_root);
+static void flist_sort_and_clean(struct file_list *flist, int flags);
 static void output_flist(struct file_list *flist);
 
 void init_flist(void)
@@ -329,6 +351,235 @@ static void flist_done_allocating(struct file_list *flist)
 		flist->pool_boundary = ptr;
 }
 
+void reset_checksum_cache()
+{
+	int slot, slots = am_sender ? 1 : basis_dir_cnt + 1;
+
+	if (!csum_cache)
+		csum_cache = new_array0(struct csum_cache, slots);
+
+	for (slot = 0; slot < slots; slot++) {
+		struct file_list *flist = csum_cache[slot].flist;
+
+		if (flist) {
+			/* Reset the pool memory and empty the file-list array. */
+			pool_free_old(flist->file_pool,
+				      pool_boundary(flist->file_pool, 0));
+			flist->used = 0;
+		} else
+			flist = csum_cache[slot].flist = flist_new(FLIST_TEMP, "reset_checksum_cache");
+
+		flist->low = 0;
+		flist->high = -1;
+		flist->next = NULL;
+	}
+}
+
+/* The basename_len count is the length of the basename + 1 for the '\0'. */
+static int add_checksum(struct file_list *flist, const char *dirname,
+			const char *basename, int basename_len, OFF_T file_length,
+			time_t mtime, uint32 ctime, uint32 inode,
+			const char *sum)
+{
+	struct file_struct *file;
+	int alloc_len, extra_len;
+	char *bp;
+
+	if (basename_len == RSYNCSUMS_LEN+1 && *basename == '.'
+	 && strcmp(basename, RSYNCSUMS_FILE) == 0)
+		return 0;
+
+	/* "2" is for a 32-bit ctime num and an 32-bit inode num. */
+	extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT + 2)
+		  * EXTRA_LEN;
+#if EXTRA_ROUNDING > 0
+	if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
+		extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
+#endif
+	alloc_len = FILE_STRUCT_LEN + extra_len + basename_len;
+	bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum");
+
+	memset(bp, 0, extra_len + FILE_STRUCT_LEN);
+	bp += extra_len;
+	file = (struct file_struct *)bp;
+	bp += FILE_STRUCT_LEN;
+
+	memcpy(bp, basename, basename_len);
+
+	file->mode = S_IFREG;
+	file->modtime = mtime;
+	file->len32 = (uint32)file_length;
+	if (file_length > 0xFFFFFFFFu) {
+		file->flags |= FLAG_LENGTH64;
+		OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32);
+	}
+	file->dirname = dirname;
+	F_CTIME(file) = ctime;
+	F_INODE(file) = inode;
+	bp = F_SUM(file);
+	memcpy(bp, sum, flist_csum_len);
+
+	flist_expand(flist, 1);
+	flist->files[flist->used++] = file;
+
+	flist->sorted = flist->files;
+
+	return 1;
+}
+
+/* The "dirname" arg's data must remain unchanged during the lifespan of
+ * the created csum_cache[].flist object because we use it directly. */
+static void read_checksums(int slot, struct file_list *flist, const char *dirname)
+{
+	char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
+	FILE *fp;
+	char *cp;
+	int len, i;
+	time_t mtime;
+	OFF_T file_length;
+	uint32 ctime, inode;
+	int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0;
+
+	if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN))
+		return;
+	if (dlen)
+		fbuf[dlen++] = '/';
+	else
+		dirname = NULL;
+	strlcpy(fbuf+dlen, RSYNCSUMS_FILE, sizeof fbuf - dlen);
+	if (slot) {
+		pathjoin(line, sizeof line, basis_dir[slot-1], fbuf);
+		cp = line;
+	} else
+		cp = fbuf;
+	if (!(fp = fopen(cp, "r")))
+		return;
+
+	while (fgets(line, sizeof line, fp)) {
+		cp = line;
+		if (file_sum_nni->num == CSUM_MD5) {
+			char *alt_sum = cp;
+			if (*cp == '=')
+				while (*++cp == '=') {}
+			else
+				while (isHexDigit(cp)) cp++;
+			if (cp - alt_sum != MD4_DIGEST_LEN*2 || *cp != ' ')
+				break;
+			while (*++cp == ' ') {}
+		}
+
+		if (*cp == '=') {
+			continue;
+		} else {
+			for (i = 0; i < flist_csum_len*2; i++, cp++) {
+				int x;
+				if (isHexDigit(cp)) {
+					if (isDigit(cp))
+						x = *cp - '0';
+					else
+						x = (*cp & 0xF) + 9;
+				} else {
+					cp = "";
+					break;
+				}
+				if (i & 1)
+					sum[i/2] |= x;
+				else
+					sum[i/2] = x << 4;
+			}
+		}
+		if (*cp != ' ')
+			break;
+		while (*++cp == ' ') {}
+
+		if (file_sum_nni->num < CSUM_MD5) {
+			char *alt_sum = cp;
+			if (*cp == '=')
+				while (*++cp == '=') {}
+			else
+				while (isHexDigit(cp)) cp++;
+			if (cp - alt_sum != MD5_DIGEST_LEN*2 || *cp != ' ')
+				break;
+			while (*++cp == ' ') {}
+		}
+
+		file_length = 0;
+		while (isDigit(cp))
+			file_length = file_length * 10 + *cp++ - '0';
+		if (*cp != ' ')
+			break;
+		while (*++cp == ' ') {}
+
+		mtime = 0;
+		while (isDigit(cp))
+			mtime = mtime * 10 + *cp++ - '0';
+		if (*cp != ' ')
+			break;
+		while (*++cp == ' ') {}
+
+		ctime = 0;
+		while (isDigit(cp))
+			ctime = ctime * 10 + *cp++ - '0';
+		if (*cp != ' ')
+			break;
+		while (*++cp == ' ') {}
+
+		inode = 0;
+		while (isDigit(cp))
+			inode = inode * 10 + *cp++ - '0';
+		if (*cp != ' ')
+			break;
+		while (*++cp == ' ') {}
+
+		len = strlen(cp);
+		while (len && (cp[len-1] == '\n' || cp[len-1] == '\r'))
+			len--;
+		if (!len)
+			break;
+		cp[len++] = '\0'; /* len now counts the null */
+		if (strchr(cp, '/'))
+			break;
+		if (len > MAXPATHLEN)
+			continue;
+
+		strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
+
+		add_checksum(flist, dirname, cp, len, file_length,
+			     mtime, ctime, inode,
+			     sum);
+	}
+	fclose(fp);
+
+	flist_sort_and_clean(flist, CLEAN_KEEP_LAST);
+}
+
+void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
+			 STRUCT_STAT *stp, char *sum_buf)
+{
+	struct file_list *flist = csum_cache[slot].flist;
+	int j;
+
+	if (!flist->next) {
+		flist->next = cur_flist; /* next points from checksum flist to file flist */
+		read_checksums(slot, flist, file->dirname);
+	}
+
+	if ((j = flist_find(flist, file)) >= 0) {
+		struct file_struct *fp = flist->sorted[j];
+
+		if (F_LENGTH(fp) == stp->st_size
+		 && fp->modtime == stp->st_mtime
+		 && (checksum_files & CSF_LAX
+		  || (F_CTIME(fp) == (uint32)stp->st_ctime
+		   && F_INODE(fp) == (uint32)stp->st_ino))) {
+			memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
+			return;
+		}
+	}
+
+	file_checksum(fname, stp, sum_buf);
+}
+
 /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's
  * F_PATHNAME(), or (2) "NULL, dir, dirlen" to chdir() to the supplied dir,
  * with dir == NULL taken to be the starting directory, and dirlen < 0
@@ -1231,7 +1482,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 			      STRUCT_STAT *stp, int flags, int filter_level)
 {
 	static char *lastdir;
-	static int lastdir_len = -1;
+	static int lastdir_len = -2;
 	struct file_struct *file;
 	char thisname[MAXPATHLEN];
 	char linkname[MAXPATHLEN];
@@ -1377,9 +1628,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 			memcpy(lastdir, thisname, len);
 			lastdir[len] = '\0';
 			lastdir_len = len;
+			if (checksum_files && am_sender && flist)
+				reset_checksum_cache();
 		}
-	} else
+	} else {
 		basename = thisname;
+		if (checksum_files && am_sender && flist && lastdir_len == -2) {
+			lastdir_len = -1;
+			reset_checksum_cache();
+		}
+	}
 	basename_len = strlen(basename) + 1; /* count the '\0' */
 
 #ifdef SUPPORT_LINKS
@@ -1409,11 +1667,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		extra_len += EXTRA_LEN;
 #endif
 
-	if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
-		file_checksum(thisname, &st, tmp_sum);
-		if (sender_keeps_checksum)
-			extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
-	}
+	if (sender_keeps_checksum && S_ISREG(st.st_mode))
+		extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
 
 #if EXTRA_ROUNDING > 0
 	if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
@@ -1502,8 +1757,14 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		return NULL;
 	}
 
-	if (sender_keeps_checksum && S_ISREG(st.st_mode))
-		memcpy(F_SUM(file), tmp_sum, flist_csum_len);
+	if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
+		if (flist && checksum_files)
+			get_cached_checksum(0, thisname, file, &st, tmp_sum);
+		else
+			file_checksum(thisname, &st, tmp_sum);
+		if (sender_keeps_checksum)
+			memcpy(F_SUM(file), tmp_sum, flist_csum_len);
+	}
 
 	if (unsort_ndx)
 		F_NDX(file) = stats.num_dirs;
@@ -2720,7 +2981,7 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 	/* The --relative option sends paths with a leading slash, so we need
 	 * to specify the strip_root option here.  We rejected leading slashes
 	 * for a non-relative transfer in recv_file_entry(). */
-	flist_sort_and_clean(flist, relative_paths);
+	flist_sort_and_clean(flist, relative_paths ? CLEAN_STRIP_ROOT : 0);
 
 	if (protocol_version < 30) {
 		/* Recv the io_error flag */
@@ -2965,7 +3226,7 @@ void flist_free(struct file_list *flist)
 
 /* This routine ensures we don't have any duplicate names in our file list.
  * duplicate names can cause corruption because of the pipelining. */
-static void flist_sort_and_clean(struct file_list *flist, int strip_root)
+static void flist_sort_and_clean(struct file_list *flist, int flags)
 {
 	char fbuf[MAXPATHLEN];
 	int i, prev_i;
@@ -3016,7 +3277,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root)
 			/* If one is a dir and the other is not, we want to
 			 * keep the dir because it might have contents in the
 			 * list.  Otherwise keep the first one. */
-			if (S_ISDIR(file->mode)) {
+			if (S_ISDIR(file->mode) || flags & CLEAN_KEEP_LAST) {
 				struct file_struct *fp = flist->sorted[j];
 				if (!S_ISDIR(fp->mode))
 					keep = i, drop = j;
@@ -3032,8 +3293,8 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root)
 			} else
 				keep = j, drop = i;
 
-			if (!am_sender) {
-				if (DEBUG_GTE(DUP, 1)) {
+			if (!am_sender || flags & CLEAN_KEEP_LAST) {
+				if (DEBUG_GTE(DUP, 1) && !(flags & CLEAN_KEEP_LAST)) {
 					rprintf(FINFO,
 					    "removing duplicate name %s from file list (%d)\n",
 					    f_name(file, fbuf), drop + flist->ndx_start);
@@ -3055,7 +3316,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root)
 	}
 	flist->high = prev_i;
 
-	if (strip_root) {
+	if (flags & CLEAN_STRIP_ROOT) {
 		/* We need to strip off the leading slashes for relative
 		 * paths, but this must be done _after_ the sorting phase. */
 		for (i = flist->low; i <= flist->high; i++) {
diff --git a/generator.c b/generator.c
--- a/generator.c
+++ b/generator.c
@@ -54,6 +54,7 @@ extern int delete_after;
 extern int missing_args;
 extern int msgdone_cnt;
 extern int ignore_errors;
+extern int checksum_files;
 extern int remove_source_files;
 extern int delay_updates;
 extern int update_only;
@@ -614,7 +615,7 @@ static enum filetype get_file_type(mode_t mode)
 }
 
 /* Perform our quick-check heuristic for determining if a file is unchanged. */
-int quick_check_ok(enum filetype ftype, const char *fn, struct file_struct *file, STRUCT_STAT *st)
+int quick_check_ok(enum filetype ftype, const char *fn, struct file_struct *file, STRUCT_STAT *st, int slot)
 {
 	switch (ftype) {
 	  case FT_REG:
@@ -625,7 +626,10 @@ int quick_check_ok(enum filetype ftype, const char *fn, struct file_struct *file
 		 * of the file mtime to determine whether to sync. */
 		if (always_checksum > 0) {
 			char sum[MAX_DIGEST_LEN];
-			file_checksum(fn, st, sum);
+			if (checksum_files && slot >= 0)
+				get_cached_checksum(slot, fn, file, st, sum);
+			else
+				file_checksum(fn, st, sum);
 			return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
 		}
 
@@ -956,7 +960,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
 			best_match = j;
 			match_level = 1;
 		}
-		if (!quick_check_ok(FT_REG, cmpbuf, file, &sxp->st))
+		if (!quick_check_ok(FT_REG, cmpbuf, file, &sxp->st, j+1))
 			continue;
 		if (match_level == 1) {
 			best_match = j;
@@ -1079,7 +1083,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 			match_level = 1;
 			best_match = j;
 		}
-		if (!quick_check_ok(ftype, cmpbuf, file, &sxp->st))
+		if (!quick_check_ok(ftype, cmpbuf, file, &sxp->st, j+1))
 			continue;
 		if (match_level < 2) {
 			match_level = 2;
@@ -1215,7 +1219,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	 * --ignore-non-existing, daemon exclude, or mkdir failure. */
 	static struct file_struct *skip_dir = NULL;
 	static struct file_list *fuzzy_dirlist[MAX_BASIS_DIRS+1];
-	static int need_fuzzy_dirlist = 0;
+	static int need_new_dirscan = 0;
 	struct file_struct *fuzzy_file = NULL;
 	int fd = -1, f_copy = -1;
 	stat_x sx, real_sx;
@@ -1332,8 +1336,9 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 						fuzzy_dirlist[i] = NULL;
 					}
 				}
-				need_fuzzy_dirlist = 1;
-			}
+				need_new_dirscan = 1;
+			} else if (checksum_files)
+				need_new_dirscan = 1;
 #ifdef SUPPORT_ACLS
 			if (!preserve_perms)
 				dflt_perms = default_perms_for_dir(dn);
@@ -1341,6 +1346,24 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		}
 		parent_dirname = dn;
 
+		if (need_new_dirscan && ftype == FT_REG) {
+			int i;
+			strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf);
+			for (i = 0; i < fuzzy_basis; i++) {
+				if (i && pathjoin(fnamecmpbuf, MAXPATHLEN, basis_dir[i-1], dn) >= MAXPATHLEN)
+					continue;
+				fuzzy_dirlist[i] = get_dirlist(fnamecmpbuf, -1, GDL_IGNORE_FILTER_RULES | GDL_PERHAPS_DIR);
+				if (fuzzy_dirlist[i] && fuzzy_dirlist[i]->used == 0) {
+					flist_free(fuzzy_dirlist[i]);
+					fuzzy_dirlist[i] = NULL;
+				}
+			}
+			if (checksum_files) {
+				reset_checksum_cache();
+			}
+			need_new_dirscan = 0;
+		}
+
 		statret = link_stat(fname, &sx.st, keep_dirlinks && is_dir);
 		stat_errno = errno;
 	}
@@ -1387,7 +1410,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 			if (INFO_GTE(SKIP, 2)) {
 				if (ftype != stype)
 					suf = " (type change)";
-				else if (!quick_check_ok(ftype, fname, file, &sx.st))
+				else if (!quick_check_ok(ftype, fname, file, &sx.st, 0))
 					suf = always_checksum ? " (sum change)" : " (file change)";
 				else if (!unchanged_attrs(fname, file, &sx))
 					suf = " (attr change)";
@@ -1558,7 +1581,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 			goto cleanup;
 		}
 		if (statret == 0) {
-			if (stype == FT_SYMLINK && quick_check_ok(stype, fname, file, &sx.st)) {
+			if (stype == FT_SYMLINK && quick_check_ok(stype, fname, file, &sx.st, 0)) {
 				/* The link is pointing to the right place. */
 				set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
 				if (itemizing)
@@ -1627,7 +1650,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		if (statret == 0) {
 			if (ftype != stype)
 				statret = -1;
-			else if (quick_check_ok(ftype, fname, file, &sx.st)) {
+			else if (quick_check_ok(ftype, fname, file, &sx.st, 0)) {
 				/* The device or special file is identical. */
 				set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
 				if (itemizing)
@@ -1752,22 +1775,6 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		partialptr = NULL;
 
 	if (statret != 0 && fuzzy_basis) {
-		if (need_fuzzy_dirlist) {
-			const char *dn = file->dirname ? file->dirname : ".";
-			int i;
-			strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf);
-			for (i = 0; i < fuzzy_basis; i++) {
-				if (i && pathjoin(fnamecmpbuf, MAXPATHLEN, basis_dir[i-1], dn) >= MAXPATHLEN)
-					continue;
-				fuzzy_dirlist[i] = get_dirlist(fnamecmpbuf, -1, GDL_IGNORE_FILTER_RULES | GDL_PERHAPS_DIR);
-				if (fuzzy_dirlist[i] && fuzzy_dirlist[i]->used == 0) {
-					flist_free(fuzzy_dirlist[i]);
-					fuzzy_dirlist[i] = NULL;
-				}
-			}
-			need_fuzzy_dirlist = 0;
-		}
-
 		/* Sets fnamecmp_type to FNAMECMP_FUZZY or above. */
 		fuzzy_file = find_fuzzy(file, fuzzy_dirlist, &fnamecmp_type);
 		if (fuzzy_file) {
@@ -1806,7 +1813,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		;
 	else if (fnamecmp_type >= FNAMECMP_FUZZY)
 		;
-	else if (quick_check_ok(FT_REG, fnamecmp, file, &sx.st)) {
+	else if (quick_check_ok(FT_REG, fnamecmp, file, &sx.st, fnamecmp_type == FNAMECMP_FNAME ? 0 : -1)) {
 		if (partialptr) {
 			do_unlink(partialptr);
 			handle_partial_dir(partialptr, PDIR_DELETE);
diff --git a/hlink.c b/hlink.c
--- a/hlink.c
+++ b/hlink.c
@@ -406,7 +406,7 @@ int hard_link_check(struct file_struct *file, int ndx, char *fname,
 				}
 				break;
 			}
-			if (!quick_check_ok(FT_REG, cmpbuf, file, &alt_sx.st))
+			if (!quick_check_ok(FT_REG, cmpbuf, file, &alt_sx.st, j+1))
 				continue;
 			statret = 1;
 			if (unchanged_attrs(cmpbuf, file, &alt_sx))
diff --git a/loadparm.c b/loadparm.c
--- a/loadparm.c
+++ b/loadparm.c
@@ -162,6 +162,13 @@ static struct enum_list enum_syslog_facility[] = {
 	{ -1, NULL }
 };
 
+static struct enum_list enum_checksum_files[] = {
+	{ CSF_IGNORE_FILES, "none" },
+	{ CSF_LAX_MODE, "lax" },
+	{ CSF_STRICT_MODE, "strict" },
+	{ -1, NULL }
+};
+
 /* Expand %VAR% references.  Any unknown vars or unrecognized
  * syntax leaves the raw chars unchanged. */
 static char *expand_vars(const char *str)
diff --git a/options.c b/options.c
--- a/options.c
+++ b/options.c
@@ -126,6 +126,7 @@ size_t bwlimit_writemax = 0;
 int ignore_existing = 0;
 int ignore_non_existing = 0;
 int need_messages_from_generator = 0;
+int checksum_files = CSF_IGNORE_FILES;
 int max_delete = INT_MIN;
 OFF_T max_size = -1;
 OFF_T min_size = -1;
@@ -582,7 +583,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
       OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
       OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
       OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE,
-      OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR,
+      OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, OPT_SUMFILES,
       OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS,
       OPT_STOP_AFTER, OPT_STOP_AT,
       OPT_REFUSED_BASE = 9000};
@@ -739,6 +740,7 @@ static struct poptOption long_options[] = {
   {"no-c",             0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
   {"checksum-choice",  0,  POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
   {"cc",               0,  POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
+  {"sumfiles",         0,  POPT_ARG_STRING, 0, OPT_SUMFILES, 0, 0 },
   {"block-size",      'B', POPT_ARG_STRING, 0, OPT_BLOCK_SIZE, 0, 0 },
   {"compare-dest",     0,  POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
   {"copy-dest",        0,  POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
@@ -1751,6 +1753,23 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			}
 			break;
 
+		case OPT_SUMFILES:
+			arg = poptGetOptArg(pc);
+			checksum_files = 0;
+			if (strcmp(arg, "lax") == 0)
+				checksum_files |= CSF_LAX_MODE;
+			else if (strcmp(arg, "strict") == 0)
+				checksum_files |= CSF_STRICT_MODE;
+			else if (strcmp(arg, "none") == 0)
+				checksum_files = CSF_IGNORE_FILES;
+			else {
+				snprintf(err_buf, sizeof err_buf,
+				    "Invalid argument passed to --sumfiles (%s)\n",
+				    arg);
+				return 0;
+			}
+			break;
+
 		case OPT_INFO:
 			arg = poptGetOptArg(pc);
 			parse_output_words(info_words, info_levels, arg, USER_PRIORITY);
@@ -2118,6 +2137,9 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 	}
 #endif
 
+	if (!always_checksum)
+		checksum_files = CSF_IGNORE_FILES;
+
 	if (write_batch && read_batch) {
 		snprintf(err_buf, sizeof err_buf,
 			"--write-batch and --read-batch can not be used together\n");
diff --git a/rsync.1.md b/rsync.1.md
--- a/rsync.1.md
+++ b/rsync.1.md
@@ -422,6 +422,7 @@ has its own detailed description later in this manpage.
 --quiet, -q              suppress non-error messages
 --no-motd                suppress daemon-mode MOTD
 --checksum, -c           skip based on checksum, not mod-time & size
+--sumfiles=MODE          use .rsyncsums to speedup --checksum mode
 --archive, -a            archive mode is -rlptgoD (no -A,-X,-U,-N,-H)
 --no-OPTION              turn off an implied OPTION (e.g. --no-D)
 --recursive, -r          recurse into directories
@@ -814,6 +815,8 @@ expand it.
     file that has the same size as the corresponding sender's file: files with
     either a changed size or a changed checksum are selected for transfer.
 
+    See also the [`--sumfiles`](#opt) option for a way to use cached checksum data.
+
     Note that rsync always verifies that each _transferred_ file was correctly
     reconstructed on the receiving side by checking a whole-file checksum that
     is generated as the file is transferred, but that automatic
@@ -825,6 +828,38 @@ expand it.
     option or an environment variable that is discussed in that option's
     section.
 
+0.  `--sumfiles=MODE`
+
+    This option tells rsync to make use of any cached checksum information it
+    finds in per-directory .rsyncsums files when the current transfer is using
+    the [`--checksum`](#opt) option.  If the checksum data is up-to-date, it is
+    used instead of recomputing it, saving both disk I/O and CPU time.  If the
+    checksum data is missing or outdated, the checksum is computed just as it
+    would be if `--sumfiles` was not specified.
+
+    The MODE value is either "lax", for relaxed checking (which compares size
+    and mtime), "strict" (which also compares ctime and inode), or "none" to
+    ignore any .rsyncsums files ("none" is the default).  Rsync does not create
+    or update these files, but there is a perl script in the support directory
+    named "rsyncsums" that can be used for that.
+
+    This option has no effect unless [`--checksum`](#opt) (`-c`) was also
+    specified.  It also only affects the current side of the transfer, so if
+    you want the remote side to parse its own .rsyncsums files, specify the
+    option via [`--remote-option`](#opt) (`-M`) (e.g. "`-M--sumfiles=lax`").
+
+    To avoid transferring the system's checksum files, you can use an exclude
+    (e.g. [`--exclude=.rsyncsums`](#opt)).  To make this easier to type, you
+    can use a popt alias.  For instance, adding the following line in your
+    ~/.popt file defines a `--cs` option that enables lax checksum files and
+    excludes the checksum files:
+
+    >     rsync alias --cs -c --sumfiles=lax -M--sumfiles=lax -f-_.rsyncsums
+
+    An rsync daemon does not allow the client to control this setting, so see
+    the "checksum files" daemon parameter for information on how to make a
+    daemon use cached checksum data.
+
 0.  `--archive`, `-a`
 
     This is equivalent to `-rlptgoD`.  It is a quick way of saying you want
diff --git a/rsync.h b/rsync.h
--- a/rsync.h
+++ b/rsync.h
@@ -897,6 +897,10 @@ extern int file_sum_extra_cnt;
 #define F_SUM(f) ((char*)OPT_EXTRA(f, START_BUMP(f) + HLINK_BUMP(f) \
 				    + SUM_EXTRA_CNT - 1))
 
+/* These are only valid on an entry derived from a checksum file. */
+#define F_CTIME(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT)->unum
+#define F_INODE(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT + 1)->unum
+
 /* Some utility defines: */
 #define F_IS_ACTIVE(f) (f)->basename[0]
 #define F_IS_HLINKED(f) ((f)->flags & FLAG_HLINKED)
@@ -1111,6 +1115,13 @@ typedef struct {
 #define RELNAMECACHE_LEN (offsetof(relnamecache, fname))
 #endif
 
+#define CSF_ENABLE (1<<1)
+#define CSF_LAX (1<<2)
+
+#define CSF_IGNORE_FILES 0
+#define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX)
+#define CSF_STRICT_MODE (CSF_ENABLE)
+
 #include "byteorder.h"
 #include "lib/mdigest.h"
 #include "lib/wildmatch.h"
diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md
--- a/rsyncd.conf.5.md
+++ b/rsyncd.conf.5.md
@@ -453,6 +453,19 @@ in the values of parameters.  See that section for details.
     the max connections limit is not exceeded for the modules sharing the lock
     file.  The default is `/var/run/rsyncd.lock`.
 
+0.  `checksum files`
+
+    This parameter tells rsync to make use of any cached checksum information
+    it finds in per-directory .rsyncsums files when the current transfer is
+    using the `--checksum` option.  The value can be set to either "lax",
+    "strict", or "none".  See the client's `--sumfiles` option for what these
+    choices do.
+
+    Note also that the client's command-line option, `--sumfiles`, has no
+    effect on a daemon.  A daemon will only access checksum files if this
+    config option tells it to.  See also the `exclude` directive for a way to
+    hide the .rsyncsums files from the user.
+
 0.  `read only`
 
     This parameter determines whether clients will be able to upload files or
diff --git a/support/rsyncsums b/support/rsyncsums
new file mode 100755
--- /dev/null
+++ b/support/rsyncsums
@@ -0,0 +1,201 @@
+#!/usr/bin/perl -w
+use strict;
+
+use Getopt::Long;
+use Cwd qw(abs_path cwd);
+use Digest::MD4;
+use Digest::MD5;
+
+our $SUMS_FILE = '.rsyncsums';
+
+&Getopt::Long::Configure('bundling');
+&usage if !&GetOptions(
+    'recurse|r' => \( my $recurse_opt ),
+    'mode|m=s' => \( my $cmp_mode = 'strict' ),
+    'check|c' => \( my $check_opt ),
+    'verbose|v+' => \( my $verbosity = 0 ),
+    'help|h' => \( my $help_opt ),
+);
+&usage if $help_opt || $cmp_mode !~ /^(lax|strict)$/;
+
+my $ignore_ctime_and_inode = $cmp_mode eq 'lax' ? 0 : 1;
+
+my $start_dir = cwd();
+
+my @dirs = @ARGV;
+@dirs = '.' unless @dirs;
+foreach (@dirs) {
+    $_ = abs_path($_);
+}
+
+$| = 1;
+
+my $exit_code = 0;
+
+my $md4 = Digest::MD4->new;
+my $md5 = Digest::MD5->new;
+
+while (@dirs) {
+    my $dir = shift @dirs;
+
+    if (!chdir($dir)) {
+	warn "Unable to chdir to $dir: $!\n";
+	next;
+    }
+    if (!opendir(DP, '.')) {
+	warn "Unable to opendir $dir: $!\n";
+	next;
+    }
+
+    my $reldir = $dir;
+    $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
+    if ($verbosity) {
+	print "$reldir ... ";
+	print "\n" if $check_opt;
+    }
+
+    my %cache;
+    my $f_cnt = 0;
+    if (open(FP, '<', $SUMS_FILE)) {
+	while (<FP>) {
+	    chomp;
+	    my($sum4, $sum5, $size, $mtime, $ctime, $inode, $fn) = split(' ', $_, 7);
+	    $cache{$fn} = [ 0, $sum4, $sum5, $size, $mtime, $ctime & 0xFFFFFFFF, $inode & 0xFFFFFFFF ];
+	    $f_cnt++;
+	}
+	close FP;
+    }
+
+    my @subdirs;
+    my $d_cnt = 0;
+    my $update_cnt = 0;
+    while (defined(my $fn = readdir(DP))) {
+	next if $fn =~ /^\.\.?$/ || $fn =~ /^\Q$SUMS_FILE\E$/o || -l $fn;
+	if (-d _) {
+	    push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/;
+	    next;
+	}
+	next unless -f _;
+
+	my($size,$mtime,$ctime,$inode) = (stat(_))[7,9,10,1];
+	$ctime &= 0xFFFFFFFF;
+	$inode &= 0xFFFFFFFF;
+	my $ref = $cache{$fn};
+	$d_cnt++;
+
+	if (!$check_opt) {
+	    if (defined $ref) {
+		$$ref[0] = 1;
+		if ($$ref[3] == $size
+		 && $$ref[4] == $mtime
+		 && ($ignore_ctime_and_inode || ($$ref[5] == $ctime && $$ref[6] == $inode))
+		 && $$ref[1] !~ /=/ && $$ref[2] !~ /=/) {
+		    next;
+		}
+	    }
+	    if (!$update_cnt++) {
+		print "UPDATING\n" if $verbosity;
+	    }
+	}
+
+	if (!open(IN, $fn)) {
+	    print STDERR "Unable to read $fn: $!\n";
+	    if (defined $ref) {
+		delete $cache{$fn};
+		$f_cnt--;
+	    }
+	    next;
+	}
+
+	my($sum4, $sum5);
+	while (1) {
+	    while (sysread(IN, $_, 64*1024)) {
+		$md4->add($_);
+		$md5->add($_);
+	    }
+	    $sum4 = $md4->hexdigest;
+	    $sum5 = $md5->hexdigest;
+	    print " $sum4 $sum5" if $verbosity > 2;
+	    print " $fn" if $verbosity > 1;
+	    my($size2,$mtime2,$ctime2,$inode2) = (stat(IN))[7,9,10,1];
+	    $ctime2 &= 0xFFFFFFFF;
+	    $inode2 &= 0xFFFFFFFF;
+	    last if $size == $size2 && $mtime == $mtime2
+	     && ($ignore_ctime_and_inode || ($ctime == $ctime2 && $inode == $inode2));
+	    $size = $size2;
+	    $mtime = $mtime2;
+	    $ctime = $ctime2;
+	    $inode = $inode2;
+	    sysseek(IN, 0, 0);
+	    print " REREADING\n" if $verbosity > 1;
+	}
+
+	close IN;
+
+	if ($check_opt) {
+	    my $dif;
+	    if (!defined $ref) {
+		$dif = 'MISSING';
+	    } elsif ($sum4 ne $$ref[1] || $sum5 ne $$ref[2]) {
+		$dif = 'FAILED';
+	    } else {
+		print " OK\n" if $verbosity > 1;
+		next;
+	    }
+	    if ($verbosity < 2) {
+		print $verbosity ? ' ' : "$reldir/";
+		print $fn;
+	    }
+	    print " $dif\n";
+	    $exit_code = 1;
+	} else {
+	    print "\n" if $verbosity > 1;
+	    $cache{$fn} = [ 1, $sum4, $sum5, $size, $mtime, $ctime, $inode ];
+	}
+    }
+
+    closedir DP;
+
+    unshift(@dirs, sort @subdirs) if $recurse_opt;
+
+    if ($check_opt) {
+	;
+    } elsif ($d_cnt == 0) {
+	if ($f_cnt) {
+	    print "(removed $SUMS_FILE) " if $verbosity;
+	    unlink($SUMS_FILE);
+	}
+	print "empty\n" if $verbosity;
+    } elsif ($update_cnt || $d_cnt != $f_cnt) {
+	print "UPDATING\n" if $verbosity && !$update_cnt;
+	open(FP, '>', $SUMS_FILE) or die "Unable to write $dir/$SUMS_FILE: $!\n";
+
+	foreach my $fn (sort keys %cache) {
+	    my $ref = $cache{$fn};
+	    my($found, $sum4, $sum5, $size, $mtime, $ctime, $inode) = @$ref;
+	    next unless $found;
+	    printf FP '%s %s %10d %10d %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $inode, $fn;
+	}
+	close FP;
+    } else {
+	print "ok\n" if $verbosity;
+    }
+}
+
+exit $exit_code;
+
+sub usage
+{
+    die <<EOT;
+Usage: rsyncsums [OPTIONS] [DIRS]
+
+Options:
+ -r, --recurse     Update $SUMS_FILE files in subdirectories too.
+ -m, --mode=MODE   Compare entries in either "lax" or "strict" mode.  Using
+                   "lax" compares size and mtime, while "strict" additionally
+                   compares ctime and inode.  Default:  strict.
+ -c, --check       Check if the checksums are right (doesn't update).
+ -v, --verbose     Mention what we're doing.  Repeat for more info.
+ -h, --help        Display this help message.
+EOT
+}