Port #403: --min-filesize to skip small files

duperemove.c

@@ -209,6 +209,7 @@ enum {
 	EXCLUDE_OPTION,
 	BATCH_SIZE_OPTION,
 	NO_COLOR_OPTION,
+	MIN_FILESIZE_OPTION,
 };
 
 static int process_fdupes(void)
@@ -322,14 +323,15 @@ static int parse_options(int argc, char **argv, int *filelist_idx)
 		{ "exclude", 1, NULL, EXCLUDE_OPTION },
 		{ "batchsize", 1, NULL, BATCH_SIZE_OPTION },
 		{ "no-color", 0, NULL, NO_COLOR_OPTION },
+		{ "min-filesize", 1, NULL, MIN_FILESIZE_OPTION },
 		{ NULL, 0, NULL, 0}
 	};
 
 	if (argc < 2) {
 		help(); /* Never returns */
 	}
 
-	while ((c = getopt_long(argc, argv, "b:vdDrh?LRqB:", long_ops, NULL))
+	while ((c = getopt_long(argc, argv, "b:vdDrh?LRqB:m:", long_ops, NULL))
 	       != -1) {
 		switch (c) {
 		case 'b':
@@ -412,6 +414,14 @@ static int parse_options(int argc, char **argv, int *filelist_idx)
 		case NO_COLOR_OPTION:
 			opt_no_color = 1;
 			break;
+		case MIN_FILESIZE_OPTION:
+		case 'm':
+			options.min_filesize = parse_size(optarg);
+			if (options.min_filesize == 0) {
+				eprintf("Error: --min-filesize must be greater than zero\n");
+				return EINVAL;
+			}
+			break;
 		case EXCLUDE_OPTION:
 			if (add_exclude_pattern(optarg))
 				eprintf("Error: cannot exclude %s\n", optarg);

file_scan.c

@@ -604,7 +604,7 @@ bool is_fs_supported(char *path)
 /* Check if path should be processed:
  * - is path not excluded ?
  * - is path a file or directory ?
- * - is path not an empty file ?
+ * - is path at least --min-filesize bytes (empty files by default) ?
  * - does path lives on our locked filesystem ?
  *   for files, we only do that check if the parent is not checked
  *
@@ -625,8 +625,9 @@ bool check_file(struct dbhandle *db, char *path, struct statx *st, bool parent_c
 		return false;
 	}
 
-	if (S_ISREG(st->stx_mode) && st->stx_size == 0) {
-		vprintf("Skipping empty file %s\n", path);
+	if (S_ISREG(st->stx_mode) && st->stx_size < options.min_filesize) {
+		vprintf("Skipping file below --min-filesize: %s (%llu < %"PRIu64")\n",
+			path, st->stx_size, options.min_filesize);
 		return false;
 	}
 

markdown/duperemove.md

@@ -116,6 +116,12 @@ running `duperemove` on very large files (like virtual machines etc).
 
     By default, batching is set to 1024.
 
+**-m** `N`, **\--min-filesize**=`N`
+  ~ Skip all regular files smaller than `N` bytes (suffixes like `K`, `M`, `G`
+are accepted). Trees with many tiny files can be scanned much faster this way,
+since such files rarely dedupe usefully. The default of `1` only skips empty
+files.
+
 **-h**
   ~ Print numbers in human-readable format.
 

opt.c

@@ -24,4 +24,5 @@ struct options options = {
 	.dedupe_same_file = true,
 	.batch_size = 1024,
 	.fdupes_mode = false,
+	.min_filesize = 1,	/* default: only skip empty files */
 };

opt.h

@@ -15,6 +15,7 @@
 #define	__OPT_H__
 
 #include <stdbool.h>
+#include <stdint.h>
 
 struct options {
 	int run_dedupe;
@@ -28,6 +29,7 @@ struct options {
 	unsigned int batch_size;
 	bool fdupes_mode : 1;
 	char *hashfile;
+	uint64_t min_filesize;	/* skip regular files smaller than this */
 };
 
 extern struct options options;

tests/integration/test_min_filesize.py

@@ -0,0 +1,25 @@
+"""--min-filesize skips regular files below the threshold; the default (1) only
+skips empty files."""
+
+from harness import DuperemoveTest
+
+
+class MinFilesizeTest(DuperemoveTest):
+    def _scan_count(self, *extra):
+        self.write("tree/tiny", b"x" * 500)
+        self.mkrand("tree/big", 200000)
+        self.write("tree/empty", b"")
+        self.scan(self.path("tree"), *extra)
+        self.assertDmOk()
+        return self.hf_count("files")
+
+    def test_default_skips_only_empty(self):
+        # tiny + big recorded, empty skipped
+        self.assertEqual(2, self._scan_count())
+
+    def test_skips_below_threshold(self):
+        # only big (200K) survives a 1K floor
+        self.assertEqual(1, self._scan_count("--min-filesize", "1K"))
+
+    def test_dash_m_alias(self):
+        self.assertEqual(1, self._scan_count("-m", "1K"))