From bbcc2142ef70555825f22e74b8b77f5fc2efbb09 Mon Sep 17 00:00:00 2001 From: Martin Leitner-Ankerl Date: Wed, 15 Jul 2026 19:31:06 +0200 Subject: [PATCH] duperemove: add --min-filesize to skip small files Ports markfasheh/duperemove#403 by Andreas K. Huettel. Add -m/--min-filesize=N: regular files smaller than N bytes are skipped during scanning. Trees with many tiny files (which rarely dedupe usefully) scan much faster. Implemented by folding the existing empty-file skip into one threshold check in check_file(); min_filesize defaults to 1, so the default behaviour (skip only empty files) is unchanged. Rejects 0. Adds a man-page entry and an integration test. Co-authored-by: Andreas K. Huettel Co-Authored-By: Claude Fable 5 --- duperemove.c | 12 +++++++++++- file_scan.c | 7 ++++--- markdown/duperemove.md | 6 ++++++ opt.c | 1 + opt.h | 2 ++ tests/integration/test_min_filesize.py | 25 +++++++++++++++++++++++++ 6 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_min_filesize.py diff --git a/duperemove.c b/duperemove.c index 6c51d6b30402..d243eb5da48a 100644 --- a/duperemove.c +++ b/duperemove.c @@ -209,6 +209,7 @@ enum { EXCLUDE_OPTION, BATCH_SIZE_OPTION, NO_COLOR_OPTION, + MIN_FILESIZE_OPTION, }; static int process_fdupes(void) @@ -322,6 +323,7 @@ static int parse_options(int argc, char **argv, int *filelist_idx) { "exclude", 1, NULL, EXCLUDE_OPTION }, { "batchsize", 1, NULL, BATCH_SIZE_OPTION }, { "no-color", 0, NULL, NO_COLOR_OPTION }, + { "min-filesize", 1, NULL, MIN_FILESIZE_OPTION }, { NULL, 0, NULL, 0} }; @@ -329,7 +331,7 @@ static int parse_options(int argc, char **argv, int *filelist_idx) help(); /* Never returns */ } - while ((c = getopt_long(argc, argv, "b:vdDrh?LRqB:", long_ops, NULL)) + while ((c = getopt_long(argc, argv, "b:vdDrh?LRqB:m:", long_ops, NULL)) != -1) { switch (c) { case 'b': @@ -412,6 +414,14 @@ static int parse_options(int argc, char **argv, int *filelist_idx) case NO_COLOR_OPTION: opt_no_color = 1; break; + case MIN_FILESIZE_OPTION: + case 'm': + options.min_filesize = parse_size(optarg); + if (options.min_filesize == 0) { + eprintf("Error: --min-filesize must be greater than zero\n"); + return EINVAL; + } + break; case EXCLUDE_OPTION: if (add_exclude_pattern(optarg)) eprintf("Error: cannot exclude %s\n", optarg); diff --git a/file_scan.c b/file_scan.c index 2e71e3826eb3..588284c1ebee 100644 --- a/file_scan.c +++ b/file_scan.c @@ -604,7 +604,7 @@ bool is_fs_supported(char *path) /* Check if path should be processed: * - is path not excluded ? * - is path a file or directory ? - * - is path not an empty file ? + * - is path at least --min-filesize bytes (empty files by default) ? * - does path lives on our locked filesystem ? * for files, we only do that check if the parent is not checked * @@ -625,8 +625,9 @@ bool check_file(struct dbhandle *db, char *path, struct statx *st, bool parent_c return false; } - if (S_ISREG(st->stx_mode) && st->stx_size == 0) { - vprintf("Skipping empty file %s\n", path); + if (S_ISREG(st->stx_mode) && st->stx_size < options.min_filesize) { + vprintf("Skipping file below --min-filesize: %s (%llu < %"PRIu64")\n", + path, st->stx_size, options.min_filesize); return false; } diff --git a/markdown/duperemove.md b/markdown/duperemove.md index ab41d1ae57fb..fc10f8ceb8fe 100644 --- a/markdown/duperemove.md +++ b/markdown/duperemove.md @@ -116,6 +116,12 @@ running `duperemove` on very large files (like virtual machines etc). By default, batching is set to 1024. +**-m** `N`, **\--min-filesize**=`N` + ~ Skip all regular files smaller than `N` bytes (suffixes like `K`, `M`, `G` +are accepted). Trees with many tiny files can be scanned much faster this way, +since such files rarely dedupe usefully. The default of `1` only skips empty +files. + **-h** ~ Print numbers in human-readable format. diff --git a/opt.c b/opt.c index a1813b289b15..153bb542c64e 100644 --- a/opt.c +++ b/opt.c @@ -24,4 +24,5 @@ struct options options = { .dedupe_same_file = true, .batch_size = 1024, .fdupes_mode = false, + .min_filesize = 1, /* default: only skip empty files */ }; diff --git a/opt.h b/opt.h index 6812397e389c..622f8e3d90d0 100644 --- a/opt.h +++ b/opt.h @@ -15,6 +15,7 @@ #define __OPT_H__ #include +#include struct options { int run_dedupe; @@ -28,6 +29,7 @@ struct options { unsigned int batch_size; bool fdupes_mode : 1; char *hashfile; + uint64_t min_filesize; /* skip regular files smaller than this */ }; extern struct options options; diff --git a/tests/integration/test_min_filesize.py b/tests/integration/test_min_filesize.py new file mode 100644 index 000000000000..244bdf3511d0 --- /dev/null +++ b/tests/integration/test_min_filesize.py @@ -0,0 +1,25 @@ +"""--min-filesize skips regular files below the threshold; the default (1) only +skips empty files.""" + +from harness import DuperemoveTest + + +class MinFilesizeTest(DuperemoveTest): + def _scan_count(self, *extra): + self.write("tree/tiny", b"x" * 500) + self.mkrand("tree/big", 200000) + self.write("tree/empty", b"") + self.scan(self.path("tree"), *extra) + self.assertDmOk() + return self.hf_count("files") + + def test_default_skips_only_empty(self): + # tiny + big recorded, empty skipped + self.assertEqual(2, self._scan_count()) + + def test_skips_below_threshold(self): + # only big (200K) survives a 1K floor + self.assertEqual(1, self._scan_count("--min-filesize", "1K")) + + def test_dash_m_alias(self): + self.assertEqual(1, self._scan_count("-m", "1K"))