diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 029548b1..32abbd10 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -58,9 +58,9 @@ jobs: - name: "Test it" if: ${{ env.RUN_TEST == 'true' }} run: | - RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v + RM_TS_PEDANTIC=0 python -m pytest -s -v - - name: CoW tests + - name: CoW tests - BTRFS if: ${{ env.RUN_TEST == 'true' }} shell: bash run: | @@ -69,22 +69,43 @@ jobs: sudo mkfs.btrfs -f /dev/ram0 sudo mount /dev/ram0 "${RM_TS_DIR}" sudo chmod 0777 "${RM_TS_DIR}" - cat <<'EOF' >> tests/conftest.py - def pytest_collection_modifyitems(items, config): - selected_items = [] - deselected_items = [] + RM_TS_PEDANTIC=0 python -m pytest -s -v tests/test_mains/test_is_reflink.py + RM_TS_PEDANTIC=0 python -m pytest -s -v tests/test_mains/test_dedupe.py + - name: Build ZFS + if: ${{ env.RUN_TEST == 'true' }} + shell: bash + run: | + sudo apt install alien autoconf automake build-essential debhelper-compat dh-autoreconf dh-dkms dh-python dkms fakeroot gawk git libaio-dev libattr1-dev libblkid-dev libcurl4-openssl-dev libelf-dev libffi-dev libpam0g-dev libssl-dev libtirpc-dev libtool libudev-dev linux-headers-generic parallel po-debconf python3 python3-all-dev python3-cffi python3-dev python3-packaging python3-setuptools python3-sphinx uuid-dev zlib1g-dev + git clone -b zfs-2.4-release --single-branch https://github.com/openzfs/zfs.git + cd ./zfs + sh autogen.sh + ./configure + make -s -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod + sudo dpkg -i *.deb + # Update order of directories to search for modules, otherwise + # Ubuntu will load kernel-shipped ones. + sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf + sudo depmod + sudo modprobe zfs + echo "ZFS versions installed:" + sudo zfs -V + echo "Bclone enabled: $(cat /sys/module/zfs/parameters/zfs_bclone_enabled)" + - name: CoW tests - ZFS + if: ${{ env.RUN_TEST == 'true' }} + shell: bash +# TODO: change to blkdiscard when the runner will be running 6.10+ kernel. + run: | + sudo umount "${RM_TS_DIR}" + sudo rmmod brd + sudo modprobe brd rd_nr=1 rd_size=12582912 + + sudo zpool create -m "${RM_TS_DIR}" rmlint ram0 + + sudo chmod 0777 "${RM_TS_DIR}" + + RM_TS_PEDANTIC=0 python -m pytest -s -v tests/test_mains/test_is_reflink.py - for item in items: - if "needs_reflink_fs" in getattr(item, "fixturenames", ()): - selected_items.append(item) - else: - deselected_items.append(item) - config.hook.pytest_deselected(items=deselected_items) - items[:] = selected_items - EOF - RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v - - name: "Cleanup" if: ${{ env.RUN_TEST == 'true' }} run: | diff --git a/docs/rmlint.1.rst b/docs/rmlint.1.rst index 69fa218a..86233d7e 100644 --- a/docs/rmlint.1.rst +++ b/docs/rmlint.1.rst @@ -664,9 +664,10 @@ FORMATTERS This will free up duplicate extents. Needs at least kernel 4.2. Use this option when you only have read-only access to a btrfs filesystem but still want to deduplicate it. This is usually the case for snapshots. + Note: This does not work on ZFS as it doesn't support the FIDEDUPERANGE ioctl. * ``reflink``: Try to reflink the duplicate file to the original. See also ``--reflink`` in ``man 1 cp``. Fails if the filesystem does not support - it. + it. Works on btrfs, XFS (with reflink enabled), and ZFS (OpenZFS 2.2+). * ``hardlink``: Replace the duplicate file with a hardlink to the original file. The resulting files will have the same inode number. Fails if both files are not on the same partition. You can use ``ls -i`` to show the @@ -682,7 +683,8 @@ FORMATTERS Default is ``remove``. * *link*: Shortcut for ``-c sh:handler=clone,reflink,hardlink,symlink``. - Use this if you are on a reflink-capable system. + Use this if you are on a reflink-capable system. On ZFS, the clone + handler will be skipped (as it's not supported), and reflink will be used instead. * *hardlink*: Shortcut for ``-c sh:handler=hardlink,symlink``. Use this if you want to hardlink files, but want to fallback for duplicates that lie on different devices. diff --git a/lib/formats/sh.c.in b/lib/formats/sh.c.in index 1b9c08c2..7e7f91e2 100644 --- a/lib/formats/sh.c.in +++ b/lib/formats/sh.c.in @@ -86,7 +86,7 @@ static bool rm_sh_emit_handler_clone(RmFmtHandlerShScript *self, char **out, RmF return false; } - if (!rm_mounts_can_reflink(self->session->mounts, file->dev, self->last_original->dev) ) { + if (!rm_mounts_can_dedupe(self->session->mounts, file->dev, self->last_original->dev) ) { return false; } diff --git a/lib/utilities.c b/lib/utilities.c index dd65a9de..3fe45a5c 100644 --- a/lib/utilities.c +++ b/lib/utilities.c @@ -579,7 +579,8 @@ static RmMountEntry *rm_mount_list_next(RmMountEntries *self) { } } -static bool fs_supports_reflinks(char *fstype, char *mountpoint) { +static bool fs_supports_dedupe(char *fstype, char *mountpoint) { + /* Check if filesystem supports FIDEDUPERANGE ioctl for deduplication */ if(strcmp(fstype, "btrfs")==0) { return true; } @@ -596,6 +597,49 @@ static bool fs_supports_reflinks(char *fstype, char *mountpoint) { return false; } +static bool fs_supports_reflinks(char *fstype, char *mountpoint, char *fsname) { + /* Check if filesystem supports any form of reflinks/clones */ + if(fs_supports_dedupe(fstype, mountpoint)) { + /* If it supports dedupe ioctl, it supports reflinks */ + return true; + } + if(strcmp(fstype, "zfs")==0) { + /* ZFS supports reflinks/clones in newer versions (OpenZFS 2.2+). + * Note: ZFS does not support the FIDEDUPERANGE ioctl used by + * 'rmlint --dedupe', but does support 'cp --reflink=always'. + * For sh:handler=reflink, the generated script will use cp --reflink. + * For sh:handler=clone, the dedupe operation will fail gracefully. */ + + /* Check that the zpool has block_cloning feature enabled. + * Get the pool name from fsname (the part before the first '/', if any). + * It can be disabled, or not supported because the pool has not been + * upgraded yet, or because ZFS is too old. */ + char *fs = g_strdup(fsname); + g_strdelimit(fs, "/", '\0'); /* modifies 'fs' inplace */ + + gchar *argv[] = {"zpool", "get", "-H", "-o", "value", "feature@block_cloning", fs, NULL}; + gchar *standard_output = NULL; + gint exit_status = 0; + bool supports_cloning = false; + + if(g_spawn_sync(NULL, argv, NULL, G_SPAWN_SEARCH_PATH | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &standard_output, NULL, &exit_status, NULL)) { + if(exit_status == 0 && standard_output != NULL) { + /* Check if the feature is "active" or "enabled" */ + if(strstr(standard_output, "active") != NULL || + strstr(standard_output, "enabled") != NULL) { + supports_cloning = true; + } + } + g_free(standard_output); + } + + g_free(fs); + return supports_cloning; + } + return false; +} + static RmMountEntries *rm_mount_list_open(RmMountTable *table) { RmMountEntries *self = g_slice_new(RmMountEntries); @@ -675,13 +719,21 @@ static RmMountEntries *rm_mount_list_open(RmMountTable *table) { evilfs_found->name, wrap_entry->dir, (unsigned)dir_stat.st_dev); } - if(fs_supports_reflinks(wrap_entry->type, wrap_entry->dir)) { + if(fs_supports_reflinks(wrap_entry->type, wrap_entry->dir, wrap_entry->fsname)) { RmStat dir_stat; if(rm_sys_stat(wrap_entry->dir, &dir_stat) == 0) { g_hash_table_insert(table->reflinkfs_table, GUINT_TO_POINTER(dir_stat.st_dev), wrap_entry->type); rm_log_debug_line("Filesystem %s: reflink capable", wrap_entry->dir); + + /* Also check if it supports dedupe ioctl */ + if(fs_supports_dedupe(wrap_entry->type, wrap_entry->dir)) { + g_hash_table_insert(table->dedupefs_table, + GUINT_TO_POINTER(dir_stat.st_dev), + wrap_entry->type); + rm_log_debug_line("Filesystem %s: dedupe capable", wrap_entry->dir); + } continue; } } @@ -712,6 +764,7 @@ static bool rm_mounts_create_tables(RmMountTable *self, bool force_fiemap) { /* Mapping dev_t => true (used as set) */ self->evilfs_table = g_hash_table_new(NULL, NULL); self->reflinkfs_table = g_hash_table_new(NULL, NULL); + self->dedupefs_table = g_hash_table_new(NULL, NULL); RmMountEntry *entry = NULL; RmMountEntries *mnt_entries = rm_mount_list_open(self); @@ -838,6 +891,7 @@ void rm_mounts_table_destroy(RmMountTable *self) { g_hash_table_unref(self->nfs_table); g_hash_table_unref(self->evilfs_table); g_hash_table_unref(self->reflinkfs_table); + g_hash_table_unref(self->dedupefs_table); g_slice_free(RmMountTable, self); } @@ -977,6 +1031,25 @@ bool rm_mounts_can_reflink(RmMountTable *self, dev_t source, dev_t dest) { } } +bool rm_mounts_can_dedupe(RmMountTable *self, dev_t source, dev_t dest) { + g_assert(self); + if(g_hash_table_contains(self->dedupefs_table, GUINT_TO_POINTER(source))) { + if(source == dest) { + return true; + } else { + RmPartitionInfo *source_part = + g_hash_table_lookup(self->part_table, GINT_TO_POINTER(source)); + RmPartitionInfo *dest_part = + g_hash_table_lookup(self->part_table, GINT_TO_POINTER(dest)); + g_assert(source_part); + g_assert(dest_part); + return(source_part->disk == dest_part->disk); + } + } else { + return false; + } +} + ///////////////////////////////// // FIEMAP IMPLEMENTATION // ///////////////////////////////// diff --git a/lib/utilities.h b/lib/utilities.h index 2fa39ba1..d52a9ae2 100644 --- a/lib/utilities.h +++ b/lib/utilities.h @@ -352,6 +352,7 @@ typedef struct RmMountTable { GHashTable *nfs_table; GHashTable *evilfs_table; GHashTable *reflinkfs_table; + GHashTable *dedupefs_table; } RmMountTable; /** @@ -409,6 +410,12 @@ bool rm_mounts_is_evil(RmMountTable *self, dev_t to_check); */ bool rm_mounts_can_reflink(RmMountTable *self, dev_t source, dev_t dest); +/** + * @brief Indicates true if source and dest are on same partition, and the + * partition supports the FIDEDUPERANGE ioctl for deduplication. + */ +bool rm_mounts_can_dedupe(RmMountTable *self, dev_t source, dev_t dest); + ///////////////////////////////// // FIEMAP IMPLEMENTATION // ///////////////////////////////// diff --git a/tests/test_mains/test_dedupe.py b/tests/test_mains/test_dedupe.py index 81a3d0b3..3409e0fb 100644 --- a/tests/test_mains/test_dedupe.py +++ b/tests/test_mains/test_dedupe.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from tests.utils import * -def test_equal_files(usual_setup_usual_teardown, needs_reflink_fs): +def test_equal_files(usual_setup_usual_teardown, needs_dedupe_fs): path_a = create_file('1234', 'a') path_b = create_file('1234', 'b') @@ -21,7 +21,7 @@ def test_equal_files(usual_setup_usual_teardown, needs_reflink_fs): with_json=False) -def test_different_files(usual_setup_usual_teardown, needs_reflink_fs): +def test_different_files(usual_setup_usual_teardown, needs_dedupe_fs): path_a = create_file('1234', 'a') path_b = create_file('4321', 'b') @@ -34,7 +34,7 @@ def test_different_files(usual_setup_usual_teardown, needs_reflink_fs): verbosity="") -def test_bad_arguments(usual_setup_usual_teardown, needs_reflink_fs): +def test_bad_arguments(usual_setup_usual_teardown, needs_dedupe_fs): path_a = create_file('1234', 'a') path_b = create_file('1234', 'b') path_c = create_file('1234', 'c') @@ -52,7 +52,7 @@ def test_bad_arguments(usual_setup_usual_teardown, needs_reflink_fs): verbosity="") -def test_directories(usual_setup_usual_teardown, needs_reflink_fs): +def test_directories(usual_setup_usual_teardown, needs_dedupe_fs): path_a = os.path.dirname(create_dirs('dir_a')) path_b = os.path.dirname(create_dirs('dir_b')) @@ -65,7 +65,7 @@ def test_directories(usual_setup_usual_teardown, needs_reflink_fs): verbosity="") -def test_dedupe_works(usual_setup_usual_teardown, needs_reflink_fs): +def test_dedupe_works(usual_setup_usual_teardown, needs_dedupe_fs): # test files need to be larger than btrfs node size to prevent inline extents path_a = create_file('1' * 100000, 'a') @@ -99,7 +99,7 @@ def test_dedupe_works(usual_setup_usual_teardown, needs_reflink_fs): ) -def test_clone_handler(usual_setup_usual_teardown, needs_reflink_fs): +def test_clone_handler(usual_setup_usual_teardown, needs_dedupe_fs): # test files need to be larger than btrfs node size to prevent inline extents path_a = create_file('1' * 100000, 'a') path_b = create_file('1' * 100000, 'b') diff --git a/tests/utils.py b/tests/utils.py index 65cbcfd5..31115689 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,8 +46,8 @@ 'paranoid', ] -_REFLINK_CAPABLE_FILESYSTEMS = {'btrfs', 'xfs', 'ocfs2'} - +_REFLINK_CAPABLE_FILESYSTEMS = {'btrfs', 'xfs', 'ocfs2', 'zfs'} +_DEDUPE_CAPABLE_FILESYSTEMS = {'btrfs', 'xfs', 'ocfs2'} def runs_as_root(): return os.geteuid() == 0 @@ -470,6 +470,19 @@ def is_on_reflink_fs(path): return False +def is_on_dedupe_fs(path): + parts = psutil.disk_partitions(all=True) + + for up_path in _up(path): + for part in parts: + if up_path == part.mountpoint: + print("{0} is {1} mounted at {2}".format(path, part.fstype, part.mountpoint)) + return (part.fstype in _DEDUPE_CAPABLE_FILESYSTEMS) + + print("No mountpoint found for {}".format(path)) + return False + + @pytest.fixture # fixture for tests dependent on reflink-capable testdir def needs_reflink_fs(): @@ -479,6 +492,18 @@ def needs_reflink_fs(): pytest.skip("testdir is not on reflink-capable filesystem") yield + +@pytest.fixture +# fixture for tests dependent on dedupe-capable testdir +def needs_dedupe_fs(): + # TODO: generalise the name to capability instead of a specific fs + if not has_feature('btrfs-support'): + pytest.skip("btrfs not supported") + elif not is_on_dedupe_fs(TESTDIR_NAME): + pytest.skip("testdir is not on dedupe-capable filesystem") + yield + + # count the number of line in a file which start with each pattern def pattern_count(path, patterns): counts = [0] * len(patterns)