Add ZFS filesystem reflink support

.github/workflows/build-and-test.yml

@@ -58,9 +58,9 @@ jobs:
       - name: "Test it"
         if: ${{ env.RUN_TEST == 'true' }}
         run: |
-          RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v
+          RM_TS_PEDANTIC=0 python -m pytest -s -v
 
-      - name: CoW tests
+      - name: CoW tests - BTRFS
         if: ${{ env.RUN_TEST == 'true' }}
         shell: bash
         run: |
@@ -69,22 +69,43 @@ jobs:
           sudo mkfs.btrfs -f /dev/ram0
           sudo mount /dev/ram0 "${RM_TS_DIR}"
           sudo chmod 0777 "${RM_TS_DIR}"
-          cat <<'EOF' >> tests/conftest.py
 
-          def pytest_collection_modifyitems(items, config):
-              selected_items = []
-              deselected_items = []
+          RM_TS_PEDANTIC=0 python -m pytest -s -v tests/test_mains/test_is_reflink.py
+          RM_TS_PEDANTIC=0 python -m pytest -s -v tests/test_mains/test_dedupe.py
+      - name: Build ZFS
+        if: ${{ env.RUN_TEST == 'true' }}
+        shell: bash
+        run: |
+          sudo apt install alien autoconf automake build-essential debhelper-compat dh-autoreconf dh-dkms dh-python dkms fakeroot gawk git libaio-dev libattr1-dev libblkid-dev libcurl4-openssl-dev libelf-dev libffi-dev libpam0g-dev libssl-dev libtirpc-dev libtool libudev-dev linux-headers-generic parallel po-debconf python3 python3-all-dev python3-cffi python3-dev python3-packaging python3-setuptools python3-sphinx uuid-dev zlib1g-dev
+          git clone -b zfs-2.4-release --single-branch https://github.com/openzfs/zfs.git
+          cd ./zfs
+          sh autogen.sh
+          ./configure
+          make -s -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod
+          sudo dpkg -i *.deb
+          # Update order of directories to search for modules, otherwise
+          #   Ubuntu will load kernel-shipped ones.
+          sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
+          sudo depmod
+          sudo modprobe zfs
+          echo "ZFS versions installed:"
+          sudo zfs -V
+          echo "Bclone enabled: $(cat /sys/module/zfs/parameters/zfs_bclone_enabled)"
+      - name: CoW tests - ZFS
+        if: ${{ env.RUN_TEST == 'true' }}
+        shell: bash
+# TODO: change to blkdiscard when the runner will be running 6.10+ kernel.
+        run: |
+          sudo umount "${RM_TS_DIR}"
+          sudo rmmod brd
+          sudo modprobe brd rd_nr=1 rd_size=12582912
+
+          sudo zpool create -m "${RM_TS_DIR}" rmlint ram0
+
+          sudo chmod 0777 "${RM_TS_DIR}"
+
+          RM_TS_PEDANTIC=0 python -m pytest -s -v tests/test_mains/test_is_reflink.py
 
-              for item in items:
-                  if "needs_reflink_fs" in getattr(item, "fixturenames", ()):
-                      selected_items.append(item)
-                  else:
-                      deselected_items.append(item)
-              config.hook.pytest_deselected(items=deselected_items)
-              items[:] = selected_items
-          EOF
-          RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v
-
       - name: "Cleanup"
         if: ${{ env.RUN_TEST == 'true' }}
         run: |

docs/rmlint.1.rst

@@ -664,9 +664,10 @@ FORMATTERS
       This will free up duplicate extents. Needs at least kernel 4.2.
       Use this option when you only have read-only access to a btrfs filesystem but still
       want to deduplicate it. This is usually the case for snapshots.
+      Note: This does not work on ZFS as it doesn't support the FIDEDUPERANGE ioctl.
     * ``reflink``: Try to reflink the duplicate file to the original. See also
       ``--reflink`` in ``man 1 cp``. Fails if the filesystem does not support
-      it.
+      it. Works on btrfs, XFS (with reflink enabled), and ZFS (OpenZFS 2.2+).
     * ``hardlink``: Replace the duplicate file with a hardlink to the original
       file. The resulting files will have the same inode number. Fails if both
       files are not on the same partition. You can use ``ls -i`` to show the
@@ -682,7 +683,8 @@ FORMATTERS
     Default is ``remove``.
 
   * *link*: Shortcut for ``-c sh:handler=clone,reflink,hardlink,symlink``.
-    Use this if you are on a reflink-capable system.
+    Use this if you are on a reflink-capable system. On ZFS, the clone
+    handler will be skipped (as it's not supported), and reflink will be used instead.
   * *hardlink*: Shortcut for ``-c sh:handler=hardlink,symlink``.
     Use this if you want to hardlink files, but want to fallback
     for duplicates that lie on different devices.

lib/formats/sh.c.in

@@ -86,7 +86,7 @@ static bool rm_sh_emit_handler_clone(RmFmtHandlerShScript *self, char **out, RmF
         return false;
     }
 
-    if (!rm_mounts_can_reflink(self->session->mounts, file->dev, self->last_original->dev) ) {
+    if (!rm_mounts_can_dedupe(self->session->mounts, file->dev, self->last_original->dev) ) {
         return false;
     }
 

lib/utilities.c

@@ -579,7 +579,8 @@ static RmMountEntry *rm_mount_list_next(RmMountEntries *self) {
     }
 }
 
-static bool fs_supports_reflinks(char *fstype, char *mountpoint) {
+static bool fs_supports_dedupe(char *fstype, char *mountpoint) {
+    /* Check if filesystem supports FIDEDUPERANGE ioctl for deduplication */
     if(strcmp(fstype, "btrfs")==0) {
         return true;
     }
@@ -596,6 +597,49 @@ static bool fs_supports_reflinks(char *fstype, char *mountpoint) {
     return false;
 }
 
+static bool fs_supports_reflinks(char *fstype, char *mountpoint, char *fsname) {
+    /* Check if filesystem supports any form of reflinks/clones */
+    if(fs_supports_dedupe(fstype, mountpoint)) {
+        /* If it supports dedupe ioctl, it supports reflinks */
+        return true;
+    }
+    if(strcmp(fstype, "zfs")==0) {
+        /* ZFS supports reflinks/clones in newer versions (OpenZFS 2.2+).
+         * Note: ZFS does not support the FIDEDUPERANGE ioctl used by
+         * 'rmlint --dedupe', but does support 'cp --reflink=always'.
+         * For sh:handler=reflink, the generated script will use cp --reflink.
+         * For sh:handler=clone, the dedupe operation will fail gracefully. */
+
+        /* Check that the zpool has block_cloning feature enabled.
+         * Get the pool name from fsname (the part before the first '/', if any).
+         * It can be disabled, or not supported because the pool has not been
+         * upgraded yet, or because ZFS is too old. */
+        char *fs = g_strdup(fsname);
+        g_strdelimit(fs, "/", '\0'); /* modifies 'fs' inplace */
+
+        gchar *argv[] = {"zpool", "get", "-H", "-o", "value", "feature@block_cloning", fs, NULL};
+        gchar *standard_output = NULL;
+        gint exit_status = 0;
+        bool supports_cloning = false;
+
+        if(g_spawn_sync(NULL, argv, NULL, G_SPAWN_SEARCH_PATH | G_SPAWN_STDERR_TO_DEV_NULL,
+                        NULL, NULL, &standard_output, NULL, &exit_status, NULL)) {
+            if(exit_status == 0 && standard_output != NULL) {
+                /* Check if the feature is "active" or "enabled" */
+                if(strstr(standard_output, "active") != NULL ||
+                   strstr(standard_output, "enabled") != NULL) {
+                    supports_cloning = true;
+                }
+            }
+            g_free(standard_output);
+        }
+
+        g_free(fs);
+        return supports_cloning;
+    }
+    return false;
+}
+
 static RmMountEntries *rm_mount_list_open(RmMountTable *table) {
     RmMountEntries *self = g_slice_new(RmMountEntries);
 
@@ -675,13 +719,21 @@ static RmMountEntries *rm_mount_list_open(RmMountTable *table) {
                   evilfs_found->name, wrap_entry->dir, (unsigned)dir_stat.st_dev);
         }
 
-        if(fs_supports_reflinks(wrap_entry->type, wrap_entry->dir)) {
+        if(fs_supports_reflinks(wrap_entry->type, wrap_entry->dir, wrap_entry->fsname)) {
             RmStat dir_stat;
             if(rm_sys_stat(wrap_entry->dir, &dir_stat) == 0) {
                 g_hash_table_insert(table->reflinkfs_table,
                                     GUINT_TO_POINTER(dir_stat.st_dev),
                                     wrap_entry->type);
                 rm_log_debug_line("Filesystem %s: reflink capable", wrap_entry->dir);
+
+                /* Also check if it supports dedupe ioctl */
+                if(fs_supports_dedupe(wrap_entry->type, wrap_entry->dir)) {
+                    g_hash_table_insert(table->dedupefs_table,
+                                        GUINT_TO_POINTER(dir_stat.st_dev),
+                                        wrap_entry->type);
+                    rm_log_debug_line("Filesystem %s: dedupe capable", wrap_entry->dir);
+                }
                 continue;
             }
         }
@@ -712,6 +764,7 @@ static bool rm_mounts_create_tables(RmMountTable *self, bool force_fiemap) {
     /* Mapping dev_t => true (used as set) */
     self->evilfs_table = g_hash_table_new(NULL, NULL);
     self->reflinkfs_table = g_hash_table_new(NULL, NULL);
+    self->dedupefs_table = g_hash_table_new(NULL, NULL);
 
     RmMountEntry *entry = NULL;
     RmMountEntries *mnt_entries = rm_mount_list_open(self);
@@ -838,6 +891,7 @@ void rm_mounts_table_destroy(RmMountTable *self) {
     g_hash_table_unref(self->nfs_table);
     g_hash_table_unref(self->evilfs_table);
     g_hash_table_unref(self->reflinkfs_table);
+    g_hash_table_unref(self->dedupefs_table);
     g_slice_free(RmMountTable, self);
 }
 
@@ -977,6 +1031,25 @@ bool rm_mounts_can_reflink(RmMountTable *self, dev_t source, dev_t dest) {
     }
 }
 
+bool rm_mounts_can_dedupe(RmMountTable *self, dev_t source, dev_t dest) {
+    g_assert(self);
+    if(g_hash_table_contains(self->dedupefs_table, GUINT_TO_POINTER(source))) {
+        if(source == dest) {
+            return true;
+        } else {
+            RmPartitionInfo *source_part =
+                g_hash_table_lookup(self->part_table, GINT_TO_POINTER(source));
+            RmPartitionInfo *dest_part =
+                g_hash_table_lookup(self->part_table, GINT_TO_POINTER(dest));
+            g_assert(source_part);
+            g_assert(dest_part);
+            return(source_part->disk == dest_part->disk);
+        }
+    } else {
+        return false;
+    }
+}
+
 /////////////////////////////////
 //    FIEMAP IMPLEMENTATION     //
 /////////////////////////////////

lib/utilities.h

@@ -352,6 +352,7 @@ typedef struct RmMountTable {
     GHashTable *nfs_table;
     GHashTable *evilfs_table;
     GHashTable *reflinkfs_table;
+    GHashTable *dedupefs_table;
 } RmMountTable;
 
 /**
@@ -409,6 +410,12 @@ bool rm_mounts_is_evil(RmMountTable *self, dev_t to_check);
  */
 bool rm_mounts_can_reflink(RmMountTable *self, dev_t source, dev_t dest);
 
+/**
+ * @brief Indicates true if source and dest are on same partition, and the
+ * partition supports the FIDEDUPERANGE ioctl for deduplication.
+ */
+bool rm_mounts_can_dedupe(RmMountTable *self, dev_t source, dev_t dest);
+
 /////////////////////////////////
 //    FIEMAP IMPLEMENTATION     //
 /////////////////////////////////

tests/test_mains/test_dedupe.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 from tests.utils import *
 
-def test_equal_files(usual_setup_usual_teardown, needs_reflink_fs):
+def test_equal_files(usual_setup_usual_teardown, needs_dedupe_fs):
     path_a = create_file('1234', 'a')
     path_b = create_file('1234', 'b')
 
@@ -21,7 +21,7 @@ def test_equal_files(usual_setup_usual_teardown, needs_reflink_fs):
             with_json=False)
 
 
-def test_different_files(usual_setup_usual_teardown, needs_reflink_fs):
+def test_different_files(usual_setup_usual_teardown, needs_dedupe_fs):
     path_a = create_file('1234', 'a')
     path_b = create_file('4321', 'b')
 
@@ -34,7 +34,7 @@ def test_different_files(usual_setup_usual_teardown, needs_reflink_fs):
             verbosity="")
 
 
-def test_bad_arguments(usual_setup_usual_teardown, needs_reflink_fs):
+def test_bad_arguments(usual_setup_usual_teardown, needs_dedupe_fs):
     path_a = create_file('1234', 'a')
     path_b = create_file('1234', 'b')
     path_c = create_file('1234', 'c')
@@ -52,7 +52,7 @@ def test_bad_arguments(usual_setup_usual_teardown, needs_reflink_fs):
                 verbosity="")
 
 
-def test_directories(usual_setup_usual_teardown, needs_reflink_fs):
+def test_directories(usual_setup_usual_teardown, needs_dedupe_fs):
     path_a = os.path.dirname(create_dirs('dir_a'))
     path_b = os.path.dirname(create_dirs('dir_b'))
 
@@ -65,7 +65,7 @@ def test_directories(usual_setup_usual_teardown, needs_reflink_fs):
             verbosity="")
 
 
-def test_dedupe_works(usual_setup_usual_teardown, needs_reflink_fs):
+def test_dedupe_works(usual_setup_usual_teardown, needs_dedupe_fs):
 
     # test files need to be larger than btrfs node size to prevent inline extents
     path_a = create_file('1' * 100000, 'a')
@@ -99,7 +99,7 @@ def test_dedupe_works(usual_setup_usual_teardown, needs_reflink_fs):
         )
 
 
-def test_clone_handler(usual_setup_usual_teardown, needs_reflink_fs):
+def test_clone_handler(usual_setup_usual_teardown, needs_dedupe_fs):
     # test files need to be larger than btrfs node size to prevent inline extents
     path_a = create_file('1' * 100000, 'a')
     path_b = create_file('1' * 100000, 'b')

tests/utils.py

@@ -46,8 +46,8 @@
     'paranoid',
 ]
 
-_REFLINK_CAPABLE_FILESYSTEMS = {'btrfs', 'xfs', 'ocfs2'}
-
+_REFLINK_CAPABLE_FILESYSTEMS = {'btrfs', 'xfs', 'ocfs2', 'zfs'}
+_DEDUPE_CAPABLE_FILESYSTEMS = {'btrfs', 'xfs', 'ocfs2'}
 
 def runs_as_root():
     return os.geteuid() == 0
@@ -470,6 +470,19 @@ def is_on_reflink_fs(path):
     return False
 
 
+def is_on_dedupe_fs(path):
+    parts = psutil.disk_partitions(all=True)
+
+    for up_path in _up(path):
+        for part in parts:
+            if up_path == part.mountpoint:
+                print("{0} is {1} mounted at {2}".format(path, part.fstype, part.mountpoint))
+                return (part.fstype in _DEDUPE_CAPABLE_FILESYSTEMS)
+
+    print("No mountpoint found for {}".format(path))
+    return False
+
+
 @pytest.fixture
 # fixture for tests dependent on reflink-capable testdir
 def needs_reflink_fs():
@@ -479,6 +492,18 @@ def needs_reflink_fs():
         pytest.skip("testdir is not on reflink-capable filesystem")
     yield
 
+
+@pytest.fixture
+# fixture for tests dependent on dedupe-capable testdir
+def needs_dedupe_fs():
+    # TODO: generalise the name to capability instead of a specific fs
+    if not has_feature('btrfs-support'):
+        pytest.skip("btrfs not supported")
+    elif not is_on_dedupe_fs(TESTDIR_NAME):
+        pytest.skip("testdir is not on dedupe-capable filesystem")
+    yield
+
+
 # count the number of line in a file which start with each pattern
 def pattern_count(path, patterns):
     counts = [0] * len(patterns)