feat(cli): add 'check files' command with optimized git grouping (#163)

sushant-suse · web-flow · commit 2d2078f52a90 · 2026-02-12T12:03:07.000+05:30
* feat(cli): add 'check files' command with optimized git grouping (#78) Signed-off-by: sushant-suse <sushant.gaurav@suse.com> * feat(check): implement doctype filtering and pinpoint missing files (#78) Signed-off-by: sushant-suse <sushant.gaurav@suse.com> * refactor(check): use Deliverable models and implement contextual error reporting (#78) Signed-off-by: sushant-suse <sushant.gaurav@suse.com> * refactor(check): added expanduser() (#78) Signed-off-by: sushant-suse <sushant.gaurav@suse.com> * feat(check): add success message and finalize path handling (#78) Signed-off-by: sushant-suse <sushant.gaurav@suse.com> --------- Signed-off-by: sushant-suse <sushant.gaurav@suse.com>
diff --git a/changelog.d/78.feature.rst b/changelog.d/78.feature.rst
@@ -0,0 +1 @@
+Added ``check files`` subcommand to verify that all DC files defined in XML configurations exist in their respective remote repositories. The implementation includes an optimization to group checks by repository, significantly reducing Git network overhead.
diff --git a/src/docbuild/cli/cmd_check/__init__.py b/src/docbuild/cli/cmd_check/__init__.py
@@ -0,0 +1,37 @@
+import asyncio
+from typing import TYPE_CHECKING
+
+import click
+
+from docbuild.cli.context import DocBuildContext
+
+# Import Doctype for the type hint
+from docbuild.models.doctype import Doctype
+
+from ..callback import validate_doctypes
+from .process import process_check_files
+
+
+@click.group(name="check")
+def cmd_check() -> None:
+    """Check the environment or configuration for consistency."""
+    pass
+
+
+@cmd_check.command(name="files")
+@click.argument("doctypes",
+   nargs=-1,
+   callback=validate_doctypes,
+)
+@click.pass_obj
+def check_files(ctx: DocBuildContext, doctypes: tuple[Doctype, ...]) -> None:
+    """Verify that DC files exist. Optional: specify 'product/version/lang'."""
+    # Execute the logic via asyncio, passing the optional doctype filter
+    # doctypes is a tuple of Doctype objects here
+    missing: list[str] = asyncio.run(process_check_files(ctx, doctypes))
+
+    if missing:
+        missing_str = "\n- ".join(str(f) for f in missing if f)
+        raise click.ClickException(
+            f"DC file verification failed. The following files are missing:\n- {missing_str}"
+        )
diff --git a/src/docbuild/cli/cmd_check/process.py b/src/docbuild/cli/cmd_check/process.py
@@ -0,0 +1,100 @@
+"""Logic for checking DC file availability in remote repositories."""
+
+from collections.abc import Sequence
+import logging
+from pathlib import Path
+from typing import cast
+
+from docbuild.cli.cmd_metadata.metaprocess import get_deliverable_from_doctype
+from docbuild.cli.context import DocBuildContext
+from docbuild.config.xml.stitch import create_stitchfile
+from docbuild.constants import DEFAULT_DELIVERABLES
+from docbuild.models.config.env import EnvConfig
+from docbuild.models.deliverable import Deliverable
+from docbuild.models.doctype import Doctype
+from docbuild.utils.git import ManagedGitRepo
+
+log = logging.getLogger(__name__)
+
+
+async def _verify_repository_files(
+    repo_url: str,
+    branch: str,
+    deliverables: list[Deliverable],
+    repo_root: Path,
+) -> list[str]:
+    """Clone a repository and check for the existence of specific DC files."""
+    missing = []
+    # Use the first deliverable to get the repo's abbreviated name (surl)
+    repo_surl = deliverables[0].git.surl
+    log.info(f"Checking Repo: {repo_surl} [{branch}]")
+
+    repo_handler = ManagedGitRepo(repo_url, repo_root)
+
+    if not await repo_handler.clone_bare():
+        log.error(f"Repository inaccessible: {repo_surl}")
+        for d in deliverables:
+            # Format: [repo] product/version/lang:file
+            missing.append(f"[{repo_surl}] {d.productid}/{d.docsetid}/{d.lang}:{d.dcfile}")
+        return missing
+
+    available_files = await repo_handler.ls_tree(branch)
+    for d in deliverables:
+        display_name = f"[{repo_surl}] {d.productid}/{d.docsetid}/{d.lang}:{d.dcfile}"
+        if d.dcfile in available_files:
+            log.info(f"Found: {display_name}")
+        else:
+            log.error(f"Missing: {display_name}")
+            missing.append(display_name)
+    return missing
+
+
+async def process_check_files(
+    ctx: DocBuildContext,
+    doctypes: Sequence[Doctype] | None
+) -> list[str]:
+    """Verify DC file existence using official Deliverable models."""
+    log.info("Starting DC file availability check...")
+
+    env_config = cast(EnvConfig, ctx.envconfig)
+    config_dir = env_config.paths.config_dir.expanduser()
+    repo_root = env_config.paths.repo_dir.expanduser()
+
+    # 1. Prepare XML and Stitch Tree
+    xml_files = list(config_dir.rglob("*.xml"))
+    if not xml_files:
+        log.warning(f"No XML files found in {config_dir}")
+        return []
+
+    stitch_tree = await create_stitchfile(xml_files)
+
+    # 2. Identify target doctypes (use defaults if none provided)
+    if not doctypes:
+        doctypes = [Doctype.from_str(DEFAULT_DELIVERABLES)]
+
+    # 3. Use official logic to extract Deliverable objects
+    all_deliverables: list[Deliverable] = []
+    for dt in doctypes:
+        all_deliverables.extend(get_deliverable_from_doctype(stitch_tree, dt))
+
+    if not all_deliverables:
+        log.error("No deliverables found for the selected doctypes.")
+        return []
+
+    # 4. Group by repository and branch to optimize network calls
+    groups: dict[tuple[str, str], list[Deliverable]] = {}
+    for d in all_deliverables:
+        key = (d.git.url, d.branch or "main")
+        groups.setdefault(key, []).append(d)
+
+    # 5. Verification Loop
+    missing_files: list[str] = []
+    for (url, branch), deli_list in groups.items():
+        results = await _verify_repository_files(url, branch, deli_list, repo_root)
+        missing_files.extend(results)
+
+    # Final success message if no files were missing
+    if not missing_files:
+        log.info("All DC files are available in remote repositories.")
+
+    return missing_files
diff --git a/src/docbuild/cli/cmd_cli.py b/src/docbuild/cli/cmd_cli.py
@@ -26,6 +26,7 @@
 from ..utils.pidlock import LockAcquisitionError, PidFileLock
 from .cmd_build import build
 from .cmd_c14n import c14n
+from .cmd_check import cmd_check
 from .cmd_config import config
 from .cmd_metadata import metadata
 from .cmd_repo import repo
@@ -228,3 +229,4 @@ def cli(
 cli.add_command(repo)
 cli.add_command(metadata)
 cli.add_command(validate)
+cli.add_command(cmd_check)
diff --git a/src/docbuild/utils/git.py b/src/docbuild/utils/git.py
@@ -199,3 +199,43 @@ async def fetch_updates(self: Self) -> bool:
         except RuntimeError as e:
             log.error("Failed to fetch updates for '%s': %s", self.slug, e)
             return False
+
+    async def ls_tree(self: Self, branch: str, recursive: bool = True) -> list[str]:
+        """List all files in a specific branch of the bare repository.
+
+        :param branch: The branch name to inspect.
+        :param recursive: Whether to list files in subdirectories.
+        :return: A list of file paths found in the branch.
+        """
+        if not self.bare_repo_path.exists():
+            log.warning(
+                "Cannot run ls-tree: Bare repository does not exist at %s",
+                self.bare_repo_path,
+            )
+            return []
+
+        args = ["ls-tree", "--name-only"]
+        if recursive:
+            args.append("-r")
+        args.append(branch)
+
+        try:
+            # We use execute_git_command which already handles the 'git' prefix
+            # and uses the bare_repo_path as the current working directory.
+            self.result = await execute_git_command(
+                *args,
+                cwd=self.bare_repo_path,
+                gitconfig=self._gitconfig,
+            )
+
+            if not self.result.stdout:
+                return []
+
+            return self.result.stdout.strip().splitlines()
+
+        except RuntimeError as e:
+            log.error(
+                "Failed to run ls-tree on branch '%s' in '%s': %s",
+                branch, self.slug, e
+            )
+            return []
diff --git a/tests/cli/cmd_check/__init__.py b/tests/cli/cmd_check/__init__.py
diff --git a/tests/cli/cmd_check/test_process.py b/tests/cli/cmd_check/test_process.py
@@ -0,0 +1,133 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from docbuild.cli.cmd_check import process
+from docbuild.cli.cmd_check.process import process_check_files
+
+
+@pytest.fixture
+def mock_deliverable():
+    """Create a mock Deliverable object with required attributes."""
+    deli = MagicMock()
+    deli.productid = "sles"
+    deli.docsetid = "16.0"
+    deli.lang = "en-us"
+    deli.dcfile = "README.md"
+    deli.branch = "main"
+    deli.git.url = "https://github.com/org/repo.git"
+    deli.git.surl = "gh://org/repo"
+    return deli
+
+@patch.object(process, "create_stitchfile", new_callable=AsyncMock)
+@patch.object(process, "get_deliverable_from_doctype")
+@patch.object(process, "ManagedGitRepo")
+async def test_process_check_files_all_found(
+    mock_repo_class, mock_get_deli, mock_stitch, tmp_path, mock_deliverable
+):
+    """Test full process when all files exist in the repo."""
+    # 1. Setup mocks
+    mock_stitch.return_value = MagicMock()
+    mock_get_deli.return_value = [mock_deliverable]
+
+    mock_repo = AsyncMock()
+    mock_repo.clone_bare.return_value = True
+    mock_repo.ls_tree.return_value = ["README.md"]
+    mock_repo_class.return_value = mock_repo
+
+    # 2. Setup Context
+    ctx = MagicMock()
+    config_dir = tmp_path / "config.d"
+    config_dir.mkdir()
+    (config_dir / "test.xml").write_text("<xml/>")
+    ctx.envconfig.paths.config_dir = config_dir
+    ctx.envconfig.paths.repo_dir = tmp_path / "repos"
+
+    # 3. Execute
+    result = await process_check_files(ctx, doctypes=None)
+
+    # 4. Assertions
+    assert result == []
+    mock_repo.clone_bare.assert_called_once()
+    mock_repo.ls_tree.assert_called_with("main")
+
+@patch.object(process, "create_stitchfile", new_callable=AsyncMock)
+@patch.object(process, "get_deliverable_from_doctype")
+@patch.object(process, "ManagedGitRepo")
+async def test_process_check_files_missing(
+    mock_repo_class, mock_get_deli, mock_stitch, tmp_path, mock_deliverable
+):
+    """Test full process when a file is missing in the repo."""
+    mock_stitch.return_value = MagicMock()
+    mock_get_deli.return_value = [mock_deliverable]
+
+    mock_repo = AsyncMock()
+    mock_repo.clone_bare.return_value = True
+    mock_repo.ls_tree.return_value = ["LICENSE"] # README.md is missing
+    mock_repo_class.return_value = mock_repo
+
+    ctx = MagicMock()
+    config_dir = tmp_path / "config.d"
+    config_dir.mkdir()
+    (config_dir / "test.xml").write_text("<xml/>")
+    ctx.envconfig.paths.config_dir = config_dir
+    ctx.envconfig.paths.repo_dir = tmp_path / "repos"
+
+    result = await process_check_files(ctx, doctypes=None)
+
+    # Check for the new formatted string output
+    expected_error = "[gh://org/repo] sles/16.0/en-us:README.md"
+    assert expected_error in result
+
+@patch.object(process, "create_stitchfile", new_callable=AsyncMock)
+@patch.object(process, "get_deliverable_from_doctype")
+@patch.object(process, "ManagedGitRepo")
+async def test_process_git_failure(
+    mock_repo_class, mock_get_deli, mock_stitch, tmp_path, mock_deliverable
+):
+    """Test coverage for the branch where Git cloning/fetching fails."""
+    mock_stitch.return_value = MagicMock()
+    mock_get_deli.return_value = [mock_deliverable]
+
+    mock_repo = AsyncMock()
+    mock_repo.clone_bare.return_value = False # Simulate failure
+    mock_repo_class.return_value = mock_repo
+
+    ctx = MagicMock()
+    config_dir = tmp_path / "config.d"
+    config_dir.mkdir()
+    (config_dir / "test.xml").write_text("<xml/>")
+    ctx.envconfig.paths.config_dir = config_dir
+    ctx.envconfig.paths.repo_dir = tmp_path / "repos"
+
+    result = await process_check_files(ctx, doctypes=None)
+
+    expected_error = "[gh://org/repo] sles/16.0/en-us:README.md"
+    assert expected_error in result
+
+@patch.object(process, "create_stitchfile", new_callable=AsyncMock)
+@patch.object(process, "get_deliverable_from_doctype")
+async def test_process_no_deliverables_found(mock_get_deli, mock_stitch, tmp_path):
+    """Test path where stitch tree returns no deliverables."""
+    mock_stitch.return_value = MagicMock()
+    mock_get_deli.return_value = [] # No deliverables
+
+    ctx = MagicMock()
+    config_dir = tmp_path / "config.d"
+    config_dir.mkdir()
+    (config_dir / "test.xml").write_text("<xml/>")
+    ctx.envconfig.paths.config_dir = config_dir
+
+    result = await process_check_files(ctx, doctypes=None)
+    assert result == []
+
+@patch.object(process, "create_stitchfile", new_callable=AsyncMock)
+async def test_process_no_xml_files(mock_stitch, tmp_path):
+    """Verify behavior when no XML files are present at all."""
+    ctx = MagicMock()
+    config_dir = tmp_path / "empty"
+    config_dir.mkdir()
+    ctx.envconfig.paths.config_dir = config_dir
+
+    result = await process_check_files(ctx, doctypes=None)
+    assert result == []

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Added ``check files`` subcommand to verify that all DC files defined in XML configurations exist in their respective remote repositories. The implementation includes an optimization to group checks by repository, significantly reducing Git network overhead.