PDC integration: one downloader for both data commons

.github/scripts/assert-public-image.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Fail unless a Quay repository is anonymously readable.
+#
+# Quay creates a repository on first push and makes it PRIVATE by default. A
+# private GaCDI image is not a build failure -- the push succeeds and the tag is
+# there -- but every Galaxy job that requires the container then dies at pull
+# time with an authentication error that says nothing about visibility. This
+# check turns that into an immediate, explanatory CI failure the first time an
+# image is published.
+#
+# Visibility is queried without credentials on purpose: the workflow is logged in
+# to Quay, so any authenticated check would pass on a private repository and
+# prove nothing about what Galaxy can pull.
+set -euo pipefail
+
+ORG="${1:?usage: assert-public-image.sh ORG IMAGE}"
+IMAGE="${2:?usage: assert-public-image.sh ORG IMAGE}"
+# Overridable so the branches can be exercised against a local fixture server.
+API="${QUAY_API_BASE:-https://quay.io/api/v1}/repository/${ORG}/${IMAGE}"
+
+response="$(curl -sS -o /tmp/quay-repo.json -w '%{http_code}' "$API" || true)"
+is_public="$(jq -r 'if .is_public == null then "unknown" else (.is_public | tostring) end' /tmp/quay-repo.json 2>/dev/null || echo unknown)"
+
+if [[ "$response" == "200" && "$is_public" == "true" ]]; then
+  echo "quay.io/${ORG}/${IMAGE} is public."
+  exit 0
+fi
+
+cat >&2 <<EOF
+quay.io/${ORG}/${IMAGE} is not anonymously readable (HTTP ${response}, is_public=${is_public}).
+
+The image was pushed, but Galaxy cannot pull it. Quay makes a repository private
+when it is first created, so this is expected on a brand-new image and must be
+changed once by hand:
+
+  https://quay.io/repository/${ORG}/${IMAGE}?tab=settings
+  -> Repository Visibility -> Make Public
+
+Re-run this workflow afterwards to confirm.
+EOF
+exit 1

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [main, manifest_tool]
+    branches: [main, manifest_tool, PD_integration]
   pull_request:
 
 jobs:
@@ -36,4 +36,4 @@ jobs:
       - name: Install planemo
         run: python -m pip install planemo
       - name: Lint
-        run: planemo lint --skip citations tools/manifest_gdc
+        run: planemo lint tools/manifest_gdc tools/gacdi-downloader

.github/workflows/containers.yml

@@ -1,61 +1,106 @@
-name: Build and push manifest container
+name: Build and push GaCDI containers
 
-# Builds the manifest-builder image and pushes it to Quay whenever
-# cli_tools/gacdi_manifest changes on main, or manually. The image tag is
-# always taken from gacdi_manifest/__init__.py's __version__ (the single
-# source of truth for versioning; pyproject.toml's version is derived from it
-# via hatchling) and the job refuses to overwrite a tag that already exists on
-# Quay, so a forgotten version bump fails the build instead of silently
-# clobbering the published image. Requires repository secrets QUAY_USERNAME
-# and QUAY_TOKEN (Quay robot account).
 on:
   push:
     branches: ["main"]
     paths:
       - "cli_tools/gacdi_manifest/**"
+      - "tools/gacdi-downloader/**"
+      - ".github/workflows/containers.yml"
   workflow_dispatch:
 
 env:
   REGISTRY: quay.io
-  # Must match the <container> namespace in tools/manifest_gdc/macros.xml.
   ORG: goeckslab
-  IMAGE: gacdi-manifest
 
 jobs:
-  manifest:
+  changes:
     runs-on: ubuntu-latest
+    outputs:
+      manifest: ${{ steps.filter.outputs.manifest }}
+      downloader: ${{ steps.filter.outputs.downloader }}
     steps:
       - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            manifest:
+              - 'cli_tools/gacdi_manifest/**'
+            downloader:
+              - 'cli_tools/gacdi_manifest/**'
+              - 'tools/gacdi-downloader/**'
 
-      - name: Read version from package __init__.py
+  manifest:
+    needs: changes
+    if: github.event_name == 'workflow_dispatch' || needs.changes.outputs.manifest == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Read version from package
         id: version
         run: |
           VERSION=$(grep -m1 '^__version__' cli_tools/gacdi_manifest/gacdi_manifest/__init__.py | sed -E 's/__version__ = "(.*)"/\1/')
           echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
-
       - uses: docker/setup-buildx-action@v3
-
       - uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_TOKEN }}
-
       - name: Refuse to overwrite an existing tag
         run: |
-          IMAGE_REF="${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }}:${{ steps.version.outputs.version }}"
+          IMAGE_REF="${{ env.REGISTRY }}/${{ env.ORG }}/gacdi-manifest:${{ steps.version.outputs.version }}"
           if docker buildx imagetools inspect "$IMAGE_REF" >/dev/null 2>&1; then
             echo "Tag ${{ steps.version.outputs.version }} already exists at $IMAGE_REF." >&2
             echo "Bump __version__ in gacdi_manifest/__init__.py before pushing again." >&2
             exit 1
           fi
-
-      - name: Build & push
+      - name: Build and push manifest image
         uses: docker/build-push-action@v6
         with:
           context: cli_tools/gacdi_manifest
           file: cli_tools/gacdi_manifest/Dockerfile
           push: true
           build-args: |
             GACDI_BUILD=${{ github.sha }}
-          tags: ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }}:${{ steps.version.outputs.version }}
+          tags: ${{ env.REGISTRY }}/${{ env.ORG }}/gacdi-manifest:${{ steps.version.outputs.version }}
+      - name: Verify the image is publicly pullable
+        run: .github/scripts/assert-public-image.sh "${{ env.ORG }}" gacdi-manifest
+
+  downloader:
+    needs: changes
+    if: github.event_name == 'workflow_dispatch' || needs.changes.outputs.downloader == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Read version from package
+        id: version
+        run: |
+          VERSION=$(grep -m1 '^__version__' cli_tools/gacdi_manifest/gacdi_manifest/__init__.py | sed -E 's/__version__ = "(.*)"/\1/')
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_TOKEN }}
+      - name: Refuse to overwrite an existing tag
+        run: |
+          IMAGE_REF="${{ env.REGISTRY }}/${{ env.ORG }}/gacdi-downloader:${{ steps.version.outputs.version }}"
+          if docker buildx imagetools inspect "$IMAGE_REF" >/dev/null 2>&1; then
+            echo "Tag ${{ steps.version.outputs.version }} already exists at $IMAGE_REF." >&2
+            echo "Bump __version__ in gacdi_manifest/__init__.py before pushing again." >&2
+            exit 1
+          fi
+      - name: Build and push downloader image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: tools/gacdi-downloader/dependencies/Dockerfile
+          push: true
+          build-args: |
+            GACDI_BUILD=${{ github.sha }}
+          tags: ${{ env.REGISTRY }}/${{ env.ORG }}/gacdi-downloader:${{ steps.version.outputs.version }}
+      - name: Verify the image is publicly pullable
+        run: .github/scripts/assert-public-image.sh "${{ env.ORG }}" gacdi-downloader

cli_tools/gacdi_manifest/README.md

@@ -55,23 +55,60 @@ repeatable custom facets (`--extra-filter "field=…;op=in|exclude;values=a,b"`)
 a raw GDC filters JSON (`--raw-filters`). The manifest is emitted in a deterministic
 (sorted) order for reproducible workflows.
 
-### End-to-end with the GaCDI GDC importer
+## Data Commons Downloader
 
-This tool is designed to feed directly into the **GaCDI GDC importer** (the
-manifest-download branch), so a single Galaxy workflow goes *filter → manifest →
-download → analysis*:
+The same package provides `gacdi-download`, the runtime behind the Galaxy **GaCDI
+Data Commons Downloader**. It accepts either a GDC TSV manifest or a PDC file
+manifest in CSV/TSV form and detects the source from the first non-blank header.
+
+```bash
+gacdi-download --manifest portal_manifest.tsv --outdir downloads
+```
+
+Detection is intentionally strict:
+
+- GDC requires `id`, `filename`, `md5`, and `size` (the portal also emits `state`).
+- PDC is recognized from multiple PDC-specific file-manifest columns such as
+  `File ID`, `File Name`, `PDC Study ID`, `Md5sum`, and `File Download Link`.
+- A header that matches both or neither format is rejected, and the error lists
+  the columns that were actually observed.
+
+For GDC, the command invokes `gdc-client`. If `GDC_AUTH_TOKEN` is non-empty it is
+passed through a mode-0600 FIFO, so the token is never written to a regular file
+or exposed in process arguments. For PDC, files are streamed from each `File
+Download Link` and checked against `File Size (in bytes)` and `Md5sum`. Existing
+files with the expected MD5 are skipped. Partial or corrupt downloads are removed.
+
+PDC download links expire after seven days, and PDC limits repeated downloads of
+one file from an IP address to 10 attempts per 24 hours. The downloader reports
+both conditions with actionable messages. Re-export an expired file manifest from
+PDC **Explore → Files → Export File Manifest**.
+
+Expected command exit codes are stable for Galaxy and workflow callers:
+
+| Code | Meaning |
+| ---: | --- |
+| 0 | Success (including a valid header-only manifest) |
+| 1 | Other expected manifest-package failure |
+| 2 | Invalid, unknown, or ambiguous input manifest |
+| 4 | Remote API failure while building a manifest |
+| 5 | GDC/PDC transfer or integrity-check failure |
+
+### End-to-end with the GaCDI downloader
+
+This tool is designed to feed directly into the **GaCDI Data Commons Downloader**,
+so a single Galaxy workflow goes *filter → manifest → download → analysis*:
 
 ```
-[GaCDI Manifest Builder]--gdc_manifest.txt-->[GaCDI GDC importer]--collection-->[analysis tools]
+[GaCDI Manifest Builder]--gdc_manifest.txt-->[GaCDI Data Commons Downloader]--collection-->[analysis tools]
                         \--metadata.tsv------------------------------(join)----/
 ```
 
 **Compatibility contract (locked by `tests/test_importer_contract.py`):**
 
-1. **Manifest → importer.** `gdc_manifest.txt` is a TSV whose header
-   (`id, filename, md5, size, state`) is a superset of what the importer's
-   `parse_gdc_manifest` requires (`id/filename/md5/size`); its datatype (`txt`)
-   is accepted by the importer's manifest input (`tabular,txt`). Rows with no
+1. **Manifest → downloader.** `gdc_manifest.txt` is a TSV whose header
+   (`id, filename, md5, size, state`) satisfies the downloader's detection rule;
+   its datatype (`txt`) is accepted by the manifest input (`txt,tabular,csv`). Rows with no
    `id` are dropped so the manifest and metadata stay row-aligned. The same file
    also works with `gdc-client download -m gdc_manifest.txt`.
 2. **Metadata ↔ history.** `metadata.tsv` leads with `file_id` and `filename` —
@@ -86,31 +123,31 @@ annotations (e.g. labels for an image ML model).
 
 ## Runtime environment
 
-The tool ships a pinned container (`quay.io/<org>/gacdi-manifest`) referenced from
-the wrapper, with Python + `requests` Conda requirements as a fallback. The Quay
-namespace (`paulocilasjr`) is a placeholder — update `@QUAY_ORG@` in
-`tools/manifest_gdc/macros.xml`, `containers/Dockerfile.manifest`, and the workflow
-before publishing.
+The manifest builder and downloader ship pinned containers in the `goeckslab`
+Quay namespace. The downloader image combines the Python package with the pinned
+GDC Data Transfer Tool 2.3 binary.
 
 ```bash
-docker build -f containers/Dockerfile.manifest -t gacdi-manifest:dev .
+docker build -f cli_tools/gacdi_manifest/Dockerfile -t gacdi-manifest:dev cli_tools/gacdi_manifest
 docker run --rm gacdi-manifest:dev gacdi-manifest gdc --help
+docker build -f tools/gacdi-downloader/dependencies/Dockerfile -t gacdi-downloader:dev .
+docker run --rm gacdi-downloader:dev gacdi-download --help
 ```
 
 ## Development
 
 ```bash
-python -m pip install -e '.[dev]'
+python -m pip install -e 'cli_tools/gacdi_manifest[dev]'
 pytest -q                            # mocked; no network
-planemo lint tools/manifest_gdc
+planemo lint tools/manifest_gdc tools/gacdi-downloader
 ```
 
 ## Roadmap
 
-- **Phase 1 (this branch):** GDC manifest builder + enrichment + join/QC.
-- **Phase 2:** CRDC GDC-style commons (PDC/IDC/ICDC/CDS/CTDC) reusing the filter/
-  join core.
-- **Phase 3:** GEO/SRA accession-list builders; on merge with the importer branch,
+- **Phase 1:** GDC manifest builder + enrichment + join/QC.
+- **Phase 2 (current):** Unified GDC/PDC manifest downloader.
+- **Phase 3:** Additional CRDC commons (IDC/ICDC/CDS/CTDC) reusing the filter/join core.
+- **Phase 4:** GEO/SRA accession-list builders; on merge with the importer branch,
   fold shared HTTP utilities into the `gacdi` package.
 
 ## License

cli_tools/gacdi_manifest/gacdi_manifest/__init__.py

@@ -13,7 +13,7 @@
 
 import os
 
-__version__ = "0.1.1"
+__version__ = "0.3.0"
 
 # Build identifier baked into the container image at build time (e.g. the git
 # commit SHA). Lets you confirm the exact code a run used, even when the version

cli_tools/gacdi_manifest/gacdi_manifest/download/__init__.py

@@ -0,0 +1,5 @@
+"""Manifest detection and download backends for GDC and PDC."""
+
+from .detect import detect_source
+
+__all__ = ["detect_source"]

cli_tools/gacdi_manifest/gacdi_manifest/download/cli.py

@@ -0,0 +1,61 @@
+"""Command-line dispatcher for GDC and PDC download manifests."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+
+from .. import version_string
+from ..errors import ManifestError
+from .detect import detect_source
+from .gdc import download_gdc
+from .pdc import download_pdc
+
+log = logging.getLogger("gacdi_manifest.download")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="gacdi-download",
+        description="Download files from an auto-detected GDC or PDC file manifest.",
+    )
+    parser.add_argument("--version", action="version", version=f"gacdi-download {version_string()}")
+    parser.add_argument("--manifest", required=True, help="GDC or PDC manifest (CSV or TSV).")
+    parser.add_argument("--outdir", required=True, help="Directory in which to place downloaded files.")
+    parser.add_argument(
+        "--keep-compressed",
+        action="store_true",
+        help=(
+            "Leave gzipped text and XML payloads compressed. By default they are expanded "
+            "after checksum verification so that formats such as mzML and mzIdentML are "
+            "directly usable by downstream tools. Formats that are compressed by design "
+            "(BAM, BGZF VCF, tabix indexes) are never expanded either way."
+        ),
+    )
+    parser.add_argument("--verbose", action="store_true")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(levelname)s %(name)s: %(message)s",
+    )
+    log.info("gacdi-download %s", version_string())
+    try:
+        source = detect_source(args.manifest)
+        log.info("Detected %s manifest.", source.upper())
+        decompress = not args.keep_compressed
+        if source == "gdc":
+            return download_gdc(args.manifest, args.outdir, decompress=decompress)
+        download_pdc(args.manifest, args.outdir, decompress=decompress)
+        return 0
+    except ManifestError as exc:
+        log.error("%s", exc)
+        return exc.exit_code
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())