add libzstd-dev to dockerfile

.github/workflows/README.md

@@ -0,0 +1,53 @@
+# GitHub Actions Workflows
+
+This directory contains the GitHub Actions workflows used by the LDZip repository.
+
+## Workflows
+
+### `ci.yaml`
+
+Runs the LDZip automated tests on:
+
+- Pushes to `main`
+- Pull requests targeting `main`
+
+All jobs run inside the shared Docker image:
+
+```text
+ghcr.io/23andme/ldzip/ldzip-ci:latest
+```
+
+### `build-image.yaml`
+
+Builds and publishes the CI Docker image to GitHub Container Registry (GHCR):
+
+```text
+ghcr.io/23andme/ldzip/ldzip-ci:latest
+```
+
+This workflow is currently disabled (via `if: false` in the `build-and-push` job) because the GitHub organization does not allow the third-party Docker GitHub Actions it requires. Until the workflow is enabled, the CI Docker image must be built and published manually.
+
+## Manually Updating the CI Docker Image
+
+Run the following commands from the root of the LDZip repository (where `Dockerfile.ci` is located):
+
+```bash
+docker login ghcr.io -u USERNAME
+
+docker buildx build \
+  --platform linux/amd64 \
+  -f Dockerfile.ci \
+  -t ghcr.io/23andme/ldzip/ldzip-ci:latest \
+  .
+
+docker images | grep ldzip-ci
+
+docker push ghcr.io/23andme/ldzip/ldzip-ci:latest
+```
+
+### Notes
+
+- Replace `USERNAME` with your GitHub username.
+- Authenticate using a GitHub Personal Access Token when prompted for your password.
+- The token must have permission to publish packages to the `23andMe` organization.
+- After the image is pushed, subsequent runs of `ci.yaml` will automatically use the updated image.

.github/workflows/build-image.yaml

@@ -21,6 +21,7 @@ concurrency:
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
+    if: false  # Disabled - set to true to re-enable
     steps:
       - name: Checkout
         uses: actions/checkout@v4

Dockerfile.ci

@@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
     git \
     openjdk-17-jre-headless \
+    libzstd-dev \
   && rm -rf /var/lib/apt/lists/*
 
 # ---- Install Miniforge (conda-forge) ----
@@ -43,6 +44,8 @@ RUN mamba create -y -n ci-env -c conda-forge -c bioconda \
       r-rcpp \
       r-dbi \
       r-rsqlite \
+      numpy \
+      scipy \
  && conda clean -a -y
 
 ENV CONDA_DEFAULT_ENV=ci-env
@@ -55,5 +58,6 @@ RUN nextflow -version
 RUN nf-test version || nf-test --help >/dev/null
 
 RUN Rscript -e "library(argparse); library(jsonlite); library(Matrix); library(tinytest); library(Rcpp); library(DBI); library(RSQLite); cat('R deps OK\n')"
+RUN python3 -c "import numpy; import scipy; print('Python deps OK')"
 
 WORKDIR /work

pipelines/wholeGenomeLD/Makefile

@@ -1,6 +1,6 @@
-
+export NXF_SYNTAX_PARSER=v1
 
 test:
-	nf-test clean
-	nf-test test tests/stub.nf.test --verbose
-	nf-test test tests/unit.nf.test --verbose
+	rm -rf .nf-test .nf-test.log
+	CONSTRAIN_MEMORY=1 nf-test test tests/stub.nf.test --verbose
+	CONSTRAIN_MEMORY=1 nf-test test tests/unit.nf.test --verbose

pipelines/wholeGenomeLD/main.nf

@@ -156,7 +156,7 @@ process compressLD {
 
 process convertNpzToBinary {
     tag { "chr${chr}-chunk${chunk_id}" }
-    memory { 16.GB * task.attempt }
+    memory { System.getenv('CONSTRAIN_MEMORY') ? 8.GB : 16.GB * task.attempt }
     publishDir "${params.outdir}/logs/${task.process}/", mode: 'copy', pattern: ".command.log", overwrite: true, saveAs: {"${task.tag}.log"}
     publishDir "${params.outdir}/plinkLD/", mode: 'link', overwrite: true, pattern: "plink.chr${chr}_${chunk_id}.*", enabled: params.stage_binary
 
@@ -265,7 +265,7 @@ process concatGenome {
 
 process indexVariants {
     tag "sqlite"
-    memory { 8.GB * (4 ** (task.attempt - 1)) }
+    memory { System.getenv('CONSTRAIN_MEMORY') ? 8.GB : 8.GB * (4 ** (task.attempt - 1)) }
     publishDir "${params.outdir}/whole_genome/", pattern: "*.sqlite", mode: 'link', overwrite: true, saveAs: { filename -> filename.replace('concat', params.prefix) }
     publishDir "${params.outdir}/logs/${task.process}/", mode: 'copy', pattern: ".command.log", overwrite: true, saveAs: {"${task.tag}.log"}