diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..95fa254 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,53 @@ +# GitHub Actions Workflows + +This directory contains the GitHub Actions workflows used by the LDZip repository. + +## Workflows + +### `ci.yaml` + +Runs the LDZip automated tests on: + +- Pushes to `main` +- Pull requests targeting `main` + +All jobs run inside the shared Docker image: + +```text +ghcr.io/23andme/ldzip/ldzip-ci:latest +``` + +### `build-image.yaml` + +Builds and publishes the CI Docker image to GitHub Container Registry (GHCR): + +```text +ghcr.io/23andme/ldzip/ldzip-ci:latest +``` + +This workflow is currently disabled (via `if: false` in the `build-and-push` job) because the GitHub organization does not allow the third-party Docker GitHub Actions it requires. Until the workflow is enabled, the CI Docker image must be built and published manually. + +## Manually Updating the CI Docker Image + +Run the following commands from the root of the LDZip repository (where `Dockerfile.ci` is located): + +```bash +docker login ghcr.io -u USERNAME + +docker buildx build \ + --platform linux/amd64 \ + -f Dockerfile.ci \ + -t ghcr.io/23andme/ldzip/ldzip-ci:latest \ + . + +docker images | grep ldzip-ci + +docker push ghcr.io/23andme/ldzip/ldzip-ci:latest +``` + +### Notes + +- Replace `USERNAME` with your GitHub username. +- Authenticate using a GitHub Personal Access Token when prompted for your password. +- The token must have permission to publish packages to the `23andMe` organization. +- After the image is pushed, subsequent runs of `ci.yaml` will automatically use the updated image. \ No newline at end of file diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index eab4f97..2cdd03d 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -21,6 +21,7 @@ concurrency: jobs: build-and-push: runs-on: ubuntu-latest + if: false # Disabled - set to true to re-enable steps: - name: Checkout uses: actions/checkout@v4 diff --git a/Dockerfile.ci b/Dockerfile.ci index 87c3851..ac2ae05 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ git \ openjdk-17-jre-headless \ + libzstd-dev \ && rm -rf /var/lib/apt/lists/* # ---- Install Miniforge (conda-forge) ---- @@ -43,6 +44,8 @@ RUN mamba create -y -n ci-env -c conda-forge -c bioconda \ r-rcpp \ r-dbi \ r-rsqlite \ + numpy \ + scipy \ && conda clean -a -y ENV CONDA_DEFAULT_ENV=ci-env @@ -55,5 +58,6 @@ RUN nextflow -version RUN nf-test version || nf-test --help >/dev/null RUN Rscript -e "library(argparse); library(jsonlite); library(Matrix); library(tinytest); library(Rcpp); library(DBI); library(RSQLite); cat('R deps OK\n')" +RUN python3 -c "import numpy; import scipy; print('Python deps OK')" WORKDIR /work \ No newline at end of file diff --git a/pipelines/wholeGenomeLD/Makefile b/pipelines/wholeGenomeLD/Makefile index 95190cc..7d6b50c 100644 --- a/pipelines/wholeGenomeLD/Makefile +++ b/pipelines/wholeGenomeLD/Makefile @@ -1,6 +1,6 @@ - +export NXF_SYNTAX_PARSER=v1 test: - nf-test clean - nf-test test tests/stub.nf.test --verbose - nf-test test tests/unit.nf.test --verbose + rm -rf .nf-test .nf-test.log + CONSTRAIN_MEMORY=1 nf-test test tests/stub.nf.test --verbose + CONSTRAIN_MEMORY=1 nf-test test tests/unit.nf.test --verbose diff --git a/pipelines/wholeGenomeLD/main.nf b/pipelines/wholeGenomeLD/main.nf index 37caaa4..49bdb1c 100755 --- a/pipelines/wholeGenomeLD/main.nf +++ b/pipelines/wholeGenomeLD/main.nf @@ -156,7 +156,7 @@ process compressLD { process convertNpzToBinary { tag { "chr${chr}-chunk${chunk_id}" } - memory { 16.GB * task.attempt } + memory { System.getenv('CONSTRAIN_MEMORY') ? 8.GB : 16.GB * task.attempt } publishDir "${params.outdir}/logs/${task.process}/", mode: 'copy', pattern: ".command.log", overwrite: true, saveAs: {"${task.tag}.log"} publishDir "${params.outdir}/plinkLD/", mode: 'link', overwrite: true, pattern: "plink.chr${chr}_${chunk_id}.*", enabled: params.stage_binary @@ -265,7 +265,7 @@ process concatGenome { process indexVariants { tag "sqlite" - memory { 8.GB * (4 ** (task.attempt - 1)) } + memory { System.getenv('CONSTRAIN_MEMORY') ? 8.GB : 8.GB * (4 ** (task.attempt - 1)) } publishDir "${params.outdir}/whole_genome/", pattern: "*.sqlite", mode: 'link', overwrite: true, saveAs: { filename -> filename.replace('concat', params.prefix) } publishDir "${params.outdir}/logs/${task.process}/", mode: 'copy', pattern: ".command.log", overwrite: true, saveAs: {"${task.tag}.log"}