From 75d046ca729a474d45267158a2a157d1c4b6b979 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:41:54 +0200 Subject: [PATCH 01/95] add bgzip module --- modules/bgzip.nf | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 modules/bgzip.nf diff --git a/modules/bgzip.nf b/modules/bgzip.nf new file mode 100644 index 0000000..09f6198 --- /dev/null +++ b/modules/bgzip.nf @@ -0,0 +1,28 @@ +process BGZIP { + label 'process_low' + tag "VCF->VCF.gz - $sample_id" + + conda "$projectDir/envs/samtools/environment.yaml" + container params.container.samtools + + publishDir "${params.outputDir}/norm_bgzip_idx/", pattern: "${sample_id}.vcf.*", mode:'copy' + + input: + tuple val(sample_id), path(vcf_file) + + output: + tuple val(sample_id), path("${vcf_file}.gz"), path("${vcf_file}.gz.tbi"), emit: bgzf_file + + script: + """ + bgzip ${vcf_file} + tabix -p vcf ${vcf_file}.gz + """ + + stub: + """ + touch "${sample_id}.vcf.gz" + touch "${sample_id}.vcf.gz.tbi" + """ +} + From 3847f18b38d8b8d2a52ddf7c3b70fb32fcb0d79b Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:43:37 +0200 Subject: [PATCH 02/95] add matrix context --- bin/build_matrix.py | 211 ++++++++++++++++++++++++++++++++++++++ modules/matrix_context.nf | 31 ++++++ 2 files changed, 242 insertions(+) create mode 100755 bin/build_matrix.py create mode 100644 modules/matrix_context.nf diff --git a/bin/build_matrix.py b/bin/build_matrix.py new file mode 100755 index 0000000..2e9c6ce --- /dev/null +++ b/bin/build_matrix.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +""" +Build a matrix context from pools (ground truth) and an optional decode table. + +Inputs +------ +1) Pools TSV (required): columns: pool_iddimension + - dimension must be "row" or "column" + - pool_id must be unique + +2) Decode TSV (optional): columns: sample_idrow_pool_idcol_pool_id + - each (row_pool_id, col_pool_id) should appear at most once + +Outputs +------- +A) JSON context (for machines): + - schema_version, layout_hash + - pools (rows/columns with index + labels) + - matrix metadata and full cell list (row/col pool, indices, labels, alias, sample_id) + +B) TSV (for humans / spreadsheets): + - one row per intersection with: row/col pool IDs, indices, labels, cell_label (A01), + alias (same as cell_label by default), sample_id (if any) + +Usage +----- +python build_matrix.py \ + --pools pools.tsv \ + --decode decode.tsv \ + --out-json matrix_context.json \ + --out-tsv matrix_cells.tsv \ + --pad-width auto +""" +import argparse, csv, hashlib, json, sys, re +from pathlib import Path +from typing import Dict, List, Tuple, Optional + +def natural_key(s: str): + return [int(t) if t.isdigit() else t.lower() for t in re.findall(r"\d+|\D+", s)] + +def read_pools(pools_path: Path) -> Tuple[List[str], List[str]]: + rows, cols = [], [] + with pools_path.open() as fh: + reader = csv.reader(fh, delimiter="\t") + first = next(reader, None) + if first is None: + raise ValueError("Pools TSV is empty.") + records = [first] + for r in reader: + records.append(r) + for rec in records: + if len(rec) < 2: + raise ValueError(f"Bad pools row (need 2 columns): {rec}") + pool_id, dim = rec[0].strip(), rec[1].strip().lower() + if dim not in {"row", "column"}: + raise ValueError(f"Invalid dimension '{dim}' for pool '{pool_id}'") + (rows if dim == "row" else cols).append(pool_id) + # Check uniqueness + all_ids = rows + cols + if len(all_ids) != len(set(all_ids)): + dup = [p for p in set(all_ids) if all_ids.count(p) > 1] + raise ValueError(f"Duplicate pool_id(s) in pools TSV: {dup}") + if not rows or not cols: + raise ValueError("Need at least one row pool and one column pool.") + return rows, cols + +def read_decode(decode_path: Optional[Path]) -> Dict[Tuple[str,str], str]: + mapping = {} + if not decode_path: return mapping + with decode_path.open() as fh: + reader = csv.reader(fh, delimiter="\t") + + first = next(reader, None) + if first is None: + return mapping + + records = [first] + for r in reader: + records.append(r) + for rec in records: + if len(rec) < 3: + raise ValueError(f"Bad decode row (need 3 columns): {rec}") + sample_id, rpid, cpid = rec[0].strip(), rec[1].strip(), rec[2].strip() + key = (rpid, cpid) + if key in mapping and mapping[key] != sample_id: + raise ValueError(f"Decode conflict for {key}: '{mapping[key]}' vs '{sample_id}'") + mapping[key] = sample_id + return mapping + +def column_label_from_index(idx: int) -> str: + """Create excel style column label 1->A, 26->Z, 27->AA""" + label = [] + n = idx + while n > 0: + n -= 1 + label.append(chr(65 + (n % 26))) + n //= 26 + return "".join(reversed(label)) + +def layout_hash(rows_ordered: List[str], cols_ordered: List[str]) -> str: + m = hashlib.sha256() + m.update(b"rows\0"); [m.update((r+"\0").encode()) for r in rows_ordered] + m.update(b"cols\0"); [m.update((c+"\0").encode()) for c in cols_ordered] + return m.hexdigest()[:16] + +def build_context(rows: List[str], + cols: List[str], + decode: Dict[Tuple[str,str], str], + pad_width: Optional[int]) -> dict: + # Force deterministic ordering + rows_ord = sorted(rows, key=natural_key) + cols_ord = sorted(cols, key=natural_key) + + # Indices and labels + row_index = {pid: i for i, pid in enumerate(rows_ord)} + col_index = {pid: i for i, pid in enumerate(cols_ord)} + col_label = {pid: column_label_from_index(col_index[pid] +1 ) for pid in cols_ord} + n_rows = len(rows_ord) + width = pad_width if pad_width and pad_width > 0 else len(str(n_rows)) + row_label = {pid: str(row_index[pid] + 1).zfill(width) for pid in rows_ord} + + # Pools section + pools_rows = [{"pool_id": pid, "index": row_index[pid], "label": row_label[pid]} for pid in rows_ord] + pools_cols = [{"pool_id": pid, "index": col_index[pid], "label": col_label[pid]} for pid in cols_ord] + + # Cells (Cartesian product, ie. all combinations) + cells = [] + for rpid in rows_ord: + for cpid in cols_ord: + ri, ci = row_index[rpid], col_index[cpid] + rl, cl = row_label[rpid], col_label[cpid] + cell = { + "row_pool_id": rpid, + "col_pool_id": cpid, + "row_index": ri, + "col_index": ci, + "row_label": rl, + "col_label": cl, + "cell_label": f"{cl}{rl}", + "alias": f"{cl}{rl}", + "sample_id": decode.get((rpid, cpid), None), + } + cells.append(cell) + + ctx = { + "schema_version": "1.0", + "layout_hash": layout_hash(rows_ord, cols_ord), + "pools": { + "rows": pools_rows, + "columns": pools_cols, + }, + "matrix": { + "n_rows": len(rows_ord), + "n_cols": len(cols_ord), + "row_label_width": width, + "cells": cells, + }, + } + return ctx + +def write_json(ctx: dict, path: Path): + with path.open("w") as fh: + json.dump(ctx, fh, indent=2, sort_keys=False) + +def write_tsv(ctx: dict, path: Path): + fields = [ + "row_pool_id","col_pool_id", + "row_index","col_index", + "row_label","col_label", + "cell_label","alias","sample_id" + ] + with path.open("w", newline="") as fh: + w = csv.writer(fh, delimiter="\t") + w.writerow(fields) + for cell in ctx["matrix"]["cells"]: + w.writerow([ + cell["row_pool_id"], cell["col_pool_id"], + cell["row_index"], cell["col_index"], + cell["row_label"], cell["col_label"], + cell["cell_label"], cell["alias"], cell.get("sample_id") + ]) + +def main(argv=None): + p = argparse.ArgumentParser(description="Build matrix context from pools and optional decode.") + p.add_argument("--pools", required=True, type=Path, help="Pools TSV (pool_id, dimension)") + p.add_argument("--decode", required=False, type=Path, help="Decode TSV (sample_id, row_pool_id, col_pool_id)") + p.add_argument("--out-json", required=True, type=Path, help="Output JSON path") + p.add_argument("--out-tsv", required=True, type=Path, help="Output TSV path") + p.add_argument("--pad-width", default="auto", help='Column label zero-padding width (int or "auto")') + args = p.parse_args(argv) + + rows, cols = read_pools(args.pools) + decode = read_decode(args.decode) if args.decode else {} + pad_width = None if str(args.pad_width).lower() == "auto" else int(args.pad_width) + + # Validate decode keys are known pools + for (rpid, cpid) in decode.keys(): + if rpid not in rows: + raise ValueError(f"Decode references unknown row_pool_id '{rpid}'") + if cpid not in cols: + raise ValueError(f"Decode references unknown col_pool_id '{cpid}'") + + ctx = build_context(rows, cols, decode, pad_width) + args.out_json.parent.mkdir(parents=True, exist_ok=True) + args.out_tsv.parent.mkdir(parents=True, exist_ok=True) + write_json(ctx, args.out_json) + write_tsv(ctx, args.out_tsv) + +if __name__ == "__main__": + main() + \ No newline at end of file diff --git a/modules/matrix_context.nf b/modules/matrix_context.nf new file mode 100644 index 0000000..1773a59 --- /dev/null +++ b/modules/matrix_context.nf @@ -0,0 +1,31 @@ +process MATRIX_CONTEXT { + label 'process_low' + conda "$projectDir/envs/pinpy/environment.yaml" + container params.container.pinpy + + publishDir "${params.outputDir}/context/", mode:'copy', pattern: "matrix_context.*" + + input: + path pooltable + path decodetable + + output: + path "matrix_context.tsv" + path "matrix_context.json", emit: json + + script: + def decode = decodetable ? "--decode ${decodetable}" : "" + """ + build_matrix.py \ + --pools ${pooltable} \ + --out-json matrix_context.json \ + --out-tsv matrix_context.tsv \ + ${decode} + """ + + stub: + """ + touch matrix_context.json + touch matrix_context.tsv + """ +} \ No newline at end of file From 963b05c33c195303fb4383e0981093da20e31ecd Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:44:09 +0200 Subject: [PATCH 03/95] update pooltable to include dimension --- assets/data/test_data/pooltable.tsv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assets/data/test_data/pooltable.tsv b/assets/data/test_data/pooltable.tsv index 77ba74d..1b733c3 100644 --- a/assets/data/test_data/pooltable.tsv +++ b/assets/data/test_data/pooltable.tsv @@ -1,4 +1,4 @@ -B0_H0 assets/data/test_data/pools/B0_H0_1.fq.gz assets/data/test_data/pools/B0_H0_2.fq.gz -B0_H1 assets/data/test_data/pools/B0_H1_1.fq.gz assets/data/test_data/pools/B0_H1_2.fq.gz -B0_V0 assets/data/test_data/pools/B0_V0_1.fq.gz assets/data/test_data/pools/B0_V0_2.fq.gz -B0_V1 assets/data/test_data/pools/B0_V1_1.fq.gz assets/data/test_data/pools/B0_V1_2.fq.gz +B0_H0 row assets/data/test_data/pools/B0_H0_1.fq.gz assets/data/test_data/pools/B0_H0_2.fq.gz +B0_H1 row assets/data/test_data/pools/B0_H1_1.fq.gz assets/data/test_data/pools/B0_H1_2.fq.gz +B0_V0 column assets/data/test_data/pools/B0_V0_1.fq.gz assets/data/test_data/pools/B0_V0_2.fq.gz +B0_V1 column assets/data/test_data/pools/B0_V1_1.fq.gz assets/data/test_data/pools/B0_V1_2.fq.gz From 551445a67ef1c35c06dc70b4c661485c1150c9b8 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:45:11 +0200 Subject: [PATCH 04/95] Change load format of pooltable and add to pinpoint --- main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index 1037035..4c53c7f 100644 --- a/main.nf +++ b/main.nf @@ -47,7 +47,7 @@ include { TEST } from './modules/test' pooltable_ch = Channel .fromPath(params.pooltable) .splitCsv(sep: '\t') - .map { row -> tuple(row[0], [file(row[1]), file(row[2])]) } + .map { row -> tuple(row[0], [file(row[2]), file(row[3])]) } if (params.step == 'calling') { cramtable_ch = Channel @@ -118,7 +118,7 @@ workflow { PILOT_PINPOINT(vcf_ch.collect(), file(params.pooltable), file(params.decodetable)) pin_ch = PILOT_PINPOINT.out.pinned_variants } else if (params.pinpoint_method == 'new') { - PINPOINT(gatk_ch, file(params.decodetable), reference_genome_ch) + PINPOINT(gatk_ch, file(params.pooltable), file(params.decodetable), reference_genome_ch) pin_ch = PINPOINT.out.pinned_variants } } From 7f12a2dda62ca6126d957a2c397d8afb89276b72 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:48:36 +0200 Subject: [PATCH 05/95] add new method for pinpointing and combining call output --- bin/pin_basic.py | 425 +++++++++++++++++++++++++++++++++++++++++++ modules/pin_basic.nf | 29 +++ 2 files changed, 454 insertions(+) create mode 100755 bin/pin_basic.py create mode 100644 modules/pin_basic.nf diff --git a/bin/pin_basic.py b/bin/pin_basic.py new file mode 100755 index 0000000..18dbfd7 --- /dev/null +++ b/bin/pin_basic.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +import json, sys, argparse +from pathlib import Path +from dataclasses import dataclass, fields +from pysam import VariantFile +from typing import Dict, Set, List, Tuple, Optional, Any, Union + +## This is pin_basic.py - script for processing pipeline output for easy downstream analysis. +# mads - 2025-09-05 + +@dataclass +class VariantEntry: + VARID: str + CHROM: str + POS: int + REF: str + ALT: str + QUAL: float = 0.0 + AC: int = 0 + FS: float = 0.0 + SOR: float = 0.0 + GQ: int = 0 + DP: int = 0 + REF_AD: int = 0 + ALT_AD: int = 0 + +def get_other_variants(pools, idx): + return set().union(*(pools[i] for i in range(len(pools)) if i != idx)) + +def pin(vcf_folder: Path, caller: str, matrix_context: str) -> Dict[str, Dict[str, Set[str]]]: + """ + Performs the actual pinpointing logic and returns a dictionary of variants per sample. + + Parameters: + vcf_folder (Path): Path to the folder containing pool vcfs. + caller (str): Variant caller name. + + Returns: + A dictionary mapping sample IDs to their corresponding unique variants and all pinpointable variants. + """ + sample_variants: Dict[str, Dict[str, Set[str]]] = {} + + row_vcfs = [vcf_folder / f"{pool}.{caller}.vcf.gz" for pool in matrix_context.row_pools] + column_vcfs = [vcf_folder / f"{pool}.{caller}.vcf.gz" for pool in matrix_context.col_pools] + + row_pools = [set(collect_variant_ids(vcf)) for vcf in row_vcfs] + column_pools = [set(collect_variant_ids(vcf)) for vcf in column_vcfs] + + # Pinning logic: + for cell in matrix_context._ctx["matrix"]["cells"]: + sample_id = cell["alias"] + row_idx = cell["row_index"] + col_idx = cell["col_index"] + # Extract all variants from all other pools in each dimension + other_row_variants = get_other_variants(row_pools, row_idx) + other_col_variants = get_other_variants(column_pools, col_idx) + + # Extract unique and pool-specific variants + unique_row_variants = row_pools[row_idx].difference(other_row_variants) + unique_col_variants = column_pools[col_idx].difference(other_col_variants) + unique_pins = unique_row_variants.intersection(unique_col_variants) + + # Extract all pinnable variants. Only required to be unique in one dimension + unique_one_dimension_row = unique_row_variants.intersection(column_pools[col_idx]) + unique_one_dimension_col = unique_col_variants.intersection(row_pools[row_idx]) + all_pins = unique_one_dimension_row.union(unique_one_dimension_col) + + sample_variants[sample_id] = { + "unique_pins": unique_pins, + "all_pins": all_pins + } + + return sample_variants + + +def collect_variant_ids(vcf_path: str, sample_id: Optional[str] = None) -> Set[str]: + """ + Read VCF and return set of variant IDs for a single sample in format 'chrom:pos:ref:alt'. + """ + with VariantFile(vcf_path) as vcf: + if sample_id is not None: + if sample_id not in vcf.header.samples: + raise ValueError(f"Sample '{sample_id}' not found in VCF header.") + chosen_sample = sample_id + else: + if len(vcf.header.samples) == 0: + raise ValueError("No samples found in VCF header.") + elif len(vcf.header.samples) > 1: + raise ValueError( + f"Multi-sample VCF ({len(vcf.header.samples)} samples) " + f"requires specifying sample_id. Available: {list(vcf.header.samples)}" + ) + chosen_sample = next(iter(vcf.header.samples)) + + variant_ids: Set[str] = set() + + for rec in vcf.fetch(): + if not rec.alts or len(rec.alts) != 1: + raise ValueError( + f"Non-biallelic record at {rec.chrom}:{rec.pos} " + f"(REF={rec.ref}, ALTS={rec.alts}). Expected exactly one ALT." + ) + + # Only include if sample has a non-missing call + sample = rec.samples[chosen_sample] + if sample.alleles and None not in sample.alleles: + variant_id = f"{rec.chrom}:{rec.pos}:{rec.ref}:{rec.alts[0]}" + variant_ids.add(variant_id) + + return variant_ids + +def _first_scalar(x, default=0): + if x is None: + return default + if isinstance(x, (list, tuple)): + return x[0] if x else default + return x + +def collect_variant_entries(vcf_path: str, sample_id: Optional[str] = None) -> List[VariantEntry]: + """ + Read VCF and return VariantEntry per record. + + If sample_id is given, extract that sample's fields; otherwise use the first sample if present. + For site-only VCFs, sample-level fields remain 0. + """ + vcf = VariantFile(vcf_path) + + # Choose sample once (or None if site-only) + if sample_id is not None: + if sample_id not in vcf.header.samples: + raise ValueError(f"Requested sample_id '{sample_id}' not found in VCF header.") + chosen_sample = sample_id + else: + chosen_sample = next(iter(vcf.header.samples)) if len(vcf.header.samples) > 0 else None + + out: List[VariantEntry] = [] + + for rec in vcf.fetch(): + # Only accept one alt allele - multi-allelic sites is split before input. + if not rec.alts or len(rec.alts) != 1: + raise ValueError( + f"Non-biallelic record at {rec.chrom}:{rec.pos} " + f"(REF={rec.ref}, ALTS={rec.alts}). Expected exactly one ALT." + ) + alt = rec.alts[0] + + # Site-level fields + entry = VariantEntry( + VARID=f"{rec.chrom}:{rec.pos}:{rec.ref}:{alt}", + CHROM=rec.chrom, + POS=rec.pos, + REF=rec.ref, + ALT=alt, + QUAL=rec.qual or 0.0, + AC=int(_first_scalar(rec.info.get("AC") if "AC" in vcf.header.info else 0, 0)), + FS=float(_first_scalar(rec.info.get("FS") if "FS" in vcf.header.info else 0.0, 0.0)), + SOR=float(_first_scalar(rec.info.get("SOR") if "SOR" in vcf.header.info else 0.0, 0.0)), + ) + + # Sample-level fields (if any) + if chosen_sample is not None: + s = rec.samples[chosen_sample] + entry.GQ = int(s.get("GQ", 0) or 0) + entry.DP = int(s.get("DP", 0) or 0) + ad = s.get("AD", []) + + entry.REF_AD = int(ad[0]) if isinstance(ad, (list, tuple)) and len(ad) > 0 else 0 + entry.ALT_AD = int(ad[1]) if isinstance(ad, (list, tuple)) and len(ad) > 1 else 0 + out.append(entry) + return out + +def get_variant_entry(vcf_path: str, variant_id: str, sample_id: Optional[str] = None) -> VariantEntry: + """ + Locate a specific variant, extract and return a VariantEntry. + """ + try: + chrom, pos_str, ref, alt = variant_id.split(":") + pos = int(pos_str) + except ValueError: + raise ValueError(f"Invalid variant_id format '{variant_id}'. Expected 'chrom:pos:ref:alt'") + + with VariantFile(vcf_path) as vcf: + if sample_id is not None: + if sample_id not in vcf.header.samples: + raise ValueError(f"Requested sample_id '{sample_id}' not found in VCF header.") + chosen_sample = sample_id + else: + chosen_sample = next(iter(vcf.header.samples)) if len(vcf.header.samples) > 0 else None + + # Search for the specific variant + for rec in vcf.fetch(chrom, pos - 1, pos): + if (rec.pos == pos and + rec.ref == ref and + rec.alts == (alt,)): + + # Only accept one alt allele - multi-allelic sites should be split before input + if not rec.alts or len(rec.alts) != 1: + raise ValueError( + f"Non-biallelic record at {rec.chrom}:{rec.pos} " + f"(REF={rec.ref}, ALTS={rec.alts}). Expected exactly one ALT." + ) + + # Site-level fields + entry = VariantEntry( + VARID=variant_id, + CHROM=rec.chrom, + POS=rec.pos, + REF=rec.ref, + ALT=rec.alts[0], + QUAL=rec.qual or 0.0, + AC=int(_first_scalar(rec.info.get("AC") if "AC" in vcf.header.info else 0, 0)), + FS=float(_first_scalar(rec.info.get("FS") if "FS" in vcf.header.info else 0.0, 0.0)), + SOR=float(_first_scalar(rec.info.get("SOR") if "SOR" in vcf.header.info else 0.0, 0.0)), + ) + + # Sample-level fields (if any) + if chosen_sample is not None: + s = rec.samples[chosen_sample] + entry.GQ = int(s.get("GQ", 0) or 0) + entry.DP = int(s.get("DP", 0) or 0) + ad = s.get("AD", []) + + entry.REF_AD = int(ad[0]) if isinstance(ad, (list, tuple)) and len(ad) > 0 else 0 + entry.ALT_AD = int(ad[1]) if isinstance(ad, (list, tuple)) and len(ad) > 1 else 0 + return entry + raise ValueError(f"Variant '{variant_id}' not found in VCF '{vcf_path}'") + +class MatrixContext: + """ + A helper class for accessing matrix context data generated by build_matrix.py. + + Provides convenience functions to navigate DoBSeq matrices safely. + """ + + def __init__(self, ctx: dict): + self._ctx = ctx + + # Sort pools by index for consistent ordering + self._rows = sorted(ctx["pools"]["rows"], key=lambda r: r["index"]) + self._cols = sorted(ctx["pools"]["columns"], key=lambda c: c["index"]) + + # Pool ID lists (ordered) + self.row_pools: List[str] = [r["pool_id"] for r in self._rows] + self.col_pools: List[str] = [c["pool_id"] for c in self._cols] + + # Index lookups + self.row_index_by_pool: Dict[str, int] = {r["pool_id"]: r["index"] for r in self._rows} + self.col_index_by_pool: Dict[str, int] = {c["pool_id"]: c["index"] for c in self._cols} + + # Label lookups + self.row_label_by_pool: Dict[str, str] = {r["pool_id"]: r["label"] for r in self._rows} + self.col_label_by_pool: Dict[str, str] = {c["pool_id"]: c["label"] for c in self._cols} + + # Combined dimension lookup + self._dim_by_pool: Dict[str, str] = { + **{p: "row" for p in self.row_pools}, + **{p: "column" for p in self.col_pools} + } + + # Label lists (ordered) + self.row_labels: List[str] = [r["label"] for r in self._rows] + self.col_labels: List[str] = [c["label"] for c in self._cols] + + # Cell lookups - single source of truth + cells = ctx["matrix"]["cells"] + self.cells: Dict[Tuple[str, str], Dict[str, Any]] = { + (c["row_pool_id"], c["col_pool_id"]): c for c in cells + } + + # Alternative access methods + self.by_alias: Dict[str, Dict[str, Any]] = { + c["alias"]: c for c in cells + } + self.by_sample: Dict[str, Dict[str, Any]] = { + c["sample_id"]: c for c in cells if c.get("sample_id") + } + + @classmethod + def load(cls, path: Union[Path, str]) -> "MatrixContext": + """Load matrix context from JSON file.""" + with open(path, "r") as fh: + data = json.load(fh) + return cls(data) + + @property + def shape(self) -> Tuple[int, int]: + """Return matrix shape as (n_rows, n_cols).""" + return (self._ctx["matrix"]["n_rows"], self._ctx["matrix"]["n_cols"]) + + @property + def n_rows(self) -> int: + """Number of rows in the matrix.""" + return self._ctx["matrix"]["n_rows"] + + @property + def n_cols(self) -> int: + """Number of columns in the matrix.""" + return self._ctx["matrix"]["n_cols"] + + @property + def layout_hash(self) -> str: + """Get the layout hash for this matrix configuration.""" + return self._ctx["layout_hash"] + + def pool_dim(self, pool_id: str) -> str: + """Get dimension ('row' or 'column') for a pool ID.""" + if pool_id not in self._dim_by_pool: + raise ValueError(f"Unknown pool_id: {pool_id}") + return self._dim_by_pool[pool_id] + + def pool_index(self, pool_id: str) -> int: + """Get the 1-based index for a pool ID.""" + if pool_id in self.row_index_by_pool: + return self.row_index_by_pool[pool_id] + elif pool_id in self.col_index_by_pool: + return self.col_index_by_pool[pool_id] + else: + raise ValueError(f"Unknown pool_id: {pool_id}") + + def pool_label(self, pool_id: str) -> str: + """Get the formatted label for a pool ID.""" + if pool_id in self.row_label_by_pool: + return self.row_label_by_pool[pool_id] + elif pool_id in self.col_label_by_pool: + return self.col_label_by_pool[pool_id] + else: + raise ValueError(f"Unknown pool_id: {pool_id}") + + def cell(self, row_pool: str, col_pool: str) -> Optional[Dict[str, Any]]: + """Get cell data for the intersection of row and column pools.""" + return self.cells.get((row_pool, col_pool)) + + def cell_from_alias(self, alias: str) -> Optional[Dict[str, Any]]: + """Get cell data from cell alias (e.g., 'A01').""" + return self.by_alias.get(alias) + + def cell_from_sample(self, sample_id: str) -> Optional[Dict[str, Any]]: + """Get cell data from sample ID.""" + return self.by_sample.get(sample_id) + + def cell_label(self, row_pool_id: str, col_pool_id: str) -> Optional[str]: + """Get the cell label (e.g., 'A01') for pool intersection.""" + cell = self.cell(row_pool_id, col_pool_id) + return cell["cell_label"] if cell else None + + def get_samples(self) -> List[str]: + """Get all sample IDs in the matrix.""" + return list(self.by_sample.keys()) + + def __repr__(self) -> str: + return (f"MatrixContext({self.n_rows}×{self.n_cols}, " + f"{len(self.get_samples())} samples)") + +def main(): + p = argparse.ArgumentParser(description="Generate pool and pinpoint variant tables from matrix context + pool VCFs") + p.add_argument("--context", "-c", type=Path, required=True, help="Path to matrix_context.json") + p.add_argument("--vcf-folder", "-v", type=Path, required=True, help="Folder containing per-pool VCFs") + p.add_argument("--output", "-o", type=Path, required=True, help="Output folder for result TSVs") + p.add_argument("--caller", "-C", type=str, default="GATK", help="Variant caller name (used in file naming, default: GATK)") + p.add_argument("--pin-type", "-p", type=str, choices=["unique_pins", "all_pins"], default="unique_pins", help="Which pinpoint set to use in pinpoint_variants.tsv (default: unique_pins)") + args = p.parse_args() + + output_path = args.output + matrix_context = args.context + pool_vcfs_path = args.vcf_folder + pin_type = args.pin_type + caller = args.caller + + output_path.mkdir(parents=True, exist_ok=True) + mc = MatrixContext.load(matrix_context) + + pools = (mc.row_pools,mc.col_pools) + + ###### + ## Create table with all variants: + + variant_entry_fields = [f.name for f in fields(VariantEntry)] + header = ["dim", "pool_id", "pool_index", "pool_label", *variant_entry_fields] + + with open(output_path / 'all_pool_variants.tsv', 'w') as fout: + print(*header, sep="\t", file=fout) + for n, dim in enumerate(['row','column']): + for pool_id in pools[n]: + filepath = pool_vcfs_path / (f"{pool_id}.{caller}.vcf.gz") + variants = collect_variant_entries(filepath) + for v in variants: + values = [getattr(v, col) for col in variant_entry_fields] + print(dim, pool_id, mc.pool_index(pool_id), mc.pool_label(pool_id), *values, sep="\t", file=fout) + + ###### + # Create table with pinpointables only: + pins = pin(vcf_folder=pool_vcfs_path, matrix_context=mc, caller=caller) + + common_fields = ['VARID','CHROM','POS','REF','ALT'] + variant_entry_header_fields = [f"{dim}.{f.name}" for f in fields(VariantEntry) for dim in ["row","column"] if f not in common_fields] + header = ["uvarid", "sample_alias", "sample_id", "row_id", "row_index", "row_label", "column_id", "column_index", "column_label", *common_fields, *variant_entry_header_fields] + + with open(output_path / 'pinpoint_variants.tsv', 'w') as fout: + print(*header, sep="\t", file=fout) + for cell in mc._ctx["matrix"]["cells"]: + sample_alias = cell["alias"] + sample_id = cell["sample_id"] + row_label = cell["row_label"] + col_label = cell["col_label"] + row_idx = cell["row_index"] + col_idx = cell["col_index"] + row_id = cell["row_pool_id"] + col_id = cell["col_pool_id"] + for pinpoint in pins[sample_alias][pin_type]: + row_info = get_variant_entry(vcf_path=pool_vcfs_path / (f"{row_id}.{caller}.vcf.gz"), variant_id=pinpoint) + col_info = get_variant_entry(vcf_path=pool_vcfs_path / (f"{col_id}.{caller}.vcf.gz"), variant_id=pinpoint) + + assert row_info.VARID == col_info.VARID, "Row and column variant IDs do not match" + + uvarid = f"{sample_alias}:{row_info.VARID}" + print( + uvarid, sample_alias, sample_id, row_id, row_idx, row_label, col_id, col_idx, col_label, + *[getattr(row_info, col) for col in variant_entry_fields], + *[getattr(col_info, col) for col in variant_entry_fields if col not in common_fields], + sep='\t', + file=fout + ) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/modules/pin_basic.nf b/modules/pin_basic.nf new file mode 100644 index 0000000..dc44483 --- /dev/null +++ b/modules/pin_basic.nf @@ -0,0 +1,29 @@ +process PIN_BASIC { + label 'process_low' + conda "$projectDir/envs/filter_variants/environment.yaml" + container params.container.filter_variants + + publishDir "${params.outputDir}/variant_compilation/", mode:'copy', pattern: "*variants.tsv" + + input: + path vcf_files + path matrix_context + + output: + path "all_pool_variants.tsv" + path "pinpoint_variants.tsv" + + script: + """ + pin_basic.py \ + --context ${matrix_context} \ + --vcf-folder . \ + --output . + """ + + stub: + """ + touch all_pool_variants.tsv + touch pinpoint_variants.tsv + """ +} \ No newline at end of file From f69dff2a416597f032ebc79701cd32defce0915f Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:50:23 +0200 Subject: [PATCH 06/95] add module for capturing all unique variants in vcf collection --- modules/unique_vcf.nf | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 modules/unique_vcf.nf diff --git a/modules/unique_vcf.nf b/modules/unique_vcf.nf new file mode 100644 index 0000000..21c9c0b --- /dev/null +++ b/modules/unique_vcf.nf @@ -0,0 +1,31 @@ +process UNIQUE_VCF { + label 'process_single' + // Merge, deduplicate, remove sample information, and index a set of VCF files. + + conda "$projectDir/envs/bcftools/environment.yaml" + container params.container.bcftools + + publishDir "${params.outputDir}/vep_annotate/", pattern: "unique.vcf.*", mode:'copy' + + input: + path vcf_files + + output: + tuple path("unique.vcf.gz"), path("unique.vcf.gz.tbi"), path('variant_keys.txt'), emit: vcf_file + + script: + """ + bcftools merge -m none --force-samples *.vcf.gz \ + | bcftools norm -d exact -O u \ + | bcftools view -G -Oz -o unique.vcf.gz + bcftools index --tbi unique.vcf.gz + bcftools query -f '%CHROM:%POS:%REF:%ALT\n' unique.vcf.gz > variant_keys.txt + """ + + stub: + """ + touch "unique.vcf.gz" + touch "unique.vcf.gz.tbi" + touch "variant_keys.txt" + """ +} From 34baf96fe2bab8030a6184773f332d945f7189c4 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:51:47 +0200 Subject: [PATCH 07/95] add module for annotating unique variants using vep --- modules/vep.nf | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 modules/vep.nf diff --git a/modules/vep.nf b/modules/vep.nf new file mode 100644 index 0000000..b36b8f5 --- /dev/null +++ b/modules/vep.nf @@ -0,0 +1,79 @@ +process VEP { + label 'process_single' + // Annotate using VEP + + conda "$projectDir/envs/vep/environment.yaml" + container params.container.vep + + publishDir "${params.outputDir}/vep_annotate/", pattern: "annotation*.tsv", mode:'copy' + + input: + tuple path(vcf_file), path(vcf_index), path(variant_keys) + path reference_genome + path cache_dir // optional cache directory + path utr_annotation // optional UTR annotation file + path alphamissense // optional AlphaMissense file + path clinvar // optional ClinVar VCF + path danmac // optional DanMac VCF + path blacklist // optional blacklist BED + path repeatmasks // optional repeat masks BED + path gnomad // optional gnomAD VCF + path loftee_gerp_bigwig // optional LOFTEE GERP BigWig + path loftee_human_ancestor // optional LOFTEE human ancestor FASTA + path loftee_conservation // optional LOFTEE conservation SQL + + output: + path "annotations.tsv" + path "annotations_w_varid.tsv", emit: annotations_file + + script: + def db = file(params.reference_genome).getName() + ".fna" + def mode = cache_dir ? "--cache --offline --dir_cache ${cache_dir}" : "--database" + def utr = utr_annotation ? "--plugin UTRAnnotator,file=${utr_annotation}" : "" + def am = alphamissense ? "--plugin AlphaMissense,file=${alphamissense}" : "" + def clinv = clinvar ? "--custom ${clinvar},ClinVar,vcf,exact,0,CLNSIG,CLNREVSTAT,CLNDN" : "" + def danm = danmac ? "--custom ${danmac},DanMac,vcf,exact,0,AC,AN" : "" + def blackl = blacklist ? "--custom ${blacklist},blacklist,bed,overlap,0,4" : "" + def repeatm = repeatmasks ? "--custom ${repeatmasks},repeats,bed,overlap,0,4" : "" + def gnom = gnomad ? "--custom ${gnomad},gnomAD,vcf,exact,0,AC_joint_nfe,AN_joint_nfe,AF_joint_nfe,AC_genomes_nfe,AN_genomes_nfe,AF_genomes_nfe,AC_exomes_nfe,AN_exomes_nfe,AF_exomes_nfe,grpmax_joint,AC_grpmax_joint,AF_grpmax_joint,grpmax_genomes,AC_grpmax_genomes,AF_grpmax_genomes,grpmax_exomes,AC_grpmax_exomes,AF_grpmax_exomes" : "" + def loftee_plugin_dir = workflow.containerEngine ? "/plugins" : "./loftee" + def loft = loftee_gerp_bigwig && loftee_human_ancestor && loftee_conservation ? "--plugin LoF,loftee_path:/plugins,gerp_bigwig:${loftee_gerp_bigwig},human_ancestor_fa:${loftee_human_ancestor},conservation_file:${loftee_conservation}" : "" + """ + vep \ + --assembly GRCh38 \ + --format vcf \ + --tab \ + --input_file "${vcf_file}" \ + --output_file "annotations.tsv" \ + --fasta "${db}" \ + --check_ref --dont_skip --show_ref_allele \ + --hgvs --canonical --minimal --mane --biotype --numbers --allele_number --symbol \ + --pick --pick_order mane_select,mane_plus_clinical,canonical,appris,tsl,biotype,ccds,rank,length \ + --force_overwrite \ + --plugin SpliceRegion,extended \ + ${mode} \ + ${utr} \ + ${am} \ + ${clinv} \ + ${danm} \ + ${blackl} \ + ${repeatm} \ + ${gnom} \ + ${loft} + + # Header + printf 'varid\t' > annotations_w_varid.tsv + grep '^#[^#]' "annotations.tsv" >> annotations_w_varid.tsv + # Body + paste \ + <(grep -F -v '*' "variant_keys.txt") \ + <(grep -v '^#' "annotations.tsv") \ + >> annotations_w_varid.tsv + """ + + stub: + """ + touch "annotations.tsv" + touch "annotations_w_varid.tsv" + """ +} From c7d174f84ef4ac089ab89cf972ac9da36aa0707a Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:53:44 +0200 Subject: [PATCH 08/95] add vep dependencies --- envs/vep/Dockerfile | 73 +++++++++++++++++++++++++++++++++++++++ envs/vep/build | 1 + envs/vep/environment.yaml | 7 ++++ 3 files changed, 81 insertions(+) create mode 100644 envs/vep/Dockerfile create mode 100644 envs/vep/build create mode 100644 envs/vep/environment.yaml diff --git a/envs/vep/Dockerfile b/envs/vep/Dockerfile new file mode 100644 index 0000000..b71116b --- /dev/null +++ b/envs/vep/Dockerfile @@ -0,0 +1,73 @@ +FROM ubuntu:20.04 AS build +ENV DEBIAN_FRONTEND=noninteractive + +# Install basic requirements +RUN apt-get update && apt-get -y install \ + wget \ + libncurses5-dev \ + libncursesw5-dev \ + libbz2-dev \ + liblzma-dev \ + build-essential \ + libz-dev \ + git \ + libncurses5 \ + libbz2-1.0 \ + liblzma5 \ + && rm -rf /var/lib/apt/lists/* + +# Install samtools +RUN wget https://github.com/samtools/samtools/releases/download/1.7/samtools-1.7.tar.bz2 && \ + tar xjvf samtools-1.7.tar.bz2 && \ + cd samtools-1.7 && \ + make && \ + make install && \ + cd .. && \ + rm -rf samtools-1.7* + +# Clone LOFTEE +RUN git clone -b grch38 https://github.com/konradjk/loftee.git /loftee + +FROM ensemblorg/ensembl-vep:release_111.0 AS runtime + +# Switch to root to install system packages and dependencies +USER root + +# Set up Perl environment +ENV PERL_MM_USE_DEFAULT=1 +ENV PERL5LIB=/opt/vep/perl5/lib/perl5:$PERL5LIB + +# Install system dependencies +RUN apt-get update && apt-get -y install \ + libncurses5 \ + libtinfo5 \ + libbz2-1.0 \ + liblzma5 \ + libsqlite3-dev \ + zlib1g \ + && rm -rf /var/lib/apt/lists/* + +# Create directory and set permissions +RUN mkdir -p /opt/vep/perl5 && \ + chmod -R 777 /opt/vep/perl5 + +# Copy samtools binary and make it executable +COPY --from=build /usr/local/bin/samtools /usr/local/bin/samtools +RUN chmod +x /usr/local/bin/samtools + +# Copy LOFTEE plugins +COPY --from=build /loftee/ /plugins/loftee/ + +# Install additional Perl dependencies as root +RUN cpanm --local-lib=/opt/vep/perl5 local::lib && \ + cpanm --local-lib=/opt/vep/perl5 --force DBD::SQLite Bio::Perl + +# Switch back to vep user +USER vep + +# Verify installations +RUN samtools --version && \ + ls -la /plugins/loftee/ + +# Set working directory +WORKDIR /opt/vep \ No newline at end of file diff --git a/envs/vep/build b/envs/vep/build new file mode 100644 index 0000000..329203d --- /dev/null +++ b/envs/vep/build @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/vep_w_loftee:111.0 --push . \ No newline at end of file diff --git a/envs/vep/environment.yaml b/envs/vep/environment.yaml new file mode 100644 index 0000000..87b3315 --- /dev/null +++ b/envs/vep/environment.yaml @@ -0,0 +1,7 @@ +name: vep +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::ensembl-vep=111.0 \ No newline at end of file From 69472e143a3d0bf9c16d1a47edc1e904d6a8d4e3 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:56:27 +0200 Subject: [PATCH 09/95] update pinpoint subworkflow with simplified output --- subworkflows/pinpoint.nf | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/subworkflows/pinpoint.nf b/subworkflows/pinpoint.nf index d97da61..a6a2261 100644 --- a/subworkflows/pinpoint.nf +++ b/subworkflows/pinpoint.nf @@ -14,6 +14,11 @@ include { DISCARD } from '../modules/discard' include { ANNOTATION } from '../subworkflows/annotation' include { VARTABLE_PINS } from '../modules/vartable_pins' include { MERGE_PINS } from '../modules/mergepins' +include { MATRIX_CONTEXT } from '../modules/matrix_context' +include { BGZIP } from '../modules/bgzip' +include { PIN_BASIC } from '../modules/pin_basic' +include { UNIQUE_VCF } from '../modules/unique_vcf' +include { VEP } from '../modules/vep' if (params.filter) { @@ -25,6 +30,7 @@ if (params.filter) { workflow PINPOINT { take: vcf_file + pooltable decode_table reference_genome @@ -82,10 +88,35 @@ workflow PINPOINT { vartables, decode_table, 'GATK') - VARTABLE_PINS(PINPY.out.vcf_unique_pins.flatten()) MERGE_PINS(PINPY.out.vcf_unique_2d_pins) + // New pinpoint-flow + // If alone, add normalisation here first.. + BGZIP(vcf_ch) + BGZIP.out.bgzf_file + .map { sample, vcf, index -> tuple(vcf, index) } + .collect() + .set { basic_input } + MATRIX_CONTEXT(pooltable,[]) + PIN_BASIC(basic_input,MATRIX_CONTEXT.out.json) + UNIQUE_VCF(basic_input) + VEP( + UNIQUE_VCF.out.vcf_file, + reference_genome, + [], // cache + [], // utr + [], // alphamissense + [], // clinvar + [], // danmac + [], // blacklist + [], // repeatmasker + [], // gnomad + [], // loftee gerp + [], // loftee human ancestor + [] // loftee conservation + ) + emit: pinned_variants = PINPY.out.lookup_table } \ No newline at end of file From b941d46dfc6a498eb7af4d5dcaf3235806b3fb25 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 10 Sep 2025 11:58:39 +0200 Subject: [PATCH 10/95] add vep container --- conf/container.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/container.config b/conf/container.config index 9713ad4..5e0c285 100644 --- a/conf/container.config +++ b/conf/container.config @@ -26,5 +26,6 @@ params { deepvariant = "quay.io/biocontainers/deepvariant:1.5.0--py36hf3e76ba_0" r_env = "rocker/tidyverse:latest" filter_variants = "madscort/dswf_filter:1.0" + vep = "madscort/vep_w_loftee:111.0" } } \ No newline at end of file From e8272cbe3496a588fe4b4ccfca4bcb68e8d22944 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 11:44:17 +0200 Subject: [PATCH 11/95] add pileup calling method --- main.nf | 6 ++++- modules/mpileup.nf | 36 ++++++++++++++++++++++++++++++ nextflow.config | 3 +++ subworkflows/pileup_calling.nf | 40 ++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 modules/mpileup.nf create mode 100644 subworkflows/pileup_calling.nf diff --git a/main.nf b/main.nf index 4c53c7f..b58f79f 100644 --- a/main.nf +++ b/main.nf @@ -33,6 +33,7 @@ include { INDEX } from './modules/index' // Pinpoint methods include { PILOT_PINPOINT } from './modules/pilot_pinpoint' include { PINPOINT } from './subworkflows/pinpoint' +include { PILEUP_CALLING } from './subworkflows/pileup_calling' // Test include { TEST } from './modules/test' @@ -85,7 +86,7 @@ workflow { } else { bam_file_w_index_ch = MAPPING(pooltable_ch, reference_genome_ch, bedfile_ch) } - } else if (params.step == 'calling') { + } else if (params.step == 'calling' || (params.step == 'pinpoint' && params.pileup_calling)) { BAM(cramtable_ch, reference_genome_ch) bam_file_w_index_ch = INDEX(BAM.out.bam_file) } @@ -120,6 +121,9 @@ workflow { } else if (params.pinpoint_method == 'new') { PINPOINT(gatk_ch, file(params.pooltable), file(params.decodetable), reference_genome_ch) pin_ch = PINPOINT.out.pinned_variants + if (params.pileup_calling) { + PILEUP_CALLING(gatk_ch, bam_file_w_index_ch, file(params.pooltable), file(params.decodetable), reference_genome_ch, bedfile_ch) + } } } diff --git a/modules/mpileup.nf b/modules/mpileup.nf new file mode 100644 index 0000000..4032db4 --- /dev/null +++ b/modules/mpileup.nf @@ -0,0 +1,36 @@ +process MPILEUP { + label 'process_low' + tag "BAM->mpileup - $sample_id" + + conda "$projectDir/envs/samtools/environment.yaml" + container params.container.samtools + + publishDir "${params.outputDir}/mpileup/", mode:'copy', pattern: "${sample_id}.mpileup.gz" + + input: + tuple val(sample_id), path(bam_file), path(index) + path(reference) + path(bedfile) + + output: + tuple val(sample_id), path("${sample_id}.mpileup.gz"), emit: mpileup_file + + script: + def db = file(params.reference_genome).getName() + ".fna" + """ + samtools mpileup \ + -l ${bedfile} \ + --max-depth 50000 \ + --min-BQ 0 \ + --output-MQ \ + --fasta-ref ${db} \ + ${bam_file} \ + | gzip > ${sample_id}.mpileup.gz + """ + + stub: + """ + touch "${sample_id}.mpileup.gz" + """ +} + diff --git a/nextflow.config b/nextflow.config index ff24e4a..936eb3f 100644 --- a/nextflow.config +++ b/nextflow.config @@ -84,6 +84,9 @@ params { // Pinpoint method (pilot or new) pinpoint_method = 'new' + // Pileup_calling (true or false) + pileup_calling = true + // Annotation configurations annotate = true snpeff_db = 'GRCh38.99' diff --git a/subworkflows/pileup_calling.nf b/subworkflows/pileup_calling.nf new file mode 100644 index 0000000..0159c42 --- /dev/null +++ b/subworkflows/pileup_calling.nf @@ -0,0 +1,40 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + PILEUP_CALLING SUB-WORKFLOW +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { INDEX_VCF } from '../modules/index_vcf' +include { DROP_PL } from '../modules/drop_pl' +include { NORMALISE_VCF } from '../modules/normalise_vcf' +include { MATRIX_CONTEXT } from '../modules/matrix_context' +include { BGZIP } from '../modules/bgzip' +include { MPILEUP } from '../modules/mpileup' + +workflow PILEUP_CALLING { + take: + vcf_file + bam_file_w_index + pooltable + decode_table + reference_genome + bedfile + + main: + INDEX_VCF(vcf_file, 'GATK') + DROP_PL(INDEX_VCF.out.vcf_w_index, 'GATK') + NORMALISE_VCF(DROP_PL.out.vcf_file, reference_genome, 'GATK') + + BGZIP(NORMALISE_VCF.out.norm_vcf) + BGZIP.out.bgzf_file + .map { sample, vcf, index -> tuple(vcf, index) } + .collect() + .set { basic_input } + MATRIX_CONTEXT(pooltable,[]) + MPILEUP(bam_file_w_index, reference_genome, bedfile) + + + + emit: + pinned_variants = Channel.empty() +} \ No newline at end of file From e9cd1431bbaabe10d120751245ce1fe28b6d09c3 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 13:17:56 +0200 Subject: [PATCH 12/95] bump marbl version --- conf/container.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/container.config b/conf/container.config index 5e0c285..3b3a5d5 100644 --- a/conf/container.config +++ b/conf/container.config @@ -27,5 +27,6 @@ params { r_env = "rocker/tidyverse:latest" filter_variants = "madscort/dswf_filter:1.0" vep = "madscort/vep_w_loftee:111.0" + marbl = "madscort/marbl:0.2.0" } } \ No newline at end of file From a72d228f20c5e7be771374c14981ed0fafd6dbbd Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 13:19:09 +0200 Subject: [PATCH 13/95] rename --- main.nf | 6 ++-- modules/rescue.nf | 29 +++++++++++++++++++ nextflow.config | 4 +-- .../{pileup_calling.nf => variant_rescue.nf} | 10 +++---- 4 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 modules/rescue.nf rename subworkflows/{pileup_calling.nf => variant_rescue.nf} (83%) diff --git a/main.nf b/main.nf index b58f79f..514b55e 100644 --- a/main.nf +++ b/main.nf @@ -33,7 +33,7 @@ include { INDEX } from './modules/index' // Pinpoint methods include { PILOT_PINPOINT } from './modules/pilot_pinpoint' include { PINPOINT } from './subworkflows/pinpoint' -include { PILEUP_CALLING } from './subworkflows/pileup_calling' +include { VARIANT_RESCUE } from './subworkflows/variant_rescue' // Test include { TEST } from './modules/test' @@ -121,8 +121,8 @@ workflow { } else if (params.pinpoint_method == 'new') { PINPOINT(gatk_ch, file(params.pooltable), file(params.decodetable), reference_genome_ch) pin_ch = PINPOINT.out.pinned_variants - if (params.pileup_calling) { - PILEUP_CALLING(gatk_ch, bam_file_w_index_ch, file(params.pooltable), file(params.decodetable), reference_genome_ch, bedfile_ch) + if (params.variant_rescue) { + VARIANT_RESCUE(gatk_ch, bam_file_w_index_ch, file(params.pooltable), file(params.decodetable), reference_genome_ch, bedfile_ch) } } } diff --git a/modules/rescue.nf b/modules/rescue.nf new file mode 100644 index 0000000..b2a43cb --- /dev/null +++ b/modules/rescue.nf @@ -0,0 +1,29 @@ +process RESCUE { + label 'process_low' + conda "$projectDir/envs/rescue/environment.yaml" + container params.container.marbl + + publishDir "${params.outputDir}/rescue_probabilities/", mode:'copy', pattern: "predictions.tsv" + + input: + path sampletable + path vcf_files + path mpileup + + output: + path "predictions.tsv" + + script: + """ + marbl \ + --vcf-folder . \ + --mpileup-folder . \ + --sampletable ${sampletable} \ + --output . + """ + + stub: + """ + touch predictions.tsv + """ +} \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index 936eb3f..24308db 100644 --- a/nextflow.config +++ b/nextflow.config @@ -84,8 +84,8 @@ params { // Pinpoint method (pilot or new) pinpoint_method = 'new' - // Pileup_calling (true or false) - pileup_calling = true + // Use variant rescue model (true or false) + variant_rescue = true // Annotation configurations annotate = true diff --git a/subworkflows/pileup_calling.nf b/subworkflows/variant_rescue.nf similarity index 83% rename from subworkflows/pileup_calling.nf rename to subworkflows/variant_rescue.nf index 0159c42..31c1d0f 100644 --- a/subworkflows/pileup_calling.nf +++ b/subworkflows/variant_rescue.nf @@ -1,6 +1,6 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - PILEUP_CALLING SUB-WORKFLOW + VARIANT_RESCUE SUB-WORKFLOW ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ @@ -10,8 +10,9 @@ include { NORMALISE_VCF } from '../modules/normalise_vcf' include { MATRIX_CONTEXT } from '../modules/matrix_context' include { BGZIP } from '../modules/bgzip' include { MPILEUP } from '../modules/mpileup' +include { RESCUE } from '../modules/rescue' -workflow PILEUP_CALLING { +workflow VARIANT_RESCUE { take: vcf_file bam_file_w_index @@ -29,11 +30,10 @@ workflow PILEUP_CALLING { BGZIP.out.bgzf_file .map { sample, vcf, index -> tuple(vcf, index) } .collect() - .set { basic_input } + .set { vcf_files } MATRIX_CONTEXT(pooltable,[]) MPILEUP(bam_file_w_index, reference_genome, bedfile) - - + RESCUE(pooltable, vcf_files, MPILEUP.out.mpileup_file.collect()) emit: pinned_variants = Channel.empty() From f9afdcaa2c35a4f7328324f659634aeb9ca8b17c Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 13:19:42 +0200 Subject: [PATCH 14/95] fix output naming --- modules/bgzip.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bgzip.nf b/modules/bgzip.nf index 09f6198..b598c0b 100644 --- a/modules/bgzip.nf +++ b/modules/bgzip.nf @@ -5,7 +5,7 @@ process BGZIP { conda "$projectDir/envs/samtools/environment.yaml" container params.container.samtools - publishDir "${params.outputDir}/norm_bgzip_idx/", pattern: "${sample_id}.vcf.*", mode:'copy' + publishDir "${params.outputDir}/norm_bgzip_idx/", pattern: "*vcf.*", mode:'copy' input: tuple val(sample_id), path(vcf_file) From 4ad05ad64bfe5a0c86c1c2c12a0625088d6b1593 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 13:20:00 +0200 Subject: [PATCH 15/95] change output structure --- modules/mpileup.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mpileup.nf b/modules/mpileup.nf index 4032db4..866b5ca 100644 --- a/modules/mpileup.nf +++ b/modules/mpileup.nf @@ -13,7 +13,7 @@ process MPILEUP { path(bedfile) output: - tuple val(sample_id), path("${sample_id}.mpileup.gz"), emit: mpileup_file + path("${sample_id}.mpileup.gz"), emit: mpileup_file script: def db = file(params.reference_genome).getName() + ".fna" From ef6e7ae0768e50bc388d8af3eed29ff16a611030 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 13:20:25 +0200 Subject: [PATCH 16/95] add environment for rescue --- envs/rescue/Dockerfile | 10 ++++++++++ envs/rescue/build | 1 + envs/rescue/environment.yaml | 9 +++++++++ 3 files changed, 20 insertions(+) create mode 100644 envs/rescue/Dockerfile create mode 100644 envs/rescue/build create mode 100644 envs/rescue/environment.yaml diff --git a/envs/rescue/Dockerfile b/envs/rescue/Dockerfile new file mode 100644 index 0000000..713996f --- /dev/null +++ b/envs/rescue/Dockerfile @@ -0,0 +1,10 @@ +FROM continuumio/miniconda3:latest + +COPY environment.yaml /tmp/environment.yaml + +RUN conda env create --name docker_env --file /tmp/environment.yaml + +ENV PATH /opt/conda/envs/docker_env/bin:$PATH +SHELL ["conda", "run", "-n", "docker_env", "/bin/bash", "-c"] + +WORKDIR /app \ No newline at end of file diff --git a/envs/rescue/build b/envs/rescue/build new file mode 100644 index 0000000..3321b77 --- /dev/null +++ b/envs/rescue/build @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/marbl:0.2.0 --push . \ No newline at end of file diff --git a/envs/rescue/environment.yaml b/envs/rescue/environment.yaml new file mode 100644 index 0000000..7558ae3 --- /dev/null +++ b/envs/rescue/environment.yaml @@ -0,0 +1,9 @@ +name: rescue +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - python=3.11.0 + - pip: + - marbl-pool==0.2.0 \ No newline at end of file From 5d2e81911b9864d5d7c2fdd326c89e1facb5d1e9 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 17:48:28 +0200 Subject: [PATCH 17/95] add decodetable creation from pooltable --- bin/build_matrix.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bin/build_matrix.py b/bin/build_matrix.py index 2e9c6ce..ab246e7 100755 --- a/bin/build_matrix.py +++ b/bin/build_matrix.py @@ -180,12 +180,21 @@ def write_tsv(ctx: dict, path: Path): cell["cell_label"], cell["alias"], cell.get("sample_id") ]) +def write_decode(ctx: dict, path: Path): + with path.open("w", newline="") as fh: + w = csv.writer(fh, delimiter="\t") + for cell in ctx["matrix"]["cells"]: + w.writerow([ + cell["alias"], cell["row_pool_id"], cell["col_pool_id"] + ]) + def main(argv=None): p = argparse.ArgumentParser(description="Build matrix context from pools and optional decode.") p.add_argument("--pools", required=True, type=Path, help="Pools TSV (pool_id, dimension)") p.add_argument("--decode", required=False, type=Path, help="Decode TSV (sample_id, row_pool_id, col_pool_id)") p.add_argument("--out-json", required=True, type=Path, help="Output JSON path") p.add_argument("--out-tsv", required=True, type=Path, help="Output TSV path") + p.add_argument("--out-decode", required=True, type=Path, help="Output decode TSV path") p.add_argument("--pad-width", default="auto", help='Column label zero-padding width (int or "auto")') args = p.parse_args(argv) @@ -203,8 +212,10 @@ def main(argv=None): ctx = build_context(rows, cols, decode, pad_width) args.out_json.parent.mkdir(parents=True, exist_ok=True) args.out_tsv.parent.mkdir(parents=True, exist_ok=True) + args.out_decode.parent.mkdir(parents=True, exist_ok=True) write_json(ctx, args.out_json) write_tsv(ctx, args.out_tsv) + write_decode(ctx, args.out_decode) if __name__ == "__main__": main() From 86ba641534ce477e4c2c654cadef8e3efa66eaae Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 17:50:15 +0200 Subject: [PATCH 18/95] make module emit decode table --- modules/matrix_context.nf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/matrix_context.nf b/modules/matrix_context.nf index 1773a59..fdcc1bf 100644 --- a/modules/matrix_context.nf +++ b/modules/matrix_context.nf @@ -4,14 +4,16 @@ process MATRIX_CONTEXT { container params.container.pinpy publishDir "${params.outputDir}/context/", mode:'copy', pattern: "matrix_context.*" + publishDir "${params.outputDir}/context/", mode:'copy', pattern: "decodetable.tsv" input: path pooltable - path decodetable + path decodetable, stageAs: 'input_table/*' output: path "matrix_context.tsv" path "matrix_context.json", emit: json + path "decodetable.tsv", emit: decodetable script: def decode = decodetable ? "--decode ${decodetable}" : "" @@ -20,6 +22,7 @@ process MATRIX_CONTEXT { --pools ${pooltable} \ --out-json matrix_context.json \ --out-tsv matrix_context.tsv \ + --out-decode decodetable.tsv \ ${decode} """ @@ -27,5 +30,6 @@ process MATRIX_CONTEXT { """ touch matrix_context.json touch matrix_context.tsv + touch decodetable.tsv """ } \ No newline at end of file From 4ab2894383d1f550e35dc7c1e456dd6357233563 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 15 Sep 2025 17:51:00 +0200 Subject: [PATCH 19/95] add matrix context as optional source of decodetable to workflow --- main.nf | 19 ++++++++++++++++--- subworkflows/pinpoint.nf | 5 ++--- subworkflows/variant_rescue.nf | 3 --- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/main.nf b/main.nf index 514b55e..9637da3 100644 --- a/main.nf +++ b/main.nf @@ -30,6 +30,9 @@ include { ANNOTATION } from './subworkflows/annotation' include { BAM } from './modules/bam' include { INDEX } from './modules/index' +// Method for creating decode table and matrix context +include { MATRIX_CONTEXT } from './modules/matrix_context' + // Pinpoint methods include { PILOT_PINPOINT } from './modules/pilot_pinpoint' include { PINPOINT } from './subworkflows/pinpoint' @@ -111,18 +114,28 @@ workflow { } if (params.step == 'pinpoint' || params.step == 'all' || params.step == '') { + pooltable = file(params.pooltable) if (params.pinpoint_method == 'pilot') { vcf_ch = gatk_ch .mix(lofreq_ch) .map { pool_id, vcf_file -> vcf_file } - PILOT_PINPOINT(vcf_ch.collect(), file(params.pooltable), file(params.decodetable)) + PILOT_PINPOINT(vcf_ch.collect(), pooltable, file(params.decodetable)) pin_ch = PILOT_PINPOINT.out.pinned_variants } else if (params.pinpoint_method == 'new') { - PINPOINT(gatk_ch, file(params.pooltable), file(params.decodetable), reference_genome_ch) + if (params.decodetable == "") { + MATRIX_CONTEXT(pooltable,[]) + matrix_context = MATRIX_CONTEXT.out.json + decode_table = MATRIX_CONTEXT.out.decodetable + } else { + MATRIX_CONTEXT(pooltable,file(params.decodetable)) + matrix_context = MATRIX_CONTEXT.out.json + decode_table = file(params.decodetable) + } + PINPOINT(gatk_ch, pooltable, decode_table, matrix_context, reference_genome_ch) pin_ch = PINPOINT.out.pinned_variants if (params.variant_rescue) { - VARIANT_RESCUE(gatk_ch, bam_file_w_index_ch, file(params.pooltable), file(params.decodetable), reference_genome_ch, bedfile_ch) + VARIANT_RESCUE(gatk_ch, bam_file_w_index_ch, pooltable, reference_genome_ch, bedfile_ch) } } } diff --git a/subworkflows/pinpoint.nf b/subworkflows/pinpoint.nf index a6a2261..3445831 100644 --- a/subworkflows/pinpoint.nf +++ b/subworkflows/pinpoint.nf @@ -14,7 +14,6 @@ include { DISCARD } from '../modules/discard' include { ANNOTATION } from '../subworkflows/annotation' include { VARTABLE_PINS } from '../modules/vartable_pins' include { MERGE_PINS } from '../modules/mergepins' -include { MATRIX_CONTEXT } from '../modules/matrix_context' include { BGZIP } from '../modules/bgzip' include { PIN_BASIC } from '../modules/pin_basic' include { UNIQUE_VCF } from '../modules/unique_vcf' @@ -32,6 +31,7 @@ workflow PINPOINT { vcf_file pooltable decode_table + matrix_context reference_genome main: @@ -98,8 +98,7 @@ workflow PINPOINT { .map { sample, vcf, index -> tuple(vcf, index) } .collect() .set { basic_input } - MATRIX_CONTEXT(pooltable,[]) - PIN_BASIC(basic_input,MATRIX_CONTEXT.out.json) + PIN_BASIC(basic_input,matrix_context) UNIQUE_VCF(basic_input) VEP( UNIQUE_VCF.out.vcf_file, diff --git a/subworkflows/variant_rescue.nf b/subworkflows/variant_rescue.nf index 31c1d0f..74ddca4 100644 --- a/subworkflows/variant_rescue.nf +++ b/subworkflows/variant_rescue.nf @@ -7,7 +7,6 @@ include { INDEX_VCF } from '../modules/index_vcf' include { DROP_PL } from '../modules/drop_pl' include { NORMALISE_VCF } from '../modules/normalise_vcf' -include { MATRIX_CONTEXT } from '../modules/matrix_context' include { BGZIP } from '../modules/bgzip' include { MPILEUP } from '../modules/mpileup' include { RESCUE } from '../modules/rescue' @@ -17,7 +16,6 @@ workflow VARIANT_RESCUE { vcf_file bam_file_w_index pooltable - decode_table reference_genome bedfile @@ -31,7 +29,6 @@ workflow VARIANT_RESCUE { .map { sample, vcf, index -> tuple(vcf, index) } .collect() .set { vcf_files } - MATRIX_CONTEXT(pooltable,[]) MPILEUP(bam_file_w_index, reference_genome, bedfile) RESCUE(pooltable, vcf_files, MPILEUP.out.mpileup_file.collect()) From aee0127d6f2dcb80d695b7a59dbd0f23ee680d0c Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 16 Sep 2025 09:05:53 +0200 Subject: [PATCH 20/95] Add annotations config for ngc --- conf/ngc.config | 12 +++++++++++- conf/profiles.config | 3 --- nextflow.config | 13 +++++++++++++ subworkflows/annotation.nf | 8 +++++--- subworkflows/pinpoint.nf | 23 +++++++++++------------ 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/conf/ngc.config b/conf/ngc.config index 694be07..86e81f5 100644 --- a/conf/ngc.config +++ b/conf/ngc.config @@ -10,7 +10,17 @@ profiles { snpsift = "java -Xmx4g -jar /services/tools/snpeff/5.0e/SnpSift.jar" snpeff_config = "/ngc/projects2/dp_00005/data/dwf/databases/snpeff/snpEff.config" snpeff_cache = "/ngc/projects2/dp_00005/data/dwf/databases/snpeff/data" - clinvar_db = "/ngc/projects2/dp_00005/data/dwf/databases/clinvar_20230903.vcf.gz" + vep_cache = "/ngc/shared/nf_tools/references/vep_cache" + clinvar_db = "/ngc/projects2/dp_00005/data/dwf/databases/clinvar_20230903.vcf.gz" + danmac_db = "/ngc/projects2/dp_00005/data/dwf/databases/danmac/danmac.vcf.gz" + blacklist_bed = "/ngc/projects2/dp_00005/data/dwf/databases/encode_blacklist/hg38-blacklist.v2.sorted.bed.gz" + repeatmasker_bed = "/ngc/projects2/dp_00005/data/dwf/databases/repeat_masker/repeatmasker.sorted.bed.gz" + gnomad_vcf = "/ngc/projects2/dp_00005/data/dwf/databases/gnomad/4.1/twist_large_gnomad.vcf.bgz" + utr_file = "/ngc/projects2/dp_00005/data/dwf/databases/utr_annotation/uORF_5UTR_GRCh38_PUBLIC.txt" + alphamissense_tsv = "/ngc/projects2/dp_00005/data/dwf/databases/alphamissense/AlphaMissense_hg38.tsv.gz" + loftee_gerp_bw = "/ngc/projects2/dp_00005/data/dwf/databases/loftee/gerp_conservation_scores.homo_sapiens.GRCh38.bw" + loftee_human_ancestor= "/ngc/projects2/dp_00005/data/dwf/databases/loftee/human_ancestor.fa.gz" + loftee_sqlite = "/ngc/projects2/dp_00005/data/dwf/databases/loftee/loftee.sql" } process { diff --git a/conf/profiles.config b/conf/profiles.config index 5ba94e8..4bf3713 100644 --- a/conf/profiles.config +++ b/conf/profiles.config @@ -85,9 +85,6 @@ profiles { snv_list = "$projectDir/assets/data/test_data/snvlist.tsv" mills = "$projectDir/assets/data/test_data/databases/mills.vcf" g1000 = "$projectDir/assets/data/test_data/databases/g1000.vcf" - clinvar_db = "$projectDir/assets/data/test_data/databases/clinvar.vcf" - snpeff_config = "$projectDir/assets/data/test_data/databases/snpeff/snpeff.config" - snpeff_cache = "$projectDir/assets/data/test_data/databases/snpeff/" } } diff --git a/nextflow.config b/nextflow.config index 24308db..5a5f3d5 100644 --- a/nextflow.config +++ b/nextflow.config @@ -96,6 +96,19 @@ params { // Add gene annotation based on third column in bedfile bedfile_gene_annotation = true + // VEP annotations + vep_cache = "" + clinvar_db = "" + danmac_db = "" + blacklist_bed = "" + repeatmasker_bed = "" + gnomad_vcf = "" + utr_file = "" + alphamissense_tsv = "" + loftee_gerp_bw = "" + loftee_human_ancestor= "" + loftee_sqlite = "" + // Misc (never edit here) help = false testing = false diff --git a/subworkflows/annotation.nf b/subworkflows/annotation.nf index 9a7e9e8..862b4d7 100644 --- a/subworkflows/annotation.nf +++ b/subworkflows/annotation.nf @@ -13,9 +13,11 @@ if (params.bedfile_gene_annotation) { bedfile = Channel.fromPath(params.bedfile, checkIfExists: true).collect() } -snpeff_cache_ch = Channel.fromPath(params.snpeff_cache, checkIfExists: true).collect() -clinvardb_ch = Channel.fromPath(params.clinvar_db + "*", checkIfExists: true).collect() -snpeff_config_ch = Channel.fromPath(params.snpeff_config, checkIfExists: true).collect() +if (params.annotate) { + snpeff_cache_ch = Channel.fromPath(params.snpeff_cache, checkIfExists: true).collect() + clinvardb_ch = Channel.fromPath(params.clinvar_db + "*", checkIfExists: true).collect() + snpeff_config_ch = Channel.fromPath(params.snpeff_config, checkIfExists: true).collect() +} workflow ANNOTATION { take: diff --git a/subworkflows/pinpoint.nf b/subworkflows/pinpoint.nf index 3445831..1ec442d 100644 --- a/subworkflows/pinpoint.nf +++ b/subworkflows/pinpoint.nf @@ -92,7 +92,6 @@ workflow PINPOINT { MERGE_PINS(PINPY.out.vcf_unique_2d_pins) // New pinpoint-flow - // If alone, add normalisation here first.. BGZIP(vcf_ch) BGZIP.out.bgzf_file .map { sample, vcf, index -> tuple(vcf, index) } @@ -103,17 +102,17 @@ workflow PINPOINT { VEP( UNIQUE_VCF.out.vcf_file, reference_genome, - [], // cache - [], // utr - [], // alphamissense - [], // clinvar - [], // danmac - [], // blacklist - [], // repeatmasker - [], // gnomad - [], // loftee gerp - [], // loftee human ancestor - [] // loftee conservation + params.vep_cache ?: [], // cache + params.utr_file ?: [], // utr + params.alphamissense_tsv ?: [], // alphamissense + params.clinvar_db ?: [], // clinvar + params.danmac_db ?: [], // danmac + params.blacklist_bed ?: [], // blacklist + params.repeatmasker_bed ?: [], // repeatmasker + params.gnomad_vcf ?: [], // gnomad + params.loftee_gerp_bw ?: [], // loftee gerp + params.loftee_human_ancestor ?: [],// loftee human ancestor + params.loftee_sqlite ?: [] // loftee conservation (sqlite) ) emit: From cd985dfa3acacca76b107a358fb8973f57e39c86 Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 09:23:31 +0200 Subject: [PATCH 21/95] update singularity vs docker configuration --- conf/container.config | 81 ++++++++++++++++++-------- envs/snpsift/environment.yaml | 7 +++ modules/addreadgroup.nf | 2 +- modules/alignment.nf | 4 +- modules/alignment_metrics.nf | 2 +- modules/alignment_umi.nf | 1 + modules/apply_bqsr.nf | 2 +- modules/bam.nf | 2 +- modules/bed_annotate.nf | 2 +- modules/bgzip.nf | 2 +- modules/bqsr.nf | 2 +- modules/call_consensus.nf | 2 +- modules/clean.nf | 2 +- modules/combinegvcfs.nf | 2 +- modules/cram.nf | 2 +- modules/deepvariant.nf | 5 +- modules/downsample.nf | 2 +- modules/drop_pl.nf | 2 +- modules/duplicate_metrics.nf | 2 +- modules/extract_umi.nf | 2 +- modules/fastq.nf | 2 +- modules/fastqc.nf | 2 +- modules/filter.nf | 2 +- modules/filter_variants.nf | 2 +- modules/flagstat.nf | 2 +- modules/freebayes.nf | 2 +- modules/gc_metrics.nf | 2 +- modules/genomicsdb.nf | 2 +- modules/genotypegvcf.nf | 2 +- modules/genotypegvcf_split.nf | 2 +- modules/group_umi.nf | 2 +- modules/haplotypecaller.nf | 2 +- modules/haplotypecaller_split.nf | 2 +- modules/haplotypecaller_truth.nf | 2 +- modules/haplotypecaller_truth_joint.nf | 2 +- modules/hs_metrics.nf | 2 +- modules/indelqual.nf | 2 +- modules/index.nf | 2 +- modules/index_vcf.nf | 2 +- modules/insert_size_metrics.nf | 2 +- modules/intervals.nf | 2 +- modules/lofreq.nf | 2 +- modules/markduplicates.nf | 2 +- modules/markduplicates_fast.nf | 2 +- modules/markduplicates_spark.nf | 2 +- modules/matrix_context.nf | 2 +- modules/merge_consensus.nf | 2 +- modules/mergebam.nf | 2 +- modules/mergevcfs.nf | 2 +- modules/mosdepth.nf | 2 +- modules/mpileup.nf | 2 +- modules/multiqc.nf | 2 +- modules/normalise_vcf.nf | 2 +- modules/octopus.nf | 2 +- modules/pilot_pinpoint.nf | 2 +- modules/pin_basic.nf | 2 +- modules/pinpy.nf | 2 +- modules/rescue.nf | 3 +- modules/snpeff.nf | 4 +- modules/snpsift_clinvar.nf | 6 +- modules/snpsift_filter.nf | 10 ++-- modules/subset.nf | 2 +- modules/test.nf | 2 +- modules/ubam.nf | 2 +- modules/umi_metrics.nf | 4 +- modules/unique_vcf.nf | 2 +- modules/validate.nf | 2 +- modules/vartable.nf | 2 +- modules/vartable_pins.nf | 2 +- modules/vep.nf | 2 +- 70 files changed, 144 insertions(+), 101 deletions(-) create mode 100644 envs/snpsift/environment.yaml diff --git a/conf/container.config b/conf/container.config index 3b3a5d5..477fd6f 100644 --- a/conf/container.config +++ b/conf/container.config @@ -3,30 +3,65 @@ // Base container image for shell scripts process { - container = 'ubuntu:20.04' + container = 'ubuntu:24.04' } params { - 'container' { - gatk = "quay.io/biocontainers/gatk4:4.5.0.0--py36hdfd78af_0" - samtools = "quay.io/biocontainers/samtools:1.21--h50ea8bc_0" - fastqc = "quay.io/biocontainers/fastqc:0.12.1--hdfd78af_0" - bwa = "quay.io/biocontainers/mulled-v2-e5d375990341c5aef3c9aff74f96f66f65375ef6:2d15960ccea84e249a150b7f5d4db3a42fc2d6c3-0" - bbmap = "quay.io/biocontainers/bbmap:38.89-h1296035_0" - fgbio = "quay.io/biocontainers/fgbio:1.5.1-0" - multiqc = "quay.io/biocontainers/multiqc:1.26--pyhdfd78af_0" - mosdepth = "quay.io/biocontainers/mosdepth:0.3.10--h4e814b3_1" - lofreq = "quay.io/biocontainers/lofreq:2.1.5--py37h3a01142_1" - freebayes = "quay.io/biocontainers/freebayes:1.2.0--py35h82df9c4_2" - bcftools = "quay.io/biocontainers/bcftools:1.19--h8b25389_1" - pinpy = "madscort/dswf_pinpy:1.0" - snpeff = "quay.io/biocontainers/snpeff:4.3--2" - sambamba = "quay.io/biocontainers/sambamba:1.0.1--h6f6fda4_1" - octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" - deepvariant = "quay.io/biocontainers/deepvariant:1.5.0--py36hf3e76ba_0" - r_env = "rocker/tidyverse:latest" - filter_variants = "madscort/dswf_filter:1.0" - vep = "madscort/vep_w_loftee:111.0" - marbl = "madscort/marbl:0.2.0" + container { + docker { + gatk = "quay.io/biocontainers/gatk4:4.5.0.0--py36hdfd78af_0" + samtools = "quay.io/biocontainers/samtools:1.21--h50ea8bc_0" + fastqc = "quay.io/biocontainers/fastqc:0.12.1--hdfd78af_0" + bwa = "quay.io/biocontainers/mulled-v2-e5d375990341c5aef3c9aff74f96f66f65375ef6:2d15960ccea84e249a150b7f5d4db3a42fc2d6c3-0" + bbmap = "quay.io/biocontainers/bbmap:38.89-h1296035_0" + fgbio = "quay.io/biocontainers/fgbio:1.5.1-0" + multiqc = "quay.io/biocontainers/multiqc:1.26--pyhdfd78af_0" + mosdepth = "quay.io/biocontainers/mosdepth:0.3.10--h4e814b3_1" + lofreq = "quay.io/biocontainers/lofreq:2.1.5--py37h3a01142_1" + freebayes = "quay.io/biocontainers/freebayes:1.2.0--py35h82df9c4_2" + bcftools = "quay.io/biocontainers/bcftools:1.19--h8b25389_1" + snpeff = "quay.io/biocontainers/snpeff:4.3--2" + snpsift = "quay.io/biocontainers/snpsift:5.1d--hdfd78af_0" + sambamba = "quay.io/biocontainers/sambamba:1.0.1--h6f6fda4_1" + deepvariant = "quay.io/biocontainers/deepvariant:1.5.0--py36hf3e76ba_0" + + // Custom and non-singularity containers + octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" + r_env = "rocker/tidyverse:latest" + pinpy = "madscort/dswf_pinpy:1.0" + filter_variants = "madscort/dswf_filter:1.0" + vep = "madscort/vep_w_loftee:111.0" + marbl = "madscort/marbl:0.2.0" + } + singularity { + gatk = "https://depot.galaxyproject.org-singularity-gatk4-4.5.0.0--py36hdfd78af_0" + bwa = "https://depot.galaxyproject.org-singularity-mulled-v2-e5d375990341c5aef3c9aff74f96f66f65375ef6-2d15960ccea84e249a150b7f5d4db3a42fc2d6c3-0" + samtools = "https://depot.galaxyproject.org-singularity-samtools-1.20--h50ea8bc_0" + picard = "https://depot.galaxyproject.org-singularity-picard-3.1.1--hdfd78af_0" + fastqc = "https://depot.galaxyproject.org-singularity-fastqc-0.12.1--hdfd78af_0" + fastp = "https://depot.galaxyproject.org-singularity-fastp-0.23.4--h5f740d0_0" + mosdepth = "https://depot.galaxyproject.org-singularity-mosdepth-0.3.8--hd299d5a_0" + multiqc = "https://depot.galaxyproject.org-singularity-multiqc-1.23--pyhdfd78af_0" + deepvariant = "https://depot.galaxyproject.org-nf-core-deepvariant-1.5.0" + strelka = "https://depot.galaxyproject.org-singularity-strelka-2.9.10--h9ee0642_1" + freebayes = "https://depot.galaxyproject.org-singularity-freebayes-1.3.6--hbfe0e7f_2" + lofreq = "https://depot.galaxyproject.org-singularity-lofreq-2.1.5--py38h588ecb2_4" + manta = "https://depot.galaxyproject.org-singularity-manta-1.6.0--h9ee0642_1" + bcftools = "https://depot.galaxyproject.org-singularity-bcftools-1.20--h8b25389_0" + sambamba = "https://depot.galaxyproject.org/singularity/sambamba:1.0.1--h6f6fda4_0" + snpeff = "https://depot.galaxyproject.org-singularity-snpeff-5.1--hdfd78af_2" + snpsift = "https://depot.galaxyproject.org/singularity/snpsift:5.1d--hdfd78af_0" + bedtools = "https://depot.galaxyproject.org-singularity-bedtools-2.31.1--hf5e1c6e_0" + sentieon = "https://depot.galaxyproject.org-singularity-sentieon-202308.02--h43eeafb_0" + fgbio = "https://depot.galaxyproject.org-singularity-fgbio-2.1.0--hdfd78af_0" + + // Custom and non-singularity containers + octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" + r_env = "rocker/tidyverse:latest" + pinpy = "madscort/dswf_pinpy:1.0" + filter_variants = "madscort/dswf_filter:1.0" + vep = "madscort/vep_w_loftee:111.0" + marbl = "madscort/marbl:0.2.0" + } } -} \ No newline at end of file +} diff --git a/envs/snpsift/environment.yaml b/envs/snpsift/environment.yaml new file mode 100644 index 0000000..23c6529 --- /dev/null +++ b/envs/snpsift/environment.yaml @@ -0,0 +1,7 @@ +name: snpsift +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - snpsift=5.1 \ No newline at end of file diff --git a/modules/addreadgroup.nf b/modules/addreadgroup.nf index f6c8889..d218d3a 100644 --- a/modules/addreadgroup.nf +++ b/modules/addreadgroup.nf @@ -3,7 +3,7 @@ process ADDREADGROUP { tag "Add read group - $sample_id" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk input: tuple val(sample_id), path(bam_file, stageAs: "raw/*") diff --git a/modules/alignment.nf b/modules/alignment.nf index b2649c4..1464fe1 100644 --- a/modules/alignment.nf +++ b/modules/alignment.nf @@ -4,8 +4,8 @@ process ALIGNMENT { // Align reads to the reference genome using BWA, convert to BAM and sort. conda "$projectDir/envs/bwa/environment.yaml" - container params.container.bwa - + container workflow.containerEngine == 'singularity' ? params.container.singularity.bwa : params.container.docker.bwa + publishDir "${params.outputDir}/log/mapping/", pattern: "${sample_id}.log", mode:'copy' input: diff --git a/modules/alignment_metrics.nf b/modules/alignment_metrics.nf index f642b22..7fac332 100644 --- a/modules/alignment_metrics.nf +++ b/modules/alignment_metrics.nf @@ -4,7 +4,7 @@ process ALIGNMENT_METRICS { // Collect alignment metrics for bam file conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/alignment_metrics/", pattern: "${sample_id}*.align.txt", mode:'copy' diff --git a/modules/alignment_umi.nf b/modules/alignment_umi.nf index 3809349..8a4a197 100644 --- a/modules/alignment_umi.nf +++ b/modules/alignment_umi.nf @@ -4,6 +4,7 @@ process ALIGNMENT_UMI { // Align reads to the reference genome using BWA, convert to BAM and sort by QUERY NAME. conda "$projectDir/envs/bwa_umi/environment.yaml" + container workflow.containerEngine == 'singularity' ? params.container.singularity.fgbio : params.container.docker.fgbio publishDir "${params.outputDir}/log/mapping/", pattern: "${sample_id}.log", mode:'copy' diff --git a/modules/apply_bqsr.nf b/modules/apply_bqsr.nf index 707e7ff..f1cf669 100644 --- a/modules/apply_bqsr.nf +++ b/modules/apply_bqsr.nf @@ -3,7 +3,7 @@ process APPLY_BQSR { tag "Apply score recalibration - $sample_id" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk input: tuple val(sample_id), path(bam_file, stageAs: "raw/*"), path(bqsr_table) diff --git a/modules/bam.nf b/modules/bam.nf index f34b97e..4b3f3e7 100644 --- a/modules/bam.nf +++ b/modules/bam.nf @@ -3,7 +3,7 @@ process BAM { tag "CRAM->BAM - $sample_id" conda "$projectDir/envs/samtools/environment.yaml" - container params.container.samtools + container workflow.containerEngine == 'singularity' ? params.container.singularity.samtools : params.container.docker.samtools input: tuple val(sample_id), path(cram_file) diff --git a/modules/bed_annotate.nf b/modules/bed_annotate.nf index a3d323a..bc3e0a9 100644 --- a/modules/bed_annotate.nf +++ b/modules/bed_annotate.nf @@ -4,7 +4,7 @@ process BED_ANNOTATE { // Annotate VCF by third column in BED file conda "$projectDir/envs/bcftools/environment.yaml" - container params.container.bcftools + container workflow.containerEngine == 'singularity' ? params.container.singularity.bcftools : params.container.docker.bcftools publishDir "${params.outputDir}/log/bed_annotate/${sample_id}/", pattern: "${sample_id}.${caller}.log", mode:'copy' diff --git a/modules/bgzip.nf b/modules/bgzip.nf index b598c0b..a592996 100644 --- a/modules/bgzip.nf +++ b/modules/bgzip.nf @@ -3,7 +3,7 @@ process BGZIP { tag "VCF->VCF.gz - $sample_id" conda "$projectDir/envs/samtools/environment.yaml" - container params.container.samtools + container workflow.containerEngine == 'singularity' ? params.container.singularity.samtools : params.container.docker.samtools publishDir "${params.outputDir}/norm_bgzip_idx/", pattern: "*vcf.*", mode:'copy' diff --git a/modules/bqsr.nf b/modules/bqsr.nf index 800960d..bee3274 100644 --- a/modules/bqsr.nf +++ b/modules/bqsr.nf @@ -3,7 +3,7 @@ process BQSR { tag "Calculate score recalibration - $sample_id" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk input: tuple val(sample_id), path(bam_file) diff --git a/modules/call_consensus.nf b/modules/call_consensus.nf index 01217bc..1558f63 100644 --- a/modules/call_consensus.nf +++ b/modules/call_consensus.nf @@ -3,7 +3,7 @@ process CALL_CONSENSUS { tag "Call consensus reads - $sample_id" conda "$projectDir/envs/fgbio/environment.yaml" - container params.container.fgbio + container workflow.containerEngine == 'singularity' ? params.container.singularity.fgbio : params.container.docker.fgbio input: tuple val(sample_id), path(bam_file) diff --git a/modules/clean.nf b/modules/clean.nf index 6bbbe08..bdb4435 100644 --- a/modules/clean.nf +++ b/modules/clean.nf @@ -3,7 +3,7 @@ process CLEAN { tag "Clean bam file - $sample_id" conda "$projectDir/envs/samtools/environment.yaml" - container params.container.samtools + container workflow.containerEngine == 'singularity' ? params.container.singularity.samtools : params.container.docker.samtools input: tuple val(sample_id), path(bam_file) diff --git a/modules/combinegvcfs.nf b/modules/combinegvcfs.nf index 538c3ce..b1eb4d9 100644 --- a/modules/combinegvcfs.nf +++ b/modules/combinegvcfs.nf @@ -4,7 +4,7 @@ process COMBINEGVCFS { // Merge gVCF files into a single GenomicsDB conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/", pattern: "genomicsdb.log", mode:'copy' publishDir "${params.outputDir}/splits/", pattern: "cohort.${interval}.g.vcf.gz", mode:'copy' diff --git a/modules/cram.nf b/modules/cram.nf index 9908cd8..9e02208 100644 --- a/modules/cram.nf +++ b/modules/cram.nf @@ -3,7 +3,7 @@ process CRAM { tag "BAM->CRAM - $sample_id" conda "$projectDir/envs/samtools/environment.yaml" - container params.container.samtools + container workflow.containerEngine == 'singularity' ? params.container.singularity.samtools : params.container.docker.samtools publishDir "${params.outputDir}/cram/", pattern: "${sample_id}.cram", mode:'copy' diff --git a/modules/deepvariant.nf b/modules/deepvariant.nf index 76e326b..2db9bd2 100644 --- a/modules/deepvariant.nf +++ b/modules/deepvariant.nf @@ -7,7 +7,7 @@ process DEEPVARIANT { // This version works on NGC only. To run locally, comment out "singularity exec" and use "run_deepvariant" directly. conda "$projectDir/envs/deepvar/environment.yaml" - container params.container.deepvariant + container workflow.containerEngine == 'singularity' ? params.container.singularity.deepvariant : params.container.docker.deepvariant publishDir "${params.outputDir}/log/deepvariant/", pattern: "${sample_id}.DV.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.DV.vcf.gz", mode:'copy' @@ -26,8 +26,7 @@ process DEEPVARIANT { def ref_dir = file(params.reference_genome).getParent() def target_dir = file(params.bedfile).getParent() """ - # singularity exec --bind ${PWD} --bind ${ref_dir} --bind ${target_dir} /services/tools/deepvariant/1.5.0/deepvariant_1.5.0.sif \ - run_deepvariant \ + /opt/deepvariant/bin/run_deepvariant --ref=${db} \ --reads=$bam_file \ --output_vcf=${sample_id}.DV.vcf.gz \ diff --git a/modules/downsample.nf b/modules/downsample.nf index f5f7162..87ce742 100644 --- a/modules/downsample.nf +++ b/modules/downsample.nf @@ -9,7 +9,7 @@ process DOWNSAMPLE { // Approximately 40% is on target - so cap set at 400M reads. 200M per file. conda "$projectDir/envs/bbmap/environment.yaml" - container params.container.bbmap + container workflow.containerEngine == 'singularity' ? params.container.singularity.bbmap : params.container.docker.bbmap input: tuple val(sample_id), path(reads, stageAs: 'raw/*') diff --git a/modules/drop_pl.nf b/modules/drop_pl.nf index 086810b..fdfb91d 100644 --- a/modules/drop_pl.nf +++ b/modules/drop_pl.nf @@ -4,7 +4,7 @@ process DROP_PL { // Drop PL tag from VCF file. conda "$projectDir/envs/bcftools/environment.yaml" - container params.container.bcftools + container workflow.containerEngine == 'singularity' ? params.container.singularity.bcftools : params.container.docker.bcftools input: tuple val(sample_id), path(vcf_file, stageAs: "input/*"), path(index, stageAs: "input/*") diff --git a/modules/duplicate_metrics.nf b/modules/duplicate_metrics.nf index 9d0bfa8..1088c03 100644 --- a/modules/duplicate_metrics.nf +++ b/modules/duplicate_metrics.nf @@ -4,7 +4,7 @@ process DUPLICATE_METRICS { // Mark duplicate reads in BAM files conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/duplicate_metrics/", pattern: "${sample_id}*.txt", mode:'copy' diff --git a/modules/extract_umi.nf b/modules/extract_umi.nf index b974fd8..bf310d7 100644 --- a/modules/extract_umi.nf +++ b/modules/extract_umi.nf @@ -10,7 +10,7 @@ process EXTRACT_UMI { // Twist UMI adapters are 5 base pairs with a 2 base pair skip, resulting in the read structure 5M2S+T conda "$projectDir/envs/fgbio/environment.yaml" - container params.container.fgbio + container workflow.containerEngine == 'singularity' ? params.container.singularity.fgbio : params.container.docker.fgbio input: tuple val(sample_id), path(bam_file, stageAs: "raw/*") diff --git a/modules/fastq.nf b/modules/fastq.nf index 1566c70..2631777 100644 --- a/modules/fastq.nf +++ b/modules/fastq.nf @@ -4,7 +4,7 @@ process FASTQ { // Convert unaligned uBAM files to FastQ. conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk input: tuple val(sample_id), path(bam_file) diff --git a/modules/fastqc.nf b/modules/fastqc.nf index fb8bc94..e800808 100644 --- a/modules/fastqc.nf +++ b/modules/fastqc.nf @@ -3,7 +3,7 @@ process FASTQC { tag "FastQC - ${sample_id}" conda "$projectDir/envs/fastqc/environment.yaml" - container params.container.fastqc + container workflow.containerEngine == 'singularity' ? params.container.singularity.fastqc : params.container.docker.fastqc publishDir "${params.outputDir}/log/fastqc/", pattern: "${sample_id}_fastqc.html", mode:'copy' publishDir "${params.outputDir}/log/fastqc/", pattern: "${sample_id}_fastqc.zip", mode:'copy' diff --git a/modules/filter.nf b/modules/filter.nf index 6bffa31..a9d0aea 100644 --- a/modules/filter.nf +++ b/modules/filter.nf @@ -1,7 +1,7 @@ process FILTER { label 'process_low' conda "$projectDir/envs/bcftools/environment.yaml" - container params.container.bcftools + container workflow.containerEngine == 'singularity' ? params.container.singularity.bcftools : params.container.docker.bcftools publishDir "${params.outputDir}/variants/filtered/", pattern: "${sample_id}.lofreq.vcf.gz", mode:'copy' diff --git a/modules/filter_variants.nf b/modules/filter_variants.nf index c92a19a..0cbbadb 100644 --- a/modules/filter_variants.nf +++ b/modules/filter_variants.nf @@ -1,7 +1,7 @@ process FILTER_VARIANTS { label 'process_low' conda "$projectDir/envs/filter_variants/environment.yaml" - container params.container.filter_variants + container workflow.containerEngine == 'singularity' ? params.container.singularity.filter_variants : params.container.docker.filter_variants publishDir "${params.outputDir}/filtered_variants/", pattern: "*.vcf", mode:'copy' diff --git a/modules/flagstat.nf b/modules/flagstat.nf index 50ca95f..9adbe99 100644 --- a/modules/flagstat.nf +++ b/modules/flagstat.nf @@ -3,7 +3,7 @@ process FLAGSTAT { tag "Flagstat - $sample_id" conda "$projectDir/envs/samtools/environment.yaml" - container params.container.samtools + container workflow.containerEngine == 'singularity' ? params.container.singularity.samtools : params.container.docker.samtools publishDir "${params.outputDir}/log/flagstat/", pattern: "${sample_id}*.flagstat", mode:'copy' diff --git a/modules/freebayes.nf b/modules/freebayes.nf index b0414c9..85c7d51 100644 --- a/modules/freebayes.nf +++ b/modules/freebayes.nf @@ -4,7 +4,7 @@ process FREEBAYES { // Call variants using Lofreq - call parallel conda "$projectDir/envs/freebayes/environment.yaml" - container params.container.freebayes + container workflow.containerEngine == 'singularity' ? params.container.singularity.freebayes : params.container.docker.freebayes publishDir "${params.outputDir}/log/freebayes/", pattern: "${sample_id}.freebayes.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.freebayes.vcf.gz", mode:'copy' diff --git a/modules/gc_metrics.nf b/modules/gc_metrics.nf index 710d72b..86750bf 100644 --- a/modules/gc_metrics.nf +++ b/modules/gc_metrics.nf @@ -4,7 +4,7 @@ process GC_METRICS { // Collect gc bias metrics for bam file conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/gc_bias_metrics/", pattern: "${sample_id}*.gc.txt", mode:'copy' publishDir "${params.outputDir}/log/gc_bias_metrics/", pattern: "${sample_id}*.gc.summary.txt", mode:'copy' diff --git a/modules/genomicsdb.nf b/modules/genomicsdb.nf index 975d78d..d0c72ea 100644 --- a/modules/genomicsdb.nf +++ b/modules/genomicsdb.nf @@ -4,7 +4,7 @@ process GENOMICSDB { // Merge gVCF files into a single GenomicsDB conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/", pattern: "genomicsdb.log", mode:'copy' publishDir "${params.outputDir}/", pattern: "./genomicsdb", mode:'copy' diff --git a/modules/genotypegvcf.nf b/modules/genotypegvcf.nf index 342821f..de4c6be 100644 --- a/modules/genotypegvcf.nf +++ b/modules/genotypegvcf.nf @@ -4,7 +4,7 @@ process GENOTYPEGVCF { // Call variants on multisample gVCF db conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/genotypegvcf/", pattern: "${sample_id}.genotypegvcf.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.GATK.vcf.gz", mode:'copy' diff --git a/modules/genotypegvcf_split.nf b/modules/genotypegvcf_split.nf index c0c760b..2b4bef9 100644 --- a/modules/genotypegvcf_split.nf +++ b/modules/genotypegvcf_split.nf @@ -4,7 +4,7 @@ process GENOTYPEGVCF_SPLIT { // Call variants on multisample gVCF db conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/genotypegvcf/${sample_id}/", pattern: "${sample_id}*.genotypegvcf.log", mode:'copy' diff --git a/modules/group_umi.nf b/modules/group_umi.nf index a68e542..5bb8154 100644 --- a/modules/group_umi.nf +++ b/modules/group_umi.nf @@ -3,7 +3,7 @@ process GROUP_UMI { tag "Group reads by UMI - $sample_id" conda "$projectDir/envs/fgbio/environment.yaml" - container params.container.fgbio + container workflow.containerEngine == 'singularity' ? params.container.singularity.fgbio : params.container.docker.fgbio publishDir "${params.outputDir}/log/group_umi/", pattern: "${sample_id}.family_size_histogram.txt", mode:'copy' diff --git a/modules/haplotypecaller.nf b/modules/haplotypecaller.nf index cdde1de..5544426 100644 --- a/modules/haplotypecaller.nf +++ b/modules/haplotypecaller.nf @@ -4,7 +4,7 @@ process HAPLOTYPECALLER { // Call variants using GATK - HaplotypeCaller conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/haplotypecaller/", pattern: "${sample_id}.g.haplotypecaller.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.GATK.g.vcf.gz", mode:'copy' diff --git a/modules/haplotypecaller_split.nf b/modules/haplotypecaller_split.nf index df310a3..16a492b 100644 --- a/modules/haplotypecaller_split.nf +++ b/modules/haplotypecaller_split.nf @@ -4,7 +4,7 @@ process HAPLOTYPECALLER_SPLIT { // Call variants using GATK - HaplotypeCaller conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/haplotypecaller/${sample_id}/", pattern: "${sample_id}*.g.haplotypecaller.log", mode:'copy' publishDir "${params.outputDir}/variants/gvcf/${sample_id}/", pattern: "${sample_id}*.GATK.g.vcf.gz", mode:'copy' diff --git a/modules/haplotypecaller_truth.nf b/modules/haplotypecaller_truth.nf index 73b5808..a0c6f55 100644 --- a/modules/haplotypecaller_truth.nf +++ b/modules/haplotypecaller_truth.nf @@ -4,7 +4,7 @@ process HC_TRUTH { // Call variants using GATK - HaplotypeCaller conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/haplotypecaller/", pattern: "${sample_id}.haplotypecaller.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.GATK.vcf.gz", mode:'copy' diff --git a/modules/haplotypecaller_truth_joint.nf b/modules/haplotypecaller_truth_joint.nf index e99ab2b..c169bb3 100644 --- a/modules/haplotypecaller_truth_joint.nf +++ b/modules/haplotypecaller_truth_joint.nf @@ -4,7 +4,7 @@ process HC_TRUTH_JOINT { // Call variants using GATK - HaplotypeCaller conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/haplotypecaller/", pattern: "${sample_id}.g.haplotypecaller.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.GATK.g.vcf.gz", mode:'copy' diff --git a/modules/hs_metrics.nf b/modules/hs_metrics.nf index febd737..f67d8bb 100644 --- a/modules/hs_metrics.nf +++ b/modules/hs_metrics.nf @@ -4,7 +4,7 @@ process HS_METRICS { // Collect HSMetrics for bam file conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/hs_metrics/", pattern: "${sample_id}*.hs.txt", mode:'copy' diff --git a/modules/indelqual.nf b/modules/indelqual.nf index 5aca450..94cee3c 100644 --- a/modules/indelqual.nf +++ b/modules/indelqual.nf @@ -4,7 +4,7 @@ process INDELQUAL { // Tag indel quality using Lofreq - indelqual conda "$projectDir/envs/lofreq/environment.yaml" - container params.container.lofreq + container workflow.containerEngine == 'singularity' ? params.container.singularity.lofreq : params.container.docker.lofreq publishDir "${params.outputDir}/log/lofreq_indelqual/", pattern: "${sample_id}.lofreq.iq.log", mode:'copy' diff --git a/modules/index.nf b/modules/index.nf index 8407e1b..fe8cbd5 100644 --- a/modules/index.nf +++ b/modules/index.nf @@ -3,7 +3,7 @@ process INDEX { tag "Index bam file - $sample_id" conda "$projectDir/envs/samtools/environment.yaml" - container params.container.samtools + container workflow.containerEngine == 'singularity' ? params.container.singularity.samtools : params.container.docker.samtools input: tuple val(sample_id), path(bam_file) diff --git a/modules/index_vcf.nf b/modules/index_vcf.nf index f1c65e0..676f423 100644 --- a/modules/index_vcf.nf +++ b/modules/index_vcf.nf @@ -3,7 +3,7 @@ process INDEX_VCF { tag "Index VCF $sample_id" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/index_vcf/", pattern: "${sample_id}.${caller}.log", mode:'copy' diff --git a/modules/insert_size_metrics.nf b/modules/insert_size_metrics.nf index 6f34489..6960097 100644 --- a/modules/insert_size_metrics.nf +++ b/modules/insert_size_metrics.nf @@ -4,7 +4,7 @@ process INSERT_SIZE_METRICS { // Collect insert size metrics for bam file conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/insert_size_metrics/", pattern: "${sample_id}*.insert.txt", mode:'copy' publishDir "${params.outputDir}/log/insert_size_metrics/", pattern: "${sample_id}.pdf", mode:'copy' diff --git a/modules/intervals.nf b/modules/intervals.nf index 2b335f3..15b2484 100644 --- a/modules/intervals.nf +++ b/modules/intervals.nf @@ -3,7 +3,7 @@ process INTERVALS { tag "BedFileToIntervalList" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk input: path reference_genome diff --git a/modules/lofreq.nf b/modules/lofreq.nf index 9272533..6e8131e 100644 --- a/modules/lofreq.nf +++ b/modules/lofreq.nf @@ -4,7 +4,7 @@ process LOFREQ { // Call variants using Lofreq - call parallel conda "$projectDir/envs/lofreq/environment.yaml" - container params.container.lofreq + container workflow.containerEngine == 'singularity' ? params.container.singularity.lofreq : params.container.docker.lofreq publishDir "${params.outputDir}/log/lofreq/", pattern: "${sample_id}.lofreq.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.lofreq.vcf.gz", mode:'copy' diff --git a/modules/markduplicates.nf b/modules/markduplicates.nf index 0a165ac..69c437f 100644 --- a/modules/markduplicates.nf +++ b/modules/markduplicates.nf @@ -4,7 +4,7 @@ process MARKDUPLICATES { // Mark duplicate reads in BAM files conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/markduplicates/", pattern: "${sample_id}*.log", mode:'copy' diff --git a/modules/markduplicates_fast.nf b/modules/markduplicates_fast.nf index 91b961d..1dea1e0 100644 --- a/modules/markduplicates_fast.nf +++ b/modules/markduplicates_fast.nf @@ -4,7 +4,7 @@ process MARKDUPLICATES_FAST { // Mark duplicate reads in BAM files conda "$projectDir/envs/sambamba/environment.yaml" - container params.container.sambamba + container workflow.containerEngine == 'singularity' ? params.container.singularity.sambamba : params.container.docker.sambamba publishDir "${params.outputDir}/log/markdup_sambamba/", pattern: "${sample_id}*.log", mode:'copy' diff --git a/modules/markduplicates_spark.nf b/modules/markduplicates_spark.nf index acebd17..f544f76 100644 --- a/modules/markduplicates_spark.nf +++ b/modules/markduplicates_spark.nf @@ -4,7 +4,7 @@ process MARKDUPLICATES_SPARK { // Mark duplicate reads in BAM files conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/markduplicates/", pattern: "${sample_id}*.log", mode:'copy' diff --git a/modules/matrix_context.nf b/modules/matrix_context.nf index fdcc1bf..5d7ee64 100644 --- a/modules/matrix_context.nf +++ b/modules/matrix_context.nf @@ -1,7 +1,7 @@ process MATRIX_CONTEXT { label 'process_low' conda "$projectDir/envs/pinpy/environment.yaml" - container params.container.pinpy + container workflow.containerEngine == 'singularity' ? params.container.singularity.pinpy : params.container.docker.pinpy publishDir "${params.outputDir}/context/", mode:'copy', pattern: "matrix_context.*" publishDir "${params.outputDir}/context/", mode:'copy', pattern: "decodetable.tsv" diff --git a/modules/merge_consensus.nf b/modules/merge_consensus.nf index d7707e4..8235981 100644 --- a/modules/merge_consensus.nf +++ b/modules/merge_consensus.nf @@ -3,7 +3,7 @@ process MERGE_CONSENSUS { tag "Merge consensus bam files - $sample_id" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk input: tuple val(sample_id), path(bam_file, stageAs: "raw/*"), path(ubam_file, stageAs: "raw/*") diff --git a/modules/mergebam.nf b/modules/mergebam.nf index e03b84f..d1348a2 100644 --- a/modules/mergebam.nf +++ b/modules/mergebam.nf @@ -3,7 +3,7 @@ process MERGEBAM { tag "Merge bam files - $sample_id" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk input: tuple val(sample_id), path(bam_file, stageAs: "raw/*"), path(ubam_file, stageAs: "raw/*") diff --git a/modules/mergevcfs.nf b/modules/mergevcfs.nf index 3845fba..4cdceff 100644 --- a/modules/mergevcfs.nf +++ b/modules/mergevcfs.nf @@ -4,7 +4,7 @@ process MERGEVCFS { // Call variants using GATK - HaplotypeCaller conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/mergevcfs/", pattern: "${sample_id}.mergevcfs.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.GATK.vcf.gz", mode:'copy' diff --git a/modules/mosdepth.nf b/modules/mosdepth.nf index 7cb93fb..6ef7d80 100644 --- a/modules/mosdepth.nf +++ b/modules/mosdepth.nf @@ -6,7 +6,7 @@ process MOSDEPTH { // 5x allele depth (240), 10x allele depth (480), 30x allele depth (1440), 50x allele depth (2400) conda "$projectDir/envs/mosdepth/environment.yaml" - container params.container.mosdepth + container workflow.containerEngine == 'singularity' ? params.container.singularity.mosdepth : params.container.docker.mosdepth publishDir "${params.outputDir}/log/mosdepth/", pattern: "${sample_id}*.per-base.bed.gz", mode:'copy' publishDir "${params.outputDir}/log/mosdepth/", pattern: "${sample_id}*.regions.bed.gz", mode:'copy' diff --git a/modules/mpileup.nf b/modules/mpileup.nf index 866b5ca..ef12e8a 100644 --- a/modules/mpileup.nf +++ b/modules/mpileup.nf @@ -3,7 +3,7 @@ process MPILEUP { tag "BAM->mpileup - $sample_id" conda "$projectDir/envs/samtools/environment.yaml" - container params.container.samtools + container workflow.containerEngine == 'singularity' ? params.container.singularity.samtools : params.container.docker.samtools publishDir "${params.outputDir}/mpileup/", mode:'copy', pattern: "${sample_id}.mpileup.gz" diff --git a/modules/multiqc.nf b/modules/multiqc.nf index beb6a61..42ce000 100644 --- a/modules/multiqc.nf +++ b/modules/multiqc.nf @@ -3,7 +3,7 @@ process MULTIQC { tag "MultiQC everything" conda "$projectDir/envs/multiqc/environment.yaml" - container params.container.multiqc + container workflow.containerEngine == 'singularity' ? params.container.singularity.multiqc : params.container.docker.multiqc publishDir "${params.outputDir}/log/multiqc/", mode:'copy' diff --git a/modules/normalise_vcf.nf b/modules/normalise_vcf.nf index 540649d..5e3662b 100644 --- a/modules/normalise_vcf.nf +++ b/modules/normalise_vcf.nf @@ -6,7 +6,7 @@ process NORMALISE_VCF { // 3. Checks refs against reference genome conda "$projectDir/envs/bcftools/environment.yaml" - container params.container.bcftools + container workflow.containerEngine == 'singularity' ? params.container.singularity.bcftools : params.container.docker.bcftools publishDir "${params.outputDir}/log/normalise_vcf/${sample_id}/", pattern: "${sample_id}.${caller}.log", mode:'copy' publishDir "${params.outputDir}/normalised_variants/", pattern: "${sample_id}.${caller}.vcf", mode:'copy' diff --git a/modules/octopus.nf b/modules/octopus.nf index 00aad12..3cd4b4e 100644 --- a/modules/octopus.nf +++ b/modules/octopus.nf @@ -8,7 +8,7 @@ process OCTOPUS { // This version works on NGC only. To run locally, comment out "singularity exec" and use "octopus" directly. conda "$projectDir/envs/octopus/environment.yaml" - container params.container.octopus + container workflow.containerEngine == 'singularity' ? params.container.singularity.octopus : params.container.docker.octopus publishDir "${params.outputDir}/log/octopus/", pattern: "${sample_id}.octopus.log", mode:'copy' publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.octopus.vcf.gz", mode:'copy' diff --git a/modules/pilot_pinpoint.nf b/modules/pilot_pinpoint.nf index bcdb9ad..a4030fe 100644 --- a/modules/pilot_pinpoint.nf +++ b/modules/pilot_pinpoint.nf @@ -1,7 +1,7 @@ process PILOT_PINPOINT { label 'process_low' conda "$projectDir/envs/r_env/environment.yaml" - container params.container.r_env + container workflow.containerEngine == 'singularity' ? params.container.singularity.r_env : params.container.docker.r_env publishDir "${params.outputDir}/pinned_variants/", pattern: "*.tsv", mode:'copy' publishDir "${params.outputDir}/pinned_variants/outlier_plots/", pattern: "*.pdf", mode: 'copy' diff --git a/modules/pin_basic.nf b/modules/pin_basic.nf index dc44483..34061ee 100644 --- a/modules/pin_basic.nf +++ b/modules/pin_basic.nf @@ -1,7 +1,7 @@ process PIN_BASIC { label 'process_low' conda "$projectDir/envs/filter_variants/environment.yaml" - container params.container.filter_variants + container workflow.containerEngine == 'singularity' ? params.container.singularity.filter_variants : params.container.docker.filter_variants publishDir "${params.outputDir}/variant_compilation/", mode:'copy', pattern: "*variants.tsv" diff --git a/modules/pinpy.nf b/modules/pinpy.nf index 0fdc6b1..ac0b261 100644 --- a/modules/pinpy.nf +++ b/modules/pinpy.nf @@ -1,7 +1,7 @@ process PINPY { label 'process_low' conda "$projectDir/envs/pinpy/environment.yaml" - container params.container.pinpy + container workflow.containerEngine == 'singularity' ? params.container.singularity.pinpy : params.container.docker.pinpy publishDir "${params.outputDir}/", mode:'copy' diff --git a/modules/rescue.nf b/modules/rescue.nf index b2a43cb..6e71562 100644 --- a/modules/rescue.nf +++ b/modules/rescue.nf @@ -1,7 +1,8 @@ process RESCUE { label 'process_low' conda "$projectDir/envs/rescue/environment.yaml" - container params.container.marbl + container workflow.containerEngine == 'singularity' ? params.container.singularity.marbl : params.container.docker.marbl + publishDir "${params.outputDir}/rescue_probabilities/", mode:'copy', pattern: "predictions.tsv" diff --git a/modules/snpeff.nf b/modules/snpeff.nf index 09745ae..8b8e90b 100644 --- a/modules/snpeff.nf +++ b/modules/snpeff.nf @@ -8,7 +8,7 @@ process SNPEFF { // -noMotif avoids adding motif annotations. conda "$projectDir/envs/snpeff/environment.yaml" - container params.container.snpeff + container workflow.containerEngine == 'singularity' ? params.container.singularity.snpeff : params.container.docker.snpeff publishDir "${params.outputDir}/log/snpeff/${sample_id}/", pattern: "${sample_id}.${caller}.log", mode:'copy' publishDir "${params.outputDir}/log/snpeff/stats/", pattern: "${sample_id}.${caller}_snpeff_stats.csv", mode:'copy' @@ -27,7 +27,7 @@ process SNPEFF { script: """ - ${params.snpeff} \ + snpEff \ ${snpeff_db} \ -c ${snpeff_config} \ -csvStats "${sample_id}.${caller}_snpeff_stats.csv" \ diff --git a/modules/snpsift_clinvar.nf b/modules/snpsift_clinvar.nf index e50c87a..af13739 100644 --- a/modules/snpsift_clinvar.nf +++ b/modules/snpsift_clinvar.nf @@ -4,8 +4,8 @@ process SNPSIFT_CLINVAR { // Add ClinVar annotations to a VCF file using SnpSift. - conda "$projectDir/envs/snpeff/environment.yaml" - container params.container.snpeff + conda "$projectDir/envs/snpsift/environment.yaml" + container workflow.containerEngine == 'singularity' ? params.container.singularity.snpsift : params.container.docker.snpsift publishDir "${params.outputDir}/log/snpsift_clinvar/${sample_id}/", pattern: "${sample_id}.${caller}.log", mode:'copy' publishDir "${params.outputDir}/annotated_variants/", pattern: "${sample_id}.${caller}.vcf", mode:'copy' @@ -22,7 +22,7 @@ process SNPSIFT_CLINVAR { script: def db = file(params.clinvar_db).getName() """ - ${params.snpsift} annotate \ + SnpSift annotate \ ${db} \ ${vcf_file} \ > ${sample_id}.${caller}.vcf \ diff --git a/modules/snpsift_filter.nf b/modules/snpsift_filter.nf index 35720fc..ea2dd31 100644 --- a/modules/snpsift_filter.nf +++ b/modules/snpsift_filter.nf @@ -4,8 +4,8 @@ process SNPSIFT_FILTER { // Filter variants based on ClinVar and SNPEff annotations using SnpSift. - conda "$projectDir/envs/snpeff/environment.yaml" - container params.container.snpeff + conda "$projectDir/envs/snpsift/environment.yaml" + container workflow.containerEngine == 'singularity' ? params.container.singularity.snpsift : params.container.docker.snpsift publishDir "${params.outputDir}/annotated_variants/lof/", pattern: "${sample_id}.${caller}.lof.vcf", mode:'copy' publishDir "${params.outputDir}/annotated_variants/pathogenic/", pattern: "${sample_id}.${caller}.p.vcf", mode:'copy' @@ -22,17 +22,17 @@ process SNPSIFT_FILTER { script: """ - ${params.snpsift} filter \ + SnpSift filter \ "(exists LOF)" \ ${vcf_file} \ > ${sample_id}.${caller}.lof.vcf - ${params.snpsift} filter \ + SnpSift filter \ "CLNSIG =~ 'Pathogenic'" \ ${vcf_file} \ > ${sample_id}.${caller}.p.vcf - ${params.snpsift} filter \ + SnpSift filter \ "((exists LOF) | (CLNSIG =~ 'Pathogenic'))" \ ${vcf_file} \ > ${sample_id}.${caller}.lofp.vcf diff --git a/modules/subset.nf b/modules/subset.nf index e960ab7..d13fbcb 100644 --- a/modules/subset.nf +++ b/modules/subset.nf @@ -3,7 +3,7 @@ process SUBSET { tag "Subset alignment and convert to bam - $sample_id" conda "$projectDir/envs/samtools/environment.yaml" - container params.container.samtools + container workflow.containerEngine == 'singularity' ? params.container.singularity.samtools : params.container.docker.samtools input: tuple val(sample_id), path(bam_file, stageAs: 'raw/*') diff --git a/modules/test.nf b/modules/test.nf index 357b284..049c96d 100644 --- a/modules/test.nf +++ b/modules/test.nf @@ -1,7 +1,7 @@ process TEST { label 'process_single' conda "$projectDir/envs/pinpy/environment.yaml" - container params.container.pinpy + container workflow.containerEngine == 'singularity' ? params.container.singularity.pinpy : params.container.docker.pinpy publishDir "${params.outputDir}/test/", pattern: "test.*", mode:'copy' diff --git a/modules/ubam.nf b/modules/ubam.nf index fb6cd90..6704ea3 100644 --- a/modules/ubam.nf +++ b/modules/ubam.nf @@ -4,7 +4,7 @@ process UBAM { // Convert FastQ files to unaligned BAM files. conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk input: tuple val(sample_id), path(reads) diff --git a/modules/umi_metrics.nf b/modules/umi_metrics.nf index 1200b1b..91d9494 100644 --- a/modules/umi_metrics.nf +++ b/modules/umi_metrics.nf @@ -17,7 +17,7 @@ process UMI_METRICS { // DS: double-stranded tag families where membership is collapsed across single-stranded tag families from the same double-stranded source molecule; i.e. 50/A and 50/B become one family conda "$projectDir/envs/fgbio/environment.yaml" - container params.container.fgbio + container workflow.containerEngine == 'singularity' ? params.container.singularity.fgbio : params.container.docker.fgbio publishDir "${params.outputDir}/log/umi_metrics/logs/", pattern: "${sample_id}.*.txt", mode:'copy' publishDir "${params.outputDir}/log/umi_metrics/plots/", pattern: "${sample_id}.*.pdf", mode:'copy' @@ -34,7 +34,7 @@ process UMI_METRICS { script: """ - ${params.fgbio} --tmp-dir=. --compression 1 --async-io CollectDuplexSeqMetrics \ + fgbio --tmp-dir=. --compression 1 --async-io CollectDuplexSeqMetrics \ --input=${bam_file} \ --output="${sample_id}" """ diff --git a/modules/unique_vcf.nf b/modules/unique_vcf.nf index 21c9c0b..8af2bcb 100644 --- a/modules/unique_vcf.nf +++ b/modules/unique_vcf.nf @@ -3,7 +3,7 @@ process UNIQUE_VCF { // Merge, deduplicate, remove sample information, and index a set of VCF files. conda "$projectDir/envs/bcftools/environment.yaml" - container params.container.bcftools + container workflow.containerEngine == 'singularity' ? params.container.singularity.bcftools : params.container.docker.bcftools publishDir "${params.outputDir}/vep_annotate/", pattern: "unique.vcf.*", mode:'copy' diff --git a/modules/validate.nf b/modules/validate.nf index 2a55e29..7084d4e 100644 --- a/modules/validate.nf +++ b/modules/validate.nf @@ -3,7 +3,7 @@ process VALIDATE { tag "Validate bam file - $sample_id" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/validation/", pattern: "${sample_id}.validation.log", mode:'copy' diff --git a/modules/vartable.nf b/modules/vartable.nf index 1ed860b..203a590 100644 --- a/modules/vartable.nf +++ b/modules/vartable.nf @@ -3,7 +3,7 @@ process VARTABLE { tag "VCF to TSV $sample_id" conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/log/variant_tables/${sample_id}/", pattern: "${sample_id}.${caller}.log", mode:'copy' publishDir "${params.outputDir}/variant_tables/", pattern: "${sample_id}.${caller}.tsv", mode:'copy' diff --git a/modules/vartable_pins.nf b/modules/vartable_pins.nf index c64c755..2c1509a 100644 --- a/modules/vartable_pins.nf +++ b/modules/vartable_pins.nf @@ -1,7 +1,7 @@ process VARTABLE_PINS { label 'process_single' conda "$projectDir/envs/gatk4/environment.yaml" - container params.container.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk publishDir "${params.outputDir}/pinpoint_variant_tables/", mode:'copy' diff --git a/modules/vep.nf b/modules/vep.nf index b36b8f5..a088349 100644 --- a/modules/vep.nf +++ b/modules/vep.nf @@ -3,7 +3,7 @@ process VEP { // Annotate using VEP conda "$projectDir/envs/vep/environment.yaml" - container params.container.vep + container workflow.containerEngine == 'singularity' ? params.container.singularity.vep : params.container.docker.vep publishDir "${params.outputDir}/vep_annotate/", pattern: "annotation*.tsv", mode:'copy' From df81e71acd4e5b4cda1386b3e2b184839474fe1e Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:17:23 +0200 Subject: [PATCH 22/95] fix gene naming in bed --- assets/data/test_data/target_calling.bed | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/data/test_data/target_calling.bed b/assets/data/test_data/target_calling.bed index d1ac8d8..64865d4 100644 --- a/assets/data/test_data/target_calling.bed +++ b/assets/data/test_data/target_calling.bed @@ -1,2 +1,2 @@ -small_ref 1 710 -small_ref 720 1300 +small_ref 1 710 gene1 +small_ref 720 1300 gene2 From 4cffa947c23005df9302998e385d7208d619f265 Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:34:32 +0200 Subject: [PATCH 23/95] add missing container --- modules/mergepins.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mergepins.nf b/modules/mergepins.nf index 1900f33..09600f1 100644 --- a/modules/mergepins.nf +++ b/modules/mergepins.nf @@ -4,7 +4,7 @@ process MERGE_PINS { // Merge pinpointables into a single VCF conda "$projectDir/envs/bcftools/environment.yaml" - container params.container.bcftools + container workflow.containerEngine == 'singularity' ? params.container.singularity.bcftools : params.container.docker.bcftools publishDir "${params.outputDir}/", mode:'copy' From 7310aa3cc1e68ff07a5836ad841a52c912fe23d2 Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:35:02 +0200 Subject: [PATCH 24/95] add confirmation of lof plugin availability --- envs/vep/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/vep/Dockerfile b/envs/vep/Dockerfile index b71116b..1273c30 100644 --- a/envs/vep/Dockerfile +++ b/envs/vep/Dockerfile @@ -35,7 +35,7 @@ USER root # Set up Perl environment ENV PERL_MM_USE_DEFAULT=1 -ENV PERL5LIB=/opt/vep/perl5/lib/perl5:$PERL5LIB +ENV PERL5LIB=/plugins/loftee:/opt/vep/perl5/lib/perl5:$PERL5LIB # Install system dependencies RUN apt-get update && apt-get -y install \ From 373c7ec7622c9274e314f59e748e155a95bcc49c Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:35:43 +0200 Subject: [PATCH 25/95] remove fgbio launch param --- modules/call_consensus.nf | 2 +- modules/extract_umi.nf | 2 +- modules/group_umi.nf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/call_consensus.nf b/modules/call_consensus.nf index 1558f63..32d054e 100644 --- a/modules/call_consensus.nf +++ b/modules/call_consensus.nf @@ -13,7 +13,7 @@ process CALL_CONSENSUS { script: """ - ${params.fgbio} --tmp-dir=. --compression 1 --async-io CallDuplexConsensusReads \ + fgbio --tmp-dir=. --compression 1 --async-io CallDuplexConsensusReads \ --input=${bam_file} \ --output="${sample_id}.ubam" \ --error-rate-pre-umi=45 \ diff --git a/modules/extract_umi.nf b/modules/extract_umi.nf index bf310d7..9ad4f89 100644 --- a/modules/extract_umi.nf +++ b/modules/extract_umi.nf @@ -20,7 +20,7 @@ process EXTRACT_UMI { script: """ - ${params.fgbio} --tmp-dir=. --compression 1 --async-io ExtractUmisFromBam \ + fgbio --tmp-dir=. --compression 1 --async-io ExtractUmisFromBam \ --input=${bam_file} \ --output=${sample_id}.ubam \ --read-structure=5M2S+T 5M2S+T \ diff --git a/modules/group_umi.nf b/modules/group_umi.nf index 5bb8154..eb01406 100644 --- a/modules/group_umi.nf +++ b/modules/group_umi.nf @@ -16,7 +16,7 @@ process GROUP_UMI { script: """ - ${params.fgbio} --tmp-dir=. --compression 1 --async-io GroupReadsByUmi \ + fgbio --tmp-dir=. --compression 1 --async-io GroupReadsByUmi \ --strategy=paired \ --input=${bam_file} \ --output="${sample_id}.bam" \ From 11e7ccea53ef17d4dc066a68fa8dfbea35fb191f Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:36:28 +0200 Subject: [PATCH 26/95] Remove modules from conf --- conf/ngc.config | 128 +++++++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 66 deletions(-) diff --git a/conf/ngc.config b/conf/ngc.config index 86e81f5..682a8b8 100644 --- a/conf/ngc.config +++ b/conf/ngc.config @@ -5,9 +5,6 @@ profiles { outputDir = "results" project = "icope_staging_r" cacheDir = "/ngc/projects2/dp_00005/scratch/nextflow" - fgbio = "java -Xmx4g -XX:+AggressiveHeap -jar /services/tools/fgbio/1.5.1/fgbio.jar" - snpeff = "java -Xmx4g -jar /services/tools/snpeff/5.0e/snpEff.jar" - snpsift = "java -Xmx4g -jar /services/tools/snpeff/5.0e/SnpSift.jar" snpeff_config = "/ngc/projects2/dp_00005/data/dwf/databases/snpeff/snpEff.config" snpeff_cache = "/ngc/projects2/dp_00005/data/dwf/databases/snpeff/data" vep_cache = "/ngc/shared/nf_tools/references/vep_cache" @@ -24,32 +21,31 @@ profiles { } process { - executor = 'pbs' + executor = 'local' scratch = '/scratch/nf_tmp' queueSize = 30 beforeScript = "export _JAVA_OPTIONS=-Djava.io.tmpdir=$params.cacheDir" clusterOptions = "-A $params.project -W group_list=$params.project" - // UMI mapping: withName: DOWNSAMPLE { executor = 'local' - module = ['tools','java/1.8.0','bbmap/38.89'] + // module = ['tools','java/1.8.0','bbmap/38.89'] cpus = 1 memory = 4.GB time = 2.hour } withName: UBAM { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 8 memory = { 20.GB * task.attempt } time = 4.hour } withName: ALIGNMENT_UMI { - module = ['tools', 'intel/perflibs/2020_update4','bwa-mem2/2.2.1','samtools/1.18'] + // module = ['tools', 'intel/perflibs/2020_update4','bwa-mem2/2.2.1','samtools/1.18'] cpus = 40 memory = 160.GB time = 2.hour @@ -57,7 +53,7 @@ profiles { withName: EXTRACT_UMI { executor = 'local' - module = ['tools','java/1.8.0','fgbio/1.5.1'] + // module = ['tools','java/1.8.0','fgbio/1.5.1'] cpus = 2 memory = 4.GB time = 2.hour @@ -65,14 +61,14 @@ profiles { withName: FASTQ { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 4.GB time = 2.hour } withName: MERGEBAM { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 8 memory = { 20.GB * task.attempt } time = 4.hour @@ -80,7 +76,7 @@ profiles { withName: ALIGNMENT_METRICS { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 2.hour @@ -88,7 +84,7 @@ profiles { withName: GC_METRICS { executor = 'local' - module = ['tools','java/1.8.0','gatk/4.3.0.0','gcc/7.4.0','intel/perflibs/2020_update4','R/4.3.0'] + // module = ['tools','java/1.8.0','gatk/4.3.0.0','gcc/7.4.0','intel/perflibs/2020_update4','R/4.3.0'] cpus = 2 memory = 8.GB time = 2.hour @@ -96,7 +92,7 @@ profiles { withName: INSERT_SIZE_METRICS { executor = 'local' - module = ['tools','java/1.8.0','gatk/4.3.0.0','gcc/7.4.0','intel/perflibs/2020_update4','R/4.3.0'] + // module = ['tools','java/1.8.0','gatk/4.3.0.0','gcc/7.4.0','intel/perflibs/2020_update4','R/4.3.0'] cpus = 2 memory = 8.GB time = 2.hour @@ -104,21 +100,21 @@ profiles { withName: HS_METRICS { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 2.hour } withName: DUPLICATE_METRICS { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 4 memory = 16.GB time = 2.hour } withName: GROUP_UMI { - module = ['tools','java/1.8.0','fgbio/1.5.1'] + // module = ['tools','java/1.8.0','fgbio/1.5.1'] cpus = 8 memory = { 20.GB * task.attempt } time = 2.hour @@ -126,21 +122,21 @@ profiles { withName: UMI_METRICS { executor = 'local' - module = ['tools','java/1.8.0','fgbio/1.5.1'] + // module = ['tools','java/1.8.0','fgbio/1.5.1'] cpus = 2 memory = 8.GB time = 2.hour } withName: CALL_CONSENSUS { - module = ['tools','java/1.8.0','fgbio/1.5.1'] + // module = ['tools','java/1.8.0','fgbio/1.5.1'] cpus = 2 memory = 8.GB time = 2.hour } withName: MERGE_CONSENSUS { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 8 memory = { 20.GB * task.attempt } time = 4.hour @@ -148,7 +144,7 @@ profiles { withName: FASTQC { executor = 'local' - module = ['tools','perl/5.20.1','fastqc/0.11.8'] + // module = ['tools','perl/5.20.1','fastqc/0.11.8'] cpus = 2 memory = 4.GB time = 4.hour @@ -156,7 +152,7 @@ profiles { withName: MOSDEPTH { executor = 'local' - module = ['tools','htslib/1.16','mosdepth/0.3.3'] + // module = ['tools','htslib/1.16','mosdepth/0.3.3'] cpus = 2 memory = 4.GB time = 2.hour @@ -164,7 +160,7 @@ profiles { withName: FLAGSTAT { executor = 'local' - module = ['tools','samtools/1.18'] + // module = ['tools','samtools/1.18'] cpus = 2 memory = 3.GB time = 2.hour @@ -172,35 +168,35 @@ profiles { withName: MULTIQC { executor = 'local' - module = ['tools','multiqc/1.23'] + // module = ['tools','multiqc/1.23'] cpus = 1 memory = 4.GB time = 4.hour } withName: ALIGNMENT { - module = ['tools', 'intel/perflibs/2020_update4','bwa-mem2/2.2.1','samtools/1.18'] + // module = ['tools', 'intel/perflibs/2020_update4','bwa-mem2/2.2.1','samtools/1.18'] cpus = 40 memory = 160.GB time = 4.hour } withName: MARKDUPLICATES { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 8 memory = { 16.GB * task.attempt } time = 4.hour } withName: MARKDUPLICATES_SPARK { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 8 memory = { 16.GB * task.attempt } time = 4.hour } withName: MARKDUPLICATES_FAST { - module = ['tools','sambamba/0.8.0'] + // module = ['tools','sambamba/0.8.0'] cpus = 8 memory = { 16.GB * task.attempt } time = 4.hour @@ -208,7 +204,7 @@ profiles { withName: CLEAN { executor = 'local' - module = ['tools','samtools/1.18'] + // module = ['tools','samtools/1.18'] cpus = 2 memory = 4.GB time = 2.hour @@ -216,7 +212,7 @@ profiles { withName: ADDREADGROUP { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 1 memory = 4.GB time = 2.hour @@ -224,7 +220,7 @@ profiles { withName: CRAM { executor = 'local' - module = ['tools','samtools/1.18'] + // module = ['tools','samtools/1.18'] cpus = 2 memory = 4.GB time = 2.hour @@ -232,7 +228,7 @@ profiles { withName: BAM { executor = 'local' - module = ['tools','samtools/1.18'] + // module = ['tools','samtools/1.18'] cpus = 2 memory = 4.GB time = 2.hour @@ -240,7 +236,7 @@ profiles { withName: INDEX { executor = 'local' - module = ['tools','samtools/1.18'] + // module = ['tools','samtools/1.18'] cpus = 2 memory = 4.GB time = 2.hour @@ -248,7 +244,7 @@ profiles { withName: VALIDATE { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 4.GB time = 2.hour @@ -256,21 +252,21 @@ profiles { withName: INTERVALS { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 1 memory = 2.GB time = 1.hour } withName: HAPLOTYPECALLER { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = { 5.hour * task.attempt } } withName: HAPLOTYPECALLER_SPLIT { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 4.hour @@ -278,7 +274,7 @@ profiles { withName: DEEPVARIANT { array = 20 - module = ['tools','singularity/4.0.2'] + // module = ['tools','singularity/4.0.2'] cpus = 2 memory = 8.GB time = 4.hour @@ -286,7 +282,7 @@ profiles { withName: INDELQUAL { executor = 'local' - module = ['tools','anaconda2/4.0.0','lofreq/2.1.3.1'] + // module = ['tools','anaconda2/4.0.0','lofreq/2.1.3.1'] cpus = 2 memory = 8.GB time = 2.hour @@ -294,7 +290,7 @@ profiles { withName: LOFREQ { maxForks = 24 - module = ['tools','anaconda2/4.0.0','lofreq/2.1.3.1'] + // module = ['tools','anaconda2/4.0.0','lofreq/2.1.3.1'] cpus = 2 memory = 8.GB time = 4.hour @@ -302,7 +298,7 @@ profiles { withName: FILTER { executor = 'local' - module = ['tools','bcftools/1.16'] + // module = ['tools','bcftools/1.16'] cpus = 2 memory = 4.GB time = 2.hour @@ -310,14 +306,14 @@ profiles { withName: PILOT_PINPOINT { executor = 'local' - module = ['tools','gcc/7.4.0','intel/perflibs/2020_update4','apache-arrow/11.0.0_CPP','R/4.3.0'] + // module = ['tools','gcc/7.4.0','intel/perflibs/2020_update4','apache-arrow/11.0.0_CPP','R/4.3.0'] cpus = 2 memory = 10.GB time = 1.hour } withName: TEST { - module = ['tools','anaconda3/2023.03'] + // module = ['tools','anaconda3/2023.03'] cpus = 1 memory = 10.GB time = 1.hour @@ -325,7 +321,7 @@ profiles { withName: SUBSET { executor = 'local' - module = ['tools','samtools/1.18'] + // module = ['tools','samtools/1.18'] cpus = 2 memory = 4.GB time = 2.hour @@ -333,7 +329,7 @@ profiles { withName: BQSR { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 4.hour @@ -341,14 +337,14 @@ profiles { withName: APPLY_BQSR { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 4.hour } withName: HC_TRUTH { - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 4.hour @@ -356,7 +352,7 @@ profiles { withName: HC_TRUTH_JOINT { array = 20 - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 4.hour @@ -364,7 +360,7 @@ profiles { withName: GENOMICSDB { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 40 memory = 160.GB time = 4.hour @@ -372,7 +368,7 @@ profiles { withName: GENOTYPEGVCF { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 40 memory = 160.GB time = 4.hour @@ -381,7 +377,7 @@ profiles { withName: GENOTYPEGVCF_SPLIT { executor = 'local' scratch = false - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 4.hour @@ -389,7 +385,7 @@ profiles { withName: CRISP { executor = 'local' - module = ['tools','crisp/20181122'] + // module = ['tools','crisp/20181122'] cpus = 40 memory = 160.GB time = 10.hour @@ -397,7 +393,7 @@ profiles { withName: OCTOPUS { executor = 'local' - module = ['tools','singularity', 'octopus/0.7.4'] + // module = ['tools','singularity', 'octopus/0.7.4'] cpus = 2 memory = 8.GB time = 6.hour @@ -405,7 +401,7 @@ profiles { withName: MERGEVCFS { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 2 memory = 8.GB time = 4.hour @@ -413,7 +409,7 @@ profiles { withName: VARTABLE { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 1 memory = 4.GB time = 1.hour @@ -421,7 +417,7 @@ profiles { withName: INDEX_VCF { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 1 memory = 4.GB time = 1.hour @@ -429,14 +425,14 @@ profiles { withName: NORMALISE_VCF { executor = 'local' - module = ['tools','bcftools/1.16'] + // module = ['tools','bcftools/1.16'] cpus = 2 memory = 4.GB time = 2.hour } withName: FILTER_VARIANTS { - module = ['tools','anaconda3/2023.03'] + // module = ['tools','anaconda3/2023.03'] cpus = 1 memory = 10.GB time = 1.hour @@ -444,14 +440,14 @@ profiles { withName: PINPY { executor = 'local' - module = ['tools','anaconda3/2022.10','bcftools/1.16','htslib/1.16'] + // module = ['tools','anaconda3/2022.10','bcftools/1.16','htslib/1.16'] cpus = 2 memory = 10.GB time = 2.hour } withName: SNPEFF { - module = ['tools', 'anaconda3/2022.10', 'perl/5.20.1', 'jdk/21.0.1', 'ngctools', 'snpeff/5.0e'] + // module = ['tools', 'anaconda3/2022.10', 'perl/5.20.1', 'jdk/21.0.1', 'ngctools', 'snpeff/5.0e'] executor = 'local' cpus = 2 memory = 4.GB @@ -459,7 +455,7 @@ profiles { } withName: SNPSIFT_CLINVAR { - module = ['tools', 'anaconda3/2022.10', 'perl/5.20.1', 'jdk/21.0.1', 'ngctools', 'snpeff/5.0e'] + // module = ['tools', 'anaconda3/2022.10', 'perl/5.20.1', 'jdk/21.0.1', 'ngctools', 'snpeff/5.0e'] executor = 'local' cpus = 2 memory = 4.GB @@ -467,7 +463,7 @@ profiles { } withName: SNPSIFT_FILTER { - module = ['tools', 'anaconda3/2022.10', 'perl/5.20.1', 'jdk/21.0.1', 'ngctools', 'snpeff/5.0e'] + // module = ['tools', 'anaconda3/2022.10', 'perl/5.20.1', 'jdk/21.0.1', 'ngctools', 'snpeff/5.0e'] executor = 'local' cpus = 2 memory = 4.GB @@ -476,7 +472,7 @@ profiles { withName: VARTABLE_PINS { executor = 'local' - module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] + // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] cpus = 1 memory = 4.GB time = 1.hour @@ -484,7 +480,7 @@ profiles { withName: BED_ANNOTATE { executor = 'local' - module = ['tools','bcftools/1.16'] + // module = ['tools','bcftools/1.16'] cpus = 1 memory = 4.GB time = 2.hour @@ -492,7 +488,7 @@ profiles { withName: DROP_PL { executor = 'local' - module = ['tools','bcftools/1.16'] + // module = ['tools','bcftools/1.16'] cpus = 1 memory = 4.GB time = 2.hour @@ -500,7 +496,7 @@ profiles { withName: MERGE_PINS { executor = 'local' - module = ['tools','bcftools/1.16','htslib/1.16'] + // module = ['tools','bcftools/1.16','htslib/1.16'] cpus = 1 memory = 4.GB time = 2.hour From 6b8f1061b31773cbf79c0cb089142c3de7d9ed6e Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:37:04 +0200 Subject: [PATCH 27/95] remove fbio launch param --- conf/profiles.config | 2 -- 1 file changed, 2 deletions(-) diff --git a/conf/profiles.config b/conf/profiles.config index 4bf3713..1a8785a 100644 --- a/conf/profiles.config +++ b/conf/profiles.config @@ -15,7 +15,6 @@ profiles { params { outputDir = "results" - fgbio = "fgbio" annotate = false spark_markdup = false } @@ -68,7 +67,6 @@ profiles { params { outputDir = "results" - fgbio = "fgbio" spark_markdup = false filter = true ploidy = 2 From e94d5262e27fbfaad7fe92598e68065518347e2d Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:40:30 +0200 Subject: [PATCH 28/95] add correct loftee plugin dir --- lib/Utils.groovy | 17 +++++++++++++++++ modules/vep.nf | 16 ++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 lib/Utils.groovy diff --git a/lib/Utils.groovy b/lib/Utils.groovy new file mode 100644 index 0000000..b80041e --- /dev/null +++ b/lib/Utils.groovy @@ -0,0 +1,17 @@ +import nextflow.Channel +import static nextflow.Nextflow.* + +class Utils { + /** + * Return a value channel carrying (mainFile, indexOrEmpty). + * indexExt is a single extension like '.bai' or '.tbi'. + */ + static def indexedFileChannel(String filePath, String indexExt = '.tbi') { + if (!filePath) return Channel.value(tuple([], [])) + + def main = file(filePath) + def index = file("${filePath}${indexExt}") + + return Channel.value(index.exists() ? tuple(main, index) : tuple(main, [])) + } +} \ No newline at end of file diff --git a/modules/vep.nf b/modules/vep.nf index a088349..2daddfb 100644 --- a/modules/vep.nf +++ b/modules/vep.nf @@ -12,12 +12,12 @@ process VEP { path reference_genome path cache_dir // optional cache directory path utr_annotation // optional UTR annotation file - path alphamissense // optional AlphaMissense file - path clinvar // optional ClinVar VCF - path danmac // optional DanMac VCF - path blacklist // optional blacklist BED - path repeatmasks // optional repeat masks BED - path gnomad // optional gnomAD VCF + tuple path(alphamissense), path(am_index) // optional AlphaMissense file + tuple path(clinvar), path(clinvar_index) // optional ClinVar VCF + tuple path(danmac), path(danmar_index) // optional DanMac VCF + tuple path(blacklist), path(blacklist_index) // optional blacklist BED + tuple path(repeatmasks), path(repeat_index) // optional repeat masks BED + tuple path(gnomad), path(gnomad_index) // optional gnomAD VCF path loftee_gerp_bigwig // optional LOFTEE GERP BigWig path loftee_human_ancestor // optional LOFTEE human ancestor FASTA path loftee_conservation // optional LOFTEE conservation SQL @@ -36,8 +36,8 @@ process VEP { def blackl = blacklist ? "--custom ${blacklist},blacklist,bed,overlap,0,4" : "" def repeatm = repeatmasks ? "--custom ${repeatmasks},repeats,bed,overlap,0,4" : "" def gnom = gnomad ? "--custom ${gnomad},gnomAD,vcf,exact,0,AC_joint_nfe,AN_joint_nfe,AF_joint_nfe,AC_genomes_nfe,AN_genomes_nfe,AF_genomes_nfe,AC_exomes_nfe,AN_exomes_nfe,AF_exomes_nfe,grpmax_joint,AC_grpmax_joint,AF_grpmax_joint,grpmax_genomes,AC_grpmax_genomes,AF_grpmax_genomes,grpmax_exomes,AC_grpmax_exomes,AF_grpmax_exomes" : "" - def loftee_plugin_dir = workflow.containerEngine ? "/plugins" : "./loftee" - def loft = loftee_gerp_bigwig && loftee_human_ancestor && loftee_conservation ? "--plugin LoF,loftee_path:/plugins,gerp_bigwig:${loftee_gerp_bigwig},human_ancestor_fa:${loftee_human_ancestor},conservation_file:${loftee_conservation}" : "" + def loftee_plugin_dir = workflow.containerEngine ? "/plugins/loftee" : "./loftee" + def loft = loftee_gerp_bigwig && loftee_human_ancestor && loftee_conservation ? "--plugin LoF,loftee_path:${loftee_plugin_dir},gerp_bigwig:${loftee_gerp_bigwig},human_ancestor_fa:${loftee_human_ancestor},conservation_file:${loftee_conservation}" : "" """ vep \ --assembly GRCh38 \ From 8380cd2fc685d70a0a7be8510e843ab9230a051f Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:41:19 +0200 Subject: [PATCH 29/95] add function for file and index tuple import --- .gitignore | 1 - subworkflows/pinpoint.nf | 20 ++++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 1c7e94f..8a2bf23 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ lib64/ parts/ sdist/ diff --git a/subworkflows/pinpoint.nf b/subworkflows/pinpoint.nf index 1ec442d..2aa114c 100644 --- a/subworkflows/pinpoint.nf +++ b/subworkflows/pinpoint.nf @@ -4,6 +4,8 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +import static Utils.indexedFileChannel + include { INDEX_VCF } from '../modules/index_vcf' include { DROP_PL } from '../modules/drop_pl' include { VARTABLE } from '../modules/vartable' @@ -19,6 +21,12 @@ include { PIN_BASIC } from '../modules/pin_basic' include { UNIQUE_VCF } from '../modules/unique_vcf' include { VEP } from '../modules/vep' +alphamissense_ch = indexedFileChannel(params.alphamissense_tsv, '.tbi') +clinvar_ch = indexedFileChannel(params.clinvar_db, '.tbi') +danmac_db_ch = indexedFileChannel(params.danmac_db, '.tbi') +blacklist_bed_ch = indexedFileChannel(params.blacklist_bed, '.tbi') +repeatmasker_bed_ch = indexedFileChannel(params.repeatmasker_bed, '.tbi') +gnomad_vcf_ch = indexedFileChannel(params.gnomad_vcf, '.tbi') if (params.filter) { snv_model = Channel.fromPath("$projectDir/assets/filter/logistic_regression_snv_model.joblib", checkIfExists: true).collect() @@ -104,12 +112,12 @@ workflow PINPOINT { reference_genome, params.vep_cache ?: [], // cache params.utr_file ?: [], // utr - params.alphamissense_tsv ?: [], // alphamissense - params.clinvar_db ?: [], // clinvar - params.danmac_db ?: [], // danmac - params.blacklist_bed ?: [], // blacklist - params.repeatmasker_bed ?: [], // repeatmasker - params.gnomad_vcf ?: [], // gnomad + alphamissense_ch ?: [], // alphamissense + clinvar_ch ?: [], // clinvar + danmac_db_ch ?: [], // danmac + blacklist_bed_ch ?: [], // blacklist + repeatmasker_bed_ch ?: [], // repeatmasker + gnomad_vcf_ch ?: [], // gnomad params.loftee_gerp_bw ?: [], // loftee gerp params.loftee_human_ancestor ?: [],// loftee human ancestor params.loftee_sqlite ?: [] // loftee conservation (sqlite) From 2503bbee3bd62142bd6cd0a75651f25a2a8b9929 Mon Sep 17 00:00:00 2001 From: Mads Cort Nielsen Date: Wed, 17 Sep 2025 11:42:47 +0200 Subject: [PATCH 30/95] update container cache to private cache, and add new modules --- next.pbs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/next.pbs b/next.pbs index 78e78b8..b93290b 100644 --- a/next.pbs +++ b/next.pbs @@ -20,6 +20,8 @@ #PBS -j oe #PBS -N nextflow +module load nextflow/25.04.6 GitPython + ## exit if any errors or unset variables are encountered set -euo pipefail @@ -29,7 +31,7 @@ version=2.0 root=/ngc/tools/ngctools/${name}/${version} data=/ngc/shared/${name} -export NXF_SINGULARITY_CACHEDIR=${data}/containers +export NXF_SINGULARITY_CACHEDIR=/ngc/projects2/dp_00005/data/dwf/bin/container_cache export NXF_ASSETS=${root}/pipelines export NXF_SCM_FILE=${root}/scm export NXF_PLUGINS_DIR=${data}/plugins @@ -37,8 +39,6 @@ export NXF_PLUGINS_DIR=${data}/plugins # Disable interactive logging #export NXF_ANSI_LOG=false -module load tools ngctools jdk/21.0.1 nextflow/24.08.0-edge - nextflow_essential_params='-profile ngc -resume' nextflow_command="nextflow run main.nf $nextflow_essential_params $@" From fe0b130b6b4491289e044d360d7f569003bfeba3 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 17 Sep 2025 11:50:41 +0200 Subject: [PATCH 31/95] update ubuntu --- conf/container.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/container.config b/conf/container.config index 3b3a5d5..3f9250e 100644 --- a/conf/container.config +++ b/conf/container.config @@ -3,7 +3,7 @@ // Base container image for shell scripts process { - container = 'ubuntu:20.04' + container = 'ubuntu:24.04' } params { From 7cbc9839dbe736a3c5006ec2a692ad2779880d11 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 17 Sep 2025 11:51:04 +0200 Subject: [PATCH 32/95] verify loftee --- envs/vep/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/envs/vep/Dockerfile b/envs/vep/Dockerfile index b71116b..0f4be64 100644 --- a/envs/vep/Dockerfile +++ b/envs/vep/Dockerfile @@ -35,7 +35,7 @@ USER root # Set up Perl environment ENV PERL_MM_USE_DEFAULT=1 -ENV PERL5LIB=/opt/vep/perl5/lib/perl5:$PERL5LIB +ENV PERL5LIB=/plugins/loftee:/opt/vep/perl5/lib/perl5:$PERL5LIB # Install system dependencies RUN apt-get update && apt-get -y install \ @@ -66,8 +66,8 @@ RUN cpanm --local-lib=/opt/vep/perl5 local::lib && \ USER vep # Verify installations -RUN samtools --version && \ - ls -la /plugins/loftee/ +RUN samtools --version && ls -la /plugins/loftee/ && \ + perl -e 'use lib q(/plugins/loftee); require q(LoF.pm); require q(utr_splice.pl); print qq(LoFTEE load OK\n)' # Set working directory WORKDIR /opt/vep \ No newline at end of file From ec0f1eb8047624701b00440f1618c72085d13943 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 17 Sep 2025 11:52:22 +0200 Subject: [PATCH 33/95] separate pinpoint output methods --- main.nf | 12 +++-- nextflow.config | 7 +++ subworkflows/pinpoint_tab.nf | 50 +++++++++++++++++++ subworkflows/{pinpoint.nf => pinpoint_vcf.nf} | 31 +----------- 4 files changed, 67 insertions(+), 33 deletions(-) create mode 100644 subworkflows/pinpoint_tab.nf rename subworkflows/{pinpoint.nf => pinpoint_vcf.nf} (70%) diff --git a/main.nf b/main.nf index 9637da3..9bd656b 100644 --- a/main.nf +++ b/main.nf @@ -35,7 +35,8 @@ include { MATRIX_CONTEXT } from './modules/matrix_context' // Pinpoint methods include { PILOT_PINPOINT } from './modules/pilot_pinpoint' -include { PINPOINT } from './subworkflows/pinpoint' +include { PINPOINT_VCF } from './subworkflows/pinpoint_vcf' +include { PINPOINT_TAB } from './subworkflows/pinpoint_tab' include { VARIANT_RESCUE } from './subworkflows/variant_rescue' // Test @@ -132,8 +133,13 @@ workflow { matrix_context = MATRIX_CONTEXT.out.json decode_table = file(params.decodetable) } - PINPOINT(gatk_ch, pooltable, decode_table, matrix_context, reference_genome_ch) - pin_ch = PINPOINT.out.pinned_variants + PINPOINT_VCF(gatk_ch, pooltable, decode_table, matrix_context, reference_genome_ch) + pin_ch = PINPOINT_VCF.out.pinned_variants + + if (params.pinpoint_tab) { + PINPOINT_TAB(gatk_ch, matrix_context, reference_genome_ch) + } + if (params.variant_rescue) { VARIANT_RESCUE(gatk_ch, bam_file_w_index_ch, pooltable, reference_genome_ch, bedfile_ch) } diff --git a/nextflow.config b/nextflow.config index 5a5f3d5..241cee0 100644 --- a/nextflow.config +++ b/nextflow.config @@ -84,9 +84,15 @@ params { // Pinpoint method (pilot or new) pinpoint_method = 'new' + // Use new tabular output method (true or false) + pinpoint_tab = true + // Use variant rescue model (true or false) variant_rescue = true + // Make pinpoint report (true or false) + pinpoint_report = true + // Annotation configurations annotate = true snpeff_db = 'GRCh38.99' @@ -97,6 +103,7 @@ params { bedfile_gene_annotation = true // VEP annotations + annotate_vep = true vep_cache = "" clinvar_db = "" danmac_db = "" diff --git a/subworkflows/pinpoint_tab.nf b/subworkflows/pinpoint_tab.nf new file mode 100644 index 0000000..2168e8b --- /dev/null +++ b/subworkflows/pinpoint_tab.nf @@ -0,0 +1,50 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + PINPOINTING WITH TABULAR OUTPUT SUB-WORKFLOW +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { INDEX_VCF } from '../modules/index_vcf' +include { DROP_PL } from '../modules/drop_pl' +include { VARTABLE } from '../modules/vartable' +include { NORMALISE_VCF } from '../modules/normalise_vcf' +include { BGZIP } from '../modules/bgzip' +include { PIN_BASIC } from '../modules/pin_basic' +include { UNIQUE_VCF } from '../modules/unique_vcf' +include { VEP } from '../modules/vep' + +workflow PINPOINT_TAB { + take: + vcf_file + matrix_context + reference_genome + + main: + INDEX_VCF(vcf_file, 'GATK') + DROP_PL(INDEX_VCF.out.vcf_w_index, 'GATK') + NORMALISE_VCF(DROP_PL.out.vcf_file, reference_genome, 'GATK') + BGZIP(NORMALISE_VCF.out.norm_vcf) + BGZIP.out.bgzf_file + .map { sample, vcf, index -> tuple(vcf, index) } + .collect() + .set { basic_input } + PIN_BASIC(basic_input,matrix_context) + UNIQUE_VCF(basic_input) + if (params.annotate_vep) { + VEP( + UNIQUE_VCF.out.vcf_file, + reference_genome, + params.vep_cache ?: [], // cache + params.utr_file ?: [], // utr + params.alphamissense_tsv ?: [], // alphamissense + params.clinvar_db ?: [], // clinvar + params.danmac_db ?: [], // danmac + params.blacklist_bed ?: [], // blacklist + params.repeatmasker_bed ?: [], // repeatmasker + params.gnomad_vcf ?: [], // gnomad + params.loftee_gerp_bw ?: [], // loftee gerp + params.loftee_human_ancestor ?: [],// loftee human ancestor + params.loftee_sqlite ?: [] // loftee conservation (sqlite) + ) + } +} \ No newline at end of file diff --git a/subworkflows/pinpoint.nf b/subworkflows/pinpoint_vcf.nf similarity index 70% rename from subworkflows/pinpoint.nf rename to subworkflows/pinpoint_vcf.nf index 1ec442d..3e1f74b 100644 --- a/subworkflows/pinpoint.nf +++ b/subworkflows/pinpoint_vcf.nf @@ -14,11 +14,6 @@ include { DISCARD } from '../modules/discard' include { ANNOTATION } from '../subworkflows/annotation' include { VARTABLE_PINS } from '../modules/vartable_pins' include { MERGE_PINS } from '../modules/mergepins' -include { BGZIP } from '../modules/bgzip' -include { PIN_BASIC } from '../modules/pin_basic' -include { UNIQUE_VCF } from '../modules/unique_vcf' -include { VEP } from '../modules/vep' - if (params.filter) { snv_model = Channel.fromPath("$projectDir/assets/filter/logistic_regression_snv_model.joblib", checkIfExists: true).collect() @@ -26,7 +21,7 @@ if (params.filter) { model_class = Channel.fromPath("$projectDir/assets/filter/model.py", checkIfExists: true).collect() } -workflow PINPOINT { +workflow PINPOINT_VCF { take: vcf_file pooltable @@ -91,30 +86,6 @@ workflow PINPOINT { VARTABLE_PINS(PINPY.out.vcf_unique_pins.flatten()) MERGE_PINS(PINPY.out.vcf_unique_2d_pins) - // New pinpoint-flow - BGZIP(vcf_ch) - BGZIP.out.bgzf_file - .map { sample, vcf, index -> tuple(vcf, index) } - .collect() - .set { basic_input } - PIN_BASIC(basic_input,matrix_context) - UNIQUE_VCF(basic_input) - VEP( - UNIQUE_VCF.out.vcf_file, - reference_genome, - params.vep_cache ?: [], // cache - params.utr_file ?: [], // utr - params.alphamissense_tsv ?: [], // alphamissense - params.clinvar_db ?: [], // clinvar - params.danmac_db ?: [], // danmac - params.blacklist_bed ?: [], // blacklist - params.repeatmasker_bed ?: [], // repeatmasker - params.gnomad_vcf ?: [], // gnomad - params.loftee_gerp_bw ?: [], // loftee gerp - params.loftee_human_ancestor ?: [],// loftee human ancestor - params.loftee_sqlite ?: [] // loftee conservation (sqlite) - ) - emit: pinned_variants = PINPY.out.lookup_table } \ No newline at end of file From 56e95b189fa53a4370a308771f28c05668c31748 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 17 Sep 2025 12:09:34 +0200 Subject: [PATCH 34/95] adjust input channelse --- subworkflows/pinpoint_tab.nf | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/subworkflows/pinpoint_tab.nf b/subworkflows/pinpoint_tab.nf index 2168e8b..53e8688 100644 --- a/subworkflows/pinpoint_tab.nf +++ b/subworkflows/pinpoint_tab.nf @@ -4,6 +4,8 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +import static Utils.indexedFileChannel + include { INDEX_VCF } from '../modules/index_vcf' include { DROP_PL } from '../modules/drop_pl' include { VARTABLE } from '../modules/vartable' @@ -13,6 +15,13 @@ include { PIN_BASIC } from '../modules/pin_basic' include { UNIQUE_VCF } from '../modules/unique_vcf' include { VEP } from '../modules/vep' +alphamissense_ch = indexedFileChannel(params.alphamissense_tsv, '.tbi') +clinvar_ch = indexedFileChannel(params.clinvar_db, '.tbi') +danmac_db_ch = indexedFileChannel(params.danmac_db, '.tbi') +blacklist_bed_ch = indexedFileChannel(params.blacklist_bed, '.tbi') +repeatmasker_bed_ch = indexedFileChannel(params.repeatmasker_bed, '.tbi') +gnomad_vcf_ch = indexedFileChannel(params.gnomad_vcf, '.tbi') + workflow PINPOINT_TAB { take: vcf_file @@ -34,17 +43,17 @@ workflow PINPOINT_TAB { VEP( UNIQUE_VCF.out.vcf_file, reference_genome, - params.vep_cache ?: [], // cache - params.utr_file ?: [], // utr - params.alphamissense_tsv ?: [], // alphamissense - params.clinvar_db ?: [], // clinvar - params.danmac_db ?: [], // danmac - params.blacklist_bed ?: [], // blacklist - params.repeatmasker_bed ?: [], // repeatmasker - params.gnomad_vcf ?: [], // gnomad - params.loftee_gerp_bw ?: [], // loftee gerp - params.loftee_human_ancestor ?: [],// loftee human ancestor - params.loftee_sqlite ?: [] // loftee conservation (sqlite) + params.vep_cache ?: [], + params.utr_file ?: [], + alphamissense_ch ?: [], + clinvar_ch, + danmac_db_ch, + blacklist_bed_ch, + repeatmasker_bed_ch, + gnomad_vcf_ch, + params.loftee_gerp_bw ?: [], + params.loftee_human_ancestor ?: [], + params.loftee_sqlite ?: [] ) } } \ No newline at end of file From e336514a9da9bda218e2d6682f91e0b88075b06c Mon Sep 17 00:00:00 2001 From: madscort <61156331+madscort@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:25:24 +0200 Subject: [PATCH 35/95] add backslash --- modules/deepvariant.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/deepvariant.nf b/modules/deepvariant.nf index 2db9bd2..f993828 100644 --- a/modules/deepvariant.nf +++ b/modules/deepvariant.nf @@ -26,7 +26,7 @@ process DEEPVARIANT { def ref_dir = file(params.reference_genome).getParent() def target_dir = file(params.bedfile).getParent() """ - /opt/deepvariant/bin/run_deepvariant + /opt/deepvariant/bin/run_deepvariant \ --ref=${db} \ --reads=$bam_file \ --output_vcf=${sample_id}.DV.vcf.gz \ From 9af0cc7f6ca876f9e7bc8bbd75b3b171aff23bc4 Mon Sep 17 00:00:00 2001 From: madscort <61156331+madscort@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:26:53 +0200 Subject: [PATCH 36/95] Update envs/rescue/environment.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- envs/rescue/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/rescue/environment.yaml b/envs/rescue/environment.yaml index 7558ae3..1ab1d4f 100644 --- a/envs/rescue/environment.yaml +++ b/envs/rescue/environment.yaml @@ -4,6 +4,6 @@ channels: - bioconda - defaults dependencies: - - python=3.11.0 + - python=3.11 - pip: - marbl-pool==0.2.0 \ No newline at end of file From 5d0e7987a99ea9dd8a86daca757def723856ae79 Mon Sep 17 00:00:00 2001 From: madscort <61156331+madscort@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:27:44 +0200 Subject: [PATCH 37/95] change to input variable file naming Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- modules/vep.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vep.nf b/modules/vep.nf index 2daddfb..7dc85f7 100644 --- a/modules/vep.nf +++ b/modules/vep.nf @@ -66,7 +66,7 @@ process VEP { grep '^#[^#]' "annotations.tsv" >> annotations_w_varid.tsv # Body paste \ - <(grep -F -v '*' "variant_keys.txt") \ + <(grep -F -v '*' "${variant_keys}") \ <(grep -v '^#' "annotations.tsv") \ >> annotations_w_varid.tsv """ From b8ec1eec2d89b9b7c35cb15e889b016a26d87c4e Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 17 Sep 2025 12:32:40 +0200 Subject: [PATCH 38/95] fix input params for pinpoint step --- main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index 9bd656b..86a8e98 100644 --- a/main.nf +++ b/main.nf @@ -54,7 +54,7 @@ pooltable_ch = Channel .splitCsv(sep: '\t') .map { row -> tuple(row[0], [file(row[2]), file(row[3])]) } -if (params.step == 'calling') { +if (params.step == 'calling' || (params.step == 'pinpoint' && params.variant_rescue)) { cramtable_ch = Channel .fromPath(params.cramtable) .splitCsv(sep: '\t') @@ -90,7 +90,7 @@ workflow { } else { bam_file_w_index_ch = MAPPING(pooltable_ch, reference_genome_ch, bedfile_ch) } - } else if (params.step == 'calling' || (params.step == 'pinpoint' && params.pileup_calling)) { + } else if (params.step == 'calling' || (params.step == 'pinpoint' && params.variant_rescue)) { BAM(cramtable_ch, reference_genome_ch) bam_file_w_index_ch = INDEX(BAM.out.bam_file) } From 882ce7c035d2a15bbafa24d2d9f68f8d0e1a781f Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 14:38:52 +0200 Subject: [PATCH 39/95] fix column naming --- bin/pin_basic.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/pin_basic.py b/bin/pin_basic.py index 18dbfd7..e871043 100755 --- a/bin/pin_basic.py +++ b/bin/pin_basic.py @@ -10,7 +10,7 @@ @dataclass class VariantEntry: - VARID: str + varid: str CHROM: str POS: int REF: str @@ -146,7 +146,7 @@ def collect_variant_entries(vcf_path: str, sample_id: Optional[str] = None) -> L # Site-level fields entry = VariantEntry( - VARID=f"{rec.chrom}:{rec.pos}:{rec.ref}:{alt}", + varid=f"{rec.chrom}:{rec.pos}:{rec.ref}:{alt}", CHROM=rec.chrom, POS=rec.pos, REF=rec.ref, @@ -202,7 +202,7 @@ def get_variant_entry(vcf_path: str, variant_id: str, sample_id: Optional[str] = # Site-level fields entry = VariantEntry( - VARID=variant_id, + varid=variant_id, CHROM=rec.chrom, POS=rec.pos, REF=rec.ref, @@ -391,8 +391,8 @@ def main(): # Create table with pinpointables only: pins = pin(vcf_folder=pool_vcfs_path, matrix_context=mc, caller=caller) - common_fields = ['VARID','CHROM','POS','REF','ALT'] - variant_entry_header_fields = [f"{dim}.{f.name}" for f in fields(VariantEntry) for dim in ["row","column"] if f not in common_fields] + common_fields = ['varid','CHROM','POS','REF','ALT'] + variant_entry_header_fields = [f"{dim}.{f.name}" for dim in ["row","column"] for f in fields(VariantEntry) if f.name not in common_fields] header = ["uvarid", "sample_alias", "sample_id", "row_id", "row_index", "row_label", "column_id", "column_index", "column_label", *common_fields, *variant_entry_header_fields] with open(output_path / 'pinpoint_variants.tsv', 'w') as fout: @@ -410,9 +410,9 @@ def main(): row_info = get_variant_entry(vcf_path=pool_vcfs_path / (f"{row_id}.{caller}.vcf.gz"), variant_id=pinpoint) col_info = get_variant_entry(vcf_path=pool_vcfs_path / (f"{col_id}.{caller}.vcf.gz"), variant_id=pinpoint) - assert row_info.VARID == col_info.VARID, "Row and column variant IDs do not match" + assert row_info.varid == col_info.varid, "Row and column variant IDs do not match" - uvarid = f"{sample_alias}:{row_info.VARID}" + uvarid = f"{sample_alias}:{row_info.varid}" print( uvarid, sample_alias, sample_id, row_id, row_idx, row_label, col_id, col_idx, col_label, *[getattr(row_info, col) for col in variant_entry_fields], From 465f1a2992bc46e28854d69e1aba09dbfcb0d5a2 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 14:40:19 +0200 Subject: [PATCH 40/95] add arm64 support for R container --- conf/container.config | 4 ++-- envs/r_env/Dockerfile | 11 +++++++++++ envs/r_env/build | 1 + envs/r_env/environment.yaml | 21 +++++++++++++++++---- 4 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 envs/r_env/Dockerfile create mode 100644 envs/r_env/build diff --git a/conf/container.config b/conf/container.config index 477fd6f..3b5218a 100644 --- a/conf/container.config +++ b/conf/container.config @@ -27,7 +27,7 @@ params { // Custom and non-singularity containers octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" - r_env = "rocker/tidyverse:latest" + r_env = "madscort/dswf_r:1.0" pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" vep = "madscort/vep_w_loftee:111.0" @@ -57,7 +57,7 @@ params { // Custom and non-singularity containers octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" - r_env = "rocker/tidyverse:latest" + r_env = "madscort/dswf_r:1.0" pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" vep = "madscort/vep_w_loftee:111.0" diff --git a/envs/r_env/Dockerfile b/envs/r_env/Dockerfile new file mode 100644 index 0000000..adcbb29 --- /dev/null +++ b/envs/r_env/Dockerfile @@ -0,0 +1,11 @@ +FROM mambaorg/micromamba:2.3-ubuntu22.04 + +WORKDIR /work + +COPY environment.yaml /tmp/environment.yml + +RUN micromamba install -y -n base -f /tmp/environment.yml && micromamba clean -a -y + +SHELL ["micromamba", "run", "-n", "base", "/bin/bash", "-c"] + +CMD ["R", "--version"] \ No newline at end of file diff --git a/envs/r_env/build b/envs/r_env/build new file mode 100644 index 0000000..4b98af6 --- /dev/null +++ b/envs/r_env/build @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/dswf_r:1.0 --push . \ No newline at end of file diff --git a/envs/r_env/environment.yaml b/envs/r_env/environment.yaml index c18dd66..2e313e2 100644 --- a/envs/r_env/environment.yaml +++ b/envs/r_env/environment.yaml @@ -1,15 +1,28 @@ name: r_env channels: - conda-forge - - bioconda - - defaults +channel_priority: strict dependencies: + # R runtime and HTML utils - r-base=4.3.0 - - r-data.table + - r-rmarkdown + - r-knitr + - pandoc + # R packages - r-dplyr=1.1.4 - r-ggplot2=3.4.4 - r-readr=2.1.5 - r-tidyr=1.3.1 - r-stringr=1.5.1 - r-purrr=1.0.2 - - r-optparse=1.7.4 \ No newline at end of file + - r-data.table + - r-optparse=1.7.4 + - r-tibble + - r-forcats + - r-scales + - r-fs + - r-here + - r-janitor + - r-ggthemes + - r-ggbeeswarm + - r-jsonlite \ No newline at end of file From 8782f1a0e56fc7569085b1888fa942c851f5fd32 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 14:45:36 +0200 Subject: [PATCH 41/95] add simple html reporting --- bin/report.Rmd | 284 +++++++++++++++++++++++++++++++++ main.nf | 22 ++- modules/pin_basic.nf | 4 +- modules/report.nf | 42 +++++ modules/rescue.nf | 4 +- subworkflows/pinpoint_tab.nf | 7 + subworkflows/variant_rescue.nf | 5 +- 7 files changed, 360 insertions(+), 8 deletions(-) create mode 100644 bin/report.Rmd create mode 100644 modules/report.nf diff --git a/bin/report.Rmd b/bin/report.Rmd new file mode 100644 index 0000000..5fd1f6f --- /dev/null +++ b/bin/report.Rmd @@ -0,0 +1,284 @@ +--- +title: "DoBSeq Report" +output: html_document +date: "2025-09-17" +params: + matrix_context: null + pinpoints: null + all_variants: null + annotations: null + rescue: null + output: null +--- + +```{r setup, include=FALSE} +knitr::opts_chunk$set(echo = F, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = params$project_root) +``` + +```{r} +library('ggthemes') +library('jsonlite') +library("dplyr") +library("stringr") +library("ggplot2") +library("ggbeeswarm") +library("tidyr") +library("readr") +library("tibble") +library("forcats") +library("scales") +``` + + +```{r} +have <- function(p) !is.null(p) && length(p) == 1 && fs::file_exists(p) +``` + + +```{r eval=have(params$pinpoints)} +matrix_context_path <- params$matrix_context +pinpoints_path <- params$pinpoints +all_variants_path <- params$all_variants +annotations_path <- params$annotations +rescue_path <- params$rescue +output_path <- params$output +``` + +```{r eval=!have(params$pinpoints)} +here::i_am("bin/report.Rmd") + +matrix_context_path <- here::here('results','context','matrix_context.json') +pinpoints_path <- here::here('results','variant_compilation','pinpoint_variants.tsv') +all_variants_path <- here::here('results','variant_compilation','all_pool_variants.tsv') +annotations_path <- here::here('results','vep_annotate','annotations_w_varid.tsv') +rescue_path <- here::here('results','rescue_probabilities','predictions.tsv') +output_path <- here::here('reports') +``` + +```{r} +pinvars <- + read_tsv(pinpoints_path, + col_types = cols(.default = col_character())) %>% + type_convert(guess_integer = TRUE) + +poolvars <- + read_tsv(all_variants_path, + col_types = cols(.default = col_character())) %>% + type_convert(guess_integer = TRUE) +``` + +```{r eval=have(params$rescue)} +rescue <- + read_tsv(rescue_path) %>% + select(-sample_id) + +pinvars <- + pinvars %>% + full_join(rescue, by=c('uvarid','row_id','column_id','varid')) +``` + +```{r eval=have(params$annotations)} + +annotations <- + read_tsv(annotations_path, + na = '-', col_types = cols(.default = col_character())) %>% + type_convert(guess_integer = TRUE) + +pinvars <- + pinvars %>% + left_join(annotations, by='varid') + +poolvars <- + poolvars %>% + left_join(annotations, by='varid') +``` + + +```{r eval=F} +annotations <- + read_tsv(annotations_path, + na = '-', col_types = cols(.default = col_character())) %>% + type_convert(guess_integer = TRUE) %>% + mutate(clinvar_stars = case_when( + str_detect(ClinVar_CLNREVSTAT, "practice_guideline") ~ 4, + str_detect(ClinVar_CLNREVSTAT, "reviewed_by_expert_panel") ~ 3, + str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_multiple_submitters,_no_conflicts") ~ 2, + str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_conflicting_interpretations") ~ 1, + str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_single_submitter") ~ 1, + TRUE ~ NA_real_ + )) %>% + mutate(is_p = if_else(str_detect(ClinVar_CLNSIG, 'Pathogenic') | str_detect(ClinVar_CLNSIG, "Likely_pathogenic"), 1,0)) %>% + mutate(is_lofp = if_else(is_lof == 1 | is_p == 1,1,0)) + +rescue <- + read_tsv(rescue_path) %>% + select(-sample_id) + +pinvars <- + read_tsv(pinpoints_path, + col_types = cols(.default = col_character())) %>% + type_convert(guess_integer = TRUE) %>% + left_join(rescue, by=c('uvarid','row_id','column_id','varid')) %>% + left_join(annotations, by='varid') + +poolvars <- + read_tsv(all_variants_path, + col_types = cols(.default = col_character())) %>% + type_convert(guess_integer = TRUE) %>% + left_join(annotations, by='varid') + +``` + +```{r} +matrix_ctx <- fromJSON(matrix_context_path) +matrix_size <- matrix_ctx$matrix$n_rows + +all_sample_ids <- + bind_rows(matrix_ctx$matrix$cells) %>% + filter(!is.na(sample_id) & sample_id != "") %>% + pull(sample_id) %>% + unique() + +all_sample_alias <- + bind_rows(matrix_ctx$matrix$cells) %>% + filter(!is.na(alias) & alias != "") %>% + pull(alias) %>% + unique() + +all_pool_alias <- + bind_rows(matrix_ctx$matrix$cells) %>% + filter(!is.na(sample_id) & sample_id != "") %>% + pull(sample_id) %>% + unique() + +# Create dynamic factor levels +row_levels <- 1:matrix_size +col_levels <- LETTERS[1:matrix_size] + +pinvars <- + pinvars %>% + mutate(row_factor = fct_rev(factor(row_label, levels = row_levels, ordered = TRUE)), + column_factor = factor(column_label, levels = col_levels, ordered = TRUE)) +``` + +#### Assigned variants per individual +```{r} +plot_data <- + pinvars %>% + group_by(sample_id,row_factor, column_factor) %>% + summarise(private_variants = n()) +plot_data %>% + ggplot(aes(x = column_factor, y = row_factor, fill = private_variants)) + + geom_point(shape = 22, size = 14, color='black') + + scale_x_discrete(drop = FALSE, position = "top") + + scale_y_discrete(drop = FALSE, limits = levels(plot_data$row_factor)) + + geom_text(aes(label=private_variants),size=5, color='white') + + ggthemes::scale_fill_continuous_tableau() + + coord_fixed() + + labs(x = "COLUMN POOL", y = "ROW POOL") + + theme_minimal() + + theme(legend.title = element_blank(), + legend.position = '', + panel.grid.minor = element_line(size = 2), + panel.grid.major = element_line(size = 2), + axis.title = element_text(face = 'bold', color = '#7f8c8d', size = 20), + axis.text = element_text(face = 'bold', color = '#7f8c8d', size = 15)) + + ggtitle('All assigned') +``` + +```{r eval=have(params$annotations)} +plot_data <- + pinvars %>% + mutate(row_factor = fct_rev(factor(row_label, levels = 1:2, ordered = TRUE)), + column_factor = factor(column_label, levels = LETTERS[1:2], ordered = TRUE)) %>% + group_by(sample_id,row_factor, column_factor) %>% + summarise(private_variants = n()) + +plot_data %>% + ggplot(aes(x = column_factor, y = row_factor, fill = private_variants)) + + geom_point(shape = 22, size = 14, color='black') + + scale_x_discrete(drop = FALSE, position = "top") + + scale_y_discrete(drop = FALSE, limits = levels(plot_data$row_factor)) + + geom_text(aes(label=private_variants),size=5, color='white') + + ggthemes::scale_fill_continuous_tableau() + + coord_fixed() + + labs(x = "COLUMN POOL", y = "ROW POOL") + + theme_minimal() + + theme(legend.title = element_blank(), + legend.position = '', + panel.grid.minor = element_line(size = 2), + panel.grid.major = element_line(size = 2), + axis.title = element_text(face = 'bold', color = '#7f8c8d', size = 20), + axis.text = element_text(face = 'bold', color = '#7f8c8d', size = 15)) + + ggtitle('pLoF/p only') +``` + +```{r eval=have(params$annotations) & have(params$rescue)} +plot_data <- + pinvars %>% + mutate(row_factor = fct_rev(factor(row_label, levels = 1:2, ordered = TRUE)), + column_factor = factor(column_label, levels = LETTERS[1:2], ordered = TRUE)) %>% + group_by(sample_id,row_factor, column_factor) %>% + summarise(private_variants = n()) + +plot_data %>% + ggplot(aes(x = column_factor, y = row_factor, fill = private_variants)) + + geom_point(shape = 22, size = 14, color='black') + + scale_x_discrete(drop = FALSE, position = "top") + + scale_y_discrete(drop = FALSE, limits = levels(plot_data$row_factor)) + + geom_text(aes(label=private_variants),size=5, color='white') + + ggthemes::scale_fill_continuous_tableau() + + coord_fixed() + + labs(x = "COLUMN POOL", y = "ROW POOL") + + theme_minimal() + + theme(legend.title = element_blank(), + legend.position = '', + panel.grid.minor = element_line(size = 2), + panel.grid.major = element_line(size = 2), + axis.title = element_text(face = 'bold', color = '#7f8c8d', size = 20), + axis.text = element_text(face = 'bold', color = '#7f8c8d', size = 15)) + + ggtitle('pLoF/p only after rescue') +``` + +#### Variant count by pool +```{r} +poolvars %>% + group_by(pool_id, dim) %>% + summarise(n = n()) %>% + ungroup() %>% + mutate(din = if_else(dim == 'column', 1,0)) %>% + ggplot(aes(x=reorder(pool_id,din),y=n,fill=dim)) + + geom_col() + + coord_flip() + + theme_minimal() + + xlab('Pool ID') + + ylab('Variant count') + + ggthemes::scale_fill_tableau() + + theme_bw() + + theme(axis.text=element_text(size=10), + legend.position = 'bottom', + legend.title = element_blank()) +``` + +#### Mean VAF count by individual +```{r} +plot_df <- + pinvars %>% + group_by(sample_alias) %>% + mutate(mean_vaf = (column.ALT_AD+row.ALT_AD)/(column.DP+row.DP)) + +ggplot(plot_df, aes(x = sample_alias, y = mean_vaf)) + + ggbeeswarm::geom_beeswarm(corral='wrap',corral.width = 0.9,alpha=0.6) + + geom_hline(yintercept = 1/(matrix_size*2), linetype = 'dashed', color='darkgrey') + + geom_hline(yintercept = 1/matrix_size, linetype = 'dashed', color='darkgrey') + + coord_flip() + + theme_minimal() + + ggthemes::scale_color_fivethirtyeight() + + theme(legend.title = element_blank()) + + ylab('Mean VAF') + + xlab('Sample ID') +``` + + + diff --git a/main.nf b/main.nf index 86a8e98..cbaef9d 100644 --- a/main.nf +++ b/main.nf @@ -37,7 +37,9 @@ include { MATRIX_CONTEXT } from './modules/matrix_context' include { PILOT_PINPOINT } from './modules/pilot_pinpoint' include { PINPOINT_VCF } from './subworkflows/pinpoint_vcf' include { PINPOINT_TAB } from './subworkflows/pinpoint_tab' + include { VARIANT_RESCUE } from './subworkflows/variant_rescue' +include { REPORT } from './modules/report' // Test include { TEST } from './modules/test' @@ -68,6 +70,9 @@ if (params.step == 'pinpoint') { .map { row -> tuple(row[0], file(row[1]), row[2]) } } +rescue_probabilities = Channel.value([]) +report_rmd = Channel.fromPath("$projectDir/bin/report.Rmd", checkIfExists: true) + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ REFERENCE FILE CHANNELS @@ -138,10 +143,21 @@ workflow { if (params.pinpoint_tab) { PINPOINT_TAB(gatk_ch, matrix_context, reference_genome_ch) - } - if (params.variant_rescue) { - VARIANT_RESCUE(gatk_ch, bam_file_w_index_ch, pooltable, reference_genome_ch, bedfile_ch) + if (params.variant_rescue) { + VARIANT_RESCUE(gatk_ch, bam_file_w_index_ch, pooltable, decode_table, reference_genome_ch, bedfile_ch) + rescue_probabilities = VARIANT_RESCUE.out.rescue_probabilities + } + + if (params.pinpoint_report) { + REPORT( + report_rmd, + matrix_context, + PINPOINT_TAB.out.pool_variants, + PINPOINT_TAB.out.pinpoint_variants, + PINPOINT_TAB.out.annotations, + rescue_probabilities) + } } } } diff --git a/modules/pin_basic.nf b/modules/pin_basic.nf index 34061ee..6ccf3e5 100644 --- a/modules/pin_basic.nf +++ b/modules/pin_basic.nf @@ -10,8 +10,8 @@ process PIN_BASIC { path matrix_context output: - path "all_pool_variants.tsv" - path "pinpoint_variants.tsv" + path "all_pool_variants.tsv", emit: pool_variants + path "pinpoint_variants.tsv", emit: pinpoint_variants script: """ diff --git a/modules/report.nf b/modules/report.nf new file mode 100644 index 0000000..6a212b4 --- /dev/null +++ b/modules/report.nf @@ -0,0 +1,42 @@ +process REPORT { + label 'process_low' + conda "$projectDir/envs/r_env/environment.yaml" + container workflow.containerEngine == 'singularity' ? params.container.singularity.r_env : params.container.docker.r_env + + publishDir "${params.outputDir}/", mode:'copy', pattern: "variant_report.html" + + input: + path rmd + path matrix_context + path all_variants + path pinpoints + path annotations + path rescue + + output: + path "variant_report.html" + + script: + def annotate = annotations ? "annotations='${annotations}'," : "annotations=NULL," + def rescue_probs = rescue ? "rescue='${rescue}'" : "rescue=NULL" + """ + Rscript -e \ + "rmarkdown::render( + '${rmd}', + params = list( + matrix_context = '${matrix_context}', + pinpoints = '${pinpoints}', + all_variants = '${all_variants}', + ${annotate} + ${rescue_probs} + ), + output_file = 'variant_report.html', + quiet = TRUE + )" + """ + + stub: + """ + touch variant_report.html + """ +} \ No newline at end of file diff --git a/modules/rescue.nf b/modules/rescue.nf index 6e71562..7af9836 100644 --- a/modules/rescue.nf +++ b/modules/rescue.nf @@ -8,13 +8,15 @@ process RESCUE { input: path sampletable + path decodetable path vcf_files path mpileup output: - path "predictions.tsv" + path "predictions.tsv", emit: probabilities script: + def decode = decodetable ? "--decode ${decodetable}" : "" """ marbl \ --vcf-folder . \ diff --git a/subworkflows/pinpoint_tab.nf b/subworkflows/pinpoint_tab.nf index 53e8688..b969aa5 100644 --- a/subworkflows/pinpoint_tab.nf +++ b/subworkflows/pinpoint_tab.nf @@ -22,6 +22,8 @@ blacklist_bed_ch = indexedFileChannel(params.blacklist_bed, '.tbi') repeatmasker_bed_ch = indexedFileChannel(params.repeatmasker_bed, '.tbi') gnomad_vcf_ch = indexedFileChannel(params.gnomad_vcf, '.tbi') +annotations_ch = Channel.value([]) + workflow PINPOINT_TAB { take: vcf_file @@ -55,5 +57,10 @@ workflow PINPOINT_TAB { params.loftee_human_ancestor ?: [], params.loftee_sqlite ?: [] ) + annotations_ch = VEP.out.annotations_file } + emit: + pinpoint_variants = PIN_BASIC.out.pinpoint_variants + pool_variants = PIN_BASIC.out.pool_variants + annotations = annotations_ch } \ No newline at end of file diff --git a/subworkflows/variant_rescue.nf b/subworkflows/variant_rescue.nf index 74ddca4..b72c1b1 100644 --- a/subworkflows/variant_rescue.nf +++ b/subworkflows/variant_rescue.nf @@ -16,6 +16,7 @@ workflow VARIANT_RESCUE { vcf_file bam_file_w_index pooltable + decodetable reference_genome bedfile @@ -30,8 +31,8 @@ workflow VARIANT_RESCUE { .collect() .set { vcf_files } MPILEUP(bam_file_w_index, reference_genome, bedfile) - RESCUE(pooltable, vcf_files, MPILEUP.out.mpileup_file.collect()) + RESCUE(pooltable, decodetable, vcf_files, MPILEUP.out.mpileup_file.collect()) emit: - pinned_variants = Channel.empty() + rescue_probabilities = RESCUE.out.probabilities } \ No newline at end of file From 9e4e2b7e572551fc4fd3a1f529f72146b27b251a Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 15:54:58 +0200 Subject: [PATCH 42/95] fix CI container owner confusion --- conf/container.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/container.config b/conf/container.config index 3b5218a..50baa67 100644 --- a/conf/container.config +++ b/conf/container.config @@ -2,8 +2,10 @@ // mads 2024-02-09 // Base container image for shell scripts +// Docker runoptions fixes ownership confusion in some containers process { container = 'ubuntu:24.04' + docker.runOptions = '-u $(id -u):$(id -g)' } params { From c771218070f8ff52970a958cb8d2b3de7be168e5 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 17:33:51 +0200 Subject: [PATCH 43/95] force CI writability --- .github/workflows/nextflow-test-container.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/nextflow-test-container.yml b/.github/workflows/nextflow-test-container.yml index 45b9eed..9bace9c 100644 --- a/.github/workflows/nextflow-test-container.yml +++ b/.github/workflows/nextflow-test-container.yml @@ -35,6 +35,9 @@ jobs: sudo mv nextflow /usr/local/bin/ nextflow -version + - name: Ensure workspace writable + run: sudo chmod -R a+rwx "$GITHUB_WORKSPACE" + - name: Run Nextflow pipeline with Docker run: | nextflow run main.nf -profile test -with-docker From 17b6dcc57f5939e3b18b369af9b51baa6ece13cd Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 18:03:06 +0200 Subject: [PATCH 44/95] test user override with env variable --- .github/workflows/nextflow-test-container.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nextflow-test-container.yml b/.github/workflows/nextflow-test-container.yml index 9bace9c..cf2367a 100644 --- a/.github/workflows/nextflow-test-container.yml +++ b/.github/workflows/nextflow-test-container.yml @@ -28,17 +28,21 @@ jobs: key: docker-cache-${{ runner.os }}-${{ hashFiles('**/nextflow.config', '**/modules.config') }} restore-keys: | docker-cache-${{ runner.os }}- - + + - name: Capture runner UID/GID + run: | + echo "UID=$(id -u)" >> $GITHUB_ENV + echo "GID=$(id -g)" >> $GITHUB_ENV + - name: Install Nextflow run: | curl -s https://get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ nextflow -version - - name: Ensure workspace writable - run: sudo chmod -R a+rwx "$GITHUB_WORKSPACE" - - name: Run Nextflow pipeline with Docker + env: + NXF_DEFAULT_DOCKER_OPTS: "-u ${{ env.UID }}:${{ env.GID }}" run: | nextflow run main.nf -profile test -with-docker From 784c6b490bd99e422c5db7be75fc3abf3377778b Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 18:49:53 +0200 Subject: [PATCH 45/95] change user of docker --- .github/workflows/nextflow-test-container.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/nextflow-test-container.yml b/.github/workflows/nextflow-test-container.yml index cf2367a..3f5a3d6 100644 --- a/.github/workflows/nextflow-test-container.yml +++ b/.github/workflows/nextflow-test-container.yml @@ -29,11 +29,6 @@ jobs: restore-keys: | docker-cache-${{ runner.os }}- - - name: Capture runner UID/GID - run: | - echo "UID=$(id -u)" >> $GITHUB_ENV - echo "GID=$(id -g)" >> $GITHUB_ENV - - name: Install Nextflow run: | curl -s https://get.nextflow.io | bash @@ -41,8 +36,6 @@ jobs: nextflow -version - name: Run Nextflow pipeline with Docker - env: - NXF_DEFAULT_DOCKER_OPTS: "-u ${{ env.UID }}:${{ env.GID }}" run: | nextflow run main.nf -profile test -with-docker From d12ff7f1f460bccf7d0dfa4fca7b94d4c0d10884 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 19:46:53 +0200 Subject: [PATCH 46/95] Update container versions --- conf/container.config | 8 ++++---- envs/r_env/Dockerfile | 2 +- envs/rescue/build | 2 +- envs/rescue/environment.yaml | 2 +- envs/vep/Dockerfile | 12 ++++-------- envs/vep/build | 2 +- 6 files changed, 12 insertions(+), 16 deletions(-) diff --git a/conf/container.config b/conf/container.config index 50baa67..b938d92 100644 --- a/conf/container.config +++ b/conf/container.config @@ -32,8 +32,8 @@ params { r_env = "madscort/dswf_r:1.0" pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" - vep = "madscort/vep_w_loftee:111.0" - marbl = "madscort/marbl:0.2.0" + vep = "madscort/vep:111.0" + marbl = "madscort/marbl:0.3.0" } singularity { gatk = "https://depot.galaxyproject.org-singularity-gatk4-4.5.0.0--py36hdfd78af_0" @@ -62,8 +62,8 @@ params { r_env = "madscort/dswf_r:1.0" pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" - vep = "madscort/vep_w_loftee:111.0" - marbl = "madscort/marbl:0.2.0" + vep = "madscort/vep:111.0" + marbl = "madscort/marbl:0.3.0" } } } diff --git a/envs/r_env/Dockerfile b/envs/r_env/Dockerfile index adcbb29..24ed8d7 100644 --- a/envs/r_env/Dockerfile +++ b/envs/r_env/Dockerfile @@ -6,6 +6,6 @@ COPY environment.yaml /tmp/environment.yml RUN micromamba install -y -n base -f /tmp/environment.yml && micromamba clean -a -y -SHELL ["micromamba", "run", "-n", "base", "/bin/bash", "-c"] +ENV PATH="/opt/conda/bin:${PATH}" CMD ["R", "--version"] \ No newline at end of file diff --git a/envs/rescue/build b/envs/rescue/build index 3321b77..fb63d5e 100644 --- a/envs/rescue/build +++ b/envs/rescue/build @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t madscort/marbl:0.2.0 --push . \ No newline at end of file +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/marbl:0.3.0 --push . \ No newline at end of file diff --git a/envs/rescue/environment.yaml b/envs/rescue/environment.yaml index 1ab1d4f..9d9cf0b 100644 --- a/envs/rescue/environment.yaml +++ b/envs/rescue/environment.yaml @@ -6,4 +6,4 @@ channels: dependencies: - python=3.11 - pip: - - marbl-pool==0.2.0 \ No newline at end of file + - marbl-pool==0.3.0 \ No newline at end of file diff --git a/envs/vep/Dockerfile b/envs/vep/Dockerfile index 0f4be64..592482f 100644 --- a/envs/vep/Dockerfile +++ b/envs/vep/Dockerfile @@ -30,12 +30,11 @@ RUN git clone -b grch38 https://github.com/konradjk/loftee.git /loftee FROM ensemblorg/ensembl-vep:release_111.0 AS runtime -# Switch to root to install system packages and dependencies USER root # Set up Perl environment ENV PERL_MM_USE_DEFAULT=1 -ENV PERL5LIB=/plugins/loftee:/opt/vep/perl5/lib/perl5:$PERL5LIB +ENV PERL5LIB=/plugins:/opt/vep/perl5/lib/perl5:$PERL5LIB # Install system dependencies RUN apt-get update && apt-get -y install \ @@ -56,18 +55,15 @@ COPY --from=build /usr/local/bin/samtools /usr/local/bin/samtools RUN chmod +x /usr/local/bin/samtools # Copy LOFTEE plugins -COPY --from=build /loftee/ /plugins/loftee/ +COPY --from=build /loftee/ /plugins # Install additional Perl dependencies as root RUN cpanm --local-lib=/opt/vep/perl5 local::lib && \ cpanm --local-lib=/opt/vep/perl5 --force DBD::SQLite Bio::Perl -# Switch back to vep user -USER vep - # Verify installations -RUN samtools --version && ls -la /plugins/loftee/ && \ - perl -e 'use lib q(/plugins/loftee); require q(LoF.pm); require q(utr_splice.pl); print qq(LoFTEE load OK\n)' +RUN samtools --version && ls -la /plugins/ && \ + perl -e 'use lib q(/plugins); require q(LoF.pm); require q(utr_splice.pl); print qq(LoFTEE load OK\n)' # Set working directory WORKDIR /opt/vep \ No newline at end of file diff --git a/envs/vep/build b/envs/vep/build index 329203d..1d9f1f8 100644 --- a/envs/vep/build +++ b/envs/vep/build @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t madscort/vep_w_loftee:111.0 --push . \ No newline at end of file +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/vep:111.0 --push . \ No newline at end of file From 3b8b754a925c17fa8e3f1490a0327536ea49d71c Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 19:47:14 +0200 Subject: [PATCH 47/95] add decode to rescue --- modules/rescue.nf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/rescue.nf b/modules/rescue.nf index 7af9836..2aa11a8 100644 --- a/modules/rescue.nf +++ b/modules/rescue.nf @@ -16,12 +16,13 @@ process RESCUE { path "predictions.tsv", emit: probabilities script: - def decode = decodetable ? "--decode ${decodetable}" : "" + def decode = decodetable ? "--decodetable ${decodetable}" : "" """ marbl \ --vcf-folder . \ --mpileup-folder . \ --sampletable ${sampletable} \ + ${decode} \ --output . """ From 2514f22523f444d4fd62cb72215240649b6f6772 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 19:49:24 +0200 Subject: [PATCH 48/95] update loftee args --- modules/vep.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vep.nf b/modules/vep.nf index 7dc85f7..e024151 100644 --- a/modules/vep.nf +++ b/modules/vep.nf @@ -36,7 +36,7 @@ process VEP { def blackl = blacklist ? "--custom ${blacklist},blacklist,bed,overlap,0,4" : "" def repeatm = repeatmasks ? "--custom ${repeatmasks},repeats,bed,overlap,0,4" : "" def gnom = gnomad ? "--custom ${gnomad},gnomAD,vcf,exact,0,AC_joint_nfe,AN_joint_nfe,AF_joint_nfe,AC_genomes_nfe,AN_genomes_nfe,AF_genomes_nfe,AC_exomes_nfe,AN_exomes_nfe,AF_exomes_nfe,grpmax_joint,AC_grpmax_joint,AF_grpmax_joint,grpmax_genomes,AC_grpmax_genomes,AF_grpmax_genomes,grpmax_exomes,AC_grpmax_exomes,AF_grpmax_exomes" : "" - def loftee_plugin_dir = workflow.containerEngine ? "/plugins/loftee" : "./loftee" + def loftee_plugin_dir = workflow.containerEngine ? "/plugins" : "./loftee" def loft = loftee_gerp_bigwig && loftee_human_ancestor && loftee_conservation ? "--plugin LoF,loftee_path:${loftee_plugin_dir},gerp_bigwig:${loftee_gerp_bigwig},human_ancestor_fa:${loftee_human_ancestor},conservation_file:${loftee_conservation}" : "" """ vep \ From b0cb6086b460d3b77a506ae1abe2774bf3d218b7 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 20:29:33 +0200 Subject: [PATCH 49/95] update merge by columns --- bin/report.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/report.Rmd b/bin/report.Rmd index 5fd1f6f..de84690 100644 --- a/bin/report.Rmd +++ b/bin/report.Rmd @@ -75,7 +75,7 @@ rescue <- pinvars <- pinvars %>% - full_join(rescue, by=c('uvarid','row_id','column_id','varid')) + full_join(rescue, by=c('uvarid','row_id','column_id','varid','row_label','column_label','sample_alias')) ``` ```{r eval=have(params$annotations)} From 51a6d75f388b5dfe2e372e7b5851016d6e604e04 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Thu, 18 Sep 2025 20:29:46 +0200 Subject: [PATCH 50/95] force root in docker --- envs/r_env/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/envs/r_env/Dockerfile b/envs/r_env/Dockerfile index 24ed8d7..8693d39 100644 --- a/envs/r_env/Dockerfile +++ b/envs/r_env/Dockerfile @@ -4,6 +4,8 @@ WORKDIR /work COPY environment.yaml /tmp/environment.yml +USER root + RUN micromamba install -y -n base -f /tmp/environment.yml && micromamba clean -a -y ENV PATH="/opt/conda/bin:${PATH}" From bbcaf8789f6333f60219ba2d9e44839529803e49 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Fri, 19 Sep 2025 13:49:32 +0200 Subject: [PATCH 51/95] Switch GATK to picard container for including R. Remove duplicate metrics --- conf/container.config | 6 +++++- modules/duplicate_metrics.nf | 35 ---------------------------------- modules/gc_metrics.nf | 4 ++-- modules/insert_size_metrics.nf | 6 +++--- subworkflows/mapping.nf | 2 -- subworkflows/mapping_umi.nf | 3 --- 6 files changed, 10 insertions(+), 46 deletions(-) delete mode 100644 modules/duplicate_metrics.nf diff --git a/conf/container.config b/conf/container.config index b938d92..b637508 100644 --- a/conf/container.config +++ b/conf/container.config @@ -15,7 +15,6 @@ params { samtools = "quay.io/biocontainers/samtools:1.21--h50ea8bc_0" fastqc = "quay.io/biocontainers/fastqc:0.12.1--hdfd78af_0" bwa = "quay.io/biocontainers/mulled-v2-e5d375990341c5aef3c9aff74f96f66f65375ef6:2d15960ccea84e249a150b7f5d4db3a42fc2d6c3-0" - bbmap = "quay.io/biocontainers/bbmap:38.89-h1296035_0" fgbio = "quay.io/biocontainers/fgbio:1.5.1-0" multiqc = "quay.io/biocontainers/multiqc:1.26--pyhdfd78af_0" mosdepth = "quay.io/biocontainers/mosdepth:0.3.10--h4e814b3_1" @@ -29,6 +28,8 @@ params { // Custom and non-singularity containers octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" + bbmap = "community.wave.seqera.io/library/bbmap_pigz:07416fe99b090fa9" + picard = "community.wave.seqera.io/library/picard:3.4.0--e9963040df0a9bf6" r_env = "madscort/dswf_r:1.0" pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" @@ -56,9 +57,12 @@ params { bedtools = "https://depot.galaxyproject.org-singularity-bedtools-2.31.1--hf5e1c6e_0" sentieon = "https://depot.galaxyproject.org-singularity-sentieon-202308.02--h43eeafb_0" fgbio = "https://depot.galaxyproject.org-singularity-fgbio-2.1.0--hdfd78af_0" + bbmap = "https://depot.galaxyproject.org/singularity/bbmap:38.22--h14c3975_1" // Custom and non-singularity containers octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" + bbmap = "community.wave.seqera.io/library/bbmap_pigz:07416fe99b090fa9" + picard = "community.wave.seqera.io/library/picard:3.4.0--e9963040df0a9bf6" r_env = "madscort/dswf_r:1.0" pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" diff --git a/modules/duplicate_metrics.nf b/modules/duplicate_metrics.nf deleted file mode 100644 index 1088c03..0000000 --- a/modules/duplicate_metrics.nf +++ /dev/null @@ -1,35 +0,0 @@ -process DUPLICATE_METRICS { - label 'process_medium' - tag "$sample_id" - // Mark duplicate reads in BAM files - - conda "$projectDir/envs/gatk4/environment.yaml" - container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk - - publishDir "${params.outputDir}/log/duplicate_metrics/", pattern: "${sample_id}*.txt", mode:'copy' - - - input: - tuple val(sample_id), path(bam_file) - val log_suffix - - output: - path "${sample_id}*.txt", emit: metrics_file - - script: - def log_filename = log_suffix == "" ? "${sample_id}.dup.txt" : "${sample_id}_${log_suffix}.dup.txt" - """ - gatk MarkDuplicates \ - --I ${bam_file} \ - --M ${log_filename} \ - --O ${sample_id}.marked.bam - - rm ${sample_id}.marked.bam - """ - - stub: - def log_filename = log_suffix == "" ? "${sample_id}.dup.txt" : "${sample_id}_${log_suffix}.dup.txt" - """ - touch "${log_filename}" - """ -} \ No newline at end of file diff --git a/modules/gc_metrics.nf b/modules/gc_metrics.nf index 86750bf..38aaeed 100644 --- a/modules/gc_metrics.nf +++ b/modules/gc_metrics.nf @@ -4,7 +4,7 @@ process GC_METRICS { // Collect gc bias metrics for bam file conda "$projectDir/envs/gatk4/environment.yaml" - container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.picard : params.container.docker.picard publishDir "${params.outputDir}/log/gc_bias_metrics/", pattern: "${sample_id}*.gc.txt", mode:'copy' publishDir "${params.outputDir}/log/gc_bias_metrics/", pattern: "${sample_id}*.gc.summary.txt", mode:'copy' @@ -24,7 +24,7 @@ process GC_METRICS { def log_filename = log_suffix == "" ? "${sample_id}.gc.txt" : "${sample_id}_${log_suffix}.gc.txt" def log_filename_summary = log_suffix == "" ? "${sample_id}.gc.summary.txt" : "${sample_id}_${log_suffix}.gc.summary.txt" """ - gatk CollectGcBiasMetrics \ + picard CollectGcBiasMetrics \ -I ${bam_file} \ -O ${log_filename} \ -S ${log_filename_summary} \ diff --git a/modules/insert_size_metrics.nf b/modules/insert_size_metrics.nf index 6960097..e9079ab 100644 --- a/modules/insert_size_metrics.nf +++ b/modules/insert_size_metrics.nf @@ -4,7 +4,7 @@ process INSERT_SIZE_METRICS { // Collect insert size metrics for bam file conda "$projectDir/envs/gatk4/environment.yaml" - container workflow.containerEngine == 'singularity' ? params.container.singularity.gatk : params.container.docker.gatk + container workflow.containerEngine == 'singularity' ? params.container.singularity.picard : params.container.docker.picard publishDir "${params.outputDir}/log/insert_size_metrics/", pattern: "${sample_id}*.insert.txt", mode:'copy' publishDir "${params.outputDir}/log/insert_size_metrics/", pattern: "${sample_id}.pdf", mode:'copy' @@ -19,10 +19,10 @@ process INSERT_SIZE_METRICS { script: def log_filename = log_suffix == "" ? "${sample_id}.insert.txt" : "${sample_id}_${log_suffix}.insert.txt" """ - gatk CollectInsertSizeMetrics \ + picard CollectInsertSizeMetrics \ -I ${bam_file} \ -O ${log_filename} \ - -H ${sample_id}.pdf + -H plot.pdf """ stub: diff --git a/subworkflows/mapping.nf b/subworkflows/mapping.nf index 41ee236..c6c482a 100644 --- a/subworkflows/mapping.nf +++ b/subworkflows/mapping.nf @@ -27,7 +27,6 @@ include { HS_METRICS } from '../modules/hs_metrics' include { ALIGNMENT_METRICS } from '../modules/alignment_metrics' include { GC_METRICS } from '../modules/gc_metrics' include { INSERT_SIZE_METRICS } from '../modules/insert_size_metrics' -include { DUPLICATE_METRICS } from '../modules/duplicate_metrics' include { MULTIQC } from '../modules/multiqc' @@ -75,7 +74,6 @@ workflow MAPPING { ALIGNMENT_METRICS(ALIGNMENT.out.raw_bam_file, reference_genome, "") GC_METRICS(ALIGNMENT.out.raw_bam_file, reference_genome, "") INSERT_SIZE_METRICS(ALIGNMENT.out.raw_bam_file, "") - DUPLICATE_METRICS(ALIGNMENT.out.raw_bam_file, "") qc_ch = qc_ch.mix( RAW_DEPTH.out.region_dist, diff --git a/subworkflows/mapping_umi.nf b/subworkflows/mapping_umi.nf index c284f0d..551c989 100644 --- a/subworkflows/mapping_umi.nf +++ b/subworkflows/mapping_umi.nf @@ -31,7 +31,6 @@ include { HS_METRICS } from '../modules/hs_metrics' include { ALIGNMENT_METRICS } from '../modules/alignment_metrics' include { GC_METRICS } from '../modules/gc_metrics' include { INSERT_SIZE_METRICS } from '../modules/insert_size_metrics' -include { DUPLICATE_METRICS } from '../modules/duplicate_metrics' include { INDEX as RAW_INDEX } from '../modules/index' include { FLAGSTAT as RAW_FLAGSTAT } from '../modules/flagstat' include { MOSDEPTH as RAW_DEPTH } from '../modules/mosdepth' @@ -137,7 +136,6 @@ workflow MAPPING_UMI { ALIGNMENT_METRICS(MERGEBAM.out.bam_file, reference_genome, "") GC_METRICS(MERGEBAM.out.bam_file, reference_genome, "") INSERT_SIZE_METRICS(MERGEBAM.out.bam_file, "") - DUPLICATE_METRICS(MERGEBAM.out.bam_file, "") UMI_METRICS(GROUP_UMI.out.grouped_bam_file) // After consensus calling (HS metrics) CONS_METRIC(ADDREADGROUP.out.bam_file, reference_genome, bedfile, "deduplicated") @@ -148,7 +146,6 @@ workflow MAPPING_UMI { GC_METRICS.out.metrics_file, GC_METRICS.out.summary_file, INSERT_SIZE_METRICS.out.metrics_file, - DUPLICATE_METRICS.out.metrics_file, CONS_METRIC.out.metrics_file) } From 9a7e99a430b0a94a9d0d4fce2beceef96e5043ab Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Fri, 19 Sep 2025 13:53:53 +0200 Subject: [PATCH 52/95] update memory allocation for java to reflect task availability --- conf/profiles.config | 6 ++++++ modules/genomicsdb.nf | 3 ++- modules/genotypegvcf.nf | 3 ++- modules/genotypegvcf_split.nf | 3 ++- modules/haplotypecaller.nf | 3 ++- modules/haplotypecaller_split.nf | 3 ++- modules/haplotypecaller_truth.nf | 3 ++- modules/haplotypecaller_truth_joint.nf | 3 ++- modules/markduplicates.nf | 3 ++- modules/markduplicates_spark.nf | 4 +++- modules/merge_consensus.nf | 3 ++- 11 files changed, 27 insertions(+), 10 deletions(-) diff --git a/conf/profiles.config b/conf/profiles.config index 1a8785a..a2753c9 100644 --- a/conf/profiles.config +++ b/conf/profiles.config @@ -63,6 +63,12 @@ profiles { memory: '4.GB', time: '1.h' ] + executor = 'local' + + // Base configuration for unlabelled process + cpus = { 2 * task.attempt } + memory = { 4.GB * task.attempt } + time = { 1.h * task.attempt } } params { diff --git a/modules/genomicsdb.nf b/modules/genomicsdb.nf index d0c72ea..5760428 100644 --- a/modules/genomicsdb.nf +++ b/modules/genomicsdb.nf @@ -19,9 +19,10 @@ process GENOMICSDB { path "genomicsdb.log" script: + def avail_mem = (task.memory.mega*0.8).intValue() input_files_command = vcfs.collect(){"--variant ${it}"}.join(' ') """ - gatk --java-options "-Xmx8g -XX:-UsePerfData" \ + gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" \ GenomicsDBImport \ $input_files_command \ --genomicsdb-workspace-path genomicsdb \ diff --git a/modules/genotypegvcf.nf b/modules/genotypegvcf.nf index de4c6be..73ae286 100644 --- a/modules/genotypegvcf.nf +++ b/modules/genotypegvcf.nf @@ -21,8 +21,9 @@ process GENOTYPEGVCF { script: def db = file(params.reference_genome).getName() + ".fna" def publishDir = file(params.outputDir + "/variants/" + sample_id + ".GATK.vcf.gz") + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options "-Xmx8g -XX:-UsePerfData" \ + gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" \ GenotypeGVCFs \ -R ${db} \ -V ${gvcf_file} \ diff --git a/modules/genotypegvcf_split.nf b/modules/genotypegvcf_split.nf index 2b4bef9..28baf35 100644 --- a/modules/genotypegvcf_split.nf +++ b/modules/genotypegvcf_split.nf @@ -19,8 +19,9 @@ process GENOTYPEGVCF_SPLIT { script: def db = file(params.reference_genome).getName() + ".fna" def id = interval ? "${sample_id}.${interval}" : sample_id + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options "-Xmx8g -XX:-UsePerfData" \ + gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" \ GenotypeGVCFs \ -R ${db} \ -V ${vcf_file} \ diff --git a/modules/haplotypecaller.nf b/modules/haplotypecaller.nf index 5544426..b6ee430 100644 --- a/modules/haplotypecaller.nf +++ b/modules/haplotypecaller.nf @@ -20,8 +20,9 @@ process HAPLOTYPECALLER { script: def db = file(params.reference_genome).getName() + ".fna" + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options "-Xmx8g -XX:-UsePerfData" HaplotypeCaller \ + gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" HaplotypeCaller \ -R ${db} \ -I ${bam_file} \ -L ${bedfile} \ diff --git a/modules/haplotypecaller_split.nf b/modules/haplotypecaller_split.nf index 16a492b..f01d92a 100644 --- a/modules/haplotypecaller_split.nf +++ b/modules/haplotypecaller_split.nf @@ -23,8 +23,9 @@ process HAPLOTYPECALLER_SPLIT { def id = interval ? "${sample_id}.${interval}" : sample_id def add_intervals = interval ? "-L ${interval}" : "" def add_intersection = interval ? "--interval-set-rule INTERSECTION" : "" + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options "-Xmx8g -XX:-UsePerfData" HaplotypeCaller \ + gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" HaplotypeCaller \ -R ${db} \ -I ${bam_file} \ -L ${interval_list} \ diff --git a/modules/haplotypecaller_truth.nf b/modules/haplotypecaller_truth.nf index a0c6f55..24e576b 100644 --- a/modules/haplotypecaller_truth.nf +++ b/modules/haplotypecaller_truth.nf @@ -20,8 +20,9 @@ process HC_TRUTH { script: def db = file(params.reference_genome).getName() + ".fna" + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options "-Xmx8g -XX:-UsePerfData" \ + gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" \ HaplotypeCaller \ -R ${db} \ -I ${bam_file} \ diff --git a/modules/haplotypecaller_truth_joint.nf b/modules/haplotypecaller_truth_joint.nf index c169bb3..de4c378 100644 --- a/modules/haplotypecaller_truth_joint.nf +++ b/modules/haplotypecaller_truth_joint.nf @@ -21,8 +21,9 @@ process HC_TRUTH_JOINT { script: def db = file(params.reference_genome).getName() + ".fna" + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options "-Xmx8g -XX:-UsePerfData" \ + gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" \ HaplotypeCaller \ -R ${db} \ -L ${bedfile} \ diff --git a/modules/markduplicates.nf b/modules/markduplicates.nf index 69c437f..0936086 100644 --- a/modules/markduplicates.nf +++ b/modules/markduplicates.nf @@ -20,8 +20,9 @@ process MARKDUPLICATES { script: def log_filename = log_suffix == "" ? "${sample_id}.dupMetric.log" : "${sample_id}_${log_suffix}.dupMetric.log" def optical_only = optical_only_tag ? "--duplicate-tagging-policy OpticalOnly" : "" + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options -Xmx16g MarkDuplicates \ + gatk --java-options -Xmx${avail_mem}M MarkDuplicates \ --OPTICAL_DUPLICATE_PIXEL_DISTANCE 2500 \ ${optical_only} \ --TMP_DIR . \ diff --git a/modules/markduplicates_spark.nf b/modules/markduplicates_spark.nf index f544f76..26c075d 100644 --- a/modules/markduplicates_spark.nf +++ b/modules/markduplicates_spark.nf @@ -20,8 +20,10 @@ process MARKDUPLICATES_SPARK { script: def log_filename = log_suffix == "" ? "${sample_id}.dupMetric.log" : "${sample_id}_${log_suffix}.dupMetric.log" def optical_only = optical_only_tag ? "--duplicate-tagging-policy OpticalOnly" : "" + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options -Xmx16g MarkDuplicatesSpark \ + gatk --java-options -Xmx${avail_mem}M \ + MarkDuplicatesSpark \ --optical-duplicate-pixel-distance 2500 \ ${optical_only} \ --tmp-dir . \ diff --git a/modules/merge_consensus.nf b/modules/merge_consensus.nf index 8235981..d04d332 100644 --- a/modules/merge_consensus.nf +++ b/modules/merge_consensus.nf @@ -14,8 +14,9 @@ process MERGE_CONSENSUS { script: def db = file(params.reference_genome).getName() + ".fna" + def avail_mem = (task.memory.mega*0.8).intValue() """ - gatk --java-options -Xmx20g MergeBamAlignment \ + gatk --java-options -Xmx${avail_mem}M MergeBamAlignment \ TMP_DIR=. \ UNMAPPED=${ubam_file} \ ALIGNED=${bam_file} \ From f3cb7d35aca8fc6d280db2c0b5fc5041b4b8aba5 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Fri, 19 Sep 2025 13:57:52 +0200 Subject: [PATCH 53/95] Add conda picard environment --- envs/picard/environment.yaml | 8 ++++++++ modules/gc_metrics.nf | 2 +- modules/insert_size_metrics.nf | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 envs/picard/environment.yaml diff --git a/envs/picard/environment.yaml b/envs/picard/environment.yaml new file mode 100644 index 0000000..971bbe5 --- /dev/null +++ b/envs/picard/environment.yaml @@ -0,0 +1,8 @@ +name: picard +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - r-base=4.3.0 + - bioconda::picard=4.6.1.0 \ No newline at end of file diff --git a/modules/gc_metrics.nf b/modules/gc_metrics.nf index 38aaeed..b9c7e8d 100644 --- a/modules/gc_metrics.nf +++ b/modules/gc_metrics.nf @@ -3,7 +3,7 @@ process GC_METRICS { tag "$sample_id" // Collect gc bias metrics for bam file - conda "$projectDir/envs/gatk4/environment.yaml" + conda "$projectDir/envs/picard/environment.yaml" container workflow.containerEngine == 'singularity' ? params.container.singularity.picard : params.container.docker.picard publishDir "${params.outputDir}/log/gc_bias_metrics/", pattern: "${sample_id}*.gc.txt", mode:'copy' diff --git a/modules/insert_size_metrics.nf b/modules/insert_size_metrics.nf index e9079ab..c7eed54 100644 --- a/modules/insert_size_metrics.nf +++ b/modules/insert_size_metrics.nf @@ -3,7 +3,7 @@ process INSERT_SIZE_METRICS { tag "$sample_id" // Collect insert size metrics for bam file - conda "$projectDir/envs/gatk4/environment.yaml" + conda "$projectDir/envs/picard/environment.yaml" container workflow.containerEngine == 'singularity' ? params.container.singularity.picard : params.container.docker.picard publishDir "${params.outputDir}/log/insert_size_metrics/", pattern: "${sample_id}*.insert.txt", mode:'copy' From b7767bfde9efdd48b363bb176b1929c4f11cc674 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Sat, 20 Sep 2025 13:52:33 +0200 Subject: [PATCH 54/95] fix bgzip stub --- modules/bgzip.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/bgzip.nf b/modules/bgzip.nf index a592996..c128374 100644 --- a/modules/bgzip.nf +++ b/modules/bgzip.nf @@ -21,8 +21,8 @@ process BGZIP { stub: """ - touch "${sample_id}.vcf.gz" - touch "${sample_id}.vcf.gz.tbi" + touch "${vcf_file}.gz" + touch "${vcf_file}.gz.tbi" """ } From c391086319bb2272edba71d6e5b24ffee6bfd9d2 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Sat, 20 Sep 2025 13:53:09 +0200 Subject: [PATCH 55/95] Update conda picard version --- conf/container.config | 4 ++-- envs/picard/environment.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/container.config b/conf/container.config index b637508..83cd901 100644 --- a/conf/container.config +++ b/conf/container.config @@ -34,7 +34,7 @@ params { pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" vep = "madscort/vep:111.0" - marbl = "madscort/marbl:0.3.0" + marbl = "madscort/marbl:0.4.0" } singularity { gatk = "https://depot.galaxyproject.org-singularity-gatk4-4.5.0.0--py36hdfd78af_0" @@ -67,7 +67,7 @@ params { pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" vep = "madscort/vep:111.0" - marbl = "madscort/marbl:0.3.0" + marbl = "madscort/marbl:0.4.0" } } } diff --git a/envs/picard/environment.yaml b/envs/picard/environment.yaml index 971bbe5..13baff6 100644 --- a/envs/picard/environment.yaml +++ b/envs/picard/environment.yaml @@ -5,4 +5,4 @@ channels: - defaults dependencies: - r-base=4.3.0 - - bioconda::picard=4.6.1.0 \ No newline at end of file + - bioconda::picard=3.4.0 \ No newline at end of file From 649045c0e7ed6f20c79a2c9b1c5c6983cf5b162b Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Sat, 20 Sep 2025 13:53:30 +0200 Subject: [PATCH 56/95] update readme --- README.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index aa772aa..bf0955f 100644 --- a/README.md +++ b/README.md @@ -43,19 +43,24 @@ Configurations can be supplied a configuration file, see ```config.json```, and nextflow run main.nf \ (-with-docker/-with-apptainer/-with-conda) \ --pooltable \ - --decodetable \ + (--decodetable \) --reference_genome \ --bedfile \ --ploidy ``` -The ```pooltable.tsv``` should connect (user assigned) pool id's to input FASTQ files; one entry for each pool. +The ```pooltable.tsv``` should connect (user assigned) pool id's and their row/column arrangement to input FASTQ files; one tab-separated line for each pool. ```Bash -pool_row_1 path/to/sample1_R1.fq.gz path/to/sample1_R2.fq.gz -pool_column_1 path/to/sample2_R1.fq.gz path/to/sample2_R2.fq.gz +pool_1 row path/to/sample1_R1.fq.gz path/to/sample1_R2.fq.gz +pool_2 row path/to/sample2_R1.fq.gz path/to/sample2_R2.fq.gz +pool_3 column path/to/sample3_R1.fq.gz path/to/sample3_R2.fq.gz +pool_4 column path/to/sample4_R1.fq.gz path/to/sample4_R2.fq.gz ``` -The ```decodetable.tsv``` should map (user assigned) individual id's in the matrix to the corresponding row and column id's of each pool; one entry for each element in the matrix. +The optional ```decodetable.tsv``` should map (user assigned) individual id's in the matrix to the corresponding row and column id's of each pool; one entry for each element in the matrix. ```Bash -individual1 pool_row_1 pool_column_1 +individual1 pool_1 pool_3 +individual2 pool_2 pool_3 +individual3 pool_1 pool_4 +individual4 pool_2 pool_4 ``` ### 6. Pipeline output The workflow will output a results folder containing multiple config dependent output files: @@ -63,9 +68,11 @@ The workflow will output a results folder containing multiple config dependent o results ├── pinpointables.vcf # Merged VCF file containing all assigned variants ├── cram/ # CRAM files for each pool +├── context/ # TSV and JSON files with matrix information and pool-individual linkage. ├── logs/ # Log files for each process ├── variants/ # VCF files for each pool ├── variant_tables/ # TSV files converted from pool VCFs +├── variant_compilation/ # TSV files with aggregated variants, annotations and rescue probabilities └── pinpoint_variants/ ├── all_pins/ # All pinpointables for each sample in individual vcfs (*note) ├── unique_pins/ # All unique pinpointables for each sample in individual vcfs (*note) @@ -75,6 +82,34 @@ results ``` A central files is the ```pinpointables.vcf```. This file contains all individually assigned variants. Since each variant contains information from two pools, these a presented as the sample columns: ROW and COLUMN. +## Variant annotation +Two annotations workflows are currently available. SnpEff for VCF output files and VEP for tabular compiled output. They can be applied by adding the following configurations and absolute paths to the JSON params file. +```Bash +config.json +{ + # Annotate output VCFs with SnpEff and clinvar + annotate: true + snpeff_db: "GRCh38.99" + snpeff_config: "snpEff.config" + snpeff_cache: "cache/" + clinvar_db: "clinvar_20230903.vcf.gz" # (*.tbi in same folder) + + # Annotate tabular output with VEP + annotate_vep: true + vep_cache: "cache/" # (Current version 111.x) + # Optional VEP input: + danmac_db: "danmac.vcf.gz" # (*.tbi in same folder) + blacklist_bed: "hg38-blacklist.v2.sorted.bed.gz" # (*.tbi in same folder) + repeatmasker_bed: "repeatmasker.sorted.bed.gz" # (*.tbi in same folder) + gnomad_vcf: "gnomad.vcf.bgz" # (*.tbi in same folder) + utr_file: "uORF_5UTR_GRCh38_PUBLIC.txt" + alphamissense_tsv: "AlphaMissense_hg38.tsv.gz" + loftee_gerp_bw: "gerp_conservation_scores.homo_sapiens.GRCh38.bw" + loftee_human_ancestor: "human_ancestor.fa.gz" + loftee_sqlite: "loftee.sql" +} +``` + # Workflow repository contents: ```Bash @@ -95,6 +130,7 @@ DoBSeqWF │ │ ├── pooltable.tsv │ │ ├── snvlist.tsv │ │ └── target_calling.bed +│ ├── filter/ # Filter model modules and parameters │ └── helper_scripts │ └── simulator.py # Script for simulating minimal pipeline data ├── bin # Executable pipeline scripts @@ -106,11 +142,12 @@ DoBSeqWF ├── envs │ └── / │ └── environment.yaml # Conda environment definitions +├── lib/ # Pipeline groovy utility functions ├── main.nf # Main workflow ├── modules/ │ └── .nf # Module scripts ├── subworkflows/ -│ └── .nf # Module scripts +│ └── .nf # Subworkflow scripts ├── next.pbs # Helper script for running on NGC-HPC └── nextflow.config # Workflow parameters ``` @@ -180,8 +217,8 @@ predisposed ├── / │ ├── DoBSeqWF/ # Clone repository here │ │ ├── config.json # Configuration file - │ │ ├── pooltable.tsv # Pool table (create with helper script) - │ │ └── decodetable.tsv # Decode table (we need a convention for this) + │ │ ├── pooltable.tsv # Pool table + │ │ └── decodetable.tsv # Decode table │ └── results │ ├── cram/ # CRAM files for each pool │ ├── logs/ # Log files for each process From a51273227949b3e7ea6a102202521f2cdcf29c19 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Sat, 20 Sep 2025 13:54:48 +0200 Subject: [PATCH 57/95] add snpeff version to ngc config --- conf/ngc.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/ngc.config b/conf/ngc.config index 682a8b8..ea9d1d7 100644 --- a/conf/ngc.config +++ b/conf/ngc.config @@ -5,6 +5,7 @@ profiles { outputDir = "results" project = "icope_staging_r" cacheDir = "/ngc/projects2/dp_00005/scratch/nextflow" + snpeff_db = 'GRCh38.99' snpeff_config = "/ngc/projects2/dp_00005/data/dwf/databases/snpeff/snpEff.config" snpeff_cache = "/ngc/projects2/dp_00005/data/dwf/databases/snpeff/data" vep_cache = "/ngc/shared/nf_tools/references/vep_cache" From a5d354337298c559956a6623dca6cede739943bf Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Sat, 20 Sep 2025 13:58:52 +0200 Subject: [PATCH 58/95] bump rescue --- envs/rescue/build | 2 +- envs/rescue/environment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/envs/rescue/build b/envs/rescue/build index fb63d5e..6130634 100644 --- a/envs/rescue/build +++ b/envs/rescue/build @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t madscort/marbl:0.3.0 --push . \ No newline at end of file +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/marbl:0.4.0 --push . \ No newline at end of file diff --git a/envs/rescue/environment.yaml b/envs/rescue/environment.yaml index 9d9cf0b..5f56ac6 100644 --- a/envs/rescue/environment.yaml +++ b/envs/rescue/environment.yaml @@ -6,4 +6,4 @@ channels: dependencies: - python=3.11 - pip: - - marbl-pool==0.3.0 \ No newline at end of file + - marbl-pool==0.4.0 \ No newline at end of file From 88eba6264368152ad0a20a850f6255ecf69ed876 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Sun, 21 Sep 2025 14:05:17 -0400 Subject: [PATCH 59/95] change default config --- nextflow.config | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/nextflow.config b/nextflow.config index 241cee0..59f9901 100644 --- a/nextflow.config +++ b/nextflow.config @@ -4,9 +4,6 @@ params { - // ID for VCF output files: - outputId = "" - // Table containing pool ids and corresponding FastQ files pooltable = "" @@ -69,11 +66,11 @@ params { // Variant calling methods lofreq = false - gatk_joint_calling = false + gatk_joint_calling = false // WGS only crisp = false // Pool only (not available through conda) - octopus = false // Pool only (linux only) + octopus = false // Pool only (container only) freebayes = false // Pool only - deepvariant = false // WGS only (linux only) + deepvariant = false // WGS only (container only) // Variant filtering. Either optimised for sensitivity or balance filter = false @@ -100,10 +97,10 @@ params { snpeff_cache = "" clinvar_db = "" // Add gene annotation based on third column in bedfile - bedfile_gene_annotation = true + bedfile_gene_annotation = false // VEP annotations - annotate_vep = true + annotate_vep = false vep_cache = "" clinvar_db = "" danmac_db = "" From e1e094fd03b3a3467d25bacc269cfc2bd2b44110 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Sun, 21 Sep 2025 15:12:06 -0400 Subject: [PATCH 60/95] update base config files --- config.json | 3 +-- config_qc.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/config.json b/config.json index 189fa33..94dc2a1 100644 --- a/config.json +++ b/config.json @@ -1,8 +1,7 @@ { "ploidy": 48, - "step": "mapping", + "step": "all", "pooltable": "pooltable.tsv", - "decodetable": "decodetable.tsv", "outputDir": "../results", "reference_genome": "/ngc/projects2/dp_00005/data/predisposed/resources/reference/Homo_sapiens_assembly38_noAlt", "bedfile": "/ngc/projects2/dp_00005/data/predisposed/resources/targets/Probes_merged_ok_SSI_PREDiSPOSED-v1_1X_TE-93129556_hg38.bed" diff --git a/config_qc.json b/config_qc.json index 969eb3c..f5c23a5 100644 --- a/config_qc.json +++ b/config_qc.json @@ -2,9 +2,9 @@ "ploidy": 48, "step": "mapping", "downsample": 4, + "outputDir": "../results_qc_150x", "fullQC": "true", "pooltable": "pooltable.tsv", - "decodetable": "decodetable.tsv", "reference_genome": "/ngc/projects2/dp_00005/data/predisposed/resources/reference/Homo_sapiens_assembly38_noAlt", "bedfile": "/ngc/projects2/dp_00005/data/predisposed/resources/targets/Probes_merged_ok_SSI_PREDiSPOSED-v1_1X_TE-93129556_hg38.bed" } \ No newline at end of file From 736ebb956924f6d3e02129204da71bb7d936577a Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 22 Sep 2025 06:24:22 -0400 Subject: [PATCH 61/95] update base config --- nextflow.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nextflow.config b/nextflow.config index 59f9901..3f47c6b 100644 --- a/nextflow.config +++ b/nextflow.config @@ -88,10 +88,10 @@ params { variant_rescue = true // Make pinpoint report (true or false) - pinpoint_report = true + pinpoint_report = false // Annotation configurations - annotate = true + annotate = false snpeff_db = 'GRCh38.99' snpeff_config = "" snpeff_cache = "" From 5e6b6b7335d9207f26fad9b9fb5b605786897dcd Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 22 Sep 2025 06:39:15 -0400 Subject: [PATCH 62/95] update report --- bin/report.Rmd | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/bin/report.Rmd b/bin/report.Rmd index de84690..26b0e2c 100644 --- a/bin/report.Rmd +++ b/bin/report.Rmd @@ -75,7 +75,8 @@ rescue <- pinvars <- pinvars %>% - full_join(rescue, by=c('uvarid','row_id','column_id','varid','row_label','column_label','sample_alias')) + full_join(rescue, by=c('uvarid','row_id','column_id','row_label','column_label','varid','sample_alias')) %>% + mutate(is_pool_pin = if_else(is.na(is_pool_pin),0,is_pool_pin)) ``` ```{r eval=have(params$annotations)} @@ -83,7 +84,18 @@ pinvars <- annotations <- read_tsv(annotations_path, na = '-', col_types = cols(.default = col_character())) %>% - type_convert(guess_integer = TRUE) + type_convert(guess_integer = TRUE) %>% + mutate(is_lof = if_else(LoF == 'HC',1,0)) %>% + mutate(clinvar_stars = case_when( + str_detect(ClinVar_CLNREVSTAT, "practice_guideline") ~ 4, + str_detect(ClinVar_CLNREVSTAT, "reviewed_by_expert_panel") ~ 3, + str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_multiple_submitters,_no_conflicts") ~ 2, + str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_conflicting_interpretations") ~ 1, + str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_single_submitter") ~ 1, + TRUE ~ NA_real_ + )) %>% + mutate(is_p = if_else(str_detect(ClinVar_CLNSIG, 'Pathogenic') | str_detect(ClinVar_CLNSIG, "Likely_pathogenic"), 1,0)) %>% + mutate(is_lofp = if_else(is_lof == 1 | is_p == 1,1,0)) pinvars <- pinvars %>% @@ -100,6 +112,7 @@ annotations <- read_tsv(annotations_path, na = '-', col_types = cols(.default = col_character())) %>% type_convert(guess_integer = TRUE) %>% + mutate(is_lof = if_else(LoF == 'HC',1,0)) %>% mutate(clinvar_stars = case_when( str_detect(ClinVar_CLNREVSTAT, "practice_guideline") ~ 4, str_detect(ClinVar_CLNREVSTAT, "reviewed_by_expert_panel") ~ 3, @@ -166,8 +179,10 @@ pinvars <- ```{r} plot_data <- pinvars %>% + filter(is_pool_pin == 1) %>% group_by(sample_id,row_factor, column_factor) %>% summarise(private_variants = n()) + plot_data %>% ggplot(aes(x = column_factor, y = row_factor, fill = private_variants)) + geom_point(shape = 22, size = 14, color='black') + @@ -190,8 +205,8 @@ plot_data %>% ```{r eval=have(params$annotations)} plot_data <- pinvars %>% - mutate(row_factor = fct_rev(factor(row_label, levels = 1:2, ordered = TRUE)), - column_factor = factor(column_label, levels = LETTERS[1:2], ordered = TRUE)) %>% + filter(is_pool_pin == 1) %>% + filter(is_lofp == 1) %>% group_by(sample_id,row_factor, column_factor) %>% summarise(private_variants = n()) @@ -214,11 +229,15 @@ plot_data %>% ggtitle('pLoF/p only') ``` + + + + ```{r eval=have(params$annotations) & have(params$rescue)} plot_data <- pinvars %>% - mutate(row_factor = fct_rev(factor(row_label, levels = 1:2, ordered = TRUE)), - column_factor = factor(column_label, levels = LETTERS[1:2], ordered = TRUE)) %>% + filter((probability >= 0.5 & is_single) | is_pool_pin == 1) %>% + filter(is_lofp == 1) %>% group_by(sample_id,row_factor, column_factor) %>% summarise(private_variants = n()) From 572811ecd095db69f02e1c94fdf947fe221818d5 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 23 Sep 2025 13:51:39 -0400 Subject: [PATCH 63/95] change sort method --- bin/build_matrix.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bin/build_matrix.py b/bin/build_matrix.py index ab246e7..77c8e8f 100755 --- a/bin/build_matrix.py +++ b/bin/build_matrix.py @@ -35,9 +35,6 @@ from pathlib import Path from typing import Dict, List, Tuple, Optional -def natural_key(s: str): - return [int(t) if t.isdigit() else t.lower() for t in re.findall(r"\d+|\D+", s)] - def read_pools(pools_path: Path) -> Tuple[List[str], List[str]]: rows, cols = [], [] with pools_path.open() as fh: @@ -108,8 +105,8 @@ def build_context(rows: List[str], decode: Dict[Tuple[str,str], str], pad_width: Optional[int]) -> dict: # Force deterministic ordering - rows_ord = sorted(rows, key=natural_key) - cols_ord = sorted(cols, key=natural_key) + rows_ord = sorted(rows, key=str.lower) + cols_ord = sorted(cols, key=str.lower) # Indices and labels row_index = {pid: i for i, pid in enumerate(rows_ord)} From 06bacbe623d8889c4176255d46f455629fe5e17e Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 23 Sep 2025 13:52:07 -0400 Subject: [PATCH 64/95] remove dup metrics leftover --- conf/ngc.config | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conf/ngc.config b/conf/ngc.config index ea9d1d7..1783eb5 100644 --- a/conf/ngc.config +++ b/conf/ngc.config @@ -107,12 +107,6 @@ profiles { time = 2.hour } - withName: DUPLICATE_METRICS { - // module = ['tools','oracle_jdk/21.0.2','gatk/4.6.0.0'] - cpus = 4 - memory = 16.GB - time = 2.hour - } withName: GROUP_UMI { // module = ['tools','java/1.8.0','fgbio/1.5.1'] From ea8d107653d1992d30a118601fb360cd37ab9885 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 23 Sep 2025 13:53:29 -0400 Subject: [PATCH 65/95] update marbl --- conf/container.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/container.config b/conf/container.config index 83cd901..6aa01f7 100644 --- a/conf/container.config +++ b/conf/container.config @@ -34,7 +34,7 @@ params { pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" vep = "madscort/vep:111.0" - marbl = "madscort/marbl:0.4.0" + marbl = "madscort/marbl:0.6.0" } singularity { gatk = "https://depot.galaxyproject.org-singularity-gatk4-4.5.0.0--py36hdfd78af_0" @@ -67,7 +67,7 @@ params { pinpy = "madscort/dswf_pinpy:1.0" filter_variants = "madscort/dswf_filter:1.0" vep = "madscort/vep:111.0" - marbl = "madscort/marbl:0.4.0" + marbl = "madscort/marbl:0.6.0" } } } From ce908cd1495f20e346d212a122cd7fe00bb5f17c Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 23 Sep 2025 14:22:41 -0400 Subject: [PATCH 66/95] update container versions --- envs/pinpy/build | 2 +- envs/pinpy/environment.yaml | 5 +++-- envs/rescue/build | 2 +- envs/rescue/environment.yaml | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/envs/pinpy/build b/envs/pinpy/build index 3418c0b..7efe5aa 100644 --- a/envs/pinpy/build +++ b/envs/pinpy/build @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64,windows/amd64 -t madscort/dswf_pinpy:1.0 --push . \ No newline at end of file +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/dswf_pinpy:1.0 --push . \ No newline at end of file diff --git a/envs/pinpy/environment.yaml b/envs/pinpy/environment.yaml index 7e89b5e..523699a 100644 --- a/envs/pinpy/environment.yaml +++ b/envs/pinpy/environment.yaml @@ -4,5 +4,6 @@ channels: - bioconda - defaults dependencies: - - bcftools=1.19 - - python=3.9 \ No newline at end of file + - bcftools=1.22 + - python=3.9 + - samtools=1.22 \ No newline at end of file diff --git a/envs/rescue/build b/envs/rescue/build index 6130634..c58e67d 100644 --- a/envs/rescue/build +++ b/envs/rescue/build @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t madscort/marbl:0.4.0 --push . \ No newline at end of file +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/marbl:0.6.0 --push . \ No newline at end of file diff --git a/envs/rescue/environment.yaml b/envs/rescue/environment.yaml index 5f56ac6..8c93ebb 100644 --- a/envs/rescue/environment.yaml +++ b/envs/rescue/environment.yaml @@ -6,4 +6,4 @@ channels: dependencies: - python=3.11 - pip: - - marbl-pool==0.4.0 \ No newline at end of file + - marbl-pool==0.6.0 \ No newline at end of file From 8c4740cf43baafa026cea48440f62b51413ba183 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 23 Sep 2025 15:25:52 -0400 Subject: [PATCH 67/95] remove node type pbs script --- next.pbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/next.pbs b/next.pbs index b93290b..a6cfa3b 100644 --- a/next.pbs +++ b/next.pbs @@ -14,7 +14,7 @@ # myqsub [-F "optional arguments to Nexflow"] next.pbs # -#PBS -l nodes=1:ppn=40:thinnode +#PBS -l nodes=1:ppn=40 #PBS -l walltime=24:00:00 #PBS -l mem=160gb #PBS -j oe From 8caae32668d2a4b654dc400ecf73d93fc2eaa1c7 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 23 Sep 2025 16:30:05 -0400 Subject: [PATCH 68/95] update report --- bin/report.Rmd | 41 +++-------------------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/bin/report.Rmd b/bin/report.Rmd index 26b0e2c..1a39c5d 100644 --- a/bin/report.Rmd +++ b/bin/report.Rmd @@ -60,7 +60,8 @@ output_path <- here::here('reports') pinvars <- read_tsv(pinpoints_path, col_types = cols(.default = col_character())) %>% - type_convert(guess_integer = TRUE) + type_convert(guess_integer = TRUE) %>% + mutate(row_label = as.integer(row_label)) poolvars <- read_tsv(all_variants_path, @@ -71,6 +72,7 @@ poolvars <- ```{r eval=have(params$rescue)} rescue <- read_tsv(rescue_path) %>% + mutate(row_label = as.integer(row_label)) %>% select(-sample_id) pinvars <- @@ -106,43 +108,6 @@ poolvars <- left_join(annotations, by='varid') ``` - -```{r eval=F} -annotations <- - read_tsv(annotations_path, - na = '-', col_types = cols(.default = col_character())) %>% - type_convert(guess_integer = TRUE) %>% - mutate(is_lof = if_else(LoF == 'HC',1,0)) %>% - mutate(clinvar_stars = case_when( - str_detect(ClinVar_CLNREVSTAT, "practice_guideline") ~ 4, - str_detect(ClinVar_CLNREVSTAT, "reviewed_by_expert_panel") ~ 3, - str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_multiple_submitters,_no_conflicts") ~ 2, - str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_conflicting_interpretations") ~ 1, - str_detect(ClinVar_CLNREVSTAT, "criteria_provided,_single_submitter") ~ 1, - TRUE ~ NA_real_ - )) %>% - mutate(is_p = if_else(str_detect(ClinVar_CLNSIG, 'Pathogenic') | str_detect(ClinVar_CLNSIG, "Likely_pathogenic"), 1,0)) %>% - mutate(is_lofp = if_else(is_lof == 1 | is_p == 1,1,0)) - -rescue <- - read_tsv(rescue_path) %>% - select(-sample_id) - -pinvars <- - read_tsv(pinpoints_path, - col_types = cols(.default = col_character())) %>% - type_convert(guess_integer = TRUE) %>% - left_join(rescue, by=c('uvarid','row_id','column_id','varid')) %>% - left_join(annotations, by='varid') - -poolvars <- - read_tsv(all_variants_path, - col_types = cols(.default = col_character())) %>% - type_convert(guess_integer = TRUE) %>% - left_join(annotations, by='varid') - -``` - ```{r} matrix_ctx <- fromJSON(matrix_context_path) matrix_size <- matrix_ctx$matrix$n_rows From a4af788a2650eefda7950ebfa0183a21818da62f Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Wed, 15 Oct 2025 14:01:33 +0200 Subject: [PATCH 69/95] add new module ngsep --- .gitignore | 1 + conf/container.config | 2 ++ envs/ngsep/Dockerfile | 31 ++++++++++++++++++++++++++++ envs/ngsep/build | 1 + envs/ngsep/environment.yaml | 7 +++++++ modules/ngsep.nf | 41 +++++++++++++++++++++++++++++++++++++ modules/test.nf | 6 +++--- nextflow.config | 5 +++-- subworkflows/calling.nf | 5 +++++ 9 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 envs/ngsep/Dockerfile create mode 100644 envs/ngsep/build create mode 100644 envs/ngsep/environment.yaml create mode 100644 modules/ngsep.nf diff --git a/.gitignore b/.gitignore index 8a2bf23..3f436c8 100644 --- a/.gitignore +++ b/.gitignore @@ -184,3 +184,4 @@ vcftable.tsv pixi.toml pixi.lock .DS_Store +.vscode diff --git a/conf/container.config b/conf/container.config index 6aa01f7..9843955 100644 --- a/conf/container.config +++ b/conf/container.config @@ -27,6 +27,7 @@ params { deepvariant = "quay.io/biocontainers/deepvariant:1.5.0--py36hf3e76ba_0" // Custom and non-singularity containers + ngsep = "madscort/ngsep:5.1.0" octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" bbmap = "community.wave.seqera.io/library/bbmap_pigz:07416fe99b090fa9" picard = "community.wave.seqera.io/library/picard:3.4.0--e9963040df0a9bf6" @@ -60,6 +61,7 @@ params { bbmap = "https://depot.galaxyproject.org/singularity/bbmap:38.22--h14c3975_1" // Custom and non-singularity containers + ngsep = "madscort/ngsep:5.1.0" octopus = "quay.io/biocontainers/octopus:0.7.4--hacbf28e_1" bbmap = "community.wave.seqera.io/library/bbmap_pigz:07416fe99b090fa9" picard = "community.wave.seqera.io/library/picard:3.4.0--e9963040df0a9bf6" diff --git a/envs/ngsep/Dockerfile b/envs/ngsep/Dockerfile new file mode 100644 index 0000000..302ef38 --- /dev/null +++ b/envs/ngsep/Dockerfile @@ -0,0 +1,31 @@ +# Docker container for NGSEP +FROM eclipse-temurin:21-jre + +# Add metadata +LABEL maintainer="bioinformatics" +LABEL description="NGSEP - Next Generation Sequencing Experience Platform" +LABEL version="5.1.0" + +# Install wget for downloading +RUN apt-get update && \ + apt-get install -y wget && \ + rm -rf /var/lib/apt/lists/* + +# Download NGSEP JAR from GitHub +RUN wget -O /usr/local/bin/NGSEPcore_5.1.0.jar \ + https://github.com/NGSEP/NGSEPcore/raw/refs/heads/master/NGSEPcore_5.1.0.jar && \ + chmod 644 /usr/local/bin/NGSEPcore_5.1.0.jar && \ + ln -s /usr/local/bin/NGSEPcore_5.1.0.jar /usr/local/bin/NGSEPcore.jar + +# Create a convenience script for running NGSEP +RUN echo '#!/bin/bash\njava $JAVA_OPTS -jar /usr/local/bin/NGSEPcore.jar "$@"' > /usr/local/bin/ngsep && \ + chmod +x /usr/local/bin/ngsep + +# Create working directory +WORKDIR /data + +# Set environment variable for Java options (can be overridden) +ENV JAVA_OPTS="-Xmx4g" + +# Default command shows help +CMD ["ngsep", "--help"] \ No newline at end of file diff --git a/envs/ngsep/build b/envs/ngsep/build new file mode 100644 index 0000000..4fc9334 --- /dev/null +++ b/envs/ngsep/build @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64 -t madscort/ngsep:5.1.0 --push . \ No newline at end of file diff --git a/envs/ngsep/environment.yaml b/envs/ngsep/environment.yaml new file mode 100644 index 0000000..bbe76fc --- /dev/null +++ b/envs/ngsep/environment.yaml @@ -0,0 +1,7 @@ +name: ngsep +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::ngsep=4.0.1 \ No newline at end of file diff --git a/modules/ngsep.nf b/modules/ngsep.nf new file mode 100644 index 0000000..5999760 --- /dev/null +++ b/modules/ngsep.nf @@ -0,0 +1,41 @@ +process NGSEP { + label 'process_low' + tag "NGSEP - $sample_id" + // Call variants using NGSEP + + conda "$projectDir/envs/ngsep/environment.yaml" + container workflow.containerEngine == 'singularity' ? params.container.singularity.ngsep : params.container.docker.ngsep + + publishDir "${params.outputDir}/log/ngsep/", pattern: "${sample_id}.ngsep.log", mode:'copy' + publishDir "${params.outputDir}/variants/", pattern: "${sample_id}.ngsep.vcf", mode:'copy' + + input: + tuple val(sample_id), path(bam_file), path(bam_index_file) + path reference_genome + path bedfile + + output: + tuple val(sample_id), path("${sample_id}.ngsep.vcf"), emit: vcf_file + path "${sample_id}.ngsep.log" + + script: + def db = file(params.reference_genome).getName() + ".fna" + """ + ngsep \ + SingleSampleVariantsDetector \ + -maxAlnsPerStartPos 50000 \ + -h 0.1 \ + -ploidy ${params.ploidy} \ + -r ${db} \ + -i ${bam_file} \ + -o "${sample_id}.ngsep" \ + > >(tee -a "${sample_id}.ngsep.log") \ + 2> >(tee -a "${sample_id}.ngsep.log" >&2) + """ + + stub: + """ + touch "${sample_id}.ngsep.vcf" "${sample_id}.ngsep.log" + """ +} + diff --git a/modules/test.nf b/modules/test.nf index 049c96d..eaa4736 100644 --- a/modules/test.nf +++ b/modules/test.nf @@ -11,8 +11,8 @@ process TEST { val pinpoint_method output: - path "test.passed" optional true - path "test.failed" optional true + path "test.passed", optional: true + path "test.failed", optional: true path "test.log" script: @@ -29,4 +29,4 @@ process TEST { touch test.passed touch test.log """ -} \ No newline at end of file +} diff --git a/nextflow.config b/nextflow.config index 3f47c6b..1ec9888 100644 --- a/nextflow.config +++ b/nextflow.config @@ -67,10 +67,11 @@ params { // Variant calling methods lofreq = false gatk_joint_calling = false // WGS only + deepvariant = false // WGS only (container only) crisp = false // Pool only (not available through conda) octopus = false // Pool only (container only) - freebayes = false // Pool only - deepvariant = false // WGS only (container only) + freebayes = true // Pool only + ngsep = true // Pool only // Variant filtering. Either optimised for sensitivity or balance filter = false diff --git a/subworkflows/calling.nf b/subworkflows/calling.nf index da4456f..72ded77 100644 --- a/subworkflows/calling.nf +++ b/subworkflows/calling.nf @@ -12,6 +12,7 @@ include { DEEPVARIANT } from '../modules/deepvariant' include { FREEBAYES } from '../modules/freebayes' include { CRISP } from '../modules/crisp' include { OCTOPUS } from '../modules/octopus' +include { NGSEP } from '../modules/ngsep' include { FILTER } from '../modules/filter' include { GENOMICSDB } from '../modules/genomicsdb' include { GENOTYPEGVCF } from '../modules/genotypegvcf' @@ -57,6 +58,10 @@ workflow CALLING { FREEBAYES(bam_file_w_index, reference_genome, bedfile) } + if (params.ngsep) { + NGSEP(bam_file_w_index, reference_genome, bedfile) + } + if (params.runHCParallel) { // Split by interval list to multi-thread calling Channel From 4f2c856b3306a9cab12d937e540cbeaddda9a93e Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:22:49 +0200 Subject: [PATCH 70/95] update report + resource limits --- bin/report.Rmd | 5 +++-- conf/profiles.config | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/report.Rmd b/bin/report.Rmd index 1a39c5d..4bf6141 100644 --- a/bin/report.Rmd +++ b/bin/report.Rmd @@ -61,7 +61,8 @@ pinvars <- read_tsv(pinpoints_path, col_types = cols(.default = col_character())) %>% type_convert(guess_integer = TRUE) %>% - mutate(row_label = as.integer(row_label)) + mutate(row_label = as.integer(row_label), + is_pool_pin = 1) poolvars <- read_tsv(all_variants_path, @@ -77,7 +78,7 @@ rescue <- pinvars <- pinvars %>% - full_join(rescue, by=c('uvarid','row_id','column_id','row_label','column_label','varid','sample_alias')) %>% + full_join(rescue, by=c('uvarid','row_id','column_id','row_label','column_label','varid','sample_alias', 'is_pool_pin')) %>% mutate(is_pool_pin = if_else(is.na(is_pool_pin),0,is_pool_pin)) ``` diff --git a/conf/profiles.config b/conf/profiles.config index a2753c9..f295055 100644 --- a/conf/profiles.config +++ b/conf/profiles.config @@ -21,7 +21,10 @@ profiles { process { executor = 'local' - + resourceLimits = [ + cpus: 5, + memory: '30.GB' + ] // Base configuration for unlabelled process cpus = { 1 * task.attempt } memory = { 6.GB * task.attempt } From f4009708ca1c72554fd07bbf1503eb6bef09d0a5 Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:23:55 +0200 Subject: [PATCH 71/95] downgrade samtools for pinpy --- envs/pinpy/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/pinpy/environment.yaml b/envs/pinpy/environment.yaml index 523699a..c77e65f 100644 --- a/envs/pinpy/environment.yaml +++ b/envs/pinpy/environment.yaml @@ -6,4 +6,4 @@ channels: dependencies: - bcftools=1.22 - python=3.9 - - samtools=1.22 \ No newline at end of file + - samtools=1.20 From 2d0380fbf1da6fba7c4e261a4f791e5aa022abab Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:26:41 +0200 Subject: [PATCH 72/95] fix shebang --- bin/pin_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/pin_basic.py b/bin/pin_basic.py index e871043..2538f39 100755 --- a/bin/pin_basic.py +++ b/bin/pin_basic.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python import json, sys, argparse from pathlib import Path from dataclasses import dataclass, fields @@ -422,4 +422,4 @@ def main(): ) if __name__ == '__main__': - main() \ No newline at end of file + main() From 541674e5283f591638127c52695b00d798681656 Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:27:33 +0200 Subject: [PATCH 73/95] prevent compiler issues --- envs/rescue/environment.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/envs/rescue/environment.yaml b/envs/rescue/environment.yaml index 8c93ebb..4bcecfc 100644 --- a/envs/rescue/environment.yaml +++ b/envs/rescue/environment.yaml @@ -5,5 +5,13 @@ channels: - defaults dependencies: - python=3.11 + - pip + - numpy + - pandas>=2.3.2 + - pysam=0.23.3 + - htslib>=1.18 + - zlib + - libdeflate + - compilers - pip: - marbl-pool==0.6.0 \ No newline at end of file From 0281fec6c6a068dcfe405a99e226365e21b91df3 Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:28:13 +0200 Subject: [PATCH 74/95] update vep --- envs/vep/environment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/envs/vep/environment.yaml b/envs/vep/environment.yaml index 87b3315..5729966 100644 --- a/envs/vep/environment.yaml +++ b/envs/vep/environment.yaml @@ -4,4 +4,5 @@ channels: - bioconda - defaults dependencies: - - bioconda::ensembl-vep=111.0 \ No newline at end of file + - bioconda::ensembl-vep=114.2 + - bioconda::samtools=1.20 From 829c881332d148dc84f351bdfc480d903cf7c3be Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:29:31 +0200 Subject: [PATCH 75/95] fix loftee path --- modules/vep.nf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/vep.nf b/modules/vep.nf index e024151..6937ae5 100644 --- a/modules/vep.nf +++ b/modules/vep.nf @@ -18,6 +18,7 @@ process VEP { tuple path(blacklist), path(blacklist_index) // optional blacklist BED tuple path(repeatmasks), path(repeat_index) // optional repeat masks BED tuple path(gnomad), path(gnomad_index) // optional gnomAD VCF + path loftee_plugin_path, stageAs: "loftee" // optional LOFTEE github repo path path loftee_gerp_bigwig // optional LOFTEE GERP BigWig path loftee_human_ancestor // optional LOFTEE human ancestor FASTA path loftee_conservation // optional LOFTEE conservation SQL @@ -27,7 +28,7 @@ process VEP { path "annotations_w_varid.tsv", emit: annotations_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" def mode = cache_dir ? "--cache --offline --dir_cache ${cache_dir}" : "--database" def utr = utr_annotation ? "--plugin UTRAnnotator,file=${utr_annotation}" : "" def am = alphamissense ? "--plugin AlphaMissense,file=${alphamissense}" : "" From 117180c5dea435b1d5372e815bc65298e165d95e Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:31:15 +0200 Subject: [PATCH 76/95] add loftee path --- nextflow.config | 2 ++ subworkflows/pinpoint_tab.nf | 1 + 2 files changed, 3 insertions(+) diff --git a/nextflow.config b/nextflow.config index 3f47c6b..c2592eb 100644 --- a/nextflow.config +++ b/nextflow.config @@ -109,6 +109,7 @@ params { gnomad_vcf = "" utr_file = "" alphamissense_tsv = "" + loftee_path = "" loftee_gerp_bw = "" loftee_human_ancestor= "" loftee_sqlite = "" @@ -127,6 +128,7 @@ process.shell = ['/bin/bash', '-euo', 'pipefail'] includeConfig 'conf/profiles.config' includeConfig 'conf/container.config' includeConfig 'conf/ngc.config' +includeConfig 'conf/ssi.config' // Add additional log and tracing. Trace can be adjusted to specific outputs. diff --git a/subworkflows/pinpoint_tab.nf b/subworkflows/pinpoint_tab.nf index b969aa5..9d4de33 100644 --- a/subworkflows/pinpoint_tab.nf +++ b/subworkflows/pinpoint_tab.nf @@ -53,6 +53,7 @@ workflow PINPOINT_TAB { blacklist_bed_ch, repeatmasker_bed_ch, gnomad_vcf_ch, + params.loftee_path ?: [], params.loftee_gerp_bw ?: [], params.loftee_human_ancestor ?: [], params.loftee_sqlite ?: [] From 5c8f3cbed7d62e812a58a6a242d63763c77ad408 Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:31:38 +0200 Subject: [PATCH 77/95] add ssi specifc conf --- conf/ssi.config | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 conf/ssi.config diff --git a/conf/ssi.config b/conf/ssi.config new file mode 100644 index 0000000..1aef6a0 --- /dev/null +++ b/conf/ssi.config @@ -0,0 +1,70 @@ +profiles { + + ssi { + + conda.enabled = true + + params { + outputDir = "../results" + cacheDir = "/srv/data/scratch/nf_tmp/" + reference_genome = "/srv/data/Reference/GATK_bundle/hg38_noScaffoldsMasked/Homo_sapiens_assembly38_noScaffoldsMasked" + bedfile = "/srv/data/bedFiles/BabyCompare/Target_bases_covered_by_probes_SSI_BabyPrecision-v2p1_1X_TE-95845496_hg38.bed" + + snpeff_db = 'GRCh38.99' + snpeff_config = "" + snpeff_cache = "" + vep_cache = "/srv/data/VCF_annotations/VEP/vep" + clinvar_db = "/srv/data/VCF_annotations/CLINVAR/clinvar.vcf.gz" + danmac_db = "" + blacklist_bed = "/srv/data/Projects/BabyCompare/vep_optional_annotation/hg38-blacklist.v2.sorted.bed.gz" + repeatmasker_bed = "/srv/data/Projects/BabyCompare/vep_optional_annotation/repeatmasker.sorted.bed.gz" + gnomad_vcf = "/srv/data/Projects/BabyCompare/vep_optional_annotation/baby_compare_gnomad.vcf.bgz" + utr_file = "/srv/data/Projects/BabyCompare/vep_optional_annotation/uORF_5UTR_GRCh38_PUBLIC.txt" + alphamissense_tsv = "/srv/data/Projects/BabyCompare/vep_optional_annotation/AlphaMissense_hg38.tsv.gz" + loftee_path = "/srv/data/Projects/BabyCompare/vep_optional_annotation/loftee" + loftee_gerp_bw = "/srv/data/Projects/BabyCompare/vep_optional_annotation/gerp_conservation_scores.homo_sapiens.GRCh38.bw" + loftee_human_ancestor= "/srv/data/Projects/BabyCompare/vep_optional_annotation/human_ancestor.fa.gz" + loftee_sqlite = "/srv/data/Projects/BabyCompare/vep_optional_annotation/loftee.sql" + } + + process { + executor = 'local' + beforeScript = "export _JAVA_OPTIONS=-Djava.io.tmpdir=$params.cacheDir" + + resourceLimits = [ + cpus: 5, + memory: '30.GB' + ] + + // Base configuration for unlabelled process + cpus = { 2 * task.attempt } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } + + // Labelled configurations + withLabel:process_single { + cpus = { 1 } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } + } + + withLabel:process_low { + cpus = { 2 * task.attempt } + memory = { 12.GB * task.attempt } + time = { 4.h * task.attempt } + } + + withLabel:process_medium { + cpus = { 6 * task.attempt } + memory = { 36.GB * task.attempt } + time = { 8.h * task.attempt } + } + + withLabel:process_multi { + cpus = { 24 * task.attempt } + memory = { 30.GB * task.attempt } + time = { 8.h * task.attempt } + } + } + } +} \ No newline at end of file From ad201e8c6befa488b8dd96174619c7a229c59a07 Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 20 Oct 2025 20:37:25 +0200 Subject: [PATCH 78/95] tmp fasta suffix fix --- modules/alignment.nf | 2 +- modules/bam.nf | 2 +- modules/cram.nf | 2 +- modules/genotypegvcf.nf | 2 +- modules/haplotypecaller.nf | 2 +- modules/indelqual.nf | 2 +- modules/mpileup.nf | 2 +- modules/normalise_vcf.nf | 2 +- modules/rescue.nf | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/alignment.nf b/modules/alignment.nf index 1464fe1..47c33e1 100644 --- a/modules/alignment.nf +++ b/modules/alignment.nf @@ -17,7 +17,7 @@ process ALIGNMENT { path "${sample_id}.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" // int bwa_cpus = Math.max((task.cpus * 0.75) as int, 1) // int samtools_cpus = Math.max(task.cpus - bwa_cpus, 1) diff --git a/modules/bam.nf b/modules/bam.nf index 4b3f3e7..e8032ef 100644 --- a/modules/bam.nf +++ b/modules/bam.nf @@ -13,7 +13,7 @@ process BAM { tuple val(sample_id), path("${sample_id}.bam"), emit: bam_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" """ samtools view \ -@ ${task.cpus-1} \ diff --git a/modules/cram.nf b/modules/cram.nf index 9e02208..dad7342 100644 --- a/modules/cram.nf +++ b/modules/cram.nf @@ -16,7 +16,7 @@ process CRAM { path("${sample_id}_cram.tsv"), emit: cram_info script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" def publishDir = file(params.outputDir + "/cram/" + sample_id + ".cram") """ samtools view \ diff --git a/modules/genotypegvcf.nf b/modules/genotypegvcf.nf index 73ae286..7fc321d 100644 --- a/modules/genotypegvcf.nf +++ b/modules/genotypegvcf.nf @@ -19,7 +19,7 @@ process GENOTYPEGVCF { path "${sample_id}.genotypegvcf.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" def publishDir = file(params.outputDir + "/variants/" + sample_id + ".GATK.vcf.gz") def avail_mem = (task.memory.mega*0.8).intValue() """ diff --git a/modules/haplotypecaller.nf b/modules/haplotypecaller.nf index b6ee430..f63db4b 100644 --- a/modules/haplotypecaller.nf +++ b/modules/haplotypecaller.nf @@ -19,7 +19,7 @@ process HAPLOTYPECALLER { path "${sample_id}.g.haplotypecaller.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" def avail_mem = (task.memory.mega*0.8).intValue() """ gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" HaplotypeCaller \ diff --git a/modules/indelqual.nf b/modules/indelqual.nf index 94cee3c..1b31897 100644 --- a/modules/indelqual.nf +++ b/modules/indelqual.nf @@ -17,7 +17,7 @@ process INDELQUAL { path "${sample_id}.lofreq.iq.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" """ lofreq indelqual \ --dindel \ diff --git a/modules/mpileup.nf b/modules/mpileup.nf index ef12e8a..317cc2d 100644 --- a/modules/mpileup.nf +++ b/modules/mpileup.nf @@ -16,7 +16,7 @@ process MPILEUP { path("${sample_id}.mpileup.gz"), emit: mpileup_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" """ samtools mpileup \ -l ${bedfile} \ diff --git a/modules/normalise_vcf.nf b/modules/normalise_vcf.nf index 5e3662b..04bc77d 100644 --- a/modules/normalise_vcf.nf +++ b/modules/normalise_vcf.nf @@ -21,7 +21,7 @@ process NORMALISE_VCF { path "${sample_id}.${caller}.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).getName() + ".fasta" """ bcftools norm \ --check-ref e \ diff --git a/modules/rescue.nf b/modules/rescue.nf index 2aa11a8..fe9b3d5 100644 --- a/modules/rescue.nf +++ b/modules/rescue.nf @@ -30,4 +30,4 @@ process RESCUE { """ touch predictions.tsv """ -} \ No newline at end of file +} From e5d5a431b27297d7085fd35aee8dca984d837dfb Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 18 Nov 2025 14:24:21 +0100 Subject: [PATCH 79/95] update freebayes --- conf/container.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/container.config b/conf/container.config index 9843955..c54b2d0 100644 --- a/conf/container.config +++ b/conf/container.config @@ -19,7 +19,7 @@ params { multiqc = "quay.io/biocontainers/multiqc:1.26--pyhdfd78af_0" mosdepth = "quay.io/biocontainers/mosdepth:0.3.10--h4e814b3_1" lofreq = "quay.io/biocontainers/lofreq:2.1.5--py37h3a01142_1" - freebayes = "quay.io/biocontainers/freebayes:1.2.0--py35h82df9c4_2" + freebayes = "quay.io/biocontainers/freebayes:1.3.6--hbfe0e7f_2" bcftools = "quay.io/biocontainers/bcftools:1.19--h8b25389_1" snpeff = "quay.io/biocontainers/snpeff:4.3--2" snpsift = "quay.io/biocontainers/snpsift:5.1d--hdfd78af_0" From bc43659a526adf3e19d003b5b6ffae913edc726c Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Sat, 22 Nov 2025 07:56:29 +0100 Subject: [PATCH 80/95] update sensible defaults --- conf/ssi.config | 16 +++++++++------- next.ssi | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 next.ssi diff --git a/conf/ssi.config b/conf/ssi.config index 1aef6a0..42dd71b 100644 --- a/conf/ssi.config +++ b/conf/ssi.config @@ -6,10 +6,10 @@ profiles { params { outputDir = "../results" + pooltable = "pooltable.tsv" cacheDir = "/srv/data/scratch/nf_tmp/" reference_genome = "/srv/data/Reference/GATK_bundle/hg38_noScaffoldsMasked/Homo_sapiens_assembly38_noScaffoldsMasked" - bedfile = "/srv/data/bedFiles/BabyCompare/Target_bases_covered_by_probes_SSI_BabyPrecision-v2p1_1X_TE-95845496_hg38.bed" - + bedfile = "/srv/data/Projects/BabyCompare/vep_optional_annotation/Target_bases_covered_by_probes_SSI_BabyPrecision-v2p1_1X_TE-95845496_hg38_ext151_merged.bed" snpeff_db = 'GRCh38.99' snpeff_config = "" snpeff_cache = "" @@ -32,10 +32,12 @@ profiles { beforeScript = "export _JAVA_OPTIONS=-Djava.io.tmpdir=$params.cacheDir" resourceLimits = [ - cpus: 5, - memory: '30.GB' + cpus: 16, + memory: '48.GB' ] + maxRetries = 2 + // Base configuration for unlabelled process cpus = { 2 * task.attempt } memory = { 6.GB * task.attempt } @@ -50,8 +52,8 @@ profiles { withLabel:process_low { cpus = { 2 * task.attempt } - memory = { 12.GB * task.attempt } - time = { 4.h * task.attempt } + memory = { 6.GB * task.attempt } + time = { 12.h * task.attempt } } withLabel:process_medium { @@ -67,4 +69,4 @@ profiles { } } } -} \ No newline at end of file +} diff --git a/next.ssi b/next.ssi new file mode 100644 index 0000000..8bb459c --- /dev/null +++ b/next.ssi @@ -0,0 +1,51 @@ +#!/bin/bash +# +# next.ssi +# +# mads +# 2025-10-20 +# +# This is a simple wrapper that that runs the DoBSeq workflow on SSI. +# The pipeline profile uses environment modules and runs all processes as submitted jobs. +# +# +# Usage: +# +# myqsub [-F "optional arguments to Nexflow"] next.pbs +# + +## exit if any errors or unset variables are encountered +set -euo pipefail + +# Disable interactive logging +#export NXF_ANSI_LOG=false + +nextflow_essential_params='-profile ssi -resume' + +nextflow_command="nextflow run main.nf $nextflow_essential_params $@" + +echo "-------- Log generated by nextflow.pbs -------" | tee -a nextflow.log +echo "Command: $0 $@" | tee -a nextflow.log +echo "User:" `whoami` | tee -a nextflow.log +echo "Date:" `date -Is` | tee -a nextflow.log +echo "PBS job ID: ${PBS_JOBID-none}" | tee -a nextflow.log +echo 'DoBSeq version:' `cat VERSION` | tee -a nextflow.log +echo 'git commit:' `(git rev-parse HEAD)` | tee -a nextflow.log +echo 'git status:' | tee -a nextflow.log +(git status -bs) | sed "s/^/ /" | tee -a nextflow.log +echo 'Nextflow version:' `nextflow -version` 2>&1 | tee -a nextflow.log +echo "Nextflow command: ${nextflow_command}" | tee -a nextflow.log + +echo `date -Is` "--------------------- Starting Nextflow ---------------------" | tee -a nextflow.log + +set +eo pipefail + +${nextflow_command} 2>&1 | tee -a nextflow.log + +echo `date -Is` "--------------------- Nextflow finished ---------------------" | tee -a nextflow.log + +if [ -z ${PBS_JOBID+x} ]; then + echo "This was not run as a PBS job" | tee -a nextflow.log +else + echo "This was run as PBS job ${PBS_JOBID}" | tee -a nextflow.log +fi From 8077bc46b62e082e7ae0d572fcd543acd67daa53 Mon Sep 17 00:00:00 2001 From: madscort <61156331+madscort@users.noreply.github.com> Date: Sat, 22 Nov 2025 08:07:29 +0100 Subject: [PATCH 81/95] Update next.ssi Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- next.ssi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/next.ssi b/next.ssi index 8bb459c..3bd2f48 100644 --- a/next.ssi +++ b/next.ssi @@ -24,7 +24,7 @@ nextflow_essential_params='-profile ssi -resume' nextflow_command="nextflow run main.nf $nextflow_essential_params $@" -echo "-------- Log generated by nextflow.pbs -------" | tee -a nextflow.log +echo "-------- Log generated by next.ssi -------" | tee -a nextflow.log echo "Command: $0 $@" | tee -a nextflow.log echo "User:" `whoami` | tee -a nextflow.log echo "Date:" `date -Is` | tee -a nextflow.log From e1446dca25b5d02989e10b86a5cf61da7fdfe326 Mon Sep 17 00:00:00 2001 From: madscort <61156331+madscort@users.noreply.github.com> Date: Sat, 22 Nov 2025 08:07:36 +0100 Subject: [PATCH 82/95] Update next.ssi Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- next.ssi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/next.ssi b/next.ssi index 3bd2f48..b6aeeac 100644 --- a/next.ssi +++ b/next.ssi @@ -3,7 +3,7 @@ # next.ssi # # mads -# 2025-10-20 +# 2024-10-20 # # This is a simple wrapper that that runs the DoBSeq workflow on SSI. # The pipeline profile uses environment modules and runs all processes as submitted jobs. From 562bdce96ce85238b0c740d9a4c9452c7a4ffacf Mon Sep 17 00:00:00 2001 From: madscort <61156331+madscort@users.noreply.github.com> Date: Sat, 22 Nov 2025 08:07:55 +0100 Subject: [PATCH 83/95] Update next.ssi Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- next.ssi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/next.ssi b/next.ssi index b6aeeac..de18fb1 100644 --- a/next.ssi +++ b/next.ssi @@ -11,7 +11,7 @@ # # Usage: # -# myqsub [-F "optional arguments to Nexflow"] next.pbs +# myqsub [-F "optional arguments to Nextflow"] next.pbs # ## exit if any errors or unset variables are encountered From 3486dc5f60e59c9247b1b936148eb09a06bd1cf0 Mon Sep 17 00:00:00 2001 From: madscort <61156331+madscort@users.noreply.github.com> Date: Sat, 22 Nov 2025 08:08:32 +0100 Subject: [PATCH 84/95] Update next.ssi Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- next.ssi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/next.ssi b/next.ssi index de18fb1..b9bb59d 100644 --- a/next.ssi +++ b/next.ssi @@ -5,7 +5,7 @@ # mads # 2024-10-20 # -# This is a simple wrapper that that runs the DoBSeq workflow on SSI. +# This is a simple wrapper that runs the DoBSeq workflow on SSI. # The pipeline profile uses environment modules and runs all processes as submitted jobs. # # From 4ccea02627fb9a98c5e9e111a21f3dd1a093e6b7 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 10 Feb 2026 08:47:33 +0100 Subject: [PATCH 85/95] minor resource update --- conf/ssi.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/ssi.config b/conf/ssi.config index 42dd71b..afc59e4 100644 --- a/conf/ssi.config +++ b/conf/ssi.config @@ -32,7 +32,7 @@ profiles { beforeScript = "export _JAVA_OPTIONS=-Djava.io.tmpdir=$params.cacheDir" resourceLimits = [ - cpus: 16, + cpus: 12, memory: '48.GB' ] @@ -53,7 +53,7 @@ profiles { withLabel:process_low { cpus = { 2 * task.attempt } memory = { 6.GB * task.attempt } - time = { 12.h * task.attempt } + time = { 24.h * task.attempt } } withLabel:process_medium { From f4e61b596c15fcaa1683d90b79eaee0566a60ad9 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 10 Feb 2026 09:40:12 +0100 Subject: [PATCH 86/95] remove hard-coding ref genome --- conf/ngc.config | 1 + conf/profiles.config | 2 +- conf/ssi.config | 2 +- config.json | 1 - config_qc.json | 1 - config_test.json | 2 +- main.nf | 7 ++++++- modules/alignment.nf | 2 +- modules/alignment_metrics.nf | 2 +- modules/alignment_umi.nf | 2 +- modules/apply_bqsr.nf | 2 +- modules/bam.nf | 2 +- modules/bqsr.nf | 2 +- modules/combinegvcfs.nf | 2 +- modules/cram.nf | 2 +- modules/crisp.nf | 2 +- modules/deepvariant.nf | 2 +- modules/freebayes.nf | 2 +- modules/gc_metrics.nf | 2 +- modules/genotypegvcf.nf | 2 +- modules/genotypegvcf_split.nf | 2 +- modules/haplotypecaller.nf | 4 ++-- modules/haplotypecaller_split.nf | 2 +- modules/haplotypecaller_truth.nf | 2 +- modules/haplotypecaller_truth_joint.nf | 2 +- modules/hs_metrics.nf | 2 +- modules/indelqual.nf | 2 +- modules/intervals.nf | 2 +- modules/lofreq.nf | 2 +- modules/merge_consensus.nf | 2 +- modules/mergebam.nf | 2 +- modules/mpileup.nf | 2 +- modules/normalise_vcf.nf | 2 +- modules/subset.nf | 2 +- modules/test.nf | 4 ++-- modules/vep.nf | 2 +- nextflow.config | 3 +++ 37 files changed, 44 insertions(+), 37 deletions(-) diff --git a/conf/ngc.config b/conf/ngc.config index 1783eb5..f55def4 100644 --- a/conf/ngc.config +++ b/conf/ngc.config @@ -5,6 +5,7 @@ profiles { outputDir = "results" project = "icope_staging_r" cacheDir = "/ngc/projects2/dp_00005/scratch/nextflow" + reference_genome = "/ngc/projects2/dp_00005/data/predisposed/resources/reference/Homo_sapiens_assembly38_noAlt.fna" snpeff_db = 'GRCh38.99' snpeff_config = "/ngc/projects2/dp_00005/data/dwf/databases/snpeff/snpEff.config" snpeff_cache = "/ngc/projects2/dp_00005/data/dwf/databases/snpeff/data" diff --git a/conf/profiles.config b/conf/profiles.config index f295055..20872df 100644 --- a/conf/profiles.config +++ b/conf/profiles.config @@ -85,7 +85,7 @@ profiles { pooltable = "$projectDir/assets/data/test_data/pooltable.tsv" fqtable = "$projectDir/assets/data/test_data/pooltable.tsv" decodetable = "$projectDir/assets/data/test_data/decodetable.tsv" - reference_genome = "$projectDir/assets/data/reference_genomes/small/small_reference" + reference_genome = "$projectDir/assets/data/reference_genomes/small/small_reference.fna" bedfile = "$projectDir/assets/data/test_data/target_calling.bed" bedfile_bam_extraction = "$projectDir/assets/data/test_data/target_calling.bed" dictfile = "$projectDir/assets/data/test_data/target_calling.dict" diff --git a/conf/ssi.config b/conf/ssi.config index afc59e4..b5cd0e9 100644 --- a/conf/ssi.config +++ b/conf/ssi.config @@ -8,7 +8,7 @@ profiles { outputDir = "../results" pooltable = "pooltable.tsv" cacheDir = "/srv/data/scratch/nf_tmp/" - reference_genome = "/srv/data/Reference/GATK_bundle/hg38_noScaffoldsMasked/Homo_sapiens_assembly38_noScaffoldsMasked" + reference_genome = "/srv/data/Reference/GATK_bundle/hg38_noScaffoldsMasked/Homo_sapiens_assembly38_noScaffoldsMasked.fasta" bedfile = "/srv/data/Projects/BabyCompare/vep_optional_annotation/Target_bases_covered_by_probes_SSI_BabyPrecision-v2p1_1X_TE-95845496_hg38_ext151_merged.bed" snpeff_db = 'GRCh38.99' snpeff_config = "" diff --git a/config.json b/config.json index 94dc2a1..34bb187 100644 --- a/config.json +++ b/config.json @@ -3,6 +3,5 @@ "step": "all", "pooltable": "pooltable.tsv", "outputDir": "../results", - "reference_genome": "/ngc/projects2/dp_00005/data/predisposed/resources/reference/Homo_sapiens_assembly38_noAlt", "bedfile": "/ngc/projects2/dp_00005/data/predisposed/resources/targets/Probes_merged_ok_SSI_PREDiSPOSED-v1_1X_TE-93129556_hg38.bed" } \ No newline at end of file diff --git a/config_qc.json b/config_qc.json index f5c23a5..64cb398 100644 --- a/config_qc.json +++ b/config_qc.json @@ -5,6 +5,5 @@ "outputDir": "../results_qc_150x", "fullQC": "true", "pooltable": "pooltable.tsv", - "reference_genome": "/ngc/projects2/dp_00005/data/predisposed/resources/reference/Homo_sapiens_assembly38_noAlt", "bedfile": "/ngc/projects2/dp_00005/data/predisposed/resources/targets/Probes_merged_ok_SSI_PREDiSPOSED-v1_1X_TE-93129556_hg38.bed" } \ No newline at end of file diff --git a/config_test.json b/config_test.json index be7454e..b805818 100644 --- a/config_test.json +++ b/config_test.json @@ -5,7 +5,7 @@ "step": "all", "pooltable": "assets/data/test_data/pooltable.tsv", "decodetable": "assets/data/test_data/decodetable.tsv", - "reference_genome": "assets/data/reference_genomes/small/small_reference", + "reference_genome": "assets/data/reference_genomes/small/small_reference.fna", "bedfile": "assets/data/test_data/target_calling.bed", "bedfile_bam_extraction": "assets/data/test_data/target_calling.bed", "dictfile": "assets/data/test_data/target_calling.dict", diff --git a/main.nf b/main.nf index cbaef9d..b412944 100644 --- a/main.nf +++ b/main.nf @@ -79,7 +79,12 @@ report_rmd = Channel.fromPath("$projectDir/bin/report.Rmd", checkIfExists: true) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -reference_genome_ch = Channel.fromPath(params.reference_genome + "*", checkIfExists: true).collect() +ref_file = file(params.reference_genome) +ref_base = ref_file.parent.resolve(ref_file.baseName) +reference_genome_ch = Channel + .fromPath(["${ref_file}*", "${ref_base}.*"], checkIfExists: true) + .unique { it.name } + .collect() bedfile_ch = Channel.fromPath(params.bedfile, checkIfExists: true).collect() /* diff --git a/modules/alignment.nf b/modules/alignment.nf index 47c33e1..57f3ee4 100644 --- a/modules/alignment.nf +++ b/modules/alignment.nf @@ -17,7 +17,7 @@ process ALIGNMENT { path "${sample_id}.log" script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name // int bwa_cpus = Math.max((task.cpus * 0.75) as int, 1) // int samtools_cpus = Math.max(task.cpus - bwa_cpus, 1) diff --git a/modules/alignment_metrics.nf b/modules/alignment_metrics.nf index 7fac332..9c79cfa 100644 --- a/modules/alignment_metrics.nf +++ b/modules/alignment_metrics.nf @@ -17,7 +17,7 @@ process ALIGNMENT_METRICS { path("${sample_id}*.align.txt"), emit: metrics_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def log_filename = log_suffix == "" ? "${sample_id}.align.txt" : "${sample_id}_${log_suffix}.align.txt" """ gatk CollectAlignmentSummaryMetrics \ diff --git a/modules/alignment_umi.nf b/modules/alignment_umi.nf index 8a4a197..7e4e47a 100644 --- a/modules/alignment_umi.nf +++ b/modules/alignment_umi.nf @@ -17,7 +17,7 @@ process ALIGNMENT_UMI { path "${sample_id}.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name // int bwa_cpus = Math.max((task.cpus * 0.75) as int, 1) // int samtools_cpus = Math.max(task.cpus - bwa_cpus, 1) diff --git a/modules/apply_bqsr.nf b/modules/apply_bqsr.nf index f1cf669..e5b12d8 100644 --- a/modules/apply_bqsr.nf +++ b/modules/apply_bqsr.nf @@ -13,7 +13,7 @@ process APPLY_BQSR { tuple val(sample_id), path("${sample_id}.bam"), emit: corrected_bam_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name """ gatk ApplyBQSR \ -R ${db} \ diff --git a/modules/bam.nf b/modules/bam.nf index e8032ef..67b61e4 100644 --- a/modules/bam.nf +++ b/modules/bam.nf @@ -13,7 +13,7 @@ process BAM { tuple val(sample_id), path("${sample_id}.bam"), emit: bam_file script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name """ samtools view \ -@ ${task.cpus-1} \ diff --git a/modules/bqsr.nf b/modules/bqsr.nf index bee3274..ffc237a 100644 --- a/modules/bqsr.nf +++ b/modules/bqsr.nf @@ -15,7 +15,7 @@ process BQSR { tuple val(sample_id), path(bam_file), path("${sample_id}.BQSR.table"), emit: bqsr_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def mills_db = file(params.mills).getName() def g1000_db = file(params.g1000).getName() """ diff --git a/modules/combinegvcfs.nf b/modules/combinegvcfs.nf index b1eb4d9..df4ac8b 100644 --- a/modules/combinegvcfs.nf +++ b/modules/combinegvcfs.nf @@ -21,7 +21,7 @@ process COMBINEGVCFS { path "genomicsdb.log" script: - db = file(params.reference_genome).getName() + ".fna" + db = file(params.reference_genome).name input_files_command = vcfs.collect(){"--variant ${it}"}.join(' ') avail_mem = (task.memory.mega*0.8).intValue() """ diff --git a/modules/cram.nf b/modules/cram.nf index dad7342..c984a09 100644 --- a/modules/cram.nf +++ b/modules/cram.nf @@ -16,7 +16,7 @@ process CRAM { path("${sample_id}_cram.tsv"), emit: cram_info script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name def publishDir = file(params.outputDir + "/cram/" + sample_id + ".cram") """ samtools view \ diff --git a/modules/crisp.nf b/modules/crisp.nf index 9103112..15c014f 100644 --- a/modules/crisp.nf +++ b/modules/crisp.nf @@ -22,7 +22,7 @@ process CRISP { script: def input_files_command = bam_files.collect(){"--bam ${it}"}.join(' ') - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name """ CRISP \ --ref ${db} \ diff --git a/modules/deepvariant.nf b/modules/deepvariant.nf index f993828..9a04d01 100644 --- a/modules/deepvariant.nf +++ b/modules/deepvariant.nf @@ -22,7 +22,7 @@ process DEEPVARIANT { path "${sample_id}.DV.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def ref_dir = file(params.reference_genome).getParent() def target_dir = file(params.bedfile).getParent() """ diff --git a/modules/freebayes.nf b/modules/freebayes.nf index 85c7d51..5cea51c 100644 --- a/modules/freebayes.nf +++ b/modules/freebayes.nf @@ -20,7 +20,7 @@ process FREEBAYES { path "${sample_id}.freebayes.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def publishDir = file(params.outputDir + "/variants/" + sample_id + ".freebayes.vcf.gz") """ freebayes \ diff --git a/modules/gc_metrics.nf b/modules/gc_metrics.nf index b9c7e8d..36cf28d 100644 --- a/modules/gc_metrics.nf +++ b/modules/gc_metrics.nf @@ -20,7 +20,7 @@ process GC_METRICS { path("${sample_id}*.gc.summary.txt"), emit: summary_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def log_filename = log_suffix == "" ? "${sample_id}.gc.txt" : "${sample_id}_${log_suffix}.gc.txt" def log_filename_summary = log_suffix == "" ? "${sample_id}.gc.summary.txt" : "${sample_id}_${log_suffix}.gc.summary.txt" """ diff --git a/modules/genotypegvcf.nf b/modules/genotypegvcf.nf index 7fc321d..29971f8 100644 --- a/modules/genotypegvcf.nf +++ b/modules/genotypegvcf.nf @@ -19,7 +19,7 @@ process GENOTYPEGVCF { path "${sample_id}.genotypegvcf.log" script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name def publishDir = file(params.outputDir + "/variants/" + sample_id + ".GATK.vcf.gz") def avail_mem = (task.memory.mega*0.8).intValue() """ diff --git a/modules/genotypegvcf_split.nf b/modules/genotypegvcf_split.nf index 28baf35..d43d82e 100644 --- a/modules/genotypegvcf_split.nf +++ b/modules/genotypegvcf_split.nf @@ -17,7 +17,7 @@ process GENOTYPEGVCF_SPLIT { path "${sample_id}*.genotypegvcf.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def id = interval ? "${sample_id}.${interval}" : sample_id def avail_mem = (task.memory.mega*0.8).intValue() """ diff --git a/modules/haplotypecaller.nf b/modules/haplotypecaller.nf index f63db4b..77881c0 100644 --- a/modules/haplotypecaller.nf +++ b/modules/haplotypecaller.nf @@ -19,7 +19,7 @@ process HAPLOTYPECALLER { path "${sample_id}.g.haplotypecaller.log" script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name def avail_mem = (task.memory.mega*0.8).intValue() """ gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" HaplotypeCaller \ @@ -43,7 +43,7 @@ process HAPLOTYPECALLER { -A AlleleFraction \ --disable-read-filter NotDuplicateReadFilter \ --max-alternate-alleles 3 \ - --max-num-haplotypes-in-population 1000 \ + --max-num-haplotypes-in-population ${params.max_haplotypes} \ --max-reads-per-alignment-start 0 \ --create-output-variant-index \ --tmp-dir . \ diff --git a/modules/haplotypecaller_split.nf b/modules/haplotypecaller_split.nf index f01d92a..f46cf0c 100644 --- a/modules/haplotypecaller_split.nf +++ b/modules/haplotypecaller_split.nf @@ -19,7 +19,7 @@ process HAPLOTYPECALLER_SPLIT { path "${sample_id}*.g.haplotypecaller.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def id = interval ? "${sample_id}.${interval}" : sample_id def add_intervals = interval ? "-L ${interval}" : "" def add_intersection = interval ? "--interval-set-rule INTERSECTION" : "" diff --git a/modules/haplotypecaller_truth.nf b/modules/haplotypecaller_truth.nf index 24e576b..0d5f4a9 100644 --- a/modules/haplotypecaller_truth.nf +++ b/modules/haplotypecaller_truth.nf @@ -19,7 +19,7 @@ process HC_TRUTH { path "${sample_id}.haplotypecaller.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def avail_mem = (task.memory.mega*0.8).intValue() """ gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" \ diff --git a/modules/haplotypecaller_truth_joint.nf b/modules/haplotypecaller_truth_joint.nf index de4c378..15b0711 100644 --- a/modules/haplotypecaller_truth_joint.nf +++ b/modules/haplotypecaller_truth_joint.nf @@ -20,7 +20,7 @@ process HC_TRUTH_JOINT { path "${sample_id}.g.haplotypecaller.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def avail_mem = (task.memory.mega*0.8).intValue() """ gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" \ diff --git a/modules/hs_metrics.nf b/modules/hs_metrics.nf index f67d8bb..e3207f3 100644 --- a/modules/hs_metrics.nf +++ b/modules/hs_metrics.nf @@ -18,7 +18,7 @@ process HS_METRICS { path("${sample_id}*.hs.txt"), emit: metrics_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def log_filename = log_suffix == "" ? "${sample_id}.hs.txt" : "${sample_id}_${log_suffix}.hs.txt" """ gatk BedToIntervalList \ diff --git a/modules/indelqual.nf b/modules/indelqual.nf index 1b31897..495e808 100644 --- a/modules/indelqual.nf +++ b/modules/indelqual.nf @@ -17,7 +17,7 @@ process INDELQUAL { path "${sample_id}.lofreq.iq.log" script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name """ lofreq indelqual \ --dindel \ diff --git a/modules/intervals.nf b/modules/intervals.nf index 15b2484..2d2f873 100644 --- a/modules/intervals.nf +++ b/modules/intervals.nf @@ -13,7 +13,7 @@ process INTERVALS { path "target_region.interval_list", emit: target_list script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name """ gatk BedToIntervalList \ I=${bedfile} \ diff --git a/modules/lofreq.nf b/modules/lofreq.nf index 6e8131e..c600568 100644 --- a/modules/lofreq.nf +++ b/modules/lofreq.nf @@ -20,7 +20,7 @@ process LOFREQ { path "${sample_id}.lofreq.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def publishDir = file(params.outputDir + "/variants/" + sample_id + ".lofreq.vcf.gz") """ lofreq call-parallel \ diff --git a/modules/merge_consensus.nf b/modules/merge_consensus.nf index d04d332..5d9d40a 100644 --- a/modules/merge_consensus.nf +++ b/modules/merge_consensus.nf @@ -13,7 +13,7 @@ process MERGE_CONSENSUS { tuple val(sample_id), path("${sample_id}.bam"), emit: bam_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name def avail_mem = (task.memory.mega*0.8).intValue() """ gatk --java-options -Xmx${avail_mem}M MergeBamAlignment \ diff --git a/modules/mergebam.nf b/modules/mergebam.nf index d1348a2..d270b94 100644 --- a/modules/mergebam.nf +++ b/modules/mergebam.nf @@ -13,7 +13,7 @@ process MERGEBAM { tuple val(sample_id), path("${sample_id}_raw.bam"), emit: bam_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name """ gatk MergeBamAlignment \ TMP_DIR=. \ diff --git a/modules/mpileup.nf b/modules/mpileup.nf index 317cc2d..c75e4b6 100644 --- a/modules/mpileup.nf +++ b/modules/mpileup.nf @@ -16,7 +16,7 @@ process MPILEUP { path("${sample_id}.mpileup.gz"), emit: mpileup_file script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name """ samtools mpileup \ -l ${bedfile} \ diff --git a/modules/normalise_vcf.nf b/modules/normalise_vcf.nf index 04bc77d..78d37c7 100644 --- a/modules/normalise_vcf.nf +++ b/modules/normalise_vcf.nf @@ -21,7 +21,7 @@ process NORMALISE_VCF { path "${sample_id}.${caller}.log" script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name """ bcftools norm \ --check-ref e \ diff --git a/modules/subset.nf b/modules/subset.nf index d13fbcb..4877106 100644 --- a/modules/subset.nf +++ b/modules/subset.nf @@ -14,7 +14,7 @@ process SUBSET { tuple val(sample_id), path("${sample_id}.bam"), emit: bam_file script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name """ samtools view \ -@ ${task.cpus} \ diff --git a/modules/test.nf b/modules/test.nf index 049c96d..5a966f1 100644 --- a/modules/test.nf +++ b/modules/test.nf @@ -11,8 +11,8 @@ process TEST { val pinpoint_method output: - path "test.passed" optional true - path "test.failed" optional true + path "test.passed", optional: true + path "test.failed", optional: true path "test.log" script: diff --git a/modules/vep.nf b/modules/vep.nf index 6937ae5..10e11bf 100644 --- a/modules/vep.nf +++ b/modules/vep.nf @@ -28,7 +28,7 @@ process VEP { path "annotations_w_varid.tsv", emit: annotations_file script: - def db = file(params.reference_genome).getName() + ".fasta" + def db = file(params.reference_genome).name def mode = cache_dir ? "--cache --offline --dir_cache ${cache_dir}" : "--database" def utr = utr_annotation ? "--plugin UTRAnnotator,file=${utr_annotation}" : "" def am = alphamissense ? "--plugin AlphaMissense,file=${alphamissense}" : "" diff --git a/nextflow.config b/nextflow.config index c2592eb..0cb232b 100644 --- a/nextflow.config +++ b/nextflow.config @@ -29,6 +29,9 @@ params { // Experiment ploidy ploidy = 48 + // HaplotypeCaller maximum number of haplotypes to evaluate: + max_haplotypes = 1000 + // Run HaplotypeCaller by intervals. Default is by chromosome. runHCParallel = true intervalList = (1..22).collect { "chr${it}" } + ["chrX"] From 8a969f188a4dc6f2f90df495c52bb8ef1d87130c Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Tue, 10 Feb 2026 09:56:43 +0100 Subject: [PATCH 87/95] limit haplotypes for ssi hpc --- conf/ssi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/ssi.config b/conf/ssi.config index b5cd0e9..2fe1823 100644 --- a/conf/ssi.config +++ b/conf/ssi.config @@ -10,6 +10,7 @@ profiles { cacheDir = "/srv/data/scratch/nf_tmp/" reference_genome = "/srv/data/Reference/GATK_bundle/hg38_noScaffoldsMasked/Homo_sapiens_assembly38_noScaffoldsMasked.fasta" bedfile = "/srv/data/Projects/BabyCompare/vep_optional_annotation/Target_bases_covered_by_probes_SSI_BabyPrecision-v2p1_1X_TE-95845496_hg38_ext151_merged.bed" + max_haplotypes = 256 snpeff_db = 'GRCh38.99' snpeff_config = "" snpeff_cache = "" From 826ebcaeaddcd2468f1b8c25ba649bf7f795c570 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Fri, 6 Mar 2026 11:07:38 +0100 Subject: [PATCH 88/95] re-add reference --- config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config.json b/config.json index 34bb187..0debb96 100644 --- a/config.json +++ b/config.json @@ -3,5 +3,6 @@ "step": "all", "pooltable": "pooltable.tsv", "outputDir": "../results", + "reference_genome": "/ngc/projects2/dp_00005/data/predisposed/resources/reference/Homo_sapiens_assembly38_noAlt.fna", "bedfile": "/ngc/projects2/dp_00005/data/predisposed/resources/targets/Probes_merged_ok_SSI_PREDiSPOSED-v1_1X_TE-93129556_hg38.bed" } \ No newline at end of file From 7982ccf24242f639c2ee1fc33c28560d6eaca68d Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Mon, 9 Mar 2026 13:58:24 +0100 Subject: [PATCH 89/95] remove ngsep and freebayes as defaults --- nextflow.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nextflow.config b/nextflow.config index e64f498..66240ba 100644 --- a/nextflow.config +++ b/nextflow.config @@ -73,8 +73,8 @@ params { deepvariant = false // WGS only (container only) crisp = false // Pool only (not available through conda) octopus = false // Pool only (container only) - freebayes = true // Pool only - ngsep = true // Pool only + freebayes = false // Pool only + ngsep = false // Pool only // Variant filtering. Either optimised for sensitivity or balance filter = false From 3fe82cbd5faef68fd98ff00347620e5af0eeccc2 Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Mon, 25 May 2026 19:58:40 +0200 Subject: [PATCH 90/95] update configs --- conf/ssi.config | 8 +++++--- envs/rescue/environment.yaml | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/conf/ssi.config b/conf/ssi.config index 2fe1823..6fae7ea 100644 --- a/conf/ssi.config +++ b/conf/ssi.config @@ -10,10 +10,12 @@ profiles { cacheDir = "/srv/data/scratch/nf_tmp/" reference_genome = "/srv/data/Reference/GATK_bundle/hg38_noScaffoldsMasked/Homo_sapiens_assembly38_noScaffoldsMasked.fasta" bedfile = "/srv/data/Projects/BabyCompare/vep_optional_annotation/Target_bases_covered_by_probes_SSI_BabyPrecision-v2p1_1X_TE-95845496_hg38_ext151_merged.bed" - max_haplotypes = 256 + runHCParallel = false + max_haplotypes = 128 snpeff_db = 'GRCh38.99' snpeff_config = "" snpeff_cache = "" + annotate_vep = true vep_cache = "/srv/data/VCF_annotations/VEP/vep" clinvar_db = "/srv/data/VCF_annotations/CLINVAR/clinvar.vcf.gz" danmac_db = "" @@ -34,7 +36,7 @@ profiles { resourceLimits = [ cpus: 12, - memory: '48.GB' + memory: '64.GB' ] maxRetries = 2 @@ -64,7 +66,7 @@ profiles { } withLabel:process_multi { - cpus = { 24 * task.attempt } + cpus = { 8 * task.attempt } memory = { 30.GB * task.attempt } time = { 8.h * task.attempt } } diff --git a/envs/rescue/environment.yaml b/envs/rescue/environment.yaml index 4bcecfc..8db4fb6 100644 --- a/envs/rescue/environment.yaml +++ b/envs/rescue/environment.yaml @@ -7,6 +7,7 @@ dependencies: - python=3.11 - pip - numpy + - scipy>=1.16.1,<2 - pandas>=2.3.2 - pysam=0.23.3 - htslib>=1.18 From 4e818c2110a33501d72055e2274ac5e8cbc4ef38 Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Tue, 26 May 2026 12:45:25 +0200 Subject: [PATCH 91/95] update attributes for line-endings --- .gitattributes | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitattributes b/.gitattributes index 8f61a8e..ac1e604 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,18 @@ # SCM syntax highlighting pixi.lock linguist-language=YAML linguist-generated=true + +# Force windows not to reformat line-endings +*.sh text eol=lf +*.bash text eol=lf +*.py text eol=lf +*.pl text eol=lf +*.R text eol=lf +*.pbs text eol=lf +*.slurm text eol=lf +*.config text eol=lf +*.yaml text eol=lf +*.nf text eol=lf +*.md text eol=lf +*.Rmd text eol=lf +*.groovy text eol=lf +*.json text eol=lf From 7c053873106ca122f3eaa187a899ba04864fefbc Mon Sep 17 00:00:00 2001 From: "Mads Cort Nielsen (MACNI)" Date: Tue, 26 May 2026 12:50:20 +0200 Subject: [PATCH 92/95] update to lf eol --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index ac1e604..59c24d9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,3 +16,4 @@ pixi.lock linguist-language=YAML linguist-generated=true *.Rmd text eol=lf *.groovy text eol=lf *.json text eol=lf +*.ssi text eol=lf From 90eb7ad920ef1147484ea0b58a7bdcc841bdde96 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Fri, 26 Jun 2026 20:31:48 +0200 Subject: [PATCH 93/95] DSL compliance --- conf/container.config | 2 +- main.nf | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/conf/container.config b/conf/container.config index c54b2d0..9b162af 100644 --- a/conf/container.config +++ b/conf/container.config @@ -5,7 +5,7 @@ // Docker runoptions fixes ownership confusion in some containers process { container = 'ubuntu:24.04' - docker.runOptions = '-u $(id -u):$(id -g)' + containerOptions = '-u $(id -u):$(id -g)' } params { diff --git a/main.nf b/main.nf index b412944..550c00e 100644 --- a/main.nf +++ b/main.nf @@ -5,13 +5,6 @@ // Use newest nextflow dsl nextflow.enable.dsl = 2 -log.info """\ - =================================== - D o B S e q - W F - =================================== - """ - .stripIndent() - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT MODULES AND SUBWORKFLOWS @@ -94,6 +87,15 @@ bedfile_ch = Channel.fromPath(params.bedfile, checkIfExists: true).collect() */ workflow { + + log.info """\ + =================================== + D o B S e q - W F + =================================== + """ + .stripIndent() + + if (params.step == 'mapping' || params.step == 'all' || params.step == '') { if (params.umi) { bam_file_w_index_ch = MAPPING_UMI(pooltable_ch, reference_genome_ch, bedfile_ch) From 2ab4806f67443648905ce1c5fdca64a080d1c2e9 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Fri, 26 Jun 2026 21:41:25 +0200 Subject: [PATCH 94/95] downgrade parser --- .github/workflows/nextflow-test-conda.yml | 3 +++ .github/workflows/nextflow-test-container.yml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nextflow-test-conda.yml b/.github/workflows/nextflow-test-conda.yml index 81bb8ec..81164fd 100644 --- a/.github/workflows/nextflow-test-conda.yml +++ b/.github/workflows/nextflow-test-conda.yml @@ -11,6 +11,9 @@ jobs: test: runs-on: ubuntu-latest + env: + NXF_SYNTAX_PARSER: v1 + steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/nextflow-test-container.yml b/.github/workflows/nextflow-test-container.yml index 3f5a3d6..8d332dc 100644 --- a/.github/workflows/nextflow-test-container.yml +++ b/.github/workflows/nextflow-test-container.yml @@ -10,7 +10,10 @@ on: jobs: test: runs-on: ubuntu-latest - + + env: + NXF_SYNTAX_PARSER: v1 + steps: - name: Checkout repository uses: actions/checkout@v4 From 45a50c48086240bec73488f82391bd8bde9bb5b0 Mon Sep 17 00:00:00 2001 From: Mads Nielsen Date: Sat, 27 Jun 2026 07:18:33 +0200 Subject: [PATCH 95/95] fix referencing in ngsep --- modules/ngsep.nf | 2 +- nextflow.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ngsep.nf b/modules/ngsep.nf index 5999760..1a23405 100644 --- a/modules/ngsep.nf +++ b/modules/ngsep.nf @@ -19,7 +19,7 @@ process NGSEP { path "${sample_id}.ngsep.log" script: - def db = file(params.reference_genome).getName() + ".fna" + def db = file(params.reference_genome).name """ ngsep \ SingleSampleVariantsDetector \ diff --git a/nextflow.config b/nextflow.config index 66240ba..7b42900 100644 --- a/nextflow.config +++ b/nextflow.config @@ -89,7 +89,7 @@ params { pinpoint_tab = true // Use variant rescue model (true or false) - variant_rescue = true + variant_rescue = false // Make pinpoint report (true or false) pinpoint_report = false