HTML report/nfcore addons

.gitattributes

@@ -1,4 +1,7 @@
 *.config linguist-language=nextflow
 *.nf.test linguist-language=nextflow
+*.bwt binary
+*.pac binary
+*.sa binary
 modules/nf-core/** linguist-generated
 subworkflows/nf-core/** linguist-generated

.github/workflows/linting.yml

@@ -32,6 +32,13 @@ jobs:
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v2
+        with:
+          version: "26.04.6"
+
+      - name: Lint Nextflow parser-v2 syntax
+        env:
+          NXF_SYNTAX_PARSER: v2
+        run: nextflow lint -exclude nf-test.config .
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:

.github/workflows/nf-test.yml

@@ -58,7 +58,7 @@ jobs:
           echo ${{ steps.set-shards.outputs.total_shards }}
 
   nf-test:
-    name: "${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/${{ needs.nf-test-changes.outputs.total_shards }}"
+    name: "${{ matrix.profile }} | ${{ matrix.NXF_VER }} | parser ${{ matrix.NXF_PARSER }} | ${{ matrix.shard }}/${{ needs.nf-test-changes.outputs.total_shards }}"
     needs: [nf-test-changes]
     if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }}
     runs-on: # use self-hosted runners
@@ -77,11 +77,17 @@ jobs:
             profile: "conda"
           - isMain: false
             profile: "singularity"
+          - NXF_VER: "24.10.0"
+            NXF_PARSER: "v2"
         NXF_VER:
           - "24.10.0"
-          - "latest-everything"
+          - "26.04.6"
+        NXF_PARSER:
+          - "v1"
+          - "v2"
     env:
       NXF_ANSI_LOG: false
+      NXF_SYNTAX_PARSER: ${{ matrix.NXF_PARSER }}
       TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }}
 
     steps:
@@ -92,7 +98,6 @@ jobs:
       - name: Run nf-test
         id: run_nf_test
         uses: ./.github/actions/nf-test
-        continue-on-error: ${{ matrix.NXF_VER == 'latest-everything' }}
         env:
           NFT_WORKDIR: ${{ env.NFT_WORKDIR }}
           NXF_VERSION: ${{ matrix.NXF_VER }}
@@ -105,21 +110,16 @@ jobs:
         if: ${{ always() }}
         run: |
           if [[ "${{ steps.run_nf_test.outcome }}" == "failure" ]]; then
-            echo "::error::Test with ${{ matrix.NXF_VER }} failed"
+            echo "::error::Test with ${{ matrix.NXF_VER }} and parser ${{ matrix.NXF_PARSER }} failed"
             # Add to workflow summary
-            echo "## ❌ Test failed: ${{ matrix.profile }} | ${{ matrix.NXF_VER }} | Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}" >> $GITHUB_STEP_SUMMARY
-            if [[ "${{ matrix.NXF_VER }}" == "latest-everything" ]]; then
-              echo "::warning::Test with latest-everything failed but will not cause workflow failure. Please check if the error is expected or if it needs fixing."
-            fi
-            if [[ "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
-              exit 1
-            fi
+            echo "## ❌ Test failed: ${{ matrix.profile }} | ${{ matrix.NXF_VER }} | parser ${{ matrix.NXF_PARSER }} | Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}" >> $GITHUB_STEP_SUMMARY
+            exit 1
           fi
 
   confirm-pass:
     needs: [nf-test]
     if: always()
-    runs-on:  # use self-hosted runners
+    runs-on: # use self-hosted runners
       - runs-on=${{ github.run_id }}-confirm-pass
       - runner=2cpu-linux-x64
     steps:

.pre-commit-config.yaml

@@ -1,3 +1,14 @@
+exclude: |
+  (?x)^(
+      \.nf-test/|
+      results/|
+      work/|
+      tmp/|
+      \.codex$|
+      \.codex/|
+      assets/testdata/.*\.(amb|ann|bwt|pac|sa)$
+  )
+
 repos:
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: "v3.1.0"

README.md

@@ -5,9 +5,13 @@
 
 TrESFlow is a Nextflow DSL2 pipeline for the preprocessing of TrES-seq data from FASTQs to cell by feature matrices.
 
+The same source tree supports Nextflow's parser v1 and parser v2. Nextflow
+24.10 or later is required; parser v2 is the default in Nextflow 26.04.
+
 ## Install
 
 Install your conda/mamba/micromamba env as follows (conda-forge & bioconda channels):
+
 ```bash
 micromamba env create -n tres
 micromamba activate tres
@@ -16,12 +20,14 @@ micromamba install screen samtools bwa-mem2 star fastqc multiqc trim-galore deep
 ```
 
 Download the repo and cd in it:
+
 ```bash
 git clone git@github.com:CSOgroup/TrESFlow.git
 cd TrESFlow
 ```
 
 Install codon in your env:
+
 ```bash
 ./scripts/install_codon_0.16.3.sh --prefix /path/to/env/prefix
 ```
@@ -110,16 +116,25 @@ RNA publishes:
 - `rna_split_fastqs/`
 - `rna_align/`
 - `TrES_Stats/`
+- `qc/samtools/`
+- `multiqc/`
+- `tres_report/`
 - `pipeline_info/`
 
 DNA publishes:
 
 - `dna_split_fastqs/`
 - `dna_align/`
 - `TrES_Stats/`
+- `qc/samtools/`
+- `multiqc/`
+- `tres_report/`
 - `pipeline_info/`
 
-`TrES_Stats/` includes RNA and DNA sequencing-efficiency UpSet PDF plots. Sankey plots, HTML reports, count tables, combined RNA+DNA reports, and sequencing-efficiency warning TSVs are not produced. Optional unavailable BAM-derived categories are skipped with warnings in the process log.
+The pipeline also writes two end-of-run HTML reports:
+
+- `tres_report/tres_report.html`: compact TrESFlow-specific RNA/DNA mapping and barcode summary
+- `multiqc/multiqc_report.html`: nf-core MultiQC aggregation of supported logs and QC files
 
 ## Runtime Contract
 
@@ -162,7 +177,7 @@ Default local CPU budget:
 
 Work-directory cleanup is intentionally aggressive: `--cleanup_work true` uses Nextflow's successful-run cleanup to remove task work directories after outputs have been published and downstream tasks have completed. This substantially reduces retained `work/` storage, but cleaned tasks are not expected to be usable with `--resume`. Set `--cleanup_work false` when you need the previous resume-friendly behavior for debugging or iterative development.
 
-DNA alignment no longer removes low-count cell barcodes during `ALIGN_DNA`. The aligned BAM still keeps proper-pair mapped, non-blacklisted reads; duplicate removal is represented later by `*_NoDup.bam`, and duplicate status appears in DNA sequencing-efficiency plots as `Unique +`.
+DNA alignment no longer removes low-count cell barcodes during `ALIGN_DNA`. The aligned BAM still keeps proper-pair mapped, non-blacklisted reads; duplicate removal is represented later by `*_NoDup.bam`.
 
 Every run writes:
 
@@ -171,6 +186,21 @@ Every run writes:
 - `${outdir}/pipeline_info/execution_trace.tsv`
 - `${outdir}/pipeline_info/flowchart.html`
 - `${outdir}/pipeline_info/runtime_contract.tsv`
+- `${outdir}/tres_report/tres_report.html` with per-library main statistics, detailed QC tables, and CSV/Excel export buttons
+- `${outdir}/tres_report/tres_report_metrics.json`
+- `${outdir}/multiqc/multiqc_report.html`
+
+Runs with real BAM outputs also write nf-core samtools sidecar QC under:
+
+- `${outdir}/qc/samtools/*.flagstat`
+- `${outdir}/qc/samtools/*.stats`
+- `${outdir}/qc/samtools/*.idxstats`
+- `${outdir}/qc/samtools/*.quickcheck.tsv`
+
+Raw FASTQ QC from nf-core FastQC is written under:
+
+- `${outdir}/qc/fastqc/*_fastqc.html`
+- `${outdir}/qc/fastqc/*_fastqc.zip`
 
 The active runtime scripts live under [`scripts/core_runtime/`](scripts/core_runtime/). `upstream/source_scripts/` is kept only as provenance for the vendored core code.
 
@@ -182,3 +212,12 @@ NXF_OFFLINE=true nextflow run . \
   --outdir /path/to/TrESFlow_results \
   --max_cpus 32
 ```
+
+Relative command-line and launch-config paths, including `--samplesheet`,
+`--outdir`, and an explicit `--core_scripts_dir`, are resolved from the
+directory where Nextflow is launched. Omitting `--outdir` writes to
+`<launch-directory>/results`. Relative paths inside the samplesheet remain
+relative to the samplesheet itself. Bundled scripts, modules, assets, and the
+default `scripts/core_runtime` directory remain relative to the pipeline
+repository, so invoking `main.nf` by absolute path from another directory is
+supported.

assets/test_realdata/ligation_barcode_whitelist.txt

@@ -0,0 +1,48 @@
+ATCACGTT
+CGATGTTT
+TTAGGCAT
+TGACCACT
+ACAGTGGT
+GCCAATGT
+CAGATCTG
+ACTTGATG
+GATCAGCG
+TAGCTTGT
+GGCTACAG
+CTTGTACT
+TGGTTGTT
+TCTCGGTT
+TAAGCGTT
+TCCGTCTT
+TGTACCTT
+TTCTGTGT
+TCTGCTGT
+TTGGAGGT
+TCGAGCGT
+TGATACGT
+TGCATAGT
+TTGACTCT
+TGCGATCT
+TTCCTGCT
+TAGTGACT
+TACAGGAT
+TCCTCAAT
+TGTGGTTG
+TACTAGTC
+TTCCATTG
+TCGAAGTG
+TAACGCTG
+TTGGTATG
+TGAACTGG
+TACTTCGG
+TCTCACGG
+TCAGGAGG
+TAAGTTCG
+TCCAGTCG
+TGTATGCG
+TCATTGAG
+TGGCTCAG
+TATGCCAG
+TCAGATTC
+TAGTCTTG
+TTCAGCTC