Fix join barcode step for Opossum (#1397)

* Fix: Use CSI instead of TBI indexing in tabix command to support chromosomes larger than 512 Mbp. * Updated pipeline_versions.txt with all pipeline version information * Updated pipeline_versions.txt with all pipeline version information * updated changelogs * Updated pipeline_versions.txt with all pipeline version information * updated pairedtag version * Updated pipeline_versions.txt with all pipeline version information * CreateFragmentFile task now uses csi index format * Update pipelines/skylab/multiome/Multiome.changelog.md Co-authored-by: Elizabeth Kiernan <[email protected]> * Update pipelines/skylab/optimus/Optimus.changelog.md Co-authored-by: Elizabeth Kiernan <[email protected]> * Update pipelines/skylab/slideseq/SlideSeq.changelog.md Co-authored-by: Elizabeth Kiernan <[email protected]> * Update pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md Co-authored-by: Elizabeth Kiernan <[email protected]> * Update pipelines/skylab/paired_tag/PairedTag.changelog.md Co-authored-by: Elizabeth Kiernan <[email protected]> * Updated pipeline_versions.txt with all pipeline version information * updating indexing for pairedtag and changing index file name * Updated outputs to reflect the csi index. Renamed the index from tbi to index * Update Multiome.wdl changed JoinBarcodes task index file name * Update pipelines/skylab/atac/atac.wdl Co-authored-by: Elizabeth Kiernan <[email protected]> --------- Co-authored-by: GitHub Action <[email protected]> Co-authored-by: Elizabeth Kiernan <[email protected]> Co-authored-by: ekiernan <[email protected]>
broadinstitute · Nov 6, 2024 · 88b1e48 · 88b1e48
1 parent d7c2443
commit 88b1e48
Show file tree

Hide file tree

Showing 15 changed files with 50 additions and 20 deletions.
diff --git a/pipeline_versions.txt b/pipeline_versions.txt
@@ -30,11 +30,11 @@ ExomeReprocessing	 3.3.3	2024-11-04
 BuildIndices	 3.0.0	2023-12-06 
 scATAC	 1.3.2	2023-08-03 
 snm3C	 4.0.4	2024-08-06 
-Multiome	 5.8.0	2024-10-23 
-PairedTag	 1.8.0	2024-10-23 
+Multiome	 5.9.0	2024-10-21 
+PairedTag	 1.8.1	2024-11-04 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
-MultiSampleSmartSeq2SingleNucleus	 2.0.2	2024-10-23 
-Optimus	 7.8.0	2024-10-23 
-atac	 2.4.0	2024-10-23 
+MultiSampleSmartSeq2SingleNucleus	 2.0.3	2024-11-04 
+Optimus	 7.8.1	2024-11-04 
+atac	 2.5.0	2024-10-23 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
-SlideSeq	 3.4.3	2024-10-24 
+SlideSeq	 3.4.4	2024-11-04 
diff --git a/pipelines/skylab/atac/atac.changelog.md b/pipelines/skylab/atac/atac.changelog.md
@@ -1,3 +1,8 @@
+# 2.5.0
+2024-10-23 (Date of Last Commit)
+
+* Updated the tabix flag in CreateFragmentFile task to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp
+
 # 2.4.0
 2024-10-23 (Date of Last Commit)
 

diff --git a/pipelines/skylab/atac/atac.wdl b/pipelines/skylab/atac/atac.wdl
@@ -49,7 +49,7 @@ workflow ATAC {
     String adapter_seq_read3 = "TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG"
   }
 
-  String pipeline_version = "2.4.0"
+  String pipeline_version = "2.5.0"
 
   # Determine docker prefix based on cloud provider
   String gcr_docker_prefix = "us.gcr.io/broad-gotc-prod/"
@@ -614,7 +614,7 @@ task CreateFragmentFile {
     echo "Starting bgzip"
     bgzip "~{input_id}.fragments.sorted.tsv"
     echo "Starting tabix"
-    tabix -s 1 -b 2 -e 3 "~{input_id}.fragments.sorted.tsv.gz"
+    tabix -s 1 -b 2 -e 3 -C "~{input_id}.fragments.sorted.tsv.gz"
   >>>
 
   runtime {
@@ -627,6 +627,8 @@ task CreateFragmentFile {
 
   output {
     File fragment_file = "~{input_id}.fragments.sorted.tsv.gz"
+    File fragment_file_index = "~{input_id}.fragments.sorted.tsv.gz.csi"
+
     File Snap_metrics = "~{input_id}.metrics.h5ad"
     File atac_library_metrics = "~{input_id}_~{atac_nhash_id}_library_metrics.csv"
   }

diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md
@@ -1,3 +1,7 @@
+# 5.9.0
+2024-10-21 (Date of Last Commit)
+* Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task changes the format for the ATAC fragment file index 
+* Renamed the fragment file index from atac_fragment_tsv_tbi to atac_fragment_tsv_index
 # 5.8.0
 2024-10-23 (Date of Last Commit)
 

diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl
@@ -9,8 +9,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Multiome {
 
-    String pipeline_version = "5.8.0"
-
+    String pipeline_version = "5.9.0"
 
     input {
         String cloud_provider
@@ -181,7 +180,7 @@ workflow Multiome {
         # atac outputs
         File bam_aligned_output_atac = Atac.bam_aligned_output
         File fragment_file_atac = JoinBarcodes.atac_fragment_tsv
-        File fragment_file_index = JoinBarcodes.atac_fragment_tsv_tbi
+        File fragment_file_index = JoinBarcodes.atac_fragment_tsv_index
         File snap_metrics_atac = JoinBarcodes.atac_h5ad_file
         File atac_library_metrics = Atac.library_metrics_file
 

diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md
@@ -1,3 +1,8 @@
+# 7.8.1
+2024-11-04 (Date of Last Commit)
+
+* Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task should not affect the Optimus pipeline
+
 # 7.8.0
 2024-10-23 (Date of Last Commit)
 

diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl
@@ -71,7 +71,7 @@ workflow Optimus {
   # version of this pipeline
 
 
-  String pipeline_version = "7.8.0"
+  String pipeline_version = "7.8.1"
 
 
   # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays

diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md
@@ -1,3 +1,8 @@
+# 1.8.1
+2024-11-04 (Date of Last Commit)
+
+* Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task should not affect the Paired-Tag pipeline
+
 # 1.8.0
 2024-10-23 (Date of Last Commit)
 

diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl
@@ -8,7 +8,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow PairedTag {
 
-    String pipeline_version = "1.8.0"
+    String pipeline_version = "1.8.1"
 
 
     input {

diff --git a/pipelines/skylab/slideseq/SlideSeq.changelog.md b/pipelines/skylab/slideseq/SlideSeq.changelog.md
@@ -1,3 +1,8 @@
+# 3.4.4
+2024-11-04 (Date of Last Commit)
+
+* Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task should not affect the Slide-seq pipeline
+
 # 3.4.3
 2024-10-24 (Date of Last Commit)
 

diff --git a/pipelines/skylab/slideseq/SlideSeq.wdl b/pipelines/skylab/slideseq/SlideSeq.wdl
@@ -25,7 +25,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow SlideSeq {
 
-    String pipeline_version = "3.4.3"
+    String pipeline_version = "3.4.4"
 
     input {
         Array[File] r1_fastq

diff --git a/...tseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md b/...tseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md
@@ -1,3 +1,8 @@
+# 2.0.3
+2024-11-04 (Date of Last Commit)
+
+* Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task should not affect the snSS2 pipeline
+
 # 2.0.2
 2024-10-23 (Date of Last Commit)
 

diff --git a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl
@@ -57,7 +57,7 @@ workflow MultiSampleSmartSeq2SingleNucleus {
   }
 
   # Version of this pipeline
-  String pipeline_version = "2.0.2"
+  String pipeline_version = "2.0.3"
 
   if (false) {
      String? none = "None"

diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl
@@ -328,7 +328,7 @@ task JoinMultiomeBarcodes {
     echo "Starting bgzip"
     bgzip "~{atac_fragment_base}.sorted.tsv"
     echo "Starting tabix"
-    tabix -s 1 -b 2 -e 3 "~{atac_fragment_base}.sorted.tsv.gz"
+    tabix -s 1 -b 2 -e 3 -C "~{atac_fragment_base}.sorted.tsv.gz"
 
   >>>
 
@@ -343,7 +343,7 @@ task JoinMultiomeBarcodes {
     File gex_h5ad_file = "~{gex_base_name}.h5ad"
     File atac_h5ad_file = "~{atac_base_name}.h5ad"
     File atac_fragment_tsv = "~{atac_fragment_base}.sorted.tsv.gz"
-    File atac_fragment_tsv_tbi = "~{atac_fragment_base}.sorted.tsv.gz.tbi"
+    File atac_fragment_tsv_index = "~{atac_fragment_base}.sorted.tsv.gz.csi"
   }
 }
 
@@ -660,4 +660,4 @@ task AggregateSmartSeq2H5ad {
         preemptible: 3
         maxRetries: 1
     }
-}
+}
diff --git a/tasks/skylab/PairedTagUtils.wdl b/tasks/skylab/PairedTagUtils.wdl
@@ -278,7 +278,7 @@ task ParseBarcodes {
     echo "Starting bgzip"
     bgzip "~{atac_fragment_base}.sorted.tsv"
     echo "Starting tabix"
-    tabix -s 1 -b 2 -e 3 "~{atac_fragment_base}.sorted.tsv.gz"
+    tabix -s 1 -b 2 -e 3 -C "~{atac_fragment_base}.sorted.tsv.gz"
 
   >>>
 
@@ -292,6 +292,6 @@ task ParseBarcodes {
   output {
       File atac_h5ad_file = "~{atac_base_name}.h5ad"
       File atac_fragment_tsv = "~{atac_fragment_base}.sorted.tsv.gz"
-      File atac_fragment_tsv_tbi = "~{atac_fragment_base}.sorted.tsv.gz.tbi"
+      File atac_fragment_tsv_tbi = "~{atac_fragment_base}.sorted.tsv.gz.csi"
   }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -8,7 +8,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils @@
     workflow PairedTag {
-        String pipeline_version = "1.8.0"
+        String pipeline_version = "1.8.1"
         input {
@@ Expand Down @@