Reducing memory required for ValidateVCF in Reblock pipeline (#1400)

* Reducing memory required for ValidateVCF when the interval list to check is a GVCF * changelogs and pipeline versions * Updated pipeline_versions.txt with all pipeline version information --------- Co-authored-by: Nikelle Petrillo <[email protected]> Co-authored-by: npetrill <[email protected]> Co-authored-by: GitHub Action <[email protected]>
broadinstitute · Oct 30, 2024 · 672993e · 672993e
1 parent 1cd67fe
commit 672993e
Show file tree

Hide file tree

Showing 34 changed files with 114 additions and 34 deletions.
diff --git a/pipeline_versions.txt b/pipeline_versions.txt
@@ -1,32 +1,32 @@
 Pipeline Name	Version	Date of Last Commit
-CheckFingerprint	 1.0.21	2024-09-06 
+CheckFingerprint	 1.0.22	2024-10-28 
 RNAWithUMIsPipeline	 1.0.17	2024-09-06 
 AnnotationFiltration	 1.2.6	2024-09-06 
-UltimaGenomicsWholeGenomeGermline	 1.1.0	2024-09-06 
-WholeGenomeGermlineSingleSample	 3.3.1	2024-09-17 
-ExomeGermlineSingleSample	 3.2.1	2024-09-17 
+UltimaGenomicsWholeGenomeGermline	 1.1.1	2024-10-28 
+WholeGenomeGermlineSingleSample	 3.3.2	2024-10-28 
+ExomeGermlineSingleSample	 3.2.2	2024-10-28 
 JointGenotypingByChromosomePartTwo	 1.5.1	2024-09-10 
 JointGenotypingByChromosomePartOne	 1.5.1	2024-09-10 
-ReblockGVCF	 2.3.0	2024-09-06 
+ReblockGVCF	 2.3.1	2024-10-28 
 JointGenotyping	 1.7.1	2024-09-10 
 UltimaGenomicsJointGenotyping	 1.2.1	2024-09-10 
-VariantCalling	 2.2.2	2024-09-06 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.21	2024-09-06 
+VariantCalling	 2.2.3	2024-10-28 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.22	2024-10-28 
 GDCWholeGenomeSomaticSingleSample	 1.3.3	2024-09-06 
-BroadInternalRNAWithUMIs	 1.0.34	2024-09-06 
-BroadInternalUltimaGenomics	 1.1.0	2024-09-06 
-BroadInternalArrays	 1.1.12	2024-09-06 
+BroadInternalRNAWithUMIs	 1.0.35	2024-09-06 
+BroadInternalUltimaGenomics	 1.1.1	2024-10-28 
+BroadInternalArrays	 1.1.13	2024-10-28 
 BroadInternalImputation	 1.1.13	2024-09-06 
-Arrays	 2.6.28	2024-09-06 
+Arrays	 2.6.29	2024-10-28 
 ValidateChip	 1.16.6	2024-09-06 
 MultiSampleArrays	 1.6.2	2024-08-02 
 Imputation	 1.1.14	2024-09-06 
-IlluminaGenotypingArray	 1.12.22	2024-09-06 
-ExternalWholeGenomeReprocessing	 2.3.1	2024-09-17 
-ExternalExomeReprocessing	 3.3.1	2024-09-17 
+IlluminaGenotypingArray	 1.12.23	2024-10-28 
+ExternalWholeGenomeReprocessing	 2.3.2	2024-10-28 
+ExternalExomeReprocessing	 3.3.2	2024-10-28 
 CramToUnmappedBams	 1.1.3	2024-08-02 
-WholeGenomeReprocessing	 3.3.1	2024-09-17 
-ExomeReprocessing	 3.3.1	2024-09-17 
+WholeGenomeReprocessing	 3.3.2	2024-10-28 
+ExomeReprocessing	 3.3.2	2024-10-28 
 BuildIndices	 3.0.0	2023-12-06 
 scATAC	 1.3.2	2023-08-03 
 snm3C	 4.0.4	2024-08-06 

diff --git a/pipelines/broad/arrays/single_sample/Arrays.changelog.md b/pipelines/broad/arrays/single_sample/Arrays.changelog.md
@@ -1,3 +1,8 @@
+# 2.6.29
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 2.6.28
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/arrays/single_sample/Arrays.wdl b/pipelines/broad/arrays/single_sample/Arrays.wdl
@@ -23,7 +23,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Arrays {
 
-  String pipeline_version = "2.6.28"
+  String pipeline_version = "2.6.29"
 
   input {
     String chip_well_barcode

diff --git a/...nes/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.changelog.md b/...nes/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.changelog.md
@@ -1,3 +1,8 @@
+# 2.3.1
+2024-10-28 (Date of Last Commit)
+
+* Updated GATK for Validate Variants, which reduces the memory requirements for the task when an interval list is not provided
+
 # 2.3.0
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl b/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl
@@ -6,7 +6,7 @@ import "../../../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow ReblockGVCF {
 
-  String pipeline_version = "2.3.0"
+  String pipeline_version = "2.3.1"
 
 
   input {

diff --git a/...oad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md b/...oad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md
@@ -1,3 +1,8 @@
+# 3.2.2
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 3.2.1
 2024-09-17 (Date of Last Commit)
 

diff --git a/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl b/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl
@@ -45,7 +45,7 @@ import "../../../../../../tasks/broad/Utilities.wdl" as utils
 # WORKFLOW DEFINITION
 workflow ExomeGermlineSingleSample {
 
-  String pipeline_version = "3.2.1"
+  String pipeline_version = "3.2.2"
 
 
   input {

diff --git a/...seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md b/...seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md
@@ -1,3 +1,8 @@
+# 1.1.1
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.1.0
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl b/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl
@@ -50,7 +50,7 @@ workflow UltimaGenomicsWholeGenomeGermline {
     filtering_model_no_gt_name: "String describing the optional filtering model; default set to rf_model_ignore_gt_incl_hpol_runs"
   }
 
-  String pipeline_version = "1.1.0"
+  String pipeline_version = "1.1.1"
 
 
   References references = alignment_references.references

diff --git a/...dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md b/...dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md
@@ -1,3 +1,8 @@
+# 3.3.2
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 3.3.1
 2024-09-17 (Date of Last Commit)
 

diff --git a/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl b/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl
@@ -40,7 +40,7 @@ import "../../../../../../structs/dna_seq/DNASeqStructs.wdl"
 workflow WholeGenomeGermlineSingleSample {
 
 
-  String pipeline_version = "3.3.1"
+  String pipeline_version = "3.3.2"
 
 
   input {

diff --git a/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.changelog.md b/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.changelog.md
@@ -1,3 +1,8 @@
+# 2.2.3
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 2.2.2
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl b/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl
@@ -9,7 +9,7 @@ import "../../../../../tasks/broad/DragenTasks.wdl" as DragenTasks
 workflow VariantCalling {
 
 
-  String pipeline_version = "2.2.2"
+  String pipeline_version = "2.2.3"
 
 
   input {

diff --git a/..._seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md b/..._seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md
@@ -1,3 +1,8 @@
+# 1.0.22
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.0.21
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl b/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl
@@ -43,7 +43,7 @@ workflow UltimaGenomicsWholeGenomeCramOnly {
     save_bam_file: "If true, then save intermeidate ouputs used by germline pipeline (such as the output BAM) otherwise they won't be kept as outputs."
   }
 
-  String pipeline_version = "1.0.21"
+  String pipeline_version = "1.0.22"
 
   References references = alignment_references.references
 

diff --git a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.changelog.md b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.changelog.md
@@ -1,3 +1,8 @@
+# 1.12.23
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.12.22
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
@@ -21,7 +21,7 @@ import "../../../../tasks/broad/Qc.wdl" as Qc
 
 workflow IlluminaGenotypingArray {
 
-  String pipeline_version = "1.12.22"
+  String pipeline_version = "1.12.23"
 
   input {
     String sample_alias

diff --git a/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.changelog.md b/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.changelog.md
@@ -1,3 +1,8 @@
+# 1.1.13
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.1.12
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.wdl b/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.wdl
@@ -9,7 +9,7 @@ workflow BroadInternalArrays {
         description: "Push outputs of Arrays.wdl to TDR dataset table ArraysOutputsTable."
     }
 
-    String pipeline_version = "1.1.12"
+    String pipeline_version = "1.1.13"
 
     input {
         # inputs to wrapper task

diff --git a/.../germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.changelog.md b/.../germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.changelog.md
@@ -1,3 +1,8 @@
+# 1.1.1
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.1.0
 2024-09-06 (Date of Last Commit)
 

diff --git a/...ad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.wdl b/...ad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.wdl
@@ -6,7 +6,7 @@ import "../../../../../../../pipelines/broad/qc/CheckFingerprint.wdl" as FP
 
 workflow BroadInternalUltimaGenomics {
 
-  String pipeline_version = "1.1.0"
+  String pipeline_version = "1.1.1"
 
   input {
 

diff --git a/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.changelog.md b/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.changelog.md
@@ -1,3 +1,8 @@
+# 1.0.35
+2024-09-06 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.0.34
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.wdl b/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.wdl
@@ -7,7 +7,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow BroadInternalRNAWithUMIs {
 
-  String pipeline_version = "1.0.34"
+  String pipeline_version = "1.0.35"
 
   input {
     # input needs to be either "hg19" or "hg38"

diff --git a/pipelines/broad/qc/CheckFingerprint.changelog.md b/pipelines/broad/qc/CheckFingerprint.changelog.md
@@ -1,3 +1,8 @@
+# 1.0.22
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.0.21
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/qc/CheckFingerprint.wdl b/pipelines/broad/qc/CheckFingerprint.wdl
@@ -24,7 +24,7 @@ import "../../../tasks/broad/Qc.wdl" as Qc
 
 workflow CheckFingerprint {
 
-  String pipeline_version = "1.0.21"
+  String pipeline_version = "1.0.22"
 
   input {
     File? input_vcf

diff --git a/pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md b/pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md
@@ -1,3 +1,8 @@
+# 3.3.2
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 3.3.1
 2024-09-17 (Date of Last Commit)
 

diff --git a/pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl b/pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl
@@ -7,7 +7,7 @@ import "../../../../structs/dna_seq/DNASeqStructs.wdl"
 workflow ExomeReprocessing {
 
 
-  String pipeline_version = "3.3.1"
+  String pipeline_version = "3.3.2"
 
   input {
     File? input_cram

diff --git a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md
@@ -1,3 +1,8 @@
+# 3.3.2
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 3.3.1
 2024-09-17 (Date of Last Commit)
 

diff --git a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl
@@ -5,7 +5,7 @@ import "../../../../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow ExternalExomeReprocessing {
 
-  String pipeline_version = "3.3.1"
+  String pipeline_version = "3.3.2"
 
 
   input {

diff --git a/...es/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md b/...es/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md
@@ -1,3 +1,8 @@
+# 2.3.2
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 2.3.1
 2024-09-17 (Date of Last Commit)
 

diff --git a/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl b/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl
@@ -6,7 +6,7 @@ import "../../../../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
 workflow ExternalWholeGenomeReprocessing {
 
 
-  String pipeline_version = "2.3.1"
+  String pipeline_version = "2.3.2"
 
   input {
     File? input_cram

diff --git a/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md b/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md
@@ -1,3 +1,8 @@
+# 3.3.2
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 3.3.1
 2024-09-17 (Date of Last Commit)
 

diff --git a/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl b/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl
@@ -6,7 +6,7 @@ import "../../../../structs/dna_seq/DNASeqStructs.wdl"
 
 workflow WholeGenomeReprocessing {
 
-  String pipeline_version = "3.3.1"
+  String pipeline_version = "3.3.2"
 
   input {
     File? input_cram

diff --git a/tasks/broad/Qc.wdl b/tasks/broad/Qc.wdl
@@ -623,7 +623,7 @@ task ValidateVCF {
     Boolean is_gvcf = true
     String? extra_args
     #Setting default docker value for workflows that haven't yet been azurized.
-    String docker_path = "us.gcr.io/broad-gatk/gatk:4.6.0.0"
+    String docker_path = "us.gcr.io/broad-gatk/gatk:4.6.1.0"
     Int machine_mem_mb = 7000
   }
 
@@ -642,7 +642,7 @@ task ValidateVCF {
       ln -s ~{calling_interval_list} ~{calling_interval_list_basename}
       ln -s ~{calling_interval_list_index} ~{calling_interval_list_index_basename}
       gatk --java-options "-Xms~{command_mem_mb}m -Xmx~{command_mem_mb}m" \
-        VcfToIntervalList -I ~{calling_interval_list_basename} -O intervals_from_gvcf.interval_list
+        VcfToIntervalList -I ~{calling_interval_list_basename} -O intervals_from_gvcf.interval_list --VARIANT_ID_METHOD USE_FIRST
       INTERVALS="intervals_from_gvcf.interval_list"
     else
       INTERVALS="~{calling_interval_list}"
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,7 +6,7 @@ import "../../../../../../tasks/broad/Utilities.wdl" as utils @@
     workflow ReblockGVCF {
-      String pipeline_version = "2.3.0"
+      String pipeline_version = "2.3.1"
       input {
@@ Expand Down @@