Merge branch 'develop' into dependabot/npm_and_yarn/website/micromatc…

…h-4.0.8

verification/VerifyMultiome.wdl

@@ -29,6 +29,9 @@ workflow VerifyMultiome {
         File test_library_metrics
         File truth_library_metrics
 
+        File test_atac_library_metrics
+        File truth_atac_library_metrics
+
         Boolean? done
     }
 
@@ -72,9 +75,14 @@ workflow VerifyMultiome {
             test_h5ad  = test_optimus_h5ad,
             truth_h5ad = truth_optimus_h5ad
     }
-    call VerifyTasks.CompareTextFiles as CompareLibraryMetrics {
+    call VerifyTasks.CompareLibraryFiles as CompareLibraryMetrics {
+        input:
+            test_text_file = test_library_metrics,
+            truth_text_file = truth_library_metrics
+    }
+    call VerifyTasks.CompareTextFiles as CompareAtacLibraryMetrics {
         input:
-            test_text_files = select_all([test_library_metrics]),
-            truth_text_files = select_all([truth_library_metrics])
+            test_text_files = select_all([test_atac_library_metrics]),
+            truth_text_files = select_all([truth_atac_library_metrics])
     }
 }

verification/VerifyOptimus.wdl

@@ -48,9 +48,9 @@ workflow VerifyOptimus {
             truth_h5ad = truth_h5ad
     }
 
-  call VerifyTasks.CompareTextFiles as CompareLibraryMetrics {
+  call VerifyTasks.CompareLibraryFiles as CompareLibraryMetrics {
           input:
-            test_text_files = select_all([test_library_metrics]),
-            truth_text_files = select_all([truth_library_metrics])
+            test_text_file = test_library_metrics,
+            truth_text_file = truth_library_metrics
 }
 }

verification/VerifyPairedTag.wdl

@@ -72,9 +72,9 @@ workflow VerifyPairedTag {
             test_h5ad  = test_optimus_h5ad,
             truth_h5ad = truth_optimus_h5ad
     }
-    call VerifyTasks.CompareTextFiles as CompareLibraryMetrics {
+    call VerifyTasks.CompareLibraryFiles as CompareLibraryMetrics {
         input:
-            test_text_files = select_all([test_library_metrics]),
-            truth_text_files = select_all([truth_library_metrics])
+            test_text_file = test_library_metrics,
+            truth_text_file = truth_library_metrics
     }
 }

verification/VerifyTasks.wdl

@@ -613,3 +613,53 @@ task CompareSnapTextFiles {
 }
 
 
+task CompareLibraryFiles {
+  input {
+    File test_text_file
+    File truth_text_file
+  }
+
+  command {
+    exit_code=0
+
+    a=~{test_text_file}
+    b=~{truth_text_file}
+
+    echo "Sorting files $a and $b"
+    sort "$a" > "a.sorted"
+    sort "$b" > "b.sorted"
+
+    echo "Calculating md5sums for $a and $b"
+    md5_a=$(md5sum "a.sorted" | cut -d ' ' -f1)
+    md5_b=$(md5sum "b.sorted" | cut -d ' ' -f1)
+
+    if [ $md5_a = $md5_b ]; then
+      echo "Files $a.sorted and $b.sorted have matching md5sums and are the same."
+    else
+      echo "Files $a.sorted and $b.sorted have different md5sums."
+
+      # Compare the files, excluding specific lines
+      excluded_lines="percent_doublets|keeper_cells|keeper_mean_reads_per_cell|keeper_median_genes|percent_keeper|percent_usable"
+        
+      # Store the diff result, but only check non-excluded lines
+      diff_output=$(diff <(grep -v -E $excluded_lines a.sorted) <(grep -v -E $excluded_lines b.sorted))
+
+      if [ -z "$diff_output" ]; then
+          echo "Files a.sorted and $b.sorted are the same when excluding specified lines."
+      else
+        echo "Files a.sorted and b.sorted have differences in non-excluded lines."
+        echo "$diff_output"
+        exit_code=2
+      fi
+    fi
+    echo "Exiting with code $exit_code"
+    exit $exit_code
+  }
+
+  runtime {
+    docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf"
+    disks: "local-disk 100 HDD"
+    memory: "50 GiB"
+    preemptible: 3
+  }
+}

verification/test-wdls/TestMultiome.wdl

@@ -114,7 +114,8 @@ workflow TestMultiome {
     Array[String] pipeline_metrics = flatten([
                                     [ # File outputs
                                     Multiome.gene_metrics_gex,
-                                    Multiome.cell_metrics_gex
+                                    Multiome.cell_metrics_gex,
+                                    Multiome.atac_library_metrics
                                     ],
                                     # File? outputs
                                     select_all([Multiome.library_metrics]),
@@ -194,6 +195,13 @@ workflow TestMultiome {
             }
         }
 
+        call Utilities.GetValidationInputs as GetAtacLibraryMetrics {
+            input:
+            input_file = Multiome.atac_library_metrics,
+            results_path = results_path,
+            truth_path = truth_path
+        }
+
       call VerifyMultiome.VerifyMultiome as Verify {
         input:
           truth_optimus_h5ad = GetOptimusH5ad.truth_file,
@@ -212,6 +220,8 @@ workflow TestMultiome {
           test_atac_h5ad = GetSnapMetrics.results_file,
           test_library_metrics =  select_first([GetLibraryMetrics.results_file, ""]),
           truth_library_metrics = select_first([GetLibraryMetrics.truth_file, ""]),
+          test_atac_library_metrics = GetAtacLibraryMetrics.results_file,
+          truth_atac_library_metrics = GetAtacLibraryMetrics.truth_file,
           done = CopyToTestResults.done
       }
     }